Adding BlockGSPreconditioner.

This now matches results from old 2D templated code, used for the NLA paper.

.github/stale.yml

@@ -1,61 +0,0 @@
-# Configuration for probot-stale - https://github.com/probot/stale
-
-# Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 30
-
-# Number of days of inactivity before an Issue or Pull Request with the stale
-# label is closed.  Set to false to disable. If disabled, issues still need to
-# be closed manually, but will remain marked as stale.
-daysUntilClose: 7
-
-# Only issues or pull requests with all of these labels are check if stale.
-# Defaults to `[]` (disabled)
-onlyLabels: []
-
-# Issues or Pull Requests with these labels will never be considered stale. Set
-# to `[]` to disable
-exemptLabels:
-  - bug
-  - WIP
-  - ready-for-review
-  - in-review
-  - in-next
-
-# Set to true to ignore issues in a project (defaults to false)
-exemptProjects: false
-
-# Set to true to ignore issues in a milestone (defaults to false)
-exemptMilestones: false
-
-# Set to true to ignore issues with an assignee (defaults to false)
-exemptAssignees: false
-
-# Label to use when marking an issue as stale
-staleLabel: stale
-
-# Comment to post when marking an issue as stale. Set to `false` to disable
-markComment: >
-  :warning: This issue or PR has been automatically marked as stale because it has not
-  had any activity in the last month. *If no activity occurs in the next week, it will 
-  be automatically closed.* Thank you for your contributions.
-
-# Comment to post when closing a stale issue. Set to `false` to disable
-closeComment: false
-
-# Limit the number of actions per hour, from 1-30. Default is 30
-limitPerRun: 30
-
-# Limit to only `issues` or `pulls`
-# only: issues
-
-# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
-# pulls:
-#   daysUntilStale: 30
-#   markComment: >
-#     This pull request has been automatically marked as stale because it has not had
-#     recent activity. It will be closed if no further activity occurs. Thank you
-#     for your contributions.
-
-# issues:
-#   exemptLabels:
-#     - confirmed

.gitignore

@@ -102,29 +102,28 @@ examples/ex25.mesh
 examples/ex25-*.gf
 examples/ex25p-*.*
 
-examples/amgx/ex1
-examples/amgx/ex1p
-examples/amgx/.logamgx
-examples/amgx/refined.mesh
-examples/amgx/sol.gf
-examples/amgx/mesh.*
-examples/amgx/sol.*
+examples/sundials/ex9
+examples/sundials/ex1[06]
+examples/sundials/ex9p
+examples/sundials/ex1[06]p
 
-examples/gingko/ex1
-examples/gingko/refined.mesh
-examples/gingko/sol.gf
-examples/gingko/mesh.*
-examples/gingko/sol.*
-
-examples/hiop/ex9
-examples/hiop/ex9p
-examples/hiop/ex9.mesh
-examples/hiop/ex9-mesh.*
-examples/hiop/ex9-init.*
-examples/hiop/ex9-final.*
+examples/sundials/ex9.mesh
+examples/sundials/ex9-mesh.*
+examples/sundials/ex9-init.*
+examples/sundials/ex9-final.*
+examples/sundials/Example9*
+examples/sundials/deformed.*
+examples/sundials/velocity.*
+examples/sundials/elastic_energy.*
+examples/sundials/ex16.mesh
+examples/sundials/ex16-mesh.*
+examples/sundials/ex16-init.*
+examples/sundials/ex16-final.*
+examples/sundials/Example16*
 
 examples/petsc/ex[1-69]p
 examples/petsc/ex1[0-1]p
+
 examples/petsc/mesh.*
 examples/petsc/sol.*
 examples/petsc/sol_p.*
@@ -142,51 +141,28 @@ examples/petsc/mode_*
 
 examples/pumi/ex1
 examples/pumi/ex[126]p
+
+examples/hiop/ex9.mesh
+examples/hiop/ex9-mesh.*
+examples/hiop/ex9-init.*
+examples/hiop/ex9-final.*
+
 examples/pumi/refined.mesh
 examples/pumi/sol.gf
 examples/pumi/mesh.*
 examples/pumi/sol.*
 examples/pumi/displaced.mesh
 
-examples/sundials/ex9
-examples/sundials/ex1[06]
-examples/sundials/ex9p
-examples/sundials/ex1[06]p
-examples/sundials/ex9.mesh
-examples/sundials/ex9-mesh.*
-examples/sundials/ex9-init.*
-examples/sundials/ex9-final.*
-examples/sundials/Example9*
-examples/sundials/deformed.*
-examples/sundials/velocity.*
-examples/sundials/elastic_energy.*
-examples/sundials/ex16.mesh
-examples/sundials/ex16-mesh.*
-examples/sundials/ex16-init.*
-examples/sundials/ex16-final.*
-examples/sundials/Example16*
-
-examples/superlu/ex1p
-examples/superlu/mesh.*
-examples/superlu/sol.*
-
-miniapps/adjoint/cvsRoberts_ASAi_dns
-miniapps/adjoint/adjoint_advection_diffusion
-
 miniapps/electromagnetics/volta
 miniapps/electromagnetics/tesla
 miniapps/electromagnetics/maxwell
 miniapps/electromagnetics/joule
+
 miniapps/electromagnetics/Volta-AMR*
 miniapps/electromagnetics/Tesla-AMR*
 miniapps/electromagnetics/Maxwell-Parallel*
 miniapps/electromagnetics/Joule_*
 
-miniapps/gslib/field-diff
-miniapps/gslib/field-interp
-miniapps/gslib/findpts
-miniapps/gslib/pfindpts
-
 miniapps/meshing/mobius-strip
 miniapps/meshing/klein-bottle
 miniapps/meshing/toroid
@@ -200,6 +176,7 @@ miniapps/meshing/pmesh-optimizer
 miniapps/meshing/minimal-surface
 miniapps/meshing/pminimal-surface
 miniapps/meshing/polar-nc
+
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
@@ -213,25 +190,9 @@ miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
 miniapps/meshing/polar-nc.mesh
 
-miniapps/navier/navier_mms
-miniapps/navier/navier_kovasznay
-miniapps/navier/navier_tgv
-miniapps/navier/navier_shear
-miniapps/navier/navier_3dfoc
-miniapps/navier/tgv_out*.txt
-miniapps/navier/*_output
-
-miniapps/nurbs/nurbs_ex1
-miniapps/nurbs/nurbs_ex1p
-miniapps/nurbs/nurbs_ex11p
-miniapps/nurbs/refined.mesh
-miniapps/nurbs/mesh.*
-miniapps/nurbs/sol.*
-miniapps/nurbs/mode_*
-miniapps/nurbs/Example1*
-
 miniapps/performance/ex1
 miniapps/performance/ex1p
+
 miniapps/performance/refined.mesh
 miniapps/performance/mesh.*
 miniapps/performance/sol.*
@@ -249,6 +210,7 @@ miniapps/toys/rubik
 miniapps/toys/snake
 miniapps/toys/lissajous
 miniapps/toys/mondrian
+
 miniapps/toys/snake-init.mesh
 miniapps/toys/snake-user.mesh
 miniapps/toys/snake-joined.mesh
@@ -263,13 +225,37 @@ miniapps/toys/lissajous.mesh
 miniapps/toys/lissajous.gf
 miniapps/toys/mondrian.mesh
 
+miniapps/nurbs/nurbs_ex1
+miniapps/nurbs/nurbs_ex1p
+miniapps/nurbs/nurbs_ex11p
+miniapps/nurbs/refined.mesh
+miniapps/nurbs/mesh.*
+miniapps/nurbs/sol.*
+miniapps/nurbs/mode_*
+miniapps/nurbs/Example1*
+
+miniapps/gslib/field-diff
+miniapps/gslib/field-interp
+miniapps/gslib/findpts
+miniapps/gslib/pfindpts
+
+miniapps/navier/navier_mms
+miniapps/navier/navier_kovasznay
+miniapps/navier/navier_tgv
+miniapps/navier/navier_shear
+miniapps/navier/navier_3dfoc
+miniapps/navier/tgv_out*.txt
+miniapps/navier/*_output
+
+miniapps/adjoint/cvsRoberts_ASAi_dns
+miniapps/adjoint/adjoint_advection_diffusion
+
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
 tests/unit/punit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
-tests/unit/ceed_tests
 
 # Test script output
 tests/scripts/*.err

.gitlab/lassen.yml

@@ -37,7 +37,7 @@ build_mfem_debug_ser_lassen:
     - mkdir -p ${BUILD_PATH}
     - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
     - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
-    - lalloc 1 -W 5 -q pdebug make -j cuda MFEM_DEBUG="YES" CPPFLAGS=-O2 CUDA_ARCH=sm_70
+    - lalloc 1 -W 5 -q pdebug make -j cuda MFEM_DEBUG="YES" CUDA_ARCH=sm_70
 
 # Sanity check
 sanitycheck_mfem_ser_lassen:

CHANGELOG

@@ -11,16 +11,6 @@
 Version 4.1.1 (development)
 ===========================
 
-libCEED integration improvements
---------------------------------
-- Add support for all types of (scalar) Coefficient.
-
-- Add support for VectorMassIntegrator and VectorDiffusionIntegrator.
-
-- Add support for AssemblyLevel::NONE for MassIntegrator, DiffusionIntegrator,
-  VectorMassIntegrator, and VectorDiffusionIntegrator. This level of assembly
-  fully applies on the fly the finite element operator.
-
 Meshing improvements
 --------------------
 - The graph linear ordering library Gecko, previously an external dependency, is
@@ -66,30 +56,10 @@ Performance improvements
 
 Improved GPU capabilities
 -------------------------
-- Added a new solver class for simple integration with NVIDIA's multigrid
-  library, AmgX. The AmgX class is designed to work as a standalone solver or
-  preconditioner for existing MFEM solvers. It uses MFEM's sparse matrix format
-  for serial runs and the HypreParMatrix format for parallel runs.
-
-  The new solver may be configured to run with one GPU per MPI rank or with more
-  MPI ranks than GPUs. In the latter case, matrices and vectors are consolidated
-  to ranks communicating with the GPUs and the solution is then broadcasted.
-
-  Although CUDA is required to build, the AmgX support is compatible with the
-  MFEM CPU device configuration. The examples/amgx folder illustrates how to
-  integrate AmgX in existing MFEM applications.
-
-  The AmgX solver class is partially based on: "AmgXWrapper: An interface
-  between PETSc and the NVIDIA AmgX library", by Pi-Yueh Chuang and Lorena
-  A. Barba, doi:10.21105/joss.00280.
-
 - Added support for Chebyshev accelerated polynomial smoother on GPU.
 
 - Optimized AMD/HIP kernel support.
 
-- Enabled HIP support in the libCEED integration, which is now available via the
-  "ceed-hip" device backend.
-
 - Added a Full Assembly mode compatible with Device kernel execution. This
   assembly level builds on top of the current Element Assembly kernels to
   compute a global sparse matrix. All integrators supported by element assembly
@@ -97,15 +67,14 @@ Improved GPU capabilities
 
 - Added CUDA support for sparse matrix-vector multiplication with cuSPARSE.
 
+- Added AmgX solver class for simple integration with NVIDIA's multigrid library.
+
 - Added support for BlockOperator on GPU. See the updated Example 5.
 
 - Added partial assembly and GPU support for complex operators, including the
   classes ComplexOperator, [Par]ComplexGridFunction, [Par]ComplexLinearForm, and
   [Par]SesquilinearForm. See the updated Example 22.
 
-- Added CUDA support for SUNDIALS ODE integrators. See the updated SUNDIALS
-  modification of Example 9/9p.
-
 Discretization improvements
 ---------------------------
 - Added support for matrix-free interpolation and restriction operators between
@@ -132,9 +101,6 @@ Discretization improvements
   Additionally, new LinearForm integrators were also added which make use of
   these new QuadratureFunction coefficient classes.
 
-- Non-conforming meshes are now supported with block nonlinear forms. See the
-  updated Example 19/19p.
-
 - Added support face integrals on the boundaries of NURBS meshes.
 
 - Added support for interpolation of functions in L2, H(div) and H(curl)
@@ -145,12 +111,6 @@ Discretization improvements
   member methods in GridFunction and ParGridFunction. See the rates.cpp file in
   the tests/convergence directory for sample usage.
 
-- The C-function based coefficient classes (FunctionCoefficient,
-  VectorFunctionCoefficient, and MatrixFunctionCoefficient) now use the more
-  general std::function class template. This allows the classes to be backward
-  compatible (i.e. they can still work with C-functions) and, in addition,
-  support any "callable", e.g. lambda functions.
-
 Linear and nonlinear solvers
 ----------------------------
 - Added power method to iteratively estimate the largest eigenvalue and the
@@ -159,9 +119,6 @@ Linear and nonlinear solvers
 - Added initial support for h- and p-multigrid solvers and preconditioners for
   matrix-based and matrix-free discretizations with basic GPU capability.
 
-- Added wrappers for Hypre's flexible GMRES solver and the new parallel ILU
-  preconditioner. The latter requires hypre version 2.19.0 or later.
-
 - Added a new IterativeSolverMonitor class that allows to monitor the residual
   and solution during the solving process of an IterativeSolver after every
   iteration.
@@ -182,15 +139,6 @@ Linear and nonlinear solvers
 - Added partially assembled convergent diagonal preconditioner for adaptively
   refined meshes (i.e. non-conforming finite element spaces), see Example 6/6p.
 
-- Upgraded SuperLU interface to use SuperLU_DIST 6.3.1. Added a simple SuperLU
-  example in the new directory examples/superlu.
-
-- Extended the KINSOL (SUNDIALS) nonlinear solver interface to support the
-  Jacobian-free Newton-Krylov method. A usage example is shown in Example 10p.
-
-- Added an interface to the MKL CPardiso solver -- an MPI-parallel sparse direct
-  solver developed by Intel. See Example 11p for an illustration of its usage.
-
 New and updated examples and miniapps
 -------------------------------------
 - Added a new example, Example 25/25p, to demonstrate the use of a Perfectly
@@ -265,8 +213,6 @@ Improved testing
   triggered only by LLNL developers, see .gitlab-ci.yml, the .gitlab directory
   and the updated CONTRIBUTING.md file.
 
-- Add tests for the libCEED integration in MFEM.
-
 - Added testing of the parallel mesh format in tests/par-mesh-format.
 
 Miscellaneous
@@ -279,9 +225,6 @@ Miscellaneous
   ParaView v5.8.1. In addition, ADIOS2 allows for setting a user-defined number
   of data substreams/subfiles at scale. See examples 5, 9, 12, 16.
 
-- Added VTU output of boundary elements and attributes and parallel VTU (PVTU)
-  output of parallel meshes for visualization using ParaView.
-
 - The integration order used in the ComputeLpError and ComputeElementLpError
   methods of class GridFunction has been increased.
 
@@ -290,9 +233,6 @@ Miscellaneous
 - Renamed "Backend::DEBUG" to "Backend::DEBUG_DEVICE" to avoid conflicts,
   as DEBUG is sometimes used as a macro.
 
-- Change the IntegrationRule inside VectorDiffusionIntegrator to use the same
-  quadrature as DiffusionIntegrator.
-
 
 Version 4.1, released on March 10, 2020
 =======================================

CMakeLists.txt

@@ -240,14 +240,12 @@ endif()
 
 # SUNDIALS
 if (MFEM_USE_SUNDIALS)
-  set(SUNDIALS_COMPONENTS CVODES ARKODE KINSOL NVector_Serial)
-  if (MFEM_USE_MPI)
-    list(APPEND SUNDIALS_COMPONENTS NVector_Parallel NVector_MPIPlusX)
+  if (NOT MFEM_USE_MPI)
+    find_package(SUNDIALS REQUIRED NVector_Serial CVODES ARKODE KINSOL)
+  else()
+    find_package(SUNDIALS REQUIRED
+      NVector_Serial NVector_Parallel NVector_ParHyp CVODES ARKODE KINSOL)
   endif()
-  if (MFEM_USE_CUDA)
-    list(APPEND SUNDIALS_COMPONENTS NVector_Cuda)
-  endif()
-  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()
 
 # Mesquite
@@ -298,7 +296,8 @@ if (MFEM_USE_CEED)
 endif()
 
 if (MFEM_USE_AMGX)
-  find_package(AMGX REQUIRED)
+  include_directories(${AMGX_DIR}/include)
+  link_directories(${AMGX_DIR}/lib64 -L${AMGX_DIR}/lib -Xlinker=-rpath=${CUDA_HOME}/lib64)
 endif()
 
 if (MFEM_USE_CONDUIT)
@@ -352,12 +351,6 @@ if (MFEM_USE_ADIOS2)
    find_package(ADIOS2 REQUIRED)
 endif()
 
-if (MFEM_USE_MKL_CPARDISO)
-  if (MFEM_USE_MPI)
-   find_package(MKL_CPARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64 MKL_MPI_WRAPPER)
-  endif()
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -384,7 +377,7 @@ endif()
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
     SLEPC MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
     MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE MKL_CPARDISO AMGX)
+    CUSPARSE AMGX)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")

CONTRIBUTING.md

@@ -98,18 +98,15 @@ The MFEM source code has the following structure:
   │   └── web
   │       └── examples
   ├── examples
-  │   ├── amgx
   │   ├── ginkgo
   │   ├── hiop
   │   ├── petsc
   │   ├── pumi
   │   └── sundials
-  |   └── superlu
   ├── fem
   │   └── libceed
   ├── general
   ├── linalg
-  │   └── simd
   ├── mesh
   ├── miniapps
   │   ├── adjoint
@@ -117,14 +114,11 @@ The MFEM source code has the following structure:
   │   ├── electromagnetics
   │   ├── gslib
   │   ├── meshing
-  │   ├── navier
   │   ├── nurbs
   │   ├── performance
   │   ├── tools
   │   └── toys
   └── tests
-      ├── convergence
-      ├── par-mesh-format
       ├── scripts
       ├── unit
       │   ├── ...

INSTALL

@@ -344,27 +344,24 @@ MFEM_USE_SUPERLU = YES/NO
    SuperLURowLocMatrix a distributed CSR matrix class needed by SuperLU. When
    enabled, this option uses the SUPERLU_* library options, see below.
 
-MFEM_USE_SUPERLU5 = YES/NO
-   If SuperLU functionality is enabled, use the older 5.1.0 version rather than
-   the more recent 6+ versions.
-
 MFEM_USE_STRUMPACK = YES/NO
    Enable MFEM functionality based on the STRUMPACK sparse direct solver and
    preconditioner through the STRUMPACKSolver and STRUMPACKRowLocMatrix
    classes. When enabled, this option uses the STRUMPACK_* library options, see
    below.
 
+MFEM_USE_AMGX = YES/NO
+   Enable MFEM functionality based on the AMGX multigrid library from 
+   NVIDIA. When enabled, this options enables the user to use SparseMatrices
+   and HypreParMatrices to solve linear systems using routines from the AMGX
+   library.
+
 MFEM_USE_GINKGO = YES/NO
    Enable MFEM functionality based on the Ginkgo library, which provides
    iterative linear solvers and preconditioners with OpenMP, CUDA backends, see
    https://github.com/ginkgo-project/ginkgo. When enabled, the user can use
    Ginkgo's solvers and preconditioners as shown in examples/ginkgo/.
 
-MFEM_USE_AMGX = YES/NO
-   Enable MFEM functionality based on the AmgX multigrid library from NVIDIA.
-   Allows the user to use SparseMatrices and HypreParMatrices to solve linear
-   systems with the routines from the AmgX library.
-
 MFEM_USE_GNUTLS = YES/NO
    Enable secure socket support in class socketstream, using the auxiliary
    GnuTLS_* classes, based on the GnuTLS library. This option may be useful in
@@ -495,13 +492,6 @@ MFEM_USE_CEED = YES/NO
    library for performant high-order operator evaluation developed by the Center
    for Efficient Exascale Discretizations in the Exascale Computing Project.
 
-MFEM_USE_MKL_CPARDISO = YES/NO
-   Enables the interface to MKL CPardiso: the Intel MKL Parallel Direct Sparse
-   Solver for Clusters. Make sure to set the correct values for MKL_MPI_WRAPPER
-   and MKL_LIBRARY_SUBDIR as shown in defaults.mk. If you configure MFEM with
-   MFEM_USE_LAPACK=YES, verify that the MKL LAPACK libraries are used. The
-   OpenMP capabilities are disabled at link time.
-
 MFEM_BUILD_TAG = (any value)
    An optional tag to characterize the build. Exported to config/config.mk.
    Can be used to identify the MFEM build from other makefiles.
@@ -551,14 +541,11 @@ The specific libraries and their options are:
   Beginning with MFEM v3.3, SUNDIALS v2.7.0 is supported.
   Beginning with MFEM v3.3.2, SUNDIALS v3.0.0 is also supported.
   Beginning with MFEM v4.1, only SUNDIALS v5.0.0+ is supported.
-  When MFEM_USE_CUDA is enabled, only SUNDIALS v5.4.0+ is supported.
   If MFEM_USE_MPI is enabled, we expect that SUNDIALS is built with support for
   both MPI and hypre.
-  If MFEM_USE_CUDA is enabled, we expect that SUNDIALS is built with support
-  for CUDA.
   URL: http://computation.llnl.gov/projects/sundials/sundials-software
   Options: SUNDIALS_OPT, SUNDIALS_LIB.
-  Versions: SUNDIALS >= 5.0.0, SUNDIALS >= 5.4.0 for CUDA suppport.
+  Versions: SUNDIALS >= 5.0.0.
 
 - Mesquite (optional), used when MFEM_USE_MESQUITE = YES.
   URL: http://trilinos.org/oldsite/packages/mesquite
@@ -593,11 +580,6 @@ The specific libraries and their options are:
   URL: https://ginkgo-project.github.io
   Options: GINKGO_OPT (Not used), GINKGO_LIB.
 
-- AmgX (optional), used when MFEM_USE_AMGX = YES.
-  URL: https://github.com/NVIDIA/AMGX
-  Options: AMGX_OPT, AMGX_LIB.
-  Versions: AmgX >= 2.1, older versions may work too.
-
 - GnuTLS (optional), used when MFEM_USE_GNUTLS = YES. On most Linux systems,
   GnuTLS is available as a development package, e.g. gnutls-devel. On Mac OS X,
   one can get the library through the Homebrew package manager (http://brew.sh).
@@ -669,16 +651,16 @@ The specific libraries and their options are:
   Options: GSLIB_OPT, GSLIB_LIB.
   Versions: GSLIB >= 1.0.5.
 
-- MKL CPardiso (optional), used when MFEM_USE_MKL_CPARDISO = YES.
-  URL: https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html
-  Options: MKL_CPARDISO_OPT, MKL_CPARDISO_LIB.
-  Versions: Intel MKL >= 2020.
-
 - CUDA (optional), used when MFEM_USE_CUDA = YES.
   URL: https://developer.nvidia.com/cuda-toolkit
   Options: CUDA_CXX, CUDA_ARCH, CUDA_OPT, CUDA_LIB.
   Versions: CUDA >= 9.1, older versions may work too.
 
+- AMGX (optional), used when MFEM_USE_AMGX = YES.
+  URL: https://github.com/NVIDIA/AMGX
+  Options: AMGX_OPT, AMGX_LIB.
+  Versions: AMGX >= 2.1, older versions may work too.
+
 - HIP (optional), used when MFEM_USE_HIP = YES.
   URL: https://rocm.github.io/ROCmInstall.html
   Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB.
@@ -686,13 +668,13 @@ The specific libraries and their options are:
 - OCCA (optional), used when MFEM_USE_OCCA = YES.
   URL: https://libocca.org
   Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.
-  Versions: OCCA >= 1.1.0.
+  Versions: OCCA >= 1.0.9.
 
 - libCEED (optional), used when MFEM_USE_CEED = YES.
   URL: https://github.com/CEED/libCEED
        https://ceed.exascaleproject.org/libceed
   Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.7.
+  Versions: libCEED > 0.6, git-hash bdfed75.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
   Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
@@ -835,8 +817,8 @@ MFEM_USE_MESQUITE
 MFEM_USE_SUITESPARSE
 MFEM_USE_SUPERLU
 MFEM_USE_STRUMPACK
-MFEM_USE_GINKGO
 MFEM_USE_AMGX
+MFEM_USE_GINKGO
 MFEM_USE_GNUTLS
 MFEM_USE_NETCDF
 MFEM_USE_MPFR
@@ -885,12 +867,11 @@ The CMake build system adds auto-detection for the following packages/libraries:
 
  - HYPRE
  - METIS - The option MFEM_USE_METIS_5 is auto-detected.
- - ParMETIS
  - MESQUITE
  - SuiteSparse
  - SuperLUDist, STRUMPACK
+ - ParMETIS
  - Ginkgo
- - AMGX
  - GNUTLS - Extends the built-in CMake support, to search GNUTLS_DIR as well.
  - NETCDF
  - MPFR

NOTICE

@@ -25,8 +25,7 @@ This project distributes the sources of several external software products with
 their own respective licenses which can be found in their code and attached
 license files. These software products and their licenses are as follows:
 
-* AmgXWrapper (linalg/amgxsolver.{hpp,cpp}) -- MIT license
-* Catch++ (tests/unit/catch.hpp) -- Boost 1.0 license
 * Gecko (general/gecko.{cpp,hpp}) -- BSD 3-clause license
 * Picojson (fem/picojson.h) -- Custom 2-clause license
+* Catch++  (tests/unit/catch.hpp) -- Boost 1.0 license
 * Zstr (general/zstr.hpp) -- MIT license

config/cmake/MFEMConfig.cmake.in

@@ -34,7 +34,6 @@ set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
-set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
 set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)

config/cmake/config.hpp.in

@@ -74,9 +74,6 @@
 // [Deprecated] Enable experimental OpenMP support. Requires MFEM_THREAD_SAFE.
 #cmakedefine MFEM_USE_LEGACY_OPENMP
 
-// Internal MFEM option: enable group/batch allocation for some small objects.
-#cmakedefine MFEM_USE_MEMALLOC
-
 // Enable MFEM functionality based on the Mesquite library.
 #cmakedefine MFEM_USE_MESQUITE
 
@@ -89,12 +86,12 @@
 // Enable MFEM functionality based on the STRUMPACK library.
 #cmakedefine MFEM_USE_STRUMPACK
 
+// Internal MFEM option: enable group/batch allocation for some small objects.
+#cmakedefine MFEM_USE_MEMALLOC
+
 // Enable functionality based on the Ginkgo library
 #cmakedefine MFEM_USE_GINKGO
 
-// Enable MFEM functionality based on the AmgX library
-#cmakedefine MFEM_USE_AMGX
-
 // Enable MFEM functionality based on the GnuTLS library
 #cmakedefine MFEM_USE_GNUTLS
 
@@ -129,6 +126,9 @@
 // Requires a CUDA compiler (nvcc).
 #cmakedefine MFEM_USE_CUDA
 
+// Enable MFEM functionality based on the AMGX library
+#cmakedefine MFEM_USE_AMGX
+
 // Enable MFEM functionality based on the RAJA library
 #cmakedefine MFEM_USE_RAJA
 
@@ -159,7 +159,4 @@
 // library.
 #cmakedefine MFEM_USE_SIMMETRIX
 
-// Enable interface to the MKL CPardiso library.
-#cmakedefine MFEM_USE_MKL_CPARDISO
-
 #endif // MFEM_CONFIG_HEADER

config/cmake/modules/FindAMGX.cmake

@@ -1,20 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - AMGX_FOUND
-#   - AMGX_LIBRARIES
-#   - AMGX_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-set(AMGX_REQUIRED_LIBRARIES cusparse cusolver cublas nvToolsExt)
-mfem_find_package(AMGX AMGX AMGX_DIR "include" "amgx_c.h" "lib" "amgx"
-  "Paths to headers required by AMGX." "Libraries required by AMGX.")

config/cmake/modules/FindMKL_CPARDISO.cmake

@@ -1,32 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - MKL_CPARDISO_FOUND
-#   - MKL_CPARDISO_LIBRARIES
-#   - MKL_CPARDISO_INCLUDE_DIRS
-
-if(NOT MKL_MPI_WRAPPER_LIB)
-  message(FATAL_ERROR "MKL CPardiso enabled but no MKL MPI Wrapper lib specified")
-endif()
-
-if(NOT MKL_LIBRARY_DIR)
-  message(WARNING "Using default MKL library path. Double check the variable MKL_LIBRARY_DIR")
-  set(MKL_LIBRARY_DIR "lib")
-endif()
-
-include(MfemCmakeUtilities)
-mfem_find_package(MKL_CPARDISO MKL_CPARDISO
-    MKL_CPARDISO_DIR "include" mkl_cluster_sparse_solver.h ${MKL_LIBRARY_DIR} mkl_core
-  "Paths to headers required by MKL CPardiso." "Libraries required by MKL CPARDISO."
-  ADD_COMPONENT MKL_LP64 "include" "" ${MKL_LIBRARY_DIR} mkl_intel_lp64
-  ADD_COMPONENT MKL_SEQUENTIAL "include" "" ${MKL_LIBRARY_DIR} mkl_sequential
-  ADD_COMPONENT MKL_MPI_WRAPPER "include" "" ${MKL_LIBRARY_DIR} ${MKL_MPI_WRAPPER_LIB})

config/cmake/modules/FindSUNDIALS.cmake

@@ -20,14 +20,10 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
   "Paths to headers required by SUNDIALS." "Libraries required by SUNDIALS."
   ADD_COMPONENT NVector_Serial
     "include" nvector/nvector_serial.h "lib" sundials_nvecserial
-  ADD_COMPONENT NVector_Cuda
-    "include" nvector/nvector_cuda.h "lib" sundials_nveccuda
-  ADD_COMPONENT NVector_ParHyp
-    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
   ADD_COMPONENT NVector_Parallel
     "include" nvector/nvector_parallel.h "lib" sundials_nvecparallel
-  ADD_COMPONENT NVector_MPIPlusX
-    "include" nvector/nvector_mpiplusx.h "lib" sundials_nvecmpiplusx
+  ADD_COMPONENT NVector_ParHyp
+    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
   ADD_COMPONENT CVODE "include" cvode/cvode.h "lib" sundials_cvode
   ADD_COMPONENT CVODES "include" cvodes/cvodes.h "lib" sundials_cvodes
   ADD_COMPONENT ARKODE "include" arkode/arkode.h "lib" sundials_arkode

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -738,11 +738,10 @@ function(mfem_export_mk_files)
       MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
       MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
       MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
-      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX
-      MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
-      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI
-      MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD
-      MFEM_USE_ADIOS2)
+      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GNUTLS
+      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE
+      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA
+      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2)
   foreach(var ${CONFIG_MK_BOOL_VARS})
     if (${var})
       set(${var} YES)

config/config.hpp

@@ -45,9 +45,6 @@
 #ifdef MFEM_USE_STRUMPACK
 #error Building with STRUMPACK (MFEM_USE_STRUMPACK=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
-#ifdef MFEM_USE_MKL_CPARDISO
-#error Building with MKL CPARDISO (MFEM_USE_MKL_CPARDISO=YES) requires MPI (MFEM_USE_MPI=YES)
-#endif
 #ifdef MFEM_USE_PETSC
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif

config/config.hpp.in

@@ -93,7 +93,6 @@
 
 // Enable MFEM functionality based on the SuperLU library.
 // #define MFEM_USE_SUPERLU
-// #define MFEM_USE_SUPERLU5
 
 // Enable MFEM functionality based on the STRUMPACK library.
 // #define MFEM_USE_STRUMPACK
@@ -101,9 +100,6 @@
 // Enable MFEM features based on the Ginkgo library
 // #define MFEM_USE_GINKGO
 
-// Enable MFEM functionality based on the AmgX library.
-// #define MFEM_USE_AMGX
-
 // Enable secure socket streams based on the GNUTLS library
 // #define MFEM_USE_GNUTLS
 
@@ -141,6 +137,9 @@
 // Requires a CUDA compiler (nvcc).
 // #define MFEM_USE_CUDA
 
+// Enable MFEM functionality based on the AMGX library.
+// #define MFEM_USE_AMGX
+
 // Build the AMD GPU/HIP-enabled version of the MFEM library.
 // Requires a HIP compiler (hipcc).
 // #define MFEM_USE_HIP
@@ -167,7 +166,4 @@
 // library.
 // #define MFEM_USE_SIMMETRIX
 
-// Enable interface to the MKL CPardiso library.
-// #define MFEM_USE_MKL_CPARDISO
-
 #endif // MFEM_CONFIG_HEADER

config/config.mk.in

@@ -32,10 +32,8 @@ MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
 MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
-MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
 MFEM_USE_STRUMPACK     = @MFEM_USE_STRUMPACK@
 MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
-MFEM_USE_AMGX          = @MFEM_USE_AMGX@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
 MFEM_USE_PETSC         = @MFEM_USE_PETSC@
@@ -47,6 +45,7 @@ MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_HIOP          = @MFEM_USE_HIOP@
 MFEM_USE_GSLIB         = @MFEM_USE_GSLIB@
 MFEM_USE_CUDA          = @MFEM_USE_CUDA@
+MFEM_USE_AMGX          = @MFEM_USE_AMGX@
 MFEM_USE_HIP           = @MFEM_USE_HIP@
 MFEM_USE_RAJA          = @MFEM_USE_RAJA@
 MFEM_USE_OCCA          = @MFEM_USE_OCCA@
@@ -54,7 +53,6 @@ MFEM_USE_CEED          = @MFEM_USE_CEED@
 MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
-MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@
 
 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@

config/defaults.cmake

@@ -33,10 +33,8 @@ option(MFEM_USE_SUNDIALS "Enable SUNDIALS usage" OFF)
 option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
 option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
-option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
 option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
-option(MFEM_USE_AMGX "Enable AmgX usage" OFF)
 option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
 option(MFEM_USE_GSLIB "Enable GSLIB usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
@@ -54,7 +52,6 @@ option(MFEM_USE_CEED "Enable CEED" OFF)
 option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
 option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
-option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)
 
 set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")
 
@@ -86,12 +83,17 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
+if (MFEM_USE_CUDA)
+   # This is only necessary when hypre is built with cuda:
+   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
+       "Libraries that HYPRE depends on.")
+endif()
 
 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")
 
 set(LIBUNWIND_DIR "" CACHE PATH "Path to Libunwind.")
 
-# For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
+# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
 # and modify cmake variables for hypre for sundials
 set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
     "Path to the SUNDIALS library.")
@@ -112,7 +114,7 @@ set(ParMETIS_DIR "${MFEM_DIR}/../parmetis-4.0.3" CACHE PATH
 set(ParMETIS_REQUIRED_PACKAGES "METIS" CACHE STRING
     "Additional packages required by ParMETIS.")
 
-set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_6.3.1" CACHE PATH
+set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_5.1.0" CACHE PATH
     "Path to the SuperLU_DIST library.")
 # SuperLU_DIST may also depend on "OpenMP", depending on how it was compiled.
 set(SuperLUDist_REQUIRED_PACKAGES "MPI" "BLAS" "ParMETIS" CACHE STRING
@@ -148,8 +150,6 @@ set(ScaLAPACK_TARGET_NAMES scalapack)
 
 set(Ginkgo_DIR "${MFEM_DIR}/../ginkgo" CACHE PATH "Path to the Ginkgo library.")
 
-set(AMGX_DIR "${MFEM_DIR}/../amgx" CACHE PATH "Path to AmgX")
-
 set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")
 
 set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")
@@ -185,12 +185,9 @@ set(HIOP_DIR "${MFEM_DIR}/../hiop/install" CACHE STRING
 set(HIOP_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
     "Packages that HiOp depends on.")
 
-set(MKL_CPARDISO_DIR "" CACHE STRING "MKL installation path.")
-set(MKL_MPI_WRAPPER_LIB "mkl_blacs_mpich_lp64" CACHE STRING "MKL MPI wrapper library")
-set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")
-
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
+set(AMGX_DIR "${MFEM_DIR}/../amgx" CACHE PATH "Path to AMGX")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
 

config/defaults.mk

@@ -120,10 +120,8 @@ MFEM_USE_SUNDIALS      = NO
 MFEM_USE_MESQUITE      = NO
 MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
-MFEM_USE_SUPERLU5      = NO
 MFEM_USE_STRUMPACK     = NO
 MFEM_USE_GINKGO        = NO
-MFEM_USE_AMGX          = NO
 MFEM_USE_GNUTLS        = NO
 MFEM_USE_NETCDF        = NO
 MFEM_USE_PETSC         = NO
@@ -135,6 +133,7 @@ MFEM_USE_PUMI          = NO
 MFEM_USE_HIOP          = NO
 MFEM_USE_GSLIB         = NO
 MFEM_USE_CUDA          = NO
+MFEM_USE_AMGX          = NO
 MFEM_USE_HIP           = NO
 MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
@@ -142,7 +141,6 @@ MFEM_USE_CEED          = NO
 MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
-MFEM_USE_MKL_CPARDISO  = NO
 
 # Compile and link options for zlib.
 ZLIB_DIR =
@@ -157,6 +155,10 @@ LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
 HYPRE_DIR = @MFEM_DIR@/../hypre/src/hypre
 HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
+ifeq (YES,$(MFEM_USE_CUDA))
+   # This is only necessary when hypre is built with cuda:
+   HYPRE_LIB += -lcusparse -lcurand
+endif
 
 # METIS library configuration
 ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK),NONO)
@@ -192,19 +194,15 @@ OPENMP_LIB =
 POSIX_CLOCKS_LIB = -lrt
 
 # SUNDIALS library configuration
-# For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
+# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
 # and modify cmake variables for hypre for sundials
-SUNDIALS_DIR    = @MFEM_DIR@/../sundials-5.0.0/instdir
-SUNDIALS_OPT    = -I$(SUNDIALS_DIR)/include
-SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*)
-SUNDIALS_LIB    = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\
+SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
+SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
+SUNDIALS_LIB = -Wl,-rpath,$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib64\
  -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol
 
 ifeq ($(MFEM_USE_MPI),YES)
-   SUNDIALS_LIB += -lsundials_nvecparallel -lsundials_nvecmpiplusx
-endif
-ifeq ($(MFEM_USE_CUDA),YES)
-   SUNDIALS_LIB += -lsundials_nveccuda
+   SUNDIALS_LIB += -lsundials_nvecparhyp -lsundials_nvecparallel
 endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES
@@ -223,15 +221,9 @@ SUITESPARSE_LIB = -Wl,-rpath,$(SUITESPARSE_DIR)/lib -L$(SUITESPARSE_DIR)/lib\
  -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB)
 
 # SuperLU library configuration
-ifeq ($(MFEM_USE_SUPERLU5),YES)
-   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0
-   SUPERLU_OPT = -I$(SUPERLU_DIR)/include
-   SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib -lsuperlu_dist_5.1.0   
-else
-   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_6.3.1
-   SUPERLU_OPT = -I$(SUPERLU_DIR)/include
-   SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64 -lsuperlu_dist -lblas
-endif
+SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0
+SUPERLU_OPT = -I$(SUPERLU_DIR)/SRC
+SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib -lsuperlu_dist_5.1.0
 
 # SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in
 # STRUMPACK >= v2.2.0)
@@ -264,13 +256,7 @@ STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\
 # Ginkgo library configuration (currently not needed)
 GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
 GINKGO_OPT = -isystem $(GINKGO_DIR)/include
-GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_DIR)/lib -L$(GINKGO_DIR)/lib -lginkgo\
- -lginkgo_omp -lginkgo_cuda -lginkgo_reference
-
-# AmgX library configuration
-AMGX_DIR = @MFEM_DIR@/../amgx
-AMGX_OPT = -I$(AMGX_DIR)/include
-AMGX_LIB = -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx
+GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_DIR)/lib -L$(GINKGO_DIR)/lib -lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference
 
 # GnuTLS library configuration
 GNUTLS_OPT =
@@ -391,14 +377,10 @@ UMPIRE_DIR = @MFEM_DIR@/../umpire
 UMPIRE_OPT = -I$(UMPIRE_DIR)/include
 UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire
 
-# MKL CPardiso library configuration
-MKL_CPARDISO_DIR ?=
-MKL_MPI_WRAPPER ?= mkl_blacs_mpich_lp64
-MKL_LIBRARY_SUBDIR ?= lib
-MKL_CPARDISO_OPT = -I$(MKL_CPARDISO_DIR)/include
-MKL_CPARDISO_LIB = -Wl,-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
-                   -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
-                   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
+# AMGX library configuration
+AMGX_DIR = @MFEM_DIR@/../amgx
+AMGX_OPT = -I$(AMGX_DIR)/include
+AMGX_LIB = -L$(CUDA_HOME)/lib64 -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx -Xlinker=-rpath=$(CUDA_HOME)/lib64
 
 # If YES, enable some informational messages
 VERBOSE = NO

examples/CMakeLists.txt

@@ -127,11 +127,6 @@ if (MFEM_USE_GINKGO)
   add_subdirectory(ginkgo)
 endif()
 
-# Include the examples/amgx directory if AmgX is enabled
-if (MFEM_USE_AMGX)
-  add_subdirectory(amgx)
-endif()
-
 # Include the examples/petsc directory if PETSc is enabled.
 if (MFEM_USE_PETSC)
   add_subdirectory(petsc)
@@ -145,3 +140,4 @@ endif()
 if (MFEM_USE_HIOP)
   add_subdirectory(hiop)
 endif()
+

examples/amgx/CMakeLists.txt

@@ -1,79 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-set(AMGX_EXAMPLES_SRCS)
-list(APPEND AMGX_EXAMPLES_SRCS
-  ex1.cpp
-  )
-
-if (MFEM_USE_MPI)
-  list(APPEND AMGX_EXAMPLES_SRCS
-    ex1p.cpp
-    )
-endif()
-
-set(AMGX_JSON_FILES amg_pcg.json multi_gs.json precon.json)
-
-# Include the source directory where mfem.hpp and mfem-performance.hpp are.
-include_directories(BEFORE ${PROJECT_BINARY_DIR})
-
-# Add targets to copy *.json files from the source directory
-foreach(JSON_FILE ${AMGX_JSON_FILES})
-  add_custom_command(OUTPUT ${JSON_FILE}
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different
-    ${CMAKE_CURRENT_SOURCE_DIR}/${JSON_FILE} ${JSON_FILE}
-    COMMENT "copy ${JSON_FILE}")
-endforeach()
-add_custom_target(copy_amgx_json_files DEPENDS ${AMGX_JSON_FILES}
-  COMMENT "Copying AMGX example json files ...")
-
-# Add "test_amgx" target, see below.
-add_custom_target(test_amgx
-  ${CMAKE_CTEST_COMMAND} -R amgx USES_TERMINAL)
-
-# Add one executable per cpp file, adding "amgx_" as prefix. Sets
-# "copy_amgx_json_files" as a prerequisite for the given examples. Also, sets
-# "test_amgx" as a target that depends on the given examples.
-set(PFX amgx_)
-add_mfem_examples(AMGX_EXAMPLES_SRCS ${PFX} copy_amgx_json_files test_amgx)
-
-# Testing.
-# The AMGX tests can be run separately using the target "test_amgx"
-# which builds the examples and runs:
-#   ctest -R amgx
-
-# Command line options for the tests.
-# Example 1/1p:
-set(EX1_TEST_OPTS)
-set(EX1P_TEST_OPTS)
-
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${AMGX_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
-
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()

examples/amgx/README

@@ -1,18 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-This directory contains modifications of the example codes that illustrate the
-use of MFEM features based on NVIDIA's multigrid library AmgX.
-
-To build these examples, make sure that MFEM is configured with the option
-"MFEM_USE_AMGX = YES", see the top-level INSTALL file for details (version
-2.1 of AmgX is recommended, though older versions may work too.
-
-We recommend comparing the original example codes with the corresponding files
-in the current directory.

examples/amgx/amg_pcg.json

@@ -17,7 +17,7 @@
             "interpolator": "D2",
             "max_row_sum" : 0.9,
             "strength_threshold" : 0.25,
-            "max_iters": 2,
+            "max_iters": 1,
             "monitor_residual": 0,
             "store_res_history": 0,
             "scope": "amg",

examples/amgx/ex1.cpp

@@ -1,15 +1,50 @@
 //                                MFEM Example 1
-//                              AmgX Modification
 //
 // Compile with: make ex1
 //
+// Sample runs:  ex1 -m ../../data/square-disc.mesh
+//               ex1 -m ../../data/star.mesh
+//               ex1 -m ../../data/star-mixed.mesh
+//               ex1 -m ../../data/escher.mesh
+//               ex1 -m ../../data/fichera.mesh
+//               ex1 -m ../../data/fichera-mixed.mesh
+//               ex1 -m ../../data/toroid-wedge.mesh
+//               ex1 -m ../../data/periodic-annulus-sector.msh
+//               ex1 -m ../../data/periodic-torus-sector.msh
+//               ex1 -m ../../data/square-disc-p2.vtk -o 2
+//               ex1 -m ../../data/square-disc-p3.mesh -o 3
+//               ex1 -m ../../data/square-disc-nurbs.mesh -o -1
+//               ex1 -m ../../data/star-mixed-p2.mesh -o 2
+//               ex1 -m ../../data/disc-nurbs.mesh -o -1
+//               ex1 -m ../../data/pipe-nurbs.mesh -o -1
+//               ex1 -m ../../data/fichera-mixed-p2.mesh -o 2
+//               ex1 -m ../../data/star-surf.mesh
+//               ex1 -m ../../data/square-disc-surf.mesh
+//               ex1 -m ../../data/inline-segment.mesh
+//               ex1 -m ../../data/amr-quad.mesh
+//               ex1 -m ../../data/amr-hex.mesh
+//               ex1 -m ../../data/fichera-amr.mesh
+//               ex1 -m ../../data/mobius-strip.mesh
+//               ex1 -m ../../data/mobius-strip.mesh -o -1 -sc
+//
+// Device sample runs:
+//               ex1 -pa -d cuda
+//               ex1 -pa -d raja-cuda
+//               ex1 -pa -d occa-cuda
+//               ex1 -pa -d raja-omp
+//               ex1 -pa -d occa-omp
+//               ex1 -pa -d ceed-cpu
+//             * ex1 -pa -d ceed-cuda
+//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
+//               ex1 -m ../../data/beam-hex.mesh -pa -d cuda
+//               ex1 -m ../../data/beam-tet.mesh -pa -d ceed-cpu
+//               ex1 -m ../../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
+//
 // AmgX sample runs:
-//               ex1
-//               ex1 -d cuda
-//               ex1 --amgx-file multi_gs.json --amgx-solver
-//               ex1 --amgx-file precon.json --amgx-preconditioner
-//               ex1 --amgx-file multi_gs.json --amgx-solver -d cuda
-//               ex1 --amgx-file precon.json --amgx-preconditioner -d cuda
+//               ./ex1 --amgx-file multi_gs.json --amgx-solver
+//               ./ex1 --amgx-file precon.json --amgx-preconditioner
+//               ./ex1 --amgx-file multi_gs.json --amgx-solver -d cuda
+//               ./ex1 --amgx-file precon.json --amgx-preconditioner -d cuda
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
@@ -42,9 +77,8 @@ int main(int argc, char *argv[])
    bool pa = false;
    const char *device_config = "cpu";
    bool visualization = true;
-   bool amgx_lib = true;
    bool amgx_solver = true;
-   const char* amgx_json_file = ""; // JSON file for AmgX
+   const char* amgx_json_file = ""; // jason file for amgx
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -56,12 +90,11 @@ int main(int argc, char *argv[])
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&amgx_lib, "-amgx", "--amgx-lib", "-no-amgx",
-                  "--no-amgx-lib", "Use AmgX in example.");
    args.AddOption(&amgx_json_file, "--amgx-file", "--amgx-file",
                   "AMGX solver config file (overrides --amgx-solver, --amgx-verbose)");
    args.AddOption(&amgx_solver, "--amgx-solver", "--amgx-solver",
-                  "--amgx-preconditioner", "--amgx-preconditioner",
+                  "--amgx-preconditioner",
+                  "--amgx-preconditioner",
                   "Configure AMGX as solver or preconditioner.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
@@ -76,6 +109,9 @@ int main(int argc, char *argv[])
    }
    args.PrintOptions(cout);
 
+   MFEM_VERIFY(!pa && strcmp(amgx_json_file,"") != 0,
+               "An AmgX json file is needed for this example \n");
+
    // 2. Enable hardware devices such as GPUs, and programming models such as
    //    CUDA, OCCA, RAJA and OpenMP based on command line options.
    Device device(device_config);
@@ -172,29 +208,10 @@ int main(int argc, char *argv[])
    cout << "Size of linear system: " << A->Height() << endl;
 
    // 11. Solve the linear system A X = B.
-   if (pa)
-   {
-      // Jacobi preconditioning in partial assembly mode
-      if (UsesTensorBasis(fespace))
-      {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
-         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-      }
-      else
-      {
-         CG(*A, B, X, 1, 400, 1e-12, 0.0);
-      }
-   }
-   else if (amgx_lib && strcmp(amgx_json_file,"") == 0)
-   {
-      bool amgx_verbose = false;
-      AmgXSolver amgx(AmgXSolver::PRECONDITIONER, amgx_verbose);
-      amgx.SetOperator(*A.As<SparseMatrix>());
-      PCG(*A, amgx, B, X, 1, 200, 1e-12, 0.0);
-   }
-   else if (amgx_lib && strcmp(amgx_json_file,"") != 0)
+   if (!pa)
    {
       AmgXSolver amgx;
+
       amgx.ReadParameters(amgx_json_file, AmgXSolver::EXTERNAL);
       amgx.InitSerial();
       amgx.SetOperator(*A.As<SparseMatrix>());
@@ -207,20 +224,19 @@ int main(int argc, char *argv[])
       {
          PCG(*A.As<SparseMatrix>(), amgx, B, X, 3, 40, 1e-12, 0.0);
       }
+
    }
-   else
+   else // Jacobi preconditioning in partial assembly mode
    {
-#ifndef MFEM_USE_SUITESPARSE
-      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-      GSSmoother M((SparseMatrix&)(*A));
-      PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
-#else
-      // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-      UMFPackSolver umf_solver;
-      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver.SetOperator(*A);
-      umf_solver.Mult(B, X);
-#endif
+      if (UsesTensorBasis(fespace))
+      {
+         OperatorJacobiSmoother M(a, ess_tdof_list);
+         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
+      }
+      else
+      {
+         CG(*A, B, X, 1, 400, 1e-12, 0.0);
+      }
    }
 
    // 12. Recover the solution as a finite element grid function.

examples/amgx/ex1p.cpp

@@ -1,13 +1,45 @@
 //                       MFEM Example 1 - Parallel Version
-//                              AmgX Modification
 //
 // Compile with: make ex1p
 //
+// Sample runs:  mpirun -np 4 ex1p -m ../../data/square-disc.mesh
+//               mpirun -np 4 ex1p -m ../../data/star.mesh
+//               mpirun -np 4 ex1p -m ../../data/star-mixed.mesh
+//               mpirun -np 4 ex1p -m ../../data/escher.mesh
+//               mpirun -np 4 ex1p -m ../../data/fichera.mesh
+//               mpirun -np 4 ex1p -m ../../data/fichera-mixed.mesh
+//               mpirun -np 4 ex1p -m ../../data/toroid-wedge.mesh
+//               mpirun -np 4 ex1p -m ../../data/periodic-annulus-sector.msh
+//               mpirun -np 4 ex1p -m ../../data/periodic-torus-sector.msh
+//               mpirun -np 4 ex1p -m ../../data/square-disc-p2.vtk -o 2
+//               mpirun -np 4 ex1p -m ../../data/square-disc-p3.mesh -o 3
+//               mpirun -np 4 ex1p -m ../../data/square-disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../../data/star-mixed-p2.mesh -o 2
+//               mpirun -np 4 ex1p -m ../../data/disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../../data/pipe-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../../data/ball-nurbs.mesh -o 2
+//               mpirun -np 4 ex1p -m ../../data/fichera-mixed-p2.mesh -o 2
+//               mpirun -np 4 ex1p -m ../../data/star-surf.mesh
+//               mpirun -np 4 ex1p -m ../../data/square-disc-surf.mesh
+//               mpirun -np 4 ex1p -m ../../data/inline-segment.mesh
+//               mpirun -np 4 ex1p -m ../../data/amr-quad.mesh
+//               mpirun -np 4 ex1p -m ../../data/amr-hex.mesh
+//               mpirun -np 4 ex1p -m ../../data/mobius-strip.mesh
+//               mpirun -np 4 ex1p -m ../../data/mobius-strip.mesh -o -1 -sc
+//
+// Device sample runs:
+//               mpirun -np 4 ex1p -pa -d cuda
+//               mpirun -np 4 ex1p -pa -d occa-cuda
+//               mpirun -np 4 ex1p -pa -d raja-omp
+//               mpirun -np 4 ex1p -pa -d ceed-cpu
+//             * mpirun -np 4 ex1p -pa -d ceed-cuda
+//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
+//               mpirun -np 4 ex1p -m ../../data/beam-tet.mesh -pa -d ceed-cpu
+//
 // AmgX sample runs:
-//               mpirun -np 4 ex1p
-//               mpirun -np 4 ex1p -d cuda
-//               mpirun -np 10 ex1p --amgx-file amg_pcg.json --amgx-mpi-teams
-//               mpirun -np 4 ex1p --amgx-file amg_pcg.json
+//
+//               mpirun -n 40 ./ex1p --amgx-file amg_pcg.json
+//               lrun -n 4 ./ex1p --amgx-file amg_pcg.json --amgx-mpi-gpu-exclusive
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
@@ -46,9 +78,8 @@ int main(int argc, char *argv[])
    bool pa = false;
    const char *device_config = "cpu";
    bool visualization = true;
-   bool amgx_lib = true;
-   bool amgx_mpi_teams = false;
-   const char* amgx_json_file = ""; // JSON file for AmgX
+   bool amgx_mpi_teams = true;
+   const char* amgx_json_file = ""; // jason file for amgx
    int ndevices = 1;
 
    OptionsParser args(argc, argv);
@@ -61,20 +92,17 @@ int main(int argc, char *argv[])
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&amgx_lib, "-amgx", "--amgx-lib", "-no-amgx",
-                  "--no-amgx-lib", "Use AmgX in example.");
    args.AddOption(&amgx_json_file, "--amgx-file", "--amgx-file",
                   "AMGX solver config file (overrides --amgx-solver, --amgx-verbose)");
    args.AddOption(&amgx_mpi_teams, "--amgx-mpi-teams", "--amgx-mpi-teams",
                   "--amgx-mpi-gpu-exclusive", "--amgx-mpi-gpu-exclusive",
-                  "Create MPI teams when using AmgX to load balance between ranks and GPUs.");
+                  "Create MPI teams when using AMGX.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
-   args.AddOption(&ndevices, "-nd","--gpus-per-node-in-teams-mode",
-                  "Number of GPU devices per node (Only used if amgx_mpi_teams is true).");
+   args.AddOption(&ndevices, "-nd","--nd","Number of GPU devices.");
 
    args.Parse();
    if (!args.Good())
@@ -89,6 +117,9 @@ int main(int argc, char *argv[])
    if (myid == 0)
    {
       args.PrintOptions(cout);
+
+      MFEM_VERIFY(!pa == true && strcmp(amgx_json_file,"") != 0,
+                  "An AmgX json file is needed for this example \n");
    }
 
    // 3. Enable hardware devices such as GPUs, and programming models such as
@@ -205,11 +236,34 @@ int main(int argc, char *argv[])
 
    // 13. Solve the linear system A X = B.
    //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-   //     * If AmgX is available solve using amg preconditioner.
    //     * With partial assembly, use Jacobi smoothing, for now.
    Solver *prec = NULL;
-   if (pa)
+   if (!pa)
    {
+
+      AmgXSolver amgx;
+
+      amgx.ReadParameters(amgx_json_file, AmgXSolver::EXTERNAL);
+
+      if (amgx_mpi_teams)
+      {
+         //Forms MPI teams to load balance between mpi ranks and gpus
+         amgx.InitMPITeams(MPI_COMM_WORLD, ndevices);
+      }
+      else
+      {
+         //Assumes MPI == number of devices
+         amgx.InitExclusiveGPU(MPI_COMM_WORLD);
+      }
+
+      amgx.SetOperator(*A.As<HypreParMatrix>());
+
+      amgx.Mult(B, X);
+
+   }
+   else
+   {
+
       if (UsesTensorBasis(fespace))
       {
          prec = new OperatorJacobiSmoother(a, ess_tdof_list);
@@ -223,60 +277,7 @@ int main(int argc, char *argv[])
       cg.SetOperator(*A);
       cg.Mult(B, X);
       delete prec;
-   }
-   else if (amgx_lib && strcmp(amgx_json_file,"") == 0)
-   {
-      MFEM_VERIFY(!amgx_mpi_teams,
-                  "Please add JSON file to try AmgX with MPI teams mode");
 
-      bool amgx_verbose = false;
-      prec = new AmgXSolver(MPI_COMM_WORLD, AmgXSolver::PRECONDITIONER,
-                            amgx_verbose);
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(1);
-      if (prec) { cg.SetPreconditioner(*prec); }
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
-
-   }
-   else if (amgx_lib && strcmp(amgx_json_file,"") != 0)
-   {
-      AmgXSolver amgx;
-      amgx.ReadParameters(amgx_json_file, AmgXSolver::EXTERNAL);
-
-      if (amgx_mpi_teams)
-      {
-         // Forms MPI teams to load balance between MPI ranks and GPUs
-         amgx.InitMPITeams(MPI_COMM_WORLD, ndevices);
-      }
-      else
-      {
-         // Assumes each MPI rank is paired with a GPU
-         amgx.InitExclusiveGPU(MPI_COMM_WORLD);
-      }
-
-      amgx.SetOperator(*A.As<HypreParMatrix>());
-      amgx.Mult(B, X);
-
-      // Release MPI communicators and resources created by AmgX
-      amgx.Finalize();
-   }
-   else
-   {
-      prec = new HypreBoomerAMG;
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(1);
-      if (prec) { cg.SetPreconditioner(*prec); }
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
    }
 
    // 14. Recover the parallel grid function corresponding to X. This is the

examples/amgx/makefile

@@ -70,6 +70,3 @@ clean: clean-build
 clean-build:
 	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
 	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
-	@rm -f .logamgx refined.mesh sol.gf mesh.* sol.*

examples/ex1.cpp

@@ -35,7 +35,6 @@
 //               ex1 -pa -d occa-omp
 //               ex1 -pa -d ceed-cpu
 //             * ex1 -pa -d ceed-cuda
-//             * ex1 -pa -d ceed-hip
 //               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
 //               ex1 -m ../data/beam-hex.mesh -pa -d cuda
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
@@ -72,6 +71,7 @@ int main(int argc, char *argv[])
    bool pa = false;
    const char *device_config = "cpu";
    bool visualization = true;
+   bool amgx = false;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -83,6 +83,8 @@ int main(int argc, char *argv[])
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&amgx, "-amgx", "--amgx-precon", "-no-amgx",
+                  "--no-amgx-precon", "Use AmgX V-cycle as preconditioner for CG.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -192,7 +194,31 @@ int main(int argc, char *argv[])
    cout << "Size of linear system: " << A->Height() << endl;
 
    // 11. Solve the linear system A X = B.
-   if (!pa)
+   if (pa)
+   {
+      // Jacobi preconditioning in partial assembly mode
+      if (UsesTensorBasis(fespace))
+      {
+         OperatorJacobiSmoother M(a, ess_tdof_list);
+         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
+      }
+      else
+      {
+         CG(*A, B, X, 1, 400, 1e-12, 0.0);
+      }
+   }
+   else if (amgx)
+   {
+#if defined(MFEM_USE_AMGX)
+      bool amgx_verbose = false;
+      AmgXSolver amgx(AmgXSolver::PRECONDITIONER, amgx_verbose);
+      amgx.SetOperator(*A.As<SparseMatrix>());
+      PCG(*A, amgx, B, X, 1, 200, 1e-12, 0.0);
+#else
+      mfem_error("MFEM not configured with AMGX \n");
+#endif
+   }
+   else
    {
 #ifndef MFEM_USE_SUITESPARSE
       // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
@@ -206,18 +232,6 @@ int main(int argc, char *argv[])
       umf_solver.Mult(B, X);
 #endif
    }
-   else // Jacobi preconditioning in partial assembly mode
-   {
-      if (UsesTensorBasis(fespace))
-      {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
-         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-      }
-      else
-      {
-         CG(*A, B, X, 1, 400, 1e-12, 0.0);
-      }
-   }
 
    // 12. Recover the solution as a finite element grid function.
    a.RecoverFEMSolution(X, b, x);

examples/ex11p.cpp

@@ -72,7 +72,6 @@ int main(int argc, char *argv[])
    int seed = 75;
    bool slu_solver  = false;
    bool sp_solver = false;
-   bool cpardiso_solver = false;
    bool visualization = 1;
 
    OptionsParser args(argc, argv);
@@ -96,10 +95,6 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_STRUMPACK
    args.AddOption(&sp_solver, "-sp", "--strumpack", "-no-sp",
                   "--no-strumpack", "Use the STRUMPACK Solver.");
-#endif
-#ifdef MFEM_USE_MKL_CPARDISO
-   args.AddOption(&cpardiso_solver, "-cpardiso", "--cpardiso", "-no-cpardiso",
-                  "--no-cpardiso", "Use the MKL CPardiso Solver.");
 #endif
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
@@ -241,7 +236,7 @@ int main(int argc, char *argv[])
    //    preconditioner for A to be used within the solver. Set the matrices
    //    which define the generalized eigenproblem A x = lambda M x.
    Solver * precond = NULL;
-   if (!slu_solver && !sp_solver && !cpardiso_solver)
+   if (!slu_solver && !sp_solver)
    {
       HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
       amg->SetPrintLevel(0);
@@ -273,19 +268,10 @@ int main(int argc, char *argv[])
          strumpack->SetFromCommandLine();
          precond = strumpack;
       }
-#endif
-#ifdef MFEM_USE_MKL_CPARDISO
-      if (cpardiso_solver)
-      {
-         auto cpardiso = new CPardisoSolver(A->GetComm());
-         cpardiso->SetMatrixType(CPardisoSolver::MatType::REAL_STRUCTURE_SYMMETRIC);
-         cpardiso->SetPrintLevel(1);
-         cpardiso->SetOperator(*A);
-         precond = cpardiso;
-      }
 #endif
    }
 
+
    HypreLOBPCG * lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
    lobpcg->SetNumModes(nev);
    lobpcg->SetRandomSeed(seed);

examples/ex19.cpp

@@ -8,7 +8,6 @@
 //    ex19 -m ../data/beam-hex.mesh
 //    ex19 -m ../data/beam-tet.mesh
 //    ex19 -m ../data/beam-wedge.mesh
-//    ex19 -m ../data/beam-quad-amr.mesh
 //
 // Description:  This examples solves a quasi-static incompressible nonlinear
 //               elasticity problem of the form 0 = H(x), where H is an
@@ -97,7 +96,7 @@ protected:
    Array<FiniteElementSpace *> spaces;
 
    // Offsets for extracting block vector segments
-   Array<int> &block_trueOffsets;
+   Array<int> &block_offsets;
 
    // Jacobian for block access
    BlockOperator *jacobian;
@@ -153,7 +152,7 @@ protected:
    Coefficient &mu;
 
    // Block offsets for variable access
-   Array<int> &block_trueOffsets;
+   Array<int> &block_offsets;
 
 public:
    RubberOperator(Array<FiniteElementSpace *> &fes, Array<Array<int> *>&ess_bdr,
@@ -247,8 +246,8 @@ int main(int argc, char *argv[])
    spaces[0] = &R_space;
    spaces[1] = &W_space;
 
-   int R_size = R_space.GetTrueVSize();
-   int W_size = W_space.GetTrueVSize();
+   int R_size = R_space.GetVSize();
+   int W_size = W_space.GetVSize();
 
    // 6. Define the Dirichlet conditions (set to boundary attribute 1 and 2)
    Array<Array<int> *> ess_bdr(2);
@@ -272,13 +271,13 @@ int main(int argc, char *argv[])
    std::cout << "***********************************************************\n";
 
    // 8. Define the block structure of the solution vector (u then p)
-   Array<int> block_trueOffsets(3);
-   block_trueOffsets[0] = 0;
-   block_trueOffsets[1] = R_space.GetTrueVSize();
-   block_trueOffsets[2] = W_space.GetTrueVSize();
-   block_trueOffsets.PartialSum();
+   Array<int> block_offsets(3);
+   block_offsets[0] = 0;
+   block_offsets[1] = R_space.GetVSize();
+   block_offsets[2] = W_space.GetVSize();
+   block_offsets.PartialSum();
 
-   BlockVector xp(block_trueOffsets);
+   BlockVector xp(block_offsets);
 
    // 9. Define grid functions for the current configuration, reference
    //    configuration, final deformation, and pressure
@@ -287,8 +286,8 @@ int main(int argc, char *argv[])
    GridFunction x_def(&R_space);
    GridFunction p_gf(&W_space);
 
-   x_gf.MakeTRef(&R_space, xp.GetBlock(0), 0);
-   p_gf.MakeTRef(&W_space, xp.GetBlock(1), 0);
+   x_gf.MakeRef(&R_space, xp.GetBlock(0), 0);
+   p_gf.MakeRef(&W_space, xp.GetBlock(1), 0);
 
    VectorFunctionCoefficient deform(dim, InitialDeformation);
    VectorFunctionCoefficient refconfig(dim, ReferenceConfiguration);
@@ -297,19 +296,14 @@ int main(int argc, char *argv[])
    x_ref.ProjectCoefficient(refconfig);
    p_gf = 0.0;
 
-   x_gf.SetTrueVector();
-   p_gf.SetTrueVector();
-
    // 10. Initialize the incompressible neo-Hookean operator
-   RubberOperator oper(spaces, ess_bdr, block_trueOffsets,
+   RubberOperator oper(spaces, ess_bdr, block_offsets,
                        newton_rel_tol, newton_abs_tol, newton_iter, c_mu);
 
    // 11. Solve the Newton system
    oper.Solve(xp);
 
    // 12. Compute the final deformation
-   x_gf.SetFromTrueVector();
-   p_gf.SetFromTrueVector();
    subtract(x_gf, x_ref, x_def);
 
    // 13. Visualize the results if requested
@@ -355,7 +349,7 @@ int main(int argc, char *argv[])
 JacobianPreconditioner::JacobianPreconditioner(Array<FiniteElementSpace *> &fes,
                                                SparseMatrix &mass,
                                                Array<int> &offsets)
-   : Solver(offsets[2]), block_trueOffsets(offsets), pressure_mass(&mass)
+   : Solver(offsets[2]), block_offsets(offsets), pressure_mass(&mass)
 {
    fes.Copy(spaces);
 
@@ -387,18 +381,18 @@ JacobianPreconditioner::JacobianPreconditioner(Array<FiniteElementSpace *> &fes,
 void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
 {
    // Extract the blocks from the input and output vectors
-   Vector disp_in(k.GetData() + block_trueOffsets[0],
-                  block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_in(k.GetData() + block_trueOffsets[1],
-                  block_trueOffsets[2]-block_trueOffsets[1]);
+   Vector disp_in(k.GetData() + block_offsets[0],
+                  block_offsets[1]-block_offsets[0]);
+   Vector pres_in(k.GetData() + block_offsets[1],
+                  block_offsets[2]-block_offsets[1]);
 
-   Vector disp_out(y.GetData() + block_trueOffsets[0],
-                   block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_out(y.GetData() + block_trueOffsets[1],
-                   block_trueOffsets[2]-block_trueOffsets[1]);
+   Vector disp_out(y.GetData() + block_offsets[0],
+                   block_offsets[1]-block_offsets[0]);
+   Vector pres_out(y.GetData() + block_offsets[1],
+                   block_offsets[2]-block_offsets[1]);
 
-   Vector temp(block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector temp2(block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector temp(block_offsets[1]-block_offsets[0]);
+   Vector temp2(block_offsets[1]-block_offsets[0]);
 
    // Perform the block elimination for the preconditioner
    mass_pcg->Mult(pres_in, pres_out);
@@ -453,9 +447,9 @@ RubberOperator::RubberOperator(Array<FiniteElementSpace *> &fes,
                                double abs_tol,
                                int iter,
                                Coefficient &c_mu)
-   : Operator(fes[0]->GetTrueVSize() + fes[1]->GetTrueVSize()),
+   : Operator(fes[0]->GetVSize() + fes[1]->GetVSize()),
      newton_solver(), newton_monitor("Newton", 1),
-     j_monitor("  GMRES", 3), mu(c_mu), block_trueOffsets(offsets)
+     j_monitor("  GMRES", 3), mu(c_mu), block_offsets(offsets)
 {
    Array<Vector *> rhs(2);
    rhs = NULL; // Set all entries in the array
@@ -477,16 +471,12 @@ RubberOperator::RubberOperator(Array<FiniteElementSpace *> &fes,
    a->AddDomainIntegrator(new MassIntegrator(one));
    a->Assemble();
    a->Finalize();
-
-   OperatorPtr op;
-   Array<int> p_ess_tdofs;
-   a->FormSystemMatrix(p_ess_tdofs, op);
    pressure_mass = a->LoseMat();
    delete a;
 
    // Initialize the Jacobian preconditioner
    JacobianPreconditioner *jac_prec =
-      new JacobianPreconditioner(fes, *pressure_mass, block_trueOffsets);
+      new JacobianPreconditioner(fes, *pressure_mass, block_offsets);
    j_prec = jac_prec;
 
    // Set up the Jacobian solver

examples/ex19p.cpp

@@ -8,7 +8,6 @@
 //    mpirun -np 2 ex19p -m ../data/beam-hex.mesh
 //    mpirun -np 2 ex19p -m ../data/beam-tet.mesh
 //    mpirun -np 2 ex19p -m ../data/beam-wedge.mesh
-//    mpirun -np 2 ex19p -m ../data/beam-quad-amr.mesh
 //
 // Description:  This examples solves a quasi-static incompressible nonlinear
 //               elasticity problem of the form 0 = H(x), where H is an
@@ -197,8 +196,10 @@ void InitialDeformation(const Vector &x, Vector &y);
 int main(int argc, char *argv[])
 {
    // 1. Initialize MPI
-   MPI_Session mpi;
-   const int myid = mpi.WorldRank();
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
 
    // 2. Parse command-line options
    const char *mesh_file = "../data/beam-tet.mesh";
@@ -238,6 +239,7 @@ int main(int argc, char *argv[])
       {
          args.PrintUsage(cout);
       }
+      MPI_Finalize();
       return 1;
    }
    if (myid == 0)
@@ -397,6 +399,8 @@ int main(int argc, char *argv[])
    // 19. Free the used memory
    delete pmesh;
 
+   MPI_Finalize();
+
    return 0;
 }
 
@@ -470,11 +474,7 @@ void JacobianPreconditioner::SetOperator(const Operator &op)
    {
       HypreBoomerAMG *stiff_prec_amg = new HypreBoomerAMG();
       stiff_prec_amg->SetPrintLevel(0);
-
-      if (!spaces[0]->GetParMesh()->Nonconforming())
-      {
-         stiff_prec_amg->SetElasticityOptions(spaces[0]);
-      }
+      stiff_prec_amg->SetElasticityOptions(spaces[0]);
 
       stiff_prec = stiff_prec_amg;
 

examples/ex1p.cpp

@@ -33,7 +33,6 @@
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
 //             * mpirun -np 4 ex1p -pa -d ceed-cuda
-//             * mpirun -np 4 ex1p -pa -d ceed-hip
 //               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
 //               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //
@@ -74,6 +73,7 @@ int main(int argc, char *argv[])
    bool pa = false;
    const char *device_config = "cpu";
    bool visualization = true;
+   bool amgx = false;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -85,6 +85,8 @@ int main(int argc, char *argv[])
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&amgx, "-amgx", "--amgx-precon", "-no-amgx",
+                  "--no-amgx-precon", "Use AmgX V-cycle as preconditioner for CG.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -219,6 +221,7 @@ int main(int argc, char *argv[])
 
    // 13. Solve the linear system A X = B.
    //     * With full assembly, use the BoomerAMG preconditioner from hypre.
+   //     * If AmgX is available solve using amg preconditioner.
    //     * With partial assembly, use Jacobi smoothing, for now.
    Solver *prec = NULL;
    if (pa)
@@ -228,6 +231,16 @@ int main(int argc, char *argv[])
          prec = new OperatorJacobiSmoother(a, ess_tdof_list);
       }
    }
+   else if (amgx)
+   {
+#if defined(MFEM_USE_AMGX)
+      bool amgx_verbose = false;
+      prec = new AmgXSolver(MPI_COMM_WORLD, AmgXSolver::PRECONDITIONER,
+                            amgx_verbose);
+#else
+      mfem_error("MFEM not configured with AMGX \n");
+#endif
+   }
    else
    {
       prec = new HypreBoomerAMG;

examples/ex1p_helmholtz.cpp

@@ -0,0 +1,503 @@
+//                       MFEM Example 1 - Parallel Version
+//
+// Compile with: make ex1p
+//
+// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
+//               mpirun -np 4 ex1p -m ../data/star.mesh
+//               mpirun -np 4 ex1p -m ../data/star-mixed.mesh
+//               mpirun -np 4 ex1p -m ../data/escher.mesh
+//               mpirun -np 4 ex1p -m ../data/fichera.mesh
+//               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
+//               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
+//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
+//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
+//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/star-mixed-p2.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/fichera-mixed-p2.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Laplace problem
+//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
+//               Specifically, we discretize using a FE space of the specified
+//               order, or if order < 1 using an isoparametric/isogeometric
+//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of mesh refinement, finite
+//               element grid functions, as well as linear and bilinear forms
+//               corresponding to the left-hand side and right-hand side of the
+//               discrete linear system. We also cover the explicit elimination
+//               of essential boundary conditions, static condensation, and the
+//               optional connection to the GLVis tool for visualization.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+double u_exact(const Vector &x);
+double f_exact(const Vector &x);
+
+// #define FORM_DEFINITE
+#define USE_GMRES
+
+#define USE_CSL
+
+#define K2 250.0
+
+int dim;
+double kappa;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/beam-tet.mesh";
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+
+   kappa = 2.0 * M_PI;
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 10,000 elements.
+   {
+      int ref_levels =
+         (int)floor(log(100000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   {
+      int par_ref_levels = 2;
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh->UniformRefinement();
+      }
+   }
+
+   {
+      double minsize = pmesh->GetElementSize(0);
+      double maxsize = minsize;
+      for (int i=1; i<pmesh->GetNE(); ++i)
+      {
+         const double size_i = pmesh->GetElementSize(i);
+         minsize = std::min(minsize, size_i);
+         maxsize = std::max(maxsize, size_i);
+      }
+
+      cout << myid << ": Element size range: (" << minsize << ", " << maxsize << ")"
+           << endl;
+   }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+      if (myid == 0)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (1,phi_i) where phi_i are the basis functions in fespace.
+   ParLinearForm *b = new ParLinearForm(fespace);
+
+   //ConstantCoefficient bcoef(1.0);
+   FunctionCoefficient bcoef(f_exact);
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   ConstantCoefficient neg(-K2);
+   ConstantCoefficient pos(K2);
+   b->AddDomainIntegrator(new DomainLFIntegrator(bcoef));
+   b->Assemble();
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   ParGridFunction x(fespace);
+   x = 0.0;
+
+   // 10. Set up the parallel bilinear form a(.,.) on the finite element space
+   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //     domain integrator.
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   a->AddDomainIntegrator(new MassIntegrator(neg));
+
+#ifdef FORM_DEFINITE
+   ParBilinearForm *adef = new ParBilinearForm(fespace);
+   adef->AddDomainIntegrator(new DiffusionIntegrator(one));
+   adef->AddDomainIntegrator(new MassIntegrator(pos));
+
+   if (static_cond) { adef->EnableStaticCondensation(); }
+   adef->Assemble();
+
+   ParGridFunction xdef(fespace);
+   xdef = 0.0;
+
+   ParLinearForm *bdef = new ParLinearForm(fespace);
+   bdef->AddDomainIntegrator(new DomainLFIntegrator(bcoef));
+   bdef->Assemble();
+
+   HypreParMatrix Adef;
+   Vector Bdef, Xdef;
+   adef->FormLinearSystem(ess_tdof_list, xdef, *bdef, Adef, Xdef, Bdef);
+#endif
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   HypreParMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+   }
+
+   //A.Print("helmholtz");
+
+   // 12. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+   //     preconditioner from hypre.
+#ifdef FORM_DEFINITE
+   HypreSolver *amg = new HypreBoomerAMG(Adef);
+#else
+   HypreSolver *amg = new HypreBoomerAMG(A);
+#endif
+
+   const bool fullDirect = true;
+
+   if (fullDirect)
+   {
+#ifdef USE_CSL
+      Vector Bdef, Xdef;
+
+      ParBilinearForm *Mform =  new ParBilinearForm(fespace);
+      Mform->AddDomainIntegrator(new MassIntegrator(pos));
+      Mform->Assemble();
+
+      HypreParMatrix Mmat, Smat, Mcopy;
+      Mform->FormLinearSystem(ess_tdof_list, x, *b, Mmat, Xdef, Bdef);
+      Mform->FormLinearSystem(ess_tdof_list, x, *b, Mcopy, Xdef,
+                              Bdef);  // There must be a better way than creating two identical matrices.
+
+      ParBilinearForm *Sform =  new ParBilinearForm(fespace);
+      Sform->AddDomainIntegrator(new DiffusionIntegrator(one));
+      Sform->Assemble();
+
+      Sform->FormLinearSystem(ess_tdof_list, x, *b, Smat, Xdef, Bdef);
+
+      const double beta1 = 1.0;
+      const double beta2 = 1.0;
+
+      Mmat *= -beta1;
+
+      HypreParMatrix * cslRe = ParAdd(&Smat, &Mmat);
+
+      Mcopy *= beta2;
+
+      ComplexHypreParMatrix chpm(cslRe, &Mcopy, false, false);
+
+      HypreParMatrix *cSysMat = chpm.GetSystemMatrix();
+
+      Array<int> block_trueOffsets(3); // number of variables + 1
+      block_trueOffsets[0] = 0;
+      block_trueOffsets[1] = fespace->TrueVSize();
+      block_trueOffsets[2] = fespace->TrueVSize();
+      block_trueOffsets.PartialSum();
+
+      // Note that B is of true size.
+      BlockVector trueY(block_trueOffsets), trueX(block_trueOffsets),
+                  trueRhs(block_trueOffsets);
+
+      trueRhs.GetBlock(0) = B;
+      trueRhs.GetBlock(1) = 0.0;
+
+      Operator * Arow = new STRUMPACKRowLocMatrix(*cSysMat);
+
+      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+      strumpack->SetPrintFactorStatistics(true);
+      strumpack->SetPrintSolveStatistics(false);
+      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
+      // strumpack->SetSymmetricPattern(true);
+      strumpack->SetOperator(*Arow);
+      strumpack->SetFromCommandLine();
+      //Solver * precond = strumpack;
+
+      // strumpack->Mult(B, X);
+
+      BlockOperator blockDiagA(block_trueOffsets);
+      for (int i=0; i<2; ++i)
+      {
+         blockDiagA.SetDiagonalBlock(i, &A);
+      }
+
+      ProductOperator prod(&blockDiagA, strumpack, false, false);
+      //GMRESSolver *gmres = new GMRESSolver(fespace->GetComm());
+      BiCGSTABSolver *gmres = new BiCGSTABSolver(fespace->GetComm());
+
+      gmres->SetOperator(prod);
+      gmres->SetRelTol(1e-8);
+      gmres->SetMaxIter(10000);
+      gmres->SetPrintLevel(1);
+
+      gmres->Mult(trueRhs, trueY);
+      strumpack->Mult(trueY, trueX);
+
+      X = trueX.GetBlock(0);
+      double xim2 = trueX.GetBlock(1).Norml2();
+      xim2 *= xim2;
+      double sumxim2 = 0.0;
+
+      MPI_Allreduce(&xim2, &sumxim2, 1, MPI_DOUBLE, MPI_SUM, fespace->GetComm());
+
+      if (myid == 0)
+      {
+         cout << myid << ": norm of Xim " << trueX.GetBlock(1).Norml2() << ", global " <<
+              sqrt(sumxim2) << endl;
+      }
+
+      delete gmres;
+      delete strumpack;
+      delete Arow;
+#else
+      Operator * Arow = new STRUMPACKRowLocMatrix(A);
+
+      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+      strumpack->SetPrintFactorStatistics(true);
+      strumpack->SetPrintSolveStatistics(false);
+      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
+      // strumpack->SetSymmetricPattern(true);
+      strumpack->SetOperator(*Arow);
+      strumpack->SetFromCommandLine();
+      //Solver * precond = strumpack;
+
+      strumpack->Mult(B, X);
+
+      delete strumpack;
+      delete Arow;
+#endif
+   }
+   else
+   {
+#ifdef USE_GMRES
+      HypreGMRES *gmres = new HypreGMRES(A);
+      gmres->SetTol(1e-12);
+      gmres->SetMaxIter(1000);
+      gmres->SetPrintLevel(10);
+      gmres->SetPreconditioner(*amg);
+      gmres->Mult(B, X);
+      delete gmres;
+#else
+      HyprePCG *pcg = new HyprePCG(A);
+      pcg->SetTol(1e-12);
+      pcg->SetMaxIter(100);
+      pcg->SetPrintLevel(2);
+      pcg->SetPreconditioner(*amg);
+      pcg->Mult(B, X);
+#endif
+   }
+
+   /*
+   HYPRE_ParCSRMatrix* amgP = amg->Get_Restriction();
+   HypreParMatrix P0(amgP[0], false);
+   HypreParMatrix P1(amgP[1], false);
+   HypreParMatrix P2(amgP[2], false);
+   //HypreParMatrix P3(amgP[3], false);
+
+   P0.Print("P0");
+   */
+
+   // 13. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a->RecoverFEMSolution(X, *b, x);
+
+   // Compute and print the L^2 norm of the error.
+   {
+      FunctionCoefficient uex(u_exact);
+
+      double err = x.ComputeL2Error(uex);
+      double xnrm = x.ComputeL2Error(zero);
+      ParGridFunction zerogf(fespace);
+      zerogf = 0.0;
+      double normE = zerogf.ComputeL2Error(uex);
+      if (myid == 0)
+      {
+         cout << "|| E_h - E ||_{L^2} = " << err << endl;
+         cout << "|| E_h ||_{L^2} = " << xnrm << endl;
+         cout << "|| E ||_{L^2} = " << normE << endl;
+      }
+   }
+
+   // 14. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 15. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x << flush;
+   }
+
+   // 16. Free the used memory.
+   //delete pcg;
+   delete amg;
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+
+double u_exact(const Vector & x)
+{
+   double xi(x(0));
+   double yi(x(1));
+   double zi(1.0);
+
+   if (x.Size() == 3)
+   {
+      zi = x(2);
+   }
+
+   return sin(kappa*xi)*sin(kappa*yi)*sin(kappa*zi);
+}
+
+double f_exact(const Vector &x)
+{
+   double xi(x(0));
+   double yi(x(1));
+   double zi(1.0);
+
+   if (x.Size() == 3)
+   {
+      zi = x(2);
+   }
+
+   const double s = 1.0;
+
+   return ((3.0*kappa*kappa) - (s*K2)) * sin(kappa*xi)*sin(kappa*yi)*sin(
+             kappa*zi) / s;
+}

examples/ex3p.cpp

@@ -158,9 +158,9 @@ int main(int argc, char *argv[])
    //    by marking all the boundary attributes from the mesh as essential
    //    (Dirichlet) and converting them to a list of true dofs.
    Array<int> ess_tdof_list;
+   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
    if (pmesh->bdr_attributes.Size())
    {
-      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
       ess_bdr = 1;
       fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
@@ -207,18 +207,36 @@ int main(int argc, char *argv[])
    // 13. Solve the system AX=B using PCG with the AMS preconditioner from hypre
    //     (in the full assembly case) or CG with Jacobi preconditioner (in the
    //     partial assembly case).
-
-   if (pa) // Jacobi preconditioning in partial assembly mode
+   if (pa) // matrix-free auxiliary space solver with PA
    {
-      OperatorJacobiSmoother Jacobi(*a, ess_tdof_list);
+      StopWatch sw;
+      sw.Clear();
+      sw.Start();
+
+      MatrixFreeAMS *ams = new MatrixFreeAMS(*a, *A, *fespace, muinv, sigma, NULL,
+                                             ess_bdr); //, 2, 2);
+
+      //OperatorJacobiSmoother jacobi(*a, ess_tdof_list);
 
       CGSolver cg(MPI_COMM_WORLD);
       cg.SetRelTol(1e-12);
       cg.SetMaxIter(1000);
       cg.SetPrintLevel(1);
       cg.SetOperator(*A);
-      cg.SetPreconditioner(Jacobi);
+      cg.SetPreconditioner(*ams);
+      //cg.SetPreconditioner(jacobi);
+      //cg.iterative_mode = false;
+      sw.Stop();
+      cout << myid << ": MatrixFreeAMS-CG setup time " << sw.RealTime() << endl;
+
+      sw.Clear();
+      sw.Start();
+
       cg.Mult(B, X);
+
+      sw.Stop();
+      cout << myid << ": MatrixFreeAMS-CG solve time " << sw.RealTime() << endl;
+      ams->PrintTimings(myid);
    }
    else
    {

examples/ex3p_airy.cpp

@@ -0,0 +1,425 @@
+//                       MFEM Example 3 - Parallel Version
+//
+// Compile with: make ex3p
+//
+// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/escher.mesh
+//               mpirun -np 4 ex3p -m ../data/escher.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/fichera.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
+//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
+//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
+//
+// Description:  This example code solves a simple electromagnetic diffusion
+//               problem corresponding to the second order definite Maxwell
+//               equation curl curl E + E = f with boundary condition
+//               E x n = <given tangential field>. Here, we use a given exact
+//               solution E and compute the corresponding r.h.s. f.
+//               We discretize with Nedelec finite elements in 2D or 3D.
+//
+//               The example demonstrates the use of H(curl) finite element
+//               spaces with the curl-curl and the (vector finite element) mass
+//               bilinear form, as well as the computation of discretization
+//               error when the exact solution is known. Static condensation is
+//               also illustrated.
+//
+//               We recommend viewing examples 1-2 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+#define TEST_AIRY
+
+#ifdef TEST_AIRY
+#include "gsl_sf_airy.h"
+
+#define XSHIFT 0.0 // 0.25
+#endif
+
+// Exact solution, E, and r.h.s., f. See below for implementation.
+void E_exact(const Vector &, Vector &);
+void f_exact(const Vector &, Vector &);
+double freq = 1.0, kappa;
+int dim;
+
+//#define K2_VALUE 11.9
+//#define K2_VALUE 2.1
+//#define K2_VALUE 1500.9
+#define K2_VALUE 10981.41589009910  // 104.792251097584^2 or 5 GHz
+
+void test_Airy_epsilon(const Vector &x, Vector &e)
+{
+   e(0) = 1.0;
+   e(1) = 1.0;
+   e(2) = (4.0 * (x(0) + XSHIFT)) - 1.0;
+
+   e *= -K2_VALUE;
+}
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   //const char *mesh_file = "../data/beam-tet.mesh";
+   const char *mesh_file = "../data/inline-tetHalf.mesh";
+   //const char *mesh_file = "inline-tetSlab.mesh";
+   //const char *mesh_file = "../data/inline-hexHalf.mesh";
+   //const char *mesh_file = "../data/inline-tet.mesh";
+   int order = 2;
+   bool static_cond = false;
+   bool visualization = 1;
+#ifdef MFEM_USE_STRUMPACK
+   bool use_strumpack = false;
+#endif
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
+                  " solution.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+#ifdef MFEM_USE_STRUMPACK
+   args.AddOption(&use_strumpack, "-strumpack", "--strumpack-solver",
+                  "-no-strumpack", "--no-strumpack-solver",
+                  "Use STRUMPACK's double complex linear solver.");
+#endif
+
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+      cout << "Using k2 " << K2_VALUE << endl;
+   }
+   kappa = freq * M_PI;
+
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 1,000 elements.
+   {
+      int ref_levels =
+         (int)floor(log(100000./mesh->GetNE())/log(2.)/dim);
+      ref_levels = 0;
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   {
+      int par_ref_levels = 2;
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh->UniformRefinement();
+      }
+   }
+   pmesh->ReorientTetMesh();
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Nedelec finite elements of the specified order.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   long globalNE = pmesh->GetGlobalNE();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+      cout << "Number of mesh elements: " << globalNE << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
+   //    basis functions in the finite element fespace.
+   VectorFunctionCoefficient f(sdim, f_exact);
+   ParLinearForm *b = new ParLinearForm(fespace);
+   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
+   b->Assemble();
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary edges will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
+   ParGridFunction solproj(fespace);
+   VectorFunctionCoefficient E(sdim, E_exact);
+   x.ProjectCoefficient(E);
+
+   solproj = x;
+
+   // 10. Set up the parallel bilinear form corresponding to the EM diffusion
+   //     operator curl muinv curl + sigma I, by adding the curl-curl and the
+   //     mass domain integrators.
+   Coefficient *muinv = new ConstantCoefficient(1.0);
+   Coefficient *sigma = new ConstantCoefficient(-K2_VALUE);
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
+#ifdef TEST_AIRY
+   VectorFunctionCoefficient epsilon(3, test_Airy_epsilon);
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(epsilon));
+#else
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
+#endif
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   HypreParMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+   }
+
+   StopWatch chrono;
+   chrono.Clear();
+   chrono.Start();
+
+#ifdef MFEM_USE_STRUMPACK
+   if (use_strumpack)
+   {
+      Operator * Arow = new STRUMPACKRowLocMatrix(A);
+
+      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+      strumpack->SetPrintFactorStatistics(true);
+      strumpack->SetPrintSolveStatistics(false);
+      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
+      // strumpack->SetSymmetricPattern(true);
+      strumpack->SetOperator(*Arow);
+      strumpack->SetFromCommandLine();
+      //Solver * precond = strumpack;
+
+      strumpack->Mult(B, X);
+      //X = 0.0;
+
+      {
+         // Check residual
+         Vector res(X.Size());
+         Vector ssol(X.Size());
+         ssol = X;
+
+         const double Bnrm = B.Norml2();
+         const double Bnrm2 = Bnrm*Bnrm;
+
+         A.Mult(ssol, res);
+         res -= B;
+
+         const double Rnrm = res.Norml2();
+         const double Rnrm2 = Rnrm*Rnrm;
+
+         double sumBnrm2 = 0.0;
+         double sumRnrm2 = 0.0;
+         MPI_Allreduce(&Bnrm2, &sumBnrm2, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+         MPI_Allreduce(&Rnrm2, &sumRnrm2, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+
+         if (myid == 0)
+         {
+            cout << myid << ": STRUMPACK residual norm " << sqrt(sumRnrm2) << ", B norm " <<
+                 sqrt(sumBnrm2) << endl;
+         }
+      }
+
+      delete strumpack;
+      delete Arow;
+   }
+   else
+#endif
+   {
+      // 12. Define and apply a parallel PCG solver for AX=B with the AMS
+      //     preconditioner from hypre.
+      ParFiniteElementSpace *prec_fespace =
+         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+      HypreSolver *ams = new HypreAMS(A, prec_fespace);
+      HyprePCG *pcg = new HyprePCG(A);
+      pcg->SetTol(1e-12);
+      pcg->SetMaxIter(500);
+      pcg->SetPrintLevel(2);
+      pcg->SetPreconditioner(*ams);
+      pcg->Mult(B, X);
+
+      delete pcg;
+      delete ams;
+   }
+
+   chrono.Stop();
+   if (myid == 0)
+   {
+      cout << "Solver time " << chrono.RealTime() << endl;
+   }
+
+   // 13. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a->RecoverFEMSolution(X, *b, x);
+
+   // 14. Compute and print the L^2 norm of the error.
+   {
+      double err = x.ComputeL2Error(E);
+      ParGridFunction x0(fespace);
+      x0 = 0.0;
+
+      double solnrm = x0.ComputeL2Error(E);
+
+      if (myid == 0)
+      {
+         cout << "\n|| E_h - E ||_{L^2} = " << err << ", relative error " << err / solnrm
+              << endl;
+      }
+   }
+
+   // 15. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+      //solproj.Save(sol_ofs);
+   }
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x << flush;
+   }
+
+   // 17. Free the used memory.
+   delete a;
+   delete sigma;
+   delete muinv;
+   delete b;
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+
+void E_exact(const Vector &x, Vector &E)
+{
+   if (dim == 3)
+   {
+#ifdef TEST_AIRY
+      const double y = (4.0 * (x(0) + XSHIFT)) - 1.0;
+      const double k = sqrt(K2_VALUE);
+      const double beta = pow(0.25 * k, 2.0/3.0);
+
+      E(0) = 0.0;
+      E(1) = 0.0;
+      E(2) = gsl_sf_airy_Ai(-beta * y, GSL_PREC_DOUBLE);
+#else
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(2));
+      E(2) = sin(kappa * x(0));
+#endif
+   }
+   else
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(0));
+      if (x.Size() == 3) { E(2) = 0.0; }
+   }
+}
+
+void f_exact(const Vector &x, Vector &f)
+{
+   if (dim == 3)
+   {
+#ifdef TEST_AIRY
+      f = 0.0;
+#else
+      f(0) = (-K2_VALUE + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (-K2_VALUE + kappa * kappa) * sin(kappa * x(2));
+      f(2) = (-K2_VALUE + kappa * kappa) * sin(kappa * x(0));
+#endif
+   }
+   else
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
+      if (x.Size() == 3) { f(2) = 0.0; }
+   }
+}

examples/ex3p_cfosls.cpp

@@ -0,0 +1,848 @@
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#include "gsl_sf_airy.h"
+
+#include "multigrid.hpp"
+
+using namespace std;
+using namespace mfem;
+
+
+//#define DIRECT_SOLVER
+
+// Define exact solution
+void E_exact(const Vector & x, Vector & E);
+void H_exact(const Vector & x, Vector & H);
+void f_exact_H(const Vector & x, Vector & f_H);
+void get_maxwell_solution(const Vector & x, double E[], double curlE[],
+                          double curl2E[]);
+void epsilon_func(const Vector &x, Vector &M);
+void epsilon2_func(const Vector &x, Vector &M);
+void epsilon_func_mat(const Vector &x, DenseMatrix &M);
+
+int dim;
+double omega;
+int sol = 3;
+
+
+class FOSLSSolver : public Solver
+{
+public:
+   FOSLSSolver(ParFiniteElementSpace *fespace_, std::vector<HypreParMatrix*>& P,
+               const int myid_)
+      : Solver(2 * fespace_->GetTrueVSize()), M_inv(MPI_COMM_WORLD),
+        fespace(fespace_),
+        n(fespace_->GetTrueVSize()), nfull(fespace_->GetVSize()), LSpcg(MPI_COMM_WORLD),
+        myid(myid_)
+   {
+      z.SetSize(n);
+      Minv_x.SetSize(n);
+
+      ParMesh *pmesh = fespace->GetParMesh();
+      int dim = pmesh->Dimension();
+      int sdim = pmesh->SpaceDimension();
+
+      VectorFunctionCoefficient epsilon(dim, epsilon_func);
+      VectorFunctionCoefficient epsilonT(epsilon);  // transpose of epsilon
+      VectorFunctionCoefficient epsilon2(dim,epsilon2_func);
+      ConstantCoefficient pos(omega);
+      ConstantCoefficient sigma(omega*omega);
+      ScalarVectorProductCoefficient coeff(pos,epsilon);
+      ScalarVectorProductCoefficient coeffT(pos,epsilonT);
+      ScalarVectorProductCoefficient coeff2(sigma,epsilon2);
+
+      bM = new ParBilinearForm(fespace);
+      bM->AddDomainIntegrator(new VectorFEMassIntegrator());
+      bM->Assemble();
+      bM->Finalize();
+
+      bM_eps = new ParBilinearForm(fespace);
+      bM_eps->AddDomainIntegrator(new VectorFEMassIntegrator(epsilonT));
+      bM_eps->Assemble();
+      bM_eps->Finalize();
+
+      bM_curl = new ParMixedBilinearForm(fespace,fespace);
+      bM_curl->AddDomainIntegrator(new MixedVectorWeakCurlIntegrator());
+      bM_curl->Assemble();
+      bM_curl->Finalize();
+
+      Array<int> ess_tdof_list;  // empty
+
+      bM->FormSystemMatrix(ess_tdof_list, M);
+      bM_eps->FormSystemMatrix(ess_tdof_list, M_eps);
+      //bM_curl->FormColSystemMatrix(ess_tdof_list, M_curl);
+      {
+         OperatorPtr M_curl_ptr;
+         bM_curl->FormRectangularSystemMatrix(ess_tdof_list, ess_tdof_list, M_curl_ptr);
+         M_curl = M_curl_ptr.As<HypreParMatrix>();
+      }
+
+      M_inv.SetAbsTol(1.0e-12);
+      M_inv.SetRelTol(1.0e-12);
+      M_inv.SetMaxIter(100);
+      M_inv.SetOperator(M);
+      M_inv.SetPrintLevel(0);
+
+      block_trueOffsets.SetSize(5);
+
+      block_trueOffsets[0] = 0;
+      block_trueOffsets[1] = n;
+      block_trueOffsets[2] = n;
+      block_trueOffsets[3] = n;
+      block_trueOffsets[4] = n;
+      block_trueOffsets.PartialSum();
+
+      trueRhs = new BlockVector(block_trueOffsets);
+      trueSol = new BlockVector(block_trueOffsets);
+
+      //    _           _    _  _       _  _
+      //   |             |  |    |     |    |
+      //   |  A00   A01  |  | E  |     |F_E |
+      //   |             |  |    |  =  |    |
+      //   |  A10   A11  |  | H  |     |F_G |
+      //   |_           _|  |_  _|     |_  _|
+      //
+      // A00 = (curl E, curl F) + \omega^2 (E,F)
+      // A01 = - \omega *( (curl E, F) + (E,curl F)
+      // A10 = - \omega *( (curl H, G) + (H,curl G)
+      // A11 = (curl H, curl G) + \omega^2 (H,G)
+
+      ParBilinearForm *a_EE = new ParBilinearForm(fespace);
+      a_EE->AddDomainIntegrator(new CurlCurlIntegrator());
+      a_EE->AddDomainIntegrator(new VectorFEMassIntegrator(coeff2));
+      a_EE->AddBoundaryIntegrator(new VectorFEMassIntegrator());
+      a_EE->Assemble();
+      a_EE->Finalize();
+      HypreParMatrix *A_EE = new HypreParMatrix;
+      a_EE->FormSystemMatrix(ess_tdof_list, *A_EE);
+
+      ParBilinearForm *a_HH = new ParBilinearForm(fespace);
+      a_HH->AddDomainIntegrator(new CurlCurlIntegrator());
+      a_HH->AddDomainIntegrator(new VectorFEMassIntegrator(sigma));
+      a_HH->AddBoundaryIntegrator(new VectorFEMassIntegrator());
+      a_HH->Assemble();
+      a_HH->Finalize();
+      HypreParMatrix *A_HH = new HypreParMatrix;
+      a_HH->FormSystemMatrix(ess_tdof_list, *A_HH);
+
+      ParBilinearForm *a_tang = new ParBilinearForm(fespace);
+      a_tang->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator(1.0));
+      a_tang->Assemble();
+      a_tang->Finalize();
+      OperatorHandle A_tang_ptr;
+      a_tang->FormSystemMatrix(ess_tdof_list, A_tang_ptr);
+      HypreParMatrix *A_tang = A_tang_ptr.As<HypreParMatrix>();
+
+      // (k curl u, eps v) + (k u, curl v)
+      ParMixedBilinearForm *a_mix1 = new ParMixedBilinearForm(fespace,fespace);
+      a_mix1->AddDomainIntegrator(new MixedVectorCurlIntegrator(coeffT));
+      a_mix1->AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(pos));
+      a_mix1->Assemble();
+      a_mix1->Finalize();
+      HypreParMatrix *A_mix1 = NULL; // new HypreParMatrix;
+      //a_mix1->FormColSystemMatrix(ess_tdof_list, *A_mix1);
+      {
+         OperatorPtr A_mix1_ptr;
+         a_mix1->FormRectangularSystemMatrix(ess_tdof_list, ess_tdof_list, A_mix1_ptr);
+         A_mix1 = A_mix1_ptr.As<HypreParMatrix>();
+      }
+
+      // (k curl u, v) + (k eps u, curl v)
+      ParMixedBilinearForm *a_mix2 = new ParMixedBilinearForm(fespace,fespace);
+      a_mix2->AddDomainIntegrator(new MixedVectorCurlIntegrator(pos));
+      a_mix2->AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(coeff));
+      a_mix2->Assemble();
+      a_mix2->Finalize();
+      HypreParMatrix *A_mix2 = NULL; // new HypreParMatrix;
+      //a_mix2->FormColSystemMatrix(ess_tdof_list, *A_mix2);
+      {
+         OperatorPtr A_mix2_ptr;
+         a_mix2->FormRectangularSystemMatrix(ess_tdof_list, ess_tdof_list, A_mix2_ptr);
+         A_mix2 = A_mix2_ptr.As<HypreParMatrix>();
+      }
+
+      BlockOperator *LS_Maxwellop = new BlockOperator(block_trueOffsets);
+      const int numBlocks = 4;
+
+      LS_Maxwellop->SetBlock(0, 0, A_EE);
+      LS_Maxwellop->SetBlock(1, 0, A_mix2, -1.0); // no bc
+      LS_Maxwellop->SetBlock(3, 0, A_tang, -1.0);
+      LS_Maxwellop->SetBlock(0, 1, A_mix1, -1.0);  // no bc
+      LS_Maxwellop->SetBlock(1, 1, A_HH);
+      LS_Maxwellop->SetBlock(2, 1, A_tang, -1.0);  // other rotation
+      LS_Maxwellop->SetBlock(1, 2, A_tang);
+      LS_Maxwellop->SetBlock(2, 2, A_EE);
+      LS_Maxwellop->SetBlock(3, 2, A_mix2, -1.0);  // no bc
+      LS_Maxwellop->SetBlock(0, 3, A_tang);  // other rotation
+      LS_Maxwellop->SetBlock(2, 3, A_mix1, -1.0);  // no bc
+      LS_Maxwellop->SetBlock(3, 3, A_HH);
+
+      // Set up the preconditioner
+      Array2D<HypreParMatrix*> blockA(numBlocks, numBlocks);
+      Array2D<double> blockAcoef(numBlocks, numBlocks);
+
+      for (int i=0; i<numBlocks; ++i)
+      {
+         for (int j=0; j<numBlocks; ++j)
+         {
+            if (LS_Maxwellop->IsZeroBlock(i,j) == 0)
+            {
+               blockA(i,j) = static_cast<HypreParMatrix *>(&LS_Maxwellop->GetBlock(i,j));
+               blockAcoef(i,j) = LS_Maxwellop->GetBlockCoef(i,j);
+            }
+            else
+            {
+               blockA(i,j) = NULL;
+               blockAcoef(i,j) = 1.0;
+            }
+         }
+      }
+
+      LSpcg.SetAbsTol(1.0e-12);
+      LSpcg.SetRelTol(1.0e-8);
+      LSpcg.SetMaxIter(2000);
+      LSpcg.SetOperator(*LS_Maxwellop);
+      LSpcg.SetPrintLevel(1);
+
+      BlockMGSolver * precMG = NULL;
+
+#ifdef DIRECT_SOLVER
+      std::vector<std::vector<int> > blockProcOffsets(numBlocks);
+      std::vector<std::vector<int> > all_block_num_loc_rows(numBlocks);
+      Array2D<SparseMatrix*> Asp;
+      Asp.SetSize(numBlocks,numBlocks);
+
+      {
+         int nprocs, rank;
+         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+         MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
+
+         std::vector<int> allnumrows(nprocs);
+         const int blockNumRows = n;
+         MPI_Allgather(&blockNumRows, 1, MPI_INT, allnumrows.data(), 1, MPI_INT,
+                       MPI_COMM_WORLD);
+
+         for (int b=0; b<numBlocks; ++b)
+         {
+            blockProcOffsets[b].resize(nprocs);
+            all_block_num_loc_rows[b].resize(nprocs);
+
+            for (int j=0; j<numBlocks; ++j)
+            {
+               Asp(b,j) = NULL;
+            }
+         }
+
+         blockProcOffsets[0][0] = 0;
+         for (int i=0; i<nprocs-1; ++i)
+         {
+            blockProcOffsets[0][i+1] = blockProcOffsets[0][i] + allnumrows[i];
+         }
+
+         for (int i=0; i<nprocs; ++i)
+         {
+            for (int b=0; b<numBlocks; ++b)
+            {
+               all_block_num_loc_rows[b][i] = allnumrows[i];
+            }
+
+            for (int b=1; b<numBlocks; ++b)
+            {
+               blockProcOffsets[b][i] = blockProcOffsets[0][i];
+            }
+         }
+      }
+
+      LSH = CreateHypreParMatrixFromBlocks(MPI_COMM_WORLD, block_trueOffsets, blockA,
+                                           Asp,
+                                           blockAcoef, blockProcOffsets, all_block_num_loc_rows);
+
+      invLSH = CreateStrumpackSolver(new STRUMPACKRowLocMatrix(*LSH), MPI_COMM_WORLD);
+#else
+      precMG = new BlockMGSolver(LS_Maxwellop->Height(), LS_Maxwellop->Width(),
+                                 blockA, blockAcoef, P);
+      precMG->SetTheta(0.5);
+      LSpcg.SetPreconditioner(*precMG);
+#endif
+   }
+
+   void SetOperator(const Operator &op) { }
+
+   void Mult(const Vector &x, Vector &y) const
+   {
+      // Solve (curl E, curl v) - k^2 (eps E, v) + ik <pi(u), pi(v)> = (x, v), with no BC,
+      // where x is complex, using FOSLS. This is the Galerkin discretization of
+      // curl curl u - k^2 eps u = x, with ik n x u x n - n x curl u = 0 on the boundary.
+
+      MFEM_VERIFY(x.Size() == 2*n, "");
+
+      (*trueRhs) = 0.0;
+
+      for (int i=0; i<n; ++i)
+      {
+         z[i] = x[i];   // Set z = x_Re
+      }
+
+      M_inv.Mult(z, Minv_x);
+      M_eps.Mult(Minv_x, z);
+
+      trueRhs->GetBlock(0) -= z;
+
+      M_curl->Mult(Minv_x, z);
+      z *= 1.0 / omega;
+
+      trueRhs->GetBlock(1) = z;
+
+      for (int i=0; i<n; ++i)
+      {
+         z[i] = x[n + i];   // Set z = x_Im
+      }
+
+      M_inv.Mult(z, Minv_x);
+      M_eps.Mult(Minv_x, z);
+
+      trueRhs->GetBlock(2) -= z;
+
+      M_curl->Mult(Minv_x, z);
+      z *= 1.0 / omega;
+
+      trueRhs->GetBlock(3) += z;
+
+#ifdef DIRECT_SOLVER
+      invLSH->Mult(*trueRhs, *trueSol);
+#else
+      LSpcg.Mult(*trueRhs, *trueSol);
+#endif
+
+      for (int i=0; i<n; ++i)
+      {
+         y[i] = trueSol->GetBlock(0)[i];   // Set y_Re = E_Re
+      }
+
+      for (int i=0; i<n; ++i)
+      {
+         y[n + i] = trueSol->GetBlock(2)[i];   // Set y_Im = E_Im
+      }
+   }
+
+   void ComplexFOSLSTest()
+   {
+      Array<int> block_offsets;
+
+      block_offsets.SetSize(5);
+
+      block_offsets[0] = 0;
+      block_offsets[1] = nfull;
+      block_offsets[2] = nfull;
+      block_offsets[3] = nfull;
+      block_offsets[4] = nfull;
+      block_offsets.PartialSum();
+
+      BlockVector rhs(block_offsets);
+      BlockVector rhsIm(block_offsets);
+
+      rhs = 0.0;
+      rhsIm = 0.0;
+
+      const double ci = 0.0; // 3.3;
+
+      // Exact complex solution: E = Er + i Ei = Epw + ci i Epw, where Epw is E_exact.
+
+      // Set up the linear form with the real part Fr only.
+      const int sdim = 3;
+      VectorFunctionCoefficient Eex(sdim, E_exact);
+      VectorFunctionCoefficient Hex(sdim, H_exact);
+      ConstantCoefficient negOne(-1.0);
+      VectorFunctionCoefficient mEex(sdim, E_exact, &negOne);
+      VectorFunctionCoefficient mHex(sdim, H_exact, &negOne);
+
+      ConstantCoefficient neg(-omega);
+      ConstantCoefficient pos(omega);
+      VectorFunctionCoefficient f_H(3,f_exact_H);  // f / omega
+      ScalarVectorProductCoefficient sf_H(neg,f_H);
+      ScalarVectorProductCoefficient spf_H(pos,f_H);
+      ScalarVectorProductCoefficient mf_H(negOne,f_H);
+
+      VectorFunctionCoefficient epsilon(3, epsilon_func);
+      VectorFunctionCoefficient epsilonT(3, epsilon_func);  // transpose of epsilon
+      MatrixFunctionCoefficient epsilonTmat(3,
+                                            epsilon_func_mat);  // transpose of epsilon
+
+      MatVecCoefficient epsT_spf_H(epsilonTmat, spf_H);
+      MatVecCoefficient epsT_sf_H(epsilonTmat, sf_H);
+
+      ParLinearForm *b_E = new ParLinearForm;
+      b_E->Update(fespace, rhs.GetBlock(0), 0);
+      b_E->AddDomainIntegrator(new VectorFEDomainLFIntegrator(
+                                  epsT_sf_H)); // (k^{-1} Fr, -k eps Qr)
+      b_E->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
+                                    Eex));  // <n x E_Re x n, n x Q_Re x n>
+      b_E->Assemble();
+
+      ParLinearForm *b_H = new ParLinearForm;
+      b_H->Update(fespace, rhs.GetBlock(1), 0);
+      b_H->AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(
+                                  f_H));  // (k^{-1} Fr, curl Rr)
+      b_H->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
+                                    Hex));  // <n x H_Re, n x R_Re>
+      b_H->Assemble();
+
+      ParLinearForm *b_E_Im = new ParLinearForm;
+      b_E_Im->Update(fespace, rhs.GetBlock(2), 0);
+      b_E_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
+                                       mHex));  // <n x H_Re, n x Q_Im x n>
+      b_E_Im->Assemble();
+
+      ParLinearForm *b_H_Im = new ParLinearForm;
+      b_H_Im->Update(fespace, rhs.GetBlock(3), 0);
+      b_H_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
+                                       mEex));  // -<n x E_Re x n, n x R_Im>
+      b_H_Im->Assemble();
+
+      // Add the imaginary part Fi.
+
+      ParLinearForm *b_Ei = new ParLinearForm;
+      b_Ei->Update(fespace, rhsIm.GetBlock(0), 0);
+      b_Ei->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
+                                     Hex));  // -<n x H_Im, n x Q_Re x n>
+      b_Ei->Assemble();
+
+      ParLinearForm *b_Hi = new ParLinearForm;
+      b_Hi->Update(fespace, rhsIm.GetBlock(1), 0);
+      b_Hi->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
+                                     Eex));  // <n x E_Im x n, n x R_Re>
+      b_Hi->Assemble();
+
+      ParLinearForm *b_Ei_Im = new ParLinearForm;
+      b_Ei_Im->Update(fespace, rhsIm.GetBlock(2), 0);
+      b_Ei_Im->AddDomainIntegrator(new VectorFEDomainLFIntegrator(
+                                      epsT_sf_H)); // -(k^{-1} Fi, k eps Qi)
+      b_Ei_Im->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
+                                        Eex));  // <n x E_Im x n, n x Q_Im x n>
+      b_Ei_Im->Assemble();
+
+      ParLinearForm *b_Hi_Im = new ParLinearForm;
+      b_Hi_Im->Update(fespace, rhsIm.GetBlock(3), 0);
+      b_Hi_Im->AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(
+                                      f_H)); // (k^{-1} Fi, curl Ri)
+      b_Hi_Im->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
+                                        Hex));  // <n x H_Im, n x R_Im>
+      b_Hi_Im->Assemble();
+
+      rhsIm *= ci;
+      rhs += rhsIm;
+
+      for (int i=0; i<4; ++i)
+      {
+         fespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(i),
+                                                         trueRhs->GetBlock(i));
+      }
+
+#ifdef DIRECT_SOLVER
+      invLSH->Mult(*trueRhs, *trueSol);
+#else
+      LSpcg.Mult(*trueRhs, *trueSol);
+#endif
+
+      // Check error
+      ParGridFunction E_gf(fespace);
+      int order = 2;
+      int order_quad = std::max(2, 2*order+1);
+      const IntegrationRule *irs[Geometry::NumGeom];
+      for (int i=0; i < Geometry::NumGeom; ++i)
+      {
+         irs[i] = &(IntRules.Get(i, order_quad));
+      }
+
+      ParMesh *pmesh = fespace->GetParMesh();
+
+      // Check error of real part
+
+      E_gf.SetFromTrueDofs(trueSol->GetBlock(0));
+      double Error_E = E_gf.ComputeL2Error(Eex, irs);
+      double norm_E = ComputeGlobalLpNorm(2, Eex, *pmesh, irs);
+
+      cout << myid << ": real error " << Error_E << " relative to " << norm_E << endl;
+
+      {
+         ostringstream mesh_name, sol_name;
+         mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+         sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+         ofstream mesh_ofs(mesh_name.str().c_str());
+         mesh_ofs.precision(8);
+         pmesh->Print(mesh_ofs);
+
+         ofstream sol_ofs(sol_name.str().c_str());
+         sol_ofs.precision(8);
+         E_gf.Save(sol_ofs);
+      }
+
+      // Check error of imaginary part
+      E_gf.SetFromTrueDofs(trueSol->GetBlock(2));
+      const double iml2 = E_gf.Norml2();
+      E_gf *= ci == 0.0 ? 0.0 : (1.0 / ci);
+      Error_E = E_gf.ComputeL2Error(Eex, irs);
+
+      cout << myid << ": imag error " << Error_E << " relative to " << norm_E <<
+           ", l2 norm " << iml2 << endl;
+   }
+
+private:
+
+   BlockVector *trueRhs, *trueSol;
+
+   Array<int> block_trueOffsets;
+
+   ParBilinearForm *bM, *bM_eps;
+   ParMixedBilinearForm *bM_curl;
+
+   HypreParMatrix M, M_eps;
+   HypreParMatrix *M_curl;
+
+   CGSolver M_inv;
+
+   const int n;
+   const int nfull;
+   const int myid;
+
+   mutable Vector z, Minv_x;
+
+   CGSolver LSpcg;
+
+   STRUMPACKSolver *invLSH;
+   HypreParMatrix *LSH;
+
+   ParFiniteElementSpace *fespace;
+};
+
+int main(int argc, char *argv[])
+{
+   StopWatch chrono;
+
+   // 1. Initialize MPI
+   int num_procs, myid;
+   MPI_Init(&argc, &argv); // Initialize MPI
+   MPI_Comm_size(MPI_COMM_WORLD,
+                 &num_procs); //total number of processors available
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid); // Determine process identifier
+   // 1. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../data/inline-hex.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // static condensation flag
+   bool static_cond = false;
+   // visualization flag
+   bool visualization = 1;
+   // number of wavelengths
+   double k = 1.0;
+   // number of mg levels
+   int ref_levels = 1;
+   // number of initial ref
+   int initref = 1;
+
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&ref_levels, "-ref", "--ref_levels",
+                  "Number of Refinements.");
+   args.AddOption(&initref, "-initref", "--initref",
+                  "Number of initial refinements.");
+   args.AddOption(&sol, "-sol", "--exact",
+                  "Exact solution flag - "
+                  " 1:sinusoidal, 2: point source, 3: plane wave");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // Angular frequency
+   //omega = 2.0*k*M_PI;
+   omega = k;
+
+   // 2. Read the mesh from the given mesh file.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+
+   if (sol == 4)
+   {
+      mesh->EnsureNodes();
+      GridFunction *nodes = mesh->GetNodes();
+      (*nodes) *= 0.5;
+   }
+
+   dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   // 3. Executing uniform h-refinement
+   for (int i = 0; i < initref; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   // 4. Define a finite element space on the mesh.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   std::vector<ParFiniteElementSpace * > fespaces(ref_levels+1);
+   std::vector<ParMesh * > ParMeshes(ref_levels+1);
+   std::vector<HypreParMatrix*> P(ref_levels);
+
+   for (int i = 0; i < ref_levels; i++)
+   {
+      ParMeshes[i] =new ParMesh(*pmesh);
+      fespaces[i] = new ParFiniteElementSpace(*fespace, *ParMeshes[i]);
+      pmesh->UniformRefinement();
+      // Update fespace
+      fespace->Update();
+      OperatorHandle Tr(Operator::Hypre_ParCSR);
+      fespace->GetTrueTransferOperator(*fespaces[i], Tr);
+      Tr.SetOperatorOwner(false);
+      Tr.Get(P[i]);
+   }
+   fespaces[ref_levels] = new ParFiniteElementSpace(*fespace);
+
+   FOSLSSolver fosls(fespace, P, myid);
+
+   fosls.ComplexFOSLSTest();
+
+   for (auto p: ParMeshes) { delete p; }
+   for (auto p: fespaces) { delete p; }
+   for (auto p: P) { delete p; }
+   ParMeshes.clear();
+   fespaces.clear();
+   P.clear();
+   delete fec;
+   delete fespace;
+   delete pmesh;
+
+   MPI_Finalize();
+   return 0;
+}
+
+//define exact solution
+void E_exact(const Vector &x, Vector &E)
+{
+   double curlE[3], curl2E[3];
+   get_maxwell_solution(x, E, curlE, curl2E);
+}
+
+void H_exact(const Vector &x, Vector &H)
+{
+   double E[3], curlE[3], curl2E[3];
+   get_maxwell_solution(x, E, curlE, curl2E);
+   for (int i = 0; i<3; i++) { H(i) = curlE[i]/omega; }
+}
+
+
+void f_exact_H(const Vector &x, Vector &f)
+{
+   // curl H - omega E = f
+   // = curl (curl E / omega) - omega E
+   f = 0.0;
+   if (sol !=4)
+   {
+      double E[3], curlE[3], curl2E[3];
+      get_maxwell_solution(x, E, curlE, curl2E);
+      f(0) = curl2E[0] / omega - omega * E[0];
+      f(1) = curl2E[1] / omega - omega * E[1];
+      f(2) = curl2E[2] / omega - omega * E[2];
+   }
+}
+
+void get_maxwell_solution(const Vector &X, double E[], double curlE[],
+                          double curl2E[])
+{
+   double x = X[0];
+   double y = X[1];
+   double z = X[2];
+
+   if (sol ==-1)
+   {
+      E[0] = y * z * (1.0 - y) * (1.0 - z);
+      E[1] = x * y * z * (1.0 - x) * (1.0 - z);
+      E[2] = x * y * (1.0 - x) * (1.0 - y);
+
+      curlE[0] = -(x-1.0) * x * (y*(2.0*z-3.0)+1.0);
+      curlE[1] = -2.0*(y-1.0)*y*(x-z);
+      curlE[2] = (z-1)*z*(1.0+y*(2.0*x-3.0));
+
+      curl2E[0] = 2.0 * y * (1.0 - y) - (2.0 * x - 3.0) * z * (1 - z);
+      curl2E[1] = 2.0 * y * (x * (1.0 - x) + (1.0 - z) * z);
+      curl2E[2] = 2.0 * y * (1.0 - y) + x * (3.0 - 2.0 * z) * (1.0 - x);
+   }
+   else if (sol == 0) // polynomial
+   {
+      // Polynomial vanishing on the boundary
+      E[0] = y * z * (1.0 - y) * (1.0 - z);
+      E[1] = (1.0 - x) * x * y * (1.0 - z) * z;
+      E[2] = (1.0 - x) * x * (1.0 - y) * y;
+      //
+      curlE[0] = -(-1.0 + x) * x * (1.0 + y * (-3.0 + 2.0 * z));
+      curlE[1] = -2.0 * (-1.0 + y) * y * (x - z);
+      curlE[2] = (1.0 + (-3.0 + 2.0 * x) * y) * (-1.0 + z) * z;
+
+      curl2E[0] = -2.0 * (-1.0 + y) * y + (-3.0 + 2.0 * x) * (-1.0 + z) * z;
+      curl2E[1] = -2.0 * y * (-x + x * x + (-1.0 + z) * z);
+      curl2E[2] = -2.0 * (-1.0 + y) * y + (-1.0 + x) * x * (-3.0 + 2.0 * z);
+   }
+   else if (sol == 1) // sinusoidal
+   {
+      E[0] = sin(omega * y);
+      E[1] = sin(omega * z);
+      E[2] = sin(omega * x);
+
+      curlE[0] = -omega * cos(omega * z);
+      curlE[1] = -omega * cos(omega * x);
+      curlE[2] = -omega * cos(omega * y);
+
+      curl2E[0] = omega * omega * E[0];
+      curl2E[1] = omega * omega * E[1];
+      curl2E[2] = omega * omega * E[2];
+   }
+   else if (sol == 2) // point source
+   {
+      // shift to avoid singularity
+      double x0 = x + 0.1;
+      double x1 = y + 0.1;
+      double x2 = z + 0.1;
+      double r = sqrt(x0 * x0 + x1 * x1 + x2 * x2);
+
+      E[0] = cos(omega * r);
+      E[1] = 0.0;
+      E[2] = 0.0;
+
+      double r_x = x0 / r;
+      double r_y = x1 / r;
+      double r_z = x2 / r;
+      double r_xy = -(r_x / r) * r_y;
+      double r_xz = -(r_x / r) * r_z;
+      double r_yx = r_xy;
+      double r_yy = (1.0 / r) * (1.0 - r_y * r_y);
+      double r_zx = r_xz;
+      double r_zz = (1.0 / r) * (1.0 - r_z * r_z);
+
+      curlE[0] = 0.0;
+      curlE[1] = -omega * r_z * sin(omega * r);
+      curlE[2] =  omega * r_y * sin(omega * r);
+
+      curl2E[0] = omega * ((r_yy + r_zz) * sin(omega * r) +
+                           (omega * r_y * r_y + omega * r_z * r_z) * cos(omega * r));
+      curl2E[1] = -omega * (r_yx * sin(omega * r) + omega * r_y * r_x * cos(
+                               omega * r));
+      curl2E[2] = -omega * (r_zx * sin(omega * r) + omega * r_z * r_x * cos(
+                               omega * r));
+   }
+   else if (sol == 3) // plane wave
+   {
+      double coeff = omega / sqrt(3.0);
+      E[0] = cos(coeff * (x + y + z));
+      E[1] = 0.0;
+      E[2] = 0.0;
+
+      curlE[0] = 0.0;
+      curlE[1] = -coeff * sin(coeff * (x + y + z));
+      curlE[2] = coeff * sin(coeff * (x + y + z));
+
+      curl2E[0] = 2.0 * coeff * coeff * E[0];
+      curl2E[1] = -coeff * coeff * E[0];
+      curl2E[2] = -coeff * coeff * E[0];
+   }
+   else if (sol == -1)
+   {
+      E[0] = cos(omega * y);
+      E[1] = 0.0;
+
+      curlE[0] = 0.0;
+      curlE[1] = 0.0;
+      curlE[2] = -omega * sin(omega * y);
+
+      curl2E[0] = omega*omega * cos(omega*y);
+      curl2E[1] = 0.0;
+      curl2E[2] = 0.0;
+   }
+   else if (sol == 4) // Airy function
+   {
+      E[0] = 0;
+      E[1] = 0;
+      // double b = -pow(omega/4.0,2.0/3.0)*(4.0*x(0)-1.0);
+      double b = -pow(omega/4.0,2.0/3.0)*(4.0*x-1.0);
+      //E[2] = boost::math::airy_ai(b);
+      E[2] = gsl_sf_airy_Ai(b, GSL_PREC_DOUBLE);
+
+      curlE[0] = 0.0;
+      curlE[1] = 4.0 * pow(omega/4.0,2.0/3.0) * gsl_sf_airy_Ai_deriv(b,
+                                                                     GSL_PREC_DOUBLE);
+      curlE[2] = 0.0;
+
+      // not used
+      curl2E[0] = 0.0;
+      curl2E[1] = 0.0;
+      curl2E[2] = 0.0;
+   }
+}
+
+void epsilon_func(const Vector &x, Vector &M)
+{
+   M.SetSize(3);
+
+   M = 1.0;
+   if (sol == 4)
+   {
+      M[2] = 4.0*x(0)-1.0;
+   }
+}
+
+void epsilon2_func(const Vector &x, Vector &M)
+{
+   M.SetSize(3);
+
+   M = 1.0;
+   if (sol == 4)
+   {
+      M[2] = (4.0*x(0)-1.0) * (4.0*x(0)-1.0);
+   }
+}
+
+void epsilon_func_mat(const Vector &x, DenseMatrix &M)
+{
+   M.SetSize(3);
+
+   M = 0.0;
+   M(0,0) = 1.0;
+   M(1,1) = 1.0;
+   if (sol != 4)
+   {
+      M(2,2) = 1.0;
+   }
+   else
+   {
+      M(2,2) = 4.0*x(0)-1.0;
+   }
+}

examples/ex3p_fosls.cpp

@@ -0,0 +1,764 @@
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#include "gsl_sf_airy.h"
+
+#include "multigrid.hpp"
+
+using namespace std;
+using namespace mfem;
+
+#define COMPLEX_IMPEDANCE  // Whether to use impedance BC and solve a complex system
+
+// Define exact solution
+void E_exact(const Vector & x, Vector & E);
+void H_exact(const Vector & x, Vector & H);
+void f_exact_H(const Vector & x, Vector & f_H);
+void get_maxwell_solution(const Vector & x, double E[], double curlE[],
+                          double curl2E[]);
+void epsilon_func(const Vector &x, DenseMatrix &M);
+void epsilon2_func(const Vector &x, DenseMatrix &M);
+
+int dim;
+double omega;
+int sol = 4;
+
+int main(int argc, char *argv[])
+{
+   StopWatch chrono;
+
+   // 1. Initialize MPI
+   int num_procs, myid;
+   MPI_Init(&argc, &argv); // Initialize MPI
+   MPI_Comm_size(MPI_COMM_WORLD,
+                 &num_procs); //total number of processors available
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid); // Determine process identifier
+   // 1. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../data/inline-hex.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // static condensation flag
+   bool static_cond = false;
+   // visualization flag
+   bool visualization = 1;
+   // number of wavelengths
+   double k = 1.0;
+   // number of mg levels
+   int ref_levels = 1;
+   // number of initial ref
+   int initref = 1;
+
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&ref_levels, "-ref", "--ref_levels",
+                  "Number of Refinements.");
+   args.AddOption(&initref, "-initref", "--initref",
+                  "Number of initial refinements.");
+   args.AddOption(&sol, "-sol", "--exact",
+                  "Exact solution flag - "
+                  " 1:sinusoidal, 2: point source, 3: plane wave");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // Angular frequency
+   //omega = 2.0*k*M_PI;
+   omega = k;
+
+   // 2. Read the mesh from the given mesh file.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+
+   if (sol == 4)
+   {
+      mesh->EnsureNodes();
+      GridFunction *nodes = mesh->GetNodes();
+      (*nodes) *= 0.5;
+   }
+
+   dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   // 3. Executing uniform h-refinement
+   for (int i = 0; i < initref; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   // 4. Define a finite element space on the mesh.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   std::vector<ParFiniteElementSpace * > fespaces(ref_levels+1);
+   std::vector<ParMesh * > ParMeshes(ref_levels+1);
+   std::vector<HypreParMatrix*> P(ref_levels);
+
+   for (int i = 0; i < ref_levels; i++)
+   {
+      ParMeshes[i] =new ParMesh(*pmesh);
+      fespaces[i] = new ParFiniteElementSpace(*fespace, *ParMeshes[i]);
+      pmesh->UniformRefinement();
+      // Update fespace
+      fespace->Update();
+      OperatorHandle Tr(Operator::Hypre_ParCSR);
+      fespace->GetTrueTransferOperator(*fespaces[i], Tr);
+      Tr.SetOperatorOwner(false);
+      Tr.Get(P[i]);
+   }
+   fespaces[ref_levels] = new ParFiniteElementSpace(*fespace);
+
+   Array<int> ess_tdof_listE;
+   Array<int> ess_tdof_listH;
+   Array<int> ess_bdrE(pmesh->bdr_attributes.Max());
+   Array<int> ess_bdrH(pmesh->bdr_attributes.Max());
+#ifdef COMPLEX_IMPEDANCE
+   ess_bdrE = 0;
+   ess_bdrH = 0;  // Neumann
+#else
+   ess_bdrE = 1;
+   ess_bdrH = 0;  // Neumann
+#endif
+   fespace->GetEssentialTrueDofs(ess_bdrE, ess_tdof_listE);
+   fespace->GetEssentialTrueDofs(ess_bdrH, ess_tdof_listH);
+
+#ifdef COMPLEX_IMPEDANCE
+   Array<int> bdr_attr(pmesh->bdr_attributes.Max());
+   bdr_attr = 1;
+   ConstantCoefficient impedance(omega);
+   RestrictedCoefficient imp_rest(impedance, bdr_attr);
+   Array<int> block_offsets(5);
+   Array<int> block_trueOffsets(5);
+#else
+   Array<int> block_offsets(3);
+   Array<int> block_trueOffsets(3);
+#endif
+   block_offsets[0] = 0;
+   block_offsets[1] = fespace->GetVSize();
+   block_offsets[2] = fespace->GetVSize();
+#ifdef COMPLEX_IMPEDANCE
+   block_offsets[3] = fespace->GetVSize();
+   block_offsets[4] = fespace->GetVSize();
+#endif
+   block_offsets.PartialSum();
+
+   block_trueOffsets[0] = 0;
+   block_trueOffsets[1] = fespace->TrueVSize();
+   block_trueOffsets[2] = fespace->TrueVSize();
+#ifdef COMPLEX_IMPEDANCE
+   block_trueOffsets[3] = fespace->TrueVSize();
+   block_trueOffsets[4] = fespace->TrueVSize();
+#endif
+   block_trueOffsets.PartialSum();
+
+   //    _           _    _  _       _  _
+   //   |             |  |    |     |    |
+   //   |  A00   A01  |  | E  |     |F_E |
+   //   |             |  |    |  =  |    |
+   //   |  A10   A11  |  | H  |     |F_G |
+   //   |_           _|  |_  _|     |_  _|
+   //
+   // A00 = (curl E, curl F) + \omega^2 (E,F)
+   // A01 = - \omega *( (curl E, F) + (E,curl F)
+   // A10 = - \omega *( (curl H, G) + (H,curl G)
+   // A11 = (curl H, curl G) + \omega^2 (H,G)
+
+   BlockVector x(block_offsets), rhs(block_offsets);
+   BlockVector trueX(block_trueOffsets), trueRhs(block_trueOffsets);
+
+   x = 0.0;
+   rhs = 0.0;
+   trueX = 0.0;
+   trueRhs = 0.0;
+
+   VectorFunctionCoefficient Eex(sdim, E_exact);
+   ConstantCoefficient negOne(-1.0);
+   VectorFunctionCoefficient mEex(sdim, E_exact, &negOne);
+
+   ParGridFunction * E_gf = new ParGridFunction;
+   ParGridFunction * Exact_gf = new ParGridFunction(fespace);
+   E_gf->MakeRef(fespace, x.GetBlock(0));
+   E_gf->ProjectCoefficient(Eex);
+   Exact_gf->ProjectCoefficient(Eex);
+
+   VectorFunctionCoefficient Hex(sdim, H_exact);
+   ParGridFunction * H_gf = new ParGridFunction;
+   H_gf->MakeRef(fespace, x.GetBlock(1));
+   H_gf->ProjectCoefficient(Hex);
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(pow(omega, 2));
+   ConstantCoefficient neg(-abs(omega));
+   ConstantCoefficient pos(abs(omega));
+
+   MatrixFunctionCoefficient epsilon(dim,epsilon_func);
+   TransposeMatrixCoefficient epsilonT(epsilon);  // transpose of epsilon
+   MatrixFunctionCoefficient epsilon2(dim,epsilon2_func);
+   ScalarMatrixProductCoefficient coeff(neg,epsilon);
+   ScalarMatrixProductCoefficient coeff2(sigma,epsilon2);
+
+   // 6. Set up the linear form
+   VectorFunctionCoefficient f_H(sdim,f_exact_H);
+   ScalarVectorProductCoefficient sf_H(neg,f_H);
+
+   MatVecCoefficient epsT_sf_H(epsilonT, sf_H);
+
+   ParLinearForm *b_E = new ParLinearForm;
+   b_E->Update(fespace, rhs.GetBlock(0), 0);
+   b_E->AddDomainIntegrator(new VectorFEDomainLFIntegrator(epsT_sf_H));
+#ifdef COMPLEX_IMPEDANCE
+   b_E->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
+                                 Eex));  // <g_Im, n x Q x n> = <n x E_Re x n, n x Q x n>
+
+   ParLinearForm *b_E_Im = new ParLinearForm;
+   b_E_Im->Update(fespace, rhs.GetBlock(2), 0);
+   b_E_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
+                                    Hex));  // -<g_Re, n x Q x n> = <n x H_Re, n x Q x n>
+   b_E_Im->Assemble();
+#endif
+   b_E->Assemble();
+
+   ParLinearForm *b_H = new ParLinearForm;
+   b_H->Update(fespace, rhs.GetBlock(1), 0);
+   b_H->AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(f_H));
+#ifdef COMPLEX_IMPEDANCE
+   b_H->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
+                                 Hex));  // -<g_Re, n x R> = <n x H_Re, n x R>
+
+   ParLinearForm *b_H_Im = new ParLinearForm;
+   b_H_Im->Update(fespace, rhs.GetBlock(3), 0);
+   b_H_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
+                                    mEex));  // -<g_Im, n x R> = -<n x E_Re x n, n x R>
+   b_H_Im->Assemble();
+#endif
+   b_H->Assemble();
+
+#ifdef COMPLEX_IMPEDANCE
+   fespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(2),
+                                                   trueRhs.GetBlock(2));
+   fespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(3),
+                                                   trueRhs.GetBlock(3));
+#endif
+
+   // 7. Bilinear form a(.,.) on the finite element space
+   ParBilinearForm *a_EE = new ParBilinearForm(fespace);
+   a_EE->AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a_EE->AddDomainIntegrator(new VectorFEMassIntegrator(coeff2));
+
+#ifdef COMPLEX_IMPEDANCE
+   a_EE->AddBoundaryIntegrator(new VectorFEMassIntegrator());
+   //a_EE->AddBoundaryIntegrator(new VectorFEMassIntegrator(imp_rest));
+   //a_EE->AddBoundaryIntegrator(new BoundaryMassIntegrator(imp_rest));
+   //a_EE->AddBdrFaceIntegrator(new BoundaryMassIntegrator(imp_rest));
+#endif
+
+   a_EE->Assemble();
+   a_EE->Finalize();
+   HypreParMatrix *A_EE = new HypreParMatrix;
+   a_EE->FormLinearSystem(ess_tdof_listE, x.GetBlock(0), rhs.GetBlock(0), *A_EE,
+                          trueX.GetBlock(0), trueRhs.GetBlock(0));
+
+   ParBilinearForm *a_HH = new ParBilinearForm(fespace);
+   a_HH->AddDomainIntegrator(new CurlCurlIntegrator(one)); // one is the coeff
+   a_HH->AddDomainIntegrator(new VectorFEMassIntegrator(sigma));
+
+#ifdef COMPLEX_IMPEDANCE
+   a_HH->AddBoundaryIntegrator(new VectorFEMassIntegrator());
+   //a_HH->AddBoundaryIntegrator(new VectorFEMassIntegrator(imp_rest));
+   //a_HH->AddBoundaryIntegrator(new BoundaryMassIntegrator(imp_rest));
+#endif
+
+   a_HH->Assemble();
+   a_HH->Finalize();
+   HypreParMatrix *A_HH = new HypreParMatrix;
+   a_HH->FormLinearSystem(ess_tdof_listH, x.GetBlock(1), rhs.GetBlock(1), *A_HH,
+                          trueX.GetBlock(1), trueRhs.GetBlock(1));
+
+   ParMixedBilinearForm *a_HE = new ParMixedBilinearForm(fespace,fespace);
+   a_HE->AddDomainIntegrator(new MixedVectorCurlIntegrator(neg));
+   a_HE->AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(coeff));
+   a_HE->Assemble();
+   a_HE->Finalize();
+   HypreParMatrix *A_HE = new HypreParMatrix;
+   a_HE->FormColLinearSystem(ess_tdof_listE,x.GetBlock(0),rhs.GetBlock(1),*A_HE,
+                             trueX.GetBlock(0),trueRhs.GetBlock(1));
+
+   HypreParMatrix *A_EH = A_HE->Transpose();
+
+#ifdef COMPLEX_IMPEDANCE
+   ParBilinearForm *a_EH_Im = new ParBilinearForm(fespace);
+   //a_EH_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator(imp_rest));
+   //a_EH_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator(impedance));
+   //a_EH_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator(omega));
+   a_EH_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator());
+   a_EH_Im->Assemble();
+   a_EH_Im->Finalize();
+
+   OperatorHandle A_EH_Im_ptr;
+   a_EH_Im->FormSystemMatrix(ess_tdof_listE,
+                             A_EH_Im_ptr);  // empty ess_tdof_list for impedance
+
+   HypreParMatrix *A_EH_Im = A_EH_Im_ptr.As<HypreParMatrix>();
+
+   HypreParMatrix *A_HE_Im = A_EH_Im->Transpose();
+#endif
+
+   BlockOperator *LS_Maxwellop = new BlockOperator(block_trueOffsets);
+#ifdef COMPLEX_IMPEDANCE
+   const int numBlocks = 4;
+#else
+   const int numBlocks = 2;
+#endif
+
+   LS_Maxwellop->SetBlock(0, 0, A_EE);
+   LS_Maxwellop->SetBlock(0, 1, A_EH);
+   LS_Maxwellop->SetBlock(1, 0, A_HE);
+   LS_Maxwellop->SetBlock(1, 1, A_HH);
+
+#ifdef COMPLEX_IMPEDANCE
+   LS_Maxwellop->SetBlock(0, 3, A_EH_Im);
+   LS_Maxwellop->SetBlock(1, 2, A_HE_Im, -1.0);
+
+   LS_Maxwellop->SetBlock(2, 1, A_EH_Im, -1.0);
+   LS_Maxwellop->SetBlock(3, 0, A_HE_Im);
+
+   LS_Maxwellop->SetBlock(2, 2, A_EE);
+   LS_Maxwellop->SetBlock(2, 3, A_EH);
+   LS_Maxwellop->SetBlock(3, 2, A_HE);
+   LS_Maxwellop->SetBlock(3, 3, A_HH);
+#endif
+
+   if (myid == 0)
+   {
+      cout << "Size of fine grid system: "
+           << 2.0 * A_EE->GetGlobalNumRows() << " x " << 2.0* A_EE->GetGlobalNumCols() <<
+           endl;
+   }
+
+   // Set up the preconditioner
+   Array2D<HypreParMatrix*> blockA(numBlocks, numBlocks);
+   Array2D<double> blockAcoef(numBlocks, numBlocks);
+
+   for (int i=0; i<numBlocks; ++i)
+   {
+      for (int j=0; j<numBlocks; ++j)
+      {
+         if (LS_Maxwellop->IsZeroBlock(i,j) == 0)
+         {
+            blockA(i,j) = static_cast<HypreParMatrix *>(&LS_Maxwellop->GetBlock(i,j));
+            blockAcoef(i,j) = LS_Maxwellop->GetBlockCoef(i,j);
+         }
+         else
+         {
+            blockA(i,j) = NULL;
+            blockAcoef(i,j) = 1.0;
+         }
+      }
+   }
+
+   // // double nnz = A_HH->NNZ();
+   // // double ndof = A_HH->GetGlobalNumRows();
+   // // double est_mem_b = nnz*12.0 + (ndof+1.0)*4;
+   // // double gb = est_mem_b*4.0/pow(1024.0,3);
+
+   // // mfem::out << "Estimated memory taken by the global matrix: " <<  gb << endl;
+
+   int maxit(2000);
+   double rtol(1.e-8);
+   double atol(1.e-12);
+
+   // trueX = 0.0;
+   CGSolver pcg(MPI_COMM_WORLD);
+   pcg.SetAbsTol(atol);
+   pcg.SetRelTol(rtol);
+   pcg.SetMaxIter(maxit);
+   pcg.SetOperator(*LS_Maxwellop);
+   pcg.SetPrintLevel(1);
+   chrono.Clear();
+   chrono.Start();
+   BlockMGSolver * precMG = new BlockMGSolver(LS_Maxwellop->Height(),
+                                              LS_Maxwellop->Width(), blockA, blockAcoef, P);
+   //precMG->SetTheta(0.5);
+   // // int lv_coarse = min(ref_levels,ref_levels-1);
+   // // int levels = ref_levels - lv_coarse;
+   // // BlkParSchwarzSmoother * precAS = new BlkParSchwarzSmoother(fespaces[lv_coarse]->GetParMesh(),levels,fespaces[ref_levels],LS_Maxwellop);
+   chrono.Stop();
+   if (myid == 0)
+   {
+      cout << "MG Setup time: " << chrono.RealTime() << endl;
+   }
+
+   // Randomize H RHS
+   //trueRhs = 1.0;
+   /*
+   for (int i=block_trueOffsets[1]; i<block_trueOffsets[2]; ++i)
+     trueRhs[i] = i % 53;
+   */
+
+   chrono.Clear();
+   chrono.Start();
+   pcg.SetPreconditioner(*precMG);
+   // // pcg.SetPreconditioner(*precAS);
+   pcg.Mult(trueRhs, trueX);
+   chrono.Stop();
+   delete precMG;
+   // // delete precAS;
+
+   // // trueX = 0.0;
+   // // invA->Mult(trueRhs,trueX);
+
+   if (myid == 0)
+   {
+      cout << "MG Solution time time: " << chrono.RealTime() << endl;
+   }
+   // // cin.get();
+   // //  if(myid == 0)
+   // //    cout << "MG prec Solution time: " << chrono.RealTime() << endl;
+
+
+   // // chrono.Clear();
+   // // chrono.Start();
+   // // Block_AMSSolver * precAMS = new Block_AMSSolver(block_trueOffsets,fespaces);
+   // // precAMS->SetSmootherType(Block_AMS::BlkSmootherType::SCHWARZ);
+   // // precAMS->SetSmootherType(Block_AMS::BlkSmootherType::HYPRE);
+   // // precAMS->SetOperator(LS_Maxwellop);
+   // // precAMS->SetTheta(1.0/5.0);
+   // // // 0-Smoother, 1-Grad, 2,3,4-Pix,Piy,Piz
+   // // precAMS->SetCycleType("023414320");
+   // // precAMS->SetNumberofCycles(1);
+   // // chrono.Stop();
+   // // if(myid == 0)
+   // //    cout << "BlkAMS Setup time: " << chrono.RealTime() << endl;
+
+   // // // resolve with block AMS
+   // // trueX = 0;
+   // // chrono.Clear();
+   // // chrono.Start();
+   // // pcg.SetPreconditioner(*precAMS);
+   // // pcg.Mult(trueRhs, trueX);
+   // // chrono.Stop();
+   // // delete precAMS;
+
+   // // if(myid == 0)
+   // //    cout << "BlockAMS Solution time: " << chrono.RealTime() << endl;
+
+
+   a_EE->RecoverFEMSolution(trueX.GetBlock(0), *b_E, *E_gf);
+   a_HH->RecoverFEMSolution(trueX.GetBlock(1), *b_H, *H_gf);
+
+
+   int order_quad = max(2, 2*order+1);
+   const IntegrationRule *irs[Geometry::NumGeom];
+   for (int i=0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+
+   double Error_E = E_gf->ComputeL2Error(Eex, irs);
+   double norm_E = ComputeGlobalLpNorm(2, Eex, *pmesh, irs);
+
+   double Error_H = H_gf->ComputeL2Error(Hex, irs);
+   double norm_H = ComputeGlobalLpNorm(2, Hex , *pmesh, irs);
+
+   if (myid == 0)
+   {
+      cout << "|| E_h - E || = " << Error_E  << "\n";
+      cout << "|| E_h - E ||/||E|| = " << Error_E/norm_E  << "\n";
+      cout << "|| H_h - H || = " << Error_H  << "\n";
+      cout << "|| H_h - H ||/||H|| = " << Error_H/norm_H  << "\n";
+      cout << "Total error = " << setprecision(15) << sqrt(Error_H*Error_H
+                                                           +Error_E*Error_E) << "\n";
+   }
+
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      E_gf->Save(sol_ofs);
+   }
+
+   //ParGridFunction ExactE(fespace);
+   /*
+   if (visualization)
+     {
+       // 8. Connect to GLVis.
+       char vishost[] = "localhost";
+       int  visport   = 19916;
+       // socketstream E_sock(vishost, visport);
+       // E_sock << "parallel " << num_procs << " " << myid << "\n";
+       // E_sock.precision(8);
+       // E_sock << "solution\n" << *pmesh << *E_gf << "window_title 'Electric field'" << endl;
+       socketstream Exact_sock(vishost, visport);
+       Exact_sock << "parallel " << num_procs << " " << myid << "\n";
+       Exact_sock.precision(8);
+       Exact_sock << "solution\n" << *pmesh << *Exact_gf << "window_title 'Electric field'" << endl;
+
+       // MPI_Barrier(pmesh->GetComm());
+       // socketstream Eex_sock(vishost, visport);
+       // Eex_sock << "parallel " << num_procs << " " << myid << "\n";
+       // Eex_sock.precision(8);
+       // Eex_sock << "solution\n" << *pmesh << *Exact_gf << "window_title 'Exact Electric field'" << endl;
+     }
+   */
+   // delete A_EE;
+   // delete A_HE;
+   // delete A_EH;
+   // delete A_HH;
+   // delete LS_Maxwellop;
+   // delete a_EE;
+   // delete a_HE;
+   // delete a_HH;
+   // delete b_E;
+   // delete b_H;
+   delete E_gf;
+   delete Exact_gf;
+   for (auto p: ParMeshes) { delete p; }
+   for (auto p: fespaces) { delete p; }
+   for (auto p: P) { delete p; }
+   ParMeshes.clear();
+   fespaces.clear();
+   P.clear();
+   delete fec;
+   delete fespace;
+   delete pmesh;
+
+   // cout << "Freed memory: " << endl;
+   // cin.get();
+   MPI_Finalize();
+   return 0;
+}
+
+//define exact solution
+void E_exact(const Vector &x, Vector &E)
+{
+   double curlE[3], curl2E[3];
+   get_maxwell_solution(x, E, curlE, curl2E);
+}
+
+void H_exact(const Vector &x, Vector &H)
+{
+   double E[3], curlE[3], curl2E[3];
+   get_maxwell_solution(x, E, curlE, curl2E);
+   for (int i = 0; i<3; i++) { H(i) = curlE[i]/omega; }
+}
+
+
+void f_exact_H(const Vector &x, Vector &f)
+{
+   // curl H - omega E = f
+   // = curl (curl E / omega) - omega E
+   f = 0.0;
+   if (sol !=4)
+   {
+      double E[3], curlE[3], curl2E[3];
+      get_maxwell_solution(x, E, curlE, curl2E);
+      f(0) = curl2E[0] / omega - omega * E[0];
+      f(1) = curl2E[1] / omega - omega * E[1];
+      f(2) = curl2E[2] / omega - omega * E[2];
+   }
+}
+
+void get_maxwell_solution(const Vector &X, double E[], double curlE[],
+                          double curl2E[])
+{
+   double x = X[0];
+   double y = X[1];
+   double z = X[2];
+
+
+
+   if (sol ==-1)
+   {
+      E[0] = y * z * (1.0 - y) * (1.0 - z);
+      E[1] = x * y * z * (1.0 - x) * (1.0 - z);
+      E[2] = x * y * (1.0 - x) * (1.0 - y);
+
+      curlE[0] = -(x-1.0) * x * (y*(2.0*z-3.0)+1.0);
+      curlE[1] = -2.0*(y-1.0)*y*(x-z);
+      curlE[2] = (z-1)*z*(1.0+y*(2.0*x-3.0));
+
+      curl2E[0] = 2.0 * y * (1.0 - y) - (2.0 * x - 3.0) * z * (1 - z);
+      curl2E[1] = 2.0 * y * (x * (1.0 - x) + (1.0 - z) * z);
+      curl2E[2] = 2.0 * y * (1.0 - y) + x * (3.0 - 2.0 * z) * (1.0 - x);
+   }
+   else if (sol == 0) // polynomial
+   {
+      // Polynomial vanishing on the boundary
+      E[0] = y * z * (1.0 - y) * (1.0 - z);
+      E[1] = (1.0 - x) * x * y * (1.0 - z) * z;
+      E[2] = (1.0 - x) * x * (1.0 - y) * y;
+      //
+      curlE[0] = -(-1.0 + x) * x * (1.0 + y * (-3.0 + 2.0 * z));
+      curlE[1] = -2.0 * (-1.0 + y) * y * (x - z);
+      curlE[2] = (1.0 + (-3.0 + 2.0 * x) * y) * (-1.0 + z) * z;
+
+      curl2E[0] = -2.0 * (-1.0 + y) * y + (-3.0 + 2.0 * x) * (-1.0 + z) * z;
+      curl2E[1] = -2.0 * y * (-x + x * x + (-1.0 + z) * z);
+      curl2E[2] = -2.0 * (-1.0 + y) * y + (-1.0 + x) * x * (-3.0 + 2.0 * z);
+   }
+   else if (sol == 1) // sinusoidal
+   {
+      E[0] = sin(omega * y);
+      E[1] = sin(omega * z);
+      E[2] = sin(omega * x);
+
+      curlE[0] = -omega * cos(omega * z);
+      curlE[1] = -omega * cos(omega * x);
+      curlE[2] = -omega * cos(omega * y);
+
+      curl2E[0] = omega * omega * E[0];
+      curl2E[1] = omega * omega * E[1];
+      curl2E[2] = omega * omega * E[2];
+   }
+   else if (sol == 2) // point source
+   {
+      // shift to avoid singularity
+      double x0 = x + 0.1;
+      double x1 = y + 0.1;
+      double x2 = z + 0.1;
+      //
+      double r = sqrt(x0 * x0 + x1 * x1 + x2 * x2);
+
+      E[0] = cos(omega * r);
+      E[1] = 0.0;
+      E[2] = 0.0;
+
+      double r_x = x0 / r;
+      double r_y = x1 / r;
+      double r_z = x2 / r;
+      double r_xy = -(r_x / r) * r_y;
+      double r_xz = -(r_x / r) * r_z;
+      double r_yx = r_xy;
+      double r_yy = (1.0 / r) * (1.0 - r_y * r_y);
+      double r_zx = r_xz;
+      double r_zz = (1.0 / r) * (1.0 - r_z * r_z);
+
+      curlE[0] = 0.0;
+      curlE[1] = -omega * r_z * sin(omega * r);
+      curlE[2] =  omega * r_y * sin(omega * r);
+
+      curl2E[0] = omega * ((r_yy + r_zz) * sin(omega * r) +
+                           (omega * r_y * r_y + omega * r_z * r_z) * cos(omega * r));
+      curl2E[1] = -omega * (r_yx * sin(omega * r) + omega * r_y * r_x * cos(
+                               omega * r));
+      curl2E[2] = -omega * (r_zx * sin(omega * r) + omega * r_z * r_x * cos(
+                               omega * r));
+   }
+   else if (sol == 3) // plane wave
+   {
+      double coeff = omega / sqrt(3.0);
+      E[0] = cos(coeff * (x + y + z));
+      E[1] = 0.0;
+      E[2] = 0.0;
+
+      curlE[0] = 0.0;
+      curlE[1] = -coeff * sin(coeff * (x + y + z));
+      curlE[2] = coeff * sin(coeff * (x + y + z));
+
+      curl2E[0] = 2.0 * coeff * coeff * E[0];
+      curl2E[1] = -coeff * coeff * E[0];
+      curl2E[2] = -coeff * coeff * E[0];
+   }
+   else if (sol == -1)
+   {
+      E[0] = cos(omega * y);
+      E[1] = 0.0;
+
+      curlE[0] = 0.0;
+      curlE[1] = 0.0;
+      curlE[2] = -omega * sin(omega * y);
+
+      curl2E[0] = omega*omega * cos(omega*y);
+      curl2E[1] = 0.0;
+      curl2E[2] = 0.0;
+   }
+   else if (sol == 4) // Airy function
+   {
+      E[0] = 0;
+      E[1] = 0;
+      // double b = -pow(omega/4.0,2.0/3.0)*(4.0*x(0)-1.0);
+      double b = -pow(omega/4.0,2.0/3.0)*(4.0*x-1.0);
+      //E[2] = boost::math::airy_ai(b);
+      E[2] = gsl_sf_airy_Ai(b, GSL_PREC_DOUBLE);
+
+      curlE[0] = 0.0;
+      curlE[1] = 4.0 * pow(omega/4.0,2.0/3.0) * gsl_sf_airy_Ai_deriv(b,
+                                                                     GSL_PREC_DOUBLE);
+      curlE[2] = 0.0;
+
+      //  not used
+      curl2E[0] = 0.0;
+      curl2E[1] = 0.0;
+      curl2E[2] = 0.0;
+   }
+}
+
+
+void epsilon_func(const Vector &x, DenseMatrix &M)
+{
+   M.SetSize(3);
+
+   M = 0.0;
+   M(0,0) = 1.0;
+   M(1,1) = 1.0;
+   if (sol != 4)
+   {
+      M(2,2) = 1.0;
+   }
+   else
+   {
+      M(2,2) = 4.0*x(0)-1.0;
+      // M(2,2) = 2.0;
+   }
+}
+
+void epsilon2_func(const Vector &x, DenseMatrix &M)
+{
+   M.SetSize(3);
+
+   M = 0.0;
+   M(0,0) = 1.0;
+   M(1,1) = 1.0;
+   if (sol != 4)
+   {
+      M(2,2) = 1.0;
+   }
+   else
+   {
+      M(2,2) = (4.0*x(0)-1.0) * (4.0*x(0)-1.0);
+      // M(2,2) = 4.0;
+   }
+}

examples/ex3p_sp.cpp

@@ -0,0 +1,915 @@
+//                       MFEM Example 3 - Parallel Version
+//
+// Compile with: make ex3p
+//
+// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/escher.mesh
+//               mpirun -np 4 ex3p -m ../data/escher.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/fichera.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
+//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
+//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
+//
+// Description:  This example code solves a simple electromagnetic diffusion
+//               problem corresponding to the second order definite Maxwell
+//               equation curl curl E + E = f with boundary condition
+//               E x n = <given tangential field>. Here, we use a given exact
+//               solution E and compute the corresponding r.h.s. f.
+//               We discretize with Nedelec finite elements in 2D or 3D.
+//
+//               The example demonstrates the use of H(curl) finite element
+//               spaces with the curl-curl and the (vector finite element) mass
+//               bilinear form, as well as the computation of discretization
+//               error when the exact solution is known. Static condensation is
+//               also illustrated.
+//
+//               We recommend viewing examples 1-2 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution, E, and r.h.s., f. See below for implementation.
+void E_exact(const Vector &, Vector &);
+void f_exact(const Vector &, Vector &);
+double freq = 1.0, kappa;
+int dim;
+
+#define SIGMAVAL -250.0
+//#define FORM_DEFINITE
+//#define SOLVE_A2
+//#define ITER_A2
+
+//#define USE_CSL
+
+//#define USE_HELMHOLTZ
+
+//#define TEST_MULTIPLE_SP
+
+#ifdef USE_HELMHOLTZ
+void GetHelmholtzMatrix(ParMesh *pmesh, const int dir, HypreParMatrix *A)
+{
+   const int order = 1;
+   FiniteElementCollection *fec;
+   fec = new H1_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+
+   Array<int> ess_tdof_list;
+
+   const bool homogeneousBCeverywhere = false;
+   if (homogeneousBCeverywhere)
+   {
+      if (pmesh->bdr_attributes.Size())
+      {
+         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+         ess_bdr = 1;
+         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+   }
+   else
+   {
+      // Set boundary conditions, depending on dir.
+      MFEM_VERIFY(dim == 3, "");
+      for (int i=0; i<pmesh->GetNBE(); ++i)
+      {
+         Element *elem = pmesh->GetBdrElement(i);
+         MFEM_VERIFY(elem->GetNVertices() >= 3, "");
+         const int *vertices = elem->GetVertices();
+         double *v[3];
+         for (int j=0; j<3; ++j)
+         {
+            v[j] = pmesh->GetVertex(vertices[j]);
+         }
+
+         double u[3];
+         double w[3];
+         for (int j=0; j<3; ++j)
+         {
+            u[j] = v[1][j] - v[0][j];  // An edge tangent
+            w[j] = v[2][j] - v[1][j];  // Another edge tangent, not parallel to u.
+         }
+
+         double n[3];  // normal vector, taken as the cross product u x v
+         n[0] = (u[1]*w[2]) - (u[2]*w[1]);
+         n[1] = (u[2]*w[0]) - (u[0]*w[2]);
+         n[2] = (u[0]*w[1]) - (u[1]*w[0]);
+
+         double t = sqrt((n[0]*n[0]) + (n[1]*n[1]) + (n[2]*n[2]));
+
+         int d = -1;
+         for (int j=0; j<3; ++j)
+         {
+            n[j] /= t;  // normalize
+            if (fabs(fabs(n[j]) - 1.0) < 1.0e-8)
+            {
+               d = j;
+            }
+         }
+
+         MFEM_VERIFY(d >= 0, "");
+
+         if (d != dir)  // face has essential BC at all DOF's.
+         {
+            elem->SetAttribute(1);
+         }
+         else
+         {
+            elem->SetAttribute(0);
+         }
+      }
+
+      Array<int> ess_bdr(2);
+      ess_bdr = 0;
+      ess_bdr[1] = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient neg(SIGMAVAL);
+
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   a->AddDomainIntegrator(new MassIntegrator(neg));
+
+   ParLinearForm *b = new ParLinearForm(fespace);
+   ConstantCoefficient zero(0.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(zero));
+   b->Assemble();
+
+   bool static_cond = false;
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   ParGridFunction x(fespace);
+   x = 0.0;
+
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, *A, X, B);
+}
+#endif
+
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/beam-tet.mesh";
+   int order = 2;
+   bool static_cond = false;
+   bool visualization = 1;
+#ifdef MFEM_USE_STRUMPACK
+   bool use_strumpack = false;
+#endif
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
+                  " solution.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+#ifdef MFEM_USE_STRUMPACK
+   args.AddOption(&use_strumpack, "-strumpack", "--strumpack-solver",
+                  "-no-strumpack", "--no-strumpack-solver",
+                  "Use STRUMPACK's double complex linear solver.");
+#endif
+
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+   kappa = freq * M_PI;
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 1,000 elements.
+   {
+      int ref_levels =
+         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+      //(int)floor(log(100000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   {
+      int par_ref_levels = 2;
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh->UniformRefinement();
+      }
+   }
+   pmesh->ReorientTetMesh();
+
+   {
+      double minsize = pmesh->GetElementSize(0);
+      double maxsize = minsize;
+      for (int i=1; i<pmesh->GetNE(); ++i)
+      {
+         const double size_i = pmesh->GetElementSize(i);
+         minsize = std::min(minsize, size_i);
+         maxsize = std::max(maxsize, size_i);
+      }
+
+      cout << myid << ": Element size range: (" << minsize << ", " << maxsize << ")"
+           << endl;
+   }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Nedelec finite elements of the specified order.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   long globalNE = pmesh->GetGlobalNE();
+   if (myid == 0)
+   {
+      cout << "Number of mesh elements: " << globalNE << endl;
+      cout << "Number of finite element unknowns: " << size << endl;
+      cout << "Root local number of finite element unknowns: " << fespace->TrueVSize()
+           << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
+   //    basis functions in the finite element fespace.
+   VectorFunctionCoefficient f(sdim, f_exact);
+   ParLinearForm *b = new ParLinearForm(fespace);
+   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
+   b->Assemble();
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary edges will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
+   VectorFunctionCoefficient E(sdim, E_exact);
+   x.ProjectCoefficient(E);
+
+   // 10. Set up the parallel bilinear form corresponding to the EM diffusion
+   //     operator curl muinv curl + sigma I, by adding the curl-curl and the
+   //     mass domain integrators.
+   Coefficient *muinv = new ConstantCoefficient(1.0);
+   Coefficient *sigma = new ConstantCoefficient(SIGMAVAL);
+   Coefficient *sigmaAbs = new ConstantCoefficient(fabs(SIGMAVAL));
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
+
+   //cout << myid << ": NBE " << pmesh->GetNBE() << endl;
+
+#ifdef FORM_DEFINITE
+   ParBilinearForm *adef = new ParBilinearForm(fespace);
+   adef->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
+   adef->AddDomainIntegrator(new VectorFEMassIntegrator(*sigmaAbs));
+
+   if (static_cond) { adef->EnableStaticCondensation(); }
+   adef->Assemble();
+
+   HypreParMatrix Adef;
+   Vector Bdef, Xdef;
+   adef->FormLinearSystem(ess_tdof_list, x, *b, Adef, Xdef, Bdef);
+#endif
+
+#ifdef USE_CSL
+   Vector Bdef, Xdef;
+
+   ParBilinearForm *Mform =  new ParBilinearForm(fespace);
+   Mform->AddDomainIntegrator(new VectorFEMassIntegrator(*sigmaAbs));
+   Mform->Assemble();
+
+   // Mform->Finalize();
+
+   HypreParMatrix Mmat, Smat, Mcopy;
+   Mform->FormLinearSystem(ess_tdof_list, x, *b, Mmat, Xdef, Bdef);
+   Mform->FormLinearSystem(ess_tdof_list, x, *b, Mcopy, Xdef,
+                           Bdef);  // There must be a better way than creating two identical matrices.
+
+   ParBilinearForm *Sform =  new ParBilinearForm(fespace);
+   Sform->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
+   Sform->Assemble();
+
+   Sform->FormLinearSystem(ess_tdof_list, x, *b, Smat, Xdef, Bdef);
+
+   ParBilinearForm *agrad = new ParBilinearForm(fespace);
+   //agrad->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
+   agrad->AddDomainIntegrator(new VectorFEMassIntegrator(*muinv));
+
+   if (static_cond) { agrad->EnableStaticCondensation(); }
+   agrad->Assemble();
+   HypreParMatrix Agrad;
+   agrad->FormLinearSystem(ess_tdof_list, x, *b, Agrad, Xdef, Bdef);
+#endif
+
+#ifdef ITER_A2
+   Vector Bdef, Xdef;
+
+   ParBilinearForm *Mform =  new ParBilinearForm(fespace);
+   Mform->AddDomainIntegrator(new VectorFEMassIntegrator(*sigmaAbs));
+   Mform->Assemble();
+
+   Mform->Finalize();
+
+   HypreParMatrix Mmat, Mcopy;
+   Mform->FormLinearSystem(ess_tdof_list, x, *b, Mmat, Xdef, Bdef);
+   Mform->FormLinearSystem(ess_tdof_list, x, *b, Mcopy, Xdef,
+                           Bdef);  // There must be a better way to implement M^2.
+
+   /*
+   HypreParMatrix *Mmat = Mform->ParallelAssemble();
+   HypreParMatrix *Mcopy = Mform->ParallelAssemble();  // There must be a better way to implement M^2.
+   */
+
+   ParBilinearForm *Sform =  new ParBilinearForm(fespace);
+   Sform->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
+   Sform->Assemble();
+
+   HypreParMatrix Smat, Scopy;
+   Sform->FormLinearSystem(ess_tdof_list, x, *b, Smat, Xdef, Bdef);
+   Sform->FormLinearSystem(ess_tdof_list, x, *b, Scopy, Xdef, Bdef);
+#endif
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   HypreParMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+#ifdef SOLVE_A2
+   HypreParMatrix Acopy;
+   {
+      Vector Bdum, Xdum;
+      a->FormLinearSystem(ess_tdof_list, x, *b, Acopy, Xdum, Bdum);
+   }
+#endif
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+   }
+
+   StopWatch chrono;
+   chrono.Clear();
+   chrono.Start();
+
+   //A.Print("maxwell1000_2");
+
+#ifdef MFEM_USE_STRUMPACK
+   if (use_strumpack)
+   {
+      const bool fullDirect = true;
+
+#ifdef USE_CSL
+      const double beta1 = 1.0;
+      const double beta2 = 0.5;
+
+      Mmat *= -beta1;
+
+      // HypreParMatrix *cslRe = Add(1.0, Smat, -beta1, Mmat);
+      HypreParMatrix * cslRe = ParAdd(&Smat, &Mmat);
+
+      Mcopy *= beta2;
+
+      //ComplexHypreParMatrix chpm(cslRe, &Mcopy, false, false);
+      ComplexHypreParMatrix chpm(&A, &Mcopy, false,
+                                 false);  // For the case beta1 = 1.
+
+      HypreParMatrix *cSysMat = chpm.GetSystemMatrix();
+
+      Array<int> block_offsets(3); // number of variables + 1
+      block_offsets[0] = 0;
+      block_offsets[1] = fespace->GetVSize();
+      block_offsets[2] = fespace->GetVSize();
+      block_offsets.PartialSum();
+
+      Array<int> block_trueOffsets(3); // number of variables + 1
+      block_trueOffsets[0] = 0;
+      block_trueOffsets[1] = fespace->TrueVSize();
+      block_trueOffsets[2] = fespace->TrueVSize();
+      block_trueOffsets.PartialSum();
+
+      //cout << myid << ": V size " << fespace->GetVSize() << ", true " << fespace->TrueVSize() << ", global true " << size << ", B size "
+      //<< B.Size() << ", X size " << X.Size() << endl;
+
+      // Note that B is of true size.
+      BlockVector trueY(block_trueOffsets), trueX(block_trueOffsets),
+                  trueRhs(block_trueOffsets);
+
+      trueRhs.GetBlock(0) = B;
+      trueRhs.GetBlock(1) = 0.0;
+
+      Operator * Arow = new STRUMPACKRowLocMatrix(*cSysMat);
+
+      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+      strumpack->SetPrintFactorStatistics(true);
+      strumpack->SetPrintSolveStatistics(false);
+      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
+      // strumpack->SetSymmetricPattern(true);
+      strumpack->SetOperator(*Arow);
+      strumpack->SetFromCommandLine();
+#endif
+
+      if (fullDirect)
+      {
+#ifdef USE_CSL
+         //Solver * precond = strumpack;
+
+         // strumpack->Mult(B, X);
+
+         BlockOperator blockDiagA(block_trueOffsets);
+         for (int i=0; i<2; ++i)
+         {
+            blockDiagA.SetDiagonalBlock(i, &A);
+         }
+
+         ParFiniteElementSpace *prec_fespace =
+            (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+         HypreSolver *amsgrad = new HypreAMS(Agrad, prec_fespace);
+
+#ifdef HYPRE_DYLAN
+         {
+            Vector Xtmp(X);
+            amsgrad->Mult(B,
+                          Xtmp);  // Just a hack to get ams to run its setup function. There should be a better way.
+         }
+
+         HypreAMSG *amsg = new HypreAMSG((HypreAMS*) amsgrad, argc, argv);
+         BlockOperator blockDiagP(block_trueOffsets);
+         for (int i=0; i<2; ++i)
+         {
+            blockDiagP.SetDiagonalBlock(i, amsg);
+         }
+
+         TripleProductOperator strumpackProj(&blockDiagP, strumpack, &blockDiagP, false,
+                                             false, false);
+         ProductOperator prod(&blockDiagA, &strumpackProj, false, false);
+#else
+         ProductOperator prod(&blockDiagA, strumpack, false, false);
+#endif
+
+         GMRESSolver *gmres = new GMRESSolver(fespace->GetComm());
+         //BiCGSTABSolver *gmres = new BiCGSTABSolver(fespace->GetComm());
+
+         gmres->SetOperator(prod);
+         gmres->SetRelTol(1e-12);
+         gmres->SetMaxIter(1000);
+         gmres->SetPrintLevel(1);
+
+         gmres->Mult(trueRhs, trueY);
+         strumpack->Mult(trueY, trueX);
+
+         X = trueX.GetBlock(0);
+         double xim2 = trueX.GetBlock(1).Norml2();
+         xim2 *= xim2;
+         double sumxim2 = 0.0;
+
+         MPI_Allreduce(&xim2, &sumxim2, 1, MPI_DOUBLE, MPI_SUM, fespace->GetComm());
+
+         if (myid == 0)
+         {
+            cout << myid << ": norm of Xim " << trueX.GetBlock(1).Norml2() << ", global " <<
+                 sqrt(sumxim2) << endl;
+         }
+
+         delete gmres;
+         delete strumpack;
+         delete Arow;
+#else
+         cout << "Solving with STRUMPACK" << endl;
+
+#ifdef TEST_MULTIPLE_SP
+         const int Ns = 2;
+         std::vector<Operator*> Arows(Ns);
+         std::vector<STRUMPACKSolver*> strumpacks(Ns);
+
+         //Operator * Arow = new STRUMPACKRowLocMatrix(A);
+
+         for (int m=0; m<Ns; ++m)
+         {
+            Arows[m] = new STRUMPACKRowLocMatrix(A);
+
+            //STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+            strumpacks[m] = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+            strumpacks[m]->SetPrintFactorStatistics(true);
+            strumpacks[m]->SetPrintSolveStatistics(false);
+            strumpacks[m]->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+            strumpacks[m]->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+            // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
+            // strumpack->SetSymmetricPattern(true);
+            strumpacks[m]->SetOperator(*Arows[m]);
+            strumpacks[m]->SetFromCommandLine();
+            //Solver * precond = strumpack;
+
+            strumpacks[m]->Mult(B, X);
+
+            //delete strumpack;
+            //delete Arow;
+         }
+#else
+         Operator * Arow = new STRUMPACKRowLocMatrix(A);
+
+         STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+         strumpack->SetPrintFactorStatistics(true);
+         strumpack->SetPrintSolveStatistics(false);
+         strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+         strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+         // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
+         // strumpack->SetSymmetricPattern(true);
+         strumpack->SetOperator(*Arow);
+         strumpack->SetFromCommandLine();
+         //Solver * precond = strumpack;
+
+         cout << "Solving with strumpack one time" << endl;
+
+         strumpack->Mult(B, X);
+
+         delete strumpack;
+         delete Arow;
+#endif
+#endif
+      }
+      else
+      {
+         ParFiniteElementSpace *prec_fespace =
+            (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+         HypreSolver *ams = new HypreAMS(A, prec_fespace);
+
+#ifdef HYPRE_DYLAN
+         {
+            Vector Xtmp(X);
+            ams->Mult(B,
+                      Xtmp);  // Just a hack to get ams to run its setup function. There should be a better way.
+         }
+
+         HypreParMatrix H[3];
+#ifdef USE_HELMHOLTZ
+         for (int i=0; i<3; ++i)
+         {
+            GetHelmholtzMatrix(pmesh, i, &(H[i]));
+         }
+#endif
+
+#ifdef USE_CSL
+         HypreIAMS *iams = new HypreIAMS(A, H, strumpack, &trueX, &trueY,
+                                         (HypreAMS*) ams, argc, argv);
+#else
+         HypreIAMS *iams = new HypreIAMS(A, H, strumpack, NULL, NULL, (HypreAMS*) ams,
+                                         argc, argv);
+#endif
+
+         GMRESSolver *gmres = new GMRESSolver(fespace->GetComm());
+         //FGMRESSolver *gmres = new FGMRESSolver(fespace->GetComm());
+         //BiCGSTABSolver *gmres = new BiCGSTABSolver(fespace->GetComm());
+         //MINRESSolver *gmres = new MINRESSolver(fespace->GetComm());
+
+         gmres->SetOperator(A);
+         gmres->SetRelTol(1e-16);
+         gmres->SetMaxIter(1000);
+         gmres->SetPrintLevel(1);
+
+#ifdef SOLVE_A2
+         {
+            StopWatch chronoA2;
+            chronoA2.Clear();
+            chronoA2.Start();
+
+            HypreParMatrix * A2 = ParMult(&A, &Acopy);
+
+            chronoA2.Stop();
+            cout << "A2 setup time " << chronoA2.RealTime() << endl;
+
+            Vector AB(B);
+            A.Mult(B, AB);
+            gmres->SetOperator(*A2);
+
+            HypreSolver *ams2 = new HypreAMS(A, prec_fespace);
+            {
+               Vector Xtmp(X);
+               ams2->Mult(B,
+                          Xtmp);  // Just a hack to get ams to run its setup function. There should be a better way.
+            }
+
+#ifdef ITER_A2
+            // Iteratively solve 0.5 (A^2 + S^2 + M^2) u^{k+1} = 0.5 (SM + MS) u^k + Ab
+
+            StopWatch chronoIterA2;
+            chronoIterA2.Clear();
+            chronoIterA2.Start();
+
+            HypreParMatrix * M2 = ParMult(&Mmat, &Mcopy);
+            HypreParMatrix * S2 = ParMult(&Smat, &Scopy);
+
+            HypreParMatrix * MS = ParMult(&Mmat, &Scopy);
+            HypreParMatrix * SM = ParMult(&Smat, &Mcopy);
+
+            HypreParMatrix * Bmat = ParAdd(SM, MS);
+            (*Bmat) *= 0.5;
+
+            // TODO: there must be a better way to form a sum of three matrices. Of course, we could define an operator that does 3 mat-vecs.
+            //HypreParMatrix * S2M2 = ParAdd(S2, M2);
+            //HypreParMatrix * iterMat = ParAdd(A2, S2M2);
+            HypreParMatrix * iterMat = ParAdd(A2, Bmat);
+
+            chronoIterA2.Stop();
+            cout << "Iter A2 setup time " << chronoIterA2.RealTime() << endl;
+
+            /*
+            HypreSolver *ams3 = new HypreAMS(*iterMat, prec_fespace);
+            {
+              Vector Xtmp(X);
+              ams3->Mult(B, Xtmp);  // Just a hack to get ams to run its setup function. There should be a better way.
+            }
+            */
+
+            /*
+            // GMRES
+            gmres->SetOperator(*iterMat);
+            gmres->SetPreconditioner(*ams2);
+            */
+
+
+            //HypreBoomerAMG *amg = new HypreBoomerAMG(*iterMat);
+            HypreBoomerAMG *amg = new HypreBoomerAMG(*A2);
+
+            // PCG
+            HyprePCG *pcg = new HyprePCG(*iterMat);
+            //HyprePCG *pcg = new HyprePCG(*A2);
+            pcg->SetTol(1e-12);
+            pcg->SetMaxIter(10);
+            pcg->SetPrintLevel(2);
+            pcg->SetPreconditioner(*amg);
+
+            /*
+            // Strumpack linear solver
+            Operator * Arow = new STRUMPACKRowLocMatrix(*iterMat);
+
+            STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+            strumpack->SetPrintFactorStatistics(true);
+            strumpack->SetPrintSolveStatistics(false);
+            strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+            strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+            strumpack->SetOperator(*Arow);
+            strumpack->SetFromCommandLine();
+            */
+
+            Vector iterRHS(AB);
+            Vector iterU(AB);
+            Vector iterU0(AB);
+
+            iterU = 0.0;
+            iterU0 = 0.0;
+
+            bool iterate = true;
+            int numIter = 0;
+            while (iterate)
+            {
+               iterRHS = iterU;
+               iterRHS.Add(-1.0, iterU0);
+
+               cout << "Iteration " << numIter + 1 << ": diff norm " << iterRHS.Norml2() <<
+                    endl;
+
+               iterU0 = iterU;
+
+               Bmat->Mult(iterU0, iterRHS);
+               //iterRHS.Add(2.0, AB);
+               iterRHS.Add(1.0, AB);
+
+               //gmres->Mult(iterRHS, iterU);
+               pcg->Mult(iterRHS, iterU);
+               //strumpack->Mult(iterRHS, iterU);
+
+               numIter++;
+
+               if (numIter > 100)
+               {
+                  iterate = false;
+               }
+            }
+
+            //delete strumpack;
+            //delete Arow;
+
+            delete pcg;
+
+            X = iterU;
+#else
+            //HypreIAMS *iams2 = new HypreIAMS(*A2, (HypreAMS*) ams2, argc, argv);
+            //gmres->SetPreconditioner(*iams2);
+            cout << myid << ": Solving" << endl;
+            gmres->SetPreconditioner(*ams2);
+            gmres->Mult(AB, X);
+            cout << myid << ": Solved" << endl;
+            return 3;
+#endif
+         }
+#else
+         gmres->SetPreconditioner(*iams);
+         gmres->Mult(B, X);
+#endif
+#else
+         HypreGMRES *gmres = new HypreGMRES(A);
+         gmres->SetTol(1e-12);
+         gmres->SetMaxIter(100);
+         gmres->SetPrintLevel(10);
+
+#ifdef FORM_DEFINITE
+         HypreSolver *amsdef = new HypreAMS(Adef, prec_fespace);
+         gmres->SetPreconditioner(*amsdef);
+#else
+         gmres->SetPreconditioner(*ams);
+#endif
+         gmres->Mult(B, X);
+#endif
+
+         delete gmres;
+         //delete iams;
+         //delete ams;
+      }
+   }
+   else
+#endif
+   {
+      // 12. Define and apply a parallel PCG solver for AX=B with the AMS
+      //     preconditioner from hypre.
+      ParFiniteElementSpace *prec_fespace =
+         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+      HypreSolver *ams = new HypreAMS(A, prec_fespace);
+      HyprePCG *pcg = new HyprePCG(A);
+      pcg->SetTol(1e-12);
+      pcg->SetMaxIter(500);
+      pcg->SetPrintLevel(2);
+      pcg->SetPreconditioner(*ams);
+      pcg->Mult(B, X);
+
+      delete pcg;
+      delete ams;
+   }
+
+   chrono.Stop();
+   cout << myid << ": Solver time " << chrono.RealTime() << endl;
+
+   // 13. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a->RecoverFEMSolution(X, *b, x);
+
+   // 14. Compute and print the L^2 norm of the error.
+   {
+      double err = x.ComputeL2Error(E);
+      Vector zeroVec(3);
+      zeroVec = 0.0;
+      VectorConstantCoefficient vzero(zeroVec);
+      ParGridFunction zerogf(fespace);
+      zerogf = 0.0;
+      double normE = zerogf.ComputeL2Error(E);
+      double normX = x.ComputeL2Error(vzero);
+      if (myid == 0)
+      {
+         cout << "|| E_h - E ||_{L^2} = " << err << endl;
+         cout << "|| E_h ||_{L^2} = " << normX << endl;
+         cout << "|| E ||_{L^2} = " << normE << endl;
+      }
+   }
+
+   // 15. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x << flush;
+   }
+
+   // 17. Free the used memory.
+   delete a;
+   delete sigma;
+   delete muinv;
+   delete b;
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+
+void E_exact(const Vector &x, Vector &E)
+{
+   if (dim == 3)
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(2));
+      E(2) = sin(kappa * x(0));
+   }
+   else
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(0));
+      if (x.Size() == 3) { E(2) = 0.0; }
+   }
+}
+
+void f_exact(const Vector &x, Vector &f)
+{
+   if (dim == 3)
+   {
+      f(0) = (SIGMAVAL + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (SIGMAVAL + kappa * kappa) * sin(kappa * x(2));
+      f(2) = (SIGMAVAL + kappa * kappa) * sin(kappa * x(0));
+   }
+   else
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
+      if (x.Size() == 3) { f(2) = 0.0; }
+   }
+}

examples/ginkgo/README

@@ -5,7 +5,7 @@
                       | | | | | ||  _||  __/| | | | | |
                       |_| |_| |_||_|   \___||_| |_| |_|
 
-                               https://mfem.org
+                               http://mfem.org
 
 This directory contains modifications of the example codes that illustrate the
 use of MFEM features based on the Ginkgo high-performance linear algebra library

examples/hiop/README

@@ -5,7 +5,7 @@
                       | | | | | ||  _||  __/| | | | | |
                       |_| |_| |_||_|   \___||_| |_| |_|
 
-                               https://mfem.org
+                               http://mfem.org
 
 This directory contains modifications of the example codes that illustrate the
 use of MFEM for solving nonlinear constrained optimization problems, including

examples/makefile

@@ -48,12 +48,6 @@ endif
 ifeq ($(MFEM_USE_GINKGO),YES)
    SUBDIRS += ginkgo
 endif
-ifeq ($(MFEM_USE_AMGX),YES)
-   SUBDIRS += amgx
-endif
-ifeq ($(MFEM_USE_SUPERLU),YES)
-   SUBDIRS += superlu
-endif
 
 SUBDIRS_ALL = $(addsuffix /all,$(SUBDIRS))
 SUBDIRS_TEST = $(addsuffix /test,$(SUBDIRS))
@@ -125,11 +119,6 @@ ex11p-test-superlu: ex11p
 	@$(call mfem-test,$<, $(RUN_MPI), SuperLU_DIST example,--superlu)
 test-par-YES: ex11p-test-superlu
 endif
-ifeq ($(MFEM_USE_MKL_CPARDISO),YES)
-ex11p-test-cpardiso: ex11p
-	@$(call mfem-test,$<, $(RUN_MPI), MKL_CPARDISO example,--cpardiso)
-test-par-YES: ex11p-test-cpardiso
-endif
 
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk
 

examples/multigrid.hpp

@@ -0,0 +1,517 @@
+#pragma once
+
+#include "mfem.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+STRUMPACKSolver* CreateStrumpackSolver(Operator *Arow, MPI_Comm comm)
+{
+   //STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, comm);
+   STRUMPACKSolver * strumpack = new STRUMPACKSolver(0, NULL, comm);
+   strumpack->SetPrintFactorStatistics(true);
+   strumpack->SetPrintSolveStatistics(false);
+   strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+   strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+   strumpack->SetOperator(*Arow);
+   strumpack->SetFromCommandLine();
+   return strumpack;
+}
+
+hypre_CSRMatrix* GetHypreParMatrixData(const HypreParMatrix & hypParMat)
+{
+   // First cast the parameter to a hypre_ParCSRMatrix
+   hypre_ParCSRMatrix * parcsr_op =
+      (hypre_ParCSRMatrix *)const_cast<HypreParMatrix&>(hypParMat);
+
+   MFEM_ASSERT(parcsr_op != NULL,"STRUMPACK: const_cast failed in SetOperator");
+
+   // Create the CSRMatrixMPI A_ by borrowing the internal data from a hypre_CSRMatrix.
+   return hypre_MergeDiagAndOffd(parcsr_op);
+}
+
+// Row and column offsets are assumed to be the same, for each process.
+// Array offsets stores process-local offsets with respect to the blocks. Process offsets are not included.
+HypreParMatrix* CreateHypreParMatrixFromBlocks(MPI_Comm comm,
+                                               Array<int> const& offsets, Array2D<HypreParMatrix*> const& blocks,
+                                               Array2D<SparseMatrix*> const& blocksSp,
+                                               Array2D<double> const& coefficient,
+                                               std::vector<std::vector<int> > const& blockProcOffsets,
+                                               std::vector<std::vector<int> > const& all_block_num_loc_rows)
+{
+   const int numBlocks = offsets.Size() - 1;
+   const int num_loc_rows = offsets[numBlocks];
+
+   int nprocs, rank;
+   MPI_Comm_rank(comm, &rank);
+   MPI_Comm_size(comm, &nprocs);
+
+   std::vector<int> all_num_loc_rows(nprocs);
+   std::vector<int> procOffsets(nprocs);
+   std::vector<std::vector<int> > procBlockOffsets(nprocs);
+
+   MPI_Allgather(&num_loc_rows, 1, MPI_INT, all_num_loc_rows.data(), 1, MPI_INT,
+                 comm);
+
+   int first_loc_row = 0;
+   int glob_nrows = 0;
+   procOffsets[0] = 0;
+   for (int i=0; i<nprocs; ++i)
+   {
+      glob_nrows += all_num_loc_rows[i];
+      if (i < rank)
+      {
+         first_loc_row += all_num_loc_rows[i];
+      }
+
+      if (i < nprocs-1)
+      {
+         procOffsets[i+1] = procOffsets[i] + all_num_loc_rows[i];
+      }
+
+      if (numBlocks > 0)
+      {
+         procBlockOffsets[i].resize(numBlocks);
+         procBlockOffsets[i][0] = 0;
+      }
+
+      for (int j=1; j<numBlocks; ++j)
+      {
+         procBlockOffsets[i][j] = procBlockOffsets[i][j-1] + all_block_num_loc_rows[j
+                                                                                    -1][i];
+      }
+   }
+
+   const int glob_ncols = glob_nrows;
+
+   std::vector<int> opI(num_loc_rows+1);
+   std::vector<int> cnt(num_loc_rows);
+
+   for (int i=0; i<num_loc_rows; ++i)
+   {
+      opI[i] = 0;
+      cnt[i] = 0;
+   }
+
+   opI[num_loc_rows] = 0;
+
+   Array2D<hypre_CSRMatrix*> csr_blocks(numBlocks, numBlocks);
+
+   // Loop over all blocks, to determine nnz for each row.
+   for (int i=0; i<numBlocks; ++i)
+   {
+      for (int j=0; j<numBlocks; ++j)
+      {
+         if (blocks(i, j) == NULL)
+         {
+            csr_blocks(i, j) = NULL;
+
+            if (blocksSp(i, j) != NULL)
+            {
+               const int nrows = blocksSp(i, j)->Height();
+               for (int k=0; k<nrows; ++k)
+               {
+                  const int rowg = offsets[i] + k;
+                  opI[rowg + 1] += blocksSp(i, j)->GetI()[k+1] - blocksSp(i, j)->GetI()[k];
+               }
+            }
+         }
+         else
+         {
+            MFEM_VERIFY(blocksSp(i, j) == NULL, "");
+
+            csr_blocks(i, j) = GetHypreParMatrixData(*(blocks(i, j)));
+
+            const int nrows = csr_blocks(i, j)->num_rows;
+
+            for (int k=0; k<nrows; ++k)
+            {
+               const int rowg = offsets[i] + k;
+               //(*(leftInjection(i, j)))[k]
+               opI[rowg + 1] += csr_blocks(i, j)->i[k+1] - csr_blocks(i, j)->i[k];
+            }
+         }
+      }
+   }
+
+   // Now opI[i] is nnz for row i-1. Do a partial sum to get offsets.
+   for (int i=0; i<num_loc_rows; ++i)
+   {
+      opI[i+1] += opI[i];
+   }
+
+   const int nnz = opI[num_loc_rows];
+
+   std::vector<HYPRE_Int> opJ(nnz);
+   std::vector<double> data(nnz);
+
+   // Loop over all blocks, to set matrix data.
+   for (int i=0; i<numBlocks; ++i)
+   {
+      for (int j=0; j<numBlocks; ++j)
+      {
+         if (csr_blocks(i, j) != NULL || blocksSp(i, j) != NULL)
+         {
+            const bool useCSR = (csr_blocks(i, j) != NULL);
+
+            const int nrows = useCSR ? csr_blocks(i, j)->num_rows : blocksSp(i,
+                                                                             j)->Height();
+            const double coef = coefficient(i, j);
+
+            int *Iarray = useCSR ? csr_blocks(i, j)->i : blocksSp(i, j)->GetI();
+
+            //const bool failure = (nrows != offsets[i+1] - offsets[i]);
+
+            MFEM_VERIFY(nrows == offsets[i+1] - offsets[i], "");
+
+            for (int k=0; k<nrows; ++k)
+            {
+               const int rowg = offsets[i] + k;  // process-local row
+               const int nnz_k = Iarray[k+1] - Iarray[k];
+               const int osk = Iarray[k];
+
+               for (int l=0; l<nnz_k; ++l)
+               {
+                  // Find the column process offset for the block.
+                  const int bcol = useCSR ? csr_blocks(i, j)->j[osk + l] : blocksSp(i,
+                                                                                    j)->GetJ()[osk + l];
+                  int bcolproc = 0;
+
+                  for (int p=1; p<nprocs; ++p)
+                  {
+                     if (blockProcOffsets[j][p] > bcol)
+                     {
+                        bcolproc = p-1;
+                        break;
+                     }
+                  }
+
+                  if (blockProcOffsets[j][nprocs - 1] <= bcol)
+                  {
+                     bcolproc = nprocs - 1;
+                  }
+
+                  const int colg = procOffsets[bcolproc] + procBlockOffsets[bcolproc][j] +
+                                   (bcol - blockProcOffsets[j][bcolproc]);
+
+                  if (colg < 0)
+                  {
+                     cout << "BUG, negative global column index" << endl;
+                  }
+
+                  opJ[opI[rowg] + cnt[rowg]] = colg;
+                  data[opI[rowg] + cnt[rowg]] = useCSR ? coef * csr_blocks(i,
+                                                                           j)->data[osk + l] : coef * blocksSp(i, j)->GetData()[osk + l];
+                  cnt[rowg]++;
+               }
+            }
+         }
+      }
+   }
+
+   bool cntCheck = true;
+   for (int i=0; i<num_loc_rows; ++i)
+   {
+      if (cnt[i] != opI[i+1] - opI[i])
+      {
+         cntCheck = false;
+      }
+   }
+
+   MFEM_VERIFY(cntCheck, "");
+
+   for (int i=0; i<numBlocks; ++i)
+   {
+      for (int j=0; j<numBlocks; ++j)
+      {
+         if (csr_blocks(i, j) != NULL)
+         {
+            hypre_CSRMatrixDestroy(csr_blocks(i, j));
+         }
+      }
+   }
+
+   std::vector<HYPRE_Int> rowStarts2(2);
+   rowStarts2[0] = first_loc_row;
+   rowStarts2[1] = first_loc_row + all_num_loc_rows[rank];
+
+   if (nnz > 0)
+   {
+      HYPRE_Int minJ = opJ[0];
+      HYPRE_Int maxJ = opJ[0];
+      for (int i=0; i<nnz; ++i)
+      {
+         minJ = std::min(minJ, opJ[i]);
+         maxJ = std::max(maxJ, opJ[i]);
+
+         if (opJ[i] >= glob_ncols)
+         {
+            cout << "Column indices out of range" << endl;
+         }
+      }
+   }
+
+   HypreParMatrix *hmat = new HypreParMatrix(comm, num_loc_rows, glob_nrows,
+                                             glob_ncols, (int*) opI.data(), (HYPRE_Int*) opJ.data(), (double*) data.data(),
+                                             (HYPRE_Int*) rowStarts2.data(), (HYPRE_Int*) rowStarts2.data());
+
+   return hmat;
+}
+
+class BlockMGSolver : public Solver
+{
+private:
+   /// The linear system matrix
+   Array2D<HypreParMatrix *>&
+   Af;  // TODO: remove this, as it is used only in the constructor
+   Array2D<double>&
+   Acoef;  // TODO: remove this, as it is used only in the constructor
+   vector<Array<int>> Aoffsets;
+   vector<Array<int>> Poffsets_i;
+   vector<Array<int>> Poffsets_j;
+   std::vector<Array2D<HypreParMatrix *>> A;
+   std::vector<HypreParMatrix *>& P;
+   std::vector<BlockOperator *> BlkP;
+   std::vector<BlockOperator *> BlkA;
+   std::vector<BlockOperator *> S;
+   HypreParMatrix * Ac;
+   int numGrids, numBlocks;
+   STRUMPACKSolver *invAc = nullptr;
+   double theta = 0.5;
+
+public:
+   BlockMGSolver(const int height, const int width, Array2D<HypreParMatrix *>& Af_,
+                 Array2D<double>& Acoef_, std::vector<HypreParMatrix *>& P_);
+
+   virtual void SetOperator(const Operator &op) {}
+
+   virtual void SetTheta(const double a) { theta = a; }
+
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~BlockMGSolver();
+};
+
+BlockMGSolver::BlockMGSolver(const int height, const int width,
+                             Array2D<HypreParMatrix *>& Af_, Array2D<double>& Acoef_,
+                             std::vector<HypreParMatrix *>& P_)
+   : Solver(height, width), Af(Af_), Acoef(Acoef_), P(P_)
+{
+   numBlocks = Af.NumRows();
+   MFEM_VERIFY(Af.NumCols() == numBlocks, "");
+   numGrids = P.size();
+   BlkP.resize(numGrids);
+   BlkA.resize(numGrids+1);
+   S.resize(numGrids);
+   A.resize(numGrids + 1);
+   A[numGrids] = Af;
+   Aoffsets.resize(numGrids+1);
+   Poffsets_i.resize(numGrids);
+   Poffsets_j.resize(numGrids);
+   // Construct Bilinear form Matrices on each level
+   for (int k = numGrids ; k > 0; k--)
+   {
+      A[k - 1].SetSize(numBlocks,numBlocks);
+      Aoffsets[k].SetSize(numBlocks+1); Aoffsets[k][0] = 0;
+      for (int i=0; i<numBlocks; i++)
+      {
+         Aoffsets[k][i+1] = A[k](i,i)->Height();
+      }
+
+      Aoffsets[k].PartialSum();
+      BlkA[k] = new BlockOperator(Aoffsets[k]);
+      S[k-1] = new BlockOperator(Aoffsets[k]);  // Smoother
+
+      for (int i=0; i<numBlocks; i++)
+      {
+         for (int j=0; j<numBlocks; j++)
+         {
+            if (A[k](i,j) == NULL)
+            {
+               A[k - 1](i,j) = NULL;
+            }
+            else
+            {
+               A[k - 1](i,j) = RAP(A[k](i,j), P[k - 1]);
+               BlkA[k]->SetBlock(i, j, A[k](i,j), Acoef(i,j));
+            }
+         }
+
+         HypreSmoother *S_i = new HypreSmoother;
+         S_i->SetType(HypreSmoother::Jacobi);
+         S_i->SetOperator(*(A[k](i,i)));
+
+         S[k - 1]->SetBlock(i,i,S_i);
+      }
+
+      Poffsets_i[k-1].SetSize(numBlocks+1); Poffsets_i[k-1][0] = 0;
+      Poffsets_j[k-1].SetSize(numBlocks+1); Poffsets_j[k-1][0] = 0;
+      for (int i=0; i<numBlocks; i++)
+      {
+         Poffsets_i[k-1][i+1] = P[k-1]->Height();
+         Poffsets_j[k-1][i+1] = P[k-1]->Width();
+      }
+      Poffsets_i[k-1].PartialSum();
+      Poffsets_j[k-1].PartialSum();
+
+      BlkP[k-1] = new BlockOperator(Poffsets_i[k-1],Poffsets_j[k-1]);
+      for (int i=0; i<numBlocks; i++)
+      {
+         BlkP[k-1]->SetBlock(i,i,P[k-1]);
+      }
+   }
+   // Set up coarse solve operator
+   // Convert the coarse grid blockmatrix to a HypreParMatrix
+   Array<int> offsets(numBlocks+1);
+   offsets[0]=0;
+   for (int i=0; i<numBlocks; i++)
+   {
+      offsets[i+1]=A[0](i,i)->Height();
+   }
+
+   offsets.PartialSum();
+
+   BlkA[0] = new BlockOperator(offsets);
+
+   Array2D<SparseMatrix*> Asp;
+   //Array2D<double> Acoef;
+   Asp.SetSize(numBlocks,numBlocks);
+   //Acoef.SetSize(numBlocks,numBlocks);
+   for (int i=0; i<numBlocks; i++)
+   {
+      for (int j=0; j<numBlocks; j++)
+      {
+         if (A[0](i,j) != NULL)
+         {
+            BlkA[0]->SetBlock(i, j, A[0](i,j), Acoef(i,j));
+         }
+
+         Asp(i,j) = NULL;
+         //Acoef(i,j) = 1.0;
+      }
+   }
+
+   // Convert to HypreParMatrix
+   HypreParMatrix * Ac;
+
+   std::vector<std::vector<int> > blockProcOffsets(numBlocks);
+   std::vector<std::vector<int> > all_block_num_loc_rows(numBlocks);
+
+   {
+      int nprocs, rank;
+      MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+      MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
+
+      std::vector<int> allnumrows(nprocs);
+      const int blockNumRows = A[0](0,0)->Height();
+      MPI_Allgather(&blockNumRows, 1, MPI_INT, allnumrows.data(), 1, MPI_INT,
+                    MPI_COMM_WORLD);
+
+      for (int b=0; b<numBlocks; ++b)
+      {
+         blockProcOffsets[b].resize(nprocs);
+         all_block_num_loc_rows[b].resize(nprocs);
+      }
+
+      blockProcOffsets[0][0] = 0;
+      for (int i=0; i<nprocs-1; ++i)
+      {
+         blockProcOffsets[0][i+1] = blockProcOffsets[0][i] + allnumrows[i];
+      }
+
+      for (int i=0; i<nprocs; ++i)
+      {
+         for (int b=0; b<numBlocks; ++b)
+         {
+            all_block_num_loc_rows[b][i] = allnumrows[i];
+         }
+
+         for (int b=1; b<numBlocks; ++b)
+         {
+            blockProcOffsets[b][i] = blockProcOffsets[0][i];
+         }
+      }
+   }
+
+   Ac = CreateHypreParMatrixFromBlocks(MPI_COMM_WORLD, offsets, A[0], Asp,
+                                       Acoef, blockProcOffsets, all_block_num_loc_rows);
+
+   invAc = CreateStrumpackSolver(new STRUMPACKRowLocMatrix(*Ac), MPI_COMM_WORLD);
+
+   delete Ac;
+}
+
+void BlockMGSolver::Mult(const Vector &r, Vector &z) const
+{
+   // Residual vectors
+   std::vector<Vector> rv(numGrids + 1);
+   // correction vectors
+   std::vector<Vector> zv(numGrids + 1);
+   // allocation
+   for (int i = 0; i <= numGrids ; i++)
+   {
+      int n = (i==0) ? invAc->Height(): BlkA[i]->Width();
+      rv[i].SetSize(n);
+      zv[i].SetSize(n);
+   }
+   // Initial residual
+   rv[numGrids] = r;
+
+   // smooth and update residuals down to the coarsest level
+   for (int i = numGrids; i > 0 ; i--)
+   {
+      // Pre smooth
+      S[i - 1]->Mult(rv[i], zv[i]); zv[i] *= theta;
+      // compute residual
+      int n = BlkA[i]->Width();
+      Vector w(n);
+      BlkA[i]->Mult(zv[i], w);
+      rv[i] -= w;
+      // Restrict
+      BlkP[i - 1]->MultTranspose(rv[i], rv[i - 1]);
+   }
+
+   // Coarse grid Solve
+   invAc->Mult(rv[0], zv[0]);
+   //
+   for (int i = 1; i <= numGrids ; i++)
+   {
+      // Prolong correction
+      Vector u(BlkP[i - 1]->Height());
+      BlkP[i - 1]->Mult(zv[i - 1], u);
+      // Update correction
+      zv[i] += u;
+      // Update residual
+      Vector v(BlkA[i]->Height());
+      BlkA[i]->Mult(u, v); rv[i] -= v;
+      // Post smooth
+      S[i - 1]->Mult(rv[i], v); v *= theta;
+      // Update correction
+      zv[i] += v;
+   }
+   z = zv[numGrids];
+}
+
+BlockMGSolver::~BlockMGSolver()
+{
+   for (int i = numGrids - 1; i >= 0 ; i--)
+   {
+      delete S[i];
+      delete BlkP[i];
+      delete BlkA[i];
+      for (int j=0; j<numBlocks; j++)
+      {
+         for (int k=0; k<numBlocks; k++)
+         {
+            delete A[i](j,k);
+         }
+      }
+      A[i].DeleteAll();
+   }
+   delete BlkA[numGrids];
+   delete invAc;
+   A.clear();
+}
+
+} // namespace mfem

examples/multigriddd.hpp

@@ -0,0 +1,869 @@
+#ifndef BGMULTIGRID
+#define BGMULTIGRID
+
+#include "mfem.hpp"
+
+using namespace std;
+
+//namespace mfem {
+namespace blockgmg
+{
+
+// Row and column offsets are assumed to be the same, for each process.
+// Array offsets stores process-local offsets with respect to the blocks. Process offsets are not included.
+HypreParMatrix* CreateHypreParMatrixFromBlocks2(MPI_Comm comm,
+                                                Array<int> const& offsets, Array2D<HypreParMatrix*> const& blocks,
+                                                Array2D<SparseMatrix*> const& blocksSp,
+                                                Array2D<double> const& coefficient,
+                                                std::vector<std::vector<int> > const& blockProcOffsets,
+                                                std::vector<std::vector<int> > const& all_block_num_loc_rows);
+
+class BlockMGSolver : public Solver
+{
+private:
+   /// The linear system matrix
+   Array2D<HypreParMatrix *>&
+   Af;  // TODO: remove this, as it is used only in the constructor
+   Array2D<double>&
+   Acoef;  // TODO: remove this, as it is used only in the constructor
+   vector<Array<int>> Aoffsets;
+   vector<Array<int>> Poffsets_i;
+   vector<Array<int>> Poffsets_j;
+   std::vector<Array2D<HypreParMatrix *>> A;
+   std::vector<HypreParMatrix *> const& P;
+   std::vector<BlockOperator *> BlkP;
+   std::vector<BlockOperator *> BlkA;
+   std::vector<BlockOperator *> S;
+   HypreParMatrix * Ac;
+   SparseMatrix AcSp;
+   int numGrids, numBlocks;
+   //STRUMPACKSolver *invAc = nullptr;
+   Operator *invAc = nullptr;
+   double theta = 0.5;
+
+public:
+   BlockMGSolver(MPI_Comm comm, const int height, const int width,
+                 Array2D<HypreParMatrix *>& Af_,
+                 Array2D<double>& Acoef_, std::vector<HypreParMatrix *> const& P_)
+      : Solver(height, width), Af(Af_), Acoef(Acoef_), P(P_)
+   {
+      numBlocks = Af.NumRows();
+      MFEM_VERIFY(Af.NumCols() == numBlocks, "");
+      numGrids = P.size();
+      BlkP.resize(numGrids);
+      BlkA.resize(numGrids+1);
+      S.resize(numGrids);
+      A.resize(numGrids + 1);
+      A[numGrids] = Af;
+      Aoffsets.resize(numGrids+1);
+      Poffsets_i.resize(numGrids);
+      Poffsets_j.resize(numGrids);
+      // Construct Bilinear form Matrices on each level
+      for (int k = numGrids ; k > 0; k--)
+      {
+         A[k - 1].SetSize(numBlocks,numBlocks);
+         Aoffsets[k].SetSize(numBlocks+1); Aoffsets[k][0] = 0;
+         for (int i=0; i<numBlocks; i++)
+         {
+            Aoffsets[k][i+1] = A[k](i,i)->Height();
+         }
+
+         Aoffsets[k].PartialSum();
+         BlkA[k] = new BlockOperator(Aoffsets[k]);
+         S[k-1] = new BlockOperator(Aoffsets[k]);  // Smoother
+
+         for (int i=0; i<numBlocks; i++)
+         {
+            for (int j=0; j<numBlocks; j++)
+            {
+               if (A[k](i,j) == NULL)
+               {
+                  A[k - 1](i,j) = NULL;
+               }
+               else
+               {
+                  A[k - 1](i,j) = RAP(A[k](i,j), P[k - 1]);
+                  BlkA[k]->SetBlock(i, j, A[k](i,j), Acoef(i,j));
+               }
+            }
+
+            HypreSmoother *S_i = new HypreSmoother;
+            S_i->SetType(HypreSmoother::Jacobi);
+            S_i->SetOperator(*(A[k](i,i)));
+
+            S[k - 1]->SetBlock(i,i,S_i);
+         }
+
+         Poffsets_i[k-1].SetSize(numBlocks+1); Poffsets_i[k-1][0] = 0;
+         Poffsets_j[k-1].SetSize(numBlocks+1); Poffsets_j[k-1][0] = 0;
+         for (int i=0; i<numBlocks; i++)
+         {
+            Poffsets_i[k-1][i+1] = P[k-1]->Height();
+            Poffsets_j[k-1][i+1] = P[k-1]->Width();
+         }
+         Poffsets_i[k-1].PartialSum();
+         Poffsets_j[k-1].PartialSum();
+
+         BlkP[k-1] = new BlockOperator(Poffsets_i[k-1],Poffsets_j[k-1]);
+         for (int i=0; i<numBlocks; i++)
+         {
+            BlkP[k-1]->SetBlock(i,i,P[k-1]);
+         }
+      }
+      // Set up coarse solve operator
+      // Convert the coarse grid blockmatrix to a HypreParMatrix
+      Array<int> offsets(numBlocks+1);
+      offsets[0]=0;
+      for (int i=0; i<numBlocks; i++)
+      {
+         offsets[i+1]=A[0](i,i)->Height();
+      }
+
+      offsets.PartialSum();
+
+      BlkA[0] = new BlockOperator(offsets);
+
+      Array2D<SparseMatrix*> Asp;
+      //Array2D<double> Acoef;
+      Asp.SetSize(numBlocks,numBlocks);
+      //Acoef.SetSize(numBlocks,numBlocks);
+      for (int i=0; i<numBlocks; i++)
+      {
+         for (int j=0; j<numBlocks; j++)
+         {
+            if (A[0](i,j) != NULL)
+            {
+               BlkA[0]->SetBlock(i, j, A[0](i,j), Acoef(i,j));
+            }
+
+            Asp(i,j) = NULL;
+            //Acoef(i,j) = 1.0;
+         }
+      }
+
+      // Convert to HypreParMatrix
+      //HypreParMatrix * Ac;
+
+      std::vector<std::vector<int> > blockProcOffsets(numBlocks);
+      std::vector<std::vector<int> > all_block_num_loc_rows(numBlocks);
+
+      {
+         int nprocs, rank;
+         MPI_Comm_rank(comm, &rank);
+         MPI_Comm_size(comm, &nprocs);
+
+         std::vector<int> allnumrows(nprocs);
+         const int blockNumRows = A[0](0,
+                                       0)->Height();    // TODO: Not valid if blocks are of different size
+         MPI_Allgather(&blockNumRows, 1, MPI_INT, allnumrows.data(), 1, MPI_INT, comm);
+
+         for (int b=0; b<numBlocks; ++b)
+         {
+            blockProcOffsets[b].resize(nprocs);
+            all_block_num_loc_rows[b].resize(nprocs);
+         }
+
+         blockProcOffsets[0][0] = 0;
+         for (int i=0; i<nprocs-1; ++i)
+         {
+            blockProcOffsets[0][i+1] = blockProcOffsets[0][i] + allnumrows[i];
+         }
+
+         for (int i=0; i<nprocs; ++i)
+         {
+            for (int b=0; b<numBlocks; ++b)
+            {
+               all_block_num_loc_rows[b][i] = allnumrows[i];
+            }
+
+            for (int b=1; b<numBlocks; ++b)
+            {
+               blockProcOffsets[b][i] = blockProcOffsets[0][i];
+            }
+         }
+      }
+
+      Ac = CreateHypreParMatrixFromBlocks2(comm, offsets, A[0], Asp,
+                                           Acoef, blockProcOffsets, all_block_num_loc_rows);
+
+#ifdef MFEM_USE_STRUMPACK
+      invAc = CreateStrumpackSolver(new STRUMPACKRowLocMatrix(*Ac), comm);
+      delete Ac;
+#else
+      Ac->GetDiag(AcSp);  // AcSp does not own the data
+      UMFPackSolver *umf_solver = new UMFPackSolver();
+      umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver->SetOperator(AcSp);
+      invAc = umf_solver;
+#endif
+   }
+
+   virtual void SetOperator(const Operator &op) {}
+
+   virtual void SetTheta(const double a) { theta = a; }
+
+   virtual void Mult(const Vector &r, Vector &z) const
+   {
+      // Residual vectors
+      std::vector<Vector> rv(numGrids + 1);
+      // correction vectors
+      std::vector<Vector> zv(numGrids + 1);
+      // allocation
+      for (int i = 0; i <= numGrids ; i++)
+      {
+         int n = (i==0) ? invAc->Height(): BlkA[i]->Width();
+         rv[i].SetSize(n);
+         zv[i].SetSize(n);
+      }
+      // Initial residual
+      rv[numGrids] = r;
+
+      // smooth and update residuals down to the coarsest level
+      for (int i = numGrids; i > 0 ; i--)
+      {
+         // Pre smooth
+         S[i - 1]->Mult(rv[i], zv[i]); zv[i] *= theta;
+         // compute residual
+         int n = BlkA[i]->Width();
+         Vector w(n);
+         BlkA[i]->Mult(zv[i], w);
+         rv[i] -= w;
+         // Restrict
+         BlkP[i - 1]->MultTranspose(rv[i], rv[i - 1]);
+      }
+
+      // Coarse grid Solve
+      invAc->Mult(rv[0], zv[0]);
+      //
+      for (int i = 1; i <= numGrids ; i++)
+      {
+         // Prolong correction
+         Vector u(BlkP[i - 1]->Height());
+         BlkP[i - 1]->Mult(zv[i - 1], u);
+         // Update correction
+         zv[i] += u;
+         // Update residual
+         Vector v(BlkA[i]->Height());
+         BlkA[i]->Mult(u, v); rv[i] -= v;
+         // Post smooth
+         S[i - 1]->Mult(rv[i], v); v *= theta;
+         // Update correction
+         zv[i] += v;
+      }
+      z = zv[numGrids];
+   }
+
+   virtual ~BlockMGSolver()
+   {
+      for (int i = numGrids - 1; i >= 0 ; i--)
+      {
+         delete S[i];
+         delete BlkP[i];
+         delete BlkA[i];
+         for (int j=0; j<numBlocks; j++)
+         {
+            for (int k=0; k<numBlocks; k++)
+            {
+               delete A[i](j,k);
+            }
+         }
+         A[i].DeleteAll();
+      }
+      delete BlkA[numGrids];
+      delete invAc;
+      A.clear();
+   }
+
+#ifdef MFEM_USE_STRUMPACK
+   STRUMPACKSolver* CreateStrumpackSolver(Operator *Arow, MPI_Comm comm)
+   {
+      //STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, comm);
+      STRUMPACKSolver * strumpack = new STRUMPACKSolver(0, NULL, comm);
+      strumpack->SetPrintFactorStatistics(true);
+      strumpack->SetPrintSolveStatistics(false);
+      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      strumpack->SetOperator(*Arow);
+      strumpack->SetFromCommandLine();
+      return strumpack;
+   }
+#endif
+
+};
+
+#define SWTIMING
+#define SERIAL_PROLONGATION
+#define ITERATIVE_COARSE_SOLVE
+#define SPARSE_JACOBI
+//#define SPARSE_ICHOLESKY
+//#define SPARSE_ILU
+//#define COARSE_PA
+//#define BLOCK_DIAG
+
+//#define COARSE_AMS
+
+class BlockMGPASolver : public Solver
+{
+private:
+   /// The linear system matrix
+   Array2D<Operator*>&
+   Af;  // TODO: remove this, as it is used only in the constructor
+   Array2D<double>&
+   Acoef;  // TODO: remove this, as it is used only in the constructor
+   vector<Array<int>> Aoffsets;
+   vector<Array<int>> Poffsets_i;
+   vector<Array<int>> Poffsets_j;
+   Array<int> Coffsets;
+
+   std::vector<Array2D<Operator*>> A;
+#ifdef SERIAL_PROLONGATION
+   std::vector<SparseMatrix *> P;
+#else
+   std::vector<HypreParMatrix *> const& P;
+#endif
+   std::vector<Vector*> const& diag;
+   std::vector<Operator*> Pt;
+   std::vector<BlockOperator *> BlkP;
+   std::vector<BlockOperator *> BlkA;
+   std::vector<BlockOperator *> S;
+   std::vector<OperatorJacobiSmoother*> Jacobi;
+
+   HypreParMatrix * Ac;
+   SparseMatrix AcSp;
+   int numGrids, numBlocks;
+   //STRUMPACKSolver *invAc = nullptr;
+   Operator *invAc = nullptr;
+   double theta = 0.5;
+
+   mutable std::vector<Vector> rv, zv;
+   mutable Vector u, v, w;
+
+   Vector diagAc;
+   Array<int> emptyEssDof;
+   OperatorJacobiSmoother *JacobiAc;
+#ifdef SPARSE_ICHOLESKY
+   Solver *iCholAc;
+#endif
+#ifdef SPARSE_ILU
+   Solver *iluAc;
+#endif
+
+#ifdef BLOCK_DIAG
+   HypreParMatrix *AcDiag;
+   SparseMatrix AcDiagSp;
+#endif
+
+public:
+   BlockMGPASolver(MPI_Comm comm, const int height, const int width,
+                   Array2D<Operator*>& Af_, Array2D<double>& Acoef_,
+#ifdef COARSE_PA
+                   BlockOperator *BlkAc,
+#else
+                   Array2D<HypreParMatrix*> const& BlkAc,
+#endif
+#ifdef COARSE_AMS
+                   ParFiniteElementSpace* coarseFespace,
+#endif
+                   std::vector<HypreParMatrix *> const& P_, std::vector<Vector*> const& diag_,
+                   Array<int>& ess_tdof_list)
+#ifdef SERIAL_PROLONGATION
+      : Solver(height, width), Af(Af_), Acoef(Acoef_), diag(diag_)
+#else
+      : Solver(height, width), Af(Af_), Acoef(Acoef_), P(P_), diag(diag_)
+#endif
+   {
+      timeMult = 0.0;
+      timeMultAc = 0.0;
+      timeMultPresmooth = 0.0;
+      timeMultResidual = 0.0;
+      timeMultRestrict = 0.0;
+
+#ifdef SERIAL_PROLONGATION
+      P.resize(P_.size());
+      for (int i=0; i<P_.size(); ++i)
+      {
+         P[i] = new SparseMatrix();
+         P_[i]->GetDiag(*P[i]);
+         P[i]->BuildTranspose();
+      }
+#endif
+
+      numBlocks = Af.NumRows();
+      MFEM_VERIFY(Af.NumCols() == numBlocks, "");
+#ifndef COARSE_PA
+      MFEM_VERIFY(BlkAc.NumCols() == numBlocks && BlkAc.NumRows() == numBlocks, "");
+#endif
+      numGrids = P.size();
+      MFEM_VERIFY(diag.size() == numBlocks, "");
+      BlkP.resize(numGrids);
+      BlkA.resize(numGrids+1);
+      Pt.resize(numGrids);
+      S.resize(numGrids);
+      A.resize(numGrids + 1);
+      A[numGrids] = Af;
+      Aoffsets.resize(numGrids+1);
+      Poffsets_i.resize(numGrids);
+      Poffsets_j.resize(numGrids);
+
+      Jacobi.resize(numBlocks);
+      for (int i=0; i<numBlocks; ++i)
+      {
+         Jacobi[i] = new OperatorJacobiSmoother(*(diag[i]), ess_tdof_list, 1.0);
+      }
+
+      // Construct Bilinear form Matrices on each level
+      for (int k = numGrids ; k > 0; k--)
+      {
+         A[k - 1].SetSize(numBlocks,numBlocks);
+         Aoffsets[k].SetSize(numBlocks+1); Aoffsets[k][0] = 0;
+         for (int i=0; i<numBlocks; i++)
+         {
+            Aoffsets[k][i+1] = A[k](i,i)->Height();
+         }
+
+         Aoffsets[k].PartialSum();
+         BlkA[k] = new BlockOperator(Aoffsets[k]);
+         S[k-1] = new BlockOperator(Aoffsets[k]);  // Smoother
+         Pt[k - 1] = new TransposeOperator(P[k - 1]);
+
+         for (int i=0; i<numBlocks; i++)
+         {
+            for (int j=0; j<numBlocks; j++)
+            {
+               if (A[k](i,j) == NULL)
+               {
+                  A[k - 1](i,j) = NULL;
+               }
+               else
+               {
+                  //A[k - 1](i,j) = RAP(A[k](i,j), P[k - 1]);
+                  A[k - 1](i,j) = new TripleProductOperator(Pt[k - 1], A[k](i,j), P[k - 1], false,
+                                                            false, false);
+                  BlkA[k]->SetBlock(i, j, A[k](i,j), Acoef(i,j));
+               }
+            }
+
+            if (k == numGrids)  // finest level
+            {
+               S[k - 1]->SetBlock(i,i,Jacobi[i]);
+            }
+            else
+            {
+               S[k - 1]->SetBlock(i,i, new TripleProductOperator(Pt[k], &(S[k]->GetBlock(i,i)),
+                                                                 P[k], false,
+                                                                 false, false));   // TODO: get a diagonal on each level
+            }
+         }
+
+         Poffsets_i[k-1].SetSize(numBlocks+1); Poffsets_i[k-1][0] = 0;
+         Poffsets_j[k-1].SetSize(numBlocks+1); Poffsets_j[k-1][0] = 0;
+         for (int i=0; i<numBlocks; i++)
+         {
+            Poffsets_i[k-1][i+1] = P[k-1]->Height();
+            Poffsets_j[k-1][i+1] = P[k-1]->Width();
+         }
+         Poffsets_i[k-1].PartialSum();
+         Poffsets_j[k-1].PartialSum();
+
+         BlkP[k-1] = new BlockOperator(Poffsets_i[k-1],Poffsets_j[k-1]);
+         for (int i=0; i<numBlocks; i++)
+         {
+            BlkP[k-1]->SetBlock(i,i,P[k-1]);
+         }
+      }
+      // Set up coarse solve operator
+      // Convert the coarse grid blockmatrix to a HypreParMatrix
+      Coffsets.SetSize(numBlocks+1);
+      Coffsets[0]=0;
+      for (int i=0; i<numBlocks; i++)
+      {
+         Coffsets[i+1]=A[0](i,i)->Height();
+#ifndef COARSE_PA
+         MFEM_VERIFY(BlkAc(i,i)->Height() == A[0](i,i)->Height(), "");
+         MFEM_VERIFY(BlkAc(i,i)->Width() == A[0](i,i)->Width(), "");
+#endif
+      }
+      Coffsets.PartialSum();
+
+#ifdef COARSE_PA
+      BlkA[0] = BlkAc;
+#else
+      BlkA[0] = new BlockOperator(Coffsets);
+#endif
+
+      Array2D<SparseMatrix*> Asp;
+      //Array2D<double> Acoef;
+      Asp.SetSize(numBlocks,numBlocks);
+      //Acoef.SetSize(numBlocks,numBlocks);
+      for (int i=0; i<numBlocks; i++)
+      {
+         for (int j=0; j<numBlocks; j++)
+         {
+#ifndef COARSE_PA
+            MFEM_VERIFY((A[0](i,j) == NULL) == (BlkAc(i,j) == NULL), "");
+
+            if (BlkAc(i,j) != NULL)
+            {
+               BlkA[0]->SetBlock(i, j, BlkAc(i,j), Acoef(i,j));
+            }
+#endif
+
+            Asp(i,j) = NULL;
+            //Acoef(i,j) = 1.0;
+         }
+      }
+
+#ifdef COARSE_PA
+      CGSolver *cg_solver = new CGSolver();
+      cg_solver->SetAbsTol(1.0e-6);
+      cg_solver->SetRelTol(1.0e-6);
+      cg_solver->SetMaxIter(1000);
+      cg_solver->SetOperator(*BlkAc);
+      cg_solver->SetPrintLevel(0);
+      cg_solver->iterative_mode = false;
+      invAc = cg_solver;
+#else
+      // Convert to HypreParMatrix
+      HypreParMatrix * Ac;
+
+      std::vector<std::vector<int> > blockProcOffsets(numBlocks);
+      std::vector<std::vector<int> > all_block_num_loc_rows(numBlocks);
+
+      {
+         int nprocs, rank;
+         MPI_Comm_rank(comm, &rank);
+         MPI_Comm_size(comm, &nprocs);
+
+         std::vector<int> allnumrows(nprocs);
+         // TODO: Not valid if blocks are of different size
+         const int blockNumRows = BlkAc(0,0)->Height();
+
+         MPI_Allgather(&blockNumRows, 1, MPI_INT, allnumrows.data(), 1, MPI_INT, comm);
+
+         for (int b=0; b<numBlocks; ++b)
+         {
+            blockProcOffsets[b].resize(nprocs);
+            all_block_num_loc_rows[b].resize(nprocs);
+         }
+
+         blockProcOffsets[0][0] = 0;
+         for (int i=0; i<nprocs-1; ++i)
+         {
+            blockProcOffsets[0][i+1] = blockProcOffsets[0][i] + allnumrows[i];
+         }
+
+         for (int i=0; i<nprocs; ++i)
+         {
+            for (int b=0; b<numBlocks; ++b)
+            {
+               all_block_num_loc_rows[b][i] = allnumrows[i];
+            }
+
+            for (int b=1; b<numBlocks; ++b)
+            {
+               blockProcOffsets[b][i] = blockProcOffsets[0][i];
+            }
+         }
+      }
+
+      Ac = CreateHypreParMatrixFromBlocks2(comm, Coffsets, BlkAc, Asp,
+                                           Acoef, blockProcOffsets, all_block_num_loc_rows);
+
+#ifdef BLOCK_DIAG
+      {
+         Array2D<HypreParMatrix*> BlkAcDiag(numBlocks, numBlocks);
+         Array2D<double> DiagCoef(numBlocks, numBlocks);
+         DiagCoef = 0.0;
+         for (int i=0; i<numBlocks; ++i)
+         {
+            DiagCoef(i,i) = Acoef(i,i);
+            for (int j=0; j<numBlocks; ++j)
+            {
+               BlkAcDiag(i,j) = NULL;
+            }
+
+            BlkAcDiag(i,i) = BlkAc(i,i);
+         }
+
+         AcDiag = CreateHypreParMatrixFromBlocks2(comm, Coffsets, BlkAcDiag, Asp,
+                                                  DiagCoef, blockProcOffsets, all_block_num_loc_rows);
+
+         AcDiag->GetDiag(AcDiagSp);
+         //delete AcDiag;
+      }
+#endif
+
+#ifdef MFEM_USE_STRUMPACK
+      invAc = CreateStrumpackSolver(new STRUMPACKRowLocMatrix(*Ac), comm);
+      delete Ac;
+#else
+      Ac->GetDiag(AcSp);  // AcSp does not own the data
+      delete Ac;
+
+#ifdef ITERATIVE_COARSE_SOLVE
+      //CGSolver *cg_solver = new CGSolver(comm);
+      CGSolver *cg_solver = new CGSolver();
+      cg_solver->SetAbsTol(1.0e-6);
+      cg_solver->SetRelTol(1.0e-6);
+      cg_solver->SetMaxIter(1000);
+      cg_solver->SetOperator(AcSp);
+      cg_solver->SetPrintLevel(0);
+      cg_solver->iterative_mode = false;
+#ifdef SPARSE_JACOBI
+      AcSp.GetDiag(diagAc);
+      JacobiAc = new OperatorJacobiSmoother(diagAc, emptyEssDof);
+      cg_solver->SetPreconditioner(*JacobiAc);
+#endif
+#ifdef SPARSE_ICHOLESKY
+      {
+         Vector tmpX(AcSp.Height());
+         Vector tmpY(AcSp.Height());
+         tmpX = 1.0;
+         tmpY = 0.0;
+#ifdef BLOCK_DIAG
+         AcDiagSp.Finalize();
+         AcDiagSp.SortColumnIndices();
+
+         AcDiagSp.Mult(tmpX, tmpY);
+#else
+         AcSp.Finalize();
+         AcSp.SortColumnIndices();
+
+         AcSp.Mult(tmpX, tmpY);
+#endif
+      }
+#ifdef BLOCK_DIAG
+      iCholAc = new IncompleteCholesky(AcDiagSp);
+#else
+      iCholAc = new IncompleteCholesky(AcSp);
+#endif
+      cg_solver->SetPreconditioner(*iCholAc);
+      cg_solver->SetPrintLevel(-1);
+#endif // SPARSE_ICHOLESKY
+
+#ifdef SPARSE_ILU
+      {
+         Vector tmpX(AcSp.Height());
+         Vector tmpY(AcSp.Height());
+         tmpX = 1.0;
+         tmpY = 0.0;
+#ifdef BLOCK_DIAG
+         AcDiagSp.Finalize();
+         AcDiagSp.SortColumnIndices();
+
+         AcDiagSp.Mult(tmpX, tmpY);
+#else
+         AcSp.Finalize();
+         AcSp.SortColumnIndices();
+
+         AcSp.Mult(tmpX, tmpY);
+#endif
+      }
+#ifdef BLOCK_DIAG
+      iluAc = new ILUcusparse(AcDiagSp);
+#else
+      iluAc = new ILUcusparse(AcSp);
+#endif
+      cg_solver->SetPreconditioner(*iluAc);
+      cg_solver->SetPrintLevel(0);
+#endif // SPARSE_ILU
+
+#ifdef COARSE_AMS
+#ifndef COARSE_PA
+      MFEM_VERIFY(numBlocks == 4, "");
+      // TODO: just set 2 AMS solvers for E and H.
+
+      if (coarseFespace != NULL)
+      {
+         BlockDiagonalPreconditioner *blkAMS = new BlockDiagonalPreconditioner(Coffsets);
+
+         for (int i=0; i<numBlocks; ++i)
+         {
+            HypreAMS *ams = new HypreAMS(*BlkAc(i,i), coarseFespace);
+            blkAMS->SetDiagonalBlock(i, ams);
+         }
+
+         cg_solver->SetPreconditioner(*blkAMS);
+      }
+#endif
+#endif
+
+      invAc = cg_solver;
+#else
+      UMFPackSolver *umf_solver = new UMFPackSolver();
+      umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver->SetOperator(AcSp);
+      invAc = umf_solver;
+#endif
+#endif
+#endif  // COARSE_PA
+
+      // Residual vectors
+      rv.resize(numGrids + 1);
+      // correction vectors
+      zv.resize(numGrids + 1);
+      // allocation
+      for (int i = 0; i <= numGrids ; i++)
+      {
+         int n = (i==0) ? invAc->Height(): BlkA[i]->Width();
+
+         rv[i].SetSize(n);
+         zv[i].SetSize(n);
+         rv[i].UseDevice(true);
+         zv[i].UseDevice(true);
+      }
+   }
+
+   virtual void SetOperator(const Operator &op) {}
+
+   virtual void SetTheta(const double a) { theta = a; }
+
+   virtual void Mult(const Vector &r, Vector &z) const
+   {
+#ifdef SWTIMING
+      StopWatch sw;
+      sw.Clear();
+      sw.Start();
+#endif
+
+      // Initial residual
+      rv[numGrids] = r;
+
+      // smooth and update residuals down to the coarsest level
+      for (int i = numGrids; i > 0 ; i--)
+      {
+         // Pre smooth
+#ifdef SWTIMING
+         StopWatch sws;
+         sws.Clear();
+         sws.Start();
+#endif
+
+         S[i - 1]->Mult(rv[i], zv[i]); zv[i] *= theta;
+
+#ifdef SWTIMING
+         sws.Stop();
+         timeMultPresmooth += sws.RealTime();
+#endif
+
+         // compute residual
+         int n = BlkA[i]->Width();
+         w.SetSize(n);
+         w.UseDevice(true);
+#ifdef SWTIMING
+         StopWatch swop;
+         swop.Clear();
+         swop.Start();
+#endif
+         BlkA[i]->Mult(zv[i], w);
+         rv[i] -= w;
+#ifdef SWTIMING
+         swop.Stop();
+         timeMultResidual += swop.RealTime();
+#endif
+         // Restrict
+#ifdef SWTIMING
+         StopWatch swr;
+         swr.Clear();
+         swr.Start();
+#endif
+         BlkP[i - 1]->MultTranspose(rv[i], rv[i - 1]);
+#ifdef SWTIMING
+         swr.Stop();
+         timeMultRestrict += swr.RealTime();
+#endif
+      }
+
+#ifdef SWTIMING
+      StopWatch swAc;
+      swAc.Clear();
+      swAc.Start();
+#endif
+
+      // Coarse grid Solve
+      invAc->Mult(rv[0], zv[0]);
+      //
+
+#ifdef SWTIMING
+      swAc.Stop();
+      timeMultAc += swAc.RealTime();
+#endif
+
+      for (int i = 1; i <= numGrids ; i++)
+      {
+         // Prolong correction
+         u.SetSize(BlkP[i - 1]->Height());
+         u.UseDevice(true);
+
+         BlkP[i - 1]->Mult(zv[i - 1], u);
+         // Update correction
+         zv[i] += u;
+         // Update residual
+         v.SetSize(BlkA[i]->Height());
+         v.UseDevice(true);
+
+         BlkA[i]->Mult(u, v); rv[i] -= v;
+         // Post smooth
+         S[i - 1]->Mult(rv[i], v); v *= theta;
+         // Update correction
+         zv[i] += v;
+      }
+      z = zv[numGrids];
+
+#ifdef SWTIMING
+      sw.Stop();
+      timeMult += sw.RealTime();
+#endif
+   }
+
+   virtual ~BlockMGPASolver()
+   {
+      for (int i = numGrids - 1; i >= 0 ; i--)
+      {
+         delete S[i];
+         delete BlkP[i];
+         delete BlkA[i];
+         for (int j=0; j<numBlocks; j++)
+         {
+            for (int k=0; k<numBlocks; k++)
+            {
+               delete A[i](j,k);
+            }
+         }
+         A[i].DeleteAll();
+      }
+      delete BlkA[numGrids];
+      delete invAc;
+      A.clear();
+
+#ifdef SPARSE_ICHOLESKY
+      delete iCholAc;
+#endif
+#ifdef SPARSE_ILU
+      delete iluAc;
+#endif
+   }
+
+#ifdef MFEM_USE_STRUMPACK
+   STRUMPACKSolver* CreateStrumpackSolver(Operator *Arow, MPI_Comm comm)
+   {
+      //STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, comm);
+      STRUMPACKSolver * strumpack = new STRUMPACKSolver(0, NULL, comm);
+      strumpack->SetPrintFactorStatistics(true);
+      strumpack->SetPrintSolveStatistics(false);
+      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      strumpack->SetOperator(*Arow);
+      strumpack->SetFromCommandLine();
+      return strumpack;
+   }
+#endif
+
+   mutable double timeMult, timeMultAc, timeMultPresmooth, timeMultResidual,
+   timeMultRestrict;
+};
+
+}
+//} // namespace mfem
+
+#endif // BGMULTIGRID

examples/petsc/README

@@ -5,7 +5,7 @@
                       | | | | | ||  _||  __/| | | | | |
                       |_| |_| |_||_|   \___||_| |_| |_|
 
-                               https://mfem.org
+                               http://mfem.org
 
 This directory contains modifications of the example codes that illustrate the
 use of MFEM features based on the PETSc suite.

examples/petsc/ex11p.cpp

@@ -410,16 +410,21 @@ int main(int argc, char *argv[])
    }
 
    // 12. Free the used memory.
-   delete lobpcg;
-   delete slepc;
+   if (!use_slepc)
+   {
+      delete lobpcg;
+   }
+   else
+   {
+      delete slepc;
+   }
    delete precond;
    delete M;
    delete A;
-   delete pA;
-   delete pM;
 #if defined(MFEM_USE_SUPERLU) || defined(MFEM_USE_STRUMPACK)
    delete Arow;
 #endif
+
    delete fespace;
    if (order > 0)
    {

examples/pumi/README

@@ -5,7 +5,7 @@
                       | | | | | ||  _||  __/| | | | | |
                       |_| |_| |_||_|   \___||_| |_| |_|
 
-                               https://mfem.org
+                               http://mfem.org
 
 This directory contains modifications of the example codes that illustrate the
 use of MFEM features based on the Parallel Unstructured Mesh Infrastructure,

examples/remapBlockMatrix.cpp

@@ -0,0 +1,363 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <vector>
+#include <cstdlib>
+#include <cmath>
+
+using namespace std;
+
+class HypreMat
+{
+private:
+   std::vector<int> I, J;
+   std::vector<double> d;
+   int size;
+
+public:
+   HypreMat() : size(0)
+   {
+
+   }
+
+   int Size() const { return size; }
+
+   void ReadMatrixParallel(std::string filename, const int np)
+   {
+      int lastRow = 0;
+      int count = 0;
+
+      std::vector<int> R;
+
+      for (int p=0; p<np; ++p)
+      {
+         std::ifstream f(filename.c_str() + std::to_string(p), std::ifstream::in);
+
+         int rowFirst, rowLast, colFirst, colLast, row, col;
+         double v;
+
+         f >> rowFirst >> rowLast >> colFirst >> colLast;
+
+         I.resize(rowLast+2);
+
+         for (row=rowFirst+1; row<=rowLast+1; ++row)
+         {
+            I[row] = 0;
+         }
+
+         while (f.good())
+         {
+            f >> row >> col >> v;
+
+            if (row >= 2485)
+            {
+               abort();
+            }
+
+            //if (row < lastRow)  // verify ascending order of rows
+            //  abort();
+
+            lastRow = row;
+
+            I[row+1]++;
+            R.push_back(row);
+            J.push_back(col);
+            d.push_back(v);
+
+            count++;
+         }
+
+         f.close();
+      }
+
+      // Partial sum of I
+
+      size = I.size() - 1;
+
+      I[0] = 0;
+      for (int j=1; j<size; ++j)
+      {
+         I[j+1] += I[j];
+      }
+
+      //int *Idata = I.data();
+
+      if (I[size] != J.size() || I[size] != d.size())
+      {
+         abort();
+      }
+
+      {
+         // Reorder J and d
+         const int nnz = J.size();
+         if (nnz != d.size() || nnz != R.size())
+         {
+            abort();
+         }
+
+         std::vector<int> JJ(nnz);
+         std::vector<double> dd(nnz);
+         std::vector<int> cnt;
+
+         JJ = J;
+         dd = d;
+
+         cnt.assign(I.size(), 0); // slightly larger than necessary
+
+         for (int j=0; j<nnz; ++j)
+         {
+            J[I[R[j]] + cnt[R[j]]] = JJ[j];
+            d[I[R[j]] + cnt[R[j]]] = dd[j];
+            cnt[R[j]]++;
+         }
+      }
+   }
+
+   void CopyReorder(HypreMat const& A, std::vector<int> const& permRow,
+                    std::vector<int> const& permCol)
+   {
+      if (permRow.size() != A.Size() || permCol.size() != A.Size())
+      {
+         abort();
+      }
+
+      size = A.Size();
+      I.resize(size+1);
+
+      I[0] = 0;
+
+      for (int j=0; j<size; ++j)
+      {
+         const std::size_t nnz_j = A.I[j+1] - A.I[j];
+         I[permRow[j]+1] = nnz_j;
+      }
+
+      for (int j=1; j<size; ++j)
+      {
+         I[j+1] += I[j];
+      }
+
+      const std::size_t nnz = I[size];
+      J.resize(nnz);
+      d.resize(nnz);
+
+      std::vector<std::size_t> cnt;
+      cnt.assign(size, 0);
+
+      for (int j=0; j<size; ++j)
+      {
+         const int pj = permRow[j];
+         const std::size_t nnz_j = A.I[j+1] - A.I[j];
+
+         for (int k=0; k<nnz_j; ++k, cnt[pj]++)
+         {
+            J[I[pj] + cnt[pj]] = permCol[A.J[A.I[j] + k]];
+            d[I[pj] + cnt[pj]] = A.d[A.I[j] + k];
+         }
+      }
+   }
+
+   void Print(std::string filename)
+   {
+      std::ofstream f(filename.c_str());
+
+      for (int j=0; j<size; ++j)
+      {
+         for (int k=I[j]; k<I[j+1]; ++k)
+         {
+            f << j << " " << J[k] << " " << d[k] << endl;
+         }
+      }
+
+      f.close();
+   }
+
+   void Compare(HypreMat const& A)
+   {
+      if (size != A.Size())
+      {
+         abort();
+      }
+
+      for (int j=0; j<size; ++j)
+      {
+         for (int k=I[j]; k<I[j+1]; ++k)
+         {
+            int m = -1;
+
+            for (int l=A.I[j]; l<A.I[j+1]; ++l)
+            {
+               if (A.J[l] == J[k])
+               {
+                  m = l;
+               }
+            }
+
+            if (m < 0)
+            {
+               cout << "row " << j << " column " << J[k] << " not found" << endl;
+            }
+            //else if (fabs(A.d[m] - d[k]) > 1.0e-8)
+            else if (fabs(fabs(A.d[m]) - fabs(d[k])) > 0.0)
+            {
+               cout << "entry (" << j << ", " << J[k] << ") " << d[k] << " != " << A.d[m] <<
+                    endl;
+            }
+         }
+      }
+   }
+};
+
+void ReadCrd(std::string filename, std::vector<double>& crd)
+{
+   std::ifstream f(filename.c_str(), std::ifstream::in);
+
+   double c;
+
+   while (f.good())
+   {
+      f >> c;
+      crd.push_back(c);
+   }
+
+   // for some reason, the last value gets pushed twice
+   crd.pop_back();
+
+   f.close();
+}
+
+void ReadCrdParallel(std::string filenamebase, const int np,
+                     std::vector<double>& crd)
+{
+   for (int p=0; p<np; ++p)
+   {
+      ReadCrd(filenamebase + std::to_string(p),
+              crd);  // results get concatenated in crd
+   }
+}
+
+// crd is the DOF coordinates in serial, crdp in parallel.
+void SerialToParallelDOFMap(std::vector<double> const& crd,
+                            std::vector<double> const& crdp,
+                            std::vector<int>& sp)
+{
+   if (crdp.size() != crd.size())
+   {
+      abort();
+   }
+
+   const int dim = 3;
+   const int n = crd.size() / dim;  // number of DOF's
+
+   sp.resize(n);
+
+   const double tol = 1.0e-8;
+
+   // Simple O(n^2) search
+
+   for (int i=0; i<n; ++i)
+   {
+      sp[i] = -1;
+
+      for (int j=0; j<n; ++j)
+      {
+         bool eq = true;
+         for (int l=0; l<dim; ++l)
+         {
+            if (fabs(crd[(dim*i)+l] - crdp[(dim*j)+l]) > tol)
+            {
+               eq = false;
+            }
+         }
+
+         if (eq)
+         {
+            if (sp[i] != -1)
+            {
+               abort();
+            }
+
+            sp[i] = j;
+         }
+      }
+
+      if (sp[i] < 0)
+      {
+         abort();
+      }
+   }
+}
+
+int main()
+{
+   const int np = 2;
+
+   std::vector<double> crd0, crdp0, crd1, crdp1;
+
+   ReadCrd("dofcrd0Ser", crd0);
+   ReadCrd("dofcrd1Ser", crd1);
+
+   const int numCrd = crd0.size() / 3;
+
+   if (3*numCrd != crd0.size())
+   {
+      abort();
+   }
+
+   //for (int i=7440; i<crd.size(); ++i)
+   //cout << "crd[" << i << "] " << crd[i] << endl;
+
+   ReadCrdParallel("dofcrd0Par", np, crdp0);
+   ReadCrdParallel("dofcrd1Par", np, crdp1);
+
+   if (3*numCrd != crdp0.size())
+   {
+      abort();
+   }
+
+   std::vector<int> sp0, sp1;  // serial to parallel DOF map for a subdomain
+   SerialToParallelDOFMap(crd0, crdp0, sp0);
+   SerialToParallelDOFMap(crd1, crdp1, sp1);
+
+   /*
+   for (int i=0; i<sp.size(); ++i)
+     cout << i << " " << sp[i] << endl;
+   */
+
+   cout << "sp0" << endl;
+   for (int i=0; i<sp0.size(); ++i)
+   {
+      cout << i << " " << sp0[i] << endl;
+   }
+
+   cout << "sp1" << endl;
+   for (int i=0; i<sp1.size(); ++i)
+   {
+      cout << i << " " << sp1[i] << endl;
+   }
+
+   HypreMat Aser, Apar, AserToPar;
+
+   /*
+   Aser.ReadMatrixParallel("HypreAsdComplexIm1_Serial.0000", 1);
+   Apar.ReadMatrixParallel("HypreAsdComplexIm1_Par5.0000", np);
+   */
+
+   Aser.ReadMatrixParallel("ifopSer.0000", 1);
+   Apar.ReadMatrixParallel("ifopPar.0000", np);
+
+   //AserToPar.CopyReorder(Aser, sp, sp);
+   AserToPar.CopyReorder(Aser, sp0, sp1);
+
+   /*
+   Aser.Print("rbSer");
+   AserToPar.Print("rbSerToPar");
+   Apar.Print("rbPar");
+   */
+
+   cout << "Comparing " << endl;
+
+   AserToPar.Compare(Apar);
+
+   return 0;
+}

examples/sundials/README

@@ -5,7 +5,7 @@
                       | | | | | ||  _||  __/| | | | | |
                       |_| |_| |_||_|   \___||_| |_| |_|
 
-                               https://mfem.org
+                               http://mfem.org
 
 This directory contains modifications of the example codes that illustrate the
 use of MFEM features based on the SUNDIALS suite of time integration and

examples/sundials/ex10p.cpp

@@ -721,14 +721,12 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
 
    if (nls_type == KINSOL)
    {
-      KINSolver *kinsolver = new KINSolver(f.GetComm(), KIN_LINESEARCH, true);
-      kinsolver->SetJFNK(true);
-      kinsolver->SetLSMaxIter(100);
+      KINSolver *kinsolver = new KINSolver(f.GetComm(), KIN_NONE, true);
       newton_solver = kinsolver;
       newton_solver->SetOperator(*reduced_oper);
       newton_solver->SetMaxIter(200);
       newton_solver->SetRelTol(rel_tol);
-      newton_solver->SetPrintLevel(1);
+      newton_solver->SetPrintLevel(0);
       kinsolver->SetMaxSetupCalls(4);
    }
    else

examples/sundials/ex9.cpp

@@ -13,27 +13,17 @@
 //    ex9 -m ../../data/disc-nurbs.mesh       -p 1 -r 3 -s 7 -dt 0.005 -tf 9
 //    ex9 -m ../../data/periodic-cube.mesh    -p 0 -r 2 -s 8 -dt 0.02 -tf 8 -o 2
 //
-// Device sample runs:
-//    ex9 -pa
-//    ex9 -ea
-//    ex9 -fa
-//    ex9 -pa -m ../../data/periodic-cube.mesh
-//    ex9 -pa -m ../../data/periodic-cube.mesh -d cuda
-//    ex9 -ea -m ../../data/periodic-cube.mesh -d cuda
-//    ex9 -fa -m ../../data/periodic-cube.mesh -d cuda
-//
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
 //               u0(x)=u(0,x) is a given initial condition.
 //
 //               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of implicit
-//               and explicit ODE time integrators, the definition of periodic
-//               boundary conditions through periodic meshes, as well as the use
-//               of GLVis for persistent visualization of a time-evolving
-//               solution. The saving of time-dependent data files for external
-//               visualization with VisIt (visit.llnl.gov) and ParaView
-//               (paraview.org) is also illustrated.
+//               bilinear forms in MFEM (face integrators), the use of explicit
+//               ODE time integrators, the definition of periodic boundary
+//               conditions through periodic meshes, as well as the use of GLVis
+//               for persistent visualization of a time-evolving solution. The
+//               saving of time-dependent data files for external visualization
+//               with VisIt (visit.llnl.gov) is also illustrated.
 
 #include "mfem.hpp"
 #include <fstream>
@@ -63,54 +53,6 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;
 
-class DG_Solver : public Solver
-{
-private:
-   SparseMatrix &M, &K, A;
-   GMRESSolver linear_solver;
-   BlockILU prec;
-   double dt;
-public:
-   DG_Solver(SparseMatrix &M_, SparseMatrix &K_, const FiniteElementSpace &fes)
-      : M(M_),
-        K(K_),
-        prec(fes.GetFE(0)->GetDof(),
-             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
-        dt(-1.0)
-   {
-      linear_solver.iterative_mode = false;
-      linear_solver.SetRelTol(1e-9);
-      linear_solver.SetAbsTol(0.0);
-      linear_solver.SetMaxIter(100);
-      linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(prec);
-   }
-
-   void SetTimeStep(double dt_)
-   {
-      if (dt_ != dt)
-      {
-         dt = dt_;
-         // Form operator A = M - dt*K
-         A = K;
-         A *= -dt;
-         A += M;
-
-         // this will also call SetOperator on the preconditioner
-         linear_solver.SetOperator(A);
-      }
-   }
-
-   void SetOperator(const Operator &op)
-   {
-      linear_solver.SetOperator(op);
-   }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      linear_solver.Mult(x, y);
-   }
-};
 
 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
     form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
@@ -120,21 +62,19 @@ public:
 class FE_Evolution : public TimeDependentOperator
 {
 private:
-   BilinearForm &M, &K;
+   SparseMatrix &M, &K;
    const Vector &b;
-   Solver *M_prec;
+   DSmoother M_prec;
    CGSolver M_solver;
-   DG_Solver *dg_solver;
 
    mutable Vector z;
 
 public:
-   FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b);
+   FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b);
 
    virtual void Mult(const Vector &x, Vector &y) const;
-   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
 
-   virtual ~FE_Evolution();
+   virtual ~FE_Evolution() { }
 };
 
 
@@ -145,16 +85,11 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../../data/periodic-hexagon.mesh";
    int ref_levels = 2;
    int order = 3;
-   bool pa = false;
-   bool ea = false;
-   bool fa = false;
-   const char *device_config = "cpu";
-   int ode_solver_type = 7;
+   int ode_solver_type = 4;
    double t_final = 10.0;
    double dt = 0.01;
-   bool visualization = false;
+   bool visualization = true;
    bool visit = false;
-   bool paraview = false;
    bool binary = false;
    int vis_steps = 5;
 
@@ -173,14 +108,6 @@ int main(int argc, char *argv[])
                   "Number of times to refine the mesh uniformly.");
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
-                  "--no-element-assembly", "Enable Element Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                   "ODE solver:\n\t"
                   "1 - Forward Euler,\n\t"
@@ -201,9 +128,6 @@ int main(int argc, char *argv[])
    args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
                   "--no-visit-datafiles",
                   "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
-                  "--no-paraview-datafiles",
-                  "Save data files for ParaView (paraview.org) visualization.");
    args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
                   "--ascii-datafiles",
                   "Use binary (Sidre) or ascii format for VisIt data files.");
@@ -223,9 +147,6 @@ int main(int argc, char *argv[])
    }
    args.PrintOptions(cout);
 
-   Device device(device_config);
-   device.Print();
-
    // 2. Read the mesh from the given mesh file. We can handle geometrically
    //    periodic meshes in this code.
    Mesh mesh(mesh_file, 1, 1);
@@ -247,7 +168,7 @@ int main(int argc, char *argv[])
 
    // 4. Define the discontinuous DG finite element space of the given
    //    polynomial order on the refined mesh.
-   DG_FECollection fec(order, dim, BasisType::GaussLobatto);
+   DG_FECollection fec(order, dim);
    FiniteElementSpace fes(&mesh, &fec);
 
    cout << "Number of unknowns: " << fes.GetVSize() << endl;
@@ -260,23 +181,8 @@ int main(int argc, char *argv[])
    FunctionCoefficient u0(u0_function);
 
    BilinearForm m(&fes);
-   BilinearForm k(&fes);
-   if (pa)
-   {
-      m.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      k.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   }
-   else if (ea)
-   {
-      m.SetAssemblyLevel(AssemblyLevel::ELEMENT);
-      k.SetAssemblyLevel(AssemblyLevel::ELEMENT);
-   }
-   else if (fa)
-   {
-      m.SetAssemblyLevel(AssemblyLevel::FULL);
-      k.SetAssemblyLevel(AssemblyLevel::FULL);
-   }
    m.AddDomainIntegrator(new MassIntegrator);
+   BilinearForm k(&fes);
    k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
    k.AddInteriorFaceIntegrator(
       new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
@@ -288,11 +194,11 @@ int main(int argc, char *argv[])
       new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));
 
    m.Assemble();
+   m.Finalize();
    int skip_zeros = 0;
    k.Assemble(skip_zeros);
-   b.Assemble();
-   m.Finalize();
    k.Finalize(skip_zeros);
+   b.Assemble();
 
    // 6. Define the initial conditions, save the corresponding grid function to
    //    a file and (optionally) save data in the VisIt format and initialize
@@ -333,20 +239,6 @@ int main(int argc, char *argv[])
       dc->Save();
    }
 
-   ParaViewDataCollection *pd = NULL;
-   if (paraview)
-   {
-      pd = new ParaViewDataCollection("Example9", &mesh);
-      pd->SetPrefixPath("ParaView");
-      pd->RegisterField("solution", &u);
-      pd->SetLevelsOfDetail(order);
-      pd->SetDataFormat(VTKFormat::BINARY);
-      pd->SetHighOrderOutput(true);
-      pd->SetCycle(0);
-      pd->SetTime(0.0);
-      pd->Save();
-   }
-
    socketstream sout;
    if (visualization)
    {
@@ -373,7 +265,7 @@ int main(int argc, char *argv[])
 
    // 7. Define the time-dependent evolution operator describing the ODE
    //    right-hand side, and define the ODE solver used for time integration.
-   FE_Evolution adv(m, k, b);
+   FE_Evolution adv(m.SpMat(), k.SpMat(), b);
 
    double t = 0.0;
    adv.SetTime(t);
@@ -397,18 +289,12 @@ int main(int argc, char *argv[])
          cvode->UseSundialsLinearSolver();
          ode_solver = cvode; break;
       case 8:
-         arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
-         arkode->Init(adv);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         arkode->SetOrder(4);
-         ode_solver = arkode; break;
       case 9:
          arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
          arkode->Init(adv);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
-         arkode->SetERKTableNum(FEHLBERG_13_7_8);
+         if (ode_solver_type == 9) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
          ode_solver = arkode; break;
    }
 
@@ -443,13 +329,6 @@ int main(int argc, char *argv[])
             dc->SetTime(t);
             dc->Save();
          }
-
-         if (paraview)
-         {
-            pd->SetCycle(ti);
-            pd->SetTime(t);
-            pd->Save();
-         }
       }
    }
 
@@ -463,7 +342,6 @@ int main(int argc, char *argv[])
 
    // 10. Free the used memory.
    delete ode_solver;
-   delete pd;
    delete dc;
 
    return 0;
@@ -471,23 +349,12 @@ int main(int argc, char *argv[])
 
 
 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b)
-   : TimeDependentOperator(_M.Height()), M(_M), K(_K), b(_b), z(_M.Height())
+FE_Evolution::FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b)
+   : TimeDependentOperator(_M.Size()), M(_M), K(_K), b(_b), z(_M.Size())
 {
-   Array<int> ess_tdof_list;
-   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACYFULL)
-   {
-      M_prec = new DSmoother(M.SpMat());
-      M_solver.SetOperator(M.SpMat());
-      dg_solver = new DG_Solver(M.SpMat(), K.SpMat(), *M.FESpace());
-   }
-   else
-   {
-      M_prec = new OperatorJacobiSmoother(M, ess_tdof_list);
-      M_solver.SetOperator(M);
-      dg_solver = NULL;
-   }
-   M_solver.SetPreconditioner(*M_prec);
+   M_solver.SetPreconditioner(M_prec);
+   M_solver.SetOperator(M);
+
    M_solver.iterative_mode = false;
    M_solver.SetRelTol(1e-9);
    M_solver.SetAbsTol(0.0);
@@ -503,21 +370,6 @@ void FE_Evolution::Mult(const Vector &x, Vector &y) const
    M_solver.Mult(z, y);
 }
 
-void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
-{
-   MFEM_VERIFY(dg_solver != NULL,
-               "Implicit time integration is not supported with partial assembly");
-   K.Mult(x, z);
-   z += b;
-   dg_solver->SetTimeStep(dt);
-   dg_solver->Mult(z, k);
-}
-
-FE_Evolution::~FE_Evolution()
-{
-   delete M_prec;
-   delete dg_solver;
-}
 
 // Velocity coefficient
 void velocity_function(const Vector &x, Vector &v)

examples/sundials/ex9p.cpp

@@ -13,28 +13,17 @@
 //    mpirun -np 4 ex9p -m ../../data/disc-nurbs.mesh       -p 1 -rp 2 -s 7 -dt 0.0025 -tf 9
 //    mpirun -np 4 ex9p -m ../../data/periodic-cube.mesh    -p 0 -rp 1 -s 8 -dt 0.01 -tf 8 -o 2
 //
-// Device sample runs:
-//    mpirun -np 4 ex9p -pa
-//    mpirun -np 4 ex9p -ea
-//    mpirun -np 4 ex9p -fa
-//    mpirun -np 4 ex9p -pa -m ../../data/periodic-cube.mesh
-//    mpirun -np 4 ex9p -pa -m ../../data/periodic-cube.mesh -d cuda
-//    mpirun -np 4 ex9p -ea -m ../../data/periodic-cube.mesh -d cuda
-//    mpirun -np 4 ex9p -fa -m ../../data/periodic-cube.mesh -d cuda
-//
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
 //               u0(x)=u(0,x) is a given initial condition.
 //
 //               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of implicit
-//               and explicit ODE time integrators, the definition of periodic
-//               boundary conditions through periodic meshes, as well as the use
-//               of GLVis for persistent visualization of a time-evolving
-//               solution. Saving of time-dependent data files for visualization
-//               with VisIt (visit.llnl.gov) and ParaView (paraview.org), as
-//               well as the optional saving with ADIOS2 (adios2.readthedocs.io)
-//               are also illustrated.
+//               bilinear forms in MFEM (face integrators), the use of explicit
+//               ODE time integrators, the definition of periodic boundary
+//               conditions through periodic meshes, as well as the use of GLVis
+//               for persistent visualization of a time-evolving solution. The
+//               saving of time-dependent data files for external visualization
+//               with VisIt (visit.llnl.gov) is also illustrated.
 
 #include "mfem.hpp"
 #include <fstream>
@@ -63,66 +52,6 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;
 
-class DG_Solver : public Solver
-{
-private:
-   HypreParMatrix &M, &K;
-   SparseMatrix M_diag;
-   HypreParMatrix *A;
-   GMRESSolver linear_solver;
-   BlockILU prec;
-   double dt;
-public:
-   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes)
-      : M(M_),
-        K(K_),
-        A(NULL),
-        linear_solver(M.GetComm()),
-        prec(fes.GetFE(0)->GetDof(),
-             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
-        dt(-1.0)
-   {
-      linear_solver.iterative_mode = false;
-      linear_solver.SetRelTol(1e-9);
-      linear_solver.SetAbsTol(0.0);
-      linear_solver.SetMaxIter(100);
-      linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(prec);
-
-      M.GetDiag(M_diag);
-   }
-
-   void SetTimeStep(double dt_)
-   {
-      if (dt_ != dt)
-      {
-         dt = dt_;
-         // Form operator A = M - dt*K
-         delete A;
-         A = Add(-dt, K, 0.0, K);
-         SparseMatrix A_diag;
-         A->GetDiag(A_diag);
-         A_diag.Add(1.0, M_diag);
-         // this will also call SetOperator on the preconditioner
-         linear_solver.SetOperator(*A);
-      }
-   }
-
-   void SetOperator(const Operator &op)
-   {
-      linear_solver.SetOperator(op);
-   }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      linear_solver.Mult(x, y);
-   }
-
-   ~DG_Solver()
-   {
-      delete A;
-   }
-};
 
 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
     form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
@@ -132,21 +61,19 @@ public:
 class FE_Evolution : public TimeDependentOperator
 {
 private:
-   OperatorHandle M, K;
+   HypreParMatrix &M, &K;
    const Vector &b;
-   Solver *M_prec;
+   HypreSmoother M_prec;
    CGSolver M_solver;
-   DG_Solver *dg_solver;
 
    mutable Vector z;
 
 public:
-   FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K, const Vector &_b);
+   FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K, const Vector &_b);
 
    virtual void Mult(const Vector &x, Vector &y) const;
-   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
 
-   virtual ~FE_Evolution();
+   virtual ~FE_Evolution() { }
 };
 
 
@@ -164,17 +91,11 @@ int main(int argc, char *argv[])
    int ser_ref_levels = 2;
    int par_ref_levels = 0;
    int order = 3;
-   bool pa = false;
-   bool ea = false;
-   bool fa = false;
-   const char *device_config = "cpu";
-   int ode_solver_type = 7;
+   int ode_solver_type = 4;
    double t_final = 10.0;
    double dt = 0.01;
-   bool visualization = false;
+   bool visualization = true;
    bool visit = false;
-   bool paraview = false;
-   bool adios2 = false;
    bool binary = false;
    int vis_steps = 5;
 
@@ -195,14 +116,6 @@ int main(int argc, char *argv[])
                   "Number of times to refine the mesh uniformly in parallel.");
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
-                  "--no-element-assembly", "Enable Element Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                   "ODE solver:\n\t"
                   "1 - Forward Euler,\n\t"
@@ -223,12 +136,6 @@ int main(int argc, char *argv[])
    args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
                   "--no-visit-datafiles",
                   "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
-                  "--no-paraview-datafiles",
-                  "Save data files for ParaView (paraview.org) visualization.");
-   args.AddOption(&adios2, "-adios2", "--adios2-streams", "-no-adios2",
-                  "--no-adios2-streams",
-                  "Save data using adios2 streams.");
    args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
                   "--ascii-datafiles",
                   "Use binary (Sidre) or ascii format for VisIt data files.");
@@ -248,7 +155,6 @@ int main(int argc, char *argv[])
    {
       args.PrintOptions(cout);
    }
-
    // check for vaild ODE solver option
    if (ode_solver_type < 1 || ode_solver_type > 9)
    {
@@ -260,9 +166,6 @@ int main(int argc, char *argv[])
       return 3;
    }
 
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
    // 3. Read the serial mesh from the given mesh file on all processors. We can
    //    handle geometrically periodic meshes in this code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
@@ -294,7 +197,7 @@ int main(int argc, char *argv[])
 
    // 6. Define the parallel discontinuous DG finite element space on the
    //    parallel refined mesh of the given polynomial order.
-   DG_FECollection fec(order, dim, BasisType::GaussLobatto);
+   DG_FECollection fec(order, dim);
    ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);
 
    HYPRE_Int global_vSize = fes->GlobalTrueVSize();
@@ -311,24 +214,8 @@ int main(int argc, char *argv[])
    FunctionCoefficient u0(u0_function);
 
    ParBilinearForm *m = new ParBilinearForm(fes);
-   ParBilinearForm *k = new ParBilinearForm(fes);
-   if (pa)
-   {
-      m->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      k->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   }
-   else if (ea)
-   {
-      m->SetAssemblyLevel(AssemblyLevel::ELEMENT);
-      k->SetAssemblyLevel(AssemblyLevel::ELEMENT);
-   }
-   else if (fa)
-   {
-      m->SetAssemblyLevel(AssemblyLevel::FULL);
-      k->SetAssemblyLevel(AssemblyLevel::FULL);
-   }
-
    m->AddDomainIntegrator(new MassIntegrator);
+   ParBilinearForm *k = new ParBilinearForm(fes);
    k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
    k->AddInteriorFaceIntegrator(
       new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
@@ -339,13 +226,15 @@ int main(int argc, char *argv[])
    b->AddBdrFaceIntegrator(
       new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));
 
-   int skip_zeros = 0;
    m->Assemble();
-   k->Assemble(skip_zeros);
-   b->Assemble();
    m->Finalize();
+   int skip_zeros = 0;
+   k->Assemble(skip_zeros);
    k->Finalize(skip_zeros);
+   b->Assemble();
 
+   HypreParMatrix *M = m->ParallelAssemble();
+   HypreParMatrix *K = k->ParallelAssemble();
    HypreParVector *B = b->ParallelAssemble();
 
    // 8. Define the initial conditions, save the corresponding grid function to
@@ -384,8 +273,6 @@ int main(int argc, char *argv[])
       {
          dc = new VisItDataCollection("Example9-Parallel", pmesh);
          dc->SetPrecision(precision);
-         // To save the mesh using MFEM's parallel mesh format:
-         // dc->SetFormat(DataCollection::PARALLEL_FORMAT);
       }
       dc->RegisterField("solution", u);
       dc->SetCycle(0);
@@ -393,41 +280,6 @@ int main(int argc, char *argv[])
       dc->Save();
    }
 
-   ParaViewDataCollection *pd = NULL;
-   if (paraview)
-   {
-      pd = new ParaViewDataCollection("Example9P", pmesh);
-      pd->SetPrefixPath("ParaView");
-      pd->RegisterField("solution", u);
-      pd->SetLevelsOfDetail(order);
-      pd->SetDataFormat(VTKFormat::BINARY);
-      pd->SetHighOrderOutput(true);
-      pd->SetCycle(0);
-      pd->SetTime(0.0);
-      pd->Save();
-   }
-
-   // Optionally output a BP (binary pack) file using ADIOS2. This can be
-   // visualized with the ParaView VTX reader.
-#ifdef MFEM_USE_ADIOS2
-   ADIOS2DataCollection *adios2_dc = NULL;
-   if (adios2)
-   {
-      std::string postfix(mesh_file);
-      postfix.erase(0, std::string("../data/").size() );
-      postfix += "_o" + std::to_string(order);
-      const std::string collection_name = "ex9-p-" + postfix + ".bp";
-
-      adios2_dc = new ADIOS2DataCollection(MPI_COMM_WORLD, collection_name, pmesh);
-      // output data substreams are half the number of mpi processes
-      adios2_dc->SetParameter("SubStreams", std::to_string(num_procs/2) );
-      adios2_dc->RegisterField("solution", u);
-      adios2_dc->SetCycle(0);
-      adios2_dc->SetTime(0.0);
-      adios2_dc->Save();
-   }
-#endif
-
    socketstream sout;
    if (visualization)
    {
@@ -460,7 +312,7 @@ int main(int argc, char *argv[])
 
    // 9. Define the time-dependent evolution operator describing the ODE
    //    right-hand side, and define the ODE solver used for time integration.
-   FE_Evolution adv(*m, *k, *B);
+   FE_Evolution adv(*M, *K, *B);
 
    double t = 0.0;
    adv.SetTime(t);
@@ -532,23 +384,6 @@ int main(int argc, char *argv[])
             dc->SetTime(t);
             dc->Save();
          }
-
-         if (paraview)
-         {
-            pd->SetCycle(ti);
-            pd->SetTime(t);
-            pd->Save();
-         }
-
-#ifdef MFEM_USE_ADIOS2
-         // transient solutions can be visualized with ParaView
-         if (adios2)
-         {
-            adios2_dc->SetCycle(ti);
-            adios2_dc->SetTime(t);
-            adios2_dc->Save();
-         }
-#endif
       }
    }
 
@@ -568,18 +403,13 @@ int main(int argc, char *argv[])
    delete u;
    delete B;
    delete b;
+   delete K;
    delete k;
+   delete M;
    delete m;
    delete fes;
    delete pmesh;
    delete ode_solver;
-   delete pd;
-#ifdef MFEM_USE_ADIOS2
-   if (adios2)
-   {
-      delete adios2_dc;
-   }
-#endif
    delete dc;
 
    MPI_Finalize();
@@ -588,43 +418,15 @@ int main(int argc, char *argv[])
 
 
 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
+FE_Evolution::FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
                            const Vector &_b)
    : TimeDependentOperator(_M.Height()),
-     b(_b),
-     M_solver(_M.ParFESpace()->GetComm()),
-     z(_M.Height())
+     M(_M), K(_K), b(_b), M_solver(M.GetComm()), z(_M.Height())
 {
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
-   {
-      M.Reset(_M.ParallelAssemble(), true);
-      K.Reset(_K.ParallelAssemble(), true);
-   }
-   else
-   {
-      M.Reset(&_M, false);
-      K.Reset(&_K, false);
-   }
+   M_prec.SetType(HypreSmoother::Jacobi);
+   M_solver.SetPreconditioner(M_prec);
+   M_solver.SetOperator(M);
 
-   M_solver.SetOperator(*M);
-
-   Array<int> ess_tdof_list;
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
-   {
-      HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
-      HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
-      HypreSmoother *hypre_prec = new HypreSmoother(M_mat, HypreSmoother::Jacobi);
-      M_prec = hypre_prec;
-
-      dg_solver = new DG_Solver(M_mat, K_mat, *_M.FESpace());
-   }
-   else
-   {
-      M_prec = new OperatorJacobiSmoother(_M, ess_tdof_list);
-      dg_solver = NULL;
-   }
-
-   M_solver.SetPreconditioner(*M_prec);
    M_solver.iterative_mode = false;
    M_solver.SetRelTol(1e-9);
    M_solver.SetAbsTol(0.0);
@@ -632,28 +434,14 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
    M_solver.SetPrintLevel(0);
 }
 
-void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
-{
-   K->Mult(x, z);
-   z += b;
-   dg_solver->SetTimeStep(dt);
-   dg_solver->Mult(z, k);
-}
-
 void FE_Evolution::Mult(const Vector &x, Vector &y) const
 {
    // y = M^{-1} (K x + b)
-   K->Mult(x, z);
+   K.Mult(x, z);
    z += b;
    M_solver.Mult(z, y);
 }
 
-FE_Evolution::~FE_Evolution()
-{
-   delete M_prec;
-   delete dg_solver;
-}
-
 
 // Velocity coefficient
 void velocity_function(const Vector &x, Vector &v)

examples/superlu/CMakeLists.txt

@@ -1,62 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-set(SUPERLU_EXAMPLES_SRCS)
-if (MFEM_USE_MPI)
-  list(APPEND SUPERLU_EXAMPLES_SRCS
-    ex1p.cpp
-    )
-endif()
-
-# Include the source directory where mfem.hpp and mfem-performance.hpp are.
-include_directories(BEFORE ${PROJECT_BINARY_DIR})
-
-# Add "test_superlu" target, see below.
-add_custom_target(test_superlu
-  ${CMAKE_CTEST_COMMAND} -R superlu USES_TERMINAL)
-
-# Add one executable per cpp file, adding "superlu_" as prefix. Sets
-# "test_superlu" as a target that depends on the given examples.
-set(PFX superlu_)
-add_mfem_examples(SUPERLU_EXAMPLES_SRCS ${PFX} "" test_superlu)
-
-# Testing.
-# The SuperLU tests can be run separately using the target "test_superlu"
-# which builds the examples and runs:
-#   ctest -R superlu
-
-# Command line options for the tests.
-# Example 1: Test SuperLU on the simple Poisson problem
-set(EX1_COMMON_OPTS -m ../../data/star.mesh -p 2)
-set(EX1P_TEST_OPTS ${EX1_COMMON_OPTS})
-
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${SUPERLU_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
-
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()

examples/superlu/ex1p.cpp

@@ -1,321 +0,0 @@
-//                       MFEM Example 1 - Parallel Version
-//                             SuperLU Modification
-//
-// Compile with: make ex1p
-//
-// Sample runs:  mpirun -np 4 ex1p -m ../../data/square-disc.mesh
-//               mpirun -np 4 ex1p -m ../../data/star.mesh
-//               mpirun -np 4 ex1p -m ../../data/star-mixed.mesh
-//               mpirun -np 4 ex1p -m ../../data/escher.mesh
-//               mpirun -np 4 ex1p -m ../../data/fichera.mesh
-//               mpirun -np 4 ex1p -m ../../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -m ../../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../../data/periodic-annulus-sector.msh
-//               mpirun -np 4 ex1p -m ../../data/periodic-torus-sector.msh
-//               mpirun -np 4 ex1p -m ../../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex1p -m ../../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/star-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../../data/disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex1p -m ../../data/star-surf.mesh
-//               mpirun -np 4 ex1p -m ../../data/square-disc-surf.mesh
-//               mpirun -np 4 ex1p -m ../../data/inline-segment.mesh
-//               mpirun -np 4 ex1p -m ../../data/amr-quad.mesh
-//               mpirun -np 4 ex1p -m ../../data/amr-hex.mesh
-//               mpirun -np 4 ex1p -m ../../data/mobius-strip.mesh
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   int slu_colperm = 4;
-   int slu_rowperm = 1;
-   int slu_iterref = 2;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&slu_colperm, "-cp", "--colperm",
-                  "SuperLU Column Permutation Method:  0-NATURAL, 1-MMD-ATA "
-                  "2-MMD_AT_PLUS_A, 3-COLAMD, 4-METIS_AT_PLUS_A, 5-PARMETIS "
-                  "6-ZOLTAN");
-   args.AddOption(&slu_rowperm, "-rp", "--rowperm",
-                  "SuperLU Row Permutation Method:  0-NOROWPERM, 1-LargeDiag");
-   args.AddOption(&slu_iterref, "-rp", "--rowperm",
-                  "SuperLU Iterative Refinement:  0-NOREFINE, 1-Single, "
-                  "2-Double, 3-Extra");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(1000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh.UniformRefinement();
-      }
-   }
-
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   bool delete_fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
-   }
-   else if (pmesh.GetNodes())
-   {
-      fec = pmesh.GetNodes()->OwnFEC();
-      delete_fec = false;
-      if (myid == 0)
-      {
-         cout << "Using isoparametric FEs: " << fec->Name() << endl;
-      }
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
-   }
-   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_Int size = fespace.GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 8. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 9. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 10. Define the solution vector x as a parallel finite element grid function
-   //     corresponding to fespace. Initialize x with initial guess of zero,
-   //     which satisfies the boundary conditions.
-   ParGridFunction x(&fespace);
-   x = 0.0;
-
-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
-   ParBilinearForm a(&fespace);
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 12. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   // 13. Solve the linear system A X = B utilizing SuperLU.
-   SuperLUSolver *superlu = new SuperLUSolver(MPI_COMM_WORLD);
-   Operator *SLU_A = new SuperLURowLocMatrix(*A.As<HypreParMatrix>());
-   superlu->SetPrintStatistics(true);
-   superlu->SetSymmetricPattern(false);
-
-   if (slu_colperm == 0)
-   {
-      superlu->SetColumnPermutation(superlu::NATURAL);
-   }
-   else if (slu_colperm == 1)
-   {
-      superlu->SetColumnPermutation(superlu::MMD_ATA);
-   }
-   else if (slu_colperm == 2)
-   {
-      superlu->SetColumnPermutation(superlu::MMD_AT_PLUS_A);
-   }
-   else if (slu_colperm == 3)
-   {
-      superlu->SetColumnPermutation(superlu::COLAMD);
-   }
-   else if (slu_colperm == 4)
-   {
-      superlu->SetColumnPermutation(superlu::METIS_AT_PLUS_A);
-   }
-   else if (slu_colperm == 5)
-   {
-      superlu->SetColumnPermutation(superlu::PARMETIS);
-   }
-   else if (slu_colperm == 6)
-   {
-      superlu->SetColumnPermutation(superlu::ZOLTAN);
-   }
-
-   if (slu_rowperm == 0)
-   {
-      superlu->SetRowPermutation(superlu::NOROWPERM);
-   }
-   else if (slu_rowperm == 1)
-   {
-#ifdef MFEM_USE_SUPERLU5
-      superlu->SetRowPermutation(superlu::LargeDiag);
-#else
-      superlu->SetRowPermutation(superlu::LargeDiag_MC64);
-#endif
-   }
-
-   if (slu_iterref == 0)
-   {
-      superlu->SetIterativeRefine(superlu::NOREFINE);
-   }
-   else if (slu_iterref == 1)
-   {
-      superlu->SetIterativeRefine(superlu::SLU_SINGLE);
-   }
-   else if (slu_iterref == 2)
-   {
-      superlu->SetIterativeRefine(superlu::SLU_DOUBLE);
-   }
-   else if (slu_iterref == 3)
-   {
-      superlu->SetIterativeRefine(superlu::SLU_EXTRA);
-   }
-
-   superlu->SetOperator(*SLU_A);
-   superlu->SetPrintStatistics(true);
-   superlu->Mult(B, X);
-
-   // 14. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
-   MPI_Finalize();
-
-   return 0;
-}

examples/superlu/makefile

@@ -1,79 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../..
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/superlu/,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-# Use the MFEM install directory
-# MFEM_INSTALL_DIR = ../../mfem
-# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
--include $(CONFIG_MK)
-
-SEQ_EXAMPLES = 
-PAR_EXAMPLES = ex1p
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES)
-
-ifeq ($(MFEM_USE_SUPERLU),NO)
-$(EXAMPLES):
-	$(error MFEM is not configured with SuperLU)
-endif
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-PARALLEL_NAME := Parallel SuperLU example
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
-
-# Testing: Specific execution options:
-# Example 1: Test SuperLU on the simple poisson problem
-EX1_COMMON_ARGS := -m ../../data/star.mesh
-EX1P_ARGS := $(EX1_COMMON_ARGS)
-ex1p-test-par: ex1p
-	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME),$(EX1P_ARGS))
-
-# Testing: "test" target and mfem-test* variables are defined in config/test.mk
-
-# Generate an error message if the MFEM library is not built and exit
-$(MFEM_LIB_FILE):
-	$(error The MFEM library is not built)
-
-clean: clean-build clean-exec
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
-	@rm -f mesh.* sol.*

examples/testStrumpack.hpp

@@ -0,0 +1,105 @@
+#ifndef TESTSTRUMPACK_HPP
+#define TESTSTRUMPACK_HPP
+
+#include "mfem.hpp"
+
+using namespace mfem;
+using namespace std;
+
+
+void TestStrumpackConstructor()
+{
+   int num_procs, rank;
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+   const int num_loc_rows = 100;
+   const int first_loc_row = num_loc_rows * rank;
+   const int glob_nrows = num_loc_rows * num_procs;
+   const int glob_ncols = glob_nrows;
+
+   int *opI = new int[num_loc_rows+1];
+
+   for (int i=0; i<num_loc_rows+1; ++i)
+   {
+      opI[i] = 0;
+   }
+
+   for (int i=0; i<num_loc_rows; ++i)
+   {
+      int nnz_i = 3;
+
+      if ((first_loc_row + i) == 0 ||
+          (first_loc_row + i) == glob_nrows-1)  // if first or last row
+      {
+         nnz_i = 2;
+      }
+
+      opI[i+1] = opI[i] + nnz_i;
+   }
+
+   const int nnz = opI[num_loc_rows];
+
+   int *opJ = new int[nnz];
+   double *data = new double[nnz];
+
+   int cnt = 0;
+   for (int i=0; i<num_loc_rows; ++i)
+   {
+      const int globalRow = first_loc_row + i;
+
+      // Diagonal entry
+
+      opJ[cnt] = first_loc_row + i;
+
+      if (globalRow == 0 || globalRow == glob_nrows-1)
+      {
+         data[cnt] = 2.0;
+      }
+      else
+      {
+         data[cnt] = 3.0;
+      }
+
+      cnt++;
+
+      // Upper diagonal
+      if (globalRow < glob_nrows-1)
+      {
+         opJ[cnt] = first_loc_row + i + 1;
+         data[cnt] = -1.0;
+         cnt++;
+      }
+
+      // Lower diagonal
+      if (globalRow > 0)
+      {
+         opJ[cnt] = first_loc_row + i - 1;
+         data[cnt] = -1.0;
+         cnt++;
+      }
+   }
+
+   Operator *op = new STRUMPACKRowLocMatrix(MPI_COMM_WORLD, num_loc_rows,
+                                            first_loc_row, glob_nrows, glob_ncols, opI, opJ, data);
+
+   STRUMPACKSolver * strumpack = new STRUMPACKSolver(0, NULL, MPI_COMM_WORLD);
+   strumpack->SetPrintFactorStatistics(true);
+   strumpack->SetPrintSolveStatistics(false);
+   strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+   strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+   strumpack->SetOperator(*op);
+   strumpack->SetFromCommandLine();
+
+   Vector x(num_loc_rows);
+   Vector y(num_loc_rows);
+
+   x = 1.0;
+   strumpack->Mult(x, y);
+
+   delete opI;
+   delete opJ;
+   delete data;
+}
+
+#endif // TESTSTRUMPACK_HPP

fem/CMakeLists.txt

@@ -17,7 +17,6 @@ set(SRCS
   bilininteg_convection_ea.cpp
   bilininteg_dgtrace_pa.cpp
   bilininteg_dgtrace_ea.cpp
-  bilininteg_diffusion_mf.cpp
   bilininteg_diffusion_pa.cpp
   bilininteg_diffusion_ea.cpp
   bilininteg_divergence.cpp
@@ -25,14 +24,11 @@ set(SRCS
   bilininteg_hdiv.cpp
   bilininteg_vectorfe.cpp
   bilininteg_gradient.cpp
-  bilininteg_mass_mf.cpp
   bilininteg_mass_pa.cpp
   bilininteg_mass_ea.cpp
   bilininteg_transpose_ea.cpp
   bilininteg_vecdiffusion.cpp
-  bilininteg_vecdiffusion_mf.cpp
   bilininteg_vecmass.cpp
-  bilininteg_vecmass_mf.cpp
   coefficient.cpp
   complex_fem.cpp
   convergence.cpp
@@ -46,9 +42,6 @@ set(SRCS
   gridfunc.cpp
   hybridization.cpp
   intrules.cpp
-  libceed/ceed.cpp
-  libceed/diffusion.cpp
-  libceed/mass.cpp
   linearform.cpp
   lininteg.cpp
   multigrid.cpp
@@ -85,9 +78,6 @@ set(HDRS
   gridfunc.hpp
   hybridization.hpp
   intrules.hpp
-  libceed/ceed.hpp
-  libceed/diffusion.hpp
-  libceed/mass.hpp
   linearform.hpp
   lininteg.hpp
   multigrid.hpp
@@ -147,6 +137,17 @@ if (MFEM_USE_MPI)
     prestriction.hpp)
 endif()
 
+if (MFEM_USE_CEED)
+   list(APPEND SRCS
+     libceed/ceed.cpp
+     libceed/diffusion.cpp
+     libceed/mass.cpp)
+   list(APPEND HDRS
+     libceed/ceed.hpp
+     libceed/diffusion.hpp
+     libceed/mass.hpp)
+endif()
+
 convert_filenames_to_full_paths(SRCS)
 convert_filenames_to_full_paths(HDRS)
 

fem/bilinearform.cpp

@@ -133,7 +133,8 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
          ext = new PABilinearFormExtension(this);
          break;
       case AssemblyLevel::NONE:
-         ext = new MFBilinearFormExtension(this);
+         mfem_error("Matrix-free action not supported yet... stay tuned!");
+         // ext = new MFBilinearFormExtension(this);
          break;
       default:
          mfem_error("Unknown assembly level");
@@ -1770,9 +1771,40 @@ MixedBilinearForm::~MixedBilinearForm()
    delete ext;
 }
 
+void DiscreteLinearOperator::SetAssemblyLevel(AssemblyLevel assembly_level)
+{
+   if (ext)
+   {
+      MFEM_ABORT("the assembly level has already been set!");
+   }
+   assembly = assembly_level;
+   switch (assembly)
+   {
+      case AssemblyLevel::FULL:
+         // Use the original BilinearForm implementation for now
+         break;
+      case AssemblyLevel::ELEMENT:
+         mfem_error("Element assembly not supported yet... stay tuned!");
+         break;
+      case AssemblyLevel::PARTIAL:
+         ext = new PADiscreteLinearOperatorExtension(this);
+         break;
+      case AssemblyLevel::NONE:
+         mfem_error("Matrix-free action not supported yet... stay tuned!");
+         break;
+      default:
+         mfem_error("Unknown assembly level");
+   }
+}
 
 void DiscreteLinearOperator::Assemble(int skip_zeros)
 {
+   if (ext)
+   {
+      ext->Assemble();
+      return;
+   }
+
    Array<int> dom_vdofs, ran_vdofs;
    ElementTransformation *T;
    const FiniteElement *dom_fe, *ran_fe;

fem/bilinearform.hpp

@@ -157,8 +157,7 @@ public:
    /// Set the desired assembly level.
    /** Valid choices are:
 
-       - AssemblyLevel::LEGACYFULL (default)
-       - AssemblyLevel::FULL
+       - AssemblyLevel::FULL  (default)
        - AssemblyLevel::PARTIAL
        - AssemblyLevel::ELEMENT
        - AssemblyLevel::NONE
@@ -376,6 +375,9 @@ public:
    /// Get the output finite element space prolongation matrix
    virtual const Operator *GetOutputProlongation() const
    { return GetProlongation(); }
+   /// Get the output finite element space prolongation matrix (local diagonal)
+   virtual const Operator *GetLocalOutputProlongation() const
+   { return GetOutputProlongation(); }
    /// Get the output finite element space restriction matrix
    virtual const Operator *GetOutputRestriction() const
    { return GetRestriction(); }
@@ -416,7 +418,9 @@ public:
        returned in the variable @a A, of type OpType, holding a *reference* to
        the system matrix (created with the method OpType::MakeRef()). The
        reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
+       destructor is called.
+
+       Currently, this method can be used only with AssemblyLevel::FULL. */
    template <typename OpType>
    void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
                          OpType &A, Vector &X, Vector &B,
@@ -438,7 +442,9 @@ public:
        returned in the variable @a A, of type OpType, holding a *reference* to
        the system matrix (created with the method OpType::MakeRef()). The
        reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
+       destructor is called.
+
+       Currently, this method can be used only with AssemblyLevel::FULL. */
    template <typename OpType>
    void FormSystemMatrix(const Array<int> &ess_tdof_list, OpType &A)
    {
@@ -588,6 +594,18 @@ public:
    /// Indicate that integrators are not owned by the BilinearForm
    void UseExternalIntegrators() { extern_bfs = 1; };
 
+   void GetTimings(double& t1, double& t2)
+   {
+      t1 = 0.0;
+      t2 = 0.0;
+      PABilinearFormExtension *paext = dynamic_cast<PABilinearFormExtension*>(ext);
+      if (paext != NULL)
+      {
+         t1 = paext->timingDomain;
+         t2 = paext->timingBoundary;
+      }
+   }
+
    /// Destroys bilinear form.
    virtual ~BilinearForm();
 };
@@ -756,7 +774,7 @@ public:
    /// Sets all sparse values of \f$ M \f$ to @a a.
    void operator=(const double a) { *mat = a; }
 
-   /// Set the desired assembly level. The default is AssemblyLevel::LEGACYFULL.
+   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
    /** This method must be called before assembly. */
    void SetAssemblyLevel(AssemblyLevel assembly_level);
 
@@ -858,7 +876,9 @@ public:
        returned in the variable @a A, of type OpType, holding a *reference* to
        the system matrix (created with the method OpType::MakeRef()). The
        reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
+       destructor is called.
+
+       Currently, this method can be used only with AssemblyLevel::FULL. */
    template <typename OpType>
    void FormRectangularSystemMatrix(const Array<int> &trial_tdof_list,
                                     const Array<int> &test_tdof_list, OpType &A)
@@ -888,7 +908,9 @@ public:
        returned in the variable @a A, of type OpType, holding a *reference* to
        the system matrix (created with the method OpType::MakeRef()). The
        reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
+       destructor is called.
+
+       Currently, this method can be used only with AssemblyLevel::FULL. */
    template <typename OpType>
    void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
                                     const Array<int> &test_tdof_list,
@@ -977,9 +999,17 @@ public:
    /// Access all interpolators added with AddDomainInterpolator().
    Array<BilinearFormIntegrator*> *GetDI() { return &dbfi; }
 
+   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
+   /** This method must be called before assembly. */
+   void SetAssemblyLevel(AssemblyLevel assembly_level);
+
    /** @brief Construct the internal matrix representation of the discrete
        linear operator. */
    virtual void Assemble(int skip_zeros = 1);
+
+   /// Get the output finite element space prolongation matrix (local diagonal)
+   virtual const Operator *GetLocalOutputProlongation() const
+   { return test_fes->GetLocalProlongationMatrix(); }
 };
 
 }

fem/bilinearform_ext.cpp

@@ -17,6 +17,8 @@
 #include "libceed/ceed.hpp"
 #include "pgridfunc.hpp"
 
+#include "../general/tic_toc.hpp"
+
 namespace mfem
 {
 
@@ -36,206 +38,6 @@ const Operator *BilinearFormExtension::GetRestriction() const
    return a->GetRestriction();
 }
 
-// Data and methods for partially-assembled bilinear forms
-MFBilinearFormExtension::MFBilinearFormExtension(BilinearForm *form)
-   : BilinearFormExtension(form),
-     trialFes(a->FESpace()),
-     testFes(a->FESpace())
-{
-   elem_restrict = NULL;
-   int_face_restrict_lex = NULL;
-   bdr_face_restrict_lex = NULL;
-}
-
-void MFBilinearFormExtension::Assemble()
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int integratorCount = integrators.Size();
-   for (int i = 0; i < integratorCount; ++i)
-   {
-      integrators[i]->AssembleMF(*a->FESpace());
-   }
-}
-
-void MFBilinearFormExtension::AssembleDiagonal(Vector &y) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-
-   const int iSz = integrators.Size();
-   if (elem_restrict && !DeviceCanUseCeed())
-   {
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AssembleDiagonalMF(localY);
-      }
-      const ElementRestriction* H1elem_restrict =
-         dynamic_cast<const ElementRestriction*>(elem_restrict);
-      if (H1elem_restrict)
-      {
-         H1elem_restrict->MultTransposeUnsigned(localY, y);
-      }
-      else
-      {
-         elem_restrict->MultTranspose(localY, y);
-      }
-   }
-   else
-   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AssembleDiagonalMF(y);
-      }
-   }
-}
-
-void MFBilinearFormExtension::Update()
-{
-   FiniteElementSpace *fes = a->FESpace();
-   height = width = fes->GetVSize();
-   trialFes = fes;
-   testFes = fes;
-
-   elem_restrict = nullptr;
-   int_face_restrict_lex = nullptr;
-   bdr_face_restrict_lex = nullptr;
-}
-
-void MFBilinearFormExtension::FormSystemMatrix(const Array<int> &ess_tdof_list,
-                                               OperatorHandle &A)
-{
-   Operator *oper;
-   Operator::FormSystemOperator(ess_tdof_list, oper);
-   A.Reset(oper); // A will own oper
-}
-
-void MFBilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
-                                               Vector &x, Vector &b,
-                                               OperatorHandle &A,
-                                               Vector &X, Vector &B,
-                                               int copy_interior)
-{
-   Operator *oper;
-   Operator::FormLinearSystem(ess_tdof_list, x, b, oper, X, B, copy_interior);
-   A.Reset(oper); // A will own oper
-}
-
-void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-
-   const int iSz = integrators.Size();
-   if (DeviceCanUseCeed() || !elem_restrict)
-   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultMF(x, y);
-      }
-   }
-   else
-   {
-      elem_restrict->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultMF(localX, localY);
-      }
-      elem_restrict->MultTranspose(localY, y);
-   }
-
-   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int iFISz = intFaceIntegrators.Size();
-   if (int_face_restrict_lex && iFISz>0)
-   {
-      int_face_restrict_lex->Mult(x, faceIntX);
-      if (faceIntX.Size()>0)
-      {
-         faceIntY = 0.0;
-         for (int i = 0; i < iFISz; ++i)
-         {
-            intFaceIntegrators[i]->AddMultMF(faceIntX, faceIntY);
-         }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
-      }
-   }
-
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int bFISz = bdrFaceIntegrators.Size();
-   if (bdr_face_restrict_lex && bFISz>0)
-   {
-      bdr_face_restrict_lex->Mult(x, faceBdrX);
-      if (faceBdrX.Size()>0)
-      {
-         faceBdrY = 0.0;
-         for (int i = 0; i < bFISz; ++i)
-         {
-            bdrFaceIntegrators[i]->AddMultMF(faceBdrX, faceBdrY);
-         }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
-      }
-   }
-}
-
-void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int iSz = integrators.Size();
-   if (elem_restrict)
-   {
-      elem_restrict->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultTransposeMF(localX, localY);
-      }
-      elem_restrict->MultTranspose(localY, y);
-   }
-   else
-   {
-      y.UseDevice(true);
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultTransposeMF(x, y);
-      }
-   }
-
-   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int iFISz = intFaceIntegrators.Size();
-   if (int_face_restrict_lex && iFISz>0)
-   {
-      int_face_restrict_lex->Mult(x, faceIntX);
-      if (faceIntX.Size()>0)
-      {
-         faceIntY = 0.0;
-         for (int i = 0; i < iFISz; ++i)
-         {
-            intFaceIntegrators[i]->AddMultTransposeMF(faceIntX, faceIntY);
-         }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
-      }
-   }
-
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int bFISz = bdrFaceIntegrators.Size();
-   if (bdr_face_restrict_lex && bFISz>0)
-   {
-      bdr_face_restrict_lex->Mult(x, faceBdrX);
-      if (faceBdrX.Size()>0)
-      {
-         faceBdrY = 0.0;
-         for (int i = 0; i < bFISz; ++i)
-         {
-            bdrFaceIntegrators[i]->AddMultTransposeMF(faceBdrX, faceBdrY);
-         }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
-      }
-   }
-}
 
 // Data and methods for partially-assembled bilinear forms
 PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
@@ -246,6 +48,9 @@ PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
    elem_restrict = NULL;
    int_face_restrict_lex = NULL;
    bdr_face_restrict_lex = NULL;
+
+   timingDomain = 0.0;
+   timingBoundary = 0.0;
 }
 
 void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
@@ -296,8 +101,12 @@ void PABilinearFormExtension::Assemble()
       integrators[i]->AssemblePA(*a->FESpace());
    }
 
-   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
-               "Partial assembly does not support AddBoundaryIntegrator yet.");
+   Array<BilinearFormIntegrator*> &bdryIntegrators = *a->GetBBFI();
+   const int bdryIntegratorCount = bdryIntegrators.Size();
+   for (int i = 0; i < bdryIntegratorCount; ++i)
+   {
+      bdryIntegrators[i]->AssemblePA(*a->FESpace());
+   }
 
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
    const int intFaceIntegratorCount = intFaceIntegrators.Size();
@@ -312,6 +121,96 @@ void PABilinearFormExtension::Assemble()
    {
       bdrFaceIntegrators[i]->AssemblePABoundaryFaces(*a->FESpace());
    }
+
+   timingDomain = 0.0;
+   timingBoundary = 0.0;
+
+   if (bdryIntegratorCount > 0)
+   {
+      FiniteElementSpace *fes = a->GetFES();
+      const int nbe = fes->GetNBE();
+      const int bedofs = nbe > 0 ? nbe * fes->GetVDim() * fes->GetBE(0)->GetDof() : 0;
+      gatherMap.SetSize(bedofs);
+      indices.SetSize(bedofs);
+
+      ndofs = fes->GetNDofs();
+      offsets.SetSize(ndofs+1);
+      for (int i = 0; i <= ndofs; ++i)
+      {
+         offsets[i] = 0;
+      }
+
+      for (int i = 0; i < nbe; i++)
+      {
+         const FiniteElement &be = *fes->GetBE(i);
+         Array<int> vdofs;
+         fes -> GetBdrElementVDofs (i, vdofs);
+
+         const TensorBasisElement* el =
+            dynamic_cast<const TensorBasisElement*>(&be);
+
+         MFEM_VERIFY(el != NULL, "");
+
+         const Array<int> &fe_dof_map = el->GetDofMap();
+
+         MFEM_VERIFY(fe_dof_map.Size() == fes->GetBE(i)->GetDof(), "");
+         MFEM_VERIFY(vdofs.Size() == fes->GetBE(i)->GetDof(), "");
+
+         for (int j=0; j<vdofs.Size(); ++j)
+         {
+            const int sidj = fe_dof_map[j];
+            const int idj = sidj >= 0 ? sidj : -1 - sidj;
+            const int dof_j = vdofs[idj];
+            const int d = dof_j >= 0 ? dof_j : -1-dof_j;
+            offsets[d+1]++;
+         }
+      }
+
+      for (int i = 1; i <= ndofs; ++i)
+      {
+         // Partial sum
+         offsets[i] += offsets[i - 1];
+      }
+
+      int os = 0;
+      Array<int> cnt(ndofs);
+      cnt = 0;
+      indices = 0;
+
+      for (int i = 0; i < nbe; i++)
+      {
+         const FiniteElement &be = *fes->GetBE(i);
+         Array<int> vdofs;
+         fes -> GetBdrElementVDofs (i, vdofs);
+
+         const TensorBasisElement* el =
+            dynamic_cast<const TensorBasisElement*>(&be);
+
+         MFEM_VERIFY(el != NULL, "");
+
+         const Array<int> &fe_dof_map = el->GetDofMap();
+
+         MFEM_VERIFY(fe_dof_map.Size() == fes->GetBE(i)->GetDof(), "");
+         MFEM_VERIFY(vdofs.Size() == fes->GetBE(i)->GetDof(), "");
+
+         for (int j=0; j<vdofs.Size(); ++j)
+         {
+            const int sidj = fe_dof_map[j];
+            const int idj = sidj >= 0 ? sidj : -1 - sidj;
+            const int dof_j = vdofs[idj];
+            const bool plus = (sidj >= 0 && dof_j >= 0) || (sidj < 0 && dof_j < 0);
+            const int d = dof_j >= 0 ? dof_j : -1-dof_j;
+            const int lid = os + j;
+            gatherMap[lid] = plus ? d : -1-d;
+            indices[offsets[d] + cnt[d]] = plus ? lid : -1-lid;
+            cnt[d]++;
+         }
+
+         os += vdofs.Size();
+      }
+
+      MFEM_VERIFY(os == bedofs, "");
+   }
 }
 
 void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
@@ -379,10 +278,18 @@ void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
    A.Reset(oper); // A will own oper
 }
 
+//#define SWTIMING
+
 void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
 {
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
 
+#ifdef SWTIMING
+   StopWatch swd;
+   swd.Clear();
+   swd.Start();
+#endif
+
    const int iSz = integrators.Size();
    if (DeviceCanUseCeed() || !elem_restrict)
    {
@@ -404,6 +311,93 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
       elem_restrict->MultTranspose(localY, y);
    }
 
+#ifdef SWTIMING
+   swd.Stop();
+   timingDomain += swd.RealTime();
+
+   StopWatch swb;
+   swb.Clear();
+   swb.Start();
+#endif
+
+   Array<BilinearFormIntegrator*> &bdryIntegrators = *a->GetBBFI();
+
+   const int biSz = bdryIntegrators.Size();
+   if (DeviceCanUseCeed() || !elem_restrict)
+   {
+      MFEM_ABORT("Not implemented");
+   }
+   else if (biSz > 0)
+   {
+      FiniteElementSpace *fes = a->GetFES();
+
+      const int nbe = fes->GetNBE();
+      const int bedofs = nbe > 0 ? nbe * fes->GetVDim() * fes->GetBE(0)->GetDof() : 0;
+      bdryX.SetSize(bedofs);
+      bdryY.SetSize(bedofs);
+      bdryX.UseDevice(true);
+      bdryY.UseDevice(true);
+      bdryX = 0.0;
+      bdryY = 0.0;
+
+      {
+         MFEM_VERIFY(gatherMap.Size() == bedofs, "");
+         auto d_gatherMap = gatherMap.Read();
+         auto d_x = x.Read();
+         auto d_bdryX = bdryX.Write();
+
+         MFEM_FORALL(i, bedofs,
+         {
+            const int gid = d_gatherMap[i];
+            const bool plus = gid >= 0;
+            const int j = plus ? gid : -1-gid;
+
+            d_bdryX[i] = plus ? d_x[j] : -d_x[j];
+         });
+      }
+
+      for (int i = 0; i < biSz; ++i)
+      {
+         bdryIntegrators[i]->AddMultPA(bdryX, bdryY);
+      }
+      //elem_restrict->MultTranspose(bdryY, y);
+
+      // bdryY contains quantities on all boundary elements. Now add them to y.
+
+      {
+         const int xsize = x.Size();
+         MFEM_VERIFY(y.Size() == xsize, "");
+         MFEM_VERIFY(gatherMap.Size() == bedofs, "");
+         auto d_offsets = offsets.Read();
+         auto d_indices = indices.Read();
+         auto d_y = y.ReadWrite();
+         auto d_bdryY = bdryY.Read();
+
+         MFEM_FORALL(i, ndofs,
+         {
+            const int offset = d_offsets[i];
+            const int nextOffset = d_offsets[i + 1];
+
+            double val = 0.0;
+            for (int j = offset; j < nextOffset; ++j)
+            {
+               const int id = d_indices[j];
+               const bool plus = id >= 0;
+               const int idj = plus ? id : -1-id;
+               const double yval = d_bdryY[idj];
+               val += plus ? yval : -yval;
+            }
+
+            d_y[i] += val;
+         });
+      }
+   }
+
+#ifdef SWTIMING
+   swb.Stop();
+   timingBoundary += swb.RealTime();
+#endif
+
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
    const int iFISz = intFaceIntegrators.Size();
    if (int_face_restrict_lex && iFISz>0)
@@ -461,6 +455,10 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
       }
    }
 
+   Array<BilinearFormIntegrator*> &bdryIntegrators = *a->GetBBFI();
+   const int biSz = bdryIntegrators.Size();
+   MFEM_VERIFY(biSz == 0, "TODO");
+
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
    const int iFISz = intFaceIntegrators.Size();
    if (int_face_restrict_lex && iFISz>0)
@@ -1021,7 +1019,6 @@ void PAMixedBilinearFormExtension::Update()
       localTrial.UseDevice(true);
       localTrial.SetSize(elem_restrict_trial->Height(),
                          Device::GetMemoryType());
-
    }
    if (elem_restrict_test)
    {
@@ -1221,4 +1218,133 @@ void PAMixedBilinearFormExtension::AssembleDiagonal_ADAt(const Vector &D,
    }
 }
 
+PADiscreteLinearOperatorExtension::PADiscreteLinearOperatorExtension(
+   DiscreteLinearOperator *linop) :
+   PAMixedBilinearFormExtension(linop)
+{
+}
+
+const Operator *PADiscreteLinearOperatorExtension::GetLocalOutputProlongation()
+const
+{
+   return a->GetLocalOutputProlongation();
+}
+
+void PADiscreteLinearOperatorExtension::Assemble()
+{
+   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int integratorCount = integrators.Size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->AssemblePA(*trialFes, *testFes);
+   }
+
+   test_multiplicity.UseDevice(true);
+   test_multiplicity.SetSize(elem_restrict_test->Width()); // l-vector
+   Vector ones(elem_restrict_test->Height()); // e-vector
+   ones = 1.0;
+
+   const ElementRestriction* elem_restrict =
+      dynamic_cast<const ElementRestriction*>(elem_restrict_test);
+   if (elem_restrict)
+   {
+      elem_restrict->MultTransposeUnsigned(ones, test_multiplicity);
+   }
+   else
+   {
+      mfem_error("A real ElementRestriction is required in this setting!");
+   }
+
+   auto tm = test_multiplicity.ReadWrite();
+   MFEM_FORALL(i, test_multiplicity.Size(),
+   {
+      tm[i] = 1.0 / tm[i];
+   });
+}
+
+void PADiscreteLinearOperatorExtension::AddMult(
+   const Vector &x, Vector &y, const double c) const
+{
+   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int iSz = integrators.Size();
+
+   // * G operation
+   SetupMultInputs(elem_restrict_trial, x, localTrial,
+                   elem_restrict_test, y, localTest, c);
+
+   // * B^TDB operation
+   for (int i = 0; i < iSz; ++i)
+   {
+      integrators[i]->AddMultPA(localTrial, localTest);
+   }
+
+   // do a kind of "set" rather than "add" in the below
+   // operation as compared to the BilinearForm case
+   // * G^T operation (kind of...)
+   const ElementRestriction* elem_restrict =
+      dynamic_cast<const ElementRestriction*>(elem_restrict_test);
+   if (elem_restrict)
+   {
+      tempY.SetSize(y.Size());
+      elem_restrict->MultLeftInverse(localTest, tempY);
+      y += tempY;
+   }
+   else
+   {
+      mfem_error("In this setting you need a real ElementRestriction!");
+   }
+}
+
+void PADiscreteLinearOperatorExtension::AddMultTranspose(
+   const Vector &x, Vector &y, const double c) const
+{
+   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int iSz = integrators.Size();
+
+   // do a kind of "set" rather than "add" in the below
+   // operation as compared to the BilinearForm case
+   // * G operation (kinda)
+   Vector xscaled(x);
+   MFEM_VERIFY(x.Size() == test_multiplicity.Size(), "Input vector of wrong size");
+   auto xs = xscaled.ReadWrite();
+   auto tm = test_multiplicity.Read();
+   MFEM_FORALL(i, x.Size(),
+   {
+      xs[i] *= tm[i];
+   });
+   SetupMultInputs(elem_restrict_test, xscaled, localTest,
+                   elem_restrict_trial, y, localTrial, c);
+
+   // * B^TD^TB operation
+   for (int i = 0; i < iSz; ++i)
+   {
+      integrators[i]->AddMultTransposePA(localTest, localTrial);
+   }
+
+   // * G^T operation
+   if (elem_restrict_trial)
+   {
+      tempY.SetSize(y.Size());
+      elem_restrict_trial->MultTranspose(localTrial, tempY);
+      y += tempY;
+   }
+   else
+   {
+      mfem_error("Trial ElementRestriction not defined");
+   }
+}
+
+void PADiscreteLinearOperatorExtension::FormRectangularSystemOperator(
+   const Array<int>& ess1, const Array<int>& ess2, OperatorHandle &A)
+{
+   const Operator *Pi = this->GetProlongation();
+   const Operator *Po = this->GetLocalOutputProlongation();
+   Operator *rap = SetupRAP(Pi, Po);
+
+   RectangularConstrainedOperator *Arco
+      = new RectangularConstrainedOperator(rap, ess1, ess2, rap != this);
+
+   A.Reset(Arco);
+}
+
 } // namespace mfem

fem/bilinearform_ext.hpp

@@ -21,6 +21,7 @@ namespace mfem
 
 class BilinearForm;
 class MixedBilinearForm;
+class DiscreteLinearOperator;
 
 /// Class extending the BilinearForm class to support different AssemblyLevels.
 /**  FA - Full Assembly
@@ -68,11 +69,16 @@ class PABilinearFormExtension : public BilinearFormExtension
 protected:
    const FiniteElementSpace *trialFes, *testFes; // Not owned
    mutable Vector localX, localY;
+   mutable Vector bdryX, bdryY;
    mutable Vector faceIntX, faceIntY;
    mutable Vector faceBdrX, faceBdrY;
    const Operator *elem_restrict; // Not owned
    const Operator *int_face_restrict_lex; // Not owned
    const Operator *bdr_face_restrict_lex; // Not owned
+   Array<int> gatherMap;
+   Array<int> indices;
+   Array<int> offsets;
+   int ndofs;
 
 public:
    PABilinearFormExtension(BilinearForm*);
@@ -88,6 +94,8 @@ public:
    void MultTranspose(const Vector &x, Vector &y) const;
    void Update();
 
+   mutable double timingDomain, timingBoundary;
+
 protected:
    void SetupRestrictionOperators(const L2FaceValues m);
 };
@@ -130,31 +138,24 @@ public:
    void MultTranspose(const Vector &x, Vector &y) const;
 };
 
-/// Data and methods for matrix-free bilinear forms
+/// Data and methods for matrix-free bilinear forms NOT YET IMPLEMENTED.
 class MFBilinearFormExtension : public BilinearFormExtension
 {
-protected:
-   const FiniteElementSpace *trialFes, *testFes; // Not owned
-   mutable Vector localX, localY;
-   mutable Vector faceIntX, faceIntY;
-   mutable Vector faceBdrX, faceBdrY;
-   const Operator *elem_restrict; // Not owned
-   const Operator *int_face_restrict_lex; // Not owned
-   const Operator *bdr_face_restrict_lex; // Not owned
-
 public:
-   MFBilinearFormExtension(BilinearForm *form);
+   MFBilinearFormExtension(BilinearForm *form)
+      : BilinearFormExtension(form) { }
 
-   void Assemble();
-   void AssembleDiagonal(Vector &diag) const;
-   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
+   /// TODO
+   void Assemble() {}
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A) {}
    void FormLinearSystem(const Array<int> &ess_tdof_list,
                          Vector &x, Vector &b,
                          OperatorHandle &A, Vector &X, Vector &B,
-                         int copy_interior = 0);
-   void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
-   void Update();
+                         int copy_interior = 0) {}
+   void Mult(const Vector &x, Vector &y) const {}
+   void MultTranspose(const Vector &x, Vector &y) const {}
+   void Update() {}
+   ~MFBilinearFormExtension() {}
 };
 
 /// Class extending the MixedBilinearForm class to support different AssemblyLevels.
@@ -212,7 +213,7 @@ protected:
    mutable Vector localTrial, localTest, tempY;
    const Operator *elem_restrict_trial; // Not owned
    const Operator *elem_restrict_test;  // Not owned
-private:
+
    /// Helper function to set up inputs/outputs for Mult or MultTranspose
    void SetupMultInputs(const Operator *elem_restrict_x,
                         const Vector &x, Vector &localX,
@@ -258,6 +259,33 @@ public:
    void Update();
 };
 
+
+/**
+   This acts very much like PAMixedBilinearFormExtension, but its
+   FormRectangularSystemOperator implementation emulates 'Set' rather than
+   'Add' in the assembly case.
+*/
+class PADiscreteLinearOperatorExtension : public PAMixedBilinearFormExtension
+{
+public:
+   PADiscreteLinearOperatorExtension(DiscreteLinearOperator *linop);
+
+   /// Partial assembly of all internal integrators
+   void Assemble();
+
+   void AddMult(const Vector &x, Vector &y, const double c) const;
+
+   void AddMultTranspose(const Vector &x, Vector &y, const double c=1.0) const;
+
+   void FormRectangularSystemOperator(const Array<int>&, const Array<int>&,
+                                      OperatorHandle& A);
+
+   const Operator * GetLocalOutputProlongation() const;
+
+private:
+   Vector test_multiplicity;
+};
+
 }
 
 #endif

fem/bilininteg.cpp

@@ -22,14 +22,14 @@ namespace mfem
 
 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePA(...)\n"
+   mfem_error ("BilinearFormIntegrator::AssemblePA(fes)\n"
                "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&,
                                         const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePA(...)\n"
+   mfem_error ("BilinearFormIntegrator::AssemblePA(fes, fes)\n"
                "   is not implemented for this class.");
 }
 
@@ -92,31 +92,7 @@ void BilinearFormIntegrator::AddMultPA(const Vector &, Vector &) const
 
 void BilinearFormIntegrator::AddMultTransposePA(const Vector &, Vector &) const
 {
-   mfem_error ("BilinearFormIntegrator::MultAssembledTranspose(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AssembleMF(const FiniteElementSpace &fes)
-{
-   mfem_error ("BilinearFormIntegrator::AssembleMF(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AddMultMF(const Vector &, Vector &) const
-{
-   mfem_error ("BilinearFormIntegrator::AddMultMF(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AddMultTransposeMF(const Vector &, Vector &) const
-{
-   mfem_error ("BilinearFormIntegrator::AddMultTransposeMF(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AssembleDiagonalMF(Vector &)
-{
-   mfem_error ("BilinearFormIntegrator::AssembleDiagonalMF(...)\n"
+   mfem_error ("BilinearFormIntegrator::AddMultTransposePA(...)\n"
                "   is not implemented for this class.");
 }
 
@@ -157,22 +133,8 @@ void BilinearFormIntegrator::AssembleElementVector(
    const FiniteElement &el, ElementTransformation &Tr, const Vector &elfun,
    Vector &elvect)
 {
-   // Note: This default implementation is general but not efficient
-   DenseMatrix elmat;
-   AssembleElementMatrix(el, Tr, elmat);
-   elvect.SetSize(elmat.Height());
-   elmat.Mult(elfun, elvect);
-}
-
-void BilinearFormIntegrator::AssembleFaceVector(
-   const FiniteElement &el1, const FiniteElement &el2,
-   FaceElementTransformations &Tr, const Vector &elfun, Vector &elvect)
-{
-   // Note: This default implementation is general but not efficient
-   DenseMatrix elmat;
-   AssembleFaceMatrix(el1, el2, Tr, elmat);
-   elvect.SetSize(elmat.Height());
-   elmat.Mult(elfun, elvect);
+   mfem_error("BilinearFormIntegrator::AssembleElementVector\n"
+              "   is not implemented for this class.");
 }
 
 
@@ -554,12 +516,10 @@ void DiffusionIntegrator::AssembleElementMatrix
 
 #ifdef MFEM_THREAD_SAFE
    DenseMatrix dshape(nd,dim), dshapedxt(nd,spaceDim), invdfdx(dim,spaceDim);
-   Vector D(VQ ? VQ->GetVDim() : 0);
 #else
    dshape.SetSize(nd,dim);
    dshapedxt.SetSize(nd,spaceDim);
    invdfdx.SetSize(dim,spaceDim);
-   D.SetSize(VQ ? VQ->GetVDim() : 0);
 #endif
    elmat.SetSize(nd);
 
@@ -577,20 +537,7 @@ void DiffusionIntegrator::AssembleElementMatrix
       // AdjugateJacobian = / adj(J),         if J is square
       //                    \ adj(J^t.J).J^t, otherwise
       Mult(dshape, Trans.AdjugateJacobian(), dshapedxt);
-      if (MQ)
-      {
-         MQ->Eval(invdfdx, Trans, ip);
-         invdfdx *= w;
-         Mult(dshapedxt, invdfdx, dshape);
-         AddMultABt(dshape, dshapedxt, elmat);
-      }
-      else if (VQ)
-      {
-         VQ->Eval(D, Trans, ip);
-         D *= w;
-         AddMultADAt(dshapedxt, D, elmat);
-      }
-      else
+      if (!MQ)
       {
          if (Q)
          {
@@ -598,6 +545,13 @@ void DiffusionIntegrator::AssembleElementMatrix
          }
          AddMult_a_AAt(w, dshapedxt, elmat);
       }
+      else
+      {
+         MQ->Eval(invdfdx, Trans, ip);
+         invdfdx *= w;
+         Mult(dshapedxt, invdfdx, dshape);
+         AddMultABt(dshape, dshapedxt, elmat);
+      }
    }
 }
 
@@ -616,14 +570,12 @@ void DiffusionIntegrator::AssembleElementMatrix2(
    DenseMatrix dshape(tr_nd, dim), dshapedxt(tr_nd, spaceDim);
    DenseMatrix te_dshape(te_nd, dim), te_dshapedxt(te_nd, spaceDim);
    DenseMatrix invdfdx(dim, spaceDim);
-   Vector D(VQ ? VQ->GetVDim() : 0);
 #else
    dshape.SetSize(tr_nd, dim);
    dshapedxt.SetSize(tr_nd, spaceDim);
    te_dshape.SetSize(te_nd, dim);
    te_dshapedxt.SetSize(te_nd, spaceDim);
    invdfdx.SetSize(dim, spaceDim);
-   D.SetSize(VQ ? VQ->GetVDim() : 0);
 #endif
    elmat.SetSize(te_nd, tr_nd);
 
@@ -643,20 +595,7 @@ void DiffusionIntegrator::AssembleElementMatrix2(
       Mult(dshape, invdfdx, dshapedxt);
       Mult(te_dshape, invdfdx, te_dshapedxt);
       // invdfdx, dshape, and te_dshape no longer needed
-      if (MQ)
-      {
-         MQ->Eval(invdfdx, Trans, ip);
-         invdfdx *= w;
-         Mult(te_dshapedxt, invdfdx, te_dshape);
-         AddMultABt(te_dshape, dshapedxt, elmat);
-      }
-      else if (VQ)
-      {
-         VQ->Eval(D, Trans, ip);
-         D *= w;
-         AddMultADAt(dshapedxt, D, elmat);
-      }
-      else
+      if (!MQ)
       {
          if (Q)
          {
@@ -665,6 +604,13 @@ void DiffusionIntegrator::AssembleElementMatrix2(
          dshapedxt *= w;
          AddMultABt(te_dshapedxt, dshapedxt, elmat);
       }
+      else
+      {
+         MQ->Eval(invdfdx, Trans, ip);
+         invdfdx *= w;
+         Mult(te_dshapedxt, invdfdx, te_dshape);
+         AddMultABt(te_dshape, dshapedxt, elmat);
+      }
    }
 }
 
@@ -676,19 +622,12 @@ void DiffusionIntegrator::AssembleElementVector(
    int dim = el.GetDim();
    double w;
 
-   if (VQ)
-   {
-      MFEM_VERIFY(VQ->GetVDim() == dim, "Unexpected dimension for VectorCoefficient");
-   }
-
 #ifdef MFEM_THREAD_SAFE
    DenseMatrix dshape(nd,dim), invdfdx(dim), mq(dim);
-   Vector D(VQ ? VQ->GetVDim() : 0);
 #else
    dshape.SetSize(nd,dim);
    invdfdx.SetSize(dim);
    mq.SetSize(dim);
-   D.SetSize(VQ ? VQ->GetVDim() : 0);
 #endif
    vec.SetSize(dim);
    pointflux.SetSize(dim);
@@ -707,7 +646,7 @@ void DiffusionIntegrator::AssembleElementVector(
       CalcAdjugate(Tr.Jacobian(), invdfdx); // invdfdx = adj(J)
       w = ip.weight / Tr.Weight();
 
-      if (!MQ && !VQ)
+      if (!MQ)
       {
          dshape.MultTranspose(elfun, vec);
          invdfdx.MultTranspose(vec, pointflux);
@@ -718,21 +657,11 @@ void DiffusionIntegrator::AssembleElementVector(
       }
       else
       {
+
          dshape.MultTranspose(elfun, pointflux);
          invdfdx.MultTranspose(pointflux, vec);
-         if (MQ)
-         {
-            MQ->Eval(mq, Tr, ip);
-            mq.Mult(vec, pointflux);
-         }
-         else
-         {
-            VQ->Eval(D, Tr, ip);
-            for (int j=0; j<dim; ++j)
-            {
-               pointflux[j] *= D[j];
-            }
-         }
+         MQ->Eval(mq, Tr, ip);
+         mq.Mult(vec, pointflux);
       }
       pointflux *= w;
       invdfdx.Mult(pointflux, vec);
@@ -971,6 +900,44 @@ const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
    return IntRules.Get(trial_fe.GetGeomType(), order);
 }
 
+/// alpha (n x u, v)
+void VectorFEBoundaryTangentIntegrator::AssembleElementMatrix
+(const FiniteElement &el, ElementTransformation &Trans,
+ DenseMatrix &elmat)
+{
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, Trans);
+
+   const int nd1 = el.GetDof();
+
+   DenseMatrix vshape(nd1, 3);
+   DenseMatrix vshapeRotated(nd1, 3);
+
+   elmat.SetSize(nd1);
+
+   elmat = 0.0;
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      Trans.SetIntPoint (&ip);
+
+      Vector n(3);
+      CalcOrtho(Trans.Jacobian(), n);
+
+      el.CalcVShape(Trans, vshape);
+
+      for (int j=0; j<nd1; ++j)
+      {
+         // Set vshapeRotated(j) = n x vshape
+         vshapeRotated(j, 0) = (n[1] * vshape(j, 2)) - (n[2] * vshape(j, 1));
+         vshapeRotated(j, 1) = (n[2] * vshape(j, 0)) - (n[0] * vshape(j, 2));
+         vshapeRotated(j, 2) = (n[0] * vshape(j, 1)) - (n[1] * vshape(j, 0));
+      }
+
+      const double w = alpha * ip.weight; // Trans.Weight() is included in n
+
+      AddMult_a_ABt(w, vshape, vshapeRotated, elmat);
+   }
+}
 
 void BoundaryMassIntegrator::AssembleFaceMatrix(
    const FiniteElement &el1, const FiniteElement &el2,
@@ -2265,7 +2232,16 @@ void VectorDiffusionIntegrator::AssembleElementMatrix(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      ir = &DiffusionIntegrator::GetRule(el,el);
+      // integrand is rational function if det(J) is not constant
+      int order = 2 * Trans.OrderGrad(&el); // order of the numerator
+      if (el.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(el.GetGeomType(), order);
+      }
    }
 
    elmat = 0.0;
@@ -2316,7 +2292,11 @@ void VectorDiffusionIntegrator::AssembleElementVector(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      ir = &DiffusionIntegrator::GetRule(el,el);
+      // integrand is rational function if det(J) is not constant
+      int order = 2 * Tr.OrderGrad(&el); // order of the numerator
+      ir = (el.Space() == FunctionSpace::rQk) ?
+           &RefinedIntRules.Get(el.GetGeomType(), order) :
+           &IntRules.Get(el.GetGeomType(), order);
    }
 
    elvect = 0.0;

fem/bilininteg.hpp

@@ -17,6 +17,8 @@
 #include "fespace.hpp"
 #include "libceed/ceed.hpp"
 
+//#define SETUPONHOST
+
 namespace mfem
 {
 
@@ -95,30 +97,6 @@ public:
    //                         const FiniteElementSpace &test_fes,
    //                         Vector &emat);
 
-   /// Method defining matrix-free assembly.
-   /** The result of fully matrix-free assembly is stored internally so that it
-       can be used later in the methods AddMultMF() and AddMultTransposeMF(). */
-   virtual void AssembleMF(const FiniteElementSpace &fes);
-
-   /** Perform the action of integrator on the input @a x and add the result to
-       the output @a y. Both @a x and @a y are E-vectors, i.e. they represent
-       the element-wise discontinuous version of the FE space.
-
-       This method can be called only after the method AssembleMF() has been
-       called. */
-   virtual void AddMultMF(const Vector &x, Vector &y) const;
-
-   /** Perform the transpose action of integrator on the input @a x and add the
-       result to the output @a y. Both @a x and @a y are E-vectors, i.e. they
-       represent the element-wise discontinuous version of the FE space.
-
-       This method can be called only after the method AssemblePA() has been
-       called. */
-   virtual void AddMultTransposeMF(const Vector &x, Vector &y) const;
-
-   /// Assemble diagonal and add it to Vector @a diag.
-   virtual void AssembleDiagonalMF(Vector &diag);
-
    virtual void AssembleEAInteriorFaces(const FiniteElementSpace &fes,
                                         Vector &ea_data_int,
                                         Vector &ea_data_ext,
@@ -155,22 +133,11 @@ public:
                                    FaceElementTransformations &Trans,
                                    DenseMatrix &elmat);
 
-   /// @brief Perform the local action of the BilinearFormIntegrator.
-   /// Note that the default implementation in the base class is general but not
-   /// efficient.
+   /// Perform the local action of the BilinearFormIntegrator
    virtual void AssembleElementVector(const FiniteElement &el,
                                       ElementTransformation &Tr,
                                       const Vector &elfun, Vector &elvect);
 
-   /// @brief Perform the local action of the BilinearFormIntegrator resulting
-   /// from a face integral term.
-   /// Note that the default implementation in the base class is general but not
-   /// efficient.
-   virtual void AssembleFaceVector(const FiniteElement &el1,
-                                   const FiniteElement &el2,
-                                   FaceElementTransformations &Tr,
-                                   const Vector &elfun, Vector &elvect);
-
    virtual void AssembleElementGrad(const FiniteElement &el,
                                     ElementTransformation &Tr,
                                     const Vector &elfun, DenseMatrix &elmat)
@@ -242,6 +209,9 @@ public:
    { return 0.0; }
 
    virtual ~BilinearFormIntegrator() { }
+
+   bool isBdryInteg = false;
+   Array<int> *el_marker = NULL;
 };
 
 /** Wraps a given @a BilinearFormIntegrator and transposes the resulting element
@@ -1844,8 +1814,10 @@ protected:
 };
 
 /** Class for integrating the bilinear form a(u,v) := (Q grad u, v) where Q is a
-    scalar coefficient, and v is a vector with components v_i in the same space
-    as u. */
+    scalar coefficient, and v is a vector with components v_i in the same (H1) space
+    as u.
+
+    See also MixedVectorGradientIntegrator when v is in H(curl). */
 class GradientIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -1898,7 +1870,6 @@ class DiffusionIntegrator: public BilinearFormIntegrator
 {
 protected:
    Coefficient *Q;
-   VectorCoefficient *VQ;
    MatrixCoefficient *MQ;
 
 private:
@@ -1906,7 +1877,6 @@ private:
 #ifndef MFEM_THREAD_SAFE
    DenseMatrix dshape, dshapedxt, invdfdx, mq;
    DenseMatrix te_dshape, te_dshapedxt;
-   Vector D;
 #endif
 
    // PA extension
@@ -1915,30 +1885,54 @@ private:
    const GeometricFactors *geom;  ///< Not owned
    int dim, ne, dofs1D, quad1D;
    Vector pa_data;
-   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
+
+#ifdef MFEM_USE_CEED
    // CEED extension
    CeedData* ceedDataPtr;
+#endif
 
 public:
    /// Construct a diffusion integrator with coefficient Q = 1
    DiffusionIntegrator()
-      : Q(NULL), VQ(NULL), MQ(NULL), maps(NULL), geom(NULL), ceedDataPtr(NULL) { }
+   {
+      Q = NULL;
+      MQ = NULL;
+      maps = NULL;
+      geom = NULL;
+#ifdef MFEM_USE_CEED
+      ceedDataPtr = NULL;
+#endif
+   }
 
    /// Construct a diffusion integrator with a scalar coefficient q
    DiffusionIntegrator(Coefficient &q)
-      : Q(&q), VQ(NULL), MQ(NULL), maps(NULL), geom(NULL), ceedDataPtr(NULL) { }
-
-   /// Construct a diffusion integrator with a vector coefficient q
-   DiffusionIntegrator(VectorCoefficient &q)
-      : Q(NULL), VQ(&q), MQ(NULL), maps(NULL), geom(NULL), ceedDataPtr(NULL) { }
+      : Q(&q)
+   {
+      MQ = NULL;
+      maps = NULL;
+      geom = NULL;
+#ifdef MFEM_USE_CEED
+      ceedDataPtr = NULL;
+#endif
+   }
 
    /// Construct a diffusion integrator with a matrix coefficient q
    DiffusionIntegrator(MatrixCoefficient &q)
-      : Q(NULL), VQ(NULL), MQ(&q), maps(NULL), geom(NULL), ceedDataPtr(NULL) { }
+      : MQ(&q)
+   {
+      Q = NULL;
+      maps = NULL;
+      geom = NULL;
+#ifdef MFEM_USE_CEED
+      ceedDataPtr = NULL;
+#endif
+   }
 
    virtual ~DiffusionIntegrator()
    {
+#ifdef MFEM_USE_CEED
       delete ceedDataPtr;
+#endif
    }
 
    /** Given a particular Finite Element computes the element stiffness matrix
@@ -1969,8 +1963,6 @@ public:
 
    using BilinearFormIntegrator::AssemblePA;
 
-   virtual void AssembleMF(const FiniteElementSpace &fes);
-
    virtual void AssemblePA(const FiniteElementSpace &fes);
 
    virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
@@ -1978,14 +1970,12 @@ public:
 
    virtual void AssembleDiagonalPA(Vector &diag);
 
-   virtual void AssembleDiagonalMF(Vector &diag);
-
-   virtual void AddMultMF(const Vector&, Vector&) const;
-
    virtual void AddMultPA(const Vector&, Vector&) const;
 
    static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
                                          const FiniteElement &test_fe);
+
+   void SetupPA(const FiniteElementSpace &fes);
 };
 
 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
@@ -2003,22 +1993,39 @@ protected:
    const GeometricFactors *geom;  ///< Not owned
    int dim, ne, nq, dofs1D, quad1D;
 
+#ifdef MFEM_USE_CEED
    // CEED extension
    CeedData* ceedDataPtr;
+#endif
 
 public:
    MassIntegrator(const IntegrationRule *ir = NULL)
-      : BilinearFormIntegrator(ir), Q(NULL), maps(NULL), geom(NULL),
-        ceedDataPtr(NULL) { }
+      : BilinearFormIntegrator(ir)
+   {
+      Q = NULL;
+      maps = NULL;
+      geom = NULL;
+#ifdef MFEM_USE_CEED
+      ceedDataPtr = NULL;
+#endif
+   }
 
    /// Construct a mass integrator with coefficient q
    MassIntegrator(Coefficient &q, const IntegrationRule *ir = NULL)
-      : BilinearFormIntegrator(ir), Q(&q), maps(NULL), geom(NULL),
-        ceedDataPtr(NULL) { }
+      : BilinearFormIntegrator(ir), Q(&q)
+   {
+      maps = NULL;
+      geom = NULL;
+#ifdef MFEM_USE_CEED
+      ceedDataPtr = NULL;
+#endif
+   }
 
    virtual ~MassIntegrator()
    {
+#ifdef MFEM_USE_CEED
       delete ceedDataPtr;
+#endif
    }
    /** Given a particular Finite Element computes the element mass matrix
        elmat. */
@@ -2032,8 +2039,6 @@ public:
 
    using BilinearFormIntegrator::AssemblePA;
 
-   virtual void AssembleMF(const FiniteElementSpace &fes);
-
    virtual void AssemblePA(const FiniteElementSpace &fes);
 
    virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
@@ -2041,10 +2046,6 @@ public:
 
    virtual void AssembleDiagonalPA(Vector &diag);
 
-   virtual void AssembleDiagonalMF(Vector &diag);
-
-   virtual void AddMultMF(const Vector&, Vector&) const;
-
    virtual void AddMultPA(const Vector&, Vector&) const;
 
    static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
@@ -2068,6 +2069,20 @@ public:
                                    DenseMatrix &elmat);
 };
 
+/// alpha (n x u, v)
+class VectorFEBoundaryTangentIntegrator : public MassIntegrator
+{
+private:
+   double alpha;
+
+public:
+   VectorFEBoundaryTangentIntegrator(double a = 1.0) : alpha(a)
+   { }
+
+   void AssembleElementMatrix(const FiniteElement &el,
+                              ElementTransformation &Trans, DenseMatrix &elmat);
+};
+
 /// alpha (q . grad u, v)
 class ConvectionIntegrator : public BilinearFormIntegrator
 {
@@ -2151,34 +2166,24 @@ protected:
    const GeometricFactors *geom;  ///< Not owned
    int dim, ne, nq, dofs1D, quad1D;
 
-   // CEED extension
-   CeedData* ceedDataPtr;
-
 public:
    /// Construct an integrator with coefficient 1.0
    VectorMassIntegrator()
-      : vdim(-1), Q_order(0), Q(NULL), VQ(NULL), MQ(NULL), ceedDataPtr(NULL) { }
+      : vdim(-1), Q_order(0), Q(NULL), VQ(NULL), MQ(NULL) { }
    /** Construct an integrator with scalar coefficient q.  If possible, save
        memory by using a scalar integrator since the resulting matrix is block
        diagonal with the same diagonal block repeated. */
    VectorMassIntegrator(Coefficient &q, int qo = 0)
-      : vdim(-1), Q_order(qo), Q(&q), VQ(NULL), MQ(NULL), ceedDataPtr(NULL) { }
+      : vdim(-1), Q(&q) { VQ = NULL; MQ = NULL; Q_order = qo; }
    VectorMassIntegrator(Coefficient &q, const IntegrationRule *ir)
-      : BilinearFormIntegrator(ir), vdim(-1), Q_order(0), Q(&q), VQ(NULL),
-        MQ(NULL), ceedDataPtr(NULL) { }
+      : BilinearFormIntegrator(ir), vdim(-1), Q(&q)
+   { VQ = NULL; MQ = NULL; Q_order = 0; }
    /// Construct an integrator with diagonal coefficient q
    VectorMassIntegrator(VectorCoefficient &q, int qo = 0)
-      : vdim(q.GetVDim()), Q_order(qo), Q(NULL), VQ(&q), MQ(NULL),
-        ceedDataPtr(NULL) { }
+      : vdim(q.GetVDim()), VQ(&q) { Q = NULL; MQ = NULL; Q_order = qo; }
    /// Construct an integrator with matrix coefficient q
    VectorMassIntegrator(MatrixCoefficient &q, int qo = 0)
-      : vdim(q.GetVDim()), Q_order(qo), Q(NULL), VQ(NULL), MQ(&q),
-        ceedDataPtr(NULL) { }
-
-   virtual ~VectorMassIntegrator()
-   {
-      delete ceedDataPtr;
-   }
+      : vdim(q.GetVDim()), MQ(&q) { Q = NULL; VQ = NULL; Q_order = qo; }
 
    int GetVDim() const { return vdim; }
    void SetVDim(int vdim) { this->vdim = vdim; }
@@ -2192,11 +2197,8 @@ public:
                                        DenseMatrix &elmat);
    using BilinearFormIntegrator::AssemblePA;
    virtual void AssemblePA(const FiniteElementSpace &fes);
-   virtual void AssembleMF(const FiniteElementSpace &fes);
    virtual void AssembleDiagonalPA(Vector &diag);
-   virtual void AssembleDiagonalMF(Vector &diag);
    virtual void AddMultPA(const Vector &x, Vector &y) const;
-   virtual void AddMultMF(const Vector &x, Vector &y) const;
 };
 
 
@@ -2560,23 +2562,13 @@ protected:
    int dim, sdim, ne, dofs1D, quad1D;
    Vector pa_data;
 
-   // CEED extension
-   CeedData* ceedDataPtr;
-
 private:
    DenseMatrix dshape, dshapedxt, pelmat;
    DenseMatrix Jinv, gshape;
 
 public:
-   VectorDiffusionIntegrator()
-      : Q(NULL), ceedDataPtr(NULL) { }
-   VectorDiffusionIntegrator(Coefficient &q)
-      : Q(&q), ceedDataPtr(NULL) { }
-
-   virtual ~VectorDiffusionIntegrator()
-   {
-      delete ceedDataPtr;
-   }
+   VectorDiffusionIntegrator() { Q = NULL; }
+   VectorDiffusionIntegrator(Coefficient &q) { Q = &q; }
 
    virtual void AssembleElementMatrix(const FiniteElement &el,
                                       ElementTransformation &Trans,
@@ -2586,11 +2578,8 @@ public:
                                       const Vector &elfun, Vector &elvect);
    using BilinearFormIntegrator::AssemblePA;
    virtual void AssemblePA(const FiniteElementSpace &fes);
-   virtual void AssembleMF(const FiniteElementSpace &fes);
    virtual void AssembleDiagonalPA(Vector &diag);
-   virtual void AssembleDiagonalMF(Vector &diag);
    virtual void AddMultPA(const Vector &x, Vector &y) const;
-   virtual void AddMultMF(const Vector &x, Vector &y) const;
 };
 
 /** Integrator for the linear elasticity form:
@@ -2910,11 +2899,33 @@ class DiscreteInterpolator : public BilinearFormIntegrator { };
 class GradientInterpolator : public DiscreteInterpolator
 {
 public:
+   GradientInterpolator() : fake_fe(NULL) { }
+   virtual ~GradientInterpolator() { delete fake_fe; }
+
    virtual void AssembleElementMatrix2(const FiniteElement &h1_fe,
                                        const FiniteElement &nd_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat)
    { nd_fe.ProjectGrad(h1_fe, Trans, elmat); }
+
+   using BilinearFormIntegrator::AssemblePA;
+   /**
+      trial_fes should be H1 Lagrange
+      test_fes should be Nedelec
+   */
+   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
+                           const FiniteElementSpace &test_fes);
+
+   virtual void AddMultPA(const Vector &x, Vector &y) const;
+   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
+
+private:
+   /// 1D finit element that generates and owns the 1D DofToQuad maps below
+   FiniteElement * fake_fe;
+
+   const DofToQuad *maps_C_C; // one-d map with Lobatto rows, Lobatto columns
+   const DofToQuad *maps_O_C; // one-d map with Legendre rows, Lobatto columns
+   int dim, ne, o_dofs1D, c_dofs1D;
 };
 
 
@@ -2929,6 +2940,24 @@ public:
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat)
    { ran_fe.Project(dom_fe, Trans, elmat); }
+
+   using BilinearFormIntegrator::AssemblePA;
+
+   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
+                           const FiniteElementSpace &test_fes);
+
+   virtual void AddMultPA(const Vector &x, Vector &y) const;
+   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
+
+private:
+   /// 1D finit element that generates and owns the 1D DofToQuad maps below
+   FiniteElement * fake_fe;
+
+   const DofToQuad *maps_C_C; // one-d map with Lobatto rows, Lobatto columns
+   const DofToQuad *maps_O_C; // one-d map with Legendre rows, Lobatto columns
+   int dim, ne, o_dofs1D, c_dofs1D;
+
+   Vector pa_data;
 };
 
 
@@ -3061,17 +3090,6 @@ protected:
    VectorCoefficient *VQ;
 };
 
-
-
-// PA Diffusion Assemble 2D kernel
-template<const int T_SDIM>
-void PADiffusionSetup2D(const int Q1D,
-                        const int coeffDim,
-                        const int NE,
-                        const Array<double> &w,
-                        const Vector &j,
-                        const Vector &c,
-                        Vector &d);
-
 }
+
 #endif

fem/bilininteg_diffusion_mf.cpp

@@ -1,70 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "libceed/diffusion.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-void DiffusionIntegrator::AssembleMF(const FiniteElementSpace &fes)
-{
-#ifdef MFEM_USE_CEED
-   // Assuming the same element type
-   fespace = &fes;
-   Mesh *mesh = fes.GetMesh();
-   if (mesh->GetNE() == 0) { return; }
-   const FiniteElement &el = *fes.GetFE(0);
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
-   if (DeviceCanUseCeed())
-   {
-      delete ceedDataPtr;
-      ceedDataPtr = new CeedData;
-      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
-      return CeedMFDiffusionAssemble(fes, *ir, *ceedDataPtr);
-   }
-#endif
-   mfem_error("Error: DiffusionIntegrator::AssembleMF only implemented with libCEED");
-}
-
-void DiffusionIntegrator::AssembleDiagonalMF(Vector &diag)
-{
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAssembleDiagonal(ceedDataPtr, diag);
-   }
-   else
-#endif
-   {
-      mfem_error("Error: DiffusionIntegrator::AssembleDiagonalMF only implemented with libCEED");
-   }
-}
-
-void DiffusionIntegrator::AddMultMF(const Vector &x, Vector &y) const
-{
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAddMult(ceedDataPtr, x, y);
-   }
-   else
-#endif
-   {
-      mfem_error("Error: DiffusionIntegrator::AddMultMF only implemented with libCEED");
-   }
-}
-
-}

fem/bilininteg_diffusion_pa.cpp

@@ -80,24 +80,28 @@ static void OccaPADiffusionSetup3D(const int D1D,
 }
 #endif // MFEM_USE_OCCA
 
+// PA Diffusion Assemble 2D kernel
+template<const int T_SDIM>
+static void PADiffusionSetup2D(const int Q1D,
+                               const int NE,
+                               const Array<double> &w,
+                               const Vector &j,
+                               const Vector &c,
+                               Vector &d);
 template<>
 void PADiffusionSetup2D<2>(const int Q1D,
-                           const int coeffDim,
                            const int NE,
                            const Array<double> &w,
                            const Vector &j,
                            const Vector &c,
                            Vector &d)
 {
-   const bool symmetric = (coeffDim != 4);
    const bool const_c = c.Size() == 1;
-   MFEM_VERIFY(coeffDim < 3 ||
-               !const_c, "Constant matrix coefficient not supported");
    const auto W = Reshape(w.Read(), Q1D,Q1D);
    const auto J = Reshape(j.Read(), Q1D,Q1D,2,2,NE);
-   const auto C = const_c ? Reshape(c.Read(), 1,1,1,1) :
-                  Reshape(c.Read(), coeffDim,Q1D,Q1D,NE);
-   auto D = Reshape(d.Write(), Q1D,Q1D, symmetric ? 3 : 4, NE);
+   const auto C = const_c ? Reshape(c.Read(), 1,1,1) :
+                  Reshape(c.Read(), Q1D,Q1D,NE);
+   auto D = Reshape(d.Write(), Q1D,Q1D, 3, NE);
    MFEM_FORALL_2D(e, NE, Q1D,Q1D,1,
    {
       MFEM_FOREACH_THREAD(qx,x,Q1D)
@@ -108,39 +112,11 @@ void PADiffusionSetup2D<2>(const int Q1D,
             const double J21 = J(qx,qy,1,0,e);
             const double J12 = J(qx,qy,0,1,e);
             const double J22 = J(qx,qy,1,1,e);
-            const double w_detJ = W(qx,qy) / ((J11*J22)-(J21*J12));
-            if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient
-            {
-               // First compute entries of R = MJ^{-T}, without det J factor.
-               const double M11 = C(0,qx,qy,e);
-               const double M12 = C(1,qx,qy,e);
-               const double M21 = symmetric ? M12 : C(2,qx,qy,e);
-               const double M22 = symmetric ? C(2,qx,qy,e) : C(3,qx,qy,e);
-               const double R11 = M11*J22 - M12*J12;
-               const double R21 = M21*J22 - M22*J12;
-               const double R12 = -M11*J21 + M12*J11;
-               const double R22 = -M21*J21 + M22*J11;
-
-               // Now set y to J^{-1}R.
-               D(qx,qy,0,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
-               D(qx,qy,1,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
-               D(qx,qy,2,e) = w_detJ * (symmetric ? (-J21*R12 + J11*R22) :
-               (J22*R12 - J12*R22)); // 2,2 or 1,2
-               if (!symmetric)
-               {
-                  D(qx,qy,3,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
-               }
-            }
-            else // Vector or scalar coefficient
-            {
-               const double C1 = const_c ? C(0,0,0,0) : C(0,qx,qy,e);
-               const double C2 = const_c ? C(0,0,0,0) :
-                                 (coeffDim == 2 ? C(1,qx,qy,e) : C(0,qx,qy,e));
-
-               D(qx,qy,0,e) =  w_detJ * (C2*J12*J12 + C1*J22*J22); // 1,1
-               D(qx,qy,1,e) = -w_detJ * (C2*J12*J11 + C1*J22*J21); // 1,2
-               D(qx,qy,2,e) =  w_detJ * (C2*J11*J11 + C1*J21*J21); // 2,2
-            }
+            const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
+            const double c_detJ = W(qx,qy) * coeff / ((J11*J22)-(J21*J12));
+            D(qx,qy,0,e) =  c_detJ * (J12*J12 + J22*J22); // 1,1
+            D(qx,qy,1,e) = -c_detJ * (J12*J11 + J22*J21); // 1,2
+            D(qx,qy,2,e) =  c_detJ * (J11*J11 + J21*J21); // 2,2
          }
       }
    });
@@ -149,14 +125,12 @@ void PADiffusionSetup2D<2>(const int Q1D,
 // PA Diffusion Assemble 2D kernel with 3D node coords
 template<>
 void PADiffusionSetup2D<3>(const int Q1D,
-                           const int coeffDim,
                            const int NE,
                            const Array<double> &w,
                            const Vector &j,
                            const Vector &c,
                            Vector &d)
 {
-   MFEM_VERIFY(coeffDim == 1, "Matrix and vector coefficients not supported");
    constexpr int DIM = 2;
    constexpr int SDIM = 3;
    const bool const_c = c.Size() == 1;
@@ -193,23 +167,19 @@ void PADiffusionSetup2D<3>(const int Q1D,
 }
 
 // PA Diffusion Assemble 3D kernel
-void PADiffusionSetup3D(const int Q1D,
-                        const int coeffDim,
-                        const int NE,
-                        const Array<double> &w,
-                        const Vector &j,
-                        const Vector &c,
-                        Vector &d)
+static void PADiffusionSetup3D(const int Q1D,
+                               const int NE,
+                               const Array<double> &w,
+                               const Vector &j,
+                               const Vector &c,
+                               Vector &d)
 {
-   const bool symmetric = (coeffDim != 9);
    const bool const_c = c.Size() == 1;
-   MFEM_VERIFY(coeffDim < 6 ||
-               !const_c, "Constant matrix coefficient not supported");
    const auto W = Reshape(w.Read(), Q1D,Q1D,Q1D);
    const auto J = Reshape(j.Read(), Q1D,Q1D,Q1D,3,3,NE);
-   const auto C = const_c ? Reshape(c.Read(), 1,1,1,1,1) :
-                  Reshape(c.Read(), coeffDim,Q1D,Q1D,Q1D,NE);
-   auto D = Reshape(d.Write(), Q1D,Q1D,Q1D, symmetric ? 6 : 9, NE);
+   const auto C = const_c ? Reshape(c.Read(), 1,1,1,1) :
+                  Reshape(c.Read(), Q1D,Q1D,Q1D,NE);
+   auto D = Reshape(d.Write(), Q1D,Q1D,Q1D, 6, NE);
    MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
    {
       MFEM_FOREACH_THREAD(qx,x,Q1D)
@@ -230,7 +200,8 @@ void PADiffusionSetup3D(const int Q1D,
                const double detJ = J11 * (J22 * J33 - J32 * J23) -
                /* */               J21 * (J12 * J33 - J32 * J13) +
                /* */               J31 * (J12 * J23 - J22 * J13);
-               const double w_detJ = W(qx,qy,qz) / detJ;
+               const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
+               const double c_detJ = W(qx,qy,qz) * coeff / detJ;
                // adj(J)
                const double A11 = (J22 * J33) - (J23 * J32);
                const double A12 = (J32 * J13) - (J12 * J33);
@@ -241,69 +212,13 @@ void PADiffusionSetup3D(const int Q1D,
                const double A31 = (J21 * J32) - (J31 * J22);
                const double A32 = (J31 * J12) - (J11 * J32);
                const double A33 = (J11 * J22) - (J12 * J21);
-
-               if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
-               {
-                  // Compute entries of R = MJ^{-T} = M adj(J)^T, without det J.
-                  const double M11 = C(0, qx,qy,qz, e);
-                  const double M12 = C(1, qx,qy,qz, e);
-                  const double M13 = C(2, qx,qy,qz, e);
-                  const double M21 = (!symmetric) ? C(3, qx,qy,qz, e) : M12;
-                  const double M22 = (!symmetric) ? C(4, qx,qy,qz, e) : C(3, qx,qy,qz, e);
-                  const double M23 = (!symmetric) ? C(5, qx,qy,qz, e) : C(4, qx,qy,qz, e);
-                  const double M31 = (!symmetric) ? C(6, qx,qy,qz, e) : M13;
-                  const double M32 = (!symmetric) ? C(7, qx,qy,qz, e) : M23;
-                  const double M33 = (!symmetric) ? C(8, qx,qy,qz, e) : C(5, qx,qy,qz, e);
-
-                  const double R11 = M11*A11 + M12*A12 + M13*A13;
-                  const double R12 = M11*A21 + M12*A22 + M13*A23;
-                  const double R13 = M11*A31 + M12*A32 + M13*A33;
-                  const double R21 = M21*A11 + M22*A12 + M23*A13;
-                  const double R22 = M21*A21 + M22*A22 + M23*A23;
-                  const double R23 = M21*A31 + M22*A32 + M23*A33;
-                  const double R31 = M31*A11 + M32*A12 + M33*A13;
-                  const double R32 = M31*A21 + M32*A22 + M33*A23;
-                  const double R33 = M31*A31 + M32*A32 + M33*A33;
-
-                  // Now set D to J^{-1} R = adj(J) R
-                  D(qx,qy,qz,0,e) = w_detJ * (A11*R11 + A12*R21 + A13*R31); // 1,1
-                  const double D12 = w_detJ * (A11*R12 + A12*R22 + A13*R32);
-                  D(qx,qy,qz,1,e) = D12; // 1,2
-                  D(qx,qy,qz,2,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
-
-                  const double D21 = w_detJ * (A21*R11 + A22*R21 + A23*R31);
-                  const double D22 = w_detJ * (A21*R12 + A22*R22 + A23*R32);
-                  const double D23 = w_detJ * (A21*R13 + A22*R23 + A23*R33);
-
-                  const double D33 = w_detJ * (A31*R13 + A32*R23 + A33*R33);
-
-                  D(qx,qy,qz,3,e) = symmetric ? D22 : D21; // 2,2 or 2,1
-                  D(qx,qy,qz,4,e) = symmetric ? D23 : D22; // 2,3 or 2,2
-                  D(qx,qy,qz,5,e) = symmetric ? D33 : D23; // 3,3 or 2,3
-
-                  if (!symmetric)
-                  {
-                     D(qx,qy,qz,6,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
-                     D(qx,qy,qz,7,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
-                     D(qx,qy,qz,8,e) = D33; // 3,3
-                  }
-               }
-               else  // Vector or scalar coefficient version
-               {
-                  const double C1 = const_c ? C(0,0,0,0,0) : C(0,qx,qy,qz,e);
-                  const double C2 = const_c ? C(0,0,0,0,0) :
-                                    (coeffDim == 3 ? C(1,qx,qy,qz,e) : C(0,qx,qy,qz,e));
-                  const double C3 = const_c ? C(0,0,0,0,0) :
-                                    (coeffDim == 3 ? C(2,qx,qy,qz,e) : C(0,qx,qy,qz,e));
-
-                  // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
-                  D(qx,qy,qz,0,e) = w_detJ * (C1*A11*A11 + C2*A12*A12 + C3*A13*A13); // 1,1
-                  D(qx,qy,qz,1,e) = w_detJ * (C1*A11*A21 + C2*A12*A22 + C3*A13*A23); // 2,1
-                  D(qx,qy,qz,2,e) = w_detJ * (C1*A11*A31 + C2*A12*A32 + C3*A13*A33); // 3,1
-                  D(qx,qy,qz,3,e) = w_detJ * (C1*A21*A21 + C2*A22*A22 + C3*A23*A23); // 2,2
-                  D(qx,qy,qz,4,e) = w_detJ * (C1*A21*A31 + C2*A22*A32 + C3*A23*A33); // 3,2
-                  D(qx,qy,qz,5,e) = w_detJ * (C1*A31*A31 + C2*A32*A32 + C3*A33*A33); // 3,3
-               }
+               // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
+               D(qx,qy,qz,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
+               D(qx,qy,qz,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
+               D(qx,qy,qz,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
+               D(qx,qy,qz,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
+               D(qx,qy,qz,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
+               D(qx,qy,qz,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
             }
          }
       }
@@ -314,7 +229,6 @@ static void PADiffusionSetup(const int dim,
                              const int sdim,
                              const int D1D,
                              const int Q1D,
-                             const int coeffDim,
                              const int NE,
                              const Array<double> &W,
                              const Vector &J,
@@ -333,8 +247,8 @@ static void PADiffusionSetup(const int dim,
 #else
       MFEM_CONTRACT_VAR(D1D);
 #endif // MFEM_USE_OCCA
-      if (sdim == 2) { PADiffusionSetup2D<2>(Q1D, coeffDim, NE, W, J, C, D); }
-      if (sdim == 3) { PADiffusionSetup2D<3>(Q1D, coeffDim, NE, W, J, C, D); }
+      if (sdim == 2) { PADiffusionSetup2D<2>(Q1D, NE, W, J, C, D); }
+      if (sdim == 3) { PADiffusionSetup2D<3>(Q1D, NE, W, J, C, D); }
    }
    if (dim == 3)
    {
@@ -345,11 +259,11 @@ static void PADiffusionSetup(const int dim,
          return;
       }
 #endif // MFEM_USE_OCCA
-      PADiffusionSetup3D(Q1D, coeffDim, NE, W, J, C, D);
+      PADiffusionSetup3D(Q1D, NE, W, J, C, D);
    }
 }
 
-void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
+void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes)
 {
    // Assuming the same element type
    fespace = &fes;
@@ -357,13 +271,16 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    if (mesh->GetNE() == 0) { return; }
    const FiniteElement &el = *fes.GetFE(0);
    const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
+#ifdef MFEM_USE_CEED
    if (DeviceCanUseCeed())
    {
-      delete ceedDataPtr;
-      ceedDataPtr = new CeedData;
-      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
-      return CeedPADiffusionAssemble(fes, *ir, *ceedDataPtr);
+      if (ceedDataPtr) { delete ceedDataPtr; }
+      CeedData* ptr = new CeedData();
+      ceedDataPtr = ptr;
+      InitCeedCoeff(Q, ptr);
+      return CeedPADiffusionAssemble(fes, *ir, *ptr);
    }
+#endif
    const int dims = el.GetDim();
    const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
    const int nq = ir->GetNPoints();
@@ -374,80 +291,9 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
-   int coeffDim = 1;
+   pa_data.SetSize(symmDims * nq * ne, Device::GetDeviceMemoryType());
    Vector coeff;
-   const int MQfullDim = MQ ? MQ->GetHeight() * MQ->GetWidth() : 0;
-   if (MQ)
-   {
-      MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
-      const int MQsymmDim = MQ->GetWidth() * (MQ->GetWidth() + 1) / 2;
-
-      const int MQdim = MQ->IsSymmetric() ? MQsymmDim : MQfullDim;
-      coeffDim = MQdim;
-
-      coeff.SetSize(MQdim * nq * ne);
-      symmetric = MQ ? MQ->IsSymmetric() : true;
-
-      DenseMatrix M;
-      Vector Msymm;
-      if (symmetric)
-      {
-         Msymm.SetSize(MQsymmDim);
-      }
-      else
-      {
-         M.SetSize(dim);
-      }
-
-      auto C = Reshape(coeff.HostWrite(), MQdim, nq, ne);
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            if (MQ->IsSymmetric())
-            {
-               MQ->EvalSymmetric(Msymm, *tr, ir->IntPoint(p));
-
-               for (int i=0; i<MQsymmDim; ++i)
-               {
-                  C(i, p, e) = Msymm[i];
-               }
-            }
-            else
-            {
-               MQ->Eval(M, *tr, ir->IntPoint(p));
-
-               for (int i=0; i<dim; ++i)
-                  for (int j=0; j<dim; ++j)
-                  {
-                     C(j+(i*dim), p, e) = M(i,j);
-                  }
-            }
-         }
-      }
-   }
-   else if (VQ)
-   {
-      MFEM_VERIFY(VQ->GetVDim() == dim, "");
-      coeffDim = VQ->GetVDim();
-      coeff.SetSize(coeffDim * nq * ne);
-      auto C = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-      Vector D(coeffDim);
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            VQ->Eval(D, *tr, ir->IntPoint(p));
-            for (int i=0; i<coeffDim; ++i)
-            {
-               C(i, p, e) = D[i];
-            }
-         }
-      }
-   }
-   else if (Q == nullptr)
+   if (Q == nullptr)
    {
       coeff.SetSize(1);
       coeff(0) = 1.0;
@@ -483,15 +329,18 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
          }
       }
    }
-   pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne,
-                   Device::GetDeviceMemoryType());
-   PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeffDim, ne, ir->GetWeights(),
-                    geom->J, coeff, pa_data);
+   PADiffusionSetup(dim, sdim, dofs1D, quad1D, ne, ir->GetWeights(), geom->J,
+                    coeff, pa_data);
 }
 
+void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
+{
+   SetupPA(fes);
+}
+
+
 template<int T_D1D = 0, int T_Q1D = 0>
 static void PADiffusionDiagonal2D(const int NE,
-                                  const bool symmetric,
                                   const Array<double> &b,
                                   const Array<double> &g,
                                   const Vector &d,
@@ -505,9 +354,9 @@ static void PADiffusionDiagonal2D(const int NE,
    MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
-   // note the different shape for D, if this is a symmetric matrix we only
+   // note the different shape for D, this is a (symmetric) matrix so we only
    // store necessary entries
-   auto D = Reshape(d.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
+   auto D = Reshape(d.Read(), Q1D*Q1D, 3, NE);
    auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
    MFEM_FORALL(e, NE,
    {
@@ -529,13 +378,12 @@ static void PADiffusionDiagonal2D(const int NE,
             for (int qy = 0; qy < Q1D; ++qy)
             {
                const int q = qx + qy * Q1D;
-               const double D00 = D(q,0,e);
-               const double D10 = D(q,1,e);
-               const double D01 = symmetric ? D10 : D(q,2,e);
-               const double D11 = symmetric ? D(q,2,e) : D(q,3,e);
-               QD0[qx][dy] += B(qy, dy) * B(qy, dy) * D00;
-               QD1[qx][dy] += B(qy, dy) * G(qy, dy) * (D01 + D10);
-               QD2[qx][dy] += G(qy, dy) * G(qy, dy) * D11;
+               const double D0 = D(q,0,e);
+               const double D1 = D(q,1,e);
+               const double D2 = D(q,2,e);
+               QD0[qx][dy] += B(qy, dy) * B(qy, dy) * D0;
+               QD1[qx][dy] += B(qy, dy) * G(qy, dy) * D1;
+               QD2[qx][dy] += G(qy, dy) * G(qy, dy) * D2;
             }
          }
       }
@@ -547,6 +395,7 @@ static void PADiffusionDiagonal2D(const int NE,
             {
                Y(dx,dy,e) += G(qx, dx) * G(qx, dx) * QD0[qx][dy];
                Y(dx,dy,e) += G(qx, dx) * B(qx, dx) * QD1[qx][dy];
+               Y(dx,dy,e) += B(qx, dx) * G(qx, dx) * QD1[qx][dy];
                Y(dx,dy,e) += B(qx, dx) * B(qx, dx) * QD2[qx][dy];
             }
          }
@@ -557,7 +406,6 @@ static void PADiffusionDiagonal2D(const int NE,
 // Shared memory PA Diffusion Diagonal 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
 static void SmemPADiffusionDiagonal2D(const int NE,
-                                      const bool symmetric,
                                       const Array<double> &b_,
                                       const Array<double> &g_,
                                       const Vector &d_,
@@ -574,7 +422,7 @@ static void SmemPADiffusionDiagonal2D(const int NE,
    MFEM_VERIFY(Q1D <= MQ1, "");
    auto b = Reshape(b_.Read(), Q1D, D1D);
    auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
+   auto D = Reshape(d_.Read(), Q1D*Q1D, 3, NE);
    auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
    MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
    {
@@ -587,10 +435,10 @@ static void SmemPADiffusionDiagonal2D(const int NE,
       MFEM_SHARED double BG[2][MQ1*MD1];
       double (*B)[MD1] = (double (*)[MD1]) (BG+0);
       double (*G)[MD1] = (double (*)[MD1]) (BG+1);
-      MFEM_SHARED double QD[3][NBZ][MD1][MQ1];
+      MFEM_SHARED double QD[4][NBZ][MD1][MQ1];
       double (*QD0)[MD1] = (double (*)[MD1])(QD[0] + tidz);
       double (*QD1)[MD1] = (double (*)[MD1])(QD[1] + tidz);
-      double (*QD2)[MD1] = (double (*)[MD1])(QD[2] + tidz);
+      double (*QD2)[MD1] = (double (*)[MD1])(QD[3] + tidz);
       if (tidz == 0)
       {
          MFEM_FOREACH_THREAD(d,y,D1D)
@@ -613,18 +461,17 @@ static void SmemPADiffusionDiagonal2D(const int NE,
             for (int qy = 0; qy < Q1D; ++qy)
             {
                const int q = qx + qy * Q1D;
-               const double D00 = D(q,0,e);
-               const double D10 = D(q,1,e);
-               const double D01 = symmetric ? D10 : D(q,2,e);
-               const double D11 = symmetric ? D(q,2,e) : D(q,3,e);
+               const double D0 = D(q,0,e);
+               const double D1 = D(q,1,e);
+               const double D2 = D(q,2,e);
                const double By = B[qy][dy];
                const double Gy = G[qy][dy];
                const double BB = By * By;
                const double BG = By * Gy;
                const double GG = Gy * Gy;
-               QD0[qx][dy] += BB * D00;
-               QD1[qx][dy] += BG * (D01 + D10);
-               QD2[qx][dy] += GG * D11;
+               QD0[qx][dy] += BB * D0;
+               QD1[qx][dy] += BG * D1;
+               QD2[qx][dy] += GG * D2;
             }
          }
       }
@@ -642,6 +489,7 @@ static void SmemPADiffusionDiagonal2D(const int NE,
                const double GG = Gx * Gx;
                Y(dx,dy,e) += GG * QD0[qx][dy];
                Y(dx,dy,e) += BG * QD1[qx][dy];
+               Y(dx,dy,e) += BG * QD1[qx][dy];
                Y(dx,dy,e) += BB * QD2[qx][dy];
             }
          }
@@ -651,7 +499,6 @@ static void SmemPADiffusionDiagonal2D(const int NE,
 
 template<int T_D1D = 0, int T_Q1D = 0>
 static void PADiffusionDiagonal3D(const int NE,
-                                  const bool symmetric,
                                   const Array<double> &b,
                                   const Array<double> &g,
                                   const Vector &d,
@@ -668,7 +515,7 @@ static void PADiffusionDiagonal3D(const int NE,
    MFEM_VERIFY(Q1D <= MQ1, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Q = Reshape(d.Read(), Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
+   auto Q = Reshape(d.Read(), Q1D*Q1D*Q1D, 6, NE);
    auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
    MFEM_FORALL(e, NE,
    {
@@ -693,10 +540,9 @@ static void PADiffusionDiagonal3D(const int NE,
                      for (int qz = 0; qz < Q1D; ++qz)
                      {
                         const int q = qx + (qy + qz * Q1D) * Q1D;
-                        const int ksym = j >= i ?
+                        const int k = j >= i ?
                         3 - (3-i)*(2-i)/2 + j:
                         3 - (3-j)*(2-j)/2 + i;
-                        const int k = symmetric ? ksym : (i*DIM) + j;
                         const double O = Q(q,k,e);
                         const double Bz = B(qz,dz);
                         const double Gz = G(qz,dz);
@@ -752,7 +598,6 @@ static void PADiffusionDiagonal3D(const int NE,
 // Shared memory PA Diffusion Diagonal 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
 static void SmemPADiffusionDiagonal3D(const int NE,
-                                      const bool symmetric,
                                       const Array<double> &b_,
                                       const Array<double> &g_,
                                       const Vector &d_,
@@ -769,7 +614,7 @@ static void SmemPADiffusionDiagonal3D(const int NE,
    MFEM_VERIFY(Q1D <= MQ1, "");
    auto b = Reshape(b_.Read(), Q1D, D1D);
    auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
+   auto D = Reshape(d_.Read(), Q1D*Q1D*Q1D, 6, NE);
    auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
    MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
    {
@@ -810,10 +655,9 @@ static void SmemPADiffusionDiagonal3D(const int NE,
                      for (int qz = 0; qz < Q1D; ++qz)
                      {
                         const int q = qx + (qy + qz * Q1D) * Q1D;
-                        const int ksym = j >= i ?
-                                         3 - (3-i)*(2-i)/2 + j:
-                                         3 - (3-j)*(2-j)/2 + i;
-                        const int k = symmetric ? ksym : (i*DIM) + j;
+                        const int k = j >= i ?
+                                      3 - (3-i)*(2-i)/2 + j:
+                                      3 - (3-j)*(2-j)/2 + i;
                         const double O = D(q,k,e);
                         const double Bz = B[qz][dz];
                         const double Gz = G[qz][dz];
@@ -872,7 +716,6 @@ static void PADiffusionAssembleDiagonal(const int dim,
                                         const int D1D,
                                         const int Q1D,
                                         const int NE,
-                                        const bool symm,
                                         const Array<double> &B,
                                         const Array<double> &G,
                                         const Vector &D,
@@ -882,30 +725,30 @@ static void PADiffusionAssembleDiagonal(const int dim,
    {
       switch ((D1D << 4 ) | Q1D)
       {
-         case 0x22: return SmemPADiffusionDiagonal2D<2,2,8>(NE,symm,B,G,D,Y);
-         case 0x33: return SmemPADiffusionDiagonal2D<3,3,8>(NE,symm,B,G,D,Y);
-         case 0x44: return SmemPADiffusionDiagonal2D<4,4,4>(NE,symm,B,G,D,Y);
-         case 0x55: return SmemPADiffusionDiagonal2D<5,5,4>(NE,symm,B,G,D,Y);
-         case 0x66: return SmemPADiffusionDiagonal2D<6,6,2>(NE,symm,B,G,D,Y);
-         case 0x77: return SmemPADiffusionDiagonal2D<7,7,2>(NE,symm,B,G,D,Y);
-         case 0x88: return SmemPADiffusionDiagonal2D<8,8,1>(NE,symm,B,G,D,Y);
-         case 0x99: return SmemPADiffusionDiagonal2D<9,9,1>(NE,symm,B,G,D,Y);
-         default: return PADiffusionDiagonal2D(NE,symm,B,G,D,Y,D1D,Q1D);
+         case 0x22: return SmemPADiffusionDiagonal2D<2,2,8>(NE,B,G,D,Y);
+         case 0x33: return SmemPADiffusionDiagonal2D<3,3,8>(NE,B,G,D,Y);
+         case 0x44: return SmemPADiffusionDiagonal2D<4,4,4>(NE,B,G,D,Y);
+         case 0x55: return SmemPADiffusionDiagonal2D<5,5,4>(NE,B,G,D,Y);
+         case 0x66: return SmemPADiffusionDiagonal2D<6,6,2>(NE,B,G,D,Y);
+         case 0x77: return SmemPADiffusionDiagonal2D<7,7,2>(NE,B,G,D,Y);
+         case 0x88: return SmemPADiffusionDiagonal2D<8,8,1>(NE,B,G,D,Y);
+         case 0x99: return SmemPADiffusionDiagonal2D<9,9,1>(NE,B,G,D,Y);
+         default: return PADiffusionDiagonal2D(NE,B,G,D,Y,D1D,Q1D);
       }
    }
    else if (dim == 3)
    {
       switch ((D1D << 4 ) | Q1D)
       {
-         case 0x23: return SmemPADiffusionDiagonal3D<2,3>(NE,symm,B,G,D,Y);
-         case 0x34: return SmemPADiffusionDiagonal3D<3,4>(NE,symm,B,G,D,Y);
-         case 0x45: return SmemPADiffusionDiagonal3D<4,5>(NE,symm,B,G,D,Y);
-         case 0x56: return SmemPADiffusionDiagonal3D<5,6>(NE,symm,B,G,D,Y);
-         case 0x67: return SmemPADiffusionDiagonal3D<6,7>(NE,symm,B,G,D,Y);
-         case 0x78: return SmemPADiffusionDiagonal3D<7,8>(NE,symm,B,G,D,Y);
-         case 0x89: return SmemPADiffusionDiagonal3D<8,9>(NE,symm,B,G,D,Y);
-         case 0x9A: return SmemPADiffusionDiagonal3D<9,10>(NE,symm,B,G,D,Y);
-         default: return PADiffusionDiagonal3D(NE,symm,B,G,D,Y,D1D,Q1D);
+         case 0x23: return SmemPADiffusionDiagonal3D<2,3>(NE,B,G,D,Y);
+         case 0x34: return SmemPADiffusionDiagonal3D<3,4>(NE,B,G,D,Y);
+         case 0x45: return SmemPADiffusionDiagonal3D<4,5>(NE,B,G,D,Y);
+         case 0x56: return SmemPADiffusionDiagonal3D<5,6>(NE,B,G,D,Y);
+         case 0x67: return SmemPADiffusionDiagonal3D<6,7>(NE,B,G,D,Y);
+         case 0x78: return SmemPADiffusionDiagonal3D<7,8>(NE,B,G,D,Y);
+         case 0x89: return SmemPADiffusionDiagonal3D<8,9>(NE,B,G,D,Y);
+         case 0x9A: return SmemPADiffusionDiagonal3D<9,10>(NE,B,G,D,Y);
+         default: return PADiffusionDiagonal3D(NE,B,G,D,Y,D1D,Q1D);
       }
    }
    MFEM_ABORT("Unknown kernel.");
@@ -913,14 +756,15 @@ static void PADiffusionAssembleDiagonal(const int dim,
 
 void DiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
 {
+#ifdef MFEM_USE_CEED
    if (DeviceCanUseCeed())
    {
-      CeedAssembleDiagonal(ceedDataPtr, diag);
+      CeedAssembleDiagonalPA(ceedDataPtr, diag);
    }
    else
+#endif
    {
-      if (pa_data.Size()==0) { AssemblePA(*fespace); }
-      PADiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne, symmetric,
+      PADiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne,
                                   maps->B, maps->G, pa_data, diag);
    }
 }
@@ -1029,7 +873,6 @@ static void OccaPADiffusionApply3D(const int D1D,
 // PA Diffusion Apply 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
 static void PADiffusionApply2D(const int NE,
-                               const bool symmetric,
                                const Array<double> &b_,
                                const Array<double> &g_,
                                const Array<double> &bt_,
@@ -1048,7 +891,7 @@ static void PADiffusionApply2D(const int NE,
    auto G = Reshape(g_.Read(), Q1D, D1D);
    auto Bt = Reshape(bt_.Read(), D1D, Q1D);
    auto Gt = Reshape(gt_.Read(), D1D, Q1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
+   auto D = Reshape(d_.Read(), Q1D*Q1D, 3, NE);
    auto X = Reshape(x_.Read(), D1D, D1D, NE);
    auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
    MFEM_FORALL(e, NE,
@@ -1104,15 +947,14 @@ static void PADiffusionApply2D(const int NE,
             const int q = qx + qy * Q1D;
 
             const double O11 = D(q,0,e);
-            const double O21 = D(q,1,e);
-            const double O12 = symmetric ? O21 : D(q,2,e);
-            const double O22 = symmetric ? D(q,2,e) : D(q,3,e);
+            const double O12 = D(q,1,e);
+            const double O22 = D(q,2,e);
 
             const double gradX = grad[qy][qx][0];
             const double gradY = grad[qy][qx][1];
 
             grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
-            grad[qy][qx][1] = (O21 * gradX) + (O22 * gradY);
+            grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
          }
       }
       for (int qy = 0; qy < Q1D; ++qy)
@@ -1151,7 +993,6 @@ static void PADiffusionApply2D(const int NE,
 // Shared memory PA Diffusion Apply 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
 static void SmemPADiffusionApply2D(const int NE,
-                                   const bool symmetric,
                                    const Array<double> &b_,
                                    const Array<double> &g_,
                                    const Vector &d_,
@@ -1169,7 +1010,7 @@ static void SmemPADiffusionApply2D(const int NE,
    MFEM_VERIFY(Q1D <= MQ1, "");
    auto b = Reshape(b_.Read(), Q1D, D1D);
    auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
+   auto D = Reshape(d_.Read(), Q1D*Q1D, 3, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, NE);
    auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
    MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
@@ -1251,13 +1092,12 @@ static void SmemPADiffusionApply2D(const int NE,
          {
             const int q = (qx + ((qy) * Q1D));
             const double O11 = D(q,0,e);
-            const double O21 = D(q,1,e);
-            const double O12 = symmetric ? O21 : D(q,2,e);
-            const double O22 = symmetric ? D(q,2,e) : D(q,3,e);
+            const double O12 = D(q,1,e);
+            const double O22 = D(q,2,e);
             const double gX = QQ0[qy][qx];
             const double gY = QQ1[qy][qx];
             QQ0[qy][qx] = (O11 * gX) + (O12 * gY);
-            QQ1[qy][qx] = (O21 * gX) + (O22 * gY);
+            QQ1[qy][qx] = (O12 * gX) + (O22 * gY);
          }
       }
       MFEM_SYNC_THREAD;
@@ -1309,7 +1149,6 @@ static void SmemPADiffusionApply2D(const int NE,
 // PA Diffusion Apply 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
 static void PADiffusionApply3D(const int NE,
-                               const bool symmetric,
                                const Array<double> &b,
                                const Array<double> &g,
                                const Array<double> &bt,
@@ -1327,7 +1166,7 @@ static void PADiffusionApply3D(const int NE,
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
    auto Gt = Reshape(gt.Read(), D1D, Q1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
+   auto D = Reshape(d_.Read(), Q1D*Q1D*Q1D, 6, NE);
    auto X = Reshape(x_.Read(), D1D, D1D, D1D, NE);
    auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
    MFEM_FORALL(e, NE,
@@ -1418,18 +1257,15 @@ static void PADiffusionApply3D(const int NE,
                const double O11 = D(q,0,e);
                const double O12 = D(q,1,e);
                const double O13 = D(q,2,e);
-               const double O21 = symmetric ? O12 : D(q,3,e);
-               const double O22 = symmetric ? D(q,3,e) : D(q,4,e);
-               const double O23 = symmetric ? D(q,4,e) : D(q,5,e);
-               const double O31 = symmetric ? O13 : D(q,6,e);
-               const double O32 = symmetric ? O23 : D(q,7,e);
-               const double O33 = symmetric ? D(q,5,e) : D(q,8,e);
+               const double O22 = D(q,3,e);
+               const double O23 = D(q,4,e);
+               const double O33 = D(q,5,e);
                const double gradX = grad[qz][qy][qx][0];
                const double gradY = grad[qz][qy][qx][1];
                const double gradZ = grad[qz][qy][qx][2];
                grad[qz][qy][qx][0] = (O11*gradX)+(O12*gradY)+(O13*gradZ);
-               grad[qz][qy][qx][1] = (O21*gradX)+(O22*gradY)+(O23*gradZ);
-               grad[qz][qy][qx][2] = (O31*gradX)+(O32*gradY)+(O33*gradZ);
+               grad[qz][qy][qx][1] = (O12*gradX)+(O22*gradY)+(O23*gradZ);
+               grad[qz][qy][qx][2] = (O13*gradX)+(O23*gradY)+(O33*gradZ);
             }
          }
       }
@@ -1528,7 +1364,6 @@ static MFEM_HOST_DEVICE inline double sign(const int q, const int d)
 
 template<int T_D1D = 0, int T_Q1D = 0>
 static void SmemPADiffusionApply3D(const int NE,
-                                   const bool symmetric,
                                    const Array<double> &b_,
                                    const Array<double> &g_,
                                    const Vector &d_,
@@ -1545,7 +1380,7 @@ static void SmemPADiffusionApply3D(const int NE,
    MFEM_VERIFY(Q1D <= M1Q, "");
    auto b = Reshape(b_.Read(), Q1D, D1D);
    auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
+   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, 6, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
    MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
@@ -1692,18 +1527,15 @@ static void SmemPADiffusionApply3D(const int NE,
                const double O11 = d(qx,qy,qz,0,e);
                const double O12 = d(qx,qy,qz,1,e);
                const double O13 = d(qx,qy,qz,2,e);
-               const double O21 = symmetric ? O12 : d(qx,qy,qz,3,e);
-               const double O22 = symmetric ? d(qx,qy,qz,3,e) : d(qx,qy,qz,4,e);
-               const double O23 = symmetric ? d(qx,qy,qz,4,e) : d(qx,qy,qz,5,e);
-               const double O31 = symmetric ? O13 : d(qx,qy,qz,6,e);
-               const double O32 = symmetric ? O23 : d(qx,qy,qz,7,e);
-               const double O33 = symmetric ? d(qx,qy,qz,5,e) : d(qx,qy,qz,8,e);
+               const double O22 = d(qx,qy,qz,3,e);
+               const double O23 = d(qx,qy,qz,4,e);
+               const double O33 = d(qx,qy,qz,5,e);
                const double gX = u[qz];
                const double gY = v[qz];
                const double gZ = w[qz];
                QQQ0[qz][qy][qx] = (O11*gX) + (O12*gY) + (O13*gZ);
-               QQQ1[qz][qy][qx] = (O21*gX) + (O22*gY) + (O23*gZ);
-               QQQ2[qz][qy][qx] = (O31*gX) + (O32*gY) + (O33*gZ);
+               QQQ1[qz][qy][qx] = (O12*gX) + (O22*gY) + (O23*gZ);
+               QQQ2[qz][qy][qx] = (O13*gX) + (O23*gY) + (O33*gZ);
             }
          }
       }
@@ -1824,7 +1656,6 @@ static void PADiffusionApply(const int dim,
                              const int D1D,
                              const int Q1D,
                              const int NE,
-                             const bool symm,
                              const Array<double> &B,
                              const Array<double> &G,
                              const Array<double> &Bt,
@@ -1855,15 +1686,15 @@ static void PADiffusionApply(const int dim,
    {
       switch (ID)
       {
-         case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,symm,B,G,D,X,Y);
-         case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,symm,B,G,D,X,Y);
-         case 0x44: return SmemPADiffusionApply2D<4,4,8>(NE,symm,B,G,D,X,Y);
-         case 0x55: return SmemPADiffusionApply2D<5,5,8>(NE,symm,B,G,D,X,Y);
-         case 0x66: return SmemPADiffusionApply2D<6,6,4>(NE,symm,B,G,D,X,Y);
-         case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,symm,B,G,D,X,Y);
-         case 0x88: return SmemPADiffusionApply2D<8,8,2>(NE,symm,B,G,D,X,Y);
-         case 0x99: return SmemPADiffusionApply2D<9,9,2>(NE,symm,B,G,D,X,Y);
-         default:   return PADiffusionApply2D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
+         case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,B,G,D,X,Y);
+         case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,B,G,D,X,Y);
+         case 0x44: return SmemPADiffusionApply2D<4,4,8>(NE,B,G,D,X,Y);
+         case 0x55: return SmemPADiffusionApply2D<5,5,8>(NE,B,G,D,X,Y);
+         case 0x66: return SmemPADiffusionApply2D<6,6,4>(NE,B,G,D,X,Y);
+         case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,B,G,D,X,Y);
+         case 0x88: return SmemPADiffusionApply2D<8,8,2>(NE,B,G,D,X,Y);
+         case 0x99: return SmemPADiffusionApply2D<9,9,2>(NE,B,G,D,X,Y);
+         default:   return PADiffusionApply2D(NE,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
       }
    }
 
@@ -1871,16 +1702,16 @@ static void PADiffusionApply(const int dim,
    {
       switch (ID)
       {
-         case 0x23: return SmemPADiffusionApply3D<2,3>(NE,symm,B,G,D,X,Y);
-         case 0x34: return SmemPADiffusionApply3D<3,4>(NE,symm,B,G,D,X,Y);
-         case 0x45: return SmemPADiffusionApply3D<4,5>(NE,symm,B,G,D,X,Y);
-         case 0x46: return SmemPADiffusionApply3D<4,6>(NE,symm,B,G,D,X,Y);
-         case 0x56: return SmemPADiffusionApply3D<5,6>(NE,symm,B,G,D,X,Y);
-         case 0x58: return SmemPADiffusionApply3D<5,8>(NE,symm,B,G,D,X,Y);
-         case 0x67: return SmemPADiffusionApply3D<6,7>(NE,symm,B,G,D,X,Y);
-         case 0x78: return SmemPADiffusionApply3D<7,8>(NE,symm,B,G,D,X,Y);
-         case 0x89: return SmemPADiffusionApply3D<8,9>(NE,symm,B,G,D,X,Y);
-         default:   return PADiffusionApply3D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
+         case 0x23: return SmemPADiffusionApply3D<2,3>(NE,B,G,D,X,Y);
+         case 0x34: return SmemPADiffusionApply3D<3,4>(NE,B,G,D,X,Y);
+         case 0x45: return SmemPADiffusionApply3D<4,5>(NE,B,G,D,X,Y);
+         case 0x46: return SmemPADiffusionApply3D<4,6>(NE,B,G,D,X,Y);
+         case 0x56: return SmemPADiffusionApply3D<5,6>(NE,B,G,D,X,Y);
+         case 0x58: return SmemPADiffusionApply3D<5,8>(NE,B,G,D,X,Y);
+         case 0x67: return SmemPADiffusionApply3D<6,7>(NE,B,G,D,X,Y);
+         case 0x78: return SmemPADiffusionApply3D<7,8>(NE,B,G,D,X,Y);
+         case 0x89: return SmemPADiffusionApply3D<8,9>(NE,B,G,D,X,Y);
+         default:   return PADiffusionApply3D(NE,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
       }
    }
    MFEM_ABORT("Unknown kernel.");
@@ -1889,13 +1720,15 @@ static void PADiffusionApply(const int dim,
 // PA Diffusion Apply kernel
 void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
+#ifdef MFEM_USE_CEED
    if (DeviceCanUseCeed())
    {
-      CeedAddMult(ceedDataPtr, x, y);
+      CeedAddMultPA(ceedDataPtr, x, y);
    }
    else
+#endif
    {
-      PADiffusionApply(dim, dofs1D, quad1D, ne, symmetric,
+      PADiffusionApply(dim, dofs1D, quad1D, ne,
                        maps->B, maps->G, maps->Bt, maps->Gt,
                        pa_data, x, y);
    }

fem/bilininteg_gradient.cpp

@@ -334,7 +334,7 @@ static void PAGradientApplyTranspose2D(const int NE,
                                        const int q1d = 0)
 {
    // TODO
-   MFEM_ASSERT(false, "GradientPAApplyTranspose 3D not implemented.");
+   MFEM_ASSERT(false, "PAGradientApplyTranspose2D not implemented.");
 }
 
 // PA Gradient Apply 3D kernel

fem/bilininteg_mass_mf.cpp

@@ -1,71 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "libceed/mass.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-void MassIntegrator::AssembleMF(const FiniteElementSpace &fes)
-{
-#ifdef MFEM_USE_CEED
-   // Assuming the same element type
-   fespace = &fes;
-   Mesh *mesh = fes.GetMesh();
-   if (mesh->GetNE() == 0) { return; }
-   const FiniteElement &el = *fes.GetFE(0);
-   ElementTransformation *T = mesh->GetElementTransformation(0);
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T);
-   if (DeviceCanUseCeed())
-   {
-      delete ceedDataPtr;
-      ceedDataPtr = new CeedData;
-      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
-      return CeedMFMassAssemble(fes, *ir, *ceedDataPtr);
-   }
-#endif
-   mfem_error("Error: MassIntegrator::AssembleMF only implemented with libCEED");
-}
-
-void MassIntegrator::AddMultMF(const Vector &x, Vector &y) const
-{
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAddMult(ceedDataPtr, x, y);
-   }
-   else
-#endif
-   {
-      mfem_error("Error: MassIntegrator::AddMultMF only implemented with libCEED");
-   }
-}
-
-void MassIntegrator::AssembleDiagonalMF(Vector &diag)
-{
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAssembleDiagonal(ceedDataPtr, diag);
-   }
-   else
-#endif
-   {
-      mfem_error("Error: MassIntegrator::AssembleDiagonalMF only implemented with libCEED");
-   }
-}
-
-} // namespace mfem

fem/bilininteg_mass_pa.cpp

@@ -1,13 +1,13 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
 //
 // This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
+// availability see http://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
 
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
@@ -23,7 +23,7 @@ namespace mfem
 
 // PA Mass Assemble kernel
 
-void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
+void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
 {
    // Assuming the same element type
    fespace = &fes;
@@ -32,13 +32,16 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    const FiniteElement &el = *fes.GetFE(0);
    ElementTransformation *T = mesh->GetElementTransformation(0);
    const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T);
+#ifdef MFEM_USE_CEED
    if (DeviceCanUseCeed())
    {
-      delete ceedDataPtr;
-      ceedDataPtr = new CeedData;
-      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
-      return CeedPAMassAssemble(fes, *ir, *ceedDataPtr);
+      if (ceedDataPtr) { delete ceedDataPtr; }
+      CeedData* ptr = new CeedData();
+      ceedDataPtr = ptr;
+      InitCeedCoeff(Q, ptr);
+      return CeedPAMassAssemble(fes, *ir, *ptr);
    }
+#endif
    dim = mesh->Dimension();
    ne = fes.GetMesh()->GetNE();
    nq = ir->GetNPoints();
@@ -47,7 +50,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
-   pa_data.SetSize(ne*nq, Device::GetDeviceMemoryType());
+   pa_data.SetSize(ne*nq, Device::GetMemoryType());
    Vector coeff;
    if (Q == nullptr)
    {
@@ -152,6 +155,12 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    }
 }
 
+void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
+{
+   SetupPA(fes);
+}
+
+
 template<int T_D1D = 0, int T_Q1D = 0>
 static void PAMassAssembleDiagonal2D(const int NE,
                                      const Array<double> &b,
@@ -459,11 +468,13 @@ static void PAMassAssembleDiagonal(const int dim, const int D1D,
 
 void MassIntegrator::AssembleDiagonalPA(Vector &diag)
 {
+#ifdef MFEM_USE_CEED
    if (DeviceCanUseCeed())
    {
-      CeedAssembleDiagonal(ceedDataPtr, diag);
+      CeedAssembleDiagonalPA(ceedDataPtr, diag);
    }
    else
+#endif
    {
       PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
    }
@@ -939,8 +950,9 @@ static void SmemPAMassApply3D(const int NE,
    auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
    {
+      const int tidz = MFEM_THREAD_ID(z);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
@@ -957,188 +969,130 @@ static void SmemPAMassApply3D(const int NE,
       double (*QQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm1;
       double (*QQD)[MQ1][MD1] = (double (*)[MQ1][MD1]) sm0;
       double (*QDD)[MD1][MD1] = (double (*)[MD1][MD1]) sm1;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
          {
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
             {
                X[dz][dy][dx] = x(dx,dy,dz,e);
             }
          }
-         MFEM_FOREACH_THREAD(dx,x,Q1D)
+      }
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
          {
-            B[dx][dy] = b(dx,dy);
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+            }
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
          {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
             {
-               u[dz] = 0;
+               double u = 0.0;
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  u += X[dz][dy][dx] * B[qx][dx];
+               }
+               DDQ[dz][dy][qx] = u;
             }
-            MFEM_UNROLL(MD1)
-            for (int dx = 0; dx < D1D; ++dx)
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
             {
-               MFEM_UNROLL(MD1)
+               double u = 0.0;
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  u += DDQ[dz][dy][qx] * B[qy][dy];
+               }
+               DQQ[dz][qy][qx] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u = 0.0;
                for (int dz = 0; dz < D1D; ++dz)
                {
-                  u[dz] += X[dz][dy][dx] * B[qx][dx];
+                  u += DQQ[dz][qy][qx] * B[qz][dz];
                }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               DDQ[dz][dy][qx] = u[dz];
+               QQQ[qz][qy][qx] = u * d(qx,qy,qz,e);
             }
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      if (tidz == 0)
       {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(d,y,D1D)
          {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
+            MFEM_FOREACH_THREAD(q,x,Q1D)
             {
-               u[dz] = 0;
+               Bt[d][q] = b(q,d);
             }
-            MFEM_UNROLL(MD1)
-            for (int dy = 0; dy < D1D; ++dy)
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
             {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; dz++)
+               double u = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
                {
-                  u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
+                  u += QQQ[qz][qy][qx] * Bt[dx][qx];
                }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               DQQ[dz][qy][qx] = u[dz];
+               QQD[qz][qy][dx] = u;
             }
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
       {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
          {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
             {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; qz++)
+               double u = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
                {
-                  u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
+                  u += QQD[qz][qy][dx] * Bt[dy][qy];
                }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
-            {
-               QQQ[qz][qy][qx] = u[qz] * d(qx,qy,qz,e);
+               QDD[qz][dy][dx] = u;
             }
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(d,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
          {
-            Bt[d][q] = b(q,d);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
             {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               MFEM_UNROLL(MQ1)
+               double u = 0.0;
                for (int qz = 0; qz < Q1D; ++qz)
                {
-                  u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
+                  u += QDD[qz][dy][dx] * Bt[dz][qz];
                }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QQD[qz][qy][dx] = u[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QDD[qz][dy][dx] = u[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               y(dx,dy,dz,e) += u[dz];
+               y(dx,dy,dz,e) += u;
             }
          }
       }
@@ -1169,20 +1123,14 @@ static void PAMassApply(const int dim,
       MFEM_ABORT("OCCA PA Mass Apply unknown kernel!");
    }
 #endif // MFEM_USE_OCCA
-   const int id = (D1D << 4) | Q1D;
    if (dim == 2)
    {
-      switch (id)
+      switch ((D1D << 4) | Q1D)
       {
          case 0x22: return SmemPAMassApply2D<2,2,16>(NE,B,Bt,D,X,Y);
-         case 0x24: return SmemPAMassApply2D<2,4,16>(NE,B,Bt,D,X,Y);
          case 0x33: return SmemPAMassApply2D<3,3,16>(NE,B,Bt,D,X,Y);
-         case 0x34: return SmemPAMassApply2D<3,4,16>(NE,B,Bt,D,X,Y);
-         case 0x36: return SmemPAMassApply2D<3,6,16>(NE,B,Bt,D,X,Y);
          case 0x44: return SmemPAMassApply2D<4,4,8>(NE,B,Bt,D,X,Y);
-         case 0x48: return SmemPAMassApply2D<4,8,4>(NE,B,Bt,D,X,Y);
          case 0x55: return SmemPAMassApply2D<5,5,8>(NE,B,Bt,D,X,Y);
-         case 0x58: return SmemPAMassApply2D<5,8,2>(NE,B,Bt,D,X,Y);
          case 0x66: return SmemPAMassApply2D<6,6,4>(NE,B,Bt,D,X,Y);
          case 0x77: return SmemPAMassApply2D<7,7,4>(NE,B,Bt,D,X,Y);
          case 0x88: return SmemPAMassApply2D<8,8,2>(NE,B,Bt,D,X,Y);
@@ -1192,35 +1140,30 @@ static void PAMassApply(const int dim,
    }
    else if (dim == 3)
    {
-      switch (id)
+      switch ((D1D << 4) | Q1D)
       {
          case 0x23: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
-         case 0x24: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
          case 0x34: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
-         case 0x36: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
          case 0x45: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
-         case 0x46: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
-         case 0x48: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
          case 0x56: return SmemPAMassApply3D<5,6>(NE,B,Bt,D,X,Y);
-         case 0x58: return SmemPAMassApply3D<5,8>(NE,B,Bt,D,X,Y);
          case 0x67: return SmemPAMassApply3D<6,7>(NE,B,Bt,D,X,Y);
          case 0x78: return SmemPAMassApply3D<7,8>(NE,B,Bt,D,X,Y);
          case 0x89: return SmemPAMassApply3D<8,9>(NE,B,Bt,D,X,Y);
-         case 0x9A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
          default:   return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
       }
    }
-   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
    MFEM_ABORT("Unknown kernel.");
 }
 
 void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
+#ifdef MFEM_USE_CEED
    if (DeviceCanUseCeed())
    {
-      CeedAddMult(ceedDataPtr, x, y);
+      CeedAddMultPA(ceedDataPtr, x, y);
    }
    else
+#endif
    {
       PAMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
    }

fem/bilininteg_vecdiffusion.cpp

@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "libceed/diffusion.hpp"
 
 using namespace std;
 
@@ -130,13 +129,6 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    const FiniteElement &el = *fes.GetFE(0);
    const IntegrationRule *ir
       = IntRule ? IntRule : &DiffusionIntegrator::GetRule(el, el);
-   if (DeviceCanUseCeed())
-   {
-      delete ceedDataPtr;
-      ceedDataPtr = new CeedData;
-      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
-      return CeedPADiffusionAssemble(fes, *ir, * ceedDataPtr);
-   }
    const int dims = el.GetDim();
    const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
    const int nq = ir->GetNPoints();
@@ -514,40 +506,33 @@ void PAVectorDiffusionApply3D(const int NE,
 // PA Diffusion Apply kernel
 void VectorDiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   if (DeviceCanUseCeed())
-   {
-      CeedAddMult(ceedDataPtr, x, y);
-   }
-   else
-   {
-      const int D1D = dofs1D;
-      const int Q1D = quad1D;
-      const Array<double> &B = maps->B;
-      const Array<double> &G = maps->G;
-      const Array<double> &Bt = maps->Bt;
-      const Array<double> &Gt = maps->Gt;
-      const Vector &D = pa_data;
+   const int D1D = dofs1D;
+   const int Q1D = quad1D;
+   const Array<double> &B = maps->B;
+   const Array<double> &G = maps->G;
+   const Array<double> &Bt = maps->Bt;
+   const Array<double> &Gt = maps->Gt;
+   const Vector &D = pa_data;
 
-      if (dim == 2 && sdim == 3)
+   if (dim == 2 && sdim == 3)
+   {
+      switch ((dofs1D << 4 ) | quad1D)
       {
-         switch ((dofs1D << 4 ) | quad1D)
-         {
-            case 0x22: return PAVectorDiffusionApply2D<2,2,3>(ne,B,G,Bt,Gt,D,x,y);
-            case 0x33: return PAVectorDiffusionApply2D<3,3,3>(ne,B,G,Bt,Gt,D,x,y);
-            case 0x44: return PAVectorDiffusionApply2D<4,4,3>(ne,B,G,Bt,Gt,D,x,y);
-            case 0x55: return PAVectorDiffusionApply2D<5,5,3>(ne,B,G,Bt,Gt,D,x,y);
-            default:
-               return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim);
-         }
+         case 0x22: return PAVectorDiffusionApply2D<2,2,3>(ne,B,G,Bt,Gt,D,x,y);
+         case 0x33: return PAVectorDiffusionApply2D<3,3,3>(ne,B,G,Bt,Gt,D,x,y);
+         case 0x44: return PAVectorDiffusionApply2D<4,4,3>(ne,B,G,Bt,Gt,D,x,y);
+         case 0x55: return PAVectorDiffusionApply2D<5,5,3>(ne,B,G,Bt,Gt,D,x,y);
+         default:
+            return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim);
       }
-      if (dim == 2 && sdim == 2)
-      { return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim); }
-
-      if (dim == 3 && sdim == 3)
-      { return PAVectorDiffusionApply3D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D); }
-
-      MFEM_ABORT("Unknown kernel.");
    }
+   if (dim == 2 && sdim == 2)
+   { return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim); }
+
+   if (dim == 3 && sdim == 3)
+   { return PAVectorDiffusionApply3D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D); }
+
+   MFEM_ABORT("Unknown kernel.");
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
@@ -741,21 +726,14 @@ static void PAVectorDiffusionAssembleDiagonal(const int dim,
 
 void VectorDiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-   if (DeviceCanUseCeed())
-   {
-      CeedAssembleDiagonal(ceedDataPtr, diag);
-   }
-   else
-   {
-      PAVectorDiffusionAssembleDiagonal(dim,
-                                        dofs1D,
-                                        quad1D,
-                                        ne,
-                                        maps->B,
-                                        maps->G,
-                                        pa_data,
-                                        diag);
-   }
+   PAVectorDiffusionAssembleDiagonal(dim,
+                                     dofs1D,
+                                     quad1D,
+                                     ne,
+                                     maps->B,
+                                     maps->G,
+                                     pa_data,
+                                     diag);
 }
 
 } // namespace mfem

fem/bilininteg_vecdiffusion_mf.cpp

@@ -1,69 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "libceed/diffusion.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-void VectorDiffusionIntegrator::AssembleMF(const FiniteElementSpace &fes)
-{
-#ifdef MFEM_USE_CEED
-   // Assumes tensor-product elements
-   Mesh *mesh = fes.GetMesh();
-   const FiniteElement &el = *fes.GetFE(0);
-   const IntegrationRule *ir
-      = IntRule ? IntRule : &DiffusionIntegrator::GetRule(el, el);
-   if (DeviceCanUseCeed())
-   {
-      delete ceedDataPtr;
-      ceedDataPtr = new CeedData;
-      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
-      return CeedMFDiffusionAssemble(fes, *ir, * ceedDataPtr);
-   }
-#endif
-   mfem_error("Error: VectorDiffusionIntegrator::AssembleMF only implemented with libCEED");
-}
-
-void VectorDiffusionIntegrator::AddMultMF(const Vector &x, Vector &y) const
-{
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAddMult(ceedDataPtr, x, y);
-   }
-   else
-#endif
-   {
-      mfem_error("Error: VectorDiffusionIntegrator::AssembleDiagonalMF only implemented with libCEED");
-   }
-}
-
-void VectorDiffusionIntegrator::AssembleDiagonalMF(Vector &diag)
-{
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAssembleDiagonal(ceedDataPtr, diag);
-   }
-   else
-#endif
-   {
-      mfem_error("Error: VectorDiffusionIntegrator::AddMultMF only implemented with libCEED");
-   }
-}
-
-} // namespace mfem

fem/bilininteg_vecmass.cpp

@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "libceed/mass.hpp"
 
 using namespace std;
 
@@ -31,13 +30,6 @@ void VectorMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    ElementTransformation *T = mesh->GetElementTransformation(0);
    const IntegrationRule *ir
       = IntRule ? IntRule : &MassIntegrator::GetRule(el, el, *T);
-   if (DeviceCanUseCeed())
-   {
-      delete ceedDataPtr;
-      ceedDataPtr = new CeedData;
-      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
-      return CeedPAMassAssemble(fes, *ir, *ceedDataPtr);
-   }
    dim = mesh->Dimension();
    ne = fes.GetMesh()->GetNE();
    nq = ir->GetNPoints();
@@ -369,14 +361,7 @@ static void PAVectorMassApply(const int dim,
 
 void VectorMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   if (DeviceCanUseCeed())
-   {
-      CeedAddMult(ceedDataPtr, x, y);
-   }
-   else
-   {
-      PAVectorMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
-   }
+   PAVectorMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
 }
 
 template<const int T_D1D = 0, const int T_Q1D = 0>
@@ -529,21 +514,14 @@ static void PAVectorMassAssembleDiagonal(const int dim,
 
 void VectorMassIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-   if (DeviceCanUseCeed())
-   {
-      CeedAssembleDiagonal(ceedDataPtr, diag);
-   }
-   else
-   {
-      PAVectorMassAssembleDiagonal(dim,
-                                   dofs1D,
-                                   quad1D,
-                                   ne,
-                                   maps->B,
-                                   maps->Bt,
-                                   pa_data,
-                                   diag);
-   }
+   PAVectorMassAssembleDiagonal(dim,
+                                dofs1D,
+                                quad1D,
+                                ne,
+                                maps->B,
+                                maps->Bt,
+                                pa_data,
+                                diag);
 }
 
 } // namespace mfem

fem/bilininteg_vecmass_mf.cpp

@@ -1,74 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "libceed/mass.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// MF Mass Integrator
-
-// MF Mass Assemble kernel
-void VectorMassIntegrator::AssembleMF(const FiniteElementSpace &fes)
-{
-#ifdef MFEM_USE_CEED
-   // Assuming the same element type
-   Mesh *mesh = fes.GetMesh();
-   if (mesh->GetNE() == 0) { return; }
-   const FiniteElement &el = *fes.GetFE(0);
-   ElementTransformation *T = mesh->GetElementTransformation(0);
-   const IntegrationRule *ir
-      = IntRule ? IntRule : &MassIntegrator::GetRule(el, el, *T);
-   if (DeviceCanUseCeed())
-   {
-      delete ceedDataPtr;
-      ceedDataPtr = new CeedData;
-      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
-      return CeedMFMassAssemble(fes, *ir, *ceedDataPtr);
-   }
-#endif
-   mfem_error("Error: VectorMassIntegrator::AssembleMF only implemented with libCEED");
-}
-
-void VectorMassIntegrator::AddMultMF(const Vector &x, Vector &y) const
-{
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAddMult(ceedDataPtr, x, y);
-   }
-   else
-#endif
-   {
-      mfem_error("Error: VectorMassIntegrator::AssembleDiagonalMF only implemented with libCEED");
-   }
-}
-
-void VectorMassIntegrator::AssembleDiagonalMF(Vector &diag)
-{
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAssembleDiagonal(ceedDataPtr, diag);
-   }
-   else
-#endif
-   {
-      mfem_error("Error: VectorMassIntegrator::AddMultMF only implemented with libCEED");
-   }
-}
-
-} // namespace mfem

fem/bilininteg_vectorfe.cpp

@@ -15,14 +15,29 @@
 namespace mfem
 {
 
-void PADiffusionSetup3D(const int Q1D,
-                        const int coeffDim,
+void PAHcurlSetup2D(const int Q1D,
+                    const int coeffDim,
+                    const int NE,
+                    const Array<double> &w,
+                    const Vector &j,
+                    Vector &coeff,
+                    Vector &op);
+
+void PAHcurlSetup2Din3D(const int Q1D,
                         const int NE,
                         const Array<double> &w,
                         const Vector &j,
-                        const Vector &_coeff,
+                        Vector &_coeff,
                         Vector &op);
 
+void PAHcurlSetup3D(const int Q1D,
+                    const int coeffDim,
+                    const int NE,
+                    const Array<double> &w,
+                    const Vector &j,
+                    Vector &coeff,
+                    Vector &op);
+
 void PAHcurlMassAssembleDiagonal2D(const int D1D,
                                    const int Q1D,
                                    const int NE,
@@ -41,6 +56,14 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
                                    const Vector &pa_data,
                                    Vector &diag);
 
+void PAHcurlMassAssembleDiagonal3DHost(const int D1D,
+                                       const int Q1D,
+                                       const int NE,
+                                       const Array<double> &_Bo,
+                                       const Array<double> &_Bc,
+                                       const Vector &_op,
+                                       Vector &_diag);
+
 template<int T_D1D = 0, int T_Q1D = 0>
 void SmemPAHcurlMassAssembleDiagonal3D(const int D1D,
                                        const int Q1D,
@@ -181,11 +204,12 @@ void PAHcurlHdivSetup3D(const int Q1D,
                         Vector &_coeff,
                         Vector &op)
 {
+   const int NQ = Q1D*Q1D*Q1D;
    const bool symmetric = (coeffDim != 9);
-   auto W = Reshape(_w.Read(), Q1D, Q1D, Q1D);
-   auto J = Reshape(j.Read(), Q1D, Q1D, Q1D, 3, 3, NE);
-   auto coeff = Reshape(_coeff.Read(), coeffDim, Q1D, Q1D, Q1D, NE);
-   auto y = Reshape(op.Write(), 9, Q1D, Q1D, Q1D, NE);
+   auto W = _w.Read();
+   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
+   auto coeff = Reshape(_coeff.Read(), coeffDim, NQ, NE);
+   auto y = Reshape(op.Write(), 9, NQ, NE);
 
    const int i11 = 0;
    const int i12 = transpose ? 3 : 1;
@@ -197,89 +221,83 @@ void PAHcurlHdivSetup3D(const int Q1D,
    const int i32 = transpose ? 5 : 7;
    const int i33 = 8;
 
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   MFEM_FORALL(e, NE,
    {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      for (int q = 0; q < NQ; ++q)
       {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J31 = J(q,2,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double J32 = J(q,2,1,e);
+         const double J13 = J(q,0,2,e);
+         const double J23 = J(q,1,2,e);
+         const double J33 = J(q,2,2,e);
+         const double detJ = J11 * (J22 * J33 - J32 * J23) -
+         /* */               J21 * (J12 * J33 - J32 * J13) +
+         /* */               J31 * (J12 * J23 - J22 * J13);
+         const double w_detJ = W[q] / detJ;
+         // adj(J)
+         const double A11 = (J22 * J33) - (J23 * J32);
+         const double A12 = (J32 * J13) - (J12 * J33);
+         const double A13 = (J12 * J23) - (J22 * J13);
+         const double A21 = (J31 * J23) - (J21 * J33);
+         const double A22 = (J11 * J33) - (J13 * J31);
+         const double A23 = (J21 * J13) - (J11 * J23);
+         const double A31 = (J21 * J32) - (J31 * J22);
+         const double A32 = (J31 * J12) - (J11 * J32);
+         const double A33 = (J11 * J22) - (J12 * J21);
+
+         if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
          {
-            MFEM_FOREACH_THREAD(qz,z,Q1D)
-            {
-               const double J11 = J(qx,qy,qz,0,0,e);
-               const double J21 = J(qx,qy,qz,1,0,e);
-               const double J31 = J(qx,qy,qz,2,0,e);
-               const double J12 = J(qx,qy,qz,0,1,e);
-               const double J22 = J(qx,qy,qz,1,1,e);
-               const double J32 = J(qx,qy,qz,2,1,e);
-               const double J13 = J(qx,qy,qz,0,2,e);
-               const double J23 = J(qx,qy,qz,1,2,e);
-               const double J33 = J(qx,qy,qz,2,2,e);
-               const double detJ = J11 * (J22 * J33 - J32 * J23) -
-               /* */               J21 * (J12 * J33 - J32 * J13) +
-               /* */               J31 * (J12 * J23 - J22 * J13);
-               const double w_detJ = W(qx,qy,qz) / detJ;
-               // adj(J)
-               const double A11 = (J22 * J33) - (J23 * J32);
-               const double A12 = (J32 * J13) - (J12 * J33);
-               const double A13 = (J12 * J23) - (J22 * J13);
-               const double A21 = (J31 * J23) - (J21 * J33);
-               const double A22 = (J11 * J33) - (J13 * J31);
-               const double A23 = (J21 * J13) - (J11 * J23);
-               const double A31 = (J21 * J32) - (J31 * J22);
-               const double A32 = (J31 * J12) - (J11 * J32);
-               const double A33 = (J11 * J22) - (J12 * J21);
+            // First compute entries of R = MJ
+            const double M11 = (!symmetric) ? coeff(i11, q, e) : coeff(0, q, e);
+            const double M12 = (!symmetric) ? coeff(i12, q, e) : coeff(1, q, e);
+            const double M13 = (!symmetric) ? coeff(i13, q, e) : coeff(2, q, e);
+            const double M21 = (!symmetric) ? coeff(i21, q, e) : M12;
+            const double M22 = (!symmetric) ? coeff(i22, q, e) : coeff(3, q, e);
+            const double M23 = (!symmetric) ? coeff(i23, q, e) : coeff(4, q, e);
+            const double M31 = (!symmetric) ? coeff(i31, q, e) : M13;
+            const double M32 = (!symmetric) ? coeff(i32, q, e) : M23;
+            const double M33 = (!symmetric) ? coeff(i33, q, e) : coeff(5, q, e);
 
-               if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
-               {
-                  // First compute entries of R = MJ
-                  const double M11 = (!symmetric) ? coeff(i11,qx,qy,qz,e) : coeff(0,qx,qy,qz,e);
-                  const double M12 = (!symmetric) ? coeff(i12,qx,qy,qz,e) : coeff(1,qx,qy,qz,e);
-                  const double M13 = (!symmetric) ? coeff(i13,qx,qy,qz,e) : coeff(2,qx,qy,qz,e);
-                  const double M21 = (!symmetric) ? coeff(i21,qx,qy,qz,e) : M12;
-                  const double M22 = (!symmetric) ? coeff(i22,qx,qy,qz,e) : coeff(3,qx,qy,qz,e);
-                  const double M23 = (!symmetric) ? coeff(i23,qx,qy,qz,e) : coeff(4,qx,qy,qz,e);
-                  const double M31 = (!symmetric) ? coeff(i31,qx,qy,qz,e) : M13;
-                  const double M32 = (!symmetric) ? coeff(i32,qx,qy,qz,e) : M23;
-                  const double M33 = (!symmetric) ? coeff(i33,qx,qy,qz,e) : coeff(5,qx,qy,qz,e);
+            const double R11 = M11*J11 + M12*J12 + M13*J13;
+            const double R12 = M11*J21 + M12*J22 + M13*J23;
+            const double R13 = M11*J31 + M12*J32 + M13*J33;
+            const double R21 = M21*J11 + M22*J12 + M23*J13;
+            const double R22 = M21*J21 + M22*J22 + M23*J23;
+            const double R23 = M21*J31 + M22*J32 + M23*J33;
+            const double R31 = M31*J11 + M32*J12 + M33*J13;
+            const double R32 = M31*J21 + M32*J22 + M33*J23;
+            const double R33 = M31*J31 + M32*J32 + M33*J33;
 
-                  const double R11 = M11*J11 + M12*J12 + M13*J13;
-                  const double R12 = M11*J21 + M12*J22 + M13*J23;
-                  const double R13 = M11*J31 + M12*J32 + M13*J33;
-                  const double R21 = M21*J11 + M22*J12 + M23*J13;
-                  const double R22 = M21*J21 + M22*J22 + M23*J23;
-                  const double R23 = M21*J31 + M22*J32 + M23*J33;
-                  const double R31 = M31*J11 + M32*J12 + M33*J13;
-                  const double R32 = M31*J21 + M32*J22 + M33*J23;
-                  const double R33 = M31*J31 + M32*J32 + M33*J33;
-
-                  // Now set y to detJ J^{-1} R = adj(J) R
-                  y(i11,qx,qy,qz,e) = w_detJ * (A11*R11 + A12*R21 + A13*R31); // 1,1
-                  y(i12,qx,qy,qz,e) = w_detJ * (A11*R12 + A12*R22 + A13*R32); // 1,2
-                  y(i13,qx,qy,qz,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
-                  y(i21,qx,qy,qz,e) = w_detJ * (A21*R11 + A22*R21 + A23*R31); // 2,1
-                  y(i22,qx,qy,qz,e) = w_detJ * (A21*R12 + A22*R22 + A23*R32); // 2,2
-                  y(i23,qx,qy,qz,e) = w_detJ * (A21*R13 + A22*R23 + A23*R33); // 2,3
-                  y(i31,qx,qy,qz,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
-                  y(i32,qx,qy,qz,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
-                  y(i33,qx,qy,qz,e) = w_detJ * (A31*R13 + A32*R23 + A33*R33); // 3,3
-               }
-               else if (coeffDim == 3)  // Vector coefficient version
-               {
-                  const double D1 = coeff(0,qx,qy,qz,e);
-                  const double D2 = coeff(1,qx,qy,qz,e);
-                  const double D3 = coeff(2,qx,qy,qz,e);
-                  // detJ J^{-1} DJ = adj(J) DJ
-                  y(i11,qx,qy,qz,e) = w_detJ * (D1*A11*J11 + D2*A12*J21 + D3*A13*J31); // 1,1
-                  y(i12,qx,qy,qz,e) = w_detJ * (D1*A11*J12 + D2*A12*J22 + D3*A13*J32); // 1,2
-                  y(i13,qx,qy,qz,e) = w_detJ * (D1*A11*J13 + D2*A12*J23 + D3*A13*J33); // 1,3
-                  y(i21,qx,qy,qz,e) = w_detJ * (D1*A21*J11 + D2*A22*J21 + D3*A23*J31); // 2,1
-                  y(i22,qx,qy,qz,e) = w_detJ * (D1*A21*J12 + D2*A22*J22 + D3*A23*J32); // 2,2
-                  y(i23,qx,qy,qz,e) = w_detJ * (D1*A21*J13 + D2*A22*J23 + D3*A23*J33); // 2,3
-                  y(i31,qx,qy,qz,e) = w_detJ * (D1*A31*J11 + D2*A32*J21 + D3*A33*J31); // 3,1
-                  y(i32,qx,qy,qz,e) = w_detJ * (D1*A31*J12 + D2*A32*J22 + D3*A33*J32); // 3,2
-                  y(i33,qx,qy,qz,e) = w_detJ * (D1*A31*J13 + D2*A32*J23 + D3*A33*J33); // 3,3
-               }
-            }
+            // Now set y to detJ J^{-1} R = adj(J) R
+            y(i11,q,e) = w_detJ * (A11*R11 + A12*R21 + A13*R31); // 1,1
+            y(i12,q,e) = w_detJ * (A11*R12 + A12*R22 + A13*R32); // 1,2
+            y(i13,q,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
+            y(i21,q,e) = w_detJ * (A21*R11 + A22*R21 + A23*R31); // 2,1
+            y(i22,q,e) = w_detJ * (A21*R12 + A22*R22 + A23*R32); // 2,2
+            y(i23,q,e) = w_detJ * (A21*R13 + A22*R23 + A23*R33); // 2,3
+            y(i31,q,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
+            y(i32,q,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
+            y(i33,q,e) = w_detJ * (A31*R13 + A32*R23 + A33*R33); // 3,3
+         }
+         else if (coeffDim == 3)  // Vector coefficient version
+         {
+            const double D1 = coeff(0, q, e);
+            const double D2 = coeff(1, q, e);
+            const double D3 = coeff(2, q, e);
+            // detJ J^{-1} DJ = adj(J) DJ
+            y(i11,q,e) = w_detJ * (D1*A11*J11 + D2*A12*J21 + D3*A13*J31); // 1,1
+            y(i12,q,e) = w_detJ * (D1*A11*J12 + D2*A12*J22 + D3*A13*J32); // 1,2
+            y(i13,q,e) = w_detJ * (D1*A11*J13 + D2*A12*J23 + D3*A13*J33); // 1,3
+            y(i21,q,e) = w_detJ * (D1*A21*J11 + D2*A22*J21 + D3*A23*J31); // 2,1
+            y(i22,q,e) = w_detJ * (D1*A21*J12 + D2*A22*J22 + D3*A23*J32); // 2,2
+            y(i23,q,e) = w_detJ * (D1*A21*J13 + D2*A22*J23 + D3*A23*J33); // 2,3
+            y(i31,q,e) = w_detJ * (D1*A31*J11 + D2*A32*J21 + D3*A33*J31); // 3,1
+            y(i32,q,e) = w_detJ * (D1*A31*J12 + D2*A32*J22 + D3*A33*J32); // 3,2
+            y(i33,q,e) = w_detJ * (D1*A31*J13 + D2*A32*J23 + D3*A33*J33); // 3,3
          }
       }
    });
@@ -297,61 +315,59 @@ void PAHcurlHdivSetup2D(const int Q1D,
                         Vector &_coeff,
                         Vector &op)
 {
+   const int NQ = Q1D*Q1D;
    const bool symmetric = (coeffDim != 4);
-   auto W = Reshape(_w.Read(), Q1D, Q1D);
-   auto J = Reshape(j.Read(), Q1D, Q1D, 2, 2, NE);
-   auto coeff = Reshape(_coeff.Read(), coeffDim, Q1D, Q1D, NE);
-   auto y = Reshape(op.Write(), 4, Q1D, Q1D, NE);
+   auto W = _w.Read();
+   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
+   auto coeff = Reshape(_coeff.Read(), coeffDim, NQ, NE);
+   auto y = Reshape(op.Write(), 4, NQ, NE);
 
    const int i11 = 0;
    const int i12 = transpose ? 2 : 1;
    const int i21 = transpose ? 1 : 2;
    const int i22 = 3;
 
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, 1,
+   MFEM_FORALL(e, NE,
    {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      for (int q = 0; q < NQ; ++q)
       {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double w_detJ = W[q] / (J11*J22) - (J21*J12);
+
+         if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient version
          {
-            const double J11 = J(qx,qy,0,0,e);
-            const double J21 = J(qx,qy,1,0,e);
-            const double J12 = J(qx,qy,0,1,e);
-            const double J22 = J(qx,qy,1,1,e);
-            const double w_detJ = W(qx,qy) / (J11*J22) - (J21*J12);
+            // First compute entries of R = MJ
+            const double M11 = coeff(i11, q, e);
+            const double M12 = (!symmetric) ? coeff(i12, q, e) : coeff(1, q, e);
+            const double M21 = (!symmetric) ? coeff(i21, q, e) : M12;
+            const double M22 = (!symmetric) ? coeff(i22, q, e) : coeff(2, q, e);
 
-            if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient version
-            {
-               // First compute entries of R = MJ
-               const double M11 = coeff(i11,qx,qy,e);
-               const double M12 = (!symmetric) ? coeff(i12,qx,qy,e) : coeff(1,qx,qy,e);
-               const double M21 = (!symmetric) ? coeff(i21,qx,qy,e) : M12;
-               const double M22 = (!symmetric) ? coeff(i22,qx,qy,e) : coeff(2,qx,qy,e);
+            const double R11 = M11*J11 + M12*J21;
+            const double R12 = M11*J12 + M12*J22;
+            const double R21 = M21*J11 + M22*J21;
+            const double R22 = M21*J12 + M22*J22;
 
-               const double R11 = M11*J11 + M12*J21;
-               const double R12 = M11*J12 + M12*J22;
-               const double R21 = M21*J11 + M22*J21;
-               const double R22 = M21*J12 + M22*J22;
-
-               // Now set y to J^{-1} R
-               y(i11,qx,qy,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
-               y(i12,qx,qy,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
-               y(i21,qx,qy,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
-               y(i22,qx,qy,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
-            }
-            else if (coeffDim == 2) // Vector coefficient version
-            {
-               const double D1 = coeff(0,qx,qy,e);
-               const double D2 = coeff(1,qx,qy,e);
-               const double R11 = D1*J11;
-               const double R12 = D1*J12;
-               const double R21 = D2*J21;
-               const double R22 = D2*J22;
-               y(i11,qx,qy,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
-               y(i12,qx,qy,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
-               y(i21,qx,qy,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
-               y(i22,qx,qy,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
-            }
+            // Now set y to J^{-1} R
+            y(i11,q,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
+            y(i12,q,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
+            y(i21,q,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
+            y(i22,q,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
+         }
+         else if (coeffDim == 2) // Vector coefficient version
+         {
+            const double D1 = coeff(0, q, e);
+            const double D2 = coeff(1, q, e);
+            const double R11 = D1*J11;
+            const double R12 = D1*J12;
+            const double R21 = D2*J21;
+            const double R22 = D2*J22;
+            y(i11,q,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
+            y(i12,q,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
+            y(i21,q,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
+            y(i22,q,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
          }
       }
    });
@@ -721,8 +737,14 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 {
    // Assumes tensor-product elements
    Mesh *mesh = trial_fes.GetMesh();
+   dim = mesh->Dimension();
+   const bool bdryInteg = isBdryInteg;
+
+   // TODO: this implementation applies boundary markers just by setting coeff to zero on boundary elements. It would be better to skip those elements in AddMultPA.
+
+   const FiniteElement *trial_fel = bdryInteg ? trial_fes.GetBE(
+                                       0) : trial_fes.GetFE(0);
 
-   const FiniteElement *trial_fel = trial_fes.GetFE(0);
    const VectorTensorFiniteElement *trial_el =
       dynamic_cast<const VectorTensorFiniteElement*>(trial_fel);
    MFEM_VERIFY(trial_el != NULL, "Only VectorTensorFiniteElement is supported!");
@@ -732,21 +754,38 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
       dynamic_cast<const VectorTensorFiniteElement*>(test_fel);
    MFEM_VERIFY(test_el != NULL, "Only VectorTensorFiniteElement is supported!");
 
+   if (bdryInteg)
+   {
+      MFEM_VERIFY(trial_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3 &&
+                  mesh->SpaceDimension() == 3 && trial_el->GetDim() == 2, "");
+   }
+
    const IntegrationRule *ir
       = IntRule ? IntRule : &MassIntegrator::GetRule(*trial_el, *trial_el,
+                                                     bdryInteg ? *mesh->GetBdrElementTransformation(0) :
                                                      *mesh->GetElementTransformation(0));
    const int dims = trial_el->GetDim();
+
    MFEM_VERIFY(dims == 2 || dims == 3, "");
 
    const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
    const int nq = ir->GetNPoints();
-   dim = mesh->Dimension();
    MFEM_VERIFY(dim == 2 || dim == 3, "");
 
-   ne = trial_fes.GetNE();
-   MFEM_VERIFY(ne == test_fes.GetNE(),
+   ne = bdryInteg ? trial_fes.GetNBE() : trial_fes.GetNE();
+   Vector bdryJac;
+   if (bdryInteg)
+   {
+      bdryJac.SetSize(ne*nq*6);
+   }
+   else
+   {
+      geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+   }
+
+   MFEM_VERIFY(ne == (bdryInteg ? test_fes.GetNBE() : test_fes.GetNE()),
                "Different meshes for test and trial spaces");
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+
    mapsC = &trial_el->GetDofToQuad(*ir, DofToQuad::TENSOR);
    mapsO = &trial_el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
    dofs1D = mapsC->ndof;
@@ -780,7 +819,12 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
       pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne,
                       Device::GetMemoryType());
 
-   Vector coeff(coeffDim * ne * nq);
+   if (bdryInteg)
+   {
+      MFEM_VERIFY(coeffDim == 1 && dims == 2, "");
+   }
+
+   Vector coeff(coeffDim * nq * ne);
    coeff = 1.0;
    auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
    if (Q || VQ || MQ)
@@ -812,7 +856,8 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 
       for (int e=0; e<ne; ++e)
       {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         ElementTransformation *tr = bdryInteg ? mesh->GetBdrElementTransformation(
+                                        e) : mesh->GetElementTransformation(e);
          for (int p=0; p<nq; ++p)
          {
             if (MQ)
@@ -853,15 +898,77 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
       }
    }
 
-   if (trial_curl && test_curl && dim == 3)
+   if (el_marker)
    {
-      PADiffusionSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+      MFEM_VERIFY(bdryInteg, "");
+      MFEM_VERIFY(el_marker->Size() == mesh->bdr_attributes.Max(), "");
+
+      for (int e=0; e<ne; ++e)
+      {
+         const int el_attr = (bdryInteg) ? mesh->GetBdrAttribute(e) : -1;
+         MFEM_VERIFY(el_attr > 0, "");
+
+         if ((*el_marker)[el_attr-1] == 0)
+         {
+            for (int p=0; p<nq; ++p)
+            {
+               for (int i=0; i<coeffDim; ++i)
+               {
+                  coeffh(i, p, e) = 0.0;
+               }
+            }
+         }
+      }
+   }
+
+   if (isBdryInteg || (trial_el->GetDerivType() == mfem::FiniteElement::CURL &&
+                       dim == 2 && mesh->SpaceDimension() == 3))
+   {
+      MFEM_VERIFY(coeffDim == 1,
+                  "");  // Vector coefficient not implemented in this case
+
+      if (isBdryInteg)
+      {
+         for (int e=0; e<ne; ++e)
+         {
+            ElementTransformation *tr = mesh->GetBdrElementTransformation(e);
+            for (int q = 0; q < nq; ++q)
+            {
+               tr->SetIntPoint(&(ir->IntPoint(q)));
+               const DenseMatrix& jac = tr->Jacobian();
+               for (int i=0; i<3; ++i)
+                  for (int j=0; j<2; ++j)
+                  {
+                     bdryJac[(6*nq*e) + (3*nq*j) + (i*nq) + q] = jac(i,j);
+                  }
+            }
+         }
+      }
+      else
+      {
+         // J is allocated as 3x3, but entries (i,j) are set only for 0 <= i < 3, 0 <= j < 2.
+         // That is, on each element J is 3x2 with 6 entries, not 9, so the last third of geom->J is unused.
+         MFEM_VERIFY(geom->J.Size() == 6 * ne * nq, "");
+      }
+
+      PAHcurlSetup2Din3D(quad1D, ne, ir->GetWeights(),
+                         isBdryInteg ? bdryJac : geom->J,
                          coeff, pa_data);
    }
+   else if (trial_curl && test_curl && dim == 3)
+   {
+#ifdef SETUPONHOST
+      PAHcurlSetup3DHost(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+                         coeff, pa_data);
+#else
+      PAHcurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+                     coeff, pa_data);
+#endif
+   }
    else if (trial_curl && test_curl && dim == 2)
    {
-      PADiffusionSetup2D<2>(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
-                            coeff, pa_data);
+      PAHcurlSetup2D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+                     coeff, pa_data);
    }
    else if (trial_div && test_div && dim == 3)
    {
@@ -966,7 +1073,7 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
    const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
    const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
 
-   if (dim == 3)
+   if (dim == 3 && !isBdryInteg)
    {
       if (trial_curl && test_curl)
       {
@@ -1108,13 +1215,13 @@ void MixedVectorGradientIntegrator::AssemblePA(const FiniteElementSpace
    // Use the same setup functions as VectorFEMassIntegrator.
    if (test_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3)
    {
-      PADiffusionSetup3D(quad1D, 1, ne, ir->GetWeights(), geom->J,
-                         coeff, pa_data);
+      PAHcurlSetup3D(quad1D, 1, ne, ir->GetWeights(), geom->J,
+                     coeff, pa_data);
    }
    else if (test_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 2)
    {
-      PADiffusionSetup2D<2>(quad1D, 1, ne, ir->GetWeights(), geom->J,
-                            coeff, pa_data);
+      PAHcurlSetup2D(quad1D, 1, ne, ir->GetWeights(), geom->J,
+                     coeff, pa_data);
    }
    else
    {

fem/coefficient.cpp

@@ -38,11 +38,11 @@ double FunctionCoefficient::Eval(ElementTransformation & T,
 
    if (Function)
    {
-      return Function(transip);
+      return ((*Function)(transip));
    }
    else
    {
-      return TDFunction(transip, GetTime());
+      return (*TDFunction)(transip, GetTime());
    }
 }
 
@@ -112,11 +112,11 @@ void VectorFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
    V.SetSize(vdim);
    if (Function)
    {
-      Function(transip, V);
+      (*Function)(transip, V);
    }
    else
    {
-      TDFunction(transip, GetTime(), V);
+      (*TDFunction)(transip, GetTime(), V);
    }
    if (Q)
    {
@@ -301,45 +301,18 @@ void MatrixFunctionCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
 
    K.SetSize(height, width);
 
-   if (symmetric) // Use SymmFunction
+   if (Function)
    {
-      MFEM_VERIFY(height == width && SymmFunction,
-                  "MatrixFunctionCoefficient is not symmetric");
-
-      Vector Ksym((width * (width + 1)) / 2); // 1x1: 1, 2x2: 3, 3x3: 6
-
-      SymmFunction(transip, Ksym);
-
-      // Copy upper triangular values from Ksym to the full matrix K
-      int os = 0;
-      for (int i=0; i<height; ++i)
-      {
-         for (int j=i; j<width; ++j)
-         {
-            const double Kij = Ksym[j - i + os];
-            K(i,j) = Kij;
-            if (j != i) { K(j,i) = Kij; }
-         }
-
-         os += width - i;
-      }
+      (*Function)(transip, K);
+   }
+   else if (TDFunction)
+   {
+      (*TDFunction)(transip, GetTime(), K);
    }
    else
    {
-      if (Function)
-      {
-         Function(transip, K);
-      }
-      else if (TDFunction)
-      {
-         TDFunction(transip, GetTime(), K);
-      }
-      else
-      {
-         K = mat;
-      }
+      K = mat;
    }
-
    if (Q)
    {
       K *= Q->Eval(T, ip, GetTime());
@@ -350,7 +323,7 @@ void MatrixFunctionCoefficient::EvalSymmetric(Vector &K,
                                               ElementTransformation &T,
                                               const IntegrationPoint &ip)
 {
-   MFEM_VERIFY(symmetric && height == width && SymmFunction,
+   MFEM_VERIFY(symmetric && height == width && height < 4 && SymmFunction,
                "MatrixFunctionCoefficient is not symmetric");
 
    double x[3];
@@ -362,7 +335,7 @@ void MatrixFunctionCoefficient::EvalSymmetric(Vector &K,
 
    if (SymmFunction)
    {
-      SymmFunction(transip, K);
+      (*SymmFunction)(transip, K);
    }
 
    if (Q)

fem/coefficient.hpp

@@ -12,8 +12,6 @@
 #ifndef MFEM_COEFFICIENT
 #define MFEM_COEFFICIENT
 
-#include <functional>
-
 #include "../config/config.hpp"
 #include "../linalg/linalg.hpp"
 #include "intrules.hpp"
@@ -125,25 +123,28 @@ public:
                        const IntegrationPoint &ip);
 };
 
-/// A general function coefficient
+
+/// A general C-function coefficient
 class FunctionCoefficient : public Coefficient
 {
 protected:
-   std::function<double(const Vector &)> Function;
-   std::function<double(const Vector &, double)> TDFunction;
+   double (*Function)(const Vector &);
+   double (*TDFunction)(const Vector &, double);
 
 public:
-   /// Define a time-independent coefficient from a std function
-   /** \param F time-independent std::function */
-   FunctionCoefficient(std::function<double(const Vector &)> F)
-      : Function(std::move(F))
-   { }
+   /// Define a time-independent coefficient from a pointer to a C-function
+   FunctionCoefficient(double (*f)(const Vector &))
+   {
+      Function = f;
+      TDFunction = NULL;
+   }
 
-   /// Define a time-dependent coefficient from a std function
-   /** \param TDF time-dependent function */
-   FunctionCoefficient(std::function<double(const Vector &, double)> TDF)
-      : TDFunction(std::move(TDF))
-   { }
+   /// Define a time-dependent coefficient from a pointer to a C-function
+   FunctionCoefficient(double (*tdf)(const Vector &, double))
+   {
+      Function = NULL;
+      TDFunction = tdf;
+   }
 
    /// (DEPRECATED) Define a time-independent coefficient from a C-function
    /** @deprecated Use the method where the C-function, @a f, uses a const
@@ -404,34 +405,33 @@ public:
    const Vector& GetVec() { return vec; }
 };
 
-/// A general vector function coefficient
+/// A general C-function vector coefficient
 class VectorFunctionCoefficient : public VectorCoefficient
 {
 private:
-   std::function<void(const Vector &, Vector &)> Function;
-   std::function<void(const Vector &, double, Vector &)> TDFunction;
+   void (*Function)(const Vector &, Vector &);
+   void (*TDFunction)(const Vector &, double, Vector &);
    Coefficient *Q;
 
 public:
-   /// Define a time-independent vector coefficient from a std function
-   /** \param dim - the size of the vector
-       \param F - time-independent function
-       \param q - optional scalar Coefficient to scale the vector coefficient */
-   VectorFunctionCoefficient(int dim,
-                             std::function<void(const Vector &, Vector &)> F,
-                             Coefficient *q = nullptr)
-      : VectorCoefficient(dim), Function(std::move(F)), Q(q)
-   { }
+   /// Construct a time-independent vector coefficient from a C-function
+   VectorFunctionCoefficient(int dim, void (*F)(const Vector &, Vector &),
+                             Coefficient *q = NULL)
+      : VectorCoefficient(dim), Q(q)
+   {
+      Function = F;
+      TDFunction = NULL;
+   }
 
-   /// Define a time-dependent vector coefficient from a std function
-   /** \param dim - the size of the vector
-       \param TDF - time-dependent function
-       \param q - optional scalar Coefficient to scale the vector coefficient */
+   /// Construct a time-dependent vector coefficient from a C-function
    VectorFunctionCoefficient(int dim,
-                             std::function<void(const Vector &, double, Vector &)> TDF,
-                             Coefficient *q = nullptr)
-      : VectorCoefficient(dim), TDFunction(std::move(TDF)), Q(q)
-   { }
+                             void (*TDF)(const Vector &, double, Vector &),
+                             Coefficient *q = NULL)
+      : VectorCoefficient(dim), Q(q)
+   {
+      Function = NULL;
+      TDFunction = TDF;
+   }
 
    using VectorCoefficient::Eval;
    /// Evaluate the vector coefficient at @a ip.
@@ -721,6 +721,7 @@ public:
    /// For backward compatibility get the width of the matrix.
    int GetVDim() const { return width; }
 
+   void SetSymmetric(bool s) { symmetric = s; }
    bool IsSymmetric() const { return symmetric; }
 
    /** @brief Evaluate the matrix coefficient in the element described by @a T
@@ -761,56 +762,61 @@ public:
 
 
 /** @brief A matrix coefficient with an optional scalar coefficient multiplier
-    \a q.  The matrix function can either be represented by a std function or
-    a constant matrix provided when constructing this object.  */
+    \a q.  The matrix function can either be represented by a C-function or a
+    constant matrix provided when constructing this object.  */
 class MatrixFunctionCoefficient : public MatrixCoefficient
 {
 private:
-   std::function<void(const Vector &, DenseMatrix &)> Function;
-   std::function<void(const Vector &, Vector &)> SymmFunction;
-   std::function<void(const Vector &, double, DenseMatrix &)> TDFunction;
-
+   void (*Function)(const Vector &, DenseMatrix &);
+   void (*SymmFunction)(const Vector &, Vector &);
+   void (*TDFunction)(const Vector &, double, DenseMatrix &);
    Coefficient *Q;
    DenseMatrix mat;
 
 public:
-   /// Define a time-independent square matrix coefficient from a std function
-   /** \param dim - the size of the matrix
-       \param F - time-independent function
-       \param q - optional scalar Coefficient to scale the matrix coefficient */
-   MatrixFunctionCoefficient(int dim,
-                             std::function<void(const Vector &, DenseMatrix &)> F,
-                             Coefficient *q = nullptr)
-      : MatrixCoefficient(dim), Function(std::move(F)), Q(q), mat(0)
-   { }
-
-   /// Define a constant matrix coefficient times a scalar Coefficient
-   /** \param m - constant matrix
-       \param q - optional scalar Coefficient to scale the matrix coefficient */
-   MatrixFunctionCoefficient(const DenseMatrix &m, Coefficient &q)
-      : MatrixCoefficient(m.Height(), m.Width()), Q(&q), mat(m)
-   { }
-
-   /// Define a time-dependent square matrix coefficient from a std function
-   /** \param dim - the size of the matrix
-       \param TDF - time-dependent function
-       \param q - optional scalar Coefficient to scale the matrix coefficient */
-   MatrixFunctionCoefficient(int dim,
-                             std::function<void(const Vector &, double, DenseMatrix &)> TDF,
-                             Coefficient *q = nullptr)
-      : MatrixCoefficient(dim), TDFunction(std::move(TDF)), Q(q)
-   { }
-
-   /** @brief Define a time-independent symmetric square matrix coefficient from
-       a std function */
-   /** \param dim - the size of the matrix
-       \param SymmF - function used in EvalSymmetric
-       \param q - optional scalar Coefficient to scale the matrix coefficient */
-   MatrixFunctionCoefficient(int dim,
-                             std::function<void(const Vector &, Vector &)> SymmF,
+   /// Construct a square matrix coefficient from a C-function without time
+   /// dependence.
+   MatrixFunctionCoefficient(int dim, void (*F)(const Vector &, DenseMatrix &),
                              Coefficient *q = NULL)
-      : MatrixCoefficient(dim, true), SymmFunction(std::move(SymmF)), Q(q), mat(0)
-   { }
+      : MatrixCoefficient(dim), Q(q)
+   {
+      Function = F;
+      TDFunction = NULL;
+      mat.SetSize(0);
+   }
+
+   /// Construct a constant matrix coefficient times a scalar Coefficient
+   MatrixFunctionCoefficient(const DenseMatrix &m, Coefficient &q)
+      : MatrixCoefficient(m.Height(), m.Width()), Q(&q)
+   {
+      Function = NULL;
+      TDFunction = NULL;
+      mat = m;
+   }
+
+   /// Construct a square matrix coefficient from a C-function with
+   /// time-dependence.
+   MatrixFunctionCoefficient(int dim,
+                             void (*TDF)(const Vector &, double, DenseMatrix &),
+                             Coefficient *q = NULL)
+      : MatrixCoefficient(dim), Q(q)
+   {
+      Function = NULL;
+      TDFunction = TDF;
+      mat.SetSize(0);
+   }
+
+   /// Construct a symmetric square matrix coefficient from a C-function
+   /// defining a vector function used by EvalSymmetric
+   MatrixFunctionCoefficient(int dim, void (*F)(const Vector &, Vector &),
+                             Coefficient *q = NULL)
+      : MatrixCoefficient(dim, true), Q(q)
+   {
+      SymmFunction = F;
+      Function = NULL;
+      TDFunction = NULL;
+      mat.SetSize(0);
+   }
 
    /// Evaluate the matrix coefficient at @a ip.
    virtual void Eval(DenseMatrix &K, ElementTransformation &T,
@@ -883,7 +889,7 @@ public:
 
 /// Coefficients based on sums, products, or other functions of coefficients.
 ///@{
-/** @brief Scalar coefficient defined as the linear combination of two scalar
+/** Scalar coefficient defined as the linear combination of two scalar
     coefficients or a scalar and a scalar coefficient */
 class SumCoefficient : public Coefficient
 {
@@ -940,8 +946,8 @@ public:
    }
 };
 
-/** @brief Scalar coefficient defined as the product of two scalar coefficients
-    or a scalar and a scalar coefficient. */
+/** Scalar coefficient defined as the product of two scalar coefficients or
+    a scalar and a scalar coefficient. */
 class ProductCoefficient : public Coefficient
 {
 private:
@@ -979,8 +985,8 @@ public:
    { return ((a == NULL ) ? aConst : a->Eval(T, ip) ) * b->Eval(T, ip); }
 };
 
-/** @brief Scalar coefficient defined as the ratio of two scalars where one or
-    both scalars are scalar coefficients. */
+/** Scalar coefficient defined as the ratio of two scalars where one or both
+    scalars are scalar coefficients. */
 class RatioCoefficient : public Coefficient
 {
 private:

fem/complex_fem.cpp

@@ -968,7 +968,7 @@ ParComplexLinearForm::ParallelAssemble()
                                            2*(pfes->GlobalTrueVSize()),
                                            tdof_offsets);
 
-   tv->Write();
+   ((Vector*) tv)->Write(true);
    Vector tvr; tvr.MakeRef(*tv, 0, tvsize);
    Vector tvi; tvi.MakeRef(*tv, tvsize, tvsize);
 

fem/complex_fem.hpp

@@ -238,8 +238,7 @@ public:
    /// Set the desired assembly level.
    /** Valid choices are:
 
-       - AssemblyLevel::LEGACYFULL (default)
-       - AssemblyLevel::FULL
+       - AssemblyLevel::FULL  (default)
        - AssemblyLevel::PARTIAL
        - AssemblyLevel::ELEMENT
        - AssemblyLevel::NONE
@@ -576,8 +575,7 @@ public:
    /// Set the desired assembly level.
    /** Valid choices are:
 
-       - AssemblyLevel::LEGACYFULL (default)
-       - AssemblyLevel::FULL
+       - AssemblyLevel::FULL  (default)
        - AssemblyLevel::PARTIAL
        - AssemblyLevel::ELEMENT
        - AssemblyLevel::NONE

fem/datacollection.cpp

@@ -908,7 +908,7 @@ void ParaViewDataCollection::Save()
 
       // CELL DATA
       out << "<PCellData>\n";
-      out << "\t<PDataArray type=\"Int32\" Name=\"" << "attribute"
+      out << "\t<PDataArray type=\"Int32\" Name=\"" << "material"
           << "\" NumberOfComponents=\"1\""
           << " format=\"" << GetDataFormatString() << "\"/>\n";
       out << "</PCellData>\n";

fem/datacollection.hpp

@@ -382,10 +382,6 @@ public:
    int Error() const { return error; }
    /// Reset the error state
    void ResetError(int err = NO_ERROR) { error = err; }
-
-#ifdef MFEM_USE_MPI
-   friend class ParMesh;
-#endif
 };
 
 

fem/fespace.cpp

@@ -14,7 +14,6 @@
 #include "../general/text.hpp"
 #include "../general/forall.hpp"
 #include "../mesh/mesh_headers.hpp"
-#include "../fem/libceed/ceed.hpp"
 #include "fem.hpp"
 
 #include <cmath>
@@ -699,7 +698,7 @@ void FiniteElementSpace::BuildConformingInterpolation() const
    for (int entity = 1; entity <= 2; entity++)
    {
       const NCMesh::NCList &list = mesh->ncmesh->GetNCList(entity);
-      if (!list.masters.Size()) { continue; }
+      if (!list.masters.size()) { continue; }
 
       Array<int> master_dofs, slave_dofs;
 
@@ -707,7 +706,7 @@ void FiniteElementSpace::BuildConformingInterpolation() const
       DenseMatrix I;
 
       // loop through all master edges/faces, constrain their slave edges/faces
-      for (int mi = 0; mi < list.masters.Size(); mi++)
+      for (unsigned mi = 0; mi < list.masters.size(); mi++)
       {
          const NCMesh::Master &master = list.masters[mi];
 
@@ -731,7 +730,7 @@ void FiniteElementSpace::BuildConformingInterpolation() const
             GetEntityDofs(entity, slave.index, slave_dofs, master.Geom());
             if (!slave_dofs.Size()) { continue; }
 
-            list.OrientedPointMatrix(slave, T.GetPointMat());
+            slave.OrientedPointMatrix(T.GetPointMat());
             fe->GetLocalInterpolation(T, I);
 
             // make each slave DOF dependent on all master DOFs
@@ -2161,7 +2160,6 @@ void FiniteElementSpace::Destroy()
       delete [] bdofs;
       delete [] fdofs;
    }
-   RemoveCeedBasisAndRestriction(this);
 }
 
 void FiniteElementSpace::GetTransferOperator(

fem/fespace.hpp

@@ -330,6 +330,11 @@ public:
    virtual const Operator *GetProlongationMatrix() const
    { return GetConformingProlongation(); }
 
+   /// Return a prolongation operator that only acts on local dofs
+   /** The returned operator is owned by the FiniteElementSpace. */
+   virtual const Operator *GetLocalProlongationMatrix() const
+   { return GetConformingProlongation(); }
+
    /// The returned SparseMatrix is owned by the FiniteElementSpace.
    virtual const SparseMatrix *GetRestrictionMatrix() const
    { return GetConformingRestriction(); }
@@ -984,7 +989,6 @@ inline bool UsesTensorBasis(const FiniteElementSpace& fes)
 {
    return dynamic_cast<const mfem::TensorBasisElement *>(fes.GetFE(0))!=nullptr;
 }
-
 }
 
 #endif

fem/hybridization.cpp

@@ -68,7 +68,7 @@ void Hybridization::ConstructC()
       {
          const int dim = pmesh->Dimension();
          const NCMesh::NCList &shared = pmesh->pncmesh->GetSharedList(dim-1);
-         num_shared_slave_faces = (HYPRE_Int) shared.slaves.Size();
+         num_shared_slave_faces = (HYPRE_Int)shared.slaves.size();
          MPI_Allreduce(&num_shared_slave_faces, &glob_num_shared_slave_faces, 1,
                        HYPRE_MPI_INT, MPI_SUM, pmesh->GetComm());
          MFEM_ASSERT(glob_num_shared_slave_faces%2 == 0, "");

fem/libceed/ceed.hpp

@@ -14,29 +14,68 @@
 
 #include "../../config/config.hpp"
 
+#ifdef MFEM_USE_CEED
 #include "../../general/device.hpp"
 #include "../../linalg/vector.hpp"
-#ifdef MFEM_USE_CEED
 #include <ceed.h>
 #include <ceed-hash.h>
-#endif
 #include <tuple>
 #include <unordered_map>
 
 namespace mfem
 {
 
-class Mesh;
 class FiniteElementSpace;
 class GridFunction;
 class IntegrationRule;
 class Coefficient;
 
-#ifdef MFEM_USE_CEED
-/// A structure used to pass additional data to f_build_diff and f_apply_diff
-struct BuildContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; };
+// Hash table for CeedBasis
+using CeedBasisKey =
+   std::tuple<const FiniteElementSpace*, const IntegrationRule*, int, int, int>;
+struct CeedBasisHash
+{
+   std::size_t operator()(const CeedBasisKey& k) const
+   {
+      return CeedHashCombine(CeedHashCombine(CeedHashInt(
+                                                reinterpret_cast<CeedHash64_t>(std::get<0>(k))),
+                                             CeedHashInt(
+                                                reinterpret_cast<CeedHash64_t>(std::get<1>(k)))),
+                             CeedHashCombine(CeedHashCombine(CeedHashInt(std::get<2>(k)),
+                                                             CeedHashInt(std::get<3>(k))),
+                                             CeedHashInt(std::get<4>(k))));
+   }
+};
+using CeedBasisMap =
+   std::unordered_map<const CeedBasisKey, CeedBasis, CeedBasisHash>;
 
-enum class CeedCoeff { Const, Grid, Quad };
+// Hash table for CeedElemRestriction
+using CeedRestrKey = std::tuple<const FiniteElementSpace*, int, int, int>;
+struct CeedRestrHash
+{
+   std::size_t operator()(const CeedRestrKey& k) const
+   {
+      return CeedHashCombine(CeedHashCombine(CeedHashInt(
+                                                reinterpret_cast<CeedHash64_t>(std::get<0>(k))),
+                                             CeedHashInt(std::get<1>(k))),
+                             CeedHashCombine(CeedHashInt(std::get<2>(k)),
+                                             CeedHashInt(std::get<3>(k))));
+   }
+};
+using CeedRestrMap =
+   std::unordered_map<const CeedRestrKey, CeedElemRestriction, CeedRestrHash>;
+
+namespace internal
+{
+extern Ceed ceed; // defined in device.cpp
+extern CeedBasisMap basis_map;
+extern CeedRestrMap restr_map;
+}
+
+/// A structure used to pass additional data to f_build_diff and f_apply_diff
+struct BuildContext { CeedInt dim, space_dim; CeedScalar coeff; };
+
+enum class CeedCoeff { Const, Grid };
 
 struct CeedConstCoeff
 {
@@ -51,17 +90,8 @@ struct CeedGridCoeff
    CeedVector coeffVector;
 };
 
-struct CeedQuadCoeff
-{
-   Vector coeff;
-   CeedElemRestriction restr;
-   CeedVector coeffVector;
-};
-#endif
-
 struct CeedData
 {
-#ifdef MFEM_USE_CEED
    CeedOperator build_oper, oper;
    CeedBasis basis, mesh_basis;
    CeedElemRestriction restr, mesh_restr, restr_i, mesh_restr_i;
@@ -74,51 +104,39 @@ struct CeedData
 
    CeedVector u, v;
 
-   CeedData()
-      : build_oper(nullptr), oper(nullptr), basis(nullptr), mesh_basis(nullptr),
-        restr(nullptr), mesh_restr(nullptr), restr_i(nullptr), mesh_restr_i(nullptr),
-        apply_qfunc(nullptr), build_qfunc(nullptr), node_coords(nullptr),
-        rho(nullptr), coeff(nullptr), build_ctx(nullptr) { }
-
    ~CeedData()
    {
       CeedOperatorDestroy(&build_oper);
       CeedOperatorDestroy(&oper);
+      CeedElemRestrictionDestroy(&restr_i);
+      CeedElemRestrictionDestroy(&mesh_restr_i);
       CeedQFunctionDestroy(&apply_qfunc);
       CeedQFunctionDestroy(&build_qfunc);
-      CeedQFunctionContextDestroy(&build_ctx);
       CeedVectorDestroy(&node_coords);
       CeedVectorDestroy(&rho);
-      if (coeff_type==CeedCoeff::Const)
+      if (coeff_type==CeedCoeff::Grid)
       {
-         delete static_cast<CeedConstCoeff*>(coeff);
-      }
-      else if (coeff_type==CeedCoeff::Grid)
-      {
-         CeedGridCoeff* c = static_cast<CeedGridCoeff*>(coeff);
+         CeedGridCoeff* c = (CeedGridCoeff*)coeff;
          CeedVectorDestroy(&c->coeffVector);
          delete c;
       }
-      else if (coeff_type==CeedCoeff::Quad)
+      else
       {
-         CeedQuadCoeff* c = static_cast<CeedQuadCoeff*>(coeff);
-         CeedVectorDestroy(&c->coeffVector);
-         delete c;
+         delete (CeedConstCoeff*)coeff;
       }
       CeedVectorDestroy(&u);
       CeedVectorDestroy(&v);
    }
-#endif
+
 };
 
 /** This structure contains the data to assemble a PA operator with libCEED.
     See libceed/mass.cpp or libceed/diffusion.cpp for examples. */
 struct CeedPAOperator
 {
-#ifdef MFEM_USE_CEED
    /** The finite element space for the trial and test functions. */
    const FiniteElementSpace &fes;
-   /** The Integration Rule to use to compute the operator. */
+   /** The Integration Rule to use to compote the operator. */
    const IntegrationRule &ir;
    /** The number of quadrature data at each quadrature point. */
    int qdatasize;
@@ -129,11 +147,11 @@ struct CeedPAOperator
    std::string const_func;
    /** The Qfunction to build the quadrature data with constant coefficient. */
    CeedQFunctionUser const_qf;
-   /** The name of the Qfunction to build the quadrature data with a coefficient
-       evaluated at quadrature points. */
-   std::string quad_func;
-   /** The Qfunction to build the quad. data with a coefficient. */
-   CeedQFunctionUser quad_qf;
+   /** The name of the Qfunction to build the quadrature data with grid function
+       coefficient. */
+   std::string grid_func;
+   /** The Qfunction to build the quad. data with grid function coefficient. */
+   CeedQFunctionUser grid_qf;
    /** The name of the Qfunction to apply the operator. */
    std::string apply_func;
    /** The Qfunction to apply the operator. */
@@ -144,84 +162,11 @@ struct CeedPAOperator
    /** The evaluation mode to apply to the test function ( CEED_EVAL_INTERP,
        CEED_EVAL_GRAD, etc.)*/
    CeedEvalMode test_op;
-#endif
-};
-
-struct CeedMFOperator
-{
-#ifdef MFEM_USE_CEED
-   /** The finite element space for the trial and test functions. */
-   const FiniteElementSpace &fes;
-   /** The Integration Rule to use to compote the operator. */
-   const IntegrationRule &ir;
-   /** The path to the header containing the functions for libCEED. */
-   std::string header;
-   /** The name of the Qfunction to apply the quadrature data with a constant
-       coefficient.*/
-   std::string const_func;
-   /** The Qfunction to apply the quadrature data with constant coefficient. */
-   CeedQFunctionUser const_qf;
-   /** The name of the Qfunction to apply the quadrature data with a coefficient
-       evaluated at quadrature point. */
-   std::string quad_func;
-   /** The Qfunction to apply the quad. data with a coefficient evaluated at
-       quadrature point. */
-   CeedQFunctionUser quad_qf;
-   /** The evaluation mode to apply to the trial function (CEED_EVAL_INTERP,
-       CEED_EVAL_GRAD, etc.) */
-   CeedEvalMode trial_op;
-   /** The evaluation mode to apply to the test function ( CEED_EVAL_INTERP,
-       CEED_EVAL_GRAD, etc.)*/
-   CeedEvalMode test_op;
-#endif
 };
 
 /** @brief Identifies the type of coefficient of the Integrator to initialize
     accordingly the CeedData. */
-void InitCeedCoeff(Coefficient* Q, Mesh &mesh, const IntegrationRule &ir,
-                   CeedData* ptr);
-
-/** This function initializes an arbitrary linear operator using the partial
-    assembly decomposition in libCEED. The operator details are described by the
-    struct CEEDPAOperator input. */
-void CeedPAAssemble(const CeedPAOperator& op,
-                    CeedData& ceedData);
-
-/** This function initializes an arbitrary linear operator using a fully matrix
-    free decomposition. The operator details are described by the
-    struct CEEDPAOperator input. */
-void CeedMFAssemble(const CeedMFOperator& op,
-                    CeedData& ceedData);
-
-/** @brief Function that applies a libCEED operator. */
-void CeedAddMult(const CeedData *ceedDataPtr,
-                 const Vector &x,
-                 Vector &y);
-
-/** @brief Function that assembles a libCEED operator diagonal. */
-void CeedAssembleDiagonal(const CeedData *ceedDataPtr,
-                          Vector &diag);
-
-/** @brief Function that determines if a CEED kernel should be used, based on
-    the current mfem::Device configuration. */
-inline bool DeviceCanUseCeed()
-{
-   return Device::Allows(Backend::CEED_MASK);
-}
-
-/** @brief Remove from ceed_basis_map and ceed_restr_map the entries associated
-    with the given @a fes. */
-void RemoveCeedBasisAndRestriction(const FiniteElementSpace *fes);
-
-#ifdef MFEM_USE_CEED
-/// Initialize a CeedVector from a Vector
-void InitCeedVector(const Vector &v, CeedVector &cv);
-
-/// Initialize a strided CeedElemRestriction
-void InitCeedStridedRestriction(const FiniteElementSpace &fes,
-                                CeedInt nelem, CeedInt nqpts, CeedInt qdatasize,
-                                const CeedInt *strides,
-                                CeedElemRestriction *restr);
+void InitCeedCoeff(Coefficient* Q, CeedData* ptr);
 
 /// Initialize a CeedBasis and a CeedElemRestriction
 void InitCeedBasisAndRestriction(const FiniteElementSpace &fes,
@@ -232,52 +177,43 @@ void InitCeedBasisAndRestriction(const FiniteElementSpace &fes,
 /// Return the path to the libCEED q-function headers.
 const std::string &GetCeedPath();
 
-// Hash table for CeedBasis
-using CeedBasisKey =
-   std::tuple<const FiniteElementSpace*, const IntegrationRule*, int, int, int>;
-struct CeedBasisHash
-{
-   std::size_t operator()(const CeedBasisKey& k) const
-   {
-      return CeedHashCombine(
-                CeedHashCombine(
-                   CeedHashInt(reinterpret_cast<CeedHash64_t>(std::get<0>(k))),
-                   CeedHashInt(reinterpret_cast<CeedHash64_t>(std::get<1>(k)))),
-                CeedHashCombine(
-                   CeedHashCombine(CeedHashInt(std::get<2>(k)),
-                                   CeedHashInt(std::get<3>(k))),
-                   CeedHashInt(std::get<4>(k))));
-   }
-};
-using CeedBasisMap =
-   std::unordered_map<const CeedBasisKey, CeedBasis, CeedBasisHash>;
+/** This function initializes an arbitrary linear operator using the partial
+    assembly decomposition in libCEED. The operator details are described by the
+    struct CEEDPAOperator input. */
+void CeedPAAssemble(const CeedPAOperator& op,
+                    CeedData& ceedData);
 
-// Hash table for CeedElemRestriction
-using CeedRestrKey = std::tuple<const FiniteElementSpace*, int, int, int>;
-struct CeedRestrHash
-{
-   std::size_t operator()(const CeedRestrKey& k) const
-   {
-      return CeedHashCombine(
-                CeedHashCombine(
-                   CeedHashInt(reinterpret_cast<CeedHash64_t>(std::get<0>(k))),
-                   CeedHashInt(std::get<1>(k))),
-                CeedHashCombine(CeedHashInt(std::get<2>(k)),
-                                CeedHashInt(std::get<3>(k))));
-   }
-};
-using CeedRestrMap =
-   std::unordered_map<const CeedRestrKey, CeedElemRestriction, CeedRestrHash>;
+/** @brief Function that applies a libCEED PA operator. */
+void CeedAddMultPA(const CeedData *ceedDataPtr,
+                   const Vector &x,
+                   Vector &y);
 
-namespace internal
+/** @brief Function that assembles a libCEED PA operator diagonal. */
+void CeedAssembleDiagonalPA(const CeedData *ceedDataPtr,
+                            Vector &diag);
+
+/** @brief Function that determines if a CEED kernel should be used, based on
+    the current mfem::Device configuration. */
+inline bool DeviceCanUseCeed()
 {
-extern Ceed ceed; // defined in device.cpp
-extern CeedBasisMap basis_map;
-extern CeedRestrMap restr_map;
+   return Device::Allows(Backend::CEED_CUDA) ||
+          (Device::Allows(Backend::CEED_CPU) &&
+           !Device::Allows(Backend::DEVICE_MASK|Backend::OMP_MASK));
 }
 
-#endif // MFEM_USE_CEED
-
 } // namespace mfem
 
+#else // MFEM_USE_CEED
+
+namespace mfem
+{
+inline bool DeviceCanUseCeed()
+{
+   return false;
+}
+
+} // namespace mfem
+
+#endif // MFEM_USE_CEED
+
 #endif // MFEM_LIBCEED_HPP

fem/libceed/diffusion.cpp

@@ -11,10 +11,9 @@
 
 #include "diffusion.hpp"
 
-#include "ceed.hpp"
 #ifdef MFEM_USE_CEED
+#include "ceed.hpp"
 #include "diffusion.h"
-#endif
 
 namespace mfem
 {
@@ -22,42 +21,18 @@ namespace mfem
 void CeedPADiffusionAssemble(const FiniteElementSpace &fes,
                              const mfem::IntegrationRule &irm, CeedData& ceedData)
 {
-#ifdef MFEM_USE_CEED
-   Mesh &mesh = *fes.GetMesh();
-   // Perform checks for some assumptions made in the Q-functions.
-   MFEM_VERIFY(mesh.Dimension() == mesh.SpaceDimension(), "case not supported");
-   MFEM_VERIFY(fes.GetVDim() == 1 || fes.GetVDim() == mesh.Dimension(),
-               "case not supported");
-   int dim = mesh.Dimension();
+   CeedInt dim = fes.GetMesh()->SpaceDimension();
    CeedPAOperator diffOp = {fes, irm,
                             dim * (dim + 1) / 2, "/diffusion.h",
                             ":f_build_diff_const", f_build_diff_const,
-                            ":f_build_diff_quad", f_build_diff_quad,
+                            ":f_build_diff_grid", f_build_diff_grid,
                             ":f_apply_diff", f_apply_diff,
                             CEED_EVAL_GRAD,
                             CEED_EVAL_GRAD
                            };
    CeedPAAssemble(diffOp, ceedData);
-#else
-   mfem_error("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
-void CeedMFDiffusionAssemble(const FiniteElementSpace &fes,
-                             const mfem::IntegrationRule &irm, CeedData& ceedData)
-{
-#ifdef MFEM_USE_CEED
-   CeedMFOperator diffOp = {fes, irm,
-                            "/diffusion.h",
-                            ":f_apply_diff_mf_const", f_apply_diff_mf_const,
-                            ":f_apply_diff_mf_quad", f_apply_diff_mf_quad,
-                            CEED_EVAL_GRAD,
-                            CEED_EVAL_GRAD
-                           };
-   CeedMFAssemble(diffOp, ceedData);
-#else
-   mfem_error("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
 }
 
 } // namespace mfem
+
+#endif // MFEM_USE_CEED

fem/libceed/diffusion.h

@@ -10,7 +10,7 @@
 // CONTRIBUTING.md for details.
 
 /// A structure used to pass additional data to f_build_diff and f_apply_diff
-struct BuildContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; };
+struct BuildContext { CeedInt dim, space_dim; CeedScalar coeff; };
 
 /// libCEED Q-function for building quadrature data for a diffusion operator
 /// with a constant coefficient
@@ -88,8 +88,8 @@ CEED_QFUNCTION(f_build_diff_const)(void *ctx, CeedInt Q,
 }
 
 /// libCEED Q-function for building quadrature data for a diffusion operator
-/// coefficient evaluated at quadrature points.
-CEED_QFUNCTION(f_build_diff_quad)(void *ctx, CeedInt Q,
+/// with a grid function coefficient
+CEED_QFUNCTION(f_build_diff_grid)(void *ctx, CeedInt Q,
                                   const CeedScalar *const *in,
                                   CeedScalar *const *out)
 {
@@ -172,15 +172,15 @@ CEED_QFUNCTION(f_apply_diff)(void *ctx, CeedInt Q,
    // in[0], out[0] have shape [dim, nc=1, Q]
    const CeedScalar *ug = in[0], *qd = in[1];
    CeedScalar *vg = out[0];
-   switch (10*bc->dim + bc->vdim)
+   switch (bc->dim)
    {
-      case 11:
+      case 1:
          for (CeedInt i = 0; i < Q; i++)
          {
             vg[i] = ug[i] * qd[i];
          }
          break;
-      case 21:
+      case 2:
          for (CeedInt i = 0; i < Q; i++)
          {
             const CeedScalar ug0 = ug[i + Q * 0];
@@ -189,23 +189,7 @@ CEED_QFUNCTION(f_apply_diff)(void *ctx, CeedInt Q,
             vg[i + Q * 1] = qd[i + Q * 1] * ug0 + qd[i + Q * 2] * ug1;
          }
          break;
-      case 22:
-         for (CeedInt i = 0; i < Q; i++)
-         {
-            const CeedScalar qd00 = qd[i + Q * 0];
-            const CeedScalar qd01 = qd[i + Q * 1];
-            const CeedScalar qd10 = qd01;
-            const CeedScalar qd11 = qd[i + Q * 2];
-            for (CeedInt c = 0; c < 2; c++)
-            {
-               const CeedScalar ug0 = ug[i + Q * (c+2*0)];
-               const CeedScalar ug1 = ug[i + Q * (c+2*1)];
-               vg[i + Q * (c+2*0)] = qd00 * ug0 + qd01 * ug1;
-               vg[i + Q * (c+2*1)] = qd10 * ug0 + qd11 * ug1;
-            }
-         }
-         break;
-      case 31:
+      case 3:
          for (CeedInt i = 0; i < Q; i++)
          {
             const CeedScalar ug0 = ug[i + Q * 0];
@@ -216,337 +200,6 @@ CEED_QFUNCTION(f_apply_diff)(void *ctx, CeedInt Q,
             vg[i + Q * 2] = qd[i + Q * 2] * ug0 + qd[i + Q * 4] * ug1 + qd[i + Q * 5] * ug2;
          }
          break;
-      case 33:
-         for (CeedInt i = 0; i < Q; i++)
-         {
-            const CeedScalar qd00 = qd[i + Q * 0];
-            const CeedScalar qd01 = qd[i + Q * 1];
-            const CeedScalar qd02 = qd[i + Q * 2];
-            const CeedScalar qd10 = qd01;
-            const CeedScalar qd11 = qd[i + Q * 3];
-            const CeedScalar qd12 = qd[i + Q * 4];
-            const CeedScalar qd20 = qd02;
-            const CeedScalar qd21 = qd12;
-            const CeedScalar qd22 = qd[i + Q * 5];
-            for (CeedInt c = 0; c < 3; c++)
-            {
-               const CeedScalar ug0 = ug[i + Q * (c+3*0)];
-               const CeedScalar ug1 = ug[i + Q * (c+3*1)];
-               const CeedScalar ug2 = ug[i + Q * (c+3*2)];
-               vg[i + Q * (c+3*0)] = qd00 * ug0 + qd01 * ug1 + qd02 * ug2;
-               vg[i + Q * (c+3*1)] = qd10 * ug0 + qd11 * ug1 + qd12 * ug2;
-               vg[i + Q * (c+3*2)] = qd20 * ug0 + qd21 * ug1 + qd22 * ug2;
-            }
-         }
-         break;
-   }
-   return 0;
-}
-
-/// libCEED Q-function for applying a diff operator
-CEED_QFUNCTION(f_apply_diff_mf_const)(void *ctx, CeedInt Q,
-                                      const CeedScalar *const *in,
-                                      CeedScalar *const *out)
-{
-   BuildContext *bc = (BuildContext*)ctx;
-   // in[0], out[0] have shape [dim, nc=1, Q]
-   // in[1] is Jacobians with shape [dim, nc=dim, Q]
-   // in[2] is quadrature weights, size (Q)
-   //
-   // At every quadrature point, compute qw/det(J).adj(J).adj(J)^T
-   const CeedScalar coeff = bc->coeff;
-   const CeedScalar *ug = in[0], *J = in[1], *qw = in[2];
-   CeedScalar *vg = out[0];
-   switch (10 * bc->dim + bc->vdim)
-   {
-      case 11:
-         for (CeedInt i = 0; i < Q; i++)
-         {
-            const CeedScalar qd = coeff * qw[i] / J[i];
-            vg[i] = ug[i] * qd;
-         }
-         break;
-      case 21:
-         for (CeedInt i = 0; i < Q; i++)
-         {
-            // J: 0 2   qd: 0 1   adj(J):  J22 -J12
-            //    1 3       1 2           -J21  J11
-            const CeedScalar J11 = J[i + Q * 0];
-            const CeedScalar J21 = J[i + Q * 1];
-            const CeedScalar J12 = J[i + Q * 2];
-            const CeedScalar J22 = J[i + Q * 3];
-            const CeedScalar w = qw[i] / (J11 * J22 - J21 * J12);
-            CeedScalar qd[3];
-            qd[0] =   coeff * w * (J12 * J12 + J22 * J22);
-            qd[1] = - coeff * w * (J11 * J12 + J21 * J22);
-            qd[2] =   coeff * w * (J11 * J11 + J21 * J21);
-            const CeedScalar ug0 = ug[i + Q * 0];
-            const CeedScalar ug1 = ug[i + Q * 1];
-            vg[i + Q * 0] = qd[0] * ug0 + qd[1] * ug1;
-            vg[i + Q * 1] = qd[1] * ug0 + qd[2] * ug1;
-         }
-         break;
-      case 22:
-         for (CeedInt i = 0; i < Q; i++)
-         {
-            // J: 0 2   qd: 0 1   adj(J):  J22 -J12
-            //    1 3       1 2           -J21  J11
-            const CeedScalar J11 = J[i + Q * 0];
-            const CeedScalar J21 = J[i + Q * 1];
-            const CeedScalar J12 = J[i + Q * 2];
-            const CeedScalar J22 = J[i + Q * 3];
-            const CeedScalar w = qw[i] / (J11 * J22 - J21 * J12);
-            CeedScalar qd[3];
-            qd[0] =   coeff * w * (J12 * J12 + J22 * J22);
-            qd[1] = - coeff * w * (J11 * J12 + J21 * J22);
-            qd[2] =   coeff * w * (J11 * J11 + J21 * J21);
-            for (CeedInt c = 0; c < 2; c++)
-            {
-               const CeedScalar ug0 = ug[i + Q * (c+2*0)];
-               const CeedScalar ug1 = ug[i + Q * (c+2*1)];
-               vg[i + Q * (c+2*0)] = qd[0] * ug0 + qd[1] * ug1;
-               vg[i + Q * (c+2*1)] = qd[1] * ug0 + qd[2] * ug1;
-            }
-         }
-         break;
-      case 31:
-         for (CeedInt i = 0; i < Q; i++)
-         {
-            // J: 0 3 6   qd: 0 1 2
-            //    1 4 7       1 3 4
-            //    2 5 8       2 4 5
-            const CeedScalar J11 = J[i + Q * 0];
-            const CeedScalar J21 = J[i + Q * 1];
-            const CeedScalar J31 = J[i + Q * 2];
-            const CeedScalar J12 = J[i + Q * 3];
-            const CeedScalar J22 = J[i + Q * 4];
-            const CeedScalar J32 = J[i + Q * 5];
-            const CeedScalar J13 = J[i + Q * 6];
-            const CeedScalar J23 = J[i + Q * 7];
-            const CeedScalar J33 = J[i + Q * 8];
-            const CeedScalar A11 = J22 * J33 - J23 * J32;
-            const CeedScalar A12 = J13 * J32 - J12 * J33;
-            const CeedScalar A13 = J12 * J23 - J13 * J22;
-            const CeedScalar A21 = J23 * J31 - J21 * J33;
-            const CeedScalar A22 = J11 * J33 - J13 * J31;
-            const CeedScalar A23 = J13 * J21 - J11 * J23;
-            const CeedScalar A31 = J21 * J32 - J22 * J31;
-            const CeedScalar A32 = J12 * J31 - J11 * J32;
-            const CeedScalar A33 = J11 * J22 - J12 * J21;
-            const CeedScalar w = qw[i] / (J11 * A11 + J21 * A12 + J31 * A13);
-            CeedScalar qd[6];
-            qd[0] = coeff * w * (A11 * A11 + A12 * A12 + A13 * A13);
-            qd[1] = coeff * w * (A11 * A21 + A12 * A22 + A13 * A23);
-            qd[2] = coeff * w * (A11 * A31 + A12 * A32 + A13 * A33);
-            qd[3] = coeff * w * (A21 * A21 + A22 * A22 + A23 * A23);
-            qd[4] = coeff * w * (A21 * A31 + A22 * A32 + A23 * A33);
-            qd[5] = coeff * w * (A31 * A31 + A32 * A32 + A33 * A33);
-            const CeedScalar ug0 = ug[i + Q * 0];
-            const CeedScalar ug1 = ug[i + Q * 1];
-            const CeedScalar ug2 = ug[i + Q * 2];
-            vg[i + Q * 0] = qd[0] * ug0 + qd[1] * ug1 + qd[2] * ug2;
-            vg[i + Q * 1] = qd[1] * ug0 + qd[3] * ug1 + qd[4] * ug2;
-            vg[i + Q * 2] = qd[2] * ug0 + qd[4] * ug1 + qd[5] * ug2;
-         }
-         break;
-      case 33:
-         for (CeedInt i = 0; i < Q; i++)
-         {
-            // J: 0 3 6   qd: 0 1 2
-            //    1 4 7       1 3 4
-            //    2 5 8       2 4 5
-            const CeedScalar J11 = J[i + Q * 0];
-            const CeedScalar J21 = J[i + Q * 1];
-            const CeedScalar J31 = J[i + Q * 2];
-            const CeedScalar J12 = J[i + Q * 3];
-            const CeedScalar J22 = J[i + Q * 4];
-            const CeedScalar J32 = J[i + Q * 5];
-            const CeedScalar J13 = J[i + Q * 6];
-            const CeedScalar J23 = J[i + Q * 7];
-            const CeedScalar J33 = J[i + Q * 8];
-            const CeedScalar A11 = J22 * J33 - J23 * J32;
-            const CeedScalar A12 = J13 * J32 - J12 * J33;
-            const CeedScalar A13 = J12 * J23 - J13 * J22;
-            const CeedScalar A21 = J23 * J31 - J21 * J33;
-            const CeedScalar A22 = J11 * J33 - J13 * J31;
-            const CeedScalar A23 = J13 * J21 - J11 * J23;
-            const CeedScalar A31 = J21 * J32 - J22 * J31;
-            const CeedScalar A32 = J12 * J31 - J11 * J32;
-            const CeedScalar A33 = J11 * J22 - J12 * J21;
-            const CeedScalar w = qw[i] / (J11 * A11 + J21 * A12 + J31 * A13);
-            CeedScalar qd[6];
-            qd[0] = coeff * w * (A11 * A11 + A12 * A12 + A13 * A13);
-            qd[1] = coeff * w * (A11 * A21 + A12 * A22 + A13 * A23);
-            qd[2] = coeff * w * (A11 * A31 + A12 * A32 + A13 * A33);
-            qd[3] = coeff * w * (A21 * A21 + A22 * A22 + A23 * A23);
-            qd[4] = coeff * w * (A21 * A31 + A22 * A32 + A23 * A33);
-            qd[5] = coeff * w * (A31 * A31 + A32 * A32 + A33 * A33);
-            for (CeedInt c = 0; c < 3; c++)
-            {
-               const CeedScalar ug0 = ug[i + Q * (c+3*0)];
-               const CeedScalar ug1 = ug[i + Q * (c+3*1)];
-               const CeedScalar ug2 = ug[i + Q * (c+3*2)];
-               vg[i + Q * (c+3*0)] = qd[0] * ug0 + qd[1] * ug1 + qd[2] * ug2;
-               vg[i + Q * (c+3*1)] = qd[1] * ug0 + qd[3] * ug1 + qd[4] * ug2;
-               vg[i + Q * (c+3*2)] = qd[2] * ug0 + qd[4] * ug1 + qd[5] * ug2;
-            }
-         }
-         break;
-   }
-   return 0;
-}
-
-CEED_QFUNCTION(f_apply_diff_mf_quad)(void *ctx, CeedInt Q,
-                                     const CeedScalar *const *in,
-                                     CeedScalar *const *out)
-{
-   BuildContext *bc = (BuildContext*)ctx;
-   // in[0], out[0] have shape [dim, nc=1, Q]
-   // in[1] is Jacobians with shape [dim, nc=dim, Q]
-   // in[2] is quadrature weights, size (Q)
-   //
-   // At every quadrature point, compute qw/det(J).adj(J).adj(J)^T
-   const CeedScalar *c = in[0], *ug = in[1], *J = in[2], *qw = in[3];
-   CeedScalar *vg = out[0];
-   switch (10 * bc->dim + bc->vdim)
-   {
-      case 11:
-         for (CeedInt i = 0; i < Q; i++)
-         {
-            const CeedScalar qd = c[i] * qw[i] / J[i];
-            vg[i] = ug[i] * qd;
-         }
-         break;
-      case 21:
-         for (CeedInt i = 0; i < Q; i++)
-         {
-            // J: 0 2   qd: 0 1   adj(J):  J22 -J12
-            //    1 3       1 2           -J21  J11
-            const CeedScalar J11 = J[i + Q * 0];
-            const CeedScalar J21 = J[i + Q * 1];
-            const CeedScalar J12 = J[i + Q * 2];
-            const CeedScalar J22 = J[i + Q * 3];
-            const CeedScalar w = qw[i] / (J11 * J22 - J21 * J12);
-            CeedScalar qd[3];
-            const CeedScalar coeff = c[i];
-            qd[0] =   coeff * w * (J12 * J12 + J22 * J22);
-            qd[1] = - coeff * w * (J11 * J12 + J21 * J22);
-            qd[2] =   coeff * w * (J11 * J11 + J21 * J21);
-            const CeedScalar ug0 = ug[i + Q * 0];
-            const CeedScalar ug1 = ug[i + Q * 1];
-            vg[i + Q * 0] = qd[0] * ug0 + qd[1] * ug1;
-            vg[i + Q * 1] = qd[1] * ug0 + qd[2] * ug1;
-         }
-         break;
-      case 22:
-         for (CeedInt i = 0; i < Q; i++)
-         {
-            // J: 0 2   qd: 0 1   adj(J):  J22 -J12
-            //    1 3       1 2           -J21  J11
-            const CeedScalar J11 = J[i + Q * 0];
-            const CeedScalar J21 = J[i + Q * 1];
-            const CeedScalar J12 = J[i + Q * 2];
-            const CeedScalar J22 = J[i + Q * 3];
-            const CeedScalar w = qw[i] / (J11 * J22 - J21 * J12);
-            CeedScalar qd[3];
-            const CeedScalar coeff = c[i];
-            qd[0] =   coeff * w * (J12 * J12 + J22 * J22);
-            qd[1] = - coeff * w * (J11 * J12 + J21 * J22);
-            qd[2] =   coeff * w * (J11 * J11 + J21 * J21);
-            for (CeedInt c = 0; c < 2; c++)
-            {
-               const CeedScalar ug0 = ug[i + Q * (c+2*0)];
-               const CeedScalar ug1 = ug[i + Q * (c+2*1)];
-               vg[i + Q * (c+2*0)] = qd[0] * ug0 + qd[1] * ug1;
-               vg[i + Q * (c+2*1)] = qd[1] * ug0 + qd[2] * ug1;
-            }
-         }
-         break;
-      case 31:
-         for (CeedInt i = 0; i < Q; i++)
-         {
-            // J: 0 3 6   qd: 0 1 2
-            //    1 4 7       1 3 4
-            //    2 5 8       2 4 5
-            const CeedScalar J11 = J[i + Q * 0];
-            const CeedScalar J21 = J[i + Q * 1];
-            const CeedScalar J31 = J[i + Q * 2];
-            const CeedScalar J12 = J[i + Q * 3];
-            const CeedScalar J22 = J[i + Q * 4];
-            const CeedScalar J32 = J[i + Q * 5];
-            const CeedScalar J13 = J[i + Q * 6];
-            const CeedScalar J23 = J[i + Q * 7];
-            const CeedScalar J33 = J[i + Q * 8];
-            const CeedScalar A11 = J22 * J33 - J23 * J32;
-            const CeedScalar A12 = J13 * J32 - J12 * J33;
-            const CeedScalar A13 = J12 * J23 - J13 * J22;
-            const CeedScalar A21 = J23 * J31 - J21 * J33;
-            const CeedScalar A22 = J11 * J33 - J13 * J31;
-            const CeedScalar A23 = J13 * J21 - J11 * J23;
-            const CeedScalar A31 = J21 * J32 - J22 * J31;
-            const CeedScalar A32 = J12 * J31 - J11 * J32;
-            const CeedScalar A33 = J11 * J22 - J12 * J21;
-            const CeedScalar w = qw[i] / (J11 * A11 + J21 * A12 + J31 * A13);
-            CeedScalar qd[6];
-            const CeedScalar coeff = c[i];
-            qd[0] = coeff * w * (A11 * A11 + A12 * A12 + A13 * A13);
-            qd[1] = coeff * w * (A11 * A21 + A12 * A22 + A13 * A23);
-            qd[2] = coeff * w * (A11 * A31 + A12 * A32 + A13 * A33);
-            qd[3] = coeff * w * (A21 * A21 + A22 * A22 + A23 * A23);
-            qd[4] = coeff * w * (A21 * A31 + A22 * A32 + A23 * A33);
-            qd[5] = coeff * w * (A31 * A31 + A32 * A32 + A33 * A33);
-            const CeedScalar ug0 = ug[i + Q * 0];
-            const CeedScalar ug1 = ug[i + Q * 1];
-            const CeedScalar ug2 = ug[i + Q * 2];
-            vg[i + Q * 0] = qd[0] * ug0 + qd[1] * ug1 + qd[2] * ug2;
-            vg[i + Q * 1] = qd[1] * ug0 + qd[3] * ug1 + qd[4] * ug2;
-            vg[i + Q * 2] = qd[2] * ug0 + qd[4] * ug1 + qd[5] * ug2;
-         }
-         break;
-      case 33:
-         for (CeedInt i = 0; i < Q; i++)
-         {
-            // J: 0 3 6   qd: 0 1 2
-            //    1 4 7       1 3 4
-            //    2 5 8       2 4 5
-            const CeedScalar J11 = J[i + Q * 0];
-            const CeedScalar J21 = J[i + Q * 1];
-            const CeedScalar J31 = J[i + Q * 2];
-            const CeedScalar J12 = J[i + Q * 3];
-            const CeedScalar J22 = J[i + Q * 4];
-            const CeedScalar J32 = J[i + Q * 5];
-            const CeedScalar J13 = J[i + Q * 6];
-            const CeedScalar J23 = J[i + Q * 7];
-            const CeedScalar J33 = J[i + Q * 8];
-            const CeedScalar A11 = J22 * J33 - J23 * J32;
-            const CeedScalar A12 = J13 * J32 - J12 * J33;
-            const CeedScalar A13 = J12 * J23 - J13 * J22;
-            const CeedScalar A21 = J23 * J31 - J21 * J33;
-            const CeedScalar A22 = J11 * J33 - J13 * J31;
-            const CeedScalar A23 = J13 * J21 - J11 * J23;
-            const CeedScalar A31 = J21 * J32 - J22 * J31;
-            const CeedScalar A32 = J12 * J31 - J11 * J32;
-            const CeedScalar A33 = J11 * J22 - J12 * J21;
-            const CeedScalar w = qw[i] / (J11 * A11 + J21 * A12 + J31 * A13);
-            CeedScalar qd[6];
-            const CeedScalar coeff = c[i];
-            qd[0] = coeff * w * (A11 * A11 + A12 * A12 + A13 * A13);
-            qd[1] = coeff * w * (A11 * A21 + A12 * A22 + A13 * A23);
-            qd[2] = coeff * w * (A11 * A31 + A12 * A32 + A13 * A33);
-            qd[3] = coeff * w * (A21 * A21 + A22 * A22 + A23 * A23);
-            qd[4] = coeff * w * (A21 * A31 + A22 * A32 + A23 * A33);
-            qd[5] = coeff * w * (A31 * A31 + A32 * A32 + A33 * A33);
-            for (CeedInt c = 0; c < 3; c++)
-            {
-               const CeedScalar ug0 = ug[i + Q * (c+3*0)];
-               const CeedScalar ug1 = ug[i + Q * (c+3*1)];
-               const CeedScalar ug2 = ug[i + Q * (c+3*2)];
-               vg[i + Q * (c+3*0)] = qd[0] * ug0 + qd[1] * ug1 + qd[2] * ug2;
-               vg[i + Q * (c+3*1)] = qd[1] * ug0 + qd[3] * ug1 + qd[4] * ug2;
-               vg[i + Q * (c+3*2)] = qd[2] * ug0 + qd[4] * ug1 + qd[5] * ug2;
-            }
-         }
-         break;
    }
    return 0;
 }

fem/libceed/diffusion.hpp

@@ -14,6 +14,7 @@
 
 #include "ceed.hpp"
 
+#ifdef MFEM_USE_CEED
 #include "../fespace.hpp"
 
 namespace mfem
@@ -24,10 +25,8 @@ void CeedPADiffusionAssemble(const FiniteElementSpace &fes,
                              const mfem::IntegrationRule &ir,
                              CeedData& ceedData);
 
-void CeedMFDiffusionAssemble(const FiniteElementSpace &fes,
-                             const mfem::IntegrationRule &ir,
-                             CeedData& ceedData);
-
 }
 
+#endif // MFEM_USE_CEED
+
 #endif // MFEM_LIBCEED_DIFF_HPP

fem/libceed/mass.cpp

@@ -11,10 +11,9 @@
 
 #include "mass.hpp"
 
-#include "ceed.hpp"
 #ifdef MFEM_USE_CEED
+#include "ceed.hpp"
 #include "mass.h"
-#endif
 
 namespace mfem
 {
@@ -22,41 +21,17 @@ namespace mfem
 void CeedPAMassAssemble(const FiniteElementSpace &fes,
                         const mfem::IntegrationRule &irm, CeedData& ceedData)
 {
-#ifdef MFEM_USE_CEED
-   Mesh &mesh = *fes.GetMesh();
-   // Perform checks for some assumptions made in the Q-functions.
-   MFEM_VERIFY(mesh.Dimension() == mesh.SpaceDimension(), "case not supported");
-   MFEM_VERIFY(1 <= fes.GetVDim() && fes.GetVDim() <= 3, "case not supported");
    CeedPAOperator massOp = {fes, irm,
                             1, "/mass.h",
                             ":f_build_mass_const", f_build_mass_const,
-                            ":f_build_mass_quad", f_build_mass_quad,
+                            ":f_build_mass_grid", f_build_mass_grid,
                             ":f_apply_mass", f_apply_mass,
                             CEED_EVAL_INTERP,
                             CEED_EVAL_INTERP
                            };
    CeedPAAssemble(massOp, ceedData);
-#else
-   mfem_error("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
-void CeedMFMassAssemble(const FiniteElementSpace &fes,
-                        const mfem::IntegrationRule &irm, CeedData& ceedData)
-{
-#ifdef MFEM_USE_CEED
-   CeedMFOperator massOp = {fes, irm,
-                            "/mass.h",
-                            ":f_apply_mass_mf_const", f_apply_mass_mf_const,
-                            ":f_apply_mass_mf_quad", f_apply_mass_mf_quad,
-                            CEED_EVAL_INTERP,
-                            CEED_EVAL_INTERP
-                           };
-   CeedMFAssemble(massOp, ceedData);
-#else
-   mfem_error("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
 }
 
 } // namespace mfem
 
+#endif // MFEM_USE_CEED

fem/libceed/mass.h

@@ -10,7 +10,7 @@
 // CONTRIBUTING.md for details.
 
 /// A structure used to pass additional data to f_build_diff and f_apply_diff
-struct BuildContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; };
+struct BuildContext { CeedInt dim, space_dim; CeedScalar coeff; };
 
 /// libCEED Q-function for building quadrature data for a mass operator with a
 /// constant coefficient
@@ -56,8 +56,8 @@ CEED_QFUNCTION(f_build_mass_const)(void *ctx, CeedInt Q,
 }
 
 /// libCEED Q-function for building quadrature data for a mass operator with a
-/// coefficient evaluated at quadrature points.
-CEED_QFUNCTION(f_build_mass_quad)(void *ctx, CeedInt Q,
+/// grid function coefficient
+CEED_QFUNCTION(f_build_mass_grid)(void *ctx, CeedInt Q,
                                   const CeedScalar *const *in,
                                   CeedScalar *const *out)
 {
@@ -102,168 +102,11 @@ CEED_QFUNCTION(f_apply_mass)(void *ctx, CeedInt Q,
                              const CeedScalar *const *in,
                              CeedScalar *const *out)
 {
-   BuildContext *bc = (BuildContext *)ctx;
    const CeedScalar *u = in[0], *w = in[1];
    CeedScalar *v = out[0];
-   switch (bc->vdim)
+   for (CeedInt i=0; i<Q; i++)
    {
-      case 1:
-         for (CeedInt i=0; i<Q; i++)
-         {
-            v[i] = w[i] * u[i];
-         }
-         break;
-      case 2:
-         for (CeedInt i=0; i<Q; i++)
-         {
-            const CeedScalar W = w[i];
-            for (CeedInt c = 0; c < 2; c++)
-            {
-               v[i+c*Q] = W * u[i+c*Q];
-            }
-         }
-         break;
-      case 3:
-         for (CeedInt i=0; i<Q; i++)
-         {
-            const CeedScalar W = w[i];
-            for (CeedInt c = 0; c < 3; c++)
-            {
-               v[i+c*Q] = W * u[i+c*Q];
-            }
-         }
-         break;
-   }
-   return 0;
-}
-
-/// libCEED Q-function for applying a diff operator
-CEED_QFUNCTION(f_apply_mass_mf_const)(void *ctx, CeedInt Q,
-                                      const CeedScalar *const *in, CeedScalar *const *out)
-{
-   BuildContext *bc = (BuildContext*)ctx;
-   const CeedScalar coeff = bc->coeff;
-   const CeedScalar *u = in[0], *J = in[1], *qw = in[2];
-   CeedScalar *v = out[0];
-   switch (10 * bc->dim + bc->vdim)
-   {
-      case 11:
-         for (CeedInt i = 0; i < Q; i++)
-         {
-            const CeedScalar rho = coeff * qw[i] / J[i];
-            v[i] = rho * u[i];
-         }
-         break;
-      case 21:
-         for (CeedInt i = 0; i < Q; i++)
-         {
-            const CeedScalar rho = coeff * (J[i+Q*0]*J[i+Q*3] - J[i+Q*1]*J[i+Q*2]) * qw[i];
-            v[i] = rho * u[i];
-         }
-         break;
-      case 22:
-         for (CeedInt i=0; i<Q; i++)
-         {
-            // 0 2
-            // 1 3
-            const CeedScalar rho = coeff * (J[i+Q*0]*J[i+Q*3] - J[i+Q*1]*J[i+Q*2]) * qw[i];
-            for (CeedInt c = 0; c < 2; c++)
-            {
-               v[i+c*Q] = rho * u[i+c*Q];
-            }
-         }
-         break;
-      case 31:
-         for (CeedInt i = 0; i < Q; i++)
-         {
-            const CeedScalar rho = (J[i+Q*0]*(J[i+Q*4]*J[i+Q*8] - J[i+Q*5]*J[i+Q*7]) -
-                                    J[i+Q*1]*(J[i+Q*3]*J[i+Q*8] - J[i+Q*5]*J[i+Q*6]) +
-                                    J[i+Q*2]*(J[i+Q*3]*J[i+Q*7] - J[i+Q*4]*J[i+Q*6])) * coeff * qw[i];
-            v[i] = rho * u[i];
-         }
-         break;
-      case 33:
-         for (CeedInt i=0; i<Q; i++)
-         {
-            // 0 3 6
-            // 1 4 7
-            // 2 5 8
-            const CeedScalar rho = (J[i+Q*0]*(J[i+Q*4]*J[i+Q*8] - J[i+Q*5]*J[i+Q*7]) -
-                                    J[i+Q*1]*(J[i+Q*3]*J[i+Q*8] - J[i+Q*5]*J[i+Q*6]) +
-                                    J[i+Q*2]*(J[i+Q*3]*J[i+Q*7] - J[i+Q*4]*J[i+Q*6])) * coeff * qw[i];
-            for (CeedInt c = 0; c < 3; c++)
-            {
-               v[i+c*Q] = rho * u[i+c*Q];
-            }
-         }
-         break;
-   }
-   return 0;
-}
-
-CEED_QFUNCTION(f_apply_mass_mf_quad)(void *ctx, CeedInt Q,
-                                     const CeedScalar *const *in, CeedScalar *const *out)
-{
-   BuildContext *bc = (BuildContext*)ctx;
-   const CeedScalar *c = in[0], *u = in[1], *J = in[2], *qw = in[3];
-   CeedScalar *v = out[0];
-   switch (10 * bc->dim + bc->vdim)
-   {
-      case 11:
-         for (CeedInt i=0; i<Q; i++)
-         {
-            const CeedScalar rho = c[i] * J[i] * qw[i];
-            v[i] = rho * u[i];
-         }
-         break;
-      case 21:
-         for (CeedInt i=0; i<Q; i++)
-         {
-            // 0 2
-            // 1 3
-            const CeedScalar rho = c[i] * (J[i+Q*0]*J[i+Q*3] - J[i+Q*1]*J[i+Q*2]) * qw[i];
-            v[i] = rho * u[i];
-         }
-         break;
-      case 22:
-         for (CeedInt i=0; i<Q; i++)
-         {
-            // 0 2
-            // 1 3
-            const CeedScalar rho = c[i] * (J[i+Q*0]*J[i+Q*3] - J[i+Q*1]*J[i+Q*2]) * qw[i];
-            for (CeedInt c = 0; c < 2; c++)
-            {
-               v[i+c*Q] = rho * u[i+c*Q];
-            }
-         }
-         break;
-      case 31:
-         for (CeedInt i=0; i<Q; i++)
-         {
-            // 0 3 6
-            // 1 4 7
-            // 2 5 8
-            const CeedScalar rho = (J[i+Q*0]*(J[i+Q*4]*J[i+Q*8] - J[i+Q*5]*J[i+Q*7]) -
-                                    J[i+Q*1]*(J[i+Q*3]*J[i+Q*8] - J[i+Q*5]*J[i+Q*6]) +
-                                    J[i+Q*2]*(J[i+Q*3]*J[i+Q*7] - J[i+Q*4]*J[i+Q*6])) * c[i] * qw[i];
-            v[i] = rho * u[i];
-         }
-         break;
-      case 33:
-         for (CeedInt i=0; i<Q; i++)
-         {
-            // 0 3 6
-            // 1 4 7
-            // 2 5 8
-            const CeedScalar rho = (J[i+Q*0]*(J[i+Q*4]*J[i+Q*8] - J[i+Q*5]*J[i+Q*7]) -
-                                    J[i+Q*1]*(J[i+Q*3]*J[i+Q*8] - J[i+Q*5]*J[i+Q*6]) +
-                                    J[i+Q*2]*(J[i+Q*3]*J[i+Q*7] - J[i+Q*4]*J[i+Q*6])) * c[i] * qw[i];
-            for (CeedInt c = 0; c < 3; c++)
-            {
-               v[i+c*Q] = rho * u[i+c*Q];
-            }
-         }
-         break;
+      v[i] = w[i] * u[i];
    }
    return 0;
 }

fem/libceed/mass.hpp

@@ -14,6 +14,7 @@
 
 #include "ceed.hpp"
 
+#ifdef MFEM_USE_CEED
 #include "../fespace.hpp"
 
 namespace mfem
@@ -24,10 +25,8 @@ void CeedPAMassAssemble(const FiniteElementSpace &fes,
                         const mfem::IntegrationRule &ir,
                         CeedData& ceedData);
 
-void CeedMFMassAssemble(const FiniteElementSpace &fes,
-                        const mfem::IntegrationRule &irm,
-                        CeedData& ceedData);
-
 }
 
+#endif // MFEM_USE_CEED
+
 #endif // MFEM_LIBCEED_MASS_HPP

fem/nonlinearform.cpp

@@ -131,8 +131,8 @@ void NonlinearForm::Mult(const Vector &x, Vector &y) const
    const Vector &px = Prolongate(x);
    if (P) { aux2.SetSize(P->Height()); }
 
-   // If we are in parallel, ParNonLinearForm::Mult uses the aux2 vector. In
-   // serial, place the result directly in y.
+   // If we are in parallel, ParNonLinearForm::Mult uses the aux2 vector.
+   // In serial, place the result directly in y.
    Vector &py = P ? aux2 : y;
 
    if (ext)
@@ -456,12 +456,11 @@ void BlockNonlinearForm::SetSpaces(Array<FiniteElementSpace *> &f)
       for (int j=0; j<Grads.NumCols(); ++j)
       {
          delete Grads(i,j);
-         delete cGrads(i,j);
       }
    }
-   for (int i = 0; i < ess_tdofs.Size(); ++i)
+   for (int i = 0; i < ess_vdofs.Size(); ++i)
    {
-      delete ess_tdofs[i];
+      delete ess_vdofs[i];
    }
 
    height = 0;
@@ -487,33 +486,10 @@ void BlockNonlinearForm::SetSpaces(Array<FiniteElementSpace *> &f)
    Grads.SetSize(fes.Size(), fes.Size());
    Grads = NULL;
 
-   cGrads.SetSize(fes.Size(), fes.Size());
-   cGrads = NULL;
-
-   P.SetSize(fes.Size());
-   cP.SetSize(fes.Size());
-   ess_tdofs.SetSize(fes.Size());
+   ess_vdofs.SetSize(fes.Size());
    for (int s = 0; s < fes.Size(); ++s)
    {
-      // Retrieve prolongation matrix for each FE space
-      P[s] = fes[s]->GetProlongationMatrix();
-      cP[s] = dynamic_cast<const SparseMatrix *>(P[s]);
-
-      // If the P Operator exists and its type is not SparseMatrix, this
-      // indicates the Operator is part of parallel run.
-      if (P[s] && !cP[s])
-      {
-         is_serial = false;
-      }
-
-      // If the P Operator exists and its type is SparseMatrix, this indicates
-      // the Operator is serial but needs prolongation on assembly.
-      if (cP[s])
-      {
-         needs_prolongation = true;
-      }
-
-      ess_tdofs[s] = new Array<int>;
+      ess_vdofs[s] = new Array<int>;
    }
 }
 
@@ -530,18 +506,45 @@ void BlockNonlinearForm::AddBdrFaceIntegrator(BlockNonlinearFormIntegrator *nfi,
    bfnfi_marker.Append(&bdr_attr_marker);
 }
 
-void BlockNonlinearForm::SetEssentialBC(
-   const Array<Array<int> *> &bdr_attr_is_ess, Array<Vector *> &rhs)
+void BlockNonlinearForm::SetEssentialBC(const
+                                        Array<Array<int> *>&bdr_attr_is_ess,
+                                        Array<Vector *> &rhs)
 {
-   for (int s = 0; s < fes.Size(); ++s)
-   {
-      ess_tdofs[s]->SetSize(ess_tdofs.Size());
+   int i, j, vsize, nv;
 
-      fes[s]->GetEssentialTrueDofs(*bdr_attr_is_ess[s], *ess_tdofs[s]);
+   for (int s=0; s<fes.Size(); ++s)
+   {
+      // First, set u variables
+      vsize = fes[s]->GetVSize();
+      Array<int> vdof_marker(vsize);
+
+      // virtual call, works in parallel too
+      fes[s]->GetEssentialVDofs(*(bdr_attr_is_ess[s]), vdof_marker);
+      nv = 0;
+      for (i = 0; i < vsize; ++i)
+      {
+         if (vdof_marker[i])
+         {
+            nv++;
+         }
+      }
+
+      ess_vdofs[s]->SetSize(nv);
+
+      for (i = j = 0; i < vsize; ++i)
+      {
+         if (vdof_marker[i])
+         {
+            (*ess_vdofs[s])[j++] = i;
+         }
+      }
 
       if (rhs[s])
       {
-         rhs[s]->SetSubVector(*ess_tdofs[s], 0.0);
+         for (i = 0; i < nv; ++i)
+         {
+            (*rhs[s])[(*ess_vdofs[s])[i]] = 0.0;
+         }
       }
    }
 }
@@ -754,52 +757,18 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,
       delete vdofs[s];
       delete el_y[s];
       delete el_x[s];
+      by.GetBlock(s).SetSubVector(*ess_vdofs[s], 0.0);
    }
 }
 
-const BlockVector &BlockNonlinearForm::Prolongate(const BlockVector &bx) const
-{
-   MFEM_VERIFY(bx.Size() == Width(), "invalid input BlockVector size");
-
-   if (needs_prolongation)
-   {
-      aux1.Update(block_offsets);
-      for (int s = 0; s < fes.Size(); s++)
-      {
-         P[s]->Mult(bx.GetBlock(s), aux1.GetBlock(s));
-      }
-      return aux1;
-   }
-   return bx;
-}
-
 void BlockNonlinearForm::Mult(const Vector &x, Vector &y) const
 {
-   BlockVector bx(x.GetData(), block_trueOffsets);
-   BlockVector by(y.GetData(), block_trueOffsets);
-
-   const BlockVector &pbx = Prolongate(bx);
-   if (needs_prolongation)
-   {
-      aux2.Update(block_offsets);
-   }
-   BlockVector &pby = needs_prolongation ? aux2 : by;
-
-   xs.Update(pbx.GetData(), block_offsets);
-   ys.Update(pby.GetData(), block_offsets);
+   xs.Update(x.GetData(), block_offsets);
+   ys.Update(y.GetData(), block_offsets);
    MultBlocked(xs, ys);
-
-   for (int s = 0; s < fes.Size(); s++)
-   {
-      if (cP[s])
-      {
-         cP[s]->MultTranspose(pby.GetBlock(s), by.GetBlock(s));
-      }
-      by.GetBlock(s).SetSubVector(*ess_tdofs[s], 0.0);
-   }
 }
 
-void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
+Operator &BlockNonlinearForm::GetGradientBlocked(const BlockVector &bx) const
 {
    const int skip_zeros = 0;
    Array<Array<int> *> vdofs(fes.Size());
@@ -811,6 +780,13 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
    Array<const FiniteElement *>fe2(fes.Size());
    ElementTransformation * T;
 
+   if (BlockGrad != NULL)
+   {
+      delete BlockGrad;
+   }
+
+   BlockGrad = new BlockOperator(block_offsets);
+
    for (int i=0; i<fes.Size(); ++i)
    {
       el_x_const[i] = el_x[i] = new Vector();
@@ -975,70 +951,44 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
       }
    }
 
+   for (int s=0; s<fes.Size(); ++s)
+   {
+      for (int i = 0; i < ess_vdofs[s]->Size(); ++i)
+      {
+         for (int j=0; j<fes.Size(); ++j)
+         {
+            if (s==j)
+            {
+               Grads(s,s)->EliminateRowCol((*ess_vdofs[s])[i], Matrix::DIAG_ONE);
+            }
+            else
+            {
+               Grads(s,j)->EliminateRow((*ess_vdofs[s])[i]);
+               Grads(j,s)->EliminateCol((*ess_vdofs[s])[i]);
+            }
+         }
+      }
+   }
+
    for (int i=0; i<fes.Size(); ++i)
    {
       for (int j=0; j<fes.Size(); ++j)
       {
+         BlockGrad->SetBlock(i,j,Grads(i,j));
          delete elmats(i,j);
       }
       delete vdofs2[i];
       delete vdofs[i];
       delete el_x[i];
    }
+
+   return *BlockGrad;
 }
 
 Operator &BlockNonlinearForm::GetGradient(const Vector &x) const
 {
-   BlockVector bx(x.GetData(), block_trueOffsets);
-   const BlockVector &pbx = Prolongate(bx);
-
-   ComputeGradientBlocked(pbx);
-
-   Array2D<SparseMatrix *> mGrads(fes.Size(), fes.Size());
-   mGrads = Grads;
-   if (needs_prolongation)
-   {
-      for (int s1 = 0; s1 < fes.Size(); ++s1)
-      {
-         for (int s2 = 0; s2 < fes.Size(); ++s2)
-         {
-            delete cGrads(s1, s2);
-            cGrads(s1, s2) = RAP(*cP[s1], *Grads(s1, s2), *cP[s2]);
-            mGrads(s1, s2) = cGrads(s1, s2);
-         }
-      }
-   }
-
-   for (int s = 0; s < fes.Size(); ++s)
-   {
-      for (int i = 0; i < ess_tdofs[s]->Size(); ++i)
-      {
-         for (int j = 0; j < fes.Size(); ++j)
-         {
-            if (s == j)
-            {
-               mGrads(s, s)->EliminateRowCol((*ess_tdofs[s])[i],
-                                             Matrix::DIAG_ONE);
-            }
-            else
-            {
-               mGrads(s, j)->EliminateRow((*ess_tdofs[s])[i]);
-               mGrads(j, s)->EliminateCol((*ess_tdofs[s])[i]);
-            }
-         }
-      }
-   }
-
-   delete BlockGrad;
-   BlockGrad = new BlockOperator(block_trueOffsets);
-   for (int i = 0; i < fes.Size(); ++i)
-   {
-      for (int j = 0; j < fes.Size(); ++j)
-      {
-         BlockGrad->SetBlock(i, j, mGrads(i, j));
-      }
-   }
-   return *BlockGrad;
+   xs.Update(x.GetData(), block_offsets);
+   return GetGradientBlocked(xs);
 }
 
 BlockNonlinearForm::~BlockNonlinearForm()
@@ -1049,9 +999,8 @@ BlockNonlinearForm::~BlockNonlinearForm()
       for (int j=0; j<fes.Size(); ++j)
       {
          delete Grads(i,j);
-         delete cGrads(i,j);
       }
-      delete ess_tdofs[i];
+      delete ess_vdofs[i];
    }
 
    for (int i = 0; i < dnfi.Size(); ++i)

fem/nonlinearform.hpp

@@ -199,41 +199,24 @@ protected:
        GridFunction-like block-vector data (e.g. in parallel). */
    mutable BlockVector xs, ys;
 
-   mutable Array2D<SparseMatrix*> Grads, cGrads;
+   mutable Array2D<SparseMatrix*> Grads;
    mutable BlockOperator *BlockGrad;
 
    // A list of the offsets
    Array<int> block_offsets;
    Array<int> block_trueOffsets;
 
-   // Array of Arrays of tdofs for each space in 'fes'
-   Array<Array<int> *> ess_tdofs;
-
-   /// Array of pointers to the prolongation matrix of fes, may be NULL
-   Array<const Operator *> P;
-
-   /// Array of results of dynamic-casting P to SparseMatrix pointer
-   Array<const SparseMatrix *> cP;
-
-   /// Indicator if the Operator is part of a parallel run
-   bool is_serial = true;
-
-   /// Indicator if the Operator needs prolongation on assembly
-   bool needs_prolongation = false;
-
-   mutable BlockVector aux1, aux2;
-
-   const BlockVector &Prolongate(const BlockVector &bx) const;
+   // Essential vdofs: one list of vdofs for each space in 'fes'
+   Array<Array<int> *> ess_vdofs;
 
    /// Specialized version of GetEnergy() for BlockVectors
    double GetEnergyBlocked(const BlockVector &bx) const;
 
    /// Specialized version of Mult() for BlockVector%s
-   /// Block L-Vector to Block L-Vector
    void MultBlocked(const BlockVector &bx, BlockVector &by) const;
 
    /// Specialized version of GetGradient() for BlockVector
-   void ComputeGradientBlocked(const BlockVector &bx) const;
+   Operator &GetGradientBlocked(const BlockVector &bx) const;
 
 public:
    /// Construct an empty BlockNonlinearForm. Initialize with SetSpaces().
@@ -278,12 +261,8 @@ public:
 
    virtual double GetEnergy(const Vector &x) const;
 
-   /// Method is only called in serial, the parallel version calls MultBlocked
-   /// directly.
    virtual void Mult(const Vector &x, Vector &y) const;
 
-   /// Method is only called in serial, the parallel version calls
-   /// GetGradientBlocked directly.
    virtual Operator &GetGradient(const Vector &x) const;
 
    /// Destructor.

fem/pbilinearform.cpp

@@ -565,6 +565,53 @@ HypreParMatrix* ParDiscreteLinearOperator::ParallelAssemble() const
    return RAP;
 }
 
+void ParDiscreteLinearOperator::ParallelAssemble(OperatorHandle &A)
+{
+   // construct the rectangular block-diagonal matrix dA
+   OperatorHandle dA(A.Type());
+   dA.MakeRectangularBlockDiag(domain_fes->GetComm(),
+                               range_fes->GlobalVSize(),
+                               domain_fes->GlobalVSize(),
+                               range_fes->GetDofOffsets(),
+                               domain_fes->GetDofOffsets(),
+                               mat);
+
+   OperatorHandle P_test(A.Type()), P_trial(A.Type());
+
+   // TODO - construct the Dof_TrueDof_Matrix directly in the required format.
+   P_test.ConvertFrom(range_fes->Dof_TrueDof_Matrix());
+   P_trial.ConvertFrom(domain_fes->Dof_TrueDof_Matrix());
+
+   A.MakeRAP(P_test, dA, P_trial);
+}
+
+//// @todo copied from ParMixedBilinearForm, should be some inheritance?
+void ParDiscreteLinearOperator::FormRectangularSystemMatrix(OperatorHandle &A)
+{
+   if (ext)
+   {
+      Array<int> empty;
+      ext->FormRectangularSystemOperator(empty, empty, A);
+      return;
+   }
+
+   /*
+   if (mat)
+   {
+      Finalize();
+      ParallelAssemble(p_mat);
+      delete mat;
+      mat = NULL;
+      delete mat_e;
+      mat_e = NULL;
+      p_mat_e = NULL;
+   }
+   A = p_mat;
+   */
+
+   mfem_error("not implemented!");
+}
+
 void ParDiscreteLinearOperator::GetParBlocks(Array2D<HypreParMatrix *> &blocks)
 const
 {

fem/pbilinearform.hpp

@@ -76,7 +76,7 @@ public:
    void KeepNbrBlock(bool knb = true) { keep_nbr_block = knb; }
 
    /** @brief Set the operator type id for the parallel matrix/operator when
-       using AssemblyLevel::LEGACYFULL. */
+       using AssemblyLevel::FULL. */
    /** If using static condensation or hybridization, call this method *after*
        enabling it. */
    void SetOperatorType(Operator::Type tid)
@@ -160,6 +160,9 @@ public:
    /// Get the parallel finite element space prolongation matrix
    virtual const Operator *GetProlongation() const
    { return pfes->GetProlongationMatrix(); }
+   /// Get the diagonal part of the parallel finite element space prolongation matrix
+   virtual const Operator *GetLocalProlongation() const
+   { return pfes->GetLocalProlongationMatrix(); }
    /// Get the parallel finite element space restriction matrix
    virtual const Operator *GetRestriction() const
    { return pfes->GetRestrictionMatrix(); }
@@ -301,10 +304,18 @@ public:
    /// Returns the matrix "assembled" on the true dofs
    HypreParMatrix *ParallelAssemble() const;
 
+   /** @brief Returns the matrix assembled on the true dofs, i.e.
+       @a A = P_test^t A_local P_trial, in the format (type id) specified by
+       @a A. */
+   void ParallelAssemble(OperatorHandle &A);
+
    /** Extract the parallel blocks corresponding to the vector dimensions of the
        domain and range parallel finite element spaces */
    void GetParBlocks(Array2D<HypreParMatrix *> &blocks) const;
 
+   /** @brief Return in @a A a parallel (on truedofs) version of this operator. */
+   virtual void FormRectangularSystemMatrix(OperatorHandle &A);
+
    virtual ~ParDiscreteLinearOperator() { }
 };
 

fem/pfespace.cpp

@@ -101,6 +101,7 @@ void ParFiniteElementSpace::ParInit(ParMesh *pm)
 
    P = NULL;
    Pconf = NULL;
+   Pconf_local = NULL;
    R = NULL;
 
    num_face_nbr_dofs = -1;
@@ -921,6 +922,39 @@ const Operator *ParFiniteElementSpace::GetProlongationMatrix() const
    }
 }
 
+const Operator *ParFiniteElementSpace::GetLocalProlongationMatrix() const
+{
+   if (Conforming())
+   {
+      if (Pconf_local) { return Pconf_local; }
+
+      if (NRanks == 1)
+      {
+         Pconf_local = new IdentityOperator(GetTrueVSize());
+      }
+      else
+      {
+         if (!Device::Allows(Backend::DEVICE_MASK))
+         {
+            Pconf_local = new ConformingProlongationOperator(*this, true);
+         }
+         else
+         {
+            // Pconf = new DeviceConformingProlongationOperator(*this);
+            mfem_error("Not implemented!");
+         }
+      }
+      return Pconf_local;
+   }
+   else
+   {
+      // return Dof_TrueDof_Matrix();
+      // just need diagonal portion, not too hard
+      mfem_error("Not implemented!");
+      return NULL;
+   }
+}
+
 void ParFiniteElementSpace::ExchangeFaceNbrData()
 {
    if (num_face_nbr_dofs >= 0) { return; }
@@ -1999,13 +2033,13 @@ int ParFiniteElementSpace
       for (int entity = 0; entity <= 2; entity++)
       {
          const NCMesh::NCList &list = pncmesh->GetNCList(entity);
-         if (!list.masters.Size()) { continue; }
+         if (!list.masters.size()) { continue; }
 
          IsoparametricTransformation T;
          DenseMatrix I;
 
          // process masters that we own or that affect our edges/faces
-         for (int mi = 0; mi < list.masters.Size(); mi++)
+         for (unsigned mi = 0; mi < list.masters.size(); mi++)
          {
             const NCMesh::Master &mf = list.masters[mi];
 
@@ -2036,7 +2070,7 @@ int ParFiniteElementSpace
                GetEntityDofs(entity, sf.index, slave_dofs, mf.Geom());
                if (!slave_dofs.Size()) { continue; }
 
-               list.OrientedPointMatrix(sf, T.GetPointMat());
+               sf.OrientedPointMatrix(T.GetPointMat());
                fe->GetLocalInterpolation(T, I);
 
                // make each slave DOF dependent on all master DOFs
@@ -2064,12 +2098,12 @@ int ParFiniteElementSpace
       {
          const NCMesh::NCList &list = pncmesh->GetNCList(entity);
 
-         int lsize[3] =
-         { list.conforming.Size(), list.masters.Size(), list.slaves.Size() };
+         std::size_t lsize[3] =
+         { list.conforming.size(), list.masters.size(), list.slaves.size() };
 
          for (int l = 0; l < 3; l++)
          {
-            for (int i = 0; i < lsize[l]; i++)
+            for (std::size_t i = 0; i < lsize[l]; i++)
             {
                const MeshId &id =
                   (l == 0) ? list.conforming[i] :
@@ -2834,6 +2868,7 @@ void ParFiniteElementSpace::Destroy()
 
    delete P; P = NULL;
    delete Pconf; Pconf = NULL;
+   delete Pconf_local; Pconf_local = NULL;
    delete R; R = NULL;
 
    delete gcomm; gcomm = NULL;
@@ -2959,12 +2994,12 @@ void ParFiniteElementSpace::Update(bool want_transform)
    }
 }
 
-
 ConformingProlongationOperator::ConformingProlongationOperator(
-   const ParFiniteElementSpace &pfes)
+   const ParFiniteElementSpace &pfes, bool local_)
    : Operator(pfes.GetVSize(), pfes.GetTrueVSize()),
      external_ldofs(),
-     gc(pfes.GroupComm())
+     gc(pfes.GroupComm()),
+     local(local_)
 {
    MFEM_VERIFY(pfes.Conforming(), "");
    const Table &group_ldof = gc.GroupLDofTable();
@@ -3013,7 +3048,14 @@ void ConformingProlongationOperator::Mult(const Vector &x, Vector &y) const
    const int m = external_ldofs.Size();
 
    const int in_layout = 2; // 2 - input is ltdofs array
-   gc.BcastBegin(const_cast<double*>(xdata), in_layout);
+   if (local)
+   {
+      y = 0.0;
+   }
+   else
+   {
+      gc.BcastBegin(const_cast<double*>(xdata), in_layout);
+   }
 
    int j = 0;
    for (int i = 0; i < m; i++)
@@ -3025,7 +3067,10 @@ void ConformingProlongationOperator::Mult(const Vector &x, Vector &y) const
    std::copy(xdata+j-m, xdata+Width(), ydata+j);
 
    const int out_layout = 0; // 0 - output is ldofs array
-   gc.BcastEnd(ydata, out_layout);
+   if (!local)
+   {
+      gc.BcastEnd(ydata, out_layout);
+   }
 }
 
 void ConformingProlongationOperator::MultTranspose(
@@ -3038,7 +3083,10 @@ void ConformingProlongationOperator::MultTranspose(
    double *ydata = y.HostWrite();
    const int m = external_ldofs.Size();
 
-   gc.ReduceBegin(xdata);
+   if (!local)
+   {
+      gc.ReduceBegin(xdata);
+   }
 
    int j = 0;
    for (int i = 0; i < m; i++)
@@ -3050,7 +3098,10 @@ void ConformingProlongationOperator::MultTranspose(
    std::copy(xdata+j, xdata+Height(), ydata+j-m);
 
    const int out_layout = 2; // 2 - output is an array on all ltdofs
-   gc.ReduceEnd<double>(ydata, out_layout, GroupCommunicator::Sum);
+   if (!local)
+   {
+      gc.ReduceEnd<double>(ydata, out_layout, GroupCommunicator::Sum);
+   }
 }
 
 DeviceConformingProlongationOperator::DeviceConformingProlongationOperator(

fem/pfespace.hpp

@@ -72,6 +72,9 @@ private:
    mutable HypreParMatrix *P;
    /// Optimized action-only prolongation operator for conforming meshes. Owned.
    mutable Operator *Pconf;
+   /// Optimized action-only diagonal prolongation operator on ldofs for
+   /// conforming meshes. Owned.
+   mutable Operator *Pconf_local;
 
    /// The (block-diagonal) matrix R (restriction of dof to true dof). Owned.
    mutable SparseMatrix *R;
@@ -335,6 +338,7 @@ public:
    HYPRE_Int GetMyTDofOffset() const;
 
    virtual const Operator *GetProlongationMatrix() const;
+   virtual const Operator *GetLocalProlongationMatrix() const;
    /// Get the R matrix which restricts a local dof vector to true dof vector.
    virtual const SparseMatrix *GetRestrictionMatrix() const
    { Dof_TrueDof_Matrix(); return R; }
@@ -389,9 +393,11 @@ class ConformingProlongationOperator : public Operator
 protected:
    Array<int> external_ldofs;
    const GroupCommunicator &gc;
+   bool local;
 
 public:
-   ConformingProlongationOperator(const ParFiniteElementSpace &pfes);
+   ConformingProlongationOperator(const ParFiniteElementSpace &pfes,
+                                  bool local_=false);
 
    virtual void Mult(const Vector &x, Vector &y) const;
 

fem/pgridfunc.cpp

@@ -1093,9 +1093,6 @@ double L2ZZErrorEstimator(BilinearFormIntegrator &flux_integrator,
 
    if (xfes->GetNE())
    {
-      MFEM_VERIFY(smooth_flux_fes.GetFE(0) != NULL,
-                  "Could not obtain FE of smooth flux space.");
-
       if (smooth_flux_fes.GetFE(0)->GetRangeType() == FiniteElement::SCALAR)
       {
          VectorMassIntegrator *vmass = new VectorMassIntegrator;