pmg in lh driver

Triangle rules from Witherden & Vincent (2015), tet rules d=0-13
from Witherden & Vincent, tet rules d=14-20 from Chuluunbaatar et al.
(2022). All rules have strictly positive weights and interior points,
replacing the legacy rules which use negative weights at several
orders and fall back to Grundmann-Moller (negative weights, high
point counts) for tets at d>=9.

CHANGELOG

@@ -8,6 +8,24 @@
                                https://mfem.org
 
 
+Version 4.10 (development)
+==========================
+
+Discretization improvements
+---------------------------
+- Added symmetric positive-weight quadrature rules for triangles and
+  tetrahedra (orders 0-20). These rules guarantee all-positive weights and
+  interior quadrature points, improving numerical stability over the previous
+  Grundmann-Moller rules. Selected via SimplexQuadrature::PositiveWeights
+  (now the default). The previous rules remain available as
+  SimplexQuadrature::Legacy.
+  Triangle rules: Witherden & Vincent, Comput. Math. Appl. 69(10):1232-1241,
+  2015.
+  Tet rules (d=1-13): Witherden & Vincent (ibid).
+  Tet rules (d=14-20): Chuluunbaatar et al., Comput. Math. Appl. 124:89-97,
+  2022.
+
+
 Version 4.9.1 (development)
 ===========================
 
@@ -16,17 +34,33 @@ Discretization improvements
 - Improved the gridfunction projection routines. Projections work for Scalar,
   Vector and VectorFE, also NURBS versions. Optionally different types of
   projections can be selected, default behaviour has not changed.
+- Added gridfunction projection methods for trace spaces, i.e., project 
+  coefficients on the mesh skeleton.   
+
+- Added methods to estimate function extremum using piecewise linear bounds +
+  recursive subdivision.
 
 Meshing improvements
 --------------------
 - Improved support for 1D NURBS meshes with variable order, including using
   the patches construct for 1D NURBS meshes.
 
+Linear and nonlinear solvers
+----------------------------
+- Added interface to MUMPS direct solver for complex-valued problems.
+  Its usage is demonstrated in ex25p. See http://mumps.enseeiht.fr/ for more details.
+  Supported versions >= 5.1.1.
+
 New and updated examples and miniapps
 -------------------------------------
 - Electromagnetics/lorentz miniapp has been updated to leverage the ParticleSet
   capability.
+- Added (Complex)PRefinementMultigrid solver option in DPG miniapps. 
 
+Linear and nonlinear solvers
+----------------------------
+- Added support for trace spaces in PRefinementTransferOperator. This is used in 
+  PRefinement multigrid methods for problems posed on trace spaces (see DPG miniapps)
 
 Version 4.9, released on Dec 11, 2025
 =====================================

CMakeLists.txt

@@ -76,7 +76,9 @@ set(XSDK_ENABLE_Fortran OFF)
 # Check if we need to enable C or Fortran.
 if (MFEM_USE_CONDUIT OR
     MFEM_USE_SIDRE OR
-    MFEM_USE_PETSC)
+    MFEM_USE_PETSC OR
+    MFEM_USE_MUMPS OR
+    MFEM_USE_COMPLEX_MUMPS)
    # This seems to be needed by:
    #  * find_package(BLAS REQUIRED) and
    #  * find_package(HDF5 REQUIRED) needed, in turn, by:
@@ -84,7 +86,7 @@ if (MFEM_USE_CONDUIT OR
    #  * find_package(PETSc REQUIRED)
    set(XSDK_ENABLE_C ON)
 endif()
-if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
+if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS OR MFEM_USE_COMPLEX_MUMPS)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
@@ -342,7 +344,7 @@ if (MFEM_USE_MPI)
     endif()
   endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU MUMPS PETSC SLEPC STRUMPACK PUMI)
+  set(PKGS_NEED_MPI SUPERLU MUMPS COMPLEX_MUMPS PETSC SLEPC STRUMPACK PUMI)
   foreach(PKG IN LISTS PKGS_NEED_MPI)
     if (MFEM_USE_${PKG})
       message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -415,7 +417,7 @@ if (MFEM_USE_SUPERLU)
 endif()
 
 # MUMPS can only be enabled in parallel
-if (MFEM_USE_MUMPS)
+if (MFEM_USE_MUMPS OR MFEM_USE_COMPLEX_MUMPS)
   if (MFEM_USE_MPI)
     find_package(MUMPS REQUIRED mumps_common pord)
     set(MFEM_MUMPS_VERSION ${MUMPS_VERSION})

config/cmake/MFEMConfig.cmake.in

@@ -34,6 +34,7 @@ set(MFEM_USE_SUNDIALS @MFEM_USE_SUNDIALS@)
 set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
 set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
+set(MFEM_USE_COMPLEX_MUMPS @MFEM_USE_COMPLEX_MUMPS@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
 set(MFEM_USE_AMGX @MFEM_USE_AMGX@)

config/cmake/config.hpp.in

@@ -103,6 +103,7 @@
 
 // Enable MFEM functionality based on the MUMPS library.
 #cmakedefine MFEM_USE_MUMPS
+#cmakedefine MFEM_USE_COMPLEX_MUMPS
 #cmakedefine MFEM_MUMPS_VERSION @MFEM_MUMPS_VERSION@
 
 // Enable MFEM functionality based on the STRUMPACK library.

config/cmake/modules/FindMUMPS.cmake

@@ -17,13 +17,31 @@
 
 include(MfemCmakeUtilities)
 
-# Toggle which precision of MUMPS to use depending on the precision of MFEM.
+# Decide headers/libs by MFEM precision
 if (MFEM_USE_DOUBLE)
-  set(_mumps_header dmumps_c.h)
-  set(_mumps_lib dmumps)
-elseif(MFEM_USE_SINGLE)
-  set(_mumps_header smumps_c.h)
-  set(_mumps_lib smumps)
+  set(_rmumps_header dmumps_c.h)
+  set(_rmumps_lib    dmumps)
+  set(_cmumps_header zmumps_c.h)
+  set(_cmumps_lib    zmumps)
+elseif (MFEM_USE_SINGLE)
+  set(_rmumps_header smumps_c.h)
+  set(_rmumps_lib    smumps)
+  set(_cmumps_header cmumps_c.h)
+  set(_cmumps_lib    cmumps)
+endif()
+
+# Choose which header/lib mfem_find_package should use as the "primary" one.
+# If both enabled, prefer the real one as primary (either is fine).
+if (MFEM_USE_MUMPS)
+  set(_mumps_header ${_rmumps_header})
+  set(_mumps_lib    ${_rmumps_lib})
+elseif (MFEM_USE_COMPLEX_MUMPS)
+  set(_mumps_header ${_cmumps_header})
+  set(_mumps_lib    ${_cmumps_lib})
+else()
+  # Should not happen in practice because FindMUMPS is only called when enabled,
+  set(_mumps_header ${_rmumps_header})
+  set(_mumps_lib    ${_rmumps_lib})
 endif()
 
 mfem_find_package(MUMPS MUMPS MUMPS_DIR
@@ -31,8 +49,35 @@ mfem_find_package(MUMPS MUMPS MUMPS_DIR
   "Paths to headers required by MUMPS."
   "Libraries required by MUMPS."
   ADD_COMPONENT mumps_common "include" ${_mumps_header} "lib" mumps_common
-  ADD_COMPONENT pord "include" ${_mumps_header} "lib" pord)
+  ADD_COMPONENT pord        "include" ${_mumps_header} "lib" pord)
 
+# If BOTH real and complex are enabled, ensure BOTH solver libs are linked.
+if (MUMPS_FOUND AND MFEM_USE_MUMPS AND MFEM_USE_COMPLEX_MUMPS)
+  # Find the "other" solver library and append it.
+  find_library(_mfem_other_mumps_solver
+    NAMES ${_cmumps_lib}   
+    HINTS ${MUMPS_DIR}
+    PATH_SUFFIXES lib lib64
+    NO_DEFAULT_PATH)
+
+  if (NOT _mfem_other_mumps_solver)
+    # Fall back to system search
+    find_library(_mfem_other_mumps_solver NAMES ${_cmumps_lib})
+  endif()
+
+  if (NOT _mfem_other_mumps_solver)
+    message(FATAL_ERROR
+      "MFEM_USE_MUMPS=ON and MFEM_USE_COMPLEX_MUMPS=ON, but could not find "
+      "the complex solver library '${_cmumps_lib}' in MUMPS_DIR='${MUMPS_DIR}'.")
+  endif()
+
+  # Put solver libs first (important for static link order)
+  # MUMPS_LIBRARIES contains the primary solver already + common + pord.
+  # We prepend the other solver.
+  list(INSERT MUMPS_LIBRARIES 0 ${_mfem_other_mumps_solver})
+endif()
+
+# Version detection 
 if (MUMPS_FOUND AND (NOT MUMPS_VERSION))
   try_run(MUMPS_VERSION_RUN_RESULT MUMPS_VERSION_COMPILE_RESULT
           ${CMAKE_CURRENT_BINARY_DIR}/config

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -871,13 +871,13 @@ function(mfem_export_mk_files)
       MFEM_USE_SINGLE MFEM_USE_DOUBLE MFEM_DEBUG MFEM_USE_EXCEPTIONS
       MFEM_USE_ZLIB MFEM_USE_LIBUNWIND MFEM_USE_LAPACK MFEM_THREAD_SAFE
       MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS
-      MFEM_USE_SUITESPARSE MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS
-      MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_MAGMA
-      MFEM_USE_GNUTLS MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC
-      MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS MFEM_USE_CONDUIT MFEM_USE_PUMI
-      MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA MFEM_USE_HIP MFEM_USE_RAJA
-      MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER MFEM_USE_UMPIRE MFEM_USE_SIMD
-      MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO
+      MFEM_USE_SUITESPARSE MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS 
+      MFEM_USE_COMPLEX_MUMPS MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX 
+      MFEM_USE_MAGMA MFEM_USE_GNUTLS MFEM_USE_NETCDF MFEM_USE_PETSC 
+      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS MFEM_USE_CONDUIT 
+      MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA MFEM_USE_HIP 
+      MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER MFEM_USE_UMPIRE 
+      MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO
       MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG
       MFEM_USE_TRIBOL MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME
       MFEM_USE_HDF5)

config/config.hpp

@@ -140,6 +140,9 @@ constexpr real_t operator""_r(unsigned long long v)
 #ifdef MFEM_USE_MUMPS
 #error Building with MUMPS (MFEM_USE_MUMPS=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
+#ifdef MFEM_USE_COMPLEX_MUMPS
+#error Building with COMPLEX_MUMPS (MFEM_USE_COMPLEX_MUMPS=YES) requires MPI (MFEM_USE_MPI=YES)
+#endif
 #ifdef MFEM_USE_STRUMPACK
 #error Building with STRUMPACK (MFEM_USE_STRUMPACK=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif

config/config.hpp.in

@@ -103,6 +103,7 @@
 
 // Enable MFEM functionality based on the MUMPS library.
 // #define MFEM_USE_MUMPS
+// #define MFEM_USE_COMPLEX_MUMPS
 // #define MFEM_MUMPS_VERSION @MFEM_MUMPS_VERSION@
 
 // Enable MFEM functionality based on the STRUMPACK library.

config/defaults.cmake

@@ -37,6 +37,7 @@ option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
 option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
 option(MFEM_USE_MUMPS "Enable MUMPS usage" OFF)
+option(MFEM_USE_COMPLEX_MUMPS "Enable COMPLEX_MUMPS usage" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
 option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
 option(MFEM_USE_AMGX "Enable AmgX usage" OFF)
@@ -152,7 +153,7 @@ set(SuperLUDist_REQUIRED_PACKAGES "MPI" "ParMETIS" "METIS"
 set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.5.0" CACHE PATH
     "Path to the MUMPS library.")
 # MUMPS may also depend on "OpenMP", depending on how it was compiled.
-set(MUMPS_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "ParMETIS" "METIS"
+set(MUMPS_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "METIS"
     "ScaLAPACK" "LAPACK" "BLAS" CACHE STRING
     "Additional packages required by MUMPS.")
 # If the MPI package does not find all required Fortran libraries:

config/defaults.mk

@@ -152,6 +152,7 @@ MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
 MFEM_USE_SUPERLU5      = NO
 MFEM_USE_MUMPS         = NO
+MFEM_USE_COMPLEX_MUMPS = NO
 MFEM_USE_STRUMPACK     = NO
 MFEM_USE_GINKGO        = NO
 MFEM_USE_AMGX          = NO
@@ -248,7 +249,7 @@ ifeq (YES,$(MFEM_USE_HIP))
 endif
 
 # METIS library configuration
-ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
+ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS)$(MFEM_USE_COMPLEX_MUMPS),NONONONO)
    ifeq ($(MFEM_USE_METIS_5),NO)
      METIS_DIR = @MFEM_DIR@/../metis-4.0
      METIS_OPT =
@@ -352,13 +353,19 @@ MPI_FORTRAN_LIB = -lmpifort
 # MUMPS library configuration
 MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.5.0
 MUMPS_OPT = -I$(MUMPS_DIR)/include
-MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib
+COMPLEX_MUMPS_OPT = $(MUMPS_OPT)
+
+MUMPS_COMMON_PATH = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib
+MUMPS_COMMON_LIB = -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
 ifeq ($(MFEM_USE_SINGLE),YES)
-   MUMPS_LIB += -lsmumps
+  MUMPS_SOLVER_LIB         = -lsmumps
+  COMPLEX_MUMPS_SOLVER_LIB = -lcmumps
 else
-   MUMPS_LIB += -ldmumps
+  MUMPS_SOLVER_LIB         = -ldmumps
+  COMPLEX_MUMPS_SOLVER_LIB = -lzmumps
 endif
-MUMPS_LIB += -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
+MUMPS_LIB         = $(MUMPS_COMMON_PATH) $(MUMPS_SOLVER_LIB) $(MUMPS_COMMON_LIB)
+COMPLEX_MUMPS_LIB = $(MUMPS_COMMON_PATH) $(COMPLEX_MUMPS_SOLVER_LIB) $(MUMPS_COMMON_LIB)
 
 # STRUMPACK library configuration
 STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build

config/get_mumps_version.cpp

@@ -10,10 +10,18 @@
 // CONTRIBUTING.md for details.
 
 #ifdef MFEM_USE_SINGLE
+#ifdef MFEM_USE_COMPLEX_MUMPS
+#include "cmumps_c.h"
+#else
 #include "smumps_c.h"
+#endif
+#else
+#ifdef MFEM_USE_COMPLEX_MUMPS
+#include "zmumps_c.h"
 #else
 #include "dmumps_c.h"
 #endif
+#endif
 #include <string>
 #include <iostream>
 #include <algorithm>

config/makefile

@@ -42,7 +42,7 @@ GHV_FLAGS = $(MFEM_CXXFLAGS) $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..)
 SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)
 SMX_PATH = $(PUMI_DIR)/include/gmi_sim.h
 SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))
-MUMPS = $(MFEM_USE_MUMPS:NO=)
+MUMPS = $(if $(filter YES,$(MFEM_USE_MUMPS) $(MFEM_USE_COMPLEX_MUMPS)),YES,)
 GMV_CXX ?= $(MFEM_CXX)
 GMV = get_mumps_version
 GMV_FLAGS = $(MFEM_CXXFLAGS) $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(MUMPS_OPT))

examples/ex25p.cpp

@@ -200,7 +200,7 @@ int main(int argc, char *argv[])
    args.AddOption(&slu_solver, "-slu", "--superlu", "-no-slu",
                   "--no-superlu", "Use the SuperLU Solver.");
 #endif
-#ifdef MFEM_USE_MUMPS
+#if defined(MFEM_USE_MUMPS) || defined(MFEM_USE_COMPLEX_MUMPS)
    args.AddOption(&mumps_solver, "-mumps", "--mumps-solver", "-no-mumps",
                   "--no-mumps-solver", "Use the MUMPS Solver.");
 #endif
@@ -502,16 +502,24 @@ int main(int argc, char *argv[])
       delete A;
    }
 #endif
-#ifdef MFEM_USE_MUMPS
+#if defined(MFEM_USE_MUMPS) || defined(MFEM_USE_COMPLEX_MUMPS)
    if (!pa && mumps_solver)
    {
-      HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+      ComplexHypreParMatrix *Ahc = Ah.As<ComplexHypreParMatrix>();
+#ifdef MFEM_USE_COMPLEX_MUMPS
+      ComplexMUMPSSolver mumps(MPI_COMM_WORLD);
+      mumps.SetPrintLevel(0);
+      mumps.SetOperator(*Ahc);
+      mumps.Mult(B, X);
+#else
+      HypreParMatrix *A = Ahc->GetSystemMatrix();
       MUMPSSolver mumps(A->GetComm());
       mumps.SetPrintLevel(0);
       mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
       mumps.SetOperator(*A);
       mumps.Mult(B, X);
       delete A;
+#endif
    }
 #endif
    // 16a. Set up the parallel Bilinear form a(.,.) for the preconditioner

fem/bilininteg.cpp

@@ -2592,6 +2592,22 @@ void MixedCurlIntegrator::AssembleElementMatrix2(
       ir = &IntRules.Get(trial_fe.GetGeomType(), order);
    }
 
+   // Workspace for matrix-coefficient path
+   DenseMatrix M;
+   DenseMatrix Mcurl; // (dimc x trial_dof) when MQ is used
+
+   if (MQ)
+   {
+      // MQ only makes sense when curl(u) is vector-valued (case 1 or 3)
+      MFEM_VERIFY(dimc == dim,
+                  "MixedCurlIntegrator: MatrixCoefficient requires vector-valued curl(u) "
+                  "(3D H(curl) or 2D H1 rotated-grad case).");
+
+      M.SetSize(dimc);
+      Mcurl.SetSize(dimc, trial_dof);
+   }
+
+
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
@@ -2607,20 +2623,59 @@ void MixedCurlIntegrator::AssembleElementMatrix2(
       }
       test_fe.CalcPhysShape(Trans, shape);
       c = ip.weight*Trans.Weight();
-      if (Q)
+
+      if (MQ)
+      {
+         // apply matrix coefficient to curl(u)
+         MQ->Eval(M, Trans, ip);
+
+         for (int d = 0; d < dimc; ++d)
+         {
+            for (int jj = 0; jj < trial_dof; ++jj)
+            {
+               real_t val = 0.0;
+               for (int k = 0; k < dimc; ++k)
+               {
+                  const real_t *curl_k = &(curlshape.GetData())[k * trial_dof];
+                  val += M(d, k) * curl_k[jj];
+               }
+               Mcurl(d, jj) = val;
+            }
+         }
+      }
+      else if (Q)
       {
          c *= Q->Eval(Trans, ip);
       }
       shape *= c;
 
-      for (int d = 0; d < dimc; ++d)
+
+      if (MQ)
       {
-         real_t * curldata = &(curlshape.GetData())[d*trial_dof];
-         for (int jj = 0; jj < trial_dof; ++jj)
+         // use transformed curl
+         for (int d = 0; d < dimc; ++d)
          {
-            for (int ii = 0; ii < test_dof; ++ii)
+            for (int jj = 0; jj < trial_dof; ++jj)
             {
-               elmat(d * test_dof + ii, jj) += shape(ii) * curldata[jj];
+               const real_t cur_val = Mcurl(d, jj);
+               for (int ii = 0; ii < test_dof; ++ii)
+               {
+                  elmat(d * test_dof + ii, jj) += shape(ii) * cur_val;
+               }
+            }
+         }
+      }
+      else
+      {
+         for (int d = 0; d < dimc; ++d)
+         {
+            real_t * curldata = &(curlshape.GetData())[d*trial_dof];
+            for (int jj = 0; jj < trial_dof; ++jj)
+            {
+               for (int ii = 0; ii < test_dof; ++ii)
+               {
+                  elmat(d * test_dof + ii, jj) += shape(ii) * curldata[jj];
+               }
             }
          }
       }
@@ -3109,6 +3164,94 @@ void VectorDiffusionIntegrator::AssembleElementMatrix(
    }
 }
 
+void VectorDiffusionIntegrator::AssembleElementMatrix2(const FiniteElement
+                                                       &trial_fe,
+                                                       const FiniteElement &test_fe,
+                                                       ElementTransformation &Trans,
+                                                       DenseMatrix &elmat)
+{
+   int tr_nd = trial_fe.GetDof();
+   int te_nd = test_fe.GetDof();
+   dim = trial_fe.GetDim();
+   int spaceDim = Trans.GetSpaceDim();
+   bool square = (dim == spaceDim);
+   vdim = (vdim <= 0) ? spaceDim : vdim;
+
+   if (VQ)
+   {
+      vcoeff.SetSize(vdim);
+   }
+   else if (MQ)
+   {
+      mcoeff.SetSize(vdim);
+   }
+
+   dshape.SetSize(tr_nd, dim);
+   dshapedxt.SetSize(tr_nd, spaceDim);
+   te_dshape.SetSize(te_nd, dim);
+   te_dshapedxt.SetSize(te_nd, spaceDim);
+
+   elmat.SetSize(vdim * te_nd, vdim*tr_nd);
+   pelmat.SetSize(te_nd, tr_nd);
+
+   const IntegrationRule *ir = IntRule ? IntRule : &DiffusionIntegrator::GetRule(
+                                  trial_fe, test_fe);
+
+   elmat = 0.0;
+
+   for (int i = 0; i < ir -> GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      trial_fe.CalcDShape(ip, dshape);
+      test_fe.CalcDShape(ip, te_dshape);
+
+      Trans.SetIntPoint(&ip);
+      double w = Trans.Weight();
+      w = ip.weight / (square ? w : w*w*w);
+      // AdjugateJacobian = / adj(J),         if J is square
+      //                    \ adj(J^t.J).J^t, otherwise
+
+      Mult(dshape, Trans.AdjugateJacobian(), dshapedxt);
+      Mult(te_dshape, Trans.AdjugateJacobian(), te_dshapedxt);
+
+      if (VQ)
+      {
+         VQ->Eval(vcoeff, Trans, ip);
+         for (int k = 0; k < vdim; ++k)
+         {
+            pelmat = 0.0;
+            AddMult_a_ABt(w*vcoeff(k), te_dshapedxt, dshapedxt, pelmat);
+            elmat.AddMatrix(pelmat, te_nd*k, tr_nd*k);
+         }
+      }
+      else if (MQ)
+      {
+         MQ->Eval(mcoeff, Trans, ip);
+         for (int ii = 0; ii < vdim; ++ii)
+         {
+            for (int jj = 0; jj < vdim; ++jj)
+            {
+               pelmat = 0.0;
+               AddMult_a_ABt(w*mcoeff(ii,jj), te_dshapedxt, dshapedxt, pelmat);
+               elmat.AddMatrix(pelmat, te_nd*ii, tr_nd*jj);
+            }
+         }
+      }
+      else
+      {
+         if (Q) { w *= Q->Eval(Trans, ip); }
+         dshapedxt *= w;
+         MultABt(te_dshapedxt,dshapedxt, pelmat);
+         for (int k = 0; k < vdim; ++k)
+         {
+            elmat.AddMatrix(pelmat, te_nd*k, tr_nd*k);
+         }
+      }
+   }
+}
+
+
+
 void VectorDiffusionIntegrator::AssembleElementVector(
    const FiniteElement &el, ElementTransformation &Tr,
    const Vector &elfun, Vector &elvect)
@@ -4451,6 +4594,76 @@ void TraceIntegrator::AssembleTraceFaceMatrix(int elem,
    }
 }
 
+
+void VectorTraceIntegrator::AssembleTraceFaceMatrix(int elem,
+                                                    const FiniteElement &trial_face_fe,
+                                                    const FiniteElement &test_fe,
+                                                    FaceElementTransformations & Trans,
+                                                    DenseMatrix &elmat)
+{
+   MFEM_VERIFY(test_fe.GetMapType() == FiniteElement::VALUE,
+               "TraceIntegrator::AssembleTraceFaceMatrix: Test space should be H1");
+   MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::INTEGRAL,
+               "TraceIntegrator::AssembleTraceFaceMatrix: Trial space should be RT trace");
+
+   int i, j, face_ndof, ndof;
+   int order;
+   int spaceDim = Trans.GetSpaceDim();
+   vdim = (vdim == -1) ? spaceDim : vdim;
+
+   face_ndof = trial_face_fe.GetDof();
+   ndof = test_fe.GetDof();
+
+   face_shape.SetSize(face_ndof);
+   shape.SetSize(ndof);
+
+   elmat.SetSize(ndof*vdim, face_ndof*vdim);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      order = test_fe.GetOrder();
+      order += trial_face_fe.GetOrder();
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+   }
+
+   int iel = Trans.Elem1->ElementNo;
+   if (iel != elem)
+   {
+      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
+   }
+
+   double scale = 1.0;
+   if (iel != elem) { scale = -1.; }
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+      // Trace finite element shape function
+      trial_face_fe.CalcPhysShape(Trans,face_shape);
+
+      // Finite element shape function
+      ElementTransformation * eltrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
+      test_fe.CalcPhysShape(*eltrans, shape);
+
+      face_shape *= Trans.Weight()*ip.weight*scale;
+      for (int k = 0; k < vdim; k++)
+      {
+         for (i = 0; i < ndof; i++)
+         {
+            for (j = 0; j < face_ndof; j++)
+            {
+               elmat(i+k*ndof, j+k*face_ndof) += shape(i) * face_shape(j);
+            }
+         }
+      }
+   }
+}
+
+
 void NormalTraceIntegrator::AssembleTraceFaceMatrix(int elem,
                                                     const FiniteElement &trial_face_fe,
                                                     const FiniteElement &test_fe,

fem/bilininteg.hpp

@@ -2936,16 +2936,27 @@ class MixedCurlIntegrator : public BilinearFormIntegrator
 {
 protected:
    Coefficient *Q;
-
+   MatrixCoefficient *MQ;
 private:
    Vector shape;
    DenseMatrix dshape;
    DenseMatrix curlshape;
    DenseMatrix elmat_comp;
 public:
-   MixedCurlIntegrator() : Q{NULL} { }
-   MixedCurlIntegrator(Coefficient *q_) :  Q{q_} { }
-   MixedCurlIntegrator(Coefficient &q) :  Q{&q} { }
+   MixedCurlIntegrator() : Q(nullptr), MQ(nullptr) { }
+   // Scalar coefficient
+   explicit MixedCurlIntegrator(Coefficient *q)
+      : Q(q), MQ(nullptr) { }
+
+   explicit MixedCurlIntegrator(Coefficient &q)
+      : Q(&q), MQ(nullptr) { }
+
+   // Matrix coefficient
+   explicit MixedCurlIntegrator(MatrixCoefficient *mq)
+      : Q(nullptr), MQ(mq) { }
+
+   explicit MixedCurlIntegrator(MatrixCoefficient &mq)
+      : Q(nullptr), MQ(&mq) { }
 
    void AssembleElementMatrix2(const FiniteElement &trial_fe,
                                const FiniteElement &test_fe,
@@ -3129,9 +3140,11 @@ public:
     to be the spatial dimension (i.e. 2-dimension or 3-dimension). */
 class VectorDiffusionIntegrator : public BilinearFormIntegrator
 {
-   int vdim = -1;
+private:
    DenseMatrix dshape, dshapedxt, pelmat;
-   DenseMatrix mcoeff;
+   DenseMatrix te_dshape, te_dshapedxt;
+   int vdim = -1;
+   DenseMatrix mcoeff,dshapedxt_m;
    Vector vcoeff;
 
 protected:
@@ -3193,6 +3206,10 @@ public:
    void AssembleElementMatrix(const FiniteElement &el,
                               ElementTransformation &Trans,
                               DenseMatrix &elmat) override;
+   void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                               const FiniteElement &test_fe,
+                               ElementTransformation &Trans,
+                               DenseMatrix &elmat) override;
    void AssembleElementVector(const FiniteElement &el,
                               ElementTransformation &Tr,
                               const Vector &elfun, Vector &elvect) override;
@@ -3798,9 +3815,29 @@ public:
                                 DenseMatrix &elmat);
 };
 
-/** Integrator for the form: $ \langle v, w \cdot n \rangle $ over a face (the interface) where
-    the trial variable $v$ is defined on the interface ($H^{1/2}$, i.e., trace of $H^1$)
-    and the test variable $w$ is in an $H(div)$-conforming space. */
+/** Integrator for the DPG form: < v, w > over a face (the interface) where
+    the trial variable v is defined on the interface
+    ((H^-1/2)^vdim i.e., vᵢ :=uᵢ⋅n (for normal trace of H(div)^vdim)
+    and the test variable w is in an dim copies of H1-conforming space. */
+class VectorTraceIntegrator : public BilinearFormIntegrator
+{
+private:
+   Vector face_shape, shape;
+   int vdim = -1;
+public:
+   VectorTraceIntegrator() { }
+   void AssembleTraceFaceMatrix(int elem,
+                                const FiniteElement &trial_face_fe,
+                                const FiniteElement &test_fe,
+                                FaceElementTransformations &Trans,
+                                DenseMatrix &elmat);
+   void SetVDim(int vdim_) { vdim = vdim_; }
+};
+
+
+/** Integrator for the form: < v, w.n > over a face (the interface) where
+    the trial variable v is defined on the interface (H^1/2, i.e., trace of H1)
+    and the test variable w is in an H(div)-conforming space. */
 class NormalTraceIntegrator : public BilinearFormIntegrator
 {
 private:

fem/bounds.cpp

@@ -39,8 +39,8 @@ void PLBound::Setup(const int nb_i, const int ncp_i,
    b_type = b_type_i;
    cp_type = cp_type_i;
    tol = tol_i;
-   lbound.SetSize(nb, ncp);
-   ubound.SetSize(nb, ncp);
+   lbound.SetSize(ncp, nb);
+   ubound.SetSize(ncp, nb);
    nodes.SetSize(nb);
    weights.SetSize(nb);
    control_points.SetSize(ncp);
@@ -125,21 +125,25 @@ void PLBound::Setup(const int nb_i, const int ncp_i,
       {
          if (j == 0)
          {
-            lbound(i, j) = bv(i);
-            ubound(i, j) = bv(i);
+            lbound(j,i) = bv(i);
+            ubound(j,i) = bv(i);
          }
          else if (j == ncp-1)
          {
-            lbound(i, j) = bv(i);
-            ubound(i, j) = bv(i);
+            lbound(j,i) = bv(i);
+            ubound(j,i) = bv(i);
          }
          else
          {
             vals(0)  = bv(i);
             vals(1) =  bmv(i) +  dm*bdmv(i);
             vals(2) =  bpv(i) +  dp*bdpv(i);
-            lbound(i, j) = vals.Min()-tol; // tolerance for good measure
-            ubound(i, j) = vals.Max()+tol; // tolerance for good measure
+            lbound(j,i) = vals.Min()-tol; // tolerance for good measure
+            ubound(j,i) = vals.Max()+tol; // tolerance for good measure
+            if (b_type == 2)
+            {
+               lbound(j,i) = std::max(lbound(j,i),0_r);
+            }
          }
       }
    }
@@ -273,8 +277,7 @@ void PLBound::Get1DBounds(const Vector &coeff, Vector &intmin,
    intmax.SetSize(ncp);
    intmin = 0.0;
    intmax = 0.0;
-   Vector coeffm(nb);
-   coeffm = 0.0;
+   Vector coeffm;
 
    real_t a0 = 0.0;
    real_t a1 = 0.0;
@@ -302,6 +305,8 @@ void PLBound::Get1DBounds(const Vector &coeff, Vector &intmin,
    // compute L2 projection for linear bases: a0 + a1*x
    if (proj)
    {
+      coeffm.SetSize(nb);
+      coeffm = 0.0;
       for (int i = 0; i < nb; i++)
       {
          x = 2.0*nodes_int(i)-1;
@@ -342,8 +347,8 @@ void PLBound::Get1DBounds(const Vector &coeff, Vector &intmin,
       real_t c = coeffm(i);
       for (int j = 0; j < ncp; j++)
       {
-         intmin(j) += min(lbound(i,j)*c, ubound(i,j)*c);
-         intmax(j) += max(lbound(i,j)*c, ubound(i,j)*c);
+         intmin(j) += min(lbound(j,i)*c, ubound(j,i)*c);
+         intmax(j) += max(lbound(j,i)*c, ubound(j,i)*c);
       }
    }
 }
@@ -474,10 +479,10 @@ void PLBound::Get2DBounds(const Vector &coeff, Vector &intmin,
          real_t w1 = intmaxT(id2++);
          for (int k = 0; k < ncp; k++) // kth row
          {
-            vals(0) = w0*lbound(j,k);
-            vals(1) = w0*ubound(j,k);
-            vals(2) = w1*lbound(j,k);
-            vals(3) = w1*ubound(j,k);
+            vals(0) = w0*lbound(k,j);
+            vals(1) = w0*ubound(k,j);
+            vals(2) = w1*lbound(k,j);
+            vals(3) = w1*ubound(k,j);
             intmin(k*ncp+i) += vals.Min();
             intmax(k*ncp+i) += vals.Max();
          }
@@ -553,17 +558,17 @@ void PLBound::Get3DBounds(const Vector &coeff, Vector &intmin,
             for (int i = 0; i < nb; i++)
             {
                x = 2.0*nodes(i)-1; // x-coordinate
-               minBounds(i) -= a0V(j) + a1V(j)*x;
-               maxBounds(i) -= a0V(j) + a1V(j)*x;
+               minNodalVals(i) -= a0V(j) + a1V(j)*x;
+               maxNodalVals(i) -= a0V(j) + a1V(j)*x;
             }
             // Compute Bernstein coefficients
             LUFactors lu(basisMatLU.GetData(), lu_ip.GetData());
-            lu.Solve(nb, 1, minBounds.GetData());
-            lu.Solve(nb, 1, maxBounds.GetData());
+            lu.Solve(nb, 1, minNodalVals.GetData());
+            lu.Solve(nb, 1, maxNodalVals.GetData());
             for (int i = 0; i < nb; i++)
             {
-               intminT(i*ncp2+j) = minBounds(i);
-               intmaxT(i*ncp2+j) = maxBounds(i);
+               intminT(i*ncp2+j) = minNodalVals(i);
+               intmaxT(i*ncp2+j) = maxNodalVals(i);
             }
          }
       }
@@ -617,10 +622,10 @@ void PLBound::Get3DBounds(const Vector &coeff, Vector &intmin,
          real_t w1 = intmaxT(id2++);
          for (int k = 0; k < ncp; k++) // kth slice
          {
-            vals(0) = w0*lbound(j,k);
-            vals(1) = w0*ubound(j,k);
-            vals(2) = w1*lbound(j,k);
-            vals(3) = w1*ubound(j,k);
+            vals(0) = w0*lbound(k,j);
+            vals(1) = w0*ubound(k,j);
+            vals(2) = w1*lbound(k,j);
+            vals(3) = w1*ubound(k,j);
             intmin(k*ncp2+i) += vals.Min();
             intmax(k*ncp2+i) += vals.Max();
          }
@@ -653,7 +658,8 @@ void PLBound::SetupBernsteinBasisMat(DenseMatrix &basisMat,
                                      Vector &nodesBern) const
 {
    const int nbern = nodesBern.Size();
-   L2_SegmentElement el(nbern-1, 2); // we use L2 to leverage lexicographic order
+   L2_SegmentElement el(nbern-1, 2);
+   // we use L2 to leverage lexicographic order
    Array<int> ordering = el.GetLexicographicOrdering();
    basisMat.SetSize(nbern, nbern);
    Vector shape(nbern);
@@ -666,6 +672,39 @@ void PLBound::SetupBernsteinBasisMat(DenseMatrix &basisMat,
    }
 }
 
+DenseMatrix PLBound::GetBoundingMatrix(int dim, bool is_lower) const
+{
+   if (dim > 1)
+   {
+      const int ncpd = static_cast<int>(std::pow(ncp, dim));
+      const int nbd = static_cast<int>(std::pow(nb, dim));
+      DenseMatrix boundND(ncpd, nbd);
+      Vector phimin, phimax, col;
+      Vector coeffs(nbd);
+      coeffs = 0.0;
+      for (int j = 0; j < nbd; j++)
+      {
+         coeffs(j) = 1.0;
+         boundND.GetColumnReference(j, col);
+         GetNDBounds(dim, coeffs, phimin, phimax);
+         col = is_lower ? phimin : phimax;
+         coeffs(j) = 0.0;
+      }
+      return boundND;
+   }
+   return is_lower ? lbound : ubound;
+}
+
+DenseMatrix PLBound::GetLowerBoundMatrix(int dim) const
+{
+   return GetBoundingMatrix(dim, true);
+}
+
+DenseMatrix PLBound::GetUpperBoundMatrix(int dim) const
+{
+   return GetBoundingMatrix(dim, false);
+}
+
 constexpr int PLBound::min_ncp_gl_x[2][11];
 constexpr int PLBound::min_ncp_gll_x[2][11];
 constexpr int PLBound::min_ncp_pos_x[2][11];
@@ -716,4 +755,4 @@ void PLBound::Print(std::ostream &outp) const
    ubound.Print(outp);
 }
 
-}
+}

fem/bounds.hpp

@@ -19,14 +19,18 @@ namespace mfem
 {
 
 /** @name Piecewise linear bounds of bases
-    \brief Piecewise linear bounds of bases can be used to compute bounds on the grid function in each element. The bounds for the bases are constructed based on the following parameters:
+    \brief Piecewise linear bounds of bases can be used to compute bounds on
+    the grid function in each element. The bounds for the bases are constructed
+    based on the following parameters:
 
     (i) @b nb: number of bases/nodes in 1D (i.e. polynomial order+1),
 
-    (ii) @b b_type: bases type, 0 - Lagrange interpolants on Gauss-Legendre nodes, 1 - Lagrange interpolants on Gauss-Lobatto-Legendre nodes, and
+    (ii) @b b_type: bases type, 0 - Lagrange interpolants on Gauss-Legendre
+    nodes, 1 - Lagrange interpolants on Gauss-Lobatto-Legendre nodes, and
     2 - Positive/Bernstein bases on uniformly distributed nodes,
 
-    (iii) @b ncp: number of control points used to construct the piecewise linear bounds
+    (iii) @b ncp: number of control points used to construct the piecewise
+    linear bounds
 
     (iv) @b cp_type: control point distribution. 0 - GL + end-points,
                     1 - Chebyshev.
@@ -35,7 +39,9 @@ namespace mfem
 
     If the user does not specify @b ncp and @b cp_type, the minimum value of
     @b ncp is used that would bound the bases for the @b cp_type. We default
-    to @b cp_type = 0 as it requires fewer number of points to bound the bases. Typically, @b ncp = 2 @b nb is sufficient to get fairly compact bounds, and increasing @b ncp results in tighter bounds.
+    to @b cp_type = 0 as it requires fewer number of points to bound the bases.
+    Typically, @b ncp = 2 @b nb is sufficient to get fairly compact bounds, and
+    increasing @b ncp results in tighter bounds.
 
     Finally, only tensor-product elements are currently supported.
 
@@ -54,7 +60,7 @@ private:
    bool proj = true; // Use linear projection to compute bounds.
    real_t tol = 0.0; // offset bounds to avoid round-off errors
    Vector nodes, weights, control_points;
-   DenseMatrix lbound, ubound; // nb x ncp matrices with bounds of all bases
+   DenseMatrix lbound, ubound; // ncp x nb matrices with bounds of all bases
    // Some auxillary storage for computing the bounds with Bernstein
    DenseMatrix basisMatNodes; // Bernstein bases at equispaced nodes
    DenseMatrix basisMatInt;   // Bernstein bases at GLL nodes
@@ -80,6 +86,9 @@ private:
       {3,5,8,9,11,12,13,13,14,15,16}
    };
 
+   /// Helper function to extract lower or upper bounding matrix
+   DenseMatrix GetBoundingMatrix(int dim, bool is_lower) const;
+
 public:
    // Constructor
    PLBound(const int nb_i, const int ncp_i, const int b_type_i,
@@ -92,40 +101,82 @@ public:
    PLBound(const FiniteElementSpace *fes,
            const int ncp_i = -1, const int cp_type_i = 0);
 
-   // Get minimum number of control points needed to bound the given bases
+   /// Get minimum number of control points needed to bound the given bases
    int GetMinimumPointsForGivenBases(int nb_i, int b_type_i,
                                      int cp_type_i) const;
 
-   // Print information about the bounds
+   /// Print information about the bounds
    void Print(std::ostream &outp = mfem::out) const;
 
-   // Enable (default) or disable linear projection before bounding.
-   // This projection increases the computational cost but results in tighter
-   // bounds.
+   /** @brief Enable (default) or disable linear projection before bounding.
+    *
+    *  @details This projection increases the computational cost but results in
+    *  tighter bounds.
+    */
    void SetProjectionFlagForBounding(bool proj_) { proj = proj_; }
 
-   /// Compute piecewise linear bounds for the lexicographically-ordered
-   /// coefficients in @a coeff in 1D/2D/3D.
+   /** @brief Compute piecewise linear bounds for the lexicographically-ordered
+    *  nodal coefficients in @a coeff in 1D/2D/3D.
+    *
+    *  @param[in] rdim    The spatial dimension of the element (1, 2, or 3).
+    *  @param[in] coeff   The vector of lexicographically-ordered coefficients.
+    *                     Should be of size nb^rdim, where nb is the number of
+    *                     bases/nodes in 1D. These coefficients must correspond
+    *                     to the bases type and number of bases, used in the
+    *                     constructor of PLBound.
+    *
+    *  @param[out] intmin The vector of minimum bound for all control points.
+    *  @param[out] intmax The vector of maximum bound for all control points.
+    *                     Both intmin and intmax are of size ncp^rdim, where
+    *                     ncp is the number of control points in 1D, and are
+    *                     ordered lexicographically.
+    */
    void GetNDBounds(const int rdim, const Vector &coeff,
                     Vector &intmin, Vector &intmax) const;
 
    /// Get number of control points used to compute the bounds.
    int GetNControlPoints() const { return ncp; }
+
+   /// Get 1D control point locations (lexicographic order) in [0,1].
+   const Vector &GetControlPoints() const { return control_points; }
+
+   /** @brief Get lower and upper bounding matrix (ncp^dim x nb^dim)
+    *
+    *  @details The matrices can be used to compute the bounds at control points
+    *           by a simple matrix-vector product with the
+    *           lexicographically-ordered nodal coefficients.
+    *           The resulting output is also lexicographically-ordered.
+    *
+    *  @note These matrices do not account for the linear projection step that
+    *        is optionally done in GetNDBounds before bounding the function.
+    */
+   ///@{
+   DenseMatrix GetLowerBoundMatrix(int dim = 1) const;
+   DenseMatrix GetUpperBoundMatrix(int dim = 1) const;
+   ///@}
+
 private:
-   /// Compute piecewise linear bounds for the lexicographically-ordered
-   /// coefficients in @a coeff in 1D.
+   /** @brief Compute piecewise linear bounds for the lexicographically-ordered
+    *  nodal coefficients in @a coeff in 1D.
+    *  See GetNDBounds for details of the input and output parameters.
+    */
    void Get1DBounds(const Vector &coeff, Vector &intmin, Vector &intmax) const;
 
-   /// Compute piecewise linear bounds for the lexicographically-ordered
-   /// coefficients in @a coeff in 2D.
+   /** @brief Compute piecewise linear bounds for the lexicographically-ordered
+    *  nodal coefficients in @a coeff in 2D.
+    *  See GetNDBounds for details of the input and output parameters.
+    */
    void Get2DBounds(const Vector &coeff, Vector &intmin, Vector &intmax) const;
 
-   /// Compute piecewise linear bounds for the lexicographically-ordered
-   /// coefficients in @a coeff in 3D.
+   /** @brief Compute piecewise linear bounds for the lexicographically-ordered
+    *  nodal coefficients in @a coeff in 3D.
+    *  See GetNDBounds for details of the input and output parameters.
+    */
    void Get3DBounds(const Vector &coeff, Vector &intmin, Vector &intmax) const;
 
-   /// Setup matrix used to compute values at given 1D locations in [0,1]
-   /// for Bernstein bases.
+   /** @brief Setup matrix used to compute values at given 1D locations in [0,1]
+    *  for Bernstein bases.
+    */
    void SetupBernsteinBasisMat(DenseMatrix &basisMat, Vector &nodesBern) const;
 
    void Setup(const int nb_i, const int ncp_i, const int b_type_i,

fem/fe/fe_base.cpp

@@ -663,58 +663,59 @@ const
    #pragma omp critical (DofToQuad)
 #endif
    {
-      // If the new Dof2Quad is already present, e.g. added in a previous call
-      // or added by another omp thread, return.
+      // Do not run if the new Dof2Quad is already present, e.g. added in a
+      // previous call or added by another omp thread.
       if (DofToQuad::SearchArray(dof2quad_array, ir,
-                                 DofToQuad::LEXICOGRAPHIC_FULL))
-      { return; }
-
-      // Undo the native ordering which is what FiniteElement::GetDofToQuad
-      // returns.
-      auto *d2q_new = new DofToQuad(d2q);
-      d2q_new->mode = DofToQuad::LEXICOGRAPHIC_FULL;
-      const int nqpt = ir.GetNPoints();
-
-      const int b_dim = (range_type == VECTOR) ? dim : 1;
-
-      for (int i = 0; i < nqpt; i++)
+                                 DofToQuad::LEXICOGRAPHIC_FULL) == nullptr)
       {
-         for (int d = 0; d < b_dim; d++)
+         // Undo the native ordering which is what FiniteElement::GetDofToQuad
+         // returns.
+         auto *d2q_new = new DofToQuad(d2q);
+         d2q_new->mode = DofToQuad::LEXICOGRAPHIC_FULL;
+         const int nqpt = ir.GetNPoints();
+
+         const int b_dim = (range_type == VECTOR) ? dim : 1;
+
+         for (int i = 0; i < nqpt; i++)
          {
-            for (int j = 0; j < dof; j++)
+            for (int d = 0; d < b_dim; d++)
             {
-               const double val = d2q.B[i + nqpt*(d+b_dim*lex_ordering[j])];
-               d2q_new->B[i+nqpt*(d+b_dim*j)] = val;
-               d2q_new->Bt[j+dof*(i+nqpt*d)] = val;
+               for (int j = 0; j < dof; j++)
+               {
+                  const double val = d2q.B[i + nqpt*(d+b_dim*lex_ordering[j])];
+                  d2q_new->B[i+nqpt*(d+b_dim*j)] = val;
+                  d2q_new->Bt[j+dof*(i+nqpt*d)] = val;
+               }
             }
          }
-      }
 
-      const int g_dim = [this]()
-      {
-         switch (deriv_type)
+         const int g_dim = [this]()
          {
-            case GRAD: return dim;
-            case DIV: return 1;
-            case CURL: return cdim;
-            default: return 0;
-         }
-      }();
-
-      for (int i = 0; i < nqpt; i++)
-      {
-         for (int d = 0; d < g_dim; d++)
-         {
-            for (int j = 0; j < dof; j++)
+            switch (deriv_type)
             {
-               const double val = d2q.G[i + nqpt*(d+g_dim*lex_ordering[j])];
-               d2q_new->G[i+nqpt*(d+g_dim*j)] = val;
-               d2q_new->Gt[j+dof*(i+nqpt*d)] = val;
+               case GRAD: return dim;
+               case DIV: return 1;
+               case CURL: return cdim;
+               default: return 0;
+            }
+         }();
+
+         for (int i = 0; i < nqpt; i++)
+         {
+            for (int d = 0; d < g_dim; d++)
+            {
+               for (int j = 0; j < dof; j++)
+               {
+                  const double val = d2q.G[i + nqpt*(d+g_dim*lex_ordering[j])];
+                  d2q_new->G[i+nqpt*(d+g_dim*j)] = val;
+                  d2q_new->Gt[j+dof*(i+nqpt*d)] = val;
+               }
             }
          }
+
+         dof2quad_array.Append(d2q_new);
       }
 
-      dof2quad_array.Append(d2q_new);
    }
 }
 

fem/fe/fe_h1.cpp

@@ -589,7 +589,7 @@ void H1_TriangleElement::CalcHessian(const IntegrationPoint &ip,
    Vector   shape_x(p + 1),   shape_y(p + 1),   shape_l(p + 1);
    Vector  dshape_x(p + 1),  dshape_y(p + 1),  dshape_l(p + 1);
    Vector ddshape_x(p + 1), ddshape_y(p + 1), ddshape_l(p + 1);
-   DenseMatrix ddu(dof, dim);
+   DenseMatrix ddu(dof, (dim*(dim+1))/2);
 #endif
 
    poly1d.CalcBasis(p, ip.x, shape_x, dshape_x, ddshape_x);

fem/fe_coll.hpp

@@ -250,6 +250,12 @@ public:
        its GetOrder() method. */
    virtual FiniteElementCollection *Clone(int p) const;
 
+   virtual int GetConstructorOrder() const
+   {
+      MFEM_ABORT("Collection " << Name() << " does not support GetConstructorOrder");
+      return -1;
+   }
+
 protected:
    const int base_p; ///< Order as returned by GetOrder().
 
@@ -314,6 +320,9 @@ public:
    FiniteElementCollection *Clone(int p) const override
    { return new H1_FECollection(p, dim, b_type); }
 
+   int GetConstructorOrder() const override
+   { return base_p; }
+
    virtual ~H1_FECollection();
 };
 
@@ -343,6 +352,10 @@ class H1_Trace_FECollection : public H1_FECollection
 public:
    H1_Trace_FECollection(const int p, const int dim,
                          const int btype = BasisType::GaussLobatto);
+
+   FiniteElementCollection *Clone(int p) const override
+   { return new H1_Trace_FECollection(p, dim+1, b_type); }
+
 };
 
 /// Arbitrary order "L2-conforming" discontinuous finite elements.
@@ -396,6 +409,9 @@ public:
    FiniteElementCollection *Clone(int p) const override
    { return new L2_FECollection(p, dim, b_type, m_type); }
 
+   int GetConstructorOrder() const override
+   { return base_p; }
+
    virtual ~L2_FECollection();
 };
 
@@ -456,6 +472,9 @@ public:
    FiniteElementCollection *Clone(int p) const override
    { return new RT_FECollection(p, dim, cb_type, ob_type); }
 
+   int GetConstructorOrder() const override
+   { return base_p-1; }
+
    virtual ~RT_FECollection();
 };
 
@@ -536,6 +555,9 @@ public:
    FiniteElementCollection *Clone(int p) const override
    { return new ND_FECollection(p, dim, cb_type, ob_type); }
 
+   int GetConstructorOrder() const override
+   { return dim>1 ? base_p : base_p+1; }
+
    virtual ~ND_FECollection();
 };
 
@@ -548,6 +570,9 @@ public:
    ND_Trace_FECollection(const int p, const int dim,
                          const int cb_type = BasisType::GaussLobatto,
                          const int ob_type = BasisType::GaussLegendre);
+
+   FiniteElementCollection *Clone(int p) const override
+   { return new ND_Trace_FECollection(p, dim+1, cb_type, ob_type); }
 };
 
 /// Arbitrary order 3D H(curl)-conforming Nedelec finite elements in 1D.

fem/gridfunc.cpp

@@ -30,6 +30,7 @@
 #include <cmath>
 #include <iostream>
 #include <algorithm>
+#include <queue>
 
 namespace mfem
 {
@@ -2261,6 +2262,80 @@ void GridFunction::AccumulateAndCountBdrTangentValues(
    }
 }
 
+void GridFunction::AccumulateAndCountTraceValues(
+   Coefficient *coeff[], VectorCoefficient *vcoeff,
+   Array<int> &values_counter)
+{
+   Array<int> vdofs;
+   Vector vc;
+
+   values_counter.SetSize(Size());
+   values_counter = 0;
+
+   const int vdim = fes->GetVDim();
+   HostReadWrite();
+
+   for (int i = 0; i < fes->GetMesh()->GetNumFaces(); i++)
+   {
+
+      const FiniteElement *fe = fes->GetFaceElement(i);
+      const int fdof = fe->GetDof();
+      ElementTransformation *transf = fes->GetMesh()->GetFaceTransformation(i);
+      const IntegrationRule &ir = fe->GetNodes();
+      fes->GetFaceVDofs(i, vdofs);
+
+      for (int j = 0; j < fdof; j++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(j);
+         transf->SetIntPoint(&ip);
+         if (vcoeff) { vcoeff->Eval(vc, *transf, ip); }
+         for (int d = 0; d < vdim; d++)
+         {
+            if (!vcoeff && !coeff[d]) { continue; }
+
+            real_t val = vcoeff ? vc(d) : coeff[d]->Eval(*transf, ip);
+            int ind = vdofs[fdof*d+j];
+            if ( ind < 0 )
+            {
+               val = -val, ind = -1-ind;
+            }
+            if (++values_counter[ind] == 1)
+            {
+               (*this)(ind) = val;
+            }
+            else
+            {
+               (*this)(ind) += val;
+            }
+         }
+      }
+   }
+}
+
+void GridFunction::AccumulateAndCountTraceTangentValues(
+   VectorCoefficient &vcoeff, Array<int> &values_counter)
+{
+   const FiniteElement *fe;
+   ElementTransformation *T;
+   Array<int> dofs;
+   Vector lvec;
+
+   values_counter.SetSize(Size());
+   values_counter = 0;
+
+   HostReadWrite();
+
+   for (int i = 0; i < fes->GetMesh()->GetNumFaces(); i++)
+   {
+      fe = fes->GetFaceElement(i);
+      T = fes->GetMesh()->GetFaceTransformation(i);
+      fes->GetFaceVDofs(i, dofs);
+      lvec.SetSize(fe->GetDof());
+      fe->Project(vcoeff, *T, lvec);
+      accumulate_dofs(dofs, lvec, *this, values_counter);
+   }
+}
+
 void GridFunction::ComputeMeans(AvgType type, Array<int> &zones_per_vdof)
 {
    switch (type)
@@ -2699,6 +2774,54 @@ void GridFunction::ProjectCoefficient(VectorCoefficient &vcoeff,
    }
 }
 
+void GridFunction::ProjectTraceCoefficient(Coefficient *coeff[])
+{
+   Array<int> values_counter;
+   AccumulateAndCountTraceValues(coeff, NULL, values_counter);
+   ComputeMeans(ARITHMETIC, values_counter);
+}
+
+void GridFunction::ProjectTraceCoefficient(VectorCoefficient &vcoeff)
+{
+   Array<int> values_counter;
+   AccumulateAndCountTraceValues(NULL, &vcoeff, values_counter);
+   ComputeMeans(ARITHMETIC, values_counter);
+}
+
+void GridFunction::ProjectTraceCoefficientNormal(VectorCoefficient &vcoeff)
+{
+   const FiniteElement *fe;
+   ElementTransformation *T;
+   Array<int> dofs;
+   int dim = vcoeff.GetVDim();
+   Vector vc(dim), nor(dim), lvec;
+
+   for (int i = 0; i < fes->GetMesh()->GetNumFaces(); i++)
+   {
+      fe = fes->GetFaceElement(i);
+      T = fes->GetMesh()->GetFaceTransformation(i);
+      const IntegrationRule &ir = fe->GetNodes();
+      lvec.SetSize(fe->GetDof());
+      for (int j = 0; j < ir.GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(j);
+         T->SetIntPoint(&ip);
+         vcoeff.Eval(vc, *T, ip);
+         CalcOrtho(T->Jacobian(), nor);
+         lvec(j) = (vc * nor);
+      }
+      fes->GetFaceVDofs(i, dofs);
+      SetSubVector(dofs, lvec);
+   }
+}
+
+void GridFunction::ProjectTraceCoefficientTangent(VectorCoefficient &vcoeff)
+{
+   Array<int> values_counter;
+   AccumulateAndCountTraceTangentValues(vcoeff, values_counter);
+   ComputeMeans(ARITHMETIC, values_counter);
+}
+
 void GridFunction::ProjectCoefficientGlobalL2(VectorCoefficient &vcoeff,
                                               real_t rtol, int iter)
 {
@@ -5117,6 +5240,103 @@ void GridFunction::GetElementBoundsAtControlPoints(const int elem,
    }
 }
 
+void GridFunction::GetElementBoundsAtControlPoints(const int elem,
+                                                   const PLBound &plb,
+                                                   const Vector &ref_range,
+                                                   const int vdim,
+                                                   Vector &lower, Vector &upper,
+                                                   Vector &control_pos) const
+{
+   const FiniteElement *fe = fes->GetFE(elem);
+   const IntegrationRule ir_in = fe->GetNodes();
+   IntegrationRule ir_new(ir_in.GetNPoints());
+   const int dim = fes->GetMesh()->Dimension();
+   const L2_FECollection *l2fec = dynamic_cast<const L2_FECollection *>
+                                  (fes->FEColl());
+
+   const TensorBasisElement *tbe =
+      dynamic_cast<const TensorBasisElement *>(fe);
+   MFEM_VERIFY(tbe != NULL, "TensorBasis FiniteElement expected.");
+
+   const Array<int> &dof_map = tbe->GetDofMap();
+   bool lexico = (dof_map.Size() == 0);
+   bool bern = (tbe->GetBasisType() == BasisType::Positive);
+   bool h1   = (l2fec == nullptr);
+
+   Vector loc_data; // gridfunction values
+   // Construct an integration rule to evaluate the gridfunction in
+   // subinterval.
+   for (int i = 0; i < ir_in.GetNPoints(); i++)
+   {
+      IntegrationPoint &ip_new = ir_new.IntPoint(i);
+      const IntegrationPoint &ip_old =
+         ir_in.IntPoint((lexico || bern) ? i : dof_map[i]);
+      Vector ip_coord(dim);
+      ip_old.Get(ip_coord.GetData(), dim);
+      for (int d = 0; d < dim; d++)
+      {
+         ip_coord(d) = ref_range(d) +
+                       (ref_range(dim+d) - ref_range(d)) * ip_coord(d);
+      }
+      ip_new.Set(ip_coord.GetData(), dim);
+   }
+   GetValues(elem, ir_new, loc_data, vdim);
+   // At this point, the loc_data contains function values ordered
+   // lexicographically, unless we are using Bernstein bases.
+   // For Bernstein, we need to project and get coefficients first.
+
+   // For bernstein, we get coefficients corresponding to these function values
+   if (bern)
+   {
+      int bt = 4; // BasisType::ClosedUniform
+      int o = fe->GetOrder();
+      DenseMatrix projmat;
+      NodalTensorFiniteElement *ntfe = nullptr;
+      if (dim == 1)
+      {
+         if (h1) { ntfe = new H1_SegmentElement(o, bt); }
+         else    { ntfe = new L2_SegmentElement(o, bt); }
+      }
+      else if (dim == 2)
+      {
+         if (h1) { ntfe = new H1_QuadrilateralElement(o, bt); }
+         else    { ntfe = new L2_QuadrilateralElement(o, bt); }
+      }
+      else if (dim == 3)
+      {
+         if (h1) { ntfe = new H1_HexahedronElement(o, bt); }
+         else    { ntfe = new L2_HexahedronElement(o, bt); }
+      }
+      // projection matrix from H1 to Positive
+      ElementTransformation *eltran = fes->GetElementTransformation(elem);
+      fe->Project(*ntfe, *eltran, projmat);
+      Vector loc_data_temp(loc_data.Size());
+      projmat.Mult(loc_data, loc_data_temp);
+      for (int i = 0; i < dof_map.Size(); i++)
+      {
+         loc_data(i) = loc_data_temp(dof_map[i]);
+      }
+      if (dof_map.Size() == 0) { loc_data = loc_data_temp; }
+      delete ntfe;
+   }
+
+   // Get bounds at control points
+   plb.GetNDBounds(dim, loc_data, lower, upper);
+
+   // Save control point positions
+   int ncp = plb.GetNControlPoints();
+   control_pos.SetSize(dim * ncp);
+   const Vector control_pos_1D = plb.GetControlPoints();
+   for (int i = 0; i < ncp; i++)
+   {
+      for (int d = 0; d < dim; d++)
+      {
+         control_pos(i + d*ncp) =
+            ref_range(d) + (ref_range(dim+d)-ref_range(d))*control_pos_1D(i);
+      }
+   }
+}
+
 void GridFunction::GetElementBounds(const int elem, const PLBound &plb,
                                     Vector &lower, Vector &upper,
                                     const int vdim) const
@@ -5197,6 +5417,467 @@ PLBound GridFunction::GetBounds(Vector &lower, Vector &upper,
    return plb;
 }
 
+struct IntervalNode
+{
+   real_t val_min;
+   real_t val_max;
+   Array<IntervalNode *> child;
+   IntervalNode(real_t vmin, real_t vmax)
+      : val_min(vmin), val_max(vmax)
+   {
+      child.SetSize(0);
+   }
+   void AddChild(IntervalNode *ch) { child.Append(ch); }
+   real_t GetChildMinLower()
+   {
+      if (child.Size() == 0)
+      {
+         return val_min;
+      }
+      real_t valmin = numeric_limits<real_t>::max();
+      for (int i = 0; i < child.Size(); i++)
+      {
+         real_t candidate = child[i]->GetChildMinLower();
+         valmin = std::min(valmin, candidate);
+      }
+      return valmin;
+   }
+   real_t GetChildMinUpper()
+   {
+      if (child.Size() == 0)
+      {
+         return val_max;
+      }
+      real_t valmax = numeric_limits<real_t>::max();
+      for (int i = 0; i < child.Size(); i++)
+      {
+         real_t candidate = child[i]->GetChildMinUpper();
+         valmax = std::min(valmax, candidate);
+      }
+      return valmax;
+   }
+   real_t GetChildMaxLower()
+   {
+      if (child.Size() == 0)
+      {
+         return val_min;
+      }
+      real_t valmin = numeric_limits<real_t>::lowest();
+      for (int i = 0; i < child.Size(); i++)
+      {
+         real_t candidate = child[i]->GetChildMaxLower();
+         valmin = std::max(valmin, candidate);
+      }
+      return valmin;
+   }
+   real_t GetChildMaxUpper()
+   {
+      if (child.Size() == 0)
+      {
+         return val_max;
+      }
+      real_t valmax = numeric_limits<real_t>::lowest();
+      for (int i = 0; i < child.Size(); i++)
+      {
+         real_t candidate = child[i]->GetChildMaxUpper();
+         valmax = std::max(valmax, candidate);
+      }
+      return valmax;
+   }
+   void DeleteChildren()
+   {
+      for (int i = 0; i < child.Size(); i++)
+      {
+         child[i]->DeleteChildren();
+         delete child[i];
+      }
+      child.SetSize(0);
+   }
+};
+
+struct SearchInterval
+{
+   Vector ref_range;
+   int depth;
+   IntervalNode *node;
+   SearchInterval(const Vector &ref_range_in, int d, IntervalNode *n)
+      : ref_range(ref_range_in), depth(d), node(n)
+   { }
+};
+
+struct IntervalCompareMin
+{
+   bool operator()(const SearchInterval *a, const SearchInterval *b) const
+   {
+      return a->node->val_min > b->node->val_min;
+   }
+};
+
+struct IntervalCompareMax
+{
+   bool operator()(const SearchInterval *a, const SearchInterval *b) const
+   {
+      return a->node->val_max < b->node->val_max;
+   }
+};
+
+std::pair<real_t, real_t> GridFunction::EstimateFunctionMinimum(
+   const int elem, const PLBound &plb, const int vdim,
+   const int max_depth, const real_t tol) const
+{
+   real_t min_threshold = std::numeric_limits<real_t>::max();
+   return EstimateFunctionMinimum(elem, plb, vdim, max_depth, tol,
+                                  min_threshold);
 }
 
+std::pair<real_t, real_t> GridFunction::EstimateFunctionMinimum(
+   const int elem, const PLBound &plb, const int vdim,
+   const int max_depth, const real_t tol, real_t &min_threshold) const
+{
+   const int dim = this->FESpace()->GetMesh()->Dimension();
+   const int ncp = plb.GetNControlPoints();
+   Vector pos_range(2*dim); pos_range = 0.0;
+   for (int d = 0; d < dim; d++) { pos_range(d+dim) = 1.0; }
+   Vector lower, upper, cp_ref_loc;
 
+   GetElementBoundsAtControlPoints(elem, plb, lower, upper, vdim);
+   real_t val_min = lower.Min();
+   real_t val_max = upper.Min();
+
+   min_threshold = std::min(min_threshold, val_max);
+
+   // Pruning: if the element's lower bound is greater than the current global
+   // upper bound, this element cannot contain the global minimum.
+   if (val_min >= min_threshold)
+   {
+      return std::make_pair(val_min, val_max);
+   }
+
+   if (val_min == val_max || max_depth == 0)
+   {
+      min_threshold = std::min(min_threshold, val_min);
+      return std::make_pair(val_min, val_max);
+   }
+   real_t abs_tol = tol*(val_max-val_min);
+
+   IntervalNode *initial_node = new IntervalNode(val_min, val_max);
+   SearchInterval *initial_interval = new SearchInterval(pos_range, 0,
+                                                         initial_node);
+
+   std::priority_queue<SearchInterval*,
+       std::vector<SearchInterval*>, IntervalCompareMin> pq;
+   pq.push(initial_interval);
+
+   real_t min_upper_bound = upper.Min();
+   real_t min_lower_bound = lower.Min();
+
+   while (!pq.empty())
+   {
+      SearchInterval *current = pq.top();
+      pq.pop();
+      int curr_depth = current->depth;
+
+      // Reached max depth or this interval cannot contain the global minimum
+      if (current->node->val_min >= min_threshold || curr_depth >= max_depth)
+      {
+         delete current;
+         continue;
+      }
+
+      min_lower_bound = initial_node->GetChildMinLower();
+      if (min_upper_bound - min_lower_bound < abs_tol)
+      {
+         delete current;
+         break;
+      }
+
+      // Subdivide the interval and get bounds on it
+      GetElementBoundsAtControlPoints(elem, plb, current->ref_range,
+                                      vdim, lower, upper, cp_ref_loc);
+
+      // process the bounds and create sub-intervals
+      for (int k = 0; k < (dim == 3 ? ncp-1 : 1); k++)
+      {
+         for (int j = 0; j < (dim >= 2 ? ncp-1 : 1); j++)
+         {
+            for (int i = 0; i < ncp-1; i++)
+            {
+               real_t lv = 0.0, uv = 0.0;
+               if (dim == 1)
+               {
+                  lv = std::min(lower(i), lower(i+1));
+                  uv = std::min(upper(i), upper(i+1));
+               }
+               else if (dim == 2)
+               {
+                  lv = std::min({lower(i + j*ncp), lower((i+1) + j*ncp),
+                                 lower(i + (j+1)*ncp),
+                                 lower((i+1) + (j+1)*ncp)});
+                  uv = std::min({upper(i + j*ncp), upper((i+1) + j*ncp),
+                                 upper(i + (j+1)*ncp),
+                                 upper((i+1) + (j+1)*ncp)});
+               }
+               else if (dim == 3)
+               {
+                  lv = std::min({lower(i + j*ncp + k*ncp*ncp),
+                                 lower((i+1) + j*ncp + k*ncp*ncp),
+                                 lower(i + (j+1)*ncp + k*ncp*ncp),
+                                 lower((i+1) + (j+1)*ncp + k*ncp*ncp),
+                                 lower(i + j*ncp + (k+1)*ncp*ncp),
+                                 lower((i+1) + j*ncp + (k+1)*ncp*ncp),
+                                 lower(i + (j+1)*ncp + (k+1)*ncp*ncp),
+                                 lower((i+1) + (j+1)*ncp + (k+1)*ncp*ncp)});
+                  uv = std::min({upper(i + j*ncp + k*ncp*ncp),
+                                 upper((i+1) + j*ncp + k*ncp*ncp),
+                                 upper(i + (j+1)*ncp + k*ncp*ncp),
+                                 upper((i+1) + (j+1)*ncp + k*ncp*ncp),
+                                 upper(i + j*ncp + (k+1)*ncp*ncp),
+                                 upper((i+1) + j*ncp + (k+1)*ncp*ncp),
+                                 upper(i + (j+1)*ncp + (k+1)*ncp*ncp),
+                                 upper((i+1) + (j+1)*ncp + (k+1)*ncp*ncp)});
+               }
+               IntervalNode *child_node = new IntervalNode(lv, uv);
+               current->node->AddChild(child_node);
+
+               if (lv < min_threshold)
+               {
+                  min_upper_bound = std::min(min_upper_bound, uv);
+                  min_threshold = std::min(min_threshold, uv);
+                  if (curr_depth < max_depth)
+                  {
+                     pos_range(0) = cp_ref_loc(i);
+                     pos_range(0+dim) = cp_ref_loc(i+1);
+                     if (dim >= 2)
+                     {
+                        pos_range(1) = cp_ref_loc(ncp + j);
+                        pos_range(1+dim) = cp_ref_loc(ncp + j+1);
+                     }
+                     if (dim == 3)
+                     {
+                        pos_range(2) = cp_ref_loc(2*ncp + k);
+                        pos_range(2+dim) = cp_ref_loc(2*ncp + k+1);
+                     }
+                     SearchInterval *child_interval =
+                        new  SearchInterval(pos_range, curr_depth + 1,
+                                            child_node);
+                     pq.push(child_interval);
+                  }
+               }
+            }
+         }
+      }
+      delete current;
+   }
+
+   // clean up remaining intervals in queue
+   while (!pq.empty())
+   {
+      delete pq.top();
+      pq.pop();
+   }
+
+   min_lower_bound = initial_node->GetChildMinLower();
+   initial_node->DeleteChildren();
+   delete initial_node;
+
+   min_threshold = std::min(min_threshold, min_lower_bound);
+   return std::make_pair(min_lower_bound, min_upper_bound);
+}
+
+std::pair<real_t, real_t> GridFunction::EstimateFunctionMaximum(
+   const int elem, const PLBound &plb, const int vdim,
+   const int max_depth, const real_t tol) const
+{
+   real_t max_threshold = std::numeric_limits<real_t>::lowest();
+   return EstimateFunctionMaximum(elem, plb, vdim, max_depth, tol,
+                                  max_threshold);
+}
+
+std::pair<real_t, real_t> GridFunction::EstimateFunctionMaximum(
+   const int elem, const PLBound &plb, const int vdim,
+   const int max_depth, const real_t tol, real_t &max_threshold) const
+{
+   const int dim = this->FESpace()->GetMesh()->Dimension();
+   const int ncp = plb.GetNControlPoints();
+   Vector pos_range(2*dim); pos_range = 0.0;
+   for (int d = 0; d < dim; d++) { pos_range(d+dim) = 1.0; }
+   Vector lower, upper, cp_ref_loc;
+
+   GetElementBoundsAtControlPoints(elem, plb, lower, upper, vdim);
+   real_t val_min = lower.Max();
+   real_t val_max = upper.Max();
+
+   max_threshold = std::max(max_threshold, val_min);
+
+   // Pruning: if the element's upper bound is less than the current global
+   // lower bound, this element cannot contain the global maximum.
+   if (val_max <= max_threshold)
+   {
+      return std::make_pair(val_min, val_max);
+   }
+
+   if (val_min == val_max || max_depth == 0)
+   {
+      max_threshold = std::max(max_threshold, val_max);
+      return std::make_pair(val_min, val_max);
+   }
+   real_t abs_tol = tol*(val_max-val_min);
+
+   IntervalNode *initial_node = new IntervalNode(val_min, val_max);
+   SearchInterval *initial_interval = new SearchInterval(pos_range, 0,
+                                                         initial_node);
+
+   std::priority_queue<SearchInterval*,
+       std::vector<SearchInterval*>, IntervalCompareMax> pq;
+   pq.push(initial_interval);
+
+   real_t max_lower_bound = val_min;
+   real_t max_upper_bound = val_max;
+
+   while (!pq.empty())
+   {
+      SearchInterval *current = pq.top();
+      pq.pop();
+      int curr_depth = current->depth;
+
+      // Reached max depth or this interval cannot contain the global maximum.
+      if (current->node->val_max <= max_threshold || curr_depth >= max_depth)
+      {
+         delete current;
+         continue;
+      }
+
+      max_upper_bound = initial_node->GetChildMaxUpper();
+      if (max_upper_bound - max_lower_bound < abs_tol)
+      {
+         delete current;
+         break;
+      }
+
+      // Subdivide the interval and get bounds on it
+      GetElementBoundsAtControlPoints(elem, plb, current->ref_range,
+                                      vdim, lower, upper, cp_ref_loc);
+
+      // process the bounds and create sub-intervals
+      for (int k = 0; k < (dim == 3 ? ncp-1 : 1); k++)
+      {
+         for (int j = 0; j < (dim >= 2 ? ncp-1 : 1); j++)
+         {
+            for (int i = 0; i < ncp-1; i++)
+            {
+               real_t lv = 0.0, uv = 0.0;
+               if (dim == 1)
+               {
+                  lv = std::max(lower(i), lower(i+1));
+                  uv = std::max(upper(i), upper(i+1));
+               }
+               else if (dim == 2)
+               {
+                  lv = std::max({lower(i + j*ncp), lower((i+1) + j*ncp),
+                                 lower(i + (j+1)*ncp),
+                                 lower((i+1) + (j+1)*ncp)});
+                  uv = std::max({upper(i + j*ncp), upper((i+1) + j*ncp),
+                                 upper(i + (j+1)*ncp),
+                                 upper((i+1) + (j+1)*ncp)});
+               }
+               else if (dim == 3)
+               {
+                  lv = std::max({lower(i + j*ncp + k*ncp*ncp),
+                                 lower((i+1) + j*ncp + k*ncp*ncp),
+                                 lower(i + (j+1)*ncp + k*ncp*ncp),
+                                 lower((i+1) + (j+1)*ncp + k*ncp*ncp),
+                                 lower(i + j*ncp + (k+1)*ncp*ncp),
+                                 lower((i+1) + j*ncp + (k+1)*ncp*ncp),
+                                 lower(i + (j+1)*ncp + (k+1)*ncp*ncp),
+                                 lower((i+1) + (j+1)*ncp + (k+1)*ncp*ncp)});
+                  uv = std::max({upper(i + j*ncp + k*ncp*ncp),
+                                 upper((i+1) + j*ncp + k*ncp*ncp),
+                                 upper(i + (j+1)*ncp + k*ncp*ncp),
+                                 upper((i+1) + (j+1)*ncp + k*ncp*ncp),
+                                 upper(i + j*ncp + (k+1)*ncp*ncp),
+                                 upper((i+1) + j*ncp + (k+1)*ncp*ncp),
+                                 upper(i + (j+1)*ncp + (k+1)*ncp*ncp),
+                                 upper((i+1) + (j+1)*ncp + (k+1)*ncp*ncp)});
+               }
+               IntervalNode *child_node = new IntervalNode(lv, uv);
+               current->node->AddChild(child_node);
+
+               if (uv > max_threshold)
+               {
+                  max_lower_bound = std::max(max_lower_bound, lv);
+                  max_threshold = std::max(max_threshold, lv);
+                  if (curr_depth < max_depth)
+                  {
+                     pos_range(0) = cp_ref_loc(i);
+                     pos_range(0+dim) = cp_ref_loc(i+1);
+                     if (dim >= 2)
+                     {
+                        pos_range(1) = cp_ref_loc(ncp + j);
+                        pos_range(1+dim) = cp_ref_loc(ncp + j+1);
+                     }
+                     if (dim == 3)
+                     {
+                        pos_range(2) = cp_ref_loc(2*ncp + k);
+                        pos_range(2+dim) = cp_ref_loc(2*ncp + k+1);
+                     }
+                     SearchInterval *child_interval =
+                        new SearchInterval(pos_range, curr_depth + 1,
+                                           child_node);
+                     pq.push(child_interval);
+                  }
+               }
+            }
+         }
+      }
+      delete current;
+   }
+   // clean up remaining intervals in queue
+   while (!pq.empty())
+   {
+      delete pq.top();
+      pq.pop();
+   }
+
+   max_upper_bound = initial_node->GetChildMaxUpper();
+   initial_node->DeleteChildren();
+   delete initial_node;
+   max_threshold = std::max(max_threshold, max_upper_bound);
+
+   return std::make_pair(max_lower_bound, max_upper_bound);
+}
+
+std::pair<real_t, real_t> GridFunction::EstimateFunctionMinimum(
+   const int vdim, const PLBound &plb, const int max_depth,
+   const real_t tol) const
+{
+   real_t global_min_lower = std::numeric_limits<real_t>::max();
+   real_t global_min_upper = std::numeric_limits<real_t>::max();
+
+   for (int i = 0; i < fes->GetNE(); i++)
+   {
+      std::pair<real_t, real_t> min_pair =
+         EstimateFunctionMinimum(i, plb, vdim, max_depth, tol,
+                                 global_min_lower);
+      global_min_upper = std::min(global_min_upper, min_pair.second);
+   }
+   return std::make_pair(global_min_lower, global_min_upper);
+}
+
+std::pair<real_t, real_t> GridFunction::EstimateFunctionMaximum(
+   const int vdim, const PLBound &plb, const int max_depth,
+   const real_t tol) const
+{
+   real_t global_max_lower = std::numeric_limits<real_t>::lowest();
+   real_t global_max_upper = std::numeric_limits<real_t>::lowest();
+
+   for (int i = 0; i < fes->GetNE(); i++)
+   {
+      std::pair<real_t, real_t> max_pair =
+         EstimateFunctionMaximum(i, plb, vdim, max_depth, tol,
+                                 global_max_upper);
+      global_max_lower = std::max(global_max_lower, max_pair.first);
+   }
+   return std::make_pair(global_max_lower, global_max_upper);
+}
+
+}

fem/gridfunc.hpp

@@ -557,6 +557,13 @@ protected:
                                            const Array<int> &bdr_attr,
                                            Array<int> &values_counter);
 
+   void AccumulateAndCountTraceValues(Coefficient *coeff[],
+                                      VectorCoefficient *vcoeff,
+                                      Array<int> &values_counter);
+
+   void AccumulateAndCountTraceTangentValues(VectorCoefficient &vcoeff,
+                                             Array<int> &values_counter);
+
    // Complete the computation of averages; called e.g. after
    // AccumulateAndCountZones().
    void ComputeMeans(AvgType type, Array<int> &zones_per_vdof);
@@ -564,6 +571,70 @@ protected:
    /// P-refinement version of Update().
    void UpdatePRef();
 
+   /** @brief Estimate the minimum value of the GridFunction in element @a elem
+    *  if it is below a certain @a min_threshold.
+    *
+    *  @details For a given element \p elem and grid function component \p vdim
+    *  an estimate of the function minimum is the minimum of the piecewise
+    *  linear lower bound obtained using the given PLBound object. The actual
+    *  minimum is between [minimum lower bound, minimum upper bound]. We
+    *  improve the estimate of the function minimum by recursively
+    *  subdividing the interval with the lowest lower bound, and computing
+    *  bounds on the sub-intervals.
+    *  This process continues until (i) the maximum recursion depth is reached
+    *  or (ii) the difference between the minimum upper bound and minimum lower
+    *  bound is less than a certain tolerance (\p tol * [initial maximum
+    *  upper bound - initial minimum lower bound]).
+    *  The function also terminates if the lowest minima estimate is found
+    *  to be above the given threshold \p min_threshold. This is useful when
+    *  we are interested in computing the global minimum of the function
+    *  over all elements. In this case we can reject elements where the lowest
+    *  bound is above the current global minimum. In case the function
+    *  minimum on the element is below the global minimum, we update
+    *  \p min_threshold.
+    *
+    *  We return a pair of values that bracket the actual minimum, i.e.
+    *  [min_lower_bound, min_upper_bound].
+    */
+   std::pair<real_t,real_t> EstimateFunctionMinimum(const int elem,
+                                                    const PLBound &plb,
+                                                    const int vdim,
+                                                    const int max_depth,
+                                                    const real_t tol,
+                                                    real_t &min_threshold)const;
+
+   /** @brief Estimate the maximum value of the GridFunction in element @a elem
+    *  if it is below a certain @a max_threshold.
+    *
+    *  @details For a given element \p elem and grid function component \p vdim
+    *  an estimate of the function maximum is the maximum of the piecewise
+    *  linear upper bound obtained using the given PLBound object. The actual
+    *  maximum is between [maximum lower bound, maximum upper bound]. We
+    *  improve the estimate of the function maximum by recursively
+    *  subdividing the interval with the highest upper bound, and computing
+    *  bounds on the sub-intervals.
+    *  This process continues until (i) the maximum recursion depth is reached
+    *  or (ii) the difference between the maximum upper bound and maximum lower
+    *  bound is less than a certain tolerance (\p tol * [initial maximum
+    *  upper bound - initial maximum lower bound]).
+    *  The function also terminates if the highest maxima estimate is found
+    *  to be below the given threshold \p max_threshold. This is useful when
+    *  we are interested in computing the global maximum of the function
+    *  over all elements. In this case we can reject elements where the upper
+    *  bound is below the current global maximum. In case the function
+    *  maximum on the element is above the global maximum, we update
+    *  \p max_threshold.
+    *
+    *  We return a pair of values that bracket the actual maximum, i.e.
+    *  [max_lower_bound, max_upper_bound].
+    */
+   std::pair<real_t,real_t> EstimateFunctionMaximum(const int elem,
+                                                    const PLBound &plb,
+                                                    const int vdim,
+                                                    const int max_depth,
+                                                    const real_t tol,
+                                                    real_t &max_threshold)const;
+
 public:
    /** @brief For each vdof, counts how many elements contain the vdof,
        as containment is determined by FiniteElementSpace::GetElementVDofs(). */
@@ -578,6 +649,21 @@ public:
       ProjectBdrCoefficient(&coeff_p, attr);
    }
 
+   /// Project a Coefficient on a GridFunction defined on H1 trace space
+   void ProjectTraceCoefficient(Coefficient *coeff[]);
+   void ProjectTraceCoefficient(Coefficient &coeff)
+   {
+      Coefficient *coeff_p = &coeff;
+      ProjectTraceCoefficient(&coeff_p);
+   }
+   /// Project a VectorCoefficient on a GridFunction defined on a Vector H1 trace space
+   void ProjectTraceCoefficient(VectorCoefficient &vcoeff);
+   /// Project a VectorCoefficient on a GridFunction defined on a RT trace space
+   void ProjectTraceCoefficientNormal(VectorCoefficient &vcoeff);
+   /// Project a VectorCoefficient on a GridFunction defined on a ND trace space
+   void ProjectTraceCoefficientTangent(VectorCoefficient &vcoeff);
+
+
    /** @brief Project a VectorCoefficient on the GridFunction, modifying only
        DOFs on the boundary associated with the boundary attributes marked in
        the @a attr array. */
@@ -1662,21 +1748,21 @@ public:
     */
    ///@{
    /// Computes the \ref PLBound for the gridfunction with number of control
-   /// points based on @a ref_factor, and returns the overall bounds for each
-   /// vdim (across all elements) in @b lower and @b upper. We also return the
+   /// points based on \p ref_factor, and returns the overall bounds for each
+   /// vdim (across all elements) in \p lower and \p upper. We also return the
    /// PLBound object used to compute the bounds.
-   /// We compute the bounds for each vdim if @a vdim < 1.
+   /// We compute the bounds for each vdim if \p vdim < 1.
    /// Note: For most cases, this method/interface will be sufficient.
    virtual PLBound GetBounds(Vector &lower, Vector &upper,
                              const int ref_factor=1, const int vdim=-1) const;
 
    /// Computes the \ref PLBound for the gridfunction with number of control
-   /// points based on @a ref_factor, and returns the bounds for each element
-   /// ordered byVDim:
+   /// points based on \p ref_factor, and returns the bounds for each element
+   /// ordered byNodes:
    /// lower_{0,0}, lower_{1,0}, ..., lower_{ne-1,0},
    /// lower_{0,1}, ..., lower_{ne-1,vdim-1}. We also return the
    /// PLBound object used to compute the bounds.
-   /// We compute the bounds for each vdim if @a vdim < 1.
+   /// We compute the bounds for each vdim if \p vdim < 1.
    PLBound GetElementBounds(Vector &lower, Vector &upper,
                             const int ref_factor=1, const int vdim=-1) const;
 
@@ -1687,6 +1773,18 @@ public:
                                         Vector &lower, Vector &upper,
                                         const int vdim = -1) const;
 
+   /** @brief Gets the bounds on given reference range inside an element.
+    *
+    *  @details @a ref_range is a vector of size 2*dim that specifies the
+    *  lower and upper limits in each dimension of the reference element.
+    *  For example, in 2D, ref_range = [rmin, smin, rmax, smax].
+    */
+   void GetElementBoundsAtControlPoints(const int elem, const PLBound &plb,
+                                        const Vector &ref_range,
+                                        const int vdim,
+                                        Vector &lower, Vector &upper,
+                                        Vector &control_pos) const;
+
    /// Compute bounds on the grid function for the given element.
    /// The bounds are stored in @b lower and @b upper.
    void GetElementBounds(const int elem, const PLBound &plb,
@@ -1694,11 +1792,45 @@ public:
                          const int vdim = -1) const;
 
    /// Compute bounds on the grid function for all the elements. The bounds
-   /// are returned in @b lower and @b upper, ordered byVDim:
+   /// are returned in @b lower and @b upper, ordered byNodes:
    /// lower_{0,0}, lower_{1,0}, ..., lower_{ne-1,0},
    /// lower_{0,1}, ..., lower_{ne-1,vdim-1}
    void GetElementBounds(const PLBound &plb, Vector &lower, Vector &upper,
                          const int vdim=-1) const;
+
+   /** @brief Estimate the minimum value of the GridFunction in element @a elem.
+    *
+    *  @details See the protected version of EstimateFunctionMinimum for
+    *  details.
+    */
+   std::pair<real_t, real_t> EstimateFunctionMinimum(const int elem,
+                                                     const PLBound &plb,
+                                                     const int vdim,
+                                                     const int max_depth,
+                                                     const real_t tol) const;
+
+   /** @brief Estimate the minimum value of the GridFunction in element @a elem.
+    *
+    *  @details See the protected version of EstimateFunctionMaximum for
+    *  details.
+    */
+   std::pair<real_t, real_t> EstimateFunctionMaximum(const int elem,
+                                                     const PLBound &plb,
+                                                     const int vdim,
+                                                     const int max_depth,
+                                                     const real_t tol) const;
+
+   /** @brief Estimate the GridFunction minimum across all elements. */
+   virtual std::pair<real_t,real_t> EstimateFunctionMinimum(const int vdim,
+                                                            const PLBound &plb,
+                                                            const int max_depth,
+                                                            const real_t tol) const;
+
+   /** @brief Estimate the GridFunction maximum across all elements. */
+   virtual std::pair<real_t,real_t> EstimateFunctionMaximum(const int vdim,
+                                                            const PLBound &plb,
+                                                            const int max_depth,
+                                                            const real_t tol) const;
    ///@}
 
    /// Destroys grid function.

fem/integ/bilininteg_vecdiffusion_pa.hpp

@@ -171,15 +171,15 @@ template<int DIM, int T_SDIM, int T_D1D, int T_Q1D>
 VectorDiffusionIntegrator::ApplyKernelType
 VectorDiffusionIntegrator::ApplyPAKernels::Kernel()
 {
-   if (DIM == 2)
+   if constexpr (DIM == 2)
    {
       return internal::SmemPAVectorDiffusionApply2D<T_SDIM, T_D1D, T_Q1D>;
    }
-   else if (DIM == 3)
+   else if constexpr (DIM == 3)
    {
       return internal::SmemPAVectorDiffusionApply3D<T_SDIM, T_D1D, T_Q1D>;
    }
-   else { MFEM_ABORT("Unsupported kernel"); }
+   MFEM_ABORT("Unsupported kernel");
 }
 
 inline VectorDiffusionIntegrator::ApplyKernelType

fem/integ/bilininteg_vecmass_pa.hpp

@@ -182,15 +182,15 @@ template<int DIM, int T_D1D, int T_Q1D>
 VectorMassIntegrator::VectorMassAddMultPAType
 VectorMassIntegrator::VectorMassAddMultPA::Kernel()
 {
-   if (DIM == 2)
+   if constexpr (DIM == 2)
    {
       return internal::SmemPAVectorMassApply2D<T_D1D,T_Q1D>;
    }
-   else if (DIM == 3)
+   else if constexpr (DIM == 3)
    {
       return internal::SmemPAVectorMassApply3D<T_D1D, T_Q1D>;
    }
-   else { MFEM_ABORT("Unsupported kernel"); }
+   MFEM_ABORT("Unsupported kernel");
 }
 
 inline VectorMassIntegrator::VectorMassAddMultPAType

fem/integ/lininteg_domain_kernels.hpp

@@ -301,18 +301,14 @@ template <int DIM, int T_D1D, int T_Q1D>
 DomainLFIntegrator::AssembleKernelType
 DomainLFIntegrator::AssembleKernels::Kernel()
 {
-   switch (DIM)
-   {
-      case 1:
-         return DLFEvalAssemble1D<T_D1D, T_Q1D>;
-      case 2:
-         return DLFEvalAssemble2D<T_D1D, T_Q1D>;
-      case 3:
-         return DLFEvalAssemble3D<T_D1D, T_Q1D>;
-   }
+   if constexpr (DIM == 1) { return DLFEvalAssemble1D<T_D1D, T_Q1D>; }
+   if constexpr (DIM == 2) { return DLFEvalAssemble2D<T_D1D, T_Q1D>; }
+   if constexpr (DIM == 3) { return DLFEvalAssemble3D<T_D1D, T_Q1D>; }
    MFEM_ABORT("");
 }
+
 /// \endcond DO_NOT_DOCUMENT
 
 } // namespace mfem
-#endif
+
+#endif // MFEM_LININTEG_DOMAIN_KERNELS_HPP

fem/intrules.cpp

@@ -14,6 +14,18 @@
 // Acknowledgment: Some of the high-precision triangular and tetrahedral
 // quadrature rules below were obtained from the Encyclopaedia of Cubature
 // Formulas at http://nines.cs.kuleuven.be/research/ecf/ecf.html
+//
+// Positive-weight simplex quadrature rules from:
+//
+//   [1] F.D. Witherden, P.E. Vincent, "On the identification of symmetric
+//       quadrature rules for finite element methods", Computers & Mathematics
+//       with Applications, 69(10):1232-1241, 2015.
+//       Used for: triangles (d=0-20), tetrahedra (d=0-13).
+//
+//   [2] G. Chuluunbaatar, O. Chuluunbaatar, A.A. Gusev, S.I. Vinitsky,
+//       "PI-type fully symmetric quadrature rules on the 3-,...,6-simplexes",
+//       Computers & Mathematics with Applications, 124:89-97, 2022.
+//       Used for: tetrahedra (d=14-20).
 
 #include "fem.hpp"
 #include "../mesh/nurbs.hpp"
@@ -959,8 +971,8 @@ IntegrationRules IntRules(0, Quadrature1D::GaussLegendre);
 
 IntegrationRules RefinedIntRules(1, Quadrature1D::GaussLegendre);
 
-IntegrationRules::IntegrationRules(int ref, int type)
-   : quad_type(type)
+IntegrationRules::IntegrationRules(int ref, int type, SimplexQuadrature stype)
+   : quad_type(type), simplex_type(stype)
 {
    refined = ref;
 
@@ -1253,6 +1265,11 @@ IntegrationRule *IntegrationRules::SegmentIntegrationRule(int Order)
 // Integration rules for reference triangle {[0,0],[1,0],[0,1]}
 IntegrationRule *IntegrationRules::TriangleIntegrationRule(int Order)
 {
+   if (simplex_type == SimplexQuadrature::PositiveWeights)
+   {
+      return TrianglePositiveIntegrationRule(Order);
+   }
+
    IntegrationRule *ir = NULL;
    // Note: Set TriangleIntRules[*] to ir only *after* ir is fully constructed.
    // This is needed in multithreaded environment.
@@ -1675,6 +1692,11 @@ IntegrationRule *IntegrationRules::SquareIntegrationRule(int Order)
     {[0,0,0],[1,0,0],[0,1,0],[0,0,1]}          */
 IntegrationRule *IntegrationRules::TetrahedronIntegrationRule(int Order)
 {
+   if (simplex_type == SimplexQuadrature::PositiveWeights)
+   {
+      return TetrahedronPositiveIntegrationRule(Order);
+   }
+
    IntegrationRule *ir;
    // Note: Set TetrahedronIntRules[*] to ir only *after* ir is fully
    // constructed. This is needed in multithreaded environment.
@@ -1788,6 +1810,944 @@ IntegrationRule *IntegrationRules::TetrahedronIntegrationRule(int Order)
    }
 }
 
+// Triangle rules from Witherden & Vincent [1].
+// Orbit data from PyFR (https://pyfr.org), licensed under CC-BY 4.0.
+IntegrationRule *IntegrationRules::TrianglePositiveIntegrationRule(int Order)
+{
+   IntegrationRule *ir = NULL;
+
+   switch (Order)
+   {
+      case 0:
+      case 1:
+         ir = new IntegrationRule(1);
+         ir->AddTriMidPoint(0, 5.00000000000000000000e-01);
+         ir->SetOrder(1);
+         TriangleIntRules[0] =
+            TriangleIntRules[1] = ir;
+         return ir;
+
+      case 2:
+         ir = new IntegrationRule(3);
+         ir->AddTriPoints3(0, 1.66666666666666740682e-01, 1.66666666666666657415e-01);
+         ir->SetOrder(2);
+         TriangleIntRules[2] = ir;
+         return ir;
+
+      case 3:
+      case 4:
+         ir = new IntegrationRule(6);
+         ir->AddTriPoints3(0, 4.45948490915964890213e-01, 1.11690794839005735906e-01);
+         ir->AddTriPoints3(3, 9.15762135097707430376e-02, 5.49758718276609353870e-02);
+         ir->SetOrder(4);
+         TriangleIntRules[3] =
+            TriangleIntRules[4] = ir;
+         return ir;
+
+      case 5:
+         ir = new IntegrationRule(7);
+         ir->AddTriMidPoint(0, 1.12500000000000002776e-01);
+         ir->AddTriPoints3(1, 1.01286507323456342888e-01, 6.29695902724135697648e-02);
+         ir->AddTriPoints3(4, 4.70142064105115109474e-01, 6.61970763942530959767e-02);
+         ir->SetOrder(5);
+         TriangleIntRules[5] = ir;
+         return ir;
+
+      case 6:
+         ir = new IntegrationRule(12);
+         ir->AddTriPoints3(0, 6.30890144915022266225e-02, 2.54224531851034094010e-02);
+         ir->AddTriPoints3(3, 2.49286745170910428726e-01, 5.83931378631896841336e-02);
+         ir->AddTriPoints6(6, 6.36502499121398668258e-01, 3.10352451033784393353e-01,
+                           4.14255378091867854096e-02);
+         ir->SetOrder(6);
+         TriangleIntRules[6] = ir;
+         return ir;
+
+      case 7:
+         ir = new IntegrationRule(15);
+         ir->AddTriPoints3(0, 3.37306485545878498300e-02, 8.27252505539606552976e-03);
+         ir->AddTriPoints3(3, 2.41577382595403566956e-01, 6.39720856150777922311e-02);
+         ir->AddTriPoints3(6, 4.74309692504718327655e-01, 3.85433230929930342734e-02);
+         ir->AddTriPoints6(9, 7.54280040550053154647e-01, 1.98683314797351684433e-01,
+                           2.79393664515998896292e-02);
+         ir->SetOrder(7);
+         TriangleIntRules[7] = ir;
+         return ir;
+
+      case 8:
+         ir = new IntegrationRule(16);
+         ir->AddTriMidPoint(0, 7.21578038388935860681e-02);
+         ir->AddTriPoints3(1, 4.59292588292723236165e-01, 4.75458171336423096598e-02);
+         ir->AddTriPoints3(4, 1.70569307751760268488e-01, 5.16086852673591223173e-02);
+         ir->AddTriPoints3(7, 5.05472283170309566458e-02, 1.62292488115990396480e-02);
+         ir->AddTriPoints6(10, 7.28492392955404244326e-01, 2.63112829634638112353e-01,
+                           1.36151570872174963733e-02);
+         ir->SetOrder(8);
+         TriangleIntRules[8] = ir;
+         return ir;
+
+      case 9:
+         ir = new IntegrationRule(19);
+         ir->AddTriMidPoint(0, 4.85678981413994181882e-02);
+         ir->AddTriPoints3(1, 4.37089591492936690997e-01, 3.89137705023871391385e-02);
+         ir->AddTriPoints3(4, 1.88203535619032802373e-01, 3.98238694636051243636e-02);
+         ir->AddTriPoints3(7, 4.89682519198737620236e-01, 1.56673501135695357467e-02);
+         ir->AddTriPoints3(10, 4.47295133944527467662e-02, 1.27888378293490156262e-02);
+         ir->AddTriPoints6(13, 7.41198598784498008385e-01, 2.21962989160765733487e-01,
+                           2.16417696886446880855e-02);
+         ir->SetOrder(9);
+         TriangleIntRules[9] = ir;
+         return ir;
+
+      case 10:
+         ir = new IntegrationRule(25);
+         ir->AddTriMidPoint(0, 4.08716645731429864541e-02);
+         ir->AddTriPoints3(1, 3.20553732169435168231e-02, 6.67648440657478327992e-03);
+         ir->AddTriPoints3(4, 1.42161101056564431744e-01, 2.29789818023723654838e-02);
+         ir->AddTriPoints6(7, 5.30054118927343997925e-01, 3.21812995288835446139e-01,
+                           3.19524531982120219009e-02);
+         ir->AddTriPoints6(13, 6.01233328683459244957e-01, 3.69146781827810910315e-01,
+                           1.70923240814797143539e-02);
+         ir->AddTriPoints6(19, 8.07930600922879049719e-01, 1.63701733737182442141e-01,
+                           1.26488788536441923438e-02);
+         ir->SetOrder(10);
+         TriangleIntRules[10] = ir;
+         return ir;
+
+      case 11:
+         ir = new IntegrationRule(28);
+         ir->AddTriMidPoint(0, 4.28805898661121093207e-02);
+         ir->AddTriPoints3(1, 2.84854176143718995640e-02, 5.21593525644734826857e-03);
+         ir->AddTriPoints3(4, 2.10219956703178278978e-01, 3.52578420558582877886e-02);
+         ir->AddTriPoints3(7, 1.02635482712246428605e-01, 1.93153796185096607307e-02);
+         ir->AddTriPoints3(10, 4.95891900965890919384e-01, 8.30313652729268436570e-03);
+         ir->AddTriPoints3(13, 4.38465926764352253997e-01, 3.36580770397341480504e-02);
+         ir->AddTriPoints6(16, 8.43349783661853091843e-01, 1.49324788652082374174e-01,
+                           5.14514478647663895533e-03);
+         ir->AddTriPoints6(22, 6.64408374196864159877e-01, 2.89581125637705882880e-01,
+                           2.01662383202502772106e-02);
+         ir->SetOrder(11);
+         TriangleIntRules[11] = ir;
+         return ir;
+
+      case 12:
+         ir = new IntegrationRule(33);
+         ir->AddTriPoints3(0, 4.88203750945541581352e-01, 1.21334190407260157640e-02);
+         ir->AddTriPoints3(3, 1.09257827659354322947e-01, 1.42430260344387719235e-02);
+         ir->AddTriPoints3(6, 2.71462507014926135440e-01, 3.12706065979513822550e-02);
+         ir->AddTriPoints3(9, 2.46463634363356387524e-02, 3.96582125498681943576e-03);
+         ir->AddTriPoints3(12, 4.40111648658593201944e-01, 2.49591674640304711508e-02);
+         ir->AddTriPoints6(15, 6.85310163906391878186e-01, 2.91655679738340944951e-01,
+                           1.08917925193037796322e-02);
+         ir->AddTriPoints6(21, 6.28249751683556123538e-01, 2.55454228638517299999e-01,
+                           2.16136818297071042760e-02);
+         ir->AddTriPoints6(27, 8.51337792510240110033e-01, 1.27279717233589384495e-01,
+                           7.54183878825571887144e-03);
+         ir->SetOrder(12);
+         TriangleIntRules[12] = ir;
+         return ir;
+
+      case 13:
+         ir = new IntegrationRule(37);
+         ir->AddTriMidPoint(0, 3.39800182934158201409e-02);
+         ir->AddTriPoints3(1, 4.89076946452539351728e-01, 1.19972009644473652512e-02);
+         ir->AddTriPoints3(4, 2.21372286291832920391e-01, 2.91392425595999905730e-02);
+         ir->AddTriPoints3(7, 4.26941414259800422482e-01, 2.78009837652266646180e-02);
+         ir->AddTriPoints3(10, 2.15096811088433259584e-02, 3.02616855176958583773e-03);
+         ir->AddTriPoints6(13, 7.48507115899952224503e-01, 1.63597401067850478640e-01,
+                           1.20895199057969096601e-02);
+         ir->AddTriPoints6(19, 8.64707770295442768038e-01, 1.10922042803463405392e-01,
+                           7.48270055258283377231e-03);
+         ir->AddTriPoints6(25, 6.23545995553675513889e-01, 3.08441760892117777804e-01,
+                           1.73206380704241866275e-02);
+         ir->AddTriPoints6(31, 7.22357793124188019007e-01, 2.72515817773429591675e-01,
+                           4.79534050177163155559e-03);
+         ir->SetOrder(13);
+         TriangleIntRules[13] = ir;
+         return ir;
+
+      case 14:
+         ir = new IntegrationRule(42);
+         ir->AddTriPoints3(0, 1.77205532412543442788e-01, 2.10812943684965080349e-02);
+         ir->AddTriPoints3(3, 4.17644719340453940415e-01, 1.63941767720626740967e-02);
+         ir->AddTriPoints3(6, 6.17998830908725871325e-02, 7.21684983488833382143e-03);
+         ir->AddTriPoints3(9, 4.88963910362178677538e-01, 1.09417906847144447147e-02);
+         ir->AddTriPoints3(12, 2.73477528308838646609e-01, 2.58870522536457925433e-02);
+         ir->AddTriPoints3(15, 1.93909612487010996063e-02, 2.46170180120004094063e-03);
+         ir->AddTriPoints6(18, 6.86980167808087793802e-01, 2.98372882136257788765e-01,
+                           7.21815405676692022074e-03);
+         ir->AddTriPoints6(24, 7.70608554774996457049e-01, 1.72266687821355679588e-01,
+                           1.23328766062818367955e-02);
+         ir->AddTriPoints6(30, 5.70222290846683188548e-01, 3.36861459796344964168e-01,
+                           1.92857553935303419057e-02);
+         ir->AddTriPoints6(36, 8.79757171370171064950e-01, 1.18974497696956893478e-01,
+                           2.50511441925033596229e-03);
+         ir->SetOrder(14);
+         TriangleIntRules[14] = ir;
+         return ir;
+
+      case 15:
+         ir = new IntegrationRule(49);
+         ir->AddTriMidPoint(0, 2.21676936910920364954e-02);
+         ir->AddTriPoints3(1, 4.05362214133975495844e-01, 2.13568907857302828224e-02);
+         ir->AddTriPoints3(4, 7.01735528999860580512e-02, 8.22236878131258133728e-03);
+         ir->AddTriPoints3(7, 4.74170681438019769871e-01, 8.69807400038170690226e-03);
+         ir->AddTriPoints3(10, 2.26378713420349653163e-01, 2.33916808643548149171e-02);
+         ir->AddTriPoints3(13, 4.94996956769126195130e-01, 4.78692309123004283711e-03);
+         ir->AddTriPoints3(16, 1.58117262509887002153e-02, 1.48038731895268772104e-03);
+         ir->AddTriPoints6(19, 6.66975644801868106093e-01, 3.14648242812450851247e-01,
+                           7.80128641528798211224e-03);
+         ir->AddTriPoints6(25, 9.19912157726236134891e-01, 7.09486052364554087291e-02,
+                           2.01492668600904969653e-03);
+         ir->AddTriPoints6(31, 7.15222356931450642392e-01, 1.90535589476393929509e-01,
+                           1.43602934626006709801e-02);
+         ir->AddTriPoints6(37, 8.13292641049419229304e-01, 1.68068645222414381202e-01,
+                           5.83631059078792285844e-03);
+         ir->AddTriPoints6(43, 5.65252664877114230357e-01, 3.38950611475277163720e-01,
+                           1.56577381424846430458e-02);
+         ir->SetOrder(15);
+         TriangleIntRules[15] = ir;
+         return ir;
+
+      case 16:
+         ir = new IntegrationRule(55);
+         ir->AddTriMidPoint(0, 2.26322830369093952463e-02);
+         ir->AddTriPoints3(1, 2.45990070467141719313e-01, 2.05464615718494759966e-02);
+         ir->AddTriPoints3(4, 4.15584896885420551627e-01, 2.03559166562126796218e-02);
+         ir->AddTriPoints3(7, 8.53555665867003487968e-02, 7.39081734511220188322e-03);
+         ir->AddTriPoints3(10, 1.61918644191271221544e-01, 1.47092048494940497855e-02);
+         ir->AddTriPoints3(13, 5.00000000000000000000e-01, 2.20927315607528452004e-03);
+         ir->AddTriPoints3(16, 4.75280727545942083268e-01, 1.29871666491385793357e-02);
+         ir->AddTriPoints6(19, 7.54170061444767725334e-01, 1.91074763640529221576e-01,
+                           9.46913623220784969603e-03);
+         ir->AddTriPoints6(25, 9.68244368030958701965e-01, 2.32034277688137335893e-02,
+                           8.27233357417524097638e-04);
+         ir->AddTriPoints6(31, 6.49303698245446425652e-01, 3.31764523474147643434e-01,
+                           7.50430089214290316213e-03);
+         ir->AddTriPoints6(37, 9.00273703270429548340e-01, 8.06961669858730079596e-02,
+                           3.97379696669624901673e-03);
+         ir->AddTriPoints6(43, 5.89148840564247877616e-01, 3.08244969196354023921e-01,
+                           1.59918050396850343342e-02);
+         ir->AddTriPoints6(49, 8.06621867499395683865e-01, 1.87441782483782071189e-01,
+                           2.69559355842440570919e-03);
+         ir->SetOrder(16);
+         TriangleIntRules[16] = ir;
+         return ir;
+
+      case 17:
+         ir = new IntegrationRule(60);
+         ir->AddTriPoints3(0, 4.17103444361599295931e-01, 1.36554632640510532210e-02);
+         ir->AddTriPoints3(3, 1.47554916607539610141e-02, 1.38694378881882109979e-03);
+         ir->AddTriPoints3(6, 4.65597871618890324363e-01, 1.25097254752486782697e-02);
+         ir->AddTriPoints3(9, 1.80358116266370605008e-01, 1.31563152940089925225e-02);
+         ir->AddTriPoints3(12, 6.66540634795969033632e-02, 6.22950040115272107855e-03);
+         ir->AddTriPoints3(15, 2.85706502436586629035e-01, 1.88581185763976415248e-02);
+         ir->AddTriPoints6(18, 8.24790070165088096132e-01, 1.59192287472792681768e-01,
+                           3.98915010296479674579e-03);
+         ir->AddTriPoints6(24, 6.26369030386452196879e-01, 3.06281591746186521164e-01,
+                           1.12438862733455335191e-02);
+         ir->AddTriPoints6(30, 5.71294867944684092720e-01, 4.15475459295228999324e-01,
+                           5.19921997791976831654e-03);
+         ir->AddTriPoints6(36, 7.53235145936458128091e-01, 1.68722513495259462957e-01,
+                           1.02789491602272593102e-02);
+         ir->AddTriPoints6(42, 7.15072259110642427515e-01, 2.71791870055354878311e-01,
+                           4.34610725050059605590e-03);
+         ir->AddTriPoints6(48, 9.15919353297816929427e-01, 7.25054707990024915887e-02,
+                           2.29217420086793351869e-03);
+         ir->AddTriPoints6(54, 5.43275579596159796658e-01, 2.99218942476970228839e-01,
+                           1.30858129676684944304e-02);
+         ir->SetOrder(17);
+         TriangleIntRules[17] = ir;
+         return ir;
+
+      case 18:
+         ir = new IntegrationRule(67);
+         ir->AddTriMidPoint(0, 1.81778676507133342410e-02);
+         ir->AddTriPoints3(1, 3.99955628067576229867e-01, 1.66522350166950668104e-02);
+         ir->AddTriPoints3(4, 4.87580301574869645620e-01, 6.02332381699985548729e-03);
+         ir->AddTriPoints3(7, 4.61809506406449243876e-01, 9.47458575338943308208e-03);
+         ir->AddTriPoints3(10, 2.42264702514271956790e-01, 1.82375447044718190515e-02);
+         ir->AddTriPoints3(13, 3.88302560886856218403e-02, 3.56466300985948522304e-03);
+         ir->AddTriPoints3(16, 9.19477421216432500017e-02, 8.27957997600162372287e-03);
+         ir->AddTriPoints6(19, 7.70372376214675247397e-01, 1.83822707925463957324e-01,
+                           6.87980811747110256732e-03);
+         ir->AddTriPoints6(25, 6.70953985194234547862e-01, 2.06349257433837918185e-01,
+                           1.18909554500764153007e-02);
+         ir->AddTriPoints6(31, 6.00418954634256873959e-01, 3.95683434332269712286e-01,
+                           2.26526725112853252742e-03);
+         ir->AddTriPoints6(37, 8.78342189467521738955e-01, 1.08195793791033278985e-01,
+                           3.42005505980359086893e-03);
+         ir->AddTriPoints6(43, 6.39988092004714625993e-01, 3.19751624525377309283e-01,
+                           8.87374455101020212511e-03);
+         ir->AddTriPoints6(49, 7.58929479855198430016e-01, 2.35772184958191743931e-01,
+                           2.50533043728986106261e-03);
+         ir->AddTriPoints6(55, 9.72360728962795684005e-01, 2.70909109951620319379e-02,
+                           6.11474063480544911126e-04);
+         ir->AddTriPoints6(61, 5.45918775386194599086e-01, 3.33493529449880754534e-01,
+                           1.27410876559122202695e-02);
+         ir->SetOrder(18);
+         TriangleIntRules[18] = ir;
+         return ir;
+
+      case 19:
+         ir = new IntegrationRule(73);
+         ir->AddTriMidPoint(0, 1.72346988520061666916e-02);
+         ir->AddTriPoints3(1, 5.25238903512089683190e-02, 3.55462829889906543543e-03);
+         ir->AddTriPoints3(4, 4.92512675041336889237e-01, 5.16087757147214078873e-03);
+         ir->AddTriPoints3(7, 1.11448873323021391268e-01, 7.61717554650914990128e-03);
+         ir->AddTriPoints3(10, 4.59194201039543670184e-01, 1.14917950133708035576e-02);
+         ir->AddTriPoints3(13, 4.03969722551901222474e-01, 1.57687674465774863020e-02);
+         ir->AddTriPoints3(16, 1.78170104781764315760e-01, 1.23259574240954274116e-02);
+         ir->AddTriPoints3(19, 1.16394611837894457196e-02, 8.82661388221423837477e-04);
+         ir->AddTriPoints3(22, 2.55161632913607716588e-01, 1.58765096830015377261e-02);
+         ir->AddTriPoints6(25, 8.30156464400275351245e-01, 1.30697676268032414448e-01,
+                           4.84774224342752330097e-03);
+         ir->AddTriPoints6(31, 5.59369805720300927732e-01, 3.11317629809541251973e-01,
+                           1.31731609886953666272e-02);
+         ir->AddTriPoints6(37, 6.33313293128784149388e-01, 3.64617780974611060962e-01,
+                           1.64103827591790965915e-03);
+         ir->AddTriPoints6(43, 7.04004819966042139079e-01, 2.21434885432331141075e-01,
+                           9.05397246560622585843e-03);
+         ir->AddTriPoints6(49, 8.52566954376889230005e-01, 1.42425757365756355810e-01,
+                           1.46315755173510018451e-03);
+         ir->AddTriPoints6(55, 6.05083979068707922266e-01, 3.54028009735275261960e-01,
+                           8.05108138201205379703e-03);
+         ir->AddTriPoints6(61, 7.43181368957436361278e-01, 2.41894578960579587079e-01,
+                           4.22794374976824798712e-03);
+         ir->AddTriPoints6(67, 9.30137698876805085746e-01, 6.00862753223067036501e-02,
+                           1.66360068142969402642e-03);
+         ir->SetOrder(19);
+         TriangleIntRules[19] = ir;
+         return ir;
+
+      case 20:
+         ir = new IntegrationRule(79);
+         ir->AddTriMidPoint(0, 1.39101107014531159140e-02);
+         ir->AddTriPoints3(1, 2.54579267673339160183e-01, 1.40832013075202471669e-02);
+         ir->AddTriPoints3(4, 1.09761410283977789426e-02, 7.98840791066619858654e-04);
+         ir->AddTriPoints3(7, 1.09383596711714603522e-01, 7.83023077607453328597e-03);
+         ir->AddTriPoints3(10, 1.86294997744540946627e-01, 9.17346297425291473671e-03);
+         ir->AddTriPoints3(13, 4.45551056955924895675e-01, 9.45239993323244813428e-03);
+         ir->AddTriPoints3(16, 3.73108805988847103130e-02, 2.16127541066557732688e-03);
+         ir->AddTriPoints3(19, 3.93425347817099924086e-01, 1.37880506290704585304e-02);
+         ir->AddTriPoints3(22, 4.76245611540499047543e-01, 7.10182530340844071076e-03);
+         ir->AddTriPoints6(25, 8.33295511838236246938e-01, 1.59133707657067247077e-01,
+                           2.20289741855849742491e-03);
+         ir->AddTriPoints6(31, 7.54921502863547422280e-01, 1.98518132228788335425e-01,
+                           5.98639857895469015836e-03);
+         ir->AddTriPoints6(37, 9.31054476783942153162e-01, 6.40905856084340586065e-02,
+                           1.12986960212586558597e-03);
+         ir->AddTriPoints6(43, 6.11877703547425655373e-01, 3.33134817309587605294e-01,
+                           8.66722556721933289070e-03);
+         ir->AddTriPoints6(49, 8.61684018936486717521e-01, 9.99522962881386756173e-02,
+                           4.14571152761385782609e-03);
+         ir->AddTriPoints6(55, 6.78165737889635522606e-01, 2.15607057390094447591e-01,
+                           7.72260782209923009323e-03);
+         ir->AddTriPoints6(61, 5.70144692890973359134e-01, 4.20023758816224113133e-01,
+                           3.69568150025529782929e-03);
+         ir->AddTriPoints6(67, 5.42331804172428100230e-01, 3.17860123835772001577e-01,
+                           1.16917457318277372147e-02);
+         ir->AddTriPoints6(73, 7.08681375720323636358e-01, 2.80581411423665327831e-01,
+                           3.57820023845768515197e-03);
+         ir->SetOrder(20);
+         TriangleIntRules[20] = ir;
+         return ir;
+
+      default:
+         // Grundmann-Moller fallback for orders beyond tabulated rules
+         int i = (Order / 2) * 2 + 1;   // closest odd >= Order
+         AllocIntRule(TriangleIntRules, i);
+         ir = new IntegrationRule;
+         ir->GrundmannMollerSimplexRule(i/2, 2);
+         if (!TriangleIntRules[i-1]) { TriangleIntRules[i-1] = ir; }
+         TriangleIntRules[i] = ir;
+         return ir;
+   }
+}
+
+// Tet rules d=0-13 from Witherden & Vincent [1], orbit data from PyFR, CC-BY 4.0.
+// Tet rules d=14-20 from Chuluunbaatar et al. [2], supplementary data.
+IntegrationRule *IntegrationRules::TetrahedronPositiveIntegrationRule(int Order)
+{
+   IntegrationRule *ir = NULL;
+
+   switch (Order)
+   {
+      case 0:
+      case 1:
+         ir = new IntegrationRule(1);
+         ir->AddTetMidPoint(0, 1.66666666666666657415e-01);
+         ir->SetOrder(1);
+         TetrahedronIntRules[0] =
+            TetrahedronIntRules[1] = ir;
+         return ir;
+
+      case 2:
+         ir = new IntegrationRule(4);
+         ir->AddTetPoints4(0, 1.38196601125010531952e-01, 4.16666666666666643537e-02);
+         ir->SetOrder(2);
+         TetrahedronIntRules[2] = ir;
+         return ir;
+
+      case 3:
+         ir = new IntegrationRule(8);
+         ir->AddTetPoints4(0, 3.28163302516381705232e-01, 2.27029737561812265667e-02);
+         ir->AddTetPoints4(4, 1.08047249898428621151e-01, 1.89636929104854412564e-02);
+         ir->SetOrder(3);
+         TetrahedronIntRules[3] = ir;
+         return ir;
+
+      case 4:
+      case 5:
+         ir = new IntegrationRule(14);
+         ir->AddTetPoints4(0, 3.10885919263300669613e-01, 1.87813209530026427319e-02);
+         ir->AddTetPoints4(4, 9.27352503108912484819e-02, 1.22488405193936587129e-02);
+         ir->AddTetPoints6(8, 4.54496295874350364485e-01, 7.09100346284691120807e-03);
+         ir->SetOrder(5);
+         TetrahedronIntRules[4] =
+            TetrahedronIntRules[5] = ir;
+         return ir;
+
+      case 6:
+         ir = new IntegrationRule(24);
+         ir->AddTetPoints4(0, 4.06739585346113652342e-02, 1.67953517588677390775e-03);
+         ir->AddTetPoints4(4, 3.22337890142275540484e-01, 9.22619692394245453915e-03);
+         ir->AddTetPoints4(8, 2.14602871259152117034e-01, 6.65379170969458179352e-03);
+         ir->AddTetPoints12(12, 6.36610018750174977420e-02, 6.03005664791649187428e-01,
+                            8.03571428571428492127e-03);
+         ir->SetOrder(6);
+         TetrahedronIntRules[6] = ir;
+         return ir;
+
+      case 7:
+         ir = new IntegrationRule(35);
+         ir->AddTetMidPoint(0, 1.59142149106884754628e-02);
+         ir->AddTetPoints4(1, 3.15701149778202794227e-01, 7.05493020166117132397e-03);
+         ir->AddTetPoints6(5, 4.49510177401603649994e-01, 5.31615463880959638471e-03);
+         ir->AddTetPoints12(11, 1.88833831026001153219e-01, 5.75171637586999962011e-01,
+                            6.20118845472243662709e-03);
+         ir->AddTetPoints12(23, 2.12654725414832546093e-02, 8.10830241098548620826e-01,
+                            1.35179513831722359664e-03);
+         ir->SetOrder(7);
+         TetrahedronIntRules[7] = ir;
+         return ir;
+
+      case 8:
+         ir = new IntegrationRule(46);
+         ir->AddTetPoints4(0, 1.07952724962210866444e-01, 4.40444181806813866292e-03);
+         ir->AddTetPoints4(4, 1.85109487782586568105e-01, 8.67195792728975463348e-03);
+         ir->AddTetPoints4(8, 4.23165436847673381848e-02, 1.25420935892336655841e-03);
+         ir->AddTetPoints4(12, 3.14181709124039088010e-01, 6.96063047615581593358e-03);
+         ir->AddTetPoints6(16, 4.35591328583830206256e-01, 6.04682171021813687217e-03);
+         ir->AddTetPoints12(22, 2.14339301271305737728e-02, 7.17464063426308307214e-01,
+                            1.19281714847407210867e-03);
+         ir->AddTetPoints12(34, 2.04139333876029116510e-01, 5.83797378302144398532e-01,
+                            2.57558102516005586052e-03);
+         ir->SetOrder(8);
+         TetrahedronIntRules[8] = ir;
+         return ir;
+
+      case 9:
+         ir = new IntegrationRule(59);
+         ir->AddTetMidPoint(0, 9.66842481874670943431e-03);
+         ir->AddTetPoints4(1, 6.19817086544571793638e-10, 1.07198802932093984424e-05);
+         ir->AddTetPoints4(5, 1.60774535395261597426e-01, 3.86222307707090968185e-03);
+         ir->AddTetPoints4(9, 3.22276521821420969260e-01, 4.92715205590488116577e-03);
+         ir->AddTetPoints4(13, 4.51089183454135844720e-02, 1.34399666326936377374e-03);
+         ir->AddTetPoints6(17, 3.87703453995623947836e-01, 6.35568001728374458448e-03);
+         ir->AddTetPoints12(23, 4.58871448752459276665e-01, 7.97025232620401369310e-02,
+                            1.39740369971642539558e-03);
+         ir->AddTetPoints12(35, 3.37758706853386048152e-02, 7.18350326442074527122e-01,
+                            1.70575989212422133613e-03);
+         ir->AddTetPoints12(47, 1.83641369809927956780e-01, 5.98301349801968918030e-01,
+                            3.42081932799802312939e-03);
+         ir->SetOrder(9);
+         TetrahedronIntRules[9] = ir;
+         return ir;
+
+      case 10:
+         ir = new IntegrationRule(81);
+         ir->AddTetMidPoint(0, 7.89996225933678984654e-03);
+         ir->AddTetPoints4(1, 3.12250068695188676138e-01, 4.48950999871145037950e-03);
+         ir->AddTetPoints4(5, 1.14309653857346149586e-01, 1.64485995279889710662e-03);
+         ir->AddTetPoints12(9, 4.10430739218965501269e-01, 1.65486025619611065718e-01,
+                            1.89898020336587186781e-03);
+         ir->AddTetPoints12(21, 6.13800882479076381770e-03, 9.42988767345204870196e-01,
+                            6.03240573898756009806e-05);
+         ir->AddTetPoints12(33, 1.21050181145589408338e-01, 4.77190379904280370660e-01,
+                            4.28995533007601147213e-03);
+         ir->AddTetPoints12(45, 3.27794682164426753879e-02, 5.94256269480006982242e-01,
+                            1.68931194662596552945e-03);
+         ir->AddTetPoints12(57, 3.24852815648231096901e-02, 8.01177284658344368573e-01,
+                            1.09602454617265063913e-03);
+         ir->AddTetPoints12(69, 1.74979342183939068356e-01, 6.28071845475365986289e-01,
+                            2.15117263314366490706e-03);
+         ir->SetOrder(10);
+         TetrahedronIntRules[10] = ir;
+         return ir;
+
+      case 11:
+         ir = new IntegrationRule(96);
+         ir->AddTetPoints4(0, 2.71527207067321363354e-02, 3.30755017786941475644e-04);
+         ir->AddTetPoints4(4, 7.29513610462571016058e-02, 1.27724462275054500421e-03);
+         ir->AddTetPoints4(8, 1.16306248902001030388e-01, 2.22195840281977797376e-03);
+         ir->AddTetPoints4(12, 1.79873804986097840519e-01, 3.55549424791121128006e-03);
+         ir->AddTetPoints4(16, 2.90224794862315171873e-01, 4.27767411104971236741e-03);
+         ir->AddTetPoints4(20, 3.25420936748619160639e-01, 2.29560465583227143668e-03);
+         ir->AddTetPoints6(24, 4.99998725049884129579e-01, 1.95152894059845476377e-04);
+         ir->AddTetPoints6(30, 3.94300142842090972639e-01, 4.13762713030314983886e-03);
+         ir->AddTetPoints12(36, 1.53994139264412854828e-02, 8.20202176629804657892e-01,
+                            3.44113207868302869216e-04);
+         ir->AddTetPoints12(48, 4.36843254717693696421e-02, 6.27516751622257062948e-01,
+                            2.00540889524405963051e-03);
+         ir->AddTetPoints12(60, 1.32316796082697751835e-01, 7.35366407834604496330e-01,
+                            5.92160675031106853265e-04);
+         ir->AddTetPoints12(72, 2.14430354900043917965e-01, 5.31595425719235903372e-01,
+                            3.02373698028425954079e-03);
+         ir->AddTetPoints12(84, 4.39586615093850330283e-01, 1.15789732843376125260e-01,
+                            1.10416876556284249307e-03);
+         ir->SetOrder(11);
+         TetrahedronIntRules[11] = ir;
+         return ir;
+
+      case 12:
+         ir = new IntegrationRule(123);
+         ir->AddTetMidPoint(0, 3.73841522662751247000e-03);
+         ir->AddTetPoints4(1, 1.87550512633127830497e-02, 1.42695998696545141987e-04);
+         ir->AddTetPoints4(5, 1.08129536920462676619e-01, 2.21101382522646480733e-03);
+         ir->AddTetPoints4(9, 2.00131676822545012673e-01, 1.02716311841773611131e-03);
+         ir->AddTetPoints4(13, 3.00854293538076578152e-01, 3.84627572131096594557e-03);
+         ir->AddTetPoints4(17, 3.33333333333333259318e-01, 3.93263480259983290444e-04);
+         ir->AddTetPoints6(21, 4.61659950214442116323e-01, 3.64372815936188636666e-04);
+         ir->AddTetPoints12(27, 1.44707549187619299857e-02, 8.12825119403836504617e-01,
+                            2.78228183082975693598e-04);
+         ir->AddTetPoints12(39, 1.93543398987769954545e-02, 6.01428742996530152354e-01,
+                            5.44744358748572190913e-04);
+         ir->AddTetPoints12(51, 7.79277628532308863640e-02, 8.27642519452021385717e-01,
+                            4.96603607085240776955e-04);
+         ir->AddTetPoints12(63, 1.22055870746741623734e-01, 4.77806520042316273944e-01,
+                            3.50252598711278933380e-03);
+         ir->AddTetPoints12(75, 2.47870739372197945727e-01, 4.77761799116294072487e-01,
+                            1.92951947508030146987e-03);
+         ir->AddTetPoints12(87, 4.29731509588804683197e-01, 1.17747435901101149547e-01,
+                            1.59458000732484511328e-03);
+         ir->AddTetPoints24(99, 6.53037808968305988344e-01, 2.26776739658831050228e-01,
+                            9.77349816032284102185e-02, 1.25441443948160597475e-03);
+         ir->SetOrder(12);
+         TetrahedronIntRules[12] = ir;
+         return ir;
+
+      case 13:
+         ir = new IntegrationRule(145);
+         ir->AddTetMidPoint(0, 4.65163625751287973520e-03);
+         ir->AddTetPoints4(1, 1.83047574861928130652e-02, 1.11328544338301012079e-04);
+         ir->AddTetPoints4(5, 1.79015082630022803745e-01, 3.21788310144650391634e-03);
+         ir->AddTetPoints4(9, 3.29615853754448240309e-01, 1.11613482625956810662e-03);
+         ir->AddTetPoints6(13, 4.84258919196047465938e-01, 4.22777636660235864308e-04);
+         ir->AddTetPoints6(19, 4.37693799377281589358e-01, 1.42741910394635984974e-03);
+         ir->AddTetPoints12(25, 1.68580250819503341120e-02, 7.16622722155387803511e-01,
+                            3.27226375772531215529e-04);
+         ir->AddTetPoints12(37, 2.35257182448596058322e-02, 8.50600927605402956644e-01,
+                            4.21131904215649394228e-04);
+         ir->AddTetPoints12(49, 6.94418950495464537553e-02, 7.08574314820604622689e-01,
+                            9.98940835257684659268e-04);
+         ir->AddTetPoints12(61, 9.37005272821476720146e-02, 5.63456727822489344959e-01,
+                            1.88623398086488818989e-03);
+         ir->AddTetPoints12(73, 1.25885360164042447995e-01, 7.40105985667665167149e-01,
+                            5.36847192210583730974e-04);
+         ir->AddTetPoints12(85, 2.17955270547737667286e-01, 5.35290625276012344003e-01,
+                            1.79931440889047528954e-03);
+         ir->AddTetPoints12(97, 3.54663455472783883948e-01, 2.00014910210617791186e-01,
+                            2.91862732938839245650e-03);
+         ir->AddTetPoints12(109, 4.16689287657038387458e-01, 1.52054854976777675812e-01,
+                            9.62750257012783515476e-04);
+         ir->AddTetPoints24(121, 6.06652560730350010054e-01, 3.04158433676372519372e-01,
+                            7.79465622318078477093e-02, 6.21649861415869242447e-04);
+         ir->SetOrder(13);
+         TetrahedronIntRules[13] = ir;
+         return ir;
+
+      case 14:
+         // Chuluunbaatar et al. 2022: 175 pts, 1xCent + 6xS31 + 1xS22 + 10xS211 + 1xS1111
+         ir = new IntegrationRule(175);
+         ir->AddTetMidPoint(0, 2.79630622899013732072e-03);
+         ir->AddTetPoints4(1, 3.33328696010048830534e-01, 1.46917540892973303920e-04);
+         ir->AddTetPoints4(5, 2.03700979179134489261e-01, 1.62781576883158380503e-03);
+         ir->AddTetPoints4(9, 4.23119120487503441730e-02, 3.41825853298758448786e-04);
+         ir->AddTetPoints4(13, 1.66911321524259963212e-02, 6.86936532495300726303e-05);
+         ir->AddTetPoints4(17, 1.64429779556425403886e-01, 2.04589809259575743788e-03);
+         ir->AddTetPoints4(21, 3.05243130480787605574e-01, 2.91353212326472864671e-03);
+         ir->AddTetPoints6(25, 3.64287147870284933049e-01, 2.86344364876423311192e-03);
+         ir->AddTetPoints12(31, 2.15024351638664623643e-01, 5.09377822427890203372e-01,
+                            1.86604702493200981690e-03);
+         ir->AddTetPoints12(43, 4.08453557824531576781e-01, 2.80291884809145824820e-02,
+                            1.37079721642533879263e-03);
+         ir->AddTetPoints12(55, 2.59921479331125596102e-02, 7.58920443676681433232e-01,
+                            5.79340182017268356431e-04);
+         ir->AddTetPoints12(67, 1.49228115767079897586e-02, 6.12483481308898292106e-01,
+                            3.15838010189281473503e-04);
+         ir->AddTetPoints12(79, 8.37923462693035414617e-02, 8.22678981568125355928e-01,
+                            3.37570057080462679680e-04);
+         ir->AddTetPoints12(91, 2.67966909251860618824e-01, 1.21450733299050331326e-02,
+                            7.33968440516890967273e-04);
+         ir->AddTetPoints12(103, 7.22614743275835913483e-02, 2.93829968686904419162e-01,
+                            1.73940500456261186446e-03);
+         ir->AddTetPoints12(115, 4.61690356122462508548e-01, 6.24312988244191680032e-02,
+                            5.50156605537168688809e-04);
+         ir->AddTetPoints12(127, 1.02556247843651599492e-05, 9.09652220362116237240e-01,
+                            3.74818592914694638193e-05);
+         ir->AddTetPoints12(139, 1.30667193397036723868e-01, 6.88760930866849863108e-01,
+                            1.25403778742792012396e-03);
+         ir->AddTetPoints24(151, 4.96082264783182565887e-03, 1.13153535288820022986e-01,
+                            2.55337379104889128367e-01, 5.28969173366363918341e-04);
+         ir->SetOrder(14);
+         TetrahedronIntRules[14] = ir;
+         return ir;
+
+      case 15:
+         // Chuluunbaatar et al. 2022: 209 pts, 1xCent + 4xS31 + 2xS22 + 11xS211 + 2xS1111
+         ir = new IntegrationRule(209);
+         ir->AddTetMidPoint(0, 1.41781886024826123995e-03);
+         ir->AddTetPoints4(1, 3.28314281102506377863e-01, 8.78039594754075579386e-04);
+         ir->AddTetPoints4(5, 5.95315181460130682378e-02, 6.49096346642090296988e-04);
+         ir->AddTetPoints4(9, 1.79953296856689010097e-01, 2.51315014419990637520e-03);
+         ir->AddTetPoints4(13, 2.87467267398706316506e-01, 1.49510722611222407301e-03);
+         ir->AddTetPoints6(17, 1.63820429539269674102e-01, 1.59662635597863640340e-03);
+         ir->AddTetPoints6(23, 4.49691037286174599696e-01, 5.56101982388976408267e-04);
+         ir->AddTetPoints12(29, 4.00736540413628217205e-01, 1.44084693588290586180e-02,
+                            7.41098096825775301023e-04);
+         ir->AddTetPoints12(41, 9.30917130105696349895e-02, 5.00389099764625755462e-01,
+                            1.48911776902807633377e-03);
+         ir->AddTetPoints12(53, 8.58969713610300000806e-02, 6.54216648304909331735e-01,
+                            1.11933741359922559432e-03);
+         ir->AddTetPoints12(65, 2.10555592437809635520e-01, 6.11819979966393701076e-02,
+                            1.62698522585349981094e-03);
+         ir->AddTetPoints12(77, 8.52418251507089524965e-02, 9.51322614055337932581e-03,
+                            1.79269162929391381043e-04);
+         ir->AddTetPoints12(89, 3.39062766530538045595e-02, 1.00939010271196565223e-03,
+                            6.94170071688395131987e-05);
+         ir->AddTetPoints12(101, 3.61923170322900333851e-01, 6.94744848139675630350e-02,
+                            1.59447426536825733780e-03);
+         ir->AddTetPoints12(113, 4.82786943073602314858e-01, 2.98248878093377127463e-02,
+                            1.85989656618890347554e-04);
+         ir->AddTetPoints12(125, 1.67482762532157707092e-02, 8.28143120169573809797e-01,
+                            2.46974837616148544094e-04);
+         ir->AddTetPoints12(137, 1.77918303634979659000e-02, 2.95058306317452390122e-01,
+                            3.64978786118624204133e-04);
+         ir->AddTetPoints12(149, 2.22052218944333024098e-01, 5.47335431979886655185e-01,
+                            5.97708840203148013097e-04);
+         ir->AddTetPoints24(161, 7.01737933129022994905e-01, 1.90468590405707266511e-01,
+                            9.00527571062145620884e-02, 6.09486799192013577881e-04);
+         ir->AddTetPoints24(185, 9.30456155647334665071e-02, 3.39714197260826189506e-01,
+                            1.75628396157984228987e-02, 7.07458692200529210524e-04);
+         ir->SetOrder(15);
+         TetrahedronIntRules[15] = ir;
+         return ir;
+
+      case 16:
+         // Chuluunbaatar et al. 2022: 248 pts, 8xS31 + 2xS22 + 11xS211 + 3xS1111
+         ir = new IntegrationRule(248);
+         ir->AddTetPoints4(0, 3.27237393634992601577e-01, 1.02720766161859349518e-03);
+         ir->AddTetPoints4(4, 1.70006239733430930539e-01, 1.65526299995553852033e-03);
+         ir->AddTetPoints4(8, 1.15524594427552973475e-01, 9.38857005487496389280e-04);
+         ir->AddTetPoints4(12, 2.91444830780401391290e-02, 6.51307095609853886316e-05);
+         ir->AddTetPoints4(16, 2.99333264802760234957e-01, 1.67595204112568215392e-03);
+         ir->AddTetPoints4(20, 3.08156348381425804206e-01, 5.31861976759345615219e-04);
+         ir->AddTetPoints4(24, 1.50096498602994791322e-02, 5.70730781338430579624e-05);
+         ir->AddTetPoints4(28, 2.15377318942399170743e-01, 1.98341225672248695419e-03);
+         ir->AddTetPoints6(32, 4.31794349434656055120e-01, 1.33750640425309239717e-03);
+         ir->AddTetPoints6(38, 3.51744151127164061954e-01, 1.76194827177588689456e-03);
+         ir->AddTetPoints12(44, 8.29503118270854405969e-03, 1.02752005688054329213e-01,
+                            7.68034210054447698257e-05);
+         ir->AddTetPoints12(56, 4.80211290069074772657e-02, 1.31255041819827861227e-01,
+                            6.11232548742599826781e-04);
+         ir->AddTetPoints12(68, 1.58330800578366723275e-02, 7.34396180497698725098e-01,
+                            2.28071918651308634196e-04);
+         ir->AddTetPoints12(80, 2.33860521976982954628e-01, 5.24417475091189966285e-01,
+                            5.40957583749601829924e-04);
+         ir->AddTetPoints12(92, 4.03140399383019043533e-01, 1.61507733387133249614e-02,
+                            7.92724342585451077596e-04);
+         ir->AddTetPoints12(104, 4.62664186044103586948e-01, 8.10608626417101511830e-03,
+                            2.79154547643383486085e-04);
+         ir->AddTetPoints12(116, 9.54740058566225929804e-02, 2.29727696324949964835e-01,
+                            9.08138512826854073234e-04);
+         ir->AddTetPoints12(128, 5.22644355194657739272e-02, 2.65023714835432855352e-01,
+                            5.22825831653034723938e-04);
+         ir->AddTetPoints12(140, 1.54579380484822833525e-02, 3.89592797271225033118e-01,
+                            2.70158998322436651619e-04);
+         ir->AddTetPoints12(152, 6.51049846146104782552e-02, 1.06703843590993385781e-02,
+                            1.88285026959176838299e-04);
+         ir->AddTetPoints12(164, 1.54103896531334966236e-01, 6.61715834360067312048e-01,
+                            9.04088475565851504463e-04);
+         ir->AddTetPoints24(176, 3.06401667521507548031e-01, 9.60517630472854377910e-02,
+                            5.82737849734082380415e-01, 5.64378019946093099565e-04);
+         ir->AddTetPoints24(200, 7.52465510383990981991e-02, 7.45338308409307703783e-01,
+                            1.68928693209220324653e-04, 1.50966565378719797755e-04);
+         ir->AddTetPoints24(224, 6.67856954025341370551e-02, 1.77429946613237937703e-01,
+                            4.69596074566060506239e-01, 1.47055596469915315222e-03);
+         ir->SetOrder(16);
+         TetrahedronIntRules[16] = ir;
+         return ir;
+
+      case 17:
+         // Chuluunbaatar et al. 2022: 284 pts, 8xS31 + 2xS22 + 14xS211 + 3xS1111
+         ir = new IntegrationRule(284);
+         ir->AddTetPoints4(0, 7.70317217555786387662e-02, 5.61182432136912863994e-04);
+         ir->AddTetPoints4(4, 3.33178098937441047322e-01, 1.00689770519544758830e-04);
+         ir->AddTetPoints4(8, 4.70791056455278841830e-02, 3.67647177915098600219e-05);
+         ir->AddTetPoints4(12, 3.04818016813530989761e-01, 1.65889979279151768450e-03);
+         ir->AddTetPoints4(16, 1.30944391509640850613e-01, 1.44088986921434188127e-03);
+         ir->AddTetPoints4(20, 1.92535395691919936079e-01, 9.15395452245713805300e-04);
+         ir->AddTetPoints4(24, 2.76657577444746005657e-01, 1.54901732083429399985e-03);
+         ir->AddTetPoints4(28, 1.25812395975189866837e-02, 3.37712309004839736381e-05);
+         ir->AddTetPoints6(32, 3.79480026881957605706e-03, 6.09736907157372445766e-05);
+         ir->AddTetPoints6(38, 6.63574479091031538269e-02, 9.84476156008183339238e-04);
+         ir->AddTetPoints12(44, 1.55274078991054307469e-02, 1.62632006416904367763e-01,
+                            1.72804764846827806904e-04);
+         ir->AddTetPoints12(56, 1.50826606630984655366e-01, 2.85203367985520928052e-01,
+                            1.47139558752681400000e-03);
+         ir->AddTetPoints12(68, 2.15452134252546806392e-01, 4.98848586166224794436e-01,
+                            1.39306363547645069810e-03);
+         ir->AddTetPoints12(80, 8.76098177043343750992e-02, 5.57666868254345748923e-01,
+                            1.10287787958478406859e-03);
+         ir->AddTetPoints12(92, 2.76970146665180327883e-01, 4.27355049241211759625e-01,
+                            7.68445912031412610432e-04);
+         ir->AddTetPoints12(104, 7.84854004483451911378e-02, 8.28808438721931994841e-01,
+                            2.88969312852391376975e-04);
+         ir->AddTetPoints12(116, 4.16031236751370603333e-01, 5.86692256093802096822e-03,
+                            3.64077087845785364682e-04);
+         ir->AddTetPoints12(128, 3.11418578536613735799e-03, 2.87446864291519776913e-01,
+                            3.72979694270385682886e-05);
+         ir->AddTetPoints12(140, 1.48006973486492082737e-01, 3.23952556344107162056e-02,
+                            8.00305827281323160088e-04);
+         ir->AddTetPoints12(152, 1.43033359605475689225e-02, 6.71799737186767331742e-02,
+                            1.04156882366980408738e-04);
+         ir->AddTetPoints12(164, 4.67432600299733047589e-01, 1.06651363448138844503e-02,
+                            3.28661937749209721524e-04);
+         ir->AddTetPoints12(176, 3.84385292133539946402e-01, 1.72438767641856061097e-01,
+                            1.30629277435961670649e-03);
+         ir->AddTetPoints12(188, 4.91933575124020999736e-02, 1.88887052205218147760e-01,
+                            6.32377286180225181393e-04);
+         ir->AddTetPoints12(200, 2.12339226453523544080e-01, 4.66228370030305223209e-03,
+                            2.79290741478308349437e-04);
+         ir->AddTetPoints24(212, 5.56903978599597615506e-01, 3.03486206973905936479e-01,
+                            1.18763501187465259079e-01, 6.89373805061415279201e-04);
+         ir->AddTetPoints24(236, 7.72626757757003540528e-02, 7.24630018034681633310e-01,
+                            1.98007306189710574618e-01, 1.50128885728721429014e-04);
+         ir->AddTetPoints24(260, 3.94151894733046209707e-02, 1.37686205384732439361e-02,
+                            3.27821577026260191356e-01, 2.69135394730690453036e-04);
+         ir->SetOrder(17);
+         TetrahedronIntRules[17] = ir;
+         return ir;
+
+      case 18:
+         // Chuluunbaatar et al. 2022: 343 pts, 1xCent + 6xS31 + 1xS22 + 18xS211 + 4xS1111
+         ir = new IntegrationRule(343);
+         ir->AddTetMidPoint(0, 1.50320520665968271855e-03);
+         ir->AddTetPoints4(1, 1.48031283019549930735e-01, 1.29183701010426432026e-03);
+         ir->AddTetPoints4(5, 9.18424577295562372115e-02, 5.65165600131921114398e-04);
+         ir->AddTetPoints4(9, 1.21731006846268821620e-02, 2.96638661291863846498e-05);
+         ir->AddTetPoints4(13, 2.96287086243479214076e-01, 1.68021599975901742008e-03);
+         ir->AddTetPoints4(17, 3.26360945420223702573e-01, 5.68549421501223146459e-04);
+         ir->AddTetPoints4(21, 2.16789137320780644913e-01, 5.55087845531530756776e-04);
+         ir->AddTetPoints6(25, 4.02614199568341046831e-01, 9.23257682834909085973e-04);
+         ir->AddTetPoints12(31, 4.40000294606430919497e-01, 2.67115252330815747261e-02,
+                            5.89236226881148018354e-04);
+         ir->AddTetPoints12(43, 4.12210360146149590310e-01, 1.74700018752273367184e-01,
+                            2.20201645262970173086e-04);
+         ir->AddTetPoints12(55, 3.72351275734651154803e-01, 2.01036053559202482210e-01,
+                            1.19114067751767893286e-03);
+         ir->AddTetPoints12(67, 2.69242251358920825499e-01, 4.49869807112305730712e-01,
+                            4.91465288515259167076e-04);
+         ir->AddTetPoints12(79, 9.76627002277863226487e-02, 5.25608170470769464622e-01,
+                            9.64486286268097844226e-04);
+         ir->AddTetPoints12(91, 7.95369840699704060138e-03, 9.21057037109250575924e-01,
+                            3.50539800669945179434e-05);
+         ir->AddTetPoints12(103, 1.87267495112264620305e-01, 6.22654876406694146596e-01,
+                            2.16974411183503681708e-04);
+         ir->AddTetPoints12(115, 1.11072676172302167719e-01, 7.68063140393386190041e-01,
+                            2.34282910136374296871e-04);
+         ir->AddTetPoints12(127, 7.28479245819699250397e-02, 6.49722949416110195919e-01,
+                            4.87753175028903700802e-04);
+         ir->AddTetPoints12(139, 4.35242030819264283381e-02, 1.17645467635705727738e-01,
+                            3.42108745586317604826e-04);
+         ir->AddTetPoints12(151, 4.75773867838972852606e-01, 4.07808063375027368691e-02,
+                            1.54370387724311262630e-04);
+         ir->AddTetPoints12(163, 1.34493684207502642303e-02, 7.10473230757141527292e-01,
+                            1.53084245505138950502e-04);
+         ir->AddTetPoints12(175, 1.55188126953707594691e-01, 4.80062353992693063853e-02,
+                            7.41762550201790354931e-04);
+         ir->AddTetPoints12(187, 9.79636346189126545891e-03, 3.98937047199835026490e-01,
+                            9.70189072253606300464e-05);
+         ir->AddTetPoints12(199, 2.22972555180978554423e-01, 4.76498622927509052349e-01,
+                            1.23606250864963739498e-03);
+         ir->AddTetPoints12(211, 4.21260289724883496554e-02, 5.74400719147259319897e-01,
+                            5.42935360416797459064e-04);
+         ir->AddTetPoints12(223, 5.33405089760383491204e-02, 8.80640211270697026436e-01,
+                            1.37609067910292992955e-04);
+         ir->AddTetPoints12(235, 1.61516953402295604381e-01, 3.97825734805866804145e-01,
+                            1.38360407936932868454e-03);
+         ir->AddTetPoints24(247, 1.45407574632878761056e-01, 2.91164813093958863011e-01,
+                            2.48113931665566757323e-02, 7.08660104525119741853e-04);
+         ir->AddTetPoints24(271, 3.05950727633923398596e-03, 8.24555339410965038027e-01,
+                            2.68565028804892240444e-02, 7.44949831353088352858e-05);
+         ir->AddTetPoints24(295, 6.97926614756394281258e-01, 2.13661027507205059095e-01,
+                            1.66604565123493510159e-02, 3.48774332060316084436e-04);
+         ir->AddTetPoints24(319, 7.44110286479428006956e-02, 3.30529031579615995007e-01,
+                            5.94802303330229431566e-01, 1.27738536486342427233e-04);
+         ir->SetOrder(18);
+         TetrahedronIntRules[18] = ir;
+         return ir;
+
+      case 19:
+         // Chuluunbaatar et al. 2022: 383 pts, 1xCent + 7xS31 + 3xS22 + 18xS211 + 5xS1111
+         ir = new IntegrationRule(383);
+         ir->AddTetMidPoint(0, 1.63415516118113374362e-03);
+         ir->AddTetPoints4(1, 1.99329047511150186933e-01, 1.40358969707281050661e-03);
+         ir->AddTetPoints4(5, 3.19645815662209620278e-01, 7.59091701642911130359e-04);
+         ir->AddTetPoints4(9, 1.28431938718745111000e-02, 3.39837309564992975973e-05);
+         ir->AddTetPoints4(13, 4.48982322308715264825e-02, 2.64426237344559213801e-04);
+         ir->AddTetPoints4(17, 1.36956642483832574664e-01, 7.02856031175181001670e-04);
+         ir->AddTetPoints4(21, 2.90023140379987831583e-01, 1.36565197140369625796e-03);
+         ir->AddTetPoints4(25, 9.74727471850612287030e-02, 5.49967641751253539552e-04);
+         ir->AddTetPoints6(29, 9.71505706534206842084e-02, 8.76433367036966496331e-04);
+         ir->AddTetPoints6(35, 3.43201766155373844125e-01, 1.42791869569969145752e-03);
+         ir->AddTetPoints6(41, 4.81048682529186311108e-01, 2.14104659346860782621e-04);
+         ir->AddTetPoints12(47, 1.65183633185142647593e-01, 6.20343271948313068620e-01,
+                            7.66624121398570487762e-04);
+         ir->AddTetPoints12(59, 6.34076230480007801971e-02, 1.00712056526788096511e-06,
+                            5.83669782740338187726e-05);
+         ir->AddTetPoints12(71, 1.14358224224068777061e-03, 4.00863727471189756901e-01,
+                            2.15179213304474853136e-05);
+         ir->AddTetPoints12(83, 1.72841898844304682481e-02, 2.80974353964125900252e-01,
+                            2.11660243173845957895e-04);
+         ir->AddTetPoints12(95, 3.72064110603054443160e-01, 1.99705629942815821032e-01,
+                            8.75485553265252145448e-04);
+         ir->AddTetPoints12(107, 5.20242736384851911513e-02, 1.37531998164526964024e-01,
+                            2.41875978548309640689e-04);
+         ir->AddTetPoints12(119, 1.60391938988630439189e-01, 6.93856268828658799552e-03,
+                            2.41421983084743403854e-04);
+         ir->AddTetPoints12(131, 2.04009311788170112981e-03, 1.85612033720361502276e-01,
+                            2.06414634761428247161e-05);
+         ir->AddTetPoints12(143, 5.27148535043930888122e-02, 3.66275984765305206992e-01,
+                            5.92721175866269185152e-04);
+         ir->AddTetPoints12(155, 2.43716736043213727525e-01, 4.92302726125094514131e-01,
+                            5.87805221231469288319e-04);
+         ir->AddTetPoints12(167, 2.35953750535251915998e-01, 4.36147999140300668408e-01,
+                            1.13870552683716466137e-03);
+         ir->AddTetPoints12(179, 3.76396525093455058819e-01, 2.39487568074092604942e-01,
+                            3.83776606890474675932e-04);
+         ir->AddTetPoints12(191, 1.01345934656195998946e-01, 7.75724008450901503231e-01,
+                            3.13399374751491786861e-04);
+         ir->AddTetPoints12(203, 1.03978576485678226443e-02, 9.15876371248755760668e-01,
+                            5.00269118618190970684e-05);
+         ir->AddTetPoints12(215, 4.34825905512194077485e-01, 1.54748703679767566493e-02,
+                            5.11025086732051357814e-04);
+         ir->AddTetPoints12(227, 1.21014927057761692564e-01, 5.16611295841134299245e-01,
+                            1.01828426898903845119e-03);
+         ir->AddTetPoints12(239, 2.02221519510974792611e-02, 1.24848861415461573343e-01,
+                            1.43839758373707761420e-04);
+         ir->AddTetPoints12(251, 6.45349753497565792326e-02, 6.58797428082353198064e-01,
+                            5.65395165437599170333e-04);
+         ir->AddTetPoints24(263, 5.41751356566992220420e-02, 1.84132753560833639650e-01,
+                            7.55314661874476711567e-01, 1.46482060260151955187e-04);
+         ir->AddTetPoints24(287, 8.81662316742434920558e-02, 6.34269181940578685719e-01,
+                            1.71630199679625762565e-02, 3.35574461050504140036e-04);
+         ir->AddTetPoints24(311, 2.99876324799488391815e-01, 5.12011805461436986242e-01,
+                            1.39945756017624101109e-01, 6.87105878169750922298e-04);
+         ir->AddTetPoints24(335, 5.43033048543508201078e-01, 3.98941499659789464149e-03,
+                            2.96197429831241032527e-01, 1.95466436126630728823e-04);
+         ir->AddTetPoints24(359, 5.64748804926985426000e-01, 5.04233375578317308263e-02,
+                            5.38328049657370907161e-03, 1.64230458281612438400e-04);
+         ir->SetOrder(19);
+         TetrahedronIntRules[19] = ir;
+         return ir;
+
+      case 20:
+         // Chuluunbaatar et al. 2022: 441 pts, 1xCent + 8xS31 + 4xS22 + 20xS211 + 6xS1111
+         ir = new IntegrationRule(441);
+         ir->AddTetMidPoint(0, 1.18189152746071531389e-03);
+         ir->AddTetPoints4(1, 1.44398440418483348102e-01, 9.66793543666131616025e-04);
+         ir->AddTetPoints4(5, 7.58537944731913719304e-03, 7.60496594911488061041e-06);
+         ir->AddTetPoints4(9, 2.92814880923072839991e-01, 1.15590090717039508002e-03);
+         ir->AddTetPoints4(13, 3.21283351882928780441e-01, 6.88636083704610908220e-04);
+         ir->AddTetPoints4(17, 1.99126561548209762842e-01, 1.13432796616464034133e-03);
+         ir->AddTetPoints4(21, 9.94395843777090698845e-02, 4.86016284904959409725e-04);
+         ir->AddTetPoints4(25, 5.64411054542164752901e-02, 3.22610846546138882538e-04);
+         ir->AddTetPoints4(29, 2.29655958568571322287e-02, 5.81298999656466752642e-05);
+         ir->AddTetPoints6(33, 1.79141319969986889671e-01, 1.09529243625682828553e-03);
+         ir->AddTetPoints6(39, 1.28302591207222288494e-01, 5.73999808941768881014e-04);
+         ir->AddTetPoints6(45, 1.45390267831490144212e-02, 1.87025723866918939901e-04);
+         ir->AddTetPoints6(51, 4.21353125034043096697e-01, 7.00971843927932197066e-04);
+         ir->AddTetPoints12(57, 2.31009838674045342444e-01, 4.40089079468850674637e-01,
+                            9.76079610667369892280e-04);
+         ir->AddTetPoints12(69, 5.34968850305569260106e-03, 1.93845423004036843118e-01,
+                            3.05362840831512903964e-05);
+         ir->AddTetPoints12(81, 1.24450062776632008887e-01, 2.68040213994591769442e-01,
+                            8.86819612544235340128e-04);
+         ir->AddTetPoints12(93, 2.04325810970697151203e-02, 2.58308114426872181824e-01,
+                            1.92969809694827680807e-04);
+         ir->AddTetPoints12(105, 4.82828231821251577238e-02, 5.80396926577266859815e-03,
+                            6.19875387085781813859e-05);
+         ir->AddTetPoints12(117, 2.89149787325270579696e-01, 4.18198665244226719384e-01,
+                            1.83280536557154684341e-04);
+         ir->AddTetPoints12(129, 4.46052961749180063022e-02, 1.63432451082933805075e-01,
+                            3.33781952353497124181e-04);
+         ir->AddTetPoints12(141, 2.70746696244779198881e-03, 9.47473449934223443947e-01,
+                            9.76902870004440518315e-06);
+         ir->AddTetPoints12(153, 1.81599434536722420530e-01, 5.60679256816125279328e-02,
+                            7.16163033436471277611e-04);
+         ir->AddTetPoints12(165, 7.98241847716316815786e-02, 2.24797220241435974364e-01,
+                            6.67198343081130593700e-04);
+         ir->AddTetPoints12(177, 1.46751424308700709198e-02, 1.05996716787954137207e-01,
+                            1.02112432519835052438e-04);
+         ir->AddTetPoints12(189, 1.47988480867858707146e-01, 6.98383249617804735543e-01,
+                            1.65331156535892409174e-04);
+         ir->AddTetPoints12(201, 4.52421884404454466289e-01, 8.04532263339534647884e-02,
+                            3.75572061072812404120e-04);
+         ir->AddTetPoints12(213, 4.82988925439242506449e-03, 3.56408995372303527560e-01,
+                            3.70372995907359520520e-05);
+         ir->AddTetPoints12(225, 3.96075105354866952023e-01, 1.93643489260524576112e-01,
+                            4.20355142354041121707e-04);
+         ir->AddTetPoints12(237, 2.11473197416018027228e-01, 5.66485319771568907044e-01,
+                            3.20737452814357003415e-04);
+         ir->AddTetPoints12(249, 2.56953469781508958558e-01, 4.54914546979866607490e-01,
+                            5.44699807144207316482e-04);
+         ir->AddTetPoints12(261, 3.63762446007509787638e-01, 7.07704644682126682298e-02,
+                            8.71724711789707298014e-04);
+         ir->AddTetPoints12(273, 4.34084693566413395982e-02, 5.61826818484091217165e-01,
+                            4.66716966955154613419e-04);
+         ir->AddTetPoints12(285, 1.26096840063810999855e-01, 4.15826803180920218095e-02,
+                            4.05374493922667372432e-04);
+         ir->AddTetPoints24(297, 3.19830665436792060952e-01, 4.36032333155288651105e-02,
+                            5.02224495466759290885e-01, 7.04309544520175120040e-04);
+         ir->AddTetPoints24(321, 6.93104765295092647634e-04, 7.61588104432530443866e-01,
+                            4.88127934219945436300e-02, 6.22004488693067759562e-05);
+         ir->AddTetPoints24(345, 3.01059509765821443905e-03, 4.68001522754562040984e-02,
+                            5.92923309754523120141e-01, 1.11516752988944994277e-04);
+         ir->AddTetPoints24(369, 8.05046770737637640281e-01, 1.15008082676269607347e-01,
+                            6.71446262920421810261e-02, 1.37946350993749402127e-04);
+         ir->AddTetPoints24(393, 3.27837344763098725853e-01, 1.41845459805815643506e-01,
+                            3.73982332962941683638e-03, 1.75610557117139728066e-04);
+         ir->AddTetPoints24(417, 1.69431918115453827856e-02, 9.30651836894259287813e-02,
+                            6.46141331822991715761e-01, 3.76832469454361519961e-04);
+         ir->SetOrder(20);
+         TetrahedronIntRules[20] = ir;
+         return ir;
+
+      default:
+         // Grundmann-Moller fallback for orders beyond tabulated rules
+         int i = (Order / 2) * 2 + 1;   // closest odd >= Order
+         AllocIntRule(TetrahedronIntRules, i);
+         ir = new IntegrationRule;
+         ir->GrundmannMollerSimplexRule(i/2, 3);
+         if (!TetrahedronIntRules[i-1]) { TetrahedronIntRules[i-1] = ir; }
+         TetrahedronIntRules[i] = ir;
+         return ir;
+   }
+}
+
+
 // Integration rules for reference pyramid
 IntegrationRule *IntegrationRules::PyramidIntegrationRule(int Order)
 {

fem/intrules.hpp

@@ -219,6 +219,39 @@ private:
       AddTetPoints6(off + 6, a, b, c, weight);
    }
 
+   // add all 24 permutations of (a,b,c,d) where a+b+c+d = 1, all distinct
+   void AddTetPoints24(const int off, const real_t a, const real_t b,
+                       const real_t c, const real_t weight)
+   {
+      const real_t d = 1. - a - b - c;
+      // all 24 permutations of 4 distinct barycentric coordinates
+      // permuting which coordinate goes to x, y, z (4th is 1-x-y-z)
+      IntPoint(off +  0).Set(a, b, c, weight);
+      IntPoint(off +  1).Set(a, b, d, weight);
+      IntPoint(off +  2).Set(a, c, b, weight);
+      IntPoint(off +  3).Set(a, c, d, weight);
+      IntPoint(off +  4).Set(a, d, b, weight);
+      IntPoint(off +  5).Set(a, d, c, weight);
+      IntPoint(off +  6).Set(b, a, c, weight);
+      IntPoint(off +  7).Set(b, a, d, weight);
+      IntPoint(off +  8).Set(b, c, a, weight);
+      IntPoint(off +  9).Set(b, c, d, weight);
+      IntPoint(off + 10).Set(b, d, a, weight);
+      IntPoint(off + 11).Set(b, d, c, weight);
+      IntPoint(off + 12).Set(c, a, b, weight);
+      IntPoint(off + 13).Set(c, a, d, weight);
+      IntPoint(off + 14).Set(c, b, a, weight);
+      IntPoint(off + 15).Set(c, b, d, weight);
+      IntPoint(off + 16).Set(c, d, a, weight);
+      IntPoint(off + 17).Set(c, d, b, weight);
+      IntPoint(off + 18).Set(d, a, b, weight);
+      IntPoint(off + 19).Set(d, a, c, weight);
+      IntPoint(off + 20).Set(d, b, a, weight);
+      IntPoint(off + 21).Set(d, b, c, weight);
+      IntPoint(off + 22).Set(d, c, a, weight);
+      IntPoint(off + 23).Set(d, c, b, weight);
+   }
+
 public:
    IntegrationRule() :
       Array<IntegrationPoint>() { }
@@ -417,6 +450,14 @@ public:
    static int CheckOpen(int type);
 };
 
+/// Simplex (triangle/tetrahedron) quadrature rule selection.
+enum class SimplexQuadrature
+{
+   Legacy           = 0,  ///< Original MFEM rules (may have negative weights)
+   PositiveWeights  = 1,  ///< Positive-weight interior-point rules for simplices
+   Default          = PositiveWeights
+};
+
 /// Container class for integration rules
 class IntegrationRules
 {
@@ -426,6 +467,9 @@ private:
    /// segment, square, and cube geometries
    const int quad_type;
 
+   /// Determines which simplex (triangle/tetrahedron) quadrature rules to use
+   const SimplexQuadrature simplex_type;
+
    int own_rules, refined;
 
    Array<IntegrationRule *> PointIntRules;
@@ -471,11 +515,17 @@ private:
    IntegrationRule *PrismIntegrationRule(int Order);
    IntegrationRule *CubeIntegrationRule(int Order);
 
+   /// Witherden-Vincent triangle quadrature rules (all-positive weights)
+   IntegrationRule *TrianglePositiveIntegrationRule(int Order);
+   /// Witherden-Vincent and Chuluunbaatar tetrahedron quadrature rules
+   IntegrationRule *TetrahedronPositiveIntegrationRule(int Order);
+
 public:
    /// Sets initial sizes for the integration rule arrays, but rules
    /// are defined the first time they are requested with the Get method.
    explicit IntegrationRules(int ref = 0,
-                             int type = Quadrature1D::GaussLegendre);
+                             int type = Quadrature1D::GaussLegendre,
+                             SimplexQuadrature stype = SimplexQuadrature::Default);
 
    /// Returns an integration rule for given GeomType and Order.
    const IntegrationRule &Get(int GeomType, int Order);

fem/pgridfunc.cpp

@@ -1568,6 +1568,39 @@ PLBound ParGridFunction::GetBounds(Vector &lower, Vector &upper,
    return plb;
 }
 
+std::pair<real_t, real_t> ParGridFunction::EstimateFunctionMinimum(
+   const int vdim, const PLBound &plb, const int max_depth,
+   const real_t tol) const
+{
+   std::pair<real_t, real_t> minmax =
+      GridFunction::EstimateFunctionMinimum(vdim, plb, max_depth, tol);
+
+   real_t glob_min_lower = minmax.first;
+   real_t glob_min_upper = minmax.second;
+   MPI_Allreduce(MPI_IN_PLACE, &glob_min_lower, 1,
+                 MFEM_MPI_REAL_T, MPI_MIN, pfes->GetComm());
+   MPI_Allreduce(MPI_IN_PLACE, &glob_min_upper, 1,
+                 MFEM_MPI_REAL_T, MPI_MIN, pfes->GetComm());
+
+   return std::make_pair(glob_min_lower, glob_min_upper);
+}
+
+std::pair<real_t, real_t> ParGridFunction::EstimateFunctionMaximum(
+   const int vdim, const PLBound &plb, const int max_depth,
+   const real_t tol) const
+{
+   std::pair<real_t, real_t> minmax =
+      GridFunction::EstimateFunctionMaximum(vdim, plb, max_depth, tol);
+
+   real_t glob_max_lower = minmax.first;
+   real_t glob_max_upper = minmax.second;
+   MPI_Allreduce(MPI_IN_PLACE, &glob_max_lower, 1,
+                 MFEM_MPI_REAL_T, MPI_MAX, pfes->GetComm());
+   MPI_Allreduce(MPI_IN_PLACE, &glob_max_upper, 1,
+                 MFEM_MPI_REAL_T, MPI_MAX, pfes->GetComm());
+   return std::make_pair(glob_max_lower, glob_max_upper);
+}
+
 } // namespace mfem
 
 #endif // MFEM_USE_MPI

fem/pgridfunc.hpp

@@ -609,6 +609,18 @@ public:
    PLBound GetBounds(Vector &lower, Vector &upper,
                      const int ref_factor=1, const int vdim=-1) const override;
 
+   /** @brief Estimate the GridFunction minimum across all elements. */
+   std::pair<real_t, real_t> EstimateFunctionMinimum(const int vdim,
+                                                     const PLBound &plb,
+                                                     const int max_depth,
+                                                     const real_t tol) const override;
+
+   /** @brief Estimate the GridFunction maximum across all elements. */
+   std::pair<real_t, real_t> EstimateFunctionMaximum(const int vdim,
+                                                     const PLBound &plb,
+                                                     const int max_depth,
+                                                     const real_t tol) const override;
+
    /** Save the local portion of the ParGridFunction. This differs from the
        serial GridFunction::Save in that it takes into account the signs of
        the local dofs. */

fem/qfunction.hpp

@@ -271,10 +271,7 @@ inline void QuadratureFunction::GetValues(
    const int s_offset = qspace->Offset(idx);
    const int sl_size = qspace->Offset(idx + 1) - s_offset;
    // Make the values matrix memory an alias of the quadrature function memory
-   Memory<real_t> &values_mem = values.GetMemory();
-   values_mem.Delete();
-   values_mem.MakeAlias(GetMemory(), vdim*s_offset, vdim*sl_size);
-   values.SetSize(vdim, sl_size);
+   values.MakeRef(GetMemory(), vdim*s_offset, vdim, sl_size);
 }
 
 inline void QuadratureFunction::GetValues(

fem/qinterp/det.hpp

@@ -334,17 +334,16 @@ template<int DIM, int SDIM, int D1D, int Q1D>
 QuadratureInterpolator::DetKernelType
 QuadratureInterpolator::DetKernels::Kernel()
 {
-   if (DIM == 1)
+   if constexpr (DIM == 1)
    {
-      if (SDIM == 1) { return internal::quadrature_interpolator::Det1D; }
-      else if (SDIM == 2) { return internal::quadrature_interpolator::Det1DSurface<D1D, Q1D, 2>; }
-      else if (SDIM == 3) { return internal::quadrature_interpolator::Det1DSurface<D1D, Q1D, 3>; }
-      else { MFEM_ABORT(""); }
+      if constexpr (SDIM == 1) { return internal::quadrature_interpolator::Det1D; }
+      else if constexpr (SDIM == 2) { return internal::quadrature_interpolator::Det1DSurface<D1D, Q1D, 2>; }
+      else if constexpr (SDIM == 3) { return internal::quadrature_interpolator::Det1DSurface<D1D, Q1D, 3>; }
    }
-   else if (DIM == 2 && SDIM == 2) { return internal::quadrature_interpolator::Det2D<D1D, Q1D>; }
-   else if (DIM == 2 && SDIM == 3) { return internal::quadrature_interpolator::Det2DSurface<D1D, Q1D>; }
-   else if (DIM == 3) { return internal::quadrature_interpolator::Det3D<D1D, Q1D>; }
-   else { MFEM_ABORT(""); }
+   else if constexpr (DIM == 2 && SDIM == 2) { return internal::quadrature_interpolator::Det2D<D1D, Q1D>; }
+   else if constexpr (DIM == 2 && SDIM == 3) { return internal::quadrature_interpolator::Det2DSurface<D1D, Q1D>; }
+   else if constexpr (DIM == 3) { return internal::quadrature_interpolator::Det3D<D1D, Q1D>; }
+   MFEM_ABORT("");
 }
 
 /// @endcond

fem/qinterp/eval.hpp

@@ -203,10 +203,10 @@ template<int DIM, QVectorLayout Q_LAYOUT,
 QuadratureInterpolator::TensorEvalKernelType
 QuadratureInterpolator::TensorEvalKernels::Kernel()
 {
-   if (DIM == 1) { return internal::quadrature_interpolator::Values1D<Q_LAYOUT>; }
-   else if (DIM == 2) { return internal::quadrature_interpolator::Values2D<Q_LAYOUT, VDIM, D1D, Q1D, NBZ>; }
-   else if (DIM == 3) { return internal::quadrature_interpolator::Values3D<Q_LAYOUT, VDIM, D1D, Q1D>; }
-   else { MFEM_ABORT(""); }
+   if constexpr (DIM == 1) { return internal::quadrature_interpolator::Values1D<Q_LAYOUT>; }
+   else if constexpr (DIM == 2) { return internal::quadrature_interpolator::Values2D<Q_LAYOUT, VDIM, D1D, Q1D, NBZ>; }
+   else if constexpr (DIM == 3) { return internal::quadrature_interpolator::Values3D<Q_LAYOUT, VDIM, D1D, Q1D>; }
+   MFEM_ABORT("");
 }
 
 /// @endcond

fem/qinterp/eval_hdiv.hpp

@@ -453,8 +453,15 @@ QuadratureInterpolator::TensorEvalHDivKernels::Kernel()
 {
    using namespace internal::quadrature_interpolator;
    static_assert(DIM == 2 || DIM == 3, "only DIM=2 and DIM=3 are implemented!");
-   if (DIM == 2) { return EvalHDiv2D<Q_LAYOUT, FLAGS, D1D, Q1D>; }
-   return EvalHDiv3D<Q_LAYOUT, FLAGS, D1D, Q1D>;
+   if constexpr (DIM == 2)
+   {
+      return EvalHDiv2D<Q_LAYOUT, FLAGS, D1D, Q1D>;
+   }
+   else if constexpr (DIM == 3)
+   {
+      return EvalHDiv3D<Q_LAYOUT, FLAGS, D1D, Q1D>;
+   }
+   MFEM_ABORT("only DIM=2 and DIM=3 are implemented!");
 }
 
 /// @endcond

fem/qinterp/grad.hpp

@@ -592,10 +592,10 @@ template<int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
 QuadratureInterpolator::GradKernelType
 QuadratureInterpolator::GradKernels::Kernel()
 {
-   if (DIM == 1) { return internal::quadrature_interpolator::Derivatives1D<Q_LAYOUT, GRAD_PHYS>; }
-   else if (DIM == 2) { return internal::quadrature_interpolator::Derivatives2D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, Q1D, NBZ>; }
-   else if (DIM == 3) { return internal::quadrature_interpolator::Derivatives3D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, Q1D>; }
-   else { MFEM_ABORT(""); }
+   if constexpr (DIM == 1) { return internal::quadrature_interpolator::Derivatives1D<Q_LAYOUT, GRAD_PHYS>; }
+   else if constexpr (DIM == 2) { return internal::quadrature_interpolator::Derivatives2D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, Q1D, NBZ>; }
+   else if constexpr (DIM == 3) { return internal::quadrature_interpolator::Derivatives3D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, Q1D>; }
+   MFEM_ABORT("");
 }
 
 template<int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
@@ -603,10 +603,10 @@ template<int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
 QuadratureInterpolator::CollocatedGradKernelType
 QuadratureInterpolator::CollocatedGradKernels::Kernel()
 {
-   if (DIM == 1) { return internal::quadrature_interpolator::CollocatedDerivatives1D<Q_LAYOUT, GRAD_PHYS>; }
-   else if (DIM == 2) { return internal::quadrature_interpolator::CollocatedDerivatives2D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, NBZ>; }
-   else if (DIM == 3) { return internal::quadrature_interpolator::CollocatedDerivatives3D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D>; }
-   else { MFEM_ABORT(""); }
+   if constexpr (DIM == 1) { return internal::quadrature_interpolator::CollocatedDerivatives1D<Q_LAYOUT, GRAD_PHYS>; }
+   else if constexpr (DIM == 2) { return internal::quadrature_interpolator::CollocatedDerivatives2D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, NBZ>; }
+   else if constexpr (DIM == 3) { return internal::quadrature_interpolator::CollocatedDerivatives3D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D>; }
+   MFEM_ABORT("");
 }
 
 /// @endcond

fem/quadinterpolator.cpp

@@ -752,10 +752,10 @@ template <int DIM, int VDIM, int ND, int NQ>
 EvalKernel QuadratureInterpolator::EvalKernels::Kernel()
 {
    using namespace internal::quadrature_interpolator;
-   if (DIM == 1) { return Eval1D; }
-   else if (DIM == 2) { return Eval2D<VDIM,ND,NQ>; }
-   else if (DIM == 3) { return Eval3D<VDIM,ND,NQ>; }
-   else { MFEM_ABORT(""); }
+   if constexpr (DIM == 1) { return Eval1D; }
+   else if constexpr (DIM == 2) { return Eval2D<VDIM,ND,NQ>; }
+   else if constexpr (DIM == 3) { return Eval3D<VDIM,ND,NQ>; }
+   MFEM_ABORT("");
 }
 
 template <int DIM>

fem/tmop.cpp

@@ -4102,8 +4102,11 @@ void TMOP_Integrator::GetSurfaceFittingErrors(const Vector &d_loc,
 #ifdef MFEM_USE_MPI
       // Don't count the overlapping DOFs in parallel.
       // The pfes might be ordered byVDIM, while the loop goes consecutively.
-      const int dof_i = pfes->DofToVDof(i, 0);
-      if (parallel && pfes->GetLocalTDofNumber(dof_i) < 0) { continue; }
+      if (parallel)
+      {
+         const int dof_i = pfes->DofToVDof(i, 0);
+         if (pfes->GetLocalTDofNumber(dof_i) < 0) { continue; }
+      }
 #endif
 
       dof_cnt++;

fem/transfer.cpp

@@ -2057,6 +2057,10 @@ TransferOperator::TransferOperator(const FiniteElementSpace& lFESpace_,
    : Operator(hFESpace_.GetVSize(), lFESpace_.GetVSize())
 {
    bool isvar_order = lFESpace_.IsVariableOrder() || hFESpace_.IsVariableOrder();
+   bool is_trace_space =
+      (dynamic_cast<const H1_Trace_FECollection*>(lFESpace_.FEColl()) ||
+       dynamic_cast<const ND_Trace_FECollection*>(lFESpace_.FEColl()) ||
+       dynamic_cast<const RT_Trace_FECollection*>(lFESpace_.FEColl()));
    if (lFESpace_.FEColl() == hFESpace_.FEColl() && !isvar_order)
    {
       OperatorPtr P(Operator::ANY_TYPE);
@@ -2066,6 +2070,7 @@ TransferOperator::TransferOperator(const FiniteElementSpace& lFESpace_,
    }
    else if (lFESpace_.GetVDim() == 1
             && hFESpace_.GetVDim() == 1
+            && !is_trace_space
             && dynamic_cast<const TensorBasisElement*>(lFESpace_.GetTypicalFE())
             && dynamic_cast<const TensorBasisElement*>(hFESpace_.GetTypicalFE())
             && !isvar_order
@@ -2096,15 +2101,244 @@ void TransferOperator::MultTranspose(const Vector& x, Vector& y) const
 
 
 PRefinementTransferOperator::PRefinementTransferOperator(
-   const FiniteElementSpace& lFESpace_, const FiniteElementSpace& hFESpace_)
+   const FiniteElementSpace& lFESpace_, const FiniteElementSpace& hFESpace_,
+   bool assemble_matrix)
    : Operator(hFESpace_.GetVSize(), lFESpace_.GetVSize()), lFESpace(lFESpace_),
      hFESpace(hFESpace_)
 {
    isvar_order = lFESpace_.IsVariableOrder() || hFESpace_.IsVariableOrder();
+
+   MFEM_VERIFY(lFESpace.FEColl()->GetContType() ==
+               hFESpace.FEColl()->GetContType(),
+               "Incompatible finite element space continuity types.");
+
+   is_trace_space =
+      (dynamic_cast<const H1_Trace_FECollection*>(lFESpace.FEColl()) ||
+       dynamic_cast<const ND_Trace_FECollection*>(lFESpace.FEColl()) ||
+       dynamic_cast<const RT_Trace_FECollection*>(lFESpace.FEColl()));
+
+   if (assemble_matrix) { AssembleMatrix(); }
+
 }
 
+void PRefinementTransferOperator::AssembleMatrix()
+{
+   Mesh* mesh = hFESpace.GetMesh();
+   const int nL = lFESpace.GetVSize();
+   const int nH = hFESpace.GetVSize();
+
+   P.reset(new SparseMatrix(nH, nL));
+   Array<int> l_dofs, h_dofs, l_vdofs, h_vdofs;
+   DenseMatrix loc_prol;
+
+   Geometry::Type cached_geom = Geometry::INVALID;
+   const FiniteElement* h_fe = nullptr;
+   const FiniteElement* l_fe = nullptr;
+   IsoparametricTransformation T;
+
+   int vdim = lFESpace.GetVDim();
+
+   const int iend = (is_trace_space) ? mesh->GetNumFaces() : mesh->GetNE();
+   DofTransformation doftrans_h, doftrans_l;
+   Vector w(nH); w = 0.0;
+
+   for (int i = 0; i < iend; i++)
+   {
+      if (is_trace_space)
+      {
+         hFESpace.GetFaceDofs(i, h_dofs);
+         lFESpace.GetFaceDofs(i, l_dofs);
+      }
+      else
+      {
+         hFESpace.GetElementDofs(i, h_dofs, doftrans_h);
+         lFESpace.GetElementDofs(i, l_dofs, doftrans_l);
+      }
+
+      const Geometry::Type geom = (is_trace_space) ? mesh->GetFaceGeometry(i)
+                                  : mesh->GetElementBaseGeometry(i);
+
+      if (geom != cached_geom || isvar_order)
+      {
+         h_fe = (is_trace_space) ? hFESpace.GetFaceElement(i) : hFESpace.GetFE(i);
+         l_fe = (is_trace_space) ? lFESpace.GetFaceElement(i) : lFESpace.GetFE(i);
+         T.SetIdentityTransformation(h_fe->GetGeomType());
+         h_fe->GetTransferMatrix(*l_fe, T, loc_prol);
+         cached_geom = geom;
+      }
+
+      DenseMatrix Aeff(loc_prol);
+      TransformPrimal(doftrans_h, doftrans_l, Aeff);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         l_dofs.Copy(l_vdofs);
+         lFESpace.DofsToVDofs(vd, l_vdofs);
+
+         h_dofs.Copy(h_vdofs);
+         hFESpace.DofsToVDofs(vd, h_vdofs);
+
+         Aeff.AdjustDofDirection(h_vdofs, l_vdofs);
+
+         P->AddSubMatrix(h_vdofs, l_vdofs, Aeff);
+
+         for (int rr = 0; rr < h_vdofs.Size(); rr++)
+         {
+            w(h_vdofs[rr]) += 1.0;
+         }
+
+      }
+   }
+
+   P->Finalize();
+
+   Vector inv_w(nH);
+   for (int i = 0; i < nH; i++)
+   {
+      inv_w(i) = (w(i) > 0.0) ? (1.0 / w(i)) : 1.0;
+   }
+
+   P->ScaleRows(inv_w);
+
+   assembled = true;
+
+}
+
+std::unique_ptr<SparseMatrix>
+PRefinementTransferOperator::BuildConformingTransferMatrix() const
+{
+   MFEM_VERIFY(assembled && P, "Matrix path requires assembled P.");
+
+   const SparseMatrix *Pl = lFESpace.GetConformingProlongation();
+   const SparseMatrix *Rh = hFESpace.GetRestrictionMatrix();
+
+   if (Pl && Rh)
+   {
+      SparseMatrix *RhP   = mfem::Mult(*Rh, *P);
+      SparseMatrix *RhPPl = mfem::Mult(*RhP, *Pl);
+      delete RhP;
+      return std::unique_ptr<SparseMatrix>(RhPPl);
+   }
+   else if (Pl)
+   {
+      return std::unique_ptr<SparseMatrix>(mfem::Mult(*P, *Pl));
+   }
+   else if (Rh)
+   {
+      return std::unique_ptr<SparseMatrix>(mfem::Mult(*Rh, *P));
+   }
+   else
+   {
+      return std::make_unique<SparseMatrix>(*P);
+   }
+}
+
+std::unique_ptr<Operator>
+PRefinementTransferOperator::BuildConformingTransferOperator() const
+{
+   const Operator *Pl = lFESpace.GetProlongationMatrix();
+   const Operator *Rh = hFESpace.GetRestrictionOperator();
+
+   if (Pl && Rh)
+   {
+      return std::make_unique<TripleProductOperator>(Rh,
+                                                     const_cast<PRefinementTransferOperator*>(this), Pl,
+                                                     false, false, false);
+   }
+   else if (Pl)
+   {
+      return std::make_unique<ProductOperator>
+             (const_cast<PRefinementTransferOperator*>(this), Pl,
+              false, false);
+   }
+   else if (Rh)
+   {
+      return std::make_unique<ProductOperator>(Rh,
+                                               const_cast<PRefinementTransferOperator*>(this),
+                                               false, false);
+   }
+   else
+   {
+      // return nullptr to mean "identity/no-op wrapper", i.e. use `this`
+      return nullptr;
+   }
+}
+
+Operator *
+PRefinementTransferOperator::GetTrueTransferOperator()
+{
+   if (tP) { return tP.get(); }
+#ifdef MFEM_USE_MPI
+   const ParFiniteElementSpace* lpfes = dynamic_cast<const ParFiniteElementSpace*>
+                                        (&lFESpace);
+   const ParFiniteElementSpace* hpfes = dynamic_cast<const ParFiniteElementSpace*>
+                                        (&hFESpace);
+   bool parallel = (lpfes) && (hpfes);
+
+   if (parallel)
+   {
+      if (assembled)
+      {
+         HypreParMatrix * Pl = lpfes->Dof_TrueDof_Matrix();
+         const SparseMatrix * Rh = hpfes->GetRestrictionMatrix();
+         // Rh * P
+         SparseMatrix * RhP = mfem::Mult(*Rh, *P);
+         HypreParMatrix * RhPh = new HypreParMatrix(hpfes->GetComm(),
+                                                    hpfes->GlobalTrueVSize(), lpfes->GlobalVSize(),
+                                                    hpfes->GetTrueDofOffsets(), lpfes->GetDofOffsets(), RhP);
+         HypreStealOwnership(*RhPh, *RhP);
+         delete RhP;
+         HypreParMatrix * tmp = ParMult(RhPh, Pl, true);
+         delete RhPh;
+         tP.reset(tmp);
+         return tP.get();
+      }
+      else
+      {
+         auto Pl = lpfes->GetProlongationMatrix();
+         auto Rh = hpfes->GetRestrictionOperator();
+         tP = std::make_unique<TripleProductOperator>(Rh, this, Pl, false, false, false);
+         return tP.get();
+      }
+   }
+   else
+   {
+      if (assembled)
+      {
+         auto M = BuildConformingTransferMatrix();
+         tP.reset(M.release());
+         return tP.get();
+      }
+      else
+      {
+         tP = BuildConformingTransferOperator();
+         return tP ? tP.get() : this;
+      }
+   }
+#else
+   {
+      if (assembled)
+      {
+         auto M = BuildConformingTransferMatrix();
+         tP.reset(M.release());
+         return tP.get();
+      }
+      else
+      {
+         tP = BuildConformingTransferOperator();
+         return tP ? tP.get() : this;
+      }
+   }
+#endif
+}
+
+
 void PRefinementTransferOperator::Mult(const Vector& x, Vector& y) const
 {
+   y = 0.0;
+
+   if (assembled) { P->Mult(x, y); return; }
+
    Mesh* mesh = hFESpace.GetMesh();
    Array<int> l_dofs, h_dofs, l_vdofs, h_vdofs;
    DenseMatrix loc_prol;
@@ -2117,19 +2351,31 @@ void PRefinementTransferOperator::Mult(const Vector& x, Vector& y) const
 
    int vdim = lFESpace.GetVDim();
 
-   y = 0.0;
 
    DofTransformation doftrans_h, doftrans_l;
-   for (int i = 0; i < mesh->GetNE(); i++)
-   {
-      hFESpace.GetElementDofs(i, h_dofs, doftrans_h);
-      lFESpace.GetElementDofs(i, l_dofs, doftrans_l);
 
-      const Geometry::Type geom = mesh->GetElementBaseGeometry(i);
+   int iend = (is_trace_space) ? mesh->GetNumFaces() : mesh->GetNE();
+
+   for (int i = 0; i < iend; i++)
+   {
+      if (is_trace_space)
+      {
+         hFESpace.GetFaceDofs(i, h_dofs);
+         lFESpace.GetFaceDofs(i, l_dofs);
+      }
+      else
+      {
+         hFESpace.GetElementDofs(i, h_dofs, doftrans_h);
+         lFESpace.GetElementDofs(i, l_dofs, doftrans_l);
+      }
+
+      const Geometry::Type geom = (is_trace_space) ? mesh->GetFaceGeometry(i)
+                                  : mesh->GetElementBaseGeometry(i);
+
       if (geom != cached_geom || isvar_order)
       {
-         h_fe = hFESpace.GetFE(i);
-         l_fe = lFESpace.GetFE(i);
+         h_fe = (is_trace_space) ? hFESpace.GetFaceElement(i) : hFESpace.GetFE(i);
+         l_fe = (is_trace_space) ? lFESpace.GetFaceElement(i) : lFESpace.GetFE(i);
          T.SetIdentityTransformation(h_fe->GetGeomType());
          h_fe->GetTransferMatrix(*l_fe, T, loc_prol);
          subY.SetSize(loc_prol.Height());
@@ -2144,6 +2390,7 @@ void PRefinementTransferOperator::Mult(const Vector& x, Vector& y) const
          hFESpace.DofsToVDofs(vd, h_vdofs);
          x.GetSubVector(l_vdofs, subX);
          doftrans_l.InvTransformPrimal(subX);
+
          loc_prol.Mult(subX, subY);
          doftrans_h.TransformPrimal(subY);
          y.SetSubVector(h_vdofs, subY);
@@ -2156,6 +2403,12 @@ void PRefinementTransferOperator::MultTranspose(const Vector& x,
 {
    y = 0.0;
 
+   if (assembled)
+   {
+      P->MultTranspose(x, y);
+      return;
+   }
+
    Mesh* mesh = hFESpace.GetMesh();
    Array<int> l_dofs, h_dofs, l_vdofs, h_vdofs;
    DenseMatrix loc_prol;
@@ -2173,16 +2426,28 @@ void PRefinementTransferOperator::MultTranspose(const Vector& x,
 
    DofTransformation doftrans_h, doftrans_l;
 
-   for (int i = 0; i < mesh->GetNE(); i++)
-   {
-      hFESpace.GetElementDofs(i, h_dofs, doftrans_h);
-      lFESpace.GetElementDofs(i, l_dofs, doftrans_l);
+   int iend = (is_trace_space) ? mesh->GetNumFaces() : mesh->GetNE();
+
+   for (int i = 0; i < iend; i++)
+   {
+      if (is_trace_space)
+      {
+         hFESpace.GetFaceDofs(i, h_dofs);
+         lFESpace.GetFaceDofs(i, l_dofs);
+      }
+      else
+      {
+         hFESpace.GetElementDofs(i, h_dofs, doftrans_h);
+         lFESpace.GetElementDofs(i, l_dofs, doftrans_l);
+      }
+
+      const Geometry::Type geom = (is_trace_space) ? mesh->GetFaceGeometry(i)
+                                  : mesh->GetElementBaseGeometry(i);
 
-      const Geometry::Type geom = mesh->GetElementBaseGeometry(i);
       if (geom != cached_geom || isvar_order)
       {
-         h_fe = hFESpace.GetFE(i);
-         l_fe = lFESpace.GetFE(i);
+         h_fe = (is_trace_space) ? hFESpace.GetFaceElement(i) : hFESpace.GetFE(i);
+         l_fe = (is_trace_space) ? lFESpace.GetFaceElement(i) : lFESpace.GetFE(i);
          T.SetIdentityTransformation(h_fe->GetGeomType());
          h_fe->GetTransferMatrix(*l_fe, T, loc_prol);
          loc_prol.Transpose();

fem/transfer.hpp

@@ -569,6 +569,13 @@ private:
    const FiniteElementSpace& lFESpace;
    const FiniteElementSpace& hFESpace;
    bool isvar_order;
+   bool is_trace_space;
+   bool assembled = false;
+   std::unique_ptr<SparseMatrix> P;
+   mutable std::unique_ptr<Operator> tP;
+
+   std::unique_ptr<SparseMatrix> BuildConformingTransferMatrix() const;
+   std::unique_ptr<Operator> BuildConformingTransferOperator() const;
 
 public:
    /// @brief Constructs a transfer operator from \p lFESpace to \p hFESpace
@@ -577,11 +584,23 @@ public:
        The underlying finite elements need to implement the GetTransferMatrix
        methods. */
    PRefinementTransferOperator(const FiniteElementSpace& lFESpace_,
-                               const FiniteElementSpace& hFESpace_);
+                               const FiniteElementSpace& hFESpace_,
+                               bool assemble_matrix = false);
+
+
+   Operator * GetTrueTransferOperator();
+   const Operator * GetTrueTransferOperator() const
+   {
+      return const_cast<PRefinementTransferOperator*>(this)
+             ->GetTrueTransferOperator();
+   }
 
    /// Destructor
    virtual ~PRefinementTransferOperator() { }
 
+
+   void AssembleMatrix();
+
    /// @brief Interpolation or prolongation of a vector \p x corresponding to
    /// the coarse space to the vector \p y corresponding to the fine space.
    void Mult(const Vector& x, Vector& y) const override;

general/array.hpp

@@ -114,10 +114,22 @@ public:
    Array<T> &operator=(const Array<T> &src) { src.Copy(*this); return *this; }
 
    /// Move assignment operator
+   /** If *this is a non-owning view (e.g., from MakeRef()), the data is copied
+       so that the base is also modified. */
    Array<T> &operator=(Array<T> &&src)
    {
       if (this == &src) { return *this; }
-      Swap(src);  // Swap does not use move assignment!
+      // If *this is a non-owning view (alias), and its capacity is sufficient
+      // to contain src, then copy into *this so that the alias's base memory is
+      // modified.
+      if (!OwnsData() && Capacity() >= src.Size())
+      {
+         *this = src; // Copy assignment.
+      }
+      else
+      {
+         Swap(src); // Swap the pointers only.
+      }
       src.DeleteAll();
       return *this;
    }
@@ -251,6 +263,9 @@ public:
    /// Make this Array a reference to 'master'.
    inline void MakeRef(const Array &master);
 
+   /// Make this Array a reference to the given sub-Memory of @a base.
+   inline void MakeRef(Memory<T> &base, int offset, int size_);
+
    /// Reset the Array to use the given external Memory @a mem and size @a s.
    /** If @a own_mem is false, the Array will not own any of the pointers of
        @a mem.
@@ -1073,6 +1088,14 @@ inline void Array<T>::MakeRef(const Array &master)
    data.MakeAlias(master.GetMemory(), 0, size);
 }
 
+template <class T>
+inline void Array<T>::MakeRef(Memory<T> &base, int offset, int size_)
+{
+   data.Delete();
+   size = size_;
+   data.MakeAlias(base, offset, size_);
+}
+
 template <class T>
 inline void Array<T>::NewMemoryAndSize(
    const Memory<T> &mem, int s, bool own_mem)

linalg/blockoperator.cpp

@@ -14,6 +14,7 @@
 #include "blockvector.hpp"
 #include "blockoperator.hpp"
 
+
 namespace mfem
 {
 
@@ -129,6 +130,33 @@ void BlockOperator::MultTranspose(const Vector &x, Vector &y) const
    }
 }
 
+#ifdef MFEM_USE_MPI
+
+HypreParMatrix * BlockOperator::GetMonolithicHypreParMatrix(
+   Array2D<real_t> *blockCoeff) const
+{
+   Array2D<const HypreParMatrix*> blocks(nRowBlocks, nColBlocks);
+   for (int i = 0; i < nRowBlocks; ++i)
+   {
+      for (int j = 0; j < nColBlocks; ++j)
+      {
+         if (IsZeroBlock(i, j))
+         {
+            blocks(i, j) = nullptr;
+         }
+         else
+         {
+            auto mat = dynamic_cast<const HypreParMatrix*>(&GetBlock(i, j));
+            MFEM_VERIFY(mat,"BlockOperator block (" << i << "," << j
+                        << ") is not a HypreParMatrix.");
+            blocks(i, j) = mat;
+         }
+      }
+   }
+   return HypreParMatrixFromBlocks(blocks, blockCoeff);
+}
+#endif
+
 BlockOperator::~BlockOperator()
 {
    if (owns_blocks)
@@ -381,4 +409,161 @@ BlockLowerTriangularPreconditioner::~BlockLowerTriangularPreconditioner()
    }
 }
 
-} // namespace mfem
+BlockTriangularSymmetricPreconditioner::BlockTriangularSymmetricPreconditioner(
+   const Array<int> & offsets_)
+   : Solver(offsets_.Last()),
+     owns_blocks(0),
+     nBlocks(offsets_.Size() - 1),
+     offsets(0),
+     ops(nBlocks, nBlocks),
+     coef(nBlocks, nBlocks)
+{
+   ops = static_cast<Operator *>(NULL);
+   offsets.MakeRef(offsets_);
+}
+
+void BlockTriangularSymmetricPreconditioner::SetDiagonalBlock(int iblock,
+                                                              Operator *op, real_t c)
+{
+   MFEM_VERIFY(offsets[iblock+1] - offsets[iblock] == op->Height() &&
+               offsets[iblock+1] - offsets[iblock] == op->Width(),
+               "incompatible Operator dimensions");
+
+   SetBlock(iblock, iblock, op);
+   coef(iblock, iblock) = c;
+}
+
+void BlockTriangularSymmetricPreconditioner::SetBlock(int iRow, int iCol,
+                                                      Operator *op, real_t c)
+{
+   MFEM_VERIFY(offsets[iRow+1] - offsets[iRow] == op->NumRows() &&
+               offsets[iCol+1] - offsets[iCol] == op->NumCols(),
+               "incompatible Operator dimensions");
+
+   ops(iRow, iCol) = op;
+   coef(iRow, iCol) = c;
+}
+
+// Operator application
+
+void BlockTriangularSymmetricPreconditioner::ForwardPass(const Vector & x,
+                                                         Vector & y) const
+{
+   // Forward sweep: Solve for y1, then y2
+   for (int iRow = 0; iRow < nBlocks; ++iRow)
+   {
+      tmp.SetSize(offsets[iRow + 1] - offsets[iRow]);
+      tmp2.SetSize(offsets[iRow + 1] - offsets[iRow]);
+      tmp2 = 0.0;
+      tmp2 += xblock.GetBlock(iRow);  // tmp2 = xblock(iRow)
+
+      // Process the lower triangular part (jCol < iRow)
+      for (int jCol = 0; jCol < iRow; ++jCol)
+      {
+         if (ops(iRow, jCol))
+         {
+            ops(iRow, jCol)->Mult(yblock.GetBlock(jCol),
+                                  tmp);  // tmp = A(iRow,jCol) * yblock(jCol)
+            tmp*= coef(iRow,jCol);  // tmp *= c
+            tmp2 -= tmp;  // tmp2 -= A(iRow, jCol) * yblock(jCol)
+         }
+      }
+
+      // Apply the diagonal block
+      if (ops(iRow, iRow))
+      {
+         ops(iRow, iRow)->Mult(tmp2,
+                               yblock.GetBlock(iRow));
+         yblock.GetBlock(iRow) *= coef(iRow,
+                                       iRow);  // yblock(iRow) = A(iRow,iRow)^-1 * tmp2
+      }
+      else
+      {
+         yblock.GetBlock(iRow) = tmp2;  // If no diagonal operator, set yblock directly
+      }
+   }
+}
+
+void BlockTriangularSymmetricPreconditioner::BackwardPass(const Vector & x,
+                                                          Vector & y) const
+{
+   // Backward sweep: Adjust y1 based on y2
+   for (int iRow = nBlocks - 1; iRow >= 0; --iRow)
+   {
+      tmp.SetSize(offsets[iRow + 1] - offsets[iRow]);
+      tmp2.SetSize(offsets[iRow + 1] - offsets[iRow]);
+      tmp2 = 0.0;
+      tmp2 += xblock.GetBlock(iRow);  // tmp2 = yblock(iRow) from forward sweep
+
+      // Process the upper triangular part (jCol > iRow)
+      for (int jCol = iRow + 1; jCol < nBlocks; ++jCol)
+      {
+         if (ops(iRow, jCol))
+         {
+            ops(iRow, jCol)->Mult(yblock.GetBlock(jCol),
+                                  tmp);  // tmp = A(iRow,jCol) * yblock(jCol)
+            tmp *= coef(iRow,jCol);  // tmp *= c
+            tmp2 -= tmp;  // tmp2 -= A(iRow,jCol) * yblock(jCol)
+         }
+      }
+
+      // Reapply diagonal block to correct y1
+      if (ops(iRow, iRow))
+      {
+         ops(iRow, iRow)->Mult(tmp2,
+                               yblock.GetBlock(iRow));  // Final correction for yblock(iRow)
+         yblock.GetBlock(iRow) *= coef(iRow,
+                                       iRow);  // yblock(iRow) = A(iRow,iRow)^-1 * tmp2
+      }
+      else
+      {
+         yblock.GetBlock(iRow) = tmp2;  // If no diagonal operator, set yblock directly
+      }
+   }
+}
+
+
+
+void BlockTriangularSymmetricPreconditioner::Mult(const Vector & x,
+                                                  Vector & y) const
+{
+   MFEM_ASSERT(x.Size() == width, "incorrect input Vector size");
+   MFEM_ASSERT(y.Size() == height, "incorrect output Vector size");
+
+   // Update block views of the vectors y and x using offsets
+   yblock.Update(y.GetData(), offsets);
+   xblock.Update(x.GetData(), offsets);
+
+   // Initialize y to zero
+   y = 0.0;
+   ForwardPass(x,y);
+
+   // Update Residual
+   r.SetSize(x.Size());
+   r = 0.0; r+=x;
+   Op->AddMult(y,r,-1.0);
+
+   Vector y1(y);
+   yblock.Update(y1.GetData(), offsets);
+   xblock.Update(r.GetData(), offsets);
+   BackwardPass(r,y1);
+   y+=y1;
+}
+
+BlockTriangularSymmetricPreconditioner::~BlockTriangularSymmetricPreconditioner()
+{
+   if (owns_blocks)
+   {
+      for (int iRow=0; iRow < nBlocks; ++iRow)
+      {
+         for (int jCol=0; jCol < nBlocks; ++jCol)
+         {
+            delete ops(jCol,iRow);
+         }
+      }
+   }
+}
+
+
+
+}

linalg/blockoperator.hpp

@@ -16,6 +16,9 @@
 #include "../general/array.hpp"
 #include "operator.hpp"
 #include "blockvector.hpp"
+#ifdef MFEM_USE_MPI
+#include "hypre.hpp"
+#endif
 
 namespace mfem
 {
@@ -105,6 +108,13 @@ public:
    /// Action of the transpose operator
    void MultTranspose (const Vector & x, Vector & y) const override;
 
+#ifdef MFEM_USE_MPI
+   /// Returns a monolithic HypreParMatrix formed by merging the blocks of
+   //  this BlockOperator, assuming every block is a HypreParMatrix.
+   HypreParMatrix * GetMonolithicHypreParMatrix(Array2D<real_t> *blockCoeff=NULL)
+   const;
+#endif
+
    ~BlockOperator();
 
    //! Controls the ownership of the blocks: if nonzero, BlockOperator will
@@ -288,6 +298,82 @@ private:
    mutable Vector tmp2;
 };
 
+class BlockTriangularSymmetricPreconditioner : public Solver
+{
+private:
+   const Operator * Op;
+public:
+   //! Constructor for BlockTriangularSymmetricPreconditioners with the same
+   //! block-structure for rows and columns.
+   /**
+    *  @param offsets  Offsets that mark the start of each row/column block
+    *                  (size nBlocks+1).
+    *
+    *  @note BlockTriangularSymmetricPreconditioner will not own/copy the data
+    *  contained in @a offsets.
+    */
+   BlockTriangularSymmetricPreconditioner(const Array<int> & offsets);
+
+   //! Add block op in the block-entry (iblock, iblock).
+   /**
+    * @param iblock  The block will be inserted in location (iblock, iblock).
+    * @param op      The Operator to be inserted.
+    * @param c       The coefficient to be applied to the block.
+    */
+   void SetDiagonalBlock(int iblock, Operator *op, real_t c = 1.0);
+   //! Add a block opt in the block-entry (iblock, jblock).
+   /**
+    * @param iRow, iCol  The block will be inserted in location (iRow, iCol).
+    * @param op          The Operator to be inserted.
+    * @param c           The coefficient to be applied to the block.
+   */
+   void SetBlock(int iRow, int iCol, Operator *op, real_t c = 1.0);
+   //! This method is present since required by the abstract base class Solver
+   virtual void SetOperator(const Operator &op) {Op = &op;}
+
+   //! Return the number of blocks
+   int NumBlocks() const { return nBlocks; }
+
+   //! Return a reference to block i,j.
+   Operator & GetBlock(int iblock, int jblock)
+   { MFEM_VERIFY(ops(iblock,jblock), ""); return *ops(iblock,jblock); }
+
+   Operator & GetDiagonalBlock(int iblock)
+   { MFEM_VERIFY(ops(iblock,iblock), ""); return *ops(iblock,iblock); }
+
+   //! Return the offsets for block starts
+   Array<int> & Offsets() { return offsets; }
+
+   /// Operator application
+   virtual void Mult (const Vector & x, Vector & y) const;
+
+   ~BlockTriangularSymmetricPreconditioner();
+
+   //! Controls the ownership of the blocks: if nonzero,
+   //! BlockTriangularSymmetricPreconditioner will delete all blocks that are set
+   //! (non-NULL); the default value is zero.
+   int owns_blocks;
+
+private:
+   //! Number of block rows/columns
+   int nBlocks;
+   //! Offsets for the starting position of each block
+   Array<int> offsets;
+   //! 2D array that stores each block of the operator.
+   Array2D<Operator *> ops;
+   Array2D<real_t> coef;
+
+   //! Temporary Vectors used to efficiently apply the Mult and MultTranspose
+   //! methods.
+   mutable BlockVector xblock;
+   mutable BlockVector yblock;
+   mutable Vector tmp;
+   mutable Vector tmp2;
+   mutable Vector r;
+   void ForwardPass(const Vector & x, Vector & y) const;
+   void BackwardPass(const Vector & x, Vector & y) const;
+};
+
 }
 
 #endif /* MFEM_BLOCKOPERATOR */

linalg/complex_operator.cpp

@@ -10,8 +10,15 @@
 // CONTRIBUTING.md for details.
 
 #include "complex_operator.hpp"
+#include "../general/communication.hpp"
+#ifdef MFEM_USE_MPI
+#include "blockoperator.hpp"
+#endif
 #include <set>
 #include <map>
+#include <unordered_map>
+#include <vector>
+#include <limits>
 
 namespace mfem
 {
@@ -164,6 +171,51 @@ void ComplexOperator::MultTranspose(const Vector &x_r, const Vector &x_i,
    }
 }
 
+#ifdef MFEM_USE_MPI
+ComplexHypreParMatrix * ComplexOperator::AsComplexHypreParMatrix() const
+{
+   HypreParMatrix *Ar = nullptr;
+   HypreParMatrix *Ai = nullptr;
+   bool own_r = false;
+   bool own_i = false;
+
+   if (auto *Ahr = dynamic_cast<const HypreParMatrix*>(&real()))
+   {
+      Ar = const_cast<HypreParMatrix*>(Ahr);
+   }
+   else if (auto *Br = dynamic_cast<const BlockOperator*>(&real()))
+   {
+      Ar = Br->GetMonolithicHypreParMatrix();
+      own_r = true;
+   }
+   else
+   {
+      MFEM_ABORT("Real part is neither HypreParMatrix nor BlockOperator.");
+   }
+
+   if (auto *Ahi = dynamic_cast<const HypreParMatrix*>(&imag()))
+   {
+      Ai = const_cast<HypreParMatrix*>(Ahi);
+   }
+   else if (auto *Bi = dynamic_cast<const BlockOperator*>(&imag()))
+   {
+      Ai = Bi->GetMonolithicHypreParMatrix();
+      own_i = true;
+   }
+   else
+   {
+      MFEM_ABORT("Imag part is neither HypreParMatrix nor BlockOperator.");
+   }
+
+   return new ComplexHypreParMatrix(Ar, Ai, own_r, own_i, GetConvention());
+}
+
+
+
+#endif
+
+
+
 
 SparseMatrix & ComplexSparseMatrix::real()
 {
@@ -877,6 +929,892 @@ ComplexHypreParMatrix::getColStartStop(const HypreParMatrix * A_r,
    delete [] stat;
 }
 
-#endif // MFEM_USE_MPI
+static Array<int> Twice(const Array<int> &offs)
+{
+   Array<int> arrayout(offs.Size());
+   for (int i = 0; i < offs.Size(); i++) { arrayout[i] = 2 * offs[i]; }
+   return arrayout;
+}
+
+
+ComplexBlockOperator::ComplexBlockOperator(const ComplexOperator &A)
+   : BlockOperator(
+        Twice(dynamic_cast<const BlockOperator&>(A.real()).RowOffsets()),
+        Twice(dynamic_cast<const BlockOperator&>(A.real()).ColOffsets()))
+{
+   const BlockOperator *Ar = dynamic_cast<const BlockOperator*>(&A.real());
+   const BlockOperator *Ai = dynamic_cast<const BlockOperator*>(&A.imag());
+
+   MFEM_VERIFY(Ar && Ai,
+               "ComplexBlockOperator: expected ComplexOperator with BlockOperator Re/Im.");
+   MFEM_VERIFY(Ar->NumRowBlocks() == Ai->NumRowBlocks() &&
+               Ar->NumColBlocks() == Ai->NumColBlocks(),
+               "ComplexBlockOperator: Re/Im block layouts mismatch.");
+
+   // Populate this BlockOperator (base) with ComplexOperator blocks.
+   for (int i = 0; i < Ar->NumRowBlocks(); ++i)
+   {
+      for (int j = 0; j < Ar->NumColBlocks(); ++j)
+      {
+         HypreParMatrix *Rij = nullptr;
+         if (!Ar->IsZeroBlock(i, j))
+         {
+            Rij = const_cast<HypreParMatrix*>
+                  (dynamic_cast<const HypreParMatrix*>(&Ar->GetBlock(i, j)));
+         }
+
+         HypreParMatrix *Iij = nullptr;
+         if (!Ai->IsZeroBlock(i, j))
+         {
+            Iij = const_cast<HypreParMatrix*>
+                  (dynamic_cast<const HypreParMatrix*>(&Ai->GetBlock(i, j)));
+         }
+
+         // MFEM_VERIFY((Rij && Iij) || (!Rij && !Iij),
+         //             "ComplexBlockOperator: inconsistent sparsity at block ("
+         //             << i << "," << j << ").");
+
+         if (Rij)
+         {
+            auto *Cij = new ComplexHypreParMatrix(Rij, Iij, false, false,
+                                                  A.GetConvention());
+            SetBlock(i, j, Cij);
+         }
+      }
+   }
+}
+
+void ComplexBlockOperator::BlockComplexToComplexBlock(const Vector &xin,
+                                                      Vector &xout) const
+{
+   MFEM_VERIFY(xout.Size() == xin.Size(),
+               "BlockComplexToComplexBlock: size mismatch (xout != xin).");
+   MFEM_VERIFY(xin.Size() % 2 == 0,
+               "BlockComplexToComplexBlock: expected even-sized vector (2*N).");
+
+   // Decide whether xin is domain-sized or range-sized by matching doubled offsets.
+   const int twoNcols = ColOffsets().Last();
+   const int twoNrows = RowOffsets().Last();
+   const Array<int> *doffs = nullptr; // doubled offsets to use
+
+   if (xin.Size() == twoNcols) { doffs = &ColOffsets(); }
+   else if (xin.Size() == twoNrows) { doffs = &RowOffsets(); }
+   else
+   {
+      MFEM_ABORT("BlockComplexToComplexBlock: vector size does not match "
+                 "either doubled domain or range size.");
+   }
+
+   // xin layout: [ Re(all 0..N-1), Im(all 0..N-1) ], where N = doffs->Last()/2
+   const int N = doffs->Last() / 2;
+
+   int pos = 0; // position in per-block layout (xout), but note xout is also size 2*N
+   for (int b = 0; b < doffs->Size() - 1; b++)
+   {
+      // Doubled block segment in this BlockOperator is [(*doffs)[b], (*doffs)[b+1])
+      // The *base* (undoubled) start/length are:
+      const int s_base = (*doffs)[b] / 2;
+      const int len    = ((*doffs)[b+1] - (*doffs)[b]) / 2;
+
+      // Write per-block Re then Im, contiguous
+      for (int k = 0; k < len; k++) { xout[pos + k]       = xin[s_base + k]; }
+      for (int k = 0; k < len; k++) { xout[pos + len + k] = xin[N + s_base + k]; }
+
+      pos += 2 * len;
+   }
+}
+
+void ComplexBlockOperator::ComplexBlockToBlockComplex(const Vector &xin,
+                                                      Vector &xout) const
+{
+   MFEM_VERIFY(xout.Size() == xin.Size(),
+               "ComplexBlockToBlockComplex: size mismatch (xout != xin).");
+   MFEM_VERIFY(xin.Size() % 2 == 0,
+               "ComplexBlockToBlockComplex: expected even-sized vector (2*N).");
+
+   const int Ncols = ColOffsets().Last();
+   const int Nrows = RowOffsets().Last();
+   const Array<int> *doffs = nullptr;
+
+   if (xin.Size() == Ncols) { doffs = &ColOffsets(); }
+   else if (xin.Size() == Nrows) { doffs = &RowOffsets(); }
+   else
+   {
+      MFEM_ABORT("ComplexBlockToBlockComplex: vector size does not match "
+                 "either doubled domain or range size.");
+   }
+
+   const int N = doffs->Last() / 2;
+
+   int pos = 0; // position in per-block layout (xin)
+   for (int b = 0; b < doffs->Size() - 1; b++)
+   {
+      const int s_base = (*doffs)[b] / 2;
+      const int len    = ((*doffs)[b+1] - (*doffs)[b]) / 2;
+
+      // Read per-block Re then Im, scatter to stacked [Re(all); Im(all)]
+      for (int k = 0; k < len; k++) { xout[s_base + k]     = xin[pos + k]; }
+      for (int k = 0; k < len; k++) { xout[N + s_base + k] = xin[pos + len + k]; }
+
+      pos += 2 * len;
+   }
+}
+
+
+#ifdef MFEM_USE_COMPLEX_MUMPS
+
+// Macro so indices match MUMPS documentation
+#define MUMPS_ICNTL(I) icntl[(I) - 1]
+#define MUMPS_CNTL(I)  cntl[(I) - 1]
+#define MUMPS_INFO(I)  info[(I) - 1]
+#define MUMPS_INFOG(I) infog[(I) - 1]
+
+ComplexMUMPSSolver::ComplexMUMPSSolver(MPI_Comm comm_)
+{
+   Init(comm_);
+}
+
+ComplexMUMPSSolver::ComplexMUMPSSolver(const Operator &op)
+{
+   auto APtr = dynamic_cast<const ComplexHypreParMatrix *>(&op);
+   MFEM_VERIFY(APtr, "Not a compatible matrix type for ComplexMUMPSSolver");
+   SetOperator(op);
+}
+
+void ComplexMUMPSSolver::Init(MPI_Comm comm_)
+{
+   comm = comm_;
+   MPI_Comm_size(comm, &numProcs);
+   MPI_Comm_rank(comm, &myid);
+
+   print_level = 2;
+   row_start = 0;
+
+   id = nullptr;
+
+#if MFEM_MUMPS_VERSION >= 530
+   irhs_loc = nullptr;
+   isol_loc = nullptr;
+   rhs_loc  = nullptr;
+   sol_loc  = nullptr;
+#else
+   global_num_rows = 0;
+   recv_counts = nullptr;
+   displs = nullptr;
+   rhs_glob = nullptr;
+   rhs_glob_r = nullptr;
+   rhs_glob_i = nullptr;
+#endif
+}
+
+ComplexMUMPSSolver::~ComplexMUMPSSolver()
+{
+#if MFEM_MUMPS_VERSION >= 530
+   delete [] irhs_loc;
+   delete [] isol_loc;
+   delete [] rhs_loc;
+   delete [] sol_loc;
+#else
+   delete [] recv_counts;
+   delete [] displs;
+   delete [] rhs_glob;
+   delete [] rhs_glob_r;
+   delete [] rhs_glob_i;
+#endif
+
+   if (id)
+   {
+      id->job = -2;
+      mumps_call();
+      delete id;
+      id = nullptr;
+   }
+}
+
+void ComplexMUMPSSolver::SetOperator(const Operator &op)
+{
+   auto APtr = dynamic_cast<const ComplexHypreParMatrix *>(&op);
+   MFEM_VERIFY(APtr, "Not compatible matrix type for ComplexMUMPSSolver");
+
+   height = op.Height();
+   width  = op.Width();
+
+   const HypreParMatrix *Ar = (APtr->hasRealPart()) ? &APtr->real() : nullptr;
+   const HypreParMatrix *Ai = (APtr->hasImagPart()) ? &APtr->imag() : nullptr;
+
+   MFEM_VERIFY(Ar || Ai, "ComplexMUMPSSolver: both real and imag parts are null.");
+
+   // Pick communicator from the non-null part
+   MPI_Comm op_comm = (Ar ? Ar->GetComm() : Ai->GetComm());
+
+   // Comm setup/check
+   if (comm == MPI_COMM_NULL) { Init(op_comm); }
+   else
+   {
+      int cmp = MPI_UNEQUAL;
+      MPI_Comm_compare(comm, op_comm, &cmp);
+      MFEM_VERIFY(cmp != MPI_UNEQUAL, "MPI Comm mismatch");
+   }
+
+   // HostRead only if non-null
+   if (Ar) { Ar->HostRead(); }
+   if (Ai) { Ai->HostRead(); }
+
+   // hypre parcsr pointers
+   hypre_ParCSRMatrix *parcsr_op_r = nullptr;
+   hypre_ParCSRMatrix *parcsr_op_i = nullptr;
+
+   if (Ar) { parcsr_op_r = (hypre_ParCSRMatrix*) const_cast<HypreParMatrix&>(*Ar); }
+   if (Ai) { parcsr_op_i = (hypre_ParCSRMatrix*) const_cast<HypreParMatrix&>(*Ai); }
+
+   // Merge diag+offd for whichever exists
+   hypre_CSRMatrix *csr_op_r = nullptr;
+   hypre_CSRMatrix *csr_op_i = nullptr;
+
+   if (parcsr_op_r) { csr_op_r = hypre_MergeDiagAndOffd(parcsr_op_r); }
+   if (parcsr_op_i) { csr_op_i = hypre_MergeDiagAndOffd(parcsr_op_i); }
+
+#if MFEM_HYPRE_VERSION >= 21600
+   if (csr_op_r) { hypre_CSRMatrixBigJtoJ(csr_op_r); }
+   if (csr_op_i) { hypre_CSRMatrixBigJtoJ(csr_op_i); }
+#endif
+
+   // Determine local/global sizes and row_start from an existing part
+   const int n_loc = internal::to_int((csr_op_r ? csr_op_r->num_rows :
+                                       csr_op_i->num_rows));
+   row_start = internal::to_int((parcsr_op_r ? parcsr_op_r->first_row_index
+                                 : parcsr_op_i->first_row_index));
+   const int global_n = internal::to_int((parcsr_op_r ?
+                                          parcsr_op_r->global_num_rows
+                                          : parcsr_op_i->global_num_rows));
+
+   // Use nullptr checks
+   const int *Ir = csr_op_r ? csr_op_r->i : nullptr;
+   const int *Jr = csr_op_r ? csr_op_r->j : nullptr;
+   const real_t *Vr = csr_op_r ? (const real_t*)csr_op_r->data : nullptr;
+
+   const int *Ii = csr_op_i ? csr_op_i->i : nullptr;
+   const int *Ji = csr_op_i ? csr_op_i->j : nullptr;
+   const real_t *Vi = csr_op_i ? (const real_t*)csr_op_i->data : nullptr;
+
+   // Build union COO
+   std::vector<int> Icoo, Jcoo;
+   std::vector<mumps_complex_t> Zcoo;
+
+   size_t nnz_r = csr_op_r ? (size_t)csr_op_r->num_nonzeros : 0;
+   size_t nnz_i = csr_op_i ? (size_t)csr_op_i->num_nonzeros : 0;
+   Icoo.reserve(nnz_r + nnz_i);
+   Jcoo.reserve(nnz_r + nnz_i);
+   Zcoo.reserve(nnz_r + nnz_i);
+
+   BuildUnionCOO(n_loc, row_start, Ir, Jr, Vr, Ii, Ji, Vi, Icoo, Jcoo, Zcoo);
+
+   const int nnz = (int)Icoo.size();
+   int *I = new int[nnz];
+   int *J = new int[nnz];
+   mumps_complex_t *A = new mumps_complex_t[nnz];
+
+   std::copy(Icoo.begin(), Icoo.end(), I);
+   std::copy(Jcoo.begin(), Jcoo.end(), J);
+   std::copy(Zcoo.begin(), Zcoo.end(), A);
+
+   // New ComplexMUMPS object or reuse an existing one
+   if (!id || !reorder_reuse)
+   {
+      if (id)
+      {
+         id->job = -2;
+         mumps_call();
+         delete id;
+         id = nullptr;
+      }
+
+#ifdef MFEM_USE_SINGLE
+      id = new CMUMPS_STRUC_C();
+#else
+      id = new ZMUMPS_STRUC_C();
+#endif
+
+      id->sym = 0; // general complex
+      id->par = 1;
+      id->comm_fortran = (MUMPS_INT)MPI_Comm_c2f(comm);
+
+      // Init
+      id->job = -1;
+      mumps_call();
+
+      // Set parameters
+      SetParameters();
+
+      // Attach matrix
+      id->n       = global_n;
+      id->nnz_loc = nnz;
+      id->irn_loc = I;
+      id->jcn_loc = J;
+      id->a_loc   = A;
+
+      // Analysis (ordering + symbolic)
+      id->job = 1;
+      mumps_call();
+   }
+   else
+   {
+      // Reuse symbolic factorization / ordering
+      MFEM_VERIFY(id->n == global_n,
+                  "ReorderingReuse requires same global size (id->n mismatch)");
+
+      // Update matrix pointers (pattern is assumed compatible)
+      id->nnz_loc = nnz;
+      id->irn_loc = I;
+      id->jcn_loc = J;
+      id->a_loc   = A;
+   }
+
+   // Factorization
+   id->job = 2;
+   {
+      const int mem_relax_lim = 200;
+      while (true)
+      {
+         mumps_call();
+         if (id->MUMPS_INFOG(1) < 0)
+         {
+            if (id->MUMPS_INFOG(1) == -8 || id->MUMPS_INFOG(1) == -9)
+            {
+               id->MUMPS_ICNTL(14) += 20;
+               MFEM_VERIFY(id->MUMPS_ICNTL(14) <= mem_relax_lim,
+                           "Memory relaxation limit reached for MUMPS factorization");
+               if (myid == 0 && print_level > 0)
+               {
+                  out << "Re-running MUMPS factorization with memory relaxation "
+                      << id->MUMPS_ICNTL(14) << '\n';
+               }
+            }
+            else
+            {
+               MFEM_ABORT("Error during MUMPS numerical factorization");
+            }
+         }
+         else { break; }
+      }
+   }
+
+   // Done with input storage
+   if (csr_op_r) { hypre_CSRMatrixDestroy(csr_op_r);}
+   if (csr_op_i) { hypre_CSRMatrixDestroy(csr_op_i);}
+   delete [] I;
+   delete [] J;
+   delete [] A;
+
+   // Post-factorization RHS/SOL setup
+   id->nrhs = -1;
+
+#if MFEM_MUMPS_VERSION >= 530
+   // Distributed RHS/SOL sizes
+   id->nloc_rhs = n_loc;
+   id->lrhs_loc = n_loc;
+   id->lsol_loc = id->MUMPS_INFO(23);
+
+   delete [] irhs_loc;
+   irhs_loc = new int[id->lrhs_loc];
+   for (int i = 0; i < n_loc; i++)
+   {
+      irhs_loc[i] = row_start + i + 1;
+   }
+   id->irhs_loc = irhs_loc;
+
+   delete [] isol_loc;
+   isol_loc = new int[id->lsol_loc];
+   id->isol_loc = isol_loc;
+
+   row_starts.SetSize(numProcs);
+   MPI_Allgather(&row_start, 1, MPI_INT, row_starts, 1, MPI_INT, comm);
+
+   // Reset cached buffers
+   delete [] rhs_loc; rhs_loc = nullptr;
+   delete [] sol_loc; sol_loc = nullptr;
+   rhs1_buf.clear();
+
+#else
+   // Centralized RHS/SOL on root
+   id->lrhs = id->n;
+
+   global_num_rows = id->n;
+
+   if (myid == 0)
+   {
+      delete [] recv_counts;
+      delete [] displs;
+      recv_counts = new int[numProcs];
+      displs = new int[numProcs];
+
+      delete [] rhs_glob;   rhs_glob = nullptr;
+      delete [] rhs_glob_r; rhs_glob_r = nullptr;
+      delete [] rhs_glob_i; rhs_glob_i = nullptr;
+   }
+
+   MPI_Gather(&n_loc, 1, MPI_INT, recv_counts, 1, MPI_INT, 0, comm);
+
+   if (myid == 0)
+   {
+      displs[0] = 0;
+      int s = 0;
+      for (int k = 0; k < numProcs - 1; k++)
+      {
+         s += recv_counts[k];
+         displs[k+1] = s;
+      }
+   }
+#endif
+}
+
+void ComplexMUMPSSolver::InitRhsSol(int nrhs) const
+{
+#if MFEM_MUMPS_VERSION >= 530
+
+   MFEM_VERIFY(id, "InitRhsSol called before SetOperator");
+
+   if (id->nrhs != nrhs)
+   {
+      delete [] rhs_loc;
+      delete [] sol_loc;
+
+      rhs_loc = new mumps_complex_t[(size_t)nrhs * (size_t)id->lrhs_loc];
+      sol_loc = new mumps_complex_t[(size_t)nrhs * (size_t)id->lsol_loc];
+
+      id->rhs_loc = rhs_loc;
+      id->sol_loc = sol_loc;
+   }
+   id->nrhs = nrhs;
+
+#else
+   MFEM_VERIFY(id, "InitRhsSol called before SetOperator");
+
+   id->nrhs = nrhs;
+   id->lrhs = id->n;
+
+   if (myid == 0)
+   {
+      const size_t N = (size_t)nrhs * (size_t)global_num_rows;
+
+      delete [] rhs_glob;
+      delete [] rhs_glob_r;
+      delete [] rhs_glob_i;
+
+      rhs_glob   = new mumps_complex_t[N];
+      rhs_glob_r = new real_t[N];
+      rhs_glob_i = new real_t[N];
+
+      id->rhs = rhs_glob;
+   }
+#endif
+}
+
+void ComplexMUMPSSolver::Mult(const Vector &x, Vector &y) const
+{
+   Array<const Vector *> X(1);
+   Array<Vector *> Y(1);
+   X[0] = &x;
+   Y[0] = &y;
+   ArrayMult(X, Y);
+}
+
+void ComplexMUMPSSolver::ArrayMult(const Array<const Vector *> &X,
+                                   Array<Vector *> &Y) const
+{
+   MFEM_ASSERT(X.Size() == Y.Size(),
+               "Number of columns mismatch in ComplexMUMPSSolver::Mult!");
+   MFEM_VERIFY(id, "ComplexMUMPSSolver::ArrayMult called before SetOperator");
+
+   InitRhsSol(X.Size());
+
+#if MFEM_MUMPS_VERSION >= 530
+   MFEM_VERIFY(irhs_loc && isol_loc, "RHS/SOL maps not initialized");
+   MFEM_VERIFY(rhs_loc && sol_loc, "RHS/SOL buffers not initialized");
+   const int n_loc = id->lrhs_loc;
+   const int nrhs  = id->nrhs;
+
+   // Pack all RHS
+   for (int i = 0; i < nrhs; i++)
+   {
+      MFEM_ASSERT(X[i], "Missing Vector in Mult!");
+      X[i]->HostRead();
+      MFEM_VERIFY(X[i]->Size() == 2*n_loc, "RHS size mismatch");
+
+      const real_t *xdata = X[i]->GetData();
+      const real_t *xr = xdata;
+      const real_t *xi = xdata + n_loc;
+
+      mumps_complex_t *dst = rhs_loc + i * n_loc;
+      for (int j = 0; j < n_loc; j++)
+      {
+         dst[j].r = xr[j];
+         dst[j].i = xi[j];
+      }
+   }
+
+   id->rhs_loc  = rhs_loc;
+   id->sol_loc  = sol_loc;
+   id->irhs_loc = irhs_loc;
+   id->isol_loc = isol_loc;
+
+   // MUMPS solve
+   id->job = 3;
+   mumps_call();
+
+   const int lsol = id->lsol_loc;
+
+   // Redistribute each solution column into Y
+   for (int i = 0; i < nrhs; i++)
+   {
+      MFEM_ASSERT(Y[i], "Missing output Vector in Mult!");
+      Y[i]->HostWrite();
+      MFEM_VERIFY(Y[i]->Size() == 2*n_loc, "Output size mismatch");
+
+      const mumps_complex_t *xcol = sol_loc + i * lsol;
+      RedistributeSol(isol_loc, xcol, Y[i]->GetData(), n_loc, lsol);
+   }
+
+#else // MFEM_MUMPS_VERSION < 530
+
+   const int nrhs = id->nrhs;
+
+   MFEM_VERIFY(X.Size() > 0 && X[0], "Missing RHS");
+   const int n_loc = X[0]->Size()/2;
+
+   for (int i = 0; i < nrhs; i++)
+   {
+      MFEM_ASSERT(X[i], "Missing Vector in Mult!");
+      X[i]->HostRead();
+      MFEM_VERIFY(X[i]->Size() == 2*n_loc, "RHS size mismatch");
+   }
+
+   // Gather each RHS column (real+imag separately) into root staging
+   for (int i = 0; i < nrhs; i++)
+   {
+      const real_t *xdata = X[i]->GetData();
+
+      MPI_Gatherv(xdata, n_loc, MPITypeMap<real_t>::mpi_type,
+                  rhs_glob_r + i * global_num_rows,
+                  recv_counts, displs, MPITypeMap<real_t>::mpi_type,
+                  0, comm);
+
+      MPI_Gatherv(xdata + n_loc, n_loc, MPITypeMap<real_t>::mpi_type,
+                  rhs_glob_i + i * global_num_rows,
+                  recv_counts, displs, MPITypeMap<real_t>::mpi_type,
+                  0, comm);
+   }
+
+   // Pack into MUMPS complex RHS on root: id->rhs is in-place
+   if (myid == 0)
+   {
+      for (int i = 0; i < nrhs; i++)
+      {
+         mumps_complex_t *dst = rhs_glob + i * global_num_rows;
+         const real_t *rr = rhs_glob_r + i * global_num_rows;
+         const real_t *ri = rhs_glob_i + i * global_num_rows;
+
+         for (int j = 0; j < global_num_rows; j++)
+         {
+            dst[j].r = rr[j];
+            dst[j].i = ri[j];
+         }
+      }
+      id->rhs = rhs_glob;
+   }
+
+   // Solve
+   id->job = 3;
+   mumps_call();
+
+   // Unpack to real/imag
+   if (myid == 0)
+   {
+      for (int i = 0; i < nrhs; i++)
+      {
+         const mumps_complex_t *src = rhs_glob + i * global_num_rows;
+         real_t *rr = rhs_glob_r + i * global_num_rows;
+         real_t *ri = rhs_glob_i + i * global_num_rows;
+
+         for (int j = 0; j < global_num_rows; j++)
+         {
+            rr[j] = src[j].r;
+            ri[j] = src[j].i;
+         }
+      }
+   }
+
+   // Scatter each RHS solution
+   for (int i = 0; i < nrhs; i++)
+   {
+      MFEM_ASSERT(Y[i], "Missing Vector in Mult!");
+      Y[i]->HostWrite();
+      MFEM_VERIFY(Y[i]->Size() == 2*n_loc, "Output size mismatch");
+
+      real_t *ydata = Y[i]->GetData();
+
+      MPI_Scatterv(rhs_glob_r + i * global_num_rows,
+                   recv_counts, displs, MPITypeMap<real_t>::mpi_type,
+                   ydata, n_loc, MPITypeMap<real_t>::mpi_type,
+                   0, comm);
+
+      MPI_Scatterv(rhs_glob_i + i * global_num_rows,
+                   recv_counts, displs, MPITypeMap<real_t>::mpi_type,
+                   ydata + n_loc, n_loc, MPITypeMap<real_t>::mpi_type,
+                   0, comm);
+   }
+
+#endif
+}
+
+void ComplexMUMPSSolver::MultTranspose(const Vector &x, Vector &y) const
+{
+   MFEM_VERIFY(id, "MultTranspose called before SetOperator");
+
+   // Transpose solve
+   id->MUMPS_ICNTL(9) = 0;
+   Mult(x, y);
+   id->MUMPS_ICNTL(9) = 1;
+}
+
+void ComplexMUMPSSolver::ArrayMultTranspose(const Array<const Vector *> &X,
+                                            Array<Vector *> &Y) const
+{
+   MFEM_VERIFY(id, "ArrayMultTranspose called before SetOperator");
+
+   // Transpose solve
+   id->MUMPS_ICNTL(9) = 0;
+   ArrayMult(X, Y);
+   id->MUMPS_ICNTL(9) = 1;
+}
+
+void ComplexMUMPSSolver::SetParameters()
+{
+   // Output stream for error messages
+   id->MUMPS_ICNTL(1) = 6;
+   // Output stream for diagnostic printing local to each proc
+   id->MUMPS_ICNTL(2) = 0;
+   // Output stream for global info
+   id->MUMPS_ICNTL(3) = 6;
+   // Level of error printing
+   id->MUMPS_ICNTL(4) = print_level;
+
+   // Input matrix format (assembled)
+   id->MUMPS_ICNTL(5) = 0;
+   // Use A or A^T
+   id->MUMPS_ICNTL(9) = 1;
+   // Iterative refinement (disabled)
+   id->MUMPS_ICNTL(10) = 0;
+   // Error analysis-statistics (disabled)
+   id->MUMPS_ICNTL(11) = 0;
+   // Use of ScaLAPACK (disabled)
+   id->MUMPS_ICNTL(13) = 0;
+   // Workspace relaxation (% increase)
+   id->MUMPS_ICNTL(14) = 20;
+   // OpenMP threads (default)
+   id->MUMPS_ICNTL(16) = 0;
+   // Matrix input format (distributed)
+   id->MUMPS_ICNTL(18) = 3;
+   // Schur complement (none)
+   id->MUMPS_ICNTL(19) = 0;
+
+#if MFEM_MUMPS_VERSION >= 530
+   // Distributed RHS
+   id->MUMPS_ICNTL(20) = 10;
+   // Distributed Sol
+   id->MUMPS_ICNTL(21) = 1;
+#else
+   // Centralized RHS
+   id->MUMPS_ICNTL(20) = 0;
+   // Centralized Sol
+   id->MUMPS_ICNTL(21) = 0;
+#endif
+
+   // Out-of-core (disabled)
+   id->MUMPS_ICNTL(22) = 0;
+   // Max size of working memory (default)
+   id->MUMPS_ICNTL(23) = 0;
+
+   switch (reorder_method)
+   {
+      case ReorderingStrategy::AUTOMATIC:
+         id->MUMPS_ICNTL(28) = 0;
+         id->MUMPS_ICNTL(7) = 7;
+         id->MUMPS_ICNTL(29) = 0;
+         break;
+      case ReorderingStrategy::AMD:
+         id->MUMPS_ICNTL(28) = 1;
+         id->MUMPS_ICNTL(7) = 0;
+         break;
+      case ReorderingStrategy::AMF:
+         id->MUMPS_ICNTL(28) = 1;
+         id->MUMPS_ICNTL(7) = 2;
+         break;
+      case ReorderingStrategy::PORD:
+         id->MUMPS_ICNTL(28) = 1;
+         id->MUMPS_ICNTL(7) = 4;
+         break;
+      case ReorderingStrategy::METIS:
+         id->MUMPS_ICNTL(28) = 1;
+         id->MUMPS_ICNTL(7) = 5;
+         break;
+      case ReorderingStrategy::PARMETIS:
+         id->MUMPS_ICNTL(28) = 2;
+         id->MUMPS_ICNTL(29) = 2;
+         break;
+      case ReorderingStrategy::SCOTCH:
+         id->MUMPS_ICNTL(28) = 1;
+         id->MUMPS_ICNTL(7) = 3;
+         break;
+      case ReorderingStrategy::PTSCOTCH:
+         id->MUMPS_ICNTL(28) = 2;
+         id->MUMPS_ICNTL(29) = 1;
+         break;
+      default:
+         break; // This should be unreachable
+   }
 
 }
+
+void ComplexMUMPSSolver::BuildUnionCOO(const int n_loc,
+                                       const int row_start_,
+                                       const int *Ir, const int *Jr, const real_t *Vr,
+                                       const int *Ii, const int *Ji, const real_t *Vi,
+                                       std::vector<int> &Icoo,
+                                       std::vector<int> &Jcoo,
+                                       std::vector<mumps_complex_t> &Zcoo) const
+{
+   for (int r = 0; r < n_loc; ++r)
+   {
+      std::unordered_map<int, std::pair<real_t, real_t>> row;
+
+      const int rr0 = Ir ? Ir[r]   : 0;
+      const int rr1 = Ir ? Ir[r+1] : 0;
+      const int ii0 = Ii ? Ii[r]   : 0;
+      const int ii1 = Ii ? Ii[r+1] : 0;
+
+      row.reserve((rr1 - rr0) + (ii1 - ii0));
+
+      if (Ir)
+      {
+         for (int p = rr0; p < rr1; ++p) { row[Jr[p]].first += Vr[p]; }
+      }
+      if (Ii)
+      {
+         for (int p = ii0; p < ii1; ++p) { row[Ji[p]].second += Vi[p]; }
+      }
+
+      for (const auto &kv : row)
+      {
+         Icoo.push_back(row_start_ + r + 1);
+         Jcoo.push_back(kv.first + 1);
+         Zcoo.push_back(mumps_complex_t{kv.second.first, kv.second.second});
+      }
+   }
+}
+
+#if MFEM_MUMPS_VERSION >= 530
+int ComplexMUMPSSolver::GetRowRank(int i, const Array<int> &row_starts_) const
+{
+   if (row_starts_.Size() == 1) { return 0; }
+   auto up = std::upper_bound(row_starts_.begin(), row_starts_.end(), i);
+   return (int)std::distance(row_starts_.begin(), up) - 1;
+}
+
+void ComplexMUMPSSolver::RedistributeSol(const int *row_map,
+                                         const mumps_complex_t *x,
+                                         real_t *y_ri,
+                                         int n_loc,
+                                         int lsol_loc) const
+{
+   int *send_count = new int[numProcs]();
+   for (int i = 0; i < lsol_loc; i++)
+   {
+      const int j = row_map[i] - 1;
+      const int row_rank = GetRowRank(j, row_starts);
+      if (myid == row_rank) { continue; }
+      send_count[row_rank]++;
+   }
+
+   int *recv_count = new int[numProcs];
+   MPI_Alltoall(send_count, 1, MPI_INT, recv_count, 1, MPI_INT, comm);
+
+   int *send_displ = new int[numProcs]; send_displ[0] = 0;
+   int *recv_displ = new int[numProcs]; recv_displ[0] = 0;
+
+   int sbuff_size = send_count[numProcs-1];
+   int rbuff_size = recv_count[numProcs-1];
+   for (int k = 0; k < numProcs - 1; k++)
+   {
+      send_displ[k+1] = send_displ[k] + send_count[k];
+      recv_displ[k+1] = recv_displ[k] + recv_count[k];
+      sbuff_size += send_count[k];
+      rbuff_size += recv_count[k];
+   }
+
+   int   *sendbuf_index = new int[sbuff_size];
+   real_t *sendbuf_r    = new real_t[sbuff_size];
+   real_t *sendbuf_i    = new real_t[sbuff_size];
+   int   *soffs         = new int[numProcs]();
+
+   for (int i = 0; i < lsol_loc; i++)
+   {
+      const int j = row_map[i] - 1;
+      const int row_rank = GetRowRank(j, row_starts);
+
+      const real_t xr = (real_t)x[i].r;
+      const real_t xi = (real_t)x[i].i;
+
+      if (myid == row_rank)
+      {
+         const int local_index = j - row_start;
+         y_ri[local_index]        = xr;
+         y_ri[local_index+n_loc]  = xi;
+      }
+      else
+      {
+         const int k = send_displ[row_rank] + soffs[row_rank];
+         sendbuf_index[k] = j;
+         sendbuf_r[k] = xr;
+         sendbuf_i[k] = xi;
+         soffs[row_rank]++;
+      }
+   }
+
+   int    *recvbuf_index = new int[rbuff_size];
+   real_t *recvbuf_r     = new real_t[rbuff_size];
+   real_t *recvbuf_i     = new real_t[rbuff_size];
+
+   MPI_Alltoallv(sendbuf_index, send_count, send_displ, MPI_INT,
+                 recvbuf_index, recv_count, recv_displ, MPI_INT, comm);
+
+   MPI_Alltoallv(sendbuf_r, send_count, send_displ, MPITypeMap<real_t>::mpi_type,
+                 recvbuf_r, recv_count, recv_displ, MPITypeMap<real_t>::mpi_type, comm);
+
+   MPI_Alltoallv(sendbuf_i, send_count, send_displ, MPITypeMap<real_t>::mpi_type,
+                 recvbuf_i, recv_count, recv_displ, MPITypeMap<real_t>::mpi_type, comm);
+
+   for (int i = 0; i < rbuff_size; i++)
+   {
+      const int local_index = recvbuf_index[i] - row_start;
+      y_ri[local_index]       = recvbuf_r[i];
+      y_ri[local_index+n_loc] = recvbuf_i[i];
+   }
+
+   delete [] recvbuf_i;
+   delete [] recvbuf_r;
+   delete [] recvbuf_index;
+   delete [] soffs;
+   delete [] sendbuf_i;
+   delete [] sendbuf_r;
+   delete [] sendbuf_index;
+   delete [] recv_displ;
+   delete [] send_displ;
+   delete [] recv_count;
+   delete [] send_count;
+}
+#endif // MFEM_MUMPS_VERSION >= 530
+#endif // MFEM_USE_COMPLEX_MUMPS
+#endif // MFEM_USE_MPI
+
+} // namespace mfem

linalg/complex_operator.hpp

@@ -13,6 +13,7 @@
 #define MFEM_COMPLEX_OPERATOR
 
 #include "operator.hpp"
+#include "blockoperator.hpp"
 #include "sparsemat.hpp"
 #ifdef MFEM_USE_MPI
 #include "hypre.hpp"
@@ -22,8 +23,20 @@
 #include <umfpack.h>
 #endif
 
+#ifdef MFEM_USE_COMPLEX_MUMPS
+#ifdef MFEM_USE_SINGLE
+#include "cmumps_c.h"
+#else
+#include "zmumps_c.h"
+#endif
+#include <vector>
+#endif
+
 namespace mfem
 {
+#ifdef MFEM_USE_MPI
+class ComplexHypreParMatrix; // forward declaration
+#endif
 
 /** @brief Mimic the action of a complex operator using two real operators.
 
@@ -118,6 +131,13 @@ public:
 
    Convention GetConvention() const { return convention_; }
 
+#ifdef MFEM_USE_MPI
+   /// Return a ComplexHypreParMatrix view:
+   ///  - wraps if real/imag are HypreParMatrix
+   ///  - merges if real/imag are BlockOperator of HypreParMatrix blocks
+   ComplexHypreParMatrix *AsComplexHypreParMatrix() const;
+#endif
+
 protected:
    // Let this be hidden from the public interface since the implementation
    // depends on internal members
@@ -242,6 +262,7 @@ public:
 
 #endif
 
+
 #ifdef MFEM_USE_MPI
 
 /** @brief Specialization of the ComplexOperator built from a pair of
@@ -288,6 +309,265 @@ private:
    int myid_;
    int nranks_;
 };
+/// A BlockOperator whose blocks are ComplexOperator objects, constructed from
+/// a ComplexOperator whose Real()/Imag() parts are BlockOperator objects.
+/// It also provides layout conversions between:
+///   • BlockComplex (stacked): [ Re(all); Im(all) ]
+///   • ComplexBlock (per-block): [ Re(block_b); Im(block_b) ]
+class ComplexBlockOperator : public BlockOperator
+{
+public:
+   /// Construct from a ComplexOperator whose Real()/Imag() parts are
+   /// BlockOperator objects.
+   ComplexBlockOperator(const ComplexOperator &A);
+
+   /// Convert vector from BlockComplex (stacked) -> ComplexBlock (per-block).
+   /// Sizes must match (xout.Size() == xin.Size() == 2*N).
+   void BlockComplexToComplexBlock(const Vector &xin,
+                                   Vector &xout) const;
+   /// Convert vector from ComplexBlock (per-block) -> BlockComplex (stacked).
+   /// Sizes must match (xout.Size() == xin.Size() == 2*N).
+   void ComplexBlockToBlockComplex(const Vector &xin,
+                                   Vector &xout) const;
+private:
+};
+
+#ifdef MFEM_USE_COMPLEX_MUMPS
+/**
+ * @brief Complex MUMPS: Parallel sparse direct solver for ComplexHypreParMatrix
+ *
+ * Notes:
+ *  - Expects Operator to be a ComplexHypreParMatrix.
+ *  - Complex vectors are assumed packed as [Re; Im] in a real Vector.
+ *  - SetOperator(): analysis + factorization
+ *  - Mult()       : solve
+ */
+class ComplexMUMPSSolver : public Solver
+{
+public:
+   /// Specify the reordering strategy
+   enum ReorderingStrategy
+   {
+      /// Let MUMPS automatically decide the reordering strategy
+      AUTOMATIC = 0,
+      /// Approximate Minimum Degree with auto quasi-dense row detection is used
+      AMD,
+      /// Approximate Minimum Fill method will be used
+      AMF,
+      /// The PORD library will be used
+      PORD,
+      /// The METIS library will be used
+      METIS,
+      /// The ParMETIS library will be used
+      PARMETIS,
+      /// The Scotch library will be used
+      SCOTCH,
+      /// The PTScotch library will be used
+      PTSCOTCH
+   };
+
+   /**
+    * @brief Constructor with MPI_Comm parameter.
+    */
+   ComplexMUMPSSolver(MPI_Comm comm_);
+   /**
+    * @brief Constructor with a ComplexHypreParMatrix Operator.
+    */
+   ComplexMUMPSSolver(const Operator &op);
+
+   /**
+    * @brief Set the Operator and perform factorization
+    *
+    * @a op needs to be of type ComplexHypreParMatrix.
+    *
+    * @param op Operator used in factorization and solve
+    */
+   void SetOperator(const Operator &op);
+
+   /**
+    * @brief Solve $ y = Op^{-1} x $
+    *
+    * @param x RHS vector
+    * @param y Solution vector
+    */
+   void Mult(const Vector &x, Vector &y) const;
+   /**
+    * @brief Solve $ Y_i = Op^{-1} X_i $
+    *
+    * @param X Array of RHS vectors
+    * @param Y Array of Solution vectors
+    */
+   void ArrayMult(const Array<const Vector *> &X, Array<Vector *> &Y) const;
+   /**
+    * @brief Transpose Solve $ y = Op^{-T} x $
+    * @note This is not a Hermitian/conjugate-transpose solve.
+    *
+    * @param x RHS vector
+    * @param y Solution vector
+    */
+   void MultTranspose(const Vector &x, Vector &y) const;
+
+   /**
+    * @brief Transpose Solve $ Y_i = Op^{-T} X_i $
+    * @note This is not a Hermitian/conjugate-transpose solve.
+    *
+    * @param X Array of RHS vectors
+    * @param Y Array of Solution vectors
+    */
+   void ArrayMultTranspose(const Array<const Vector *> &X,
+                           Array<Vector *> &Y) const;
+
+   /**
+    * @brief Set the error print level for MUMPS
+    *
+    * Supported values are:
+    * - 0:  No output printed
+    * - 1:  Only errors printed
+    * - 2:  Errors, warnings, and main stats printed
+    * - 3:  Errors, warnings, main stats, and terse diagnostics printed
+    * - 4:  Errors, warnings, main stats, diagnostics, and input/output printed
+    *
+    * @param print_lvl Print level, default is 2
+    *
+    * @note This method has to be called before SetOperator
+    */
+   void SetPrintLevel(int print_lvl) { print_level = print_lvl;}
+
+   /**
+    * @brief Set the reordering strategy
+    *
+    * Supported reorderings are: ComplexMUMPSSolver::AUTOMATIC,
+    * ComplexMUMPSSolver::AMD, ComplexMUMPSSolver::AMF,
+    * ComplexMUMPSSolver::PORD, ComplexMUMPSSolver::METIS,
+    * ComplexMUMPSSolver::PARMETIS, ComplexMUMPSSolver::SCOTCH,
+    * and ComplexMUMPSSolver::PTSCOTCH
+    *
+    * @param method Reordering method
+    *
+    * @note This method has to be called before SetOperator
+    */
+   void SetReorderingStrategy(ReorderingStrategy method) { reorder_method = method; }
+
+   /**
+    * @brief Set the flag controlling reuse of the symbolic factorization
+    * for multiple operators
+    *
+    * @param reuse Flag to reuse symbolic factorization
+    *
+    * @note This method has to be called before repeated calls to SetOperator
+    */
+   void SetReorderingReuse(bool reuse) { reorder_reuse = reuse; }
+
+   ~ComplexMUMPSSolver();
+
+private:
+   // MPI communicator
+   MPI_Comm comm = MPI_COMM_NULL;
+
+   // Number of procs
+   int numProcs;
+
+   // MPI rank
+   int myid;
+
+   // Parameter controlling the printing level
+   int print_level = 0;
+
+   // Parameter controlling the reordering strategy
+   ReorderingStrategy reorder_method = ReorderingStrategy::AUTOMATIC;
+
+   // Parameter controlling whether or not to reuse the symbolic factorization
+   // for multiple calls to SetOperator
+   bool reorder_reuse = false;
+
+   // Local row offsets
+   int row_start;
+
+   // ComplexMUMPS object
+#ifdef MFEM_USE_SINGLE
+   CMUMPS_STRUC_C *id = nullptr;
+   using mumps_complex_t = mumps_complex;
+#else
+   ZMUMPS_STRUC_C *id = nullptr;
+   using mumps_complex_t = mumps_double_complex;
+#endif
+
+   /// Method for initialization
+   void Init(MPI_Comm comm_);
+
+   /// Method for setting ComplexMUMPS internal parameters
+   void SetParameters();
+
+   /// Method for configuring storage for distributed/centralized
+   /// RHS and solution
+   void InitRhsSol(int nrhs) const;
+
+   /// Method for calling the single/double ComplexMUMPS solver
+   inline void mumps_call() const
+   {
+#ifdef MFEM_USE_SINGLE
+      cmumps_c(id);
+#else
+      zmumps_c(id);
+#endif
+   }
+
+   /// Method for building the COO format of the combined complex operator
+   /// from the real and imaginary parts. This is particularly useful when
+   /// real and imaginary parts have different sparsity patterns.
+   void BuildUnionCOO(const int n_loc,
+                      const int row_start,
+                      const int *Ir, const int *Jr, const real_t *Vr,
+                      const int *Ii, const int *Ji, const real_t *Vi,
+                      std::vector<int> &Icoo,
+                      std::vector<int> &Jcoo,
+                      std::vector<mumps_complex_t> &Zcoo) const;
+
+#if MFEM_MUMPS_VERSION >= 530
+   // Row offsets on all procs
+   Array<int> row_starts;
+
+   // Local RHS row indices
+   int *irhs_loc = nullptr;
+
+   // Local solution row map returned by MUMPS
+   int *isol_loc = nullptr;
+
+   // Cached buffers
+   mutable mumps_complex_t *rhs_loc = nullptr;
+   mutable mumps_complex_t *sol_loc = nullptr;
+
+   // RHS buffers
+   mutable std::vector<mumps_complex_t> rhs1_buf;
+
+   // These two methods are needed to distribute the local solution
+   // vectors returned by MUMPS to the original MFEM parallel partition
+   int GetRowRank(int i, const Array<int> &row_starts_) const;
+
+   void RedistributeSol(const int *row_map,
+                        const mumps_complex_t *x,
+                        real_t *y_ri,
+                        int n_loc,
+                        int lsol_loc) const;
+
+#else
+   // Root-gather path
+   int global_num_rows;
+
+   // Arrays needed for MPI_Gatherv and MPI_Scatterv
+   int *recv_counts = nullptr;
+   int *displs = nullptr;
+
+   // Complex RHS/solution on root
+   mutable mumps_complex_t *rhs_glob = nullptr;
+
+   // Cached real/imag staging on root
+   mutable real_t *rhs_glob_r = nullptr;
+   mutable real_t *rhs_glob_i = nullptr;
+#endif
+};
+
+#endif // MFEM_USE_COMPLEX_MUMPS
 
 #endif // MFEM_USE_MPI
 

linalg/densemat.cpp

@@ -2185,6 +2185,45 @@ void DenseMatrix::AdjustDofDirection(Array<int> &dofs)
    }
 }
 
+void DenseMatrix::AdjustDofDirection(Array<int> &row_dofs,
+                                     Array<int> &col_dofs)
+{
+   const int nr = row_dofs.Size();
+   const int nc = col_dofs.Size();
+
+   MFEM_VERIFY(Height() == nr && Width() == nc,
+               "DenseMatrix::AdjustDofDirection: size mismatch.");
+
+   // Extract signs and convert to unsigned indices
+   Vector rsign(nr), csign(nc);
+
+   for (int i = 0; i < nr; i++)
+   {
+      const int d = row_dofs[i];
+      if (d >= 0) { rsign(i) =  1.0; }
+      else        { rsign(i) = -1.0; row_dofs[i] = -d - 1; continue; }
+      row_dofs[i] = d;
+   }
+
+   for (int j = 0; j < nc; j++)
+   {
+      const int d = col_dofs[j];
+      if (d >= 0) { csign(j) =  1.0; }
+      else        { csign(j) = -1.0; col_dofs[j] = -d - 1; continue; }
+      col_dofs[j] = d;
+   }
+
+   // Apply row/column signs
+   for (int i = 0; i < nr; i++)
+   {
+      const real_t rs = rsign(i);
+      for (int j = 0; j < nc; j++)
+      {
+         (*this)(i,j) *= rs * csign(j);
+      }
+   }
+}
+
 void DenseMatrix::SetRow(int row, real_t value)
 {
    for (int j = 0; j < Width(); j++)

linalg/densemat.hpp

@@ -82,6 +82,15 @@ public:
       }
    }
 
+   /// Make the DenseMatrix to reference the given sub-Memory of @a base.
+   /** The DenseMatrix does not assume ownership of the data array, i.e. it will
+       not delete the @a base Memory. */
+   void MakeRef(Memory<real_t> &base, int offset, int h, int w)
+   {
+      data.MakeRef(base, offset, h*w);
+      height = h; width = w;
+   }
+
    /// Change the data array and the size of the DenseMatrix.
    /** The DenseMatrix does not assume ownership of the data array, i.e. it will
        not delete the data array @a d. */
@@ -467,6 +476,10 @@ public:
        then (*this)(i,j) = -(*this)(i,j).  */
    void AdjustDofDirection(Array<int> &dofs);
 
+
+   void AdjustDofDirection(Array<int> &row_dofs,
+                           Array<int> &col_dofs);
+
    /// Replace small entries, abs(a_ij) <= eps, with zero.
    void Threshold(real_t eps);
 

linalg/hypre.cpp

@@ -3443,6 +3443,51 @@ HypreParMatrix *HypreParMatrixFromBlocks(Array2D<const HypreParMatrix*> &blocks,
    }
 }
 
+HypreParMatrix * GetSubHypreParMatrix(const Array<int> &tdofs,
+                                      const HypreParMatrix & A)
+{
+   int nrows = tdofs.Size();
+   int gncols = A.GetGlobalNumCols();
+   SparseMatrix Sr(nrows, gncols);
+
+   for (int i = 0; i<nrows; i++)
+   {
+      int col = tdofs[i] + A.RowPart()[0];
+      Sr.Set(i,col,1.0);
+   }
+   Sr.Finalize();
+
+
+   int newrows[2];
+   int newcols[2];
+
+   int row_offset;
+   MPI_Scan(&nrows,&row_offset,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
+
+   row_offset-=nrows;
+   newrows[0] = row_offset;
+   newrows[1] = row_offset+nrows;
+   newcols[0] = A.ColPart()[0];
+   newcols[1] = A.ColPart()[1];
+   int gnrows;
+   MPI_Allreduce(&nrows, &gnrows,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
+
+   HypreParMatrix * Prt = new HypreParMatrix(MPI_COMM_WORLD, nrows, gnrows,
+                                             gncols, Sr.GetI(), Sr.GetJ(),
+                                             Sr.GetData(), newrows,newcols);
+
+
+   HypreParMatrix * Pr = Prt->Transpose();
+   delete Prt;
+
+   HypreParMatrix * tmp = RAP(&A,Pr);
+   delete Pr;
+
+   return tmp;
+
+}
+
+
 HypreParMatrix * HypreParMatrixFromBlocks(Array2D<HypreParMatrix*> &blocks,
                                           Array2D<real_t> *blockCoeff)
 {
@@ -5224,7 +5269,7 @@ void HypreBoomerAMG::SetDefaultOptions()
       Pmax         = 4;    // max number of elements per row in P
 
       // AMG relaxation options:
-      relax_type   = 8;    // 8 = l1-GS, 6 = symm. GS, 3 = GS, 18 = l1-Jacobi
+      relax_type   = 88;    // 8 = l1-GS, 6 = symm. GS, 3 = GS, 18 = l1-Jacobi
       relax_sweeps = 1;    // relaxation sweeps on each level
 
       // Additional options:
@@ -5741,7 +5786,7 @@ void HypreAMS::MakeSolver(int sdim, int cycle_type)
    const bool hypre_gpu = HypreUsingGPU();
    int amg_coarsen_type = hypre_gpu ? 8 : 10;
    int amg_agg_levels   = hypre_gpu ? 0 : 1;
-   int amg_rlx_type     = hypre_gpu ? 18 : 8;
+   int amg_rlx_type     = hypre_gpu ? 18 : 88;
    int rlx_type         = hypre_gpu ? 1: 2;
    real_t theta         = 0.25;
    int amg_interp_type  = 6;

linalg/hypre.hpp

@@ -1063,6 +1063,11 @@ MFEM_DEPRECATED HypreParMatrix *HypreParMatrixFromBlocks(
    Array2D<HypreParMatrix*> &blocks,
    Array2D<real_t> *blockCoeff=NULL);
 
+/** Extract a SubMatrix of a HypreParMatrix constructed
+    from the given rows and cols (local numbering) */
+HypreParMatrix * GetSubHypreParMatrix(const Array<int> &dofs,
+                                      const HypreParMatrix & A);
+
 /** @brief Eliminate essential BC specified by @a ess_dof_list from the solution
     @a X to the r.h.s. @a B. */
 /** Here @a A is a matrix with eliminated BC, while @a Ae is such that (A+Ae) is

linalg/mumps.hpp

@@ -52,7 +52,7 @@ public:
    /// Specify the reordering strategy for the MUMPS solver
    enum ReorderingStrategy
    {
-      /// Let MUMPS automatically decide the reording strategy
+      /// Let MUMPS automatically decide the reordering strategy
       AUTOMATIC = 0,
       /// Approximate Minimum Degree with auto quasi-dense row detection is used
       AMD,
@@ -98,7 +98,7 @@ public:
    void Mult(const Vector &x, Vector &y) const;
 
    /**
-    * @brief Solve $ Y_i = Op^{-T} X_i $
+    * @brief Solve $ Y_i = Op^{-1} X_i $
     *
     * @param X Array of RHS vectors
     * @param Y Array of Solution vectors
@@ -129,8 +129,8 @@ public:
     * - 0:  No output printed
     * - 1:  Only errors printed
     * - 2:  Errors, warnings, and main stats printed
-    * - 3:  Errors, warning, main stats, and terse diagnostics printed
-    * - 4:  Errors, warning, main stats, diagnostics, and input/output printed
+    * - 3:  Errors, warnings, main stats, and terse diagnostics printed
+    * - 4:  Errors, warnings, main stats, diagnostics, and input/output printed
     *
     * @param print_lvl Print level, default is 2
     *

linalg/operator.cpp

@@ -466,10 +466,11 @@ RAPOperator::RAPOperator(const Operator &Rt_, const Operator &A_,
 
 TripleProductOperator::TripleProductOperator(
    const Operator *A, const Operator *B, const Operator *C,
-   bool ownA, bool ownB, bool ownC)
+   bool ownA, bool ownB, bool ownC, real_t alpha_)
    : Operator(A->Height(), C->Width())
    , A(A), B(B), C(C)
    , ownA(ownA), ownB(ownB), ownC(ownC)
+   , alpha(alpha_)
 {
    MFEM_VERIFY(A->Width() == B->Height(),
                "incompatible Operators: A->Width() = " << A->Width()

linalg/operator.hpp

@@ -1011,27 +1011,28 @@ public:
 };
 
 
-/// General triple product operator x -> A*B*C*x, with ownership of the factors.
+/// General triple product operator x -> α A*B*C*x, with ownership of the factors.
 class TripleProductOperator : public Operator
 {
    const Operator *A;
    const Operator *B;
    const Operator *C;
    bool ownA, ownB, ownC;
+   const real_t alpha;
    mutable Vector t1, t2;
    MemoryClass mem_class;
 
 public:
    TripleProductOperator(const Operator *A, const Operator *B,
-                         const Operator *C, bool ownA, bool ownB, bool ownC);
+                         const Operator *C, bool ownA, bool ownB, bool ownC, real_t alpha_ = 1.0);
 
    MemoryClass GetMemoryClass() const override { return mem_class; }
 
    void Mult(const Vector &x, Vector &y) const override
-   { C->Mult(x, t1); B->Mult(t1, t2); A->Mult(t2, y); }
+   { C->Mult(x, t1); B->Mult(t1, t2); A->Mult(t2, y); y*=alpha; }
 
    void MultTranspose(const Vector &x, Vector &y) const override
-   { A->MultTranspose(x, t2); B->MultTranspose(t2, t1); C->MultTranspose(t1, y); }
+   { A->MultTranspose(x, t2); B->MultTranspose(t2, t1); C->MultTranspose(t1, y); y*=alpha; }
 
    virtual ~TripleProductOperator();
 };

linalg/solvers.cpp

@@ -683,8 +683,7 @@ void SLISolver::Mult(const Vector &b, Vector &x) const
          oper->Mult(x, r);  // r = A x
          if (!zero_b) { subtract(b, r, r); } // r = b - A x
          prec->Mult(r, z);  // z = B r
-         if (!zero_b) { x += z; } // x = x + B (b - A x)
-         else { x -= z; }         // x = x - B (A x)
+         add(x, omega, z, x); // x = x + ω B (b - A x)
       }
       converged = true;
       final_iter = i;
@@ -701,7 +700,7 @@ void SLISolver::Mult(const Vector &b, Vector &x) const
          oper->Mult(x, r);  // r = A x
          subtract(b, r, r); // r = b - A x
          prec->Mult(r, z);  // z = B r
-         x += z;            // x = x + B (b - A x)
+         add(x, omega, z, x); // x = x + ω B (b - A x)
       }
       converged = true;
       final_iter = i;
@@ -754,15 +753,13 @@ void SLISolver::Mult(const Vector &b, Vector &x) const
    final_iter = max_iter;
    for (i = 1; true; )
    {
-      if (prec)
+      if (prec) //  x = x + ω B (b - A x)
       {
-         if (!zero_b) { x += z; }  // x = x + B (b - A x)
-         else { x -= z; }          // x = x - B (A x)
+         add(x, omega, z, x);
       }
       else
       {
-         if (!zero_b) { x += r; }  // x = x + (b - A x)
-         else { x -= r; }          // x = x - (A x)
+         add(x, omega, r, x);
       }
 
       oper->Mult(x, r);

linalg/solvers.hpp

@@ -586,19 +586,20 @@ private:
 };
 
 
-/// Stationary linear iteration: x <- x + B (b - A x)
+/// Stationary linear iteration: x <- x + ω B (b - A x)
 class SLISolver : public IterativeSolver
 {
 protected:
    mutable Vector r, z;
-
+   real_t omega; ///< Relaxation parameter
    void UpdateVectors();
 
 public:
-   SLISolver() { }
+   SLISolver(real_t omega_ = 1.0) : omega(omega_) { }
 
 #ifdef MFEM_USE_MPI
-   SLISolver(MPI_Comm comm_) : IterativeSolver(comm_) { }
+   SLISolver(MPI_Comm comm_, real_t omega_ = 1.0) : IterativeSolver(comm_),
+      omega(omega_) { }
 #endif
 
    void SetOperator(const Operator &op) override

makefile

@@ -224,7 +224,7 @@ CXXFLAGS ?= $(OPTIM_FLAGS)
 # MPI configuration
 ifneq ($(MFEM_USE_MPI),YES)
    HOST_CXX = $(CXX)
-   PKGS_NEED_MPI = SUPERLU MUMPS STRUMPACK PETSC PUMI SLEPC MKL_CPARDISO
+   PKGS_NEED_MPI = SUPERLU MUMPS COMPLEX_MUMPS STRUMPACK PETSC PUMI SLEPC MKL_CPARDISO
    $(foreach mpidep,$(PKGS_NEED_MPI),$(if $(MFEM_USE_$(mpidep):NO=),\
      $(warning *** [MPI is OFF] setting MFEM_USE_$(mpidep) = NO)\
      $(eval override MFEM_USE_$(mpidep)=NO),))
@@ -299,7 +299,7 @@ ifeq ($(MFEM_USE_LEGACY_OPENMP),YES)
 endif
 
 # List of MFEM dependencies, that require the *_LIB variable to be non-empty
-MFEM_REQ_LIB_DEPS = SUPERLU MUMPS METIS FMS CONDUIT SIDRE LAPACK SUNDIALS\
+MFEM_REQ_LIB_DEPS = SUPERLU MUMPS COMPLEX_MUMPS METIS FMS CONDUIT SIDRE LAPACK SUNDIALS\
  SUITESPARSE STRUMPACK GINKGO GNUTLS HDF5 NETCDF SLEPC PETSC MPFR PUMI HIOP\
  GSLIB OCCA CEED RAJA UMPIRE MKL_CPARDISO MKL_PARDISO AMGX MAGMA CALIPER PARELAG\
  TRIBOL BENCHMARK MOONOLITH ALGOIM
@@ -369,7 +369,7 @@ MFEM_DEFINES = MFEM_VERSION MFEM_VERSION_STRING MFEM_GIT_STRING MFEM_USE_MPI\
  MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA MFEM_USE_HIP\
  MFEM_USE_OCCA MFEM_USE_MOONOLITH MFEM_USE_CEED MFEM_USE_RAJA MFEM_USE_UMPIRE\
  MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO MFEM_USE_AMGX\
- MFEM_USE_MAGMA MFEM_USE_MUMPS MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_CALIPER\
+ MFEM_USE_MAGMA MFEM_USE_MUMPS MFEM_USE_COMPLEX_MUMPS MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_CALIPER\
  MFEM_USE_BENCHMARK MFEM_USE_PARELAG MFEM_USE_TRIBOL MFEM_USE_ALGOIM MFEM_USE_ENZYME\
  MFEM_SOURCE_DIR MFEM_INSTALL_DIR MFEM_SHARED_BUILD MFEM_USE_DOUBLE MFEM_USE_SINGLE
 
@@ -406,7 +406,7 @@ MFEM_INSTALL_DIR = $(abspath $(MFEM_PREFIX))
 # If we have 'config' target, export variables used by config/makefile
 ifneq (,$(filter config,$(MAKECMDGOALS)))
    export $(MFEM_DEFINES) MFEM_DEFINES $(MFEM_CONFIG_VARS) MFEM_CONFIG_VARS
-   export VERBOSE HYPRE_OPT PUMI_DIR MUMPS_OPT GSLIB_OPT
+   export VERBOSE HYPRE_OPT PUMI_DIR MUMPS_OPT COMPLEX_MUMPS_OPT GSLIB_OPT
 endif
 
 # If we have 'install' target, export variables used by config/makefile
@@ -736,6 +736,7 @@ status info:
 	$(info MFEM_USE_SUPERLU       = $(MFEM_USE_SUPERLU))
 	$(info MFEM_USE_SUPERLU5      = $(MFEM_USE_SUPERLU5))
 	$(info MFEM_USE_MUMPS         = $(MFEM_USE_MUMPS))
+	$(info MFEM_USE_COMPLEX_MUMPS = $(MFEM_USE_COMPLEX_MUMPS))
 	$(info MFEM_USE_STRUMPACK     = $(MFEM_USE_STRUMPACK))
 	$(info MFEM_USE_GINKGO        = $(MFEM_USE_GINKGO))
 	$(info MFEM_USE_AMGX          = $(MFEM_USE_AMGX))

mesh/mesh.cpp

@@ -15751,9 +15751,18 @@ Mesh PartitionMPI(int dim, int mpi_cnt, int elem_per_mpi, bool print,
 {
    MFEM_VERIFY(dim > 1, "Not implemented for 1D meshes.");
 
-   auto factor = [&](int N)
+   // Closest int divisor to the cubit root, going down.
+   auto factor3 = [](int N)
    {
-      for (int i = static_cast<int>(sqrt(N)); i > 0; i--)
+      for (int i = static_cast<int>(round(cbrt(N))); i > 0; i--)
+      {  if (N % i == 0) { return i; } }
+      return 1;
+   };
+
+   // Closest int divisor to the square root, going down.
+   auto factor2 = [](int N)
+   {
+      for (int i = static_cast<int>(round(sqrt(N))); i > 0; i--)
       { if (N % i == 0) { return i; } }
       return 1;
    };
@@ -15777,22 +15786,22 @@ Mesh PartitionMPI(int dim, int mpi_cnt, int elem_per_mpi, bool print,
    int el0_x, el0_y, el0_z;
    if (dim == 2)
    {
-      mpi_x = factor(mpi_cnt);
+      mpi_x = factor2(mpi_cnt);
       mpi_y = mpi_cnt / mpi_x;
 
       // Switch order for better balance.
-      el0_y = factor(el0);
+      el0_y = factor2(el0);
       el0_x = el0 / el0_y;
    }
    else
    {
-      mpi_x = factor(mpi_cnt);
-      mpi_y = factor(mpi_cnt / mpi_x);
+      mpi_x = factor3(mpi_cnt);
+      mpi_y = factor2(mpi_cnt / mpi_x);
       mpi_z = mpi_cnt / mpi_x / mpi_y;
 
       // Switch order for better balance.
-      el0_z = factor(el0);
-      el0_y = factor(el0 / el0_z);
+      el0_z = factor3(el0);
+      el0_y = factor2(el0 / el0_z);
       el0_x = el0 / el0_y / el0_z;
    }
 

miniapps/dpg/CMakeLists.txt

@@ -14,14 +14,16 @@ util/weakform.cpp
 util/complexweakform.cpp
 util/blockstaticcond.cpp
 util/complexstaticcond.cpp
-util/pml.cpp)
+util/pml.cpp
+util/preconditioners.cpp)
 
 list(APPEND DPG_HEADERS
 util/weakform.hpp
 util/complexweakform.hpp
 util/blockstaticcond.hpp
 util/complexstaticcond.hpp
-util/pml.hpp)
+util/pml.hpp
+util/preconditioners.hpp)
 
 if (MFEM_USE_MPI)
   list(APPEND DPG_SOURCES

miniapps/dpg/makefile

@@ -12,20 +12,19 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/dpg/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/dpg,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
-DPG_REAL_SEQ_SRC = util/weakform.cpp util/blockstaticcond.cpp
+DPG_REAL_SEQ_SRC = util/weakform.cpp util/blockstaticcond.cpp util/utils.cpp util/preconditioners.cpp
 DPG_REAL_PAR_SRC = $(DPG_REAL_SEQ_SRC) util/pweakform.cpp 
 DPG_REAL_OBJ = $(DPG_REAL_PAR_SRC:.cpp=.o)
 
-DPG_COMPLEX_SEQ_SRC = util/complexweakform.cpp util/complexstaticcond.cpp util/pml.cpp
-DPG_COMPLEX_PAR_SRC = $(DPG_COMPLEX_SEQ_SRC) util/pcomplexweakform.cpp 
+DPG_COMPLEX_SEQ_SRC = util/maxwell_utils.cpp util/utils.cpp util/complexblockform.cpp util/complexweakform.cpp util/complexstaticcond.cpp util/pml.cpp util/preconditioners.cpp
+DPG_COMPLEX_PAR_SRC = $(DPG_COMPLEX_SEQ_SRC) util/pcomplexweakform.cpp util/pcomplexblockform.cpp 
+
 DPG_COMPLEX_OBJ = $(DPG_COMPLEX_PAR_SRC:.cpp=.o)
 
 DIFFUSION_SRC = diffusion.cpp $(DPG_REAL_SEQ_SRC)
@@ -34,6 +33,12 @@ DIFFUSION_OBJ = $(DIFFUSION_SRC:.cpp=.o)
 PDIFFUSION_SRC = pdiffusion.cpp $(DPG_REAL_PAR_SRC)
 PDIFFUSION_OBJ = $(PDIFFUSION_SRC:.cpp=.o)
 
+PDIFFUSIONPRIMAL_SRC = pdiffusion-primal.cpp $(DPG_REAL_PAR_SRC)
+PDIFFUSIONPRIMAL_OBJ = $(PDIFFUSIONPRIMAL_SRC:.cpp=.o)
+
+PVDIFFUSIONPRIMAL_SRC = pvectordiffusion-primal.cpp $(DPG_REAL_PAR_SRC)
+PVDIFFUSIONPRIMAL_OBJ = $(PVDIFFUSIONPRIMAL_SRC:.cpp=.o)
+
 CONVECTIONDIFFUSION_SRC = convection-diffusion.cpp $(DPG_REAL_SEQ_SRC)
 CONVECTIONDIFFUSION_OBJ = $(CONVECTIONDIFFUSION_SRC:.cpp=.o)
 
@@ -46,14 +51,34 @@ ACOUSTICS_OBJ = $(ACOUSTICS_SRC:.cpp=.o)
 PACOUSTICS_SRC = pacoustics.cpp $(DPG_COMPLEX_PAR_SRC)
 PACOUSTICS_OBJ = $(PACOUSTICS_SRC:.cpp=.o)
 
+PACOUSTICS_PRIMAL_SRC = pacoustics-primal.cpp $(DPG_COMPLEX_PAR_SRC)
+PACOUSTICS_PRIMAL_OBJ = $(PACOUSTICS_PRIMAL_SRC:.cpp=.o)
+
 MAXWELL_SRC = maxwell.cpp $(DPG_COMPLEX_SEQ_SRC)
 MAXWELL_OBJ = $(MAXWELL_SRC:.cpp=.o)
 
 PMAXWELL_SRC = pmaxwell.cpp $(DPG_COMPLEX_PAR_SRC)
 PMAXWELL_OBJ = $(PMAXWELL_SRC:.cpp=.o)
 
+PMAXWELL_V_SRC = pmaxwell-verify.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_V_OBJ = $(PMAXWELL_V_SRC:.cpp=.o)
+
+PMAXWELL_PRIMAL_SRC = pmaxwell-primal.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_PRIMAL_OBJ = $(PMAXWELL_PRIMAL_SRC:.cpp=.o)
+
+PMAXWELL_FEM_SRC = pmaxwell-fem.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_FEM_OBJ = $(PMAXWELL_FEM_SRC:.cpp=.o)
+
+PMAXWELL_UW_COUPLED_SRC = pmaxwell-coupled.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_UW_COUPLED_OBJ = $(PMAXWELL_UW_COUPLED_SRC:.cpp=.o)
+
+PMAXWELL_FEM_COUPLED_SRC = pmaxwell-coupled-fem.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_FEM_COUPLED_OBJ = $(PMAXWELL_FEM_COUPLED_SRC:.cpp=.o)
+
 SEQ_MINIAPPS = diffusion convection-diffusion acoustics maxwell
-PAR_MINIAPPS = pdiffusion pconvection-diffusion pacoustics pmaxwell
+PAR_MINIAPPS = pdiffusion pconvection-diffusion pacoustics pmaxwell pmaxwell-verify  \
+ pacoustics-primal pmaxwell-primal pdiffusion-primal pvectordiffusion-primal \
+ pmaxwell-coupled pmaxwell-coupled-fem pmaxwell-fem
 
 ifeq ($(MFEM_USE_MPI),NO)
    MINIAPPS = $(SEQ_MINIAPPS)
@@ -79,11 +104,6 @@ COMMON_LIB += $(if $(MFEM_SHARED:YES=),,\
  $(CONFIG_MK) | lib-common
 	$(MFEM_CXX) $(MFEM_FLAGS) -c $< -o $@
 
-util/%.o: $(SRC)util/%.cpp $(wildcard $(SRC)util/%.hpp) $(MFEM_LIB_FILE)\
- $(CONFIG_MK) | lib-common
-	mkdir -p $(@D)
-	$(MFEM_CXX) $(MFEM_FLAGS) -c $< -o $@
-
 all: $(MINIAPPS)
 
 diffusion: $(DIFFUSION_OBJ)
@@ -95,6 +115,12 @@ convection-diffusion: $(CONVECTIONDIFFUSION_OBJ)
 pdiffusion: $(PDIFFUSION_OBJ)
 	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PDIFFUSION_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
 
+pdiffusion-primal: $(PDIFFUSIONPRIMAL_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PDIFFUSIONPRIMAL_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+pvectordiffusion-primal: $(PVDIFFUSIONPRIMAL_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PVDIFFUSIONPRIMAL_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
 pconvection-diffusion: $(PCONVECTIONDIFFUSION_OBJ)
 	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PCONVECTIONDIFFUSION_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
 
@@ -107,9 +133,30 @@ maxwell: $(MAXWELL_OBJ)
 pacoustics: $(PACOUSTICS_OBJ)
 	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PACOUSTICS_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
 
+pacoustics-primal: $(PACOUSTICS_PRIMAL_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PACOUSTICS_PRIMAL_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
 pmaxwell: $(PMAXWELL_OBJ)
 	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
 
+pmaxwell-fem: $(PMAXWELL_FEM_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_FEM_OBJ) $(COMMON_LIB) $(MFEM_LIBS)	
+
+pmaxwell-verify: $(PMAXWELL_V_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_V_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+
+pmaxwell-primal: $(PMAXWELL_PRIMAL_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_PRIMAL_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+pmaxwell-primal-tokamak: $(PMAXWELL_PRIMAL_TOKAMAK_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_PRIMAL_TOKAMAK_OBJ) $(COMMON_LIB) $(MFEM_LIBS)	
+
+pmaxwell-coupled: $(PMAXWELL_UW_COUPLED_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_UW_COUPLED_OBJ) $(COMMON_LIB) $(MFEM_LIBS)		
+
+pmaxwell-coupled-fem: $(PMAXWELL_FEM_COUPLED_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_FEM_COUPLED_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
 
 # Rule for building lib-common
 lib-common:

miniapps/dpg/pacoustics-primal.cpp

@@ -0,0 +1,743 @@
+//                   MFEM Ultraweak DPG acoustics example
+//
+// Compile with: make pacoustics
+//
+// sample runs
+
+// mpirun -np 4 pacoustics-primal -o 3 -m ../../data/star.mesh -sref 1 -pref 2 -rnum 1.9 -sc -prob 0
+// mpirun -np 4 pacoustics-primal -o 3 -m ../../data/inline-quad.mesh -sref 1 -pref 2  -rnum 5.2 -sc -prob 1
+// mpirun -np 4 pacoustics-primal -o 4 -m ../../data/inline-tri.mesh -sref 1 -pref 2  -rnum 7.1 -sc -prob 1
+// mpirun -np 4 pacoustics-primal -o 2 -m ../../data/inline-hex.mesh -sref 0 -pref 1 -rnum 1.9 -sc -prob 0
+
+// Description:
+// This example code demonstrates the use of MFEM to define and solve
+// the "ultraweak" (UW) DPG formulation for the Helmholtz problem
+
+//     - Δ p - ω² p = f ,   in Ω
+//                p = p₀, on ∂Ω
+
+// It solves the following kinds of problems
+//    a) f̃ = 0 and p₀ is a plane wave
+//    b) A manufactured solution problem where p_exact is a gaussian beam
+
+// The DPG Primal deals with the Second Order Equation
+//     - Δ p - ω² p = f , in Ω    (1)
+//                p = p₀, on ∂Ω
+
+// The primal-DPG formulation is obtained by integration by parts of (1)
+// and the introduction of a trace unknown on the mesh skeleton
+
+// p ∈ H¹(Ω),
+// p̂ ∈ H^-1/2(Ω)
+// (∇p,∇q) + ω²(p,q) + <p̂,q> = 0,      ∀ q ∈ H^1(Ω)
+//                         p = p₀      on ∂Ω
+
+#include "mfem.hpp"
+#include "util/pcomplexweakform.hpp"
+#include "util/pml.hpp"
+#include "../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+complex<double> acoustics_solution(const Vector & X);
+void acoustics_solution_grad(const Vector & X,vector<complex<double>> &dp);
+complex<double> acoustics_solution_laplacian(const Vector & X);
+
+double p_exact_r(const Vector &x);
+double p_exact_i(const Vector &x);
+double rhs_func_r(const Vector &x);
+double rhs_func_i(const Vector &x);
+void gradp_exact_r(const Vector &x, Vector &gradu);
+void gradp_exact_i(const Vector &x, Vector &gradu);
+double d2_exact_r(const Vector &x);
+double d2_exact_i(const Vector &x);
+
+int dim;
+double omega;
+
+enum prob_type
+{
+   plane_wave,
+   gaussian_beam
+};
+
+static const char *enum_str[] =
+{
+   "plane_wave",
+   "gaussian_beam"
+};
+
+prob_type prob;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   bool static_cond = false;
+   int iprob = 0;
+   int sr = 0;
+   int pr = 0;
+   bool paraview = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&rnum, "-rnum", "--number-of-wavelengths",
+                  "Number of wavelengths");
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: plane wave, 1: Gaussian beam");
+   args.AddOption(&delta_order, "-do", "--delta-order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&sr, "-sref", "--serial-ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&pr, "-pref", "--parallel-ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+
+   if (iprob > 1) { iprob = 0; }
+   prob = (prob_type)iprob;
+   omega = 2.*M_PI*rnum;
+
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+
+   for (int i = 0; i<sr; i++)
+   {
+      mesh.UniformRefinement();
+   }
+   dim = mesh.Dimension();
+   MFEM_VERIFY(dim > 1, "Dimension = 1 is not supported in this example");
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   enum TrialSpace
+   {
+      p_space     = 0,
+      hatp_space  = 1,
+   };
+   enum TestSpace
+   {
+      q_space = 0,
+   };
+
+   // H1 space for p
+   FiniteElementCollection *p_fec = new H1_FECollection(order,dim);
+   ParFiniteElementSpace *p_fes = new ParFiniteElementSpace(&pmesh,p_fec);
+
+   // H^-1/2 space for p̂
+   FiniteElementCollection * hatp_fec = new RT_Trace_FECollection(order-1,dim);
+   ParFiniteElementSpace *hatp_fes = new ParFiniteElementSpace(&pmesh,hatp_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+   trial_fes.Append(p_fes);
+   trial_fes.Append(hatp_fes);
+   test_fec.Append(q_fec);
+
+   // Bilinear form Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negomeg2(-omega*omega);
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_fes,test_fec);
+   a->StoreMatrices(); // needed for AMR
+
+   // Trial itegrators
+   // (∇ p,∇ q)
+   a->AddTrialIntegrator(new DiffusionIntegrator(one),nullptr,
+                         TrialSpace::p_space,TestSpace::q_space);
+   //  ω² (p,q)
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(negomeg2), nullptr,
+                         TrialSpace::p_space, TestSpace::q_space);
+
+   // < p̂,q >
+   a->AddTrialIntegrator(new TraceIntegrator,nullptr,
+                         TrialSpace::hatp_space,TestSpace::q_space);
+
+   // test integrators
+   // (∇q,∇δq)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),nullptr,
+                        TestSpace::q_space, TestSpace::q_space);
+   // (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),nullptr,
+                        TestSpace::q_space, TestSpace::q_space);
+
+   // RHS
+   FunctionCoefficient f_rhs_r(rhs_func_r);
+   FunctionCoefficient f_rhs_i(rhs_func_i);
+   if (prob == prob_type::gaussian_beam)
+   {
+      a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs_r),
+                               new DomainLFIntegrator(f_rhs_i),
+                               TestSpace::q_space);
+   }
+
+   socketstream p_out_r;
+   socketstream p_out_i;
+   if (myid == 0)
+   {
+      std::cout << "\n  Ref |"
+                << "    Dofs    |"
+                << "    ω    |" ;
+      std::cout  << "  H¹ Error  |"
+                 << "  Rate  |" ;
+      std::cout << "  Residual  |"
+                << "  Rate  |"
+                << " PCG it |" << endl;
+      std::cout << std::string(82,'-')
+                << endl;
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0 = 0;
+
+   ParGridFunction p_r, p_i;
+   FunctionCoefficient pex_r(p_exact_r);
+   FunctionCoefficient pex_i(p_exact_i);
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   if (paraview)
+   {
+      paraview_dc = new ParaViewDataCollection(enum_str[prob], &pmesh);
+      paraview_dc->SetPrefixPath("ParaViewPrimal/Acoustics");
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("p_r",&p_r);
+      paraview_dc->RegisterField("p_i",&p_i);
+   }
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+   for (int it = 0; it<=pr; it++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         p_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      Array<int> offsets(3);
+      offsets[0] = 0;
+      offsets[1] = p_fes->GetVSize();
+      offsets[2] = hatp_fes->GetVSize();
+      offsets.PartialSum();
+
+      Vector x(2*offsets.Last());
+      x = 0.;
+
+      ParGridFunction p_gf_r(p_fes, x, offsets[0]);
+      ParGridFunction p_gf_i(p_fes, x, offsets.Last()+ offsets[0]);
+      p_gf_r.ProjectBdrCoefficient(pex_r, ess_bdr);
+      p_gf_i.ProjectBdrCoefficient(pex_i, ess_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+      int num_blocks = BlockA_r->NumRowBlocks();
+      Array<int> tdof_offsets(2*num_blocks+1);
+
+      tdof_offsets[0] = 0;
+      for (int i=0; i<num_blocks; i++)
+      {
+         int h = BlockA_r->GetBlock(i,i).Height();
+         tdof_offsets[i+1] = h;
+         tdof_offsets[num_blocks+i+1] = h;
+      }
+      tdof_offsets.PartialSum();
+
+      BlockOperator blockA(tdof_offsets);
+      for (int i = 0; i<num_blocks; i++)
+      {
+         for (int j = 0; j<num_blocks; j++)
+         {
+            blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
+            blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
+         }
+      }
+
+      X = 0.;
+      BlockDiagonalPreconditioner M(tdof_offsets);
+      M.owns_blocks=0;
+
+      HypreBoomerAMG * solver_p = new HypreBoomerAMG((HypreParMatrix &)
+                                                     BlockA_r->GetBlock(0,0));
+      solver_p->SetPrintLevel(0);
+      M.SetDiagonalBlock(0,solver_p);
+      M.SetDiagonalBlock(num_blocks,solver_p);
+
+      HypreSolver * solver_hatp = nullptr;
+      if (dim == 2)
+      {
+         // AMS preconditioner for 2D H(div) (trace) space
+         solver_hatp = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(1,1),
+                                    hatp_fes);
+         dynamic_cast<HypreAMS*>(solver_hatp)->SetPrintLevel(0);
+      }
+      else
+      {
+         // ADS preconditioner for 3D H(div) (trace) space
+         solver_hatp = new HypreADS((HypreParMatrix &)BlockA_r->GetBlock(1,1),
+                                    hatp_fes);
+         dynamic_cast<HypreADS*>(solver_hatp)->SetPrintLevel(0);
+      }
+
+      M.SetDiagonalBlock(1,solver_hatp);
+      M.SetDiagonalBlock(num_blocks+1,solver_hatp);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-6);
+      cg.SetMaxIter(10000);
+      cg.SetPrintLevel(0);
+      cg.SetPreconditioner(M);
+      cg.SetOperator(blockA);
+      cg.Mult(B, X);
+
+      for (int i = 0; i<num_blocks; i++)
+      {
+         delete &M.GetDiagonalBlock(i);
+      }
+
+      int num_iter = cg.GetNumIterations();
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      double maxresidual = residuals.Max();
+      double globalresidual = residual * residual;
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+      p_r.MakeRef(p_fes, x, 0);
+      p_i.MakeRef(p_fes, x, offsets.Last());
+
+      int dofs = 0;
+      for (int i = 0; i<trial_fes.Size(); i++)
+      {
+         dofs += trial_fes[i]->GlobalTrueVSize();
+      }
+
+      double H1Error = 0.0;
+      double rate_err = 0.0;
+      VectorFunctionCoefficient pex_grad_r(dim,gradp_exact_r);
+      VectorFunctionCoefficient pex_grad_i(dim,gradp_exact_i);
+      double p_err_r = p_r.ComputeH1Error(&pex_r,&pex_grad_r);
+      double p_err_i = p_i.ComputeH1Error(&pex_i,&pex_grad_i);
+
+      H1Error = sqrt(p_err_r*p_err_r + p_err_i*p_err_i);
+
+      rate_err = (it) ? dim*log(err0/H1Error)/log((double)dof0/dofs) : 0.0;
+      err0 = H1Error;
+
+      double rate_res = (it) ? dim*log(res0/globalresidual)/log((
+                                                                   double)dof0/dofs) : 0.0;
+
+      res0 = globalresidual;
+      dof0 = dofs;
+
+      if (myid == 0)
+      {
+         std::ios oldState(nullptr);
+         oldState.copyfmt(std::cout);
+         std::cout << std::right << std::setw(5) << it << " | "
+                   << std::setw(10) <<  dof0 << " | "
+                   << std::setprecision(1) << std::fixed
+                   << std::setw(4) <<  2*rnum << " π  | ";
+         std::cout << std::setprecision(3) << std::setw(10)
+                   << std::scientific <<  err0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_err << " | " ;
+         std::cout << std::setprecision(3)
+                   << std::setw(10) << std::scientific <<  res0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_res << " | "
+                   << std::setw(6) << std::fixed << num_iter << " | "
+                   << std::endl;
+         std::cout.copyfmt(oldState);
+      }
+
+      if (visualization)
+      {
+         const char * keys = (it == 0 && dim == 2) ? "jRcml\n" : nullptr;
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         VisualizeField(p_out_r,vishost, visport, p_r,
+                        "Numerical presure (real part)", 0, 0, 500, 500, keys);
+         VisualizeField(p_out_i,vishost, visport, p_i,
+                        "Numerical presure (imaginary part)", 501, 0, 500, 500, keys);
+      }
+
+      if (paraview)
+      {
+         paraview_dc->SetCycle(it);
+         paraview_dc->SetTime((double)it);
+         paraview_dc->Save();
+      }
+
+      if (it == pr)
+      {
+         break;
+      }
+
+      pmesh.UniformRefinement();
+
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   if (paraview)
+   {
+      delete paraview_dc;
+   }
+
+   delete a;
+   delete q_fec;
+   delete hatp_fes;
+   delete hatp_fec;
+   delete p_fec;
+   delete p_fes;
+
+   return 0;
+}
+
+double p_exact_r(const Vector &x)
+{
+   return acoustics_solution(x).real();
+}
+
+double p_exact_i(const Vector &x)
+{
+   return acoustics_solution(x).imag();
+}
+
+void gradp_exact_r(const Vector &x, Vector &grad_r)
+{
+   grad_r.SetSize(x.Size());
+   vector<complex<double>> grad;
+   acoustics_solution_grad(x,grad);
+   for (unsigned i = 0; i < grad.size(); i++)
+   {
+      grad_r[i] = grad[i].real();
+   }
+}
+
+void gradp_exact_i(const Vector &x, Vector &grad_i)
+{
+   grad_i.SetSize(x.Size());
+   vector<complex<double>> grad;
+   acoustics_solution_grad(x,grad);
+   for (unsigned i = 0; i < grad.size(); i++)
+   {
+      grad_i[i] = grad[i].imag();
+   }
+}
+
+double d2_exact_r(const Vector &x)
+{
+   return acoustics_solution_laplacian(x).real();
+}
+
+double d2_exact_i(const Vector &x)
+{
+   return acoustics_solution_laplacian(x).imag();
+}
+
+
+// f = -Δ p - ω² p
+double rhs_func_r(const Vector &x)
+{
+   return -d2_exact_r(x) - omega * omega * p_exact_r(x);
+}
+
+// f = -Δ p - ω² p
+double rhs_func_i(const Vector &x)
+{
+   return -d2_exact_i(x) - omega * omega * p_exact_i(x);
+}
+
+complex<double> acoustics_solution(const Vector & X)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   switch (prob)
+   {
+      case plane_wave:
+      {
+         double beta = omega/std::sqrt((double)X.Size());
+         complex<double> alpha = beta * zi * X.Sum();
+         return exp(alpha);
+      }
+      break;
+      case gaussian_beam:
+      {
+         double rk = omega;
+         double degrees = 45;
+         double alpha = (180+degrees) * M_PI/180.;
+         double sina = sin(alpha);
+         double cosa = cos(alpha);
+         // shift the origin
+         double shift = 0.1;
+         double xprim=X(0) + shift;
+         double yprim=X(1) + shift;
+
+         double x = xprim*sina - yprim*cosa;
+         double y = xprim*cosa + yprim*sina;
+         //wavelength
+         double rl = 2.*M_PI/rk;
+
+         // beam waist radius
+         double w0 = 0.05;
+
+         // function w
+         double fact = rl/M_PI/(w0*w0);
+         double aux = 1. + (fact*y)*(fact*y);
+
+         double w = w0*sqrt(aux);
+
+         double phi0 = atan(fact*y);
+
+         double r = y + 1./y/(fact*fact);
+
+         // pressure
+         complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r +
+                              zi*phi0/2.;
+         double pf = pow(2.0/M_PI/(w*w),0.25);
+
+         return pf*exp(ze);
+      }
+      break;
+      default:
+         MFEM_ABORT("Should be unreachable");
+         return 1;
+         break;
+   }
+}
+
+void acoustics_solution_grad(const Vector & X, vector<complex<double>> & dp)
+{
+   dp.resize(X.Size());
+   complex<double> zi = complex<double>(0., 1.);
+   // initialize
+   for (int i = 0; i<X.Size(); i++) { dp[i] = 0.0; }
+   switch (prob)
+   {
+      case plane_wave:
+      {
+         double beta = omega/std::sqrt((double)X.Size());
+         complex<double> alpha = beta * zi * X.Sum();
+         complex<double> p = exp(alpha);
+         for (int i = 0; i<X.Size(); i++)
+         {
+            dp[i] = zi * beta * p;
+         }
+      }
+      break;
+      case gaussian_beam:
+      {
+         double rk = omega;
+         double degrees = 45;
+         double alpha = (180+degrees) * M_PI/180.;
+         double sina = sin(alpha);
+         double cosa = cos(alpha);
+         // shift the origin
+         double shift = 0.1;
+         double xprim=X(0) + shift;
+         double yprim=X(1) + shift;
+
+         double x = xprim*sina - yprim*cosa;
+         double y = xprim*cosa + yprim*sina;
+         double dxdxprim = sina, dxdyprim = -cosa;
+         double dydxprim = cosa, dydyprim =  sina;
+         //wavelength
+         double rl = 2.*M_PI/rk;
+
+         // beam waist radius
+         double w0 = 0.05;
+
+         // function w
+         double fact = rl/M_PI/(w0*w0);
+         double aux = 1. + (fact*y)*(fact*y);
+
+         double w = w0*sqrt(aux);
+         double dwdy = w0*fact*fact*y/sqrt(aux);
+
+         double phi0 = atan(fact*y);
+         double dphi0dy = cos(phi0)*cos(phi0)*fact;
+
+         double r = y + 1./y/(fact*fact);
+         double drdy = 1. - 1./(y*y)/(fact*fact);
+
+         // pressure
+         complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r +
+                              zi*phi0/2.;
+
+         complex<double> zdedx = -2.*x/(w*w) - 2.*zi*M_PI*x/rl/r;
+         complex<double> zdedy = 2.*x*x/(w*w*w)*dwdy - zi*rk + zi*M_PI*x*x/rl/
+                                 (r*r)*drdy + zi*dphi0dy/2.;
+
+         double pf = pow(2.0/M_PI/(w*w),0.25);
+         double dpfdy = -pow(2./M_PI/(w*w),-0.75)/M_PI/(w*w*w)*dwdy;
+
+         complex<double> zp = pf*exp(ze);
+         complex<double> zdpdx = zp*zdedx;
+         complex<double> zdpdy = dpfdy*exp(ze)+zp*zdedy;
+
+         dp[0] = (zdpdx*dxdxprim + zdpdy*dydxprim);
+         dp[1] = (zdpdx*dxdyprim + zdpdy*dydyprim);
+      }
+      break;
+      default:
+         MFEM_ABORT("Should be unreachable");
+         break;
+   }
+}
+
+complex<double> acoustics_solution_laplacian(const Vector & X)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   switch (prob)
+   {
+      case plane_wave:
+      {
+         double beta = omega/std::sqrt((double)X.Size());
+         complex<double> alpha = beta * zi * X.Sum();
+         return dim * beta * beta * exp(alpha);
+      }
+      break;
+      case gaussian_beam:
+      {
+         double rk = omega;
+         double degrees = 45;
+         double alpha = (180+degrees) * M_PI/180.;
+         double sina = sin(alpha);
+         double cosa = cos(alpha);
+         // shift the origin
+         double shift = 0.1;
+         double xprim=X(0) + shift;
+         double yprim=X(1) + shift;
+
+         double x = xprim*sina - yprim*cosa;
+         double y = xprim*cosa + yprim*sina;
+         double dxdxprim = sina, dxdyprim = -cosa;
+         double dydxprim = cosa, dydyprim =  sina;
+         //wavelength
+         double rl = 2.*M_PI/rk;
+
+         // beam waist radius
+         double w0 = 0.05;
+
+         // function w
+         double fact = rl/M_PI/(w0*w0);
+         double aux = 1. + (fact*y)*(fact*y);
+
+         double w = w0*sqrt(aux);
+         double dwdy = w0*fact*fact*y/sqrt(aux);
+         double d2wdydy = w0*fact*fact*(1. - (fact*y)*(fact*y)/aux)/sqrt(aux);
+
+         double phi0 = atan(fact*y);
+         double dphi0dy = cos(phi0)*cos(phi0)*fact;
+         double d2phi0dydy = -2.*cos(phi0)*sin(phi0)*fact*dphi0dy;
+
+         double r = y + 1./y/(fact*fact);
+         double drdy = 1. - 1./(y*y)/(fact*fact);
+         double d2rdydy = 2./(y*y*y)/(fact*fact);
+
+         // pressure
+         complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r +
+                              zi*phi0/2.;
+
+         complex<double> zdedx = -2.*x/(w*w) - 2.*zi*M_PI*x/rl/r;
+         complex<double> zdedy = 2.*x*x/(w*w*w)*dwdy - zi*rk + zi*M_PI*x*x/rl/
+                                 (r*r)*drdy + zi*dphi0dy/2.;
+         complex<double> zd2edxdx = -2./(w*w) - 2.*zi*M_PI/rl/r;
+         complex<double> zd2edxdy = 4.*x/(w*w*w)*dwdy + 2.*zi*M_PI*x/rl/(r*r)*drdy;
+         complex<double> zd2edydx = zd2edxdy;
+         complex<double> zd2edydy = -6.*x*x/(w*w*w*w)*dwdy*dwdy + 2.*x*x/
+                                    (w*w*w)*d2wdydy - 2.*zi*M_PI*x*x/rl/(r*r*r)*drdy*drdy
+                                    + zi*M_PI*x*x/rl/(r*r)*d2rdydy + zi/2.*d2phi0dydy;
+
+         double pf = pow(2.0/M_PI/(w*w),0.25);
+         double dpfdy = -pow(2./M_PI/(w*w),-0.75)/M_PI/(w*w*w)*dwdy;
+         double d2pfdydy = -1./M_PI*pow(2./M_PI,-0.75)*(-1.5*pow(w,-2.5)
+                                                        *dwdy*dwdy + pow(w,-1.5)*d2wdydy);
+
+
+         complex<double> zp = pf*exp(ze);
+         complex<double> zdpdx = zp*zdedx;
+         complex<double> zdpdy = dpfdy*exp(ze)+zp*zdedy;
+         complex<double> zd2pdxdx = zdpdx*zdedx + zp*zd2edxdx;
+         complex<double> zd2pdxdy = zdpdy*zdedx + zp*zd2edxdy;
+         complex<double> zd2pdydx = dpfdy*exp(ze)*zdedx + zdpdx*zdedy + zp*zd2edydx;
+         complex<double> zd2pdydy = d2pfdydy*exp(ze) + dpfdy*exp(
+                                       ze)*zdedy + zdpdy*zdedy + zp*zd2edydy;
+
+         return (zd2pdxdx*dxdxprim + zd2pdydx*dydxprim)*dxdxprim
+                + (zd2pdxdy*dxdxprim + zd2pdydy*dydxprim)*dydxprim
+                + (zd2pdxdx*dxdyprim + zd2pdydx*dydyprim)*dxdyprim
+                + (zd2pdxdy*dxdyprim + zd2pdydy*dydyprim)*dydyprim;
+      }
+      break;
+      default:
+         MFEM_ABORT("Should be unreachable");
+         return 1;
+         break;
+   }
+}

miniapps/dpg/pacoustics.cpp

@@ -121,6 +121,7 @@
 #include "mfem.hpp"
 #include "util/pcomplexweakform.hpp"
 #include "util/pml.hpp"
+#include "util/preconditioners.hpp"
 #include "../common/mfem-common.hpp"
 #include <fstream>
 #include <iostream>
@@ -192,6 +193,9 @@ int main(int argc, char *argv[])
    int iprob = 0;
    int sr = 0;
    int pr = 0;
+   bool pmg = false;
+   int pmg_levels = -1;
+   real_t relax_factor = 2.0/3;
    int visport = 19916;
    bool exact_known = false;
    bool with_pml = false;
@@ -216,6 +220,12 @@ int main(int argc, char *argv[])
                   "Number of parallel refinements.");
    args.AddOption(&pr, "-pref", "--parallel-ref",
                   "Number of parallel refinements.");
+   args.AddOption(&pmg, "-pmg", "--p-refinement-multigrid", "-no-pmg",
+                  "--no-p-refinement-multigrid", "Enable P-Refinement Multigrid.");
+   args.AddOption(&pmg_levels, "-pmgl","--p-refinement-multigrid-levels",
+                  "Number of levels for P-Refinement Multigrid.");
+   args.AddOption(&relax_factor, "-rf", "--relaxation-factor",
+                  "Relaxation factor for the p-multigrid smoother.");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -536,7 +546,7 @@ int main(int argc, char *argv[])
                 << "    ω    |" ;
       if (exact_known)
       {
-         std::cout  << "  L2 Error  |"
+         std::cout  << "  L² Error  |"
                     << "  Rate  |" ;
       }
       std::cout << "  Residual  |"
@@ -618,91 +628,69 @@ int main(int argc, char *argv[])
       a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
 
       ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
       BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
-      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
 
-      int num_blocks = BlockA_r->NumRowBlocks();
-      Array<int> tdof_offsets(2*num_blocks+1);
-
-      tdof_offsets[0] = 0;
-      int skip = (static_cond) ? 0 : 2;
-      int k = (static_cond) ? 2 : 0;
-      for (int i=0; i<num_blocks; i++)
+      Array<ParFiniteElementSpace *> prec_fes;
+      if (static_cond)
       {
-         tdof_offsets[i+1] = trial_fes[i+k]->GetTrueVSize();
-         tdof_offsets[num_blocks+i+1] = trial_fes[i+k]->GetTrueVSize();
-      }
-      tdof_offsets.PartialSum();
-
-      BlockOperator blockA(tdof_offsets);
-      for (int i = 0; i<num_blocks; i++)
-      {
-         for (int j = 0; j<num_blocks; j++)
-         {
-            blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
-            blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
-            blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
-            blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
-         }
-      }
-
-      X = 0.;
-      BlockDiagonalPreconditioner M(tdof_offsets);
-      M.owns_blocks=0;
-
-      if (!static_cond)
-      {
-         HypreBoomerAMG * solver_p = new HypreBoomerAMG((HypreParMatrix &)
-                                                        BlockA_r->GetBlock(0,0));
-         solver_p->SetPrintLevel(0);
-         solver_p->SetSystemsOptions(dim);
-         HypreBoomerAMG * solver_u = new HypreBoomerAMG((HypreParMatrix &)
-                                                        BlockA_r->GetBlock(1,1));
-         solver_u->SetPrintLevel(0);
-         solver_u->SetSystemsOptions(dim);
-         M.SetDiagonalBlock(0,solver_p);
-         M.SetDiagonalBlock(1,solver_u);
-         M.SetDiagonalBlock(num_blocks,solver_p);
-         M.SetDiagonalBlock(num_blocks+1,solver_u);
-      }
-
-      HypreBoomerAMG * solver_hatp = new HypreBoomerAMG((HypreParMatrix &)
-                                                        BlockA_r->GetBlock(skip,skip));
-      solver_hatp->SetPrintLevel(0);
-
-      HypreSolver * solver_hatu = nullptr;
-      if (dim == 2)
-      {
-         // AMS preconditioner for 2D H(div) (trace) space
-         solver_hatu = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(skip+1,skip+1),
-                                    hatu_fes);
-         dynamic_cast<HypreAMS*>(solver_hatu)->SetPrintLevel(0);
+         a->GetTraceFESpaces(prec_fes);
       }
       else
       {
-         // ADS preconditioner for 3D H(div) (trace) space
-         solver_hatu = new HypreADS((HypreParMatrix &)BlockA_r->GetBlock(skip+1,skip+1),
-                                    hatu_fes);
-         dynamic_cast<HypreADS*>(solver_hatu)->SetPrintLevel(0);
+         prec_fes = trial_fes;
+      }
+      Solver * cprec = nullptr;
+      if (pmg)
+      {
+#ifdef MFEM_USE_COMPLEX_MUMPS
+         bool mumps_coarse_solver = true;
+#else
+         bool mumps_coarse_solver = false;
+#endif
+         std::vector<Array<int>> ess_bdr_marker(prec_fes.Size());
+         for (int b = 0; b<prec_fes.Size(); b++)
+         {
+            if (pmesh.bdr_attributes.Size())
+            {
+               ess_bdr_marker[b].SetSize(pmesh.bdr_attributes.Max());
+               int ess_block = (static_cond) ? 0 : 2;
+               if (b == ess_block) // hatp
+               {
+                  ess_bdr_marker[b] = ess_bdr;
+               }
+               else
+               {
+                  ess_bdr_marker[b] = 0;
+               }
+            }
+         }
+         cprec = new ComplexPRefinementMultigrid(prec_fes, ess_bdr_marker, *Ahc,
+                                                 pmg_levels, relax_factor, mumps_coarse_solver );
+      }
+      else
+      {
+         BlockDiagonalPreconditioner * real_prec = new BlockDiagonalPreconditioner(
+            BlockA_r->RowOffsets());
+         real_prec->owns_blocks = 1;
+         for (int i = 0; i<BlockA_r->NumRowBlocks(); i++)
+         {
+            auto prec = MakeFESpaceDefaultSolver(prec_fes[i],0);
+            prec->SetOperator(BlockA_r->GetBlock(i,i));
+            real_prec->SetDiagonalBlock(i,prec);
+         }
+         cprec = new ComplexPreconditioner(real_prec, true);
       }
-
-      M.SetDiagonalBlock(skip,solver_hatp);
-      M.SetDiagonalBlock(skip+1,solver_hatu);
-      M.SetDiagonalBlock(skip+num_blocks,solver_hatp);
-      M.SetDiagonalBlock(skip+num_blocks+1,solver_hatu);
 
       CGSolver cg(MPI_COMM_WORLD);
       cg.SetRelTol(1e-6);
       cg.SetMaxIter(10000);
       cg.SetPrintLevel(0);
-      cg.SetPreconditioner(M);
-      cg.SetOperator(blockA);
+      cg.SetOperator(*Ahc);
+      cg.SetPreconditioner(*cprec);
       cg.Mult(B, X);
 
-      for (int i = 0; i<num_blocks; i++)
-      {
-         delete &M.GetDiagonalBlock(i);
-      }
+      delete cprec;
 
       int num_iter = cg.GetNumIterations();
 

miniapps/dpg/pconvection-diffusion.cpp

@@ -66,6 +66,7 @@
 
 #include "mfem.hpp"
 #include "util/pweakform.hpp"
+#include "util/preconditioners.hpp"
 #include "../common/mfem-common.hpp"
 #include <fstream>
 #include <iostream>
@@ -121,6 +122,9 @@ int main(int argc, char *argv[])
    real_t theta = 0.7;
    bool static_cond = false;
    epsilon = 1e0;
+   bool pmg = false;
+   int pmg_levels = -1;
+   real_t relax_factor = 2.0/3;
 
    bool visualization = true;
    int visport = 19916;
@@ -145,6 +149,12 @@ int main(int argc, char *argv[])
                   "Vector Coefficient beta");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&pmg, "-pmg", "--p-refinement-multigrid", "-no-pmg",
+                  "--no-p-refinement-multigrid", "Enable P-Refinement Multigrid.");
+   args.AddOption(&pmg_levels, "-pmgl","--p-refinement-multigrid-levels",
+                  "Number of levels for P-Refinement Multigrid.");
+   args.AddOption(&relax_factor, "-rf", "--relaxation-factor",
+                  "Relaxation factor for the p-multigrid smoother.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -453,44 +463,70 @@ int main(int argc, char *argv[])
 
       BlockOperator * A = Ah.As<BlockOperator>();
 
-      BlockDiagonalPreconditioner M(A->RowOffsets());
-      M.owns_blocks = 1;
-      int skip = 0;
-      if (!static_cond)
+      Solver * preconditioner = nullptr;
+      Array<ParFiniteElementSpace *> prec_fes;
+      if (static_cond)
       {
-         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
-         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
-         amg0->SetPrintLevel(0);
-         amg1->SetPrintLevel(0);
-         M.SetDiagonalBlock(0,amg0);
-         M.SetDiagonalBlock(1,amg1);
-         skip = 2;
-      }
-      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,
-                                                                               skip));
-      amg2->SetPrintLevel(0);
-      M.SetDiagonalBlock(skip,amg2);
-
-      HypreSolver * prec;
-      if (dim == 2)
-      {
-         // AMS preconditioner for 2D H(div) (trace) space
-         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatf_fes);
+         a->GetTraceFESpaces(prec_fes);
       }
       else
       {
-         // ADS preconditioner for 3D H(div) (trace) space
-         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatf_fes);
+         prec_fes = trial_fes;
+      }
+      if (pmg)
+      {
+#ifdef MFEM_USE_MUMPS
+         bool mumps_coarse_solver = true;
+#else
+         bool mumps_coarse_solver = false;
+#endif
+         std::vector<Array<int>> ess_bdr_marker(prec_fes.Size());
+         for (int b = 0; b<prec_fes.Size(); b++)
+         {
+            if (pmesh.bdr_attributes.Size())
+            {
+               ess_bdr_marker[b].SetSize(pmesh.bdr_attributes.Max());
+               int ess_block = (static_cond) ? 0 : 2;
+
+               if (b == ess_block) // hatu space has essential bdr conditions
+               {
+                  ess_bdr_marker[b] = ess_bdr_uhat;
+               }
+               else if (b == ess_block+1) // hatf space has essential bdr conditions
+               {
+                  ess_bdr_marker[b] = ess_bdr_fhat;
+               }
+               else
+               {
+                  ess_bdr_marker[b] = 0;
+               }
+            }
+         }
+         preconditioner = new PRefinementMultigrid(prec_fes, ess_bdr_marker, *A,
+                                                   pmg_levels, relax_factor, mumps_coarse_solver);
+      }
+      else
+      {
+         preconditioner = new BlockDiagonalPreconditioner(A->RowOffsets());
+         auto block_diag = dynamic_cast<BlockDiagonalPreconditioner*>(preconditioner);
+         block_diag->owns_blocks = 1;
+         for (int i = 0; i<A->NumRowBlocks(); i++)
+         {
+            auto prec = MakeFESpaceDefaultSolver(prec_fes[i],0);
+            prec->SetOperator(A->GetBlock(i,i));
+            block_diag->SetDiagonalBlock(i,prec);
+         }
       }
-      M.SetDiagonalBlock(skip+1,prec);
 
       CGSolver cg(MPI_COMM_WORLD);
       cg.SetRelTol(1e-12);
       cg.SetMaxIter(2000);
       cg.SetPrintLevel(0);
-      cg.SetPreconditioner(M);
       cg.SetOperator(*A);
+      cg.SetPreconditioner(*preconditioner);
       cg.Mult(B, X);
+      delete preconditioner;
+
       int num_iter = cg.GetNumIterations();
 
       a->RecoverFEMSolution(X,x);

miniapps/dpg/pdiffusion-primal.cpp

@@ -0,0 +1,329 @@
+//                 MFEM Primal DPG parallel example for diffusion
+//
+// Compile with: make pdiffusion-primal
+//
+// Sample runs
+// mpirun -np 4 pdiffusion-primal -m ../../data/inline-quad.mesh -o 3 -sref 1 -pref 2
+
+//       - Δ u = f,   in Ω
+//           u = u₀, on ∂Ω
+
+
+
+// --------------------------------------
+// |   |     u     |      σ̂    |  RHS   |
+// --------------------------------------
+// | v |  (∇u,∇v)  |  -(σ̂ₙ,v)  |  (f,v) |
+//
+// u ∈ H¹(Ω),   σ̂ₙ ∈ H^-1/2(Τ)
+
+
+#include "mfem.hpp"
+#include "util/pweakform.hpp"
+#include "../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+double exact_u(const Vector & X);
+void exact_gradu(const Vector & X, Vector &gradu);
+double exact_laplacian_u(const Vector & X);
+void exact_hatsigma(const Vector & X, Vector & hatsigma);
+double f_exact(const Vector & X);
+
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI and HYPRE.
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int sref = 0; // initial uniform mesh refinements
+   int pref = 0; // parallel mesh refinements for AMR
+   bool static_cond = false;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&sref, "-sref", "--num-serial-refinements",
+                  "Number of initial serial uniform refinements");
+   args.AddOption(&pref, "-pref", "--num-parallel-refinements",
+                  "Number of AMR refinements");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   for (int i = 0; i<sref; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+
+   // H1 space for u
+   FiniteElementCollection *u_fec = new H1_FECollection(order,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec);
+
+
+   // H^-1/2 space for σ̂
+   FiniteElementCollection * hatsigma_fec = new RT_Trace_FECollection(order-1,dim);
+   ParFiniteElementSpace *hatsigma_fes = new ParFiniteElementSpace(&pmesh,
+                                                                   hatsigma_fec);
+
+   int test_order = order+delta_order;
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(hatsigma_fes);
+   test_fec.Append(v_fec);
+
+
+   ConstantCoefficient one(1.0);
+   FunctionCoefficient f(f_exact); // rhs for the manufactured solution problem
+   FunctionCoefficient uex(exact_u);
+   VectorFunctionCoefficient graduex(dim,exact_gradu);
+
+   ParDPGWeakForm * a = new ParDPGWeakForm(trial_fes,test_fec);
+   a->StoreMatrices(true); // this is needed for estimation of residual
+
+   // (∇u,∇v)
+   a->AddTrialIntegrator(new DiffusionIntegrator(one),0,0);
+
+   // -<σ̂,v> (sign is included in σ̂)
+   a->AddTrialIntegrator(new TraceIntegrator,1,0);
+
+   // (∇v,∇δv)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+   // (v,δv)
+   a->AddTestIntegrator(new MassIntegrator(one),0,0);
+
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f),0);
+
+
+   if (myid == 0)
+   {
+      std::cout << "\n  Ref |"
+                << "    Dofs    |"
+                << "  H1 Error  |"
+                << "  Rate  |"
+                << "  Residual  |"
+                << "  Rate  |"
+                << " PCG it |" << endl;
+      std::cout << std::string(72,'-') << endl;
+   }
+
+
+   socketstream u_out;
+
+   double err0 = 0.;
+   int dof0=0.;
+   double res0=0.0;
+
+   ParGridFunction u_gf(u_fes);
+   u_gf = 0.0;
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+
+   for (int it = 0; it<=pref; it++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         u_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      Array<int> offsets(3);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = hatsigma_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      u_gf.MakeRef(u_fes,x.GetBlock(0),0);
+      u_gf.ProjectBdrCoefficient(uex,ess_bdr);
+
+      Vector X,B;
+      OperatorPtr Ah;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockOperator * A = Ah.As<BlockOperator>();
+
+      BlockDiagonalPreconditioner M(A->RowOffsets());
+      M.owns_blocks = 1;
+
+      HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+      amg0->SetPrintLevel(0);
+      M.SetDiagonalBlock(0,amg0);
+      HypreSolver * prec;
+      if (dim == 2)
+      {
+         // AMS preconditioner for 2D H(div) (trace) space
+         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(1,1), hatsigma_fes);
+      }
+      else
+      {
+         // ADS preconditioner for 3D H(div) (trace) space
+         prec = new HypreADS((HypreParMatrix &)A->GetBlock(1,1), hatsigma_fes);
+      }
+      M.SetDiagonalBlock(1,prec);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(0);
+      cg.SetPreconditioner(M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+
+      double maxresidual = residuals.Max();
+      double globalresidual = residual * residual;
+
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+      u_gf.MakeRef(u_fes,x.GetBlock(0),0);
+
+      int dofs = u_fes->GlobalTrueVSize() + hatsigma_fes->GlobalTrueVSize();
+
+      double u_err = u_gf.ComputeH1Error(&uex,&graduex);
+      double rate_err = (it) ? dim*log(err0/u_err)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (it) ? dim*log(res0/globalresidual)/log((
+                                                                   double)dof0/dofs) : 0.0;
+      err0 = u_err;
+      res0 = globalresidual;
+      dof0 = dofs;
+
+      if (myid == 0)
+      {
+         std::ios oldState(nullptr);
+         oldState.copyfmt(std::cout);
+         std::cout << std::right << std::setw(5) << it << " | "
+                   << std::setw(10) <<  dof0 << " | "
+                   << std::setprecision(3)
+                   << std::setw(10) << std::scientific <<  err0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_err << " | "
+                   << std::setprecision(3)
+                   << std::setw(10) << std::scientific <<  res0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_res << " | "
+                   << std::setw(6) << std::fixed << cg.GetNumIterations() << " | "
+                   << std::endl;
+         std::cout.copyfmt(oldState);
+      }
+
+      if (visualization)
+      {
+         const char * keys = (it == 0 && dim == 2) ? "jRcm\n" : nullptr;
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+
+         VisualizeField(u_out,vishost,visport,u_gf,
+                        "Numerical u", 0,0,500,500,keys);
+      }
+
+      if (it == pref) { break; }
+
+      pmesh.UniformRefinement();
+
+      for (int i=0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete v_fec;
+   delete hatsigma_fes;
+   delete hatsigma_fec;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+
+
+
+}
+
+
+
+double exact_u(const Vector & X)
+{
+   double alpha = M_PI * (X.Sum());
+   return sin(alpha);
+}
+
+void exact_gradu(const Vector & X, Vector & du)
+{
+   du.SetSize(X.Size());
+   double alpha = M_PI * (X.Sum());
+   du.SetSize(X.Size());
+   for (int i = 0; i<du.Size(); i++)
+   {
+      du[i] = M_PI * cos(alpha);
+   }
+}
+
+double exact_laplacian_u(const Vector & X)
+{
+   double alpha = M_PI * (X.Sum());
+   double u = sin(alpha);
+   return - M_PI*M_PI * u * X.Size();
+}
+
+void exact_hatsigma(const Vector & X, Vector & hatsigma)
+{
+   exact_gradu(X,hatsigma);
+   hatsigma *= -1.;
+}
+
+double f_exact(const Vector & X)
+{
+   return -exact_laplacian_u(X);
+}

miniapps/dpg/pdiffusion.cpp

@@ -70,6 +70,7 @@
 
 #include "mfem.hpp"
 #include "util/pweakform.hpp"
+#include "util/preconditioners.hpp"
 #include "../common/mfem-common.hpp"
 #include <fstream>
 #include <iostream>
@@ -114,6 +115,9 @@ int main(int argc, char *argv[])
    int sref = 0; // initial uniform mesh refinements
    int pref = 0; // parallel mesh refinements for AMR
    int iprob = 0;
+   bool pmg = false;
+   int pmg_levels = -1;
+   real_t relax_factor = 2.0/3;
    bool static_cond = false;
    real_t theta = 0.7;
    bool visualization = true;
@@ -137,6 +141,12 @@ int main(int argc, char *argv[])
                   " 0: manufactured, 1: L-shape");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&pmg, "-pmg", "--p-refinement-multigrid", "-no-pmg",
+                  "--no-p-refinement-multigrid", "Enable P-Refinement Multigrid.");
+   args.AddOption(&pmg_levels, "-pmgl","--p-refinement-multigrid-levels",
+                  "Number of levels for P-Refinement Multigrid.");
+   args.AddOption(&relax_factor, "-rf", "--relaxation-factor",
+                  "Relaxation factor for the p-multigrid smoother.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -186,7 +196,6 @@ int main(int argc, char *argv[])
       }
    }
 
-
    for (int i = 0; i<sref; i++)
    {
       mesh.UniformRefinement();
@@ -378,44 +387,63 @@ int main(int argc, char *argv[])
 
       BlockOperator * A = Ah.As<BlockOperator>();
 
-      BlockDiagonalPreconditioner M(A->RowOffsets());
-      M.owns_blocks = 1;
-      int skip = 0;
-      if (!static_cond)
+      Solver * preconditioner = nullptr;
+      Array<ParFiniteElementSpace *> prec_fes;
+      if (static_cond)
       {
-         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
-         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
-         amg0->SetPrintLevel(0);
-         amg1->SetPrintLevel(0);
-         M.SetDiagonalBlock(0,amg0);
-         M.SetDiagonalBlock(1,amg1);
-         skip=2;
-      }
-      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,
-                                                                               skip));
-      amg2->SetPrintLevel(0);
-      M.SetDiagonalBlock(skip,amg2);
-      HypreSolver * prec;
-      if (dim == 2)
-      {
-         // AMS preconditioner for 2D H(div) (trace) space
-         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatsigma_fes);
+         a->GetTraceFESpaces(prec_fes);
       }
       else
       {
-         // ADS preconditioner for 3D H(div) (trace) space
-         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatsigma_fes);
+         prec_fes = trial_fes;
+      }
+      if (pmg)
+      {
+#ifdef MFEM_USE_MUMPS
+         bool mumps_coarse_solver = true;
+#else
+         bool mumps_coarse_solver = false;
+#endif
+         std::vector<Array<int>> ess_bdr_marker(prec_fes.Size());
+         for (int b = 0; b<prec_fes.Size(); b++)
+         {
+            ess_bdr_marker[b].SetSize(pmesh.bdr_attributes.Max());
+            int ess_block = (static_cond) ? 0 : 2;
+            if (b == ess_block)
+            {
+               ess_bdr_marker[b] = ess_bdr;
+            }
+            else
+            {
+               ess_bdr_marker[b] = 0;
+            }
+         }
+         preconditioner = new PRefinementMultigrid(prec_fes, ess_bdr_marker, *A,
+                                                   pmg_levels, relax_factor,mumps_coarse_solver);
+      }
+      else
+      {
+         preconditioner = new BlockDiagonalPreconditioner(A->RowOffsets());
+         auto block_diag = dynamic_cast<BlockDiagonalPreconditioner*>(preconditioner);
+         block_diag->owns_blocks = 1;
+         for (int i = 0; i<A->NumRowBlocks(); i++)
+         {
+            auto prec = MakeFESpaceDefaultSolver(prec_fes[i],0);
+            prec->SetOperator(A->GetBlock(i,i));
+            block_diag->SetDiagonalBlock(i,prec);
+         }
       }
-      M.SetDiagonalBlock(skip+1,prec);
 
       CGSolver cg(MPI_COMM_WORLD);
       cg.SetRelTol(1e-12);
       cg.SetMaxIter(2000);
       cg.SetPrintLevel(0);
-      cg.SetPreconditioner(M);
       cg.SetOperator(*A);
+      cg.SetPreconditioner(*preconditioner);
       cg.Mult(B, X);
 
+      delete preconditioner;
+
       a->RecoverFEMSolution(X,x);
 
       Vector & residuals = a->ComputeResidual(x);

miniapps/dpg/plasma/lh-eld-dpg-fem.cpp

@@ -0,0 +1,976 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//                   MFEM Ultraweak DPG Maxwell parallel example
+//
+// Compile with: make lh-eld-dpg
+//
+// mpirun -np 8 ./lh-eld-dpg -o 3 -paraview -pr 0
+// mpirun -np 8 ./lh-eld-dpg -o 3 -paraview -pr 1 -sc
+
+// Electron Landau Damping
+// Strong formulation:
+//     ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵ E + i ω²ϵ₀(J₁ + J₂) = 0,   in Ω
+//                   - Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥ = 0,   in Ω     
+//                   - Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥ = 0,   in Ω 
+//                                            E×n = E₀,  on ∂Ω
+//                                            J₁  = 0,  on ∂Ω
+//                                            J₂  = 0,  on ∂Ω
+// The DPG UW deals with the First Order System
+//   i ω μ₀ H + ∇ × E                  = 0,   in Ω
+// -i ω ϵ₀ϵ E + ∇ × H - ω ϵ₀ (J₁ + J₂) = 0,   in Ω
+//         - Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥ = 0,   in Ω     
+//         - Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥ = 0,   in Ω 
+//                                 E×n = E₀,  on ∂Ω
+//                                 J₁  = 0,  on ∂Ω
+//                                 J₂  = 0,  on ∂Ω
+
+
+// in 2D
+// E is vector valued and H is scalar.
+//      (∇ × E, δE) = (E, ∇ × δE ) + < n × E , δE >
+// or (∇ ⋅ AE , δE) = (AE, ∇ δE ) + < AE ⋅ n, δE >
+// where A = [0 1; -1 0];
+
+// E ∈ (L²(Ω))² , H ∈ L²(Ω), J ∈ (H¹(Ω))²
+// Ê ∈ H^-1/2(Γₕ), Ĥ ∈ H^1/2(Γₕ)
+//     iωμ₀ (H,δE) + (E,∇×δE) + < AÊ, δE > = 0,      ∀ δE ∈ H¹(Ω)
+//  -i ωϵ₀ϵ (E,δH) + (H,∇×δH) + < Ĥ, δH×n > - ωϵ₀(J₁ + J₂,δH) = 0,  ∀ δH ∈ H(curl,Ω)
+// ( (b⋅∇)J₁,(b⋅∇) δJ₁ ) + c₁ (J₁,δJ₁) - c₁ (P(r) b⊗b E, δJ₁) = 0,  ∀ δJ₁ ∈ (H¹(Ω))²
+// ( (b⋅∇)J₂,(b⋅∇) δJ₂ ) + c₂ (J₂,δJ₂) + c₂ (P(r) b⊗b E, δJ₂) = 0,  ∀ δJ₁ ∈ (H¹(Ω))²
+//                                                                  Ê = E₀, on ∂Ω
+//                                                            J₁ = J₂ = 0,  on ∂Ω
+// ----------------------------------------------------------------------------------------------
+// |   |      E       |     H    |        J₁        |        J₂        |   Ê   |   Ĥ    |  RHS  |
+// ----------------------------------------------------------------------------------------------
+// |δE |  (E,∇ × δE)  |iωμ₀(H,δE)|                  |                  | <Ê,δE>|        |   0   |  
+// |   |              |          |                  |                  |       |        |       |  
+// |δH | -iωϵ₀ϵ(E,δH) | (H,∇×δH) |   -ωϵ₀ (J₁,δH)   |  -ωϵ₀ (J₂,δH)    |       |<Ĥ,δH×n>|   0   |  
+// |   |              |          |                  |                  |       |        |       |  
+// |δJ₁|-c₁(P(r)E,δJ₁)|          |((b⋅∇)J₁,(b⋅∇)δJ₁)|                  |       |        |   0   |  
+// |   |              |          |     + c₁ (J₁,δJ₁)|                  |       |        |       |    
+// |δJ₂| c₂(P(r)E,δJ₂)|          |                  |((b⋅∇)J₂,(b⋅∇)δJ₂)|       |        |   0   |  
+// |   |              |          |                  |     + c₂ (J₂,δJ₂)|       |        |       |    
+// where (δE,δH,δJ₁,δJ₂) ∈  H¹(Ω) × H(curl,Ω) × (H¹(Ω))² × (H¹(Ω))² 
+
+
+
+#include "mfem.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../util/pcomplexblockform.hpp"
+#include "../../common/mfem-common.hpp"
+#include "../util/blockcomplexhypremat.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "utils/lh_utils.hpp"
+#include "../util/utils.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+   int order = 1;
+   int delta_order = 1;
+   int par_ref_levels = 0;
+   int ser_ref_levels = 0;
+
+   real_t rnum=1.5;
+   real_t mu = 1.257;
+   real_t eps0 = 8.8541878128;
+   real_t cfactor = 1e-6;
+   real_t balance_scale = 1.0;
+   bool enable_balance_scale = false;
+
+   bool visualization = false;
+   bool paraview = false;
+   bool debug = false;
+   bool mumps_solver = false;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&ser_ref_levels, "-sr", "--serial-refinement_levels",
+                  "Number of serial refinement levels.");                  
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&a0, "-a0", "--a0", "P(r) first parameter.");
+   args.AddOption(&a1, "-a1", "--a1", "P(r) second parameter.");
+   args.AddOption(&delta, "-delta", "--delta", "stability parameter.");
+   args.AddOption(&mumps_solver, "-mumps", "--mumps", "-no-mumps",
+                  "--no-mumps",
+                  "Enable or disable MUMPS solver.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&debug, "-debug", "--debug", "-no-debug",
+                  "--no-debug",
+                  "Enable or disable debug mode (delta = 0.01 and no coupling).");                  
+   args.AddOption(&enable_balance_scale, "-ebs", "--enable-balance-scale", "-no-ebs",
+                  "--no-ebs",
+                  "Enable or disable balance scale.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (Mpi::Root())
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (Mpi::Root())
+   {
+      args.PrintOptions(cout);
+   }
+
+   // number of diffusion equations
+   int ndiffusionequations = 2; 
+
+   Vector cvals(ndiffusionequations);
+   Vector csigns(ndiffusionequations);
+   cvals(0)  = 25e6;  cvals(1)  = 1e6;
+   csigns(0) = -1.0;  csigns(1) = 1.0;
+   cvals *= cfactor; // scale the coefficients
+
+   real_t omega = 2.*M_PI*rnum;
+   int test_order = order+delta_order;
+
+   balance_scale = (enable_balance_scale) ? omega * eps0  : 1.0;
+
+   if (!debug) 
+   {
+      delta = 0.0; // disable delta if electron Landau damping is enabled
+      if (Mpi::Root())
+      {
+         cout << "Electron Landau damping enabled, delta set to 0.0." << endl;
+      }
+   }    
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   MFEM_VERIFY(dim == 2, "Dimension != 2 is not supported in this example");
+
+   for (int i = 0; i < ser_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // mesh.RemoveInternalBoundaries();
+
+   Array<int> int_bdr_attr;
+   for (int i = 0; i < mesh.GetNBE(); i++)
+   {
+      if (mesh.FaceIsInterior(mesh.GetBdrElementFaceIndex(i)))
+      {
+         int_bdr_attr.Append(mesh.GetBdrAttribute(i));
+      }
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   for (int i = 0; i < par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }  
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+
+   // Define coefficients
+   ConstantCoefficient muinv(1./mu);
+   ConstantCoefficient one_cf(1.0);
+   //  ωμ₀
+   ConstantCoefficient omegamu_cf(omega*mu);
+   // -ω μ₀  
+   ConstantCoefficient negomegamu_cf(-omega*mu);
+   // -ωϵ₀
+   real_t scale = (debug) ? 0.0 : 1.0;
+   ConstantCoefficient negomegeps0_cf(-omega*eps0 * scale);
+   ConstantCoefficient balancescaled_negomegeps0_cf( -omega*eps0/balance_scale * scale);
+
+   // μ₀² ω²
+   ConstantCoefficient mu2omeg2_cf((mu*mu*omega*omega));
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_vcf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   // S(r) 
+   FunctionCoefficient S_cf_r(sfunc_r), S_cf_i(sfunc_i);
+   // P(r) 
+   FunctionCoefficient P_cf_r(pfunc_r), P_cf_i(pfunc_i); 
+
+   VectorFunctionCoefficient b_cf(dim,bfunc);// b
+   ScalarVectorProductCoefficient scaled_b_cf(sqrt(cfactor), b_cf);
+
+   MatrixFunctionCoefficient bb_cf(dim,bcrossb); // b⊗b
+   MatrixSumCoefficient oneminusbb(Mone_cf, bb_cf, 1.0, -1.0); // 1 - b⊗b
+
+   // S(r) (I - b⊗b)
+   ScalarMatrixProductCoefficient Soneminusbb_r(S_cf_r, oneminusbb), Soneminusbb_i(S_cf_i, oneminusbb); 
+
+   // P(r) b⊗b 
+   ScalarMatrixProductCoefficient P_cf_bb_r(P_cf_r, bb_cf), P_cf_bb_i(P_cf_i, bb_cf); 
+
+   // ε = S(r) (I - b⊗b) + P(r) b⊗b 
+   MatrixSumCoefficient eps_r(Soneminusbb_r, P_cf_bb_r, 1.0, 1.0); 
+   MatrixSumCoefficient eps_i(Soneminusbb_i, P_cf_bb_i, 1.0, 1.0); 
+
+   coefs_r[nattr-1] = &eps_r;
+   coefs_i[nattr-1] = &eps_i;
+
+   // for (int i = 0; i < nattr-1; ++i)
+   // {
+   //    coefs_r[i] = &eps_r;
+   //    coefs_i[i] = &eps_i;
+   // }
+
+   PWMatrixCoefficient eps_cf_r(dim, attr, coefs_r);
+   PWMatrixCoefficient eps_cf_i(dim, attr, coefs_i);
+
+   ConstantCoefficient eps0omeg(omega * eps0);
+   ConstantCoefficient negeps0omeg(-omega * eps0);
+
+   // ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient eps0omeg_eps_r(eps0omeg, eps_cf_r);
+   // ω ϵ₀ ϵᵢ
+   ScalarMatrixProductCoefficient eps0omeg_eps_i(eps0omeg, eps_cf_i);
+   // -ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient negeps0omeg_eps_r(negeps0omeg, eps_cf_r);
+   // -ω ϵ₀ ϵᵢ
+   ScalarMatrixProductCoefficient negeps0omeg_eps_i(eps0omeg, eps_cf_i);
+
+   // A = [0 1; -1 0]
+   DenseMatrix rot_mat(2);
+   rot_mat(0,0) = 0.; rot_mat(0,1) = 1.;
+   rot_mat(1,0) = -1.; rot_mat(1,1) = 0.;
+   MatrixConstantCoefficient rot(rot_mat);
+
+   // ω ϵ₀ ϵᵣ A
+   MatrixProductCoefficient eps0omeg_eps_r_rot(eps0omeg_eps_r, rot);
+   // ω ϵ₀ ϵᵢ A
+   MatrixProductCoefficient eps0omeg_eps_i_rot(eps0omeg_eps_i, rot);
+   // -ω ϵ₀ ϵᵣ A
+   MatrixProductCoefficient negeps0omeg_eps_r_rot(negeps0omeg_eps_r, rot);
+   // -ω ϵ₀ ϵᵢ A
+   MatrixProductCoefficient negeps0omeg_eps_i_rot(negeps0omeg_eps_i, rot);
+
+   // (ωϵ₀ϵ)(ωϵ₀ϵ)^*  (δH, δH)
+   TransposeMatrixCoefficient eps0omeg_eps_r_t(eps0omeg_eps_r);
+   TransposeMatrixCoefficient eps0omeg_eps_i_t(eps0omeg_eps_i);
+   MatrixProductCoefficient MrMrt_cf(eps0omeg_eps_r, eps0omeg_eps_r_t);
+   MatrixProductCoefficient MiMit_cf(eps0omeg_eps_i, eps0omeg_eps_i_t);
+   MatrixProductCoefficient MiMrt_cf(eps0omeg_eps_i, eps0omeg_eps_r_t);
+   MatrixProductCoefficient MrMit_cf(eps0omeg_eps_r, eps0omeg_eps_i_t);
+
+   // (MᵣMᵣᵗ + MᵢMᵢᵗ) + i (MᵢMᵣᵗ - MᵣMᵢᵗ)
+   MatrixSumCoefficient Mreal_cf(MrMrt_cf,MiMit_cf);
+   MatrixSumCoefficient Mimag_cf(MiMrt_cf,MrMit_cf,1.0,-1.0);
+
+   // if ELD
+   Array<Vector *> c_arrays(ndiffusionequations);
+   Array<PWConstCoefficient *> pw_c_coeffs(ndiffusionequations);
+   Array<MatrixCoefficient *> cPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> cPibb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPibb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> balancescaled_signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> balancescaled_signedcPibb_cf(ndiffusionequations);
+   
+   Vector temp(nattr); temp=0.0;
+   Array<ConstantCoefficient *> c_coeffs(ndiffusionequations);
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      temp[nattr-1] = cvals(i);
+      // temp = cvals(i);
+      pw_c_coeffs[i] = new PWConstCoefficient(temp);
+      c_coeffs[i] = new ConstantCoefficient(cvals(i));
+      cPrbb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_r);
+      cPibb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_i);
+      signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPrbb_cf[i]);
+      signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPibb_cf[i]);
+      balancescaled_signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(balance_scale,*signedcPrbb_cf[i]);
+      balancescaled_signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(balance_scale,*signedcPibb_cf[i]);
+   }   
+
+   // ----------------------------------------------
+   // DPG UW formulation for Maxwell equations
+   // ----------------------------------------------
+   // Define DPG spaces for the Maxwell equations
+   // trial spaces for E, H, Ê, Ĥ
+   Array<FiniteElementCollection *> dpg_trial_fecols;
+   Array<FiniteElementCollection *> dpg_test_fecols;
+   Array<ParFiniteElementSpace *> dpg_pfes;
+
+   // Vector L2 space for E
+   dpg_trial_fecols.Append(new L2_FECollection(order-1, dim));
+   dpg_pfes.Append(new ParFiniteElementSpace(&pmesh, dpg_trial_fecols.Last(), dim));
+   // Scalar L2 space for H
+   dpg_trial_fecols.Append(new L2_FECollection(order-1, dim));
+   dpg_pfes.Append(new ParFiniteElementSpace(&pmesh, dpg_trial_fecols.Last()));
+
+   // Trial trace space for Ê 
+   dpg_trial_fecols.Append(new RT_Trace_FECollection(order-1, dim));
+   dpg_pfes.Append(new ParFiniteElementSpace(&pmesh, dpg_trial_fecols.Last()));
+   // Trial trace space for Ĥ 
+   dpg_trial_fecols.Append(new H1_Trace_FECollection(order, dim));
+   dpg_pfes.Append(new ParFiniteElementSpace(&pmesh, dpg_trial_fecols.Last()));
+
+   Array<HYPRE_BigInt> dpg_tdofs(dpg_pfes.Size());
+   for (int i = 0; i < dpg_pfes.Size(); ++i)
+   {
+      dpg_tdofs[i] = dpg_pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << "DPG ParFiniteElementSpace " << i << " has " << dpg_tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   if (Mpi::Root())
+   {
+      cout << "Total number of DPG true dofs: " << dpg_tdofs.Sum() << endl;
+   }
+
+   // test spaces for E and H
+   dpg_test_fecols.Append(new H1_FECollection(test_order, dim));
+   dpg_test_fecols.Append(new ND_FECollection(test_order, dim));
+
+   ParComplexDPGWeakForm * a_dpg = new ParComplexDPGWeakForm(dpg_pfes,dpg_test_fecols);
+
+   // (E,∇ × δE)
+   a_dpg->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one_cf)),
+                         nullptr,0, 0);
+   //  -i ω ϵ₀ (ϵE,δH) = - i ω ϵ₀(ϵᵣ + i ϵᵢ E, δH)
+   //                  =  (ω ϵ₀ ϵᵢ E, δH) + i (-ω ϵ₀ϵᵣ E, δH)
+   a_dpg->AddTrialIntegrator(
+      new TransposeIntegrator(new VectorFEMassIntegrator(eps0omeg_eps_i)), 
+      new TransposeIntegrator(new VectorFEMassIntegrator(negeps0omeg_eps_r)),
+      0, 1);
+   // iωμ₀(H,δE)
+   a_dpg->AddTrialIntegrator(nullptr, new MixedScalarMassIntegrator(omegamu_cf), 1, 0);
+   // a_dpg->AddTrialIntegrator(nullptr, new MassIntegrator(omegamu_cf), 1, 0);
+   // (H,∇ × δH)                         
+   a_dpg->AddTrialIntegrator(
+      new TransposeIntegrator(new MixedCurlIntegrator(one_cf)), nullptr, 1, 1);
+
+   // Trace integrators               
+   //  <Ê,δE>
+   a_dpg->AddTrialIntegrator(new TraceIntegrator,nullptr, 2, 0);
+   // <Ĥ,δH × n>
+   a_dpg->AddTrialIntegrator(new TangentTraceIntegrator,nullptr, 3, 1);
+
+   // test integrators
+   // (∇δE,∇δE)
+   a_dpg->AddTestIntegrator(new DiffusionIntegrator(one_cf),nullptr, 0, 0);
+   // (δE,δE)
+   a_dpg->AddTestIntegrator(new MassIntegrator(one_cf),nullptr, 0, 0);
+   // μ₀² ω² (δE,δE)
+   a_dpg->AddTestIntegrator(new MassIntegrator(mu2omeg2_cf),nullptr,0, 0);
+   // -i ω μ₀ (δE,∇ × δH) = i (δE, -ω μ₀ ∇ × δ H)
+   a_dpg->AddTestIntegrator(nullptr,
+         new TransposeIntegrator(new MixedCurlIntegrator(negomegamu_cf)),0, 1);
+   // -i ω ϵ₀ϵ(∇ × δE, δH) = -i (ωϵ₀(ϵᵣ+iϵᵢ) A ∇ δE,δE), A = [0 1; -1 0]
+   //                       =  (ω ϵ₀ ϵᵢ A ∇ δE,δE) + i (-ω ϵ₀ ϵᵣ A ∇ δE,δE)
+   a_dpg->AddTestIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_i_rot),
+                        new MixedVectorGradientIntegrator(negeps0omeg_eps_r_rot),0, 1);
+   // i ω μ₀ (∇ × δH ,δE) = i (ω μ₀ ∇ × δH, δE )
+   a_dpg->AddTestIntegrator(nullptr,new MixedCurlIntegrator(omegamu_cf),
+                        1, 0);
+   // i ω ϵ₀ϵ̄ (δH, ∇ × δE ) = i (ω ϵ₀(ϵᵣ -i ϵᵢ) δH, A ∇ δE) 
+   //                        = ( δH, ω ϵ₀ ϵᵢ A ∇ δE) + i (δH, ω ϵ₀ ϵᵣ A ∇ δE)
+   a_dpg->AddTestIntegrator(
+      new TransposeIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_i_rot)),
+      new TransposeIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_r_rot)),1, 0);
+   // (ωϵ₀ϵ)(ωϵ₀ϵ)^*  (δH, δH)
+   // (MᵣMᵣᵗ + MᵢMᵢᵗ) + i (MᵢMᵣᵗ - MᵣMᵢᵗ)
+   a_dpg->AddTestIntegrator(new VectorFEMassIntegrator(Mreal_cf),
+                        new VectorFEMassIntegrator(Mimag_cf),1, 1);
+   // (∇×δH ,∇×δH)
+   a_dpg->AddTestIntegrator(new CurlCurlIntegrator(one_cf),nullptr,1,1);
+   // (δH,δH)
+   a_dpg->AddTestIntegrator(new VectorFEMassIntegrator(one_cf),nullptr,1,1);
+
+   a_dpg->Assemble();
+
+   // ----------------------------------------------
+   // Define FEM spaces for the diffusion equations
+   // ----------------------------------------------
+
+   Array<FiniteElementCollection *> fem_fecols;
+   Array<ParFiniteElementSpace *> fem_pfes;
+   for (int i = 0; i < ndiffusionequations; ++i)
+   {
+      fem_fecols.Append(new H1_FECollection(order, dim));
+      fem_pfes.Append(new ParFiniteElementSpace(&pmesh, fem_fecols.Last(), dim));
+   }
+
+   Array<HYPRE_BigInt> fem_tdofs(fem_pfes.Size());
+   for (int i = 0; i < fem_pfes.Size(); ++i)
+   {
+      fem_tdofs[i] = fem_pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << "FEM ParFiniteElementSpace " << i << " has " << fem_tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+
+   if (Mpi::Root())
+   {
+      cout << "Total number of FEM true dofs: " << fem_tdofs.Sum() << endl;
+   }
+   
+   ParComplexBlockForm *a_fem = new ParComplexBlockForm(fem_pfes);
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      // ( (b⋅∇)Jᵢ  , (b⋅∇) Gᵢ)
+      a_fem->AddDomainIntegrator(new DirectionalVectorDiffusionIntegrator(scaled_b_cf), nullptr, i, i);
+      // cᵢ (Jᵢ , Gᵢ)
+      a_fem->AddDomainIntegrator(new VectorMassIntegrator(*pw_c_coeffs[i]), nullptr, i, i);
+   }
+   a_fem->Assemble();
+
+   // ----------------------------------------------
+   // Cross term DPG-FEM coupling
+   // ----------------------------------------------
+   //  -ωϵ₀ (J₁,δH)
+   //  -ωϵ₀ (J₂,δH)
+   ParMixedBilinearForm * a_J1H = new ParMixedBilinearForm(fem_pfes[0], dpg_pfes[1]);
+   ParMixedBilinearForm * a_J2H = new ParMixedBilinearForm(fem_pfes[1], dpg_pfes[1]);
+
+   ParMixedBilinearForm * a_J1H_i = new ParMixedBilinearForm(fem_pfes[0], dpg_pfes[1]);
+   ParMixedBilinearForm * a_J2H_i = new ParMixedBilinearForm(fem_pfes[1], dpg_pfes[1]);
+   ConstantCoefficient dummy_cf(0.0);
+   a_J1H_i->AddDomainIntegrator(new VectorMassIntegrator(dummy_cf));
+   a_J2H_i->AddDomainIntegrator(new VectorMassIntegrator(dummy_cf));
+   a_J1H_i->Assemble(0);
+   a_J2H_i->Assemble(0);
+
+   // a_J1H->AddDomainIntegrator(new VectorMassIntegrator(negomegeps0_cf));
+   // a_J2H->AddDomainIntegrator(new VectorMassIntegrator(negomegeps0_cf));
+
+   a_J1H->AddDomainIntegrator(new VectorMassIntegrator(balancescaled_negomegeps0_cf));
+   a_J2H->AddDomainIntegrator(new VectorMassIntegrator(balancescaled_negomegeps0_cf));
+
+
+   
+   a_J1H->Assemble(0);
+   a_J2H->Assemble(0);
+
+
+   // ±cᵢ(P(r) (b ⊗ b) E, δJᵢ))
+   ParMixedBilinearForm * a_EJ1 = new ParMixedBilinearForm(dpg_pfes[0], fem_pfes[0]);
+   ParMixedBilinearForm * a_EJ2 = new ParMixedBilinearForm(dpg_pfes[0], fem_pfes[1]);
+   ParMixedBilinearForm * a_EJ1_i = new ParMixedBilinearForm(dpg_pfes[0], fem_pfes[0]);
+   ParMixedBilinearForm * a_EJ2_i = new ParMixedBilinearForm(dpg_pfes[0], fem_pfes[1]);
+   // a_EJ1_i->AddDomainIntegrator(new VectorMassIntegrator(*signedcPibb_cf[0]));
+   // a_EJ2_i->AddDomainIntegrator(new VectorMassIntegrator(*signedcPibb_cf[1]));
+
+   a_EJ1_i->AddDomainIntegrator(new VectorMassIntegrator(*balancescaled_signedcPibb_cf[0]));
+   a_EJ2_i->AddDomainIntegrator(new VectorMassIntegrator(*balancescaled_signedcPibb_cf[1]));
+
+   a_EJ1_i->Assemble(0);
+   a_EJ2_i->Assemble(0);
+   // a_EJ1->AddDomainIntegrator(new VectorMassIntegrator(*signedcPrbb_cf[0]));
+   // a_EJ2->AddDomainIntegrator(new VectorMassIntegrator(*signedcPrbb_cf[1]));
+
+   a_EJ1->AddDomainIntegrator(new VectorMassIntegrator(*balancescaled_signedcPrbb_cf[0]));
+   a_EJ2->AddDomainIntegrator(new VectorMassIntegrator(*balancescaled_signedcPrbb_cf[1]));
+
+
+   a_EJ1->Assemble(0);
+   a_EJ2->Assemble(0);
+
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      delete pw_c_coeffs[i];
+      delete c_coeffs[i];
+      delete cPrbb_cf[i]; 
+      delete cPibb_cf[i]; 
+      delete signedcPrbb_cf[i]; 
+      delete signedcPibb_cf[i]; 
+   }
+
+   // Assemble all the system and get the global matrix and right-hand side
+   Array<ParFiniteElementSpace *> all_pfes;
+   all_pfes.Append(dpg_pfes);
+   all_pfes.Append(fem_pfes);
+
+   int npfes = all_pfes.Size();
+   Array<int> all_offsets(npfes + 1); all_offsets[0] = 0;
+   Array<int> all_toffsets(npfes + 1); all_toffsets[0] = 0;
+   for (int i = 0; i < npfes; ++i)
+   {
+      all_offsets[i+1] = all_pfes[i]->GetVSize();
+      all_toffsets[i+1] = all_pfes[i]->GetTrueVSize();
+   }
+   all_offsets.PartialSum();
+   all_toffsets.PartialSum();
+
+   Array<int> empty;
+   OperatorPtr Ah_dpg, Ah_fem, Ah_J1H, Ah_J2H, Ah_EJ1, Ah_EJ2;
+   OperatorPtr Ah_J1H_i, Ah_J2H_i, Ah_EJ1_i, Ah_EJ2_i;
+   // 4x4 upper left block (E, H, Ê, Ĥ)
+   a_dpg->FormSystemMatrix(empty,Ah_dpg); 
+   // 2x2 lower right block (J₁, J₂)
+   a_fem->FormSystemMatrix(empty,Ah_fem);
+   // cross term A₁₄  
+   a_J1H->FormRectangularSystemMatrix(empty, empty, Ah_J1H);
+   a_J1H_i->FormRectangularSystemMatrix(empty, empty, Ah_J1H_i);
+   // cross term A₂₄
+   a_J2H->FormRectangularSystemMatrix(empty, empty, Ah_J2H);
+   a_J2H_i->FormRectangularSystemMatrix(empty, empty, Ah_J2H_i);
+   // cross term A₄₀ 
+   a_EJ1->FormRectangularSystemMatrix(empty, empty, Ah_EJ1);
+   a_EJ1_i->FormRectangularSystemMatrix(empty, empty, Ah_EJ1_i);
+   // cross term A₅₀  
+   a_EJ2->FormRectangularSystemMatrix(empty, empty, Ah_EJ2);
+   a_EJ2_i->FormRectangularSystemMatrix(empty, empty, Ah_EJ2_i);
+
+
+   // put all the operators into a block operator
+   BlockOperator A_r(all_toffsets);
+   BlockOperator A_i(all_toffsets);
+
+   ComplexOperator * Ac_dpg = Ah_dpg.As<ComplexOperator>();
+   ComplexOperator * Ac_fem = Ah_fem.As<ComplexOperator>();
+
+   BlockOperator * BlockAdpg_r = dynamic_cast<BlockOperator *>(&Ac_dpg->real());
+   BlockOperator * BlockAdpg_i = dynamic_cast<BlockOperator *>(&Ac_dpg->imag());
+   BlockOperator * BlockAfem_r = dynamic_cast<BlockOperator *>(&Ac_fem->real());
+   BlockOperator * BlockAfem_i = dynamic_cast<BlockOperator *>(&Ac_fem->imag());
+
+
+   for (int i = 0; i < dpg_pfes.Size(); ++i)
+   {
+      for (int j = 0; j < dpg_pfes.Size(); ++j)
+      {
+         A_r.SetBlock(i, j, &BlockAdpg_r->GetBlock(i, j));
+         A_i.SetBlock(i, j, &BlockAdpg_i->GetBlock(i, j));
+      }
+   }
+   for (int i = 0; i < fem_pfes.Size(); ++i)
+   {
+      for (int j = 0; j < fem_pfes.Size(); ++j)
+      {
+         A_r.SetBlock(dpg_pfes.Size() + i, dpg_pfes.Size() + j, &BlockAfem_r->GetBlock(i, j));
+         A_i.SetBlock(dpg_pfes.Size() + i, dpg_pfes.Size() + j, &BlockAfem_i->GetBlock(i, j));
+      }
+   }  
+
+   A_r.SetBlock(1, 4, Ah_J1H.Ptr());
+   A_r.SetBlock(1, 5, Ah_J2H.Ptr());
+   A_r.SetBlock(4, 0, Ah_EJ1.Ptr());
+   A_r.SetBlock(5, 0, Ah_EJ2.Ptr());
+
+   A_i.SetBlock(1, 4, Ah_J1H_i.Ptr());
+   A_i.SetBlock(1, 5, Ah_J2H_i.Ptr());
+   A_i.SetBlock(4, 0, Ah_EJ1_i.Ptr());
+   A_i.SetBlock(5, 0, Ah_EJ2_i.Ptr());
+
+
+   ComplexOperator * A = new ComplexOperator(&A_r, &A_i, false, false);
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Complex Operator A finished successfully." << endl;
+   }
+
+   socketstream E_out_r;
+
+   Vector x(2*all_offsets.Last());
+   x = 0.;
+   
+   Array<ParGridFunction *> pgf_r(npfes);
+   Array<ParGridFunction *> pgf_i(npfes);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i] = new ParGridFunction(all_pfes[i], x, all_offsets[i]);
+      pgf_i[i] = new ParGridFunction(all_pfes[i], x, all_offsets.Last() + all_offsets[i]);
+   }
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+   ParGridFunction E_par_r(&L2_fes);
+   ParGridFunction E_par_i(&L2_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   std::string output_dir = "ParaView/UW-FEM/" + GetTimestamp();
+
+   if (paraview)
+   {
+      if (Mpi::Root()) { WriteParametersToFile(args, output_dir); }
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order
+                         << "_eld_1" ;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",pgf_r[0]);
+      paraview_dc->RegisterField("E_i",pgf_i[0]);
+      paraview_dc->RegisterField("E_par_r",&E_par_r);
+      paraview_dc->RegisterField("E_par_i",&E_par_i);
+      paraview_dc->RegisterField("H_r",pgf_r[1]);
+      paraview_dc->RegisterField("H_i",pgf_i[1]);      
+      paraview_dc->RegisterField("Jh_1_r",pgf_r[4]);
+      paraview_dc->RegisterField("Jh_1_i",pgf_i[4]);
+      paraview_dc->RegisterField("Jh_2_r",pgf_r[5]);
+      paraview_dc->RegisterField("Jh_2_i",pgf_i[5]);
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listJ;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+   
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+
+      // remove internal boundaries
+      for (int i = 0; i<int_bdr_attr.Size(); i++)
+      {
+         ess_bdr[int_bdr_attr[i]-1] = 0;
+      }
+      all_pfes[2]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += all_toffsets[2];
+      }
+      ess_bdr = 1;
+      for (int i = 0; i<ndiffusionequations;i++)
+      {
+         ess_tdof_listJ.SetSize(0);
+         all_pfes[i+4]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJ);
+         for (int j = 0; j < ess_tdof_listJ.Size(); j++)
+         {
+            ess_tdof_listJ[j] += all_toffsets[i+4];
+         }
+         ess_tdof_list.Append(ess_tdof_listJ);
+      }
+
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+   
+
+   // rotate the vector
+   // (x,y) -> (y,-x)
+   Vector rot_one_x(dim); rot_one_x = 0.0; rot_one_x(1) = -1.0;
+   Vector rot_negone_x(dim); rot_negone_x = 0.0; rot_negone_x(1) = 1.0;
+   VectorConstantCoefficient rot_one_x_cf(rot_one_x);
+   VectorConstantCoefficient rot_negone_x_cf(rot_negone_x);
+
+   pgf_r[2]->ProjectBdrCoefficientNormal(rot_one_x_cf, one_r_bdr);
+   pgf_r[2]->ProjectBdrCoefficientNormal(rot_negone_x_cf, negone_r_bdr);
+   pgf_i[2]->ProjectBdrCoefficientNormal(rot_one_x_cf, one_i_bdr);
+   pgf_i[2]->ProjectBdrCoefficientNormal(rot_negone_x_cf, negone_i_bdr);
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Boundary conditions finished." << endl;
+   }
+
+   BlockOperator * P = new BlockOperator(all_offsets, all_toffsets);
+   BlockMatrix * R = new BlockMatrix(all_toffsets, all_offsets);
+   P->owns_blocks = 0;
+   R->owns_blocks = 0;
+
+   for (int i = 0; i < npfes; i++)
+   {
+      HypreParMatrix * P_ = all_pfes[i]->Dof_TrueDof_Matrix();
+      P->SetBlock(i,i,P_);
+      const SparseMatrix * R_ = all_pfes[i]->GetRestrictionMatrix();
+      R->SetBlock(i, i, const_cast<SparseMatrix*>(R_));
+   }
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Build prolongation finished" << endl;
+   }
+
+   int n = P->Width();
+   Vector B(2*n); B = 0.0;
+
+   Vector X(2*n);
+   Vector X_r(X, 0, n);
+   Vector X_i(X, n, n);
+
+   Vector x_r(x, 0, x.Size()/2);
+   Vector x_i(x, x.Size()/2, x.Size()/2);
+
+   R->Mult(x_r, X_r);
+   R->Mult(x_i, X_i);
+
+   ParBlockComplexSystem aa(A);
+   A = aa.EliminateBC(ess_tdof_list, X, B);
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Eliminate BC finished successfully." << endl;
+   }
+
+
+   BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&A->real());
+   BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&A->imag());
+
+   int nblocks = BlockA_r->NumRowBlocks();
+   
+   Array2D<const HypreParMatrix*> A_r_matrices(nblocks, nblocks);
+   Array2D<const HypreParMatrix*> A_i_matrices(nblocks, nblocks);
+   for (int i = 0; i < nblocks; i++)
+   {
+      for (int j = 0; j < nblocks; j++)
+      {
+         if (!BlockA_r->IsZeroBlock(i,j))
+         {
+            A_r_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockA_r->GetBlock(i,j));
+         }
+         else
+         {
+            A_r_matrices(i,j) = nullptr;
+         }
+         if (!BlockA_i->IsZeroBlock(i,j))
+         {
+            A_i_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockA_i->GetBlock(i,j));
+         }
+         else
+         {
+            A_i_matrices(i,j) = nullptr;
+         }
+      }
+   }
+
+   HypreParMatrix * Ahr = HypreParMatrixFromBlocks(A_r_matrices);
+   HypreParMatrix * Ahi = HypreParMatrixFromBlocks(A_i_matrices);
+
+   ComplexHypreParMatrix * Ahc_hypre =
+      new ComplexHypreParMatrix(Ahr, Ahi,false, false);
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Getting ready for solve." << endl;
+   }
+   
+
+#ifdef MFEM_USE_COMPLEX_MUMPS
+   if (mumps_solver)
+   {
+      auto solver = new ComplexMUMPSSolver(MPI_COMM_WORLD);
+      solver->SetPrintLevel(1);
+      solver->SetOperator(*Ahc_hypre);
+      solver->Mult(B,X);
+      delete solver;
+      delete Ahc_hypre;
+   }
+#else
+   if (mumps_solver)
+   {
+      MFEM_WARNING("MFEM compiled without mumps. Switching to an iterative solver");
+   }
+   mumps_solver = false;
+#endif
+
+   Array<int> tdof_offsets(nblocks*2+1);
+   tdof_offsets[0] = 0;
+   for (int i=0; i<nblocks; i++)
+   {
+      tdof_offsets[i+1] = A_r_matrices(i,i)->Height();
+      tdof_offsets[nblocks+i+1] = tdof_offsets[i+1];
+   }
+   tdof_offsets.PartialSum();
+
+   if (!mumps_solver)
+   {
+
+      BlockDiagonalPreconditioner M(tdof_offsets);
+
+      HypreBoomerAMG * solver_E = new HypreBoomerAMG((HypreParMatrix &)
+                                                   BlockA_r->GetBlock(0,0));
+      solver_E->SetPrintLevel(0);
+      solver_E->SetSystemsOptions(dim);
+
+      HypreBoomerAMG * solver_H = new HypreBoomerAMG((HypreParMatrix &)
+                                                BlockA_r->GetBlock(1,1));
+      solver_H->SetPrintLevel(0);
+
+      M.SetDiagonalBlock(0,solver_E);
+      M.SetDiagonalBlock(1,solver_H);
+      M.SetDiagonalBlock(nblocks,solver_E);
+      M.SetDiagonalBlock(nblocks+1,solver_H);
+
+      HypreAMS * solver_hatE = 
+      new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(2,2), dpg_pfes[2]);
+
+
+      HypreBoomerAMG * solver_hatH = new HypreBoomerAMG((HypreParMatrix &)
+                 BlockA_r->GetBlock(3,3));
+      solver_hatE->SetPrintLevel(0);
+      solver_hatH->SetPrintLevel(0);
+      solver_hatH->SetRelaxType(88);
+
+
+      M.SetDiagonalBlock(2,solver_hatE);
+      M.SetDiagonalBlock(3,solver_hatH);
+      M.SetDiagonalBlock(2+nblocks,solver_hatE);
+      M.SetDiagonalBlock(3+nblocks,solver_hatH);
+
+
+      HypreBoomerAMG * solver_J1 = new HypreBoomerAMG((HypreParMatrix &)
+                                                BlockA_r->GetBlock(4,4));
+      solver_J1->SetPrintLevel(0);
+      solver_J1->SetSystemsOptions(dim);
+      solver_J1->SetRelaxType(88);
+
+      HypreBoomerAMG * solver_J2 = new HypreBoomerAMG((HypreParMatrix &)
+                                                BlockA_r->GetBlock(5,5));
+      solver_J2->SetPrintLevel(0);
+      solver_J2->SetSystemsOptions(dim);
+      solver_J2->SetRelaxType(88);
+
+
+      M.SetDiagonalBlock(4,solver_J1);
+      M.SetDiagonalBlock(5,solver_J2);
+      M.SetDiagonalBlock(4+nblocks,solver_J1);
+      M.SetDiagonalBlock(5+nblocks,solver_J2);                                                
+
+      GMRESSolver gmres(MPI_COMM_WORLD);
+      gmres.SetRelTol(1e-10);
+      gmres.SetMaxIter(1000);
+      gmres.SetPrintLevel(1);
+      gmres.SetPreconditioner(M);
+      gmres.SetOperator(*A);
+      gmres.Mult(B, X);
+   }
+
+
+   n = P->Height();
+   int m = P->Width();
+
+   x_r.MakeRef(x, 0, n);
+   x_i.MakeRef(x, n, n);
+
+   X_r.MakeRef(X, 0, m);
+   X_i.MakeRef(X, m, m);
+
+   P->Mult(X_r, x_r);
+   P->Mult(X_i, x_i);
+
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i]->MakeRef(all_pfes[i], x, all_offsets[i]);
+      pgf_i[i]->MakeRef(all_pfes[i], x, all_offsets.Last() + all_offsets[i]);
+   }
+   
+   ParallelECoefficient par_e_r(pgf_r[0]);
+   ParallelECoefficient par_e_i(pgf_i[0]);
+   E_par_r.ProjectCoefficient(par_e_r);
+   E_par_i.ProjectCoefficient(par_e_i);
+   
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, *pgf_r[0],
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((real_t)0);
+      paraview_dc->Save();
+      delete paraview_dc;
+   }
+
+
+   delete a_fem;
+   for (int i = 0; i < fem_fecols.Size(); ++i)
+   {
+      delete fem_fecols[i];
+      delete fem_pfes[i];
+   }
+
+
+   delete a_dpg;
+   for (int i = 0; i < dpg_trial_fecols.Size(); ++i)
+   {
+      delete dpg_trial_fecols[i];
+      delete dpg_pfes[i];
+   }
+   for (int i = 0; i< dpg_test_fecols.Size(); ++i)
+   {
+      delete dpg_test_fecols[i];
+   }
+
+   return 0;
+}

miniapps/dpg/plasma/lh-eld-dpg.cpp

@@ -0,0 +1,907 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//                   MFEM Ultraweak DPG Maxwell parallel example
+//
+// Compile with: make lh-eld-dpg
+//
+// mpirun -np 8 ./lh-eld-dpg -o 4 -paraview -eld -m data/quad.msh -ebs -sc
+
+// Electron Landau Damping
+// Strong formulation:
+//     ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵ E + i ω²ϵ₀(J₁ + J₂) = 0,   in Ω
+//                   - Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥ = 0,   in Ω     
+//                   - Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥ = 0,   in Ω 
+//                                            E×n = E₀,  on ∂Ω
+//                                            J₁  = 0,  on ∂Ω
+//                                            J₂  = 0,  on ∂Ω
+// The DPG UW deals with the First Order System
+//  i ω μ₀ H  + ∇ × E                  = 0,   in Ω
+// -i ω ϵ₀ϵ E + ∇ × H - ω ϵ₀ (J₁ + J₂) = 0,   in Ω
+//         - Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥ = 0,   in Ω     
+//         - Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥ = 0,   in Ω 
+//                                 E×n = E₀,  on ∂Ω
+//                                 J₁  = 0,  on ∂Ω
+//                                 J₂  = 0,  on ∂Ω
+
+
+// in 2D
+// E is vector valued and H is scalar.
+//      (∇ × E, δE) = (E, ∇ × δE ) + < n × E , δE >
+// or (∇ ⋅ AE , δE) = (AE, ∇ δE ) + < AE ⋅ n, δE >
+// where A = [0 1; -1 0];
+
+// E ∈ (L²(Ω))² , H ∈ L²(Ω), J ∈ (H¹(Ω))²
+// Ê ∈ H^-1/2(Γₕ), Ĥ ∈ H^1/2(Γₕ), Ĵ₁, Ĵ₂ ∈ (H^-1/2(Γₕ))²
+//     iωμ₀ (H,δE) + (E,∇×δE) + < AÊ, δE > = 0,      ∀ δE ∈ H¹(Ω)
+//  -i ωϵ₀ϵ (E,δH) + (H,∇×δH) + < Ĥ, δH×n > - ω ϵ₀ (J₁ + J₂,δH) = 0,  ∀ δH ∈ H(curl,Ω)
+// ( (b⋅∇)J₁,(b⋅∇) δJ₁ ) + <Ĵ₁, δJ₁> + c₁ (J₁,δJ₁) - c₁ (P(r) b⊗b E, δJ₁) = 0,  ∀ δJ₁ ∈ (H¹(Ω))²
+// ( (b⋅∇)J₂,(b⋅∇) δJ₂ ) + <Ĵ₂, δJ₂> + c₂ (J₂,δJ₂) + c₂ (P(r) b⊗b E, δJ₂) = 0,  ∀ δJ₂ ∈ (H¹(Ω))²
+//                                                                  Ê = E₀, on ∂Ω
+//                                                            Ĵ₁ = Ĵ₂ = 0,  on ∂Ω
+// ----------------------------------------------------------------------------------------------------------------
+// |   |      E       |     H    |        J₁        |        J₂        |   Ê   |   Ĥ    |   Ĵ₁   |   Ĵ₂   |  RHS  |
+// ----------------------------------------------------------------------------------------------------------------
+// |δE |  (E,∇ × δE)  |iωμ₀(H,δE)|                  |                  | <Ê,δE>|        |        |        |   0   |  
+// |   |              |          |                  |                  |       |        |        |        |       |  
+// |δH | -iωϵ₀ϵ(E,δH) | (H,∇×δH) |   -ωϵ₀ (J₁,δH)   |  -ωϵ₀ (J₂,δH)    |       |<Ĥ,δH×n>|        |        |   0   |  
+// |   |              |          |                  |                  |       |        |        |        |       |  
+// |δJ₁|-c₁(P(r)E,δJ₁)|          |((b⋅∇)J₁,(b⋅∇)δJ₁)|                  |       |        |<Ĵ₁,δJ₁>|        |   0   |  
+// |   |              |          |     + c₁ (J₁,δJ₁)|                  |       |        |        |        |       |    
+// |δJ₂| c₂(P(r)E,δJ₂)|          |                  |((b⋅∇)J₂,(b⋅∇)δJ₂)|       |        |        |<Ĵ₂,δJ₂>|   0   |  
+// |   |              |          |                  |     + c₂ (J₂,δJ₂)|       |        |        |        |       |    
+// where (δE,δH,δJ₁,δJ₂) ∈  H¹(Ω) × H(curl,Ω) × (H¹(Ω))² × (H¹(Ω))² 
+
+
+
+#include "mfem.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../../common/mfem-common.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "../util/preconditioners.hpp"
+#include "utils/lh_utils.hpp"
+#include "../util/utils.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+   int order = 1;
+   int delta_order = 1;
+   int par_ref_levels = 0;
+   int ser_ref_levels = 0;
+
+   // real_t rnum=1.5e9;
+   // real_t mu = 1.257e-6;
+   // real_t eps0 = 8.8541878128e-12;
+
+   real_t rnum=1.5;
+   real_t mu = 1.257;
+   real_t eps0 = 8.8541878128;
+   real_t cfactor = 1e-6;
+   real_t balance_scale = 1.0;
+   bool enable_balance_scale = false;
+
+   bool eld = false; // enable/disable electron Landau damping 
+   real_t delta_prec = 0.0;
+
+
+   bool static_cond = false;
+   bool visualization = false;
+   bool paraview = false;
+   bool debug = false;
+   bool mumps_solver = false;
+   bool pmg = false;
+   int pmg_levels = -1;
+   real_t relax_factor = 2.0/3;
+
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&delta_order, "-do", "--delta-order",
+                  "Finite element order for the test space");                  
+   args.AddOption(&ser_ref_levels, "-sr", "--serial-refinement_levels",
+                  "Number of serial refinement levels.");                  
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&a0, "-a0", "--a0", "P(r) first parameter.");
+   args.AddOption(&a1, "-a1", "--a1", "P(r) second parameter.");
+   args.AddOption(&delta_prec, "-dp", "--delta-prec", "stability parameter for the preconditioner.");
+   args.AddOption(&eld, "-eld", "--eld", "-no-eld",
+                  "--no-eld",
+                  "Enable or disable electron Landau damping.");
+   args.AddOption(&mumps_solver, "-mumps", "--mumps", "-no-mumps",
+                  "--no-mumps",
+                  "Enable or disable MUMPS solver.");
+   args.AddOption(&pmg, "-pmg", "--p-refinement-multigrid", "-no-pmg",
+                  "--no-p-refinement-multigrid", "Enable P-Refinement Multigrid.");
+   args.AddOption(&pmg_levels, "-pmgl","--p-refinement-multigrid-levels",
+                  "Number of levels for P-Refinement Multigrid.");
+   args.AddOption(&relax_factor, "-rf", "--relaxation-factor",
+                  "Relaxation factor for the p-multigrid smoother.");                  
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&debug, "-debug", "--debug", "-no-debug",
+                  "--no-debug",
+                  "Enable or disable debug mode (delta = 0.01 and no coupling).");         
+   args.AddOption(&enable_balance_scale, "-ebs", "--enable-balance-scale", "-no-ebs",
+                  "--no-ebs",
+                  "Enable or disable balance scale.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (Mpi::Root())
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (Mpi::Root())
+   {
+      args.PrintOptions(cout);
+   }
+
+   // number of diffusion equations
+   int ndiffusionequations = (eld) ?  2 : 0; 
+
+   Vector cvals(ndiffusionequations);
+   Vector csigns(ndiffusionequations);
+   if (eld)
+   {
+      cvals(0)  = 25e6;  cvals(1)  = 1e6;
+      csigns(0) = -1.0;  csigns(1) = 1.0;
+   }
+   cvals *= cfactor; // scale the coefficients
+   real_t omega = 2.*M_PI*rnum;
+   int test_order = order+delta_order;
+
+   balance_scale = (enable_balance_scale) ? omega * eps0 * omega : 1.0;
+
+
+   if (eld && !debug) 
+   {
+      delta = 0.0; // disable delta if electron Landau damping is enabled
+      if (Mpi::Root())
+      {
+         cout << "Electron Landau damping enabled, delta set to 0.0." << endl;
+      }
+   }    
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   MFEM_VERIFY(dim == 2, "Dimension != 2 is not supported in this example");
+
+   for (int i = 0; i < ser_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // mesh.RemoveInternalBoundaries();
+
+   Array<int> int_bdr_attr;
+   for (int i = 0; i < mesh.GetNBE(); i++)
+   {
+      if (mesh.FaceIsInterior(mesh.GetBdrElementFaceIndex(i)))
+      {
+         int_bdr_attr.Append(mesh.GetBdrAttribute(i));
+      }
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   for (int i = 0; i < par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }  
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+
+   // Define coefficients
+   ConstantCoefficient muinv(1./mu);
+   ConstantCoefficient one_cf(1.0);
+   //  ωμ₀
+   ConstantCoefficient omegamu_cf(omega*mu);
+   // -ω μ₀  
+   ConstantCoefficient negomegamu_cf(-omega*mu);
+   // -ωϵ₀
+   real_t scale = (debug) ? 0.0 : 1.0;
+   ConstantCoefficient negomegeps0_cf(-omega*eps0 * scale);
+   ConstantCoefficient balancescaled_negomegeps0_cf( -omega*eps0/balance_scale * scale);
+   // μ₀² ω²
+   ConstantCoefficient mu2omeg2_cf((mu*mu*omega*omega));
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_vcf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   // S(r) 
+   FunctionCoefficient S_cf_r(sfunc_r), S_cf_i(sfunc_i);
+   // P(r) 
+   FunctionCoefficient P_cf_r(pfunc_r), P_cf_i(pfunc_i); 
+
+   VectorFunctionCoefficient b_cf(dim,bfunc);// b
+   ScalarVectorProductCoefficient scaled_b_cf(sqrt(cfactor), b_cf);
+   ConstantCoefficient diff_coeff(cfactor);
+
+   MatrixFunctionCoefficient bb_cf(dim,bcrossb); // b⊗b
+   MatrixSumCoefficient oneminusbb(Mone_cf, bb_cf, 1.0, -1.0); // 1 - b⊗b
+
+   // S(r) (I - b⊗b)
+   ScalarMatrixProductCoefficient Soneminusbb_r(S_cf_r, oneminusbb), Soneminusbb_i(S_cf_i, oneminusbb); 
+
+   // P(r) b⊗b 
+   ScalarMatrixProductCoefficient P_cf_bb_r(P_cf_r, bb_cf), P_cf_bb_i(P_cf_i, bb_cf); 
+
+   // ε = S(r) (I - b⊗b) + P(r) b⊗b 
+   MatrixSumCoefficient eps_r(Soneminusbb_r, P_cf_bb_r, 1.0, 1.0); 
+   MatrixSumCoefficient eps_i(Soneminusbb_i, P_cf_bb_i, 1.0, 1.0); 
+
+   coefs_r[nattr-1] = &eps_r;
+   coefs_i[nattr-1] = &eps_i;
+
+   // for (int i = 0; i < nattr-1; ++i)
+   // {
+   //    coefs_r[i] = &eps_r;
+   //    coefs_i[i] = &eps_i;
+   // }
+
+   PWMatrixCoefficient eps_cf_r(dim, attr, coefs_r);
+   PWMatrixCoefficient eps_cf_i(dim, attr, coefs_i);
+
+   ConstantCoefficient eps0omeg(omega * eps0);
+   ConstantCoefficient negeps0omeg(-omega * eps0);
+
+   // ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient eps0omeg_eps_r(eps0omeg, eps_cf_r);
+   // ω ϵ₀ ϵᵢ
+   ScalarMatrixProductCoefficient eps0omeg_eps_i(eps0omeg, eps_cf_i);
+   // -ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient negeps0omeg_eps_r(negeps0omeg, eps_cf_r);
+   // -ω ϵ₀ ϵᵢ
+   ScalarMatrixProductCoefficient negeps0omeg_eps_i(eps0omeg, eps_cf_i);
+
+   // A = [0 1; -1 0]
+   DenseMatrix rot_mat(2);
+   rot_mat(0,0) = 0.; rot_mat(0,1) = 1.;
+   rot_mat(1,0) = -1.; rot_mat(1,1) = 0.;
+   MatrixConstantCoefficient rot(rot_mat);
+   TransposeMatrixCoefficient Rt(rot);
+
+   // ω ϵ₀ ϵᵣ A
+   MatrixProductCoefficient eps0omeg_eps_r_rot(eps0omeg_eps_r, rot);
+   // ω ϵ₀ ϵᵢ A
+   MatrixProductCoefficient eps0omeg_eps_i_rot(eps0omeg_eps_i, rot);
+   // ω ϵ₀ Aᵀ ϵᵢ
+   MatrixProductCoefficient eps0omeg_Rt_eps_i(Rt, eps0omeg_eps_i);
+   // ω ϵ₀ Aᵀ ϵᵣ
+   MatrixProductCoefficient eps0omeg_Rt_eps_r(Rt, eps0omeg_eps_r);
+   // -ω ϵ₀ ϵᵣ A
+   MatrixProductCoefficient negeps0omeg_eps_r_rot(negeps0omeg_eps_r, rot);
+   // -ω ϵ₀ ϵᵢ A
+   MatrixProductCoefficient negeps0omeg_eps_i_rot(negeps0omeg_eps_i, rot);
+
+   // (ωϵ₀ϵ)(ωϵ₀ϵ)^*  (δH, δH)
+   TransposeMatrixCoefficient eps0omeg_eps_r_t(eps0omeg_eps_r);
+   TransposeMatrixCoefficient eps0omeg_eps_i_t(eps0omeg_eps_i);
+   MatrixProductCoefficient MrMrt_cf(eps0omeg_eps_r, eps0omeg_eps_r_t);
+   MatrixProductCoefficient MiMit_cf(eps0omeg_eps_i, eps0omeg_eps_i_t);
+   MatrixProductCoefficient MiMrt_cf(eps0omeg_eps_i, eps0omeg_eps_r_t);
+   MatrixProductCoefficient MrMit_cf(eps0omeg_eps_r, eps0omeg_eps_i_t);
+
+   // (MᵣMᵣᵗ + MᵢMᵢᵗ) + i (MᵢMᵣᵗ - MᵣMᵢᵗ)
+   MatrixSumCoefficient Mreal_cf(MrMrt_cf,MiMit_cf);
+   MatrixSumCoefficient Mimag_cf(MiMrt_cf,MrMit_cf,1.0,-1.0);
+
+   // if ELD
+   Array<Vector *> c_arrays(ndiffusionequations);
+   Array<PWConstCoefficient *> pw_c_coeffs(ndiffusionequations);
+   Array<MatrixCoefficient *> cPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> cPibb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPibb_cf(ndiffusionequations);
+
+   Array<MatrixCoefficient *> balancescaled_signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> balancescaled_signedcPibb_cf(ndiffusionequations);
+   Vector temp(nattr); temp=0.0;
+   Array<ConstantCoefficient *> c_coeffs(ndiffusionequations);
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      temp[nattr-1] = cvals(i);
+      // temp = cvals(i);
+      pw_c_coeffs[i] = new PWConstCoefficient(temp);
+      c_coeffs[i] = new ConstantCoefficient(cvals(i));
+      cPrbb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_r);
+      cPibb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_i);
+      signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPrbb_cf[i]);
+      signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPibb_cf[i]);
+      balancescaled_signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(balance_scale,*signedcPrbb_cf[i]);
+      balancescaled_signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(balance_scale,*signedcPibb_cf[i]);
+   }
+
+   // Define the spaces
+   Array<FiniteElementCollection *> trial_fecols;
+   Array<FiniteElementCollection *> test_fecols;
+   Array<ParFiniteElementSpace *> pfes;
+
+   // Vector L2 space for E
+   trial_fecols.Append(new L2_FECollection(order-1, dim));
+   pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last(), dim));
+   // Scalar L2 space for H
+   trial_fecols.Append(new L2_FECollection(order-1, dim));
+   pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last()));
+
+   // Vector H1 spaces for Jᵢ 
+   for (int i = 0; i < ndiffusionequations; i++)
+   {
+      trial_fecols.Append(new H1_FECollection(order, dim));
+      pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last(), dim));
+   }
+
+   // Trial trace space for Ê 
+   trial_fecols.Append(new RT_Trace_FECollection(order-1, dim));
+   pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last()));
+   // Trial trace space for Ĥ 
+   trial_fecols.Append(new H1_Trace_FECollection(order, dim));
+   pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last()));
+
+   // Vector Trace spaces Ĵᵢ    
+   for (int i = 0; i < ndiffusionequations; i++)
+   {
+      trial_fecols.Append(new RT_Trace_FECollection(order-1,dim));
+      pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last(),dim));
+   }
+
+   Array<HYPRE_BigInt> tdofs(pfes.Size());
+   for (int i = 0; i < pfes.Size(); ++i)
+   {
+      tdofs[i] = pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << "ParFiniteElementSpace " << i << " has " << tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   if (Mpi::Root())
+   {
+      cout << "Total number of true dofs: " << tdofs.Sum() << endl;
+   }
+
+   // test spaces for E and H
+   test_fecols.Append(new H1_FECollection(test_order, dim));
+   test_fecols.Append(new ND_FECollection(test_order, dim));
+   // Test spaces δJs 
+   for (int i = 0; i < ndiffusionequations; i++)
+   {
+      test_fecols.Append(new H1_FECollection(test_order, dim));
+      // test_fecols.Append(new H1_FECollection(order, dim));
+   }
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(pfes,test_fecols);
+   const IntegrationRule &ir_test = IntRules.Get(pmesh.GetElementGeometry(0),
+                                           2*test_order + 4);
+   const IntegrationRule &ir_trial = IntRules.Get(pmesh.GetElementGeometry(0),
+                                           order+test_order + 4);
+   a->SetTestIntegrationRule(ir_test);
+   a->SetTrialIntegrationRule(ir_trial);
+   
+   
+   for (int i = 0; i < ndiffusionequations; i++)
+   {
+      a->SetTestFECollVdim(i+2,dim);
+   }
+
+   // (E,∇ × δE)
+   a->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one_cf)),
+                         nullptr,0, 0);
+   //  -i ω ϵ₀ (ϵE,δH) = - i ω ϵ₀(ϵᵣ + i ϵᵢ E, δH)
+   //                   =  (ω ϵ₀ ϵᵢ E, δH) + i (-ω ϵ₀ϵᵣ E, δH)
+   a->AddTrialIntegrator(
+      new TransposeIntegrator(new VectorFEMassIntegrator(eps0omeg_eps_i)), 
+      new TransposeIntegrator(new VectorFEMassIntegrator(negeps0omeg_eps_r)),
+                              0,1);
+   // iωμ₀(H,δE) 
+   a->AddTrialIntegrator(nullptr,new MixedScalarMassIntegrator(omegamu_cf),1, 0);
+   // (H,∇ × δH)                         
+   a->AddTrialIntegrator(
+      new TransposeIntegrator(new MixedCurlIntegrator(one_cf)), nullptr,1, 1);
+
+   // Trace integrators               
+   //  <Ê,δE>
+   a->AddTrialIntegrator(new TraceIntegrator,nullptr, 2 + ndiffusionequations, 0);
+   // <Ĥ,δH × n>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr, 2 + ndiffusionequations+1, 1);
+   if (eld)
+   {
+      for (int i = 0; i < ndiffusionequations; i++)
+      {
+         // ±cᵢ(P(r) (b ⊗ b) E, δJᵢ)
+         a->AddTrialIntegrator(new VectorMassIntegrator(*balancescaled_signedcPrbb_cf[i]),
+                               new VectorMassIntegrator(*balancescaled_signedcPibb_cf[i]),
+                               0, i+2);
+         // -ωϵ₀ (Jᵢ ,δH)
+         a->AddTrialIntegrator(
+            new TransposeIntegrator(new VectorFEMassIntegrator(balancescaled_negomegeps0_cf)),
+            // new TransposeIntegrator(new VectorFEMassIntegrator(negomegeps0_cf)),
+                           nullptr,i+2, 1);
+         // ((b⋅∇)Jᵢ, (b⋅∇) δJᵢ)
+         a->AddTrialIntegrator(new DirectionalVectorDiffusionIntegrator(scaled_b_cf), nullptr,
+         // a->AddTrialIntegrator(new VectorDiffusionIntegrator(diff_coeff), nullptr,
+                               i+2, i+2);
+         // cᵢ(Jᵢ, δJᵢ)
+         a->AddTrialIntegrator(new VectorMassIntegrator(*pw_c_coeffs[i]), nullptr,
+                               i+2, i+2);
+
+         // // <Ĵᵢ,δJᵢ>
+         a->AddTrialIntegrator(new VectorTraceIntegrator,nullptr,
+                               i + ndiffusionequations + 4, i+2);                      
+      }
+   }      
+
+   // test integrators
+   // (∇δE,∇δE)
+   a->AddTestIntegrator(new DiffusionIntegrator(one_cf),nullptr, 0, 0);
+   // (δE,δE)
+   a->AddTestIntegrator(new MassIntegrator(one_cf),nullptr, 0, 0);
+   // μ₀² ω² (δE,δE)
+   a->AddTestIntegrator(new MassIntegrator(mu2omeg2_cf),nullptr,0, 0);
+   // -i ω μ₀ (δE,∇ × δH) = i (δE, -ω μ₀ ∇ × δ H)
+   a->AddTestIntegrator(nullptr,
+         new TransposeIntegrator(new MixedCurlIntegrator(negomegamu_cf)),0, 1);
+   // -i ω ϵ₀ϵ(∇ × δE, δH) = -i (ωϵ₀(ϵᵣ+iϵᵢ) A ∇ δE,δE), A = [0 1; -1 0]
+   //                       =  (ω ϵ₀ ϵᵢ A ∇ δE,δE) + i (-ω ϵ₀ ϵᵣ A ∇ δE,δE)
+   a->AddTestIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_i_rot),
+                        new MixedVectorGradientIntegrator(negeps0omeg_eps_r_rot),0, 1);
+   // i ω μ₀ (∇ × δH ,δE) = i (ω μ₀ ∇ × δH, δE )
+   a->AddTestIntegrator(nullptr,new MixedCurlIntegrator(omegamu_cf),
+                        1, 0);
+   // i ω ϵ₀ϵ̄ (δH, ∇ × δE ) = i (ω ϵ₀(ϵᵣ -i ϵᵢ) δH, A ∇ δE) 
+   //                        = ( δH, ω ϵ₀ ϵᵢ A ∇ δE) + i (δH, ω ϵ₀ ϵᵣ A ∇ δE)
+
+
+   a->AddTestIntegrator(new TransposeIntegrator(new MixedVectorGradientIntegrator(eps0omeg_Rt_eps_i)),
+                        new TransposeIntegrator(new MixedVectorGradientIntegrator(eps0omeg_Rt_eps_r)),1, 0);
+
+   // a->AddTestIntegrator(
+   //    new TransposeIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_i_rot)),
+   //    new TransposeIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_r_rot)),1, 0);
+   
+   
+      // (ωϵ₀ϵ)(ωϵ₀ϵ)^*  (δH, δH)
+   // (MᵣMᵣᵗ + MᵢMᵢᵗ) + i (MᵢMᵣᵗ - MᵣMᵢᵗ)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(Mreal_cf),
+                        new VectorFEMassIntegrator(Mimag_cf),1, 1);
+   // (∇×δH ,∇×δH)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one_cf),nullptr,1,1);
+   // (δH,δH)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one_cf),nullptr,1,1);
+
+   for (int i = 0; i < ndiffusionequations; i++)
+   {
+      // (∇δJ,∇δJ) 
+      // a->AddTestIntegrator(new VectorDiffusionIntegrator(one_cf),nullptr,
+                           // i+2,i+2);
+      // (b⋅∇δJ, b⋅∇δJ)
+      a->AddTestIntegrator(new DirectionalVectorDiffusionIntegrator(scaled_b_cf),nullptr,
+      // a->AddTestIntegrator(new DirectionalVectorDiffusionIntegrator(b_cf),nullptr,
+                           i+2,i+2);                     
+      // (δJ,δJ)
+      // a->AddTestIntegrator(new VectorMassIntegrator(one_cf),nullptr, 
+      a->AddTestIntegrator(new VectorMassIntegrator(*c_coeffs[i]),nullptr, 
+                           i+2,i+2);
+   }
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble(0);
+
+
+   // delta = delta_prec;
+   // ParComplexDPGWeakForm * a_prec = new ParComplexDPGWeakForm(pfes,test_fecols);
+   // for (int i = 0; i < ndiffusionequations; i++)
+   // {
+   //    a_prec->SetTestFECollVdim(i+2,dim);
+   // }
+
+   // // (E,∇ × δE)
+   // a_prec->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one_cf)),
+   //                       nullptr,0, 0);
+   // //  -i ω ϵ₀ (ϵE,δH) = - i ω ϵ₀(ϵᵣ + i ϵᵢ E, δH)
+   // //                   =  (ω ϵ₀ ϵᵢ E, δH) + i (-ω ϵ₀ϵᵣ E, δH)
+   // a_prec->AddTrialIntegrator(
+   //    new TransposeIntegrator(new VectorFEMassIntegrator(eps0omeg_eps_i)), 
+   //    new TransposeIntegrator(new VectorFEMassIntegrator(negeps0omeg_eps_r)),
+   //    0,1);
+   // // iωμ₀(H,δE)
+   // a_prec->AddTrialIntegrator(nullptr,new MixedScalarMassIntegrator(omegamu_cf),1, 0);
+   // // (H,∇ × δH)                         
+   // a_prec->AddTrialIntegrator(
+   //    new TransposeIntegrator(new MixedCurlIntegrator(one_cf)), nullptr,1, 1);
+
+   // // Trace integrators               
+   // //  <Ê,δE>
+   // a_prec->AddTrialIntegrator(new TraceIntegrator,nullptr, 2 + ndiffusionequations, 0);
+   // // <Ĥ,δH × n>
+   // a_prec->AddTrialIntegrator(new TangentTraceIntegrator,nullptr, 2 + ndiffusionequations+1, 1);
+   // if (eld)
+   // {
+   //    for (int i = 0; i < ndiffusionequations; i++)
+   //    {
+   //       // ±cᵢ(P(r) (b ⊗ b) E, δJᵢ)
+   //       a_prec->AddTrialIntegrator(new VectorMassIntegrator(*balancescaled_signedcPrbb_cf[i]),
+   //                             new VectorMassIntegrator(*balancescaled_signedcPibb_cf[i]), 0, i+2);
+         
+   //       // a_prec->AddTrialIntegrator(
+   //       //    new TransposeIntegrator(new VectorFEMassIntegrator(balancescaled_negomegeps0_cf)),
+   //       //                   nullptr,i+2, 1);
+         
+   //       // ((b⋅∇)Jᵢ, (b⋅∇) δJᵢ)
+   //       a_prec->AddTrialIntegrator(new DirectionalVectorDiffusionIntegrator(scaled_b_cf), nullptr, i+2, i+2);
+   //       // cᵢ(Jᵢ, δJᵢ)
+   //       a_prec->AddTrialIntegrator(new VectorMassIntegrator(*pw_c_coeffs[i]), nullptr, i+2, i+2);
+   //       // <Ĵᵢ,δJᵢ>
+   //       // a_prec->AddTrialIntegrator(new VectorTraceIntegrator,nullptr, i + ndiffusionequations + 4, i+2);                      
+   //    }
+   // }      
+
+   // // test integrators
+   // // (∇δE,∇δE)
+   // a_prec->AddTestIntegrator(new DiffusionIntegrator(one_cf),nullptr, 0, 0);
+   // // (δE,δE)
+   // a_prec->AddTestIntegrator(new MassIntegrator(one_cf),nullptr, 0, 0);
+   // // μ₀² ω² (δE,δE)
+   // a_prec->AddTestIntegrator(new MassIntegrator(mu2omeg2_cf),nullptr,0, 0);
+   // // -i ω μ₀ (δE,∇ × δH) = i (δE, -ω μ₀ ∇ × δ H)
+   // a_prec->AddTestIntegrator(nullptr,
+   //       new TransposeIntegrator(new MixedCurlIntegrator(negomegamu_cf)),0, 1);
+   // // -i ω ϵ₀ϵ(∇ × δE, δH) = -i (ωϵ₀(ϵᵣ+iϵᵢ) A ∇ δE,δE), A = [0 1; -1 0]
+   // //                       =  (ω ϵ₀ ϵᵢ A ∇ δE,δE) + i (-ω ϵ₀ ϵᵣ A ∇ δE,δE)
+   // a_prec->AddTestIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_i_rot),
+   //                      new MixedVectorGradientIntegrator(negeps0omeg_eps_r_rot),0, 1);
+   // // i ω μ₀ (∇ × δH ,δE) = i (ω μ₀ ∇ × δH, δE )
+   // a_prec->AddTestIntegrator(nullptr,new MixedCurlIntegrator(omegamu_cf),
+   //                      1, 0);
+   // // i ω ϵ₀ϵ̄ (δH, ∇ × δE ) = i (ω ϵ₀(ϵᵣ -i ϵᵢ) δH, A ∇ δE) 
+   // //                        = ( δH, ω ϵ₀ ϵᵢ A ∇ δE) + i (δH, ω ϵ₀ ϵᵣ A ∇ δE)
+   // a_prec->AddTestIntegrator(
+   //    new TransposeIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_i_rot)),
+   //    new TransposeIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_r_rot)),1, 0);
+   // // (ωϵ₀ϵ)(ωϵ₀ϵ)^*  (δH, δH)
+   // // (MᵣMᵣᵗ + MᵢMᵢᵗ) + i (MᵢMᵣᵗ - MᵣMᵢᵗ)
+   // a_prec->AddTestIntegrator(new VectorFEMassIntegrator(Mreal_cf),
+   //                      new VectorFEMassIntegrator(Mimag_cf),1, 1);
+   // // (∇×δH ,∇×δH)
+   // a_prec->AddTestIntegrator(new CurlCurlIntegrator(one_cf),nullptr,1,1);
+   // // (δH,δH)
+   // a_prec->AddTestIntegrator(new VectorFEMassIntegrator(one_cf),nullptr,1,1);
+
+   // for (int i = 0; i < ndiffusionequations; i++)
+   // {
+   //    // (b⋅∇δJ, b⋅∇δJ)
+   //    a_prec->AddTestIntegrator(new DirectionalVectorDiffusionIntegrator(scaled_b_cf),nullptr,
+   //                         i+2,i+2);                     
+   //    // (δJ,δJ)
+   //    a_prec->AddTestIntegrator(new VectorMassIntegrator(*c_coeffs[i]),nullptr, 
+   //                         i+2,i+2);
+   // }
+
+   // if (static_cond) { a_prec->EnableStaticCondensation(); }
+   // a_prec->Assemble();
+
+
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      delete pw_c_coeffs[i];
+      delete c_coeffs[i];
+      delete cPrbb_cf[i]; 
+      delete cPibb_cf[i]; 
+      delete signedcPrbb_cf[i]; 
+      delete signedcPibb_cf[i]; 
+   }
+
+   socketstream E_out_r;
+
+   int npfes = pfes.Size();
+   Array<int> offsets(npfes+1);  offsets[0] = 0;
+   Array<int> toffsets(npfes+1); toffsets[0] = 0;
+   for (int i = 0; i<npfes; i++)
+   {
+      offsets[i+1] = pfes[i]->GetVSize();
+      toffsets[i+1] = pfes[i]->TrueVSize();
+   }
+   offsets.PartialSum();
+   toffsets.PartialSum();
+
+   Vector x(2*offsets.Last());
+   x = 0.;
+   
+   Array<ParGridFunction *> pgf_r(npfes);
+   Array<ParGridFunction *> pgf_i(npfes);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i] = new ParGridFunction(pfes[i], x, offsets[i]);
+      pgf_i[i] = new ParGridFunction(pfes[i], x, offsets.Last() + offsets[i]);
+   }
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+   ParGridFunction E_par_r(&L2_fes);
+   ParGridFunction E_par_i(&L2_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   std::string output_dir = "ParaView/UW/" + GetTimestamp();
+
+   if (paraview)
+   {
+      if (Mpi::Root()) { WriteParametersToFile(args, output_dir); }
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order
+                         << "_eld_" << eld;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",pgf_r[0]);
+      paraview_dc->RegisterField("E_i",pgf_i[0]);
+      paraview_dc->RegisterField("E_par_r",&E_par_r);
+      paraview_dc->RegisterField("E_par_i",&E_par_i);
+      paraview_dc->RegisterField("H_r",pgf_r[1]);
+      paraview_dc->RegisterField("H_i",pgf_i[1]);      
+      if (eld)
+      {
+         paraview_dc->RegisterField("Jh_1_r",pgf_r[2]);
+         paraview_dc->RegisterField("Jh_1_i",pgf_i[2]);
+         paraview_dc->RegisterField("Jh_2_r",pgf_r[3]);
+         paraview_dc->RegisterField("Jh_2_i",pgf_i[3]);
+      }
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listJ;
+   Array<int> ess_tdof_listJhat;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+   
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+
+      // remove internal boundaries
+      for (int i = 0; i<int_bdr_attr.Size(); i++)
+      {
+         ess_bdr[int_bdr_attr[i]-1] = 0;
+      }
+
+      pfes[2+ndiffusionequations]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += toffsets[2+ndiffusionequations];
+      }
+      // ess_bdr=1;
+      for (int i = 0; i<ndiffusionequations;i++)
+      {
+         ess_tdof_listJ.SetSize(0);
+         ess_tdof_listJhat.SetSize(0);
+         pfes[i+2]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJ);
+         // pfes[i+6]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJhat);
+         for (int j = 0; j < ess_tdof_listJ.Size(); j++)
+         {
+            ess_tdof_listJ[j] += toffsets[i+2];
+         }
+         // for (int j = 0; j < ess_tdof_listJhat.Size(); j++)
+         // {
+         //    ess_tdof_listJhat[j] += toffsets[i+6];
+         // }
+
+         ess_tdof_list.Append(ess_tdof_listJ);
+         // ess_tdof_list.Append(ess_tdof_listJhat);
+      }
+
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+   
+
+   // rotate the vector
+   // (x,y) -> (y,-x)
+   Vector rot_one_x(dim); rot_one_x = 0.0; rot_one_x(1) = -1.0;
+   Vector rot_negone_x(dim); rot_negone_x = 0.0; rot_negone_x(1) = 1.0;
+   VectorConstantCoefficient rot_one_x_cf(rot_one_x);
+   VectorConstantCoefficient rot_negone_x_cf(rot_negone_x);
+
+   pgf_r[2+ndiffusionequations]->ProjectBdrCoefficientNormal(rot_one_x_cf, one_r_bdr);
+   pgf_r[2+ndiffusionequations]->ProjectBdrCoefficientNormal(rot_negone_x_cf, negone_r_bdr);
+   pgf_i[2+ndiffusionequations]->ProjectBdrCoefficientNormal(rot_one_x_cf, one_i_bdr);
+   pgf_i[2+ndiffusionequations]->ProjectBdrCoefficientNormal(rot_negone_x_cf, negone_i_bdr);
+
+   OperatorPtr Ah;
+   Vector X,B;
+   a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+   ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+   BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+
+   Array<ParFiniteElementSpace *> prec_fes;
+   if (static_cond)
+   {
+      a->GetTraceFESpaces(prec_fes);
+   }
+   else
+   {
+      prec_fes = pfes;
+   }
+   Solver * cprec = nullptr;
+
+   if (pmg)
+   {
+#ifdef MFEM_USE_COMPLEX_MUMPS
+      bool mumps_coarse_solver = true;
+#else
+      bool mumps_coarse_solver = false;
+#endif
+      std::vector<Array<int>> ess_bdr_marker(prec_fes.Size());
+      for (int b = 0; b<prec_fes.Size(); b++)
+      {
+         if (pmesh.bdr_attributes.Size())
+         {
+            ess_bdr_marker[b].SetSize(pmesh.bdr_attributes.Max());
+            int ess_block_hatE = (static_cond) ? ndiffusionequations : ndiffusionequations + 2;
+            int ess_block_J1 = (static_cond) ? 0 : 2;
+            int ess_block_J2 = (static_cond) ? 1 : 3;
+            if (b == ess_block_hatE || b == ess_block_J1 || b == ess_block_J2) // hatE, J1, J2
+            {
+               ess_bdr_marker[b] = ess_bdr;
+            }
+            else
+            {
+               ess_bdr_marker[b] = 0;
+            }
+         }
+      }
+      cprec = new ComplexPRefinementMultigrid(prec_fes, ess_bdr_marker, *Ahc, 
+                                              pmg_levels, relax_factor, mumps_coarse_solver);
+   }
+   else
+   {
+      BlockDiagonalPreconditioner * real_prec = new BlockDiagonalPreconditioner(BlockA_r->RowOffsets());
+      real_prec->owns_blocks = 1;
+      for (int i = 0; i<BlockA_r->NumRowBlocks(); i++)
+      {
+         auto prec = MakeFESpaceDefaultSolver(prec_fes[i],0);
+         prec->SetOperator(BlockA_r->GetBlock(i,i));
+         real_prec->SetDiagonalBlock(i,prec);
+      }
+      cprec = new ComplexPreconditioner(real_prec, true);
+   }
+
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-10);
+   cg.SetMaxIter(500);
+   cg.SetPrintLevel(1);
+   cg.SetOperator(*Ahc);
+   cg.SetPreconditioner(*cprec);
+   cg.Mult(B, X);
+
+   a->RecoverFEMSolution(X, x);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i]->MakeRef(pfes[i], x, offsets[i]);
+      pgf_i[i]->MakeRef(pfes[i], x, offsets.Last() + offsets[i]);
+   }
+   
+   ParallelECoefficient par_e_r(pgf_r[0]);
+   ParallelECoefficient par_e_i(pgf_i[0]);
+   E_par_r.ProjectCoefficient(par_e_r);
+   E_par_i.ProjectCoefficient(par_e_i);
+   
+   // rescale the J solutions
+   for (int i = 0; i < ndiffusionequations; ++i)
+   {
+      (*pgf_r[2+i]) /= balance_scale;
+      (*pgf_i[2+i]) /= balance_scale;
+   }
+
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, *pgf_r[0],
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((real_t)0);
+      paraview_dc->Save();
+      delete paraview_dc;
+   }
+
+
+   delete a;
+   for (int i = 0; i < trial_fecols.Size(); ++i)
+   {
+      delete trial_fecols[i];
+      delete pfes[i];
+   }
+   for (int i = 0; i< test_fecols.Size(); ++i)
+   {
+      delete test_fecols[i];
+   }
+
+   return 0;
+}

miniapps/dpg/plasma/lh-eld-fem-symm.cpp

@@ -0,0 +1,522 @@
+//                   MFEM FEM parallel example
+//
+
+// Electron Landau Damping
+
+
+//    ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵᵣ E + i ω²ϵ₀(J₁ + J₂) = 0,   in Ω
+//                   - Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥ = 0,   in Ω     
+//                   - Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥ = 0,   in Ω 
+
+
+// Strong formulation:
+//    ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵᵣ E + 1/2 i ω²ϵ₀(J₁ + J₂) - c₁ P(r)(b ⊗ b) J₃ + c₂ P(r)(b ⊗ b) J₄ = 0,   in Ω
+//                   - Δ∥ J₁ + c₁ J₁ - c₁ P(r) (b ⊗ b) E = 0,   in Ω     
+//                   - Δ∥ J₂ + c₂ J₂ + c₂ P(r) (b ⊗ b) E = 0,   in Ω 
+//                   - Δ∥ J₃ + c₁ J₃ + 1/2 i ω² ϵ₀ E = 0,  in Ω
+//                   - Δ∥ J₄ + c₂ J₄ + 1/2 i ω² ϵ₀ E = 0,  in Ω
+//
+//                                           E×n = E₀,  on ∂Ω
+//                                           J₁  = 0,  on ∂Ω
+//                                           J₂  = 0,  on ∂Ω
+// weak formulation:
+//   Find E ∈ H(curl,Ω), J₁  ∈ H¹(Ω), J₂  ∈ H¹(Ω),  J₃ ∈ H¹(Ω), J₄  ∈ H¹(Ω) such that
+//   (1/μ₀ ∇×E, ∇ × F) - ω² ϵ₀ (ϵᵣ E, F) + i ω²ϵ₀(J₁ + J₂, F) - (c₁ P(r)(b ⊗ b) J₃, F) + (c₂ P(r)(b ⊗ b) J₄, F) = 0,  ∀ F ∈ H(curl,Ω)
+//        ( (b⋅∇)J₁ , (b⋅∇) G) + c₁ (J₁ , G) - c₁ (P(r) (b ⊗ b) E, G) = 0,  ∀ G ∈ (H¹(Ω))ᵈ
+//        ( (b⋅∇)J₂ , (b⋅∇) H) + c₂ (J₂ , H) + c₂ (P(r) (b ⊗ b) E, H) = 0,  ∀ H ∈ (H¹(Ω))ᵈ
+//        ( (b⋅∇)J₃ , (b⋅∇) I) + c₁ (J₃ , I) - i ω² ϵ₀ (E, I) = 0,  ∀ I ∈ (H¹(Ω))ᵈ
+//        ( (b⋅∇)J₄ , (b⋅∇) K) + c₂ (J₄ , K) - i ω² ϵ₀ (E, K) = 0,  ∀ K ∈ (H¹(Ω))ᵈ
+
+//     |             E             |                J₁             |               J₂             |         J₃        |         J₄      | 
+// -----------------------------------------------------------------------------------------------------------
+// δE  |(1/μ₀∇×E,∇×δE)-ω²ϵ₀(ϵᵣE,δE)|          i ω²ϵ₀(J₁,δE)        |        i ω²ϵ₀(J₂,δE)         |-(c₁P(r)(b⊗b)J₃,δE)|c₂P(r)(b⊗b)J₄,δE)|    
+// δJ₁ |                           | ((b⋅∇)J₁,(b⋅∇)δJ₁)+c₁(J₁,δJ₁) |                              |                   |    |       |
+// δJ₂ |                           |                               |((b⋅∇)J₂,(b⋅∇)δJ₂)+ c₂(J₂,δJ₂)|       |       |  
+// δJ₃ |           |        |       |       |       |  
+// δJ₄ |           |        |       |       |       |  
+
+
+
+#include "mfem.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../util/pcomplexblockform.hpp"
+#include "../util/blockcomplexhypremat.hpp"
+#include "../util/utils.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "utils/lh_utils.hpp"
+#include "../../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+#include <cstring>
+#include <filesystem>
+
+using namespace std;
+using namespace mfem;
+
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+
+   int order = 2;
+   int par_ref_levels = 0;
+   int ser_ref_levels = 0;
+   bool visualization = false;
+   // real_t rnum=4.6e9;
+   // real_t mu = 1.257e-6;
+   // real_t eps0 = 8.8541878128e-12*factor;
+   real_t rnum=1.5e9;
+   real_t mu = 1.257e-6;
+   real_t eps0 = 8.8541878128e-12;
+   bool eld = true; // enable/disable electron Landau damping 
+
+   bool paraview = false;
+   bool debug = false;
+   bool mumps_solver = true;
+   real_t delta_prec = 0.01;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&ser_ref_levels, "-sr", "--serial-refinement_levels",
+                  "Number of serial refinement levels.");                  
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&a0, "-a0", "--a0", "P(r) first parameter.");
+   args.AddOption(&a1, "-a1", "--a1", "P(r) second parameter.");
+   args.AddOption(&delta_prec, "-dp", "--delta-prec", "stability parameter for the preconditioner.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&eld, "-eld", "--eld", "-no-eld",
+                  "--no-eld",
+                  "Enable or disable electron Landau damping.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.AddOption(&mumps_solver, "-mumps", "--mumps", "-no-mumps",
+                  "--no-mumps",
+                  "Enable or disable MUMPS solver.");
+   args.AddOption(&debug, "-debug", "--debug", "-no-debug",
+                  "--no-debug",
+                  "Enable or disable debug mode (delta = 0.01 and no coupling).");                  
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   
+   // number of diffusion equations
+   int ndiffusionequations = (eld) ?  2 : 0; 
+
+   Vector cvals(ndiffusionequations);
+   Vector csigns(ndiffusionequations);
+   // real_t cfactor = 1e-6;
+   real_t cfactor = 1.0;
+   if (eld)
+   {
+      cvals(0)  = 25e6;  cvals(1)  = 1e6;
+      csigns(0) = -1.0;  csigns(1) = 1.0;
+   }
+   cvals *= cfactor; // scale the coefficients
+   
+   real_t omega = 2.*M_PI*rnum;
+   if (eld && !debug) 
+   {
+      delta = 0.0; // disable delta if electron Landau damping is enabled
+      if (Mpi::Root())
+      {
+         cout << "Electron Landau damping enabled, delta set to 0.0." << endl;
+      }
+   }   
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   for (int i = 0; i < ser_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   mesh.RemoveInternalBoundaries();
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+   
+   for (int i = 0; i<par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }
+
+   // Define the coefficients 
+   ConstantCoefficient muinv(1./mu);
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_vcf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   // S(r) 
+   FunctionCoefficient S_cf_r(sfunc_r), S_cf_i(sfunc_i);
+   // P(r) 
+   FunctionCoefficient P_cf_r(pfunc_r), P_cf_i(pfunc_i); 
+
+   VectorFunctionCoefficient b_cf(dim,bfunc);// b
+   ScalarVectorProductCoefficient scaledb_cf(sqrt(cfactor), b_cf); 
+   MatrixFunctionCoefficient bb_cf(dim,bcrossb); // b⊗b
+
+   MatrixSumCoefficient oneminusbb(Mone_cf, bb_cf, 1.0, -1.0); // 1 - b⊗b
+
+   // S(r) (I - b⊗b)
+   ScalarMatrixProductCoefficient Soneminusbb_r(S_cf_r, oneminusbb), Soneminusbb_i(S_cf_i, oneminusbb); 
+
+   // P(r) b⊗b 
+   ScalarMatrixProductCoefficient P_cf_bb_r(P_cf_r, bb_cf), P_cf_bb_i(P_cf_i, bb_cf); 
+
+   // εᵣ = S(r) (I - b⊗b) + P(r) b⊗b 
+   MatrixSumCoefficient eps_r(Soneminusbb_r, P_cf_bb_r, 1.0, 1.0); 
+   MatrixSumCoefficient eps_i(Soneminusbb_i, P_cf_bb_i, 1.0, 1.0); 
+
+   coefs_r[nattr-1] = &eps_r;
+   coefs_i[nattr-1] = &eps_i;
+
+   PWMatrixCoefficient eps_cf_r(dim, attr, coefs_r);
+   PWMatrixCoefficient eps_cf_i(dim, attr, coefs_i);
+
+   real_t scale = (debug) ? 0.0 : 1.0;
+   ConstantCoefficient eps0omeg2(eps0 * omega * omega * scale);
+   ConstantCoefficient negeps0omeg2(-eps0 * omega * omega);
+
+   ScalarMatrixProductCoefficient m_cf_r(negeps0omeg2, eps_cf_r);
+   ScalarMatrixProductCoefficient m_cf_i(negeps0omeg2, eps_cf_i);
+
+   // if ELD
+   Array<Vector *> c_arrays(ndiffusionequations);
+   Array<PWConstCoefficient *> pw_c_coeffs(ndiffusionequations);
+   Array<MatrixCoefficient *> cPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> cPibb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPibb_cf(ndiffusionequations);
+   Vector temp(nattr); temp=0.0;
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      temp[nattr-1] = cvals(i);
+      pw_c_coeffs[i] = new PWConstCoefficient(temp);
+      cPrbb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_r);
+      cPibb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_i);
+      signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPrbb_cf[i]);
+      signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPibb_cf[i]);
+   }
+
+   Array<FiniteElementCollection *> fecols;
+   Array<ParFiniteElementSpace *> pfes;
+   fecols.Append(new ND_FECollection(order, dim));
+   pfes.Append(new ParFiniteElementSpace(&pmesh, fecols[0]));
+   if (eld)
+   {
+      for (int i = 0; i < 2*ndiffusionequations; ++i)
+      {
+         fecols.Append(new H1_FECollection(order, dim));
+         pfes.Append(new ParFiniteElementSpace(&pmesh, fecols[i+1], dim));
+      }
+   }
+
+   Array<HYPRE_BigInt> tdofs(pfes.Size());
+   for (int i = 0; i < pfes.Size(); ++i)
+   {
+      tdofs[i] = pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << "ParFiniteElementSpace " << i << " has " << tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   if (Mpi::Root())
+   {
+      cout << "Total number of true dofs: " << tdofs.Sum() << endl;
+   }
+
+   ParComplexBlockForm *a = new ParComplexBlockForm(pfes);
+   // (1/μ₀ ∇×E, ∇ × F)
+   a->AddDomainIntegrator(new CurlCurlIntegrator(muinv), nullptr, 0, 0);
+   // - ω² ϵ₀ (ϵᵣ E, F)
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(m_cf_r),
+                          new VectorFEMassIntegrator(m_cf_i), 0, 0);
+   ConstantCoefficient halfeps0omeg2(0.5*eps0 * omega * omega);    
+   ConstantCoefficient neghalfeps0omeg2(-0.5*eps0 * omega * omega);                   
+   if (eld)
+   {
+      for (int i = 0; i<ndiffusionequations; i++)
+      {
+         //  1/2 i ω²ϵ₀((J₁+J₂),F)
+         a->AddDomainIntegrator(nullptr, new TransposeIntegrator(new VectorFEMassIntegrator(halfeps0omeg2)), i+1, 0);
+         //  ( (b⋅∇)J₁ , (b⋅∇) G)
+         a->AddDomainIntegrator(new DirectionalVectorDiffusionIntegrator(scaledb_cf), nullptr, i+1, i+1);
+         // cᵢ (J₁ , G)
+         a->AddDomainIntegrator(new VectorMassIntegrator(*pw_c_coeffs[i]), nullptr, i+1, i+1);
+         // ±cᵢ(P(r) (b ⊗ b) E, G)
+         a->AddDomainIntegrator(new VectorFEMassIntegrator(*signedcPrbb_cf[i]), 
+                                new VectorFEMassIntegrator(*signedcPibb_cf[i]), 0, i+1);
+         // 
+         // ±(cᵢ P(r)(b ⊗ b) Jᵢ₊₂,F)
+         a->AddDomainIntegrator(new TransposeIntegrator(
+                                new VectorFEMassIntegrator(*signedcPrbb_cf[i])), 
+                                new TransposeIntegrator(
+                                new VectorFEMassIntegrator(*signedcPibb_cf[i])), i+3, 0);
+      
+      // ( (b⋅∇)J₃ , (b⋅∇) I) + c₁ (J₃ , I) - i ω² ϵ₀ (E, I) = 0,  ∀ I ∈ (H¹(Ω))ᵈ
+      // ( (b⋅∇)J₄ , (b⋅∇) K) + c₂ (J₄ , K) - i ω² ϵ₀ (E, K) = 0,  ∀ K ∈ (H¹(Ω))ᵈ
+         a->AddDomainIntegrator(new DirectionalVectorDiffusionIntegrator(scaledb_cf), nullptr, i+3, i+3);
+         a->AddDomainIntegrator(new VectorMassIntegrator(*pw_c_coeffs[i]), nullptr, i+3, i+3);
+
+         // + i ω² ϵ₀ (E, I)                        
+         a->AddDomainIntegrator(nullptr, new VectorFEMassIntegrator(halfeps0omeg2), 0, i+3);                         
+                                 
+      }
+   }
+
+   a->Assemble();
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      delete pw_c_coeffs[i];
+      delete cPrbb_cf[i]; 
+      delete cPibb_cf[i]; 
+      delete signedcPrbb_cf[i]; 
+      delete signedcPibb_cf[i]; 
+   }
+
+   socketstream E_out_r;
+
+   int npfes = pfes.Size();
+   Array<int> offsets(npfes+1);  offsets[0] = 0;
+   Array<int> toffsets(npfes+1); toffsets[0] = 0;
+   for (int i = 0; i<npfes; i++)
+   {
+      offsets[i+1] = pfes[i]->GetVSize();
+      toffsets[i+1] = pfes[i]->TrueVSize();
+   }
+   offsets.PartialSum();
+   toffsets.PartialSum();
+
+   Vector x(2*offsets.Last());
+   x = 0.;
+
+   Array<ParGridFunction *> pgf_r(npfes);
+   Array<ParGridFunction *> pgf_i(npfes);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i] = new ParGridFunction(pfes[i], x, offsets[i]);
+      pgf_i[i] = new ParGridFunction(pfes[i], x, offsets.Last() + offsets[i]);
+   }
+
+   ParComplexGridFunction maxwell_pgf(pfes[0]); maxwell_pgf = 0.0;
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+   ParGridFunction E_par_r(&L2_fes);
+   ParGridFunction E_par_i(&L2_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   std::string output_dir = "ParaView/FEM/" + GetTimestamp();
+
+   if (paraview)
+   {
+      if (Mpi::Root()) { WriteParametersToFile(args, output_dir); }
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",pgf_r[0]);
+      paraview_dc->RegisterField("E_i",pgf_i[0]);
+      paraview_dc->RegisterField("MaxwellE_r",&maxwell_pgf.real());
+      paraview_dc->RegisterField("MaxwellE_i",&maxwell_pgf.imag());
+      paraview_dc->RegisterField("E_par_r",&E_par_r);
+      paraview_dc->RegisterField("E_par_i",&E_par_i);
+      if (eld)
+      {
+         paraview_dc->RegisterField("Jh_1_r",pgf_r[1]);
+         paraview_dc->RegisterField("Jh_1_i",pgf_i[1]);
+         paraview_dc->RegisterField("Jh_2_r",pgf_r[2]);
+         paraview_dc->RegisterField("Jh_2_i",pgf_i[2]);
+         paraview_dc->RegisterField("Jh_3_r",pgf_r[3]);
+         paraview_dc->RegisterField("Jh_3_i",pgf_i[3]);
+         paraview_dc->RegisterField("Jh_4_r",pgf_r[4]);
+         paraview_dc->RegisterField("Jh_4_i",pgf_i[4]);
+      }
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listJ;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+
+      pfes[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      for (int i = 0; i<2*ndiffusionequations;i++)
+      {
+         ess_tdof_listJ.SetSize(0);
+         pfes[i+1]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJ);
+         for (int j = 0; j < ess_tdof_listJ.Size(); j++)
+         {
+            ess_tdof_listJ[j] += toffsets[i+1];
+         }
+         ess_tdof_list.Append(ess_tdof_listJ);
+      }
+
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+
+
+   pgf_r[0]->ProjectBdrCoefficientTangent(one_x_cf, one_r_bdr);
+   pgf_r[0]->ProjectBdrCoefficientTangent(negone_x_cf, negone_r_bdr);
+   pgf_i[0]->ProjectBdrCoefficientTangent(one_x_cf, one_i_bdr);
+   pgf_i[0]->ProjectBdrCoefficientTangent(negone_x_cf, negone_i_bdr);
+
+   OperatorPtr Ah;
+   Vector B, X;
+
+   Vector b(x.Size()); b = 0.0;
+
+   a->FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);
+   ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+
+   BlockOperator * Block_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+   BlockOperator * Block_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+   int nblocks = Block_r->NumRowBlocks();
+   Array2D<const HypreParMatrix*> r_matrices(nblocks, nblocks);
+   Array2D<const HypreParMatrix*> i_matrices(nblocks, nblocks);
+   for (int i = 0; i < nblocks; i++)
+   {
+      for (int j = 0; j < nblocks; j++)
+      {
+         r_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&Block_r->GetBlock(i,j));
+         i_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&Block_i->GetBlock(i,j));
+      }
+   }
+   HypreParMatrix * hr = HypreParMatrixFromBlocks(r_matrices);
+   HypreParMatrix * hi = HypreParMatrixFromBlocks(i_matrices);
+
+   ComplexHypreParMatrix * hc_hypre =
+      new ComplexHypreParMatrix(hr, hi,false, false);
+
+
+   auto P = new ComplexMUMPSSolver(MPI_COMM_WORLD);
+   P->SetPrintLevel(1);
+   P->SetOperator(*hc_hypre);
+   P->Mult(B, X);
+
+   a->RecoverFEMSolution(X, x);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i]->MakeRef(pfes[i], x, offsets[i]);
+      pgf_i[i]->MakeRef(pfes[i], x, offsets.Last() + offsets[i]);
+   }
+
+   ParallelECoefficient par_e_r(pgf_r[0]);
+   ParallelECoefficient par_e_i(pgf_i[0]);
+   E_par_r.ProjectCoefficient(par_e_r);
+   E_par_i.ProjectCoefficient(par_e_i);
+
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, *pgf_r[0],
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   }
+
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((real_t)0);
+      paraview_dc->Save();
+      delete paraview_dc;
+   }
+
+   delete a;
+   for (int i = 0; i < fecols.Size(); ++i)
+   {
+      delete fecols[i];
+      delete pfes[i];
+      delete pgf_r[i];
+      delete pgf_i[i];
+   }
+
+   return 0;
+
+}
+
+

miniapps/dpg/plasma/lh-eld-fem.cpp

@@ -0,0 +1,705 @@
+//                   MFEM FEM parallel example
+//
+
+// Electron Landau Damping
+// Strong formulation:
+//    ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵᵣ E + i ω²ϵ₀(J₁ + J₂) = 0,   in Ω
+//                   - Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥ = 0,   in Ω     
+//                   - Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥ = 0,   in Ω 
+//                                           E×n = E₀,  on ∂Ω
+//                                           J₁  = 0,  on ∂Ω
+//                                           J₂  = 0,  on ∂Ω
+// weak formulation:
+//   Find E ∈ H(curl,Ω), J₁  ∈ H¹(Ω), J₂  ∈ H¹(Ω) such that
+//   (1/μ₀ ∇×E, ∇ × F) - ω² ϵ₀ (ϵᵣ E, F) + i ω²ϵ₀(J₁ + J₂, F) = 0,  ∀ F ∈ H(curl,Ω)
+//        ( (b⋅∇)J₁ , (b⋅∇) G) + c₁ (J₁ , G) - c₁ (P(r) (b ⊗ b) E, G) = 0,  ∀ G ∈ (H¹(Ω))ᵈ 
+//        ( (b⋅∇)J₂ , (b⋅∇) H) + c₂ (J₂ , H) + c₂ (P(r) (b ⊗ b) E, H) = 0,  ∀ H ∈ (H¹(Ω))ᵈ
+
+// mpirun -np 8 ./lh-eld-fem -o 4 -paraview -eld -m data/quad.msh 
+
+#include "mfem.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../util/pcomplexblockform.hpp"
+#include "../util/blockcomplexhypremat.hpp"
+#include "../util/utils.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "utils/lh_utils.hpp"
+#include "../../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+#include <cstring>
+#include <filesystem>
+
+
+#ifndef MFEM_USE_COMPLEX_MUMPS
+MFEM_ABORT("This example requires MFEM to be built with ComplexMUMPS.");
+#endif
+
+
+using namespace std;
+using namespace mfem;
+
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+
+   int order = 2;
+   int par_ref_levels = 0;
+   int ser_ref_levels = 0;
+   bool visualization = false;
+   // real_t rnum=4.6e9;
+   // real_t mu = 1.257e-6;
+   // real_t eps0 = 8.8541878128e-12*factor;
+   real_t rnum=1.5e9;
+   real_t mu = 1.257e-6;
+   real_t eps0 = 8.8541878128e-12;
+   bool eld = true; // enable/disable electron Landau damping 
+
+   bool paraview = false;
+   bool debug = false;
+   bool monolithic_precond = true;
+   bool direct_solve = false;
+   bool triangular_precond = false;
+   bool use_amg = false;
+   real_t delta_prec = 0.01;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&ser_ref_levels, "-sr", "--serial-refinement_levels",
+                  "Number of serial refinement levels.");                  
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&a0, "-a0", "--a0", "P(r) first parameter.");
+   args.AddOption(&a1, "-a1", "--a1", "P(r) second parameter.");
+   args.AddOption(&delta_prec, "-dp", "--delta-prec", "stability parameter for the preconditioner.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&eld, "-eld", "--eld", "-no-eld",
+                  "--no-eld",
+                  "Enable or disable electron Landau damping.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.AddOption(&direct_solve, "-direct", "--direct", "-no-direct",
+                  "--no-direct",
+                  "Enable or disable direct monolithic solver.");
+   args.AddOption(&monolithic_precond, "-monolithic", "--monolithic", "-no-monolithic",
+                  "--no-monolithic",
+                  "Enable or disable monolithic preconditioner.");
+   args.AddOption(&triangular_precond, "-triangular", "--triangular", "-no-triangular",
+                  "--no-triangular",
+                  "Enable or disable lower triangular preconditioner.");
+   args.AddOption(&use_amg, "-amg", "--amg", "-no-amg",
+                  "--no-amg",
+                  "Enable or disable AMG.");
+   args.AddOption(&debug, "-debug", "--debug", "-no-debug",
+                  "--no-debug",
+                  "Enable or disable debug mode (delta = 0.01 and no coupling).");                  
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   
+   // number of diffusion equations
+   int ndiffusionequations = (eld) ?  2 : 0; 
+
+   Vector cvals(ndiffusionequations);
+   Vector csigns(ndiffusionequations);
+   real_t cfactor = 1e-6;
+   if (eld)
+   {
+      cvals(0)  = 25e6;  cvals(1)  = 1e6;
+      csigns(0) = -1.0;  csigns(1) = 1.0;
+   }
+   cvals *= cfactor; // scale the coefficients
+   
+   real_t omega = 2.*M_PI*rnum;
+   if (eld && !debug) 
+   {
+      delta = 0.0; // disable delta if electron Landau damping is enabled
+      if (Mpi::Root())
+      {
+         cout << "Electron Landau damping enabled, delta set to 0.0." << endl;
+      }
+   }   
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   for (int i = 0; i < ser_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   mesh.RemoveInternalBoundaries();
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+   
+   for (int i = 0; i<par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }
+
+   // Define the coefficients 
+   ConstantCoefficient muinv(1./mu);
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_vcf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   // S(r) 
+   FunctionCoefficient S_cf_r(sfunc_r), S_cf_i(sfunc_i);
+   // P(r) 
+   FunctionCoefficient P_cf_r(pfunc_r), P_cf_i(pfunc_i); 
+
+   VectorFunctionCoefficient b_cf(dim,bfunc);// b
+   ScalarVectorProductCoefficient scaledb_cf(sqrt(cfactor), b_cf); 
+   MatrixFunctionCoefficient bb_cf(dim,bcrossb); // b⊗b
+
+   MatrixSumCoefficient oneminusbb(Mone_cf, bb_cf, 1.0, -1.0); // 1 - b⊗b
+
+   // S(r) (I - b⊗b)
+   ScalarMatrixProductCoefficient Soneminusbb_r(S_cf_r, oneminusbb), Soneminusbb_i(S_cf_i, oneminusbb); 
+
+   // P(r) b⊗b 
+   ScalarMatrixProductCoefficient P_cf_bb_r(P_cf_r, bb_cf), P_cf_bb_i(P_cf_i, bb_cf); 
+
+   // εᵣ = S(r) (I - b⊗b) + P(r) b⊗b 
+   MatrixSumCoefficient eps_r(Soneminusbb_r, P_cf_bb_r, 1.0, 1.0); 
+   MatrixSumCoefficient eps_i(Soneminusbb_i, P_cf_bb_i, 1.0, 1.0); 
+
+   coefs_r[nattr-1] = &eps_r;
+   coefs_i[nattr-1] = &eps_i;
+
+   PWMatrixCoefficient eps_cf_r(dim, attr, coefs_r);
+   PWMatrixCoefficient eps_cf_i(dim, attr, coefs_i);
+
+   real_t scale = (debug) ? 0.0 : 1.0;
+   ConstantCoefficient eps0omeg2(eps0 * omega * omega * scale);
+   ConstantCoefficient negeps0omeg2(-eps0 * omega * omega);
+
+   ScalarMatrixProductCoefficient m_cf_r(negeps0omeg2, eps_cf_r);
+   ScalarMatrixProductCoefficient m_cf_i(negeps0omeg2, eps_cf_i);
+
+   // if ELD
+   Array<Vector *> c_arrays(ndiffusionequations);
+   Array<PWConstCoefficient *> pw_c_coeffs(ndiffusionequations);
+   Array<MatrixCoefficient *> cPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> cPibb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPibb_cf(ndiffusionequations);
+   Vector temp(nattr); temp=0.0;
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      temp[nattr-1] = cvals(i);
+      pw_c_coeffs[i] = new PWConstCoefficient(temp);
+      cPrbb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_r);
+      cPibb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_i);
+      signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPrbb_cf[i]);
+      signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPibb_cf[i]);
+   }
+
+   Array<FiniteElementCollection *> fecols;
+   Array<ParFiniteElementSpace *> pfes;
+   fecols.Append(new ND_FECollection(order, dim));
+   pfes.Append(new ParFiniteElementSpace(&pmesh, fecols[0]));
+   if (eld)
+   {
+      for (int i = 0; i < ndiffusionequations; ++i)
+      {
+         fecols.Append(new H1_FECollection(order, dim));
+         pfes.Append(new ParFiniteElementSpace(&pmesh, fecols[i+1], dim));
+      }
+   }
+
+   Array<HYPRE_BigInt> tdofs(pfes.Size());
+   for (int i = 0; i < pfes.Size(); ++i)
+   {
+      tdofs[i] = pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << "ParFiniteElementSpace " << i << " has " << tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   if (Mpi::Root())
+   {
+      cout << "Total number of true dofs: " << tdofs.Sum() << endl;
+   }
+
+   ParComplexBlockForm *a = new ParComplexBlockForm(pfes);
+   // (1/μ₀ ∇×E, ∇ × F)
+   a->AddDomainIntegrator(new CurlCurlIntegrator(muinv), nullptr, 0, 0);
+   // - ω² ϵ₀ (ϵᵣ E, F)
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(m_cf_r),
+                          new VectorFEMassIntegrator(m_cf_i), 0, 0);
+   if (eld)
+   {
+      for (int i = 0; i<ndiffusionequations; i++)
+      {
+         //  i ω²ϵ₀((J₁+J₂),F)
+         a->AddDomainIntegrator(nullptr, new TransposeIntegrator(new VectorFEMassIntegrator(eps0omeg2)), i+1, 0);
+         //  ( (b⋅∇)J₁ , (b⋅∇) G)
+         a->AddDomainIntegrator(new DirectionalVectorDiffusionIntegrator(scaledb_cf), nullptr, i+1, i+1);
+         // cᵢ (J₁ , G)
+         a->AddDomainIntegrator(new VectorMassIntegrator(*pw_c_coeffs[i]), nullptr, i+1, i+1);
+         // ±cᵢ(P(r) (b ⊗ b) E, G)
+         a->AddDomainIntegrator(new VectorFEMassIntegrator(*signedcPrbb_cf[i]), new VectorFEMassIntegrator(*signedcPibb_cf[i]), 0, i+1);
+      }
+   }
+
+   a->Assemble();
+
+   socketstream E_out_r;
+
+   int npfes = pfes.Size();
+   Array<int> offsets(npfes+1);  offsets[0] = 0;
+   Array<int> toffsets(npfes+1); toffsets[0] = 0;
+   for (int i = 0; i<npfes; i++)
+   {
+      offsets[i+1] = pfes[i]->GetVSize();
+      toffsets[i+1] = pfes[i]->TrueVSize();
+   }
+   offsets.PartialSum();
+   toffsets.PartialSum();
+
+   Vector x(2*offsets.Last());
+   x = 0.;
+
+   Array<ParGridFunction *> pgf_r(npfes);
+   Array<ParGridFunction *> pgf_i(npfes);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i] = new ParGridFunction(pfes[i], x, offsets[i]);
+      pgf_i[i] = new ParGridFunction(pfes[i], x, offsets.Last() + offsets[i]);
+   }
+
+   ParComplexGridFunction maxwell_pgf(pfes[0]); maxwell_pgf = 0.0;
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+   ParGridFunction E_par_r(&L2_fes);
+   ParGridFunction E_par_i(&L2_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   std::string output_dir = "ParaView/FEM/" + GetTimestamp();
+
+   if (paraview)
+   {
+      if (Mpi::Root()) { WriteParametersToFile(args, output_dir); }
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",pgf_r[0]);
+      paraview_dc->RegisterField("E_i",pgf_i[0]);
+      paraview_dc->RegisterField("MaxwellE_r",&maxwell_pgf.real());
+      paraview_dc->RegisterField("MaxwellE_i",&maxwell_pgf.imag());
+      paraview_dc->RegisterField("E_par_r",&E_par_r);
+      paraview_dc->RegisterField("E_par_i",&E_par_i);
+      if (eld)
+      {
+         paraview_dc->RegisterField("Jh_1_r",pgf_r[1]);
+         paraview_dc->RegisterField("Jh_1_i",pgf_i[1]);
+         paraview_dc->RegisterField("Jh_2_r",pgf_r[2]);
+         paraview_dc->RegisterField("Jh_2_i",pgf_i[2]);
+      }
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listJ;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+
+      pfes[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      for (int i = 0; i<ndiffusionequations;i++)
+      {
+         ess_tdof_listJ.SetSize(0);
+         pfes[i+1]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJ);
+         for (int j = 0; j < ess_tdof_listJ.Size(); j++)
+         {
+            ess_tdof_listJ[j] += toffsets[i+1];
+         }
+         ess_tdof_list.Append(ess_tdof_listJ);
+      }
+
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+
+
+   pgf_r[0]->ProjectBdrCoefficientTangent(one_x_cf, one_r_bdr);
+   pgf_r[0]->ProjectBdrCoefficientTangent(negone_x_cf, negone_r_bdr);
+   pgf_i[0]->ProjectBdrCoefficientTangent(one_x_cf, one_i_bdr);
+   pgf_i[0]->ProjectBdrCoefficientTangent(negone_x_cf, negone_i_bdr);
+
+   OperatorPtr Ah;
+   Vector B, X;
+
+   Vector b(x.Size()); b = 0.0;
+
+   a->FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);
+   ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+
+   if (direct_solve)
+   {
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+      int nblocks = BlockA_r->NumRowBlocks();
+      Array2D<const HypreParMatrix*> A_r_matrices(nblocks, nblocks);
+      Array2D<const HypreParMatrix*> A_i_matrices(nblocks, nblocks);
+      for (int i = 0; i < nblocks; i++)
+      {
+         for (int j = 0; j < nblocks; j++)
+         {
+            A_r_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockA_r->GetBlock(i,j));
+            A_i_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockA_i->GetBlock(i,j));
+         }
+      }
+      HypreParMatrix * Ahr = HypreParMatrixFromBlocks(A_r_matrices);
+      HypreParMatrix * Ahi = HypreParMatrixFromBlocks(A_i_matrices);
+
+      ComplexHypreParMatrix * Ahc_hypre =
+         new ComplexHypreParMatrix(Ahr, Ahi,true, true);
+
+      ComplexMUMPSSolver cmumps(MPI_COMM_WORLD);
+      cmumps.SetPrintLevel(0);
+      cmumps.SetOperator(*Ahc_hypre);
+      cmumps.Mult(B,X);
+      delete Ahc_hypre;   
+   }
+   else 
+   {
+      // Set up the preconditioner
+      delta = delta_prec;
+      ParComplexBlockForm a_prec(pfes);
+      // (1/μ₀ ∇×E, ∇ × F)
+      a_prec.AddDomainIntegrator(new CurlCurlIntegrator(muinv), nullptr, 0, 0);
+      // - ω² ϵ₀ (ϵᵣ E, F)
+      a_prec.AddDomainIntegrator(new VectorFEMassIntegrator(m_cf_r),
+                                new VectorFEMassIntegrator(m_cf_i), 0, 0);
+      if (eld)
+      {
+         for (int i = 0; i<ndiffusionequations; i++)
+         {
+            //  ( (b⋅∇)J₁ , (b⋅∇) G)
+            a_prec.AddDomainIntegrator(new DirectionalVectorDiffusionIntegrator(scaledb_cf), nullptr, i+1, i+1);
+            // cᵢ (J₁ , G)
+            a_prec.AddDomainIntegrator(new VectorMassIntegrator(*pw_c_coeffs[i]), nullptr, i+1, i+1);
+            // ±cᵢ(P(r) (b ⊗ b) E, G)
+            a_prec.AddDomainIntegrator(new VectorFEMassIntegrator(*signedcPrbb_cf[i]), new VectorFEMassIntegrator(*signedcPibb_cf[i]), 0, i+1);
+         }
+      }
+      a_prec.Assemble();
+
+      OperatorPtr Ahprec;
+      a_prec.FormSystemMatrix(ess_tdof_list, Ahprec);
+      ComplexOperator * Ahcprec = Ahprec.As<ComplexOperator>();
+
+      GMRESSolver gmres(MPI_COMM_WORLD);
+      gmres.SetRelTol(1e-10);
+      gmres.SetMaxIter(2000);
+      gmres.SetPrintLevel(1);
+
+      Array<int> blk(3);
+      if (monolithic_precond)
+      {
+         BlockOperator * BlockPrec_r = dynamic_cast<BlockOperator *>(&Ahcprec->real());
+         BlockOperator * BlockPrec_i = dynamic_cast<BlockOperator *>(&Ahcprec->imag());
+
+         int nblocks = BlockPrec_r->NumRowBlocks();
+         Array2D<const HypreParMatrix*> prec_r_matrices(nblocks, nblocks);
+         Array2D<const HypreParMatrix*> prec_i_matrices(nblocks, nblocks);
+         for (int i = 0; i < nblocks; i++)
+         {
+            for (int j = 0; j < nblocks; j++)
+            {
+               prec_r_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockPrec_r->GetBlock(i,j));
+               prec_i_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockPrec_i->GetBlock(i,j));
+            }
+         }
+         HypreParMatrix * Prechr = HypreParMatrixFromBlocks(prec_r_matrices);
+         HypreParMatrix * Prechi = HypreParMatrixFromBlocks(prec_i_matrices);
+
+         ComplexHypreParMatrix * Prechc_hypre =
+            new ComplexHypreParMatrix(Prechr, Prechi,true, true);
+
+         ComplexMUMPSSolver cmumps(MPI_COMM_WORLD);
+         cmumps.SetPrintLevel(0);
+         cmumps.SetOperator(*Prechc_hypre);
+
+         gmres.SetOperator(*Ahc);
+         gmres.SetPreconditioner(cmumps);
+         gmres.Mult(B, X);
+         delete Prechc_hypre;
+      }
+      else   
+      {
+         ComplexBlockOperator Ac(*Ahc);
+         ComplexBlockOperator Acprec(*Ahcprec);
+
+         Vector Xc(X.Size()); Xc = 0.0;
+         Vector Bc(B.Size());
+         Ac.BlockComplexToComplexBlock(B, Bc);
+         
+         Solver * Mc = nullptr;   
+         int nblocks = Ac.NumRowBlocks();
+
+         Array<Solver *> diag_solvers(nblocks);
+         diag_solvers[0] = new ComplexMUMPSSolver(MPI_COMM_WORLD);
+         dynamic_cast<ComplexMUMPSSolver*>(diag_solvers[0])->SetPrintLevel(0);
+         diag_solvers[0]->SetOperator(Acprec.GetBlock(0,0));
+
+         for (int i = 1; i < nblocks; ++i)
+         {
+            if (use_amg)
+            {
+               HypreBoomerAMG * amg1 = new HypreBoomerAMG();
+               HypreBoomerAMG * amg2 = new HypreBoomerAMG();
+               amg1->SetPrintLevel(0); amg2->SetPrintLevel(0);
+               amg1->SetSystemsOptions(dim); amg2->SetSystemsOptions(dim);
+               amg1->SetRelaxType(88); amg2->SetRelaxType(88);
+               auto op = dynamic_cast<ComplexHypreParMatrix*>(&Acprec.GetBlock(i,i));
+               blk[0]=0;
+               blk[1]=op->real().Height();
+               blk[2]=op->real().Height();
+               blk.PartialSum();
+               BlockDiagonalPreconditioner * bprec = new BlockDiagonalPreconditioner(blk);
+               bprec->owns_blocks=1;
+               amg1->SetOperator(op->real());
+               amg2->SetOperator(op->real());
+               bprec->SetDiagonalBlock(0, amg1);
+               bprec->SetDiagonalBlock(1, amg2);
+               diag_solvers[i] = bprec;
+            }
+            else
+            {
+               diag_solvers[i] = new ComplexMUMPSSolver(MPI_COMM_WORLD);
+               dynamic_cast<ComplexMUMPSSolver*>(diag_solvers[i])->SetPrintLevel(0);
+               diag_solvers[i]->SetOperator(Acprec.GetBlock(i,i));
+            }  
+         }
+
+         if (triangular_precond)
+         {
+            Mc = new BlockLowerTriangularPreconditioner(Ac.RowOffsets());
+            auto Mclt = dynamic_cast<BlockLowerTriangularPreconditioner*>(Mc);
+            Mclt->SetBlock(1,0, &Acprec.GetBlock(1,0));
+            Mclt->SetBlock(2,0, &Acprec.GetBlock(2,0));
+            for (int i = 0; i < nblocks; ++i) { Mclt->SetDiagonalBlock(i, diag_solvers[i]); }
+         }
+         else
+         {
+            Mc = new BlockDiagonalPreconditioner(Ac.RowOffsets());
+            auto Mcdiag = dynamic_cast<BlockDiagonalPreconditioner*>(Mc);
+            for (int i = 0; i < nblocks; ++i) { Mcdiag->SetDiagonalBlock(i, diag_solvers[i]); }
+         }   
+            
+         gmres.SetPreconditioner(*Mc);
+         gmres.SetOperator(Ac);
+         gmres.Mult(Bc, Xc);
+
+
+         for (int i = 0; i < nblocks; ++i)
+         {
+            delete diag_solvers[i];
+         }
+         delete Mc;
+
+         Ac.ComplexBlockToBlockComplex(Xc, X);
+      }
+   }
+   
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      delete pw_c_coeffs[i];
+      delete cPrbb_cf[i]; 
+      delete cPibb_cf[i]; 
+      delete signedcPrbb_cf[i]; 
+      delete signedcPibb_cf[i]; 
+   }
+
+
+
+   a->RecoverFEMSolution(X, x);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i]->MakeRef(pfes[i], x, offsets[i]);
+      pgf_i[i]->MakeRef(pfes[i], x, offsets.Last() + offsets[i]);
+   }
+
+   ParallelECoefficient par_e_r(pgf_r[0]);
+   ParallelECoefficient par_e_i(pgf_i[0]);
+   E_par_r.ProjectCoefficient(par_e_r);
+   E_par_i.ProjectCoefficient(par_e_i);
+
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, *pgf_r[0],
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   }
+
+
+   delta = 0.0;
+   ParSesquilinearForm a_maxwell(pfes[0]);
+   // (1/μ₀ ∇×E, ∇ × F)
+   a_maxwell.AddDomainIntegrator(new CurlCurlIntegrator(muinv), nullptr);
+   // - ω² ϵ₀ (ϵᵣ E, F)
+   a_maxwell.AddDomainIntegrator(new VectorFEMassIntegrator(m_cf_r),
+                        new VectorFEMassIntegrator(m_cf_i));
+   a_maxwell.Assemble();                          
+
+
+   VectorGridFunctionCoefficient J1_cf_r(pgf_r[1]);
+   VectorGridFunctionCoefficient J1_cf_i(pgf_i[1]);
+   VectorGridFunctionCoefficient J2_cf_r(pgf_r[2]);
+   VectorGridFunctionCoefficient J2_cf_i(pgf_i[2]);
+
+   // -iω²ϵ₀ (Jᵣ + i Jᵢ ,F) = ω² ϵ₀ (Jᵢ - i Jᵣ,F)   
+   //                       = (ω² ϵ₀ Jᵢ, F) + i (-ω² ϵ₀Jᵢ,F)
+
+
+   ScalarVectorProductCoefficient omeg2_eps0_J1_cf_i(eps0*omega*omega, J1_cf_i);
+   ScalarVectorProductCoefficient omeg2_eps0_J2_cf_i(eps0*omega*omega, J2_cf_i);
+
+   ScalarVectorProductCoefficient negomeg2_eps0_J1_cf_r(-eps0*omega*omega, J1_cf_r);
+   ScalarVectorProductCoefficient negomeg2_eps0_J2_cf_r(-eps0*omega*omega, J2_cf_r);
+   
+   ParComplexLinearForm b_maxwell(pfes[0]);
+   b_maxwell.AddDomainIntegrator(new VectorFEDomainLFIntegrator(omeg2_eps0_J1_cf_i),
+                                 new VectorFEDomainLFIntegrator(negomeg2_eps0_J1_cf_r));
+   b_maxwell.AddDomainIntegrator(new VectorFEDomainLFIntegrator(omeg2_eps0_J2_cf_i),
+                                 new VectorFEDomainLFIntegrator(negomeg2_eps0_J2_cf_r));
+
+   b_maxwell.Assemble();
+
+
+   // remove internal boundaries
+   ess_bdr = 1;
+   // for (int i = 0; i<int_bdr_attr.Size(); i++)
+   // {
+   //    ess_bdr[int_bdr_attr[i]-1] = 0;
+   // }
+
+   pfes[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   maxwell_pgf.ProjectBdrCoefficientTangent(one_x_cf, zero_vcf, one_r_bdr);
+   maxwell_pgf.ProjectBdrCoefficientTangent(negone_x_cf, zero_vcf, negone_r_bdr);
+   maxwell_pgf.ProjectBdrCoefficientTangent(zero_vcf, one_x_cf, one_i_bdr);
+   maxwell_pgf.ProjectBdrCoefficientTangent(zero_vcf, negone_x_cf, negone_i_bdr);
+
+
+   OperatorPtr maxwell_Ah;
+   Vector maxwell_X,maxwell_B;
+   a_maxwell.FormLinearSystem(ess_tdof_list,maxwell_pgf,b_maxwell,maxwell_Ah, maxwell_X,maxwell_B);
+
+   ComplexHypreParMatrix *Maxwell_A = maxwell_Ah.As<ComplexHypreParMatrix>();
+   ComplexMUMPSSolver cmumps(MPI_COMM_WORLD);
+   cmumps.SetPrintLevel(0);
+   cmumps.SetOperator(*Maxwell_A);
+   cmumps.Mult(maxwell_B, maxwell_X);
+
+   a_maxwell.RecoverFEMSolution(maxwell_X, maxwell_B, maxwell_pgf);
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((real_t)0);
+      paraview_dc->Save();
+      delete paraview_dc;
+   }
+
+   delete a;
+   for (int i = 0; i < fecols.Size(); ++i)
+   {
+      delete fecols[i];
+      delete pfes[i];
+      delete pgf_r[i];
+      delete pgf_i[i];
+   }
+
+   return 0;
+
+}
+
+

miniapps/dpg/plasma/lh-eld-fosls-dpg.cpp

@@ -0,0 +1,669 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//                   MFEM Ultraweak DPG Maxwell parallel example
+//
+// Compile with: make lh-eld-fosls-dpg
+//
+// mpirun -np 8 ./lh-eld-fosls-dpg -o 4 -paraview -eld -m data/quad.msh -ebs -sc
+
+// Electron Landau Damping
+// Strong formulation:
+//     ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵ E + i ω²ϵ₀(J₁ + J₂) = 0,   in Ω
+//               λ₁   (- Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥) = 0,   in Ω     
+//               λ₂   (- Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥) = 0,   in Ω 
+//                                            E×n = E₀,  on ∂Ω
+//                                            J₁  = 0,  on ∂Ω
+//                                            J₂  = 0,  on ∂Ω
+// The DPG-FOSLS deals with the First Order System
+//  i ω μ₀ H  + ∇ × E                  = 0,   in Ω
+// -i ω ϵ₀ϵ E + ∇ × H - ω ϵ₀ (J₁ + J₂) = 0,   in Ω
+// - λ₁ b⋅∇Q₁ + λ₁ c₁ J₁ - λ₁ c₁ P B E = 0,   in Ω     
+//                   λ₁ Q₁ + λ₁ b⋅∇ J₁ = 0,   in Ω
+// - λ₂ b⋅∇Q₂ + λ₂ c₂ J₂ + λ₂ c₂ P B E = 0,   in Ω
+//                   λ₂ Q₂ + λ₂ b⋅∇ J₂ = 0,   in Ω
+//                                 E×n = E₀,  on ∂Ω
+//                                 J₁  = 0,  on ∂Ω
+//                                 J₂  = 0,  on ∂Ω
+
+
+// in 2D  E is vector valued and H is scalar and
+//     ∇ × E = ∇ ⋅ AE  where A = [0 1; -1 0];
+
+// E ∈ H(curl,Ω) , H ∈ H¹(Ω), Jᵢ, Qᵢ ∈ (H¹(Ω))²
+//  ( iωμ₀ H, F ) + ( ∇ × E, F) = 0,                      ∀ F  ∈  L²(Ω)
+// -(iωϵ₀ϵ E, R ) + ( ∇ × H, R) - ω ϵ₀ (J₁ + J₂, R) = 0,  ∀ R  ∈ (L²(Ω))²
+// λ₁ (b ⋅ ∇Q₁, K₁) + λ₁ c₁(J₁, K₁) - λ₁ c₁ (P B E, K₁) = 0,      ∀ K₁ ∈ (L²(Ω))²
+// λ₁ (Q₁, L₁) + λ₁ (b ⋅ ∇ J₁, L₁) = 0,                        ∀ L₁ ∈ (L²(Ω))²
+// λ₂ (b ⋅ ∇Q₂, K₂) + λ₂ c₂(J₂, K₂) + λ₂ c₂ (P B E, K₂) = 0,      ∀ K₂ ∈ (L²(Ω))²
+// λ₂ (Q₂, L₂) + λ₂ (b ⋅ ∇ J₂, L₂) = 0,                        ∀ L₂ ∈ (L²(Ω))²
+//                                    E = E₀, on ∂Ω
+//                              J₁ = J₂ = 0,  on ∂Ω
+// ----------------------------------------------------------------------------------------------------------------
+// |   |      E       |      H    |     J₁      |     Q₁     |       J₂   |      Q₂    |   RHS  |
+// ----------------------------------------------------------------------------------------------------------------
+// | F |  (∇ × E, F)  | (iωμ₀H,F) |             |            |            |            |        |        |   0   |  
+// |   |              |           |             |            |            |            |        |        |       |  
+// | R | -iωϵ₀ϵ(E,R)  |(∇ × H, R) | -ωϵ₀(J₁,R)  |            | -ωϵ₀(J₂, R)|            |        |        |   0   |  
+// |   |              |           |             |            |            |            |        |        |       |  
+// |K₁ | -λ₁c₁(PBE,K₁)|           | λ₁c₁(J₁, K₁)|(λ₁b⋅∇Q₁,K₁)|            |            |        |        |   0   |  
+// |   |              |           |             |            |            |            |        |        |       |    
+// |L₁ |              |           | λ₁(b⋅∇J₁,L₁)| λ₁(Q₁, L₁) |            |            |        |        |   0   |  
+// |   |              |           |             |            |            |            |        |        |       |    
+// |K₂ | λ₂c₂(PBE, K₂)|           |             |            |λ₂c₂(J₂, K₂)|λ₂(b⋅∇Q₂,K₂)|        |        |   0   |  
+// |   |              |           |             |            |            |            |        |        |       |    
+// |L₂ |              |           |             |            |λ₂(b⋅∇J₂,L₂)| λ₂(Q₂, L₂) |        |        |   0   |  
+// |   |              |           |             |            |            |            |        |        |       |    
+// where (F,R,K₁,L₁,K₂,L₂) ∈  L²(Ω) × (L²(Ω))² × (L²(Ω))² × (L²(Ω))² × (L²(Ω))² × (L²(Ω))² 
+
+
+
+#include "mfem.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../../common/mfem-common.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "utils/lh_utils.hpp"
+#include "../util/utils.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+   int order = 1;
+   int delta_order = 1;
+   int par_ref_levels = 0;
+   int ser_ref_levels = 0;
+
+   // real_t rnum=1.5e9;
+   // real_t mu = 1.257e-6;
+   // real_t eps0 = 8.8541878128e-12;
+
+   real_t rnum=1.5;
+   real_t mu = 1.257;
+   real_t eps0 = 8.8541878128;
+   real_t balance_scale = 1.0;
+   bool enable_balance_scale = false;
+
+   bool eld = false; // enable/disable electron Landau damping 
+   real_t delta_prec = 0.0;
+   real_t lambda1 = 1.0;
+   real_t lambda2 = 1.0;
+
+   bool static_cond = false;
+   bool visualization = false;
+   bool paraview = false;
+   bool debug = false;
+   bool mumps_solver = false;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&delta_order, "-do", "--delta-order",
+                  "Finite element order for the test space");                  
+   args.AddOption(&ser_ref_levels, "-sr", "--serial-refinement_levels",
+                  "Number of serial refinement levels.");                  
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&a0, "-a0", "--a0", "P(r) first parameter.");
+   args.AddOption(&a1, "-a1", "--a1", "P(r) second parameter.");
+   args.AddOption(&lambda1, "-l1", "--lambda1", "Lambda 1 scaling parameter.");
+   args.AddOption(&lambda2, "-l2", "--lambda2", "Lambda 2 scaling parameter.");
+   args.AddOption(&delta_prec, "-dp", "--delta-prec", "stability parameter for the preconditioner.");
+   args.AddOption(&eld, "-eld", "--eld", "-no-eld",
+                  "--no-eld",
+                  "Enable or disable electron Landau damping.");
+   args.AddOption(&mumps_solver, "-mumps", "--mumps", "-no-mumps",
+                  "--no-mumps",
+                  "Enable or disable MUMPS solver.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&debug, "-debug", "--debug", "-no-debug",
+                  "--no-debug",
+                  "Enable or disable debug mode (delta = 0.01 and no coupling).");         
+   args.AddOption(&enable_balance_scale, "-ebs", "--enable-balance-scale", "-no-ebs",
+                  "--no-ebs",
+                  "Enable or disable balance scale.");
+
+
+   args.Parse();
+   if (!args.Good())
+   {
+      if (Mpi::Root())
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (Mpi::Root())
+   {
+      args.PrintOptions(cout);
+   }
+
+   // number of diffusion equations
+   int ndiffusionequations = (eld) ?  2 : 0; 
+
+   Vector cvals(ndiffusionequations);
+   Vector csigns(ndiffusionequations);
+   if (eld)
+   {
+      cvals(0)  = 25e6;  cvals(1)  = 1e6;
+      csigns(0) = -1.0;  csigns(1) = 1.0;
+   }
+   cvals(0) *= lambda1;
+   cvals(1) *= lambda2;
+   real_t omega = 2.*M_PI*rnum;
+   int test_order = order+delta_order;
+
+   balance_scale = (enable_balance_scale) ? eps0 * omega * omega : 1.0;
+
+   if (eld && !debug) 
+   {
+      delta = 0.0; // disable delta if electron Landau damping is enabled
+      if (Mpi::Root())
+      {
+         cout << "Electron Landau damping enabled, delta set to 0.0." << endl;
+      }
+   }    
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   MFEM_VERIFY(dim == 2, "Dimension != 2 is not supported in this example");
+
+   for (int i = 0; i < ser_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   Array<int> int_bdr_attr;
+   for (int i = 0; i < mesh.GetNBE(); i++)
+   {
+      if (mesh.FaceIsInterior(mesh.GetBdrElementFaceIndex(i)))
+      {
+         int_bdr_attr.Append(mesh.GetBdrAttribute(i));
+      }
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   for (int i = 0; i < par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }  
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+   real_t scale = (debug) ? 0.0 : 1.0;
+
+   // Define coefficients
+   ConstantCoefficient one_cf(1.0);
+   //  ωμ₀
+   ConstantCoefficient omegamu_cf(omega*mu);
+   // -ωϵ₀
+   ConstantCoefficient negomegeps0_cf(-omega*eps0 * scale);
+   ConstantCoefficient balancescaled_negomegeps0_cf( -omega*eps0/balance_scale * scale);
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_vcf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   // S(r) 
+   FunctionCoefficient S_cf_r(sfunc_r), S_cf_i(sfunc_i);
+   // P(r) 
+   FunctionCoefficient P_cf_r(pfunc_r), P_cf_i(pfunc_i); 
+
+   // b
+   VectorFunctionCoefficient b_cf(dim,bfunc);
+   ScalarVectorProductCoefficient scaled1_b_cf(sqrt(lambda1), b_cf);
+   ScalarVectorProductCoefficient scaled2_b_cf(sqrt(lambda2), b_cf);
+
+   // b⊗b
+   MatrixFunctionCoefficient bb_cf(dim,bcrossb); 
+   // I - b⊗b
+   MatrixSumCoefficient oneminusbb(Mone_cf, bb_cf, 1.0, -1.0); 
+   // S(r) (I - b⊗b)
+   ScalarMatrixProductCoefficient Soneminusbb_r(S_cf_r, oneminusbb), Soneminusbb_i(S_cf_i, oneminusbb); 
+   // P(r) b⊗b 
+   ScalarMatrixProductCoefficient P_cf_bb_r(P_cf_r, bb_cf), P_cf_bb_i(P_cf_i, bb_cf); 
+
+   // ε = S(r) (I - b⊗b) + P(r) b⊗b 
+   MatrixSumCoefficient eps_r(Soneminusbb_r, P_cf_bb_r, 1.0, 1.0); 
+   MatrixSumCoefficient eps_i(Soneminusbb_i, P_cf_bb_i, 1.0, 1.0); 
+
+   coefs_r[nattr-1] = &eps_r;
+   coefs_i[nattr-1] = &eps_i;
+
+   PWMatrixCoefficient eps_cf_r(dim, attr, coefs_r);
+   PWMatrixCoefficient eps_cf_i(dim, attr, coefs_i);
+
+   ConstantCoefficient eps0omeg(omega * eps0);
+   ConstantCoefficient negeps0omeg(-omega * eps0);
+
+
+   // - iωϵ₀ϵ = ωϵ₀ϵᵢ + i (-ωϵ₀ϵᵣ)
+   // ω ϵ₀ ϵᵢ
+   ScalarMatrixProductCoefficient eps0omeg_eps_i(eps0omeg, eps_cf_i);
+   // -ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient negeps0omeg_eps_r(negeps0omeg, eps_cf_r);
+
+   // if ELD
+   Array<Vector *> c_arrays(ndiffusionequations);
+   Array<PWConstCoefficient *> pw_c_coeffs(ndiffusionequations);
+   Array<MatrixCoefficient *> cPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> cPibb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPibb_cf(ndiffusionequations);
+
+   Array<MatrixCoefficient *> balancescaled_signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> balancescaled_signedcPibb_cf(ndiffusionequations);
+   Vector temp(nattr); temp=0.0;
+   Array<ConstantCoefficient *> c_coeffs(ndiffusionequations);
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      temp[nattr-1] = cvals(i);
+      // temp = cvals(i);
+      pw_c_coeffs[i] = new PWConstCoefficient(temp);
+      c_coeffs[i] = new ConstantCoefficient(cvals(i));
+      cPrbb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_r);
+      cPibb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_i);
+      signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPrbb_cf[i]);
+      signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPibb_cf[i]);
+      balancescaled_signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(balance_scale,*signedcPrbb_cf[i]);
+      balancescaled_signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(balance_scale,*signedcPibb_cf[i]);
+   }
+
+
+   std::vector<std::string> variables = {" E  ", " H  ", " J₁ ", " J₂ ", " Q₁ ", " Q₂ "};
+   std::vector<std::string> spaces = {"H(curl,Ω) ", "   H¹(Ω)  ", 
+                                      "  (H¹(Ω))ᵈ", "  (H¹(Ω))ᵈ", 
+                                      "  (H¹(Ω))ᵈ", "  (H¹(Ω))ᵈ"};
+
+   // Define the spaces
+   Array<FiniteElementCollection *> trial_fecols;
+   Array<FiniteElementCollection *> test_fecols;
+   Array<ParFiniteElementSpace *> pfes;
+
+   // H(curl) space for E
+   trial_fecols.Append(new ND_FECollection(order, dim));
+   pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last()));
+   // Scalar H¹ space for H
+   trial_fecols.Append(new H1_FECollection(order, dim));
+   pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last()));
+
+   // Vector H¹ spaces for Jᵢ and Qᵢ
+   for (int i = 0; i < 2*ndiffusionequations; i++)
+   {
+      trial_fecols.Append(new H1_FECollection(order, dim));
+      pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last(), dim));
+   }
+
+   Array<HYPRE_BigInt> tdofs(pfes.Size());
+   for (int i = 0; i < pfes.Size(); ++i)
+   {
+      tdofs[i] = pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << "ParFiniteElementSpace " << spaces[i] << "  for " << variables[i] << " has " << tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   if (Mpi::Root())
+   {
+      cout << "Total number of true dofs: " << tdofs.Sum() << endl;
+   }
+
+   // test spaces for F and G
+   test_fecols.Append(new L2_FECollection(test_order, dim));
+   test_fecols.Append(new L2_FECollection(test_order, dim));
+   // Test spaces Kᵢ and Lᵢ 
+   for (int i = 0; i < 2*ndiffusionequations; i++)
+   {
+      test_fecols.Append(new L2_FECollection(test_order, dim));
+   }
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(pfes,test_fecols);
+   for (int i = 0; i < test_fecols.Size(); i++)
+   {  // all but the first  space are vector valued
+      if (i > 0) { a->SetTestFECollVdim(i,dim); }
+   }
+
+   // (∇ × E, F)
+   a->AddTrialIntegrator(new MixedCurlIntegrator(one_cf), nullptr, 0, 0);
+
+   // (i ω μ₀ H, F)
+   a->AddTrialIntegrator(nullptr, new MixedScalarMassIntegrator(omegamu_cf), 1, 0);
+
+   // -i ω ϵ₀ϵ (E, R) = ω ϵ₀ ϵᵢ (E, G) + i (-ω ϵ₀ ϵᵣ E, R)
+   a->AddTrialIntegrator(new VectorFEMassIntegrator(eps0omeg_eps_i),
+                         new VectorFEMassIntegrator(negeps0omeg_eps_r), 0, 1);
+
+   // (∇ × H, R)
+   a->AddTrialIntegrator(new MixedCurlIntegrator(one_cf), nullptr, 1,1);
+   
+   if (eld)
+   {
+      // - ω ϵ₀ (J₁, R)
+      a->AddTrialIntegrator(new VectorMassIntegrator(balancescaled_negomegeps0_cf), nullptr, 2, 1);
+      // - ω ϵ₀ (J₂, R)
+      a->AddTrialIntegrator(new VectorMassIntegrator(balancescaled_negomegeps0_cf), nullptr, 3, 1);
+      // -c₁(PBE, K₁)
+      a->AddTrialIntegrator(new VectorFEMassIntegrator(*balancescaled_signedcPrbb_cf[0]),
+                            new VectorFEMassIntegrator(*balancescaled_signedcPibb_cf[0]), 0, 2);
+
+      // c₁(J₁, K₁)
+      a->AddTrialIntegrator(new VectorMassIntegrator(*pw_c_coeffs[0]), nullptr, 2, 2);
+
+      // (b⋅∇Q₁, K₁)
+      a->AddTrialIntegrator(new DirectionalVectorGradientIntegrator(scaled1_b_cf), nullptr, 3, 2);
+
+      // (b⋅∇J₁, L₁)
+      a->AddTrialIntegrator(new DirectionalVectorDiffusionIntegrator(scaled1_b_cf), nullptr, 2, 3);
+
+      // (Q₁, L₁)
+      a->AddTrialIntegrator(new VectorMassIntegrator(one_cf), nullptr, 3, 3);
+
+      // c₂(PBE, K₂)
+      a->AddTrialIntegrator(new VectorFEMassIntegrator(*balancescaled_signedcPrbb_cf[1]),
+                            new VectorFEMassIntegrator(*balancescaled_signedcPibb_cf[1]), 0, 4);
+
+      // c₂(J₂, K₂)
+      a->AddTrialIntegrator(new VectorMassIntegrator(*pw_c_coeffs[1]), nullptr, 4, 4);
+   
+      // (b⋅∇Q₂, K₂)
+      a->AddTrialIntegrator(new DirectionalVectorGradientIntegrator(scaled2_b_cf), nullptr, 5, 4);
+
+      // (b⋅∇J₂, L₂)
+      a->AddTrialIntegrator(new DirectionalVectorDiffusionIntegrator(scaled2_b_cf), nullptr, 4, 5);
+
+      // (Q₂, L₂)
+      a->AddTrialIntegrator(new VectorMassIntegrator(one_cf), nullptr, 5, 5);
+   }
+   // test integrators for test norm: ||v||² = ||F||² + ||R||² + ||K₁||² + ||L₁||² + ||K₂||² + ||L₂||² 
+   // (F,δF), F, δF ∈  L²(Ω)
+   a->AddTestIntegrator(new MassIntegrator(one_cf),nullptr, 0, 0);
+   // (R,δR), R, δR ∈ (L²(Ω))²
+   a->AddTestIntegrator(new VectorMassIntegrator(one_cf),nullptr, 1, 1);
+   
+   if (eld)
+   {
+      // (K₁,δK₁), K₁, δK₁ ∈ (L²(Ω))²
+      a->AddTestIntegrator(new VectorMassIntegrator(one_cf),nullptr, 2, 2);
+      // (L₁,δL₁), L₁, δL₁ ∈ (L²(Ω))²
+      a->AddTestIntegrator(new VectorMassIntegrator(one_cf),nullptr, 3, 3);
+      // (K₂,δK₂), K₂, δK₂ ∈ (L²(Ω))²
+      a->AddTestIntegrator(new VectorMassIntegrator(one_cf),nullptr, 4, 4);
+      // (L₂,δL₂), L₂, δL₂ ∈ (L²(Ω))²
+      a->AddTestIntegrator(new VectorMassIntegrator(one_cf),nullptr, 5, 5);        
+   }
+
+   a->Assemble(0);
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      delete pw_c_coeffs[i];
+      delete c_coeffs[i];
+      delete cPrbb_cf[i]; 
+      delete cPibb_cf[i]; 
+      delete signedcPrbb_cf[i]; 
+      delete signedcPibb_cf[i]; 
+   }
+
+   socketstream E_out_r;
+
+   int npfes = pfes.Size();
+   Array<int> offsets(npfes+1);  offsets[0] = 0;
+   Array<int> toffsets(npfes+1); toffsets[0] = 0;
+   for (int i = 0; i<npfes; i++)
+   {
+      offsets[i+1] = pfes[i]->GetVSize();
+      toffsets[i+1] = pfes[i]->TrueVSize();
+   }
+   offsets.PartialSum();
+   toffsets.PartialSum();
+
+   Vector x(2*offsets.Last());
+   x = 0.;
+   
+   Array<ParGridFunction *> pgf_r(npfes);
+   Array<ParGridFunction *> pgf_i(npfes);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i] = new ParGridFunction(pfes[i], x, offsets[i]);
+      pgf_i[i] = new ParGridFunction(pfes[i], x, offsets.Last() + offsets[i]);
+   }
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+   ParGridFunction E_par_r(&L2_fes);
+   ParGridFunction E_par_i(&L2_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   std::string output_dir = "ParaView/FOSLS/" + GetTimestamp();
+
+   if (paraview)
+   {
+      if (Mpi::Root()) { WriteParametersToFile(args, output_dir); }
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order
+                         << "_eld_" << eld;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",pgf_r[0]);
+      paraview_dc->RegisterField("E_i",pgf_i[0]);
+      paraview_dc->RegisterField("E_par_r",&E_par_r);
+      paraview_dc->RegisterField("E_par_i",&E_par_i);
+      paraview_dc->RegisterField("H_r",pgf_r[1]);
+      paraview_dc->RegisterField("H_i",pgf_i[1]);      
+      if (eld)
+      {
+         paraview_dc->RegisterField("J1_r",pgf_r[2]);
+         paraview_dc->RegisterField("J1_i",pgf_i[2]);
+         paraview_dc->RegisterField("Q1_r",pgf_r[3]);
+         paraview_dc->RegisterField("Q1_i",pgf_i[3]);
+         paraview_dc->RegisterField("J2_r",pgf_r[4]);
+         paraview_dc->RegisterField("J2_i",pgf_i[4]);
+         paraview_dc->RegisterField("Q2_r",pgf_r[5]);
+         paraview_dc->RegisterField("Q2_i",pgf_i[5]);
+      }
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listJ;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+   
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+
+      // remove internal boundaries
+      for (int i = 0; i<int_bdr_attr.Size(); i++)
+      {
+         ess_bdr[int_bdr_attr[i]-1] = 0;
+      }
+
+      pfes[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      ess_bdr=1;
+      for (int i = 0; i<ndiffusionequations;i++)
+      {
+         ess_tdof_listJ.SetSize(0);
+         pfes[2*i+2]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJ); // J₁ (2), J₂ (4)
+         for (int j = 0; j < ess_tdof_listJ.Size(); j++)
+         {
+            ess_tdof_listJ[j] += toffsets[2*i+2]; 
+         }
+         ess_tdof_list.Append(ess_tdof_listJ);
+      }
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+   
+
+   pgf_r[0]->ProjectBdrCoefficientTangent(one_x_cf, one_r_bdr);
+   pgf_r[0]->ProjectBdrCoefficientTangent(negone_x_cf, negone_r_bdr);
+   pgf_i[0]->ProjectBdrCoefficientTangent(one_x_cf, one_i_bdr);
+   pgf_i[0]->ProjectBdrCoefficientTangent(negone_x_cf, negone_i_bdr);
+
+   OperatorPtr Ah;
+   Vector X,B;
+   a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+   ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+   bool direct_solve = true;
+
+   if (direct_solve)
+   {
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+      int nblocks = BlockA_r->NumRowBlocks();
+      if(Mpi::Root())
+      {
+         mfem::out << "Number of blocks: " << nblocks << std::endl;
+      }
+      Array2D<const HypreParMatrix*> A_r_matrices(nblocks, nblocks);
+      Array2D<const HypreParMatrix*> A_i_matrices(nblocks, nblocks);
+      for (int i = 0; i < nblocks; i++)
+      {
+         for (int j = 0; j < nblocks; j++)
+         {
+            A_r_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockA_r->GetBlock(i,j));
+            A_i_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockA_i->GetBlock(i,j));
+         }
+      }
+      HypreParMatrix * Ahr = HypreParMatrixFromBlocks(A_r_matrices);
+      HypreParMatrix * Ahi = HypreParMatrixFromBlocks(A_i_matrices);
+
+      ComplexHypreParMatrix * Ahc_hypre =
+         new ComplexHypreParMatrix(Ahr, Ahi,true, true);
+
+      ComplexMUMPSSolver cmumps(MPI_COMM_WORLD);
+      cmumps.SetPrintLevel(0);
+      cmumps.SetOperator(*Ahc_hypre);
+      cmumps.Mult(B,X);
+      delete Ahc_hypre;   
+
+   }
+
+   a->RecoverFEMSolution(X, x);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i]->MakeRef(pfes[i], x, offsets[i]);
+      pgf_i[i]->MakeRef(pfes[i], x, offsets.Last() + offsets[i]);
+   }
+   
+   ParallelECoefficient par_e_r(pgf_r[0]);
+   ParallelECoefficient par_e_i(pgf_i[0]);
+   E_par_r.ProjectCoefficient(par_e_r);
+   E_par_i.ProjectCoefficient(par_e_i);
+   
+   // // rescale the J solutions
+   for (int i = 0; i < 2*ndiffusionequations; ++i)
+   {
+      (*pgf_r[2+i]) /= balance_scale;
+      (*pgf_i[2+i]) /= balance_scale;
+   }
+
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, *pgf_r[0],
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((real_t)0);
+      paraview_dc->Save();
+      delete paraview_dc;
+   }
+
+
+   delete a;
+   for (int i = 0; i < trial_fecols.Size(); ++i)
+   {
+      delete trial_fecols[i];
+      delete pfes[i];
+   }
+   for (int i = 0; i< test_fecols.Size(); ++i)
+   {
+      delete test_fecols[i];
+   }
+
+   return 0;
+}

miniapps/dpg/plasma/lh-eld-fosls-fem.cpp

@@ -0,0 +1,897 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//                   MFEM FOSLS-FEM Maxwell parallel example
+//
+// Compile with: make lh-eld-fosls-fem
+//
+// mpirun -np 8 ./lh-eld-fosls-fem -o 4 -paraview -m data/quad.msh -ebs -sc
+// Electron Landau Damping
+// Strong formulation:
+//     ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵ E + i ω²ϵ₀(J₁ + J₂) = 0,   in Ω
+//     subject to the constraints
+//                   - Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥ = 0,   in Ω     
+//                   - Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥ = 0,   in Ω 
+//                                            E×n = E₀,  on ∂Ω
+//                                            J₁  = 0,  on ∂Ω
+//                                            J₂  = 0,  on ∂Ω
+// The DPG-FOSLS deals with the First Order System
+//  i ω μ₀ H  + ∇ × E                  = 0,   in Ω
+// -i ω ϵ₀ϵ E + ∇ × H - ω ϵ₀ (J₁ + J₂) = 0,   in Ω
+// subject to the constraints
+//         - Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥ = 0,   in Ω     
+//         - Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥ = 0,   in Ω 
+//                                 E×n = E₀,  on ∂Ω
+//                                 J₁  = 0,  on ∂Ω
+//                                 J₂  = 0,  on ∂Ω
+
+
+// in 2D  E is vector valued and H is scalar and
+//     ∇ × E = ∇ ⋅ AE  where A = [0 1; -1 0];
+
+// E ∈ H(curl,Ω) , H ∈ H¹(Ω), Jᵢ, Qᵢ ∈ (H¹(Ω))²
+// minimize the FOSLS functional:
+//  ( iωμ₀ H, F ) + ( ∇ × E, F) = 0,                      ∀ F  ∈  L²(Ω)
+// -(iωϵ₀ϵ E, R ) + ( ∇ × H, R) - ω ϵ₀ (J₁ + J₂, R) = 0,  ∀ R  ∈ (L²(Ω))²
+// subject to the constraints
+//  (b⋅∇J₁, b ⋅ ∇K₁) + c₁(J₁, K₁) - c₁ (P B E, K₁) = 0,      ∀ K₁ ∈ (H¹(Ω))²
+//  (b⋅∇J₂, b ⋅ ∇K₂) + c₂(J₂, K₂) + c₂ (P B E, K₂) = 0,      ∀ K₂ ∈ (H¹(Ω))²
+//                                    E = E₀, on ∂Ω
+//                              J₁ = J₂ = 0,  on ∂Ω
+
+
+// We formulate the problem as constrained minimization
+// i.e, we minimize ||A U - F|| subject to B U = 0
+// where U = [E, H, J₁, J₂]ᵀ
+// A and B are given by:
+// -----------------------------------------------
+
+// A (FOSLS)       
+// ----------------------------------------------------------
+// |   |      E       |      H    |     J₁      |     J₂    |       
+// ----------------------------------------------------------
+// | F |  (∇ × E, F)  | (iωμ₀H,F) |             |           |   
+// |   |              |           |             |           |   
+// | R | -iωϵ₀ϵ(E,R)  |(∇ × H, R) | -ωϵ₀(J₁,R)  |-ωϵ₀(J₂, R)|   
+
+// B (Constraints)
+// --------------------------------------------------------------------------------
+// |   |      E       |                 J₁           |                J₂          |       
+// --------------------------------------------------------------------------------
+// |K₁ | -c₁(PBE,K₁)  | (b⋅∇J₁, b⋅∇K₁) + c₁(J₁, K₁)  |                            |       
+// |   |              |                              |                            |       
+// |K₂ |  c₂(PBE, K₂) |                              |(b⋅∇J₂, b⋅∇K₂) + c₂(J₂, K₂) |       
+
+// The saddle point system is then given by:
+// | A   B̄ᵀ | |U| = |F|
+// | B   0  | |λ|   |0|
+
+
+#include "mfem.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../util/pcomplexblockform.hpp"
+#include "../util/blockcomplexhypremat.hpp"
+#include "../../common/mfem-common.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "utils/lh_utils.hpp"
+#include "../util/utils.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+   int order = 1;
+   int delta_order = 1;
+   int par_ref_levels = 0;
+   int ser_ref_levels = 0;
+
+   // real_t rnum=1.5e9;
+   // real_t mu = 1.257e-6;
+   // real_t eps0 = 8.8541878128e-12;
+
+   real_t rnum=1.5;
+   real_t mu = 1.257;
+   real_t eps0 = 8.8541878128;
+   real_t cfactor = 1e-6;
+   real_t balance_scale = 1.0;
+   bool enable_balance_scale = false;
+
+   real_t delta_prec = 0.0;
+
+
+   bool static_cond = false;
+   bool visualization = false;
+   bool paraview = false;
+   bool debug = false;
+   bool mumps_solver = false;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&delta_order, "-do", "--delta-order",
+                  "Finite element order for the test space");                  
+   args.AddOption(&ser_ref_levels, "-sr", "--serial-refinement_levels",
+                  "Number of serial refinement levels.");                  
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&a0, "-a0", "--a0", "P(r) first parameter.");
+   args.AddOption(&a1, "-a1", "--a1", "P(r) second parameter.");
+   args.AddOption(&delta_prec, "-dp", "--delta-prec", "stability parameter for the preconditioner.");
+   args.AddOption(&mumps_solver, "-mumps", "--mumps", "-no-mumps",
+                  "--no-mumps",
+                  "Enable or disable MUMPS solver.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&debug, "-debug", "--debug", "-no-debug",
+                  "--no-debug",
+                  "Enable or disable debug mode (delta = 0.01 and no coupling).");         
+   args.AddOption(&enable_balance_scale, "-ebs", "--enable-balance-scale", "-no-ebs",
+                  "--no-ebs",
+                  "Enable or disable balance scale.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (Mpi::Root())
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (Mpi::Root())
+   {
+      args.PrintOptions(cout);
+   }
+
+   // number of diffusion equations
+   int ndiffusionequations = 2; 
+
+   Vector cvals(ndiffusionequations);
+   Vector csigns(ndiffusionequations);
+   cvals(0)  = 25e6;  cvals(1)  = 1e6;
+   csigns(0) = -1.0;  csigns(1) = 1.0;
+
+   cvals *= cfactor; // scale the coefficients
+   real_t omega = 2.*M_PI*rnum;
+   int test_order = order+delta_order;
+
+   balance_scale = (enable_balance_scale) ? eps0 : 1.0;
+
+   if (!debug) 
+   {
+      delta = 0.0; // disable delta if electron Landau damping is enabled
+      if (Mpi::Root())
+      {
+         cout << "Electron Landau damping enabled, delta set to 0.0." << endl;
+      }
+   }    
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   MFEM_VERIFY(dim == 2, "Dimension != 2 is not supported in this example");
+
+   for (int i = 0; i < ser_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   Array<int> int_bdr_attr;
+   for (int i = 0; i < mesh.GetNBE(); i++)
+   {
+      if (mesh.FaceIsInterior(mesh.GetBdrElementFaceIndex(i)))
+      {
+         int_bdr_attr.Append(mesh.GetBdrAttribute(i));
+      }
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   for (int i = 0; i < par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }  
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+   real_t scale = (debug) ? 0.0 : 1.0;
+
+   // Define coefficients
+   ConstantCoefficient one_cf(1.0);
+   //  ωμ₀
+   ConstantCoefficient omegamu_cf(omega*mu);
+   // -ωϵ₀
+   ConstantCoefficient negomegeps0_cf(-omega*eps0 * scale);
+   ConstantCoefficient balancescaled_negomegeps0_cf( -omega*eps0/balance_scale * scale);
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_vcf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   // S(r) 
+   FunctionCoefficient S_cf_r(sfunc_r), S_cf_i(sfunc_i);
+   // P(r) 
+   FunctionCoefficient P_cf_r(pfunc_r), P_cf_i(pfunc_i); 
+
+   // b
+   VectorFunctionCoefficient b_cf(dim,bfunc);
+   ScalarVectorProductCoefficient scaled_b_cf(sqrt(cfactor), b_cf);
+   ConstantCoefficient diff_coeff(cfactor);
+
+   // b⊗b
+   MatrixFunctionCoefficient bb_cf(dim,bcrossb); 
+   // I - b⊗b
+   MatrixSumCoefficient oneminusbb(Mone_cf, bb_cf, 1.0, -1.0); 
+   // S(r) (I - b⊗b)
+   ScalarMatrixProductCoefficient Soneminusbb_r(S_cf_r, oneminusbb), Soneminusbb_i(S_cf_i, oneminusbb); 
+   // P(r) b⊗b 
+   ScalarMatrixProductCoefficient P_cf_bb_r(P_cf_r, bb_cf), P_cf_bb_i(P_cf_i, bb_cf); 
+
+   // ε = S(r) (I - b⊗b) + P(r) b⊗b 
+   MatrixSumCoefficient eps_r(Soneminusbb_r, P_cf_bb_r, 1.0, 1.0); 
+   MatrixSumCoefficient eps_i(Soneminusbb_i, P_cf_bb_i, 1.0, 1.0); 
+
+   coefs_r[nattr-1] = &eps_r;
+   coefs_i[nattr-1] = &eps_i;
+
+   PWMatrixCoefficient eps_cf_r(dim, attr, coefs_r);
+   PWMatrixCoefficient eps_cf_i(dim, attr, coefs_i);
+
+   ConstantCoefficient eps0omeg(omega * eps0);
+   ConstantCoefficient negeps0omeg(-omega * eps0);
+
+
+   // - iωϵ₀ϵ = ωϵ₀ϵᵢ + i (-ωϵ₀ϵᵣ)
+   // ω ϵ₀ ϵᵢ
+   ScalarMatrixProductCoefficient eps0omeg_eps_i(eps0omeg, eps_cf_i);
+   // -ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient negeps0omeg_eps_r(negeps0omeg, eps_cf_r);
+
+   // if ELD
+   Array<Vector *> c_arrays(ndiffusionequations);
+   Array<PWConstCoefficient *> pw_c_coeffs(ndiffusionequations);
+   Array<MatrixCoefficient *> cPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> cPibb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPibb_cf(ndiffusionequations);
+
+   Array<MatrixCoefficient *> balancescaled_signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> balancescaled_signedcPibb_cf(ndiffusionequations);
+   Vector temp(nattr); temp=0.0;
+   Array<ConstantCoefficient *> c_coeffs(ndiffusionequations);
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      temp[nattr-1] = cvals(i);
+      // temp = cvals(i);
+      pw_c_coeffs[i] = new PWConstCoefficient(temp);
+      c_coeffs[i] = new ConstantCoefficient(cvals(i));
+      cPrbb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_r);
+      cPibb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_i);
+      signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPrbb_cf[i]);
+      signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPibb_cf[i]);
+      balancescaled_signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(balance_scale,*signedcPrbb_cf[i]);
+      balancescaled_signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(balance_scale,*signedcPibb_cf[i]);
+   }
+
+
+   std::vector<std::string> fosls_variables = {" E  ", " H  ", " J₁ ", " J₂ "};
+   std::vector<std::string> fosls_spaces = {"H(curl,Ω) ", "   H¹(Ω)  ", 
+                                            "  (H¹(Ω))ᵈ", "  (H¹(Ω))ᵈ"}; 
+   std::vector<std::string> constraints_variables = {" Q₁ ", " Q₂ "};
+   std::vector<std::string> constraints_spaces = {"  (H¹(Ω))ᵈ", "  (H¹(Ω))ᵈ"};                                             
+
+   // Define the spaces
+   Array<FiniteElementCollection *> fosls_trial_fecols;
+   Array<FiniteElementCollection *> fosls_test_fecols;
+   Array<ParFiniteElementSpace *> fosls_pfes;
+   Array<FiniteElementCollection *> constraints_fecols;
+   Array<ParFiniteElementSpace *> constraints_pfes;
+
+   // H(curl) space for E
+   fosls_trial_fecols.Append(new ND_FECollection(order, dim));
+   fosls_pfes.Append(new ParFiniteElementSpace(&pmesh, fosls_trial_fecols.Last()));
+   // Scalar H¹ space for H
+   fosls_trial_fecols.Append(new H1_FECollection(order, dim));
+   fosls_pfes.Append(new ParFiniteElementSpace(&pmesh, fosls_trial_fecols.Last()));
+   // Vector H¹ spaces for Jᵢ
+   for (int i = 0; i < ndiffusionequations; i++)
+   {
+      fosls_trial_fecols.Append(new H1_FECollection(order, dim));
+      fosls_pfes.Append(new ParFiniteElementSpace(&pmesh, fosls_trial_fecols.Last(), dim));
+      constraints_fecols.Append(new H1_FECollection(order, dim));
+      constraints_pfes.Append(new ParFiniteElementSpace(&pmesh, constraints_fecols.Last(), dim));
+   }
+
+   Array<HYPRE_BigInt> tdofs(fosls_pfes.Size());
+   Array<HYPRE_BigInt> constraints_tdofs(constraints_pfes.Size());
+   for (int i = 0; i < fosls_pfes.Size(); ++i)
+   {
+      tdofs[i] = fosls_pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << " FOSLS      FE Space " << fosls_spaces[i] << "  for " << fosls_variables[i] << " has " << tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   for (int i = 0; i < constraints_pfes.Size(); ++i)
+   {
+      constraints_tdofs[i] = constraints_pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << " Constraint FE Space " << constraints_spaces[i] << "  for " << constraints_variables[i] << " has " << constraints_tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   if (Mpi::Root())
+   {
+      cout << "Total number of true dofs: " << tdofs.Sum() << endl;
+   }
+
+   // test spaces for F and R
+   fosls_test_fecols.Append(new L2_FECollection(test_order, dim));
+   fosls_test_fecols.Append(new L2_FECollection(test_order, dim));
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(fosls_pfes,fosls_test_fecols);
+   a->SetTestFECollVdim(1,dim); 
+
+   // (∇ × E, F)
+   a->AddTrialIntegrator(new MixedCurlIntegrator(one_cf), nullptr, 0, 0);
+
+   // (i ω μ₀ H, F)
+   a->AddTrialIntegrator(nullptr, new MixedScalarMassIntegrator(omegamu_cf), 1, 0);
+
+   // -i ω ϵ₀ϵ (E, R) = ω ϵ₀ ϵᵢ (E, G) + i (-ω ϵ₀ ϵᵣ E, R)
+   a->AddTrialIntegrator(new VectorFEMassIntegrator(eps0omeg_eps_i),
+                         new VectorFEMassIntegrator(negeps0omeg_eps_r), 0, 1);
+
+   // (∇ × H, R)
+   a->AddTrialIntegrator(new MixedCurlIntegrator(one_cf), nullptr, 1,1);
+   
+   // - ω ϵ₀ (J₁, R)
+   a->AddTrialIntegrator(new VectorMassIntegrator(balancescaled_negomegeps0_cf), nullptr, 2, 1);
+   // - ω ϵ₀ (J₂, R)
+   a->AddTrialIntegrator(new VectorMassIntegrator(balancescaled_negomegeps0_cf), nullptr, 3, 1);
+
+   // test integrators for test norm: ||v||² = ||F||² + ||R||² + ||K₁||² + ||L₁||² + ||K₂||² + ||L₂||² 
+   // (F,δF), F, δF ∈  L²(Ω)
+   a->AddTestIntegrator(new MassIntegrator(one_cf),nullptr, 0, 0);
+   // (R,δR), R, δR ∈ (L²(Ω))²
+   a->AddTestIntegrator(new VectorMassIntegrator(one_cf),nullptr, 1, 1);
+
+   a->Assemble(0);
+
+
+   // Constraint matrix B
+   ParMixedBilinearForm bEK1_r(fosls_pfes[0], constraints_pfes[0]);
+   ParMixedBilinearForm bEK1_i(fosls_pfes[0], constraints_pfes[0]);
+
+   // - c₁ (P B E, K₁)
+   bEK1_r.AddDomainIntegrator(new VectorFEMassIntegrator(*balancescaled_signedcPrbb_cf[0]));
+   bEK1_i.AddDomainIntegrator(new VectorFEMassIntegrator(*balancescaled_signedcPibb_cf[0]));
+
+   bEK1_r.Assemble();
+   bEK1_i.Assemble();
+
+   ParMixedBilinearForm bK2H_r(constraints_pfes[1], fosls_pfes[1]);
+   bK2H_r.Assemble();
+
+   ParMixedBilinearForm bEK2_r(fosls_pfes[0], constraints_pfes[1]);
+   ParMixedBilinearForm bEK2_i(fosls_pfes[0], constraints_pfes[1]);
+   // c₂(PBE, K₂)
+   bEK2_r.AddDomainIntegrator(new VectorFEMassIntegrator(*balancescaled_signedcPrbb_cf[1]));    
+   bEK2_i.AddDomainIntegrator(new VectorFEMassIntegrator(*balancescaled_signedcPibb_cf[1]));    
+
+   bEK2_r.Assemble();
+   bEK2_i.Assemble();
+
+   // (b⋅∇J₁, b⋅∇K₁) + c₁(J₁, K₁) 
+   ParBilinearForm bJ1K1(constraints_pfes[0]);
+   bJ1K1.AddDomainIntegrator(new DirectionalVectorDiffusionIntegrator(scaled_b_cf));
+   bJ1K1.AddDomainIntegrator(new VectorMassIntegrator(*pw_c_coeffs[0]));
+   bJ1K1.Assemble();
+
+   ParBilinearForm bJ2K2(constraints_pfes[1]);
+   bJ2K2.AddDomainIntegrator(new DirectionalVectorDiffusionIntegrator(scaled_b_cf));
+   bJ2K2.AddDomainIntegrator(new VectorMassIntegrator(*pw_c_coeffs[1]));
+   bJ2K2.Assemble();
+
+
+   int fosls_npfes = fosls_pfes.Size();
+   Array<int> fosls_offsets(fosls_npfes+1);  fosls_offsets[0] = 0;
+   Array<int> fosls_toffsets(fosls_npfes+1); fosls_toffsets[0] = 0;
+   for (int i = 0; i<fosls_npfes; i++)
+   {
+      fosls_offsets[i+1] = fosls_pfes[i]->GetVSize();
+      fosls_toffsets[i+1] = fosls_pfes[i]->TrueVSize();
+   }
+   fosls_offsets.PartialSum();
+   fosls_toffsets.PartialSum();
+
+   Array<int> empty;
+   OperatorPtr fosls_Aop;
+
+   a->FormSystemMatrix(empty, fosls_Aop);
+   ComplexOperator * fosls_Ac = fosls_Aop.As<ComplexOperator>();
+
+   OperatorPtr constraintOp_EK1_r, constraintOp_EK1_i;
+   bEK1_r.FormRectangularSystemMatrix(empty, empty, constraintOp_EK1_r);
+   bEK1_i.FormRectangularSystemMatrix(empty, empty, constraintOp_EK1_i);
+
+   OperatorPtr constraintOp_EK2_r, constraintOp_EK2_i;
+   bEK2_r.FormRectangularSystemMatrix(empty, empty, constraintOp_EK2_r);
+   bEK2_i.FormRectangularSystemMatrix(empty, empty, constraintOp_EK2_i);
+
+   OperatorPtr constraintOp_J1K1;
+   bJ1K1.FormSystemMatrix(empty, constraintOp_J1K1);
+   OperatorPtr constraintOp_J2K2;
+   bJ2K2.FormSystemMatrix(empty, constraintOp_J2K2);
+
+
+
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      delete pw_c_coeffs[i];
+      delete c_coeffs[i];
+      delete cPrbb_cf[i]; 
+      delete cPibb_cf[i]; 
+      delete signedcPrbb_cf[i]; 
+      delete signedcPibb_cf[i]; 
+   }
+
+   // Assemble all the system and get the global matrix and right-hand side
+   Array<ParFiniteElementSpace *> all_pfes;
+   all_pfes.Append(fosls_pfes);
+   all_pfes.Append(constraints_pfes);
+
+   int npfes = all_pfes.Size();
+   Array<int> all_offsets(npfes + 1); all_offsets[0] = 0;
+   Array<int> all_toffsets(npfes + 1); all_toffsets[0] = 0;
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      all_offsets[i+1] = all_pfes[i]->GetVSize();
+      all_toffsets[i+1] = all_pfes[i]->GetTrueVSize();
+   }
+   all_offsets.PartialSum();
+   all_toffsets.PartialSum();
+
+
+   // put all the operators into a block operator
+   BlockOperator A_r(all_toffsets);
+   BlockOperator A_i(all_toffsets);
+
+   BlockOperator * BlockAfosls_r = dynamic_cast<BlockOperator *>(&fosls_Ac->real());
+   BlockOperator * BlockAfosls_i = dynamic_cast<BlockOperator *>(&fosls_Ac->imag());
+
+
+   for (int i = 0; i < fosls_pfes.Size(); ++i)
+   {
+      for (int j = 0; j < fosls_pfes.Size(); ++j)
+      {
+         A_r.SetBlock(i, j, &BlockAfosls_r->GetBlock(i, j));
+         A_i.SetBlock(i, j, &BlockAfosls_i->GetBlock(i, j));
+      }
+   }
+
+
+   A_r.SetBlock(fosls_pfes.Size()+0, 0, constraintOp_EK1_r.Ptr());
+   A_i.SetBlock(fosls_pfes.Size()+0, 0, constraintOp_EK1_i.Ptr());
+
+   A_r.SetBlock(fosls_pfes.Size()+1, 0, constraintOp_EK2_r.Ptr());
+   A_i.SetBlock(fosls_pfes.Size()+1, 0, constraintOp_EK2_i.Ptr());
+
+   A_r.SetBlock(fosls_pfes.Size()+0, 2, constraintOp_J1K1.Ptr());
+   A_r.SetBlock(fosls_pfes.Size()+1, 3, constraintOp_J2K2.Ptr());
+
+   // Construct the adjoint operator B̄ᵀ
+   HypreParMatrix * constraintOp_EK1_r_t = constraintOp_EK1_r.As<HypreParMatrix>()->Transpose();
+   HypreParMatrix * constraintOp_EK1_i_t = constraintOp_EK1_i.As<HypreParMatrix>()->Transpose();
+   HypreParMatrix * constraintOp_EK2_r_t = constraintOp_EK2_r.As<HypreParMatrix>()->Transpose();
+   HypreParMatrix * constraintOp_EK2_i_t = constraintOp_EK2_i.As<HypreParMatrix>()->Transpose();
+   HypreParMatrix * constraintOp_J1K1_t = constraintOp_J1K1.As<HypreParMatrix>()->Transpose();
+   HypreParMatrix * constraintOp_J2K2_t = constraintOp_J2K2.As<HypreParMatrix>()->Transpose();
+
+   // We also need to scale the imaginary part of B̄ᵀ by -1;
+   *constraintOp_EK1_i_t *= -1.0;
+   *constraintOp_EK2_i_t *= -1.0;
+
+   A_r.SetBlock(0, fosls_pfes.Size()+0, constraintOp_EK1_r_t);
+   A_i.SetBlock(0, fosls_pfes.Size()+0, constraintOp_EK1_i_t);    
+   A_r.SetBlock(0, fosls_pfes.Size()+1, constraintOp_EK2_r_t);
+   A_i.SetBlock(0, fosls_pfes.Size()+1, constraintOp_EK2_i_t);
+   A_r.SetBlock(2, fosls_pfes.Size()+0, constraintOp_J1K1_t);
+   A_r.SetBlock(3, fosls_pfes.Size()+1, constraintOp_J2K2_t);
+
+   //    // dammies for other coupling terms in B
+   // ParMixedBilinearForm bHK1_r(fosls_pfes[1], constraints_pfes[0]); bHK1_r.Assemble();
+   // ParMixedBilinearForm bHK1_i(fosls_pfes[1], constraints_pfes[0]); bHK1_i.Assemble();
+   // OperatorPtr constraintOp_HK1_r, constraintOp_HK1_i;
+   // bHK1_r.FormRectangularSystemMatrix(empty, empty, constraintOp_HK1_r);
+   // bHK1_i.FormRectangularSystemMatrix(empty, empty, constraintOp_HK1_i);
+   // A_r.SetBlock(fosls_pfes.Size()+0, 1, constraintOp_HK1_r.Ptr());
+   // A_i.SetBlock(fosls_pfes.Size()+0, 1, constraintOp_HK1_i.Ptr());
+   
+   // ParMixedBilinearForm bHK2_r(fosls_pfes[1], constraints_pfes[1]); bHK2_r.Assemble(); 
+   // ParMixedBilinearForm bHK2_i(fosls_pfes[1], constraints_pfes[1]); bHK2_i.Assemble();
+   
+   // ParMixedBilinearForm bK1H_r(constraints_pfes[0], fosls_pfes[1]); bK1H_r.Assemble();
+   // ParMixedBilinearForm bK1H_i(constraints_pfes[0], fosls_pfes[1]); bK1H_i.Assemble();
+   // ParMixedBilinearForm bK2H_r(fosls_pfes[1], constraints_pfes[1]); bK2H_r.Assemble();
+   // ParMixedBilinearForm bK2H_i(fosls_pfes[1], constraints_pfes[0]); bK2H_i.Assemble();
+
+
+
+   // ParBilinearForm bJ1K2(constraints_pfes[0]);
+
+
+
+
+   ComplexOperator * A = new ComplexOperator(&A_r, &A_i, false, false);
+   if (Mpi::Root())
+   {
+      mfem::out << "Complex Operator A finished successfully." << endl;
+   }
+
+
+   socketstream E_out_r;
+   Vector x(2*all_offsets.Last());
+   x = 0.;
+   Array<ParGridFunction *> pgf_r(npfes);
+   Array<ParGridFunction *> pgf_i(npfes);
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i] = new ParGridFunction(all_pfes[i], x, all_offsets[i]);
+      pgf_i[i] = new ParGridFunction(all_pfes[i], x, all_offsets.Last() + all_offsets[i]);
+   }
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+   ParGridFunction E_par_r(&L2_fes);
+   ParGridFunction E_par_i(&L2_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+   std::string output_dir = "ParaView/FOSLS-FEM/" + GetTimestamp();
+
+
+   if (paraview)
+   {
+      if (Mpi::Root()) { WriteParametersToFile(args, output_dir); }
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order
+                         << "_eld_1" ;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("Er",pgf_r[0]);
+      paraview_dc->RegisterField("Ei",pgf_i[0]);
+      paraview_dc->RegisterField("E_par_r",&E_par_r);
+      paraview_dc->RegisterField("E_par_i",&E_par_i);
+      paraview_dc->RegisterField("Hr",pgf_r[1]);
+      paraview_dc->RegisterField("Hi",pgf_i[1]);      
+      paraview_dc->RegisterField("J1r",pgf_r[2]);
+      paraview_dc->RegisterField("J1i",pgf_i[2]);
+      paraview_dc->RegisterField("J2r",pgf_r[3]);
+      paraview_dc->RegisterField("J2i",pgf_i[3]);
+      paraview_dc->RegisterField("Q1r",pgf_r[4]);
+      paraview_dc->RegisterField("Q1i",pgf_i[4]);
+      paraview_dc->RegisterField("Q2r",pgf_r[5]);
+      paraview_dc->RegisterField("Q2i",pgf_i[5]);
+   }
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listJ;
+   Array<int> ess_tdof_listQ;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+
+      // remove internal boundaries
+      for (int i = 0; i<int_bdr_attr.Size(); i++)
+      {
+         ess_bdr[int_bdr_attr[i]-1] = 0;
+      }
+      all_pfes[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += all_toffsets[0];
+      }
+      ess_bdr = 1;
+      for (int i = 0; i<ndiffusionequations;i++)
+      {
+         ess_tdof_listJ.SetSize(0);
+         ess_tdof_listQ.SetSize(0);
+         all_pfes[i+2]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJ);
+         all_pfes[i+4]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listQ);
+         for (int j = 0; j < ess_tdof_listJ.Size(); j++)
+         {
+            ess_tdof_listJ[j] += all_toffsets[i+2];
+         }
+         for (int j = 0; j < ess_tdof_listQ.Size(); j++)
+         {
+            ess_tdof_listQ[j] += all_toffsets[i+4];
+         }
+         ess_tdof_list.Append(ess_tdof_listJ);
+         ess_tdof_list.Append(ess_tdof_listQ);
+      }
+
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+
+   pgf_r[0]->ProjectBdrCoefficientTangent(one_x_cf, one_r_bdr);
+   pgf_r[0]->ProjectBdrCoefficientTangent(negone_x_cf, negone_r_bdr);
+   pgf_i[0]->ProjectBdrCoefficientTangent(one_x_cf, one_i_bdr);
+   pgf_i[0]->ProjectBdrCoefficientTangent(negone_x_cf, negone_i_bdr);
+
+   BlockOperator * P = new BlockOperator(all_offsets, all_toffsets);
+   BlockMatrix * R = new BlockMatrix(all_toffsets, all_offsets);
+   P->owns_blocks = 0;
+   R->owns_blocks = 0;
+
+   for (int i = 0; i < npfes; i++)
+   {
+      HypreParMatrix * P_ = all_pfes[i]->Dof_TrueDof_Matrix();
+      P->SetBlock(i,i,P_);
+      const SparseMatrix * R_ = all_pfes[i]->GetRestrictionMatrix();
+      R->SetBlock(i, i, const_cast<SparseMatrix*>(R_));
+   }
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Build prolongation finished" << endl;
+   }
+
+   int n = P->Width();
+   Vector B(2*n); B = 0.0;
+
+   Vector X(2*n);
+   Vector X_r(X, 0, n);
+   Vector X_i(X, n, n);
+
+   Vector x_r(x, 0, x.Size()/2);
+   Vector x_i(x, x.Size()/2, x.Size()/2);
+
+   R->Mult(x_r, X_r);
+   R->Mult(x_i, X_i);
+
+   ParBlockComplexSystem aa(A);
+   A = aa.EliminateBC(ess_tdof_list, X, B);
+   ess_tdof_list.Print(mfem::out);
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Eliminate BC finished successfully." << endl;
+   }
+
+   bool direct_solve = true;
+
+   if (direct_solve)
+   {
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&A->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&A->imag());
+
+      int nblocks = BlockA_r->NumRowBlocks();
+      Array2D<const HypreParMatrix*> A_r_matrices(nblocks, nblocks);
+      Array2D<const HypreParMatrix*> A_i_matrices(nblocks, nblocks);
+      for (int i = 0; i < nblocks; i++)
+      {
+         for (int j = 0; j < nblocks; j++)
+         {
+            if (BlockA_r->IsZeroBlock(i,j))
+            {
+               A_r_matrices(i,j) = nullptr;
+            }
+            else
+            {
+               A_r_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockA_r->GetBlock(i,j));
+            }
+            if (BlockA_i->IsZeroBlock(i,j))
+            {
+               A_i_matrices(i,j) = nullptr;
+            }
+            else
+            {
+               A_i_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockA_i->GetBlock(i,j));
+            }   
+         }
+      }
+      HypreParMatrix * Ahr = HypreParMatrixFromBlocks(A_r_matrices);
+      HypreParMatrix * Ahi = HypreParMatrixFromBlocks(A_i_matrices);
+
+      ComplexHypreParMatrix * Ahc_hypre =
+         new ComplexHypreParMatrix(Ahr, Ahi,true, true);
+
+      ComplexMUMPSSolver cmumps(MPI_COMM_WORLD);
+      cmumps.SetPrintLevel(1);
+      cmumps.SetOperator(*Ahc_hypre);
+
+      if (Mpi::Root())
+      {
+         mfem::out << "Setup CMUMPS solver finished successfully." << endl;
+      }
+
+      cmumps.Mult(B,X);
+      delete Ahc_hypre;   
+   }
+   else
+   {
+      MFEM_ABORT("to be implemented: iterative solver with preconditioner");
+   }
+
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Solve finished successfully." << endl;
+   }
+
+
+   n = P->Height();
+   int m = P->Width();
+
+   x_r.MakeRef(x, 0, n);
+   x_i.MakeRef(x, n, n);
+
+   X_r.MakeRef(X, 0, m);
+   X_i.MakeRef(X, m, m);
+
+   P->Mult(X_r, x_r);
+   P->Mult(X_i, x_i);
+
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i]->MakeRef(all_pfes[i], x, all_offsets[i]);
+      pgf_i[i]->MakeRef(all_pfes[i], x, all_offsets.Last() + all_offsets[i]);
+   }
+   
+   ParallelECoefficient par_e_r(pgf_r[0]);
+   ParallelECoefficient par_e_i(pgf_i[0]);
+   E_par_r.ProjectCoefficient(par_e_r);
+   E_par_i.ProjectCoefficient(par_e_i);
+   
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, *pgf_r[0],
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((real_t)0);
+      paraview_dc->Save();
+      delete paraview_dc;
+   }
+
+
+
+   return 0;
+
+
+   
+
+
+   // a->RecoverFEMSolution(X, x);
+
+   // for (int i = 0; i < npfes; ++i)
+   // {
+   //    pgf_r[i]->MakeRef(pfes[i], x, offsets[i]);
+   //    pgf_i[i]->MakeRef(pfes[i], x, offsets.Last() + offsets[i]);
+   // }
+   
+   // ParallelECoefficient par_e_r(pgf_r[0]);
+   // ParallelECoefficient par_e_i(pgf_i[0]);
+   // E_par_r.ProjectCoefficient(par_e_r);
+   // E_par_i.ProjectCoefficient(par_e_i);
+   
+   // // // rescale the J solutions
+   // for (int i = 0; i < 2*ndiffusionequations; ++i)
+   // {
+   //    (*pgf_r[2+i]) /= balance_scale;
+   //    (*pgf_i[2+i]) /= balance_scale;
+   // }
+
+   // if (visualization)
+   // {
+   //    const char * keys = nullptr;
+   //    char vishost[] = "localhost";
+   //    int  visport   = 19916;
+   //    common::VisualizeField(E_out_r,vishost, visport, *pgf_r[0],
+   //                           "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   // }
+
+   // if (paraview)
+   // {
+   //    paraview_dc->SetCycle(0);
+   //    paraview_dc->SetTime((real_t)0);
+   //    paraview_dc->Save();
+   //    delete paraview_dc;
+   // }
+
+
+   // delete a;
+   // for (int i = 0; i < trial_fecols.Size(); ++i)
+   // {
+   //    delete trial_fecols[i];
+   //    delete pfes[i];
+   // }
+   // for (int i = 0; i< test_fecols.Size(); ++i)
+   // {
+   //    delete test_fecols[i];
+   // }
+
+   // return 0;
+}

miniapps/dpg/plasma/lh-eld-uw.cpp

@@ -0,0 +1,980 @@
+//                   MFEM UW parallel example
+//
+
+// Electron Landau Damping
+//
+//        ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵᵣ E + i ω²ϵ₀(J₁ + J₂) = 0,   in Ω
+//                   - (b⋅∇)²J₁ + c₁ J₁ - c₁ P b⊗b E = 0,   in Ω     
+//                   - (b⋅∇)²J₂ + c₂ J₂ + c₂ P b⊗b E = 0,   in Ω 
+//                                               E×n = E₀,  on ∂Ω
+//                                               J₁  = 0,   on ∂Ω
+//                                               J₂  = 0,   on ∂Ω
+// First order system:
+//     i ω μ₀ H + ∇ × E                    = 0, in Ω
+// -i ω ϵ₀ ϵᵣ E + ∇ × H - α ω ϵ₀ (J₁ + J₂) = 0, in Ω 
+//            b ⋅ ∇Q₁ + c₁ J₁ - c₁ P B E = 0, in Ω
+//                          Q₁ + b ⋅ ∇J₁ = 0, in Ω 
+//            b ⋅ ∇Q₂ + c₂ J₂ + c₂ P B E = 0, in Ω  
+//                          Q₂ + b ⋅ ∇J₂ = 0, in Ω
+// where B = b ⊗ b. Here α serves as a scaling factor. For α = 0 the system becomes 
+// lower trianular (Maxwell decouples from the diffusion equations).
+
+// Note that in 2D we have the following 2 definitions of the curl.
+// a) for a 2D vector E, ∇ × E:= ∇ ⋅ (R E) where R = [0 1; -1 0]
+// b) for a scalar H, ∇ × H:= R ∇ H
+
+// Define the group variables 
+// then the strong formulation reads:
+// (A u, v) = (i ω μ₀ H, F) + (∇ × E, F) 
+//          - (i ω ϵ₀ ϵᵣ E, W) + (∇ × H, W) - (α ω ϵ₀ (J₁ + J₂), W)
+//          + (b ⋅ ∇Q₁, K₁) + (c₁ J₁, K₁) - (c₁ P B E, K₁) + (Q₁, L₁) + (b ⋅ ∇J₁, L₁)
+//          + (b ⋅ ∇Q₂, K₂) + (c₂ J₂, K₂) + (c₂ P B E, K₂) + (Q₂, L₂) + (b ⋅ ∇J₂, L₂) 
+//
+// and the adjoint operator is defined by:
+// (u, A^⋆ v) = (E, ∇×F + i ω ϵ₀ ϵ⋆ᵣ W - c₁ P̄ B K₁ + c₂ P̄ B K₂)
+//            + (H, -i ω μ₀ F + ∇ × W)
+//            + (J₁, -α ω ϵ₀ W + c₁ K₁ - b ⋅ ∇ L₁)
+//            + (Q₁, - b ⋅ ∇ K₁ + L₁)
+//            + (J₂, -α ω ϵ₀ W + c₂ K₂ - b ⋅ ∇ L₂)
+//            + (Q₂, - b ⋅ ∇ K₂ + L₂)
+// 
+// Define the group trial field and trace variables 
+// u:=(E,H,J₁,Q₁,J₂,Q₂), û:=(Ê, Ĥ, Ĵ₁, Q̂₁, Ĵ₂, Q̂₂) and 
+// the test variable  v:=(F,W,K₁,L₁,K₂,L₂) 
+
+// The ultraweak variational formulation is then given by:
+// Find u ∈ U=((L²(Ω))² × L²(Ω) × (L²(Ω))² × (L²(Ω))² × (L²(Ω))² × (L²(Ω))²),
+// û ∈ Û=( H⁻¹/²(Γₕ) × H¹/²(Γₕ) × (H⁻¹/²(Γₕ))² × (H⁻¹/²(Γₕ))² × (H⁻¹/²(Γₕ))²× (H⁻¹/²(Γₕ))² )
+//  such that ∀ v ∈ V=(H¹(Ω) × H(curl,Ω) × (H¹(Ω))² × (H¹(Ω))² × (H¹(Ω))² × (H¹(Ω))²)
+//  (u, A^⋆ v) + b̂(û, v) = 0,
+// or equivalently
+// 
+// (i ω μ₀ H, F) + (E, ∇×F) + < R Ê, F > = 0,      ∀ F ∈ H¹(Ω)
+// -i ω ϵ₀ ϵᵣ (E, W) + (H, ∇ ×  W) + < Ĥ, W × n > - α ω ϵ₀ (J₁ + J₂, W) = 0,  ∀ W ∈ H(curl,Ω)
+// -(Q, b ⋅ ∇ K₁) + <Q̂₁, K₁> + (c₁ J₁, K₁) - c₁(P B E, K₁) = 0,  ∀ K₁ ∈ (H¹(Ω))²
+//  (Q₁, L₁) - (J₁, b ⋅ ∇L₁) + <Ĵ₁, L₁> = 0,  ∀ L₁ ∈ (H¹(Ω))²
+// -(Q, b⋅ ∇ K₂) + <Q̂₂, K₂> + (c₂ J₂, K₂) + c₂(P B E, K₂) = 0,  ∀ K₂ ∈ (H¹(Ω))²
+//  (Q₂, L₂) - (J₂, b ⋅ ∇L₂) + <Ĵ₂, L₂> = 0,  ∀ L₂ ∈ (H¹(Ω))²
+
+// with the adjoint graph test norm defined by:
+// ‖v‖²_V = ‖A^⋆ v‖² + ‖v‖²
+//        = ‖∇×F + i ω ϵ₀ ϵ⋆ᵣ W - c₁ P̄ B K₁ + c₂ P̄ B K₂‖²
+//        + ‖-i ω μ₀ F + ∇ × W‖²
+//        + ‖-α ω ϵ₀ W + c₁ K₁ - b ⋅ ∇ L₁‖²
+//        + ‖- b ⋅ ∇ K₁ + L₁‖²
+//        + ‖-α ω ϵ₀ W + c₂ K₂ - b ⋅ ∇ L₂‖²
+//        + ‖- b ⋅ ∇ K₂ + L₂‖²
+//        + ‖F‖² + ‖W‖² + ‖K₁‖² + ‖L₁‖² + ‖K₂‖² + ‖L₂‖²
+//
+
+#include "mfem.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../../common/mfem-common.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "utils/lh_utils.hpp"
+#include "../util/utils.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+// Define spaces
+enum TrialSpace
+{
+   E_space     = 0,
+   H_space     = 1,
+   J1_space    = 2,
+   Q1_space    = 3,
+   J2_space    = 4,
+   Q2_space    = 5,
+   hatE_space  = 6,
+   hatH_space  = 7,
+   hatJ1_space = 8,
+   hatQ1_space = 9,
+   hatJ2_space = 10,
+   hatQ2_space = 11
+};
+
+enum TestSpace
+{
+   F_space = 0,
+   W_space = 1,
+   K1_space = 2,
+   L1_space = 3,
+   K2_space = 4,
+   L2_space = 5
+};
+
+inline const char * ToString(TrialSpace ts)
+{
+   switch (ts)
+   {
+      case E_space:     return " L²(Ω)²  for E  ";
+      case H_space:     return " L²(Ω)   for H  ";
+      case J1_space:    return " L²(Ω)²  for J₁ ";
+      case Q1_space:    return " L²(Ω)²  for Q₁ ";
+      case J2_space:    return " L²(Ω)²  for J₂ ";
+      case Q2_space:    return " L²(Ω)²  for Q₂ ";
+      case hatE_space:  return " tr(RT)  for Ê  ";
+      case hatH_space:  return " tr(H¹)  for Ĥ  ";
+      case hatJ1_space: return " tr(RT)² for Ĵ₁ ";
+      case hatQ1_space: return " tr(RT)² for Q̂₁ ";
+      case hatJ2_space: return " tr(RT)² for Ĵ₂ ";
+      case hatQ2_space: return " tr(RT)² for Q̂₂ ";
+      default:          return "Unknown TrialSpace";
+   }
+}
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+   int order = 1;
+   int delta_order = 1;
+   int par_ref_levels = 0;
+   int ser_ref_levels = 0;
+
+   real_t rnum=1.5;
+   real_t mu = 1.257;
+   real_t eps0 = 8.8541878128;
+   real_t alpha = 1.0; // scaling factor for E, Js coupling
+   bool static_cond = false;
+   bool visualization = false;
+   bool paraview = false;
+   bool debug = false;
+   bool mumps_solver = false;
+
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&delta_order, "-do", "--delta-order",
+                  "Finite element order for the test space");                  
+   args.AddOption(&ser_ref_levels, "-sr", "--serial-refinement_levels",
+                  "Number of serial refinement levels.");                  
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&a0, "-a0", "--a0", "P(r) first parameter.");
+   args.AddOption(&a1, "-a1", "--a1", "P(r) second parameter.");
+   args.AddOption(&alpha, "-alpha", "--alpha",
+                  "Scaling factor for E, Js coupling.");
+   args.AddOption(&mumps_solver, "-mumps", "--mumps", "-no-mumps",
+                  "--no-mumps",
+                  "Enable or disable MUMPS solver.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&debug, "-debug", "--debug", "-no-debug",
+                  "--no-debug",
+                  "Enable or disable debug mode (delta = 0.01 and no coupling).");         
+   args.Parse();
+   if (!args.Good())
+   {
+      if (Mpi::Root())
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (Mpi::Root())
+   {
+      args.PrintOptions(cout);
+   }
+
+   // number of diffusion equations
+   int ndiffusionequations = 2; 
+
+   if (!debug) 
+   {
+      delta = 0.0; // disable delta if electron Landau damping is enabled
+      if (Mpi::Root())
+      {
+         cout << "Electron Landau damping enabled, delta set to 0.0." << endl;
+      }
+   }    
+   else
+   {
+      if (Mpi::Root())
+      {
+         alpha = 0.0; 
+         cout << "Setting alpha = 0.0 to disable coupling between E and Js." << endl;
+         cout << "Debug mode enabled, delta = 0.01 and no coupling." << endl;
+      }
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   MFEM_VERIFY(dim == 2, "Dimension != 2 is not supported in this example");
+
+   for (int i = 0; i < ser_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   Array<int> int_bdr_attr;
+   for (int i = 0; i < mesh.GetNBE(); i++)
+   {
+      if (mesh.FaceIsInterior(mesh.GetBdrElementFaceIndex(i)))
+      {
+         int_bdr_attr.Append(mesh.GetBdrAttribute(i));
+      }
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   for (int i = 0; i < par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }  
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+
+   real_t omega = 2.*M_PI*rnum;
+   real_t neg_omega_eps0 = -omega * eps0;
+   real_t omega_eps0 = omega * eps0;
+
+   Vector cvals(ndiffusionequations);
+   Vector csigns(ndiffusionequations);
+   real_t cfactor = 1e-6;
+   cvals(0)  = 25e6;  cvals(1)  = 1e6;
+   csigns(0) = -1.0;  csigns(1) = 1.0;
+   cvals *= cfactor; // scale the coefficients
+
+   // List of coefficients
+   // Trial integrators coefficients
+   DenseMatrix R(dim), Rt(dim);
+   R(0,0) =  0.0;  R(0,1) = 1.0;
+   R(1,0) = -1.0;  R(1,1) = 0.0;
+   R.Transpose(Rt);
+
+   VectorFunctionCoefficient no_scale_bvec_cf(dim,bfunc);// b
+   real_t scaled_cfactor = std::sqrt(cfactor);
+   ScalarVectorProductCoefficient bvec_cf(scaled_cfactor,no_scale_bvec_cf); // scaled b
+   // VectorFunctionCoefficient bvec_cf(dim,bfunc);// b
+   ScalarVectorProductCoefficient neg_bvec_cf(-1.0,bvec_cf); // -b
+   ScalarVectorProductCoefficient alpha_omega_eps0_bvec_cf(eps0*omega*alpha,bvec_cf); // α ω ϵ₀ b
+
+
+   MatrixFunctionCoefficient B_cf(dim, bcrossb);
+   FunctionCoefficient p_r_cf(pfunc_r);
+   FunctionCoefficient p_i_cf(pfunc_i);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   Array<Vector *> cf_arrays(ndiffusionequations);
+   Array<PWConstCoefficient *> c_cf(ndiffusionequations);
+   Array<ProductCoefficient *> neg_c_alpha_omega_eps0_cf(ndiffusionequations);
+   Array<ProductCoefficient *> c2_cf(ndiffusionequations);
+   Array<ProductCoefficient *> signed_c_cf(ndiffusionequations);
+   Array<MatrixCoefficient*> signed_PB_r_cf(ndiffusionequations);
+   Array<MatrixCoefficient*> signed_PB_i_cf(ndiffusionequations);
+   Array<ProductCoefficient*> neg_signed_c_omega_eps_cf(ndiffusionequations);
+   Array<ProductCoefficient*> signed_c_omega_eps_cf(ndiffusionequations);
+   Array<MatrixCoefficient*> signed_PBR_r_cf(ndiffusionequations);
+   Array<MatrixCoefficient*> signed_PBR_i_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> c2_absP2BB_cf(ndiffusionequations);
+   Array<VectorCoefficient*> neg_c_bvec_cf(ndiffusionequations);
+
+   Vector zerovec(nattr); zerovec=0.0;
+
+   // ϵᵣ in plasma (real)
+   MatrixFunctionCoefficient eps_r_temp_cf(dim, epsilon_func_r);
+   // ϵᵣ in plasma(imag)
+   MatrixFunctionCoefficient eps_i_temp_cf(dim, epsilon_func_i);
+
+   coefs_r[nattr-1] = &eps_r_temp_cf;
+   coefs_i[nattr-1] = &eps_i_temp_cf;
+
+   // ωμ₀ 
+   ConstantCoefficient omegamu_cf(mu * omega);
+   // -ωμ₀
+   ConstantCoefficient neg_omegamu_cf(-mu * omega);
+   // ω² μ₀²
+   ConstantCoefficient omega2mu2_cf(mu * mu * omega * omega);  
+   // R 
+   MatrixConstantCoefficient R_cf(R);
+   // Rᵀ
+   MatrixConstantCoefficient Rt_cf(Rt);
+   // ϵᵣ (real)
+   PWMatrixCoefficient eps_r_cf(dim, attr, coefs_r);
+   // ϵᵣ (imag)
+   PWMatrixCoefficient eps_i_cf(dim, attr, coefs_i);
+   // -ω ϵ₀ ϵᵣ (real)
+   ScalarMatrixProductCoefficient neg_omega_eps0_eps_r_cf(neg_omega_eps0, eps_r_cf);
+   // ω ϵ₀ ϵᵣ (imag)
+   ScalarMatrixProductCoefficient omega_eps0_eps_i_cf(omega_eps0, eps_i_cf);
+   // ω ϵ₀ ϵᵣᵢ R 
+   MatrixProductCoefficient omega_eps0_eps_i_R_cf(omega_eps0_eps_i_cf, R_cf);
+   // -ω ϵ₀ ϵᵣᵣ R
+   MatrixProductCoefficient neg_omega_eps0_eps_r_R_cf(neg_omega_eps0_eps_r_cf, R_cf); 
+   // 
+   // ω ϵ₀ Rᵀ ϵᵣᵢ
+   MatrixProductCoefficient omega_eps0_Rt_eps_i_cf(Rt_cf, omega_eps0_eps_i_cf);
+   // ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient omega_eps0_eps_r_cf(omega_eps0, eps_r_cf);
+   // ω ϵ₀ Rᵀ ϵᵣᵣ 
+   MatrixProductCoefficient omega_eps0_Rt_eps_r_cf(Rt_cf, omega_eps0_eps_r_cf);
+   // |ϵᵣ|² = ϵᵣᵣϵᵣᵣ + ϵᵣᵢϵᵣᵢ   
+   MatrixProductCoefficient eps_r_eps_r_cf(eps_r_cf, eps_r_cf);
+   MatrixProductCoefficient eps_i_eps_i_cf(eps_i_cf, eps_i_cf);
+
+   MatrixSumCoefficient abseps2_cf(eps_r_eps_r_cf, eps_i_eps_i_cf);
+   // ω² ϵ₀² 
+   ConstantCoefficient alpha2_omega2_eps02(alpha*alpha*omega * omega * eps0 * eps0);
+   // ω² ϵ₀² |ϵᵣ|²   
+   ScalarMatrixProductCoefficient omega2_eps02_abseps2_cf(omega*omega*eps0*eps0, abseps2_cf);
+
+   // 1
+   ConstantCoefficient one_cf(1.0);
+   // -1
+   ConstantCoefficient neg_one_cf(-1.0);
+   // - ω ϵ₀
+   ConstantCoefficient neg_omega_eps0_cf(neg_omega_eps0);
+   // - α ω ϵ₀
+   ConstantCoefficient neg_alpha_omega_eps0_cf(alpha * neg_omega_eps0); 
+
+   // P B
+   ScalarMatrixProductCoefficient PB_r_cf(p_r_cf, B_cf);
+   ScalarMatrixProductCoefficient PB_i_cf(p_i_cf, B_cf);
+
+   // BB
+   MatrixProductCoefficient BB_cf(B_cf, B_cf);
+   
+   // Pᵣ²
+   ProductCoefficient P2_r_cf(p_r_cf, p_r_cf);
+   // Pᵢ²
+   ProductCoefficient P2_i_cf(p_i_cf, p_i_cf);
+   // |P|²
+   SumCoefficient absP2_cf(P2_r_cf, P2_i_cf);
+   // |P|² B B
+   ScalarMatrixProductCoefficient absP2_BB_cf(absP2_cf, BB_cf);
+   // cᵢ  
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      zerovec[nattr-1] = cvals(i);
+      c_cf[i] = new PWConstCoefficient(zerovec);
+      neg_c_alpha_omega_eps0_cf[i] = new ProductCoefficient(-omega_eps0*alpha, *c_cf[i]);
+      c2_cf[i] = new ProductCoefficient(*c_cf[i], *c_cf[i]);
+      signed_c_cf[i] = new ProductCoefficient(csigns(i), *c_cf[i]);
+      signed_PB_r_cf[i] = new ScalarMatrixProductCoefficient(*signed_c_cf[i], PB_r_cf);
+      signed_PB_i_cf[i] = new ScalarMatrixProductCoefficient(*signed_c_cf[i], PB_i_cf);
+      signed_c_omega_eps_cf[i] = new ProductCoefficient(omega_eps0, *signed_c_cf[i]);
+      neg_signed_c_omega_eps_cf[i] = new ProductCoefficient(neg_omega_eps0, *signed_c_cf[i]);
+      // cᵢ P B R
+      signed_PBR_r_cf[i] = new MatrixProductCoefficient(*signed_PB_r_cf[i], R_cf);
+      signed_PBR_i_cf[i] = new MatrixProductCoefficient(*signed_PB_i_cf[i], R_cf);
+      // cᵢ² |P|² B B
+      c2_absP2BB_cf[i] = new ScalarMatrixProductCoefficient(*c2_cf[i], absP2_BB_cf);
+      neg_c_bvec_cf[i] = new ScalarVectorProductCoefficient(-1.0 * cvals(i), bvec_cf);
+   }   
+
+   ProductCoefficient neg_c1_c2_cf(*signed_c_cf[0], *signed_c_cf[1]);
+   // -c₁ c₂ |P|² B B
+   ScalarMatrixProductCoefficient neg_c1_c2_absP2BB_cf(neg_c1_c2_cf, absP2_BB_cf);
+
+
+   MatrixProductCoefficient PB_r_eps_r_cf(PB_r_cf, eps_r_cf);
+   MatrixProductCoefficient PB_r_eps_i_cf(PB_r_cf, eps_i_cf);
+   MatrixProductCoefficient PB_i_eps_r_cf(PB_i_cf, eps_r_cf);
+   MatrixProductCoefficient PB_i_eps_i_cf(PB_i_cf, eps_i_cf);
+
+
+   Array<FiniteElementCollection *> trial_fecols;
+   Array<ParFiniteElementSpace *> trial_pfes;
+   Array<FiniteElementCollection *> test_fecols;
+
+   // L²(Ω)² space for E
+   trial_fecols.Append(new L2_FECollection(order-1, dim));
+   trial_pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last(), dim));
+   // L²(Ω) space for H
+   trial_fecols.Append(new L2_FECollection(order-1, dim));
+   trial_pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last()));
+   // L²(Ω)² space for J₁, Q₁, J₂, Q₂
+   for (int i = 0; i < 2*ndiffusionequations; ++i)
+   {
+      trial_fecols.Append(new L2_FECollection(order-1, dim));
+      trial_pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last(), dim));
+   }
+   // H⁻¹/²(Γₕ) space for Ê
+   trial_fecols.Append(new RT_Trace_FECollection(order-1, dim));
+   trial_pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last()));
+   // H¹/²(Γₕ) space for Ĥ
+   trial_fecols.Append(new H1_Trace_FECollection(order, dim));
+   trial_pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last()));
+   // H⁻¹/²(Γₕ)² space for Ĵ₁, Q̂₁, Ĵ₂, Q̂₂
+   for (int i = 0; i < 2*ndiffusionequations; ++i)
+   {
+      trial_fecols.Append(new RT_Trace_FECollection(order-1, dim));
+      trial_pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last(), dim));
+   }
+
+   Array<HYPRE_BigInt> tdofs(trial_pfes.Size());
+   for (int i = 0; i < trial_pfes.Size(); ++i)
+   {
+      tdofs[i] = trial_pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << "ParFiniteElementSpace " << ToString(TrialSpace(i))  << " has " << tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   if (Mpi::Root())
+   {
+      cout << "Total number of true dofs: " << tdofs.Sum() << endl;
+   }
+
+   // Test spaces   
+   // H¹(Ω) space for F
+   test_fecols.Append(new H1_FECollection(order, dim));
+   // H(curl,Ω) space for W
+   test_fecols.Append(new ND_FECollection(order, dim));
+   // Test spaces for K₁, L₁, K₂, L₂
+   for (int i = 0; i < 2*ndiffusionequations; ++i)
+   {
+      test_fecols.Append(new H1_FECollection(order, dim));
+   }
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_pfes,test_fecols);
+   for (int i = 0; i < 2*ndiffusionequations; ++i)
+   {
+      a->SetTestFECollVdim(i+2,dim);
+   }
+
+   // Trial integrators
+   // i(ω μ₀ H, F)
+   a->AddTrialIntegrator(nullptr,
+                         new MixedScalarMassIntegrator(omegamu_cf),
+                         TrialSpace::H_space,TestSpace::F_space);
+
+   // (E, ∇×F)
+   a->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one_cf)),
+                         nullptr,
+                         TrialSpace::E_space,TestSpace::F_space);
+   
+   // -i ω ϵ₀ ϵᵣ (E, W) = -i(ω ϵ₀ (ϵᵣᵣ+i ϵᵣᵢ)  E, W)   
+   //                   = (ω ϵ₀ ϵᵣᵢ + i (-ωϵ₀ϵᵣᵣ) )  E, W)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(omega_eps0_eps_i_cf)),
+                         new TransposeIntegrator(new VectorFEMassIntegrator(neg_omega_eps0_eps_r_cf)),
+                         TrialSpace::E_space,TestSpace::W_space);
+   
+   // (H, ∇ ×  W)   
+   a->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one_cf)), 
+                         nullptr,
+                         TrialSpace::H_space, TestSpace::W_space);
+   
+
+
+   VectorFunctionCoefficient b_cf(dim, bfunc);
+   ScalarVectorProductCoefficient neg_b_cf(-1.0,b_cf);   
+
+   for (int i = 0; i < ndiffusionequations; i++)
+   {
+      // - α ω ϵ₀ (Jᵢ,W)   
+      a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(neg_alpha_omega_eps0_cf)),
+                            nullptr,
+                            TrialSpace::J1_space + 2*i,TestSpace::W_space);
+      // (Qᵢ, - b ⋅ ∇ Kᵢ)   
+      a->AddTrialIntegrator(new TransposeIntegrator(new DirectionalVectorGradientIntegrator(neg_b_cf)),
+                         nullptr,
+                         TrialSpace::Q1_space + 2*i,TestSpace::K1_space+ 2*i);
+
+      // (cᵢ Jᵢ, Kᵢ)
+      a->AddTrialIntegrator(new VectorMassIntegrator(*c_cf[i]),
+                            nullptr,
+                            TrialSpace::J1_space + 2*i,TestSpace::K1_space + 2*i);       
+                            
+      // (Qᵢ, Lᵢ)
+      a->AddTrialIntegrator(new VectorMassIntegrator(one_cf),
+                            nullptr,
+                            TrialSpace::Q1_space + 2*i,TestSpace::L1_space + 2*i);      
+
+      // (Jᵢ, - b ⋅ ∇Lᵢ)
+      a->AddTrialIntegrator(new TransposeIntegrator(new DirectionalVectorGradientIntegrator(neg_b_cf)),
+                            nullptr,
+                            TrialSpace::J1_space + 2*i,TestSpace::L1_space + 2*i);
+
+      // - c₁(P B E, K₁) = (- c₁ Pᵣ B E, K₁) + i (-c₁ Pᵢ B E, K₁) 
+      //   c₂(P B E, K₂) = (  c₂ Pᵣ B E, K₂) + i ( c₂ Pᵢ B E, K₂) 
+      a->AddTrialIntegrator(new VectorMassIntegrator(*signed_PB_r_cf[i]),
+                            new VectorMassIntegrator(*signed_PB_i_cf[i]),
+                            TrialSpace::E_space,TestSpace::K1_space + 2*i);
+
+      // Trace integrators 
+      // <Q̂ᵢ, Kᵢ>
+      a->AddTrialIntegrator(new VectorTraceIntegrator,nullptr,
+                            TrialSpace::hatQ1_space + 2*i,TestSpace::K1_space + 2*i);
+      // <Ĵᵢ, Lᵢ>
+      a->AddTrialIntegrator(new VectorTraceIntegrator,nullptr,
+                            TrialSpace::hatJ1_space + 2*i,TestSpace::L1_space + 2*i);                      
+
+   }
+
+   // Trace integrators
+   // < R Ê, F > (we include R in the variable)
+   a->AddTrialIntegrator(new TraceIntegrator,
+                         nullptr,
+                         TrialSpace::hatE_space,TestSpace::F_space);
+   //< Ĥ, W × n >                         
+   a->AddTrialIntegrator(new TangentTraceIntegrator,
+                         nullptr,
+                         TrialSpace::hatH_space,TestSpace::W_space);
+
+   // -------------------------------------------------------------------------------   
+   //                         Test integrators 
+   // -------------------------------------------------------------------------------   
+   // (∇×F,∇×δF) = (R ∇ F, R ∇δF) = (Rᵀ R ∇F, ∇δF) = (∇F, ∇δF)
+   a->AddTestIntegrator(new DiffusionIntegrator(one_cf),
+                        nullptr,
+                        TestSpace::F_space, TestSpace::F_space);
+   // i(-ω ϵ₀ ϵᵣ ∇ × F, δW) = i(-ω ϵ₀ (ϵᵣᵣ + i ϵᵣᵢ) ∇ × F, δW)
+   //                       = (ω ϵ₀ ϵᵣᵢ ∇ × F, δW) + i(-ω ϵ₀ ϵᵣᵣ ∇ × F, δW)
+   //                       = (ω ϵ₀ ϵᵣᵢ R ∇ F, δW) + i(-ω ϵ₀ ϵᵣᵣ R ∇ F, δW)
+   a->AddTestIntegrator(new MixedVectorGradientIntegrator(omega_eps0_eps_i_R_cf),
+                        new MixedVectorGradientIntegrator(neg_omega_eps0_eps_r_R_cf),
+                        TestSpace::F_space, TestSpace::W_space);
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      // (-c₁ P B ∇ × F, δK₁) = (-c₁ Pᵣ B ∇ × F, δK₁) + i (-c₁ Pᵢ B ∇ × F, δK₁)
+      // ( c₂ P B ∇ × F, δK₂) = ( c₂ Pᵣ B ∇ × F, δK₂) + i ( c₂ Pᵢ B ∇ × F, δK₂)
+      a->AddTestIntegrator(new MixedCurlIntegrator(*signed_PB_r_cf[i]),
+                           new MixedCurlIntegrator(*signed_PB_i_cf[i]),
+                           TestSpace::F_space, TestSpace::K1_space+ 2*i);
+
+
+
+
+   }                     
+
+   // i(ω ϵ₀ ϵᵣ⋆ W, ∇×δF) = i(ω ϵ₀ (ϵᵣᵣ - i ϵᵣᵢ) W, ∇×δF) (ϵᵣᵣ & ϵᵣᵢ are symmetric)
+   //                     = (ω ϵ₀ ϵᵣᵢ W, ∇×δF) + i(ω ϵ₀ ϵᵣᵣ W, ∇×δF) 
+   //                     = (ω ϵ₀ Rᵀ ϵᵣᵢ W, ∇δF) + i(ω ϵ₀ Rᵀ ϵᵣᵣ W, ∇δF) 
+   a->AddTestIntegrator(new TransposeIntegrator(new MixedVectorGradientIntegrator(omega_eps0_Rt_eps_i_cf)),
+                        new TransposeIntegrator(new MixedVectorGradientIntegrator(omega_eps0_Rt_eps_r_cf)),
+                        TestSpace::W_space, TestSpace::F_space);
+   // (ω² ϵ²₀ ϵᵣϵᵣ⋆ W, δ W) = (ω² ϵ²₀ |ϵᵣ|² W, δW) 
+   a->AddTestIntegrator(new VectorFEMassIntegrator(omega2_eps02_abseps2_cf),
+                        nullptr,
+                        TestSpace::W_space, TestSpace::W_space); 
+   
+   // Pᵢ B ϵᵣᵣ - Pᵣ B ϵᵣᵢ
+   MatrixSumCoefficient PB_i_eps_r_minus_PB_r_eps_i_cf(PB_i_eps_r_cf, PB_r_eps_i_cf, 1.0, -1.0);
+   // Pᵣ B ϵᵣᵣ + Pᵢ B ϵᵣᵢ)
+   MatrixSumCoefficient PB_r_eps_r_plus_PB_i_eps_i_cf(PB_r_eps_r_cf, PB_i_eps_i_cf, 1.0, 1.0);
+   //   ω ϵ₀ c₁ (Pᵢ B ϵᵣᵣ - Pᵣ B ϵᵣᵢ) (negsigned)
+   //  -ω ϵ₀ c₂ (Pᵢ B ϵᵣᵣ - Pᵣ B ϵᵣᵢ) (negsigned) 
+   // - ω ϵ₀ c₁ (Pᵣ B ϵᵣᵣ + Pᵢ B ϵᵣᵢ) (signed)
+   //   ω ϵ₀ c₂ (Pᵣ B ϵᵣᵣ + Pᵢ B ϵᵣᵢ) (signed)
+   Array<MatrixCoefficient*> neg_signed_c_PB_i_eps_r_minus_PB_r_eps_i_cf(ndiffusionequations);
+   Array<MatrixCoefficient*> signed_c_PB_i_eps_r_minus_PB_r_eps_i_cf(ndiffusionequations);
+   Array<MatrixCoefficient*> signed_c_PB_r_eps_r_plus_PB_i_eps_i_cf(ndiffusionequations);
+   Array<MatrixCoefficient*> neg_signed_c_PB_r_eps_r_plus_PB_i_eps_i_cf(ndiffusionequations);
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      neg_signed_c_PB_i_eps_r_minus_PB_r_eps_i_cf[i] = new ScalarMatrixProductCoefficient(*neg_signed_c_omega_eps_cf[i], PB_i_eps_r_minus_PB_r_eps_i_cf);
+      signed_c_PB_r_eps_r_plus_PB_i_eps_i_cf[i] = new ScalarMatrixProductCoefficient(*signed_c_omega_eps_cf[i], PB_r_eps_r_plus_PB_i_eps_i_cf);
+   
+      // i(-ω ϵ₀c₁ P B ϵᵣ⋆ W, δK₁) = ω ϵ₀ c₁ (Pᵢ B ϵᵣᵣ - Pᵣ B ϵᵣᵢ) + i (-ωϵ₀ c₁ (Pᵣ B ϵᵣᵣ + Pᵢ B ϵᵣᵢ))
+      // i( ω ϵ₀c₂ P B ϵᵣ⋆ W, δK₂) = -ω ϵ₀ c₂ (Pᵢ B ϵᵣᵣ - Pᵣ B ϵᵣᵢ) + i (ωϵ₀ c₂ (Pᵣ B ϵᵣᵣ + Pᵢ B ϵᵣᵢ))
+      // TestSpace K1_space and K2_space (2,4)
+      TestSpace tspace = static_cast<TestSpace>(2*i+2);
+      a->AddTestIntegrator(new VectorFEMassIntegrator(*neg_signed_c_PB_i_eps_r_minus_PB_r_eps_i_cf[i]),
+                           new VectorFEMassIntegrator(*signed_c_PB_r_eps_r_plus_PB_i_eps_i_cf[i]),
+                           TestSpace::W_space, tspace);
+   
+      signed_c_PB_i_eps_r_minus_PB_r_eps_i_cf[i] = new ScalarMatrixProductCoefficient(*signed_c_omega_eps_cf[i], PB_i_eps_r_minus_PB_r_eps_i_cf);
+      neg_signed_c_PB_r_eps_r_plus_PB_i_eps_i_cf[i] = new ScalarMatrixProductCoefficient(*neg_signed_c_omega_eps_cf[i], PB_r_eps_r_plus_PB_i_eps_i_cf);
+      // Note that P is scalar and also ϵᵣ B = B ϵᵣ 
+      // i( ω ϵ₀c₁ ϵᵣ P̄ B K₁, δW) = i (K₁,  ω ϵ₀ c₁ P B ϵᵣ⋆ δW) 
+      // i(-ω ϵ₀c₂ ϵᵣ P̄ B K₂, δW) = i (K₂, -ω ϵ₀ c₂ P B ϵᵣ⋆ δW)
+      a->AddTestIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(*signed_c_PB_i_eps_r_minus_PB_r_eps_i_cf[i])),
+                           new TransposeIntegrator(new VectorFEMassIntegrator(*neg_signed_c_PB_r_eps_r_plus_PB_i_eps_i_cf[i])),
+                           tspace, TestSpace::W_space);
+      // (c²₁ P P̄ B B K₁, δK₁)
+      // (c²₂ P P̄ B B K₂, δK₂)
+      a->AddTestIntegrator(new VectorMassIntegrator(*c2_absP2BB_cf[i]),
+                           nullptr,
+                           tspace, tspace);
+
+      // (c₁²K₁, δK₁) 
+      // (c₂²K₂, δK₂)
+      a->AddTestIntegrator(new VectorMassIntegrator(*c2_cf[i]),
+                           nullptr,
+                           tspace, tspace);     
+                           
+      // (-c₁ α ω ϵ₀ W, δK₁) 
+      // (-c₂ α ω ϵ₀ W, δK₂)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(*neg_c_alpha_omega_eps0_cf[i]),
+                           nullptr,
+                           TestSpace::W_space, tspace);
+      // (-c₁ α  ω ϵ₀ K₁, δW) 
+      // (-c₂ α  ω ϵ₀ K₂, δW)                            
+      a->AddTestIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(*neg_c_alpha_omega_eps0_cf[i])),
+                           nullptr,
+                           tspace, TestSpace::W_space);
+
+      // (-c₁ b⋅∇L₁, δK₁) 
+      // (-c₂ b⋅∇L₂, δK₂) 
+      TestSpace Kspace = static_cast<TestSpace>(2*i+2);
+      TestSpace Lspace = static_cast<TestSpace>(2*i+3);
+
+      a->AddTestIntegrator(new DirectionalVectorGradientIntegrator(*neg_c_bvec_cf[i]),
+                           nullptr,
+                           Lspace, Kspace);
+      // (K₁,-c₁ b ⋅ ∇δL₁)
+      // (K₂,-c₂ b ⋅ ∇δL₂)
+      a->AddTestIntegrator(new TransposeIntegrator(new DirectionalVectorGradientIntegrator(*neg_c_bvec_cf[i])),
+                           nullptr,
+                           Kspace, Lspace);
+      // (L₁, δL₁) 
+      // (L₂, δL₂)
+      a->AddTestIntegrator(new VectorMassIntegrator(one_cf),
+                           nullptr,
+                           Lspace, Lspace);   
+      // (b⋅ ∇L₁, b ⋅ ∇δL₁)   
+      // (b⋅ ∇L₂, b ⋅ ∇δL₂)
+      a->AddTestIntegrator(new DirectionalVectorDiffusionIntegrator(bvec_cf),
+                           nullptr, Lspace, Lspace);
+      // (b ⋅ ∇K₁, b ⋅ ∇δK₁)
+      // (b ⋅ ∇K₂, b ⋅ ∇δK₂)       
+      a->AddTestIntegrator(new DirectionalVectorDiffusionIntegrator(bvec_cf),
+                           nullptr, Kspace, Kspace);                    
+
+      //-(b ⋅ ∇K₁, δL₁) 
+      //-(b ⋅ ∇K₂, δL₂) 
+      a->AddTestIntegrator(new DirectionalVectorGradientIntegrator(neg_b_cf),
+                           nullptr, Kspace, Lspace);
+      //(L₁, - b ⋅ ∇δK₁) 
+      //(L₂, - b ⋅ ∇δK₂) 
+      a->AddTestIntegrator(new TransposeIntegrator(new DirectionalVectorGradientIntegrator(neg_b_cf)),
+                           nullptr, Lspace, Kspace);
+
+      // (α ω ϵ₀ b⋅ ∇L₁, δW) 
+      // (α ω ϵ₀ b⋅ ∇L₂, δW)                            
+      a->AddTestIntegrator(new MixedDirectionalVectorGradientIntegrator(alpha_omega_eps0_bvec_cf),
+                           nullptr,
+                           Lspace, TestSpace::W_space);
+      // (W, α ω ϵ₀ b ⋅ ∇δL₁)
+      // (W, α ω ϵ₀ b ⋅ ∇δL₂)
+      a->AddTestIntegrator(new TransposeIntegrator(new MixedDirectionalVectorGradientIntegrator(alpha_omega_eps0_bvec_cf)),
+                           nullptr,
+                           TestSpace::W_space, Lspace);
+
+   }
+
+   // (-c₁ P̄ B K₁, ∇×δF) = (K₁, -c₁ P B ∇×δF) = (K₁, -c₁ Pᵣ B ∇×δF) + i (K₁, -c₁ Pᵢ B ∇×δF)
+   a->AddTestIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(*signed_PB_r_cf[0])),
+                        new TransposeIntegrator(new MixedCurlIntegrator(*signed_PB_i_cf[0])),
+                        TestSpace::K1_space, TestSpace::F_space); 
+   // (c₂ P̄ B K₂, ∇×δF) = (K₂, c₂ P B ∇×δF) = (K₂, c₂ Pᵣ B ∇×δF) + i (K₂, c₂ Pᵢ B ∇×δF)
+   a->AddTestIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(*signed_PB_r_cf[1])),
+                        new TransposeIntegrator(new MixedCurlIntegrator(*signed_PB_i_cf[1])),
+                        TestSpace::K2_space, TestSpace::F_space); 
+
+
+   // (- c₁c₂ P P̄ B B K₁, δK₂)
+   // (- c₁c₂ P P̄ B B K₂, δK₁)
+   a->AddTestIntegrator(new VectorMassIntegrator(neg_c1_c2_absP2BB_cf),
+                        nullptr,
+                        TestSpace::K1_space, TestSpace::K2_space);
+   a->AddTestIntegrator(new VectorMassIntegrator(neg_c1_c2_absP2BB_cf),
+                        nullptr,
+                        TestSpace::K2_space, TestSpace::K1_space);
+
+   // (∇ × W, ∇ × δW)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one_cf),
+                        nullptr,
+                        TestSpace::W_space, TestSpace::W_space);
+   // i(ω μ₀ ∇ × W, δF)
+   a->AddTestIntegrator(nullptr,
+                        new MixedCurlIntegrator(omegamu_cf),
+                        TestSpace::W_space, TestSpace::F_space);
+   // i(-ω μ₀ F, ∇ ×δW)
+   a->AddTestIntegrator(nullptr, 
+                        new TransposeIntegrator(new MixedCurlIntegrator(neg_omegamu_cf)),
+                        TestSpace::F_space, TestSpace::W_space);
+   // (ω² μ₀² F, δF)
+   a->AddTestIntegrator(new MassIntegrator(omega2mu2_cf),
+                        nullptr,
+                        TestSpace::F_space, TestSpace::F_space);
+   // (α² ω² ϵ₀² W, δW) 
+   a->AddTestIntegrator(new VectorFEMassIntegrator(alpha2_omega2_eps02),
+                        nullptr,
+                        TestSpace::W_space,TestSpace::W_space);   
+
+   // (F, δF) 
+   a->AddTestIntegrator(new MassIntegrator(one_cf),
+                        nullptr,
+                        TestSpace::F_space,TestSpace::F_space);
+   // (W, δW) 
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one_cf),
+                        nullptr,
+                        TestSpace::W_space,TestSpace::W_space);
+   // (K₁, δK₁) 
+   a->AddTestIntegrator(new VectorMassIntegrator(one_cf),
+                        nullptr,
+                        TestSpace::K1_space,TestSpace::K1_space);                  
+   // (L₁, δL₁)
+   a->AddTestIntegrator(new VectorMassIntegrator(one_cf),
+                        nullptr,
+                        TestSpace::L1_space,TestSpace::L1_space); 
+   // (K₂, δK₂) 
+   a->AddTestIntegrator(new VectorMassIntegrator(one_cf),
+                        nullptr,
+                        TestSpace::K2_space,TestSpace::K2_space);
+   // (L₂, δL₂)
+   a->AddTestIntegrator(new VectorMassIntegrator(one_cf),
+                        nullptr,
+                        TestSpace::L2_space,TestSpace::L2_space);
+
+   a->Assemble();
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      delete c_cf[i];
+      delete signed_c_cf[i];
+      delete signed_PB_r_cf[i];
+      delete signed_PB_i_cf[i];
+      delete signed_PBR_r_cf[i];
+      delete signed_PBR_i_cf[i];
+   }   
+
+   socketstream E_out_r;
+
+   int npfes = trial_pfes.Size();
+   Array<int> offsets(npfes+1);  offsets[0] = 0;
+   Array<int> toffsets(npfes+1); toffsets[0] = 0;
+   for (int i = 0; i<npfes; i++)
+   {
+      offsets[i+1] = trial_pfes[i]->GetVSize();
+      toffsets[i+1] = trial_pfes[i]->TrueVSize();
+   }
+   offsets.PartialSum();
+   toffsets.PartialSum();
+
+   Vector x(2*offsets.Last());
+   x = 0.;
+   
+   Array<ParGridFunction *> pgf_r(npfes);
+   Array<ParGridFunction *> pgf_i(npfes);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i] = new ParGridFunction(trial_pfes[i], x, offsets[i]);
+      pgf_i[i] = new ParGridFunction(trial_pfes[i], x, offsets.Last() + offsets[i]);
+   }
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+   ParGridFunction E_par_r(&L2_fes);
+   ParGridFunction E_par_i(&L2_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   std::string output_dir = "ParaView/UW/" + GetTimestamp();
+
+   if (paraview)
+   {
+      if (Mpi::Root()) { WriteParametersToFile(args, output_dir); }
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",pgf_r[0]);
+      paraview_dc->RegisterField("E_i",pgf_i[0]);
+      paraview_dc->RegisterField("E_par_r",&E_par_r);
+      paraview_dc->RegisterField("E_par_i",&E_par_i);
+      paraview_dc->RegisterField("H_r",pgf_r[1]);
+      paraview_dc->RegisterField("H_i",pgf_i[1]);      
+      paraview_dc->RegisterField("J_1_r",pgf_r[2]);
+      paraview_dc->RegisterField("J_1_i",pgf_i[2]);
+      paraview_dc->RegisterField("Q_1_r",pgf_r[3]);
+      paraview_dc->RegisterField("Q_1_i",pgf_i[3]);
+      paraview_dc->RegisterField("J_2_r",pgf_r[4]);
+      paraview_dc->RegisterField("J_2_i",pgf_i[4]);
+      paraview_dc->RegisterField("Q_2_r",pgf_r[5]);
+      paraview_dc->RegisterField("Q_2_i",pgf_i[5]);
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listJhat;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+
+      // remove internal boundaries
+      for (int i = 0; i<int_bdr_attr.Size(); i++)
+      {
+         ess_bdr[int_bdr_attr[i]-1] = 0;
+      }
+
+      trial_pfes[2+2*ndiffusionequations]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += toffsets[2+2*ndiffusionequations];
+      }
+      // ess_bdr=1;
+      for (int i = 0; i<ndiffusionequations;i++)
+      {
+         ess_tdof_listJhat.SetSize(0);
+         trial_pfes[2*i+8]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJhat);
+         for (int j = 0; j < ess_tdof_listJhat.Size(); j++)
+         {
+            ess_tdof_listJhat[j] += toffsets[2*i+8];
+         }
+
+         ess_tdof_list.Append(ess_tdof_listJhat);
+      }
+
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+   
+   // rotate the vector
+   // (x,y) -> (y,-x)
+   Vector rot_one_x(dim); rot_one_x = 0.0; rot_one_x(1) = -1.0;
+   Vector rot_negone_x(dim); rot_negone_x = 0.0; rot_negone_x(1) = 1.0;
+   VectorConstantCoefficient rot_one_x_cf(rot_one_x);
+   VectorConstantCoefficient rot_negone_x_cf(rot_negone_x);
+
+   pgf_r[2+2*ndiffusionequations]->ProjectBdrCoefficientNormal(rot_one_x_cf, one_r_bdr);
+   pgf_r[2+2*ndiffusionequations]->ProjectBdrCoefficientNormal(rot_negone_x_cf, negone_r_bdr);
+   pgf_i[2+2*ndiffusionequations]->ProjectBdrCoefficientNormal(rot_one_x_cf, one_i_bdr);
+   pgf_i[2+2*ndiffusionequations]->ProjectBdrCoefficientNormal(rot_negone_x_cf, negone_i_bdr);
+
+   OperatorPtr Ah;
+   Vector X,B;
+   a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+   ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+
+   BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+   int nblocks = BlockA_r->NumRowBlocks();
+
+   {
+      ComplexBlockOperator Ac(*Ahc);
+      Vector Xc(X.Size()); Xc = 0.0;
+      Vector Bc(B.Size());
+      Ac.BlockComplexToComplexBlock(B, Bc);
+   
+      BlockDiagonalPreconditioner Mc(Ac.RowOffsets());
+      for (int i = 0; i < nblocks; ++i)
+      {
+         auto solver = new ComplexMUMPSSolver(MPI_COMM_WORLD);
+         solver->SetPrintLevel(0);
+         solver->SetOperator(Ac.GetBlock(i,i));
+         Mc.SetDiagonalBlock(i, solver);
+      }
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-5);
+      cg.SetMaxIter(500);
+      cg.SetPrintLevel(1);
+      cg.SetPreconditioner(Mc);
+      cg.SetOperator(Ac);
+      cg.Mult(Bc, Xc);
+      Ac.ComplexBlockToBlockComplex(Xc, X);
+   }
+
+   a->RecoverFEMSolution(X, x);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i]->MakeRef(trial_pfes[i], x, offsets[i]);
+      pgf_i[i]->MakeRef(trial_pfes[i], x, offsets.Last() + offsets[i]);
+   }
+   
+   ParallelECoefficient par_e_r(pgf_r[0]);
+   ParallelECoefficient par_e_i(pgf_i[0]);
+   E_par_r.ProjectCoefficient(par_e_r);
+   E_par_i.ProjectCoefficient(par_e_i);
+
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, *pgf_r[0],
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((real_t)0);
+      paraview_dc->Save();
+      delete paraview_dc;
+   }
+
+   delete a;
+   for (int i = 0; i < trial_fecols.Size(); ++i)
+   {
+      delete trial_fecols[i];
+      delete trial_pfes[i];
+   }
+   for (int i = 0; i< test_fecols.Size(); ++i)
+   {
+      delete test_fecols[i];
+   }
+
+
+   return 0;
+}   
+

miniapps/dpg/plasma/lh-eld-weak.cpp

@@ -0,0 +1,711 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//                   MFEM Maxwell-Vdiffusion coupling parallel example
+//
+// Compile with: make lh-eld-weak
+//
+// mpirun -np 8 ./lh-eld-weak -o 3 -paraview -pr 0
+// mpirun -np 8 ./lh-eld-weak -o 3 -paraview -pr 1 -sc
+
+// Electron Landau Damping
+
+// Weak Coupling Formulation:
+// Strong formulation:
+//     ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵ E = i ω²ϵ₀(J₁ + J₂),   in Ω
+//                         E×n = E₀,  on ∂Ω
+//             - Δ∥ J₁ + c₁ J₁ =  c₁ P(r) E∥,   in Ω     
+//             - Δ∥ J₂ + c₂ J₂ = -c₂ P(r) E∥,   in Ω 
+//                         J₁  = 0,  on ∂Ω
+//                         J₂  = 0,  on ∂Ω
+//   Find E ∈ H(curl,Ω), J₁  ∈ H¹(Ω), J₂  ∈ H¹(Ω) such that
+//   (1/μ₀ ∇×E, ∇ × F) - ω² ϵ₀ (ϵᵣ E, F) = -i ω²ϵ₀(J₁ + J₂, F),  ∀ F ∈ H(curl,Ω)
+
+// ( (b⋅∇)J₁,(b⋅∇) δJ₁ ) + c₁ (J₁,δJ₁) =  c₁(P(r) b⊗b E, δJ₁),  ∀ δJ₁ ∈ (H¹(Ω))²
+// ( (b⋅∇)J₂,(b⋅∇) δJ₂ ) + c₂ (J₂,δJ₂) = -c₂(P(r) b⊗b E, δJ₂),  ∀ δJ₂ ∈ (H¹(Ω))²
+//                                                    J₁ = J₂ = 0,  on ∂Ω
+// --------------------------------------------------------------------------------
+// |               J₁                |                 J₂           |     RHS      |
+// --------------------------------------------------------------------------------
+// |δJ₁|((b⋅∇)J₁,(b⋅∇)δJ₁)+c₁(J₁,δJ₁)|                              | c₁(P(r)E,δJ₁)|  
+// |   |                             |                              |              |   
+// |δJ₂|                             |((b⋅∇)J₂,(b⋅∇)δJ₂)+ c₂(J₂,δJ₂)|-c₂(P(r)E,δJ₂)|  
+// where (δE,δH,δJ₁,δJ₂) ∈  H¹(Ω) × H(curl,Ω) × (H¹(Ω))² × (H¹(Ω))² 
+
+
+
+#include "mfem.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../util/pcomplexblockform.hpp"
+#include "../../common/mfem-common.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "../util/utils.hpp"
+#include "utils/lh_utils.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+   int order = 1;
+   int par_ref_levels = 0;
+   int ser_ref_levels = 0;
+
+   real_t rnum=1.5;
+   real_t mu = 1.257;
+   real_t eps0 = 8.8541878128;
+   real_t cfactor = 1e-6;
+
+   bool static_cond = false;
+   bool visualization = false;
+   bool paraview = false;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&ser_ref_levels, "-sr", "--serial-refinement_levels",
+                  "Number of serial refinement levels.");                  
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&a0, "-a0", "--a0", "P(r) first parameter.");
+   args.AddOption(&a1, "-a1", "--a1", "P(r) second parameter.");
+   args.AddOption(&delta, "-delta", "--delta", "stability parameter.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (Mpi::Root())
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (Mpi::Root())
+   {
+      args.PrintOptions(cout);
+   }
+
+   // number of diffusion equations
+   int ndiffusionequations = 2;
+
+   Vector cvals(ndiffusionequations);
+   Vector csigns(ndiffusionequations);
+   cvals(0)  = 25e6;  cvals(1)  = 1e6;
+   csigns(0) = 1.0;  csigns(1) = -1.0;
+   cvals *= cfactor; // scale the coefficients
+   real_t omega = 2.*M_PI*rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   MFEM_VERIFY(dim == 2, "Dimension != 2 is not supported in this example");
+
+   for (int i = 0; i < ser_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   Array<int> int_bdr_attr;
+   for (int i = 0; i < mesh.GetNBE(); i++)
+   {
+      if (mesh.FaceIsInterior(mesh.GetBdrElementFaceIndex(i)))
+      {
+         int_bdr_attr.Append(mesh.GetBdrAttribute(i));
+      }
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   for (int i = 0; i < par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }  
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+
+   // Define coefficients
+   ConstantCoefficient muinv(1./mu);
+   ConstantCoefficient one_cf(1.0);
+
+   ConstantCoefficient negomegeps0_cf(-omega*eps0);
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_vcf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   // S(r) 
+   FunctionCoefficient S_cf_r(sfunc_r), S_cf_i(sfunc_i);
+   // P(r) 
+   FunctionCoefficient P_cf_r(pfunc_r), P_cf_i(pfunc_i); 
+
+   VectorFunctionCoefficient b_cf(dim,bfunc);// b
+   ScalarVectorProductCoefficient scaled_b_cf(sqrt(cfactor), b_cf);
+   ConstantCoefficient diff_coeff(cfactor);
+
+   MatrixFunctionCoefficient bb_cf(dim,bcrossb); // b⊗b
+   MatrixSumCoefficient oneminusbb(Mone_cf, bb_cf, 1.0, -1.0); // 1 - b⊗b
+
+   // S(r) (I - b⊗b)
+   ScalarMatrixProductCoefficient Soneminusbb_r(S_cf_r, oneminusbb), Soneminusbb_i(S_cf_i, oneminusbb); 
+
+   // P(r) b⊗b 
+   ScalarMatrixProductCoefficient P_cf_bb_r(P_cf_r, bb_cf), P_cf_bb_i(P_cf_i, bb_cf); 
+
+   // ε = S(r) (I - b⊗b) + P(r) b⊗b 
+   MatrixSumCoefficient eps_r(Soneminusbb_r, P_cf_bb_r, 1.0, 1.0); 
+   MatrixSumCoefficient eps_i(Soneminusbb_i, P_cf_bb_i, 1.0, 1.0); 
+
+   coefs_r[nattr-1] = &eps_r;
+   coefs_i[nattr-1] = &eps_i;
+
+   PWMatrixCoefficient eps_cf_r(dim, attr, coefs_r);
+   PWMatrixCoefficient eps_cf_i(dim, attr, coefs_i);
+
+   ConstantCoefficient eps0omeg(omega * eps0);
+   ConstantCoefficient negeps0omeg(-omega * eps0);
+   ConstantCoefficient negeps0omeg2(-omega * omega * eps0);
+   ConstantCoefficient eps0omeg2(omega * omega * eps0);
+
+   ScalarMatrixProductCoefficient m_cf_r(negeps0omeg2, eps_cf_r);
+   ScalarMatrixProductCoefficient m_cf_i(negeps0omeg2, eps_cf_i);
+
+   // ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient eps0omeg_eps_r(eps0omeg, eps_cf_r);
+   // ω ϵ₀ ϵᵢ
+   ScalarMatrixProductCoefficient eps0omeg_eps_i(eps0omeg, eps_cf_i);
+   // -ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient negeps0omeg_eps_r(negeps0omeg, eps_cf_r);
+   // -ω ϵ₀ ϵᵢ
+   ScalarMatrixProductCoefficient negeps0omeg_eps_i(eps0omeg, eps_cf_i);
+
+   // A = [0 1; -1 0]
+   DenseMatrix rot_mat(2);
+   rot_mat(0,0) = 0.; rot_mat(0,1) = 1.;
+   rot_mat(1,0) = -1.; rot_mat(1,1) = 0.;
+   MatrixConstantCoefficient rot(rot_mat);
+
+   // ω ϵ₀ ϵᵣ A
+   MatrixProductCoefficient eps0omeg_eps_r_rot(eps0omeg_eps_r, rot);
+   // ω ϵ₀ ϵᵢ A
+   MatrixProductCoefficient eps0omeg_eps_i_rot(eps0omeg_eps_i, rot);
+   // -ω ϵ₀ ϵᵣ A
+   MatrixProductCoefficient negeps0omeg_eps_r_rot(negeps0omeg_eps_r, rot);
+   // -ω ϵ₀ ϵᵢ A
+   MatrixProductCoefficient negeps0omeg_eps_i_rot(negeps0omeg_eps_i, rot);
+
+   Array<Vector *> c_arrays(ndiffusionequations);
+   Array<PWConstCoefficient *> pw_c_coeffs(ndiffusionequations);
+   Array<MatrixCoefficient *> cPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> cPibb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPibb_cf(ndiffusionequations);
+
+   Vector temp(nattr); temp=0.0;
+   Array<ConstantCoefficient *> c_coeffs(ndiffusionequations);
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      temp[nattr-1] = cvals(i);
+      // temp = cvals(i);
+      pw_c_coeffs[i] = new PWConstCoefficient(temp);
+      c_coeffs[i] = new ConstantCoefficient(cvals(i));
+      cPrbb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_r);
+      cPibb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_i);
+      signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPrbb_cf[i]);
+      signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPibb_cf[i]);
+   }
+
+   // Define the spaces
+   Array<FiniteElementCollection *> fem_fecols;
+   Array<ParFiniteElementSpace *> fem_pfes;
+
+
+   ND_FECollection *nd_fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *nd_pfes = new ParFiniteElementSpace(&pmesh, nd_fec);
+
+   // Vector H1 spaces for Jᵢ 
+   for (int i = 0; i < ndiffusionequations; i++)
+   {
+      fem_fecols.Append(new H1_FECollection(order, dim));
+      fem_pfes.Append(new ParFiniteElementSpace(&pmesh, fem_fecols.Last(), dim));
+   }
+
+
+   HYPRE_BigInt nd_tdofs = nd_pfes->GlobalTrueVSize();
+   if (Mpi::Root())
+   {
+      cout << "ND ParFiniteElementSpace has " << nd_tdofs
+           << " true dofs." << endl;
+   }
+   Array<HYPRE_BigInt> fem_tdofs(fem_pfes.Size());
+   for (int i = 0; i < fem_pfes.Size(); ++i)
+   {
+      fem_tdofs[i] = fem_pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << "FEM ParFiniteElementSpace " << i << " has " << fem_tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   if (Mpi::Root())
+   {
+      cout << "Total number of ND true dofs for Maxwell:     " << nd_tdofs << endl;
+      cout << "Total number of FEM true dofs for VDiffusion: " << fem_tdofs.Sum() << endl;
+   }
+
+   int nfem_pfes = fem_pfes.Size();
+   Array<int> fem_offsets(nfem_pfes+1);  fem_offsets[0] = 0;
+   Array<int> fem_toffsets(nfem_pfes+1); fem_toffsets[0] = 0;
+   for (int i = 0; i<nfem_pfes; i++)
+   {
+      fem_offsets[i+1] = fem_pfes[i]->GetVSize();
+      fem_toffsets[i+1] = fem_pfes[i]->TrueVSize();
+   }
+   fem_offsets.PartialSum();
+   fem_toffsets.PartialSum();
+
+   Vector nd_x(2*nd_pfes->GetVSize());
+   nd_x = 0.;
+   Vector fem_x(2*fem_offsets.Last());
+   fem_x = 0.;
+
+   Vector fem_b(2*fem_offsets.Last());
+   fem_b = 0.;
+
+   ParGridFunction * nd_pgf_r = new ParGridFunction(nd_pfes, nd_x, 0);
+   ParGridFunction * nd_pgf_i = new ParGridFunction(nd_pfes, nd_x, nd_x.Size()/2);
+   Array<ParGridFunction *> fem_pgf_r(nfem_pfes);
+   Array<ParGridFunction *> fem_pgf_i(nfem_pfes);
+
+   Array<ParGridFunction *> old_fem_pgf_r(nfem_pfes);
+   Array<ParGridFunction *> old_fem_pgf_i(nfem_pfes);
+
+   for (int i = 0; i < nfem_pfes; ++i)
+   {
+      fem_pgf_r[i] = new ParGridFunction(fem_pfes[i], fem_x, fem_offsets[i]);
+      fem_pgf_i[i] = new ParGridFunction(fem_pfes[i], fem_x, fem_offsets.Last() + fem_offsets[i]);
+      old_fem_pgf_r[i] = new ParGridFunction(fem_pfes[i]); *old_fem_pgf_r[i] = 0.0;
+      old_fem_pgf_i[i] = new ParGridFunction(fem_pfes[i]); *old_fem_pgf_i[i] = 0.0;
+   }
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+   ParGridFunction E_par_r(&L2_fes);
+   ParGridFunction E_par_i(&L2_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+   std::string output_dir = "ParaView/FEM/WeakCoupling/" + GetTimestamp();
+
+   if (paraview)
+   {
+      if (Mpi::Root()) { WriteParametersToFile(args, output_dir); }
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",nd_pgf_r);
+      paraview_dc->RegisterField("E_i",nd_pgf_i);
+      paraview_dc->RegisterField("E_par_r",&E_par_r);
+      paraview_dc->RegisterField("E_par_i",&E_par_i);
+      paraview_dc->RegisterField("Jh_1_r",fem_pgf_r[0]);
+      paraview_dc->RegisterField("Jh_1_i",fem_pgf_i[0]);
+      paraview_dc->RegisterField("Jh_2_r",fem_pgf_r[1]);
+      paraview_dc->RegisterField("Jh_2_i",fem_pgf_i[1]);
+      paraview_dc->Save();
+   }
+
+
+   
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listJ;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+   
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+   
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }   
+
+
+   int max_fixed_point_iter = 50;
+   // loop through fixed point iterations
+
+   for (int k = 0; k<max_fixed_point_iter; k++)
+   {
+      nd_x = 0.0;
+
+      // reset GridFunctions
+      *nd_pgf_r = 0.0;
+      *nd_pgf_i = 0.0;
+
+      delta = (k == 0) ? 0.01 : 0.01;
+      // delta = 0.01 * pow(0.5, 0.0);
+      delta = 0.01 * pow(0.9, k);
+      ParSesquilinearForm a_nd(nd_pfes);
+
+      // (1/μ₀ ∇×E, ∇ × F)
+      a_nd.AddDomainIntegrator(new CurlCurlIntegrator(muinv), nullptr);
+      // - ω² ϵ₀ (ϵᵣ E, F)
+      a_nd.AddDomainIntegrator(new VectorFEMassIntegrator(m_cf_r),
+                          new VectorFEMassIntegrator(m_cf_i));
+      a_nd.Assemble();
+
+      VectorGridFunctionCoefficient J1_cf_r(fem_pgf_r[0]);
+      VectorGridFunctionCoefficient J1_cf_i(fem_pgf_i[0]);
+      VectorGridFunctionCoefficient J2_cf_r(fem_pgf_r[1]);
+      VectorGridFunctionCoefficient J2_cf_i(fem_pgf_i[1]);
+
+      // -iω²ϵ₀ (Jᵣ + i Jᵢ ,F) = ω² ϵ₀ (Jᵢ - i Jᵣ,F)   
+      ScalarVectorProductCoefficient negomeg2eps0_J1_cf_i(negeps0omeg2, J1_cf_i);
+      ScalarVectorProductCoefficient negomeg2eps0_J2_cf_i(negeps0omeg2, J2_cf_i);
+
+      ScalarVectorProductCoefficient omeg2eps0_J1_cf_i(eps0omeg2, J1_cf_i);
+      ScalarVectorProductCoefficient omeg2eps0_J2_cf_i(eps0omeg2, J2_cf_i);
+
+      ScalarVectorProductCoefficient negomeg2eps0_J1_cf_r(negeps0omeg2, J1_cf_r);
+      ScalarVectorProductCoefficient negomeg2eps0_J2_cf_r(negeps0omeg2, J2_cf_r);
+
+
+      ScalarVectorProductCoefficient omeg2eps0_J1_cf_r(eps0omeg2, J1_cf_r);
+      ScalarVectorProductCoefficient omeg2eps0_J2_cf_r(eps0omeg2, J2_cf_r);
+
+
+      ParComplexLinearForm nd_b(nd_pfes);
+
+      // ω² ϵ₀ (Jᵢ - i Jᵣ,F) = (ω² ϵ₀ Jᵢ, F) + i (-ω² ϵ₀Jᵢ,F)  
+      // nd_b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(omeg2eps0_J1_cf_i),
+      //                          new VectorFEDomainLFIntegrator(negomeg2eps0_J1_cf_r));
+      // nd_b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(omeg2eps0_J2_cf_i),
+      //                          new VectorFEDomainLFIntegrator(negomeg2eps0_J2_cf_r));    
+      
+
+      ScalarVectorProductCoefficient omeg2_eps0_J1_cf_i(eps0*omega*omega, J1_cf_i);
+      ScalarVectorProductCoefficient omeg2_eps0_J2_cf_i(eps0*omega*omega, J2_cf_i);
+
+      ScalarVectorProductCoefficient negomeg2_eps0_J1_cf_r(-eps0*omega*omega, J1_cf_r);
+      ScalarVectorProductCoefficient negomeg2_eps0_J2_cf_r(-eps0*omega*omega, J2_cf_r);
+      
+      nd_b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(omeg2_eps0_J1_cf_i),
+                               new VectorFEDomainLFIntegrator(negomeg2_eps0_J1_cf_r));
+      nd_b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(omeg2_eps0_J2_cf_i),
+                               new VectorFEDomainLFIntegrator(negomeg2_eps0_J2_cf_r));  
+
+      // nd_b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(negomeg2eps0_J1_cf_i), nullptr);
+      // ConstantCoefficient temp(1.0);
+      // nd_b.AddDomainIntegrator(nullptr,new VectorFEDomainLFIntegrator(J1_cf_r));
+      // nd_b.AddDomainIntegrator(nullptr,new VectorFEDomainLFIntegrator(omeg2eps0_J2_cf_r));
+      
+
+      nd_b.Assemble();
+
+      // remove internal boundaries
+      ess_bdr = 1;
+      for (int i = 0; i<int_bdr_attr.Size(); i++)
+      {
+         ess_bdr[int_bdr_attr[i]-1] = 0;
+      }
+
+      nd_pfes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+      // if (k == 0)
+      // {
+      nd_pgf_r->ProjectBdrCoefficientTangent(one_x_cf, one_r_bdr);
+      nd_pgf_r->ProjectBdrCoefficientTangent(negone_x_cf, negone_r_bdr);
+      nd_pgf_i->ProjectBdrCoefficientTangent(one_x_cf, one_i_bdr);
+      nd_pgf_i->ProjectBdrCoefficientTangent(negone_x_cf, negone_i_bdr);
+      // }
+
+      OperatorPtr nd_Ah;
+      Vector nd_X,nd_B;
+      a_nd.FormLinearSystem(ess_tdof_list,nd_x,nd_b,nd_Ah, nd_X,nd_B);
+
+
+      ComplexHypreParMatrix *nd_A = nd_Ah.As<ComplexHypreParMatrix>();
+
+      ComplexMUMPSSolver nd_mumps(MPI_COMM_WORLD);
+      nd_mumps.SetPrintLevel(0);
+      nd_mumps.SetOperator(*nd_A);
+      nd_mumps.Mult(nd_B, nd_X);
+
+      a_nd.RecoverFEMSolution(nd_X, nd_b, nd_x);
+
+      nd_pgf_r->MakeRef(nd_pfes, nd_x, 0);
+      nd_pgf_i->MakeRef(nd_pfes, nd_x, nd_x.Size()/2);
+
+
+      fem_x = 0.0;
+      fem_b = 0.0;
+      for (int i = 0; i < nfem_pfes; ++i)
+      {
+         *fem_pgf_r[i] = 0.0;
+         *fem_pgf_i[i] = 0.0;
+      }
+
+      // Diffusion equations
+      ParComplexBlockForm a_fem(fem_pfes);
+      for (int i = 0; i < ndiffusionequations; i++)
+      {
+         // ((b⋅∇)Jᵢ, (b⋅∇) δJᵢ)
+         a_fem.AddDomainIntegrator(new DirectionalVectorDiffusionIntegrator(scaled_b_cf), 
+                                   nullptr,i,i);
+         // cᵢ(Jᵢ, δJᵢ)
+         a_fem.AddDomainIntegrator(new VectorMassIntegrator(*pw_c_coeffs[i]), 
+                                   nullptr,i,i);
+      }
+
+      // FEM RHS
+      // ±cᵢ(P(r) (b ⊗ b) E, δJᵢ)  
+      real_t * fembdata = fem_b.GetData();
+      ParLinearForm b_J1_r(fem_pfes[0],fembdata);
+      ParLinearForm b_J2_r(fem_pfes[1],fembdata + fem_offsets[1]);
+
+      ParLinearForm b_J1_i(fem_pfes[0],fembdata + fem_offsets.Last());
+      ParLinearForm b_J2_i(fem_pfes[1],fembdata + fem_offsets.Last() + fem_offsets[1]);
+      
+      VectorGridFunctionCoefficient E_r_cf(nd_pgf_r);
+      VectorGridFunctionCoefficient E_i_cf(nd_pgf_i);
+
+      // (a + i b) * (c + i d) = (ac - bd) + i (ad + bc)
+      MatrixVectorProductCoefficient c1_cf_rr(*signedcPrbb_cf[0], E_r_cf);
+      MatrixVectorProductCoefficient c1_cf_ii(*signedcPibb_cf[0], E_i_cf);
+      MatrixVectorProductCoefficient c1_cf_ri(*signedcPrbb_cf[0], E_i_cf);
+      MatrixVectorProductCoefficient c1_cf_ir(*signedcPibb_cf[0], E_r_cf);
+
+      VectorSumCoefficient c1_cf_r(c1_cf_rr, c1_cf_ii, 1.0, -1.0);
+      VectorSumCoefficient c1_cf_i(c1_cf_ri, c1_cf_ir, 1.0,  1.0);
+
+      MatrixVectorProductCoefficient c2_cf_rr(*signedcPrbb_cf[1], E_r_cf);
+      MatrixVectorProductCoefficient c2_cf_ii(*signedcPibb_cf[1], E_i_cf);
+      MatrixVectorProductCoefficient c2_cf_ri(*signedcPrbb_cf[1], E_i_cf);
+      MatrixVectorProductCoefficient c2_cf_ir(*signedcPibb_cf[1], E_r_cf); 
+
+      VectorSumCoefficient c2_cf_r(c2_cf_rr, c2_cf_ii, 1.0, -1.0);
+      VectorSumCoefficient c2_cf_i(c2_cf_ri, c2_cf_ir, 1.0,  1.0);
+
+
+      b_J1_r.AddDomainIntegrator(new VectorDomainLFIntegrator(c1_cf_r));
+      b_J1_i.AddDomainIntegrator(new VectorDomainLFIntegrator(c1_cf_i));
+      b_J2_r.AddDomainIntegrator(new VectorDomainLFIntegrator(c2_cf_r));
+      b_J2_i.AddDomainIntegrator(new VectorDomainLFIntegrator(c2_cf_i));
+      b_J1_r.Assemble();
+      b_J1_i.Assemble();
+      b_J2_r.Assemble();
+      b_J2_i.Assemble();
+
+
+      a_fem.Assemble();
+
+
+      ess_bdr=1;
+      ess_tdof_list.SetSize(0);
+      for (int i = 0; i<ndiffusionequations;i++)
+      {
+         ess_tdof_listJ.SetSize(0);
+         fem_pfes[i]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJ);
+         for (int j = 0; j < ess_tdof_listJ.Size(); j++)
+         {
+            ess_tdof_listJ[j] += fem_toffsets[i];
+         }
+         ess_tdof_list.Append(ess_tdof_listJ);
+      }
+
+      OperatorPtr fem_Ah;
+      Vector fem_X,fem_B;
+
+      a_fem.FormLinearSystem(ess_tdof_list,fem_x,fem_b,fem_Ah, fem_X,fem_B);
+
+
+      ComplexOperator * fem_Ahc = fem_Ah.As<ComplexOperator>();
+
+      BlockOperator * fem_BlockA_r = dynamic_cast<BlockOperator *>(&fem_Ahc->real());
+      BlockOperator * fem_BlockA_i = dynamic_cast<BlockOperator *>(&fem_Ahc->imag());
+
+      int fem_nblocks = fem_BlockA_r->NumRowBlocks();
+
+      Array2D<const HypreParMatrix*> fem_A_r_matrices(fem_nblocks, fem_nblocks);
+      Array2D<const HypreParMatrix*> fem_A_i_matrices(fem_nblocks, fem_nblocks);
+      for (int i = 0; i < fem_nblocks; i++)
+      {
+         for (int j = 0; j < fem_nblocks; j++)
+         {
+            fem_A_r_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&fem_BlockA_r->GetBlock(i,j));
+            fem_A_i_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&fem_BlockA_i->GetBlock(i,j));
+         }
+      }
+
+      HypreParMatrix * fem_Ahr = HypreParMatrixFromBlocks(fem_A_r_matrices);
+      HypreParMatrix * fem_Ahi = HypreParMatrixFromBlocks(fem_A_i_matrices);
+
+      ComplexHypreParMatrix * fem_Ahc_hypre =
+         new ComplexHypreParMatrix(fem_Ahr, fem_Ahi,false, false);
+
+      if (Mpi::Root())
+      {
+         mfem::out << "FEM Assembly finished successfully." << endl;
+      }   
+
+
+      Array<int> fem_tdof_offsets(2*fem_nblocks+1);
+      fem_tdof_offsets[0] = 0;
+      for (int i=0; i<fem_nblocks; i++)
+      {
+         fem_tdof_offsets[i+1] = fem_A_r_matrices(i,i)->Height();
+         fem_tdof_offsets[fem_nblocks+i+1] = fem_tdof_offsets[i+1];
+      }
+      fem_tdof_offsets.PartialSum();
+
+      BlockDiagonalPreconditioner fem_M(fem_tdof_offsets);
+
+      HypreBoomerAMG * solver_J1 = new HypreBoomerAMG((HypreParMatrix &)
+                                   fem_BlockA_r->GetBlock(0,0));
+      solver_J1->SetPrintLevel(0);
+      solver_J1->SetSystemsOptions(dim);
+      HypreBoomerAMG * solver_J2 = new HypreBoomerAMG((HypreParMatrix &)
+                                   fem_BlockA_r->GetBlock(1,1));
+      solver_J2->SetPrintLevel(0);
+      fem_M.SetDiagonalBlock(0,solver_J1);
+      fem_M.SetDiagonalBlock(1,solver_J2);
+      fem_M.SetDiagonalBlock(fem_nblocks,solver_J1);
+      fem_M.SetDiagonalBlock(fem_nblocks+1,solver_J2);
+
+
+      CGSolver fem_cg(MPI_COMM_WORLD);
+      fem_cg.SetRelTol(1e-16);
+      fem_cg.SetMaxIter(1000);
+      fem_cg.SetPrintLevel(1);
+      fem_cg.SetPreconditioner(fem_M);
+      fem_cg.SetOperator(*fem_Ahc_hypre);
+      fem_cg.Mult(fem_B, fem_X);
+
+      a_fem.RecoverFEMSolution(fem_X, fem_x);
+
+
+      real_t alpha = (k == 0) ? 1.0 : 0.01;
+
+      for (int i = 0; i < nfem_pfes; ++i)
+      {
+         fem_pgf_r[i]->MakeRef(fem_pfes[i], fem_x, fem_offsets[i]);
+         fem_pgf_i[i]->MakeRef(fem_pfes[i], fem_x, fem_offsets.Last() + fem_offsets[i]);
+
+         (*fem_pgf_r[i])*= alpha;
+         (*fem_pgf_i[i])*= alpha;
+         fem_pgf_r[i]->Add(1.0-alpha, *old_fem_pgf_r[i]);
+         fem_pgf_i[i]->Add(1.0-alpha, *old_fem_pgf_i[i]);
+         *old_fem_pgf_r[i] = *fem_pgf_r[i];
+         *old_fem_pgf_i[i] = *fem_pgf_i[i];
+      }
+
+
+      ParallelECoefficient par_e_r(nd_pgf_r);
+      ParallelECoefficient par_e_i(nd_pgf_i);
+      E_par_r.ProjectCoefficient(par_e_r);
+      E_par_i.ProjectCoefficient(par_e_i);      
+
+      if (paraview)
+      {
+         paraview_dc->SetCycle(k);
+         paraview_dc->SetTime((real_t)k);
+         paraview_dc->Save();
+      }
+
+   }
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      delete pw_c_coeffs[i];
+      delete c_coeffs[i];
+      delete cPrbb_cf[i]; 
+      delete cPibb_cf[i]; 
+      delete signedcPrbb_cf[i]; 
+      delete signedcPibb_cf[i]; 
+   }
+
+
+
+   delete nd_fec;
+   delete nd_pfes;
+   for (int i = 0; i < fem_fecols.Size(); ++i)
+   {
+      delete fem_fecols[i];
+      delete fem_pfes[i];
+   }
+
+   if (paraview_dc) 
+   { 
+      delete paraview_dc; 
+   }
+
+   return 0;
+
+}

miniapps/dpg/plasma/lh-primal-dpg.cpp

@@ -0,0 +1,479 @@
+
+// srun -n 256 ./pmaxwell-primal-tokamak -o 3 -sc -rnum
+// srun -n 448 ./pmaxwell-primal-tokamak -o 4 -sc -rnum 11.0 -paraview
+
+// srun -n 448 ./pmaxwell-primal-tokamak -o 4 -do 0 -sc -paraview (with the new epsilon GridFunction coefficients)
+// Description:
+// This example code demonstrates the use of MFEM to define and solve
+// the "ultraweak" (UW) DPG formulation for the Maxwell problem
+
+//      ∇×(1/μ ∇×E) - ω^2 ϵ₀ ϵ E = 0 ,   in Ω
+//                           E×n = E_0, on ∂Ω
+
+// The primal-DPG formulation is obtained by integration by parts
+// and the introduction of trace unknowns on the mesh skeleton
+
+// in 3D
+// E ∈ H(curl)
+// Ê ∈ H_0^1/2(Ω)(curl, Γ_h)
+//  1/μ (∇×E , ∇×F) + (ω^2 ϵ₀ ϵ , F) + <Ê , F × n> = 0,      ∀ F ∈ H(curl,Ω)
+//                                           Ê × n = E_0     on ∂Ω
+
+#include "mfem.hpp"
+#include "../util/utils.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "utils/lh_utils.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+   int order = 2;
+   int delta_order = 1;
+   int par_ref_levels = 0;
+   int amr_ref_levels = 0;
+
+   // real_t rnum=4.6e9;
+   // real_t mu = 1.257e-6/factor;
+   // real_t epsilon_scale = 8.8541878128e-12*factor;
+   real_t rnum=1.5;
+   real_t mu = 1.257;
+   real_t eps0 = 8.8541878128;
+   bool mumps_solver = false;
+   real_t theta = 0.0;
+
+   bool visualization = false;
+   bool static_cond = false;
+   bool paraview = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement-levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&amr_ref_levels, "-amr", "--parallel-amr-refinement-levels",
+                  "Number of parallel AMR refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+#ifdef MFEM_USE_MUMPS
+   args.AddOption(&mumps_solver, "-mumps", "--mumps-solver", "-no-mumps",
+                  "--no-mumps-solver", "Use the MUMPS Solver.");
+#endif
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+
+   double omega = 2.*M_PI*rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   mesh.RemoveInternalBoundaries();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+
+   for (int i = 0; i<par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }
+
+   FiniteElementCollection *E_fec = new ND_FECollection(order,dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh,E_fec);
+
+   // H^-1/2 (curl) space for Ê (in 2D H1 trace, in 3D ND trace)
+   int test_order = order+delta_order;
+   FiniteElementCollection * hatE_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementCollection * F_fec = new ND_FECollection(test_order, dim);
+   ParFiniteElementSpace *hatE_fes = new ParFiniteElementSpace(&pmesh,hatE_fec);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+   trial_fes.Append(E_fes);
+   trial_fes.Append(hatE_fes);
+   test_fec.Append(F_fec);
+
+   int gdofs = 0;
+   for (int i = 0; i<trial_fes.Size(); i++)
+   {
+      gdofs += trial_fes[i]->GlobalTrueVSize();
+   }
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Global number of dofs = " << gdofs << endl;
+   }
+
+   // Define the coefficients 
+   ConstantCoefficient muinv(1./mu);
+   ConstantCoefficient one(1.0);
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_vcf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   // S(r) 
+   FunctionCoefficient S_cf_r(sfunc_r), S_cf_i(sfunc_i);
+   // P(r) 
+   FunctionCoefficient P_cf_r(pfunc_r), P_cf_i(pfunc_i); 
+
+   VectorFunctionCoefficient b_cf(dim,bfunc);// b
+   MatrixFunctionCoefficient bb_cf(dim,bcrossb); // b⊗b
+
+   MatrixSumCoefficient oneminusbb(Mone_cf, bb_cf, 1.0, -1.0); // 1 - b⊗b
+
+   // S(r) (I - b⊗b)
+   ScalarMatrixProductCoefficient Soneminusbb_r(S_cf_r, oneminusbb), Soneminusbb_i(S_cf_i, oneminusbb); 
+
+   // P(r) b⊗b 
+   ScalarMatrixProductCoefficient P_cf_bb_r(P_cf_r, bb_cf), P_cf_bb_i(P_cf_i, bb_cf); 
+
+   // εᵣ = S(r) (I - b⊗b) + P(r) b⊗b 
+   MatrixSumCoefficient eps_r(Soneminusbb_r, P_cf_bb_r, 1.0, 1.0); 
+   MatrixSumCoefficient eps_i(Soneminusbb_i, P_cf_bb_i, 1.0, 1.0); 
+
+   coefs_r[nattr-1] = &eps_r;
+   coefs_i[nattr-1] = &eps_i;
+
+   PWMatrixCoefficient eps_cf_r(dim, attr, coefs_r);
+   PWMatrixCoefficient eps_cf_i(dim, attr, coefs_i);
+
+   ConstantCoefficient negeps0omeg2(-eps0 * omega * omega);
+
+   ScalarMatrixProductCoefficient m_cf_r(negeps0omeg2, eps_cf_r);
+   ScalarMatrixProductCoefficient m_cf_i(negeps0omeg2, eps_cf_i);
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_fes,test_fec);
+
+   // (1/μ₀ ∇ × E,∇ × F)
+   a->AddTrialIntegrator(new CurlCurlIntegrator(muinv), nullptr,0,0);
+
+   // -(ω^2 ϵ₀ ϵ, F)
+   const IntegrationRule *irs[Geometry::NumGeom];
+   int order_quad = 2*order + 2;
+   for (int i = 0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+   const IntegrationRule &ir = IntRules.Get(pmesh.GetElementGeometry(0),
+                                            2*test_order + 2);
+   VectorFEMassIntegrator * integ_r = new VectorFEMassIntegrator(m_cf_r);
+   VectorFEMassIntegrator * integ_i = new VectorFEMassIntegrator(m_cf_i);
+   integ_r->SetIntegrationRule(ir);
+   integ_i->SetIntegrationRule(ir);
+   a->AddTrialIntegrator(integ_r,integ_i,0,0);
+
+   // < n×Ê,F>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,1,0);
+
+   // test integrators
+   // (∇×F ,∇× δF)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,0,0);
+   // (F,δF)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,0,0);
+
+   socketstream E_out_r;
+   socketstream E_theta_out_r;
+
+   ParComplexGridFunction E_gf(E_fes);
+   E_gf.real() = 0.0;
+   E_gf.imag() = 0.0;
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+
+   ParGridFunction E_theta_r(&L2_fes);
+   ParGridFunction E_theta_i(&L2_fes);
+   ParGridFunction E_theta(&L2_fes);
+   E_theta = 0.0;
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+   // Create ParaView directory and file
+   std::string output_dir = "ParaView/PrimalDPG/" + GetTimestamp();
+   if (Mpi::Root())
+   {
+      WriteParametersToFile(args, output_dir);
+   }
+   if (paraview)
+   {
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",&E_gf.real());
+      paraview_dc->RegisterField("E_i",&E_gf.imag());
+      paraview_dc->RegisterField("E_theta_r",&E_theta_r);
+      paraview_dc->RegisterField("E_theta_i",&E_theta_i);
+   }
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+
+   a->Assemble();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+
+      E_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+
+
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = E_fes->GetVSize();
+   offsets[2] = hatE_fes->GetVSize();
+   offsets.PartialSum();
+
+   Vector x(2*offsets.Last());
+   x = 0.;
+   double * xdata = x.GetData();
+
+   E_gf.real().MakeRef(E_fes,&xdata[0]);
+   E_gf.imag().MakeRef(E_fes,&xdata[offsets.Last()]);
+
+
+   E_gf.ProjectBdrCoefficientTangent(one_x_cf,zero_vcf, one_r_bdr);
+   E_gf.ProjectBdrCoefficientTangent(negone_x_cf,zero_vcf, negone_r_bdr);
+   E_gf.ProjectBdrCoefficientTangent(zero_vcf,one_x_cf, one_i_bdr);
+   E_gf.ProjectBdrCoefficientTangent(zero_vcf,negone_x_cf, negone_i_bdr);
+
+   OperatorPtr Ah;
+   Vector X,B;
+   a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+   ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+   BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+   BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+   int num_blocks = BlockA_r->NumRowBlocks();
+   Array<int> tdof_offsets(2*num_blocks+1);
+
+   tdof_offsets[0] = 0;
+   for (int i=0; i<num_blocks; i++)
+   {
+      const int h = BlockA_r->GetBlock(i,i).Height();
+      tdof_offsets[i+1] = h;
+      tdof_offsets[num_blocks+i+1] = h;
+   }
+   tdof_offsets.PartialSum();
+
+   BlockOperator blockA(tdof_offsets);
+   for (int i = 0; i<num_blocks; i++)
+   {
+      for (int j = 0; j<num_blocks; j++)
+      {
+         blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
+         blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
+         blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
+         blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
+      }
+   }
+
+#ifdef MFEM_USE_MUMPS
+   if (mumps_solver)
+   {
+      // Monolithic real part
+      Array2D<const HypreParMatrix * > Ab_r(num_blocks,num_blocks);
+      // Monolithic imag part
+      Array2D<const HypreParMatrix * > Ab_i(num_blocks,num_blocks);
+      for (int i = 0; i<num_blocks; i++)
+      {
+         for (int j = 0; j<num_blocks; j++)
+         {
+            Ab_r(i,j) = &(HypreParMatrix &)BlockA_r->GetBlock(i,j);
+            Ab_i(i,j) = &(HypreParMatrix &)BlockA_i->GetBlock(i,j);
+         }
+      }
+      HypreParMatrix * A_r = HypreParMatrixFromBlocks(Ab_r);
+      HypreParMatrix * A_i = HypreParMatrixFromBlocks(Ab_i);
+
+      ComplexHypreParMatrix Acomplex(A_r, A_i,true,true);
+
+      HypreParMatrix * A = Acomplex.GetSystemMatrix();
+
+      MUMPSSolver mumps(MPI_COMM_WORLD);
+      mumps.SetPrintLevel(0);
+      mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
+      mumps.SetOperator(*A);
+      mumps.Mult(B,X);
+      delete A;
+   }
+#else
+   if (mumps_solver)
+   {
+      MFEM_WARNING("MFEM compiled without mumps. Switching to an iterative solver");
+   }
+   mumps_solver = false;
+#endif
+   if (!mumps_solver)
+   {
+      BlockDiagonalPreconditioner M(tdof_offsets);
+      ParFiniteElementSpace *ams_fes = nullptr;
+      if (static_cond)
+      {
+         ams_fes = new ParFiniteElementSpace(&pmesh,
+                                             E_fes->FEColl()->GetTraceCollection());
+      }
+
+      HypreAMS * solver_E = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(0,0),
+                                          (static_cond) ? ams_fes : E_fes);
+      solver_E->SetPrintLevel(0);
+      HypreBoomerAMG * solver_hatE = new HypreBoomerAMG((HypreParMatrix &)
+                                                         BlockA_r->GetBlock(1,1));
+      solver_hatE->SetPrintLevel(0);
+      solver_hatE->SetRelaxType(88);
+
+      M.SetDiagonalBlock(0,solver_E);
+      M.SetDiagonalBlock(1,solver_hatE);
+      M.SetDiagonalBlock(2,solver_E);
+      M.SetDiagonalBlock(3,solver_hatE);
+
+      if (myid == 0)
+      {
+         std::cout << "PCG iterations" << endl;
+      }
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-8);
+      cg.SetMaxIter(1000);
+      cg.SetPrintLevel(1);
+      cg.SetPreconditioner(M);
+      cg.SetOperator(blockA);
+      cg.Mult(B, X);
+
+      for (int i = 0; i<num_blocks; i++)
+      {
+         delete &M.GetDiagonalBlock(i);
+      }
+
+      //int num_iter = cg.GetNumIterations();
+
+   }
+   a->RecoverFEMSolution(X,x);
+
+   E_gf.real().MakeRef(E_fes,x.GetData());
+   E_gf.imag().MakeRef(E_fes,&x.GetData()[offsets.Last()]);
+
+   AzimuthalECoefficient az_e_r(&E_gf.real());
+   AzimuthalECoefficient az_e_i(&E_gf.imag());
+
+   E_theta_r.ProjectCoefficient(az_e_r);
+   E_theta_i.ProjectCoefficient(az_e_i);
+
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, E_gf.real(),
+                              "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+      common::VisualizeField(E_theta_out_r,vishost, visport, E_theta_r,
+                              "Numerical Electric field (Azimuthal-real)", 501, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime(0.0);
+      paraview_dc->Save();
+   }
+
+   delete a;
+   delete F_fec;
+   delete hatE_fes;
+   delete hatE_fec;
+   delete E_fec;
+   delete E_fes;
+
+   return 0;
+}
+

miniapps/dpg/plasma/makefile

@@ -0,0 +1,145 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/miniapps/dpg/plasma,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+# Include defaults.mk to get XLINKER
+DEFAULTS_MK = $(MFEM_DIR)/config/defaults.mk
+include $(DEFAULTS_MK)
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+DPG_COMPLEX_SEQ_SRC = ../util/utils.cpp ../util/complexweakform.cpp \
+	../util/complexstaticcond.cpp ../util/complexblockform.cpp ../util/preconditioners.cpp
+DPG_COMPLEX_PAR_SRC = $(DPG_COMPLEX_SEQ_SRC) ../util/pcomplexweakform.cpp ../util/pcomplexblockform.cpp ../util/blockcomplexhypremat.cpp  
+DPG_COMPLEX_OBJ = $(DPG_COMPLEX_PAR_SRC:.cpp=.o)
+
+LH_UTILS_SRC = utils/lh_utils.cpp ../util/maxwell_utils.cpp
+LH_UTILS_OBJ = $(LH_UTILS_SRC:.cpp=.o)
+
+LH_PRIMAL_SRC = lh-primal-dpg.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_PRIMAL_OBJ = $(LH_PRIMAL_SRC:.cpp=.o)
+
+LH_ELD_FEM_SRC = lh-eld-fem.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_ELD_FEM_OBJ = $(LH_ELD_FEM_SRC:.cpp=.o)
+
+LH_ELD_FEM_SYMM_SRC = lh-eld-fem-symm.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_ELD_FEM_SYMM_OBJ = $(LH_ELD_FEM_SYMM_SRC:.cpp=.o)
+
+LH_ELD_DPG_SRC = lh-eld-dpg.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_ELD_DPG_OBJ = $(LH_ELD_DPG_SRC:.cpp=.o)
+
+LH_ELD_FOSLS_DPG_SRC = lh-eld-fosls-dpg.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_ELD_FOSLS_DPG_OBJ = $(LH_ELD_FOSLS_DPG_SRC:.cpp=.o)
+
+LH_ELD_FOSLS_FEM_SRC = lh-eld-fosls-fem.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_ELD_FOSLS_FEM_OBJ = $(LH_ELD_FOSLS_FEM_SRC:.cpp=.o)
+
+LH_ELD_DPG_FEM_SRC = lh-eld-dpg-fem.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)	
+LH_ELD_DPG_FEM_OBJ = $(LH_ELD_DPG_FEM_SRC:.cpp=.o)
+
+LH_ELD_WEAK_SRC = lh-eld-weak.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_ELD_WEAK_OBJ = $(LH_ELD_WEAK_SRC:.cpp=.o)
+
+LH_ELD_DPG_FEM_WEAK_SRC = lh-eld-dpg-fem-weak.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_ELD_DPG_FEM_WEAK_OBJ = $(LH_ELD_DPG_FEM_WEAK_SRC:.cpp=.o)
+
+LH_ELD_UW_SRC = lh-eld-uw.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_ELD_UW_OBJ = $(LH_ELD_UW_SRC:.cpp=.o)
+
+
+SEQ_MINIAPPS = 
+PAR_MINIAPPS = lh-primal-dpg lh-eld-fem lh-eld-fem-symm lh-eld-dpg  \
+ lh-eld-dpg-fem lh-eld-weak lh-eld-dpg-fem-weak lh-eld-fosls-dpg\
+ lh-eld-fosls-fem lh-eld-uw
+
+ifeq ($(MFEM_USE_MPI),NO)
+   MINIAPPS = $(SEQ_MINIAPPS)
+else
+   MINIAPPS = $(PAR_MINIAPPS) $(SEQ_MINIAPPS)
+endif
+
+COMMON_LIB = -L$(MFEM_BUILD_DIR)/miniapps/common -lmfem-common
+
+# If MFEM_SHARED is set, add the ../common rpath
+COMMON_LIB += $(if $(MFEM_SHARED:YES=),,\
+   $(if $(MFEM_USE_CUDA:YES=),$(CXX_XLINKER),$(CUDA_XLINKER))-rpath,$(abspath\
+   $(MFEM_BUILD_DIR)/miniapps/common))
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all lib-common clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+%.o: %.cpp
+
+%.o: $(SRC)%.cpp $(wildcard $(SRC)%.hpp) $(MFEM_LIB_FILE)\
+ $(CONFIG_MK) | lib-common
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $< -o $@
+
+all: $(MINIAPPS)
+
+lh-primal-dpg: $(LH_PRIMAL_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_PRIMAL_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-fem: $(LH_ELD_FEM_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_FEM_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-fem-symm: $(LH_ELD_FEM_SYMM_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_FEM_SYMM_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-dpg: $(LH_ELD_DPG_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_DPG_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-fosls-dpg: $(LH_ELD_FOSLS_DPG_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_FOSLS_DPG_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-fosls-fem: $(LH_ELD_FOSLS_FEM_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_FOSLS_FEM_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-dpg-fem: $(LH_ELD_DPG_FEM_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_DPG_FEM_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-weak: $(LH_ELD_WEAK_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_WEAK_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-dpg-fem-weak: $(LH_ELD_DPG_FEM_WEAK_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_DPG_FEM_WEAK_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-uw: $(LH_ELD_UW_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_UW_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+# Rule for building lib-common
+lib-common:
+	$(MAKE) -C $(MFEM_BUILD_DIR)/miniapps/common
+
+MFEM_TESTS = MINIAPPS
+include $(MFEM_TEST_MK)
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(PAR_MINIAPPS) $(SEQ_MINIAPPS)
+	rm -f $(DPG_COMPLEX_OBJ) $(LH_UTILS_OBJ)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -rf ParaView 

miniapps/dpg/plasma/other/makefile

@@ -0,0 +1,121 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../../..
+MFEM_BUILD_DIR ?= ../../../..
+SRC = $(if $(MFEM_DIR:../../../..=),$(MFEM_DIR)/miniapps/dpg/plasma/other,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+# Include defaults.mk to get XLINKER
+DEFAULTS_MK = $(MFEM_DIR)/config/defaults.mk
+include $(DEFAULTS_MK)
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+DPG_COMPLEX_SEQ_SRC = ../../util/utils.cpp ../../util/complexweakform.cpp \
+	../../util/complexstaticcond.cpp ../../util/complexblockform.cpp \
+	../../util/preconditioners.cpp ../../util/maxwell_utils.cpp
+DPG_COMPLEX_PAR_SRC = $(DPG_COMPLEX_SEQ_SRC) ../../util/pcomplexweakform.cpp ../../util/pcomplexblockform.cpp ../../util/blockcomplexhypremat.cpp  
+DPG_COMPLEX_OBJ = $(DPG_COMPLEX_PAR_SRC:.cpp=.o)
+
+MAXWELL_SRC = ../../util/maxwell_utils.cpp
+MAXWELL_OBJ = $(MAXWELL_SRC:.cpp=.o)
+
+PMAXWELL_TOKAMAK_SRC = pmaxwell-tokamak.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_TOKAMAK_OBJ = $(PMAXWELL_TOKAMAK_SRC:.cpp=.o)
+
+PMAXWELL_FEM_TOKAMAK_SRC = pmaxwell-fem-tokamak.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_FEM_TOKAMAK_OBJ = $(PMAXWELL_FEM_TOKAMAK_SRC:.cpp=.o)
+
+PMAXWELL_PRIMAL_TOKAMAK_SRC = pmaxwell-primal-tokamak.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_PRIMAL_TOKAMAK_OBJ = $(PMAXWELL_PRIMAL_TOKAMAK_SRC:.cpp=.o)
+
+PMAXWELL_UW_TOKAMAK_SRC = pmaxwell-uw-tokamak.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_UW_TOKAMAK_OBJ = $(PMAXWELL_UW_TOKAMAK_SRC:.cpp=.o)
+
+PMAXWELL_UW_LH_SRC = pmaxwell-uw-lh.cpp $(DPG_COMPLEX_PAR_SRC) $(MAXWELL_SRC)
+PMAXWELL_UW_LH_OBJ = $(PMAXWELL_UW_LH_SRC:.cpp=.o)
+
+PMAXWELL_FEM_LH_SRC = pmaxwell-fem-lh.cpp $(DPG_COMPLEX_PAR_SRC) $(MAXWELL_SRC)
+PMAXWELL_FEM_LH_OBJ = $(PMAXWELL_FEM_LH_SRC:.cpp=.o)
+
+SEQ_MINIAPPS = 
+PAR_MINIAPPS = pmaxwell-tokamak pmaxwell-fem-tokamak \
+ pmaxwell-primal-tokamak pmaxwell-uw-tokamak  \
+ pmaxwell-uw-lh pmaxwell-fem-lh 
+
+ifeq ($(MFEM_USE_MPI),NO)
+   MINIAPPS = $(SEQ_MINIAPPS)
+else
+   MINIAPPS = $(PAR_MINIAPPS) $(SEQ_MINIAPPS)
+endif
+
+COMMON_LIB = -L$(MFEM_BUILD_DIR)/miniapps/common -lmfem-common
+
+# If MFEM_SHARED is set, add the ../common rpath
+COMMON_LIB += $(if $(MFEM_SHARED:YES=),,\
+   $(if $(MFEM_USE_CUDA:YES=),$(CXX_XLINKER),$(CUDA_XLINKER))-rpath,$(abspath\
+   $(MFEM_BUILD_DIR)/miniapps/common))
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all lib-common clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+%.o: %.cpp
+
+%.o: $(SRC)%.cpp $(wildcard $(SRC)%.hpp) $(MFEM_LIB_FILE)\
+ $(CONFIG_MK) | lib-common
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $< -o $@
+
+all: $(MINIAPPS)
+
+pmaxwell-tokamak: $(PMAXWELL_TOKAMAK_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_TOKAMAK_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+pmaxwell-fem-tokamak: $(PMAXWELL_FEM_TOKAMAK_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_FEM_TOKAMAK_OBJ) $(COMMON_LIB) $(MFEM_LIBS)	
+
+pmaxwell-primal-tokamak: $(PMAXWELL_PRIMAL_TOKAMAK_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_PRIMAL_TOKAMAK_OBJ) $(COMMON_LIB) $(MFEM_LIBS)	
+
+pmaxwell-uw-tokamak: $(PMAXWELL_UW_TOKAMAK_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_UW_TOKAMAK_OBJ) $(COMMON_LIB) $(MFEM_LIBS)		
+
+pmaxwell-uw-lh: $(PMAXWELL_UW_LH_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_UW_LH_OBJ) $(COMMON_LIB) $(MFEM_LIBS)		
+
+pmaxwell-fem-lh: $(PMAXWELL_FEM_LH_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_FEM_LH_OBJ) $(COMMON_LIB) $(MFEM_LIBS)		
+
+# Rule for building lib-common
+lib-common:
+	$(MAKE) -C $(MFEM_BUILD_DIR)/miniapps/common
+
+MFEM_TESTS = MINIAPPS
+include $(MFEM_TEST_MK)
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(PAR_MINIAPPS) $(SEQ_MINIAPPS)
+	rm -f $(DPG_COMPLEX_OBJ) $(MAXWELL_OBJ)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -rf ParaView 

miniapps/dpg/plasma/other/pmaxwell-fem-lh.cpp

@@ -0,0 +1,318 @@
+//                   MFEM Ultraweak DPG Maxwell parallel example
+//
+// the "ultraweak" (UW) DPG formulation for the Maxwell problem
+
+//      ∇×(1/μ ∇×E) - ω² ϵ E = Ĵ ,   in Ω
+//                       E×n = E₀ , on ∂Ω
+
+#include "mfem.hpp"
+#include "../../util/pcomplexweakform.hpp"
+#include "../../util/utils.hpp"
+#include "../../util/maxwell_utils.hpp"
+#include "../../../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+#include <cstring>
+#include <filesystem>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // fine mesh (trianles)
+   // default mesh
+   const char *mesh_file = "data/mesh-tri34K.mesh";
+   // coarse mesh (triangles)
+   // const char *mesh_file = "data/mesh-tri11K.mesh";
+   // coarse mesh (quadrilaterals)
+   // const char *mesh_file = "data/mesh-quad5K.mesh";
+
+   // epsilon tensor
+   const char * eps_r_file = nullptr;
+   const char * eps_i_file = nullptr;
+
+   int order = 2;
+   int par_ref_levels = 0;
+   bool visualization = false;
+   // real_t rnum=4.6e9;
+   // real_t mu = 1.257e-6;
+   // real_t epsilon_scale = 8.8541878128e-12*factor;
+   real_t rnum=4.6;
+   real_t mu = 1.257;
+   real_t epsilon_scale = 8.8541878128;
+
+   bool paraview = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   if (strcmp(mesh_file, "data/mesh-tri34K.mesh") == 0)
+   {
+      eps_r_file = "data/eps-tri34K_r.gf";
+      eps_i_file = "data/eps-tri34K_i.gf";
+   }
+   else if (strcmp(mesh_file, "data/mesh-tri11K.mesh") == 0)
+   {
+      eps_r_file = "data/eps-tri11K_r.gf";
+      eps_i_file = "data/eps-tri11K_i.gf";
+   }
+   else if (strcmp(mesh_file, "data/mesh-quad5K.mesh") == 0)
+   {
+      eps_r_file = "data/eps-quad5K_r.gf";
+      eps_i_file = "data/eps-quad5K_i.gf";
+   }
+   else
+   {
+      MFEM_ABORT("Unknown mesh file: " + string(mesh_file));
+   }
+
+   real_t omega = 2.*M_PI*rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   Array<int> int_bdr_attr;
+   for (int i = 0; i < mesh.GetNBE(); i++)
+   {
+      if (mesh.FaceIsInterior(mesh.GetBdrElementFaceIndex(i)))
+      {
+         int_bdr_attr.Append(mesh.GetBdrAttribute(i));
+      }
+   }
+   int_bdr_attr.Sort();
+   int_bdr_attr.Unique();
+
+   // mesh.RemoveInternalBoundaries();
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+
+   EpsilonMatrixCoefficient eps_r_cf(eps_r_file,&mesh,&pmesh, epsilon_scale);
+   EpsilonMatrixCoefficient eps_i_cf(eps_i_file,&mesh,&pmesh, epsilon_scale);
+
+   for (int i = 0; i<par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+      eps_r_cf.Update();
+      eps_i_cf.Update();
+   }
+
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh, fec);
+
+   // Bilinear form coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient muinv(1./mu);
+
+   ScalarMatrixProductCoefficient m_cf_r(-omega*omega, eps_r_cf);
+   ScalarMatrixProductCoefficient m_cf_i(-omega*omega, eps_i_cf);
+
+   ParComplexLinearForm *b = new ParComplexLinearForm(E_fes);
+   b->Vector::operator=(0.0);
+
+   ParSesquilinearForm *a = new ParSesquilinearForm(E_fes);
+   a->AddDomainIntegrator(new CurlCurlIntegrator(muinv),nullptr);
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(m_cf_r),
+                          new VectorFEMassIntegrator(m_cf_i));
+
+   socketstream E_out_r;
+   socketstream E_theta_out_r;
+   socketstream E_theta_out_i;
+
+   ParComplexGridFunction E_gf(E_fes);
+   E_gf.real() = 0.0;
+   E_gf.imag() = 0.0;
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+
+   ParGridFunction E_theta_r(&L2_fes);
+   ParGridFunction E_theta_i(&L2_fes);
+   ParGridFunction E_theta(&L2_fes);
+   E_theta = 0.0;
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+   // ParaViewDataCollection * paraview_tdc = nullptr;
+
+   std::string output_dir = "ParaView/FEM/" + GetTimestamp();
+   if (Mpi::Root())
+   {
+      WriteParametersToFile(args, output_dir);
+   }
+
+   if (paraview)
+   {
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",&E_gf.real());
+      paraview_dc->RegisterField("E_i",&E_gf.imag());
+      paraview_dc->RegisterField("E_theta_r",&E_theta_r);
+      paraview_dc->RegisterField("E_theta_i",&E_theta_i);
+
+      // paraview_tdc = new ParaViewDataCollection(mesh_file, &pmesh);
+      // paraview_tdc->SetPrefixPath("ParaViewFEM2D/TimeHarmonic");
+      // paraview_tdc->SetLevelsOfDetail(order);
+      // paraview_tdc->SetCycle(0);
+      // paraview_tdc->SetDataFormat(VTKFormat::BINARY);
+      // paraview_tdc->SetHighOrderOutput(true);
+      // paraview_tdc->SetTime(0.0); // set the time
+      // paraview_tdc->RegisterField("E_theta_t",&E_theta);
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      // remove internal boundaries
+      for (int i = 0; i<int_bdr_attr.Size(); i++)
+      {
+         ess_bdr[int_bdr_attr[i]-1] = 0;
+      }
+
+      E_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_cf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   E_gf.ProjectBdrCoefficientTangent(one_x_cf,zero_cf, one_r_bdr);
+   E_gf.ProjectBdrCoefficientTangent(negone_x_cf,zero_cf, negone_r_bdr);
+   E_gf.ProjectBdrCoefficientTangent(zero_cf,one_x_cf, one_i_bdr);
+   E_gf.ProjectBdrCoefficientTangent(zero_cf,negone_x_cf, negone_i_bdr);
+
+   b->Assemble();
+   a->Assemble();
+
+   OperatorPtr Ah;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, E_gf, *b, Ah, X, B);
+
+#ifdef MFEM_USE_MUMPS
+   HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+   // auto cpardiso = new CPardisoSolver(A->GetComm());
+   auto solver = new MUMPSSolver(MPI_COMM_WORLD);
+   solver->SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
+   solver->SetPrintLevel(1);
+   solver->SetOperator(*A);
+   solver->Mult(B,X);
+   delete A;
+   delete solver;
+#else
+   MFEM_ABORT("MFEM compiled without mumps");
+#endif
+
+   a->RecoverFEMSolution(X, *b, E_gf);
+
+   AzimuthalECoefficient az_e_r(&E_gf.real());
+   AzimuthalECoefficient az_e_i(&E_gf.imag());
+
+   E_theta_r.ProjectCoefficient(az_e_r);
+   E_theta_i.ProjectCoefficient(az_e_i);
+
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, E_gf.real(),
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+      common::VisualizeField(E_theta_out_r,vishost, visport, E_theta_r,
+                             "Numerical Electric field (azimuthal)", 0, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((real_t)0);
+      paraview_dc->Save();
+      delete paraview_dc;
+
+      // int num_frames = 32;
+      // for (int i = 0; i<num_frames; i++)
+      // {
+      //    real_t t = (real_t)(i % num_frames) / num_frames;
+      //    add(cos(real_t(2.0*M_PI)*t), E_theta_r,
+      //        sin(real_t(2.0*M_PI)*t), E_theta_i, E_theta);
+      //    paraview_tdc->SetCycle(i);
+      //    paraview_tdc->SetTime(t);
+      //    paraview_tdc->Save();
+      // }
+      // delete paraview_tdc;
+   }
+
+
+   delete a;
+   delete b;
+   delete E_fes;
+   delete fec;
+
+   return 0;
+
+}

miniapps/dpg/plasma/other/pmaxwell-fem-tokamak.cpp

@@ -0,0 +1,307 @@
+
+// srun -n 256 ./pmaxwell-fem-tokamak -o 3 -sc -rnum
+// srun -n 448 ./pmaxwell-fem-tokamak -o 4 -sc -rnum 11.0 -sigma 2.0 -paraview
+
+// srun -n 448 ./pmaxwell-fem-tokamak -o 4 -paraview
+// Description:
+// This example code demonstrates the use of MFEM to define and solve
+// the standard FEM formulation for the Maxwell problem
+
+//      ∇×(1/μ ∇×E) - (ω^2 ϵ + i ω σ) E = J ,   in Ω
+//                E×n = E_0, on ∂Ω
+
+#include "mfem.hpp"
+#include "../../util/pcomplexweakform.hpp"
+#include "../../../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class EpsilonMatrixCoefficient : public MatrixArrayCoefficient
+{
+private:
+   Mesh * mesh = nullptr;
+   ParMesh * pmesh = nullptr;
+   Array<ParGridFunction * > pgfs;
+   Array<GridFunctionCoefficient * > gf_cfs;
+   GridFunction * vgf = nullptr;
+   int dim;
+public:
+   EpsilonMatrixCoefficient(const char * filename, Mesh * mesh_, ParMesh * pmesh_,
+                            double scale = 1.0)
+      : MatrixArrayCoefficient(mesh_->Dimension()), mesh(mesh_), pmesh(pmesh_),
+        dim(mesh_->Dimension())
+   {
+      std::filebuf fb;
+      fb.open(filename,std::ios::in);
+      std::istream is(&fb);
+      vgf = new GridFunction(mesh,is);
+      fb.close();
+      FiniteElementSpace * vfes = vgf->FESpace();
+      int vdim = vfes->GetVDim();
+      const FiniteElementCollection * fec = vfes->FEColl();
+      FiniteElementSpace * fes = new FiniteElementSpace(mesh, fec);
+      int num_procs = Mpi::WorldSize();
+      int * partitioning = mesh->GeneratePartitioning(num_procs);
+      double *data = vgf->GetData();
+      GridFunction gf;
+      pgfs.SetSize(vdim);
+      gf_cfs.SetSize(vdim);
+      for (int i = 0; i<dim; i++)
+      {
+         for (int j = 0; j<dim; j++)
+         {
+            int k = i*dim+j;
+            // int k = j*dim+i;
+            gf.MakeRef(fes,&data[k*fes->GetVSize()]);
+            pgfs[k] = new ParGridFunction(pmesh,&gf,partitioning);
+            (*pgfs[k])*=scale;
+            gf_cfs[k] = new GridFunctionCoefficient(pgfs[k]);
+            Set(i,j,gf_cfs[k], true);
+         }
+      }
+   }
+   ~EpsilonMatrixCoefficient()
+   {
+      for (int i = 0; i<pgfs.Size(); i++)
+      {
+         delete pgfs[i];
+      }
+      pgfs.DeleteAll();
+   }
+};
+
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "data/mesh_330k.mesh";
+   const char * eps_r_file = "data/eps_r_330k.gf";
+   const char * eps_i_file = "data/eps_i_330k.gf";
+
+   int order = 1;
+   bool visualization = false;
+   double rnum=50.0e6;
+   int sr = 0;
+   int pr = 0;
+   bool paraview = false;
+   double mu = 1.257e-6;
+   double epsilon = 1.0;
+   double epsilon_scale = 8.8541878128e-12;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&sr, "-sref", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pref", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   socketstream E_out_r;
+
+
+   double omega = 2.*M_PI*rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   for (int i = 0; i<sr; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+
+
+   EpsilonMatrixCoefficient eps_r_cf(eps_r_file,&mesh,&pmesh, epsilon_scale);
+   EpsilonMatrixCoefficient eps_i_cf(eps_i_file,&mesh,&pmesh, epsilon_scale);
+
+   mesh.Clear();
+
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh, fec);
+
+   // Bilinear form coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient muinv(1./mu);
+
+   if (myid == 0)
+   {
+      std::cout << "Assembling matrix" << endl;
+   }
+
+   ScalarMatrixProductCoefficient m_cf_r(-omega*omega, eps_r_cf);
+   ScalarMatrixProductCoefficient m_cf_i(-omega*omega, eps_i_cf);
+
+   ParComplexLinearForm *b = new ParComplexLinearForm(E_fes);
+   b->Vector::operator=(0.0);
+
+   ParSesquilinearForm *a = new ParSesquilinearForm(E_fes);
+   a->AddDomainIntegrator(new CurlCurlIntegrator(muinv),nullptr);
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(m_cf_r),
+                          new VectorFEMassIntegrator(m_cf_i));
+
+   ParComplexGridFunction E_gf(E_fes);
+   E_gf.real() = 0.0;
+   E_gf.imag() = 0.0;
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   if (paraview)
+   {
+      paraview_dc = new ParaViewDataCollection(mesh_file, &pmesh);
+      paraview_dc->SetPrefixPath("ParaView");
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",&E_gf.real());
+      paraview_dc->RegisterField("E_i",&E_gf.imag());
+   }
+
+   // internal bdr attributes
+   Array<int> internal_bdr({1, 3, 6, 9, 17, 157, 185, 75, 210, 211,
+                            212, 213, 214, 215, 216, 217, 218, 219,
+                            220, 221, 222, 223, 224, 225, 226, 227,
+                            228, 229, 230, 231, 232, 233, 234, 125});
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   Array<int> one_bdr;
+   Array<int> negone_bdr;
+
+   if (myid == 0)
+   {
+      std::cout << "Attributes" << endl;
+   }
+
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      // need to exclude these attributes
+      for (int i = 0; i<internal_bdr.Size(); i++)
+      {
+         ess_bdr[internal_bdr[i]-1] = 0;
+      }
+      E_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      one_bdr = 0;
+      negone_bdr = 0;
+      one_bdr[234] = 1;
+      negone_bdr[235] = 1;
+   }
+
+   if (myid == 0)
+   {
+      std::cout << "Attributes 2" << endl;
+   }
+
+   Vector z_one(3); z_one = 0.0; z_one(2) = 1.0;
+   Vector zero(3); zero = 0.0;
+   Vector z_negone(3); z_negone = 0.0; z_negone(2) = -1.0;
+   VectorConstantCoefficient z_one_cf(z_one);
+   VectorConstantCoefficient z_negone_cf(z_negone);
+   VectorConstantCoefficient zero_cf(zero);
+
+
+   E_gf.real() = 0.0;
+   E_gf.imag() = 0.0;
+   E_gf.ProjectBdrCoefficientTangent(z_one_cf,zero_cf, one_bdr);
+   E_gf.ProjectBdrCoefficientTangent(z_negone_cf,zero_cf, negone_bdr);
+
+   if (myid == 0)
+   {
+      std::cout << "Assembly started" << endl;
+   }
+   b->Assemble();
+   a->Assemble();
+
+   OperatorPtr Ah;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, E_gf, *b, Ah, X, B);
+
+   if (myid == 0)
+   {
+      std::cout << "Assembly finished" << endl;
+   }
+
+#ifdef MFEM_USE_MUMPS
+   HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+   // auto cpardiso = new CPardisoSolver(A->GetComm());
+   auto solver = new MUMPSSolver(MPI_COMM_WORLD);
+   solver->SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
+   solver->SetPrintLevel(1);
+   solver->SetOperator(*A);
+   solver->Mult(B,X);
+   delete A;
+   delete solver;
+#else
+   MFEM_ABORT("MFEM compiled without mumps");
+#endif
+
+   a->RecoverFEMSolution(X, *b, E_gf);
+
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, E_gf.real(),
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((double)0);
+      paraview_dc->Save();
+   }
+
+   if (paraview)
+   {
+      delete paraview_dc;
+   }
+
+   delete a;
+   delete b;
+   delete E_fes;
+   delete fec;
+
+   return 0;
+}
+

miniapps/dpg/plasma/other/pmaxwell-primal-tokamak.cpp

@@ -0,0 +1,495 @@
+
+// srun -n 256 ./pmaxwell-primal-tokamak -o 3 -sc -rnum
+// srun -n 448 ./pmaxwell-primal-tokamak -o 4 -sc -rnum 11.0 -paraview
+
+// srun -n 448 ./pmaxwell-primal-tokamak -o 4 -do 0 -sc -paraview (with the new epsilon GridFunction coefficients)
+// Description:
+// This example code demonstrates the use of MFEM to define and solve
+// the "ultraweak" (UW) DPG formulation for the Maxwell problem
+
+//      ∇×(1/μ ∇×E) - ω^2 ϵ E = Ĵ ,   in Ω
+//                E×n = E_0, on ∂Ω
+
+// The primal-DPG formulation is obtained by integration by parts
+// and the introduction of trace unknowns on the mesh skeleton
+
+// in 3D
+// E ∈ H(curl)
+// Ê ∈ H_0^1/2(Ω)(curl, Γ_h)
+//  1/μ (∇×E , ∇×F) + (ω^2 ϵ , F) + <Ê , F × n> = 0,      ∀ F ∈ H(curl,Ω)
+//                                        Ê × n = E_0     on ∂Ω
+
+#include "mfem.hpp"
+#include "../../util/pcomplexweakform.hpp"
+#include "../../../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class EpsilonMatrixCoefficient : public MatrixArrayCoefficient
+{
+private:
+   Mesh * mesh = nullptr;
+   ParMesh * pmesh = nullptr;
+   Array<ParGridFunction * > pgfs;
+   Array<GridFunctionCoefficient * > gf_cfs;
+   GridFunction * vgf = nullptr;
+   int dim;
+public:
+   EpsilonMatrixCoefficient(const char * filename, Mesh * mesh_, ParMesh * pmesh_,
+                            double scale = 1.0)
+      : MatrixArrayCoefficient(mesh_->Dimension()), mesh(mesh_), pmesh(pmesh_),
+        dim(mesh_->Dimension())
+   {
+      std::filebuf fb;
+      fb.open(filename,std::ios::in);
+      std::istream is(&fb);
+      vgf = new GridFunction(mesh,is);
+      fb.close();
+      FiniteElementSpace * vfes = vgf->FESpace();
+      int vdim = vfes->GetVDim();
+      const FiniteElementCollection * fec = vfes->FEColl();
+      FiniteElementSpace * fes = new FiniteElementSpace(mesh, fec);
+      int num_procs = Mpi::WorldSize();
+      int * partitioning = mesh->GeneratePartitioning(num_procs);
+      double *data = vgf->GetData();
+      GridFunction gf;
+      pgfs.SetSize(vdim);
+      gf_cfs.SetSize(vdim);
+      for (int i = 0; i<dim; i++)
+      {
+         for (int j = 0; j<dim; j++)
+         {
+            int k = i*dim+j;
+            // int k = j*dim+i;
+            gf.MakeRef(fes,&data[k*fes->GetVSize()]);
+            pgfs[k] = new ParGridFunction(pmesh,&gf,partitioning);
+            (*pgfs[k])*=scale;
+            gf_cfs[k] = new GridFunctionCoefficient(pgfs[k]);
+            Set(i,j,gf_cfs[k], true);
+         }
+      }
+   }
+   ~EpsilonMatrixCoefficient()
+   {
+      for (int i = 0; i<pgfs.Size(); i++)
+      {
+         delete pgfs[i];
+      }
+      pgfs.DeleteAll();
+   }
+
+};
+
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "data/mesh_330k.mesh";
+   const char * eps_r_file = "data/eps_r_330k.gf";
+   const char * eps_i_file = "data/eps_i_330k.gf";
+
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = false;
+   // double rnum=50.0;
+   double rnum=50.0e6;
+   int sr = 0;
+   int pr = 0;
+   bool paraview = false;
+   double mu = 1.257e-6;
+   double epsilon = 1.0;
+   double epsilon_scale = 8.8541878128e-12;
+   // double epsilon_scale = 8.8541878128;
+   bool mumps_solver = false;
+
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&sr, "-sref", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pref", "--parallel_ref",
+                  "Number of parallel refinements.");
+#ifdef MFEM_USE_MUMPS
+   args.AddOption(&mumps_solver, "-mumps", "--mumps-solver", "-no-mumps",
+                  "--no-mumps-solver", "Use the MUMPS Solver.");
+#endif
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   double omega = 2.*M_PI*rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   for (int i = 0; i<sr; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+
+   EpsilonMatrixCoefficient eps_r_cf(eps_r_file,&mesh,&pmesh, epsilon_scale);
+   EpsilonMatrixCoefficient eps_i_cf(eps_i_file,&mesh,&pmesh, epsilon_scale);
+   mesh.Clear();
+
+   FiniteElementCollection *E_fec = new ND_FECollection(order,dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh,E_fec);
+
+   // H^-1/2 (curl) space for Ê
+   int test_order = order+delta_order;
+   FiniteElementCollection * hatE_fec = new ND_Trace_FECollection(order,dim);
+   FiniteElementCollection * F_fec = new ND_FECollection(test_order, dim);
+   ParFiniteElementSpace *hatE_fes = new ParFiniteElementSpace(&pmesh,hatE_fec);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+   trial_fes.Append(E_fes);
+   trial_fes.Append(hatE_fes);
+   test_fec.Append(F_fec);
+
+   // Bilinear form coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient invmu_cf(1./mu);
+
+   Vector z_one(3); z_one = 0.0; z_one(2) = 1.0;
+   Vector zero(3); zero = 0.0;
+   Vector z_negone(3); z_negone = 0.0; z_negone(2) = -1.0;
+   VectorConstantCoefficient z_one_cf(z_one);
+   VectorConstantCoefficient z_negone_cf(z_negone);
+   VectorConstantCoefficient zero_cf(zero);
+
+   if (myid == 0)
+   {
+      std::cout << "Assembling matrix" << endl;
+   }
+
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_fes,test_fec);
+
+   // (∇ × E,∇ × F)
+   a->AddTrialIntegrator(new MixedCurlCurlIntegrator(invmu_cf), nullptr,0,0);
+
+   // -(ω^2 ϵ, F)
+   ScalarMatrixProductCoefficient m_cf_r(-omega*omega, eps_r_cf);
+   ScalarMatrixProductCoefficient m_cf_i(-omega*omega, eps_i_cf);
+
+   const IntegrationRule *irs[Geometry::NumGeom];
+   int order_quad = 2*order + 2;
+   for (int i = 0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+   const IntegrationRule &ir = IntRules.Get(pmesh.GetElementGeometry(0),
+                                            2*test_order + 2);
+   VectorFEMassIntegrator * integ_r = new VectorFEMassIntegrator(m_cf_r);
+   VectorFEMassIntegrator * integ_i = new VectorFEMassIntegrator(m_cf_i);
+   integ_r->SetIntegrationRule(ir);
+   integ_i->SetIntegrationRule(ir);
+   a->AddTrialIntegrator(integ_r,integ_i,0,0);
+
+   // < n×Ê,F>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,1,0);
+
+   // test integrators
+   // (∇×F ,∇× δF)
+
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,0,0);
+
+   // (F,δF)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,0,0);
+
+   socketstream E_out_r;
+
+   ParComplexGridFunction E_gf(E_fes);
+   E_gf.real() = 0.0;
+   E_gf.imag() = 0.0;
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   if (paraview)
+   {
+      paraview_dc = new ParaViewDataCollection(mesh_file, &pmesh);
+      paraview_dc->SetPrefixPath("ParaViewPrimalDPG");
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",&E_gf.real());
+      paraview_dc->RegisterField("E_i",&E_gf.imag());
+   }
+
+   // internal bdr attributes
+   Array<int> internal_bdr({1, 3, 6, 9, 17, 157, 185, 75, 210, 211,
+                            212, 213, 214, 215, 216, 217, 218, 219,
+                            220, 221, 222, 223, 224, 225, 226, 227,
+                            228, 229, 230, 231, 232, 233, 234, 125});
+
+   for (int it = 0; it<=pr; it++)
+   {
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      Array<int> one_bdr;
+      Array<int> negone_bdr;
+
+      if (myid == 0)
+      {
+         std::cout << "Attributes" << endl;
+      }
+
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         one_bdr.SetSize(pmesh.bdr_attributes.Max());
+         negone_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         // need to exclude these attributes
+         for (int i = 0; i<internal_bdr.Size(); i++)
+         {
+            ess_bdr[internal_bdr[i]-1] = 0;
+         }
+         E_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+         one_bdr = 0;
+         negone_bdr = 0;
+         one_bdr[234] = 1;
+         negone_bdr[235] = 1;
+      }
+
+      if (myid == 0)
+      {
+         std::cout << "Attributes 2" << endl;
+      }
+
+      Array<int> offsets(3);
+      offsets[0] = 0;
+      offsets[1] = E_fes->GetVSize();
+      offsets[2] = hatE_fes->GetVSize();
+      offsets.PartialSum();
+
+      Vector x(2*offsets.Last());
+      x = 0.;
+      double * xdata = x.GetData();
+
+      E_gf.real().MakeRef(E_fes,&xdata[0]);
+      E_gf.imag().MakeRef(E_fes,&xdata[offsets.Last()]);
+
+      E_gf.ProjectBdrCoefficientTangent(z_one_cf,zero_cf, one_bdr);
+      E_gf.ProjectBdrCoefficientTangent(z_negone_cf,zero_cf, negone_bdr);
+
+      if (myid == 0)
+      {
+         std::cout << "Assembly started" << endl;
+      }
+
+      a->Assemble();
+
+      if (myid == 0)
+      {
+         std::cout << "Assembly finished" << endl;
+      }
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+      int num_blocks = BlockA_r->NumRowBlocks();
+      Array<int> tdof_offsets(2*num_blocks+1);
+
+      tdof_offsets[0] = 0;
+      for (int i=0; i<num_blocks; i++)
+      {
+         tdof_offsets[i+1] = trial_fes[i]->GetTrueVSize();
+         tdof_offsets[num_blocks+i+1] = trial_fes[i]->GetTrueVSize();
+      }
+      tdof_offsets.PartialSum();
+
+      BlockOperator blockA(tdof_offsets);
+      for (int i = 0; i<num_blocks; i++)
+      {
+         for (int j = 0; j<num_blocks; j++)
+         {
+            blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
+            blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
+         }
+      }
+
+
+#ifdef MFEM_USE_MUMPS
+      if (mumps_solver)
+      {
+         // Monolithic real part
+         Array2D <HypreParMatrix * > Ab_r(num_blocks,num_blocks);
+         // Monolithic imag part
+         Array2D <HypreParMatrix * > Ab_i(num_blocks,num_blocks);
+         for (int i = 0; i<num_blocks; i++)
+         {
+            for (int j = 0; j<num_blocks; j++)
+            {
+               Ab_r(i,j) = &(HypreParMatrix &)BlockA_r->GetBlock(i,j);
+               Ab_i(i,j) = &(HypreParMatrix &)BlockA_i->GetBlock(i,j);
+            }
+         }
+         HypreParMatrix * A_r = HypreParMatrixFromBlocks(Ab_r);
+         HypreParMatrix * A_i = HypreParMatrixFromBlocks(Ab_i);
+
+         ComplexHypreParMatrix Acomplex(A_r, A_i,true,true);
+
+         HypreParMatrix * A = Acomplex.GetSystemMatrix();
+
+         MUMPSSolver mumps(MPI_COMM_WORLD);
+         mumps.SetPrintLevel(0);
+         mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
+         mumps.SetOperator(*A);
+         mumps.Mult(B,X);
+         delete A;
+      }
+#else
+      if (mumps_solver)
+      {
+         MFEM_WARNING("MFEM compiled without mumps. Switching to an iterative solver");
+      }
+      mumps_solver = false;
+#endif
+      if (!mumps_solver)
+      {
+         BlockDiagonalPreconditioner M(tdof_offsets);
+
+         HypreAMS * solver_E = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(0,0),
+                                            E_fes);
+         HypreAMS * solver_hatE = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(1,1),
+                                               hatE_fes);
+         solver_E->SetPrintLevel(0);
+         solver_hatE->SetPrintLevel(0);
+
+         M.SetDiagonalBlock(0,solver_E);
+         M.SetDiagonalBlock(1,solver_hatE);
+
+         if (myid == 0)
+         {
+            std::cout << "PCG iterations" << endl;
+         }
+
+
+         double bl2norm = B.Norml2();
+         double xl2norm = X.Norml2();
+         MPI_Allreduce(MPI_IN_PLACE,&bl2norm,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+         MPI_Allreduce(MPI_IN_PLACE,&xl2norm,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+         if (myid == 0)
+         {
+            mfem::out << "X.Norm = " << xl2norm << endl;
+            mfem::out << "B.Norm = " << bl2norm << endl;
+         }
+         CGSolver cg(MPI_COMM_WORLD);
+         cg.SetRelTol(1e-8);
+         cg.SetMaxIter(1000);
+         cg.SetPrintLevel(1);
+         cg.SetPreconditioner(M);
+         cg.SetOperator(blockA);
+         cg.Mult(B, X);
+
+         for (int i = 0; i<num_blocks; i++)
+         {
+            delete &M.GetDiagonalBlock(i);
+         }
+
+         int num_iter = cg.GetNumIterations();
+
+      }
+      a->RecoverFEMSolution(X,x);
+
+      E_gf.real().MakeRef(E_fes,x.GetData());
+      E_gf.imag().MakeRef(E_fes,&x.GetData()[offsets.Last()]);
+
+      int dofs = 0;
+      for (int i = 0; i<trial_fes.Size(); i++)
+      {
+         dofs += trial_fes[i]->GlobalTrueVSize();
+      }
+
+      if (visualization)
+      {
+         const char * keys = (it == 0 && dim == 2) ? "jRcml\n" : nullptr;
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         common::VisualizeField(E_out_r,vishost, visport, E_gf.real(),
+                                "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+      }
+
+      if (paraview)
+      {
+         paraview_dc->SetCycle(it);
+         paraview_dc->SetTime((double)it);
+         paraview_dc->Save();
+      }
+
+      if (it == pr)
+      {
+         break;
+      }
+
+      pmesh.UniformRefinement();
+
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   if (paraview)
+   {
+      delete paraview_dc;
+   }
+
+   delete a;
+   delete F_fec;
+   delete hatE_fes;
+   delete hatE_fec;
+   delete E_fec;
+   delete E_fes;
+
+   return 0;
+}
+

miniapps/dpg/plasma/other/pmaxwell-tokamak.cpp

@@ -0,0 +1,782 @@
+
+// srun -n 256 ./pmaxwell-tokamak -o 3 -sc -rnum
+// srun -n 448 ./pmaxwell-tokamak -o 4 -sc -rnum 11.0 -sigma 2.0 -paraview
+
+// srun -n 448 ./pmaxwell-tokamak -o 4 -do 0 -sc -paraview (with the new epsilon GridFunction coefficients)
+// TODO do > 0 fails with non SPD G matrix
+// Description:
+// This example code demonstrates the use of MFEM to define and solve
+// the "ultraweak" (UW) DPG formulation for the Maxwell problem
+
+//      ∇×(1/μ ∇×E) - (ω^2 ϵ + i ω σ) E = Ĵ ,   in Ω
+//                E×n = E_0, on ∂Ω
+
+// The DPG UW deals with the First Order System
+//        i ω μ H + ∇ × E = 0,   in Ω (Faraday's law)
+//            M E + ∇ × H = J,   in Ω (Ampere's law)
+//            E × n = E_0, on ∂Ω
+// Note: Ĵ = -iωJ
+// where M = -(i ω ϵI + σI)
+
+// The ultraweak-DPG formulation is obtained by integration by parts of both
+// equations and the introduction of trace unknowns on the mesh skeleton
+
+// in 3D
+// E,H ∈ (L^2(Ω))^3
+// Ê ∈ H_0^1/2(Ω)(curl, Γ_h), Ĥ ∈ H^-1/2(curl, Γ_h)
+//  i ω μ (H,F) + (E,∇ × F) + < Ê, F × n > = 0,      ∀ F ∈ H(curl,Ω)
+//      M (E,G) + (H,∇ × G) + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)
+//                                   Ê × n = E_0     on ∂Ω
+// -------------------------------------------------------------------------
+// |   |       E      |      H      |      Ê       |       Ĥ      |  RHS    |
+// -------------------------------------------------------------------------
+// | F |  (E,∇ × F)   | i ω μ (H,F) | < n × Ê, F > |              |         |
+// |   |              |             |              |              |         |
+// | G |    (ME,G)    |  (H,∇ × G)  |              | < n × Ĥ, G > |  (J,G)  |
+// where (F,G) ∈  H(curl,Ω) × H(curl,Ω)
+
+// Here we use the "Adjoint Graph" norm on the test space i.e.,
+// ||(F,G)||^2_V = ||A^*(F,G)||^2 + ||(F,G)||^2 where A is the
+// maxwell operator defined by (1)
+
+
+#include "mfem.hpp"
+#include "../../util/pcomplexweakform.hpp"
+#include "../../../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class EpsilonMatrixCoefficient : public MatrixArrayCoefficient
+{
+private:
+   Mesh * mesh = nullptr;
+   ParMesh * pmesh = nullptr;
+   Array<ParGridFunction * > pgfs;
+   Array<GridFunctionCoefficient * > gf_cfs;
+   GridFunction * vgf = nullptr;
+   int dim;
+public:
+   EpsilonMatrixCoefficient(const char * filename, Mesh * mesh_, ParMesh * pmesh_,
+                            double scale = 1.0)
+      : MatrixArrayCoefficient(mesh_->Dimension()), mesh(mesh_), pmesh(pmesh_),
+        dim(mesh_->Dimension())
+   {
+      std::filebuf fb;
+      fb.open(filename,std::ios::in);
+      std::istream is(&fb);
+      vgf = new GridFunction(mesh,is);
+      fb.close();
+      FiniteElementSpace * vfes = vgf->FESpace();
+      int vdim = vfes->GetVDim();
+      const FiniteElementCollection * fec = vfes->FEColl();
+      FiniteElementSpace * fes = new FiniteElementSpace(mesh, fec);
+      int num_procs = Mpi::WorldSize();
+      int * partitioning = mesh->GeneratePartitioning(num_procs);
+      double *data = vgf->GetData();
+      GridFunction gf;
+      pgfs.SetSize(vdim);
+      gf_cfs.SetSize(vdim);
+      for (int i = 0; i<dim; i++)
+      {
+         for (int j = 0; j<dim; j++)
+         {
+            int k = i*dim+j;
+            // int k = j*dim+i;
+            gf.MakeRef(fes,&data[k*fes->GetVSize()]);
+            pgfs[k] = new ParGridFunction(pmesh,&gf,partitioning);
+            (*pgfs[k])*=scale;
+            gf_cfs[k] = new GridFunctionCoefficient(pgfs[k]);
+            Set(i,j,gf_cfs[k], true);
+         }
+      }
+   }
+   ~EpsilonMatrixCoefficient()
+   {
+      for (int i = 0; i<pgfs.Size(); i++)
+      {
+         delete pgfs[i];
+      }
+      pgfs.DeleteAll();
+   }
+
+};
+
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // const char *mesh_file = "tokamak_100k.msh";
+   // const char *mesh_file = "tokamak_200k.msh";
+   // const char *mesh_file = "meshes/tokamak_100k.msh";
+   // const char *mesh_file = "meshes/tokamak_100k.msh";
+
+   const char *mesh_file = "data/mesh_330k.mesh";
+   const char * eps_r_file = "data/eps_r_330k.gf";
+   const char * eps_i_file = "data/eps_i_330k.gf";
+
+   // const char *mesh_file = "meshes/box.msh";
+
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = false;
+   double rnum=50.0e6;
+   bool static_cond = false;
+   int sr = 0;
+   int pr = 0;
+   bool paraview = false;
+   double factor = 1.0;
+   double mu = 1.257e-6/factor;
+   // double mu = 1.0;
+   double epsilon = 1.0;
+   double sigma = 0.01*factor;
+   double epsilon_scale = 8.8541878128e-12*factor;
+   bool graph_norm = true;
+   bool mumps_solver = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&sigma, "-sigma", "--sigma",
+                  "conductivity");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&sr, "-sref", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pref", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&graph_norm, "-graph", "--graph-norm", "-no-gn",
+                  "--no-graph-norm", "Enable adjoint graph norm.");
+#ifdef MFEM_USE_MUMPS
+   args.AddOption(&mumps_solver, "-mumps", "--mumps-solver", "-no-mumps",
+                  "--no-mumps-solver", "Use the MUMPS Solver.");
+#endif
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   double omega = 2.*M_PI*rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   for (int i = 0; i<sr; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+
+   // Test first with identity matrix coefficient
+   DenseMatrix Id(dim); Id = 0.0;
+   Id(0,0) = 1; Id(0,1) = 0.0; Id(0,2) = 0.0;
+   Id(1,0) = 0.0; Id(1,1) = 1; Id(1,2) = 0.0;
+   Id(2,0) = 0.0; Id(2,1) = 0.0; Id(2,2) = 1;
+   DenseMatrix C(dim); C = 0.0;
+   C(0,0) = 1.0; C(1,1) = 1.0; C(2,2) = 1.0;
+   DenseMatrix zmat(dim); zmat = 0.0;
+   MatrixConstantCoefficient identity_cf(Id);
+
+   // FiniteElementCollection *H1_fec = new H1_FECollection(1,dim);
+   // ParFiniteElementSpace *H1_fes = new ParFiniteElementSpace(&pmesh,H1_fec);
+
+   // Array<ParGridFunction * > pgfs(dim*dim);
+   // Array<GridFunctionCoefficient * > pcfs(dim*dim);
+   // MatrixArrayCoefficient eps_r_cf(dim);
+   // MatrixArrayCoefficient eps_i_cf(dim);
+   // for (int i = 0; i<dim; i++)
+   // {
+   //    for (int j = 0; j<dim; j++)
+   //    {
+   //       int k = i*dim + j;
+   //       pgfs[k] = new ParGridFunction(H1_fes);
+   //       if (i == j)
+   //       {
+   //          (*pgfs[k]) = 1.0;
+   //       }
+   //       else
+   //       {
+   //          (*pgfs[k]) = 0.0;
+   //       }
+   //       pcfs[k] = new GridFunctionCoefficient(pgfs[k]);
+   //       eps_r_cf.Set(i,j,pcfs[k],false);
+   //       eps_i_cf.Set(i,j,pcfs[k], false);
+   //    }
+   // }
+   // MatrixConstantCoefficient eps_r_cf(C);
+
+   // MatrixConstantCoefficient eps_i_cf(zmat);
+
+   DenseMatrix mat_eps_r(dim);
+   mat_eps_r(0,0) = -3.18447132e+02; mat_eps_r(0,1) = -7.73308634e-11;
+   mat_eps_r(0,2) =  2.26832549e+02;
+   mat_eps_r(1,0) = -7.73030845e-11; mat_eps_r(1,1) = -1.26300045e+06;
+   mat_eps_r(1,2) = -7.73308634e-11;
+   mat_eps_r(2,0) = -2.26832549e+02; mat_eps_r(2,1) = -7.73030845e-11;
+   mat_eps_r(2,2) = -3.18447132e+02;
+   DenseMatrix mat_eps_i(dim);
+   mat_eps_i(0,0) =  3.06793840e+02; mat_eps_i(0,1) =  4.06300785e-11;
+   mat_eps_i(0,2) =  7.95645437e+02;
+   mat_eps_i(1,0) =  4.07275170e-11; mat_eps_i(1,1) =  6.64641976e+05;
+   mat_eps_i(1,2) =  4.06300785e-11;
+   mat_eps_i(2,0) = -7.95645437e+02; mat_eps_i(2,1) =  4.07275170e-11;
+   mat_eps_i(2,2) =  3.06793840e+02;
+
+   mat_eps_r *= epsilon_scale;
+   mat_eps_i *= epsilon_scale;
+
+   // MatrixConstantCoefficient eps_r_cf(mat_eps_r);
+   // MatrixConstantCoefficient eps_i_cf(mat_eps_i);
+
+   EpsilonMatrixCoefficient eps_r_cf(eps_r_file,&mesh,&pmesh, epsilon_scale);
+   EpsilonMatrixCoefficient eps_i_cf(eps_i_file,&mesh,&pmesh, epsilon_scale);
+
+   mesh.Clear();
+
+
+   // Matrix Coefficient (M = -i\omega \epsilon - \sigma I);
+   // M = -i * omega * (eps_r + i eps_i) - sigmaI
+   //  = omega eps_i - sigma I + i (-omega eps_r)
+   MatrixSumCoefficient Mr_cf(eps_i_cf,identity_cf,omega,-sigma);
+   ScalarMatrixProductCoefficient Mi_cf(-omega,eps_r_cf);
+
+   // Matrix Coefficient eps_mat = -i\omega \epsilon
+   //                            = omega eps_mat_i + i (-omega eps_mat_r)
+   // ScalarMatrixProductCoefficient mat_eps_r_cf(omega,eps_i_cf);
+   // ScalarMatrixProductCoefficient mat_eps_i_cf(-omega,eps_r_cf);
+
+
+   // Define spaces
+   // L2 space for E
+   FiniteElementCollection *E_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh,E_fec,dim);
+
+   // Vector L2 space for H
+   FiniteElementCollection *H_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *H_fes = new ParFiniteElementSpace(&pmesh,H_fec, dim);
+
+   // H^-1/2 (curl) space for Ê
+   FiniteElementCollection * hatE_fec = nullptr;
+   FiniteElementCollection * hatH_fec = nullptr;
+   FiniteElementCollection * F_fec = nullptr;
+   int test_order = order+delta_order;
+   hatE_fec = new ND_Trace_FECollection(order,dim);
+   hatH_fec = new ND_Trace_FECollection(order,dim);
+   F_fec = new ND_FECollection(test_order, dim);
+
+   ParFiniteElementSpace *hatE_fes = new ParFiniteElementSpace(&pmesh,hatE_fec);
+   ParFiniteElementSpace *hatH_fes = new ParFiniteElementSpace(&pmesh,hatH_fec);
+   FiniteElementCollection * G_fec = new ND_FECollection(test_order, dim);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+   trial_fes.Append(E_fes);
+   trial_fes.Append(H_fes);
+   trial_fes.Append(hatE_fes);
+   trial_fes.Append(hatH_fes);
+   test_fec.Append(F_fec);
+   test_fec.Append(G_fec);
+
+   // Bilinear form coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient eps2omeg2(epsilon*epsilon*omega*omega);
+   ConstantCoefficient mu2omeg2(mu*mu*omega*omega);
+   ConstantCoefficient muomeg(mu*omega);
+   ConstantCoefficient negepsomeg(-epsilon*omega);
+   ConstantCoefficient epsomeg(epsilon*omega);
+   ConstantCoefficient negmuomeg(-mu*omega);
+   ConstantCoefficient sigma_cf(sigma);
+   ConstantCoefficient negsigma_cf(-sigma);
+   ConstantCoefficient sigma2_cf(sigma*sigma);
+
+   Vector z_one(3); z_one = 0.0; z_one(2) = 1.0;
+   Vector zero(3); zero = 0.0;
+   Vector z_negone(3); z_negone = 0.0; z_negone(2) = -1.0;
+   VectorConstantCoefficient z_one_cf(z_one);
+   VectorConstantCoefficient z_negone_cf(z_negone);
+   VectorConstantCoefficient zero_cf(zero);
+
+   if (myid == 0)
+   {
+      std::cout << "Assembling matrix" << endl;
+   }
+
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_fes,test_fec);
+
+   // (E,∇ × F)
+   a->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one)),
+                         nullptr,0,0);
+
+
+   // --------------------------------------------------------------------------
+   // -(i ω ϵ + σ) (E , G) = i (- ω ϵ E, G) - (σ E, G)
+   // a->AddTrialIntegrator(
+   // new TransposeIntegrator(new VectorFEMassIntegrator(negsigma_cf)),
+   // new TransposeIntegrator(new VectorFEMassIntegrator(negepsomeg)),0,1);
+
+   // a->AddTrialIntegrator(
+   // new TransposeIntegrator(new VectorFEMassIntegrator(negsigma_cf)), nullptr,0,1);
+   // a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(mat_eps_r_cf)),
+   //                       new TransposeIntegrator(new VectorFEMassIntegrator(mat_eps_i_cf)), 0,1);
+
+
+   //  (M E , G) = (M_r E, G) + i (M_i E, G)
+   a->AddTrialIntegrator(
+      new TransposeIntegrator(new VectorFEMassIntegrator(Mr_cf)),
+      new TransposeIntegrator(new VectorFEMassIntegrator(Mi_cf)),0,1);
+   // --------------------------------------------------------------------------
+
+
+   //  (H,∇ × G)
+   a->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one)),
+                         nullptr,1,1);
+   // < n×Ĥ ,G>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,3,1);
+   // test integrators
+   // (∇×G ,∇× δG)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,1,1);
+
+   ConstantCoefficient l2weight(1.0);
+
+   // (G,δG)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(l2weight),nullptr,1,1);
+
+   // i ω μ (H, F)
+   a->AddTrialIntegrator(nullptr,
+                         new TransposeIntegrator(new VectorFEMassIntegrator(muomeg)),1,0);
+   // < n×Ê,F>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,2,0);
+   // test integrators
+   // (∇×F,∇×δF)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,0,0);
+   // (F,δF)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,0,0);
+
+   if (graph_norm)
+   {
+      // μ^2 ω^2 (F,δF)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(mu2omeg2),nullptr,0,0);
+      // -i ω μ (F,∇ × δG) = i (F, -ω μ ∇ × δ G)
+      a->AddTestIntegrator(nullptr,new MixedVectorWeakCurlIntegrator(negmuomeg),0,1);
+
+      // --------------------------------------------------------------------------
+      // // -i ω ϵ (∇ × F, δG) - σ (∇ × F, δG)
+      // a->AddTestIntegrator(new MixedVectorCurlIntegrator(negsigma_cf),
+      // new MixedVectorCurlIntegrator(negepsomeg),0,1);
+
+      // a->AddTestIntegrator(new MixedVectorCurlIntegrator(negsigma_cf), nullptr, 0,1);
+
+      // a->AddTestIntegrator(new MixedVectorCurlIntegrator(mat_eps_r_cf),
+      //                      new MixedVectorCurlIntegrator(mat_eps_i_cf),0,1);
+
+      // (M ∇ × F, δG) = (M_r  ∇ × F, δG) + i (M_i  ∇ × F, δG)
+      a->AddTestIntegrator(new MixedVectorCurlIntegrator(Mr_cf),
+                           new MixedVectorCurlIntegrator(Mi_cf),0,1);
+      // --------------------------------------------------------------------------
+
+      // i ω μ (∇ × G,δF)
+      a->AddTestIntegrator(nullptr,new MixedVectorCurlIntegrator(muomeg),1,0);
+   }
+   // --------------------------------------------------------------------------
+   // i ω ϵ (G, ∇ × δF ) - σ (G, ∇ × δF )
+   // a->AddTestIntegrator(new MixedVectorWeakCurlIntegrator(negsigma_cf),
+   // new MixedVectorWeakCurlIntegrator(epsomeg),1,0);
+
+   // // (Eps^* G, ∇ × δF ) - σ (G, ∇ × δF ) = (Eps_r^T G,  ∇ × δF) - i (Eps_i^T G,  ∇ × δF) - σ (G, ∇ × δF )
+   // TransposeMatrixCoefficient mat_eps_rt_cf(mat_eps_r_cf);
+   // TransposeMatrixCoefficient mat_eps_it_cf(mat_eps_i_cf);
+   // ScalarMatrixProductCoefficient neg_mat_eps_it_cf(-1.0,mat_eps_it_cf);
+   // a->AddTestIntegrator(new MixedVectorWeakCurlIntegrator(negsigma_cf), nullptr, 1,0);
+   // a->AddTestIntegrator(new MixedVectorWeakCurlIntegrator(mat_eps_rt_cf),
+   //                      new MixedVectorWeakCurlIntegrator(neg_mat_eps_it_cf), 1,0);
+
+
+
+   // (M^* G, ∇ × δF ) = (Mr^T G,  ∇ × δF) - i (Mi^T G,  ∇ × δF)
+   TransposeMatrixCoefficient Mrt_cf(Mr_cf);
+   TransposeMatrixCoefficient Mit_cf(Mi_cf);
+   ScalarMatrixProductCoefficient negMit_cf(-1.0,Mit_cf);
+   if (graph_norm)
+   {
+      a->AddTestIntegrator(new MixedVectorWeakCurlIntegrator(Mrt_cf),
+                           new MixedVectorWeakCurlIntegrator(negMit_cf),1,0);
+   }
+   // --------------------------------------------------------------------------
+
+   // --------------------------------------------------------------------------
+   // ϵ^2 ω^2 (G,δG)
+   // a->AddTestIntegrator(new VectorFEMassIntegrator(eps2omeg2),nullptr,1,1);
+   // σ^2(G,δG)
+   // a->AddTestIntegrator(new VectorFEMassIntegrator(sigma2_cf),nullptr,1,1);
+
+   // MatrixProductCoefficient ErErt_cf(mat_eps_r_cf,mat_eps_rt_cf);
+   // MatrixProductCoefficient EiEit_cf(mat_eps_i_cf,mat_eps_it_cf);
+   // MatrixProductCoefficient EiErt_cf(mat_eps_i_cf,mat_eps_rt_cf);
+   // MatrixProductCoefficient ErEit_cf(mat_eps_r_cf,mat_eps_it_cf);
+
+   // MatrixSumCoefficient EEr_cf(ErErt_cf,EiEit_cf);
+   // MatrixSumCoefficient EEi_cf(EiErt_cf,ErEit_cf,1.0,-1.0);
+
+   // a->AddTestIntegrator(new VectorFEMassIntegrator(EEr_cf),
+   //                      new VectorFEMassIntegrator(EEi_cf),1,1);
+
+   // M*M^*(G,δG) = (MrMr^t + MiMi^t) + i (MiMr^t - MrMi^t)
+   MatrixProductCoefficient MrMrt_cf(Mr_cf,Mrt_cf);
+   MatrixProductCoefficient MiMit_cf(Mi_cf,Mit_cf);
+   MatrixProductCoefficient MiMrt_cf(Mi_cf,Mrt_cf);
+   MatrixProductCoefficient MrMit_cf(Mr_cf,Mit_cf);
+
+   MatrixSumCoefficient MMr_cf(MrMrt_cf,MiMit_cf);
+   MatrixSumCoefficient MMi_cf(MiMrt_cf,MrMit_cf,1.0,-1.0);
+
+   const IntegrationRule *irs[Geometry::NumGeom];
+   int order_quad = 2*order + 2;
+   for (int i = 0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+   const IntegrationRule &ir = IntRules.Get(pmesh.GetElementGeometry(0),
+                                            2*test_order + 2);
+
+   VectorFEMassIntegrator * integ_r = new VectorFEMassIntegrator(MMr_cf);
+   integ_r->SetIntegrationRule(ir);
+   VectorFEMassIntegrator * integ_i = new VectorFEMassIntegrator(MMi_cf);
+   integ_i->SetIntegrationRule(ir);
+
+   if (graph_norm)
+   {
+      a->AddTestIntegrator(integ_r,integ_i,1,1);
+   }
+   // --------------------------------------------------------------------------
+
+
+   socketstream E_out_r;
+   socketstream H_out_r;
+
+   ParComplexGridFunction E(E_fes);
+   ParComplexGridFunction H(H_fes);
+   E.real() = 0.0;
+   E.imag() = 0.0;
+   H.real() = 0.0;
+   H.imag() = 0.0;
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   if (paraview)
+   {
+      paraview_dc = new ParaViewDataCollection(mesh_file, &pmesh);
+      paraview_dc->SetPrefixPath("ParaViewUWDPG");
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",&E.real());
+      paraview_dc->RegisterField("E_i",&E.imag());
+      paraview_dc->RegisterField("H_r",&H.real());
+      paraview_dc->RegisterField("H_i",&H.imag());
+   }
+
+   // internal bdr attributes
+   Array<int> internal_bdr({1, 3, 6, 9, 17, 157, 185, 75, 210, 211,
+                            212, 213, 214, 215, 216, 217, 218, 219,
+                            220, 221, 222, 223, 224, 225, 226, 227,
+                            228, 229, 230, 231, 232, 233, 234, 125});
+
+   for (int it = 0; it<=pr; it++)
+   {
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      Array<int> one_bdr;
+      Array<int> negone_bdr;
+
+      if (myid == 0)
+      {
+         std::cout << "Attributes" << endl;
+      }
+
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         one_bdr.SetSize(pmesh.bdr_attributes.Max());
+         negone_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         // need to exclude these attributes
+         for (int i = 0; i<internal_bdr.Size(); i++)
+         {
+            ess_bdr[internal_bdr[i]-1] = 0;
+         }
+         hatE_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+         one_bdr = 0;
+         negone_bdr = 0;
+         one_bdr[234] = 1;
+         negone_bdr[235] = 1;
+         // one_bdr[6] = 1;
+         // negone_bdr[7] = 1;
+      }
+
+      if (myid == 0)
+      {
+         std::cout << "Attributes 2" << endl;
+      }
+
+      // Set up bdr conditions
+      // shift the ess_tdofs
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += E_fes->GetTrueVSize() + H_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = E_fes->GetVSize();
+      offsets[2] = H_fes->GetVSize();
+      offsets[3] = hatE_fes->GetVSize();
+      offsets[4] = hatH_fes->GetVSize();
+      offsets.PartialSum();
+
+      Vector x(2*offsets.Last());
+      x = 0.;
+      double * xdata = x.GetData();
+
+      ParComplexGridFunction hatE_gf(hatE_fes);
+      hatE_gf.real().MakeRef(hatE_fes,&xdata[offsets[2]]);
+      hatE_gf.imag().MakeRef(hatE_fes,&xdata[offsets.Last()+ offsets[2]]);
+
+      hatE_gf.ProjectBdrCoefficientTangent(z_one_cf,zero_cf, one_bdr);
+      hatE_gf.ProjectBdrCoefficientTangent(z_negone_cf,zero_cf, negone_bdr);
+
+      if (myid == 0)
+      {
+         std::cout << "Assembly started" << endl;
+      }
+
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      if (myid == 0)
+      {
+         std::cout << "Assembly finished" << endl;
+      }
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+      int num_blocks = BlockA_r->NumRowBlocks();
+      Array<int> tdof_offsets(2*num_blocks+1);
+
+      tdof_offsets[0] = 0;
+      int skip = (static_cond) ? 0 : 2;
+      int k = (static_cond) ? 2 : 0;
+      for (int i=0; i<num_blocks; i++)
+      {
+         tdof_offsets[i+1] = trial_fes[i+k]->GetTrueVSize();
+         tdof_offsets[num_blocks+i+1] = trial_fes[i+k]->GetTrueVSize();
+      }
+      tdof_offsets.PartialSum();
+
+      BlockOperator blockA(tdof_offsets);
+      for (int i = 0; i<num_blocks; i++)
+      {
+         for (int j = 0; j<num_blocks; j++)
+         {
+            blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
+            blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
+         }
+      }
+
+      X = 0.;
+
+#ifdef MFEM_USE_MUMPS
+      if (mumps_solver)
+      {
+         // Monolithic real part
+         Array2D <HypreParMatrix * > Ab_r(num_blocks,num_blocks);
+         // Monolithic imag part
+         Array2D <HypreParMatrix * > Ab_i(num_blocks,num_blocks);
+         for (int i = 0; i<num_blocks; i++)
+         {
+            for (int j = 0; j<num_blocks; j++)
+            {
+               Ab_r(i,j) = &(HypreParMatrix &)BlockA_r->GetBlock(i,j);
+               Ab_i(i,j) = &(HypreParMatrix &)BlockA_i->GetBlock(i,j);
+            }
+         }
+         HypreParMatrix * A_r = HypreParMatrixFromBlocks(Ab_r);
+         HypreParMatrix * A_i = HypreParMatrixFromBlocks(Ab_i);
+
+         ComplexHypreParMatrix Acomplex(A_r, A_i,true,true);
+
+         HypreParMatrix * A = Acomplex.GetSystemMatrix();
+
+         MUMPSSolver mumps(MPI_COMM_WORLD);
+         mumps.SetPrintLevel(0);
+         mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
+         mumps.SetOperator(*A);
+         mumps.Mult(B,X);
+         delete A;
+      }
+#else
+      if (mumps_solver)
+      {
+         MFEM_WARNING("MFEM compiled without mumps. Switching to an iterative solver");
+      }
+      mumps_solver = false;
+#endif
+      if (!mumps_solver)
+      {
+         BlockDiagonalPreconditioner M(tdof_offsets);
+
+         if (!static_cond)
+         {
+            HypreBoomerAMG * solver_E = new HypreBoomerAMG((HypreParMatrix &)
+                                                           BlockA_r->GetBlock(0,0));
+            solver_E->SetPrintLevel(0);
+            solver_E->SetSystemsOptions(dim);
+            HypreBoomerAMG * solver_H = new HypreBoomerAMG((HypreParMatrix &)
+                                                           BlockA_r->GetBlock(1,1));
+            solver_H->SetPrintLevel(0);
+            solver_H->SetSystemsOptions(dim);
+            M.SetDiagonalBlock(0,solver_E);
+            M.SetDiagonalBlock(1,solver_H);
+            M.SetDiagonalBlock(num_blocks,solver_E);
+            M.SetDiagonalBlock(num_blocks+1,solver_H);
+         }
+         HypreAMS * solver_hatE = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(skip,
+                                                                                    skip), hatE_fes);
+         HypreAMS * solver_hatH = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(
+                                                  skip+1,skip+1), hatH_fes);
+         solver_hatE->SetPrintLevel(0);
+         solver_hatH->SetPrintLevel(0);
+
+         M.SetDiagonalBlock(skip,solver_hatE);
+         M.SetDiagonalBlock(skip+1,solver_hatH);
+         M.SetDiagonalBlock(skip+num_blocks,solver_hatE);
+         M.SetDiagonalBlock(skip+num_blocks+1,solver_hatH);
+
+         if (myid == 0)
+         {
+            std::cout << "PCG iterations" << endl;
+         }
+         CGSolver cg(MPI_COMM_WORLD);
+         // GMRESSolver cg(MPI_COMM_WORLD);
+         cg.SetRelTol(1e-6);
+         cg.SetMaxIter(500);
+         cg.SetPrintLevel(1);
+         cg.SetPreconditioner(M);
+         cg.SetOperator(blockA);
+         cg.Mult(B, X);
+
+         for (int i = 0; i<num_blocks; i++)
+         {
+            delete &M.GetDiagonalBlock(i);
+         }
+
+         int num_iter = cg.GetNumIterations();
+      }
+      a->RecoverFEMSolution(X,x);
+
+      E.real().MakeRef(E_fes,x.GetData());
+      E.imag().MakeRef(E_fes,&x.GetData()[offsets.Last()]);
+
+      H.real().MakeRef(H_fes,&x.GetData()[offsets[1]]);
+      H.imag().MakeRef(H_fes,&x.GetData()[offsets.Last()+offsets[1]]);
+
+      int dofs = 0;
+      for (int i = 0; i<trial_fes.Size(); i++)
+      {
+         dofs += trial_fes[i]->GlobalTrueVSize();
+      }
+
+      if (visualization)
+      {
+         const char * keys = (it == 0 && dim == 2) ? "jRcml\n" : nullptr;
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         common::VisualizeField(E_out_r,vishost, visport, E.real(),
+                                "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+         common::VisualizeField(H_out_r,vishost, visport, H.real(),
+                                "Numerical Magnetic field (real part)", 501, 0, 500, 500, keys);
+      }
+
+      if (paraview)
+      {
+         paraview_dc->SetCycle(it);
+         paraview_dc->SetTime((double)it);
+         paraview_dc->Save();
+      }
+
+      if (it == pr)
+      {
+         break;
+      }
+
+      pmesh.UniformRefinement();
+
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   if (paraview)
+   {
+      delete paraview_dc;
+   }
+
+   delete a;
+   delete F_fec;
+   delete G_fec;
+   delete hatH_fes;
+   delete hatH_fec;
+   delete hatE_fes;
+   delete hatE_fec;
+   delete H_fec;
+   delete E_fec;
+   delete H_fes;
+   delete E_fes;
+
+   return 0;
+}
+

miniapps/dpg/plasma/other/pmaxwell-uw-lh.cpp

@@ -0,0 +1,833 @@
+//                   MFEM Ultraweak DPG Maxwell parallel example
+//
+// Description:
+// This example code demonstrates the use of MFEM to define and solve
+// the "ultraweak" (UW) DPG formulation for the Maxwell problem
+
+//      ∇×(1/μ ∇×E) - ω² ϵ E = Ĵ ,   in Ω
+//                       E×n = E₀ , on ∂Ω
+
+// The DPG UW deals with the First Order System
+//  i ω μ H + ∇ × E = 0,   in Ω
+// -i ω ϵ E + ∇ × H = 0,   in Ω
+//            E × n = E_0, on ∂Ω
+
+// The ultraweak-DPG formulation is obtained by integration by parts of both
+// equations and the introduction of trace unknowns on the mesh skeleton
+
+// in 2D
+// E is vector valued and H is scalar.
+//    (∇ × E, F) = (E, ∇ × F) + < n × E , F>
+// or (∇ ⋅ AE , F) = (AE, ∇ F) + < AE ⋅ n, F>
+// where A = [0 1; -1 0];
+
+// E ∈ (L²(Ω))² , H ∈ L²(Ω)
+// Ê ∈ H^-1/2(Γₕ), Ĥ ∈ H^1/2(Γₕ)
+//  i ω μ (H,F) + (E, ∇ × F) + < AÊ, F > = 0,      ∀ F ∈ H¹
+// -i ω ϵ (E,G) + (H,∇ × G)  + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)
+//                                        Ê = E₀      on ∂Ω
+// -------------------------------------------------------------------------
+// |   |       E      |      H      |      Ê       |       Ĥ      |  RHS    |
+// -------------------------------------------------------------------------
+// | F |  (E,∇ × F)   | i ω μ (H,F) |   < Ê, F >   |              |         |
+// |   |              |             |              |              |         |
+// | G | -i ω ϵ (E,G) |  (H,∇ × G)  |              | < Ĥ, G × n > |  (J,G)  |
+// where (F,G) ∈  H¹ × H(curl,Ω)
+
+#include "mfem.hpp"
+#include "../../util/pcomplexweakform.hpp"
+#include "../../util/utils.hpp"
+#include "../../util/maxwell_utils.hpp"
+#include "../../../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+#include <ctime>
+#include <string>
+#include <sstream>
+#include <cstring>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   int num_procs = Mpi::WorldSize();
+   Hypre::Init();
+
+   // fine mesh (trianles)
+   // default mesh
+   const char *mesh_file = "data/mesh-tri34K.mesh";
+   // coarse mesh (triangles)
+   // const char *mesh_file = "data/mesh-tri11K.mesh";
+   // coarse mesh (quadrilaterals)
+   // const char *mesh_file = "data/mesh-quad5K.mesh";
+
+   // epsilon tensor
+   const char * eps_r_file = nullptr;
+   const char * eps_i_file = nullptr;
+
+   int order = 2;
+   int delta_order = 1;
+   int par_ref_levels = 0;
+   int amr_ref_levels = 0;
+   // real_t rnum=4.6e9;
+   // real_t mu = 1.257e-6/factor;
+   // real_t epsilon_scale = 8.8541878128e-12*factor;
+   real_t rnum=4.6;
+   real_t mu = 1.257;
+   real_t epsilon_scale = 8.8541878128;
+
+   //  ∇×(1/μ ∇×E) - ω² ϵ E = Ĵ ,   in Ω
+   //  1/1.257e-6 - 2π*2π*4.6*4.6e18*8.8541878128e-12
+   // 1/1.257e-6 - 2π*2π*4.6*4.6*8.8541878128e6
+   // 1e6/1.257 - 2π*2π*4.6*4.6*8.8541878128e6
+   // 1/1.257 - 2π*2π*4.6*4.6*8.8541878128
+
+   bool visualization = false;
+   bool static_cond = false;
+   bool graph_norm = true;
+   bool mumps_solver = false;
+   real_t theta = 0.0;
+   bool paraview = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement-levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&amr_ref_levels, "-amr", "--parallel-amr-refinement-levels",
+                  "Number of parallel AMR refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&graph_norm, "-graph", "--graph-norm", "-no-gn",
+                  "--no-graph-norm", "Enable adjoint graph norm.");
+#ifdef MFEM_USE_MUMPS
+   args.AddOption(&mumps_solver, "-mumps", "--mumps-solver", "-no-mumps",
+                  "--no-mumps-solver", "Use the MUMPS Solver.");
+#endif
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   if (strcmp(mesh_file, "data/mesh-tri34K.mesh") == 0)
+   {
+      eps_r_file = "data/eps-tri34K_r.gf";
+      eps_i_file = "data/eps-tri34K_i.gf";
+   }
+   else if (strcmp(mesh_file, "data/mesh-tri11K.mesh") == 0)
+   {
+      eps_r_file = "data/eps-tri11K_r.gf";
+      eps_i_file = "data/eps-tri11K_i.gf";
+   }
+   else if (strcmp(mesh_file, "data/mesh-quad5K.mesh") == 0)
+   {
+      eps_r_file = "data/eps-quad5K_r.gf";
+      eps_i_file = "data/eps-quad5K_i.gf";
+   }
+   else
+   {
+      MFEM_ABORT("Unknown mesh file: " + string(mesh_file));
+   }
+
+   real_t omega = 2.*M_PI*rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   Array<int> int_bdr_attr;
+   for (int i = 0; i < mesh.GetNBE(); i++)
+   {
+      if (mesh.FaceIsInterior(mesh.GetBdrElementFaceIndex(i)))
+      {
+         int_bdr_attr.Append(mesh.GetBdrAttribute(i));
+      }
+   }
+
+   // mesh.RemoveInternalBoundaries();
+   mesh.EnsureNCMesh(true);
+   int * partitioning = mesh.GeneratePartitioning(num_procs);
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh, partitioning);
+
+   EpsilonMatrixCoefficient eps_r_cf(eps_r_file,&mesh,&pmesh, epsilon_scale);
+   EpsilonMatrixCoefficient eps_i_cf(eps_i_file,&mesh,&pmesh, epsilon_scale);
+   mesh.Clear();
+
+   for (int i = 0; i<par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+      eps_r_cf.Update();
+      eps_i_cf.Update();
+   }
+
+   // eps_r_cf.VisualizeMatrixCoefficient();
+   // eps_i_cf.VisualizeMatrixCoefficient();
+
+   // return 0;
+
+   // Matrix Coefficient (M = -i\omega \epsilon);
+   // M = -i * omega * (eps_r + i eps_i)
+   //  = omega eps_i + i (-omega eps_r)
+   ScalarMatrixProductCoefficient Mr_cf(omega,eps_i_cf);
+   ScalarMatrixProductCoefficient Mi_cf(-omega,eps_r_cf);
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient muomeg(mu*omega);
+   ConstantCoefficient mu2omeg2(mu*mu*omega*omega);
+   ConstantCoefficient negmuomeg(-mu*omega);
+   DenseMatrix rot_mat(2);
+   rot_mat(0,0) = 0.; rot_mat(0,1) = 1.;
+   rot_mat(1,0) = -1.; rot_mat(1,1) = 0.;
+   MatrixConstantCoefficient rot(rot_mat);
+   MatrixProductCoefficient Mrot_r(Mr_cf,rot);
+   MatrixProductCoefficient Mrot_i(Mi_cf,rot);
+   ScalarMatrixProductCoefficient negMrot_i(-1.0,Mrot_i);
+
+   TransposeMatrixCoefficient Mrt_cf(Mr_cf);
+   TransposeMatrixCoefficient Mit_cf(Mi_cf);
+   MatrixProductCoefficient MrMrt_cf(Mr_cf,Mrt_cf);
+   MatrixProductCoefficient MiMit_cf(Mi_cf,Mit_cf);
+   MatrixProductCoefficient MiMrt_cf(Mi_cf,Mrt_cf);
+   MatrixProductCoefficient MrMit_cf(Mr_cf,Mit_cf);
+
+   MatrixSumCoefficient MMr_cf(MrMrt_cf,MiMit_cf);
+   MatrixSumCoefficient MMi_cf(MiMrt_cf,MrMit_cf,1.0,-1.0);
+
+   mesh.Clear();
+
+   // Define spaces
+   enum TrialSpace
+   {
+      E_space     = 0,
+      H_space     = 1,
+      hatE_space  = 2,
+      hatH_space  = 3
+   };
+   enum TestSpace
+   {
+      F_space = 0,
+      G_space = 1
+   };
+
+   int dimc = (dim == 3) ? 3 : 1;
+   int test_order = order+delta_order;
+
+   // Vector L2 L2 space for E
+   FiniteElementCollection *E_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh,E_fec,dim);
+
+   // Vector L2 space for H
+   FiniteElementCollection *H_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *H_fes = new ParFiniteElementSpace(&pmesh,H_fec, dimc);
+
+   // H^-1/2 (curl) space for Ê
+   FiniteElementCollection * hatE_fec = new RT_Trace_FECollection(order-1,dim);
+   FiniteElementCollection * hatH_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementCollection * F_fec = new H1_FECollection(test_order, dim);
+
+   ParFiniteElementSpace *hatE_fes = new ParFiniteElementSpace(&pmesh,hatE_fec);
+   ParFiniteElementSpace *hatH_fes = new ParFiniteElementSpace(&pmesh,hatH_fec);
+   FiniteElementCollection * G_fec = new ND_FECollection(test_order, dim);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+   trial_fes.Append(E_fes);
+   trial_fes.Append(H_fes);
+   trial_fes.Append(hatE_fes);
+   trial_fes.Append(hatH_fes);
+   test_fec.Append(F_fec);
+   test_fec.Append(G_fec);
+
+   int gdofs = 0;
+   for (int i = 0; i<trial_fes.Size(); i++)
+   {
+      gdofs += trial_fes[i]->GlobalTrueVSize();
+   }
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Global number of dofs = " << gdofs << endl;
+   }
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_fes,test_fec);
+   a->StoreMatrices(); // needed for AMR
+
+   const IntegrationRule *irs[Geometry::NumGeom];
+   int order_quad = 2*order + 2;
+   for (int i = 0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+   const IntegrationRule &ir = IntRules.Get(pmesh.GetElementGeometry(0),
+                                            2*test_order + 2);
+
+   // (E,∇ × F)
+   MixedCurlIntegrator * curl_integ = new MixedCurlIntegrator(one);
+   curl_integ->SetIntRule(&ir);
+   a->AddTrialIntegrator(new TransposeIntegrator(curl_integ), nullptr,
+                         TrialSpace::E_space,
+                         TestSpace::F_space);
+
+   //  (M E , G) = (M_r E, G) + i (M_i E, G) = (E, M_rt G) + i (E, Mit G)
+   //            = (M_rt G, E)^T + i (Mit G, E)^T
+   VectorFEMassIntegrator * Mrt_cf_integ = new VectorFEMassIntegrator(Mrt_cf);
+   VectorFEMassIntegrator * Mit_cf_integ = new VectorFEMassIntegrator(Mit_cf);
+   Mrt_cf_integ->SetIntegrationRule(ir);
+   Mit_cf_integ->SetIntegrationRule(ir);
+   a->AddTrialIntegrator(
+      new TransposeIntegrator(Mrt_cf_integ),
+      new TransposeIntegrator(Mit_cf_integ),
+      TrialSpace::E_space, TestSpace::G_space);
+
+   //  (H,∇ × G)
+   MixedCurlIntegrator * curl_integ_H = new MixedCurlIntegrator(one);
+   curl_integ_H->SetIntRule(&ir);
+   a->AddTrialIntegrator(new TransposeIntegrator(curl_integ_H), nullptr,
+                         TrialSpace::H_space, TestSpace::G_space);
+
+   // < n×Ĥ ,G>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,
+                         TrialSpace::hatH_space, TestSpace::G_space);
+
+   // i ω μ (H, F)
+   MixedScalarMassIntegrator * muomeg_integ = new MixedScalarMassIntegrator(
+      muomeg);
+   muomeg_integ->SetIntRule(&ir);
+   a->AddTrialIntegrator(nullptr,muomeg_integ,
+                         TrialSpace::H_space, TestSpace::F_space);
+
+   // < n×Ê,F>
+   a->AddTrialIntegrator(new TraceIntegrator,nullptr,
+                         TrialSpace::hatE_space, TestSpace::F_space);
+
+   // test integrators
+   // (∇×G ,∇× δG)
+   CurlCurlIntegrator * curlcurl_integ = new CurlCurlIntegrator(one);
+   curlcurl_integ->SetIntRule(&ir);
+   a->AddTestIntegrator(curlcurl_integ,nullptr,
+                        TestSpace::G_space,TestSpace::G_space);
+   // (G,δG)
+   VectorFEMassIntegrator * vfemass_integ = new VectorFEMassIntegrator(one);
+   vfemass_integ->SetIntegrationRule(ir);
+   a->AddTestIntegrator(vfemass_integ,nullptr,
+                        TestSpace::G_space,TestSpace::G_space);
+   // (∇F,∇δF)
+   DiffusionIntegrator * diff_integ = new DiffusionIntegrator(one);
+   diff_integ->SetIntRule(&ir);
+   a->AddTestIntegrator(diff_integ,nullptr,
+                        TestSpace::F_space, TestSpace::F_space);
+   // (F,δF)
+   MassIntegrator * mass_integ = new MassIntegrator(one);
+   mass_integ->SetIntRule(&ir);
+   a->AddTestIntegrator(mass_integ,nullptr,
+                        TestSpace::F_space, TestSpace::F_space);
+
+   if (graph_norm)
+   {
+      // μ^2 ω^2 (F,δF)
+      MassIntegrator * mu2omeg2_integ = new MassIntegrator(mu2omeg2);
+      mu2omeg2_integ->SetIntRule(&ir);
+      a->AddTestIntegrator(mu2omeg2_integ,nullptr,
+                           TestSpace::F_space, TestSpace::F_space);
+
+      // -i ω μ (F,∇ × δG) = i (F, -ω μ ∇ × δ G)
+      MixedCurlIntegrator * negmuomeg_integ = new MixedCurlIntegrator(negmuomeg);
+      negmuomeg_integ->SetIntRule(&ir);
+      a->AddTestIntegrator(nullptr, new TransposeIntegrator(negmuomeg_integ),
+                           TestSpace::F_space, TestSpace::G_space);
+
+      // (M ∇ × F, δG) = (M_r ∇ × F, δG) + i (M_i ∇ × F, δG)
+      //               = (M_r A ∇ F, δG) + i (M_i A ∇ F, δG), A = [0 1; -1; 0]
+      MixedVectorGradientIntegrator * Mrot_r_integ = new
+      MixedVectorGradientIntegrator(Mrot_r);
+      MixedVectorGradientIntegrator * Mrot_i_integ = new
+      MixedVectorGradientIntegrator(Mrot_i);
+      Mrot_r_integ->SetIntRule(&ir);
+      Mrot_i_integ->SetIntRule(&ir);
+      a->AddTestIntegrator(Mrot_r_integ, Mrot_i_integ,
+                           TestSpace::F_space, TestSpace::G_space);
+
+      // i ω μ (∇ × G,δF) = i (ω μ ∇ × G, δF )
+      MixedCurlIntegrator * muomeg_integ = new MixedCurlIntegrator(muomeg);
+      muomeg_integ->SetIntRule(&ir);
+      a->AddTestIntegrator(nullptr,muomeg_integ,
+                           TestSpace::G_space, TestSpace::F_space);
+
+      // (M^* G, ∇ × δF ) = (G, Mr A ∇ δF) - i (G, Mi A ∇ δF)
+      MixedVectorGradientIntegrator * Mrot_r_integ2 = new
+      MixedVectorGradientIntegrator(Mrot_r);
+      MixedVectorGradientIntegrator * negMrot_i_integ = new
+      MixedVectorGradientIntegrator(negMrot_i);
+      Mrot_r_integ2->SetIntRule(&ir);
+      negMrot_i_integ->SetIntRule(&ir);
+      a->AddTestIntegrator(new TransposeIntegrator(Mrot_r_integ2),
+                           new TransposeIntegrator(negMrot_i_integ),
+                           TestSpace::G_space, TestSpace::F_space);
+
+      // M*M^*(G,δG) = (MrMr^t + MiMi^t) + i (MiMr^t - MrMi^t)
+      VectorFEMassIntegrator * MMr_integ = new VectorFEMassIntegrator(MMr_cf);
+      VectorFEMassIntegrator * MMi_integ = new VectorFEMassIntegrator(MMi_cf);
+      MMr_integ->SetIntegrationRule(ir);
+      MMi_integ->SetIntegrationRule(ir);
+      a->AddTestIntegrator(MMr_integ, MMi_integ,
+                           TestSpace::G_space,
+                           TestSpace::G_space);
+   }
+
+   socketstream E_out_r;
+   socketstream H_out_r;
+   socketstream E_theta_out_r;
+
+   ParComplexGridFunction E(E_fes);
+   ParComplexGridFunction H(H_fes);
+   E.real() = 0.0;
+   E.imag() = 0.0;
+   H.real() = 0.0;
+   H.imag() = 0.0;
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+
+   ParGridFunction E_theta_r(&L2_fes);
+   ParGridFunction E_theta_i(&L2_fes);
+   ParGridFunction E_theta(&L2_fes);
+   E_theta = 0.0;
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+   ParaViewDataCollection * paraview_tdc = nullptr;
+
+   // Create ParaView directory and file
+   std::string output_dir = "ParaView/UW/2D" + GetTimestamp();
+   if (Mpi::Root())
+   {
+      WriteParametersToFile(args, output_dir);
+   }
+
+   if (paraview)
+   {
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",&E.real());
+      paraview_dc->RegisterField("E_i",&E.imag());
+      paraview_dc->RegisterField("H_r",&H.real());
+      paraview_dc->RegisterField("H_i",&H.imag());
+      paraview_dc->RegisterField("E_theta_r",&E_theta_r);
+      paraview_dc->RegisterField("E_theta_i",&E_theta_i);
+
+      std::ostringstream paraview_file_name_th;
+      paraview_file_name_th << filename
+                            << "_par_ref_" << par_ref_levels
+                            << "_order_" << order
+                            << "th";
+      paraview_tdc = new ParaViewDataCollection(paraview_file_name_th.str(), &pmesh);
+      paraview_tdc->SetPrefixPath(output_dir);
+      paraview_tdc->SetLevelsOfDetail(order);
+      paraview_tdc->SetCycle(0);
+      paraview_tdc->SetDataFormat(VTKFormat::BINARY);
+      paraview_tdc->SetHighOrderOutput(true);
+      paraview_tdc->SetTime(0.0); // set the time
+      paraview_tdc->RegisterField("E_theta_t",&E_theta);
+   }
+
+   real_t res0 = 0.;
+   real_t err0 = 0.;
+   int dof0 = 0; // init to suppress gcc warning
+
+   Array<int> elements_to_refine;
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+   for (int it = 0; it<=amr_ref_levels; it++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      Array<int> one_r_bdr;
+      Array<int> one_i_bdr;
+      Array<int> negone_r_bdr;
+      Array<int> negone_i_bdr;
+
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+         one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+         negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+         negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+
+         // remove internal boundaries
+         for (int i = 0; i<int_bdr_attr.Size(); i++)
+         {
+            ess_bdr[int_bdr_attr[i]-1] = 0;
+         }
+
+         hatE_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+         one_r_bdr = 0;  one_i_bdr = 0;
+         negone_r_bdr = 0;  negone_i_bdr = 0;
+
+         // attr = 30,2 (real)
+         one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+         // attr = 26,6 (imag)
+         one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+         // attr = 22,10 (real)
+         negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+         // attr = 18,14 (imag)
+         negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+      }
+
+      // Set up bdr conditions
+      // shift the ess_tdofs
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += E_fes->GetTrueVSize() + H_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = E_fes->GetVSize();
+      offsets[2] = H_fes->GetVSize();
+      offsets[3] = hatE_fes->GetVSize();
+      offsets[4] = hatH_fes->GetVSize();
+      offsets.PartialSum();
+
+      Vector x(2*offsets.Last());
+      x = 0.;
+      real_t * xdata = x.GetData();
+
+      ParComplexGridFunction hatE_gf(hatE_fes);
+      hatE_gf.real().MakeRef(hatE_fes,&xdata[offsets[2]]);
+      hatE_gf.imag().MakeRef(hatE_fes,&xdata[offsets.Last()+ offsets[2]]);
+
+
+      Vector zero(dim); zero = 0.0;
+      VectorConstantCoefficient zero_cf(zero);
+
+      // rotate the vector
+      // (x,y) -> (y,-x)
+      Vector rot_one_x(dim); rot_one_x = 0.0; rot_one_x(1) = -1.0;
+      Vector rot_negone_x(dim); rot_negone_x = 0.0; rot_negone_x(1) = 1.0;
+      VectorConstantCoefficient rot_one_x_cf(rot_one_x);
+      VectorConstantCoefficient rot_negone_x_cf(rot_negone_x);
+
+      hatE_gf.ProjectBdrCoefficientNormal(rot_one_x_cf,zero_cf, one_r_bdr);
+      hatE_gf.ProjectBdrCoefficientNormal(rot_negone_x_cf,zero_cf, negone_r_bdr);
+      hatE_gf.ProjectBdrCoefficientNormal(zero_cf,rot_one_x_cf, one_i_bdr);
+      hatE_gf.ProjectBdrCoefficientNormal(zero_cf,rot_negone_x_cf, negone_i_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+      int num_blocks = BlockA_r->NumRowBlocks();
+      Array<int> tdof_offsets(2*num_blocks+1);
+
+      tdof_offsets[0] = 0;
+      int skip = (static_cond) ? 0 : 2;
+      int k = (static_cond) ? 2 : 0;
+      for (int i=0; i<num_blocks; i++)
+      {
+         tdof_offsets[i+1] = trial_fes[i+k]->GetTrueVSize();
+         tdof_offsets[num_blocks+i+1] = trial_fes[i+k]->GetTrueVSize();
+      }
+      tdof_offsets.PartialSum();
+
+      BlockOperator blockA(tdof_offsets);
+      for (int i = 0; i<num_blocks; i++)
+      {
+         for (int j = 0; j<num_blocks; j++)
+         {
+            blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
+            blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
+         }
+      }
+
+      X = 0.;
+
+#ifdef MFEM_USE_MUMPS
+      if (mumps_solver)
+      {
+         // Monolithic real part
+         Array2D<const HypreParMatrix * > Ab_r(num_blocks,num_blocks);
+         // Monolithic imag part
+         Array2D<const HypreParMatrix * > Ab_i(num_blocks,num_blocks);
+         for (int i = 0; i<num_blocks; i++)
+         {
+            for (int j = 0; j<num_blocks; j++)
+            {
+               Ab_r(i,j) = &(HypreParMatrix &)BlockA_r->GetBlock(i,j);
+               Ab_i(i,j) = &(HypreParMatrix &)BlockA_i->GetBlock(i,j);
+            }
+         }
+         HypreParMatrix * A_r = HypreParMatrixFromBlocks(Ab_r);
+         HypreParMatrix * A_i = HypreParMatrixFromBlocks(Ab_i);
+
+         ComplexHypreParMatrix Acomplex(A_r, A_i,true,true);
+
+         HypreParMatrix * A = Acomplex.GetSystemMatrix();
+
+         MUMPSSolver mumps(MPI_COMM_WORLD);
+         mumps.SetPrintLevel(0);
+         mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
+         mumps.SetOperator(*A);
+         mumps.Mult(B,X);
+         delete A;
+      }
+#else
+      if (mumps_solver)
+      {
+         MFEM_WARNING("MFEM compiled without mumps. Switching to an iterative solver");
+      }
+      mumps_solver = false;
+#endif
+      int num_iter = -1;
+      if (!mumps_solver)
+      {
+         BlockDiagonalPreconditioner M(tdof_offsets);
+         // BlockTriangularSymmetricPreconditioner M(tdof_offsets);
+         // M.SetOperator(blockA);
+         // int nblocks = blockA.NumRowBlocks();
+         // for (int i = 0; i<nblocks; i++)
+         // {
+         //    for (int j = 0; j<nblocks; j++)
+         //    {
+         //       if (i != j)
+         //       {
+         //          M.SetBlock(i,j,&blockA.GetBlock(i,j));
+         //       }
+         //    }
+         // }
+
+
+         if (!static_cond)
+         {
+            HypreBoomerAMG * solver_E = new HypreBoomerAMG((HypreParMatrix &)
+                                                           BlockA_r->GetBlock(0,0));
+            solver_E->SetPrintLevel(0);
+            solver_E->SetSystemsOptions(dim);
+            HypreBoomerAMG * solver_H = new HypreBoomerAMG((HypreParMatrix &)
+                                                           BlockA_r->GetBlock(1,1));
+            solver_H->SetPrintLevel(0);
+            // solver_H->SetSystemsOptions(dim);
+            M.SetDiagonalBlock(0,solver_E);
+            M.SetDiagonalBlock(1,solver_H);
+            M.SetDiagonalBlock(num_blocks,solver_E);
+            M.SetDiagonalBlock(num_blocks+1,solver_H);
+         }
+         HypreAMS * solver_hatE = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(skip,
+                                                                                    skip), hatE_fes);
+         HypreBoomerAMG * solver_hatH = new HypreBoomerAMG((HypreParMatrix &)
+                                                           BlockA_r->GetBlock(
+                                                              skip+1,skip+1));
+         solver_hatE->SetPrintLevel(0);
+         solver_hatH->SetPrintLevel(0);
+         solver_hatH->SetRelaxType(88);
+
+         M.SetDiagonalBlock(skip,solver_hatE);
+         M.SetDiagonalBlock(skip+1,solver_hatH);
+         M.SetDiagonalBlock(skip+num_blocks,solver_hatE);
+         M.SetDiagonalBlock(skip+num_blocks+1,solver_hatH);
+
+         if (myid == 0)
+         {
+            std::cout << "PCG iterations" << endl;
+         }
+         CGSolver cg(MPI_COMM_WORLD);
+         cg.SetRelTol(1e-7);
+         cg.SetMaxIter(1500);
+         cg.SetPrintLevel(3);
+         cg.SetPreconditioner(M);
+         cg.SetOperator(blockA);
+         cg.Mult(B, X);
+
+         for (int i = 0; i<num_blocks; i++)
+         {
+            delete &M.GetDiagonalBlock(i);
+         }
+
+         num_iter = cg.GetNumIterations();
+      }
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+      real_t residual = residuals.Norml2();
+      real_t maxresidual = residuals.Max();
+      real_t globalresidual = residual * residual;
+      MPI_Allreduce(MPI_IN_PLACE, &maxresidual, 1, MPITypeMap<real_t>::mpi_type,
+                    MPI_MAX, MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE, &globalresidual, 1,
+                    MPITypeMap<real_t>::mpi_type, MPI_SUM, MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+      E.real().MakeRef(E_fes,x.GetData());
+      E.imag().MakeRef(E_fes,&x.GetData()[offsets.Last()]);
+
+      H.real().MakeRef(H_fes,&x.GetData()[offsets[1]]);
+      H.imag().MakeRef(H_fes,&x.GetData()[offsets.Last()+offsets[1]]);
+
+      int dofs = 0;
+      for (int i = 0; i<trial_fes.Size(); i++)
+      {
+         dofs += trial_fes[i]->GlobalTrueVSize();
+      }
+
+      real_t rate_res = (it) ? dim*log(res0/globalresidual)/log((
+                                                                   real_t)dof0/dofs) : 0.0;
+
+      res0 = globalresidual;
+      dof0 = dofs;
+
+      if (myid == 0)
+      {
+         std::ios oldState(nullptr);
+         oldState.copyfmt(std::cout);
+         std::cout << std::right << std::setw(5) << it << " | "
+                   << std::setw(10) <<  dof0 << " | "
+                   << std::setprecision(1) << std::fixed
+                   << std::setw(4) <<  2.0*rnum << " π  | "
+                   << std::setprecision(3);
+         std::cout << std::setprecision(3)
+                   << std::setw(10) << std::scientific <<  res0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_res << " | "
+                   << std::setw(6) << std::fixed << num_iter << " | "
+                   << std::endl;
+         std::cout.copyfmt(oldState);
+      }
+
+      AzimuthalECoefficient az_e_r(&E.real());
+      AzimuthalECoefficient az_e_i(&E.imag());
+
+      E_theta_r.ProjectCoefficient(az_e_r);
+      E_theta_i.ProjectCoefficient(az_e_i);
+
+      if (visualization)
+      {
+         const char * keys = nullptr;
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         common::VisualizeField(E_out_r,vishost, visport, E.real(),
+                                "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+         common::VisualizeField(H_out_r,vishost, visport, H.real(),
+                                "Numerical Magnetic field (real part)", 501, 0, 500, 500, keys);
+         common::VisualizeField(E_theta_out_r,vishost, visport, E_theta_r,
+                                "Numerical Electric field (Azimuthal-real)", 501, 0, 500, 500, keys);
+      }
+
+      if (paraview)
+      {
+         paraview_dc->SetCycle(it);
+         paraview_dc->SetTime((real_t)it);
+         paraview_dc->Save();
+      }
+      if (it == amr_ref_levels)
+      {
+         int num_frames = 32;
+         for (int i = 0; i<num_frames; i++)
+         {
+            real_t t = (real_t)(i % num_frames) / num_frames;
+            add(cos(real_t(2.0*M_PI)*t), E_theta_r,
+                sin(real_t(2.0*M_PI)*t), E_theta_i, E_theta);
+            paraview_tdc->SetCycle(i);
+            paraview_tdc->SetTime(t);
+            paraview_tdc->Save();
+         }
+         delete paraview_tdc;
+         break;
+      }
+
+      if (theta > 0.0)
+      {
+         elements_to_refine.SetSize(0);
+         for (int iel = 0; iel<pmesh.GetNE(); iel++)
+         {
+            if (residuals[iel] > theta * maxresidual)
+            {
+               elements_to_refine.Append(iel);
+            }
+         }
+         pmesh.GeneralRefinement(elements_to_refine,1,1);
+      }
+      else
+      {
+         pmesh.UniformRefinement();
+      }
+
+      eps_r_cf.Update();
+      eps_i_cf.Update();
+
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      L2_fes.Update();
+      E_theta_r.Update();
+      E_theta_i.Update();
+      E_theta.Update();
+
+      a->Update();
+
+
+   }
+
+   delete a;
+   delete F_fec;
+   delete G_fec;
+   delete hatH_fes;
+   delete hatH_fec;
+   delete hatE_fes;
+   delete hatE_fec;
+   delete H_fec;
+   delete E_fec;
+   delete H_fes;
+   delete E_fes;
+
+   return 0;
+
+}

miniapps/dpg/plasma/other/pmaxwell-uw-tokamak.cpp

@@ -0,0 +1,676 @@
+
+// srun -n 512 ./pmaxwell-uw-tokamak -o 4 -sc -prob 2 -paraview
+// srun -n 512 ./pmaxwell-uw-tokamak -o 4 -sc -prob 4 -paraview
+
+// Description:
+// This example code demonstrates the use of MFEM to define and solve
+// the "ultraweak" (UW) DPG formulation for the Maxwell problem
+
+//      ∇×(1/μ ∇×E) - (ω^2 ϵ + i ω σ) E = Ĵ ,   in Ω
+//                E×n = E_0, on ∂Ω
+
+// The DPG UW deals with the First Order System
+//        i ω μ H + ∇ × E = 0,   in Ω (Faraday's law)
+//            M E + ∇ × H = J,   in Ω (Ampere's law)
+//            E × n = E_0, on ∂Ω
+// Note: Ĵ = -iωJ
+// where M = -(i ω ϵI + σI)
+
+// The ultraweak-DPG formulation is obtained by integration by parts of both
+// equations and the introduction of trace unknowns on the mesh skeleton
+
+// in 3D
+// E,H ∈ (L^2(Ω))^3
+// Ê ∈ H_0^1/2(Ω)(curl, Γ_h), Ĥ ∈ H^-1/2(curl, Γ_h)
+//  i ω μ (H,F) + (E,∇ × F) + < Ê, F × n > = 0,      ∀ F ∈ H(curl,Ω)
+//      M (E,G) + (H,∇ × G) + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)
+//                                   Ê × n = E_0     on ∂Ω
+// -------------------------------------------------------------------------
+// |   |       E      |      H      |      Ê       |       Ĥ      |  RHS    |
+// -------------------------------------------------------------------------
+// | F |  (E,∇ × F)   | i ω μ (H,F) | < n × Ê, F > |              |         |
+// |   |              |             |              |              |         |
+// | G |    (ME,G)    |  (H,∇ × G)  |              | < n × Ĥ, G > |  (J,G)  |
+// where (F,G) ∈  H(curl,Ω) × H(curl,Ω)
+
+// Here we use the "Adjoint Graph" norm on the test space i.e.,
+// ||(F,G)||^2_V = ||A^*(F,G)||^2 + ||(F,G)||^2 where A is the
+// maxwell operator defined by (1)
+
+
+#include "mfem.hpp"
+#include "../../util/pcomplexweakform.hpp"
+#include "../../util/preconditioners.hpp"
+#include "../../../common/mfem-common.hpp"
+#include "../../util/maxwell_utils.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = false;
+   real_t rnum=5.0;
+   bool static_cond = false;
+   int pr = 0;
+   bool paraview = false;
+   real_t mu = 1.0;
+   real_t epsilon = 1.0;
+   real_t sigma = 0.1;
+   bool graph_norm = true;
+   int prob = 0;
+   bool pmg = false;
+   int pmg_levels = -1;
+   real_t relax_factor = 2.0/3;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&sigma, "-sigma", "--sigma",
+                  "conductivity");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&pr, "-pref", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&prob, "-prob", "--problem",
+                  "Choice of problem: 0: tet mesh eps = 1, 1: hex mesh eps = 1 ,  2: tet-mesh eps from file,", 
+                  "3: hex-mesh eps from file, 4: 400k-tet-mesh eps from file");
+   args.AddOption(&pmg, "-pmg", "--p-refinement-multigrid", "-no-pmg",
+                  "--no-p-refinement-multigrid", "Enable P-Refinement Multigrid.");
+   args.AddOption(&pmg_levels, "-pmgl","--p-refinement-multigrid-levels",
+                  "Number of levels for P-Refinement Multigrid.");
+   args.AddOption(&relax_factor, "-rf", "--relaxation-factor",
+                  "Relaxation factor for the p-multigrid smoother.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&graph_norm, "-graph", "--graph-norm", "-no-gn",
+                  "--no-graph-norm", "Enable adjoint graph norm.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+
+   const char *mesh_file = nullptr;
+   if (prob == 0 || prob == 2)
+   {
+      mesh_file = "meshes/tokamak-tet.mesh";
+      if (prob == 2)
+      {
+         rnum=50.0e6;
+         mu = 1.257e-6;
+         sigma = 0.0;
+      }
+   }
+   else if (prob == 1 || prob == 3)
+   {
+      mesh_file = "meshes/tokamak-hex.mesh";
+      if (prob == 3)
+      {
+         rnum=50.0e6;
+         mu = 1.257e-6;
+         sigma = 0.0;
+      }
+   }
+   else if (prob == 4)
+   {
+      mesh_file = "meshes/tokamak-400K-tet-new.mesh";
+      rnum = 54.0e6;
+      mu = 1.257e-6;
+      sigma = 0.0;
+   }
+   else
+   {
+      MFEM_ABORT("No valid problem choice given");
+   }
+
+
+   real_t omega = 2.*M_PI*rnum;
+   Mesh mesh(mesh_file, 1, 1);
+
+   mesh.RemoveInternalBoundaries();
+
+   int dim = mesh.Dimension();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+
+   DenseMatrix mat_eps_i(dim); mat_eps_i = 0.0;
+   DenseMatrix mat_eps_r(dim); mat_eps_r = 0.0;
+   mat_eps_r(0,0) = epsilon;
+   mat_eps_r(1,1) = epsilon;
+   mat_eps_r(2,2) = epsilon;
+
+   DenseMatrix I(dim); I = 0.0;
+   I(0,0) = 1.0;
+   I(1,1) = 1.0;
+   I(2,2) = 1.0;
+   MatrixConstantCoefficient identity_cf(I);
+
+
+   MatrixCoefficient * eps_r_cf = nullptr;
+   MatrixCoefficient * eps_i_cf = nullptr;
+   if (prob == 0 || prob == 1)
+   {
+      eps_r_cf = new MatrixConstantCoefficient(mat_eps_r);
+      eps_i_cf = new MatrixConstantCoefficient(mat_eps_i);
+   }
+   else
+   {
+      const char * eps_r_file = nullptr;
+      const char * eps_i_file = nullptr;
+      if (prob == 2)
+      {
+         eps_r_file = "data/tet-eps_r.gf";
+         eps_i_file = "data/tet-eps_i.gf";
+      }
+      else if (prob == 3)
+      {
+         eps_r_file = "data/hex-eps_r.gf";
+         eps_i_file = "data/hex-eps_i.gf";
+      }
+      else if (prob == 4)
+      {
+         eps_r_file = "data/tet-400k-eps_r.gf";
+         eps_i_file = "data/tet-400k-eps_i.gf";
+      }
+      real_t epsilon_scale = 8.8541878128e-12;
+      eps_r_cf = new EpsilonMatrixCoefficient(eps_r_file,&mesh,&pmesh, epsilon_scale);
+      eps_i_cf = new EpsilonMatrixCoefficient(eps_i_file,&mesh,&pmesh, epsilon_scale);
+   }
+   mesh.Clear();
+   for (int i = 0; i<pr; i++)
+   {
+      pmesh.UniformRefinement();
+      dynamic_cast<EpsilonMatrixCoefficient *>(eps_r_cf)->Update();
+      dynamic_cast<EpsilonMatrixCoefficient *>(eps_i_cf)->Update();
+   }
+
+
+   auto cf_r = dynamic_cast<EpsilonMatrixCoefficient *>(eps_r_cf);
+   auto cf_i = dynamic_cast<EpsilonMatrixCoefficient *>(eps_i_cf);
+
+   std::ostringstream cf_r_file_name;
+   cf_r_file_name << "epsilor_r" << "_prob_" << prob  << "_order_" << order;
+   std::ostringstream cf_i_file_name;
+   cf_i_file_name << "epsilor_i" << "_prob_" << prob  << "_order_" << order;
+
+
+   VisualizeMatrixArrayCoefficient(*cf_r, &pmesh, order, paraview,
+       cf_r_file_name.str().c_str());
+
+   VisualizeMatrixArrayCoefficient(*cf_i, &pmesh, order, paraview,
+       cf_i_file_name.str().c_str());
+    
+
+   int ne = pmesh.GetNE();
+   MPI_Allreduce(MPI_IN_PLACE,&ne,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
+
+   if (myid == 0)
+   {
+      mfem::out << "Number of elements = " << ne << endl;
+   }
+
+   // Matrix Coefficient (M = -i\omega \epsilon - \sigma I);
+   // M = -i * omega * (eps_r + i eps_i) - sigmaI
+   //  = omega eps_i - sigma I + i (-omega eps_r)
+   MatrixSumCoefficient Mr_cf(*eps_i_cf,identity_cf,omega,-sigma);
+   ScalarMatrixProductCoefficient Mi_cf(-omega,*eps_r_cf);
+
+   // Define spaces
+   // L2 space for E
+   FiniteElementCollection *E_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh,E_fec,dim);
+
+   // Vector L2 space for H
+   FiniteElementCollection *H_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *H_fes = new ParFiniteElementSpace(&pmesh,H_fec, dim);
+
+   // H^-1/2 (curl) space for Ê
+   FiniteElementCollection * hatE_fec = nullptr;
+   FiniteElementCollection * hatH_fec = nullptr;
+   FiniteElementCollection * F_fec = nullptr;
+   int test_order = order+delta_order;
+   hatE_fec = new ND_Trace_FECollection(order,dim);
+   hatH_fec = new ND_Trace_FECollection(order,dim);
+   F_fec = new ND_FECollection(test_order, dim);
+
+   ParFiniteElementSpace *hatE_fes = new ParFiniteElementSpace(&pmesh,hatE_fec);
+   ParFiniteElementSpace *hatH_fes = new ParFiniteElementSpace(&pmesh,hatH_fec);
+   FiniteElementCollection * G_fec = new ND_FECollection(test_order, dim);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+   trial_fes.Append(E_fes);
+   trial_fes.Append(H_fes);
+   trial_fes.Append(hatE_fes);
+   trial_fes.Append(hatH_fes);
+   test_fec.Append(F_fec);
+   test_fec.Append(G_fec);
+
+   int gdofs = 0;
+   for (int i = 0; i<trial_fes.Size(); i++)
+   {
+      gdofs += trial_fes[i]->GlobalTrueVSize();
+   }
+
+   if (myid == 0)
+   {
+      mfem::out << "Global number of dofs = " << gdofs << endl;
+   }
+
+   // Bilinear form coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient eps2omeg2(epsilon*epsilon*omega*omega);
+   ConstantCoefficient mu2omeg2(mu*mu*omega*omega);
+   ConstantCoefficient muomeg(mu*omega);
+   ConstantCoefficient negepsomeg(-epsilon*omega);
+   ConstantCoefficient epsomeg(epsilon*omega);
+   ConstantCoefficient negmuomeg(-mu*omega);
+   ConstantCoefficient sigma_cf(sigma);
+   ConstantCoefficient negsigma_cf(-sigma);
+   ConstantCoefficient sigma2_cf(sigma*sigma);
+
+   Vector z_one(3); z_one = 0.0; z_one(2) = 1.0;
+   Vector zero(3); zero = 0.0;
+   Vector z_negone(3); z_negone = 0.0; z_negone(2) = -1.0;
+   VectorConstantCoefficient z_one_cf(z_one);
+   VectorConstantCoefficient z_negone_cf(z_negone);
+   VectorConstantCoefficient zero_cf(zero);
+
+   if (myid == 0)
+   {
+      std::cout << "Assembling matrix" << endl;
+   }
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_fes,test_fec);
+   const IntegrationRule &ir_test = IntRules.Get(pmesh.GetElementGeometry(0),
+                                           2*test_order + 4);
+   const IntegrationRule &ir_trial = IntRules.Get(pmesh.GetElementGeometry(0),
+                                           order+test_order + 4);
+
+   a->SetTestIntegrationRule(ir_test);
+   // a->SetTrialIntegrationRule(ir_trial);
+   // (E,∇ × F)
+   a->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one)),
+                         nullptr,0,0);
+
+
+   // --------------------------------------------------------------------------
+   //  (M E , G) = (M_r E, G) + i (M_i E, G)
+   a->AddTrialIntegrator(
+      new TransposeIntegrator(new VectorFEMassIntegrator(Mr_cf)),
+      new TransposeIntegrator(new VectorFEMassIntegrator(Mi_cf)),0,1);
+   // --------------------------------------------------------------------------
+
+   //  (H,∇ × G)
+   a->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one)),
+                         nullptr,1,1);
+   // < n×Ĥ ,G>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,3,1);
+   // test integrators
+   // (∇×G ,∇× δG)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,1,1);
+
+   ConstantCoefficient l2weight(1.0);
+
+   // (G,δG)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(l2weight),nullptr,1,1);
+
+   // i ω μ (H, F)
+   a->AddTrialIntegrator(nullptr,
+                         new TransposeIntegrator(new VectorFEMassIntegrator(muomeg)),1,0);
+   // < n×Ê,F>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,2,0);
+   // test integrators
+   // (∇×F,∇×δF)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,0,0);
+   // (F,δF)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,0,0);
+
+   if (graph_norm)
+   {
+      // μ^2 ω^2 (F,δF)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(mu2omeg2),nullptr,0,0);
+      // -i ω μ (F,∇ × δG) = i (F, -ω μ ∇ × δ G)
+      a->AddTestIntegrator(nullptr,new MixedVectorWeakCurlIntegrator(negmuomeg),0,1);
+
+      // --------------------------------------------------------------------------
+      // (M ∇ × F, δG) = (M_r  ∇ × F, δG) + i (M_i  ∇ × F, δG)
+      a->AddTestIntegrator(new MixedVectorCurlIntegrator(Mr_cf),
+                           new MixedVectorCurlIntegrator(Mi_cf),0,1);
+      // --------------------------------------------------------------------------
+
+      // i ω μ (∇ × G,δF)
+      a->AddTestIntegrator(nullptr,new MixedVectorCurlIntegrator(muomeg),1,0);
+   }
+   // --------------------------------------------------------------------------
+   // (M^* G, ∇ × δF ) = (Mr^T G,  ∇ × δF) - i (Mi^T G,  ∇ × δF)
+   TransposeMatrixCoefficient Mrt_cf(Mr_cf);
+   TransposeMatrixCoefficient Mit_cf(Mi_cf);
+   ScalarMatrixProductCoefficient negMit_cf(-1.0,Mit_cf);
+   if (graph_norm)
+   {
+      a->AddTestIntegrator(new MixedVectorWeakCurlIntegrator(Mrt_cf),
+                           new MixedVectorWeakCurlIntegrator(negMit_cf),1,0);
+   }
+   // --------------------------------------------------------------------------
+
+   // M*M^*(G,δG) = (MrMr^t + MiMi^t) + i (MiMr^t - MrMi^t)
+   MatrixProductCoefficient MrMrt_cf(Mr_cf,Mrt_cf);
+   MatrixProductCoefficient MiMit_cf(Mi_cf,Mit_cf);
+   MatrixProductCoefficient MiMrt_cf(Mi_cf,Mrt_cf);
+   MatrixProductCoefficient MrMit_cf(Mr_cf,Mit_cf);
+
+   MatrixSumCoefficient MMr_cf(MrMrt_cf,MiMit_cf);
+   MatrixSumCoefficient MMi_cf(MiMrt_cf,MrMit_cf,1.0,-1.0);
+
+   const IntegrationRule *irs[Geometry::NumGeom];
+   int order_quad = 2*order + 2;
+   for (int i = 0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+   const IntegrationRule &ir = IntRules.Get(pmesh.GetElementGeometry(0),
+                                            2*test_order + 2);
+
+   if (graph_norm)
+   {
+      a->AddTestIntegrator(new VectorFEMassIntegrator(MMr_cf),
+                           new VectorFEMassIntegrator(MMi_cf),1,1);
+   }
+   // --------------------------------------------------------------------------
+
+
+   socketstream E_out_r;
+   socketstream H_out_r;
+
+   ParComplexGridFunction E(E_fes);
+   ParComplexGridFunction H(H_fes);
+   E.real() = 0.0;
+   E.imag() = 0.0;
+   H.real() = 0.0;
+   H.imag() = 0.0;
+
+   ParGridFunction Et(E_fes);
+   ParGridFunction Ht(H_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+   ParaViewDataCollection * paraview_dct = nullptr;
+   std::ostringstream paraview_file_name;
+   paraview_file_name << "prob_" << prob
+                      << "_order_" << order
+                      << "_pref_" << pr
+                      << "_sc_" << (int)static_cond;
+
+   std::ostringstream paraview_file_namet;
+   paraview_file_namet << "tim-harmonin-prob_" << prob
+                       << "_order_" << order
+                       << "_pref_" << pr
+                       << "_sc_" << (int)static_cond;
+
+   if (paraview)
+   {
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath("ParaView/UW/3D");
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",&E.real());
+      paraview_dc->RegisterField("E_i",&E.imag());
+      paraview_dc->RegisterField("H_r",&H.real());
+      paraview_dc->RegisterField("H_i",&H.imag());
+
+      // paraview_dct = new ParaViewDataCollection(paraview_file_namet.str(), &pmesh);
+      // paraview_dct->SetPrefixPath("ParaView/UW3D");
+      // paraview_dct->SetLevelsOfDetail(order);
+      // paraview_dct->SetCycle(0);
+      // paraview_dct->SetDataFormat(VTKFormat::BINARY);
+      // paraview_dct->SetHighOrderOutput(true);
+      // paraview_dct->SetTime(0.0); // set the time
+      // paraview_dct->RegisterField("Et",&Et);
+      // paraview_dct->RegisterField("Ht",&Ht);
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   Array<int> one_bdr;
+   Array<int> negone_bdr;
+
+   if (myid == 0)
+   {
+      std::cout << "Setting up boundary attributes" << endl;
+   }
+
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      hatE_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      one_bdr = 0;
+      negone_bdr = 0;
+      if (prob == 4)
+      {
+         one_bdr[232-1] = 1;
+         negone_bdr[231-1] = 1;
+      }
+      else
+      {
+         one_bdr[1] = 1;
+         negone_bdr[2] = 1;      
+      }
+   }
+
+
+   if (myid == 0)
+   {
+      std::cout << "Setting up boundary conditions" << endl;
+   }
+   // Set up bdr conditions
+   // shift the ess_tdofs
+   for (int j = 0; j < ess_tdof_list.Size(); j++)
+   {
+      ess_tdof_list[j] += E_fes->GetTrueVSize() + H_fes->GetTrueVSize();
+   }
+
+   Array<int> offsets(5);
+   offsets[0] = 0;
+   offsets[1] = E_fes->GetVSize();
+   offsets[2] = H_fes->GetVSize();
+   offsets[3] = hatE_fes->GetVSize();
+   offsets[4] = hatH_fes->GetVSize();
+   offsets.PartialSum();
+
+   Vector x(2*offsets.Last());
+   x = 0.;
+   real_t * xdata = x.GetData();
+
+   ParComplexGridFunction hatE_gf(hatE_fes);
+   hatE_gf.real().MakeRef(hatE_fes,&xdata[offsets[2]]);
+   hatE_gf.imag().MakeRef(hatE_fes,&xdata[offsets.Last()+ offsets[2]]);
+
+   hatE_gf.ProjectBdrCoefficientTangent(z_one_cf,zero_cf, one_bdr);
+   hatE_gf.ProjectBdrCoefficientTangent(z_negone_cf,zero_cf, negone_bdr);
+
+   if (myid == 0)
+   {
+      std::cout << "Assembly started" << endl;
+   }
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   OperatorPtr Ah;
+   Vector X,B;
+   a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+   if (myid == 0)
+   {
+      std::cout << "Assembly finished" << endl;
+   }
+
+   ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+   BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+
+   Array<ParFiniteElementSpace *> prec_fes;
+   if (static_cond)
+   {
+      a->GetTraceFESpaces(prec_fes);
+   }
+   else
+   {
+      prec_fes = trial_fes;
+   }
+   Solver * cprec = nullptr;
+   if (pmg)
+   {
+#ifdef MFEM_USE_COMPLEX_MUMPS
+      bool mumps_coarse_solver = true;
+#else
+      bool mumps_coarse_solver = false;
+#endif
+      std::vector<Array<int>> ess_bdr_marker(prec_fes.Size());
+      for (int b = 0; b<prec_fes.Size(); b++)
+      {
+         if (pmesh.bdr_attributes.Size())
+         {
+            ess_bdr_marker[b].SetSize(pmesh.bdr_attributes.Max());
+            int ess_block = (static_cond) ? 0 : 2;
+            if (b == ess_block) // hatE
+            {
+               ess_bdr_marker[b] = ess_bdr;
+            }
+            else
+            {
+               ess_bdr_marker[b] = 0;
+            }
+         }
+      }
+      cprec = new ComplexPRefinementMultigrid(prec_fes, ess_bdr_marker, *Ahc, 
+                                              pmg_levels, relax_factor, mumps_coarse_solver);
+   }
+   else
+   {
+      BlockDiagonalPreconditioner * real_prec = new BlockDiagonalPreconditioner(BlockA_r->RowOffsets());
+      real_prec->owns_blocks = 1;
+      for (int i = 0; i<BlockA_r->NumRowBlocks(); i++)
+      {
+         auto prec = MakeFESpaceDefaultSolver(prec_fes[i],0);
+         prec->SetOperator(BlockA_r->GetBlock(i,i));
+         real_prec->SetDiagonalBlock(i,prec);
+      }
+      cprec = new ComplexPreconditioner(real_prec, true);
+   }
+
+   if (myid == 0)
+   {
+      std::cout << "PCG iterations" << endl;
+   }
+   CGSolver cg(MPI_COMM_WORLD);
+   // GMRESSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-6);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   cg.SetOperator(*Ahc);
+   cg.SetPreconditioner(*cprec);
+   cg.Mult(B, X);
+
+   delete cprec;
+
+
+   a->RecoverFEMSolution(X,x);
+
+   E.real().MakeRef(E_fes,x.GetData());
+   E.imag().MakeRef(E_fes,&x.GetData()[offsets.Last()]);
+
+   H.real().MakeRef(H_fes,&x.GetData()[offsets[1]]);
+   H.imag().MakeRef(H_fes,&x.GetData()[offsets.Last()+offsets[1]]);
+
+   if (visualization)
+   {
+      const char * keys = (dim == 2) ? "jRcml\n" : nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, E.real(),
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+      common::VisualizeField(H_out_r,vishost, visport, H.real(),
+                             "Numerical Magnetic field (real part)", 501, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime(0.0);
+      paraview_dc->Save();
+
+      // Et = E.real();
+      // int num_frames = 32;
+      // for (int it = 0; it<num_frames; it++)
+      // {
+      //    real_t t = (real_t)(it % num_frames) / num_frames;
+      //    add(cos(2.0*M_PI*t), E.real(), sin(2.0*M_PI*t), E.imag(), Et);
+      //    paraview_dct->SetCycle(it);
+      //    paraview_dct->SetTime((real_t)it);
+      //    paraview_dct->Save();
+      // }
+
+
+
+   }
+
+   if (paraview)
+   {
+      delete paraview_dc;
+   }
+
+   delete eps_r_cf;
+   delete eps_i_cf;
+   delete a;
+   delete F_fec;
+   delete G_fec;
+   delete hatH_fes;
+   delete hatH_fec;
+   delete hatE_fes;
+   delete hatE_fec;
+   delete H_fec;
+   delete E_fec;
+   delete H_fes;
+   delete E_fes;
+
+   return 0;
+}
+

miniapps/dpg/plasma/utils/lh_utils.cpp

@@ -0,0 +1,96 @@
+#include "lh_utils.hpp"
+
+real_t delta = 0.01; 
+real_t a0 = -1.0;   
+real_t a1 = 5.0;    
+
+void bfunc(const Vector &x, Vector &b)
+{
+   real_t r = std::sqrt(x(0) * x(0) + x(1) * x(1));
+   int dim = x.Size();
+   b.SetSize(dim); b = 0.0;
+   b(0) = -x(1) / r;
+   b(1) =  x(0) / r;
+   if (dim == 3) b(2) = 0.0;
+}
+
+void bcrossb(const Vector &x, DenseMatrix &bb)
+{
+   Vector b;
+   bfunc(x, b);
+   bb.SetSize(b.Size());
+   MultVVt(b, bb);
+}
+
+std::complex<real_t> pfunc(const Vector &x)
+{
+   real_t r = std::sqrt(x(0) * x(0) + x(1) * x(1));
+   return std::complex<real_t>(a0 + a1 *(r-0.9), delta);
+}
+
+std::complex<real_t> sfunc(const Vector &x)
+{
+   return std::complex<real_t>(1.0, delta);
+}
+
+real_t pfunc_r(const Vector &x) { return pfunc(x).real(); }
+real_t pfunc_i(const Vector &x) { return pfunc(x).imag(); }
+
+real_t sfunc_r(const Vector &x) { return sfunc(x).real(); }
+real_t sfunc_i(const Vector &x) { return sfunc(x).imag(); }
+
+
+void epsilon_func_r(const Vector &x, DenseMatrix &eps)
+{
+   std::complex<real_t> p = pfunc(x);
+   std::complex<real_t> s = sfunc(x);
+   DenseMatrix B;
+   bcrossb(x, B);
+   int dim = x.Size();
+   eps.SetSize(dim);
+
+   for (int i = 0; i < dim; i++)
+   {
+      for (int j = 0; j < dim; j++)
+      {
+         if (i == j)
+         {
+            eps(i, j) = (s * (1.0 - B(i, j)) + p * B(i, j)).real();
+         }
+         else
+         {
+            eps(i, j) = ((p - s) * B(i, j)).real();
+         }
+      }
+   }
+}
+
+void epsilon_func_i(const Vector &x, DenseMatrix &eps)
+{
+   int dim = x.Size();
+   eps.SetSize(dim);
+
+   std::complex<real_t> p = pfunc(x);
+   std::complex<real_t> s = sfunc(x);
+   DenseMatrix B;
+   bcrossb(x, B);
+
+   for (int i = 0; i < dim; i++)
+   {
+      for (int j = 0; j < dim; j++)
+      {
+         if (i == j)
+         {
+            eps(i, j) = (s * (1.0 - B(i, j)) + p * B(i, j)).imag();
+         }
+         else
+         {
+            eps(i, j) = ((p - s) * B(i, j)).imag();
+         }
+      }
+   }
+}
+
+
+
+

miniapps/dpg/plasma/utils/lh_utils.hpp

@@ -0,0 +1,21 @@
+#pragma once
+#include "mfem.hpp"
+
+using namespace mfem;
+using namespace std;
+
+extern double delta;
+extern double a0;
+extern double a1;
+
+std::complex<real_t> pfunc(const Vector &x);
+std::complex<real_t> sfunc(const Vector &x);
+
+real_t pfunc_r(const Vector &x);
+real_t pfunc_i(const Vector &x);
+real_t sfunc_r(const Vector &x);
+real_t sfunc_i(const Vector &x);
+void bfunc(const Vector &x, Vector &b);
+void bcrossb(const Vector &x, DenseMatrix &bb);
+void epsilon_func_r(const Vector &x, DenseMatrix &eps);
+void epsilon_func_i(const Vector &x, DenseMatrix &eps);

miniapps/dpg/pmaxwell-coupled-fem.cpp

@@ -0,0 +1,642 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//                   MFEM Ultraweak DPG Maxwell parallel example
+//
+// Compile with: make pmaxwell-coupled
+//
+// mpirun -np 4 ./pmaxwell-coupled -sref 1 -pref 2 -o 2 -rnum 0.5 -m ../../data/ref-cube.mesh -sc
+
+//      ∇×(1/μ ∇×E) - ω² ϵ E + J = F̃ ,   in Ω
+//             -ΔJ + α² J + c² E = G ,   in Ω
+//                       E×n = E₀ , on ∂Ω
+//                         J = J₀ , on ∂Ω
+
+#include "mfem.hpp"
+#include "util/pcomplexweakform.hpp"
+#include "util/pcomplexblockform.hpp"
+#include "../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+void maxwell_solution(const Vector &x, std::vector<complex<double>> &E);
+void maxwell_solution_curl(const Vector &x,
+                           std::vector<complex<double>> &curlE);
+void maxwell_solution_curlcurl(const Vector &x,
+                               std::vector<complex<double>> &curlcurlE);
+void J_solution(const Vector &x,std::vector<complex<double>> &J);
+void J_solution_grad(const Vector &x,
+                     std::vector<std::vector<complex<double>>> &gradJ);
+void J_solution_laplace(const Vector &x,
+                        std::vector<complex<double>> &laplaceJ);
+
+
+void E_exact_r(const Vector &x, Vector & E_r);
+void E_exact_i(const Vector &x, Vector & E_i);
+void H_exact_r(const Vector &x, Vector & H_r);
+void H_exact_i(const Vector &x, Vector & H_i);
+void J_exact_r(const Vector &x, Vector & J_r);
+void J_exact_i(const Vector &x, Vector & J_i);
+
+void curlE_exact_r(const Vector &x, Vector &curlE_r);
+void curlE_exact_i(const Vector &x, Vector &curlE_i);
+void curlH_exact_r(const Vector &x,Vector &curlH_r);
+void curlH_exact_i(const Vector &x,Vector &curlH_i);
+void gradJ_exact_r(const Vector &x, DenseMatrix &gradJ_r);
+void gradJ_exact_i(const Vector &x, DenseMatrix &gradJ_i);
+
+void curlcurlE_exact_r(const Vector &x, Vector & curlcurlE_r);
+void curlcurlE_exact_i(const Vector &x, Vector & curlcurlE_i);
+void LaplaceJ_exact_r(const Vector &x, Vector & d2J_r);
+void LaplaceJ_exact_i(const Vector &x, Vector & d2J_i);
+
+void  rhs1_func_r(const Vector &x, Vector & rhs1_r);
+void  rhs1_func_i(const Vector &x, Vector & rhs1_i);
+void  rhs2_func_r(const Vector &x, Vector & rhs2_r);
+void  rhs2_func_i(const Vector &x, Vector & rhs2_i);
+
+int dim;
+int dimc;
+double omega;
+double mu = 1.0;
+double epsilon = 1.0;
+double alpha = 1.0;
+double c = 1.0;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "../../data/inline-square.mesh";
+   int order = 1;
+   double rnum=1.0;
+   int sr = 0;
+   int pr = 1;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&rnum, "-rnum", "--number-of-wavelengths",
+                  "Number of wavelengths");
+   args.AddOption(&sr, "-sref", "--serial-ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&pr, "-pref", "--parallel-ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+
+   omega = 2.*M_PI*rnum;
+
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+   MFEM_VERIFY(dim > 1, "Dimension = 1 is not supported in this example");
+
+   dimc = (dim == 3) ? 3 : 1;
+
+   for (int i = 0; i<sr; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   enum TrialSpace
+   {
+      E_space     = 0,
+      J_space     = 1
+   };
+
+   // Vector L2 space for E
+   FiniteElementCollection *E_fec = new ND_FECollection(order,dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh,E_fec);
+
+   // Vector H1 space for J
+   FiniteElementCollection *J_fec = new H1_FECollection(order,dim);
+   ParFiniteElementSpace *J_fes = new ParFiniteElementSpace(&pmesh,J_fec, dim);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   trial_fes.Append(E_fes);
+   trial_fes.Append(J_fes);
+
+   ParComplexBlockForm * a = new ParComplexBlockForm(trial_fes);
+
+   // // Bilinear form coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient muinv(1./mu);
+   ConstantCoefficient rec_omega(1.0/omega);
+   ConstantCoefficient muomeg_cf(mu*omega);
+   ConstantCoefficient mu2omeg2_cf(mu*mu*omega*omega);
+   ConstantCoefficient eps2omeg2_cf(epsilon*epsilon*omega*omega);
+   ConstantCoefficient negepsomeg_cf(-epsilon*omega);
+   ConstantCoefficient epsomeg_cf(epsilon*omega);
+   ConstantCoefficient negmuomeg_cf(-mu*omega);
+   ConstantCoefficient c2_cf(c*c);
+   ConstantCoefficient a2_cf(alpha*alpha);
+
+   ConstantCoefficient negepsomeg2_cf(-epsilon*omega*omega);
+
+
+   // (1/μ ∇ × E,∇ × δE)
+   a->AddDomainIntegrator(new CurlCurlIntegrator(muinv), nullptr,
+                          TrialSpace::E_space, TrialSpace::E_space);
+   // (- ω² ϵ E, δE)
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(negepsomeg2_cf), nullptr,
+                          TrialSpace::E_space, TrialSpace::E_space);
+   // //  (J, δE)
+   a->AddDomainIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(one)),
+                          nullptr,
+                          TrialSpace::J_space, TrialSpace::E_space);
+
+   // // (∇ J, ∇ δJ)
+   a->AddDomainIntegrator(new VectorDiffusionIntegrator(one), nullptr,
+                          TrialSpace::J_space, TrialSpace::J_space);
+   // // (α²J,   δJ)
+   a->AddDomainIntegrator(new VectorMassIntegrator(a2_cf), nullptr,
+                          TrialSpace::J_space, TrialSpace::J_space);
+   // // (c²E,   δJ)
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(c2_cf), nullptr,
+                          TrialSpace::E_space, TrialSpace::J_space);
+
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = E_fes->GetVSize();
+   offsets[2] = J_fes->GetVSize();
+   offsets.PartialSum();
+
+   Vector x(2*offsets.Last());
+   x = 0.;
+   Vector b(2*offsets.Last());
+   real_t * bdata = b.GetData();
+
+   ParLinearForm bE_r(E_fes, bdata);
+   ParLinearForm bE_i(E_fes, &bdata[offsets.Last()]);
+   ParLinearForm bJ_r(J_fes, &bdata[offsets[1]]);
+   ParLinearForm bJ_i(J_fes, &bdata[offsets[2]]);
+
+   VectorFunctionCoefficient f_rhs1_r(dim,rhs1_func_r);
+   VectorFunctionCoefficient f_rhs1_i(dim,rhs1_func_i);
+
+   bE_r.AddDomainIntegrator(new VectorFEDomainLFIntegrator(f_rhs1_r));
+   bE_i.AddDomainIntegrator(new VectorFEDomainLFIntegrator(f_rhs1_i));
+
+
+
+   VectorFunctionCoefficient f_rhs2_r(dim,rhs2_func_r);
+   VectorFunctionCoefficient f_rhs2_i(dim,rhs2_func_i);
+
+   bJ_r.AddDomainIntegrator(new VectorDomainLFIntegrator(f_rhs2_r));
+   bJ_i.AddDomainIntegrator(new VectorDomainLFIntegrator(f_rhs2_i));
+   bE_r.Assemble();
+   bE_i.Assemble();
+   bJ_r.Assemble();
+   bJ_i.Assemble();
+
+
+   socketstream E_out_r, E_out_i, Eex_out_r, Eex_out_i;
+   socketstream J_out_r, J_out_i, Jex_out_r, Jex_out_i;
+
+
+   ParGridFunction E_r(E_fes), E_i(E_fes);
+   ParGridFunction J_r(J_fes), J_i(J_fes);
+   ParGridFunction Eex_r(E_fes), Eex_i(E_fes);
+   ParGridFunction Jex_r(J_fes), Jex_i(J_fes);
+
+
+   a->Assemble();
+
+   // cin.get();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listE;
+   Array<int> ess_tdof_listJ;
+   Array<int> ess_bdr;
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      E_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_listE);
+      J_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJ);
+   }
+
+   // shift the ess_tdofs
+   for (int j = 0; j < ess_tdof_listJ.Size(); j++)
+   {
+      ess_tdof_listJ[j] += E_fes->GetTrueVSize();
+   }
+   ess_tdof_list.Append(ess_tdof_listE);
+   ess_tdof_list.Append(ess_tdof_listJ);
+
+   ParGridFunction E_gf_r(E_fes, x, offsets[0]); E_gf_r = 0.0;
+   ParGridFunction E_gf_i(E_fes, x, offsets.Last()); E_gf_i = 0.0;
+   VectorFunctionCoefficient Eex_cf_r(dim,E_exact_r);
+   VectorFunctionCoefficient Eex_cf_i(dim,E_exact_i);
+   E_gf_r.ProjectBdrCoefficientTangent(Eex_cf_r, ess_bdr);
+   E_gf_i.ProjectBdrCoefficientTangent(Eex_cf_i, ess_bdr);
+
+   VectorFunctionCoefficient Ecf_r(dim,E_exact_r);
+   VectorFunctionCoefficient Ecf_i(dim,E_exact_i);
+   VectorFunctionCoefficient Jcf_r(dim,J_exact_r);
+   VectorFunctionCoefficient Jcf_i(dim,J_exact_i);
+
+   ParGridFunction J_gf_r(J_fes, x, offsets[1]); J_gf_r = 0.0;
+   ParGridFunction J_gf_i(J_fes, x, offsets.Last() + offsets[1]); J_gf_i = 0.0;
+   J_gf_r.ProjectBdrCoefficient(Jcf_r,ess_bdr);
+   J_gf_i.ProjectBdrCoefficient(Jcf_i,ess_bdr);
+
+   OperatorPtr Ah;
+   Vector X,B;
+   a->FormLinearSystem(ess_tdof_list,x,b, Ah, X,B);
+
+   ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+   BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+   BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+   int nblocks = BlockA_r->NumRowBlocks();
+   Array2D<const HypreParMatrix*> A_r_matrices(nblocks, nblocks);
+   Array2D<const HypreParMatrix*> A_i_matrices(nblocks, nblocks);
+   for (int i = 0; i < nblocks; i++)
+   {
+      for (int j = 0; j < nblocks; j++)
+      {
+         A_r_matrices(i,j) = dynamic_cast<const HypreParMatrix*>(&BlockA_r->GetBlock(i,
+                                                                                     j));
+         A_i_matrices(i,j) = dynamic_cast<const HypreParMatrix*>(&BlockA_i->GetBlock(i,
+                                                                                     j));
+      }
+   }
+   HypreParMatrix * Ahr = HypreParMatrixFromBlocks(A_r_matrices);
+   HypreParMatrix * Ahi = HypreParMatrixFromBlocks(A_i_matrices);
+
+   ComplexHypreParMatrix * Ahc_hypre =
+      new ComplexHypreParMatrix(Ahr, Ahi,false, false);
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Assembly finished successfully." << endl;
+   }
+
+#ifdef MFEM_USE_COMPLEX_MUMPS
+   auto solver = new ComplexMUMPSSolver(MPI_COMM_WORLD);
+   solver->SetPrintLevel(1);
+   solver->SetOperator(*Ahc_hypre);
+   solver->Mult(B,X);
+   delete solver;
+   delete Ahc_hypre;
+#else
+   MFEM_ABORT("MFEM compiled without mumps");
+#endif
+
+   a->RecoverFEMSolution(X, x);
+
+
+   E_r.MakeRef(E_fes,x, 0);
+   E_i.MakeRef(E_fes,x, offsets.Last());
+
+   J_r.MakeRef(J_fes,x, offsets[1]);
+   J_i.MakeRef(J_fes,x, offsets.Last()+offsets[1]);
+
+   if (visualization)
+   {
+      // const char * keys = (it == 0 && dim == 2) ? "jRcml\n" : nullptr;
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      VisualizeField(E_out_r,vishost, visport, E_r,
+                     "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+      VisualizeField(J_out_r,vishost, visport, J_r,
+                     "Numerical J field (real part)", 0, 0, 500, 500, keys);
+
+   }
+
+   delete a;
+
+   return 0;
+}
+
+void maxwell_solution(const Vector & X, std::vector<complex<double>> &E)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   E.resize(dim);
+   for (int i = 0; i < dim; ++i)
+   {
+      E[i] = 0.0;
+   }
+   E[0] = exp(zi * omega * (X.Sum()));
+}
+
+void maxwell_solution_curl(const Vector & X,
+                           std::vector<complex<double>> &curlE)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   curlE.resize(dimc);
+   for (int i = 0; i < dimc; ++i)
+   {
+      curlE[i] = 0.0;
+   }
+
+   std::complex<double> pw = exp(zi * omega * (X.Sum()));
+   if (dim == 3)
+   {
+      curlE[0] = 0.0;
+      curlE[1] = zi * omega * pw;
+      curlE[2] = -zi * omega * pw;
+   }
+   else
+   {
+      curlE[0] = -zi * omega * pw;
+   }
+}
+
+void maxwell_solution_curlcurl(const Vector & X,
+                               std::vector<complex<double>> &curlcurlE)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   curlcurlE.resize(dim);
+   for (int i = 0; i < dim; ++i)
+   {
+      curlcurlE[i] = 0.0;;
+   }
+   std::complex<double> pw = exp(zi * omega * (X.Sum()));
+   if (dim == 3)
+   {
+      curlcurlE[0] = 2.0 * omega * omega * pw;
+      curlcurlE[1] = - omega * omega * pw;
+      curlcurlE[2] = - omega * omega * pw;
+   }
+   else
+   {
+      curlcurlE[0] = omega * omega * pw;
+      curlcurlE[1] = -omega * omega * pw;
+   }
+}
+
+void J_solution(const Vector &x,std::vector<complex<double>> &J)
+{
+   J.resize(dim);
+   for (int i = 0; i < dim; ++i)
+   {
+      J[i] = 0.0;
+   }
+   J[0] = x[0] * x[0];
+}
+
+void J_solution_grad(const Vector &x,
+                     std::vector<std::vector<complex<double>>> &gradJ)
+{
+   gradJ.resize(dim);
+   for (int i = 0; i < dim; ++i)
+   {
+      gradJ[i].resize(dim);
+      for (int j = 0; j < dim; ++j)
+      {
+         gradJ[i][j] = 0.0;
+      }
+   }
+   gradJ[0][0] = 2*x[0];
+}
+
+void J_solution_laplace(const Vector &x, std::vector<complex<double>> &laplaceJ)
+{
+   laplaceJ.resize(dim);
+   for (int i = 0; i < dim; ++i)
+   {
+      laplaceJ[i] = 0.0;
+   }
+   laplaceJ[0] = 2.0;
+}
+
+void E_exact_r(const Vector &x, Vector & E_r)
+{
+   std::vector<std::complex<double>> E;
+   maxwell_solution(x,E);
+   E_r.SetSize(E.size());
+   for (unsigned i = 0; i < E.size(); i++)
+   {
+      E_r[i]= E[i].real();
+   }
+}
+
+void E_exact_i(const Vector &x, Vector & E_i)
+{
+   std::vector<std::complex<double>> E;
+   maxwell_solution(x, E);
+   E_i.SetSize(E.size());
+   for (unsigned i = 0; i < E.size(); i++)
+   {
+      E_i[i]= E[i].imag();
+   }
+}
+
+void J_exact_r(const Vector &x, Vector & J_r)
+{
+   std::vector<std::complex<double>> J;
+   J_solution(x,J);
+   J_r.SetSize(J.size());
+   for (unsigned i = 0; i < J.size(); i++)
+   {
+      J_r[i]= J[i].real();
+   }
+}
+
+void J_exact_i(const Vector &x, Vector & J_i)
+{
+   std::vector<std::complex<double>> J;
+   J_solution(x,J);
+   J_i.SetSize(J.size());
+   for (unsigned i = 0; i < J.size(); i++)
+   {
+      J_i[i]= J[i].imag();
+   }
+}
+
+void curlE_exact_r(const Vector &x, Vector &curlE_r)
+{
+   std::vector<std::complex<double>> curlE;
+   maxwell_solution_curl(x, curlE);
+   curlE_r.SetSize(curlE.size());
+   for (unsigned i = 0; i < curlE.size(); i++)
+   {
+      curlE_r[i]= curlE[i].real();
+   }
+}
+
+void curlE_exact_i(const Vector &x, Vector &curlE_i)
+{
+   std::vector<std::complex<double>> curlE;
+   maxwell_solution_curl(x, curlE);
+   curlE_i.SetSize(curlE.size());
+   for (unsigned i = 0; i < curlE.size(); i++)
+   {
+      curlE_i[i]= curlE[i].imag();
+   }
+}
+
+void gradJ_exact_r(const Vector &x, DenseMatrix &gradJ_r)
+{
+   std::vector<std::vector<std::complex<double>>> gradJ;
+   J_solution_grad(x, gradJ);
+   gradJ_r.SetSize(gradJ.size());
+   for (unsigned i = 0; i < gradJ.size(); i++)
+   {
+      for (unsigned j = 0; j < gradJ.size(); j++)
+      {
+         gradJ_r(i,j)= gradJ[i][j].real();
+      }
+   }
+}
+
+void gradJ_exact_i(const Vector &x, DenseMatrix &gradJ_i)
+{
+   std::vector<std::vector<std::complex<double>>> gradJ;
+   J_solution_grad(x, gradJ);
+   gradJ_i.SetSize(gradJ.size());
+   for (unsigned i = 0; i < gradJ.size(); i++)
+   {
+      for (unsigned j = 0; j < gradJ.size(); j++)
+      {
+         gradJ_i(i,j)= gradJ[i][j].imag();
+      }
+   }
+}
+
+void curlcurlE_exact_r(const Vector &x, Vector & curlcurlE_r)
+{
+   std::vector<std::complex<double>> curlcurlE;
+   maxwell_solution_curlcurl(x, curlcurlE);
+   curlcurlE_r.SetSize(curlcurlE.size());
+   for (unsigned i = 0; i < curlcurlE.size(); i++)
+   {
+      curlcurlE_r[i]= curlcurlE[i].real();
+   }
+}
+
+void curlcurlE_exact_i(const Vector &x, Vector & curlcurlE_i)
+{
+   std::vector<std::complex<double>> curlcurlE;
+   maxwell_solution_curlcurl(x, curlcurlE);
+   curlcurlE_i.SetSize(curlcurlE.size());
+   for (unsigned i = 0; i < curlcurlE.size(); i++)
+   {
+      curlcurlE_i[i]= curlcurlE[i].imag();
+   }
+}
+
+
+void LaplaceJ_exact_r(const Vector &x, Vector & d2J_r)
+{
+   std::vector<std::complex<double>> d2J;
+   J_solution_laplace(x, d2J);
+   d2J_r.SetSize(d2J.size());
+   for (unsigned i = 0; i < d2J.size(); i++)
+   {
+      d2J_r[i]= d2J[i].real();
+   }
+}
+
+void LaplaceJ_exact_i(const Vector &x, Vector & d2J_i)
+{
+   std::vector<std::complex<double>> d2J;
+   J_solution_laplace(x, d2J);
+   d2J_i.SetSize(d2J.size());
+   for (unsigned i = 0; i < d2J.size(); i++)
+   {
+      d2J_i[i]= d2J[i].imag();
+   }
+}
+
+// F = ∇×(1/μ ∇×E) - ω² ϵ E + J
+void  rhs1_func_r(const Vector &x, Vector & F_r)
+{
+   Vector E_r, curlcurlE_r, J_r;
+   E_exact_r(x,E_r);
+   curlcurlE_exact_r(x,curlcurlE_r);
+   J_exact_r(x,J_r);
+   F_r.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      F_r(i) = 1.0/mu * curlcurlE_r(i)
+               - omega * omega * epsilon * E_r(i)
+               + J_r(i);
+   }
+}
+
+void  rhs1_func_i(const Vector &x, Vector & F_i)
+{
+   Vector E_i, curlcurlE_i, J_i;
+   E_exact_i(x,E_i);
+   curlcurlE_exact_i(x,curlcurlE_i);
+   J_exact_i(x,J_i);
+   F_i.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      F_i(i) = 1.0/mu * curlcurlE_i(i)
+               - omega * omega * epsilon * E_i(i)
+               + J_i(i);
+   }
+}
+
+
+// G = -ΔJ + α² J + c² E
+// G_r + i G_i = - Δ (J_r + i J_i) + α² (J_r + i J_i) + c² (E_r + i E_i)
+void  rhs2_func_r(const Vector &x, Vector & G_r)
+{
+   // G_r = - Δ J_r + α² J_r + c² E_r
+   Vector E_r, J_r, d2J_r;
+   E_exact_r(x,E_r);
+   J_exact_r(x,J_r);
+   LaplaceJ_exact_r(x,d2J_r);
+   G_r.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      G_r(i) = -d2J_r[i] + alpha*alpha*J_r[i] + c*c * E_r[i];
+   }
+}
+
+void  rhs2_func_i(const Vector &x, Vector & G_i)
+{
+   // G_i = - Δ J_i + α² J_i + c² E_i
+   Vector E_i, J_i, d2J_i;
+   E_exact_i(x,E_i);
+   J_exact_i(x,J_i);
+   LaplaceJ_exact_i(x,d2J_i);
+   G_i.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      G_i(i) = -d2J_i[i] + alpha*alpha*J_i[i] + c*c * E_i[i];
+   }
+}

miniapps/dpg/pmaxwell-primal.cpp

@@ -0,0 +1,595 @@
+//                   MFEM Ultraweak DPG Maxwell parallel example
+//
+// Compile with: make pmaxwell
+//
+// sample run
+// mpirun -np 4 pmaxwell -m ../../data/star.mesh -o 2 -sref 0 -pref 3 -rnum 1.0
+// mpirun -np 4 pmaxwell -m ../../data/inline-quad.mesh -o 3 -sref 0 -pref 3 -rnum 4.8 -sc
+// mpirun -np 4 pmaxwell -m ../../data/inline-hex.mesh -o 2 -sref 0 -pref 1 -rnum 0.8 -sc
+
+// Description:
+// This example code demonstrates the use of MFEM to define and solve
+// the "ultraweak" (UW) DPG formulation for the Maxwell problem
+
+//      ∇×(1/μ ∇×E) - ω² ϵ E = J ,   in Ω
+//                       E×n = E₀ , on ∂Ω
+
+// It solves the following kinds of problems
+// 1) Known exact solutions with error convergence rates
+//    a) A manufactured solution problem where E is a plane wave
+
+#include "mfem.hpp"
+#include "util/pcomplexweakform.hpp"
+#include "../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+void E_exact_r(const Vector &x, Vector & E_r);
+void E_exact_i(const Vector &x, Vector & E_i);
+
+void  rhs_func_r(const Vector &x, Vector & J_r);
+void  rhs_func_i(const Vector &x, Vector & J_i);
+
+void curlE_exact_r(const Vector &x, Vector &curlE_r);
+void curlE_exact_i(const Vector &x, Vector &curlE_i);
+
+void curlcurlE_exact_r(const Vector &x, Vector & curlcurlE_r);
+void curlcurlE_exact_i(const Vector &x, Vector & curlcurlE_i);
+
+void maxwell_solution(const Vector & X,
+                      std::vector<complex<double>> &E);
+
+void maxwell_solution_curl(const Vector & X,
+                           std::vector<complex<double>> &curlE);
+
+void maxwell_solution_curlcurl(const Vector & X,
+                               std::vector<complex<double>> &curlcurlE);
+
+int dim;
+int dimc;
+double omega;
+double mu = 1.0;
+double epsilon = 1.0;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   double rnum=1.0;
+   bool static_cond = false;
+   int sr = 0;
+   int pr = 1;
+   bool visualization = true;
+   bool paraview = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&rnum, "-rnum", "--number-of-wavelengths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&delta_order, "-do", "--delta-order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&sr, "-sref", "--serial-ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&pr, "-pref", "--parallel-ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+
+   omega = 2.*M_PI*rnum;
+
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+   MFEM_VERIFY(dim > 1, "Dimension = 1 is not supported in this example");
+
+   dimc = (dim == 3) ? 3 : 1;
+
+   for (int i = 0; i<sr; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   enum TrialSpace { E_space = 0, hatE_space = 1 };
+   enum TestSpace  { F_space = 0 };
+
+   // H(curl) space for E
+   FiniteElementCollection *E_fec = new ND_FECollection(order,dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh,E_fec);
+
+   // H^-1/2 (curl) space for Ê
+   FiniteElementCollection * hatE_fec = nullptr;
+   int test_order = order+delta_order;
+   if (dim == 2)
+   {
+      hatE_fec = new H1_Trace_FECollection(order,dim);
+   }
+   else
+   {
+      hatE_fec = new ND_Trace_FECollection(order,dim);
+   }
+   ParFiniteElementSpace *hatE_fes = new ParFiniteElementSpace(&pmesh,hatE_fec);
+   FiniteElementCollection * F_fec = new ND_FECollection(test_order, dim);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+   trial_fes.Append(E_fes);
+   trial_fes.Append(hatE_fes);
+   test_fec.Append(F_fec);
+
+   // Bilinear form coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negepsomeg2(-epsilon*omega*omega);
+   ConstantCoefficient muinv(1./mu);
+   // for the 2D case
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_fes,test_fec);
+   a->StoreMatrices(); // needed for AMR
+
+   // (∇ × E,∇ × F)
+   a->AddTrialIntegrator(new CurlCurlIntegrator(muinv), nullptr,
+                         TrialSpace::E_space, TestSpace::F_space);
+   // -ω² ϵ (E , F)
+   a->AddTrialIntegrator(new VectorFEMassIntegrator(negepsomeg2), nullptr,
+                         TrialSpace::E_space,TestSpace::F_space);
+   // < n×Ê ,F>
+   if (dim == 3)
+   {
+      a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,
+                            TrialSpace::hatE_space, TestSpace::F_space);
+   }
+   else
+   {
+      a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,
+                            TrialSpace::hatE_space, TestSpace::F_space);
+   }
+   // test integrators
+   // (∇×F ,∇×δF)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,
+                        TestSpace::F_space,TestSpace::F_space);
+   // (F,δF)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,
+                        TestSpace::F_space,TestSpace::F_space);
+
+   // RHS
+   VectorFunctionCoefficient f_rhs_r(dim,rhs_func_r);
+   VectorFunctionCoefficient f_rhs_i(dim,rhs_func_i);
+   a->AddDomainLFIntegrator(new VectorFEDomainLFIntegrator(f_rhs_r),
+                            new VectorFEDomainLFIntegrator(f_rhs_i),
+                            TestSpace::F_space);
+
+   VectorFunctionCoefficient Eex_r(dim,E_exact_r);
+   VectorFunctionCoefficient Eex_i(dim,E_exact_i);
+
+   socketstream E_out_r;
+   if (myid == 0)
+   {
+      std::cout << "\n  Ref |"
+                << "    Dofs    |"
+                << "    ω    |" ;
+      std::cout  << "  H(curl) Error  |"
+                 << "  Rate  |" ;
+      std::cout << "  Residual  |"
+                << "  Rate  |"
+                << " PCG it |" << endl;
+      std::cout << std::string(87,'-')
+                << endl;
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   ParGridFunction E_r, E_i;
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   if (paraview)
+   {
+      paraview_dc = new ParaViewDataCollection("Plane", &pmesh);
+      paraview_dc->SetPrefixPath("ParaViewPrimal/Maxwell");
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",&E_r);
+      paraview_dc->RegisterField("E_i",&E_i);
+   }
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+   for (int it = 0; it<=pr; it++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         E_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      Array<int> offsets(3);
+      offsets[0] = 0;
+      offsets[1] = E_fes->GetVSize();
+      offsets[2] = hatE_fes->GetVSize();
+      offsets.PartialSum();
+
+      Vector x(2*offsets.Last());
+      x = 0.;
+
+      ParGridFunction E_gf_r(E_fes, x, offsets[0]);
+      ParGridFunction E_gf_i(E_fes, x, offsets.Last() + offsets[0]);
+      E_gf_r.ProjectBdrCoefficientTangent(Eex_r, ess_bdr);
+      E_gf_i.ProjectBdrCoefficientTangent(Eex_i, ess_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+      int num_blocks = BlockA_r->NumRowBlocks();
+      Array<int> tdof_offsets(2*num_blocks+1);
+
+      tdof_offsets[0] = 0;
+      for (int i=0; i<num_blocks; i++)
+      {
+         const int h = BlockA_r->GetBlock(i,i).Height();
+         tdof_offsets[i+1] = h;
+         tdof_offsets[num_blocks+i+1] = h;
+      }
+      tdof_offsets.PartialSum();
+
+      BlockOperator blockA(tdof_offsets);
+      for (int i = 0; i<num_blocks; i++)
+      {
+         for (int j = 0; j<num_blocks; j++)
+         {
+            blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
+            blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
+         }
+      }
+
+      X = 0.;
+      BlockDiagonalPreconditioner M(tdof_offsets);
+
+      ParFiniteElementSpace *ams_fes = nullptr;
+      if (static_cond)
+      {
+         ams_fes = new ParFiniteElementSpace(&pmesh,
+                                             E_fes->FEColl()->GetTraceCollection());
+      }
+      HypreAMS * solver_E = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(0,0),
+                                         (static_cond) ? ams_fes : E_fes);
+      solver_E->SetPrintLevel(0);
+
+      HypreSolver * solver_hatE = nullptr;
+
+      if (dim == 2)
+      {
+         solver_hatE = new HypreBoomerAMG((HypreParMatrix &)BlockA_r->GetBlock(1,1));
+         dynamic_cast<HypreBoomerAMG*>(solver_hatE)->SetPrintLevel(0);
+      }
+      else
+      {
+         solver_hatE = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(1,1),
+                                    hatE_fes);
+         dynamic_cast<HypreAMS*>(solver_hatE)->SetPrintLevel(0);
+      }
+
+      M.SetDiagonalBlock(0,solver_E);
+      M.SetDiagonalBlock(1,solver_hatE);
+      M.SetDiagonalBlock(num_blocks,solver_E);
+      M.SetDiagonalBlock(num_blocks+1,solver_hatE);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(10000);
+      cg.SetPrintLevel(0);
+      cg.SetPreconditioner(M);
+      cg.SetOperator(blockA);
+      cg.Mult(B, X);
+      delete ams_fes;
+      for (int i = 0; i<num_blocks; i++)
+      {
+         delete &M.GetDiagonalBlock(i);
+      }
+
+      int num_iter = cg.GetNumIterations();
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      double maxresidual = residuals.Max();
+      double globalresidual = residual * residual;
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+      E_r.MakeRef(E_fes,x, 0);
+      E_i.MakeRef(E_fes,x, offsets.Last());
+
+      int dofs = 0;
+      for (int i = 0; i<trial_fes.Size(); i++)
+      {
+         dofs += trial_fes[i]->GlobalTrueVSize();
+      }
+
+      double HcurlError = 0.0;
+      double rate_err = 0.0;
+
+      VectorFunctionCoefficient curlEex_r(dim,curlE_exact_r);
+      VectorFunctionCoefficient curlEex_i(dim,curlE_exact_i);
+
+      double E_err_r = E_r.ComputeHCurlError(&Eex_r,&curlEex_r);
+      double E_err_i = E_i.ComputeHCurlError(&Eex_i,&curlEex_i);
+
+      HcurlError = sqrt(  E_err_r*E_err_r + E_err_i*E_err_i);
+      rate_err = (it) ? dim*log(err0/HcurlError)/log((double)dof0/dofs) : 0.0;
+      err0 = HcurlError;
+
+      double rate_res =
+         (it) ? dim*log(res0/globalresidual)/log((double)dof0/dofs) : 0.0;
+
+      res0 = globalresidual;
+      dof0 = dofs;
+
+      if (myid == 0)
+      {
+         std::ios oldState(nullptr);
+         oldState.copyfmt(std::cout);
+         std::cout << std::right << std::setw(5) << it << " | "
+                   << std::setw(10) <<  dof0 << " | "
+                   << std::setprecision(1) << std::fixed
+                   << std::setw(4) <<  2.0*rnum << " π  | "
+                   << std::setprecision(3);
+         std::cout << std::setw(15) << std::scientific <<  err0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_err << " | " ;
+         std::cout << std::setprecision(3)
+                   << std::setw(10) << std::scientific <<  res0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_res << " | "
+                   << std::setw(6) << std::fixed << num_iter << " | "
+                   << std::endl;
+         std::cout.copyfmt(oldState);
+      }
+
+      if (visualization)
+      {
+         const char * keys = (it == 0 && dim == 2) ? "jRcml\n" : nullptr;
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         VisualizeField(E_out_r,vishost, visport, E_r,
+                        "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+      }
+
+      if (paraview)
+      {
+         paraview_dc->SetCycle(it);
+         paraview_dc->SetTime((double)it);
+         paraview_dc->Save();
+      }
+
+      if (it == pr)
+      {
+         break;
+      }
+
+      pmesh.UniformRefinement();
+
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   if (paraview)
+   {
+      delete paraview_dc;
+   }
+
+   delete a;
+   delete F_fec;
+   delete hatE_fes;
+   delete hatE_fec;
+   delete E_fec;
+   delete E_fes;
+
+   return 0;
+}
+
+void E_exact_r(const Vector &x, Vector & E_r)
+{
+   std::vector<std::complex<double>> E;
+   maxwell_solution(x,E);
+   E_r.SetSize(E.size());
+   for (unsigned i = 0; i < E.size(); i++)
+   {
+      E_r[i]= E[i].real();
+   }
+}
+
+void E_exact_i(const Vector &x, Vector & E_i)
+{
+   std::vector<std::complex<double>> E;
+   maxwell_solution(x, E);
+   E_i.SetSize(E.size());
+   for (unsigned i = 0; i < E.size(); i++)
+   {
+      E_i[i]= E[i].imag();
+   }
+}
+
+void curlE_exact_r(const Vector &x, Vector &curlE_r)
+{
+   std::vector<std::complex<double>> curlE;
+   maxwell_solution_curl(x, curlE);
+   curlE_r.SetSize(curlE.size());
+   for (unsigned i = 0; i < curlE.size(); i++)
+   {
+      curlE_r[i]= curlE[i].real();
+   }
+}
+
+void curlE_exact_i(const Vector &x, Vector &curlE_i)
+{
+   std::vector<std::complex<double>> curlE;
+   maxwell_solution_curl(x, curlE);
+   curlE_i.SetSize(curlE.size());
+   for (unsigned i = 0; i < curlE.size(); i++)
+   {
+      curlE_i[i]= curlE[i].imag();
+   }
+}
+
+void curlcurlE_exact_r(const Vector &x, Vector & curlcurlE_r)
+{
+   std::vector<std::complex<double>> curlcurlE;
+   maxwell_solution_curlcurl(x, curlcurlE);
+   curlcurlE_r.SetSize(curlcurlE.size());
+   for (unsigned i = 0; i < curlcurlE.size(); i++)
+   {
+      curlcurlE_r[i]= curlcurlE[i].real();
+   }
+}
+
+void curlcurlE_exact_i(const Vector &x, Vector & curlcurlE_i)
+{
+   std::vector<std::complex<double>> curlcurlE;
+   maxwell_solution_curlcurl(x, curlcurlE);
+   curlcurlE_i.SetSize(curlcurlE.size());
+   for (unsigned i = 0; i < curlcurlE.size(); i++)
+   {
+      curlcurlE_i[i]= curlcurlE[i].imag();
+   }
+}
+
+void  rhs_func_r(const Vector &x, Vector & J_r)
+{
+   Vector E_r, curlcurlE_r;
+   E_exact_r(x,E_r);
+   curlcurlE_exact_r(x,curlcurlE_r);
+   J_r.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      J_r(i) = 1./mu*curlcurlE_r(i) - omega *omega * epsilon * E_r(i);
+   }
+}
+
+void  rhs_func_i(const Vector &x, Vector & J_i)
+{
+   Vector E_i, curlcurlE_i;
+   E_exact_i(x,E_i);
+   curlcurlE_exact_i(x,curlcurlE_i);
+   J_i.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      J_i(i) = 1./mu*curlcurlE_i(i) - omega *omega * epsilon * E_i(i);
+   }
+}
+
+void maxwell_solution(const Vector & X, std::vector<complex<double>> &E)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   E.resize(dim);
+   for (int i = 0; i < dim; ++i)
+   {
+      E[i] = 0.0;
+   }
+   E[0] = exp(zi * omega * (X.Sum()));
+}
+
+void maxwell_solution_curl(const Vector & X,
+                           std::vector<complex<double>> &curlE)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   curlE.resize(dimc);
+   for (int i = 0; i < dimc; ++i)
+   {
+      curlE[i] = 0.0;
+   }
+   std::complex<double> pw = exp(zi * omega * (X.Sum()));
+   if (dim == 3)
+   {
+      curlE[0] = 0.0;
+      curlE[1] = zi * omega * pw;
+      curlE[2] = -zi * omega * pw;
+   }
+   else
+   {
+      curlE[0] = -zi * omega * pw;
+   }
+}
+
+void maxwell_solution_curlcurl(const Vector & X,
+                               std::vector<complex<double>> &curlcurlE)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   curlcurlE.resize(dim);
+   for (int i = 0; i < dim; ++i)
+   {
+      curlcurlE[i] = 0.0;;
+   }
+   std::complex<double> pw = exp(zi * omega * (X.Sum()));
+   if (dim == 3)
+   {
+      curlcurlE[0] = 2.0 * omega * omega * pw;
+      curlcurlE[1] = - omega * omega * pw;
+      curlcurlE[2] = - omega * omega * pw;
+   }
+   else
+   {
+      curlcurlE[0] = omega * omega * pw;
+      curlcurlE[1] = -omega * omega * pw;
+   }
+}

miniapps/dpg/pmaxwell.cpp

@@ -44,7 +44,7 @@
 // The DPG UW deals with the First Order System
 //  i ω μ H + ∇ × E = 0,   in Ω
 // -i ω ϵ E + ∇ × H = J,   in Ω
-//            E × n = E_0, on ∂Ω
+//            E × n = E₀, on ∂Ω
 // Note: Ĵ = -iωJ
 
 // The ultraweak-DPG formulation is obtained by integration by parts of both
@@ -57,7 +57,7 @@
 // where A = [0 1; -1 0];
 
 // E ∈ (L²(Ω))² , H ∈ L²(Ω)
-// Ê ∈ H^-1/2(Ω)(Γₕ), Ĥ ∈ H^1/2(Γₕ)
+// Ê ∈ H^-1/2(Γₕ), Ĥ ∈ H^1/2(Γₕ)
 //  i ω μ (H,F) + (E, ∇ × F) + < AÊ, F > = 0,      ∀ F ∈ H¹
 // -i ω ϵ (E,G) + (H,∇ × G)  + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)
 //                                        Ê = E₀      on ∂Ω
@@ -71,7 +71,7 @@
 
 // in 3D
 // E,H ∈ (L^2(Ω))³
-// Ê ∈ H_0^1/2(Ω)(curl, Γₕ), Ĥ ∈ H^-1/2(curl, Γₕ)
+// Ê ∈ H\_0^1/2(Ω)(curl, Γₕ), Ĥ ∈ H^-1/2(curl, Γₕ)
 //  i ω μ (H,F) + (E,∇ × F) + < Ê, F × n > = 0,      ∀ F ∈ H(curl,Ω)
 // -i ω ϵ (E,G) + (H,∇ × G) + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)
 //                                   Ê × n = E₀      on ∂Ω
@@ -106,7 +106,7 @@
 // in 2D
 // E ∈ (L²(Ω))² , H ∈ L²(Ω)
 // Ê ∈ H^-1/2(Ω)(Γₕ), Ĥ ∈ H^1/2(Γₕ)
-//  i ω μ (α⁻¹ H,F) + (E, ∇ × F) + < AÊ, F > = 0,          ∀ F ∈ H¹
+//  i ω μ (α⁻¹ H,F) + (E, ∇ × F) + < AÊ, F > = 0,         ∀ F ∈ H¹
 // -i ω ϵ (β E,G)   + (H,∇ × G)  + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)
 //                                            Ê = E₀     on ∂Ω
 // ---------------------------------------------------------------------------------
@@ -121,10 +121,10 @@
 //
 // in 3D
 // E,H ∈ (L^2(Ω))³
-// Ê ∈ H_0^1/2(Ω)(curl, Γ_h), Ĥ ∈ H^-1/2(curl, Γₕ)
+// Ê ∈ H_0^1/2(Ω)(curl, Γₕ), Ĥ ∈ H^-1/2(curl, Γₕ)
 //  i ω μ (α⁻¹ H,F) + (E,∇ × F) + < Ê, F × n > = 0,      ∀ F ∈ H(curl,Ω)
-// -i ω ϵ (β E,G)    + (H,∇ × G) + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)
-//                                        Ê × n = E_0     on ∂Ω
+// -i ω ϵ (β E,G)   + (H,∇ × G) + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)
+//                                       Ê × n = E₀      on ∂Ω
 // -------------------------------------------------------------------------------
 // |   |       E      |      H           |      Ê       |       Ĥ      |  RHS    |
 // -------------------------------------------------------------------------------
@@ -138,6 +138,7 @@
 #include "mfem.hpp"
 #include "util/pcomplexweakform.hpp"
 #include "util/pml.hpp"
+#include "util/preconditioners.hpp"
 #include "../common/mfem-common.hpp"
 #include <fstream>
 #include <iostream>
@@ -146,13 +147,13 @@ using namespace std;
 using namespace mfem;
 using namespace mfem::common;
 
+
 void E_exact_r(const Vector &x, Vector & E_r);
 void E_exact_i(const Vector &x, Vector & E_i);
 
 void H_exact_r(const Vector &x, Vector & H_r);
 void H_exact_i(const Vector &x, Vector & H_i);
 
-
 void  rhs_func_r(const Vector &x, Vector & J_r);
 void  rhs_func_i(const Vector &x, Vector & J_i);
 
@@ -221,6 +222,9 @@ int main(int argc, char *argv[])
    int delta_order = 1;
    real_t rnum=1.0;
    real_t theta = 0.0;
+   bool pmg = false;
+   int pmg_levels = -1;
+   real_t relax_factor = 2.0/3;
    bool static_cond = false;
    int iprob = 0;
    int sr = 0;
@@ -255,6 +259,12 @@ int main(int argc, char *argv[])
                   "Number of parallel refinements.");
    args.AddOption(&pr, "-pref", "--parallel-ref",
                   "Number of parallel refinements.");
+   args.AddOption(&pmg, "-pmg", "--p-refinement-multigrid", "-no-pmg",
+                  "--no-p-refinement-multigrid", "Enable P-Refinement Multigrid.");
+   args.AddOption(&pmg_levels, "-pmgl","--p-refinement-multigrid-levels",
+                  "Number of levels for P-Refinement Multigrid.");
+   args.AddOption(&relax_factor, "-rf", "--relaxation-factor",
+                  "Relaxation factor for the p-multigrid smoother.");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -345,7 +355,7 @@ int main(int argc, char *argv[])
       F_space = 0,
       G_space = 1
    };
-   // L2 space for E
+   // Vector L2 L2 space for E
    FiniteElementCollection *E_fec = new L2_FECollection(order-1,dim);
    ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh,E_fec,dim);
 
@@ -503,7 +513,16 @@ int main(int argc, char *argv[])
          *negepsomeg_detJ_Jt_J_inv_r_rot, attrPML);
    }
 
+   pmesh.EnsureNodes();
+   int meshorder = pmesh.GetNodalFESpace()->FEColl()->GetOrder();
+   const IntegrationRule &test_ir = IntRules.Get(pmesh.GetElementGeometry(0),
+                                                 2*test_order + 5 + meshorder);
+   const IntegrationRule &trial_ir = IntRules.Get(pmesh.GetElementGeometry(0),
+                                                  order+test_order + 5 + meshorder);
+
    ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_fes,test_fec);
+   // a->SetTrialIntegrationRule(trial_ir);
+   // a->SetTestIntegrationRule(test_ir);
    a->StoreMatrices(); // needed for AMR
 
    // (E,∇ × F)
@@ -821,89 +840,70 @@ int main(int argc, char *argv[])
       a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
 
       ComplexOperator * Ahc = Ah.As<ComplexOperator>();
-
       BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
-      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
 
-      int num_blocks = BlockA_r->NumRowBlocks();
-      Array<int> tdof_offsets(2*num_blocks+1);
-
-      tdof_offsets[0] = 0;
-      int skip = (static_cond) ? 0 : 2;
-      int k = (static_cond) ? 2 : 0;
-      for (int i=0; i<num_blocks; i++)
+      Array<ParFiniteElementSpace *> prec_fes;
+      if (static_cond)
       {
-         tdof_offsets[i+1] = trial_fes[i+k]->GetTrueVSize();
-         tdof_offsets[num_blocks+i+1] = trial_fes[i+k]->GetTrueVSize();
-      }
-      tdof_offsets.PartialSum();
-
-      BlockOperator blockA(tdof_offsets);
-      for (int i = 0; i<num_blocks; i++)
-      {
-         for (int j = 0; j<num_blocks; j++)
-         {
-            blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
-            blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
-            blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
-            blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
-         }
-      }
-
-      X = 0.;
-      BlockDiagonalPreconditioner M(tdof_offsets);
-
-      if (!static_cond)
-      {
-         HypreBoomerAMG * solver_E = new HypreBoomerAMG((HypreParMatrix &)
-                                                        BlockA_r->GetBlock(0,0));
-         solver_E->SetPrintLevel(0);
-         solver_E->SetSystemsOptions(dim);
-         HypreBoomerAMG * solver_H = new HypreBoomerAMG((HypreParMatrix &)
-                                                        BlockA_r->GetBlock(1,1));
-         solver_H->SetPrintLevel(0);
-         solver_H->SetSystemsOptions(dim);
-         M.SetDiagonalBlock(0,solver_E);
-         M.SetDiagonalBlock(1,solver_H);
-         M.SetDiagonalBlock(num_blocks,solver_E);
-         M.SetDiagonalBlock(num_blocks+1,solver_H);
-      }
-
-      HypreSolver * solver_hatH = nullptr;
-      HypreAMS * solver_hatE = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(skip,
-                                                                                 skip),
-                                            hatE_fes);
-      solver_hatE->SetPrintLevel(0);
-      if (dim == 2)
-      {
-         solver_hatH = new HypreBoomerAMG((HypreParMatrix &)BlockA_r->GetBlock(skip+1,
-                                                                               skip+1));
-         dynamic_cast<HypreBoomerAMG*>(solver_hatH)->SetPrintLevel(0);
+         a->GetTraceFESpaces(prec_fes);
       }
       else
       {
-         solver_hatH = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(skip+1,skip+1),
-                                    hatH_fes);
-         dynamic_cast<HypreAMS*>(solver_hatH)->SetPrintLevel(0);
+         prec_fes = trial_fes;
+      }
+      Solver * cprec = nullptr;
+      if (pmg)
+      {
+#ifdef MFEM_USE_COMPLEX_MUMPS
+         bool mumps_coarse_solver = true;
+#else
+         bool mumps_coarse_solver = false;
+#endif
+         std::vector<Array<int>> ess_bdr_marker(prec_fes.Size());
+         for (int b = 0; b<prec_fes.Size(); b++)
+         {
+            if (pmesh.bdr_attributes.Size())
+            {
+               ess_bdr_marker[b].SetSize(pmesh.bdr_attributes.Max());
+               int ess_block = (static_cond) ? 0 : 2;
+               if (b == ess_block) // hatE
+               {
+                  ess_bdr_marker[b] = ess_bdr;
+               }
+               else
+               {
+                  ess_bdr_marker[b] = 0;
+               }
+            }
+         }
+         cprec = new ComplexPRefinementMultigrid(prec_fes, ess_bdr_marker, *Ahc,
+                                                 pmg_levels, relax_factor, mumps_coarse_solver);
+      }
+      else
+      {
+         BlockDiagonalPreconditioner * real_prec = new BlockDiagonalPreconditioner(
+            BlockA_r->RowOffsets());
+         real_prec->owns_blocks = 1;
+         for (int i = 0; i<BlockA_r->NumRowBlocks(); i++)
+         {
+            auto prec = MakeFESpaceDefaultSolver(prec_fes[i],0);
+            prec->SetOperator(BlockA_r->GetBlock(i,i));
+            real_prec->SetDiagonalBlock(i,prec);
+         }
+         cprec = new ComplexPreconditioner(real_prec, true);
       }
 
-      M.SetDiagonalBlock(skip,solver_hatE);
-      M.SetDiagonalBlock(skip+1,solver_hatH);
-      M.SetDiagonalBlock(skip+num_blocks,solver_hatE);
-      M.SetDiagonalBlock(skip+num_blocks+1,solver_hatH);
-
       CGSolver cg(MPI_COMM_WORLD);
+      // SLISolver cg(MPI_COMM_WORLD);
       cg.SetRelTol(1e-6);
       cg.SetMaxIter(10000);
       cg.SetPrintLevel(0);
-      cg.SetPreconditioner(M);
-      cg.SetOperator(blockA);
+      cg.SetOperator(*Ahc);
+
+      cg.SetPreconditioner(*cprec);
       cg.Mult(B, X);
 
-      for (int i = 0; i<num_blocks; i++)
-      {
-         delete &M.GetDiagonalBlock(i);
-      }
+      delete cprec;
 
       int num_iter = cg.GetNumIterations();
 
@@ -1185,7 +1185,7 @@ void hatE_exact_r(const Vector & x, Vector & hatE_r)
    {
       Vector E_r;
       E_exact_r(x,E_r);
-      hatE_r.SetSize(hatE_r.Size());
+      hatE_r.SetSize(E_r.Size());
       // rotate E_hat
       hatE_r[0] = E_r[1];
       hatE_r[1] = -E_r[0];
@@ -1202,7 +1202,7 @@ void hatE_exact_i(const Vector & x, Vector & hatE_i)
    {
       Vector E_i;
       E_exact_i(x,E_i);
-      hatE_i.SetSize(hatE_i.Size());
+      hatE_i.SetSize(E_i.Size());
       // rotate E_hat
       hatE_i[0] = E_i[1];
       hatE_i[1] = -E_i[0];

miniapps/dpg/pvectordiffusion-primal.cpp

@@ -0,0 +1,390 @@
+//                 MFEM Primal DPG parallel example for diffusion
+//
+// Compile with: make pdiffusion-primal
+//
+// Sample runs
+// mpirun -np 4 pdiffusion-primal -m ../../data/inline-quad.mesh -o 3 -sref 1 -pref 2
+
+//       - Δ u = f,   in Ω
+//           u = u₀, on ∂Ω
+
+
+
+// --------------------------------------
+// |   |     u     |      σ̂    |  RHS   |
+// --------------------------------------
+// | v |  (∇u,∇v)  |  -(σ̂ₙ,v)  |  (f,v) |
+//
+// u ∈ H¹(Ω),   σ̂ₙ ∈ H^-1/2(Τ)
+
+
+#include "mfem.hpp"
+#include "util/pweakform.hpp"
+#include "../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+void exact_u(const Vector & X, Vector & u);
+void exact_gradu(const Vector & X, DenseMatrix &du);
+void exact_laplacian_u(const Vector & X, Vector & d2u);
+void f_exact(const Vector & X, Vector & f);
+
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI and HYPRE.
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int sref = 0; // initial uniform mesh refinements
+   int pref = 0; // parallel mesh refinements for AMR
+   bool static_cond = false;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&sref, "-sref", "--num-serial-refinements",
+                  "Number of initial serial uniform refinements");
+   args.AddOption(&pref, "-pref", "--num-parallel-refinements",
+                  "Number of AMR refinements");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   for (int i = 0; i<sref; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+
+   // H1 space for u
+   FiniteElementCollection *u_fec = new H1_FECollection(order,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec,dim);
+
+
+   // H^-1/2 space for σ̂
+   FiniteElementCollection * hatsigma_fec = new RT_Trace_FECollection(order-1,dim);
+   ParFiniteElementSpace *hatsigma_fes = new ParFiniteElementSpace(&pmesh,
+                                                                   hatsigma_fec,dim);
+
+   ParFiniteElementSpace *hatsigma1_fes = new ParFiniteElementSpace(&pmesh,
+                                                                    hatsigma_fec);
+
+   int test_order = order+delta_order;
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(hatsigma_fes);
+   test_fec.Append(v_fec);
+
+
+   ConstantCoefficient one(1.0);
+   VectorFunctionCoefficient f(dim,
+                               f_exact); // rhs for the manufactured solution problem
+   VectorFunctionCoefficient uex(dim,exact_u);
+   MatrixFunctionCoefficient graduex(dim,exact_gradu);
+
+   ParDPGWeakForm * a = new ParDPGWeakForm(trial_fes,test_fec);
+   a->SetTestFECollVdim(0,dim);
+   a->StoreMatrices(true); // this is needed for estimation of residual
+
+   // (∇u,∇v)
+   a->AddTrialIntegrator(new VectorDiffusionIntegrator(one),0,0);
+
+   // // -<σ̂,v> (sign is included in σ̂)
+   a->AddTrialIntegrator(new VectorTraceIntegrator,1,0);
+
+   // // (∇v,∇δv)
+   a->AddTestIntegrator(new VectorDiffusionIntegrator(one),0,0);
+   // // (v,δv)
+   a->AddTestIntegrator(new VectorMassIntegrator(one),0,0);
+
+   a->AddDomainLFIntegrator(new VectorDomainLFIntegrator(f),0);
+
+
+   if (myid == 0)
+   {
+      std::cout << "\n  Ref |"
+                << "    Dofs    |"
+                << "  H1 Error  |"
+                << "  Rate  |"
+                << "  Residual  |"
+                << "  Rate  |"
+                << " PCG it |" << endl;
+      std::cout << std::string(72,'-') << endl;
+   }
+
+
+   socketstream u_out, ex_out;
+
+   double err0 = 0.;
+   int dof0=0.;
+   double res0=0.0;
+
+   ParGridFunction u_gf(u_fes);
+   ParGridFunction exact_gf(u_fes);
+   u_gf = 0.0;
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+
+   for (int it = 0; it<=pref; it++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         u_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      Array<int> offsets(3);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = hatsigma_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      u_gf.MakeRef(u_fes,x.GetBlock(0),0);
+      u_gf.ProjectBdrCoefficient(uex,ess_bdr);
+      exact_gf.ProjectCoefficient(uex);
+
+      Vector X,B;
+      OperatorPtr Ah;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockOperator * A = Ah.As<BlockOperator>();
+
+
+
+      // BlockDiagonalPreconditioner M(A->RowOffsets());
+      BlockDiagonalPreconditioner M(A->RowOffsets());
+      M.owns_blocks = 1;
+
+      // HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+      // amg0->SetSystemsOptions(dim);
+#ifdef MFEM_USE_MUMPS
+      MUMPSSolver * amg0 = new MUMPSSolver(MPI_COMM_WORLD);
+      amg0->SetOperator((HypreParMatrix &)A->GetBlock(0,0));
+      MUMPSSolver * prec = new MUMPSSolver(MPI_COMM_WORLD);
+      prec->SetOperator((HypreParMatrix&)A->GetBlock(1,1));
+#else
+      HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+      amg0->SetSystemsOptions(dim);
+      HypreBoomerAMG * prec = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
+      prec->SetSystemsOptions(dim);
+#endif
+      M.SetDiagonalBlock(0,amg0);
+      M.SetDiagonalBlock(1,prec);
+
+      // Array<int> toffsets(4);
+      // toffsets[0] = 0;
+      // toffsets[1] = A->RowOffsets()[1];
+      // toffsets[2] = (A->RowOffsets()[2]-A->RowOffsets()[1])/dim;
+      // toffsets[3] = (A->RowOffsets()[2]-A->RowOffsets()[1])/dim;
+      // toffsets.PartialSum();
+
+      // BlockDiagonalPreconditioner M(toffsets);
+      // M.owns_blocks = 1;
+
+      // HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+      // amg0->SetSystemsOptions(dim);
+
+      // HypreParMatrix & A11 = (HypreParMatrix &)A->GetBlock(1,1);
+      // Array<int> tdofs1(A11.Height()/2);
+      // Array<int> tdofs2(A11.Height()/2);
+      // for (int i = 0; i<tdofs1.Size(); i++)
+      // {
+      //    tdofs1[i] = i;
+      //    tdofs2[i] = tdofs1.Size() + i;
+      // }
+
+      // HypreParMatrix * S1 = GetSubHypreParMatrix(tdofs1,A11);
+      // HypreParMatrix * S2 = GetSubHypreParMatrix(tdofs2,A11);
+      // MUMPSSolver * prec1 = new MUMPSSolver(MPI_COMM_WORLD);
+      // prec1->SetOperator(*S1);
+      // MUMPSSolver * prec2 = new MUMPSSolver(MPI_COMM_WORLD);
+      // prec2->SetOperator(*S2);
+      // HypreSolver * prec1;
+      // HypreSolver * prec2;
+      // if (dim == 2)
+      // {
+      //    // AMS preconditioner for 2D H(div) (trace) space
+      //    prec1 = new HypreAMS(*S1, hatsigma1_fes);
+      //    prec2 = new HypreAMS(*S2, hatsigma1_fes);
+
+      // }
+      // else
+      // {
+      //    // ADS preconditioner for 3D H(div) (trace) space
+      //    prec1 = new HypreADS(*S1, hatsigma1_fes);
+      //    prec2 = new HypreADS(*S2, hatsigma1_fes);
+      // }
+      // M.SetDiagonalBlock(1,prec1);
+      // M.SetDiagonalBlock(2,prec2);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(0);
+      cg.SetPreconditioner(M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+      // Vector residuals;
+
+      double residual = residuals.Norml2();
+
+      double maxresidual = residuals.Max();
+      double globalresidual = residual * residual;
+
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+      u_gf.MakeRef(u_fes,x.GetBlock(0),0);
+
+      int dofs = u_fes->GlobalTrueVSize() + hatsigma_fes->GlobalTrueVSize();
+
+      double u_err = u_gf.ComputeL2Error(uex);
+
+      double rate_err = (it) ? dim*log(err0/u_err)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (it) ? dim*log(res0/globalresidual)/log((
+                                                                   double)dof0/dofs) : 0.0;
+      err0 = u_err;
+      res0 = globalresidual;
+      dof0 = dofs;
+
+      if (myid == 0)
+      {
+         std::ios oldState(nullptr);
+         oldState.copyfmt(std::cout);
+         std::cout << std::right << std::setw(5) << it << " | "
+                   << std::setw(10) <<  dof0 << " | "
+                   << std::setprecision(3)
+                   << std::setw(10) << std::scientific <<  err0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_err << " | "
+                   << std::setprecision(3)
+                   << std::setw(10) << std::scientific <<  res0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_res << " | "
+                   << std::setw(6) << std::fixed << cg.GetNumIterations() << " | "
+                   << std::endl;
+         std::cout.copyfmt(oldState);
+      }
+
+      if (visualization)
+      {
+         const char * keys = (it == 0 && dim == 2) ? "jRcm\n" : nullptr;
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+
+         VisualizeField(u_out,vishost,visport,u_gf,
+                        "Numerical u", 0,0,500,500,keys);
+
+         VisualizeField(ex_out,vishost,visport,exact_gf,
+                        "Exact u", 0,0,500,500,keys);
+      }
+
+      if (it == pref) { break; }
+
+      pmesh.UniformRefinement();
+
+      for (int i=0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      hatsigma1_fes->Update(false);
+      a->Update();
+      exact_gf.Update();
+   }
+
+   delete a;
+   delete v_fec;
+   delete hatsigma_fes;
+   delete hatsigma_fec;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+
+}
+
+
+
+void exact_u(const Vector & X, Vector & u)
+{
+   int size = X.Size();
+   for (int i = 0; i<size; i++)
+   {
+      u(i) = sin(M_PI*X(i));
+   }
+
+}
+
+void exact_gradu(const Vector & X, DenseMatrix & du)
+{
+   du.SetSize(X.Size());
+   for (int i = 0; i<du.Size(); i++)
+   {
+      du(i,i) = M_PI * cos(M_PI * X(i));
+   }
+}
+
+void exact_laplacian_u(const Vector & X, Vector & du)
+{
+   int size = X.Size();
+   du.SetSize(size);
+   for (int i = 0; i<size; i++)
+   {
+      du(i) = -M_PI*M_PI * sin(M_PI*X(i));
+   }
+}
+
+void f_exact(const Vector & X, Vector & f)
+{
+   f.SetSize(X.Size());
+   exact_laplacian_u(X,f);
+   f.Neg();
+}

miniapps/dpg/util/blockcomplexhypremat.cpp

@@ -0,0 +1,120 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "blockcomplexhypremat.hpp"
+
+#ifdef MFEM_USE_MPI
+
+namespace mfem
+{
+
+void ParBlockComplexSystem::FillEssTdofLists(const Array<int> &
+                                             ess_tdof_list)
+{
+   for (int i = 0; i < ess_tdofs.Size(); i++)
+   {
+      delete ess_tdofs[i];
+      ess_tdofs[i] = new Array<int>();
+   }
+   int j;
+   for (int i = 0; i < ess_tdof_list.Size(); i++)
+   {
+      int tdof = ess_tdof_list[i];
+      for (j = 0; j < nblocks; j++)
+      {
+         if (toffsets[j+1] > tdof) { break; }
+      }
+      ess_tdofs[j]->Append(tdof-toffsets[j]);
+   }
+}
+
+ComplexOperator * ParBlockComplexSystem::EliminateBC(const Array<int>
+                                                     ess_tdof_list, Vector &X, Vector & B)
+{
+   FillEssTdofLists(ess_tdof_list);
+   delete op_e_r;
+   delete op_e_i;
+   op_e_r = new BlockOperator(toffsets);
+   op_e_i = new BlockOperator(toffsets);
+   op_e_r->owns_blocks = 1;
+   op_e_i->owns_blocks = 1;
+
+   for (int i = 0; i < nblocks; i++)
+   {
+      for (int j = 0; j < nblocks; j++)
+      {
+         if (op_r->IsZeroBlock(i,j)) { continue; }
+         if (i == j)
+         {
+            auto mat_r = &(HypreParMatrix &)op_r->GetBlock(i,i);
+            op_e_r->SetBlock(i, i, mat_r->EliminateRowsCols(*ess_tdofs[i]));
+            if (!op_i->IsZeroBlock(i,j))
+            {
+               auto mat_i = &(HypreParMatrix &)op_i->GetBlock(i,i);
+               op_e_i->SetBlock(i, i, mat_i->EliminateCols(*ess_tdofs[i]));
+               mat_i->EliminateRows(*ess_tdofs[i]);
+            }
+         }
+         else
+         {
+            auto mat_r = &(HypreParMatrix &)op_r->GetBlock(i,j);
+            op_e_r->SetBlock(i, j, mat_r->EliminateCols(*ess_tdofs[j]));
+            mat_r->EliminateRows(*ess_tdofs[i]);
+            if (!op_i->IsZeroBlock(i,j))
+            {
+               auto mat_i = &(HypreParMatrix &)op_i->GetBlock(i,j);
+               op_e_i->SetBlock(i, j, mat_i->EliminateCols(*ess_tdofs[j]));
+               mat_i->EliminateRows(*ess_tdofs[i]);
+            }
+         }
+      }
+   }
+
+   int n = B.Size()/2;
+   Vector B_r(B, 0, n);
+   Vector B_i(B, n, n);
+
+   Vector X_r(X, 0, n);
+   Vector X_i(X, n, n);
+
+   // eliminate tdof is RHS
+   // B_r -= Ae_r*X_r + Ae_i X_i
+   // B_i -= Ae_i*X_r + Ae_r X_i
+   Vector tmp(B_r.Size());
+   op_e_r->Mult(X_r, tmp); B_r-=tmp;
+   op_e_i->Mult(X_i, tmp); B_r+=tmp;
+
+   op_e_i->Mult(X_r, tmp); B_i-=tmp;
+   op_e_r->Mult(X_i, tmp); B_i-=tmp;
+
+   for (int j = 0; j < nblocks; j++)
+   {
+      if (!ess_tdofs[j]->Size()) { continue; }
+      for (int i = 0; i < ess_tdofs[j]->Size(); i++)
+      {
+         int tdof = (*ess_tdofs[j])[i];
+         int gdof = tdof + toffsets[j];
+         B_r(gdof) = X_r(gdof); // diagonal policy is always one in parallel
+         B_i(gdof) = X_i(gdof); // diagonal policy is always one in parallel
+      }
+   }
+
+   X_r.SetSubVectorComplement(ess_tdof_list, 0.0);
+   X_i.SetSubVectorComplement(ess_tdof_list, 0.0);
+
+   return op;
+}
+
+
+
+} // namespace mfem
+
+#endif

miniapps/dpg/util/blockcomplexhypremat.hpp

@@ -0,0 +1,75 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BLOCKCOMPLEX
+#define MFEM_BLOCKCOMPLEX
+
+#include "mfem.hpp"
+
+#ifdef MFEM_USE_MPI
+
+namespace mfem
+{
+
+class ParBlockComplexSystem
+{
+
+protected:
+
+   /// ess_tdof list for each space
+   Array<Array<int> *> ess_tdofs;
+   Array<int> toffsets;
+   int nblocks;
+
+   /** split ess_tdof_list given in global tdof (for all spaces)
+    to individual lists for each space */
+   void FillEssTdofLists(const Array<int> & ess_tdof_list);
+
+   // Block Prolongation
+   BlockOperator * P = nullptr;
+   // Block Restriction
+   BlockMatrix * R = nullptr;
+
+   ComplexOperator * op = nullptr;
+   BlockOperator * op_r = nullptr;
+   BlockOperator * op_i = nullptr;
+   BlockOperator * op_e_r = nullptr;
+   BlockOperator * op_e_i = nullptr;
+
+public:
+
+   ParBlockComplexSystem() {}
+
+   /// Creates bilinear form associated with FE spaces @a trial_pfes_.
+   ParBlockComplexSystem(ComplexOperator * op_)
+      : op(op_)
+   {
+      op_r = dynamic_cast<BlockOperator *>(&op->real());
+      op_i = dynamic_cast<BlockOperator *>(&op->imag());
+      toffsets = op_r->RowOffsets(); // (assumes square blockoperator)
+      nblocks = toffsets.Size() - 1;
+      ess_tdofs.SetSize(nblocks);
+   }
+
+   ComplexOperator * EliminateBC(const Array<int> ess_tdof_list, Vector &X,
+                                 Vector & B);
+
+   virtual ~ParBlockComplexSystem() {}
+
+
+};
+
+} // namespace mfem
+
+
+#endif // MFEM_USE_MPI
+
+#endif

miniapps/dpg/util/blockstaticcond.cpp

@@ -69,12 +69,14 @@ void BlockStaticCondensation::SetSpaces(Array<FiniteElementSpace*> & fes_)
    nblocks = fes.Size();
    rblocks = 0;
    tr_fes.SetSize(nblocks);
+   tr_fec.SetSize(nblocks);
    mesh = fes[0]->GetMesh();
 
    IsTraceSpace.SetSize(nblocks);
    const FiniteElementCollection * fec;
    for (int i = 0; i < nblocks; i++)
    {
+      tr_fec[i] = nullptr;
       fec = fes[i]->FEColl();
       IsTraceSpace[i] =
          (dynamic_cast<const H1_Trace_FECollection*>(fec) ||
@@ -86,21 +88,24 @@ void BlockStaticCondensation::SetSpaces(Array<FiniteElementSpace*> & fes_)
          pmesh = dynamic_cast<ParMesh *>(mesh);
          tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
                      nullptr : (IsTraceSpace[i]) ? fes[i] :
-                     new ParFiniteElementSpace(pmesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                     new ParFiniteElementSpace(pmesh, tr_fec[i] = fec->GetTraceCollection(),
+                                               fes[i]->GetVDim(),
                                                fes[i]->GetOrdering());
       }
       else
       {
          tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
                      nullptr : (IsTraceSpace[i]) ? fes[i] :
-                     new FiniteElementSpace(mesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                     new FiniteElementSpace(mesh, tr_fec[i] = fec->GetTraceCollection(),
+                                            fes[i]->GetVDim(),
                                             fes[i]->GetOrdering());
       }
 #else
       // skip if it's an L2 space (no trace space to construct)
       tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
                   nullptr : (IsTraceSpace[i]) ? fes[i] :
-                  new FiniteElementSpace(mesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                  new FiniteElementSpace(mesh, tr_fec[i] = fec->GetTraceCollection(),
+                                         fes[i]->GetVDim(),
                                          fes[i]->GetOrdering());
 #endif
       if (tr_fes[i]) { rblocks++; }
@@ -213,39 +218,43 @@ void BlockStaticCondensation::GetReducedElementIndicesAndOffsets(int el,
    {
       int td = 0;
       int ndof;
+      int vdim = fes[i]->GetVDim();
       // if it's an L2 space (bubbles)
       if (!tr_fes[i])
       {
-         ndof = fes[i]->GetVDim()*fes[i]->GetFE(el)->GetDof();
+         ndof = fes[i]->GetFE(el)->GetDof();
          td = 0;
       }
       else if (IsTraceSpace[i])
       {
          for (int iface = 0; iface < numfaces; iface++)
          {
-            td += fes[i]->GetVDim()*fes[i]->GetFaceElement(faces[iface])->GetDof();
+            td += fes[i]->GetFaceElement(faces[iface])->GetDof();
          }
          ndof = td;
       }
       else
       {
          Array<int> trace_dofs;
-         ndof = fes[i]->GetVDim()*fes[i]->GetFE(el)->GetDof();
-         tr_fes[i]->GetElementVDofs(el, trace_dofs);
+         ndof = fes[i]->GetFE(el)->GetDof();
+         tr_fes[i]->GetElementDofs(el, trace_dofs);
          td = trace_dofs.Size(); // number of trace dofs
       }
-      offsets[i+1] = td;
-      tr_dofs.SetSize(td);
-      int_dofs.SetSize(ndof - td);
-      for (int j = 0; j<td; j++)
+      offsets[i+1] = td*vdim;
+      tr_dofs.SetSize(td*vdim);
+      int_dofs.SetSize(vdim*(ndof - td));
+      for (int k = 0; k<vdim; k++)
       {
-         tr_dofs[j] = skip + j;
+         for (int j = 0; j<td; j++)
+         {
+            tr_dofs[j+k*td] = skip + j;
+         }
+         for (int j = 0; j<ndof-td; j++)
+         {
+            int_dofs[j+k*(ndof-td)] = skip + td + j;
+         }
+         skip+=ndof;
       }
-      for (int j = 0; j<ndof-td; j++)
-      {
-         int_dofs[j] = skip + td + j;
-      }
-      skip+=ndof;
 
       trace_ldofs.Append(tr_dofs);
       interior_ldofs.Append(int_dofs);
@@ -976,6 +985,15 @@ BlockStaticCondensation::~BlockStaticCondensation()
       delete lmat[i]; lmat[i] = nullptr;
       delete lvec[i]; lvec[i] = nullptr;
    }
+
+   for (int i = 0; i<tr_fes.Size(); i++)
+   {
+      if (tr_fec[i])
+      {
+         delete tr_fes[i];
+         delete tr_fec[i];
+      }
+   }
 }
 
 } // namespace mfem

miniapps/dpg/util/blockstaticcond.hpp

@@ -40,6 +40,7 @@ class BlockStaticCondensation
    // New set of "reduced" Finite Element Spaces
    // (after static condensation)
    Array<FiniteElementSpace *> tr_fes;
+   Array<FiniteElementCollection *> tr_fec;
 
    Array<int> dof_offsets;
    Array<int> tdof_offsets;
@@ -185,6 +186,11 @@ public:
        full linear system, compute the solution of the full system 'sol'. */
    void ComputeSolution(const Vector &sc_sol, Vector &sol) const;
 
+   void GetTraceFESpaces(Array<FiniteElementSpace *> & trace_fes) const
+   {
+      trace_fes = tr_fes;
+   }
+
 };
 
 }

miniapps/dpg/util/complexblockform.cpp

@@ -0,0 +1,577 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "complexblockform.hpp"
+
+namespace mfem
+{
+
+void ComplexBlockForm::Init()
+{
+   integs_r.SetSize(fes.Size(), fes.Size());
+   integs_i.SetSize(fes.Size(), fes.Size());
+   for (int i = 0; i < integs_r.NumRows(); i++)
+   {
+      for (int j = 0; j < integs_r.NumCols(); j++)
+      {
+         integs_r(i,j) = new Array<BilinearFormIntegrator * >();
+         integs_i(i,j) = new Array<BilinearFormIntegrator * >();
+      }
+   }
+
+   ComputeOffsets();
+
+   mat_r = mat_e_r = NULL;
+   mat_i = mat_e_i = NULL;
+   diag_policy = mfem::Operator::DIAG_ONE;
+   height = dof_offsets[nblocks];
+   width = height;
+
+   initialized = true;
+
+}
+
+void ComplexBlockForm::ComputeOffsets()
+{
+   dof_offsets.SetSize(nblocks+1);
+   tdof_offsets.SetSize(nblocks+1);
+   dof_offsets[0] = 0;
+   tdof_offsets[0] = 0;
+   for (int i =0; i<nblocks; i++)
+   {
+      dof_offsets[i+1] = fes[i]->GetVSize();
+      tdof_offsets[i+1] = fes[i]->GetTrueVSize();
+   }
+   dof_offsets.PartialSum();
+   tdof_offsets.PartialSum();
+}
+
+// Allocate SparseMatrix and RHS
+void ComplexBlockForm::AllocMat()
+{
+   mat_r = new BlockMatrix(dof_offsets);
+   mat_r->owns_blocks = 1;
+   mat_i = new BlockMatrix(dof_offsets);
+   mat_i->owns_blocks = 1;
+
+   for (int i = 0; i < mat_r->NumRowBlocks(); i++)
+   {
+      int h = dof_offsets[i+1] - dof_offsets[i];
+      for (int j = 0; j < mat_r->NumColBlocks(); j++)
+      {
+         int w = dof_offsets[j+1] - dof_offsets[j];
+         mat_r->SetBlock(i,j,new SparseMatrix(h, w));
+         mat_i->SetBlock(i,j,new SparseMatrix(h, w));
+      }
+   }
+}
+
+void ComplexBlockForm::Finalize(int skip_zeros)
+{
+   if (mat_r)
+   {
+      mat_r->Finalize(skip_zeros);
+      mat_i->Finalize(skip_zeros);
+   }
+   if (mat_e_r)
+   {
+      mat_e_r->Finalize(skip_zeros);
+      mat_e_i->Finalize(skip_zeros);
+   }
+}
+
+/// Adds new Domain BF Integrator. Assumes ownership of @a bfi.
+void ComplexBlockForm::AddDomainIntegrator(
+   BilinearFormIntegrator *bfi_r,
+   BilinearFormIntegrator *bfi_i,
+   int n, int m)
+{
+   MFEM_VERIFY(n < fes.Size(),
+               "ComplexBlockFrom::AddDomainIntegrator: fespace row index out of bounds");
+   MFEM_VERIFY(m < fes.Size(),
+               "ComplexBlockFrom::AddDomainIntegrator: fespace col index out of bounds");
+   if (bfi_r) { integs_r(n,m)->Append(bfi_r); }
+   if (bfi_i) { integs_i(n,m)->Append(bfi_i); }
+}
+
+void ComplexBlockForm::BuildProlongation()
+{
+   P = new BlockMatrix(dof_offsets, tdof_offsets);
+   R = new BlockMatrix(tdof_offsets, dof_offsets);
+   P->owns_blocks = 0;
+   R->owns_blocks = 0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      const SparseMatrix *P_ = fes[i]->GetConformingProlongation();
+      if (P_)
+      {
+         const SparseMatrix *R_ = fes[i]->GetRestrictionMatrix();
+         P->SetBlock(i, i, const_cast<SparseMatrix*>(P_));
+         R->SetBlock(i, i, const_cast<SparseMatrix*>(R_));
+      }
+   }
+}
+
+void ComplexBlockForm::ConformingAssemble()
+{
+   Finalize(0);
+   if (!P) { BuildProlongation(); }
+
+   BlockMatrix * Pt = Transpose(*P);
+   BlockMatrix * PtA_r = mfem::Mult(*Pt, *mat_r);
+   BlockMatrix * PtA_i = mfem::Mult(*Pt, *mat_i);
+   mat_r->owns_blocks = 0;
+   mat_i->owns_blocks = 0;
+   for (int i = 0; i < nblocks; i++)
+   {
+      for (int j = 0; j < nblocks; j++)
+      {
+         SparseMatrix * tmp_r = &mat_r->GetBlock(i,j);
+         SparseMatrix * tmp_i = &mat_i->GetBlock(i,j);
+         if (Pt->IsZeroBlock(i, i))
+         {
+            PtA_r->SetBlock(i, j, tmp_r);
+            PtA_i->SetBlock(i, j, tmp_i);
+         }
+         else
+         {
+            delete tmp_r;
+            delete tmp_i;
+         }
+      }
+   }
+   delete mat_r;
+   delete mat_i;
+   if (mat_e_r)
+   {
+      BlockMatrix *PtAe_r = mfem::Mult(*Pt, *mat_e_r);
+      BlockMatrix *PtAe_i = mfem::Mult(*Pt, *mat_e_i);
+      mat_e_r->owns_blocks = 0;
+      mat_e_i->owns_blocks = 0;
+      for (int i = 0; i<nblocks; i++)
+      {
+         for (int j = 0; j<nblocks; j++)
+         {
+            SparseMatrix * tmp_r = &mat_e_r->GetBlock(i, j);
+            SparseMatrix * tmp_i = &mat_e_i->GetBlock(i, j);
+            if (Pt->IsZeroBlock(i, i))
+            {
+               PtAe_r->SetBlock(i, j, tmp_r);
+               PtAe_i->SetBlock(i, j, tmp_i);
+            }
+            else
+            {
+               delete tmp_r;
+               delete tmp_i;
+            }
+         }
+      }
+      delete mat_e_r;
+      delete mat_e_i;
+      mat_e_r = PtAe_r;
+      mat_e_i = PtAe_i;
+   }
+   delete Pt;
+
+   mat_r = mfem::Mult(*PtA_r, *P);
+   mat_i = mfem::Mult(*PtA_i, *P);
+
+   PtA_r->owns_blocks = 0;
+   PtA_i->owns_blocks = 0;
+   for (int i = 0; i < nblocks; i++)
+   {
+      for (int j = 0; j < nblocks; j++)
+      {
+         SparseMatrix * tmp_r = &PtA_r->GetBlock(j, i);
+         SparseMatrix * tmp_i = &PtA_i->GetBlock(j, i);
+         if (P->IsZeroBlock(i, i))
+         {
+            mat_r->SetBlock(j, i, tmp_r);
+            mat_i->SetBlock(j, i, tmp_i);
+         }
+         else
+         {
+            delete tmp_r;
+            delete tmp_i;
+         }
+      }
+   }
+   delete PtA_r;
+   delete PtA_i;
+
+   if (mat_e_r)
+   {
+      BlockMatrix *PtAeP_r = mfem::Mult(*mat_e_r, *P);
+      BlockMatrix *PtAeP_i = mfem::Mult(*mat_e_i, *P);
+      mat_e_r->owns_blocks = 0;
+      mat_e_i->owns_blocks = 0;
+      for (int i = 0; i < nblocks; i++)
+      {
+         for (int j = 0; j < nblocks; j++)
+         {
+            SparseMatrix * tmp_r = &mat_e_r->GetBlock(j, i);
+            SparseMatrix * tmp_i = &mat_e_i->GetBlock(j, i);
+            if (P->IsZeroBlock(i, i))
+            {
+               PtAeP_r->SetBlock(j, i, tmp_r);
+               PtAeP_i->SetBlock(j, i, tmp_i);
+            }
+            else
+            {
+               delete tmp_r;
+               delete tmp_i;
+            }
+         }
+      }
+
+      delete mat_e_r;
+      delete mat_e_i;
+      mat_e_r = PtAeP_r;
+      mat_e_i = PtAeP_i;
+   }
+   height = 2*mat_r->Height();
+   width = 2*mat_r->Width();
+}
+
+/// Assembles the form i.e. sums over all domain integrators.
+void ComplexBlockForm::Assemble(int skip_zeros)
+{
+   ElementTransformation *eltrans;
+   Array<int> faces, ori;
+
+   DofTransformation doftrans_i, doftrans_j;
+   if (mat_r == NULL)
+   {
+      AllocMat();
+   }
+
+   // loop through the elements
+   DenseMatrix A_r, Ae_r;
+   DenseMatrix A_i, Ae_i;
+   Array<int> vdofs;
+
+   // loop through elements
+   for (int iel = 0; iel < mesh -> GetNE(); iel++)
+   {
+      Array<int> offs(fes.Size()+1); offs = 0;
+
+      eltrans = mesh->GetElementTransformation(iel);
+
+      for (int j = 0; j < fes.Size(); j++)
+      {
+         offs[j+1] = fes[j]->GetVDim() * fes[j]->GetFE(iel)->GetDof();
+      }
+      offs.PartialSum();
+
+      A_r.SetSize(offs.Last(),offs.Last()); A_r = 0.0;
+      A_i.SetSize(offs.Last(),offs.Last()); A_i = 0.0;
+
+      for (int j = 0; j < fes.Size(); j++)
+      {
+         const FiniteElement & fe_j = *fes[j]->GetFE(iel);
+         for (int i = 0; i < fes.Size(); i++)
+         {
+            const FiniteElement & fe_i = *fes[i]->GetFE(iel);
+            // real integrators
+            for (int k = 0; k < integs_r(i,j)->Size(); k++)
+            {
+               if (i == j)
+               {
+                  (*integs_r(i,j))[k]->AssembleElementMatrix(fe_i,*eltrans,Ae_r);
+               }
+               else
+               {
+                  (*integs_r(i,j))[k]->AssembleElementMatrix2(fe_i,fe_j,*eltrans,Ae_r);
+               }
+               A_r.AddSubMatrix(offs[j], offs[i], Ae_r);
+            }
+            // imag integrators
+            for (int k = 0; k < integs_i(i,j)->Size(); k++)
+            {
+               if (i == j)
+               {
+                  (*integs_i(i,j))[k]->AssembleElementMatrix(fe_i,*eltrans,Ae_i);
+               }
+               else
+               {
+                  (*integs_i(i,j))[k]->AssembleElementMatrix2(fe_i, fe_j, *eltrans, Ae_i);
+               }
+               A_i.AddSubMatrix(offs[j], offs[i], Ae_i);
+            }
+         }
+      }
+
+      ComplexDenseMatrix A(&A_r, &A_i, false, false);
+
+      // Assembly
+      for (int i = 0; i<fes.Size(); i++)
+      {
+         Array<int> vdofs_i;
+         doftrans_i.SetDofTransformation(nullptr);
+         fes[i]->GetElementVDofs(iel, vdofs_i,doftrans_i);
+         for (int j = 0; j < fes.Size(); j++)
+         {
+            Array<int> vdofs_j;
+            doftrans_j.SetDofTransformation(nullptr);
+            fes[j]->GetElementVDofs(iel, vdofs_j,doftrans_j);
+
+            A.real().GetSubMatrix(offs[i],offs[i+1],
+                                  offs[j],offs[j+1], Ae_r);
+            A.imag().GetSubMatrix(offs[i],offs[i+1],
+                                  offs[j],offs[j+1], Ae_i);
+            TransformDual(doftrans_i, doftrans_j, Ae_r);
+            TransformDual(doftrans_i, doftrans_j, Ae_i);
+            mat_r->GetBlock(i,j).AddSubMatrix(vdofs_i,vdofs_j, Ae_r);
+            mat_i->GetBlock(i,j).AddSubMatrix(vdofs_i,vdofs_j, Ae_i);
+         }
+      }
+   } // end of loop through elements
+}
+
+void ComplexBlockForm::FormLinearSystem(const Array<int>
+                                        &ess_tdof_list,
+                                        Vector &x,
+                                        Vector &b,
+                                        OperatorHandle &A,
+                                        Vector &X,
+                                        Vector &B,
+                                        int copy_interior)
+{
+   FormSystemMatrix(ess_tdof_list, A);
+
+   Vector x_r(x, 0, x.Size()/2);
+   Vector x_i(x, x.Size()/2, x.Size()/2);
+   Vector b_r(b, 0, b.Size()/2);
+   Vector b_i(b, b.Size()/2, b.Size()/2);
+   if (!P)
+   {
+      EliminateVDofsInRHS(ess_tdof_list, x_r,x_i, b_r, b_i);
+      if (!copy_interior)
+      {
+         x_r.SetSubVectorComplement(ess_tdof_list, 0.0);
+         x_i.SetSubVectorComplement(ess_tdof_list, 0.0);
+      }
+      X.MakeRef(x, 0, x.Size());
+      B.MakeRef(b, 0, b.Size());
+   }
+   else // non conforming space
+   {
+      B.SetSize(2*P->Width());
+      Vector B_r(B, 0, P->Width());
+      Vector B_i(B, P->Width(),P->Width());
+
+      P->MultTranspose(b_r, B_r);
+      P->MultTranspose(b_i, B_i);
+      Vector tmp_r,tmp_i;
+      for (int i = 0; i<nblocks; i++)
+      {
+         if (P->IsZeroBlock(i,i))
+         {
+            int offset = tdof_offsets[i];
+            tmp_r.MakeRef(b_r, offset,tdof_offsets[i+1]-tdof_offsets[i]);
+            tmp_i.MakeRef(b_i, offset,tdof_offsets[i+1]-tdof_offsets[i]);
+            B_r.SetVector(tmp_r,offset);
+            B_i.SetVector(tmp_i,offset);
+         }
+      }
+
+      X.SetSize(2*R->Height());
+      Vector X_r(X, 0, X.Size()/2);
+      Vector X_i(X, X.Size()/2, X.Size()/2);
+
+      R->Mult(x_r, X_r);
+      R->Mult(x_i, X_i);
+      for (int i = 0; i<nblocks; i++)
+      {
+         if (R->IsZeroBlock(i,i))
+         {
+            int offset = tdof_offsets[i];
+            tmp_r.MakeRef(x_r, offset, tdof_offsets[i+1]-tdof_offsets[i]);
+            tmp_i.MakeRef(x_i, offset, tdof_offsets[i+1]-tdof_offsets[i]);
+            X_r.SetVector(tmp_r,offset);
+            X_i.SetVector(tmp_i,offset);
+         }
+      }
+
+      EliminateVDofsInRHS(ess_tdof_list, X_r, X_i, B_r, B_i);
+      if (!copy_interior)
+      {
+         X_r.SetSubVectorComplement(ess_tdof_list, 0.0);
+         X_i.SetSubVectorComplement(ess_tdof_list, 0.0);
+      }
+   }
+}
+
+void ComplexBlockForm::FormSystemMatrix(const Array<int>
+                                        &ess_tdof_list,
+                                        OperatorHandle &A)
+{
+   if (!mat_e_r)
+   {
+      bool conforming = true;
+      for (int i = 0; i<nblocks; i++)
+      {
+         const SparseMatrix *P_ = fes[i]->GetConformingProlongation();
+         if (P_)
+         {
+            conforming = false;
+            break;
+         }
+      }
+      if (!conforming) { ConformingAssemble(); }
+      const int remove_zeros = 0;
+      EliminateVDofs(ess_tdof_list, diag_policy);
+      Finalize(remove_zeros);
+   }
+   mat = new ComplexOperator(mat_r,mat_i,false,false);
+   A.Reset(mat,false);
+}
+
+void ComplexBlockForm::EliminateVDofsInRHS(
+   const Array<int> &vdofs, const Vector &x_r, const Vector & x_i,
+   Vector &b_r, Vector & b_i)
+{
+   mat_e_r->AddMult(x_r,b_r,-1.);
+   mat_e_i->AddMult(x_i,b_r,1.);
+   mat_e_r->AddMult(x_i,b_i,-1.);
+   mat_e_i->AddMult(x_r,b_i,-1.);
+   mat_r->PartMult(vdofs,x_r,b_r);
+   mat_r->PartMult(vdofs,x_i,b_i);
+}
+
+void ComplexBlockForm::EliminateVDofs(const Array<int> &vdofs,
+                                      Operator::DiagonalPolicy dpolicy)
+{
+   if (mat_e_r == NULL)
+   {
+      Array<int> offsets;
+
+      offsets.MakeRef( (P) ? tdof_offsets : dof_offsets);
+
+      mat_e_r = new BlockMatrix(offsets);
+      mat_e_r->owns_blocks = 1;
+      mat_e_i = new BlockMatrix(offsets);
+      mat_e_i->owns_blocks = 1;
+      for (int i = 0; i < mat_e_r->NumRowBlocks(); i++)
+      {
+         int h = offsets[i+1] - offsets[i];
+         for (int j = 0; j < mat_e_r->NumColBlocks(); j++)
+         {
+            int w = offsets[j+1] - offsets[j];
+            mat_e_r->SetBlock(i, j, new SparseMatrix(h, w));
+            mat_e_i->SetBlock(i, j, new SparseMatrix(h, w));
+         }
+      }
+   }
+   mat_r->EliminateRowCols(vdofs, mat_e_r, diag_policy);
+   mat_i->EliminateRowCols(vdofs, mat_e_i, Operator::DiagonalPolicy::DIAG_ZERO);
+}
+
+void ComplexBlockForm::RecoverFEMSolution(const Vector &X, Vector &x)
+{
+   if (!P)
+   {
+      x.SyncMemory(X);
+   }
+   else
+   {
+      x.SetSize(2*P->Height());
+      Vector X_r(const_cast<Vector &>(X), 0, X.Size()/2);
+      Vector X_i(const_cast<Vector &>(X), X.Size()/2, X.Size()/2);
+
+      Vector x_r(x, 0, x.Size()/2);
+      Vector x_i(x, x.Size()/2, x.Size()/2);
+
+      P->Mult(X_r, x_r);
+      P->Mult(X_i, x_i);
+
+      Vector tmp_r, tmp_i;
+      for (int i = 0; i<nblocks; i++)
+      {
+         if (P->IsZeroBlock(i,i))
+         {
+            int offset = tdof_offsets[i];
+            tmp_r.MakeRef(X_r, offset, tdof_offsets[i+1]-tdof_offsets[i]);
+            tmp_i.MakeRef(X_i, offset, tdof_offsets[i+1]-tdof_offsets[i]);
+            x_r.SetVector(tmp_r,offset);
+            x_i.SetVector(tmp_i,offset);
+         }
+      }
+   }
+}
+
+void ComplexBlockForm::ReleaseInitMemory()
+{
+   if (initialized)
+   {
+      for (int k = 0; k < integs_r.NumRows(); k++)
+      {
+         for (int l = 0; l < integs_r.NumCols(); l++)
+         {
+            for (int i = 0; i < integs_r(k,l)->Size(); i++)
+            {
+               delete (*integs_r(k,l))[i];
+            }
+            delete integs_r(k,l);
+            for (int i = 0; i < integs_i(k,l)->Size(); i++)
+            {
+               delete (*integs_i(k,l))[i];
+            }
+            delete integs_i(k,l);
+         }
+      }
+      integs_r.DeleteAll();
+      integs_i.DeleteAll();
+   }
+}
+
+void ComplexBlockForm::Update()
+{
+   delete mat_e_r; mat_e_r = nullptr;
+   delete mat_e_i; mat_e_i = nullptr;
+   delete mat; mat = nullptr;
+   delete mat_r; mat_r = nullptr;
+   delete mat_i; mat_i = nullptr;
+
+   if (P)
+   {
+      delete P; P = nullptr;
+      delete R; R = nullptr;
+   }
+
+   ComputeOffsets();
+
+   diag_policy = mfem::Operator::DIAG_ONE;
+   height = dof_offsets[nblocks];
+   width = height;
+
+   initialized = true;
+
+}
+
+ComplexBlockForm::~ComplexBlockForm()
+{
+   delete mat_e_r; mat_e_r = nullptr;
+   delete mat_e_i; mat_e_i = nullptr;
+   delete mat; mat = nullptr;
+   delete mat_r; mat_r = nullptr;
+   delete mat_i; mat_i = nullptr;
+
+   ReleaseInitMemory();
+
+   if (P)
+   {
+      delete P;
+      delete R;
+   }
+}
+
+} // namespace mfem

miniapps/dpg/util/complexblockform.hpp

@@ -0,0 +1,199 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_COMPLEX_BLOCKFORM
+#define MFEM_COMPLEX_BLOCKFORM
+
+#include "mfem.hpp"
+#include "complexstaticcond.hpp"
+
+namespace mfem
+{
+
+class ComplexBlockForm
+{
+
+protected:
+
+   bool initialized = false;
+
+   Mesh * mesh = nullptr;
+   int height, width;
+   int nblocks;
+   Array<int> dof_offsets;
+   Array<int> tdof_offsets;
+
+   /// Block matrix $ M $ to be associated with the real/imag Block bilinear form. Owned.
+   BlockMatrix *mat_r = nullptr;
+   BlockMatrix *mat_i = nullptr;
+   ComplexOperator * mat = nullptr;
+
+   /** @brief Block Matrix $ M_e $ used to store the eliminations
+        from the b.c.  Owned.
+       $ M + M_e = M_{original} $ */
+   BlockMatrix *mat_e_r = nullptr;
+   BlockMatrix *mat_e_i = nullptr;
+
+   /// FE spaces
+   Array<FiniteElementSpace * > fes;
+
+   /// Set of Trial Integrators to be applied for matrix A
+   Array2D<Array<BilinearFormIntegrator * > * > integs_r;
+   Array2D<Array<BilinearFormIntegrator * > * > integs_i;
+
+   /// Block Prolongation
+   BlockMatrix * P = nullptr;
+   /// Block Restriction
+   BlockMatrix * R = nullptr;
+
+   mfem::Operator::DiagonalPolicy diag_policy;
+
+   void Init();
+   void ReleaseInitMemory();
+
+   // Allocate appropriate SparseMatrix and assign it to mat
+   void AllocMat();
+
+   void ConformingAssemble();
+
+   void ComputeOffsets();
+
+   virtual void BuildProlongation();
+
+private:
+
+public:
+
+   ComplexBlockForm()
+   {
+      height = 0;
+      width = 0;
+   }
+
+   /// Creates bilinear form associated with FE spaces @a fes_.
+   ComplexBlockForm(Array<FiniteElementSpace* > & fes_)
+   {
+      SetSpaces(fes_);
+   }
+
+   void SetSpaces(Array<FiniteElementSpace* > & fes_)
+   {
+      fes = fes_;
+      nblocks = fes.Size();
+      mesh = fes[0]->GetMesh();
+      Init();
+   }
+
+   // Get the size of the bilinear form of the ComplexBlockForm
+   int Size() const { return height; }
+
+   // Pre-allocate the internal real and imag BlockMatrix before assembly.
+   void AllocateMatrix() { if (mat_r == nullptr) { AllocMat(); } }
+
+   ///  Finalizes the matrix initialization.
+   void Finalize(int skip_zeros = 1);
+
+   /// Returns a reference to the BlockMatrix:  $ M_r $
+   BlockMatrix &BlockMat_r()
+   {
+      MFEM_VERIFY(mat_r, "mat_r is NULL and can't be dereferenced");
+      return *mat_r;
+   }
+   /// Returns a reference to the BlockMatrix:  $ M_i $
+   BlockMatrix &BlockMat_i()
+   {
+      MFEM_VERIFY(mat_i, "mat_i is NULL and can't be dereferenced");
+      return *mat_i;
+   }
+
+   /// Returns a reference to the BlockMatrix of eliminated b.c.: $ M_e_r $
+   BlockMatrix &BlockMatElim_r()
+   {
+      MFEM_VERIFY(mat_e_r, "mat_e is NULL and can't be dereferenced");
+      return *mat_e_r;
+   }
+
+   /// Returns a reference to the BlockMatrix of eliminated b.c.: $ M_e_i $
+   BlockMatrix &BlockMatElim_i()
+   {
+      MFEM_VERIFY(mat_e_i, "mat_e is NULL and can't be dereferenced");
+      return *mat_e_i;
+   }
+
+   /** Adds new Trial Integrator. Assumes ownership of @a bfi_r and @a bfi_i.
+       @a n and @a m correspond to the trial FESpace and test FEColl
+       respectively */
+   void AddDomainIntegrator(BilinearFormIntegrator *bfi_r,
+                            BilinearFormIntegrator *bfi_i,
+                            int n, int m);
+
+   /// Assembles the form i.e. sums over all integrators.
+   void Assemble(int skip_zeros = 1);
+
+   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
+                                 Vector &x, Vector &b, OperatorHandle & A,
+                                 Vector &X, Vector &B, int copy_interior = 0);
+
+   template <typename OpType>
+   void FormLinearSystem(const Array<int> &ess_tdof_list,
+                         Vector &x, Vector &b, OpType &A,
+                         Vector &X, Vector &B, int copy_interior = 0)
+   {
+      OperatorHandle Ah;
+      FormLinearSystem(ess_tdof_list, x, b, Ah, X, B, copy_interior);
+      OpType *A_ptr = Ah.Is<OpType>();
+      MFEM_VERIFY(A_ptr, "invalid OpType used");
+      A.MakeRef(*A_ptr);
+   }
+
+   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                 OperatorHandle &A);
+
+   template <typename OpType>
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OpType &A)
+   {
+      OperatorHandle Ah;
+      FormSystemMatrix(ess_tdof_list, Ah);
+      OpType *A_ptr = Ah.Is<OpType>();
+      MFEM_VERIFY(A_ptr, "invalid OpType used");
+      A.MakeRef(*A_ptr);
+   }
+
+   void EliminateVDofs(const Array<int> &vdofs,
+                       Operator::DiagonalPolicy dpolicy = Operator::DIAG_ONE);
+
+   void EliminateVDofsInRHS(const Array<int> &vdofs,
+                            const Vector &x_r, const Vector & x_i,
+                            Vector &b_r, Vector & b_i);
+
+   virtual void RecoverFEMSolution(const Vector &X, Vector &x);
+
+   /// Sets diagonal policy used upon construction of the linear system.
+   /** Policies include:
+       - DIAG_ZERO (Set the diagonal values to zero)
+       - DIAG_ONE  (Set the diagonal values to one)
+       - DIAG_KEEP (Keep the diagonal values)
+   */
+   void SetDiagonalPolicy(Operator::DiagonalPolicy policy)
+   {
+      diag_policy = policy;
+   }
+
+   virtual void Update();
+
+   /// Destroys bilinear form.
+   virtual ~ComplexBlockForm();
+
+};
+
+} // namespace mfem
+
+#endif

miniapps/dpg/util/complexstaticcond.cpp

@@ -71,12 +71,14 @@ void ComplexBlockStaticCondensation::SetSpaces(Array<FiniteElementSpace*> &
    nblocks = fes.Size();
    rblocks = 0;
    tr_fes.SetSize(nblocks);
+   tr_fec.SetSize(nblocks);
    mesh = fes[0]->GetMesh();
 
    IsTraceSpace.SetSize(nblocks);
    const FiniteElementCollection * fec;
    for (int i = 0; i < nblocks; i++)
    {
+      tr_fec[i] = nullptr;
       fec = fes[i]->FEColl();
       IsTraceSpace[i] =
          (dynamic_cast<const H1_Trace_FECollection*>(fec) ||
@@ -88,21 +90,24 @@ void ComplexBlockStaticCondensation::SetSpaces(Array<FiniteElementSpace*> &
          pmesh = dynamic_cast<ParMesh *>(mesh);
          tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
                      nullptr : (IsTraceSpace[i]) ? fes[i] :
-                     new ParFiniteElementSpace(pmesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                     new ParFiniteElementSpace(pmesh, tr_fec[i] = fec->GetTraceCollection(),
+                                               fes[i]->GetVDim(),
                                                fes[i]->GetOrdering());
       }
       else
       {
          tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
                      nullptr : (IsTraceSpace[i]) ? fes[i] :
-                     new FiniteElementSpace(mesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                     new FiniteElementSpace(mesh, tr_fec[i] = fec->GetTraceCollection(),
+                                            fes[i]->GetVDim(),
                                             fes[i]->GetOrdering());
       }
 #else
       // skip if it's an L2 space (no trace space to construct)
       tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
                   nullptr : (IsTraceSpace[i]) ? fes[i] :
-                  new FiniteElementSpace(mesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                  new FiniteElementSpace(mesh, tr_fec[i] = fec->GetTraceCollection(),
+                                         fes[i]->GetVDim(),
                                          fes[i]->GetOrdering());
 #endif
       if (tr_fes[i]) { rblocks++; }
@@ -220,40 +225,43 @@ void ComplexBlockStaticCondensation::GetReduceElementIndicesAndOffsets(int el,
    {
       int td = 0;
       int ndof;
+      int vdim = fes[i]->GetVDim();
       // if it's an L2 space (bubbles)
       if (!tr_fes[i])
       {
-         ndof = fes[i]->GetVDim()*fes[i]->GetFE(el)->GetDof();
+         ndof = fes[i]->GetFE(el)->GetDof();
          td = 0;
       }
       else if (IsTraceSpace[i])
       {
          for (int iface = 0; iface < numfaces; iface++)
          {
-            td += fes[i]->GetVDim()*fes[i]->GetFaceElement(faces[iface])->GetDof();
+            td += fes[i]->GetFaceElement(faces[iface])->GetDof();
          }
          ndof = td;
       }
       else
       {
          Array<int> trace_dofs;
-         ndof = fes[i]->GetVDim()*fes[i]->GetFE(el)->GetDof();
-         tr_fes[i]->GetElementVDofs(el, trace_dofs);
+         ndof = fes[i]->GetFE(el)->GetDof();
+         tr_fes[i]->GetElementDofs(el, trace_dofs);
          td = trace_dofs.Size(); // number of trace dofs
       }
-      offsets[i+1] = td;
-      tr_dofs.SetSize(td);
-      int_dofs.SetSize(ndof - td);
-      for (int j = 0; j<td; j++)
+      offsets[i+1] = td*vdim;
+      tr_dofs.SetSize(td*vdim);
+      int_dofs.SetSize(vdim*(ndof - td));
+      for (int k=0; k<vdim; k++)
       {
-         tr_dofs[j] = skip + j;
+         for (int j = 0; j<td; j++)
+         {
+            tr_dofs[j+k*td] = skip + j;
+         }
+         for (int j = 0; j<ndof-td; j++)
+         {
+            int_dofs[j+k*(ndof-td)] = skip + td + j;
+         }
+         skip+=ndof;
       }
-      for (int j = 0; j<ndof-td; j++)
-      {
-         int_dofs[j] = skip + td + j;
-      }
-      skip+=ndof;
-
       trace_ldofs.Append(tr_dofs);
       interior_ldofs.Append(int_dofs);
    }
@@ -1157,6 +1165,16 @@ ComplexBlockStaticCondensation::~ComplexBlockStaticCondensation()
       delete lmat[i]; lmat[i] = nullptr;
       delete lvec[i]; lvec[i] = nullptr;
    }
+
+   for (int i = 0; i<tr_fes.Size(); i++)
+   {
+      if (tr_fec[i])
+      {
+         delete tr_fes[i];
+         delete tr_fec[i];
+      }
+   }
+
 }
 
 }

miniapps/dpg/util/complexstaticcond.hpp

@@ -36,6 +36,7 @@ class ComplexBlockStaticCondensation
    // New set of "reduced" Finite Element Spaces
    // (after static condensation)
    Array<FiniteElementSpace *> tr_fes;
+   Array<FiniteElementCollection *> tr_fec;
 
    Array<int> dof_offsets;
    Array<int> tdof_offsets;
@@ -214,6 +215,11 @@ public:
        full linear system, compute the solution of the full system 'sol'. */
    void ComputeSolution(const Vector &sc_sol, Vector &sol) const;
 
+   void GetTraceFESpaces(Array<FiniteElementSpace *> & trace_fes) const
+   {
+      trace_fes = tr_fes;
+   }
+
 };
 
 }

miniapps/dpg/util/complexweakform.cpp

@@ -314,9 +314,48 @@ void ComplexDPGWeakForm::ConformingAssemble()
    width = 2*mat_r->Width();
 }
 
+void ComplexDPGWeakForm::SetIntegrationRules()
+{
+   if (trial_ir)
+   {
+      for (int i = 0; i < trial_integs_r.NumRows(); i++)
+      {
+         for (int j = 0; j < trial_integs_r.NumCols(); j++)
+         {
+            for (int k = 0; k < trial_integs_r(i,j)->Size(); k++)
+            {
+               (*trial_integs_r(i,j))[k]->SetIntRule(trial_ir);
+            }
+            for (int k = 0; k < trial_integs_i(i,j)->Size(); k++)
+            {
+               (*trial_integs_i(i,j))[k]->SetIntRule(trial_ir);
+            }
+         }
+      }
+   }
+   if (test_ir)
+   {
+      for (int i = 0; i < test_integs_r.NumRows(); i++)
+      {
+         for (int j = 0; j < test_integs_r.NumCols(); j++)
+         {
+            for (int k = 0; k < test_integs_r(i,j)->Size(); k++)
+            {
+               (*test_integs_r(i,j))[k]->SetIntRule(test_ir);
+            }
+            for (int k = 0; k < test_integs_i(i,j)->Size(); k++)
+            {
+               (*test_integs_i(i,j))[k]->SetIntRule(test_ir);
+            }
+         }
+      }
+   }
+}
+
 /// Assembles the form i.e. sums over all domain integrators.
 void ComplexDPGWeakForm::Assemble(int skip_zeros)
 {
+   this->SetIntegrationRules();
    ElementTransformation *eltrans;
    Array<int> faces, ori;
 
@@ -505,8 +544,10 @@ void ComplexDPGWeakForm::Assemble(int skip_zeros)
       }
 
       ComplexCholeskyFactors chol(G_r.GetData(), G_i.GetData());
+
       int h = G_r.Height();
-      chol.Factor(h);
+      bool info = chol.Factor(h);
+      MFEM_VERIFY(info, "Complex Cholesky factorization of G failed");
 
       int w = B_r.Width();
       chol.LSolve(h,w,B_r.GetData(), B_i.GetData());
@@ -919,7 +960,7 @@ void ComplexDPGWeakForm::EnableStaticCondensation()
 Vector & ComplexDPGWeakForm::ComputeResidual(const Vector & x)
 {
    MFEM_VERIFY(store_matrices,
-               "Matrices needed for the residual are not store. Call ComplexDPGWeakForm::StoreMatrices()")
+               "Matrices needed for the residual are not stored. Call ComplexDPGWeakForm::StoreMatrices()")
    // wrap vector in a blockvector
    int n = x.Size()/2;
 
@@ -962,7 +1003,8 @@ Vector & ComplexDPGWeakForm::ComputeResidual(const Vector & x)
          {
             for (int ie = 0; ie < faces.Size(); ie++)
             {
-               trial_offs[j+1] += trial_fes[j]->GetFaceElement(faces[ie])->GetDof();
+               trial_offs[j+1] += trial_fes[j]->GetVDim()*trial_fes[j]->GetFaceElement(
+                                     faces[ie])->GetDof();
             }
          }
          else

miniapps/dpg/util/complexweakform.hpp

@@ -69,6 +69,10 @@ protected:
    Array2D<Array<BilinearFormIntegrator * > * > test_integs_r;
    Array2D<Array<BilinearFormIntegrator * > * > test_integs_i;
 
+   /// Integration rules for the test and trial integrators
+   const IntegrationRule *trial_ir = nullptr;
+   const IntegrationRule *test_ir = nullptr;
+
    /// Set of LinearForm Integrators to be applied.
    Array<Array<LinearFormIntegrator * > * > lfis_r;
    Array<Array<LinearFormIntegrator * > * > lfis_i;
@@ -101,7 +105,8 @@ protected:
    Vector residuals;
 
 private:
-
+   /// Enforces a unique integration rule for all trial/test integrators
+   void SetIntegrationRules();
 public:
 
    ComplexDPGWeakForm()
@@ -198,6 +203,11 @@ public:
                               LinearFormIntegrator *lfi_i,
                               int n);
 
+   /// Sets the same integration rules for all trial integrators
+   void SetTrialIntegrationRule(const IntegrationRule &ir) { trial_ir = &ir; }
+   /// Sets the same integration rules for all test integrators
+   void SetTestIntegrationRule(const IntegrationRule &ir) { test_ir = &ir; }
+
    /// Assembles the form i.e. sums over all integrators.
    void Assemble(int skip_zeros = 1);
 
@@ -273,6 +283,23 @@ public:
 
    Vector & ComputeResidual(const Vector & x);
 
+   void GetTraceFESpaces(Array<FiniteElementSpace *> & trace_fes) const
+   {
+      trace_fes.SetSize(0);
+      Array<FiniteElementSpace *> trace_fes_all;
+      if (static_cond)
+      {
+         static_cond->GetTraceFESpaces(trace_fes_all);
+         for (int i = 0; i < trace_fes_all.Size(); i++)
+         {
+            if (trace_fes_all[i])
+            {
+               trace_fes.Append(trace_fes_all[i]);
+            }
+         }
+      }
+   }
+
    /// Destroys bilinear form.
    virtual ~ComplexDPGWeakForm();
 

miniapps/dpg/util/maxwell_utils.cpp

@@ -0,0 +1,562 @@
+#include "maxwell_utils.hpp"
+
+real_t AzimuthalECoefficient::Eval(ElementTransformation &T,
+                                   const IntegrationPoint &ip)
+{
+   Vector X, E;
+   vgf->GetVectorValue(T,ip,E);
+   T.Transform(ip, X);
+   real_t x = X(0);
+   real_t y = X(1);
+   real_t r = sqrt(x*x + y*y);
+
+   real_t val = -x*E[1] + y*E[0];
+   return val/r;
+}
+
+real_t ParallelECoefficient::Eval(ElementTransformation &T,
+                                  const IntegrationPoint &ip)
+{
+   Vector X, E;
+   vgf->GetVectorValue(T,ip,E);
+   T.Transform(ip, X);
+   Vector b;
+   ComputeB(X, b);
+   return E*b;
+}
+
+EpsilonMatrixCoefficient::EpsilonMatrixCoefficient(const char * filename,
+                                                   Mesh * mesh_, ParMesh * pmesh_,
+                                                   real_t scale)
+   : MatrixArrayCoefficient(mesh_->Dimension()), mesh(mesh_), pmesh(pmesh_),
+     dim(mesh->Dimension())
+{
+   std::filebuf fb;
+   fb.open(filename,std::ios::in);
+   std::istream is(&fb);
+   vgf = new GridFunction(mesh,is);
+   fb.close();
+   FiniteElementSpace * vfes = vgf->FESpace();
+   int vdim = vfes->GetVDim();
+   const FiniteElementCollection * fec = vfes->FEColl();
+   FiniteElementSpace * fes = new FiniteElementSpace(mesh, fec);
+   int * partitioning = mesh->GeneratePartitioning(num_procs);
+   double *data = vgf->GetData();
+   GridFunction gf;
+   pgfs.SetSize(vdim);
+   gf_cfs.SetSize(vdim);
+   sdim = sqrt(vdim);
+   for (int i = 0; i<sdim; i++)
+   {
+      for (int j = 0; j<sdim; j++)
+      {
+         int k = i*sdim+j;
+         gf.MakeRef(fes,&data[k*fes->GetVSize()]);
+         pgfs[k] = new ParGridFunction(pmesh,&gf,partitioning);
+         (*pgfs[k])*=scale;
+         gf_cfs[k] = new GridFunctionCoefficient(pgfs[k]);
+         if (i<dim && j<dim)
+         {
+            Set(i,j,gf_cfs[k], true);
+         }
+      }
+   }
+}
+
+void EpsilonMatrixCoefficient::VisualizeMatrixCoefficient()
+{
+   Array<socketstream *> sol_sock(pgfs.Size());
+   for (int k = 0; k<pgfs.Size(); k++)
+   {
+      if (Mpi::Root()) { mfem::out << "Visualizing component " << k << endl; }
+      char vishost[] = "localhost";
+      int visport = 19916;
+      sol_sock[k] = new socketstream(vishost, visport);
+      sol_sock[k]->precision(8);
+      *sol_sock[k] << "parallel " << num_procs << " " << myid << "\n";
+      int i = k/sdim;
+      int j = k%sdim;
+      *sol_sock[k] << "solution\n" << *pmesh << *pgfs[k]
+                   << "window_title 'Epsilon Matrix Coefficient Component (" << i << "," << j <<
+                   ")'" << flush;
+   }
+}
+
+void EpsilonMatrixCoefficient::Update()
+{
+   pgfs[0]->ParFESpace()->Update();
+   for (int k = 0; k<pgfs.Size(); k++)
+   {
+      pgfs[k]->Update();
+   }
+}
+
+EpsilonMatrixCoefficient::~EpsilonMatrixCoefficient()
+{
+   for (int i = 0; i<pgfs.Size(); i++)
+   {
+      delete pgfs[i];
+   }
+   pgfs.DeleteAll();
+}
+
+DielectricTensorComponentCoefficient::DielectricTensorComponentCoefficient(
+   real_t delta_, real_t a0_, real_t a1_,
+   int row_, int col_,
+   bool use_imag_)
+   : delta(delta_), a0(a0_), a1(a1_), row(row_), col(col_), use_imag(use_imag_) { }
+
+
+real_t DielectricTensorComponentCoefficient::Eval(ElementTransformation &T,
+                                                  const IntegrationPoint &ip)
+{
+   Vector x;
+   T.Transform(ip, x);
+   return use_imag ? ComputeImagPart(x) : ComputeRealPart(x);
+}
+
+real_t DielectricTensorComponentCoefficient::ComputeRealPart(const Vector &x)
+{
+   Vector b;
+   ComputeB(x, b);
+
+   real_t r = std::sqrt(x(0)*x(0) + x(1)*x(1));
+   real_t S = 1.0;
+   real_t P = a0 + a1 * (r - 0.9);
+
+   real_t bb_ij = b(row) * b(col);
+   return S * (row == col) + (P - S) * bb_ij;
+}
+
+real_t DielectricTensorComponentCoefficient::ComputeImagPart(const Vector &x)
+{
+   return (row == col) ? delta : 0.0;
+}
+
+void VisualizeMatrixArrayCoefficient(MatrixArrayCoefficient &mc, ParMesh *pmesh,
+                                     int order, bool paraview, const char *name)
+{
+   MFEM_VERIFY(pmesh != nullptr, "ParMesh pointer must not be null.");
+   int dim = mc.GetVDim();
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Visualizing matrix coefficient with dimension: " << dim << endl;
+      mfem::out << "order = " << order << endl;
+      mfem::out << "pmesh dimension = " << pmesh->Dimension() << endl;
+   }
+   auto fec = new H1_FECollection(order, pmesh->Dimension());
+   auto pfes = new ParFiniteElementSpace(pmesh, fec);
+
+   Array<ParGridFunction *> pgfs(dim * dim);
+
+   Array<GridFunctionCoefficient *> gf_cfs(dim * dim);
+   ParaViewDataCollection * pvdc = nullptr;
+   std::ostringstream label;
+   if (name)
+   {
+      label << name;
+   }
+   else
+   {
+      label << "eps";
+   }
+   if (paraview)
+   {
+      pvdc = new ParaViewDataCollection(label.str(), pmesh);
+      pvdc->SetPrefixPath("ParaView");
+      pvdc->SetLevelsOfDetail(order);
+      pvdc->SetCycle(0);
+      pvdc->SetDataFormat(VTKFormat::BINARY);
+   }
+   for (int i = 0; i < dim; ++i)
+   {
+      for (int j = 0; j < dim; ++j)
+      {
+         Coefficient *c_ij = mc.GetCoeff(i, j);
+         if (!c_ij) { continue; }
+
+         pgfs[i*dim + j] = new ParGridFunction(pfes);
+         *pgfs[i*dim + j] = 0.0;
+         pgfs[i*dim + j]->ProjectCoefficient(*c_ij);
+
+         if (Mpi::Root())
+         {
+            mfem::out << "Projected component (" << i << "," << j << ")" << endl;
+         }
+
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         socketstream sol_sock(vishost, visport);
+
+         sol_sock << "parallel " << pmesh->GetNRanks() << " " << pmesh->GetMyRank() <<
+                  "\n";
+         sol_sock << "solution\n" << * pmesh <<  *pgfs[i*dim + j];
+         sol_sock << "window_title '" << label.str() <<"_" << i << j << "'\n";
+         sol_sock << flush;
+
+         if (paraview)
+         {
+            pvdc->RegisterField(label.str() + std::to_string(i) + std::to_string(j),
+                                pgfs[i*dim + j]);
+         }
+      }
+   }
+   if (paraview)
+   {
+      pvdc->Save();
+      delete pvdc;
+      for (int i = 0; i < dim * dim; ++i)
+      {
+         delete pgfs[i];
+      }
+   }
+   delete pfes;
+   delete fec;
+}
+
+void ComputeB(const Vector &x, Vector &b)
+{
+   real_t x0 = x(0), x1 = x(1);
+   real_t r = std::sqrt(x0 * x0 + x1 * x1);
+   int dim = x.Size();
+   b.SetSize(dim); b = 0.0;
+   b(0) = -x1 / r;
+   b(1) =  x0 / r;
+   if (dim == 3) { b(2) = 0.0; }
+}
+
+void DirectionalVectorDiffusionIntegrator::AssembleElementMatrix(
+   const FiniteElement &el,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   int dof = el.GetDof();
+   int dim = el.GetDim();
+   int vdim = Trans.GetSpaceDim();
+
+   elmat.SetSize(dof * vdim, dof * vdim);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = 2 * el.GetOrder(); // Integration order
+      ir = &IntRules.Get(el.GetGeomType(), order);
+   }
+
+   // Get shape functions and their derivatives
+   DenseMatrix dshape(dof, dim);
+   Vector vec(dim);
+
+   for (int k = 0; k < ir->GetNPoints(); k++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(k);
+      Trans.SetIntPoint(&ip);
+      double w = ip.weight * Trans.Weight();
+      VQ->Eval(vec, Trans, ip);
+      el.CalcPhysDShape(Trans, dshape);
+
+      // Compute (vq·∇)φ for each basis function
+      Vector vq_grad_phi(dof); vq_grad_phi = 0.0;
+      for (int j = 0; j < dof; j++)
+      {
+         for (int d = 0; d < dim; d++)
+         {
+            vq_grad_phi(j) += vec(d) * dshape(j, d);
+         }
+      }
+
+      for (int comp = 0; comp < vdim; comp++)
+      {
+         int offset = comp * dof;
+         for (int j = 0; j < dof; j++)
+         {
+            int jj = j + offset;
+            for (int i = 0; i < dof; i++)
+            {
+               int ii = i + offset;
+               elmat(jj, ii) += w * vq_grad_phi(j)
+                                * vq_grad_phi(i);
+            }
+         }
+      }
+   }
+}
+
+void DirectionalVectorDiffusionIntegrator::AssembleElementMatrix2(
+   const FiniteElement &trial_fe, const FiniteElement &test_fe,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   int trial_dof = trial_fe.GetDof();
+   int test_dof = test_fe.GetDof();
+   int dim = trial_fe.GetDim();
+   int vdim = Trans.GetSpaceDim();
+
+   elmat.SetSize(test_dof * vdim, trial_dof * vdim);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW() - 1;
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+   }
+
+   Vector vec(dim);
+   DenseMatrix trial_dshape(trial_dof, dim);
+   DenseMatrix test_dshape(test_dof, dim);
+
+   for (int k = 0; k < ir->GetNPoints(); k++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(k);
+      Trans.SetIntPoint(&ip);
+      double w = ip.weight * Trans.Weight();
+      VQ->Eval(vec, Trans, ip);
+      trial_fe.CalcPhysDShape(Trans, trial_dshape);
+      test_fe.CalcPhysDShape(Trans, test_dshape);
+
+      // Compute (vq·∇)φ for each basis function
+      Vector vq_grad_phi_trial(trial_dof); vq_grad_phi_trial = 0.0;
+      for (int j = 0; j < trial_dof; j++)
+      {
+         for (int d = 0; d < dim; d++)
+         {
+            vq_grad_phi_trial(j) += vec(d) * trial_dshape(j, d);
+         }
+      }
+
+      Vector vq_grad_phi_test(test_dof); vq_grad_phi_test = 0.0;
+      for (int j = 0; j < test_dof; j++)
+      {
+         for (int d = 0; d < dim; d++)
+         {
+            vq_grad_phi_test(j) += vec(d) * test_dshape(j, d);
+         }
+      }
+
+      for (int trial_comp = 0; trial_comp < vdim; trial_comp++)
+      {
+         int offset_trial = trial_comp * trial_dof;
+         int offset_test = trial_comp * test_dof;
+         for (int j = 0; j < test_dof; j++)
+         {
+            int jj = j + offset_test;
+            for (int i = 0; i < trial_dof; i++)
+            {
+               int ii = i + offset_trial;
+               elmat(jj, ii) += w * vq_grad_phi_test(j) * vq_grad_phi_trial(i);
+            }
+         }
+      }
+   }
+}
+
+
+
+void DirectionalVectorGradientIntegrator::AssembleElementMatrix(
+   const FiniteElement &el,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   int dof = el.GetDof();
+   int sdim = Trans.GetSpaceDim();
+
+   elmat.SetSize(dof * sdim, dof * sdim);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = 2 * el.GetOrder(); // Integration order
+      ir = &IntRules.Get(el.GetGeomType(), order);
+   }
+
+   // Get shape functions and their derivatives
+   DenseMatrix dshape(dof, sdim);
+   Vector shape(dof);
+   Vector vec(sdim);
+   Vector vq_grad_phi(dof);
+   for (int k = 0; k < ir->GetNPoints(); k++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(k);
+      Trans.SetIntPoint(&ip);
+      double w = ip.weight * Trans.Weight();
+      VQ->Eval(vec, Trans, ip);
+
+      // gradient of trial (u) in physical space
+      el.CalcPhysDShape(Trans, dshape);
+
+      // value of test (v) in physical space
+      el.CalcPhysShape(Trans, shape);
+
+      // (vq·∇)φ_i on the trial side
+      vq_grad_phi = 0.0;
+      for (int j = 0; j < dof; j++)
+      {
+         for (int d = 0; d < sdim; d++)
+         {
+            vq_grad_phi(j) += vec(d) * dshape(j, d);
+         }
+      }
+
+      for (int comp = 0; comp < sdim; comp++)
+      {
+         int offset = comp * dof;
+         for (int j = 0; j < dof; j++)
+         {
+            int jj = j + offset;
+            for (int i = 0; i < dof; i++)
+            {
+               int ii = i + offset;
+               elmat(jj, ii) += w * shape(j) * vq_grad_phi(i);
+            }
+         }
+      }
+   }
+}
+
+void DirectionalVectorGradientIntegrator::AssembleElementMatrix2(
+   const FiniteElement &trial_fe, const FiniteElement &test_fe,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   int trial_dof = trial_fe.GetDof();
+   int test_dof = test_fe.GetDof();
+   int dim = trial_fe.GetDim();
+   int sdim = Trans.GetSpaceDim();
+
+   elmat.SetSize(test_dof * sdim, trial_dof * sdim);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+   }
+
+   Vector vec(sdim);
+   DenseMatrix trial_dshape(trial_dof, sdim);
+   Vector test_shape(test_dof);
+   Vector vq_grad_phi_trial(trial_dof);
+
+   for (int k = 0; k < ir->GetNPoints(); k++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(k);
+      Trans.SetIntPoint(&ip);
+      double w = ip.weight * Trans.Weight();
+      VQ->Eval(vec, Trans, ip);
+      // gradient of trial (u) in physical space
+      trial_fe.CalcPhysDShape(Trans, trial_dshape);
+      // value of test (v) in physical space
+      test_fe.CalcPhysShape(Trans, test_shape);
+
+      // (vq·∇)φ_i on the trial side
+      vq_grad_phi_trial = 0.0;
+      for (int i = 0; i < trial_dof; i++)
+      {
+         for (int d = 0; d < dim; d++)
+         {
+            vq_grad_phi_trial(i) += vec(d) * trial_dshape(i, d);
+         }
+      }
+
+      for (int comp = 0; comp < sdim; comp++)
+      {
+         int offset_trial = comp * trial_dof;
+         int offset_test = comp * test_dof;
+         for (int j = 0; j < test_dof; j++)
+         {
+            int jj = j + offset_test;
+            for (int i = 0; i < trial_dof; i++)
+            {
+               int ii = i + offset_trial;
+               elmat(jj, ii) += w * test_shape(j) * vq_grad_phi_trial(i);
+            }
+         }
+      }
+   }
+}
+
+void MixedDirectionalVectorGradientIntegrator::AssembleElementMatrix2(
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   const int trial_dof = trial_fe.GetDof();
+   const int test_dof  = test_fe.GetDof();
+   const int sdim      = Trans.GetSpaceDim();
+
+   MFEM_VERIFY(test_fe.GetMapType() == mfem::FiniteElement::H_CURL ||
+               test_fe.GetMapType() == mfem::FiniteElement::H_DIV,
+               "MixedDirectionalVectorGradientIntegrator requires "
+               "H(curl) or H(div) test space.");
+
+   const int vdim = sdim;
+
+   // Test dofs already correspond to vector basis functions
+   elmat.SetSize(test_dof, trial_dof * vdim);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      const int order =
+         trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+   }
+
+   Vector q(vdim);
+   DenseMatrix trial_dshape(trial_dof, vdim);
+   DenseMatrix test_vshape(test_dof, vdim);
+
+   Vector q_dot_grad_phi(trial_dof);
+
+   for (int k = 0; k < ir->GetNPoints(); k++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(k);
+      Trans.SetIntPoint(&ip);
+
+      const double w = ip.weight * Trans.Weight();
+
+      // q in physical space
+      VQ->Eval(q, Trans, ip);
+
+      // ∇phi_i in physical space
+      trial_fe.CalcPhysDShape(Trans, trial_dshape);
+
+      // vector-valued test shape
+      test_fe.CalcVShape(Trans, test_vshape);
+
+      // (q · ∇) phi_i
+      q_dot_grad_phi = 0.0;
+      for (int i = 0; i < trial_dof; i++)
+      {
+         double val = 0.0;
+         for (int d = 0; d < vdim; d++)
+         {
+            val += q(d) * trial_dshape(i, d);
+         }
+         q_dot_grad_phi(i) = val;
+      }
+
+      // Assemble: v_j · (q·∇phi_i)
+      for (int j = 0; j < test_dof; j++)
+      {
+         for (int comp = 0; comp < vdim; comp++)
+         {
+            const double vj = test_vshape(j, comp);
+            const int offset_trial = comp * trial_dof;
+
+            for (int i = 0; i < trial_dof; i++)
+            {
+               elmat(j, offset_trial + i) +=
+                  w * vj * q_dot_grad_phi(i);
+            }
+         }
+      }
+   }
+}

miniapps/dpg/util/maxwell_utils.hpp

@@ -0,0 +1,135 @@
+#include "mfem.hpp"
+
+using namespace mfem;
+using namespace std;
+
+class AzimuthalECoefficient : public Coefficient
+{
+private:
+   const GridFunction * vgf;
+public:
+   AzimuthalECoefficient(const GridFunction * vgf_)
+      : Coefficient(), vgf(vgf_) {}
+   virtual real_t Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
+class ParallelECoefficient : public Coefficient
+{
+private:
+   const GridFunction * vgf;
+public:
+   ParallelECoefficient(const GridFunction * vgf_)
+      : Coefficient(), vgf(vgf_) {}
+   virtual real_t Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
+class EpsilonMatrixCoefficient : public MatrixArrayCoefficient
+{
+private:
+   Mesh * mesh = nullptr;
+   ParMesh * pmesh = nullptr;
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Array<ParGridFunction * > pgfs;
+   Array<GridFunctionCoefficient * > gf_cfs;
+   GridFunction * vgf = nullptr;
+   int dim;
+   int sdim;
+public:
+   EpsilonMatrixCoefficient(const char * filename, Mesh * mesh_, ParMesh * pmesh_,
+                            real_t scale = 1.0);
+
+   // Visualize the components of the matrix coefficient
+   // in separate GLVis windows for each component
+   void VisualizeMatrixCoefficient();
+   // Update the Gridfunctions after mesh refinement
+   void Update();
+
+   ~EpsilonMatrixCoefficient();
+
+};
+
+class DielectricTensorComponentCoefficient : public Coefficient
+{
+private:
+   real_t delta, a0, a1;
+   int row, col;
+   bool use_imag;
+
+public:
+   /// Constructor
+   /// @param delta_    Imaginary scaling factor
+   /// @param a0_       Constant in P(r)
+   /// @param a1_       Linear term in P(r)
+   /// @param row_      row index of the dielectric tensor
+   /// @param col_      column index of the dielectric tensor
+   /// @param use_imag_ If true, returns imaginary part; otherwise real part
+   DielectricTensorComponentCoefficient(real_t delta_, real_t a0_, real_t a1_,
+                                        int row_, int col_, bool use_imag_ = false);
+   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
+
+private:
+   real_t ComputeRealPart(const Vector &x);
+   real_t ComputeImagPart(const Vector &x);
+};
+
+void VisualizeMatrixArrayCoefficient(MatrixArrayCoefficient &mc, ParMesh *pmesh,
+                                     int order, bool paraview = false, const char *name = nullptr);
+
+void ComputeB(const Vector &x, Vector &b);
+
+// Integrator for (vq·∇) u · (vq·∇) v where u,v ∈ (H¹(Ω))ᵈ
+class DirectionalVectorDiffusionIntegrator : public BilinearFormIntegrator
+{
+private:
+   VectorCoefficient *VQ;
+
+public:
+   DirectionalVectorDiffusionIntegrator(VectorCoefficient &q) : VQ(&q) { }
+
+   virtual void AssembleElementMatrix(const FiniteElement &el,
+                                      ElementTransformation &Trans,
+                                      DenseMatrix &elmat);
+   virtual void AssembleElementMatrix2(const FiniteElement &tr_el,
+                                       const FiniteElement &te_el,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+
+};
+
+// Integrator for (vq·∇) u · v where u ∈ (H¹(Ω))ᵈ, v ∈ (H¹(Ω))ᵈ or (L²(Ω))ᵈ
+class DirectionalVectorGradientIntegrator : public BilinearFormIntegrator
+{
+private:
+   VectorCoefficient *VQ;
+
+public:
+   DirectionalVectorGradientIntegrator(VectorCoefficient &q) : VQ(&q) { }
+
+   virtual void AssembleElementMatrix(const FiniteElement &el,
+                                      ElementTransformation &Trans,
+                                      DenseMatrix &elmat);
+   virtual void AssembleElementMatrix2(const FiniteElement &tr_el,
+                                       const FiniteElement &te_el,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+
+};
+
+// Integrator for (vq·∇) u · v where u ∈ (H¹(Ω))ᵈ, v ∈ H(curl) or H(div)
+class MixedDirectionalVectorGradientIntegrator : public BilinearFormIntegrator
+{
+private:
+   VectorCoefficient *VQ;
+
+public:
+   MixedDirectionalVectorGradientIntegrator(VectorCoefficient &q) : VQ(&q) { }
+
+   virtual void AssembleElementMatrix2(const FiniteElement &tr_el,
+                                       const FiniteElement &te_el,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+
+};

miniapps/dpg/util/pcomplexblockform.cpp

@@ -0,0 +1,265 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "pcomplexblockform.hpp"
+
+#ifdef MFEM_USE_MPI
+
+namespace mfem
+{
+
+void ParComplexBlockForm::FillEssTdofLists(const Array<int> &
+                                           ess_tdof_list)
+{
+   int j;
+   for (int i = 0; i < ess_tdof_list.Size(); i++)
+   {
+      int tdof = ess_tdof_list[i];
+      for (j = 0; j < nblocks; j++)
+      {
+         if (tdof_offsets[j+1] > tdof) { break; }
+      }
+      ess_tdofs[j]->Append(tdof-tdof_offsets[j]);
+   }
+}
+
+void ParComplexBlockForm::Assemble(int skip_zeros)
+{
+   ComplexBlockForm::Assemble(skip_zeros);
+}
+
+void ParComplexBlockForm::ParallelAssemble(BlockMatrix *m_r,
+                                           BlockMatrix *m_i)
+{
+   if (!P) { BuildProlongation(); }
+
+   p_mat_r = new BlockOperator(tdof_offsets);
+   p_mat_i = new BlockOperator(tdof_offsets);
+   p_mat_e_r = new BlockOperator(tdof_offsets);
+   p_mat_e_i = new BlockOperator(tdof_offsets);
+   p_mat_r->owns_blocks = 1;
+   p_mat_i->owns_blocks = 1;
+   p_mat_e_r->owns_blocks = 1;
+   p_mat_e_i->owns_blocks = 1;
+   HypreParMatrix * A_r = nullptr;
+   HypreParMatrix * A_i = nullptr;
+   HypreParMatrix * PtAP_r = nullptr;
+   HypreParMatrix * PtAP_i = nullptr;
+   for (int i = 0; i < nblocks; i++)
+   {
+      HypreParMatrix * Pi = (HypreParMatrix*)(&P->GetBlock(i,i));
+      for (int j = 0; j<nblocks; j++)
+      {
+         if (m_r->IsZeroBlock(i,j)) { continue; }
+         if (i == j)
+         {
+            // Make block diagonal square hypre matrix
+            A_r = new HypreParMatrix(pfes[i]->GetComm(), pfes[i]->GlobalVSize(),
+                                     pfes[i]->GetDofOffsets(), &m_r->GetBlock(i,i));
+            PtAP_r = RAP(A_r,Pi);
+            delete A_r;
+            p_mat_e_r->SetBlock(i, i, PtAP_r->EliminateRowsCols(*ess_tdofs[i]));
+
+            A_i = new HypreParMatrix(pfes[i]->GetComm(), pfes[i]->GlobalVSize(),
+                                     pfes[i]->GetDofOffsets(), &m_i->GetBlock(i,i));
+
+            PtAP_i = RAP(A_i,Pi);
+            delete A_i;
+            p_mat_e_i->SetBlock(i, i, PtAP_i->EliminateCols(*ess_tdofs[i]));
+            PtAP_i->EliminateRows(*ess_tdofs[i]);
+         }
+         else
+         {
+            HypreParMatrix * Pj = (HypreParMatrix*)(&P->GetBlock(j,j));
+            A_r = new HypreParMatrix(pfes[i]->GetComm(), pfes[i]->GlobalVSize(),
+                                     pfes[j]->GlobalVSize(), pfes[i]->GetDofOffsets(),
+                                     pfes[j]->GetDofOffsets(), &m_r->GetBlock(i,j));
+            PtAP_r = RAP(Pi,A_r,Pj);
+            delete A_r;
+            p_mat_e_r->SetBlock(i, j, PtAP_r->EliminateCols(*ess_tdofs[j]));
+            PtAP_r->EliminateRows(*ess_tdofs[i]);
+
+            A_i = new HypreParMatrix(pfes[i]->GetComm(), pfes[i]->GlobalVSize(),
+                                     pfes[j]->GlobalVSize(), pfes[i]->GetDofOffsets(),
+                                     pfes[j]->GetDofOffsets(), &m_i->GetBlock(i,j));
+            PtAP_i = RAP(Pi,A_i,Pj);
+            delete A_i;
+            p_mat_e_i->SetBlock(i, j, PtAP_i->EliminateCols(*ess_tdofs[j]));
+            PtAP_i->EliminateRows(*ess_tdofs[i]);
+         }
+         p_mat_r->SetBlock(i, j, PtAP_r);
+         p_mat_i->SetBlock(i, j, PtAP_i);
+      }
+   }
+}
+
+void ParComplexBlockForm::BuildProlongation()
+{
+   P = new BlockOperator(dof_offsets, tdof_offsets);
+   R = new BlockMatrix(tdof_offsets, dof_offsets);
+   P->owns_blocks = 0;
+   R->owns_blocks = 0;
+
+   for (int i = 0; i < nblocks; i++)
+   {
+      HypreParMatrix * P_ = pfes[i]->Dof_TrueDof_Matrix();
+      P->SetBlock(i,i,P_);
+      const SparseMatrix * R_ = pfes[i]->GetRestrictionMatrix();
+      R->SetBlock(i, i, const_cast<SparseMatrix*>(R_));
+   }
+}
+
+void ParComplexBlockForm::FormLinearSystem(const Array<int>
+                                           &ess_tdof_list,
+                                           Vector &x, Vector &b,
+                                           OperatorHandle &A,
+                                           Vector &X, Vector &B,
+                                           int copy_interior)
+{
+   FormSystemMatrix(ess_tdof_list, A);
+
+   int n = P->Width();
+   B.SetSize(2*n);
+   Vector B_r(B, 0, n);
+   Vector B_i(B, n, n);
+
+   Vector b_r(b, 0, b.Size()/2);
+   Vector b_i(b, b.Size()/2, b.Size()/2);
+
+   P->MultTranspose(b_r, B_r);
+   P->MultTranspose(b_i, B_i);
+
+   int m = R->Height();
+   X.SetSize(2*m);
+
+   Vector X_r(X, 0, m);
+   Vector X_i(X, m, m);
+
+   Vector x_r(x, 0, x.Size()/2);
+   Vector x_i(x, x.Size()/2, x.Size()/2);
+
+   R->Mult(x_r, X_r);
+   R->Mult(x_i, X_i);
+
+   // eliminate tdof is RHS
+   // B_r -= Ae_r*X_r + Ae_i X_i
+   // B_i -= Ae_i*X_r + Ae_r X_i
+   Vector tmp(B_r.Size());
+   p_mat_e_r->Mult(X_r, tmp); B_r-=tmp;
+   p_mat_e_i->Mult(X_i, tmp); B_r+=tmp;
+
+   p_mat_e_i->Mult(X_r, tmp); B_i-=tmp;
+   p_mat_e_r->Mult(X_i, tmp); B_i-=tmp;
+
+   for (int j = 0; j < nblocks; j++)
+   {
+      if (!ess_tdofs[j]->Size()) { continue; }
+      for (int i = 0; i < ess_tdofs[j]->Size(); i++)
+      {
+         int tdof = (*ess_tdofs[j])[i];
+         int gdof = tdof + tdof_offsets[j];
+         B_r(gdof) = X_r(gdof); // diagonal policy is always one in parallel
+         B_i(gdof) = X_i(gdof); // diagonal policy is always one in parallel
+      }
+   }
+   if (!copy_interior)
+   {
+      X_r.SetSubVectorComplement(ess_tdof_list, 0.0);
+      X_i.SetSubVectorComplement(ess_tdof_list, 0.0);
+   }
+}
+
+void ParComplexBlockForm::FormSystemMatrix(const Array<int>
+                                           &ess_tdof_list,
+                                           OperatorHandle &A)
+{
+   FillEssTdofLists(ess_tdof_list);
+   if (mat_r)
+   {
+      const int remove_zeros = 0;
+      Finalize(remove_zeros);
+      ParallelAssemble(mat_r, mat_i);
+      delete mat_r;
+      delete mat_i;
+      mat_r = nullptr;
+      mat_i = nullptr;
+      delete mat_e_r;
+      delete mat_e_i;
+      mat_e_r = nullptr;
+      mat_e_i = nullptr;
+   }
+   p_mat = new ComplexOperator(p_mat_r, p_mat_i, false, false);
+   A.Reset(p_mat, false);
+}
+
+void ParComplexBlockForm::RecoverFEMSolution(const Vector &X,
+                                             Vector &x)
+{
+   int n = P->Height();
+   int m = P->Width();
+   x.SetSize(2*n);
+
+   Vector x_r(x,0,n);
+   Vector x_i(x,n,n);
+   Vector X_r(const_cast<Vector&>(X), 0, m);
+   Vector X_i(const_cast<Vector&>(X), m, m);
+
+   P->Mult(X_r, x_r);
+   P->Mult(X_i, x_i);
+}
+
+void ParComplexBlockForm::Update()
+{
+   ComplexBlockForm::Update();
+   delete p_mat_e_r;
+   delete p_mat_e_i;
+   p_mat_e_r = nullptr;
+   p_mat_e_i = nullptr;
+   delete p_mat_r;
+   delete p_mat_i;
+   p_mat_r = nullptr;
+   p_mat_i = nullptr;
+   delete p_mat;
+   p_mat = nullptr;
+   for (int i = 0; i < nblocks; i++)
+   {
+      delete ess_tdofs[i];
+      ess_tdofs[i] = new Array<int>();
+   }
+   delete P;
+   P = nullptr;
+   delete R;
+   R = nullptr;
+}
+
+ParComplexBlockForm::~ParComplexBlockForm()
+{
+   delete p_mat_e_r;
+   delete p_mat_e_i;
+   p_mat_e_r = nullptr;
+   p_mat_e_i = nullptr;
+   delete p_mat_r;
+   delete p_mat_i;
+   p_mat_r = nullptr;
+   p_mat_i = nullptr;
+   delete p_mat;
+   p_mat = nullptr;
+   for (int i = 0; i < nblocks; i++)
+   {
+      delete ess_tdofs[i];
+   }
+   delete P;
+   delete R;
+}
+
+} // namespace mfem
+
+#endif

miniapps/dpg/util/pcomplexblockform.hpp

@@ -0,0 +1,108 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_PCOMPLEX_BLOCKFORM
+#define MFEM_PCOMPLEX_BLOCKFORM
+
+#include "mfem.hpp"
+#include "complexblockform.hpp"
+
+#ifdef MFEM_USE_MPI
+
+namespace mfem
+{
+
+/** @brief Class representing the parallel weak formulation.
+   (Convenient for DPG Equations) */
+class ParComplexBlockForm : public ComplexBlockForm
+{
+
+protected:
+   /// Trial FE spaces
+   Array<ParFiniteElementSpace * > pfes;
+
+   /// ess_tdof list for each space
+   Array<Array<int> *> ess_tdofs;
+
+   /** split ess_tdof_list given in global tdof (for all spaces)
+    to individual lists for each space */
+   void FillEssTdofLists(const Array<int> & ess_tdof_list);
+
+   // Block Prolongation
+   BlockOperator * P = nullptr;
+   // Block Restriction
+   BlockMatrix * R = nullptr;
+
+   ComplexOperator * p_mat = nullptr;
+   BlockOperator * p_mat_r = nullptr;
+   BlockOperator * p_mat_i = nullptr;
+   BlockOperator * p_mat_e_r = nullptr;
+   BlockOperator * p_mat_e_i = nullptr;
+
+   void BuildProlongation();
+
+public:
+
+   ParComplexBlockForm() {}
+
+   /// Creates bilinear form associated with FE spaces @a trial_pfes_.
+   ParComplexBlockForm(Array<ParFiniteElementSpace* > & pfes_)
+      : ComplexBlockForm() { SetParSpaces(pfes_); }
+
+   void SetParSpaces(Array<ParFiniteElementSpace* > & pfes_)
+   {
+      pfes = pfes_;
+      ess_tdofs.SetSize(pfes.Size());
+
+      Array<FiniteElementSpace * > sfes(pfes.Size());
+      for (int i = 0; i<sfes.Size(); i++)
+      {
+         sfes[i] = (FiniteElementSpace *)pfes[i];
+         ess_tdofs[i] = new Array<int>();
+      }
+      SetSpaces(sfes);
+   }
+
+
+   /// Assembles the form i.e. sums over all domain integrators.
+   void Assemble(int skip_zeros = 1);
+
+   /// Returns the matrix assembled on the true dofs, i.e. P^t A P.
+   /** The returned matrix has to be deleted by the caller. */
+   void ParallelAssemble(BlockMatrix *mat_r, BlockMatrix *mat_i);
+
+   void FormLinearSystem(const Array<int> &ess_tdof_list,
+                         Vector &x, Vector &b,
+                         OperatorHandle &A,
+                         Vector &X, Vector &B,
+                         int copy_interior = 0);
+
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
+
+   /** Call this method after solving a linear system constructed using the
+       FormLinearSystem method to recover the solution as a ParGridFunction-size
+       vector in x. Use the same arguments as in the FormLinearSystem call. */
+   virtual void RecoverFEMSolution(const Vector &X, Vector &x);
+
+   virtual void Update();
+
+   /// Destroys bilinear form.
+   virtual ~ParComplexBlockForm();
+
+
+};
+
+} // namespace mfem
+
+
+#endif // MFEM_USE_MPI
+
+#endif