pmg in lh driver

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

CHANGELOG

@@ -34,6 +34,8 @@ Discretization improvements
 - Improved the gridfunction projection routines. Projections work for Scalar,
   Vector and VectorFE, also NURBS versions. Optionally different types of
   projections can be selected, default behaviour has not changed.
+- Added gridfunction projection methods for trace spaces, i.e., project 
+  coefficients on the mesh skeleton.   
 
 - Added methods to estimate function extremum using piecewise linear bounds +
   recursive subdivision.
@@ -43,11 +45,22 @@ Meshing improvements
 - Improved support for 1D NURBS meshes with variable order, including using
   the patches construct for 1D NURBS meshes.
 
+Linear and nonlinear solvers
+----------------------------
+- Added interface to MUMPS direct solver for complex-valued problems.
+  Its usage is demonstrated in ex25p. See http://mumps.enseeiht.fr/ for more details.
+  Supported versions >= 5.1.1.
+
 New and updated examples and miniapps
 -------------------------------------
 - Electromagnetics/lorentz miniapp has been updated to leverage the ParticleSet
   capability.
+- Added (Complex)PRefinementMultigrid solver option in DPG miniapps. 
 
+Linear and nonlinear solvers
+----------------------------
+- Added support for trace spaces in PRefinementTransferOperator. This is used in 
+  PRefinement multigrid methods for problems posed on trace spaces (see DPG miniapps)
 
 Version 4.9, released on Dec 11, 2025
 =====================================

CMakeLists.txt

@@ -76,7 +76,9 @@ set(XSDK_ENABLE_Fortran OFF)
 # Check if we need to enable C or Fortran.
 if (MFEM_USE_CONDUIT OR
     MFEM_USE_SIDRE OR
-    MFEM_USE_PETSC)
+    MFEM_USE_PETSC OR
+    MFEM_USE_MUMPS OR
+    MFEM_USE_COMPLEX_MUMPS)
    # This seems to be needed by:
    #  * find_package(BLAS REQUIRED) and
    #  * find_package(HDF5 REQUIRED) needed, in turn, by:
@@ -84,7 +86,7 @@ if (MFEM_USE_CONDUIT OR
    #  * find_package(PETSc REQUIRED)
    set(XSDK_ENABLE_C ON)
 endif()
-if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
+if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS OR MFEM_USE_COMPLEX_MUMPS)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
@@ -342,7 +344,7 @@ if (MFEM_USE_MPI)
     endif()
   endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU MUMPS PETSC SLEPC STRUMPACK PUMI)
+  set(PKGS_NEED_MPI SUPERLU MUMPS COMPLEX_MUMPS PETSC SLEPC STRUMPACK PUMI)
   foreach(PKG IN LISTS PKGS_NEED_MPI)
     if (MFEM_USE_${PKG})
       message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -415,7 +417,7 @@ if (MFEM_USE_SUPERLU)
 endif()
 
 # MUMPS can only be enabled in parallel
-if (MFEM_USE_MUMPS)
+if (MFEM_USE_MUMPS OR MFEM_USE_COMPLEX_MUMPS)
   if (MFEM_USE_MPI)
     find_package(MUMPS REQUIRED mumps_common pord)
     set(MFEM_MUMPS_VERSION ${MUMPS_VERSION})

config/cmake/MFEMConfig.cmake.in

@@ -34,6 +34,7 @@ set(MFEM_USE_SUNDIALS @MFEM_USE_SUNDIALS@)
 set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
 set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
+set(MFEM_USE_COMPLEX_MUMPS @MFEM_USE_COMPLEX_MUMPS@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
 set(MFEM_USE_AMGX @MFEM_USE_AMGX@)

config/cmake/config.hpp.in

@@ -103,6 +103,7 @@
 
 // Enable MFEM functionality based on the MUMPS library.
 #cmakedefine MFEM_USE_MUMPS
+#cmakedefine MFEM_USE_COMPLEX_MUMPS
 #cmakedefine MFEM_MUMPS_VERSION @MFEM_MUMPS_VERSION@
 
 // Enable MFEM functionality based on the STRUMPACK library.

config/cmake/modules/FindMUMPS.cmake

@@ -17,13 +17,31 @@
 
 include(MfemCmakeUtilities)
 
-# Toggle which precision of MUMPS to use depending on the precision of MFEM.
+# Decide headers/libs by MFEM precision
 if (MFEM_USE_DOUBLE)
-  set(_mumps_header dmumps_c.h)
-  set(_mumps_lib dmumps)
-elseif(MFEM_USE_SINGLE)
-  set(_mumps_header smumps_c.h)
-  set(_mumps_lib smumps)
+  set(_rmumps_header dmumps_c.h)
+  set(_rmumps_lib    dmumps)
+  set(_cmumps_header zmumps_c.h)
+  set(_cmumps_lib    zmumps)
+elseif (MFEM_USE_SINGLE)
+  set(_rmumps_header smumps_c.h)
+  set(_rmumps_lib    smumps)
+  set(_cmumps_header cmumps_c.h)
+  set(_cmumps_lib    cmumps)
+endif()
+
+# Choose which header/lib mfem_find_package should use as the "primary" one.
+# If both enabled, prefer the real one as primary (either is fine).
+if (MFEM_USE_MUMPS)
+  set(_mumps_header ${_rmumps_header})
+  set(_mumps_lib    ${_rmumps_lib})
+elseif (MFEM_USE_COMPLEX_MUMPS)
+  set(_mumps_header ${_cmumps_header})
+  set(_mumps_lib    ${_cmumps_lib})
+else()
+  # Should not happen in practice because FindMUMPS is only called when enabled,
+  set(_mumps_header ${_rmumps_header})
+  set(_mumps_lib    ${_rmumps_lib})
 endif()
 
 mfem_find_package(MUMPS MUMPS MUMPS_DIR
@@ -31,8 +49,35 @@ mfem_find_package(MUMPS MUMPS MUMPS_DIR
   "Paths to headers required by MUMPS."
   "Libraries required by MUMPS."
   ADD_COMPONENT mumps_common "include" ${_mumps_header} "lib" mumps_common
-  ADD_COMPONENT pord "include" ${_mumps_header} "lib" pord)
+  ADD_COMPONENT pord        "include" ${_mumps_header} "lib" pord)
 
+# If BOTH real and complex are enabled, ensure BOTH solver libs are linked.
+if (MUMPS_FOUND AND MFEM_USE_MUMPS AND MFEM_USE_COMPLEX_MUMPS)
+  # Find the "other" solver library and append it.
+  find_library(_mfem_other_mumps_solver
+    NAMES ${_cmumps_lib}   
+    HINTS ${MUMPS_DIR}
+    PATH_SUFFIXES lib lib64
+    NO_DEFAULT_PATH)
+
+  if (NOT _mfem_other_mumps_solver)
+    # Fall back to system search
+    find_library(_mfem_other_mumps_solver NAMES ${_cmumps_lib})
+  endif()
+
+  if (NOT _mfem_other_mumps_solver)
+    message(FATAL_ERROR
+      "MFEM_USE_MUMPS=ON and MFEM_USE_COMPLEX_MUMPS=ON, but could not find "
+      "the complex solver library '${_cmumps_lib}' in MUMPS_DIR='${MUMPS_DIR}'.")
+  endif()
+
+  # Put solver libs first (important for static link order)
+  # MUMPS_LIBRARIES contains the primary solver already + common + pord.
+  # We prepend the other solver.
+  list(INSERT MUMPS_LIBRARIES 0 ${_mfem_other_mumps_solver})
+endif()
+
+# Version detection 
 if (MUMPS_FOUND AND (NOT MUMPS_VERSION))
   try_run(MUMPS_VERSION_RUN_RESULT MUMPS_VERSION_COMPILE_RESULT
           ${CMAKE_CURRENT_BINARY_DIR}/config

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -871,13 +871,13 @@ function(mfem_export_mk_files)
       MFEM_USE_SINGLE MFEM_USE_DOUBLE MFEM_DEBUG MFEM_USE_EXCEPTIONS
       MFEM_USE_ZLIB MFEM_USE_LIBUNWIND MFEM_USE_LAPACK MFEM_THREAD_SAFE
       MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS
-      MFEM_USE_SUITESPARSE MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS
-      MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_MAGMA
-      MFEM_USE_GNUTLS MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC
-      MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS MFEM_USE_CONDUIT MFEM_USE_PUMI
-      MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA MFEM_USE_HIP MFEM_USE_RAJA
-      MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER MFEM_USE_UMPIRE MFEM_USE_SIMD
-      MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO
+      MFEM_USE_SUITESPARSE MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS 
+      MFEM_USE_COMPLEX_MUMPS MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX 
+      MFEM_USE_MAGMA MFEM_USE_GNUTLS MFEM_USE_NETCDF MFEM_USE_PETSC 
+      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS MFEM_USE_CONDUIT 
+      MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA MFEM_USE_HIP 
+      MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER MFEM_USE_UMPIRE 
+      MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO
       MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG
       MFEM_USE_TRIBOL MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME
       MFEM_USE_HDF5)

config/config.hpp

@@ -140,6 +140,9 @@ constexpr real_t operator""_r(unsigned long long v)
 #ifdef MFEM_USE_MUMPS
 #error Building with MUMPS (MFEM_USE_MUMPS=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
+#ifdef MFEM_USE_COMPLEX_MUMPS
+#error Building with COMPLEX_MUMPS (MFEM_USE_COMPLEX_MUMPS=YES) requires MPI (MFEM_USE_MPI=YES)
+#endif
 #ifdef MFEM_USE_STRUMPACK
 #error Building with STRUMPACK (MFEM_USE_STRUMPACK=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif

config/config.hpp.in

@@ -103,6 +103,7 @@
 
 // Enable MFEM functionality based on the MUMPS library.
 // #define MFEM_USE_MUMPS
+// #define MFEM_USE_COMPLEX_MUMPS
 // #define MFEM_MUMPS_VERSION @MFEM_MUMPS_VERSION@
 
 // Enable MFEM functionality based on the STRUMPACK library.

config/defaults.cmake

@@ -37,6 +37,7 @@ option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
 option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
 option(MFEM_USE_MUMPS "Enable MUMPS usage" OFF)
+option(MFEM_USE_COMPLEX_MUMPS "Enable COMPLEX_MUMPS usage" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
 option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
 option(MFEM_USE_AMGX "Enable AmgX usage" OFF)
@@ -152,7 +153,7 @@ set(SuperLUDist_REQUIRED_PACKAGES "MPI" "ParMETIS" "METIS"
 set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.5.0" CACHE PATH
     "Path to the MUMPS library.")
 # MUMPS may also depend on "OpenMP", depending on how it was compiled.
-set(MUMPS_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "ParMETIS" "METIS"
+set(MUMPS_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "METIS"
     "ScaLAPACK" "LAPACK" "BLAS" CACHE STRING
     "Additional packages required by MUMPS.")
 # If the MPI package does not find all required Fortran libraries:

config/defaults.mk

@@ -152,6 +152,7 @@ MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
 MFEM_USE_SUPERLU5      = NO
 MFEM_USE_MUMPS         = NO
+MFEM_USE_COMPLEX_MUMPS = NO
 MFEM_USE_STRUMPACK     = NO
 MFEM_USE_GINKGO        = NO
 MFEM_USE_AMGX          = NO
@@ -248,7 +249,7 @@ ifeq (YES,$(MFEM_USE_HIP))
 endif
 
 # METIS library configuration
-ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
+ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS)$(MFEM_USE_COMPLEX_MUMPS),NONONONO)
    ifeq ($(MFEM_USE_METIS_5),NO)
      METIS_DIR = @MFEM_DIR@/../metis-4.0
      METIS_OPT =
@@ -352,13 +353,19 @@ MPI_FORTRAN_LIB = -lmpifort
 # MUMPS library configuration
 MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.5.0
 MUMPS_OPT = -I$(MUMPS_DIR)/include
-MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib
+COMPLEX_MUMPS_OPT = $(MUMPS_OPT)
+
+MUMPS_COMMON_PATH = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib
+MUMPS_COMMON_LIB = -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
 ifeq ($(MFEM_USE_SINGLE),YES)
-   MUMPS_LIB += -lsmumps
+  MUMPS_SOLVER_LIB         = -lsmumps
+  COMPLEX_MUMPS_SOLVER_LIB = -lcmumps
 else
-   MUMPS_LIB += -ldmumps
+  MUMPS_SOLVER_LIB         = -ldmumps
+  COMPLEX_MUMPS_SOLVER_LIB = -lzmumps
 endif
-MUMPS_LIB += -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
+MUMPS_LIB         = $(MUMPS_COMMON_PATH) $(MUMPS_SOLVER_LIB) $(MUMPS_COMMON_LIB)
+COMPLEX_MUMPS_LIB = $(MUMPS_COMMON_PATH) $(COMPLEX_MUMPS_SOLVER_LIB) $(MUMPS_COMMON_LIB)
 
 # STRUMPACK library configuration
 STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build

config/get_mumps_version.cpp

@@ -10,10 +10,18 @@
 // CONTRIBUTING.md for details.
 
 #ifdef MFEM_USE_SINGLE
+#ifdef MFEM_USE_COMPLEX_MUMPS
+#include "cmumps_c.h"
+#else
 #include "smumps_c.h"
+#endif
+#else
+#ifdef MFEM_USE_COMPLEX_MUMPS
+#include "zmumps_c.h"
 #else
 #include "dmumps_c.h"
 #endif
+#endif
 #include <string>
 #include <iostream>
 #include <algorithm>

config/makefile

@@ -42,7 +42,7 @@ GHV_FLAGS = $(MFEM_CXXFLAGS) $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..)
 SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)
 SMX_PATH = $(PUMI_DIR)/include/gmi_sim.h
 SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))
-MUMPS = $(MFEM_USE_MUMPS:NO=)
+MUMPS = $(if $(filter YES,$(MFEM_USE_MUMPS) $(MFEM_USE_COMPLEX_MUMPS)),YES,)
 GMV_CXX ?= $(MFEM_CXX)
 GMV = get_mumps_version
 GMV_FLAGS = $(MFEM_CXXFLAGS) $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(MUMPS_OPT))

examples/ex25p.cpp

@@ -200,7 +200,7 @@ int main(int argc, char *argv[])
    args.AddOption(&slu_solver, "-slu", "--superlu", "-no-slu",
                   "--no-superlu", "Use the SuperLU Solver.");
 #endif
-#ifdef MFEM_USE_MUMPS
+#if defined(MFEM_USE_MUMPS) || defined(MFEM_USE_COMPLEX_MUMPS)
    args.AddOption(&mumps_solver, "-mumps", "--mumps-solver", "-no-mumps",
                   "--no-mumps-solver", "Use the MUMPS Solver.");
 #endif
@@ -502,16 +502,24 @@ int main(int argc, char *argv[])
       delete A;
    }
 #endif
-#ifdef MFEM_USE_MUMPS
+#if defined(MFEM_USE_MUMPS) || defined(MFEM_USE_COMPLEX_MUMPS)
    if (!pa && mumps_solver)
    {
-      HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+      ComplexHypreParMatrix *Ahc = Ah.As<ComplexHypreParMatrix>();
+#ifdef MFEM_USE_COMPLEX_MUMPS
+      ComplexMUMPSSolver mumps(MPI_COMM_WORLD);
+      mumps.SetPrintLevel(0);
+      mumps.SetOperator(*Ahc);
+      mumps.Mult(B, X);
+#else
+      HypreParMatrix *A = Ahc->GetSystemMatrix();
       MUMPSSolver mumps(A->GetComm());
       mumps.SetPrintLevel(0);
       mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
       mumps.SetOperator(*A);
       mumps.Mult(B, X);
       delete A;
+#endif
    }
 #endif
    // 16a. Set up the parallel Bilinear form a(.,.) for the preconditioner

fem/bilininteg.cpp

@@ -2592,6 +2592,22 @@ void MixedCurlIntegrator::AssembleElementMatrix2(
       ir = &IntRules.Get(trial_fe.GetGeomType(), order);
    }
 
+   // Workspace for matrix-coefficient path
+   DenseMatrix M;
+   DenseMatrix Mcurl; // (dimc x trial_dof) when MQ is used
+
+   if (MQ)
+   {
+      // MQ only makes sense when curl(u) is vector-valued (case 1 or 3)
+      MFEM_VERIFY(dimc == dim,
+                  "MixedCurlIntegrator: MatrixCoefficient requires vector-valued curl(u) "
+                  "(3D H(curl) or 2D H1 rotated-grad case).");
+
+      M.SetSize(dimc);
+      Mcurl.SetSize(dimc, trial_dof);
+   }
+
+
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
@@ -2607,20 +2623,59 @@ void MixedCurlIntegrator::AssembleElementMatrix2(
       }
       test_fe.CalcPhysShape(Trans, shape);
       c = ip.weight*Trans.Weight();
-      if (Q)
+
+      if (MQ)
+      {
+         // apply matrix coefficient to curl(u)
+         MQ->Eval(M, Trans, ip);
+
+         for (int d = 0; d < dimc; ++d)
+         {
+            for (int jj = 0; jj < trial_dof; ++jj)
+            {
+               real_t val = 0.0;
+               for (int k = 0; k < dimc; ++k)
+               {
+                  const real_t *curl_k = &(curlshape.GetData())[k * trial_dof];
+                  val += M(d, k) * curl_k[jj];
+               }
+               Mcurl(d, jj) = val;
+            }
+         }
+      }
+      else if (Q)
       {
          c *= Q->Eval(Trans, ip);
       }
       shape *= c;
 
-      for (int d = 0; d < dimc; ++d)
+
+      if (MQ)
       {
-         real_t * curldata = &(curlshape.GetData())[d*trial_dof];
-         for (int jj = 0; jj < trial_dof; ++jj)
+         // use transformed curl
+         for (int d = 0; d < dimc; ++d)
          {
-            for (int ii = 0; ii < test_dof; ++ii)
+            for (int jj = 0; jj < trial_dof; ++jj)
             {
-               elmat(d * test_dof + ii, jj) += shape(ii) * curldata[jj];
+               const real_t cur_val = Mcurl(d, jj);
+               for (int ii = 0; ii < test_dof; ++ii)
+               {
+                  elmat(d * test_dof + ii, jj) += shape(ii) * cur_val;
+               }
+            }
+         }
+      }
+      else
+      {
+         for (int d = 0; d < dimc; ++d)
+         {
+            real_t * curldata = &(curlshape.GetData())[d*trial_dof];
+            for (int jj = 0; jj < trial_dof; ++jj)
+            {
+               for (int ii = 0; ii < test_dof; ++ii)
+               {
+                  elmat(d * test_dof + ii, jj) += shape(ii) * curldata[jj];
+               }
             }
          }
       }
@@ -3109,6 +3164,94 @@ void VectorDiffusionIntegrator::AssembleElementMatrix(
    }
 }
 
+void VectorDiffusionIntegrator::AssembleElementMatrix2(const FiniteElement
+                                                       &trial_fe,
+                                                       const FiniteElement &test_fe,
+                                                       ElementTransformation &Trans,
+                                                       DenseMatrix &elmat)
+{
+   int tr_nd = trial_fe.GetDof();
+   int te_nd = test_fe.GetDof();
+   dim = trial_fe.GetDim();
+   int spaceDim = Trans.GetSpaceDim();
+   bool square = (dim == spaceDim);
+   vdim = (vdim <= 0) ? spaceDim : vdim;
+
+   if (VQ)
+   {
+      vcoeff.SetSize(vdim);
+   }
+   else if (MQ)
+   {
+      mcoeff.SetSize(vdim);
+   }
+
+   dshape.SetSize(tr_nd, dim);
+   dshapedxt.SetSize(tr_nd, spaceDim);
+   te_dshape.SetSize(te_nd, dim);
+   te_dshapedxt.SetSize(te_nd, spaceDim);
+
+   elmat.SetSize(vdim * te_nd, vdim*tr_nd);
+   pelmat.SetSize(te_nd, tr_nd);
+
+   const IntegrationRule *ir = IntRule ? IntRule : &DiffusionIntegrator::GetRule(
+                                  trial_fe, test_fe);
+
+   elmat = 0.0;
+
+   for (int i = 0; i < ir -> GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      trial_fe.CalcDShape(ip, dshape);
+      test_fe.CalcDShape(ip, te_dshape);
+
+      Trans.SetIntPoint(&ip);
+      double w = Trans.Weight();
+      w = ip.weight / (square ? w : w*w*w);
+      // AdjugateJacobian = / adj(J),         if J is square
+      //                    \ adj(J^t.J).J^t, otherwise
+
+      Mult(dshape, Trans.AdjugateJacobian(), dshapedxt);
+      Mult(te_dshape, Trans.AdjugateJacobian(), te_dshapedxt);
+
+      if (VQ)
+      {
+         VQ->Eval(vcoeff, Trans, ip);
+         for (int k = 0; k < vdim; ++k)
+         {
+            pelmat = 0.0;
+            AddMult_a_ABt(w*vcoeff(k), te_dshapedxt, dshapedxt, pelmat);
+            elmat.AddMatrix(pelmat, te_nd*k, tr_nd*k);
+         }
+      }
+      else if (MQ)
+      {
+         MQ->Eval(mcoeff, Trans, ip);
+         for (int ii = 0; ii < vdim; ++ii)
+         {
+            for (int jj = 0; jj < vdim; ++jj)
+            {
+               pelmat = 0.0;
+               AddMult_a_ABt(w*mcoeff(ii,jj), te_dshapedxt, dshapedxt, pelmat);
+               elmat.AddMatrix(pelmat, te_nd*ii, tr_nd*jj);
+            }
+         }
+      }
+      else
+      {
+         if (Q) { w *= Q->Eval(Trans, ip); }
+         dshapedxt *= w;
+         MultABt(te_dshapedxt,dshapedxt, pelmat);
+         for (int k = 0; k < vdim; ++k)
+         {
+            elmat.AddMatrix(pelmat, te_nd*k, tr_nd*k);
+         }
+      }
+   }
+}
+
+
+
 void VectorDiffusionIntegrator::AssembleElementVector(
    const FiniteElement &el, ElementTransformation &Tr,
    const Vector &elfun, Vector &elvect)
@@ -4451,6 +4594,76 @@ void TraceIntegrator::AssembleTraceFaceMatrix(int elem,
    }
 }
 
+
+void VectorTraceIntegrator::AssembleTraceFaceMatrix(int elem,
+                                                    const FiniteElement &trial_face_fe,
+                                                    const FiniteElement &test_fe,
+                                                    FaceElementTransformations & Trans,
+                                                    DenseMatrix &elmat)
+{
+   MFEM_VERIFY(test_fe.GetMapType() == FiniteElement::VALUE,
+               "TraceIntegrator::AssembleTraceFaceMatrix: Test space should be H1");
+   MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::INTEGRAL,
+               "TraceIntegrator::AssembleTraceFaceMatrix: Trial space should be RT trace");
+
+   int i, j, face_ndof, ndof;
+   int order;
+   int spaceDim = Trans.GetSpaceDim();
+   vdim = (vdim == -1) ? spaceDim : vdim;
+
+   face_ndof = trial_face_fe.GetDof();
+   ndof = test_fe.GetDof();
+
+   face_shape.SetSize(face_ndof);
+   shape.SetSize(ndof);
+
+   elmat.SetSize(ndof*vdim, face_ndof*vdim);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      order = test_fe.GetOrder();
+      order += trial_face_fe.GetOrder();
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+   }
+
+   int iel = Trans.Elem1->ElementNo;
+   if (iel != elem)
+   {
+      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
+   }
+
+   double scale = 1.0;
+   if (iel != elem) { scale = -1.; }
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+      // Trace finite element shape function
+      trial_face_fe.CalcPhysShape(Trans,face_shape);
+
+      // Finite element shape function
+      ElementTransformation * eltrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
+      test_fe.CalcPhysShape(*eltrans, shape);
+
+      face_shape *= Trans.Weight()*ip.weight*scale;
+      for (int k = 0; k < vdim; k++)
+      {
+         for (i = 0; i < ndof; i++)
+         {
+            for (j = 0; j < face_ndof; j++)
+            {
+               elmat(i+k*ndof, j+k*face_ndof) += shape(i) * face_shape(j);
+            }
+         }
+      }
+   }
+}
+
+
 void NormalTraceIntegrator::AssembleTraceFaceMatrix(int elem,
                                                     const FiniteElement &trial_face_fe,
                                                     const FiniteElement &test_fe,

fem/bilininteg.hpp

@@ -2936,16 +2936,27 @@ class MixedCurlIntegrator : public BilinearFormIntegrator
 {
 protected:
    Coefficient *Q;
-
+   MatrixCoefficient *MQ;
 private:
    Vector shape;
    DenseMatrix dshape;
    DenseMatrix curlshape;
    DenseMatrix elmat_comp;
 public:
-   MixedCurlIntegrator() : Q{NULL} { }
-   MixedCurlIntegrator(Coefficient *q_) :  Q{q_} { }
-   MixedCurlIntegrator(Coefficient &q) :  Q{&q} { }
+   MixedCurlIntegrator() : Q(nullptr), MQ(nullptr) { }
+   // Scalar coefficient
+   explicit MixedCurlIntegrator(Coefficient *q)
+      : Q(q), MQ(nullptr) { }
+
+   explicit MixedCurlIntegrator(Coefficient &q)
+      : Q(&q), MQ(nullptr) { }
+
+   // Matrix coefficient
+   explicit MixedCurlIntegrator(MatrixCoefficient *mq)
+      : Q(nullptr), MQ(mq) { }
+
+   explicit MixedCurlIntegrator(MatrixCoefficient &mq)
+      : Q(nullptr), MQ(&mq) { }
 
    void AssembleElementMatrix2(const FiniteElement &trial_fe,
                                const FiniteElement &test_fe,
@@ -3129,9 +3140,11 @@ public:
     to be the spatial dimension (i.e. 2-dimension or 3-dimension). */
 class VectorDiffusionIntegrator : public BilinearFormIntegrator
 {
-   int vdim = -1;
+private:
    DenseMatrix dshape, dshapedxt, pelmat;
-   DenseMatrix mcoeff;
+   DenseMatrix te_dshape, te_dshapedxt;
+   int vdim = -1;
+   DenseMatrix mcoeff,dshapedxt_m;
    Vector vcoeff;
 
 protected:
@@ -3193,6 +3206,10 @@ public:
    void AssembleElementMatrix(const FiniteElement &el,
                               ElementTransformation &Trans,
                               DenseMatrix &elmat) override;
+   void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                               const FiniteElement &test_fe,
+                               ElementTransformation &Trans,
+                               DenseMatrix &elmat) override;
    void AssembleElementVector(const FiniteElement &el,
                               ElementTransformation &Tr,
                               const Vector &elfun, Vector &elvect) override;
@@ -3798,9 +3815,29 @@ public:
                                 DenseMatrix &elmat);
 };
 
-/** Integrator for the form: $ \langle v, w \cdot n \rangle $ over a face (the interface) where
-    the trial variable $v$ is defined on the interface ($H^{1/2}$, i.e., trace of $H^1$)
-    and the test variable $w$ is in an $H(div)$-conforming space. */
+/** Integrator for the DPG form: < v, w > over a face (the interface) where
+    the trial variable v is defined on the interface
+    ((H^-1/2)^vdim i.e., vᵢ :=uᵢ⋅n (for normal trace of H(div)^vdim)
+    and the test variable w is in an dim copies of H1-conforming space. */
+class VectorTraceIntegrator : public BilinearFormIntegrator
+{
+private:
+   Vector face_shape, shape;
+   int vdim = -1;
+public:
+   VectorTraceIntegrator() { }
+   void AssembleTraceFaceMatrix(int elem,
+                                const FiniteElement &trial_face_fe,
+                                const FiniteElement &test_fe,
+                                FaceElementTransformations &Trans,
+                                DenseMatrix &elmat);
+   void SetVDim(int vdim_) { vdim = vdim_; }
+};
+
+
+/** Integrator for the form: < v, w.n > over a face (the interface) where
+    the trial variable v is defined on the interface (H^1/2, i.e., trace of H1)
+    and the test variable w is in an H(div)-conforming space. */
 class NormalTraceIntegrator : public BilinearFormIntegrator
 {
 private:

fem/fe_coll.hpp

@@ -250,6 +250,12 @@ public:
        its GetOrder() method. */
    virtual FiniteElementCollection *Clone(int p) const;
 
+   virtual int GetConstructorOrder() const
+   {
+      MFEM_ABORT("Collection " << Name() << " does not support GetConstructorOrder");
+      return -1;
+   }
+
 protected:
    const int base_p; ///< Order as returned by GetOrder().
 
@@ -314,6 +320,9 @@ public:
    FiniteElementCollection *Clone(int p) const override
    { return new H1_FECollection(p, dim, b_type); }
 
+   int GetConstructorOrder() const override
+   { return base_p; }
+
    virtual ~H1_FECollection();
 };
 
@@ -343,6 +352,10 @@ class H1_Trace_FECollection : public H1_FECollection
 public:
    H1_Trace_FECollection(const int p, const int dim,
                          const int btype = BasisType::GaussLobatto);
+
+   FiniteElementCollection *Clone(int p) const override
+   { return new H1_Trace_FECollection(p, dim+1, b_type); }
+
 };
 
 /// Arbitrary order "L2-conforming" discontinuous finite elements.
@@ -396,6 +409,9 @@ public:
    FiniteElementCollection *Clone(int p) const override
    { return new L2_FECollection(p, dim, b_type, m_type); }
 
+   int GetConstructorOrder() const override
+   { return base_p; }
+
    virtual ~L2_FECollection();
 };
 
@@ -456,6 +472,9 @@ public:
    FiniteElementCollection *Clone(int p) const override
    { return new RT_FECollection(p, dim, cb_type, ob_type); }
 
+   int GetConstructorOrder() const override
+   { return base_p-1; }
+
    virtual ~RT_FECollection();
 };
 
@@ -536,6 +555,9 @@ public:
    FiniteElementCollection *Clone(int p) const override
    { return new ND_FECollection(p, dim, cb_type, ob_type); }
 
+   int GetConstructorOrder() const override
+   { return dim>1 ? base_p : base_p+1; }
+
    virtual ~ND_FECollection();
 };
 
@@ -548,6 +570,9 @@ public:
    ND_Trace_FECollection(const int p, const int dim,
                          const int cb_type = BasisType::GaussLobatto,
                          const int ob_type = BasisType::GaussLegendre);
+
+   FiniteElementCollection *Clone(int p) const override
+   { return new ND_Trace_FECollection(p, dim+1, cb_type, ob_type); }
 };
 
 /// Arbitrary order 3D H(curl)-conforming Nedelec finite elements in 1D.

fem/gridfunc.cpp

@@ -2262,6 +2262,80 @@ void GridFunction::AccumulateAndCountBdrTangentValues(
    }
 }
 
+void GridFunction::AccumulateAndCountTraceValues(
+   Coefficient *coeff[], VectorCoefficient *vcoeff,
+   Array<int> &values_counter)
+{
+   Array<int> vdofs;
+   Vector vc;
+
+   values_counter.SetSize(Size());
+   values_counter = 0;
+
+   const int vdim = fes->GetVDim();
+   HostReadWrite();
+
+   for (int i = 0; i < fes->GetMesh()->GetNumFaces(); i++)
+   {
+
+      const FiniteElement *fe = fes->GetFaceElement(i);
+      const int fdof = fe->GetDof();
+      ElementTransformation *transf = fes->GetMesh()->GetFaceTransformation(i);
+      const IntegrationRule &ir = fe->GetNodes();
+      fes->GetFaceVDofs(i, vdofs);
+
+      for (int j = 0; j < fdof; j++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(j);
+         transf->SetIntPoint(&ip);
+         if (vcoeff) { vcoeff->Eval(vc, *transf, ip); }
+         for (int d = 0; d < vdim; d++)
+         {
+            if (!vcoeff && !coeff[d]) { continue; }
+
+            real_t val = vcoeff ? vc(d) : coeff[d]->Eval(*transf, ip);
+            int ind = vdofs[fdof*d+j];
+            if ( ind < 0 )
+            {
+               val = -val, ind = -1-ind;
+            }
+            if (++values_counter[ind] == 1)
+            {
+               (*this)(ind) = val;
+            }
+            else
+            {
+               (*this)(ind) += val;
+            }
+         }
+      }
+   }
+}
+
+void GridFunction::AccumulateAndCountTraceTangentValues(
+   VectorCoefficient &vcoeff, Array<int> &values_counter)
+{
+   const FiniteElement *fe;
+   ElementTransformation *T;
+   Array<int> dofs;
+   Vector lvec;
+
+   values_counter.SetSize(Size());
+   values_counter = 0;
+
+   HostReadWrite();
+
+   for (int i = 0; i < fes->GetMesh()->GetNumFaces(); i++)
+   {
+      fe = fes->GetFaceElement(i);
+      T = fes->GetMesh()->GetFaceTransformation(i);
+      fes->GetFaceVDofs(i, dofs);
+      lvec.SetSize(fe->GetDof());
+      fe->Project(vcoeff, *T, lvec);
+      accumulate_dofs(dofs, lvec, *this, values_counter);
+   }
+}
+
 void GridFunction::ComputeMeans(AvgType type, Array<int> &zones_per_vdof)
 {
    switch (type)
@@ -2700,6 +2774,54 @@ void GridFunction::ProjectCoefficient(VectorCoefficient &vcoeff,
    }
 }
 
+void GridFunction::ProjectTraceCoefficient(Coefficient *coeff[])
+{
+   Array<int> values_counter;
+   AccumulateAndCountTraceValues(coeff, NULL, values_counter);
+   ComputeMeans(ARITHMETIC, values_counter);
+}
+
+void GridFunction::ProjectTraceCoefficient(VectorCoefficient &vcoeff)
+{
+   Array<int> values_counter;
+   AccumulateAndCountTraceValues(NULL, &vcoeff, values_counter);
+   ComputeMeans(ARITHMETIC, values_counter);
+}
+
+void GridFunction::ProjectTraceCoefficientNormal(VectorCoefficient &vcoeff)
+{
+   const FiniteElement *fe;
+   ElementTransformation *T;
+   Array<int> dofs;
+   int dim = vcoeff.GetVDim();
+   Vector vc(dim), nor(dim), lvec;
+
+   for (int i = 0; i < fes->GetMesh()->GetNumFaces(); i++)
+   {
+      fe = fes->GetFaceElement(i);
+      T = fes->GetMesh()->GetFaceTransformation(i);
+      const IntegrationRule &ir = fe->GetNodes();
+      lvec.SetSize(fe->GetDof());
+      for (int j = 0; j < ir.GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(j);
+         T->SetIntPoint(&ip);
+         vcoeff.Eval(vc, *T, ip);
+         CalcOrtho(T->Jacobian(), nor);
+         lvec(j) = (vc * nor);
+      }
+      fes->GetFaceVDofs(i, dofs);
+      SetSubVector(dofs, lvec);
+   }
+}
+
+void GridFunction::ProjectTraceCoefficientTangent(VectorCoefficient &vcoeff)
+{
+   Array<int> values_counter;
+   AccumulateAndCountTraceTangentValues(vcoeff, values_counter);
+   ComputeMeans(ARITHMETIC, values_counter);
+}
+
 void GridFunction::ProjectCoefficientGlobalL2(VectorCoefficient &vcoeff,
                                               real_t rtol, int iter)
 {

fem/gridfunc.hpp

@@ -557,6 +557,13 @@ protected:
                                            const Array<int> &bdr_attr,
                                            Array<int> &values_counter);
 
+   void AccumulateAndCountTraceValues(Coefficient *coeff[],
+                                      VectorCoefficient *vcoeff,
+                                      Array<int> &values_counter);
+
+   void AccumulateAndCountTraceTangentValues(VectorCoefficient &vcoeff,
+                                             Array<int> &values_counter);
+
    // Complete the computation of averages; called e.g. after
    // AccumulateAndCountZones().
    void ComputeMeans(AvgType type, Array<int> &zones_per_vdof);
@@ -642,6 +649,21 @@ public:
       ProjectBdrCoefficient(&coeff_p, attr);
    }
 
+   /// Project a Coefficient on a GridFunction defined on H1 trace space
+   void ProjectTraceCoefficient(Coefficient *coeff[]);
+   void ProjectTraceCoefficient(Coefficient &coeff)
+   {
+      Coefficient *coeff_p = &coeff;
+      ProjectTraceCoefficient(&coeff_p);
+   }
+   /// Project a VectorCoefficient on a GridFunction defined on a Vector H1 trace space
+   void ProjectTraceCoefficient(VectorCoefficient &vcoeff);
+   /// Project a VectorCoefficient on a GridFunction defined on a RT trace space
+   void ProjectTraceCoefficientNormal(VectorCoefficient &vcoeff);
+   /// Project a VectorCoefficient on a GridFunction defined on a ND trace space
+   void ProjectTraceCoefficientTangent(VectorCoefficient &vcoeff);
+
+
    /** @brief Project a VectorCoefficient on the GridFunction, modifying only
        DOFs on the boundary associated with the boundary attributes marked in
        the @a attr array. */

fem/transfer.cpp

@@ -2057,6 +2057,10 @@ TransferOperator::TransferOperator(const FiniteElementSpace& lFESpace_,
    : Operator(hFESpace_.GetVSize(), lFESpace_.GetVSize())
 {
    bool isvar_order = lFESpace_.IsVariableOrder() || hFESpace_.IsVariableOrder();
+   bool is_trace_space =
+      (dynamic_cast<const H1_Trace_FECollection*>(lFESpace_.FEColl()) ||
+       dynamic_cast<const ND_Trace_FECollection*>(lFESpace_.FEColl()) ||
+       dynamic_cast<const RT_Trace_FECollection*>(lFESpace_.FEColl()));
    if (lFESpace_.FEColl() == hFESpace_.FEColl() && !isvar_order)
    {
       OperatorPtr P(Operator::ANY_TYPE);
@@ -2066,6 +2070,7 @@ TransferOperator::TransferOperator(const FiniteElementSpace& lFESpace_,
    }
    else if (lFESpace_.GetVDim() == 1
             && hFESpace_.GetVDim() == 1
+            && !is_trace_space
             && dynamic_cast<const TensorBasisElement*>(lFESpace_.GetTypicalFE())
             && dynamic_cast<const TensorBasisElement*>(hFESpace_.GetTypicalFE())
             && !isvar_order
@@ -2096,15 +2101,244 @@ void TransferOperator::MultTranspose(const Vector& x, Vector& y) const
 
 
 PRefinementTransferOperator::PRefinementTransferOperator(
-   const FiniteElementSpace& lFESpace_, const FiniteElementSpace& hFESpace_)
+   const FiniteElementSpace& lFESpace_, const FiniteElementSpace& hFESpace_,
+   bool assemble_matrix)
    : Operator(hFESpace_.GetVSize(), lFESpace_.GetVSize()), lFESpace(lFESpace_),
      hFESpace(hFESpace_)
 {
    isvar_order = lFESpace_.IsVariableOrder() || hFESpace_.IsVariableOrder();
+
+   MFEM_VERIFY(lFESpace.FEColl()->GetContType() ==
+               hFESpace.FEColl()->GetContType(),
+               "Incompatible finite element space continuity types.");
+
+   is_trace_space =
+      (dynamic_cast<const H1_Trace_FECollection*>(lFESpace.FEColl()) ||
+       dynamic_cast<const ND_Trace_FECollection*>(lFESpace.FEColl()) ||
+       dynamic_cast<const RT_Trace_FECollection*>(lFESpace.FEColl()));
+
+   if (assemble_matrix) { AssembleMatrix(); }
+
 }
 
+void PRefinementTransferOperator::AssembleMatrix()
+{
+   Mesh* mesh = hFESpace.GetMesh();
+   const int nL = lFESpace.GetVSize();
+   const int nH = hFESpace.GetVSize();
+
+   P.reset(new SparseMatrix(nH, nL));
+   Array<int> l_dofs, h_dofs, l_vdofs, h_vdofs;
+   DenseMatrix loc_prol;
+
+   Geometry::Type cached_geom = Geometry::INVALID;
+   const FiniteElement* h_fe = nullptr;
+   const FiniteElement* l_fe = nullptr;
+   IsoparametricTransformation T;
+
+   int vdim = lFESpace.GetVDim();
+
+   const int iend = (is_trace_space) ? mesh->GetNumFaces() : mesh->GetNE();
+   DofTransformation doftrans_h, doftrans_l;
+   Vector w(nH); w = 0.0;
+
+   for (int i = 0; i < iend; i++)
+   {
+      if (is_trace_space)
+      {
+         hFESpace.GetFaceDofs(i, h_dofs);
+         lFESpace.GetFaceDofs(i, l_dofs);
+      }
+      else
+      {
+         hFESpace.GetElementDofs(i, h_dofs, doftrans_h);
+         lFESpace.GetElementDofs(i, l_dofs, doftrans_l);
+      }
+
+      const Geometry::Type geom = (is_trace_space) ? mesh->GetFaceGeometry(i)
+                                  : mesh->GetElementBaseGeometry(i);
+
+      if (geom != cached_geom || isvar_order)
+      {
+         h_fe = (is_trace_space) ? hFESpace.GetFaceElement(i) : hFESpace.GetFE(i);
+         l_fe = (is_trace_space) ? lFESpace.GetFaceElement(i) : lFESpace.GetFE(i);
+         T.SetIdentityTransformation(h_fe->GetGeomType());
+         h_fe->GetTransferMatrix(*l_fe, T, loc_prol);
+         cached_geom = geom;
+      }
+
+      DenseMatrix Aeff(loc_prol);
+      TransformPrimal(doftrans_h, doftrans_l, Aeff);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         l_dofs.Copy(l_vdofs);
+         lFESpace.DofsToVDofs(vd, l_vdofs);
+
+         h_dofs.Copy(h_vdofs);
+         hFESpace.DofsToVDofs(vd, h_vdofs);
+
+         Aeff.AdjustDofDirection(h_vdofs, l_vdofs);
+
+         P->AddSubMatrix(h_vdofs, l_vdofs, Aeff);
+
+         for (int rr = 0; rr < h_vdofs.Size(); rr++)
+         {
+            w(h_vdofs[rr]) += 1.0;
+         }
+
+      }
+   }
+
+   P->Finalize();
+
+   Vector inv_w(nH);
+   for (int i = 0; i < nH; i++)
+   {
+      inv_w(i) = (w(i) > 0.0) ? (1.0 / w(i)) : 1.0;
+   }
+
+   P->ScaleRows(inv_w);
+
+   assembled = true;
+
+}
+
+std::unique_ptr<SparseMatrix>
+PRefinementTransferOperator::BuildConformingTransferMatrix() const
+{
+   MFEM_VERIFY(assembled && P, "Matrix path requires assembled P.");
+
+   const SparseMatrix *Pl = lFESpace.GetConformingProlongation();
+   const SparseMatrix *Rh = hFESpace.GetRestrictionMatrix();
+
+   if (Pl && Rh)
+   {
+      SparseMatrix *RhP   = mfem::Mult(*Rh, *P);
+      SparseMatrix *RhPPl = mfem::Mult(*RhP, *Pl);
+      delete RhP;
+      return std::unique_ptr<SparseMatrix>(RhPPl);
+   }
+   else if (Pl)
+   {
+      return std::unique_ptr<SparseMatrix>(mfem::Mult(*P, *Pl));
+   }
+   else if (Rh)
+   {
+      return std::unique_ptr<SparseMatrix>(mfem::Mult(*Rh, *P));
+   }
+   else
+   {
+      return std::make_unique<SparseMatrix>(*P);
+   }
+}
+
+std::unique_ptr<Operator>
+PRefinementTransferOperator::BuildConformingTransferOperator() const
+{
+   const Operator *Pl = lFESpace.GetProlongationMatrix();
+   const Operator *Rh = hFESpace.GetRestrictionOperator();
+
+   if (Pl && Rh)
+   {
+      return std::make_unique<TripleProductOperator>(Rh,
+                                                     const_cast<PRefinementTransferOperator*>(this), Pl,
+                                                     false, false, false);
+   }
+   else if (Pl)
+   {
+      return std::make_unique<ProductOperator>
+             (const_cast<PRefinementTransferOperator*>(this), Pl,
+              false, false);
+   }
+   else if (Rh)
+   {
+      return std::make_unique<ProductOperator>(Rh,
+                                               const_cast<PRefinementTransferOperator*>(this),
+                                               false, false);
+   }
+   else
+   {
+      // return nullptr to mean "identity/no-op wrapper", i.e. use `this`
+      return nullptr;
+   }
+}
+
+Operator *
+PRefinementTransferOperator::GetTrueTransferOperator()
+{
+   if (tP) { return tP.get(); }
+#ifdef MFEM_USE_MPI
+   const ParFiniteElementSpace* lpfes = dynamic_cast<const ParFiniteElementSpace*>
+                                        (&lFESpace);
+   const ParFiniteElementSpace* hpfes = dynamic_cast<const ParFiniteElementSpace*>
+                                        (&hFESpace);
+   bool parallel = (lpfes) && (hpfes);
+
+   if (parallel)
+   {
+      if (assembled)
+      {
+         HypreParMatrix * Pl = lpfes->Dof_TrueDof_Matrix();
+         const SparseMatrix * Rh = hpfes->GetRestrictionMatrix();
+         // Rh * P
+         SparseMatrix * RhP = mfem::Mult(*Rh, *P);
+         HypreParMatrix * RhPh = new HypreParMatrix(hpfes->GetComm(),
+                                                    hpfes->GlobalTrueVSize(), lpfes->GlobalVSize(),
+                                                    hpfes->GetTrueDofOffsets(), lpfes->GetDofOffsets(), RhP);
+         HypreStealOwnership(*RhPh, *RhP);
+         delete RhP;
+         HypreParMatrix * tmp = ParMult(RhPh, Pl, true);
+         delete RhPh;
+         tP.reset(tmp);
+         return tP.get();
+      }
+      else
+      {
+         auto Pl = lpfes->GetProlongationMatrix();
+         auto Rh = hpfes->GetRestrictionOperator();
+         tP = std::make_unique<TripleProductOperator>(Rh, this, Pl, false, false, false);
+         return tP.get();
+      }
+   }
+   else
+   {
+      if (assembled)
+      {
+         auto M = BuildConformingTransferMatrix();
+         tP.reset(M.release());
+         return tP.get();
+      }
+      else
+      {
+         tP = BuildConformingTransferOperator();
+         return tP ? tP.get() : this;
+      }
+   }
+#else
+   {
+      if (assembled)
+      {
+         auto M = BuildConformingTransferMatrix();
+         tP.reset(M.release());
+         return tP.get();
+      }
+      else
+      {
+         tP = BuildConformingTransferOperator();
+         return tP ? tP.get() : this;
+      }
+   }
+#endif
+}
+
+
 void PRefinementTransferOperator::Mult(const Vector& x, Vector& y) const
 {
+   y = 0.0;
+
+   if (assembled) { P->Mult(x, y); return; }
+
    Mesh* mesh = hFESpace.GetMesh();
    Array<int> l_dofs, h_dofs, l_vdofs, h_vdofs;
    DenseMatrix loc_prol;
@@ -2117,19 +2351,31 @@ void PRefinementTransferOperator::Mult(const Vector& x, Vector& y) const
 
    int vdim = lFESpace.GetVDim();
 
-   y = 0.0;
 
    DofTransformation doftrans_h, doftrans_l;
-   for (int i = 0; i < mesh->GetNE(); i++)
-   {
-      hFESpace.GetElementDofs(i, h_dofs, doftrans_h);
-      lFESpace.GetElementDofs(i, l_dofs, doftrans_l);
 
-      const Geometry::Type geom = mesh->GetElementBaseGeometry(i);
+   int iend = (is_trace_space) ? mesh->GetNumFaces() : mesh->GetNE();
+
+   for (int i = 0; i < iend; i++)
+   {
+      if (is_trace_space)
+      {
+         hFESpace.GetFaceDofs(i, h_dofs);
+         lFESpace.GetFaceDofs(i, l_dofs);
+      }
+      else
+      {
+         hFESpace.GetElementDofs(i, h_dofs, doftrans_h);
+         lFESpace.GetElementDofs(i, l_dofs, doftrans_l);
+      }
+
+      const Geometry::Type geom = (is_trace_space) ? mesh->GetFaceGeometry(i)
+                                  : mesh->GetElementBaseGeometry(i);
+
       if (geom != cached_geom || isvar_order)
       {
-         h_fe = hFESpace.GetFE(i);
-         l_fe = lFESpace.GetFE(i);
+         h_fe = (is_trace_space) ? hFESpace.GetFaceElement(i) : hFESpace.GetFE(i);
+         l_fe = (is_trace_space) ? lFESpace.GetFaceElement(i) : lFESpace.GetFE(i);
          T.SetIdentityTransformation(h_fe->GetGeomType());
          h_fe->GetTransferMatrix(*l_fe, T, loc_prol);
          subY.SetSize(loc_prol.Height());
@@ -2144,6 +2390,7 @@ void PRefinementTransferOperator::Mult(const Vector& x, Vector& y) const
          hFESpace.DofsToVDofs(vd, h_vdofs);
          x.GetSubVector(l_vdofs, subX);
          doftrans_l.InvTransformPrimal(subX);
+
          loc_prol.Mult(subX, subY);
          doftrans_h.TransformPrimal(subY);
          y.SetSubVector(h_vdofs, subY);
@@ -2156,6 +2403,12 @@ void PRefinementTransferOperator::MultTranspose(const Vector& x,
 {
    y = 0.0;
 
+   if (assembled)
+   {
+      P->MultTranspose(x, y);
+      return;
+   }
+
    Mesh* mesh = hFESpace.GetMesh();
    Array<int> l_dofs, h_dofs, l_vdofs, h_vdofs;
    DenseMatrix loc_prol;
@@ -2173,16 +2426,28 @@ void PRefinementTransferOperator::MultTranspose(const Vector& x,
 
    DofTransformation doftrans_h, doftrans_l;
 
-   for (int i = 0; i < mesh->GetNE(); i++)
-   {
-      hFESpace.GetElementDofs(i, h_dofs, doftrans_h);
-      lFESpace.GetElementDofs(i, l_dofs, doftrans_l);
+   int iend = (is_trace_space) ? mesh->GetNumFaces() : mesh->GetNE();
+
+   for (int i = 0; i < iend; i++)
+   {
+      if (is_trace_space)
+      {
+         hFESpace.GetFaceDofs(i, h_dofs);
+         lFESpace.GetFaceDofs(i, l_dofs);
+      }
+      else
+      {
+         hFESpace.GetElementDofs(i, h_dofs, doftrans_h);
+         lFESpace.GetElementDofs(i, l_dofs, doftrans_l);
+      }
+
+      const Geometry::Type geom = (is_trace_space) ? mesh->GetFaceGeometry(i)
+                                  : mesh->GetElementBaseGeometry(i);
 
-      const Geometry::Type geom = mesh->GetElementBaseGeometry(i);
       if (geom != cached_geom || isvar_order)
       {
-         h_fe = hFESpace.GetFE(i);
-         l_fe = lFESpace.GetFE(i);
+         h_fe = (is_trace_space) ? hFESpace.GetFaceElement(i) : hFESpace.GetFE(i);
+         l_fe = (is_trace_space) ? lFESpace.GetFaceElement(i) : lFESpace.GetFE(i);
          T.SetIdentityTransformation(h_fe->GetGeomType());
          h_fe->GetTransferMatrix(*l_fe, T, loc_prol);
          loc_prol.Transpose();

fem/transfer.hpp

@@ -569,6 +569,13 @@ private:
    const FiniteElementSpace& lFESpace;
    const FiniteElementSpace& hFESpace;
    bool isvar_order;
+   bool is_trace_space;
+   bool assembled = false;
+   std::unique_ptr<SparseMatrix> P;
+   mutable std::unique_ptr<Operator> tP;
+
+   std::unique_ptr<SparseMatrix> BuildConformingTransferMatrix() const;
+   std::unique_ptr<Operator> BuildConformingTransferOperator() const;
 
 public:
    /// @brief Constructs a transfer operator from \p lFESpace to \p hFESpace
@@ -577,11 +584,23 @@ public:
        The underlying finite elements need to implement the GetTransferMatrix
        methods. */
    PRefinementTransferOperator(const FiniteElementSpace& lFESpace_,
-                               const FiniteElementSpace& hFESpace_);
+                               const FiniteElementSpace& hFESpace_,
+                               bool assemble_matrix = false);
+
+
+   Operator * GetTrueTransferOperator();
+   const Operator * GetTrueTransferOperator() const
+   {
+      return const_cast<PRefinementTransferOperator*>(this)
+             ->GetTrueTransferOperator();
+   }
 
    /// Destructor
    virtual ~PRefinementTransferOperator() { }
 
+
+   void AssembleMatrix();
+
    /// @brief Interpolation or prolongation of a vector \p x corresponding to
    /// the coarse space to the vector \p y corresponding to the fine space.
    void Mult(const Vector& x, Vector& y) const override;

linalg/blockoperator.cpp

@@ -14,6 +14,7 @@
 #include "blockvector.hpp"
 #include "blockoperator.hpp"
 
+
 namespace mfem
 {
 
@@ -129,6 +130,33 @@ void BlockOperator::MultTranspose(const Vector &x, Vector &y) const
    }
 }
 
+#ifdef MFEM_USE_MPI
+
+HypreParMatrix * BlockOperator::GetMonolithicHypreParMatrix(
+   Array2D<real_t> *blockCoeff) const
+{
+   Array2D<const HypreParMatrix*> blocks(nRowBlocks, nColBlocks);
+   for (int i = 0; i < nRowBlocks; ++i)
+   {
+      for (int j = 0; j < nColBlocks; ++j)
+      {
+         if (IsZeroBlock(i, j))
+         {
+            blocks(i, j) = nullptr;
+         }
+         else
+         {
+            auto mat = dynamic_cast<const HypreParMatrix*>(&GetBlock(i, j));
+            MFEM_VERIFY(mat,"BlockOperator block (" << i << "," << j
+                        << ") is not a HypreParMatrix.");
+            blocks(i, j) = mat;
+         }
+      }
+   }
+   return HypreParMatrixFromBlocks(blocks, blockCoeff);
+}
+#endif
+
 BlockOperator::~BlockOperator()
 {
    if (owns_blocks)
@@ -381,4 +409,161 @@ BlockLowerTriangularPreconditioner::~BlockLowerTriangularPreconditioner()
    }
 }
 
-} // namespace mfem
+BlockTriangularSymmetricPreconditioner::BlockTriangularSymmetricPreconditioner(
+   const Array<int> & offsets_)
+   : Solver(offsets_.Last()),
+     owns_blocks(0),
+     nBlocks(offsets_.Size() - 1),
+     offsets(0),
+     ops(nBlocks, nBlocks),
+     coef(nBlocks, nBlocks)
+{
+   ops = static_cast<Operator *>(NULL);
+   offsets.MakeRef(offsets_);
+}
+
+void BlockTriangularSymmetricPreconditioner::SetDiagonalBlock(int iblock,
+                                                              Operator *op, real_t c)
+{
+   MFEM_VERIFY(offsets[iblock+1] - offsets[iblock] == op->Height() &&
+               offsets[iblock+1] - offsets[iblock] == op->Width(),
+               "incompatible Operator dimensions");
+
+   SetBlock(iblock, iblock, op);
+   coef(iblock, iblock) = c;
+}
+
+void BlockTriangularSymmetricPreconditioner::SetBlock(int iRow, int iCol,
+                                                      Operator *op, real_t c)
+{
+   MFEM_VERIFY(offsets[iRow+1] - offsets[iRow] == op->NumRows() &&
+               offsets[iCol+1] - offsets[iCol] == op->NumCols(),
+               "incompatible Operator dimensions");
+
+   ops(iRow, iCol) = op;
+   coef(iRow, iCol) = c;
+}
+
+// Operator application
+
+void BlockTriangularSymmetricPreconditioner::ForwardPass(const Vector & x,
+                                                         Vector & y) const
+{
+   // Forward sweep: Solve for y1, then y2
+   for (int iRow = 0; iRow < nBlocks; ++iRow)
+   {
+      tmp.SetSize(offsets[iRow + 1] - offsets[iRow]);
+      tmp2.SetSize(offsets[iRow + 1] - offsets[iRow]);
+      tmp2 = 0.0;
+      tmp2 += xblock.GetBlock(iRow);  // tmp2 = xblock(iRow)
+
+      // Process the lower triangular part (jCol < iRow)
+      for (int jCol = 0; jCol < iRow; ++jCol)
+      {
+         if (ops(iRow, jCol))
+         {
+            ops(iRow, jCol)->Mult(yblock.GetBlock(jCol),
+                                  tmp);  // tmp = A(iRow,jCol) * yblock(jCol)
+            tmp*= coef(iRow,jCol);  // tmp *= c
+            tmp2 -= tmp;  // tmp2 -= A(iRow, jCol) * yblock(jCol)
+         }
+      }
+
+      // Apply the diagonal block
+      if (ops(iRow, iRow))
+      {
+         ops(iRow, iRow)->Mult(tmp2,
+                               yblock.GetBlock(iRow));
+         yblock.GetBlock(iRow) *= coef(iRow,
+                                       iRow);  // yblock(iRow) = A(iRow,iRow)^-1 * tmp2
+      }
+      else
+      {
+         yblock.GetBlock(iRow) = tmp2;  // If no diagonal operator, set yblock directly
+      }
+   }
+}
+
+void BlockTriangularSymmetricPreconditioner::BackwardPass(const Vector & x,
+                                                          Vector & y) const
+{
+   // Backward sweep: Adjust y1 based on y2
+   for (int iRow = nBlocks - 1; iRow >= 0; --iRow)
+   {
+      tmp.SetSize(offsets[iRow + 1] - offsets[iRow]);
+      tmp2.SetSize(offsets[iRow + 1] - offsets[iRow]);
+      tmp2 = 0.0;
+      tmp2 += xblock.GetBlock(iRow);  // tmp2 = yblock(iRow) from forward sweep
+
+      // Process the upper triangular part (jCol > iRow)
+      for (int jCol = iRow + 1; jCol < nBlocks; ++jCol)
+      {
+         if (ops(iRow, jCol))
+         {
+            ops(iRow, jCol)->Mult(yblock.GetBlock(jCol),
+                                  tmp);  // tmp = A(iRow,jCol) * yblock(jCol)
+            tmp *= coef(iRow,jCol);  // tmp *= c
+            tmp2 -= tmp;  // tmp2 -= A(iRow,jCol) * yblock(jCol)
+         }
+      }
+
+      // Reapply diagonal block to correct y1
+      if (ops(iRow, iRow))
+      {
+         ops(iRow, iRow)->Mult(tmp2,
+                               yblock.GetBlock(iRow));  // Final correction for yblock(iRow)
+         yblock.GetBlock(iRow) *= coef(iRow,
+                                       iRow);  // yblock(iRow) = A(iRow,iRow)^-1 * tmp2
+      }
+      else
+      {
+         yblock.GetBlock(iRow) = tmp2;  // If no diagonal operator, set yblock directly
+      }
+   }
+}
+
+
+
+void BlockTriangularSymmetricPreconditioner::Mult(const Vector & x,
+                                                  Vector & y) const
+{
+   MFEM_ASSERT(x.Size() == width, "incorrect input Vector size");
+   MFEM_ASSERT(y.Size() == height, "incorrect output Vector size");
+
+   // Update block views of the vectors y and x using offsets
+   yblock.Update(y.GetData(), offsets);
+   xblock.Update(x.GetData(), offsets);
+
+   // Initialize y to zero
+   y = 0.0;
+   ForwardPass(x,y);
+
+   // Update Residual
+   r.SetSize(x.Size());
+   r = 0.0; r+=x;
+   Op->AddMult(y,r,-1.0);
+
+   Vector y1(y);
+   yblock.Update(y1.GetData(), offsets);
+   xblock.Update(r.GetData(), offsets);
+   BackwardPass(r,y1);
+   y+=y1;
+}
+
+BlockTriangularSymmetricPreconditioner::~BlockTriangularSymmetricPreconditioner()
+{
+   if (owns_blocks)
+   {
+      for (int iRow=0; iRow < nBlocks; ++iRow)
+      {
+         for (int jCol=0; jCol < nBlocks; ++jCol)
+         {
+            delete ops(jCol,iRow);
+         }
+      }
+   }
+}
+
+
+
+}

linalg/blockoperator.hpp

@@ -16,6 +16,9 @@
 #include "../general/array.hpp"
 #include "operator.hpp"
 #include "blockvector.hpp"
+#ifdef MFEM_USE_MPI
+#include "hypre.hpp"
+#endif
 
 namespace mfem
 {
@@ -105,6 +108,13 @@ public:
    /// Action of the transpose operator
    void MultTranspose (const Vector & x, Vector & y) const override;
 
+#ifdef MFEM_USE_MPI
+   /// Returns a monolithic HypreParMatrix formed by merging the blocks of
+   //  this BlockOperator, assuming every block is a HypreParMatrix.
+   HypreParMatrix * GetMonolithicHypreParMatrix(Array2D<real_t> *blockCoeff=NULL)
+   const;
+#endif
+
    ~BlockOperator();
 
    //! Controls the ownership of the blocks: if nonzero, BlockOperator will
@@ -288,6 +298,82 @@ private:
    mutable Vector tmp2;
 };
 
+class BlockTriangularSymmetricPreconditioner : public Solver
+{
+private:
+   const Operator * Op;
+public:
+   //! Constructor for BlockTriangularSymmetricPreconditioners with the same
+   //! block-structure for rows and columns.
+   /**
+    *  @param offsets  Offsets that mark the start of each row/column block
+    *                  (size nBlocks+1).
+    *
+    *  @note BlockTriangularSymmetricPreconditioner will not own/copy the data
+    *  contained in @a offsets.
+    */
+   BlockTriangularSymmetricPreconditioner(const Array<int> & offsets);
+
+   //! Add block op in the block-entry (iblock, iblock).
+   /**
+    * @param iblock  The block will be inserted in location (iblock, iblock).
+    * @param op      The Operator to be inserted.
+    * @param c       The coefficient to be applied to the block.
+    */
+   void SetDiagonalBlock(int iblock, Operator *op, real_t c = 1.0);
+   //! Add a block opt in the block-entry (iblock, jblock).
+   /**
+    * @param iRow, iCol  The block will be inserted in location (iRow, iCol).
+    * @param op          The Operator to be inserted.
+    * @param c           The coefficient to be applied to the block.
+   */
+   void SetBlock(int iRow, int iCol, Operator *op, real_t c = 1.0);
+   //! This method is present since required by the abstract base class Solver
+   virtual void SetOperator(const Operator &op) {Op = &op;}
+
+   //! Return the number of blocks
+   int NumBlocks() const { return nBlocks; }
+
+   //! Return a reference to block i,j.
+   Operator & GetBlock(int iblock, int jblock)
+   { MFEM_VERIFY(ops(iblock,jblock), ""); return *ops(iblock,jblock); }
+
+   Operator & GetDiagonalBlock(int iblock)
+   { MFEM_VERIFY(ops(iblock,iblock), ""); return *ops(iblock,iblock); }
+
+   //! Return the offsets for block starts
+   Array<int> & Offsets() { return offsets; }
+
+   /// Operator application
+   virtual void Mult (const Vector & x, Vector & y) const;
+
+   ~BlockTriangularSymmetricPreconditioner();
+
+   //! Controls the ownership of the blocks: if nonzero,
+   //! BlockTriangularSymmetricPreconditioner will delete all blocks that are set
+   //! (non-NULL); the default value is zero.
+   int owns_blocks;
+
+private:
+   //! Number of block rows/columns
+   int nBlocks;
+   //! Offsets for the starting position of each block
+   Array<int> offsets;
+   //! 2D array that stores each block of the operator.
+   Array2D<Operator *> ops;
+   Array2D<real_t> coef;
+
+   //! Temporary Vectors used to efficiently apply the Mult and MultTranspose
+   //! methods.
+   mutable BlockVector xblock;
+   mutable BlockVector yblock;
+   mutable Vector tmp;
+   mutable Vector tmp2;
+   mutable Vector r;
+   void ForwardPass(const Vector & x, Vector & y) const;
+   void BackwardPass(const Vector & x, Vector & y) const;
+};
+
 }
 
 #endif /* MFEM_BLOCKOPERATOR */

linalg/complex_operator.cpp

@@ -10,8 +10,15 @@
 // CONTRIBUTING.md for details.
 
 #include "complex_operator.hpp"
+#include "../general/communication.hpp"
+#ifdef MFEM_USE_MPI
+#include "blockoperator.hpp"
+#endif
 #include <set>
 #include <map>
+#include <unordered_map>
+#include <vector>
+#include <limits>
 
 namespace mfem
 {
@@ -164,6 +171,51 @@ void ComplexOperator::MultTranspose(const Vector &x_r, const Vector &x_i,
    }
 }
 
+#ifdef MFEM_USE_MPI
+ComplexHypreParMatrix * ComplexOperator::AsComplexHypreParMatrix() const
+{
+   HypreParMatrix *Ar = nullptr;
+   HypreParMatrix *Ai = nullptr;
+   bool own_r = false;
+   bool own_i = false;
+
+   if (auto *Ahr = dynamic_cast<const HypreParMatrix*>(&real()))
+   {
+      Ar = const_cast<HypreParMatrix*>(Ahr);
+   }
+   else if (auto *Br = dynamic_cast<const BlockOperator*>(&real()))
+   {
+      Ar = Br->GetMonolithicHypreParMatrix();
+      own_r = true;
+   }
+   else
+   {
+      MFEM_ABORT("Real part is neither HypreParMatrix nor BlockOperator.");
+   }
+
+   if (auto *Ahi = dynamic_cast<const HypreParMatrix*>(&imag()))
+   {
+      Ai = const_cast<HypreParMatrix*>(Ahi);
+   }
+   else if (auto *Bi = dynamic_cast<const BlockOperator*>(&imag()))
+   {
+      Ai = Bi->GetMonolithicHypreParMatrix();
+      own_i = true;
+   }
+   else
+   {
+      MFEM_ABORT("Imag part is neither HypreParMatrix nor BlockOperator.");
+   }
+
+   return new ComplexHypreParMatrix(Ar, Ai, own_r, own_i, GetConvention());
+}
+
+
+
+#endif
+
+
+
 
 SparseMatrix & ComplexSparseMatrix::real()
 {
@@ -877,6 +929,892 @@ ComplexHypreParMatrix::getColStartStop(const HypreParMatrix * A_r,
    delete [] stat;
 }
 
-#endif // MFEM_USE_MPI
+static Array<int> Twice(const Array<int> &offs)
+{
+   Array<int> arrayout(offs.Size());
+   for (int i = 0; i < offs.Size(); i++) { arrayout[i] = 2 * offs[i]; }
+   return arrayout;
+}
+
+
+ComplexBlockOperator::ComplexBlockOperator(const ComplexOperator &A)
+   : BlockOperator(
+        Twice(dynamic_cast<const BlockOperator&>(A.real()).RowOffsets()),
+        Twice(dynamic_cast<const BlockOperator&>(A.real()).ColOffsets()))
+{
+   const BlockOperator *Ar = dynamic_cast<const BlockOperator*>(&A.real());
+   const BlockOperator *Ai = dynamic_cast<const BlockOperator*>(&A.imag());
+
+   MFEM_VERIFY(Ar && Ai,
+               "ComplexBlockOperator: expected ComplexOperator with BlockOperator Re/Im.");
+   MFEM_VERIFY(Ar->NumRowBlocks() == Ai->NumRowBlocks() &&
+               Ar->NumColBlocks() == Ai->NumColBlocks(),
+               "ComplexBlockOperator: Re/Im block layouts mismatch.");
+
+   // Populate this BlockOperator (base) with ComplexOperator blocks.
+   for (int i = 0; i < Ar->NumRowBlocks(); ++i)
+   {
+      for (int j = 0; j < Ar->NumColBlocks(); ++j)
+      {
+         HypreParMatrix *Rij = nullptr;
+         if (!Ar->IsZeroBlock(i, j))
+         {
+            Rij = const_cast<HypreParMatrix*>
+                  (dynamic_cast<const HypreParMatrix*>(&Ar->GetBlock(i, j)));
+         }
+
+         HypreParMatrix *Iij = nullptr;
+         if (!Ai->IsZeroBlock(i, j))
+         {
+            Iij = const_cast<HypreParMatrix*>
+                  (dynamic_cast<const HypreParMatrix*>(&Ai->GetBlock(i, j)));
+         }
+
+         // MFEM_VERIFY((Rij && Iij) || (!Rij && !Iij),
+         //             "ComplexBlockOperator: inconsistent sparsity at block ("
+         //             << i << "," << j << ").");
+
+         if (Rij)
+         {
+            auto *Cij = new ComplexHypreParMatrix(Rij, Iij, false, false,
+                                                  A.GetConvention());
+            SetBlock(i, j, Cij);
+         }
+      }
+   }
+}
+
+void ComplexBlockOperator::BlockComplexToComplexBlock(const Vector &xin,
+                                                      Vector &xout) const
+{
+   MFEM_VERIFY(xout.Size() == xin.Size(),
+               "BlockComplexToComplexBlock: size mismatch (xout != xin).");
+   MFEM_VERIFY(xin.Size() % 2 == 0,
+               "BlockComplexToComplexBlock: expected even-sized vector (2*N).");
+
+   // Decide whether xin is domain-sized or range-sized by matching doubled offsets.
+   const int twoNcols = ColOffsets().Last();
+   const int twoNrows = RowOffsets().Last();
+   const Array<int> *doffs = nullptr; // doubled offsets to use
+
+   if (xin.Size() == twoNcols) { doffs = &ColOffsets(); }
+   else if (xin.Size() == twoNrows) { doffs = &RowOffsets(); }
+   else
+   {
+      MFEM_ABORT("BlockComplexToComplexBlock: vector size does not match "
+                 "either doubled domain or range size.");
+   }
+
+   // xin layout: [ Re(all 0..N-1), Im(all 0..N-1) ], where N = doffs->Last()/2
+   const int N = doffs->Last() / 2;
+
+   int pos = 0; // position in per-block layout (xout), but note xout is also size 2*N
+   for (int b = 0; b < doffs->Size() - 1; b++)
+   {
+      // Doubled block segment in this BlockOperator is [(*doffs)[b], (*doffs)[b+1])
+      // The *base* (undoubled) start/length are:
+      const int s_base = (*doffs)[b] / 2;
+      const int len    = ((*doffs)[b+1] - (*doffs)[b]) / 2;
+
+      // Write per-block Re then Im, contiguous
+      for (int k = 0; k < len; k++) { xout[pos + k]       = xin[s_base + k]; }
+      for (int k = 0; k < len; k++) { xout[pos + len + k] = xin[N + s_base + k]; }
+
+      pos += 2 * len;
+   }
+}
+
+void ComplexBlockOperator::ComplexBlockToBlockComplex(const Vector &xin,
+                                                      Vector &xout) const
+{
+   MFEM_VERIFY(xout.Size() == xin.Size(),
+               "ComplexBlockToBlockComplex: size mismatch (xout != xin).");
+   MFEM_VERIFY(xin.Size() % 2 == 0,
+               "ComplexBlockToBlockComplex: expected even-sized vector (2*N).");
+
+   const int Ncols = ColOffsets().Last();
+   const int Nrows = RowOffsets().Last();
+   const Array<int> *doffs = nullptr;
+
+   if (xin.Size() == Ncols) { doffs = &ColOffsets(); }
+   else if (xin.Size() == Nrows) { doffs = &RowOffsets(); }
+   else
+   {
+      MFEM_ABORT("ComplexBlockToBlockComplex: vector size does not match "
+                 "either doubled domain or range size.");
+   }
+
+   const int N = doffs->Last() / 2;
+
+   int pos = 0; // position in per-block layout (xin)
+   for (int b = 0; b < doffs->Size() - 1; b++)
+   {
+      const int s_base = (*doffs)[b] / 2;
+      const int len    = ((*doffs)[b+1] - (*doffs)[b]) / 2;
+
+      // Read per-block Re then Im, scatter to stacked [Re(all); Im(all)]
+      for (int k = 0; k < len; k++) { xout[s_base + k]     = xin[pos + k]; }
+      for (int k = 0; k < len; k++) { xout[N + s_base + k] = xin[pos + len + k]; }
+
+      pos += 2 * len;
+   }
+}
+
+
+#ifdef MFEM_USE_COMPLEX_MUMPS
+
+// Macro so indices match MUMPS documentation
+#define MUMPS_ICNTL(I) icntl[(I) - 1]
+#define MUMPS_CNTL(I)  cntl[(I) - 1]
+#define MUMPS_INFO(I)  info[(I) - 1]
+#define MUMPS_INFOG(I) infog[(I) - 1]
+
+ComplexMUMPSSolver::ComplexMUMPSSolver(MPI_Comm comm_)
+{
+   Init(comm_);
+}
+
+ComplexMUMPSSolver::ComplexMUMPSSolver(const Operator &op)
+{
+   auto APtr = dynamic_cast<const ComplexHypreParMatrix *>(&op);
+   MFEM_VERIFY(APtr, "Not a compatible matrix type for ComplexMUMPSSolver");
+   SetOperator(op);
+}
+
+void ComplexMUMPSSolver::Init(MPI_Comm comm_)
+{
+   comm = comm_;
+   MPI_Comm_size(comm, &numProcs);
+   MPI_Comm_rank(comm, &myid);
+
+   print_level = 2;
+   row_start = 0;
+
+   id = nullptr;
+
+#if MFEM_MUMPS_VERSION >= 530
+   irhs_loc = nullptr;
+   isol_loc = nullptr;
+   rhs_loc  = nullptr;
+   sol_loc  = nullptr;
+#else
+   global_num_rows = 0;
+   recv_counts = nullptr;
+   displs = nullptr;
+   rhs_glob = nullptr;
+   rhs_glob_r = nullptr;
+   rhs_glob_i = nullptr;
+#endif
+}
+
+ComplexMUMPSSolver::~ComplexMUMPSSolver()
+{
+#if MFEM_MUMPS_VERSION >= 530
+   delete [] irhs_loc;
+   delete [] isol_loc;
+   delete [] rhs_loc;
+   delete [] sol_loc;
+#else
+   delete [] recv_counts;
+   delete [] displs;
+   delete [] rhs_glob;
+   delete [] rhs_glob_r;
+   delete [] rhs_glob_i;
+#endif
+
+   if (id)
+   {
+      id->job = -2;
+      mumps_call();
+      delete id;
+      id = nullptr;
+   }
+}
+
+void ComplexMUMPSSolver::SetOperator(const Operator &op)
+{
+   auto APtr = dynamic_cast<const ComplexHypreParMatrix *>(&op);
+   MFEM_VERIFY(APtr, "Not compatible matrix type for ComplexMUMPSSolver");
+
+   height = op.Height();
+   width  = op.Width();
+
+   const HypreParMatrix *Ar = (APtr->hasRealPart()) ? &APtr->real() : nullptr;
+   const HypreParMatrix *Ai = (APtr->hasImagPart()) ? &APtr->imag() : nullptr;
+
+   MFEM_VERIFY(Ar || Ai, "ComplexMUMPSSolver: both real and imag parts are null.");
+
+   // Pick communicator from the non-null part
+   MPI_Comm op_comm = (Ar ? Ar->GetComm() : Ai->GetComm());
+
+   // Comm setup/check
+   if (comm == MPI_COMM_NULL) { Init(op_comm); }
+   else
+   {
+      int cmp = MPI_UNEQUAL;
+      MPI_Comm_compare(comm, op_comm, &cmp);
+      MFEM_VERIFY(cmp != MPI_UNEQUAL, "MPI Comm mismatch");
+   }
+
+   // HostRead only if non-null
+   if (Ar) { Ar->HostRead(); }
+   if (Ai) { Ai->HostRead(); }
+
+   // hypre parcsr pointers
+   hypre_ParCSRMatrix *parcsr_op_r = nullptr;
+   hypre_ParCSRMatrix *parcsr_op_i = nullptr;
+
+   if (Ar) { parcsr_op_r = (hypre_ParCSRMatrix*) const_cast<HypreParMatrix&>(*Ar); }
+   if (Ai) { parcsr_op_i = (hypre_ParCSRMatrix*) const_cast<HypreParMatrix&>(*Ai); }
+
+   // Merge diag+offd for whichever exists
+   hypre_CSRMatrix *csr_op_r = nullptr;
+   hypre_CSRMatrix *csr_op_i = nullptr;
+
+   if (parcsr_op_r) { csr_op_r = hypre_MergeDiagAndOffd(parcsr_op_r); }
+   if (parcsr_op_i) { csr_op_i = hypre_MergeDiagAndOffd(parcsr_op_i); }
+
+#if MFEM_HYPRE_VERSION >= 21600
+   if (csr_op_r) { hypre_CSRMatrixBigJtoJ(csr_op_r); }
+   if (csr_op_i) { hypre_CSRMatrixBigJtoJ(csr_op_i); }
+#endif
+
+   // Determine local/global sizes and row_start from an existing part
+   const int n_loc = internal::to_int((csr_op_r ? csr_op_r->num_rows :
+                                       csr_op_i->num_rows));
+   row_start = internal::to_int((parcsr_op_r ? parcsr_op_r->first_row_index
+                                 : parcsr_op_i->first_row_index));
+   const int global_n = internal::to_int((parcsr_op_r ?
+                                          parcsr_op_r->global_num_rows
+                                          : parcsr_op_i->global_num_rows));
+
+   // Use nullptr checks
+   const int *Ir = csr_op_r ? csr_op_r->i : nullptr;
+   const int *Jr = csr_op_r ? csr_op_r->j : nullptr;
+   const real_t *Vr = csr_op_r ? (const real_t*)csr_op_r->data : nullptr;
+
+   const int *Ii = csr_op_i ? csr_op_i->i : nullptr;
+   const int *Ji = csr_op_i ? csr_op_i->j : nullptr;
+   const real_t *Vi = csr_op_i ? (const real_t*)csr_op_i->data : nullptr;
+
+   // Build union COO
+   std::vector<int> Icoo, Jcoo;
+   std::vector<mumps_complex_t> Zcoo;
+
+   size_t nnz_r = csr_op_r ? (size_t)csr_op_r->num_nonzeros : 0;
+   size_t nnz_i = csr_op_i ? (size_t)csr_op_i->num_nonzeros : 0;
+   Icoo.reserve(nnz_r + nnz_i);
+   Jcoo.reserve(nnz_r + nnz_i);
+   Zcoo.reserve(nnz_r + nnz_i);
+
+   BuildUnionCOO(n_loc, row_start, Ir, Jr, Vr, Ii, Ji, Vi, Icoo, Jcoo, Zcoo);
+
+   const int nnz = (int)Icoo.size();
+   int *I = new int[nnz];
+   int *J = new int[nnz];
+   mumps_complex_t *A = new mumps_complex_t[nnz];
+
+   std::copy(Icoo.begin(), Icoo.end(), I);
+   std::copy(Jcoo.begin(), Jcoo.end(), J);
+   std::copy(Zcoo.begin(), Zcoo.end(), A);
+
+   // New ComplexMUMPS object or reuse an existing one
+   if (!id || !reorder_reuse)
+   {
+      if (id)
+      {
+         id->job = -2;
+         mumps_call();
+         delete id;
+         id = nullptr;
+      }
+
+#ifdef MFEM_USE_SINGLE
+      id = new CMUMPS_STRUC_C();
+#else
+      id = new ZMUMPS_STRUC_C();
+#endif
+
+      id->sym = 0; // general complex
+      id->par = 1;
+      id->comm_fortran = (MUMPS_INT)MPI_Comm_c2f(comm);
+
+      // Init
+      id->job = -1;
+      mumps_call();
+
+      // Set parameters
+      SetParameters();
+
+      // Attach matrix
+      id->n       = global_n;
+      id->nnz_loc = nnz;
+      id->irn_loc = I;
+      id->jcn_loc = J;
+      id->a_loc   = A;
+
+      // Analysis (ordering + symbolic)
+      id->job = 1;
+      mumps_call();
+   }
+   else
+   {
+      // Reuse symbolic factorization / ordering
+      MFEM_VERIFY(id->n == global_n,
+                  "ReorderingReuse requires same global size (id->n mismatch)");
+
+      // Update matrix pointers (pattern is assumed compatible)
+      id->nnz_loc = nnz;
+      id->irn_loc = I;
+      id->jcn_loc = J;
+      id->a_loc   = A;
+   }
+
+   // Factorization
+   id->job = 2;
+   {
+      const int mem_relax_lim = 200;
+      while (true)
+      {
+         mumps_call();
+         if (id->MUMPS_INFOG(1) < 0)
+         {
+            if (id->MUMPS_INFOG(1) == -8 || id->MUMPS_INFOG(1) == -9)
+            {
+               id->MUMPS_ICNTL(14) += 20;
+               MFEM_VERIFY(id->MUMPS_ICNTL(14) <= mem_relax_lim,
+                           "Memory relaxation limit reached for MUMPS factorization");
+               if (myid == 0 && print_level > 0)
+               {
+                  out << "Re-running MUMPS factorization with memory relaxation "
+                      << id->MUMPS_ICNTL(14) << '\n';
+               }
+            }
+            else
+            {
+               MFEM_ABORT("Error during MUMPS numerical factorization");
+            }
+         }
+         else { break; }
+      }
+   }
+
+   // Done with input storage
+   if (csr_op_r) { hypre_CSRMatrixDestroy(csr_op_r);}
+   if (csr_op_i) { hypre_CSRMatrixDestroy(csr_op_i);}
+   delete [] I;
+   delete [] J;
+   delete [] A;
+
+   // Post-factorization RHS/SOL setup
+   id->nrhs = -1;
+
+#if MFEM_MUMPS_VERSION >= 530
+   // Distributed RHS/SOL sizes
+   id->nloc_rhs = n_loc;
+   id->lrhs_loc = n_loc;
+   id->lsol_loc = id->MUMPS_INFO(23);
+
+   delete [] irhs_loc;
+   irhs_loc = new int[id->lrhs_loc];
+   for (int i = 0; i < n_loc; i++)
+   {
+      irhs_loc[i] = row_start + i + 1;
+   }
+   id->irhs_loc = irhs_loc;
+
+   delete [] isol_loc;
+   isol_loc = new int[id->lsol_loc];
+   id->isol_loc = isol_loc;
+
+   row_starts.SetSize(numProcs);
+   MPI_Allgather(&row_start, 1, MPI_INT, row_starts, 1, MPI_INT, comm);
+
+   // Reset cached buffers
+   delete [] rhs_loc; rhs_loc = nullptr;
+   delete [] sol_loc; sol_loc = nullptr;
+   rhs1_buf.clear();
+
+#else
+   // Centralized RHS/SOL on root
+   id->lrhs = id->n;
+
+   global_num_rows = id->n;
+
+   if (myid == 0)
+   {
+      delete [] recv_counts;
+      delete [] displs;
+      recv_counts = new int[numProcs];
+      displs = new int[numProcs];
+
+      delete [] rhs_glob;   rhs_glob = nullptr;
+      delete [] rhs_glob_r; rhs_glob_r = nullptr;
+      delete [] rhs_glob_i; rhs_glob_i = nullptr;
+   }
+
+   MPI_Gather(&n_loc, 1, MPI_INT, recv_counts, 1, MPI_INT, 0, comm);
+
+   if (myid == 0)
+   {
+      displs[0] = 0;
+      int s = 0;
+      for (int k = 0; k < numProcs - 1; k++)
+      {
+         s += recv_counts[k];
+         displs[k+1] = s;
+      }
+   }
+#endif
+}
+
+void ComplexMUMPSSolver::InitRhsSol(int nrhs) const
+{
+#if MFEM_MUMPS_VERSION >= 530
+
+   MFEM_VERIFY(id, "InitRhsSol called before SetOperator");
+
+   if (id->nrhs != nrhs)
+   {
+      delete [] rhs_loc;
+      delete [] sol_loc;
+
+      rhs_loc = new mumps_complex_t[(size_t)nrhs * (size_t)id->lrhs_loc];
+      sol_loc = new mumps_complex_t[(size_t)nrhs * (size_t)id->lsol_loc];
+
+      id->rhs_loc = rhs_loc;
+      id->sol_loc = sol_loc;
+   }
+   id->nrhs = nrhs;
+
+#else
+   MFEM_VERIFY(id, "InitRhsSol called before SetOperator");
+
+   id->nrhs = nrhs;
+   id->lrhs = id->n;
+
+   if (myid == 0)
+   {
+      const size_t N = (size_t)nrhs * (size_t)global_num_rows;
+
+      delete [] rhs_glob;
+      delete [] rhs_glob_r;
+      delete [] rhs_glob_i;
+
+      rhs_glob   = new mumps_complex_t[N];
+      rhs_glob_r = new real_t[N];
+      rhs_glob_i = new real_t[N];
+
+      id->rhs = rhs_glob;
+   }
+#endif
+}
+
+void ComplexMUMPSSolver::Mult(const Vector &x, Vector &y) const
+{
+   Array<const Vector *> X(1);
+   Array<Vector *> Y(1);
+   X[0] = &x;
+   Y[0] = &y;
+   ArrayMult(X, Y);
+}
+
+void ComplexMUMPSSolver::ArrayMult(const Array<const Vector *> &X,
+                                   Array<Vector *> &Y) const
+{
+   MFEM_ASSERT(X.Size() == Y.Size(),
+               "Number of columns mismatch in ComplexMUMPSSolver::Mult!");
+   MFEM_VERIFY(id, "ComplexMUMPSSolver::ArrayMult called before SetOperator");
+
+   InitRhsSol(X.Size());
+
+#if MFEM_MUMPS_VERSION >= 530
+   MFEM_VERIFY(irhs_loc && isol_loc, "RHS/SOL maps not initialized");
+   MFEM_VERIFY(rhs_loc && sol_loc, "RHS/SOL buffers not initialized");
+   const int n_loc = id->lrhs_loc;
+   const int nrhs  = id->nrhs;
+
+   // Pack all RHS
+   for (int i = 0; i < nrhs; i++)
+   {
+      MFEM_ASSERT(X[i], "Missing Vector in Mult!");
+      X[i]->HostRead();
+      MFEM_VERIFY(X[i]->Size() == 2*n_loc, "RHS size mismatch");
+
+      const real_t *xdata = X[i]->GetData();
+      const real_t *xr = xdata;
+      const real_t *xi = xdata + n_loc;
+
+      mumps_complex_t *dst = rhs_loc + i * n_loc;
+      for (int j = 0; j < n_loc; j++)
+      {
+         dst[j].r = xr[j];
+         dst[j].i = xi[j];
+      }
+   }
+
+   id->rhs_loc  = rhs_loc;
+   id->sol_loc  = sol_loc;
+   id->irhs_loc = irhs_loc;
+   id->isol_loc = isol_loc;
+
+   // MUMPS solve
+   id->job = 3;
+   mumps_call();
+
+   const int lsol = id->lsol_loc;
+
+   // Redistribute each solution column into Y
+   for (int i = 0; i < nrhs; i++)
+   {
+      MFEM_ASSERT(Y[i], "Missing output Vector in Mult!");
+      Y[i]->HostWrite();
+      MFEM_VERIFY(Y[i]->Size() == 2*n_loc, "Output size mismatch");
+
+      const mumps_complex_t *xcol = sol_loc + i * lsol;
+      RedistributeSol(isol_loc, xcol, Y[i]->GetData(), n_loc, lsol);
+   }
+
+#else // MFEM_MUMPS_VERSION < 530
+
+   const int nrhs = id->nrhs;
+
+   MFEM_VERIFY(X.Size() > 0 && X[0], "Missing RHS");
+   const int n_loc = X[0]->Size()/2;
+
+   for (int i = 0; i < nrhs; i++)
+   {
+      MFEM_ASSERT(X[i], "Missing Vector in Mult!");
+      X[i]->HostRead();
+      MFEM_VERIFY(X[i]->Size() == 2*n_loc, "RHS size mismatch");
+   }
+
+   // Gather each RHS column (real+imag separately) into root staging
+   for (int i = 0; i < nrhs; i++)
+   {
+      const real_t *xdata = X[i]->GetData();
+
+      MPI_Gatherv(xdata, n_loc, MPITypeMap<real_t>::mpi_type,
+                  rhs_glob_r + i * global_num_rows,
+                  recv_counts, displs, MPITypeMap<real_t>::mpi_type,
+                  0, comm);
+
+      MPI_Gatherv(xdata + n_loc, n_loc, MPITypeMap<real_t>::mpi_type,
+                  rhs_glob_i + i * global_num_rows,
+                  recv_counts, displs, MPITypeMap<real_t>::mpi_type,
+                  0, comm);
+   }
+
+   // Pack into MUMPS complex RHS on root: id->rhs is in-place
+   if (myid == 0)
+   {
+      for (int i = 0; i < nrhs; i++)
+      {
+         mumps_complex_t *dst = rhs_glob + i * global_num_rows;
+         const real_t *rr = rhs_glob_r + i * global_num_rows;
+         const real_t *ri = rhs_glob_i + i * global_num_rows;
+
+         for (int j = 0; j < global_num_rows; j++)
+         {
+            dst[j].r = rr[j];
+            dst[j].i = ri[j];
+         }
+      }
+      id->rhs = rhs_glob;
+   }
+
+   // Solve
+   id->job = 3;
+   mumps_call();
+
+   // Unpack to real/imag
+   if (myid == 0)
+   {
+      for (int i = 0; i < nrhs; i++)
+      {
+         const mumps_complex_t *src = rhs_glob + i * global_num_rows;
+         real_t *rr = rhs_glob_r + i * global_num_rows;
+         real_t *ri = rhs_glob_i + i * global_num_rows;
+
+         for (int j = 0; j < global_num_rows; j++)
+         {
+            rr[j] = src[j].r;
+            ri[j] = src[j].i;
+         }
+      }
+   }
+
+   // Scatter each RHS solution
+   for (int i = 0; i < nrhs; i++)
+   {
+      MFEM_ASSERT(Y[i], "Missing Vector in Mult!");
+      Y[i]->HostWrite();
+      MFEM_VERIFY(Y[i]->Size() == 2*n_loc, "Output size mismatch");
+
+      real_t *ydata = Y[i]->GetData();
+
+      MPI_Scatterv(rhs_glob_r + i * global_num_rows,
+                   recv_counts, displs, MPITypeMap<real_t>::mpi_type,
+                   ydata, n_loc, MPITypeMap<real_t>::mpi_type,
+                   0, comm);
+
+      MPI_Scatterv(rhs_glob_i + i * global_num_rows,
+                   recv_counts, displs, MPITypeMap<real_t>::mpi_type,
+                   ydata + n_loc, n_loc, MPITypeMap<real_t>::mpi_type,
+                   0, comm);
+   }
+
+#endif
+}
+
+void ComplexMUMPSSolver::MultTranspose(const Vector &x, Vector &y) const
+{
+   MFEM_VERIFY(id, "MultTranspose called before SetOperator");
+
+   // Transpose solve
+   id->MUMPS_ICNTL(9) = 0;
+   Mult(x, y);
+   id->MUMPS_ICNTL(9) = 1;
+}
+
+void ComplexMUMPSSolver::ArrayMultTranspose(const Array<const Vector *> &X,
+                                            Array<Vector *> &Y) const
+{
+   MFEM_VERIFY(id, "ArrayMultTranspose called before SetOperator");
+
+   // Transpose solve
+   id->MUMPS_ICNTL(9) = 0;
+   ArrayMult(X, Y);
+   id->MUMPS_ICNTL(9) = 1;
+}
+
+void ComplexMUMPSSolver::SetParameters()
+{
+   // Output stream for error messages
+   id->MUMPS_ICNTL(1) = 6;
+   // Output stream for diagnostic printing local to each proc
+   id->MUMPS_ICNTL(2) = 0;
+   // Output stream for global info
+   id->MUMPS_ICNTL(3) = 6;
+   // Level of error printing
+   id->MUMPS_ICNTL(4) = print_level;
+
+   // Input matrix format (assembled)
+   id->MUMPS_ICNTL(5) = 0;
+   // Use A or A^T
+   id->MUMPS_ICNTL(9) = 1;
+   // Iterative refinement (disabled)
+   id->MUMPS_ICNTL(10) = 0;
+   // Error analysis-statistics (disabled)
+   id->MUMPS_ICNTL(11) = 0;
+   // Use of ScaLAPACK (disabled)
+   id->MUMPS_ICNTL(13) = 0;
+   // Workspace relaxation (% increase)
+   id->MUMPS_ICNTL(14) = 20;
+   // OpenMP threads (default)
+   id->MUMPS_ICNTL(16) = 0;
+   // Matrix input format (distributed)
+   id->MUMPS_ICNTL(18) = 3;
+   // Schur complement (none)
+   id->MUMPS_ICNTL(19) = 0;
+
+#if MFEM_MUMPS_VERSION >= 530
+   // Distributed RHS
+   id->MUMPS_ICNTL(20) = 10;
+   // Distributed Sol
+   id->MUMPS_ICNTL(21) = 1;
+#else
+   // Centralized RHS
+   id->MUMPS_ICNTL(20) = 0;
+   // Centralized Sol
+   id->MUMPS_ICNTL(21) = 0;
+#endif
+
+   // Out-of-core (disabled)
+   id->MUMPS_ICNTL(22) = 0;
+   // Max size of working memory (default)
+   id->MUMPS_ICNTL(23) = 0;
+
+   switch (reorder_method)
+   {
+      case ReorderingStrategy::AUTOMATIC:
+         id->MUMPS_ICNTL(28) = 0;
+         id->MUMPS_ICNTL(7) = 7;
+         id->MUMPS_ICNTL(29) = 0;
+         break;
+      case ReorderingStrategy::AMD:
+         id->MUMPS_ICNTL(28) = 1;
+         id->MUMPS_ICNTL(7) = 0;
+         break;
+      case ReorderingStrategy::AMF:
+         id->MUMPS_ICNTL(28) = 1;
+         id->MUMPS_ICNTL(7) = 2;
+         break;
+      case ReorderingStrategy::PORD:
+         id->MUMPS_ICNTL(28) = 1;
+         id->MUMPS_ICNTL(7) = 4;
+         break;
+      case ReorderingStrategy::METIS:
+         id->MUMPS_ICNTL(28) = 1;
+         id->MUMPS_ICNTL(7) = 5;
+         break;
+      case ReorderingStrategy::PARMETIS:
+         id->MUMPS_ICNTL(28) = 2;
+         id->MUMPS_ICNTL(29) = 2;
+         break;
+      case ReorderingStrategy::SCOTCH:
+         id->MUMPS_ICNTL(28) = 1;
+         id->MUMPS_ICNTL(7) = 3;
+         break;
+      case ReorderingStrategy::PTSCOTCH:
+         id->MUMPS_ICNTL(28) = 2;
+         id->MUMPS_ICNTL(29) = 1;
+         break;
+      default:
+         break; // This should be unreachable
+   }
 
 }
+
+void ComplexMUMPSSolver::BuildUnionCOO(const int n_loc,
+                                       const int row_start_,
+                                       const int *Ir, const int *Jr, const real_t *Vr,
+                                       const int *Ii, const int *Ji, const real_t *Vi,
+                                       std::vector<int> &Icoo,
+                                       std::vector<int> &Jcoo,
+                                       std::vector<mumps_complex_t> &Zcoo) const
+{
+   for (int r = 0; r < n_loc; ++r)
+   {
+      std::unordered_map<int, std::pair<real_t, real_t>> row;
+
+      const int rr0 = Ir ? Ir[r]   : 0;
+      const int rr1 = Ir ? Ir[r+1] : 0;
+      const int ii0 = Ii ? Ii[r]   : 0;
+      const int ii1 = Ii ? Ii[r+1] : 0;
+
+      row.reserve((rr1 - rr0) + (ii1 - ii0));
+
+      if (Ir)
+      {
+         for (int p = rr0; p < rr1; ++p) { row[Jr[p]].first += Vr[p]; }
+      }
+      if (Ii)
+      {
+         for (int p = ii0; p < ii1; ++p) { row[Ji[p]].second += Vi[p]; }
+      }
+
+      for (const auto &kv : row)
+      {
+         Icoo.push_back(row_start_ + r + 1);
+         Jcoo.push_back(kv.first + 1);
+         Zcoo.push_back(mumps_complex_t{kv.second.first, kv.second.second});
+      }
+   }
+}
+
+#if MFEM_MUMPS_VERSION >= 530
+int ComplexMUMPSSolver::GetRowRank(int i, const Array<int> &row_starts_) const
+{
+   if (row_starts_.Size() == 1) { return 0; }
+   auto up = std::upper_bound(row_starts_.begin(), row_starts_.end(), i);
+   return (int)std::distance(row_starts_.begin(), up) - 1;
+}
+
+void ComplexMUMPSSolver::RedistributeSol(const int *row_map,
+                                         const mumps_complex_t *x,
+                                         real_t *y_ri,
+                                         int n_loc,
+                                         int lsol_loc) const
+{
+   int *send_count = new int[numProcs]();
+   for (int i = 0; i < lsol_loc; i++)
+   {
+      const int j = row_map[i] - 1;
+      const int row_rank = GetRowRank(j, row_starts);
+      if (myid == row_rank) { continue; }
+      send_count[row_rank]++;
+   }
+
+   int *recv_count = new int[numProcs];
+   MPI_Alltoall(send_count, 1, MPI_INT, recv_count, 1, MPI_INT, comm);
+
+   int *send_displ = new int[numProcs]; send_displ[0] = 0;
+   int *recv_displ = new int[numProcs]; recv_displ[0] = 0;
+
+   int sbuff_size = send_count[numProcs-1];
+   int rbuff_size = recv_count[numProcs-1];
+   for (int k = 0; k < numProcs - 1; k++)
+   {
+      send_displ[k+1] = send_displ[k] + send_count[k];
+      recv_displ[k+1] = recv_displ[k] + recv_count[k];
+      sbuff_size += send_count[k];
+      rbuff_size += recv_count[k];
+   }
+
+   int   *sendbuf_index = new int[sbuff_size];
+   real_t *sendbuf_r    = new real_t[sbuff_size];
+   real_t *sendbuf_i    = new real_t[sbuff_size];
+   int   *soffs         = new int[numProcs]();
+
+   for (int i = 0; i < lsol_loc; i++)
+   {
+      const int j = row_map[i] - 1;
+      const int row_rank = GetRowRank(j, row_starts);
+
+      const real_t xr = (real_t)x[i].r;
+      const real_t xi = (real_t)x[i].i;
+
+      if (myid == row_rank)
+      {
+         const int local_index = j - row_start;
+         y_ri[local_index]        = xr;
+         y_ri[local_index+n_loc]  = xi;
+      }
+      else
+      {
+         const int k = send_displ[row_rank] + soffs[row_rank];
+         sendbuf_index[k] = j;
+         sendbuf_r[k] = xr;
+         sendbuf_i[k] = xi;
+         soffs[row_rank]++;
+      }
+   }
+
+   int    *recvbuf_index = new int[rbuff_size];
+   real_t *recvbuf_r     = new real_t[rbuff_size];
+   real_t *recvbuf_i     = new real_t[rbuff_size];
+
+   MPI_Alltoallv(sendbuf_index, send_count, send_displ, MPI_INT,
+                 recvbuf_index, recv_count, recv_displ, MPI_INT, comm);
+
+   MPI_Alltoallv(sendbuf_r, send_count, send_displ, MPITypeMap<real_t>::mpi_type,
+                 recvbuf_r, recv_count, recv_displ, MPITypeMap<real_t>::mpi_type, comm);
+
+   MPI_Alltoallv(sendbuf_i, send_count, send_displ, MPITypeMap<real_t>::mpi_type,
+                 recvbuf_i, recv_count, recv_displ, MPITypeMap<real_t>::mpi_type, comm);
+
+   for (int i = 0; i < rbuff_size; i++)
+   {
+      const int local_index = recvbuf_index[i] - row_start;
+      y_ri[local_index]       = recvbuf_r[i];
+      y_ri[local_index+n_loc] = recvbuf_i[i];
+   }
+
+   delete [] recvbuf_i;
+   delete [] recvbuf_r;
+   delete [] recvbuf_index;
+   delete [] soffs;
+   delete [] sendbuf_i;
+   delete [] sendbuf_r;
+   delete [] sendbuf_index;
+   delete [] recv_displ;
+   delete [] send_displ;
+   delete [] recv_count;
+   delete [] send_count;
+}
+#endif // MFEM_MUMPS_VERSION >= 530
+#endif // MFEM_USE_COMPLEX_MUMPS
+#endif // MFEM_USE_MPI
+
+} // namespace mfem

linalg/complex_operator.hpp

@@ -13,6 +13,7 @@
 #define MFEM_COMPLEX_OPERATOR
 
 #include "operator.hpp"
+#include "blockoperator.hpp"
 #include "sparsemat.hpp"
 #ifdef MFEM_USE_MPI
 #include "hypre.hpp"
@@ -22,8 +23,20 @@
 #include <umfpack.h>
 #endif
 
+#ifdef MFEM_USE_COMPLEX_MUMPS
+#ifdef MFEM_USE_SINGLE
+#include "cmumps_c.h"
+#else
+#include "zmumps_c.h"
+#endif
+#include <vector>
+#endif
+
 namespace mfem
 {
+#ifdef MFEM_USE_MPI
+class ComplexHypreParMatrix; // forward declaration
+#endif
 
 /** @brief Mimic the action of a complex operator using two real operators.
 
@@ -118,6 +131,13 @@ public:
 
    Convention GetConvention() const { return convention_; }
 
+#ifdef MFEM_USE_MPI
+   /// Return a ComplexHypreParMatrix view:
+   ///  - wraps if real/imag are HypreParMatrix
+   ///  - merges if real/imag are BlockOperator of HypreParMatrix blocks
+   ComplexHypreParMatrix *AsComplexHypreParMatrix() const;
+#endif
+
 protected:
    // Let this be hidden from the public interface since the implementation
    // depends on internal members
@@ -242,6 +262,7 @@ public:
 
 #endif
 
+
 #ifdef MFEM_USE_MPI
 
 /** @brief Specialization of the ComplexOperator built from a pair of
@@ -288,6 +309,265 @@ private:
    int myid_;
    int nranks_;
 };
+/// A BlockOperator whose blocks are ComplexOperator objects, constructed from
+/// a ComplexOperator whose Real()/Imag() parts are BlockOperator objects.
+/// It also provides layout conversions between:
+///   • BlockComplex (stacked): [ Re(all); Im(all) ]
+///   • ComplexBlock (per-block): [ Re(block_b); Im(block_b) ]
+class ComplexBlockOperator : public BlockOperator
+{
+public:
+   /// Construct from a ComplexOperator whose Real()/Imag() parts are
+   /// BlockOperator objects.
+   ComplexBlockOperator(const ComplexOperator &A);
+
+   /// Convert vector from BlockComplex (stacked) -> ComplexBlock (per-block).
+   /// Sizes must match (xout.Size() == xin.Size() == 2*N).
+   void BlockComplexToComplexBlock(const Vector &xin,
+                                   Vector &xout) const;
+   /// Convert vector from ComplexBlock (per-block) -> BlockComplex (stacked).
+   /// Sizes must match (xout.Size() == xin.Size() == 2*N).
+   void ComplexBlockToBlockComplex(const Vector &xin,
+                                   Vector &xout) const;
+private:
+};
+
+#ifdef MFEM_USE_COMPLEX_MUMPS
+/**
+ * @brief Complex MUMPS: Parallel sparse direct solver for ComplexHypreParMatrix
+ *
+ * Notes:
+ *  - Expects Operator to be a ComplexHypreParMatrix.
+ *  - Complex vectors are assumed packed as [Re; Im] in a real Vector.
+ *  - SetOperator(): analysis + factorization
+ *  - Mult()       : solve
+ */
+class ComplexMUMPSSolver : public Solver
+{
+public:
+   /// Specify the reordering strategy
+   enum ReorderingStrategy
+   {
+      /// Let MUMPS automatically decide the reordering strategy
+      AUTOMATIC = 0,
+      /// Approximate Minimum Degree with auto quasi-dense row detection is used
+      AMD,
+      /// Approximate Minimum Fill method will be used
+      AMF,
+      /// The PORD library will be used
+      PORD,
+      /// The METIS library will be used
+      METIS,
+      /// The ParMETIS library will be used
+      PARMETIS,
+      /// The Scotch library will be used
+      SCOTCH,
+      /// The PTScotch library will be used
+      PTSCOTCH
+   };
+
+   /**
+    * @brief Constructor with MPI_Comm parameter.
+    */
+   ComplexMUMPSSolver(MPI_Comm comm_);
+   /**
+    * @brief Constructor with a ComplexHypreParMatrix Operator.
+    */
+   ComplexMUMPSSolver(const Operator &op);
+
+   /**
+    * @brief Set the Operator and perform factorization
+    *
+    * @a op needs to be of type ComplexHypreParMatrix.
+    *
+    * @param op Operator used in factorization and solve
+    */
+   void SetOperator(const Operator &op);
+
+   /**
+    * @brief Solve $ y = Op^{-1} x $
+    *
+    * @param x RHS vector
+    * @param y Solution vector
+    */
+   void Mult(const Vector &x, Vector &y) const;
+   /**
+    * @brief Solve $ Y_i = Op^{-1} X_i $
+    *
+    * @param X Array of RHS vectors
+    * @param Y Array of Solution vectors
+    */
+   void ArrayMult(const Array<const Vector *> &X, Array<Vector *> &Y) const;
+   /**
+    * @brief Transpose Solve $ y = Op^{-T} x $
+    * @note This is not a Hermitian/conjugate-transpose solve.
+    *
+    * @param x RHS vector
+    * @param y Solution vector
+    */
+   void MultTranspose(const Vector &x, Vector &y) const;
+
+   /**
+    * @brief Transpose Solve $ Y_i = Op^{-T} X_i $
+    * @note This is not a Hermitian/conjugate-transpose solve.
+    *
+    * @param X Array of RHS vectors
+    * @param Y Array of Solution vectors
+    */
+   void ArrayMultTranspose(const Array<const Vector *> &X,
+                           Array<Vector *> &Y) const;
+
+   /**
+    * @brief Set the error print level for MUMPS
+    *
+    * Supported values are:
+    * - 0:  No output printed
+    * - 1:  Only errors printed
+    * - 2:  Errors, warnings, and main stats printed
+    * - 3:  Errors, warnings, main stats, and terse diagnostics printed
+    * - 4:  Errors, warnings, main stats, diagnostics, and input/output printed
+    *
+    * @param print_lvl Print level, default is 2
+    *
+    * @note This method has to be called before SetOperator
+    */
+   void SetPrintLevel(int print_lvl) { print_level = print_lvl;}
+
+   /**
+    * @brief Set the reordering strategy
+    *
+    * Supported reorderings are: ComplexMUMPSSolver::AUTOMATIC,
+    * ComplexMUMPSSolver::AMD, ComplexMUMPSSolver::AMF,
+    * ComplexMUMPSSolver::PORD, ComplexMUMPSSolver::METIS,
+    * ComplexMUMPSSolver::PARMETIS, ComplexMUMPSSolver::SCOTCH,
+    * and ComplexMUMPSSolver::PTSCOTCH
+    *
+    * @param method Reordering method
+    *
+    * @note This method has to be called before SetOperator
+    */
+   void SetReorderingStrategy(ReorderingStrategy method) { reorder_method = method; }
+
+   /**
+    * @brief Set the flag controlling reuse of the symbolic factorization
+    * for multiple operators
+    *
+    * @param reuse Flag to reuse symbolic factorization
+    *
+    * @note This method has to be called before repeated calls to SetOperator
+    */
+   void SetReorderingReuse(bool reuse) { reorder_reuse = reuse; }
+
+   ~ComplexMUMPSSolver();
+
+private:
+   // MPI communicator
+   MPI_Comm comm = MPI_COMM_NULL;
+
+   // Number of procs
+   int numProcs;
+
+   // MPI rank
+   int myid;
+
+   // Parameter controlling the printing level
+   int print_level = 0;
+
+   // Parameter controlling the reordering strategy
+   ReorderingStrategy reorder_method = ReorderingStrategy::AUTOMATIC;
+
+   // Parameter controlling whether or not to reuse the symbolic factorization
+   // for multiple calls to SetOperator
+   bool reorder_reuse = false;
+
+   // Local row offsets
+   int row_start;
+
+   // ComplexMUMPS object
+#ifdef MFEM_USE_SINGLE
+   CMUMPS_STRUC_C *id = nullptr;
+   using mumps_complex_t = mumps_complex;
+#else
+   ZMUMPS_STRUC_C *id = nullptr;
+   using mumps_complex_t = mumps_double_complex;
+#endif
+
+   /// Method for initialization
+   void Init(MPI_Comm comm_);
+
+   /// Method for setting ComplexMUMPS internal parameters
+   void SetParameters();
+
+   /// Method for configuring storage for distributed/centralized
+   /// RHS and solution
+   void InitRhsSol(int nrhs) const;
+
+   /// Method for calling the single/double ComplexMUMPS solver
+   inline void mumps_call() const
+   {
+#ifdef MFEM_USE_SINGLE
+      cmumps_c(id);
+#else
+      zmumps_c(id);
+#endif
+   }
+
+   /// Method for building the COO format of the combined complex operator
+   /// from the real and imaginary parts. This is particularly useful when
+   /// real and imaginary parts have different sparsity patterns.
+   void BuildUnionCOO(const int n_loc,
+                      const int row_start,
+                      const int *Ir, const int *Jr, const real_t *Vr,
+                      const int *Ii, const int *Ji, const real_t *Vi,
+                      std::vector<int> &Icoo,
+                      std::vector<int> &Jcoo,
+                      std::vector<mumps_complex_t> &Zcoo) const;
+
+#if MFEM_MUMPS_VERSION >= 530
+   // Row offsets on all procs
+   Array<int> row_starts;
+
+   // Local RHS row indices
+   int *irhs_loc = nullptr;
+
+   // Local solution row map returned by MUMPS
+   int *isol_loc = nullptr;
+
+   // Cached buffers
+   mutable mumps_complex_t *rhs_loc = nullptr;
+   mutable mumps_complex_t *sol_loc = nullptr;
+
+   // RHS buffers
+   mutable std::vector<mumps_complex_t> rhs1_buf;
+
+   // These two methods are needed to distribute the local solution
+   // vectors returned by MUMPS to the original MFEM parallel partition
+   int GetRowRank(int i, const Array<int> &row_starts_) const;
+
+   void RedistributeSol(const int *row_map,
+                        const mumps_complex_t *x,
+                        real_t *y_ri,
+                        int n_loc,
+                        int lsol_loc) const;
+
+#else
+   // Root-gather path
+   int global_num_rows;
+
+   // Arrays needed for MPI_Gatherv and MPI_Scatterv
+   int *recv_counts = nullptr;
+   int *displs = nullptr;
+
+   // Complex RHS/solution on root
+   mutable mumps_complex_t *rhs_glob = nullptr;
+
+   // Cached real/imag staging on root
+   mutable real_t *rhs_glob_r = nullptr;
+   mutable real_t *rhs_glob_i = nullptr;
+#endif
+};
+
+#endif // MFEM_USE_COMPLEX_MUMPS
 
 #endif // MFEM_USE_MPI
 

linalg/densemat.cpp

@@ -2185,6 +2185,45 @@ void DenseMatrix::AdjustDofDirection(Array<int> &dofs)
    }
 }
 
+void DenseMatrix::AdjustDofDirection(Array<int> &row_dofs,
+                                     Array<int> &col_dofs)
+{
+   const int nr = row_dofs.Size();
+   const int nc = col_dofs.Size();
+
+   MFEM_VERIFY(Height() == nr && Width() == nc,
+               "DenseMatrix::AdjustDofDirection: size mismatch.");
+
+   // Extract signs and convert to unsigned indices
+   Vector rsign(nr), csign(nc);
+
+   for (int i = 0; i < nr; i++)
+   {
+      const int d = row_dofs[i];
+      if (d >= 0) { rsign(i) =  1.0; }
+      else        { rsign(i) = -1.0; row_dofs[i] = -d - 1; continue; }
+      row_dofs[i] = d;
+   }
+
+   for (int j = 0; j < nc; j++)
+   {
+      const int d = col_dofs[j];
+      if (d >= 0) { csign(j) =  1.0; }
+      else        { csign(j) = -1.0; col_dofs[j] = -d - 1; continue; }
+      col_dofs[j] = d;
+   }
+
+   // Apply row/column signs
+   for (int i = 0; i < nr; i++)
+   {
+      const real_t rs = rsign(i);
+      for (int j = 0; j < nc; j++)
+      {
+         (*this)(i,j) *= rs * csign(j);
+      }
+   }
+}
+
 void DenseMatrix::SetRow(int row, real_t value)
 {
    for (int j = 0; j < Width(); j++)

linalg/densemat.hpp

@@ -476,6 +476,10 @@ public:
        then (*this)(i,j) = -(*this)(i,j).  */
    void AdjustDofDirection(Array<int> &dofs);
 
+
+   void AdjustDofDirection(Array<int> &row_dofs,
+                           Array<int> &col_dofs);
+
    /// Replace small entries, abs(a_ij) <= eps, with zero.
    void Threshold(real_t eps);
 

linalg/hypre.cpp

@@ -3443,6 +3443,51 @@ HypreParMatrix *HypreParMatrixFromBlocks(Array2D<const HypreParMatrix*> &blocks,
    }
 }
 
+HypreParMatrix * GetSubHypreParMatrix(const Array<int> &tdofs,
+                                      const HypreParMatrix & A)
+{
+   int nrows = tdofs.Size();
+   int gncols = A.GetGlobalNumCols();
+   SparseMatrix Sr(nrows, gncols);
+
+   for (int i = 0; i<nrows; i++)
+   {
+      int col = tdofs[i] + A.RowPart()[0];
+      Sr.Set(i,col,1.0);
+   }
+   Sr.Finalize();
+
+
+   int newrows[2];
+   int newcols[2];
+
+   int row_offset;
+   MPI_Scan(&nrows,&row_offset,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
+
+   row_offset-=nrows;
+   newrows[0] = row_offset;
+   newrows[1] = row_offset+nrows;
+   newcols[0] = A.ColPart()[0];
+   newcols[1] = A.ColPart()[1];
+   int gnrows;
+   MPI_Allreduce(&nrows, &gnrows,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
+
+   HypreParMatrix * Prt = new HypreParMatrix(MPI_COMM_WORLD, nrows, gnrows,
+                                             gncols, Sr.GetI(), Sr.GetJ(),
+                                             Sr.GetData(), newrows,newcols);
+
+
+   HypreParMatrix * Pr = Prt->Transpose();
+   delete Prt;
+
+   HypreParMatrix * tmp = RAP(&A,Pr);
+   delete Pr;
+
+   return tmp;
+
+}
+
+
 HypreParMatrix * HypreParMatrixFromBlocks(Array2D<HypreParMatrix*> &blocks,
                                           Array2D<real_t> *blockCoeff)
 {
@@ -5224,7 +5269,7 @@ void HypreBoomerAMG::SetDefaultOptions()
       Pmax         = 4;    // max number of elements per row in P
 
       // AMG relaxation options:
-      relax_type   = 8;    // 8 = l1-GS, 6 = symm. GS, 3 = GS, 18 = l1-Jacobi
+      relax_type   = 88;    // 8 = l1-GS, 6 = symm. GS, 3 = GS, 18 = l1-Jacobi
       relax_sweeps = 1;    // relaxation sweeps on each level
 
       // Additional options:
@@ -5741,7 +5786,7 @@ void HypreAMS::MakeSolver(int sdim, int cycle_type)
    const bool hypre_gpu = HypreUsingGPU();
    int amg_coarsen_type = hypre_gpu ? 8 : 10;
    int amg_agg_levels   = hypre_gpu ? 0 : 1;
-   int amg_rlx_type     = hypre_gpu ? 18 : 8;
+   int amg_rlx_type     = hypre_gpu ? 18 : 88;
    int rlx_type         = hypre_gpu ? 1: 2;
    real_t theta         = 0.25;
    int amg_interp_type  = 6;

linalg/hypre.hpp

@@ -1063,6 +1063,11 @@ MFEM_DEPRECATED HypreParMatrix *HypreParMatrixFromBlocks(
    Array2D<HypreParMatrix*> &blocks,
    Array2D<real_t> *blockCoeff=NULL);
 
+/** Extract a SubMatrix of a HypreParMatrix constructed
+    from the given rows and cols (local numbering) */
+HypreParMatrix * GetSubHypreParMatrix(const Array<int> &dofs,
+                                      const HypreParMatrix & A);
+
 /** @brief Eliminate essential BC specified by @a ess_dof_list from the solution
     @a X to the r.h.s. @a B. */
 /** Here @a A is a matrix with eliminated BC, while @a Ae is such that (A+Ae) is

linalg/mumps.hpp

@@ -52,7 +52,7 @@ public:
    /// Specify the reordering strategy for the MUMPS solver
    enum ReorderingStrategy
    {
-      /// Let MUMPS automatically decide the reording strategy
+      /// Let MUMPS automatically decide the reordering strategy
       AUTOMATIC = 0,
       /// Approximate Minimum Degree with auto quasi-dense row detection is used
       AMD,
@@ -98,7 +98,7 @@ public:
    void Mult(const Vector &x, Vector &y) const;
 
    /**
-    * @brief Solve $ Y_i = Op^{-T} X_i $
+    * @brief Solve $ Y_i = Op^{-1} X_i $
     *
     * @param X Array of RHS vectors
     * @param Y Array of Solution vectors
@@ -129,8 +129,8 @@ public:
     * - 0:  No output printed
     * - 1:  Only errors printed
     * - 2:  Errors, warnings, and main stats printed
-    * - 3:  Errors, warning, main stats, and terse diagnostics printed
-    * - 4:  Errors, warning, main stats, diagnostics, and input/output printed
+    * - 3:  Errors, warnings, main stats, and terse diagnostics printed
+    * - 4:  Errors, warnings, main stats, diagnostics, and input/output printed
     *
     * @param print_lvl Print level, default is 2
     *

linalg/operator.cpp

@@ -466,10 +466,11 @@ RAPOperator::RAPOperator(const Operator &Rt_, const Operator &A_,
 
 TripleProductOperator::TripleProductOperator(
    const Operator *A, const Operator *B, const Operator *C,
-   bool ownA, bool ownB, bool ownC)
+   bool ownA, bool ownB, bool ownC, real_t alpha_)
    : Operator(A->Height(), C->Width())
    , A(A), B(B), C(C)
    , ownA(ownA), ownB(ownB), ownC(ownC)
+   , alpha(alpha_)
 {
    MFEM_VERIFY(A->Width() == B->Height(),
                "incompatible Operators: A->Width() = " << A->Width()

linalg/operator.hpp

@@ -1011,27 +1011,28 @@ public:
 };
 
 
-/// General triple product operator x -> A*B*C*x, with ownership of the factors.
+/// General triple product operator x -> α A*B*C*x, with ownership of the factors.
 class TripleProductOperator : public Operator
 {
    const Operator *A;
    const Operator *B;
    const Operator *C;
    bool ownA, ownB, ownC;
+   const real_t alpha;
    mutable Vector t1, t2;
    MemoryClass mem_class;
 
 public:
    TripleProductOperator(const Operator *A, const Operator *B,
-                         const Operator *C, bool ownA, bool ownB, bool ownC);
+                         const Operator *C, bool ownA, bool ownB, bool ownC, real_t alpha_ = 1.0);
 
    MemoryClass GetMemoryClass() const override { return mem_class; }
 
    void Mult(const Vector &x, Vector &y) const override
-   { C->Mult(x, t1); B->Mult(t1, t2); A->Mult(t2, y); }
+   { C->Mult(x, t1); B->Mult(t1, t2); A->Mult(t2, y); y*=alpha; }
 
    void MultTranspose(const Vector &x, Vector &y) const override
-   { A->MultTranspose(x, t2); B->MultTranspose(t2, t1); C->MultTranspose(t1, y); }
+   { A->MultTranspose(x, t2); B->MultTranspose(t2, t1); C->MultTranspose(t1, y); y*=alpha; }
 
    virtual ~TripleProductOperator();
 };

linalg/solvers.cpp

@@ -683,8 +683,7 @@ void SLISolver::Mult(const Vector &b, Vector &x) const
          oper->Mult(x, r);  // r = A x
          if (!zero_b) { subtract(b, r, r); } // r = b - A x
          prec->Mult(r, z);  // z = B r
-         if (!zero_b) { x += z; } // x = x + B (b - A x)
-         else { x -= z; }         // x = x - B (A x)
+         add(x, omega, z, x); // x = x + ω B (b - A x)
       }
       converged = true;
       final_iter = i;
@@ -701,7 +700,7 @@ void SLISolver::Mult(const Vector &b, Vector &x) const
          oper->Mult(x, r);  // r = A x
          subtract(b, r, r); // r = b - A x
          prec->Mult(r, z);  // z = B r
-         x += z;            // x = x + B (b - A x)
+         add(x, omega, z, x); // x = x + ω B (b - A x)
       }
       converged = true;
       final_iter = i;
@@ -754,15 +753,13 @@ void SLISolver::Mult(const Vector &b, Vector &x) const
    final_iter = max_iter;
    for (i = 1; true; )
    {
-      if (prec)
+      if (prec) //  x = x + ω B (b - A x)
       {
-         if (!zero_b) { x += z; }  // x = x + B (b - A x)
-         else { x -= z; }          // x = x - B (A x)
+         add(x, omega, z, x);
       }
       else
       {
-         if (!zero_b) { x += r; }  // x = x + (b - A x)
-         else { x -= r; }          // x = x - (A x)
+         add(x, omega, r, x);
       }
 
       oper->Mult(x, r);

linalg/solvers.hpp

@@ -586,19 +586,20 @@ private:
 };
 
 
-/// Stationary linear iteration: x <- x + B (b - A x)
+/// Stationary linear iteration: x <- x + ω B (b - A x)
 class SLISolver : public IterativeSolver
 {
 protected:
    mutable Vector r, z;
-
+   real_t omega; ///< Relaxation parameter
    void UpdateVectors();
 
 public:
-   SLISolver() { }
+   SLISolver(real_t omega_ = 1.0) : omega(omega_) { }
 
 #ifdef MFEM_USE_MPI
-   SLISolver(MPI_Comm comm_) : IterativeSolver(comm_) { }
+   SLISolver(MPI_Comm comm_, real_t omega_ = 1.0) : IterativeSolver(comm_),
+      omega(omega_) { }
 #endif
 
    void SetOperator(const Operator &op) override

makefile

@@ -224,7 +224,7 @@ CXXFLAGS ?= $(OPTIM_FLAGS)
 # MPI configuration
 ifneq ($(MFEM_USE_MPI),YES)
    HOST_CXX = $(CXX)
-   PKGS_NEED_MPI = SUPERLU MUMPS STRUMPACK PETSC PUMI SLEPC MKL_CPARDISO
+   PKGS_NEED_MPI = SUPERLU MUMPS COMPLEX_MUMPS STRUMPACK PETSC PUMI SLEPC MKL_CPARDISO
    $(foreach mpidep,$(PKGS_NEED_MPI),$(if $(MFEM_USE_$(mpidep):NO=),\
      $(warning *** [MPI is OFF] setting MFEM_USE_$(mpidep) = NO)\
      $(eval override MFEM_USE_$(mpidep)=NO),))
@@ -299,7 +299,7 @@ ifeq ($(MFEM_USE_LEGACY_OPENMP),YES)
 endif
 
 # List of MFEM dependencies, that require the *_LIB variable to be non-empty
-MFEM_REQ_LIB_DEPS = SUPERLU MUMPS METIS FMS CONDUIT SIDRE LAPACK SUNDIALS\
+MFEM_REQ_LIB_DEPS = SUPERLU MUMPS COMPLEX_MUMPS METIS FMS CONDUIT SIDRE LAPACK SUNDIALS\
  SUITESPARSE STRUMPACK GINKGO GNUTLS HDF5 NETCDF SLEPC PETSC MPFR PUMI HIOP\
  GSLIB OCCA CEED RAJA UMPIRE MKL_CPARDISO MKL_PARDISO AMGX MAGMA CALIPER PARELAG\
  TRIBOL BENCHMARK MOONOLITH ALGOIM
@@ -369,7 +369,7 @@ MFEM_DEFINES = MFEM_VERSION MFEM_VERSION_STRING MFEM_GIT_STRING MFEM_USE_MPI\
  MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA MFEM_USE_HIP\
  MFEM_USE_OCCA MFEM_USE_MOONOLITH MFEM_USE_CEED MFEM_USE_RAJA MFEM_USE_UMPIRE\
  MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO MFEM_USE_AMGX\
- MFEM_USE_MAGMA MFEM_USE_MUMPS MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_CALIPER\
+ MFEM_USE_MAGMA MFEM_USE_MUMPS MFEM_USE_COMPLEX_MUMPS MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_CALIPER\
  MFEM_USE_BENCHMARK MFEM_USE_PARELAG MFEM_USE_TRIBOL MFEM_USE_ALGOIM MFEM_USE_ENZYME\
  MFEM_SOURCE_DIR MFEM_INSTALL_DIR MFEM_SHARED_BUILD MFEM_USE_DOUBLE MFEM_USE_SINGLE
 
@@ -406,7 +406,7 @@ MFEM_INSTALL_DIR = $(abspath $(MFEM_PREFIX))
 # If we have 'config' target, export variables used by config/makefile
 ifneq (,$(filter config,$(MAKECMDGOALS)))
    export $(MFEM_DEFINES) MFEM_DEFINES $(MFEM_CONFIG_VARS) MFEM_CONFIG_VARS
-   export VERBOSE HYPRE_OPT PUMI_DIR MUMPS_OPT GSLIB_OPT
+   export VERBOSE HYPRE_OPT PUMI_DIR MUMPS_OPT COMPLEX_MUMPS_OPT GSLIB_OPT
 endif
 
 # If we have 'install' target, export variables used by config/makefile
@@ -736,6 +736,7 @@ status info:
 	$(info MFEM_USE_SUPERLU       = $(MFEM_USE_SUPERLU))
 	$(info MFEM_USE_SUPERLU5      = $(MFEM_USE_SUPERLU5))
 	$(info MFEM_USE_MUMPS         = $(MFEM_USE_MUMPS))
+	$(info MFEM_USE_COMPLEX_MUMPS = $(MFEM_USE_COMPLEX_MUMPS))
 	$(info MFEM_USE_STRUMPACK     = $(MFEM_USE_STRUMPACK))
 	$(info MFEM_USE_GINKGO        = $(MFEM_USE_GINKGO))
 	$(info MFEM_USE_AMGX          = $(MFEM_USE_AMGX))

miniapps/dpg/CMakeLists.txt

@@ -14,14 +14,16 @@ util/weakform.cpp
 util/complexweakform.cpp
 util/blockstaticcond.cpp
 util/complexstaticcond.cpp
-util/pml.cpp)
+util/pml.cpp
+util/preconditioners.cpp)
 
 list(APPEND DPG_HEADERS
 util/weakform.hpp
 util/complexweakform.hpp
 util/blockstaticcond.hpp
 util/complexstaticcond.hpp
-util/pml.hpp)
+util/pml.hpp
+util/preconditioners.hpp)
 
 if (MFEM_USE_MPI)
   list(APPEND DPG_SOURCES

miniapps/dpg/makefile

@@ -12,20 +12,19 @@
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
-MFEM_INSTALL_DIR ?= ../../mfem
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/dpg/,)
-CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
-   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/dpg,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
 
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
-DPG_REAL_SEQ_SRC = util/weakform.cpp util/blockstaticcond.cpp
+DPG_REAL_SEQ_SRC = util/weakform.cpp util/blockstaticcond.cpp util/utils.cpp util/preconditioners.cpp
 DPG_REAL_PAR_SRC = $(DPG_REAL_SEQ_SRC) util/pweakform.cpp 
 DPG_REAL_OBJ = $(DPG_REAL_PAR_SRC:.cpp=.o)
 
-DPG_COMPLEX_SEQ_SRC = util/complexweakform.cpp util/complexstaticcond.cpp util/pml.cpp
-DPG_COMPLEX_PAR_SRC = $(DPG_COMPLEX_SEQ_SRC) util/pcomplexweakform.cpp 
+DPG_COMPLEX_SEQ_SRC = util/maxwell_utils.cpp util/utils.cpp util/complexblockform.cpp util/complexweakform.cpp util/complexstaticcond.cpp util/pml.cpp util/preconditioners.cpp
+DPG_COMPLEX_PAR_SRC = $(DPG_COMPLEX_SEQ_SRC) util/pcomplexweakform.cpp util/pcomplexblockform.cpp 
+
 DPG_COMPLEX_OBJ = $(DPG_COMPLEX_PAR_SRC:.cpp=.o)
 
 DIFFUSION_SRC = diffusion.cpp $(DPG_REAL_SEQ_SRC)
@@ -34,6 +33,12 @@ DIFFUSION_OBJ = $(DIFFUSION_SRC:.cpp=.o)
 PDIFFUSION_SRC = pdiffusion.cpp $(DPG_REAL_PAR_SRC)
 PDIFFUSION_OBJ = $(PDIFFUSION_SRC:.cpp=.o)
 
+PDIFFUSIONPRIMAL_SRC = pdiffusion-primal.cpp $(DPG_REAL_PAR_SRC)
+PDIFFUSIONPRIMAL_OBJ = $(PDIFFUSIONPRIMAL_SRC:.cpp=.o)
+
+PVDIFFUSIONPRIMAL_SRC = pvectordiffusion-primal.cpp $(DPG_REAL_PAR_SRC)
+PVDIFFUSIONPRIMAL_OBJ = $(PVDIFFUSIONPRIMAL_SRC:.cpp=.o)
+
 CONVECTIONDIFFUSION_SRC = convection-diffusion.cpp $(DPG_REAL_SEQ_SRC)
 CONVECTIONDIFFUSION_OBJ = $(CONVECTIONDIFFUSION_SRC:.cpp=.o)
 
@@ -46,14 +51,34 @@ ACOUSTICS_OBJ = $(ACOUSTICS_SRC:.cpp=.o)
 PACOUSTICS_SRC = pacoustics.cpp $(DPG_COMPLEX_PAR_SRC)
 PACOUSTICS_OBJ = $(PACOUSTICS_SRC:.cpp=.o)
 
+PACOUSTICS_PRIMAL_SRC = pacoustics-primal.cpp $(DPG_COMPLEX_PAR_SRC)
+PACOUSTICS_PRIMAL_OBJ = $(PACOUSTICS_PRIMAL_SRC:.cpp=.o)
+
 MAXWELL_SRC = maxwell.cpp $(DPG_COMPLEX_SEQ_SRC)
 MAXWELL_OBJ = $(MAXWELL_SRC:.cpp=.o)
 
 PMAXWELL_SRC = pmaxwell.cpp $(DPG_COMPLEX_PAR_SRC)
 PMAXWELL_OBJ = $(PMAXWELL_SRC:.cpp=.o)
 
+PMAXWELL_V_SRC = pmaxwell-verify.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_V_OBJ = $(PMAXWELL_V_SRC:.cpp=.o)
+
+PMAXWELL_PRIMAL_SRC = pmaxwell-primal.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_PRIMAL_OBJ = $(PMAXWELL_PRIMAL_SRC:.cpp=.o)
+
+PMAXWELL_FEM_SRC = pmaxwell-fem.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_FEM_OBJ = $(PMAXWELL_FEM_SRC:.cpp=.o)
+
+PMAXWELL_UW_COUPLED_SRC = pmaxwell-coupled.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_UW_COUPLED_OBJ = $(PMAXWELL_UW_COUPLED_SRC:.cpp=.o)
+
+PMAXWELL_FEM_COUPLED_SRC = pmaxwell-coupled-fem.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_FEM_COUPLED_OBJ = $(PMAXWELL_FEM_COUPLED_SRC:.cpp=.o)
+
 SEQ_MINIAPPS = diffusion convection-diffusion acoustics maxwell
-PAR_MINIAPPS = pdiffusion pconvection-diffusion pacoustics pmaxwell
+PAR_MINIAPPS = pdiffusion pconvection-diffusion pacoustics pmaxwell pmaxwell-verify  \
+ pacoustics-primal pmaxwell-primal pdiffusion-primal pvectordiffusion-primal \
+ pmaxwell-coupled pmaxwell-coupled-fem pmaxwell-fem
 
 ifeq ($(MFEM_USE_MPI),NO)
    MINIAPPS = $(SEQ_MINIAPPS)
@@ -79,11 +104,6 @@ COMMON_LIB += $(if $(MFEM_SHARED:YES=),,\
  $(CONFIG_MK) | lib-common
 	$(MFEM_CXX) $(MFEM_FLAGS) -c $< -o $@
 
-util/%.o: $(SRC)util/%.cpp $(wildcard $(SRC)util/%.hpp) $(MFEM_LIB_FILE)\
- $(CONFIG_MK) | lib-common
-	mkdir -p $(@D)
-	$(MFEM_CXX) $(MFEM_FLAGS) -c $< -o $@
-
 all: $(MINIAPPS)
 
 diffusion: $(DIFFUSION_OBJ)
@@ -95,6 +115,12 @@ convection-diffusion: $(CONVECTIONDIFFUSION_OBJ)
 pdiffusion: $(PDIFFUSION_OBJ)
 	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PDIFFUSION_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
 
+pdiffusion-primal: $(PDIFFUSIONPRIMAL_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PDIFFUSIONPRIMAL_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+pvectordiffusion-primal: $(PVDIFFUSIONPRIMAL_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PVDIFFUSIONPRIMAL_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
 pconvection-diffusion: $(PCONVECTIONDIFFUSION_OBJ)
 	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PCONVECTIONDIFFUSION_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
 
@@ -107,9 +133,30 @@ maxwell: $(MAXWELL_OBJ)
 pacoustics: $(PACOUSTICS_OBJ)
 	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PACOUSTICS_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
 
+pacoustics-primal: $(PACOUSTICS_PRIMAL_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PACOUSTICS_PRIMAL_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
 pmaxwell: $(PMAXWELL_OBJ)
 	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
 
+pmaxwell-fem: $(PMAXWELL_FEM_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_FEM_OBJ) $(COMMON_LIB) $(MFEM_LIBS)	
+
+pmaxwell-verify: $(PMAXWELL_V_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_V_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+
+pmaxwell-primal: $(PMAXWELL_PRIMAL_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_PRIMAL_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+pmaxwell-primal-tokamak: $(PMAXWELL_PRIMAL_TOKAMAK_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_PRIMAL_TOKAMAK_OBJ) $(COMMON_LIB) $(MFEM_LIBS)	
+
+pmaxwell-coupled: $(PMAXWELL_UW_COUPLED_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_UW_COUPLED_OBJ) $(COMMON_LIB) $(MFEM_LIBS)		
+
+pmaxwell-coupled-fem: $(PMAXWELL_FEM_COUPLED_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_FEM_COUPLED_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
 
 # Rule for building lib-common
 lib-common:

miniapps/dpg/pacoustics-primal.cpp

@@ -0,0 +1,743 @@
+//                   MFEM Ultraweak DPG acoustics example
+//
+// Compile with: make pacoustics
+//
+// sample runs
+
+// mpirun -np 4 pacoustics-primal -o 3 -m ../../data/star.mesh -sref 1 -pref 2 -rnum 1.9 -sc -prob 0
+// mpirun -np 4 pacoustics-primal -o 3 -m ../../data/inline-quad.mesh -sref 1 -pref 2  -rnum 5.2 -sc -prob 1
+// mpirun -np 4 pacoustics-primal -o 4 -m ../../data/inline-tri.mesh -sref 1 -pref 2  -rnum 7.1 -sc -prob 1
+// mpirun -np 4 pacoustics-primal -o 2 -m ../../data/inline-hex.mesh -sref 0 -pref 1 -rnum 1.9 -sc -prob 0
+
+// Description:
+// This example code demonstrates the use of MFEM to define and solve
+// the "ultraweak" (UW) DPG formulation for the Helmholtz problem
+
+//     - Δ p - ω² p = f ,   in Ω
+//                p = p₀, on ∂Ω
+
+// It solves the following kinds of problems
+//    a) f̃ = 0 and p₀ is a plane wave
+//    b) A manufactured solution problem where p_exact is a gaussian beam
+
+// The DPG Primal deals with the Second Order Equation
+//     - Δ p - ω² p = f , in Ω    (1)
+//                p = p₀, on ∂Ω
+
+// The primal-DPG formulation is obtained by integration by parts of (1)
+// and the introduction of a trace unknown on the mesh skeleton
+
+// p ∈ H¹(Ω),
+// p̂ ∈ H^-1/2(Ω)
+// (∇p,∇q) + ω²(p,q) + <p̂,q> = 0,      ∀ q ∈ H^1(Ω)
+//                         p = p₀      on ∂Ω
+
+#include "mfem.hpp"
+#include "util/pcomplexweakform.hpp"
+#include "util/pml.hpp"
+#include "../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+complex<double> acoustics_solution(const Vector & X);
+void acoustics_solution_grad(const Vector & X,vector<complex<double>> &dp);
+complex<double> acoustics_solution_laplacian(const Vector & X);
+
+double p_exact_r(const Vector &x);
+double p_exact_i(const Vector &x);
+double rhs_func_r(const Vector &x);
+double rhs_func_i(const Vector &x);
+void gradp_exact_r(const Vector &x, Vector &gradu);
+void gradp_exact_i(const Vector &x, Vector &gradu);
+double d2_exact_r(const Vector &x);
+double d2_exact_i(const Vector &x);
+
+int dim;
+double omega;
+
+enum prob_type
+{
+   plane_wave,
+   gaussian_beam
+};
+
+static const char *enum_str[] =
+{
+   "plane_wave",
+   "gaussian_beam"
+};
+
+prob_type prob;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   bool static_cond = false;
+   int iprob = 0;
+   int sr = 0;
+   int pr = 0;
+   bool paraview = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&rnum, "-rnum", "--number-of-wavelengths",
+                  "Number of wavelengths");
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: plane wave, 1: Gaussian beam");
+   args.AddOption(&delta_order, "-do", "--delta-order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&sr, "-sref", "--serial-ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&pr, "-pref", "--parallel-ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+
+   if (iprob > 1) { iprob = 0; }
+   prob = (prob_type)iprob;
+   omega = 2.*M_PI*rnum;
+
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+
+   for (int i = 0; i<sr; i++)
+   {
+      mesh.UniformRefinement();
+   }
+   dim = mesh.Dimension();
+   MFEM_VERIFY(dim > 1, "Dimension = 1 is not supported in this example");
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   enum TrialSpace
+   {
+      p_space     = 0,
+      hatp_space  = 1,
+   };
+   enum TestSpace
+   {
+      q_space = 0,
+   };
+
+   // H1 space for p
+   FiniteElementCollection *p_fec = new H1_FECollection(order,dim);
+   ParFiniteElementSpace *p_fes = new ParFiniteElementSpace(&pmesh,p_fec);
+
+   // H^-1/2 space for p̂
+   FiniteElementCollection * hatp_fec = new RT_Trace_FECollection(order-1,dim);
+   ParFiniteElementSpace *hatp_fes = new ParFiniteElementSpace(&pmesh,hatp_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+   trial_fes.Append(p_fes);
+   trial_fes.Append(hatp_fes);
+   test_fec.Append(q_fec);
+
+   // Bilinear form Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negomeg2(-omega*omega);
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_fes,test_fec);
+   a->StoreMatrices(); // needed for AMR
+
+   // Trial itegrators
+   // (∇ p,∇ q)
+   a->AddTrialIntegrator(new DiffusionIntegrator(one),nullptr,
+                         TrialSpace::p_space,TestSpace::q_space);
+   //  ω² (p,q)
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(negomeg2), nullptr,
+                         TrialSpace::p_space, TestSpace::q_space);
+
+   // < p̂,q >
+   a->AddTrialIntegrator(new TraceIntegrator,nullptr,
+                         TrialSpace::hatp_space,TestSpace::q_space);
+
+   // test integrators
+   // (∇q,∇δq)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),nullptr,
+                        TestSpace::q_space, TestSpace::q_space);
+   // (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),nullptr,
+                        TestSpace::q_space, TestSpace::q_space);
+
+   // RHS
+   FunctionCoefficient f_rhs_r(rhs_func_r);
+   FunctionCoefficient f_rhs_i(rhs_func_i);
+   if (prob == prob_type::gaussian_beam)
+   {
+      a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs_r),
+                               new DomainLFIntegrator(f_rhs_i),
+                               TestSpace::q_space);
+   }
+
+   socketstream p_out_r;
+   socketstream p_out_i;
+   if (myid == 0)
+   {
+      std::cout << "\n  Ref |"
+                << "    Dofs    |"
+                << "    ω    |" ;
+      std::cout  << "  H¹ Error  |"
+                 << "  Rate  |" ;
+      std::cout << "  Residual  |"
+                << "  Rate  |"
+                << " PCG it |" << endl;
+      std::cout << std::string(82,'-')
+                << endl;
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0 = 0;
+
+   ParGridFunction p_r, p_i;
+   FunctionCoefficient pex_r(p_exact_r);
+   FunctionCoefficient pex_i(p_exact_i);
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   if (paraview)
+   {
+      paraview_dc = new ParaViewDataCollection(enum_str[prob], &pmesh);
+      paraview_dc->SetPrefixPath("ParaViewPrimal/Acoustics");
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("p_r",&p_r);
+      paraview_dc->RegisterField("p_i",&p_i);
+   }
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+   for (int it = 0; it<=pr; it++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         p_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      Array<int> offsets(3);
+      offsets[0] = 0;
+      offsets[1] = p_fes->GetVSize();
+      offsets[2] = hatp_fes->GetVSize();
+      offsets.PartialSum();
+
+      Vector x(2*offsets.Last());
+      x = 0.;
+
+      ParGridFunction p_gf_r(p_fes, x, offsets[0]);
+      ParGridFunction p_gf_i(p_fes, x, offsets.Last()+ offsets[0]);
+      p_gf_r.ProjectBdrCoefficient(pex_r, ess_bdr);
+      p_gf_i.ProjectBdrCoefficient(pex_i, ess_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+      int num_blocks = BlockA_r->NumRowBlocks();
+      Array<int> tdof_offsets(2*num_blocks+1);
+
+      tdof_offsets[0] = 0;
+      for (int i=0; i<num_blocks; i++)
+      {
+         int h = BlockA_r->GetBlock(i,i).Height();
+         tdof_offsets[i+1] = h;
+         tdof_offsets[num_blocks+i+1] = h;
+      }
+      tdof_offsets.PartialSum();
+
+      BlockOperator blockA(tdof_offsets);
+      for (int i = 0; i<num_blocks; i++)
+      {
+         for (int j = 0; j<num_blocks; j++)
+         {
+            blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
+            blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
+         }
+      }
+
+      X = 0.;
+      BlockDiagonalPreconditioner M(tdof_offsets);
+      M.owns_blocks=0;
+
+      HypreBoomerAMG * solver_p = new HypreBoomerAMG((HypreParMatrix &)
+                                                     BlockA_r->GetBlock(0,0));
+      solver_p->SetPrintLevel(0);
+      M.SetDiagonalBlock(0,solver_p);
+      M.SetDiagonalBlock(num_blocks,solver_p);
+
+      HypreSolver * solver_hatp = nullptr;
+      if (dim == 2)
+      {
+         // AMS preconditioner for 2D H(div) (trace) space
+         solver_hatp = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(1,1),
+                                    hatp_fes);
+         dynamic_cast<HypreAMS*>(solver_hatp)->SetPrintLevel(0);
+      }
+      else
+      {
+         // ADS preconditioner for 3D H(div) (trace) space
+         solver_hatp = new HypreADS((HypreParMatrix &)BlockA_r->GetBlock(1,1),
+                                    hatp_fes);
+         dynamic_cast<HypreADS*>(solver_hatp)->SetPrintLevel(0);
+      }
+
+      M.SetDiagonalBlock(1,solver_hatp);
+      M.SetDiagonalBlock(num_blocks+1,solver_hatp);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-6);
+      cg.SetMaxIter(10000);
+      cg.SetPrintLevel(0);
+      cg.SetPreconditioner(M);
+      cg.SetOperator(blockA);
+      cg.Mult(B, X);
+
+      for (int i = 0; i<num_blocks; i++)
+      {
+         delete &M.GetDiagonalBlock(i);
+      }
+
+      int num_iter = cg.GetNumIterations();
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      double maxresidual = residuals.Max();
+      double globalresidual = residual * residual;
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+      p_r.MakeRef(p_fes, x, 0);
+      p_i.MakeRef(p_fes, x, offsets.Last());
+
+      int dofs = 0;
+      for (int i = 0; i<trial_fes.Size(); i++)
+      {
+         dofs += trial_fes[i]->GlobalTrueVSize();
+      }
+
+      double H1Error = 0.0;
+      double rate_err = 0.0;
+      VectorFunctionCoefficient pex_grad_r(dim,gradp_exact_r);
+      VectorFunctionCoefficient pex_grad_i(dim,gradp_exact_i);
+      double p_err_r = p_r.ComputeH1Error(&pex_r,&pex_grad_r);
+      double p_err_i = p_i.ComputeH1Error(&pex_i,&pex_grad_i);
+
+      H1Error = sqrt(p_err_r*p_err_r + p_err_i*p_err_i);
+
+      rate_err = (it) ? dim*log(err0/H1Error)/log((double)dof0/dofs) : 0.0;
+      err0 = H1Error;
+
+      double rate_res = (it) ? dim*log(res0/globalresidual)/log((
+                                                                   double)dof0/dofs) : 0.0;
+
+      res0 = globalresidual;
+      dof0 = dofs;
+
+      if (myid == 0)
+      {
+         std::ios oldState(nullptr);
+         oldState.copyfmt(std::cout);
+         std::cout << std::right << std::setw(5) << it << " | "
+                   << std::setw(10) <<  dof0 << " | "
+                   << std::setprecision(1) << std::fixed
+                   << std::setw(4) <<  2*rnum << " π  | ";
+         std::cout << std::setprecision(3) << std::setw(10)
+                   << std::scientific <<  err0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_err << " | " ;
+         std::cout << std::setprecision(3)
+                   << std::setw(10) << std::scientific <<  res0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_res << " | "
+                   << std::setw(6) << std::fixed << num_iter << " | "
+                   << std::endl;
+         std::cout.copyfmt(oldState);
+      }
+
+      if (visualization)
+      {
+         const char * keys = (it == 0 && dim == 2) ? "jRcml\n" : nullptr;
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         VisualizeField(p_out_r,vishost, visport, p_r,
+                        "Numerical presure (real part)", 0, 0, 500, 500, keys);
+         VisualizeField(p_out_i,vishost, visport, p_i,
+                        "Numerical presure (imaginary part)", 501, 0, 500, 500, keys);
+      }
+
+      if (paraview)
+      {
+         paraview_dc->SetCycle(it);
+         paraview_dc->SetTime((double)it);
+         paraview_dc->Save();
+      }
+
+      if (it == pr)
+      {
+         break;
+      }
+
+      pmesh.UniformRefinement();
+
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   if (paraview)
+   {
+      delete paraview_dc;
+   }
+
+   delete a;
+   delete q_fec;
+   delete hatp_fes;
+   delete hatp_fec;
+   delete p_fec;
+   delete p_fes;
+
+   return 0;
+}
+
+double p_exact_r(const Vector &x)
+{
+   return acoustics_solution(x).real();
+}
+
+double p_exact_i(const Vector &x)
+{
+   return acoustics_solution(x).imag();
+}
+
+void gradp_exact_r(const Vector &x, Vector &grad_r)
+{
+   grad_r.SetSize(x.Size());
+   vector<complex<double>> grad;
+   acoustics_solution_grad(x,grad);
+   for (unsigned i = 0; i < grad.size(); i++)
+   {
+      grad_r[i] = grad[i].real();
+   }
+}
+
+void gradp_exact_i(const Vector &x, Vector &grad_i)
+{
+   grad_i.SetSize(x.Size());
+   vector<complex<double>> grad;
+   acoustics_solution_grad(x,grad);
+   for (unsigned i = 0; i < grad.size(); i++)
+   {
+      grad_i[i] = grad[i].imag();
+   }
+}
+
+double d2_exact_r(const Vector &x)
+{
+   return acoustics_solution_laplacian(x).real();
+}
+
+double d2_exact_i(const Vector &x)
+{
+   return acoustics_solution_laplacian(x).imag();
+}
+
+
+// f = -Δ p - ω² p
+double rhs_func_r(const Vector &x)
+{
+   return -d2_exact_r(x) - omega * omega * p_exact_r(x);
+}
+
+// f = -Δ p - ω² p
+double rhs_func_i(const Vector &x)
+{
+   return -d2_exact_i(x) - omega * omega * p_exact_i(x);
+}
+
+complex<double> acoustics_solution(const Vector & X)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   switch (prob)
+   {
+      case plane_wave:
+      {
+         double beta = omega/std::sqrt((double)X.Size());
+         complex<double> alpha = beta * zi * X.Sum();
+         return exp(alpha);
+      }
+      break;
+      case gaussian_beam:
+      {
+         double rk = omega;
+         double degrees = 45;
+         double alpha = (180+degrees) * M_PI/180.;
+         double sina = sin(alpha);
+         double cosa = cos(alpha);
+         // shift the origin
+         double shift = 0.1;
+         double xprim=X(0) + shift;
+         double yprim=X(1) + shift;
+
+         double x = xprim*sina - yprim*cosa;
+         double y = xprim*cosa + yprim*sina;
+         //wavelength
+         double rl = 2.*M_PI/rk;
+
+         // beam waist radius
+         double w0 = 0.05;
+
+         // function w
+         double fact = rl/M_PI/(w0*w0);
+         double aux = 1. + (fact*y)*(fact*y);
+
+         double w = w0*sqrt(aux);
+
+         double phi0 = atan(fact*y);
+
+         double r = y + 1./y/(fact*fact);
+
+         // pressure
+         complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r +
+                              zi*phi0/2.;
+         double pf = pow(2.0/M_PI/(w*w),0.25);
+
+         return pf*exp(ze);
+      }
+      break;
+      default:
+         MFEM_ABORT("Should be unreachable");
+         return 1;
+         break;
+   }
+}
+
+void acoustics_solution_grad(const Vector & X, vector<complex<double>> & dp)
+{
+   dp.resize(X.Size());
+   complex<double> zi = complex<double>(0., 1.);
+   // initialize
+   for (int i = 0; i<X.Size(); i++) { dp[i] = 0.0; }
+   switch (prob)
+   {
+      case plane_wave:
+      {
+         double beta = omega/std::sqrt((double)X.Size());
+         complex<double> alpha = beta * zi * X.Sum();
+         complex<double> p = exp(alpha);
+         for (int i = 0; i<X.Size(); i++)
+         {
+            dp[i] = zi * beta * p;
+         }
+      }
+      break;
+      case gaussian_beam:
+      {
+         double rk = omega;
+         double degrees = 45;
+         double alpha = (180+degrees) * M_PI/180.;
+         double sina = sin(alpha);
+         double cosa = cos(alpha);
+         // shift the origin
+         double shift = 0.1;
+         double xprim=X(0) + shift;
+         double yprim=X(1) + shift;
+
+         double x = xprim*sina - yprim*cosa;
+         double y = xprim*cosa + yprim*sina;
+         double dxdxprim = sina, dxdyprim = -cosa;
+         double dydxprim = cosa, dydyprim =  sina;
+         //wavelength
+         double rl = 2.*M_PI/rk;
+
+         // beam waist radius
+         double w0 = 0.05;
+
+         // function w
+         double fact = rl/M_PI/(w0*w0);
+         double aux = 1. + (fact*y)*(fact*y);
+
+         double w = w0*sqrt(aux);
+         double dwdy = w0*fact*fact*y/sqrt(aux);
+
+         double phi0 = atan(fact*y);
+         double dphi0dy = cos(phi0)*cos(phi0)*fact;
+
+         double r = y + 1./y/(fact*fact);
+         double drdy = 1. - 1./(y*y)/(fact*fact);
+
+         // pressure
+         complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r +
+                              zi*phi0/2.;
+
+         complex<double> zdedx = -2.*x/(w*w) - 2.*zi*M_PI*x/rl/r;
+         complex<double> zdedy = 2.*x*x/(w*w*w)*dwdy - zi*rk + zi*M_PI*x*x/rl/
+                                 (r*r)*drdy + zi*dphi0dy/2.;
+
+         double pf = pow(2.0/M_PI/(w*w),0.25);
+         double dpfdy = -pow(2./M_PI/(w*w),-0.75)/M_PI/(w*w*w)*dwdy;
+
+         complex<double> zp = pf*exp(ze);
+         complex<double> zdpdx = zp*zdedx;
+         complex<double> zdpdy = dpfdy*exp(ze)+zp*zdedy;
+
+         dp[0] = (zdpdx*dxdxprim + zdpdy*dydxprim);
+         dp[1] = (zdpdx*dxdyprim + zdpdy*dydyprim);
+      }
+      break;
+      default:
+         MFEM_ABORT("Should be unreachable");
+         break;
+   }
+}
+
+complex<double> acoustics_solution_laplacian(const Vector & X)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   switch (prob)
+   {
+      case plane_wave:
+      {
+         double beta = omega/std::sqrt((double)X.Size());
+         complex<double> alpha = beta * zi * X.Sum();
+         return dim * beta * beta * exp(alpha);
+      }
+      break;
+      case gaussian_beam:
+      {
+         double rk = omega;
+         double degrees = 45;
+         double alpha = (180+degrees) * M_PI/180.;
+         double sina = sin(alpha);
+         double cosa = cos(alpha);
+         // shift the origin
+         double shift = 0.1;
+         double xprim=X(0) + shift;
+         double yprim=X(1) + shift;
+
+         double x = xprim*sina - yprim*cosa;
+         double y = xprim*cosa + yprim*sina;
+         double dxdxprim = sina, dxdyprim = -cosa;
+         double dydxprim = cosa, dydyprim =  sina;
+         //wavelength
+         double rl = 2.*M_PI/rk;
+
+         // beam waist radius
+         double w0 = 0.05;
+
+         // function w
+         double fact = rl/M_PI/(w0*w0);
+         double aux = 1. + (fact*y)*(fact*y);
+
+         double w = w0*sqrt(aux);
+         double dwdy = w0*fact*fact*y/sqrt(aux);
+         double d2wdydy = w0*fact*fact*(1. - (fact*y)*(fact*y)/aux)/sqrt(aux);
+
+         double phi0 = atan(fact*y);
+         double dphi0dy = cos(phi0)*cos(phi0)*fact;
+         double d2phi0dydy = -2.*cos(phi0)*sin(phi0)*fact*dphi0dy;
+
+         double r = y + 1./y/(fact*fact);
+         double drdy = 1. - 1./(y*y)/(fact*fact);
+         double d2rdydy = 2./(y*y*y)/(fact*fact);
+
+         // pressure
+         complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r +
+                              zi*phi0/2.;
+
+         complex<double> zdedx = -2.*x/(w*w) - 2.*zi*M_PI*x/rl/r;
+         complex<double> zdedy = 2.*x*x/(w*w*w)*dwdy - zi*rk + zi*M_PI*x*x/rl/
+                                 (r*r)*drdy + zi*dphi0dy/2.;
+         complex<double> zd2edxdx = -2./(w*w) - 2.*zi*M_PI/rl/r;
+         complex<double> zd2edxdy = 4.*x/(w*w*w)*dwdy + 2.*zi*M_PI*x/rl/(r*r)*drdy;
+         complex<double> zd2edydx = zd2edxdy;
+         complex<double> zd2edydy = -6.*x*x/(w*w*w*w)*dwdy*dwdy + 2.*x*x/
+                                    (w*w*w)*d2wdydy - 2.*zi*M_PI*x*x/rl/(r*r*r)*drdy*drdy
+                                    + zi*M_PI*x*x/rl/(r*r)*d2rdydy + zi/2.*d2phi0dydy;
+
+         double pf = pow(2.0/M_PI/(w*w),0.25);
+         double dpfdy = -pow(2./M_PI/(w*w),-0.75)/M_PI/(w*w*w)*dwdy;
+         double d2pfdydy = -1./M_PI*pow(2./M_PI,-0.75)*(-1.5*pow(w,-2.5)
+                                                        *dwdy*dwdy + pow(w,-1.5)*d2wdydy);
+
+
+         complex<double> zp = pf*exp(ze);
+         complex<double> zdpdx = zp*zdedx;
+         complex<double> zdpdy = dpfdy*exp(ze)+zp*zdedy;
+         complex<double> zd2pdxdx = zdpdx*zdedx + zp*zd2edxdx;
+         complex<double> zd2pdxdy = zdpdy*zdedx + zp*zd2edxdy;
+         complex<double> zd2pdydx = dpfdy*exp(ze)*zdedx + zdpdx*zdedy + zp*zd2edydx;
+         complex<double> zd2pdydy = d2pfdydy*exp(ze) + dpfdy*exp(
+                                       ze)*zdedy + zdpdy*zdedy + zp*zd2edydy;
+
+         return (zd2pdxdx*dxdxprim + zd2pdydx*dydxprim)*dxdxprim
+                + (zd2pdxdy*dxdxprim + zd2pdydy*dydxprim)*dydxprim
+                + (zd2pdxdx*dxdyprim + zd2pdydx*dydyprim)*dxdyprim
+                + (zd2pdxdy*dxdyprim + zd2pdydy*dydyprim)*dydyprim;
+      }
+      break;
+      default:
+         MFEM_ABORT("Should be unreachable");
+         return 1;
+         break;
+   }
+}

miniapps/dpg/pacoustics.cpp

@@ -121,6 +121,7 @@
 #include "mfem.hpp"
 #include "util/pcomplexweakform.hpp"
 #include "util/pml.hpp"
+#include "util/preconditioners.hpp"
 #include "../common/mfem-common.hpp"
 #include <fstream>
 #include <iostream>
@@ -192,6 +193,9 @@ int main(int argc, char *argv[])
    int iprob = 0;
    int sr = 0;
    int pr = 0;
+   bool pmg = false;
+   int pmg_levels = -1;
+   real_t relax_factor = 2.0/3;
    int visport = 19916;
    bool exact_known = false;
    bool with_pml = false;
@@ -216,6 +220,12 @@ int main(int argc, char *argv[])
                   "Number of parallel refinements.");
    args.AddOption(&pr, "-pref", "--parallel-ref",
                   "Number of parallel refinements.");
+   args.AddOption(&pmg, "-pmg", "--p-refinement-multigrid", "-no-pmg",
+                  "--no-p-refinement-multigrid", "Enable P-Refinement Multigrid.");
+   args.AddOption(&pmg_levels, "-pmgl","--p-refinement-multigrid-levels",
+                  "Number of levels for P-Refinement Multigrid.");
+   args.AddOption(&relax_factor, "-rf", "--relaxation-factor",
+                  "Relaxation factor for the p-multigrid smoother.");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -536,7 +546,7 @@ int main(int argc, char *argv[])
                 << "    ω    |" ;
       if (exact_known)
       {
-         std::cout  << "  L2 Error  |"
+         std::cout  << "  L² Error  |"
                     << "  Rate  |" ;
       }
       std::cout << "  Residual  |"
@@ -618,91 +628,69 @@ int main(int argc, char *argv[])
       a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
 
       ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
       BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
-      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
 
-      int num_blocks = BlockA_r->NumRowBlocks();
-      Array<int> tdof_offsets(2*num_blocks+1);
-
-      tdof_offsets[0] = 0;
-      int skip = (static_cond) ? 0 : 2;
-      int k = (static_cond) ? 2 : 0;
-      for (int i=0; i<num_blocks; i++)
+      Array<ParFiniteElementSpace *> prec_fes;
+      if (static_cond)
       {
-         tdof_offsets[i+1] = trial_fes[i+k]->GetTrueVSize();
-         tdof_offsets[num_blocks+i+1] = trial_fes[i+k]->GetTrueVSize();
-      }
-      tdof_offsets.PartialSum();
-
-      BlockOperator blockA(tdof_offsets);
-      for (int i = 0; i<num_blocks; i++)
-      {
-         for (int j = 0; j<num_blocks; j++)
-         {
-            blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
-            blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
-            blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
-            blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
-         }
-      }
-
-      X = 0.;
-      BlockDiagonalPreconditioner M(tdof_offsets);
-      M.owns_blocks=0;
-
-      if (!static_cond)
-      {
-         HypreBoomerAMG * solver_p = new HypreBoomerAMG((HypreParMatrix &)
-                                                        BlockA_r->GetBlock(0,0));
-         solver_p->SetPrintLevel(0);
-         solver_p->SetSystemsOptions(dim);
-         HypreBoomerAMG * solver_u = new HypreBoomerAMG((HypreParMatrix &)
-                                                        BlockA_r->GetBlock(1,1));
-         solver_u->SetPrintLevel(0);
-         solver_u->SetSystemsOptions(dim);
-         M.SetDiagonalBlock(0,solver_p);
-         M.SetDiagonalBlock(1,solver_u);
-         M.SetDiagonalBlock(num_blocks,solver_p);
-         M.SetDiagonalBlock(num_blocks+1,solver_u);
-      }
-
-      HypreBoomerAMG * solver_hatp = new HypreBoomerAMG((HypreParMatrix &)
-                                                        BlockA_r->GetBlock(skip,skip));
-      solver_hatp->SetPrintLevel(0);
-
-      HypreSolver * solver_hatu = nullptr;
-      if (dim == 2)
-      {
-         // AMS preconditioner for 2D H(div) (trace) space
-         solver_hatu = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(skip+1,skip+1),
-                                    hatu_fes);
-         dynamic_cast<HypreAMS*>(solver_hatu)->SetPrintLevel(0);
+         a->GetTraceFESpaces(prec_fes);
       }
       else
       {
-         // ADS preconditioner for 3D H(div) (trace) space
-         solver_hatu = new HypreADS((HypreParMatrix &)BlockA_r->GetBlock(skip+1,skip+1),
-                                    hatu_fes);
-         dynamic_cast<HypreADS*>(solver_hatu)->SetPrintLevel(0);
+         prec_fes = trial_fes;
+      }
+      Solver * cprec = nullptr;
+      if (pmg)
+      {
+#ifdef MFEM_USE_COMPLEX_MUMPS
+         bool mumps_coarse_solver = true;
+#else
+         bool mumps_coarse_solver = false;
+#endif
+         std::vector<Array<int>> ess_bdr_marker(prec_fes.Size());
+         for (int b = 0; b<prec_fes.Size(); b++)
+         {
+            if (pmesh.bdr_attributes.Size())
+            {
+               ess_bdr_marker[b].SetSize(pmesh.bdr_attributes.Max());
+               int ess_block = (static_cond) ? 0 : 2;
+               if (b == ess_block) // hatp
+               {
+                  ess_bdr_marker[b] = ess_bdr;
+               }
+               else
+               {
+                  ess_bdr_marker[b] = 0;
+               }
+            }
+         }
+         cprec = new ComplexPRefinementMultigrid(prec_fes, ess_bdr_marker, *Ahc,
+                                                 pmg_levels, relax_factor, mumps_coarse_solver );
+      }
+      else
+      {
+         BlockDiagonalPreconditioner * real_prec = new BlockDiagonalPreconditioner(
+            BlockA_r->RowOffsets());
+         real_prec->owns_blocks = 1;
+         for (int i = 0; i<BlockA_r->NumRowBlocks(); i++)
+         {
+            auto prec = MakeFESpaceDefaultSolver(prec_fes[i],0);
+            prec->SetOperator(BlockA_r->GetBlock(i,i));
+            real_prec->SetDiagonalBlock(i,prec);
+         }
+         cprec = new ComplexPreconditioner(real_prec, true);
       }
-
-      M.SetDiagonalBlock(skip,solver_hatp);
-      M.SetDiagonalBlock(skip+1,solver_hatu);
-      M.SetDiagonalBlock(skip+num_blocks,solver_hatp);
-      M.SetDiagonalBlock(skip+num_blocks+1,solver_hatu);
 
       CGSolver cg(MPI_COMM_WORLD);
       cg.SetRelTol(1e-6);
       cg.SetMaxIter(10000);
       cg.SetPrintLevel(0);
-      cg.SetPreconditioner(M);
-      cg.SetOperator(blockA);
+      cg.SetOperator(*Ahc);
+      cg.SetPreconditioner(*cprec);
       cg.Mult(B, X);
 
-      for (int i = 0; i<num_blocks; i++)
-      {
-         delete &M.GetDiagonalBlock(i);
-      }
+      delete cprec;
 
       int num_iter = cg.GetNumIterations();
 

miniapps/dpg/pconvection-diffusion.cpp

@@ -66,6 +66,7 @@
 
 #include "mfem.hpp"
 #include "util/pweakform.hpp"
+#include "util/preconditioners.hpp"
 #include "../common/mfem-common.hpp"
 #include <fstream>
 #include <iostream>
@@ -121,6 +122,9 @@ int main(int argc, char *argv[])
    real_t theta = 0.7;
    bool static_cond = false;
    epsilon = 1e0;
+   bool pmg = false;
+   int pmg_levels = -1;
+   real_t relax_factor = 2.0/3;
 
    bool visualization = true;
    int visport = 19916;
@@ -145,6 +149,12 @@ int main(int argc, char *argv[])
                   "Vector Coefficient beta");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&pmg, "-pmg", "--p-refinement-multigrid", "-no-pmg",
+                  "--no-p-refinement-multigrid", "Enable P-Refinement Multigrid.");
+   args.AddOption(&pmg_levels, "-pmgl","--p-refinement-multigrid-levels",
+                  "Number of levels for P-Refinement Multigrid.");
+   args.AddOption(&relax_factor, "-rf", "--relaxation-factor",
+                  "Relaxation factor for the p-multigrid smoother.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -453,44 +463,70 @@ int main(int argc, char *argv[])
 
       BlockOperator * A = Ah.As<BlockOperator>();
 
-      BlockDiagonalPreconditioner M(A->RowOffsets());
-      M.owns_blocks = 1;
-      int skip = 0;
-      if (!static_cond)
+      Solver * preconditioner = nullptr;
+      Array<ParFiniteElementSpace *> prec_fes;
+      if (static_cond)
       {
-         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
-         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
-         amg0->SetPrintLevel(0);
-         amg1->SetPrintLevel(0);
-         M.SetDiagonalBlock(0,amg0);
-         M.SetDiagonalBlock(1,amg1);
-         skip = 2;
-      }
-      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,
-                                                                               skip));
-      amg2->SetPrintLevel(0);
-      M.SetDiagonalBlock(skip,amg2);
-
-      HypreSolver * prec;
-      if (dim == 2)
-      {
-         // AMS preconditioner for 2D H(div) (trace) space
-         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatf_fes);
+         a->GetTraceFESpaces(prec_fes);
       }
       else
       {
-         // ADS preconditioner for 3D H(div) (trace) space
-         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatf_fes);
+         prec_fes = trial_fes;
+      }
+      if (pmg)
+      {
+#ifdef MFEM_USE_MUMPS
+         bool mumps_coarse_solver = true;
+#else
+         bool mumps_coarse_solver = false;
+#endif
+         std::vector<Array<int>> ess_bdr_marker(prec_fes.Size());
+         for (int b = 0; b<prec_fes.Size(); b++)
+         {
+            if (pmesh.bdr_attributes.Size())
+            {
+               ess_bdr_marker[b].SetSize(pmesh.bdr_attributes.Max());
+               int ess_block = (static_cond) ? 0 : 2;
+
+               if (b == ess_block) // hatu space has essential bdr conditions
+               {
+                  ess_bdr_marker[b] = ess_bdr_uhat;
+               }
+               else if (b == ess_block+1) // hatf space has essential bdr conditions
+               {
+                  ess_bdr_marker[b] = ess_bdr_fhat;
+               }
+               else
+               {
+                  ess_bdr_marker[b] = 0;
+               }
+            }
+         }
+         preconditioner = new PRefinementMultigrid(prec_fes, ess_bdr_marker, *A,
+                                                   pmg_levels, relax_factor, mumps_coarse_solver);
+      }
+      else
+      {
+         preconditioner = new BlockDiagonalPreconditioner(A->RowOffsets());
+         auto block_diag = dynamic_cast<BlockDiagonalPreconditioner*>(preconditioner);
+         block_diag->owns_blocks = 1;
+         for (int i = 0; i<A->NumRowBlocks(); i++)
+         {
+            auto prec = MakeFESpaceDefaultSolver(prec_fes[i],0);
+            prec->SetOperator(A->GetBlock(i,i));
+            block_diag->SetDiagonalBlock(i,prec);
+         }
       }
-      M.SetDiagonalBlock(skip+1,prec);
 
       CGSolver cg(MPI_COMM_WORLD);
       cg.SetRelTol(1e-12);
       cg.SetMaxIter(2000);
       cg.SetPrintLevel(0);
-      cg.SetPreconditioner(M);
       cg.SetOperator(*A);
+      cg.SetPreconditioner(*preconditioner);
       cg.Mult(B, X);
+      delete preconditioner;
+
       int num_iter = cg.GetNumIterations();
 
       a->RecoverFEMSolution(X,x);

miniapps/dpg/pdiffusion-primal.cpp

@@ -0,0 +1,329 @@
+//                 MFEM Primal DPG parallel example for diffusion
+//
+// Compile with: make pdiffusion-primal
+//
+// Sample runs
+// mpirun -np 4 pdiffusion-primal -m ../../data/inline-quad.mesh -o 3 -sref 1 -pref 2
+
+//       - Δ u = f,   in Ω
+//           u = u₀, on ∂Ω
+
+
+
+// --------------------------------------
+// |   |     u     |      σ̂    |  RHS   |
+// --------------------------------------
+// | v |  (∇u,∇v)  |  -(σ̂ₙ,v)  |  (f,v) |
+//
+// u ∈ H¹(Ω),   σ̂ₙ ∈ H^-1/2(Τ)
+
+
+#include "mfem.hpp"
+#include "util/pweakform.hpp"
+#include "../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+double exact_u(const Vector & X);
+void exact_gradu(const Vector & X, Vector &gradu);
+double exact_laplacian_u(const Vector & X);
+void exact_hatsigma(const Vector & X, Vector & hatsigma);
+double f_exact(const Vector & X);
+
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI and HYPRE.
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int sref = 0; // initial uniform mesh refinements
+   int pref = 0; // parallel mesh refinements for AMR
+   bool static_cond = false;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&sref, "-sref", "--num-serial-refinements",
+                  "Number of initial serial uniform refinements");
+   args.AddOption(&pref, "-pref", "--num-parallel-refinements",
+                  "Number of AMR refinements");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   for (int i = 0; i<sref; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+
+   // H1 space for u
+   FiniteElementCollection *u_fec = new H1_FECollection(order,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec);
+
+
+   // H^-1/2 space for σ̂
+   FiniteElementCollection * hatsigma_fec = new RT_Trace_FECollection(order-1,dim);
+   ParFiniteElementSpace *hatsigma_fes = new ParFiniteElementSpace(&pmesh,
+                                                                   hatsigma_fec);
+
+   int test_order = order+delta_order;
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(hatsigma_fes);
+   test_fec.Append(v_fec);
+
+
+   ConstantCoefficient one(1.0);
+   FunctionCoefficient f(f_exact); // rhs for the manufactured solution problem
+   FunctionCoefficient uex(exact_u);
+   VectorFunctionCoefficient graduex(dim,exact_gradu);
+
+   ParDPGWeakForm * a = new ParDPGWeakForm(trial_fes,test_fec);
+   a->StoreMatrices(true); // this is needed for estimation of residual
+
+   // (∇u,∇v)
+   a->AddTrialIntegrator(new DiffusionIntegrator(one),0,0);
+
+   // -<σ̂,v> (sign is included in σ̂)
+   a->AddTrialIntegrator(new TraceIntegrator,1,0);
+
+   // (∇v,∇δv)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+   // (v,δv)
+   a->AddTestIntegrator(new MassIntegrator(one),0,0);
+
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f),0);
+
+
+   if (myid == 0)
+   {
+      std::cout << "\n  Ref |"
+                << "    Dofs    |"
+                << "  H1 Error  |"
+                << "  Rate  |"
+                << "  Residual  |"
+                << "  Rate  |"
+                << " PCG it |" << endl;
+      std::cout << std::string(72,'-') << endl;
+   }
+
+
+   socketstream u_out;
+
+   double err0 = 0.;
+   int dof0=0.;
+   double res0=0.0;
+
+   ParGridFunction u_gf(u_fes);
+   u_gf = 0.0;
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+
+   for (int it = 0; it<=pref; it++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         u_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      Array<int> offsets(3);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = hatsigma_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      u_gf.MakeRef(u_fes,x.GetBlock(0),0);
+      u_gf.ProjectBdrCoefficient(uex,ess_bdr);
+
+      Vector X,B;
+      OperatorPtr Ah;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockOperator * A = Ah.As<BlockOperator>();
+
+      BlockDiagonalPreconditioner M(A->RowOffsets());
+      M.owns_blocks = 1;
+
+      HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+      amg0->SetPrintLevel(0);
+      M.SetDiagonalBlock(0,amg0);
+      HypreSolver * prec;
+      if (dim == 2)
+      {
+         // AMS preconditioner for 2D H(div) (trace) space
+         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(1,1), hatsigma_fes);
+      }
+      else
+      {
+         // ADS preconditioner for 3D H(div) (trace) space
+         prec = new HypreADS((HypreParMatrix &)A->GetBlock(1,1), hatsigma_fes);
+      }
+      M.SetDiagonalBlock(1,prec);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(0);
+      cg.SetPreconditioner(M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+
+      double maxresidual = residuals.Max();
+      double globalresidual = residual * residual;
+
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+      u_gf.MakeRef(u_fes,x.GetBlock(0),0);
+
+      int dofs = u_fes->GlobalTrueVSize() + hatsigma_fes->GlobalTrueVSize();
+
+      double u_err = u_gf.ComputeH1Error(&uex,&graduex);
+      double rate_err = (it) ? dim*log(err0/u_err)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (it) ? dim*log(res0/globalresidual)/log((
+                                                                   double)dof0/dofs) : 0.0;
+      err0 = u_err;
+      res0 = globalresidual;
+      dof0 = dofs;
+
+      if (myid == 0)
+      {
+         std::ios oldState(nullptr);
+         oldState.copyfmt(std::cout);
+         std::cout << std::right << std::setw(5) << it << " | "
+                   << std::setw(10) <<  dof0 << " | "
+                   << std::setprecision(3)
+                   << std::setw(10) << std::scientific <<  err0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_err << " | "
+                   << std::setprecision(3)
+                   << std::setw(10) << std::scientific <<  res0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_res << " | "
+                   << std::setw(6) << std::fixed << cg.GetNumIterations() << " | "
+                   << std::endl;
+         std::cout.copyfmt(oldState);
+      }
+
+      if (visualization)
+      {
+         const char * keys = (it == 0 && dim == 2) ? "jRcm\n" : nullptr;
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+
+         VisualizeField(u_out,vishost,visport,u_gf,
+                        "Numerical u", 0,0,500,500,keys);
+      }
+
+      if (it == pref) { break; }
+
+      pmesh.UniformRefinement();
+
+      for (int i=0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete v_fec;
+   delete hatsigma_fes;
+   delete hatsigma_fec;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+
+
+
+}
+
+
+
+double exact_u(const Vector & X)
+{
+   double alpha = M_PI * (X.Sum());
+   return sin(alpha);
+}
+
+void exact_gradu(const Vector & X, Vector & du)
+{
+   du.SetSize(X.Size());
+   double alpha = M_PI * (X.Sum());
+   du.SetSize(X.Size());
+   for (int i = 0; i<du.Size(); i++)
+   {
+      du[i] = M_PI * cos(alpha);
+   }
+}
+
+double exact_laplacian_u(const Vector & X)
+{
+   double alpha = M_PI * (X.Sum());
+   double u = sin(alpha);
+   return - M_PI*M_PI * u * X.Size();
+}
+
+void exact_hatsigma(const Vector & X, Vector & hatsigma)
+{
+   exact_gradu(X,hatsigma);
+   hatsigma *= -1.;
+}
+
+double f_exact(const Vector & X)
+{
+   return -exact_laplacian_u(X);
+}

miniapps/dpg/pdiffusion.cpp

@@ -70,6 +70,7 @@
 
 #include "mfem.hpp"
 #include "util/pweakform.hpp"
+#include "util/preconditioners.hpp"
 #include "../common/mfem-common.hpp"
 #include <fstream>
 #include <iostream>
@@ -114,6 +115,9 @@ int main(int argc, char *argv[])
    int sref = 0; // initial uniform mesh refinements
    int pref = 0; // parallel mesh refinements for AMR
    int iprob = 0;
+   bool pmg = false;
+   int pmg_levels = -1;
+   real_t relax_factor = 2.0/3;
    bool static_cond = false;
    real_t theta = 0.7;
    bool visualization = true;
@@ -137,6 +141,12 @@ int main(int argc, char *argv[])
                   " 0: manufactured, 1: L-shape");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&pmg, "-pmg", "--p-refinement-multigrid", "-no-pmg",
+                  "--no-p-refinement-multigrid", "Enable P-Refinement Multigrid.");
+   args.AddOption(&pmg_levels, "-pmgl","--p-refinement-multigrid-levels",
+                  "Number of levels for P-Refinement Multigrid.");
+   args.AddOption(&relax_factor, "-rf", "--relaxation-factor",
+                  "Relaxation factor for the p-multigrid smoother.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -186,7 +196,6 @@ int main(int argc, char *argv[])
       }
    }
 
-
    for (int i = 0; i<sref; i++)
    {
       mesh.UniformRefinement();
@@ -378,44 +387,63 @@ int main(int argc, char *argv[])
 
       BlockOperator * A = Ah.As<BlockOperator>();
 
-      BlockDiagonalPreconditioner M(A->RowOffsets());
-      M.owns_blocks = 1;
-      int skip = 0;
-      if (!static_cond)
+      Solver * preconditioner = nullptr;
+      Array<ParFiniteElementSpace *> prec_fes;
+      if (static_cond)
       {
-         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
-         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
-         amg0->SetPrintLevel(0);
-         amg1->SetPrintLevel(0);
-         M.SetDiagonalBlock(0,amg0);
-         M.SetDiagonalBlock(1,amg1);
-         skip=2;
-      }
-      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,
-                                                                               skip));
-      amg2->SetPrintLevel(0);
-      M.SetDiagonalBlock(skip,amg2);
-      HypreSolver * prec;
-      if (dim == 2)
-      {
-         // AMS preconditioner for 2D H(div) (trace) space
-         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatsigma_fes);
+         a->GetTraceFESpaces(prec_fes);
       }
       else
       {
-         // ADS preconditioner for 3D H(div) (trace) space
-         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatsigma_fes);
+         prec_fes = trial_fes;
+      }
+      if (pmg)
+      {
+#ifdef MFEM_USE_MUMPS
+         bool mumps_coarse_solver = true;
+#else
+         bool mumps_coarse_solver = false;
+#endif
+         std::vector<Array<int>> ess_bdr_marker(prec_fes.Size());
+         for (int b = 0; b<prec_fes.Size(); b++)
+         {
+            ess_bdr_marker[b].SetSize(pmesh.bdr_attributes.Max());
+            int ess_block = (static_cond) ? 0 : 2;
+            if (b == ess_block)
+            {
+               ess_bdr_marker[b] = ess_bdr;
+            }
+            else
+            {
+               ess_bdr_marker[b] = 0;
+            }
+         }
+         preconditioner = new PRefinementMultigrid(prec_fes, ess_bdr_marker, *A,
+                                                   pmg_levels, relax_factor,mumps_coarse_solver);
+      }
+      else
+      {
+         preconditioner = new BlockDiagonalPreconditioner(A->RowOffsets());
+         auto block_diag = dynamic_cast<BlockDiagonalPreconditioner*>(preconditioner);
+         block_diag->owns_blocks = 1;
+         for (int i = 0; i<A->NumRowBlocks(); i++)
+         {
+            auto prec = MakeFESpaceDefaultSolver(prec_fes[i],0);
+            prec->SetOperator(A->GetBlock(i,i));
+            block_diag->SetDiagonalBlock(i,prec);
+         }
       }
-      M.SetDiagonalBlock(skip+1,prec);
 
       CGSolver cg(MPI_COMM_WORLD);
       cg.SetRelTol(1e-12);
       cg.SetMaxIter(2000);
       cg.SetPrintLevel(0);
-      cg.SetPreconditioner(M);
       cg.SetOperator(*A);
+      cg.SetPreconditioner(*preconditioner);
       cg.Mult(B, X);
 
+      delete preconditioner;
+
       a->RecoverFEMSolution(X,x);
 
       Vector & residuals = a->ComputeResidual(x);

miniapps/dpg/plasma/lh-eld-dpg-fem.cpp

@@ -0,0 +1,976 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//                   MFEM Ultraweak DPG Maxwell parallel example
+//
+// Compile with: make lh-eld-dpg
+//
+// mpirun -np 8 ./lh-eld-dpg -o 3 -paraview -pr 0
+// mpirun -np 8 ./lh-eld-dpg -o 3 -paraview -pr 1 -sc
+
+// Electron Landau Damping
+// Strong formulation:
+//     ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵ E + i ω²ϵ₀(J₁ + J₂) = 0,   in Ω
+//                   - Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥ = 0,   in Ω     
+//                   - Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥ = 0,   in Ω 
+//                                            E×n = E₀,  on ∂Ω
+//                                            J₁  = 0,  on ∂Ω
+//                                            J₂  = 0,  on ∂Ω
+// The DPG UW deals with the First Order System
+//   i ω μ₀ H + ∇ × E                  = 0,   in Ω
+// -i ω ϵ₀ϵ E + ∇ × H - ω ϵ₀ (J₁ + J₂) = 0,   in Ω
+//         - Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥ = 0,   in Ω     
+//         - Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥ = 0,   in Ω 
+//                                 E×n = E₀,  on ∂Ω
+//                                 J₁  = 0,  on ∂Ω
+//                                 J₂  = 0,  on ∂Ω
+
+
+// in 2D
+// E is vector valued and H is scalar.
+//      (∇ × E, δE) = (E, ∇ × δE ) + < n × E , δE >
+// or (∇ ⋅ AE , δE) = (AE, ∇ δE ) + < AE ⋅ n, δE >
+// where A = [0 1; -1 0];
+
+// E ∈ (L²(Ω))² , H ∈ L²(Ω), J ∈ (H¹(Ω))²
+// Ê ∈ H^-1/2(Γₕ), Ĥ ∈ H^1/2(Γₕ)
+//     iωμ₀ (H,δE) + (E,∇×δE) + < AÊ, δE > = 0,      ∀ δE ∈ H¹(Ω)
+//  -i ωϵ₀ϵ (E,δH) + (H,∇×δH) + < Ĥ, δH×n > - ωϵ₀(J₁ + J₂,δH) = 0,  ∀ δH ∈ H(curl,Ω)
+// ( (b⋅∇)J₁,(b⋅∇) δJ₁ ) + c₁ (J₁,δJ₁) - c₁ (P(r) b⊗b E, δJ₁) = 0,  ∀ δJ₁ ∈ (H¹(Ω))²
+// ( (b⋅∇)J₂,(b⋅∇) δJ₂ ) + c₂ (J₂,δJ₂) + c₂ (P(r) b⊗b E, δJ₂) = 0,  ∀ δJ₁ ∈ (H¹(Ω))²
+//                                                                  Ê = E₀, on ∂Ω
+//                                                            J₁ = J₂ = 0,  on ∂Ω
+// ----------------------------------------------------------------------------------------------
+// |   |      E       |     H    |        J₁        |        J₂        |   Ê   |   Ĥ    |  RHS  |
+// ----------------------------------------------------------------------------------------------
+// |δE |  (E,∇ × δE)  |iωμ₀(H,δE)|                  |                  | <Ê,δE>|        |   0   |  
+// |   |              |          |                  |                  |       |        |       |  
+// |δH | -iωϵ₀ϵ(E,δH) | (H,∇×δH) |   -ωϵ₀ (J₁,δH)   |  -ωϵ₀ (J₂,δH)    |       |<Ĥ,δH×n>|   0   |  
+// |   |              |          |                  |                  |       |        |       |  
+// |δJ₁|-c₁(P(r)E,δJ₁)|          |((b⋅∇)J₁,(b⋅∇)δJ₁)|                  |       |        |   0   |  
+// |   |              |          |     + c₁ (J₁,δJ₁)|                  |       |        |       |    
+// |δJ₂| c₂(P(r)E,δJ₂)|          |                  |((b⋅∇)J₂,(b⋅∇)δJ₂)|       |        |   0   |  
+// |   |              |          |                  |     + c₂ (J₂,δJ₂)|       |        |       |    
+// where (δE,δH,δJ₁,δJ₂) ∈  H¹(Ω) × H(curl,Ω) × (H¹(Ω))² × (H¹(Ω))² 
+
+
+
+#include "mfem.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../util/pcomplexblockform.hpp"
+#include "../../common/mfem-common.hpp"
+#include "../util/blockcomplexhypremat.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "utils/lh_utils.hpp"
+#include "../util/utils.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+   int order = 1;
+   int delta_order = 1;
+   int par_ref_levels = 0;
+   int ser_ref_levels = 0;
+
+   real_t rnum=1.5;
+   real_t mu = 1.257;
+   real_t eps0 = 8.8541878128;
+   real_t cfactor = 1e-6;
+   real_t balance_scale = 1.0;
+   bool enable_balance_scale = false;
+
+   bool visualization = false;
+   bool paraview = false;
+   bool debug = false;
+   bool mumps_solver = false;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&ser_ref_levels, "-sr", "--serial-refinement_levels",
+                  "Number of serial refinement levels.");                  
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&a0, "-a0", "--a0", "P(r) first parameter.");
+   args.AddOption(&a1, "-a1", "--a1", "P(r) second parameter.");
+   args.AddOption(&delta, "-delta", "--delta", "stability parameter.");
+   args.AddOption(&mumps_solver, "-mumps", "--mumps", "-no-mumps",
+                  "--no-mumps",
+                  "Enable or disable MUMPS solver.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&debug, "-debug", "--debug", "-no-debug",
+                  "--no-debug",
+                  "Enable or disable debug mode (delta = 0.01 and no coupling).");                  
+   args.AddOption(&enable_balance_scale, "-ebs", "--enable-balance-scale", "-no-ebs",
+                  "--no-ebs",
+                  "Enable or disable balance scale.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (Mpi::Root())
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (Mpi::Root())
+   {
+      args.PrintOptions(cout);
+   }
+
+   // number of diffusion equations
+   int ndiffusionequations = 2; 
+
+   Vector cvals(ndiffusionequations);
+   Vector csigns(ndiffusionequations);
+   cvals(0)  = 25e6;  cvals(1)  = 1e6;
+   csigns(0) = -1.0;  csigns(1) = 1.0;
+   cvals *= cfactor; // scale the coefficients
+
+   real_t omega = 2.*M_PI*rnum;
+   int test_order = order+delta_order;
+
+   balance_scale = (enable_balance_scale) ? omega * eps0  : 1.0;
+
+   if (!debug) 
+   {
+      delta = 0.0; // disable delta if electron Landau damping is enabled
+      if (Mpi::Root())
+      {
+         cout << "Electron Landau damping enabled, delta set to 0.0." << endl;
+      }
+   }    
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   MFEM_VERIFY(dim == 2, "Dimension != 2 is not supported in this example");
+
+   for (int i = 0; i < ser_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // mesh.RemoveInternalBoundaries();
+
+   Array<int> int_bdr_attr;
+   for (int i = 0; i < mesh.GetNBE(); i++)
+   {
+      if (mesh.FaceIsInterior(mesh.GetBdrElementFaceIndex(i)))
+      {
+         int_bdr_attr.Append(mesh.GetBdrAttribute(i));
+      }
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   for (int i = 0; i < par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }  
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+
+   // Define coefficients
+   ConstantCoefficient muinv(1./mu);
+   ConstantCoefficient one_cf(1.0);
+   //  ωμ₀
+   ConstantCoefficient omegamu_cf(omega*mu);
+   // -ω μ₀  
+   ConstantCoefficient negomegamu_cf(-omega*mu);
+   // -ωϵ₀
+   real_t scale = (debug) ? 0.0 : 1.0;
+   ConstantCoefficient negomegeps0_cf(-omega*eps0 * scale);
+   ConstantCoefficient balancescaled_negomegeps0_cf( -omega*eps0/balance_scale * scale);
+
+   // μ₀² ω²
+   ConstantCoefficient mu2omeg2_cf((mu*mu*omega*omega));
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_vcf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   // S(r) 
+   FunctionCoefficient S_cf_r(sfunc_r), S_cf_i(sfunc_i);
+   // P(r) 
+   FunctionCoefficient P_cf_r(pfunc_r), P_cf_i(pfunc_i); 
+
+   VectorFunctionCoefficient b_cf(dim,bfunc);// b
+   ScalarVectorProductCoefficient scaled_b_cf(sqrt(cfactor), b_cf);
+
+   MatrixFunctionCoefficient bb_cf(dim,bcrossb); // b⊗b
+   MatrixSumCoefficient oneminusbb(Mone_cf, bb_cf, 1.0, -1.0); // 1 - b⊗b
+
+   // S(r) (I - b⊗b)
+   ScalarMatrixProductCoefficient Soneminusbb_r(S_cf_r, oneminusbb), Soneminusbb_i(S_cf_i, oneminusbb); 
+
+   // P(r) b⊗b 
+   ScalarMatrixProductCoefficient P_cf_bb_r(P_cf_r, bb_cf), P_cf_bb_i(P_cf_i, bb_cf); 
+
+   // ε = S(r) (I - b⊗b) + P(r) b⊗b 
+   MatrixSumCoefficient eps_r(Soneminusbb_r, P_cf_bb_r, 1.0, 1.0); 
+   MatrixSumCoefficient eps_i(Soneminusbb_i, P_cf_bb_i, 1.0, 1.0); 
+
+   coefs_r[nattr-1] = &eps_r;
+   coefs_i[nattr-1] = &eps_i;
+
+   // for (int i = 0; i < nattr-1; ++i)
+   // {
+   //    coefs_r[i] = &eps_r;
+   //    coefs_i[i] = &eps_i;
+   // }
+
+   PWMatrixCoefficient eps_cf_r(dim, attr, coefs_r);
+   PWMatrixCoefficient eps_cf_i(dim, attr, coefs_i);
+
+   ConstantCoefficient eps0omeg(omega * eps0);
+   ConstantCoefficient negeps0omeg(-omega * eps0);
+
+   // ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient eps0omeg_eps_r(eps0omeg, eps_cf_r);
+   // ω ϵ₀ ϵᵢ
+   ScalarMatrixProductCoefficient eps0omeg_eps_i(eps0omeg, eps_cf_i);
+   // -ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient negeps0omeg_eps_r(negeps0omeg, eps_cf_r);
+   // -ω ϵ₀ ϵᵢ
+   ScalarMatrixProductCoefficient negeps0omeg_eps_i(eps0omeg, eps_cf_i);
+
+   // A = [0 1; -1 0]
+   DenseMatrix rot_mat(2);
+   rot_mat(0,0) = 0.; rot_mat(0,1) = 1.;
+   rot_mat(1,0) = -1.; rot_mat(1,1) = 0.;
+   MatrixConstantCoefficient rot(rot_mat);
+
+   // ω ϵ₀ ϵᵣ A
+   MatrixProductCoefficient eps0omeg_eps_r_rot(eps0omeg_eps_r, rot);
+   // ω ϵ₀ ϵᵢ A
+   MatrixProductCoefficient eps0omeg_eps_i_rot(eps0omeg_eps_i, rot);
+   // -ω ϵ₀ ϵᵣ A
+   MatrixProductCoefficient negeps0omeg_eps_r_rot(negeps0omeg_eps_r, rot);
+   // -ω ϵ₀ ϵᵢ A
+   MatrixProductCoefficient negeps0omeg_eps_i_rot(negeps0omeg_eps_i, rot);
+
+   // (ωϵ₀ϵ)(ωϵ₀ϵ)^*  (δH, δH)
+   TransposeMatrixCoefficient eps0omeg_eps_r_t(eps0omeg_eps_r);
+   TransposeMatrixCoefficient eps0omeg_eps_i_t(eps0omeg_eps_i);
+   MatrixProductCoefficient MrMrt_cf(eps0omeg_eps_r, eps0omeg_eps_r_t);
+   MatrixProductCoefficient MiMit_cf(eps0omeg_eps_i, eps0omeg_eps_i_t);
+   MatrixProductCoefficient MiMrt_cf(eps0omeg_eps_i, eps0omeg_eps_r_t);
+   MatrixProductCoefficient MrMit_cf(eps0omeg_eps_r, eps0omeg_eps_i_t);
+
+   // (MᵣMᵣᵗ + MᵢMᵢᵗ) + i (MᵢMᵣᵗ - MᵣMᵢᵗ)
+   MatrixSumCoefficient Mreal_cf(MrMrt_cf,MiMit_cf);
+   MatrixSumCoefficient Mimag_cf(MiMrt_cf,MrMit_cf,1.0,-1.0);
+
+   // if ELD
+   Array<Vector *> c_arrays(ndiffusionequations);
+   Array<PWConstCoefficient *> pw_c_coeffs(ndiffusionequations);
+   Array<MatrixCoefficient *> cPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> cPibb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPibb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> balancescaled_signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> balancescaled_signedcPibb_cf(ndiffusionequations);
+   
+   Vector temp(nattr); temp=0.0;
+   Array<ConstantCoefficient *> c_coeffs(ndiffusionequations);
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      temp[nattr-1] = cvals(i);
+      // temp = cvals(i);
+      pw_c_coeffs[i] = new PWConstCoefficient(temp);
+      c_coeffs[i] = new ConstantCoefficient(cvals(i));
+      cPrbb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_r);
+      cPibb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_i);
+      signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPrbb_cf[i]);
+      signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPibb_cf[i]);
+      balancescaled_signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(balance_scale,*signedcPrbb_cf[i]);
+      balancescaled_signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(balance_scale,*signedcPibb_cf[i]);
+   }   
+
+   // ----------------------------------------------
+   // DPG UW formulation for Maxwell equations
+   // ----------------------------------------------
+   // Define DPG spaces for the Maxwell equations
+   // trial spaces for E, H, Ê, Ĥ
+   Array<FiniteElementCollection *> dpg_trial_fecols;
+   Array<FiniteElementCollection *> dpg_test_fecols;
+   Array<ParFiniteElementSpace *> dpg_pfes;
+
+   // Vector L2 space for E
+   dpg_trial_fecols.Append(new L2_FECollection(order-1, dim));
+   dpg_pfes.Append(new ParFiniteElementSpace(&pmesh, dpg_trial_fecols.Last(), dim));
+   // Scalar L2 space for H
+   dpg_trial_fecols.Append(new L2_FECollection(order-1, dim));
+   dpg_pfes.Append(new ParFiniteElementSpace(&pmesh, dpg_trial_fecols.Last()));
+
+   // Trial trace space for Ê 
+   dpg_trial_fecols.Append(new RT_Trace_FECollection(order-1, dim));
+   dpg_pfes.Append(new ParFiniteElementSpace(&pmesh, dpg_trial_fecols.Last()));
+   // Trial trace space for Ĥ 
+   dpg_trial_fecols.Append(new H1_Trace_FECollection(order, dim));
+   dpg_pfes.Append(new ParFiniteElementSpace(&pmesh, dpg_trial_fecols.Last()));
+
+   Array<HYPRE_BigInt> dpg_tdofs(dpg_pfes.Size());
+   for (int i = 0; i < dpg_pfes.Size(); ++i)
+   {
+      dpg_tdofs[i] = dpg_pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << "DPG ParFiniteElementSpace " << i << " has " << dpg_tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   if (Mpi::Root())
+   {
+      cout << "Total number of DPG true dofs: " << dpg_tdofs.Sum() << endl;
+   }
+
+   // test spaces for E and H
+   dpg_test_fecols.Append(new H1_FECollection(test_order, dim));
+   dpg_test_fecols.Append(new ND_FECollection(test_order, dim));
+
+   ParComplexDPGWeakForm * a_dpg = new ParComplexDPGWeakForm(dpg_pfes,dpg_test_fecols);
+
+   // (E,∇ × δE)
+   a_dpg->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one_cf)),
+                         nullptr,0, 0);
+   //  -i ω ϵ₀ (ϵE,δH) = - i ω ϵ₀(ϵᵣ + i ϵᵢ E, δH)
+   //                  =  (ω ϵ₀ ϵᵢ E, δH) + i (-ω ϵ₀ϵᵣ E, δH)
+   a_dpg->AddTrialIntegrator(
+      new TransposeIntegrator(new VectorFEMassIntegrator(eps0omeg_eps_i)), 
+      new TransposeIntegrator(new VectorFEMassIntegrator(negeps0omeg_eps_r)),
+      0, 1);
+   // iωμ₀(H,δE)
+   a_dpg->AddTrialIntegrator(nullptr, new MixedScalarMassIntegrator(omegamu_cf), 1, 0);
+   // a_dpg->AddTrialIntegrator(nullptr, new MassIntegrator(omegamu_cf), 1, 0);
+   // (H,∇ × δH)                         
+   a_dpg->AddTrialIntegrator(
+      new TransposeIntegrator(new MixedCurlIntegrator(one_cf)), nullptr, 1, 1);
+
+   // Trace integrators               
+   //  <Ê,δE>
+   a_dpg->AddTrialIntegrator(new TraceIntegrator,nullptr, 2, 0);
+   // <Ĥ,δH × n>
+   a_dpg->AddTrialIntegrator(new TangentTraceIntegrator,nullptr, 3, 1);
+
+   // test integrators
+   // (∇δE,∇δE)
+   a_dpg->AddTestIntegrator(new DiffusionIntegrator(one_cf),nullptr, 0, 0);
+   // (δE,δE)
+   a_dpg->AddTestIntegrator(new MassIntegrator(one_cf),nullptr, 0, 0);
+   // μ₀² ω² (δE,δE)
+   a_dpg->AddTestIntegrator(new MassIntegrator(mu2omeg2_cf),nullptr,0, 0);
+   // -i ω μ₀ (δE,∇ × δH) = i (δE, -ω μ₀ ∇ × δ H)
+   a_dpg->AddTestIntegrator(nullptr,
+         new TransposeIntegrator(new MixedCurlIntegrator(negomegamu_cf)),0, 1);
+   // -i ω ϵ₀ϵ(∇ × δE, δH) = -i (ωϵ₀(ϵᵣ+iϵᵢ) A ∇ δE,δE), A = [0 1; -1 0]
+   //                       =  (ω ϵ₀ ϵᵢ A ∇ δE,δE) + i (-ω ϵ₀ ϵᵣ A ∇ δE,δE)
+   a_dpg->AddTestIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_i_rot),
+                        new MixedVectorGradientIntegrator(negeps0omeg_eps_r_rot),0, 1);
+   // i ω μ₀ (∇ × δH ,δE) = i (ω μ₀ ∇ × δH, δE )
+   a_dpg->AddTestIntegrator(nullptr,new MixedCurlIntegrator(omegamu_cf),
+                        1, 0);
+   // i ω ϵ₀ϵ̄ (δH, ∇ × δE ) = i (ω ϵ₀(ϵᵣ -i ϵᵢ) δH, A ∇ δE) 
+   //                        = ( δH, ω ϵ₀ ϵᵢ A ∇ δE) + i (δH, ω ϵ₀ ϵᵣ A ∇ δE)
+   a_dpg->AddTestIntegrator(
+      new TransposeIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_i_rot)),
+      new TransposeIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_r_rot)),1, 0);
+   // (ωϵ₀ϵ)(ωϵ₀ϵ)^*  (δH, δH)
+   // (MᵣMᵣᵗ + MᵢMᵢᵗ) + i (MᵢMᵣᵗ - MᵣMᵢᵗ)
+   a_dpg->AddTestIntegrator(new VectorFEMassIntegrator(Mreal_cf),
+                        new VectorFEMassIntegrator(Mimag_cf),1, 1);
+   // (∇×δH ,∇×δH)
+   a_dpg->AddTestIntegrator(new CurlCurlIntegrator(one_cf),nullptr,1,1);
+   // (δH,δH)
+   a_dpg->AddTestIntegrator(new VectorFEMassIntegrator(one_cf),nullptr,1,1);
+
+   a_dpg->Assemble();
+
+   // ----------------------------------------------
+   // Define FEM spaces for the diffusion equations
+   // ----------------------------------------------
+
+   Array<FiniteElementCollection *> fem_fecols;
+   Array<ParFiniteElementSpace *> fem_pfes;
+   for (int i = 0; i < ndiffusionequations; ++i)
+   {
+      fem_fecols.Append(new H1_FECollection(order, dim));
+      fem_pfes.Append(new ParFiniteElementSpace(&pmesh, fem_fecols.Last(), dim));
+   }
+
+   Array<HYPRE_BigInt> fem_tdofs(fem_pfes.Size());
+   for (int i = 0; i < fem_pfes.Size(); ++i)
+   {
+      fem_tdofs[i] = fem_pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << "FEM ParFiniteElementSpace " << i << " has " << fem_tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+
+   if (Mpi::Root())
+   {
+      cout << "Total number of FEM true dofs: " << fem_tdofs.Sum() << endl;
+   }
+   
+   ParComplexBlockForm *a_fem = new ParComplexBlockForm(fem_pfes);
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      // ( (b⋅∇)Jᵢ  , (b⋅∇) Gᵢ)
+      a_fem->AddDomainIntegrator(new DirectionalVectorDiffusionIntegrator(scaled_b_cf), nullptr, i, i);
+      // cᵢ (Jᵢ , Gᵢ)
+      a_fem->AddDomainIntegrator(new VectorMassIntegrator(*pw_c_coeffs[i]), nullptr, i, i);
+   }
+   a_fem->Assemble();
+
+   // ----------------------------------------------
+   // Cross term DPG-FEM coupling
+   // ----------------------------------------------
+   //  -ωϵ₀ (J₁,δH)
+   //  -ωϵ₀ (J₂,δH)
+   ParMixedBilinearForm * a_J1H = new ParMixedBilinearForm(fem_pfes[0], dpg_pfes[1]);
+   ParMixedBilinearForm * a_J2H = new ParMixedBilinearForm(fem_pfes[1], dpg_pfes[1]);
+
+   ParMixedBilinearForm * a_J1H_i = new ParMixedBilinearForm(fem_pfes[0], dpg_pfes[1]);
+   ParMixedBilinearForm * a_J2H_i = new ParMixedBilinearForm(fem_pfes[1], dpg_pfes[1]);
+   ConstantCoefficient dummy_cf(0.0);
+   a_J1H_i->AddDomainIntegrator(new VectorMassIntegrator(dummy_cf));
+   a_J2H_i->AddDomainIntegrator(new VectorMassIntegrator(dummy_cf));
+   a_J1H_i->Assemble(0);
+   a_J2H_i->Assemble(0);
+
+   // a_J1H->AddDomainIntegrator(new VectorMassIntegrator(negomegeps0_cf));
+   // a_J2H->AddDomainIntegrator(new VectorMassIntegrator(negomegeps0_cf));
+
+   a_J1H->AddDomainIntegrator(new VectorMassIntegrator(balancescaled_negomegeps0_cf));
+   a_J2H->AddDomainIntegrator(new VectorMassIntegrator(balancescaled_negomegeps0_cf));
+
+
+   
+   a_J1H->Assemble(0);
+   a_J2H->Assemble(0);
+
+
+   // ±cᵢ(P(r) (b ⊗ b) E, δJᵢ))
+   ParMixedBilinearForm * a_EJ1 = new ParMixedBilinearForm(dpg_pfes[0], fem_pfes[0]);
+   ParMixedBilinearForm * a_EJ2 = new ParMixedBilinearForm(dpg_pfes[0], fem_pfes[1]);
+   ParMixedBilinearForm * a_EJ1_i = new ParMixedBilinearForm(dpg_pfes[0], fem_pfes[0]);
+   ParMixedBilinearForm * a_EJ2_i = new ParMixedBilinearForm(dpg_pfes[0], fem_pfes[1]);
+   // a_EJ1_i->AddDomainIntegrator(new VectorMassIntegrator(*signedcPibb_cf[0]));
+   // a_EJ2_i->AddDomainIntegrator(new VectorMassIntegrator(*signedcPibb_cf[1]));
+
+   a_EJ1_i->AddDomainIntegrator(new VectorMassIntegrator(*balancescaled_signedcPibb_cf[0]));
+   a_EJ2_i->AddDomainIntegrator(new VectorMassIntegrator(*balancescaled_signedcPibb_cf[1]));
+
+   a_EJ1_i->Assemble(0);
+   a_EJ2_i->Assemble(0);
+   // a_EJ1->AddDomainIntegrator(new VectorMassIntegrator(*signedcPrbb_cf[0]));
+   // a_EJ2->AddDomainIntegrator(new VectorMassIntegrator(*signedcPrbb_cf[1]));
+
+   a_EJ1->AddDomainIntegrator(new VectorMassIntegrator(*balancescaled_signedcPrbb_cf[0]));
+   a_EJ2->AddDomainIntegrator(new VectorMassIntegrator(*balancescaled_signedcPrbb_cf[1]));
+
+
+   a_EJ1->Assemble(0);
+   a_EJ2->Assemble(0);
+
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      delete pw_c_coeffs[i];
+      delete c_coeffs[i];
+      delete cPrbb_cf[i]; 
+      delete cPibb_cf[i]; 
+      delete signedcPrbb_cf[i]; 
+      delete signedcPibb_cf[i]; 
+   }
+
+   // Assemble all the system and get the global matrix and right-hand side
+   Array<ParFiniteElementSpace *> all_pfes;
+   all_pfes.Append(dpg_pfes);
+   all_pfes.Append(fem_pfes);
+
+   int npfes = all_pfes.Size();
+   Array<int> all_offsets(npfes + 1); all_offsets[0] = 0;
+   Array<int> all_toffsets(npfes + 1); all_toffsets[0] = 0;
+   for (int i = 0; i < npfes; ++i)
+   {
+      all_offsets[i+1] = all_pfes[i]->GetVSize();
+      all_toffsets[i+1] = all_pfes[i]->GetTrueVSize();
+   }
+   all_offsets.PartialSum();
+   all_toffsets.PartialSum();
+
+   Array<int> empty;
+   OperatorPtr Ah_dpg, Ah_fem, Ah_J1H, Ah_J2H, Ah_EJ1, Ah_EJ2;
+   OperatorPtr Ah_J1H_i, Ah_J2H_i, Ah_EJ1_i, Ah_EJ2_i;
+   // 4x4 upper left block (E, H, Ê, Ĥ)
+   a_dpg->FormSystemMatrix(empty,Ah_dpg); 
+   // 2x2 lower right block (J₁, J₂)
+   a_fem->FormSystemMatrix(empty,Ah_fem);
+   // cross term A₁₄  
+   a_J1H->FormRectangularSystemMatrix(empty, empty, Ah_J1H);
+   a_J1H_i->FormRectangularSystemMatrix(empty, empty, Ah_J1H_i);
+   // cross term A₂₄
+   a_J2H->FormRectangularSystemMatrix(empty, empty, Ah_J2H);
+   a_J2H_i->FormRectangularSystemMatrix(empty, empty, Ah_J2H_i);
+   // cross term A₄₀ 
+   a_EJ1->FormRectangularSystemMatrix(empty, empty, Ah_EJ1);
+   a_EJ1_i->FormRectangularSystemMatrix(empty, empty, Ah_EJ1_i);
+   // cross term A₅₀  
+   a_EJ2->FormRectangularSystemMatrix(empty, empty, Ah_EJ2);
+   a_EJ2_i->FormRectangularSystemMatrix(empty, empty, Ah_EJ2_i);
+
+
+   // put all the operators into a block operator
+   BlockOperator A_r(all_toffsets);
+   BlockOperator A_i(all_toffsets);
+
+   ComplexOperator * Ac_dpg = Ah_dpg.As<ComplexOperator>();
+   ComplexOperator * Ac_fem = Ah_fem.As<ComplexOperator>();
+
+   BlockOperator * BlockAdpg_r = dynamic_cast<BlockOperator *>(&Ac_dpg->real());
+   BlockOperator * BlockAdpg_i = dynamic_cast<BlockOperator *>(&Ac_dpg->imag());
+   BlockOperator * BlockAfem_r = dynamic_cast<BlockOperator *>(&Ac_fem->real());
+   BlockOperator * BlockAfem_i = dynamic_cast<BlockOperator *>(&Ac_fem->imag());
+
+
+   for (int i = 0; i < dpg_pfes.Size(); ++i)
+   {
+      for (int j = 0; j < dpg_pfes.Size(); ++j)
+      {
+         A_r.SetBlock(i, j, &BlockAdpg_r->GetBlock(i, j));
+         A_i.SetBlock(i, j, &BlockAdpg_i->GetBlock(i, j));
+      }
+   }
+   for (int i = 0; i < fem_pfes.Size(); ++i)
+   {
+      for (int j = 0; j < fem_pfes.Size(); ++j)
+      {
+         A_r.SetBlock(dpg_pfes.Size() + i, dpg_pfes.Size() + j, &BlockAfem_r->GetBlock(i, j));
+         A_i.SetBlock(dpg_pfes.Size() + i, dpg_pfes.Size() + j, &BlockAfem_i->GetBlock(i, j));
+      }
+   }  
+
+   A_r.SetBlock(1, 4, Ah_J1H.Ptr());
+   A_r.SetBlock(1, 5, Ah_J2H.Ptr());
+   A_r.SetBlock(4, 0, Ah_EJ1.Ptr());
+   A_r.SetBlock(5, 0, Ah_EJ2.Ptr());
+
+   A_i.SetBlock(1, 4, Ah_J1H_i.Ptr());
+   A_i.SetBlock(1, 5, Ah_J2H_i.Ptr());
+   A_i.SetBlock(4, 0, Ah_EJ1_i.Ptr());
+   A_i.SetBlock(5, 0, Ah_EJ2_i.Ptr());
+
+
+   ComplexOperator * A = new ComplexOperator(&A_r, &A_i, false, false);
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Complex Operator A finished successfully." << endl;
+   }
+
+   socketstream E_out_r;
+
+   Vector x(2*all_offsets.Last());
+   x = 0.;
+   
+   Array<ParGridFunction *> pgf_r(npfes);
+   Array<ParGridFunction *> pgf_i(npfes);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i] = new ParGridFunction(all_pfes[i], x, all_offsets[i]);
+      pgf_i[i] = new ParGridFunction(all_pfes[i], x, all_offsets.Last() + all_offsets[i]);
+   }
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+   ParGridFunction E_par_r(&L2_fes);
+   ParGridFunction E_par_i(&L2_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   std::string output_dir = "ParaView/UW-FEM/" + GetTimestamp();
+
+   if (paraview)
+   {
+      if (Mpi::Root()) { WriteParametersToFile(args, output_dir); }
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order
+                         << "_eld_1" ;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",pgf_r[0]);
+      paraview_dc->RegisterField("E_i",pgf_i[0]);
+      paraview_dc->RegisterField("E_par_r",&E_par_r);
+      paraview_dc->RegisterField("E_par_i",&E_par_i);
+      paraview_dc->RegisterField("H_r",pgf_r[1]);
+      paraview_dc->RegisterField("H_i",pgf_i[1]);      
+      paraview_dc->RegisterField("Jh_1_r",pgf_r[4]);
+      paraview_dc->RegisterField("Jh_1_i",pgf_i[4]);
+      paraview_dc->RegisterField("Jh_2_r",pgf_r[5]);
+      paraview_dc->RegisterField("Jh_2_i",pgf_i[5]);
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listJ;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+   
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+
+      // remove internal boundaries
+      for (int i = 0; i<int_bdr_attr.Size(); i++)
+      {
+         ess_bdr[int_bdr_attr[i]-1] = 0;
+      }
+      all_pfes[2]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += all_toffsets[2];
+      }
+      ess_bdr = 1;
+      for (int i = 0; i<ndiffusionequations;i++)
+      {
+         ess_tdof_listJ.SetSize(0);
+         all_pfes[i+4]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJ);
+         for (int j = 0; j < ess_tdof_listJ.Size(); j++)
+         {
+            ess_tdof_listJ[j] += all_toffsets[i+4];
+         }
+         ess_tdof_list.Append(ess_tdof_listJ);
+      }
+
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+   
+
+   // rotate the vector
+   // (x,y) -> (y,-x)
+   Vector rot_one_x(dim); rot_one_x = 0.0; rot_one_x(1) = -1.0;
+   Vector rot_negone_x(dim); rot_negone_x = 0.0; rot_negone_x(1) = 1.0;
+   VectorConstantCoefficient rot_one_x_cf(rot_one_x);
+   VectorConstantCoefficient rot_negone_x_cf(rot_negone_x);
+
+   pgf_r[2]->ProjectBdrCoefficientNormal(rot_one_x_cf, one_r_bdr);
+   pgf_r[2]->ProjectBdrCoefficientNormal(rot_negone_x_cf, negone_r_bdr);
+   pgf_i[2]->ProjectBdrCoefficientNormal(rot_one_x_cf, one_i_bdr);
+   pgf_i[2]->ProjectBdrCoefficientNormal(rot_negone_x_cf, negone_i_bdr);
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Boundary conditions finished." << endl;
+   }
+
+   BlockOperator * P = new BlockOperator(all_offsets, all_toffsets);
+   BlockMatrix * R = new BlockMatrix(all_toffsets, all_offsets);
+   P->owns_blocks = 0;
+   R->owns_blocks = 0;
+
+   for (int i = 0; i < npfes; i++)
+   {
+      HypreParMatrix * P_ = all_pfes[i]->Dof_TrueDof_Matrix();
+      P->SetBlock(i,i,P_);
+      const SparseMatrix * R_ = all_pfes[i]->GetRestrictionMatrix();
+      R->SetBlock(i, i, const_cast<SparseMatrix*>(R_));
+   }
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Build prolongation finished" << endl;
+   }
+
+   int n = P->Width();
+   Vector B(2*n); B = 0.0;
+
+   Vector X(2*n);
+   Vector X_r(X, 0, n);
+   Vector X_i(X, n, n);
+
+   Vector x_r(x, 0, x.Size()/2);
+   Vector x_i(x, x.Size()/2, x.Size()/2);
+
+   R->Mult(x_r, X_r);
+   R->Mult(x_i, X_i);
+
+   ParBlockComplexSystem aa(A);
+   A = aa.EliminateBC(ess_tdof_list, X, B);
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Eliminate BC finished successfully." << endl;
+   }
+
+
+   BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&A->real());
+   BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&A->imag());
+
+   int nblocks = BlockA_r->NumRowBlocks();
+   
+   Array2D<const HypreParMatrix*> A_r_matrices(nblocks, nblocks);
+   Array2D<const HypreParMatrix*> A_i_matrices(nblocks, nblocks);
+   for (int i = 0; i < nblocks; i++)
+   {
+      for (int j = 0; j < nblocks; j++)
+      {
+         if (!BlockA_r->IsZeroBlock(i,j))
+         {
+            A_r_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockA_r->GetBlock(i,j));
+         }
+         else
+         {
+            A_r_matrices(i,j) = nullptr;
+         }
+         if (!BlockA_i->IsZeroBlock(i,j))
+         {
+            A_i_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockA_i->GetBlock(i,j));
+         }
+         else
+         {
+            A_i_matrices(i,j) = nullptr;
+         }
+      }
+   }
+
+   HypreParMatrix * Ahr = HypreParMatrixFromBlocks(A_r_matrices);
+   HypreParMatrix * Ahi = HypreParMatrixFromBlocks(A_i_matrices);
+
+   ComplexHypreParMatrix * Ahc_hypre =
+      new ComplexHypreParMatrix(Ahr, Ahi,false, false);
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Getting ready for solve." << endl;
+   }
+   
+
+#ifdef MFEM_USE_COMPLEX_MUMPS
+   if (mumps_solver)
+   {
+      auto solver = new ComplexMUMPSSolver(MPI_COMM_WORLD);
+      solver->SetPrintLevel(1);
+      solver->SetOperator(*Ahc_hypre);
+      solver->Mult(B,X);
+      delete solver;
+      delete Ahc_hypre;
+   }
+#else
+   if (mumps_solver)
+   {
+      MFEM_WARNING("MFEM compiled without mumps. Switching to an iterative solver");
+   }
+   mumps_solver = false;
+#endif
+
+   Array<int> tdof_offsets(nblocks*2+1);
+   tdof_offsets[0] = 0;
+   for (int i=0; i<nblocks; i++)
+   {
+      tdof_offsets[i+1] = A_r_matrices(i,i)->Height();
+      tdof_offsets[nblocks+i+1] = tdof_offsets[i+1];
+   }
+   tdof_offsets.PartialSum();
+
+   if (!mumps_solver)
+   {
+
+      BlockDiagonalPreconditioner M(tdof_offsets);
+
+      HypreBoomerAMG * solver_E = new HypreBoomerAMG((HypreParMatrix &)
+                                                   BlockA_r->GetBlock(0,0));
+      solver_E->SetPrintLevel(0);
+      solver_E->SetSystemsOptions(dim);
+
+      HypreBoomerAMG * solver_H = new HypreBoomerAMG((HypreParMatrix &)
+                                                BlockA_r->GetBlock(1,1));
+      solver_H->SetPrintLevel(0);
+
+      M.SetDiagonalBlock(0,solver_E);
+      M.SetDiagonalBlock(1,solver_H);
+      M.SetDiagonalBlock(nblocks,solver_E);
+      M.SetDiagonalBlock(nblocks+1,solver_H);
+
+      HypreAMS * solver_hatE = 
+      new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(2,2), dpg_pfes[2]);
+
+
+      HypreBoomerAMG * solver_hatH = new HypreBoomerAMG((HypreParMatrix &)
+                 BlockA_r->GetBlock(3,3));
+      solver_hatE->SetPrintLevel(0);
+      solver_hatH->SetPrintLevel(0);
+      solver_hatH->SetRelaxType(88);
+
+
+      M.SetDiagonalBlock(2,solver_hatE);
+      M.SetDiagonalBlock(3,solver_hatH);
+      M.SetDiagonalBlock(2+nblocks,solver_hatE);
+      M.SetDiagonalBlock(3+nblocks,solver_hatH);
+
+
+      HypreBoomerAMG * solver_J1 = new HypreBoomerAMG((HypreParMatrix &)
+                                                BlockA_r->GetBlock(4,4));
+      solver_J1->SetPrintLevel(0);
+      solver_J1->SetSystemsOptions(dim);
+      solver_J1->SetRelaxType(88);
+
+      HypreBoomerAMG * solver_J2 = new HypreBoomerAMG((HypreParMatrix &)
+                                                BlockA_r->GetBlock(5,5));
+      solver_J2->SetPrintLevel(0);
+      solver_J2->SetSystemsOptions(dim);
+      solver_J2->SetRelaxType(88);
+
+
+      M.SetDiagonalBlock(4,solver_J1);
+      M.SetDiagonalBlock(5,solver_J2);
+      M.SetDiagonalBlock(4+nblocks,solver_J1);
+      M.SetDiagonalBlock(5+nblocks,solver_J2);                                                
+
+      GMRESSolver gmres(MPI_COMM_WORLD);
+      gmres.SetRelTol(1e-10);
+      gmres.SetMaxIter(1000);
+      gmres.SetPrintLevel(1);
+      gmres.SetPreconditioner(M);
+      gmres.SetOperator(*A);
+      gmres.Mult(B, X);
+   }
+
+
+   n = P->Height();
+   int m = P->Width();
+
+   x_r.MakeRef(x, 0, n);
+   x_i.MakeRef(x, n, n);
+
+   X_r.MakeRef(X, 0, m);
+   X_i.MakeRef(X, m, m);
+
+   P->Mult(X_r, x_r);
+   P->Mult(X_i, x_i);
+
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i]->MakeRef(all_pfes[i], x, all_offsets[i]);
+      pgf_i[i]->MakeRef(all_pfes[i], x, all_offsets.Last() + all_offsets[i]);
+   }
+   
+   ParallelECoefficient par_e_r(pgf_r[0]);
+   ParallelECoefficient par_e_i(pgf_i[0]);
+   E_par_r.ProjectCoefficient(par_e_r);
+   E_par_i.ProjectCoefficient(par_e_i);
+   
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, *pgf_r[0],
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((real_t)0);
+      paraview_dc->Save();
+      delete paraview_dc;
+   }
+
+
+   delete a_fem;
+   for (int i = 0; i < fem_fecols.Size(); ++i)
+   {
+      delete fem_fecols[i];
+      delete fem_pfes[i];
+   }
+
+
+   delete a_dpg;
+   for (int i = 0; i < dpg_trial_fecols.Size(); ++i)
+   {
+      delete dpg_trial_fecols[i];
+      delete dpg_pfes[i];
+   }
+   for (int i = 0; i< dpg_test_fecols.Size(); ++i)
+   {
+      delete dpg_test_fecols[i];
+   }
+
+   return 0;
+}

miniapps/dpg/plasma/lh-eld-dpg.cpp

@@ -0,0 +1,907 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//                   MFEM Ultraweak DPG Maxwell parallel example
+//
+// Compile with: make lh-eld-dpg
+//
+// mpirun -np 8 ./lh-eld-dpg -o 4 -paraview -eld -m data/quad.msh -ebs -sc
+
+// Electron Landau Damping
+// Strong formulation:
+//     ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵ E + i ω²ϵ₀(J₁ + J₂) = 0,   in Ω
+//                   - Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥ = 0,   in Ω     
+//                   - Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥ = 0,   in Ω 
+//                                            E×n = E₀,  on ∂Ω
+//                                            J₁  = 0,  on ∂Ω
+//                                            J₂  = 0,  on ∂Ω
+// The DPG UW deals with the First Order System
+//  i ω μ₀ H  + ∇ × E                  = 0,   in Ω
+// -i ω ϵ₀ϵ E + ∇ × H - ω ϵ₀ (J₁ + J₂) = 0,   in Ω
+//         - Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥ = 0,   in Ω     
+//         - Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥ = 0,   in Ω 
+//                                 E×n = E₀,  on ∂Ω
+//                                 J₁  = 0,  on ∂Ω
+//                                 J₂  = 0,  on ∂Ω
+
+
+// in 2D
+// E is vector valued and H is scalar.
+//      (∇ × E, δE) = (E, ∇ × δE ) + < n × E , δE >
+// or (∇ ⋅ AE , δE) = (AE, ∇ δE ) + < AE ⋅ n, δE >
+// where A = [0 1; -1 0];
+
+// E ∈ (L²(Ω))² , H ∈ L²(Ω), J ∈ (H¹(Ω))²
+// Ê ∈ H^-1/2(Γₕ), Ĥ ∈ H^1/2(Γₕ), Ĵ₁, Ĵ₂ ∈ (H^-1/2(Γₕ))²
+//     iωμ₀ (H,δE) + (E,∇×δE) + < AÊ, δE > = 0,      ∀ δE ∈ H¹(Ω)
+//  -i ωϵ₀ϵ (E,δH) + (H,∇×δH) + < Ĥ, δH×n > - ω ϵ₀ (J₁ + J₂,δH) = 0,  ∀ δH ∈ H(curl,Ω)
+// ( (b⋅∇)J₁,(b⋅∇) δJ₁ ) + <Ĵ₁, δJ₁> + c₁ (J₁,δJ₁) - c₁ (P(r) b⊗b E, δJ₁) = 0,  ∀ δJ₁ ∈ (H¹(Ω))²
+// ( (b⋅∇)J₂,(b⋅∇) δJ₂ ) + <Ĵ₂, δJ₂> + c₂ (J₂,δJ₂) + c₂ (P(r) b⊗b E, δJ₂) = 0,  ∀ δJ₂ ∈ (H¹(Ω))²
+//                                                                  Ê = E₀, on ∂Ω
+//                                                            Ĵ₁ = Ĵ₂ = 0,  on ∂Ω
+// ----------------------------------------------------------------------------------------------------------------
+// |   |      E       |     H    |        J₁        |        J₂        |   Ê   |   Ĥ    |   Ĵ₁   |   Ĵ₂   |  RHS  |
+// ----------------------------------------------------------------------------------------------------------------
+// |δE |  (E,∇ × δE)  |iωμ₀(H,δE)|                  |                  | <Ê,δE>|        |        |        |   0   |  
+// |   |              |          |                  |                  |       |        |        |        |       |  
+// |δH | -iωϵ₀ϵ(E,δH) | (H,∇×δH) |   -ωϵ₀ (J₁,δH)   |  -ωϵ₀ (J₂,δH)    |       |<Ĥ,δH×n>|        |        |   0   |  
+// |   |              |          |                  |                  |       |        |        |        |       |  
+// |δJ₁|-c₁(P(r)E,δJ₁)|          |((b⋅∇)J₁,(b⋅∇)δJ₁)|                  |       |        |<Ĵ₁,δJ₁>|        |   0   |  
+// |   |              |          |     + c₁ (J₁,δJ₁)|                  |       |        |        |        |       |    
+// |δJ₂| c₂(P(r)E,δJ₂)|          |                  |((b⋅∇)J₂,(b⋅∇)δJ₂)|       |        |        |<Ĵ₂,δJ₂>|   0   |  
+// |   |              |          |                  |     + c₂ (J₂,δJ₂)|       |        |        |        |       |    
+// where (δE,δH,δJ₁,δJ₂) ∈  H¹(Ω) × H(curl,Ω) × (H¹(Ω))² × (H¹(Ω))² 
+
+
+
+#include "mfem.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../../common/mfem-common.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "../util/preconditioners.hpp"
+#include "utils/lh_utils.hpp"
+#include "../util/utils.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+   int order = 1;
+   int delta_order = 1;
+   int par_ref_levels = 0;
+   int ser_ref_levels = 0;
+
+   // real_t rnum=1.5e9;
+   // real_t mu = 1.257e-6;
+   // real_t eps0 = 8.8541878128e-12;
+
+   real_t rnum=1.5;
+   real_t mu = 1.257;
+   real_t eps0 = 8.8541878128;
+   real_t cfactor = 1e-6;
+   real_t balance_scale = 1.0;
+   bool enable_balance_scale = false;
+
+   bool eld = false; // enable/disable electron Landau damping 
+   real_t delta_prec = 0.0;
+
+
+   bool static_cond = false;
+   bool visualization = false;
+   bool paraview = false;
+   bool debug = false;
+   bool mumps_solver = false;
+   bool pmg = false;
+   int pmg_levels = -1;
+   real_t relax_factor = 2.0/3;
+
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&delta_order, "-do", "--delta-order",
+                  "Finite element order for the test space");                  
+   args.AddOption(&ser_ref_levels, "-sr", "--serial-refinement_levels",
+                  "Number of serial refinement levels.");                  
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&a0, "-a0", "--a0", "P(r) first parameter.");
+   args.AddOption(&a1, "-a1", "--a1", "P(r) second parameter.");
+   args.AddOption(&delta_prec, "-dp", "--delta-prec", "stability parameter for the preconditioner.");
+   args.AddOption(&eld, "-eld", "--eld", "-no-eld",
+                  "--no-eld",
+                  "Enable or disable electron Landau damping.");
+   args.AddOption(&mumps_solver, "-mumps", "--mumps", "-no-mumps",
+                  "--no-mumps",
+                  "Enable or disable MUMPS solver.");
+   args.AddOption(&pmg, "-pmg", "--p-refinement-multigrid", "-no-pmg",
+                  "--no-p-refinement-multigrid", "Enable P-Refinement Multigrid.");
+   args.AddOption(&pmg_levels, "-pmgl","--p-refinement-multigrid-levels",
+                  "Number of levels for P-Refinement Multigrid.");
+   args.AddOption(&relax_factor, "-rf", "--relaxation-factor",
+                  "Relaxation factor for the p-multigrid smoother.");                  
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&debug, "-debug", "--debug", "-no-debug",
+                  "--no-debug",
+                  "Enable or disable debug mode (delta = 0.01 and no coupling).");         
+   args.AddOption(&enable_balance_scale, "-ebs", "--enable-balance-scale", "-no-ebs",
+                  "--no-ebs",
+                  "Enable or disable balance scale.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (Mpi::Root())
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (Mpi::Root())
+   {
+      args.PrintOptions(cout);
+   }
+
+   // number of diffusion equations
+   int ndiffusionequations = (eld) ?  2 : 0; 
+
+   Vector cvals(ndiffusionequations);
+   Vector csigns(ndiffusionequations);
+   if (eld)
+   {
+      cvals(0)  = 25e6;  cvals(1)  = 1e6;
+      csigns(0) = -1.0;  csigns(1) = 1.0;
+   }
+   cvals *= cfactor; // scale the coefficients
+   real_t omega = 2.*M_PI*rnum;
+   int test_order = order+delta_order;
+
+   balance_scale = (enable_balance_scale) ? omega * eps0 * omega : 1.0;
+
+
+   if (eld && !debug) 
+   {
+      delta = 0.0; // disable delta if electron Landau damping is enabled
+      if (Mpi::Root())
+      {
+         cout << "Electron Landau damping enabled, delta set to 0.0." << endl;
+      }
+   }    
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   MFEM_VERIFY(dim == 2, "Dimension != 2 is not supported in this example");
+
+   for (int i = 0; i < ser_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // mesh.RemoveInternalBoundaries();
+
+   Array<int> int_bdr_attr;
+   for (int i = 0; i < mesh.GetNBE(); i++)
+   {
+      if (mesh.FaceIsInterior(mesh.GetBdrElementFaceIndex(i)))
+      {
+         int_bdr_attr.Append(mesh.GetBdrAttribute(i));
+      }
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   for (int i = 0; i < par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }  
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+
+   // Define coefficients
+   ConstantCoefficient muinv(1./mu);
+   ConstantCoefficient one_cf(1.0);
+   //  ωμ₀
+   ConstantCoefficient omegamu_cf(omega*mu);
+   // -ω μ₀  
+   ConstantCoefficient negomegamu_cf(-omega*mu);
+   // -ωϵ₀
+   real_t scale = (debug) ? 0.0 : 1.0;
+   ConstantCoefficient negomegeps0_cf(-omega*eps0 * scale);
+   ConstantCoefficient balancescaled_negomegeps0_cf( -omega*eps0/balance_scale * scale);
+   // μ₀² ω²
+   ConstantCoefficient mu2omeg2_cf((mu*mu*omega*omega));
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_vcf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   // S(r) 
+   FunctionCoefficient S_cf_r(sfunc_r), S_cf_i(sfunc_i);
+   // P(r) 
+   FunctionCoefficient P_cf_r(pfunc_r), P_cf_i(pfunc_i); 
+
+   VectorFunctionCoefficient b_cf(dim,bfunc);// b
+   ScalarVectorProductCoefficient scaled_b_cf(sqrt(cfactor), b_cf);
+   ConstantCoefficient diff_coeff(cfactor);
+
+   MatrixFunctionCoefficient bb_cf(dim,bcrossb); // b⊗b
+   MatrixSumCoefficient oneminusbb(Mone_cf, bb_cf, 1.0, -1.0); // 1 - b⊗b
+
+   // S(r) (I - b⊗b)
+   ScalarMatrixProductCoefficient Soneminusbb_r(S_cf_r, oneminusbb), Soneminusbb_i(S_cf_i, oneminusbb); 
+
+   // P(r) b⊗b 
+   ScalarMatrixProductCoefficient P_cf_bb_r(P_cf_r, bb_cf), P_cf_bb_i(P_cf_i, bb_cf); 
+
+   // ε = S(r) (I - b⊗b) + P(r) b⊗b 
+   MatrixSumCoefficient eps_r(Soneminusbb_r, P_cf_bb_r, 1.0, 1.0); 
+   MatrixSumCoefficient eps_i(Soneminusbb_i, P_cf_bb_i, 1.0, 1.0); 
+
+   coefs_r[nattr-1] = &eps_r;
+   coefs_i[nattr-1] = &eps_i;
+
+   // for (int i = 0; i < nattr-1; ++i)
+   // {
+   //    coefs_r[i] = &eps_r;
+   //    coefs_i[i] = &eps_i;
+   // }
+
+   PWMatrixCoefficient eps_cf_r(dim, attr, coefs_r);
+   PWMatrixCoefficient eps_cf_i(dim, attr, coefs_i);
+
+   ConstantCoefficient eps0omeg(omega * eps0);
+   ConstantCoefficient negeps0omeg(-omega * eps0);
+
+   // ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient eps0omeg_eps_r(eps0omeg, eps_cf_r);
+   // ω ϵ₀ ϵᵢ
+   ScalarMatrixProductCoefficient eps0omeg_eps_i(eps0omeg, eps_cf_i);
+   // -ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient negeps0omeg_eps_r(negeps0omeg, eps_cf_r);
+   // -ω ϵ₀ ϵᵢ
+   ScalarMatrixProductCoefficient negeps0omeg_eps_i(eps0omeg, eps_cf_i);
+
+   // A = [0 1; -1 0]
+   DenseMatrix rot_mat(2);
+   rot_mat(0,0) = 0.; rot_mat(0,1) = 1.;
+   rot_mat(1,0) = -1.; rot_mat(1,1) = 0.;
+   MatrixConstantCoefficient rot(rot_mat);
+   TransposeMatrixCoefficient Rt(rot);
+
+   // ω ϵ₀ ϵᵣ A
+   MatrixProductCoefficient eps0omeg_eps_r_rot(eps0omeg_eps_r, rot);
+   // ω ϵ₀ ϵᵢ A
+   MatrixProductCoefficient eps0omeg_eps_i_rot(eps0omeg_eps_i, rot);
+   // ω ϵ₀ Aᵀ ϵᵢ
+   MatrixProductCoefficient eps0omeg_Rt_eps_i(Rt, eps0omeg_eps_i);
+   // ω ϵ₀ Aᵀ ϵᵣ
+   MatrixProductCoefficient eps0omeg_Rt_eps_r(Rt, eps0omeg_eps_r);
+   // -ω ϵ₀ ϵᵣ A
+   MatrixProductCoefficient negeps0omeg_eps_r_rot(negeps0omeg_eps_r, rot);
+   // -ω ϵ₀ ϵᵢ A
+   MatrixProductCoefficient negeps0omeg_eps_i_rot(negeps0omeg_eps_i, rot);
+
+   // (ωϵ₀ϵ)(ωϵ₀ϵ)^*  (δH, δH)
+   TransposeMatrixCoefficient eps0omeg_eps_r_t(eps0omeg_eps_r);
+   TransposeMatrixCoefficient eps0omeg_eps_i_t(eps0omeg_eps_i);
+   MatrixProductCoefficient MrMrt_cf(eps0omeg_eps_r, eps0omeg_eps_r_t);
+   MatrixProductCoefficient MiMit_cf(eps0omeg_eps_i, eps0omeg_eps_i_t);
+   MatrixProductCoefficient MiMrt_cf(eps0omeg_eps_i, eps0omeg_eps_r_t);
+   MatrixProductCoefficient MrMit_cf(eps0omeg_eps_r, eps0omeg_eps_i_t);
+
+   // (MᵣMᵣᵗ + MᵢMᵢᵗ) + i (MᵢMᵣᵗ - MᵣMᵢᵗ)
+   MatrixSumCoefficient Mreal_cf(MrMrt_cf,MiMit_cf);
+   MatrixSumCoefficient Mimag_cf(MiMrt_cf,MrMit_cf,1.0,-1.0);
+
+   // if ELD
+   Array<Vector *> c_arrays(ndiffusionequations);
+   Array<PWConstCoefficient *> pw_c_coeffs(ndiffusionequations);
+   Array<MatrixCoefficient *> cPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> cPibb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPibb_cf(ndiffusionequations);
+
+   Array<MatrixCoefficient *> balancescaled_signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> balancescaled_signedcPibb_cf(ndiffusionequations);
+   Vector temp(nattr); temp=0.0;
+   Array<ConstantCoefficient *> c_coeffs(ndiffusionequations);
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      temp[nattr-1] = cvals(i);
+      // temp = cvals(i);
+      pw_c_coeffs[i] = new PWConstCoefficient(temp);
+      c_coeffs[i] = new ConstantCoefficient(cvals(i));
+      cPrbb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_r);
+      cPibb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_i);
+      signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPrbb_cf[i]);
+      signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPibb_cf[i]);
+      balancescaled_signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(balance_scale,*signedcPrbb_cf[i]);
+      balancescaled_signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(balance_scale,*signedcPibb_cf[i]);
+   }
+
+   // Define the spaces
+   Array<FiniteElementCollection *> trial_fecols;
+   Array<FiniteElementCollection *> test_fecols;
+   Array<ParFiniteElementSpace *> pfes;
+
+   // Vector L2 space for E
+   trial_fecols.Append(new L2_FECollection(order-1, dim));
+   pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last(), dim));
+   // Scalar L2 space for H
+   trial_fecols.Append(new L2_FECollection(order-1, dim));
+   pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last()));
+
+   // Vector H1 spaces for Jᵢ 
+   for (int i = 0; i < ndiffusionequations; i++)
+   {
+      trial_fecols.Append(new H1_FECollection(order, dim));
+      pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last(), dim));
+   }
+
+   // Trial trace space for Ê 
+   trial_fecols.Append(new RT_Trace_FECollection(order-1, dim));
+   pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last()));
+   // Trial trace space for Ĥ 
+   trial_fecols.Append(new H1_Trace_FECollection(order, dim));
+   pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last()));
+
+   // Vector Trace spaces Ĵᵢ    
+   for (int i = 0; i < ndiffusionequations; i++)
+   {
+      trial_fecols.Append(new RT_Trace_FECollection(order-1,dim));
+      pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last(),dim));
+   }
+
+   Array<HYPRE_BigInt> tdofs(pfes.Size());
+   for (int i = 0; i < pfes.Size(); ++i)
+   {
+      tdofs[i] = pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << "ParFiniteElementSpace " << i << " has " << tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   if (Mpi::Root())
+   {
+      cout << "Total number of true dofs: " << tdofs.Sum() << endl;
+   }
+
+   // test spaces for E and H
+   test_fecols.Append(new H1_FECollection(test_order, dim));
+   test_fecols.Append(new ND_FECollection(test_order, dim));
+   // Test spaces δJs 
+   for (int i = 0; i < ndiffusionequations; i++)
+   {
+      test_fecols.Append(new H1_FECollection(test_order, dim));
+      // test_fecols.Append(new H1_FECollection(order, dim));
+   }
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(pfes,test_fecols);
+   const IntegrationRule &ir_test = IntRules.Get(pmesh.GetElementGeometry(0),
+                                           2*test_order + 4);
+   const IntegrationRule &ir_trial = IntRules.Get(pmesh.GetElementGeometry(0),
+                                           order+test_order + 4);
+   a->SetTestIntegrationRule(ir_test);
+   a->SetTrialIntegrationRule(ir_trial);
+   
+   
+   for (int i = 0; i < ndiffusionequations; i++)
+   {
+      a->SetTestFECollVdim(i+2,dim);
+   }
+
+   // (E,∇ × δE)
+   a->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one_cf)),
+                         nullptr,0, 0);
+   //  -i ω ϵ₀ (ϵE,δH) = - i ω ϵ₀(ϵᵣ + i ϵᵢ E, δH)
+   //                   =  (ω ϵ₀ ϵᵢ E, δH) + i (-ω ϵ₀ϵᵣ E, δH)
+   a->AddTrialIntegrator(
+      new TransposeIntegrator(new VectorFEMassIntegrator(eps0omeg_eps_i)), 
+      new TransposeIntegrator(new VectorFEMassIntegrator(negeps0omeg_eps_r)),
+                              0,1);
+   // iωμ₀(H,δE) 
+   a->AddTrialIntegrator(nullptr,new MixedScalarMassIntegrator(omegamu_cf),1, 0);
+   // (H,∇ × δH)                         
+   a->AddTrialIntegrator(
+      new TransposeIntegrator(new MixedCurlIntegrator(one_cf)), nullptr,1, 1);
+
+   // Trace integrators               
+   //  <Ê,δE>
+   a->AddTrialIntegrator(new TraceIntegrator,nullptr, 2 + ndiffusionequations, 0);
+   // <Ĥ,δH × n>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr, 2 + ndiffusionequations+1, 1);
+   if (eld)
+   {
+      for (int i = 0; i < ndiffusionequations; i++)
+      {
+         // ±cᵢ(P(r) (b ⊗ b) E, δJᵢ)
+         a->AddTrialIntegrator(new VectorMassIntegrator(*balancescaled_signedcPrbb_cf[i]),
+                               new VectorMassIntegrator(*balancescaled_signedcPibb_cf[i]),
+                               0, i+2);
+         // -ωϵ₀ (Jᵢ ,δH)
+         a->AddTrialIntegrator(
+            new TransposeIntegrator(new VectorFEMassIntegrator(balancescaled_negomegeps0_cf)),
+            // new TransposeIntegrator(new VectorFEMassIntegrator(negomegeps0_cf)),
+                           nullptr,i+2, 1);
+         // ((b⋅∇)Jᵢ, (b⋅∇) δJᵢ)
+         a->AddTrialIntegrator(new DirectionalVectorDiffusionIntegrator(scaled_b_cf), nullptr,
+         // a->AddTrialIntegrator(new VectorDiffusionIntegrator(diff_coeff), nullptr,
+                               i+2, i+2);
+         // cᵢ(Jᵢ, δJᵢ)
+         a->AddTrialIntegrator(new VectorMassIntegrator(*pw_c_coeffs[i]), nullptr,
+                               i+2, i+2);
+
+         // // <Ĵᵢ,δJᵢ>
+         a->AddTrialIntegrator(new VectorTraceIntegrator,nullptr,
+                               i + ndiffusionequations + 4, i+2);                      
+      }
+   }      
+
+   // test integrators
+   // (∇δE,∇δE)
+   a->AddTestIntegrator(new DiffusionIntegrator(one_cf),nullptr, 0, 0);
+   // (δE,δE)
+   a->AddTestIntegrator(new MassIntegrator(one_cf),nullptr, 0, 0);
+   // μ₀² ω² (δE,δE)
+   a->AddTestIntegrator(new MassIntegrator(mu2omeg2_cf),nullptr,0, 0);
+   // -i ω μ₀ (δE,∇ × δH) = i (δE, -ω μ₀ ∇ × δ H)
+   a->AddTestIntegrator(nullptr,
+         new TransposeIntegrator(new MixedCurlIntegrator(negomegamu_cf)),0, 1);
+   // -i ω ϵ₀ϵ(∇ × δE, δH) = -i (ωϵ₀(ϵᵣ+iϵᵢ) A ∇ δE,δE), A = [0 1; -1 0]
+   //                       =  (ω ϵ₀ ϵᵢ A ∇ δE,δE) + i (-ω ϵ₀ ϵᵣ A ∇ δE,δE)
+   a->AddTestIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_i_rot),
+                        new MixedVectorGradientIntegrator(negeps0omeg_eps_r_rot),0, 1);
+   // i ω μ₀ (∇ × δH ,δE) = i (ω μ₀ ∇ × δH, δE )
+   a->AddTestIntegrator(nullptr,new MixedCurlIntegrator(omegamu_cf),
+                        1, 0);
+   // i ω ϵ₀ϵ̄ (δH, ∇ × δE ) = i (ω ϵ₀(ϵᵣ -i ϵᵢ) δH, A ∇ δE) 
+   //                        = ( δH, ω ϵ₀ ϵᵢ A ∇ δE) + i (δH, ω ϵ₀ ϵᵣ A ∇ δE)
+
+
+   a->AddTestIntegrator(new TransposeIntegrator(new MixedVectorGradientIntegrator(eps0omeg_Rt_eps_i)),
+                        new TransposeIntegrator(new MixedVectorGradientIntegrator(eps0omeg_Rt_eps_r)),1, 0);
+
+   // a->AddTestIntegrator(
+   //    new TransposeIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_i_rot)),
+   //    new TransposeIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_r_rot)),1, 0);
+   
+   
+      // (ωϵ₀ϵ)(ωϵ₀ϵ)^*  (δH, δH)
+   // (MᵣMᵣᵗ + MᵢMᵢᵗ) + i (MᵢMᵣᵗ - MᵣMᵢᵗ)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(Mreal_cf),
+                        new VectorFEMassIntegrator(Mimag_cf),1, 1);
+   // (∇×δH ,∇×δH)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one_cf),nullptr,1,1);
+   // (δH,δH)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one_cf),nullptr,1,1);
+
+   for (int i = 0; i < ndiffusionequations; i++)
+   {
+      // (∇δJ,∇δJ) 
+      // a->AddTestIntegrator(new VectorDiffusionIntegrator(one_cf),nullptr,
+                           // i+2,i+2);
+      // (b⋅∇δJ, b⋅∇δJ)
+      a->AddTestIntegrator(new DirectionalVectorDiffusionIntegrator(scaled_b_cf),nullptr,
+      // a->AddTestIntegrator(new DirectionalVectorDiffusionIntegrator(b_cf),nullptr,
+                           i+2,i+2);                     
+      // (δJ,δJ)
+      // a->AddTestIntegrator(new VectorMassIntegrator(one_cf),nullptr, 
+      a->AddTestIntegrator(new VectorMassIntegrator(*c_coeffs[i]),nullptr, 
+                           i+2,i+2);
+   }
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble(0);
+
+
+   // delta = delta_prec;
+   // ParComplexDPGWeakForm * a_prec = new ParComplexDPGWeakForm(pfes,test_fecols);
+   // for (int i = 0; i < ndiffusionequations; i++)
+   // {
+   //    a_prec->SetTestFECollVdim(i+2,dim);
+   // }
+
+   // // (E,∇ × δE)
+   // a_prec->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one_cf)),
+   //                       nullptr,0, 0);
+   // //  -i ω ϵ₀ (ϵE,δH) = - i ω ϵ₀(ϵᵣ + i ϵᵢ E, δH)
+   // //                   =  (ω ϵ₀ ϵᵢ E, δH) + i (-ω ϵ₀ϵᵣ E, δH)
+   // a_prec->AddTrialIntegrator(
+   //    new TransposeIntegrator(new VectorFEMassIntegrator(eps0omeg_eps_i)), 
+   //    new TransposeIntegrator(new VectorFEMassIntegrator(negeps0omeg_eps_r)),
+   //    0,1);
+   // // iωμ₀(H,δE)
+   // a_prec->AddTrialIntegrator(nullptr,new MixedScalarMassIntegrator(omegamu_cf),1, 0);
+   // // (H,∇ × δH)                         
+   // a_prec->AddTrialIntegrator(
+   //    new TransposeIntegrator(new MixedCurlIntegrator(one_cf)), nullptr,1, 1);
+
+   // // Trace integrators               
+   // //  <Ê,δE>
+   // a_prec->AddTrialIntegrator(new TraceIntegrator,nullptr, 2 + ndiffusionequations, 0);
+   // // <Ĥ,δH × n>
+   // a_prec->AddTrialIntegrator(new TangentTraceIntegrator,nullptr, 2 + ndiffusionequations+1, 1);
+   // if (eld)
+   // {
+   //    for (int i = 0; i < ndiffusionequations; i++)
+   //    {
+   //       // ±cᵢ(P(r) (b ⊗ b) E, δJᵢ)
+   //       a_prec->AddTrialIntegrator(new VectorMassIntegrator(*balancescaled_signedcPrbb_cf[i]),
+   //                             new VectorMassIntegrator(*balancescaled_signedcPibb_cf[i]), 0, i+2);
+         
+   //       // a_prec->AddTrialIntegrator(
+   //       //    new TransposeIntegrator(new VectorFEMassIntegrator(balancescaled_negomegeps0_cf)),
+   //       //                   nullptr,i+2, 1);
+         
+   //       // ((b⋅∇)Jᵢ, (b⋅∇) δJᵢ)
+   //       a_prec->AddTrialIntegrator(new DirectionalVectorDiffusionIntegrator(scaled_b_cf), nullptr, i+2, i+2);
+   //       // cᵢ(Jᵢ, δJᵢ)
+   //       a_prec->AddTrialIntegrator(new VectorMassIntegrator(*pw_c_coeffs[i]), nullptr, i+2, i+2);
+   //       // <Ĵᵢ,δJᵢ>
+   //       // a_prec->AddTrialIntegrator(new VectorTraceIntegrator,nullptr, i + ndiffusionequations + 4, i+2);                      
+   //    }
+   // }      
+
+   // // test integrators
+   // // (∇δE,∇δE)
+   // a_prec->AddTestIntegrator(new DiffusionIntegrator(one_cf),nullptr, 0, 0);
+   // // (δE,δE)
+   // a_prec->AddTestIntegrator(new MassIntegrator(one_cf),nullptr, 0, 0);
+   // // μ₀² ω² (δE,δE)
+   // a_prec->AddTestIntegrator(new MassIntegrator(mu2omeg2_cf),nullptr,0, 0);
+   // // -i ω μ₀ (δE,∇ × δH) = i (δE, -ω μ₀ ∇ × δ H)
+   // a_prec->AddTestIntegrator(nullptr,
+   //       new TransposeIntegrator(new MixedCurlIntegrator(negomegamu_cf)),0, 1);
+   // // -i ω ϵ₀ϵ(∇ × δE, δH) = -i (ωϵ₀(ϵᵣ+iϵᵢ) A ∇ δE,δE), A = [0 1; -1 0]
+   // //                       =  (ω ϵ₀ ϵᵢ A ∇ δE,δE) + i (-ω ϵ₀ ϵᵣ A ∇ δE,δE)
+   // a_prec->AddTestIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_i_rot),
+   //                      new MixedVectorGradientIntegrator(negeps0omeg_eps_r_rot),0, 1);
+   // // i ω μ₀ (∇ × δH ,δE) = i (ω μ₀ ∇ × δH, δE )
+   // a_prec->AddTestIntegrator(nullptr,new MixedCurlIntegrator(omegamu_cf),
+   //                      1, 0);
+   // // i ω ϵ₀ϵ̄ (δH, ∇ × δE ) = i (ω ϵ₀(ϵᵣ -i ϵᵢ) δH, A ∇ δE) 
+   // //                        = ( δH, ω ϵ₀ ϵᵢ A ∇ δE) + i (δH, ω ϵ₀ ϵᵣ A ∇ δE)
+   // a_prec->AddTestIntegrator(
+   //    new TransposeIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_i_rot)),
+   //    new TransposeIntegrator(new MixedVectorGradientIntegrator(eps0omeg_eps_r_rot)),1, 0);
+   // // (ωϵ₀ϵ)(ωϵ₀ϵ)^*  (δH, δH)
+   // // (MᵣMᵣᵗ + MᵢMᵢᵗ) + i (MᵢMᵣᵗ - MᵣMᵢᵗ)
+   // a_prec->AddTestIntegrator(new VectorFEMassIntegrator(Mreal_cf),
+   //                      new VectorFEMassIntegrator(Mimag_cf),1, 1);
+   // // (∇×δH ,∇×δH)
+   // a_prec->AddTestIntegrator(new CurlCurlIntegrator(one_cf),nullptr,1,1);
+   // // (δH,δH)
+   // a_prec->AddTestIntegrator(new VectorFEMassIntegrator(one_cf),nullptr,1,1);
+
+   // for (int i = 0; i < ndiffusionequations; i++)
+   // {
+   //    // (b⋅∇δJ, b⋅∇δJ)
+   //    a_prec->AddTestIntegrator(new DirectionalVectorDiffusionIntegrator(scaled_b_cf),nullptr,
+   //                         i+2,i+2);                     
+   //    // (δJ,δJ)
+   //    a_prec->AddTestIntegrator(new VectorMassIntegrator(*c_coeffs[i]),nullptr, 
+   //                         i+2,i+2);
+   // }
+
+   // if (static_cond) { a_prec->EnableStaticCondensation(); }
+   // a_prec->Assemble();
+
+
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      delete pw_c_coeffs[i];
+      delete c_coeffs[i];
+      delete cPrbb_cf[i]; 
+      delete cPibb_cf[i]; 
+      delete signedcPrbb_cf[i]; 
+      delete signedcPibb_cf[i]; 
+   }
+
+   socketstream E_out_r;
+
+   int npfes = pfes.Size();
+   Array<int> offsets(npfes+1);  offsets[0] = 0;
+   Array<int> toffsets(npfes+1); toffsets[0] = 0;
+   for (int i = 0; i<npfes; i++)
+   {
+      offsets[i+1] = pfes[i]->GetVSize();
+      toffsets[i+1] = pfes[i]->TrueVSize();
+   }
+   offsets.PartialSum();
+   toffsets.PartialSum();
+
+   Vector x(2*offsets.Last());
+   x = 0.;
+   
+   Array<ParGridFunction *> pgf_r(npfes);
+   Array<ParGridFunction *> pgf_i(npfes);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i] = new ParGridFunction(pfes[i], x, offsets[i]);
+      pgf_i[i] = new ParGridFunction(pfes[i], x, offsets.Last() + offsets[i]);
+   }
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+   ParGridFunction E_par_r(&L2_fes);
+   ParGridFunction E_par_i(&L2_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   std::string output_dir = "ParaView/UW/" + GetTimestamp();
+
+   if (paraview)
+   {
+      if (Mpi::Root()) { WriteParametersToFile(args, output_dir); }
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order
+                         << "_eld_" << eld;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",pgf_r[0]);
+      paraview_dc->RegisterField("E_i",pgf_i[0]);
+      paraview_dc->RegisterField("E_par_r",&E_par_r);
+      paraview_dc->RegisterField("E_par_i",&E_par_i);
+      paraview_dc->RegisterField("H_r",pgf_r[1]);
+      paraview_dc->RegisterField("H_i",pgf_i[1]);      
+      if (eld)
+      {
+         paraview_dc->RegisterField("Jh_1_r",pgf_r[2]);
+         paraview_dc->RegisterField("Jh_1_i",pgf_i[2]);
+         paraview_dc->RegisterField("Jh_2_r",pgf_r[3]);
+         paraview_dc->RegisterField("Jh_2_i",pgf_i[3]);
+      }
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listJ;
+   Array<int> ess_tdof_listJhat;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+   
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+
+      // remove internal boundaries
+      for (int i = 0; i<int_bdr_attr.Size(); i++)
+      {
+         ess_bdr[int_bdr_attr[i]-1] = 0;
+      }
+
+      pfes[2+ndiffusionequations]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += toffsets[2+ndiffusionequations];
+      }
+      // ess_bdr=1;
+      for (int i = 0; i<ndiffusionequations;i++)
+      {
+         ess_tdof_listJ.SetSize(0);
+         ess_tdof_listJhat.SetSize(0);
+         pfes[i+2]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJ);
+         // pfes[i+6]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJhat);
+         for (int j = 0; j < ess_tdof_listJ.Size(); j++)
+         {
+            ess_tdof_listJ[j] += toffsets[i+2];
+         }
+         // for (int j = 0; j < ess_tdof_listJhat.Size(); j++)
+         // {
+         //    ess_tdof_listJhat[j] += toffsets[i+6];
+         // }
+
+         ess_tdof_list.Append(ess_tdof_listJ);
+         // ess_tdof_list.Append(ess_tdof_listJhat);
+      }
+
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+   
+
+   // rotate the vector
+   // (x,y) -> (y,-x)
+   Vector rot_one_x(dim); rot_one_x = 0.0; rot_one_x(1) = -1.0;
+   Vector rot_negone_x(dim); rot_negone_x = 0.0; rot_negone_x(1) = 1.0;
+   VectorConstantCoefficient rot_one_x_cf(rot_one_x);
+   VectorConstantCoefficient rot_negone_x_cf(rot_negone_x);
+
+   pgf_r[2+ndiffusionequations]->ProjectBdrCoefficientNormal(rot_one_x_cf, one_r_bdr);
+   pgf_r[2+ndiffusionequations]->ProjectBdrCoefficientNormal(rot_negone_x_cf, negone_r_bdr);
+   pgf_i[2+ndiffusionequations]->ProjectBdrCoefficientNormal(rot_one_x_cf, one_i_bdr);
+   pgf_i[2+ndiffusionequations]->ProjectBdrCoefficientNormal(rot_negone_x_cf, negone_i_bdr);
+
+   OperatorPtr Ah;
+   Vector X,B;
+   a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+   ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+   BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+
+   Array<ParFiniteElementSpace *> prec_fes;
+   if (static_cond)
+   {
+      a->GetTraceFESpaces(prec_fes);
+   }
+   else
+   {
+      prec_fes = pfes;
+   }
+   Solver * cprec = nullptr;
+
+   if (pmg)
+   {
+#ifdef MFEM_USE_COMPLEX_MUMPS
+      bool mumps_coarse_solver = true;
+#else
+      bool mumps_coarse_solver = false;
+#endif
+      std::vector<Array<int>> ess_bdr_marker(prec_fes.Size());
+      for (int b = 0; b<prec_fes.Size(); b++)
+      {
+         if (pmesh.bdr_attributes.Size())
+         {
+            ess_bdr_marker[b].SetSize(pmesh.bdr_attributes.Max());
+            int ess_block_hatE = (static_cond) ? ndiffusionequations : ndiffusionequations + 2;
+            int ess_block_J1 = (static_cond) ? 0 : 2;
+            int ess_block_J2 = (static_cond) ? 1 : 3;
+            if (b == ess_block_hatE || b == ess_block_J1 || b == ess_block_J2) // hatE, J1, J2
+            {
+               ess_bdr_marker[b] = ess_bdr;
+            }
+            else
+            {
+               ess_bdr_marker[b] = 0;
+            }
+         }
+      }
+      cprec = new ComplexPRefinementMultigrid(prec_fes, ess_bdr_marker, *Ahc, 
+                                              pmg_levels, relax_factor, mumps_coarse_solver);
+   }
+   else
+   {
+      BlockDiagonalPreconditioner * real_prec = new BlockDiagonalPreconditioner(BlockA_r->RowOffsets());
+      real_prec->owns_blocks = 1;
+      for (int i = 0; i<BlockA_r->NumRowBlocks(); i++)
+      {
+         auto prec = MakeFESpaceDefaultSolver(prec_fes[i],0);
+         prec->SetOperator(BlockA_r->GetBlock(i,i));
+         real_prec->SetDiagonalBlock(i,prec);
+      }
+      cprec = new ComplexPreconditioner(real_prec, true);
+   }
+
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-10);
+   cg.SetMaxIter(500);
+   cg.SetPrintLevel(1);
+   cg.SetOperator(*Ahc);
+   cg.SetPreconditioner(*cprec);
+   cg.Mult(B, X);
+
+   a->RecoverFEMSolution(X, x);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i]->MakeRef(pfes[i], x, offsets[i]);
+      pgf_i[i]->MakeRef(pfes[i], x, offsets.Last() + offsets[i]);
+   }
+   
+   ParallelECoefficient par_e_r(pgf_r[0]);
+   ParallelECoefficient par_e_i(pgf_i[0]);
+   E_par_r.ProjectCoefficient(par_e_r);
+   E_par_i.ProjectCoefficient(par_e_i);
+   
+   // rescale the J solutions
+   for (int i = 0; i < ndiffusionequations; ++i)
+   {
+      (*pgf_r[2+i]) /= balance_scale;
+      (*pgf_i[2+i]) /= balance_scale;
+   }
+
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, *pgf_r[0],
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((real_t)0);
+      paraview_dc->Save();
+      delete paraview_dc;
+   }
+
+
+   delete a;
+   for (int i = 0; i < trial_fecols.Size(); ++i)
+   {
+      delete trial_fecols[i];
+      delete pfes[i];
+   }
+   for (int i = 0; i< test_fecols.Size(); ++i)
+   {
+      delete test_fecols[i];
+   }
+
+   return 0;
+}

miniapps/dpg/plasma/lh-eld-fem-symm.cpp

@@ -0,0 +1,522 @@
+//                   MFEM FEM parallel example
+//
+
+// Electron Landau Damping
+
+
+//    ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵᵣ E + i ω²ϵ₀(J₁ + J₂) = 0,   in Ω
+//                   - Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥ = 0,   in Ω     
+//                   - Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥ = 0,   in Ω 
+
+
+// Strong formulation:
+//    ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵᵣ E + 1/2 i ω²ϵ₀(J₁ + J₂) - c₁ P(r)(b ⊗ b) J₃ + c₂ P(r)(b ⊗ b) J₄ = 0,   in Ω
+//                   - Δ∥ J₁ + c₁ J₁ - c₁ P(r) (b ⊗ b) E = 0,   in Ω     
+//                   - Δ∥ J₂ + c₂ J₂ + c₂ P(r) (b ⊗ b) E = 0,   in Ω 
+//                   - Δ∥ J₃ + c₁ J₃ + 1/2 i ω² ϵ₀ E = 0,  in Ω
+//                   - Δ∥ J₄ + c₂ J₄ + 1/2 i ω² ϵ₀ E = 0,  in Ω
+//
+//                                           E×n = E₀,  on ∂Ω
+//                                           J₁  = 0,  on ∂Ω
+//                                           J₂  = 0,  on ∂Ω
+// weak formulation:
+//   Find E ∈ H(curl,Ω), J₁  ∈ H¹(Ω), J₂  ∈ H¹(Ω),  J₃ ∈ H¹(Ω), J₄  ∈ H¹(Ω) such that
+//   (1/μ₀ ∇×E, ∇ × F) - ω² ϵ₀ (ϵᵣ E, F) + i ω²ϵ₀(J₁ + J₂, F) - (c₁ P(r)(b ⊗ b) J₃, F) + (c₂ P(r)(b ⊗ b) J₄, F) = 0,  ∀ F ∈ H(curl,Ω)
+//        ( (b⋅∇)J₁ , (b⋅∇) G) + c₁ (J₁ , G) - c₁ (P(r) (b ⊗ b) E, G) = 0,  ∀ G ∈ (H¹(Ω))ᵈ
+//        ( (b⋅∇)J₂ , (b⋅∇) H) + c₂ (J₂ , H) + c₂ (P(r) (b ⊗ b) E, H) = 0,  ∀ H ∈ (H¹(Ω))ᵈ
+//        ( (b⋅∇)J₃ , (b⋅∇) I) + c₁ (J₃ , I) - i ω² ϵ₀ (E, I) = 0,  ∀ I ∈ (H¹(Ω))ᵈ
+//        ( (b⋅∇)J₄ , (b⋅∇) K) + c₂ (J₄ , K) - i ω² ϵ₀ (E, K) = 0,  ∀ K ∈ (H¹(Ω))ᵈ
+
+//     |             E             |                J₁             |               J₂             |         J₃        |         J₄      | 
+// -----------------------------------------------------------------------------------------------------------
+// δE  |(1/μ₀∇×E,∇×δE)-ω²ϵ₀(ϵᵣE,δE)|          i ω²ϵ₀(J₁,δE)        |        i ω²ϵ₀(J₂,δE)         |-(c₁P(r)(b⊗b)J₃,δE)|c₂P(r)(b⊗b)J₄,δE)|    
+// δJ₁ |                           | ((b⋅∇)J₁,(b⋅∇)δJ₁)+c₁(J₁,δJ₁) |                              |                   |    |       |
+// δJ₂ |                           |                               |((b⋅∇)J₂,(b⋅∇)δJ₂)+ c₂(J₂,δJ₂)|       |       |  
+// δJ₃ |           |        |       |       |       |  
+// δJ₄ |           |        |       |       |       |  
+
+
+
+#include "mfem.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../util/pcomplexblockform.hpp"
+#include "../util/blockcomplexhypremat.hpp"
+#include "../util/utils.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "utils/lh_utils.hpp"
+#include "../../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+#include <cstring>
+#include <filesystem>
+
+using namespace std;
+using namespace mfem;
+
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+
+   int order = 2;
+   int par_ref_levels = 0;
+   int ser_ref_levels = 0;
+   bool visualization = false;
+   // real_t rnum=4.6e9;
+   // real_t mu = 1.257e-6;
+   // real_t eps0 = 8.8541878128e-12*factor;
+   real_t rnum=1.5e9;
+   real_t mu = 1.257e-6;
+   real_t eps0 = 8.8541878128e-12;
+   bool eld = true; // enable/disable electron Landau damping 
+
+   bool paraview = false;
+   bool debug = false;
+   bool mumps_solver = true;
+   real_t delta_prec = 0.01;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&ser_ref_levels, "-sr", "--serial-refinement_levels",
+                  "Number of serial refinement levels.");                  
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&a0, "-a0", "--a0", "P(r) first parameter.");
+   args.AddOption(&a1, "-a1", "--a1", "P(r) second parameter.");
+   args.AddOption(&delta_prec, "-dp", "--delta-prec", "stability parameter for the preconditioner.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&eld, "-eld", "--eld", "-no-eld",
+                  "--no-eld",
+                  "Enable or disable electron Landau damping.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.AddOption(&mumps_solver, "-mumps", "--mumps", "-no-mumps",
+                  "--no-mumps",
+                  "Enable or disable MUMPS solver.");
+   args.AddOption(&debug, "-debug", "--debug", "-no-debug",
+                  "--no-debug",
+                  "Enable or disable debug mode (delta = 0.01 and no coupling).");                  
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   
+   // number of diffusion equations
+   int ndiffusionequations = (eld) ?  2 : 0; 
+
+   Vector cvals(ndiffusionequations);
+   Vector csigns(ndiffusionequations);
+   // real_t cfactor = 1e-6;
+   real_t cfactor = 1.0;
+   if (eld)
+   {
+      cvals(0)  = 25e6;  cvals(1)  = 1e6;
+      csigns(0) = -1.0;  csigns(1) = 1.0;
+   }
+   cvals *= cfactor; // scale the coefficients
+   
+   real_t omega = 2.*M_PI*rnum;
+   if (eld && !debug) 
+   {
+      delta = 0.0; // disable delta if electron Landau damping is enabled
+      if (Mpi::Root())
+      {
+         cout << "Electron Landau damping enabled, delta set to 0.0." << endl;
+      }
+   }   
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   for (int i = 0; i < ser_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   mesh.RemoveInternalBoundaries();
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+   
+   for (int i = 0; i<par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }
+
+   // Define the coefficients 
+   ConstantCoefficient muinv(1./mu);
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_vcf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   // S(r) 
+   FunctionCoefficient S_cf_r(sfunc_r), S_cf_i(sfunc_i);
+   // P(r) 
+   FunctionCoefficient P_cf_r(pfunc_r), P_cf_i(pfunc_i); 
+
+   VectorFunctionCoefficient b_cf(dim,bfunc);// b
+   ScalarVectorProductCoefficient scaledb_cf(sqrt(cfactor), b_cf); 
+   MatrixFunctionCoefficient bb_cf(dim,bcrossb); // b⊗b
+
+   MatrixSumCoefficient oneminusbb(Mone_cf, bb_cf, 1.0, -1.0); // 1 - b⊗b
+
+   // S(r) (I - b⊗b)
+   ScalarMatrixProductCoefficient Soneminusbb_r(S_cf_r, oneminusbb), Soneminusbb_i(S_cf_i, oneminusbb); 
+
+   // P(r) b⊗b 
+   ScalarMatrixProductCoefficient P_cf_bb_r(P_cf_r, bb_cf), P_cf_bb_i(P_cf_i, bb_cf); 
+
+   // εᵣ = S(r) (I - b⊗b) + P(r) b⊗b 
+   MatrixSumCoefficient eps_r(Soneminusbb_r, P_cf_bb_r, 1.0, 1.0); 
+   MatrixSumCoefficient eps_i(Soneminusbb_i, P_cf_bb_i, 1.0, 1.0); 
+
+   coefs_r[nattr-1] = &eps_r;
+   coefs_i[nattr-1] = &eps_i;
+
+   PWMatrixCoefficient eps_cf_r(dim, attr, coefs_r);
+   PWMatrixCoefficient eps_cf_i(dim, attr, coefs_i);
+
+   real_t scale = (debug) ? 0.0 : 1.0;
+   ConstantCoefficient eps0omeg2(eps0 * omega * omega * scale);
+   ConstantCoefficient negeps0omeg2(-eps0 * omega * omega);
+
+   ScalarMatrixProductCoefficient m_cf_r(negeps0omeg2, eps_cf_r);
+   ScalarMatrixProductCoefficient m_cf_i(negeps0omeg2, eps_cf_i);
+
+   // if ELD
+   Array<Vector *> c_arrays(ndiffusionequations);
+   Array<PWConstCoefficient *> pw_c_coeffs(ndiffusionequations);
+   Array<MatrixCoefficient *> cPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> cPibb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPibb_cf(ndiffusionequations);
+   Vector temp(nattr); temp=0.0;
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      temp[nattr-1] = cvals(i);
+      pw_c_coeffs[i] = new PWConstCoefficient(temp);
+      cPrbb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_r);
+      cPibb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_i);
+      signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPrbb_cf[i]);
+      signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPibb_cf[i]);
+   }
+
+   Array<FiniteElementCollection *> fecols;
+   Array<ParFiniteElementSpace *> pfes;
+   fecols.Append(new ND_FECollection(order, dim));
+   pfes.Append(new ParFiniteElementSpace(&pmesh, fecols[0]));
+   if (eld)
+   {
+      for (int i = 0; i < 2*ndiffusionequations; ++i)
+      {
+         fecols.Append(new H1_FECollection(order, dim));
+         pfes.Append(new ParFiniteElementSpace(&pmesh, fecols[i+1], dim));
+      }
+   }
+
+   Array<HYPRE_BigInt> tdofs(pfes.Size());
+   for (int i = 0; i < pfes.Size(); ++i)
+   {
+      tdofs[i] = pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << "ParFiniteElementSpace " << i << " has " << tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   if (Mpi::Root())
+   {
+      cout << "Total number of true dofs: " << tdofs.Sum() << endl;
+   }
+
+   ParComplexBlockForm *a = new ParComplexBlockForm(pfes);
+   // (1/μ₀ ∇×E, ∇ × F)
+   a->AddDomainIntegrator(new CurlCurlIntegrator(muinv), nullptr, 0, 0);
+   // - ω² ϵ₀ (ϵᵣ E, F)
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(m_cf_r),
+                          new VectorFEMassIntegrator(m_cf_i), 0, 0);
+   ConstantCoefficient halfeps0omeg2(0.5*eps0 * omega * omega);    
+   ConstantCoefficient neghalfeps0omeg2(-0.5*eps0 * omega * omega);                   
+   if (eld)
+   {
+      for (int i = 0; i<ndiffusionequations; i++)
+      {
+         //  1/2 i ω²ϵ₀((J₁+J₂),F)
+         a->AddDomainIntegrator(nullptr, new TransposeIntegrator(new VectorFEMassIntegrator(halfeps0omeg2)), i+1, 0);
+         //  ( (b⋅∇)J₁ , (b⋅∇) G)
+         a->AddDomainIntegrator(new DirectionalVectorDiffusionIntegrator(scaledb_cf), nullptr, i+1, i+1);
+         // cᵢ (J₁ , G)
+         a->AddDomainIntegrator(new VectorMassIntegrator(*pw_c_coeffs[i]), nullptr, i+1, i+1);
+         // ±cᵢ(P(r) (b ⊗ b) E, G)
+         a->AddDomainIntegrator(new VectorFEMassIntegrator(*signedcPrbb_cf[i]), 
+                                new VectorFEMassIntegrator(*signedcPibb_cf[i]), 0, i+1);
+         // 
+         // ±(cᵢ P(r)(b ⊗ b) Jᵢ₊₂,F)
+         a->AddDomainIntegrator(new TransposeIntegrator(
+                                new VectorFEMassIntegrator(*signedcPrbb_cf[i])), 
+                                new TransposeIntegrator(
+                                new VectorFEMassIntegrator(*signedcPibb_cf[i])), i+3, 0);
+      
+      // ( (b⋅∇)J₃ , (b⋅∇) I) + c₁ (J₃ , I) - i ω² ϵ₀ (E, I) = 0,  ∀ I ∈ (H¹(Ω))ᵈ
+      // ( (b⋅∇)J₄ , (b⋅∇) K) + c₂ (J₄ , K) - i ω² ϵ₀ (E, K) = 0,  ∀ K ∈ (H¹(Ω))ᵈ
+         a->AddDomainIntegrator(new DirectionalVectorDiffusionIntegrator(scaledb_cf), nullptr, i+3, i+3);
+         a->AddDomainIntegrator(new VectorMassIntegrator(*pw_c_coeffs[i]), nullptr, i+3, i+3);
+
+         // + i ω² ϵ₀ (E, I)                        
+         a->AddDomainIntegrator(nullptr, new VectorFEMassIntegrator(halfeps0omeg2), 0, i+3);                         
+                                 
+      }
+   }
+
+   a->Assemble();
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      delete pw_c_coeffs[i];
+      delete cPrbb_cf[i]; 
+      delete cPibb_cf[i]; 
+      delete signedcPrbb_cf[i]; 
+      delete signedcPibb_cf[i]; 
+   }
+
+   socketstream E_out_r;
+
+   int npfes = pfes.Size();
+   Array<int> offsets(npfes+1);  offsets[0] = 0;
+   Array<int> toffsets(npfes+1); toffsets[0] = 0;
+   for (int i = 0; i<npfes; i++)
+   {
+      offsets[i+1] = pfes[i]->GetVSize();
+      toffsets[i+1] = pfes[i]->TrueVSize();
+   }
+   offsets.PartialSum();
+   toffsets.PartialSum();
+
+   Vector x(2*offsets.Last());
+   x = 0.;
+
+   Array<ParGridFunction *> pgf_r(npfes);
+   Array<ParGridFunction *> pgf_i(npfes);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i] = new ParGridFunction(pfes[i], x, offsets[i]);
+      pgf_i[i] = new ParGridFunction(pfes[i], x, offsets.Last() + offsets[i]);
+   }
+
+   ParComplexGridFunction maxwell_pgf(pfes[0]); maxwell_pgf = 0.0;
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+   ParGridFunction E_par_r(&L2_fes);
+   ParGridFunction E_par_i(&L2_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   std::string output_dir = "ParaView/FEM/" + GetTimestamp();
+
+   if (paraview)
+   {
+      if (Mpi::Root()) { WriteParametersToFile(args, output_dir); }
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",pgf_r[0]);
+      paraview_dc->RegisterField("E_i",pgf_i[0]);
+      paraview_dc->RegisterField("MaxwellE_r",&maxwell_pgf.real());
+      paraview_dc->RegisterField("MaxwellE_i",&maxwell_pgf.imag());
+      paraview_dc->RegisterField("E_par_r",&E_par_r);
+      paraview_dc->RegisterField("E_par_i",&E_par_i);
+      if (eld)
+      {
+         paraview_dc->RegisterField("Jh_1_r",pgf_r[1]);
+         paraview_dc->RegisterField("Jh_1_i",pgf_i[1]);
+         paraview_dc->RegisterField("Jh_2_r",pgf_r[2]);
+         paraview_dc->RegisterField("Jh_2_i",pgf_i[2]);
+         paraview_dc->RegisterField("Jh_3_r",pgf_r[3]);
+         paraview_dc->RegisterField("Jh_3_i",pgf_i[3]);
+         paraview_dc->RegisterField("Jh_4_r",pgf_r[4]);
+         paraview_dc->RegisterField("Jh_4_i",pgf_i[4]);
+      }
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listJ;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+
+      pfes[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      for (int i = 0; i<2*ndiffusionequations;i++)
+      {
+         ess_tdof_listJ.SetSize(0);
+         pfes[i+1]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJ);
+         for (int j = 0; j < ess_tdof_listJ.Size(); j++)
+         {
+            ess_tdof_listJ[j] += toffsets[i+1];
+         }
+         ess_tdof_list.Append(ess_tdof_listJ);
+      }
+
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+
+
+   pgf_r[0]->ProjectBdrCoefficientTangent(one_x_cf, one_r_bdr);
+   pgf_r[0]->ProjectBdrCoefficientTangent(negone_x_cf, negone_r_bdr);
+   pgf_i[0]->ProjectBdrCoefficientTangent(one_x_cf, one_i_bdr);
+   pgf_i[0]->ProjectBdrCoefficientTangent(negone_x_cf, negone_i_bdr);
+
+   OperatorPtr Ah;
+   Vector B, X;
+
+   Vector b(x.Size()); b = 0.0;
+
+   a->FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);
+   ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+
+   BlockOperator * Block_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+   BlockOperator * Block_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+   int nblocks = Block_r->NumRowBlocks();
+   Array2D<const HypreParMatrix*> r_matrices(nblocks, nblocks);
+   Array2D<const HypreParMatrix*> i_matrices(nblocks, nblocks);
+   for (int i = 0; i < nblocks; i++)
+   {
+      for (int j = 0; j < nblocks; j++)
+      {
+         r_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&Block_r->GetBlock(i,j));
+         i_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&Block_i->GetBlock(i,j));
+      }
+   }
+   HypreParMatrix * hr = HypreParMatrixFromBlocks(r_matrices);
+   HypreParMatrix * hi = HypreParMatrixFromBlocks(i_matrices);
+
+   ComplexHypreParMatrix * hc_hypre =
+      new ComplexHypreParMatrix(hr, hi,false, false);
+
+
+   auto P = new ComplexMUMPSSolver(MPI_COMM_WORLD);
+   P->SetPrintLevel(1);
+   P->SetOperator(*hc_hypre);
+   P->Mult(B, X);
+
+   a->RecoverFEMSolution(X, x);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i]->MakeRef(pfes[i], x, offsets[i]);
+      pgf_i[i]->MakeRef(pfes[i], x, offsets.Last() + offsets[i]);
+   }
+
+   ParallelECoefficient par_e_r(pgf_r[0]);
+   ParallelECoefficient par_e_i(pgf_i[0]);
+   E_par_r.ProjectCoefficient(par_e_r);
+   E_par_i.ProjectCoefficient(par_e_i);
+
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, *pgf_r[0],
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   }
+
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((real_t)0);
+      paraview_dc->Save();
+      delete paraview_dc;
+   }
+
+   delete a;
+   for (int i = 0; i < fecols.Size(); ++i)
+   {
+      delete fecols[i];
+      delete pfes[i];
+      delete pgf_r[i];
+      delete pgf_i[i];
+   }
+
+   return 0;
+
+}
+
+

miniapps/dpg/plasma/lh-eld-fem.cpp

@@ -0,0 +1,705 @@
+//                   MFEM FEM parallel example
+//
+
+// Electron Landau Damping
+// Strong formulation:
+//    ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵᵣ E + i ω²ϵ₀(J₁ + J₂) = 0,   in Ω
+//                   - Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥ = 0,   in Ω     
+//                   - Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥ = 0,   in Ω 
+//                                           E×n = E₀,  on ∂Ω
+//                                           J₁  = 0,  on ∂Ω
+//                                           J₂  = 0,  on ∂Ω
+// weak formulation:
+//   Find E ∈ H(curl,Ω), J₁  ∈ H¹(Ω), J₂  ∈ H¹(Ω) such that
+//   (1/μ₀ ∇×E, ∇ × F) - ω² ϵ₀ (ϵᵣ E, F) + i ω²ϵ₀(J₁ + J₂, F) = 0,  ∀ F ∈ H(curl,Ω)
+//        ( (b⋅∇)J₁ , (b⋅∇) G) + c₁ (J₁ , G) - c₁ (P(r) (b ⊗ b) E, G) = 0,  ∀ G ∈ (H¹(Ω))ᵈ 
+//        ( (b⋅∇)J₂ , (b⋅∇) H) + c₂ (J₂ , H) + c₂ (P(r) (b ⊗ b) E, H) = 0,  ∀ H ∈ (H¹(Ω))ᵈ
+
+// mpirun -np 8 ./lh-eld-fem -o 4 -paraview -eld -m data/quad.msh 
+
+#include "mfem.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../util/pcomplexblockform.hpp"
+#include "../util/blockcomplexhypremat.hpp"
+#include "../util/utils.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "utils/lh_utils.hpp"
+#include "../../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+#include <cstring>
+#include <filesystem>
+
+
+#ifndef MFEM_USE_COMPLEX_MUMPS
+MFEM_ABORT("This example requires MFEM to be built with ComplexMUMPS.");
+#endif
+
+
+using namespace std;
+using namespace mfem;
+
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+
+   int order = 2;
+   int par_ref_levels = 0;
+   int ser_ref_levels = 0;
+   bool visualization = false;
+   // real_t rnum=4.6e9;
+   // real_t mu = 1.257e-6;
+   // real_t eps0 = 8.8541878128e-12*factor;
+   real_t rnum=1.5e9;
+   real_t mu = 1.257e-6;
+   real_t eps0 = 8.8541878128e-12;
+   bool eld = true; // enable/disable electron Landau damping 
+
+   bool paraview = false;
+   bool debug = false;
+   bool monolithic_precond = true;
+   bool direct_solve = false;
+   bool triangular_precond = false;
+   bool use_amg = false;
+   real_t delta_prec = 0.01;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&ser_ref_levels, "-sr", "--serial-refinement_levels",
+                  "Number of serial refinement levels.");                  
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&a0, "-a0", "--a0", "P(r) first parameter.");
+   args.AddOption(&a1, "-a1", "--a1", "P(r) second parameter.");
+   args.AddOption(&delta_prec, "-dp", "--delta-prec", "stability parameter for the preconditioner.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&eld, "-eld", "--eld", "-no-eld",
+                  "--no-eld",
+                  "Enable or disable electron Landau damping.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.AddOption(&direct_solve, "-direct", "--direct", "-no-direct",
+                  "--no-direct",
+                  "Enable or disable direct monolithic solver.");
+   args.AddOption(&monolithic_precond, "-monolithic", "--monolithic", "-no-monolithic",
+                  "--no-monolithic",
+                  "Enable or disable monolithic preconditioner.");
+   args.AddOption(&triangular_precond, "-triangular", "--triangular", "-no-triangular",
+                  "--no-triangular",
+                  "Enable or disable lower triangular preconditioner.");
+   args.AddOption(&use_amg, "-amg", "--amg", "-no-amg",
+                  "--no-amg",
+                  "Enable or disable AMG.");
+   args.AddOption(&debug, "-debug", "--debug", "-no-debug",
+                  "--no-debug",
+                  "Enable or disable debug mode (delta = 0.01 and no coupling).");                  
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   
+   // number of diffusion equations
+   int ndiffusionequations = (eld) ?  2 : 0; 
+
+   Vector cvals(ndiffusionequations);
+   Vector csigns(ndiffusionequations);
+   real_t cfactor = 1e-6;
+   if (eld)
+   {
+      cvals(0)  = 25e6;  cvals(1)  = 1e6;
+      csigns(0) = -1.0;  csigns(1) = 1.0;
+   }
+   cvals *= cfactor; // scale the coefficients
+   
+   real_t omega = 2.*M_PI*rnum;
+   if (eld && !debug) 
+   {
+      delta = 0.0; // disable delta if electron Landau damping is enabled
+      if (Mpi::Root())
+      {
+         cout << "Electron Landau damping enabled, delta set to 0.0." << endl;
+      }
+   }   
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   for (int i = 0; i < ser_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   mesh.RemoveInternalBoundaries();
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+   
+   for (int i = 0; i<par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }
+
+   // Define the coefficients 
+   ConstantCoefficient muinv(1./mu);
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_vcf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   // S(r) 
+   FunctionCoefficient S_cf_r(sfunc_r), S_cf_i(sfunc_i);
+   // P(r) 
+   FunctionCoefficient P_cf_r(pfunc_r), P_cf_i(pfunc_i); 
+
+   VectorFunctionCoefficient b_cf(dim,bfunc);// b
+   ScalarVectorProductCoefficient scaledb_cf(sqrt(cfactor), b_cf); 
+   MatrixFunctionCoefficient bb_cf(dim,bcrossb); // b⊗b
+
+   MatrixSumCoefficient oneminusbb(Mone_cf, bb_cf, 1.0, -1.0); // 1 - b⊗b
+
+   // S(r) (I - b⊗b)
+   ScalarMatrixProductCoefficient Soneminusbb_r(S_cf_r, oneminusbb), Soneminusbb_i(S_cf_i, oneminusbb); 
+
+   // P(r) b⊗b 
+   ScalarMatrixProductCoefficient P_cf_bb_r(P_cf_r, bb_cf), P_cf_bb_i(P_cf_i, bb_cf); 
+
+   // εᵣ = S(r) (I - b⊗b) + P(r) b⊗b 
+   MatrixSumCoefficient eps_r(Soneminusbb_r, P_cf_bb_r, 1.0, 1.0); 
+   MatrixSumCoefficient eps_i(Soneminusbb_i, P_cf_bb_i, 1.0, 1.0); 
+
+   coefs_r[nattr-1] = &eps_r;
+   coefs_i[nattr-1] = &eps_i;
+
+   PWMatrixCoefficient eps_cf_r(dim, attr, coefs_r);
+   PWMatrixCoefficient eps_cf_i(dim, attr, coefs_i);
+
+   real_t scale = (debug) ? 0.0 : 1.0;
+   ConstantCoefficient eps0omeg2(eps0 * omega * omega * scale);
+   ConstantCoefficient negeps0omeg2(-eps0 * omega * omega);
+
+   ScalarMatrixProductCoefficient m_cf_r(negeps0omeg2, eps_cf_r);
+   ScalarMatrixProductCoefficient m_cf_i(negeps0omeg2, eps_cf_i);
+
+   // if ELD
+   Array<Vector *> c_arrays(ndiffusionequations);
+   Array<PWConstCoefficient *> pw_c_coeffs(ndiffusionequations);
+   Array<MatrixCoefficient *> cPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> cPibb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPibb_cf(ndiffusionequations);
+   Vector temp(nattr); temp=0.0;
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      temp[nattr-1] = cvals(i);
+      pw_c_coeffs[i] = new PWConstCoefficient(temp);
+      cPrbb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_r);
+      cPibb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_i);
+      signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPrbb_cf[i]);
+      signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPibb_cf[i]);
+   }
+
+   Array<FiniteElementCollection *> fecols;
+   Array<ParFiniteElementSpace *> pfes;
+   fecols.Append(new ND_FECollection(order, dim));
+   pfes.Append(new ParFiniteElementSpace(&pmesh, fecols[0]));
+   if (eld)
+   {
+      for (int i = 0; i < ndiffusionequations; ++i)
+      {
+         fecols.Append(new H1_FECollection(order, dim));
+         pfes.Append(new ParFiniteElementSpace(&pmesh, fecols[i+1], dim));
+      }
+   }
+
+   Array<HYPRE_BigInt> tdofs(pfes.Size());
+   for (int i = 0; i < pfes.Size(); ++i)
+   {
+      tdofs[i] = pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << "ParFiniteElementSpace " << i << " has " << tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   if (Mpi::Root())
+   {
+      cout << "Total number of true dofs: " << tdofs.Sum() << endl;
+   }
+
+   ParComplexBlockForm *a = new ParComplexBlockForm(pfes);
+   // (1/μ₀ ∇×E, ∇ × F)
+   a->AddDomainIntegrator(new CurlCurlIntegrator(muinv), nullptr, 0, 0);
+   // - ω² ϵ₀ (ϵᵣ E, F)
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(m_cf_r),
+                          new VectorFEMassIntegrator(m_cf_i), 0, 0);
+   if (eld)
+   {
+      for (int i = 0; i<ndiffusionequations; i++)
+      {
+         //  i ω²ϵ₀((J₁+J₂),F)
+         a->AddDomainIntegrator(nullptr, new TransposeIntegrator(new VectorFEMassIntegrator(eps0omeg2)), i+1, 0);
+         //  ( (b⋅∇)J₁ , (b⋅∇) G)
+         a->AddDomainIntegrator(new DirectionalVectorDiffusionIntegrator(scaledb_cf), nullptr, i+1, i+1);
+         // cᵢ (J₁ , G)
+         a->AddDomainIntegrator(new VectorMassIntegrator(*pw_c_coeffs[i]), nullptr, i+1, i+1);
+         // ±cᵢ(P(r) (b ⊗ b) E, G)
+         a->AddDomainIntegrator(new VectorFEMassIntegrator(*signedcPrbb_cf[i]), new VectorFEMassIntegrator(*signedcPibb_cf[i]), 0, i+1);
+      }
+   }
+
+   a->Assemble();
+
+   socketstream E_out_r;
+
+   int npfes = pfes.Size();
+   Array<int> offsets(npfes+1);  offsets[0] = 0;
+   Array<int> toffsets(npfes+1); toffsets[0] = 0;
+   for (int i = 0; i<npfes; i++)
+   {
+      offsets[i+1] = pfes[i]->GetVSize();
+      toffsets[i+1] = pfes[i]->TrueVSize();
+   }
+   offsets.PartialSum();
+   toffsets.PartialSum();
+
+   Vector x(2*offsets.Last());
+   x = 0.;
+
+   Array<ParGridFunction *> pgf_r(npfes);
+   Array<ParGridFunction *> pgf_i(npfes);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i] = new ParGridFunction(pfes[i], x, offsets[i]);
+      pgf_i[i] = new ParGridFunction(pfes[i], x, offsets.Last() + offsets[i]);
+   }
+
+   ParComplexGridFunction maxwell_pgf(pfes[0]); maxwell_pgf = 0.0;
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+   ParGridFunction E_par_r(&L2_fes);
+   ParGridFunction E_par_i(&L2_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   std::string output_dir = "ParaView/FEM/" + GetTimestamp();
+
+   if (paraview)
+   {
+      if (Mpi::Root()) { WriteParametersToFile(args, output_dir); }
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",pgf_r[0]);
+      paraview_dc->RegisterField("E_i",pgf_i[0]);
+      paraview_dc->RegisterField("MaxwellE_r",&maxwell_pgf.real());
+      paraview_dc->RegisterField("MaxwellE_i",&maxwell_pgf.imag());
+      paraview_dc->RegisterField("E_par_r",&E_par_r);
+      paraview_dc->RegisterField("E_par_i",&E_par_i);
+      if (eld)
+      {
+         paraview_dc->RegisterField("Jh_1_r",pgf_r[1]);
+         paraview_dc->RegisterField("Jh_1_i",pgf_i[1]);
+         paraview_dc->RegisterField("Jh_2_r",pgf_r[2]);
+         paraview_dc->RegisterField("Jh_2_i",pgf_i[2]);
+      }
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listJ;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+
+      pfes[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      for (int i = 0; i<ndiffusionequations;i++)
+      {
+         ess_tdof_listJ.SetSize(0);
+         pfes[i+1]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJ);
+         for (int j = 0; j < ess_tdof_listJ.Size(); j++)
+         {
+            ess_tdof_listJ[j] += toffsets[i+1];
+         }
+         ess_tdof_list.Append(ess_tdof_listJ);
+      }
+
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+
+
+   pgf_r[0]->ProjectBdrCoefficientTangent(one_x_cf, one_r_bdr);
+   pgf_r[0]->ProjectBdrCoefficientTangent(negone_x_cf, negone_r_bdr);
+   pgf_i[0]->ProjectBdrCoefficientTangent(one_x_cf, one_i_bdr);
+   pgf_i[0]->ProjectBdrCoefficientTangent(negone_x_cf, negone_i_bdr);
+
+   OperatorPtr Ah;
+   Vector B, X;
+
+   Vector b(x.Size()); b = 0.0;
+
+   a->FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);
+   ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+
+   if (direct_solve)
+   {
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+      int nblocks = BlockA_r->NumRowBlocks();
+      Array2D<const HypreParMatrix*> A_r_matrices(nblocks, nblocks);
+      Array2D<const HypreParMatrix*> A_i_matrices(nblocks, nblocks);
+      for (int i = 0; i < nblocks; i++)
+      {
+         for (int j = 0; j < nblocks; j++)
+         {
+            A_r_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockA_r->GetBlock(i,j));
+            A_i_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockA_i->GetBlock(i,j));
+         }
+      }
+      HypreParMatrix * Ahr = HypreParMatrixFromBlocks(A_r_matrices);
+      HypreParMatrix * Ahi = HypreParMatrixFromBlocks(A_i_matrices);
+
+      ComplexHypreParMatrix * Ahc_hypre =
+         new ComplexHypreParMatrix(Ahr, Ahi,true, true);
+
+      ComplexMUMPSSolver cmumps(MPI_COMM_WORLD);
+      cmumps.SetPrintLevel(0);
+      cmumps.SetOperator(*Ahc_hypre);
+      cmumps.Mult(B,X);
+      delete Ahc_hypre;   
+   }
+   else 
+   {
+      // Set up the preconditioner
+      delta = delta_prec;
+      ParComplexBlockForm a_prec(pfes);
+      // (1/μ₀ ∇×E, ∇ × F)
+      a_prec.AddDomainIntegrator(new CurlCurlIntegrator(muinv), nullptr, 0, 0);
+      // - ω² ϵ₀ (ϵᵣ E, F)
+      a_prec.AddDomainIntegrator(new VectorFEMassIntegrator(m_cf_r),
+                                new VectorFEMassIntegrator(m_cf_i), 0, 0);
+      if (eld)
+      {
+         for (int i = 0; i<ndiffusionequations; i++)
+         {
+            //  ( (b⋅∇)J₁ , (b⋅∇) G)
+            a_prec.AddDomainIntegrator(new DirectionalVectorDiffusionIntegrator(scaledb_cf), nullptr, i+1, i+1);
+            // cᵢ (J₁ , G)
+            a_prec.AddDomainIntegrator(new VectorMassIntegrator(*pw_c_coeffs[i]), nullptr, i+1, i+1);
+            // ±cᵢ(P(r) (b ⊗ b) E, G)
+            a_prec.AddDomainIntegrator(new VectorFEMassIntegrator(*signedcPrbb_cf[i]), new VectorFEMassIntegrator(*signedcPibb_cf[i]), 0, i+1);
+         }
+      }
+      a_prec.Assemble();
+
+      OperatorPtr Ahprec;
+      a_prec.FormSystemMatrix(ess_tdof_list, Ahprec);
+      ComplexOperator * Ahcprec = Ahprec.As<ComplexOperator>();
+
+      GMRESSolver gmres(MPI_COMM_WORLD);
+      gmres.SetRelTol(1e-10);
+      gmres.SetMaxIter(2000);
+      gmres.SetPrintLevel(1);
+
+      Array<int> blk(3);
+      if (monolithic_precond)
+      {
+         BlockOperator * BlockPrec_r = dynamic_cast<BlockOperator *>(&Ahcprec->real());
+         BlockOperator * BlockPrec_i = dynamic_cast<BlockOperator *>(&Ahcprec->imag());
+
+         int nblocks = BlockPrec_r->NumRowBlocks();
+         Array2D<const HypreParMatrix*> prec_r_matrices(nblocks, nblocks);
+         Array2D<const HypreParMatrix*> prec_i_matrices(nblocks, nblocks);
+         for (int i = 0; i < nblocks; i++)
+         {
+            for (int j = 0; j < nblocks; j++)
+            {
+               prec_r_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockPrec_r->GetBlock(i,j));
+               prec_i_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockPrec_i->GetBlock(i,j));
+            }
+         }
+         HypreParMatrix * Prechr = HypreParMatrixFromBlocks(prec_r_matrices);
+         HypreParMatrix * Prechi = HypreParMatrixFromBlocks(prec_i_matrices);
+
+         ComplexHypreParMatrix * Prechc_hypre =
+            new ComplexHypreParMatrix(Prechr, Prechi,true, true);
+
+         ComplexMUMPSSolver cmumps(MPI_COMM_WORLD);
+         cmumps.SetPrintLevel(0);
+         cmumps.SetOperator(*Prechc_hypre);
+
+         gmres.SetOperator(*Ahc);
+         gmres.SetPreconditioner(cmumps);
+         gmres.Mult(B, X);
+         delete Prechc_hypre;
+      }
+      else   
+      {
+         ComplexBlockOperator Ac(*Ahc);
+         ComplexBlockOperator Acprec(*Ahcprec);
+
+         Vector Xc(X.Size()); Xc = 0.0;
+         Vector Bc(B.Size());
+         Ac.BlockComplexToComplexBlock(B, Bc);
+         
+         Solver * Mc = nullptr;   
+         int nblocks = Ac.NumRowBlocks();
+
+         Array<Solver *> diag_solvers(nblocks);
+         diag_solvers[0] = new ComplexMUMPSSolver(MPI_COMM_WORLD);
+         dynamic_cast<ComplexMUMPSSolver*>(diag_solvers[0])->SetPrintLevel(0);
+         diag_solvers[0]->SetOperator(Acprec.GetBlock(0,0));
+
+         for (int i = 1; i < nblocks; ++i)
+         {
+            if (use_amg)
+            {
+               HypreBoomerAMG * amg1 = new HypreBoomerAMG();
+               HypreBoomerAMG * amg2 = new HypreBoomerAMG();
+               amg1->SetPrintLevel(0); amg2->SetPrintLevel(0);
+               amg1->SetSystemsOptions(dim); amg2->SetSystemsOptions(dim);
+               amg1->SetRelaxType(88); amg2->SetRelaxType(88);
+               auto op = dynamic_cast<ComplexHypreParMatrix*>(&Acprec.GetBlock(i,i));
+               blk[0]=0;
+               blk[1]=op->real().Height();
+               blk[2]=op->real().Height();
+               blk.PartialSum();
+               BlockDiagonalPreconditioner * bprec = new BlockDiagonalPreconditioner(blk);
+               bprec->owns_blocks=1;
+               amg1->SetOperator(op->real());
+               amg2->SetOperator(op->real());
+               bprec->SetDiagonalBlock(0, amg1);
+               bprec->SetDiagonalBlock(1, amg2);
+               diag_solvers[i] = bprec;
+            }
+            else
+            {
+               diag_solvers[i] = new ComplexMUMPSSolver(MPI_COMM_WORLD);
+               dynamic_cast<ComplexMUMPSSolver*>(diag_solvers[i])->SetPrintLevel(0);
+               diag_solvers[i]->SetOperator(Acprec.GetBlock(i,i));
+            }  
+         }
+
+         if (triangular_precond)
+         {
+            Mc = new BlockLowerTriangularPreconditioner(Ac.RowOffsets());
+            auto Mclt = dynamic_cast<BlockLowerTriangularPreconditioner*>(Mc);
+            Mclt->SetBlock(1,0, &Acprec.GetBlock(1,0));
+            Mclt->SetBlock(2,0, &Acprec.GetBlock(2,0));
+            for (int i = 0; i < nblocks; ++i) { Mclt->SetDiagonalBlock(i, diag_solvers[i]); }
+         }
+         else
+         {
+            Mc = new BlockDiagonalPreconditioner(Ac.RowOffsets());
+            auto Mcdiag = dynamic_cast<BlockDiagonalPreconditioner*>(Mc);
+            for (int i = 0; i < nblocks; ++i) { Mcdiag->SetDiagonalBlock(i, diag_solvers[i]); }
+         }   
+            
+         gmres.SetPreconditioner(*Mc);
+         gmres.SetOperator(Ac);
+         gmres.Mult(Bc, Xc);
+
+
+         for (int i = 0; i < nblocks; ++i)
+         {
+            delete diag_solvers[i];
+         }
+         delete Mc;
+
+         Ac.ComplexBlockToBlockComplex(Xc, X);
+      }
+   }
+   
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      delete pw_c_coeffs[i];
+      delete cPrbb_cf[i]; 
+      delete cPibb_cf[i]; 
+      delete signedcPrbb_cf[i]; 
+      delete signedcPibb_cf[i]; 
+   }
+
+
+
+   a->RecoverFEMSolution(X, x);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i]->MakeRef(pfes[i], x, offsets[i]);
+      pgf_i[i]->MakeRef(pfes[i], x, offsets.Last() + offsets[i]);
+   }
+
+   ParallelECoefficient par_e_r(pgf_r[0]);
+   ParallelECoefficient par_e_i(pgf_i[0]);
+   E_par_r.ProjectCoefficient(par_e_r);
+   E_par_i.ProjectCoefficient(par_e_i);
+
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, *pgf_r[0],
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   }
+
+
+   delta = 0.0;
+   ParSesquilinearForm a_maxwell(pfes[0]);
+   // (1/μ₀ ∇×E, ∇ × F)
+   a_maxwell.AddDomainIntegrator(new CurlCurlIntegrator(muinv), nullptr);
+   // - ω² ϵ₀ (ϵᵣ E, F)
+   a_maxwell.AddDomainIntegrator(new VectorFEMassIntegrator(m_cf_r),
+                        new VectorFEMassIntegrator(m_cf_i));
+   a_maxwell.Assemble();                          
+
+
+   VectorGridFunctionCoefficient J1_cf_r(pgf_r[1]);
+   VectorGridFunctionCoefficient J1_cf_i(pgf_i[1]);
+   VectorGridFunctionCoefficient J2_cf_r(pgf_r[2]);
+   VectorGridFunctionCoefficient J2_cf_i(pgf_i[2]);
+
+   // -iω²ϵ₀ (Jᵣ + i Jᵢ ,F) = ω² ϵ₀ (Jᵢ - i Jᵣ,F)   
+   //                       = (ω² ϵ₀ Jᵢ, F) + i (-ω² ϵ₀Jᵢ,F)
+
+
+   ScalarVectorProductCoefficient omeg2_eps0_J1_cf_i(eps0*omega*omega, J1_cf_i);
+   ScalarVectorProductCoefficient omeg2_eps0_J2_cf_i(eps0*omega*omega, J2_cf_i);
+
+   ScalarVectorProductCoefficient negomeg2_eps0_J1_cf_r(-eps0*omega*omega, J1_cf_r);
+   ScalarVectorProductCoefficient negomeg2_eps0_J2_cf_r(-eps0*omega*omega, J2_cf_r);
+   
+   ParComplexLinearForm b_maxwell(pfes[0]);
+   b_maxwell.AddDomainIntegrator(new VectorFEDomainLFIntegrator(omeg2_eps0_J1_cf_i),
+                                 new VectorFEDomainLFIntegrator(negomeg2_eps0_J1_cf_r));
+   b_maxwell.AddDomainIntegrator(new VectorFEDomainLFIntegrator(omeg2_eps0_J2_cf_i),
+                                 new VectorFEDomainLFIntegrator(negomeg2_eps0_J2_cf_r));
+
+   b_maxwell.Assemble();
+
+
+   // remove internal boundaries
+   ess_bdr = 1;
+   // for (int i = 0; i<int_bdr_attr.Size(); i++)
+   // {
+   //    ess_bdr[int_bdr_attr[i]-1] = 0;
+   // }
+
+   pfes[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   maxwell_pgf.ProjectBdrCoefficientTangent(one_x_cf, zero_vcf, one_r_bdr);
+   maxwell_pgf.ProjectBdrCoefficientTangent(negone_x_cf, zero_vcf, negone_r_bdr);
+   maxwell_pgf.ProjectBdrCoefficientTangent(zero_vcf, one_x_cf, one_i_bdr);
+   maxwell_pgf.ProjectBdrCoefficientTangent(zero_vcf, negone_x_cf, negone_i_bdr);
+
+
+   OperatorPtr maxwell_Ah;
+   Vector maxwell_X,maxwell_B;
+   a_maxwell.FormLinearSystem(ess_tdof_list,maxwell_pgf,b_maxwell,maxwell_Ah, maxwell_X,maxwell_B);
+
+   ComplexHypreParMatrix *Maxwell_A = maxwell_Ah.As<ComplexHypreParMatrix>();
+   ComplexMUMPSSolver cmumps(MPI_COMM_WORLD);
+   cmumps.SetPrintLevel(0);
+   cmumps.SetOperator(*Maxwell_A);
+   cmumps.Mult(maxwell_B, maxwell_X);
+
+   a_maxwell.RecoverFEMSolution(maxwell_X, maxwell_B, maxwell_pgf);
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((real_t)0);
+      paraview_dc->Save();
+      delete paraview_dc;
+   }
+
+   delete a;
+   for (int i = 0; i < fecols.Size(); ++i)
+   {
+      delete fecols[i];
+      delete pfes[i];
+      delete pgf_r[i];
+      delete pgf_i[i];
+   }
+
+   return 0;
+
+}
+
+

miniapps/dpg/plasma/lh-eld-fosls-dpg.cpp

@@ -0,0 +1,669 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//                   MFEM Ultraweak DPG Maxwell parallel example
+//
+// Compile with: make lh-eld-fosls-dpg
+//
+// mpirun -np 8 ./lh-eld-fosls-dpg -o 4 -paraview -eld -m data/quad.msh -ebs -sc
+
+// Electron Landau Damping
+// Strong formulation:
+//     ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵ E + i ω²ϵ₀(J₁ + J₂) = 0,   in Ω
+//               λ₁   (- Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥) = 0,   in Ω     
+//               λ₂   (- Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥) = 0,   in Ω 
+//                                            E×n = E₀,  on ∂Ω
+//                                            J₁  = 0,  on ∂Ω
+//                                            J₂  = 0,  on ∂Ω
+// The DPG-FOSLS deals with the First Order System
+//  i ω μ₀ H  + ∇ × E                  = 0,   in Ω
+// -i ω ϵ₀ϵ E + ∇ × H - ω ϵ₀ (J₁ + J₂) = 0,   in Ω
+// - λ₁ b⋅∇Q₁ + λ₁ c₁ J₁ - λ₁ c₁ P B E = 0,   in Ω     
+//                   λ₁ Q₁ + λ₁ b⋅∇ J₁ = 0,   in Ω
+// - λ₂ b⋅∇Q₂ + λ₂ c₂ J₂ + λ₂ c₂ P B E = 0,   in Ω
+//                   λ₂ Q₂ + λ₂ b⋅∇ J₂ = 0,   in Ω
+//                                 E×n = E₀,  on ∂Ω
+//                                 J₁  = 0,  on ∂Ω
+//                                 J₂  = 0,  on ∂Ω
+
+
+// in 2D  E is vector valued and H is scalar and
+//     ∇ × E = ∇ ⋅ AE  where A = [0 1; -1 0];
+
+// E ∈ H(curl,Ω) , H ∈ H¹(Ω), Jᵢ, Qᵢ ∈ (H¹(Ω))²
+//  ( iωμ₀ H, F ) + ( ∇ × E, F) = 0,                      ∀ F  ∈  L²(Ω)
+// -(iωϵ₀ϵ E, R ) + ( ∇ × H, R) - ω ϵ₀ (J₁ + J₂, R) = 0,  ∀ R  ∈ (L²(Ω))²
+// λ₁ (b ⋅ ∇Q₁, K₁) + λ₁ c₁(J₁, K₁) - λ₁ c₁ (P B E, K₁) = 0,      ∀ K₁ ∈ (L²(Ω))²
+// λ₁ (Q₁, L₁) + λ₁ (b ⋅ ∇ J₁, L₁) = 0,                        ∀ L₁ ∈ (L²(Ω))²
+// λ₂ (b ⋅ ∇Q₂, K₂) + λ₂ c₂(J₂, K₂) + λ₂ c₂ (P B E, K₂) = 0,      ∀ K₂ ∈ (L²(Ω))²
+// λ₂ (Q₂, L₂) + λ₂ (b ⋅ ∇ J₂, L₂) = 0,                        ∀ L₂ ∈ (L²(Ω))²
+//                                    E = E₀, on ∂Ω
+//                              J₁ = J₂ = 0,  on ∂Ω
+// ----------------------------------------------------------------------------------------------------------------
+// |   |      E       |      H    |     J₁      |     Q₁     |       J₂   |      Q₂    |   RHS  |
+// ----------------------------------------------------------------------------------------------------------------
+// | F |  (∇ × E, F)  | (iωμ₀H,F) |             |            |            |            |        |        |   0   |  
+// |   |              |           |             |            |            |            |        |        |       |  
+// | R | -iωϵ₀ϵ(E,R)  |(∇ × H, R) | -ωϵ₀(J₁,R)  |            | -ωϵ₀(J₂, R)|            |        |        |   0   |  
+// |   |              |           |             |            |            |            |        |        |       |  
+// |K₁ | -λ₁c₁(PBE,K₁)|           | λ₁c₁(J₁, K₁)|(λ₁b⋅∇Q₁,K₁)|            |            |        |        |   0   |  
+// |   |              |           |             |            |            |            |        |        |       |    
+// |L₁ |              |           | λ₁(b⋅∇J₁,L₁)| λ₁(Q₁, L₁) |            |            |        |        |   0   |  
+// |   |              |           |             |            |            |            |        |        |       |    
+// |K₂ | λ₂c₂(PBE, K₂)|           |             |            |λ₂c₂(J₂, K₂)|λ₂(b⋅∇Q₂,K₂)|        |        |   0   |  
+// |   |              |           |             |            |            |            |        |        |       |    
+// |L₂ |              |           |             |            |λ₂(b⋅∇J₂,L₂)| λ₂(Q₂, L₂) |        |        |   0   |  
+// |   |              |           |             |            |            |            |        |        |       |    
+// where (F,R,K₁,L₁,K₂,L₂) ∈  L²(Ω) × (L²(Ω))² × (L²(Ω))² × (L²(Ω))² × (L²(Ω))² × (L²(Ω))² 
+
+
+
+#include "mfem.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../../common/mfem-common.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "utils/lh_utils.hpp"
+#include "../util/utils.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+   int order = 1;
+   int delta_order = 1;
+   int par_ref_levels = 0;
+   int ser_ref_levels = 0;
+
+   // real_t rnum=1.5e9;
+   // real_t mu = 1.257e-6;
+   // real_t eps0 = 8.8541878128e-12;
+
+   real_t rnum=1.5;
+   real_t mu = 1.257;
+   real_t eps0 = 8.8541878128;
+   real_t balance_scale = 1.0;
+   bool enable_balance_scale = false;
+
+   bool eld = false; // enable/disable electron Landau damping 
+   real_t delta_prec = 0.0;
+   real_t lambda1 = 1.0;
+   real_t lambda2 = 1.0;
+
+   bool static_cond = false;
+   bool visualization = false;
+   bool paraview = false;
+   bool debug = false;
+   bool mumps_solver = false;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&delta_order, "-do", "--delta-order",
+                  "Finite element order for the test space");                  
+   args.AddOption(&ser_ref_levels, "-sr", "--serial-refinement_levels",
+                  "Number of serial refinement levels.");                  
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&a0, "-a0", "--a0", "P(r) first parameter.");
+   args.AddOption(&a1, "-a1", "--a1", "P(r) second parameter.");
+   args.AddOption(&lambda1, "-l1", "--lambda1", "Lambda 1 scaling parameter.");
+   args.AddOption(&lambda2, "-l2", "--lambda2", "Lambda 2 scaling parameter.");
+   args.AddOption(&delta_prec, "-dp", "--delta-prec", "stability parameter for the preconditioner.");
+   args.AddOption(&eld, "-eld", "--eld", "-no-eld",
+                  "--no-eld",
+                  "Enable or disable electron Landau damping.");
+   args.AddOption(&mumps_solver, "-mumps", "--mumps", "-no-mumps",
+                  "--no-mumps",
+                  "Enable or disable MUMPS solver.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&debug, "-debug", "--debug", "-no-debug",
+                  "--no-debug",
+                  "Enable or disable debug mode (delta = 0.01 and no coupling).");         
+   args.AddOption(&enable_balance_scale, "-ebs", "--enable-balance-scale", "-no-ebs",
+                  "--no-ebs",
+                  "Enable or disable balance scale.");
+
+
+   args.Parse();
+   if (!args.Good())
+   {
+      if (Mpi::Root())
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (Mpi::Root())
+   {
+      args.PrintOptions(cout);
+   }
+
+   // number of diffusion equations
+   int ndiffusionequations = (eld) ?  2 : 0; 
+
+   Vector cvals(ndiffusionequations);
+   Vector csigns(ndiffusionequations);
+   if (eld)
+   {
+      cvals(0)  = 25e6;  cvals(1)  = 1e6;
+      csigns(0) = -1.0;  csigns(1) = 1.0;
+   }
+   cvals(0) *= lambda1;
+   cvals(1) *= lambda2;
+   real_t omega = 2.*M_PI*rnum;
+   int test_order = order+delta_order;
+
+   balance_scale = (enable_balance_scale) ? eps0 * omega * omega : 1.0;
+
+   if (eld && !debug) 
+   {
+      delta = 0.0; // disable delta if electron Landau damping is enabled
+      if (Mpi::Root())
+      {
+         cout << "Electron Landau damping enabled, delta set to 0.0." << endl;
+      }
+   }    
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   MFEM_VERIFY(dim == 2, "Dimension != 2 is not supported in this example");
+
+   for (int i = 0; i < ser_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   Array<int> int_bdr_attr;
+   for (int i = 0; i < mesh.GetNBE(); i++)
+   {
+      if (mesh.FaceIsInterior(mesh.GetBdrElementFaceIndex(i)))
+      {
+         int_bdr_attr.Append(mesh.GetBdrAttribute(i));
+      }
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   for (int i = 0; i < par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }  
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+   real_t scale = (debug) ? 0.0 : 1.0;
+
+   // Define coefficients
+   ConstantCoefficient one_cf(1.0);
+   //  ωμ₀
+   ConstantCoefficient omegamu_cf(omega*mu);
+   // -ωϵ₀
+   ConstantCoefficient negomegeps0_cf(-omega*eps0 * scale);
+   ConstantCoefficient balancescaled_negomegeps0_cf( -omega*eps0/balance_scale * scale);
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_vcf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   // S(r) 
+   FunctionCoefficient S_cf_r(sfunc_r), S_cf_i(sfunc_i);
+   // P(r) 
+   FunctionCoefficient P_cf_r(pfunc_r), P_cf_i(pfunc_i); 
+
+   // b
+   VectorFunctionCoefficient b_cf(dim,bfunc);
+   ScalarVectorProductCoefficient scaled1_b_cf(sqrt(lambda1), b_cf);
+   ScalarVectorProductCoefficient scaled2_b_cf(sqrt(lambda2), b_cf);
+
+   // b⊗b
+   MatrixFunctionCoefficient bb_cf(dim,bcrossb); 
+   // I - b⊗b
+   MatrixSumCoefficient oneminusbb(Mone_cf, bb_cf, 1.0, -1.0); 
+   // S(r) (I - b⊗b)
+   ScalarMatrixProductCoefficient Soneminusbb_r(S_cf_r, oneminusbb), Soneminusbb_i(S_cf_i, oneminusbb); 
+   // P(r) b⊗b 
+   ScalarMatrixProductCoefficient P_cf_bb_r(P_cf_r, bb_cf), P_cf_bb_i(P_cf_i, bb_cf); 
+
+   // ε = S(r) (I - b⊗b) + P(r) b⊗b 
+   MatrixSumCoefficient eps_r(Soneminusbb_r, P_cf_bb_r, 1.0, 1.0); 
+   MatrixSumCoefficient eps_i(Soneminusbb_i, P_cf_bb_i, 1.0, 1.0); 
+
+   coefs_r[nattr-1] = &eps_r;
+   coefs_i[nattr-1] = &eps_i;
+
+   PWMatrixCoefficient eps_cf_r(dim, attr, coefs_r);
+   PWMatrixCoefficient eps_cf_i(dim, attr, coefs_i);
+
+   ConstantCoefficient eps0omeg(omega * eps0);
+   ConstantCoefficient negeps0omeg(-omega * eps0);
+
+
+   // - iωϵ₀ϵ = ωϵ₀ϵᵢ + i (-ωϵ₀ϵᵣ)
+   // ω ϵ₀ ϵᵢ
+   ScalarMatrixProductCoefficient eps0omeg_eps_i(eps0omeg, eps_cf_i);
+   // -ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient negeps0omeg_eps_r(negeps0omeg, eps_cf_r);
+
+   // if ELD
+   Array<Vector *> c_arrays(ndiffusionequations);
+   Array<PWConstCoefficient *> pw_c_coeffs(ndiffusionequations);
+   Array<MatrixCoefficient *> cPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> cPibb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPibb_cf(ndiffusionequations);
+
+   Array<MatrixCoefficient *> balancescaled_signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> balancescaled_signedcPibb_cf(ndiffusionequations);
+   Vector temp(nattr); temp=0.0;
+   Array<ConstantCoefficient *> c_coeffs(ndiffusionequations);
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      temp[nattr-1] = cvals(i);
+      // temp = cvals(i);
+      pw_c_coeffs[i] = new PWConstCoefficient(temp);
+      c_coeffs[i] = new ConstantCoefficient(cvals(i));
+      cPrbb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_r);
+      cPibb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_i);
+      signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPrbb_cf[i]);
+      signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPibb_cf[i]);
+      balancescaled_signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(balance_scale,*signedcPrbb_cf[i]);
+      balancescaled_signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(balance_scale,*signedcPibb_cf[i]);
+   }
+
+
+   std::vector<std::string> variables = {" E  ", " H  ", " J₁ ", " J₂ ", " Q₁ ", " Q₂ "};
+   std::vector<std::string> spaces = {"H(curl,Ω) ", "   H¹(Ω)  ", 
+                                      "  (H¹(Ω))ᵈ", "  (H¹(Ω))ᵈ", 
+                                      "  (H¹(Ω))ᵈ", "  (H¹(Ω))ᵈ"};
+
+   // Define the spaces
+   Array<FiniteElementCollection *> trial_fecols;
+   Array<FiniteElementCollection *> test_fecols;
+   Array<ParFiniteElementSpace *> pfes;
+
+   // H(curl) space for E
+   trial_fecols.Append(new ND_FECollection(order, dim));
+   pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last()));
+   // Scalar H¹ space for H
+   trial_fecols.Append(new H1_FECollection(order, dim));
+   pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last()));
+
+   // Vector H¹ spaces for Jᵢ and Qᵢ
+   for (int i = 0; i < 2*ndiffusionequations; i++)
+   {
+      trial_fecols.Append(new H1_FECollection(order, dim));
+      pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last(), dim));
+   }
+
+   Array<HYPRE_BigInt> tdofs(pfes.Size());
+   for (int i = 0; i < pfes.Size(); ++i)
+   {
+      tdofs[i] = pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << "ParFiniteElementSpace " << spaces[i] << "  for " << variables[i] << " has " << tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   if (Mpi::Root())
+   {
+      cout << "Total number of true dofs: " << tdofs.Sum() << endl;
+   }
+
+   // test spaces for F and G
+   test_fecols.Append(new L2_FECollection(test_order, dim));
+   test_fecols.Append(new L2_FECollection(test_order, dim));
+   // Test spaces Kᵢ and Lᵢ 
+   for (int i = 0; i < 2*ndiffusionequations; i++)
+   {
+      test_fecols.Append(new L2_FECollection(test_order, dim));
+   }
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(pfes,test_fecols);
+   for (int i = 0; i < test_fecols.Size(); i++)
+   {  // all but the first  space are vector valued
+      if (i > 0) { a->SetTestFECollVdim(i,dim); }
+   }
+
+   // (∇ × E, F)
+   a->AddTrialIntegrator(new MixedCurlIntegrator(one_cf), nullptr, 0, 0);
+
+   // (i ω μ₀ H, F)
+   a->AddTrialIntegrator(nullptr, new MixedScalarMassIntegrator(omegamu_cf), 1, 0);
+
+   // -i ω ϵ₀ϵ (E, R) = ω ϵ₀ ϵᵢ (E, G) + i (-ω ϵ₀ ϵᵣ E, R)
+   a->AddTrialIntegrator(new VectorFEMassIntegrator(eps0omeg_eps_i),
+                         new VectorFEMassIntegrator(negeps0omeg_eps_r), 0, 1);
+
+   // (∇ × H, R)
+   a->AddTrialIntegrator(new MixedCurlIntegrator(one_cf), nullptr, 1,1);
+   
+   if (eld)
+   {
+      // - ω ϵ₀ (J₁, R)
+      a->AddTrialIntegrator(new VectorMassIntegrator(balancescaled_negomegeps0_cf), nullptr, 2, 1);
+      // - ω ϵ₀ (J₂, R)
+      a->AddTrialIntegrator(new VectorMassIntegrator(balancescaled_negomegeps0_cf), nullptr, 3, 1);
+      // -c₁(PBE, K₁)
+      a->AddTrialIntegrator(new VectorFEMassIntegrator(*balancescaled_signedcPrbb_cf[0]),
+                            new VectorFEMassIntegrator(*balancescaled_signedcPibb_cf[0]), 0, 2);
+
+      // c₁(J₁, K₁)
+      a->AddTrialIntegrator(new VectorMassIntegrator(*pw_c_coeffs[0]), nullptr, 2, 2);
+
+      // (b⋅∇Q₁, K₁)
+      a->AddTrialIntegrator(new DirectionalVectorGradientIntegrator(scaled1_b_cf), nullptr, 3, 2);
+
+      // (b⋅∇J₁, L₁)
+      a->AddTrialIntegrator(new DirectionalVectorDiffusionIntegrator(scaled1_b_cf), nullptr, 2, 3);
+
+      // (Q₁, L₁)
+      a->AddTrialIntegrator(new VectorMassIntegrator(one_cf), nullptr, 3, 3);
+
+      // c₂(PBE, K₂)
+      a->AddTrialIntegrator(new VectorFEMassIntegrator(*balancescaled_signedcPrbb_cf[1]),
+                            new VectorFEMassIntegrator(*balancescaled_signedcPibb_cf[1]), 0, 4);
+
+      // c₂(J₂, K₂)
+      a->AddTrialIntegrator(new VectorMassIntegrator(*pw_c_coeffs[1]), nullptr, 4, 4);
+   
+      // (b⋅∇Q₂, K₂)
+      a->AddTrialIntegrator(new DirectionalVectorGradientIntegrator(scaled2_b_cf), nullptr, 5, 4);
+
+      // (b⋅∇J₂, L₂)
+      a->AddTrialIntegrator(new DirectionalVectorDiffusionIntegrator(scaled2_b_cf), nullptr, 4, 5);
+
+      // (Q₂, L₂)
+      a->AddTrialIntegrator(new VectorMassIntegrator(one_cf), nullptr, 5, 5);
+   }
+   // test integrators for test norm: ||v||² = ||F||² + ||R||² + ||K₁||² + ||L₁||² + ||K₂||² + ||L₂||² 
+   // (F,δF), F, δF ∈  L²(Ω)
+   a->AddTestIntegrator(new MassIntegrator(one_cf),nullptr, 0, 0);
+   // (R,δR), R, δR ∈ (L²(Ω))²
+   a->AddTestIntegrator(new VectorMassIntegrator(one_cf),nullptr, 1, 1);
+   
+   if (eld)
+   {
+      // (K₁,δK₁), K₁, δK₁ ∈ (L²(Ω))²
+      a->AddTestIntegrator(new VectorMassIntegrator(one_cf),nullptr, 2, 2);
+      // (L₁,δL₁), L₁, δL₁ ∈ (L²(Ω))²
+      a->AddTestIntegrator(new VectorMassIntegrator(one_cf),nullptr, 3, 3);
+      // (K₂,δK₂), K₂, δK₂ ∈ (L²(Ω))²
+      a->AddTestIntegrator(new VectorMassIntegrator(one_cf),nullptr, 4, 4);
+      // (L₂,δL₂), L₂, δL₂ ∈ (L²(Ω))²
+      a->AddTestIntegrator(new VectorMassIntegrator(one_cf),nullptr, 5, 5);        
+   }
+
+   a->Assemble(0);
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      delete pw_c_coeffs[i];
+      delete c_coeffs[i];
+      delete cPrbb_cf[i]; 
+      delete cPibb_cf[i]; 
+      delete signedcPrbb_cf[i]; 
+      delete signedcPibb_cf[i]; 
+   }
+
+   socketstream E_out_r;
+
+   int npfes = pfes.Size();
+   Array<int> offsets(npfes+1);  offsets[0] = 0;
+   Array<int> toffsets(npfes+1); toffsets[0] = 0;
+   for (int i = 0; i<npfes; i++)
+   {
+      offsets[i+1] = pfes[i]->GetVSize();
+      toffsets[i+1] = pfes[i]->TrueVSize();
+   }
+   offsets.PartialSum();
+   toffsets.PartialSum();
+
+   Vector x(2*offsets.Last());
+   x = 0.;
+   
+   Array<ParGridFunction *> pgf_r(npfes);
+   Array<ParGridFunction *> pgf_i(npfes);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i] = new ParGridFunction(pfes[i], x, offsets[i]);
+      pgf_i[i] = new ParGridFunction(pfes[i], x, offsets.Last() + offsets[i]);
+   }
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+   ParGridFunction E_par_r(&L2_fes);
+   ParGridFunction E_par_i(&L2_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   std::string output_dir = "ParaView/FOSLS/" + GetTimestamp();
+
+   if (paraview)
+   {
+      if (Mpi::Root()) { WriteParametersToFile(args, output_dir); }
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order
+                         << "_eld_" << eld;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",pgf_r[0]);
+      paraview_dc->RegisterField("E_i",pgf_i[0]);
+      paraview_dc->RegisterField("E_par_r",&E_par_r);
+      paraview_dc->RegisterField("E_par_i",&E_par_i);
+      paraview_dc->RegisterField("H_r",pgf_r[1]);
+      paraview_dc->RegisterField("H_i",pgf_i[1]);      
+      if (eld)
+      {
+         paraview_dc->RegisterField("J1_r",pgf_r[2]);
+         paraview_dc->RegisterField("J1_i",pgf_i[2]);
+         paraview_dc->RegisterField("Q1_r",pgf_r[3]);
+         paraview_dc->RegisterField("Q1_i",pgf_i[3]);
+         paraview_dc->RegisterField("J2_r",pgf_r[4]);
+         paraview_dc->RegisterField("J2_i",pgf_i[4]);
+         paraview_dc->RegisterField("Q2_r",pgf_r[5]);
+         paraview_dc->RegisterField("Q2_i",pgf_i[5]);
+      }
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listJ;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+   
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+
+      // remove internal boundaries
+      for (int i = 0; i<int_bdr_attr.Size(); i++)
+      {
+         ess_bdr[int_bdr_attr[i]-1] = 0;
+      }
+
+      pfes[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      ess_bdr=1;
+      for (int i = 0; i<ndiffusionequations;i++)
+      {
+         ess_tdof_listJ.SetSize(0);
+         pfes[2*i+2]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJ); // J₁ (2), J₂ (4)
+         for (int j = 0; j < ess_tdof_listJ.Size(); j++)
+         {
+            ess_tdof_listJ[j] += toffsets[2*i+2]; 
+         }
+         ess_tdof_list.Append(ess_tdof_listJ);
+      }
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+   
+
+   pgf_r[0]->ProjectBdrCoefficientTangent(one_x_cf, one_r_bdr);
+   pgf_r[0]->ProjectBdrCoefficientTangent(negone_x_cf, negone_r_bdr);
+   pgf_i[0]->ProjectBdrCoefficientTangent(one_x_cf, one_i_bdr);
+   pgf_i[0]->ProjectBdrCoefficientTangent(negone_x_cf, negone_i_bdr);
+
+   OperatorPtr Ah;
+   Vector X,B;
+   a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+   ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+   bool direct_solve = true;
+
+   if (direct_solve)
+   {
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+      int nblocks = BlockA_r->NumRowBlocks();
+      if(Mpi::Root())
+      {
+         mfem::out << "Number of blocks: " << nblocks << std::endl;
+      }
+      Array2D<const HypreParMatrix*> A_r_matrices(nblocks, nblocks);
+      Array2D<const HypreParMatrix*> A_i_matrices(nblocks, nblocks);
+      for (int i = 0; i < nblocks; i++)
+      {
+         for (int j = 0; j < nblocks; j++)
+         {
+            A_r_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockA_r->GetBlock(i,j));
+            A_i_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockA_i->GetBlock(i,j));
+         }
+      }
+      HypreParMatrix * Ahr = HypreParMatrixFromBlocks(A_r_matrices);
+      HypreParMatrix * Ahi = HypreParMatrixFromBlocks(A_i_matrices);
+
+      ComplexHypreParMatrix * Ahc_hypre =
+         new ComplexHypreParMatrix(Ahr, Ahi,true, true);
+
+      ComplexMUMPSSolver cmumps(MPI_COMM_WORLD);
+      cmumps.SetPrintLevel(0);
+      cmumps.SetOperator(*Ahc_hypre);
+      cmumps.Mult(B,X);
+      delete Ahc_hypre;   
+
+   }
+
+   a->RecoverFEMSolution(X, x);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i]->MakeRef(pfes[i], x, offsets[i]);
+      pgf_i[i]->MakeRef(pfes[i], x, offsets.Last() + offsets[i]);
+   }
+   
+   ParallelECoefficient par_e_r(pgf_r[0]);
+   ParallelECoefficient par_e_i(pgf_i[0]);
+   E_par_r.ProjectCoefficient(par_e_r);
+   E_par_i.ProjectCoefficient(par_e_i);
+   
+   // // rescale the J solutions
+   for (int i = 0; i < 2*ndiffusionequations; ++i)
+   {
+      (*pgf_r[2+i]) /= balance_scale;
+      (*pgf_i[2+i]) /= balance_scale;
+   }
+
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, *pgf_r[0],
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((real_t)0);
+      paraview_dc->Save();
+      delete paraview_dc;
+   }
+
+
+   delete a;
+   for (int i = 0; i < trial_fecols.Size(); ++i)
+   {
+      delete trial_fecols[i];
+      delete pfes[i];
+   }
+   for (int i = 0; i< test_fecols.Size(); ++i)
+   {
+      delete test_fecols[i];
+   }
+
+   return 0;
+}

miniapps/dpg/plasma/lh-eld-fosls-fem.cpp

@@ -0,0 +1,897 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//                   MFEM FOSLS-FEM Maxwell parallel example
+//
+// Compile with: make lh-eld-fosls-fem
+//
+// mpirun -np 8 ./lh-eld-fosls-fem -o 4 -paraview -m data/quad.msh -ebs -sc
+// Electron Landau Damping
+// Strong formulation:
+//     ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵ E + i ω²ϵ₀(J₁ + J₂) = 0,   in Ω
+//     subject to the constraints
+//                   - Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥ = 0,   in Ω     
+//                   - Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥ = 0,   in Ω 
+//                                            E×n = E₀,  on ∂Ω
+//                                            J₁  = 0,  on ∂Ω
+//                                            J₂  = 0,  on ∂Ω
+// The DPG-FOSLS deals with the First Order System
+//  i ω μ₀ H  + ∇ × E                  = 0,   in Ω
+// -i ω ϵ₀ϵ E + ∇ × H - ω ϵ₀ (J₁ + J₂) = 0,   in Ω
+// subject to the constraints
+//         - Δ∥ J₁ + c₁ J₁ - c₁ P(r) E∥ = 0,   in Ω     
+//         - Δ∥ J₂ + c₂ J₂ + c₂ P(r) E∥ = 0,   in Ω 
+//                                 E×n = E₀,  on ∂Ω
+//                                 J₁  = 0,  on ∂Ω
+//                                 J₂  = 0,  on ∂Ω
+
+
+// in 2D  E is vector valued and H is scalar and
+//     ∇ × E = ∇ ⋅ AE  where A = [0 1; -1 0];
+
+// E ∈ H(curl,Ω) , H ∈ H¹(Ω), Jᵢ, Qᵢ ∈ (H¹(Ω))²
+// minimize the FOSLS functional:
+//  ( iωμ₀ H, F ) + ( ∇ × E, F) = 0,                      ∀ F  ∈  L²(Ω)
+// -(iωϵ₀ϵ E, R ) + ( ∇ × H, R) - ω ϵ₀ (J₁ + J₂, R) = 0,  ∀ R  ∈ (L²(Ω))²
+// subject to the constraints
+//  (b⋅∇J₁, b ⋅ ∇K₁) + c₁(J₁, K₁) - c₁ (P B E, K₁) = 0,      ∀ K₁ ∈ (H¹(Ω))²
+//  (b⋅∇J₂, b ⋅ ∇K₂) + c₂(J₂, K₂) + c₂ (P B E, K₂) = 0,      ∀ K₂ ∈ (H¹(Ω))²
+//                                    E = E₀, on ∂Ω
+//                              J₁ = J₂ = 0,  on ∂Ω
+
+
+// We formulate the problem as constrained minimization
+// i.e, we minimize ||A U - F|| subject to B U = 0
+// where U = [E, H, J₁, J₂]ᵀ
+// A and B are given by:
+// -----------------------------------------------
+
+// A (FOSLS)       
+// ----------------------------------------------------------
+// |   |      E       |      H    |     J₁      |     J₂    |       
+// ----------------------------------------------------------
+// | F |  (∇ × E, F)  | (iωμ₀H,F) |             |           |   
+// |   |              |           |             |           |   
+// | R | -iωϵ₀ϵ(E,R)  |(∇ × H, R) | -ωϵ₀(J₁,R)  |-ωϵ₀(J₂, R)|   
+
+// B (Constraints)
+// --------------------------------------------------------------------------------
+// |   |      E       |                 J₁           |                J₂          |       
+// --------------------------------------------------------------------------------
+// |K₁ | -c₁(PBE,K₁)  | (b⋅∇J₁, b⋅∇K₁) + c₁(J₁, K₁)  |                            |       
+// |   |              |                              |                            |       
+// |K₂ |  c₂(PBE, K₂) |                              |(b⋅∇J₂, b⋅∇K₂) + c₂(J₂, K₂) |       
+
+// The saddle point system is then given by:
+// | A   B̄ᵀ | |U| = |F|
+// | B   0  | |λ|   |0|
+
+
+#include "mfem.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../util/pcomplexblockform.hpp"
+#include "../util/blockcomplexhypremat.hpp"
+#include "../../common/mfem-common.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "utils/lh_utils.hpp"
+#include "../util/utils.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+   int order = 1;
+   int delta_order = 1;
+   int par_ref_levels = 0;
+   int ser_ref_levels = 0;
+
+   // real_t rnum=1.5e9;
+   // real_t mu = 1.257e-6;
+   // real_t eps0 = 8.8541878128e-12;
+
+   real_t rnum=1.5;
+   real_t mu = 1.257;
+   real_t eps0 = 8.8541878128;
+   real_t cfactor = 1e-6;
+   real_t balance_scale = 1.0;
+   bool enable_balance_scale = false;
+
+   real_t delta_prec = 0.0;
+
+
+   bool static_cond = false;
+   bool visualization = false;
+   bool paraview = false;
+   bool debug = false;
+   bool mumps_solver = false;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&delta_order, "-do", "--delta-order",
+                  "Finite element order for the test space");                  
+   args.AddOption(&ser_ref_levels, "-sr", "--serial-refinement_levels",
+                  "Number of serial refinement levels.");                  
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&a0, "-a0", "--a0", "P(r) first parameter.");
+   args.AddOption(&a1, "-a1", "--a1", "P(r) second parameter.");
+   args.AddOption(&delta_prec, "-dp", "--delta-prec", "stability parameter for the preconditioner.");
+   args.AddOption(&mumps_solver, "-mumps", "--mumps", "-no-mumps",
+                  "--no-mumps",
+                  "Enable or disable MUMPS solver.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&debug, "-debug", "--debug", "-no-debug",
+                  "--no-debug",
+                  "Enable or disable debug mode (delta = 0.01 and no coupling).");         
+   args.AddOption(&enable_balance_scale, "-ebs", "--enable-balance-scale", "-no-ebs",
+                  "--no-ebs",
+                  "Enable or disable balance scale.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (Mpi::Root())
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (Mpi::Root())
+   {
+      args.PrintOptions(cout);
+   }
+
+   // number of diffusion equations
+   int ndiffusionequations = 2; 
+
+   Vector cvals(ndiffusionequations);
+   Vector csigns(ndiffusionequations);
+   cvals(0)  = 25e6;  cvals(1)  = 1e6;
+   csigns(0) = -1.0;  csigns(1) = 1.0;
+
+   cvals *= cfactor; // scale the coefficients
+   real_t omega = 2.*M_PI*rnum;
+   int test_order = order+delta_order;
+
+   balance_scale = (enable_balance_scale) ? eps0 : 1.0;
+
+   if (!debug) 
+   {
+      delta = 0.0; // disable delta if electron Landau damping is enabled
+      if (Mpi::Root())
+      {
+         cout << "Electron Landau damping enabled, delta set to 0.0." << endl;
+      }
+   }    
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   MFEM_VERIFY(dim == 2, "Dimension != 2 is not supported in this example");
+
+   for (int i = 0; i < ser_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   Array<int> int_bdr_attr;
+   for (int i = 0; i < mesh.GetNBE(); i++)
+   {
+      if (mesh.FaceIsInterior(mesh.GetBdrElementFaceIndex(i)))
+      {
+         int_bdr_attr.Append(mesh.GetBdrAttribute(i));
+      }
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   for (int i = 0; i < par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }  
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+   real_t scale = (debug) ? 0.0 : 1.0;
+
+   // Define coefficients
+   ConstantCoefficient one_cf(1.0);
+   //  ωμ₀
+   ConstantCoefficient omegamu_cf(omega*mu);
+   // -ωϵ₀
+   ConstantCoefficient negomegeps0_cf(-omega*eps0 * scale);
+   ConstantCoefficient balancescaled_negomegeps0_cf( -omega*eps0/balance_scale * scale);
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_vcf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   // S(r) 
+   FunctionCoefficient S_cf_r(sfunc_r), S_cf_i(sfunc_i);
+   // P(r) 
+   FunctionCoefficient P_cf_r(pfunc_r), P_cf_i(pfunc_i); 
+
+   // b
+   VectorFunctionCoefficient b_cf(dim,bfunc);
+   ScalarVectorProductCoefficient scaled_b_cf(sqrt(cfactor), b_cf);
+   ConstantCoefficient diff_coeff(cfactor);
+
+   // b⊗b
+   MatrixFunctionCoefficient bb_cf(dim,bcrossb); 
+   // I - b⊗b
+   MatrixSumCoefficient oneminusbb(Mone_cf, bb_cf, 1.0, -1.0); 
+   // S(r) (I - b⊗b)
+   ScalarMatrixProductCoefficient Soneminusbb_r(S_cf_r, oneminusbb), Soneminusbb_i(S_cf_i, oneminusbb); 
+   // P(r) b⊗b 
+   ScalarMatrixProductCoefficient P_cf_bb_r(P_cf_r, bb_cf), P_cf_bb_i(P_cf_i, bb_cf); 
+
+   // ε = S(r) (I - b⊗b) + P(r) b⊗b 
+   MatrixSumCoefficient eps_r(Soneminusbb_r, P_cf_bb_r, 1.0, 1.0); 
+   MatrixSumCoefficient eps_i(Soneminusbb_i, P_cf_bb_i, 1.0, 1.0); 
+
+   coefs_r[nattr-1] = &eps_r;
+   coefs_i[nattr-1] = &eps_i;
+
+   PWMatrixCoefficient eps_cf_r(dim, attr, coefs_r);
+   PWMatrixCoefficient eps_cf_i(dim, attr, coefs_i);
+
+   ConstantCoefficient eps0omeg(omega * eps0);
+   ConstantCoefficient negeps0omeg(-omega * eps0);
+
+
+   // - iωϵ₀ϵ = ωϵ₀ϵᵢ + i (-ωϵ₀ϵᵣ)
+   // ω ϵ₀ ϵᵢ
+   ScalarMatrixProductCoefficient eps0omeg_eps_i(eps0omeg, eps_cf_i);
+   // -ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient negeps0omeg_eps_r(negeps0omeg, eps_cf_r);
+
+   // if ELD
+   Array<Vector *> c_arrays(ndiffusionequations);
+   Array<PWConstCoefficient *> pw_c_coeffs(ndiffusionequations);
+   Array<MatrixCoefficient *> cPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> cPibb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPibb_cf(ndiffusionequations);
+
+   Array<MatrixCoefficient *> balancescaled_signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> balancescaled_signedcPibb_cf(ndiffusionequations);
+   Vector temp(nattr); temp=0.0;
+   Array<ConstantCoefficient *> c_coeffs(ndiffusionequations);
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      temp[nattr-1] = cvals(i);
+      // temp = cvals(i);
+      pw_c_coeffs[i] = new PWConstCoefficient(temp);
+      c_coeffs[i] = new ConstantCoefficient(cvals(i));
+      cPrbb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_r);
+      cPibb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_i);
+      signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPrbb_cf[i]);
+      signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPibb_cf[i]);
+      balancescaled_signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(balance_scale,*signedcPrbb_cf[i]);
+      balancescaled_signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(balance_scale,*signedcPibb_cf[i]);
+   }
+
+
+   std::vector<std::string> fosls_variables = {" E  ", " H  ", " J₁ ", " J₂ "};
+   std::vector<std::string> fosls_spaces = {"H(curl,Ω) ", "   H¹(Ω)  ", 
+                                            "  (H¹(Ω))ᵈ", "  (H¹(Ω))ᵈ"}; 
+   std::vector<std::string> constraints_variables = {" Q₁ ", " Q₂ "};
+   std::vector<std::string> constraints_spaces = {"  (H¹(Ω))ᵈ", "  (H¹(Ω))ᵈ"};                                             
+
+   // Define the spaces
+   Array<FiniteElementCollection *> fosls_trial_fecols;
+   Array<FiniteElementCollection *> fosls_test_fecols;
+   Array<ParFiniteElementSpace *> fosls_pfes;
+   Array<FiniteElementCollection *> constraints_fecols;
+   Array<ParFiniteElementSpace *> constraints_pfes;
+
+   // H(curl) space for E
+   fosls_trial_fecols.Append(new ND_FECollection(order, dim));
+   fosls_pfes.Append(new ParFiniteElementSpace(&pmesh, fosls_trial_fecols.Last()));
+   // Scalar H¹ space for H
+   fosls_trial_fecols.Append(new H1_FECollection(order, dim));
+   fosls_pfes.Append(new ParFiniteElementSpace(&pmesh, fosls_trial_fecols.Last()));
+   // Vector H¹ spaces for Jᵢ
+   for (int i = 0; i < ndiffusionequations; i++)
+   {
+      fosls_trial_fecols.Append(new H1_FECollection(order, dim));
+      fosls_pfes.Append(new ParFiniteElementSpace(&pmesh, fosls_trial_fecols.Last(), dim));
+      constraints_fecols.Append(new H1_FECollection(order, dim));
+      constraints_pfes.Append(new ParFiniteElementSpace(&pmesh, constraints_fecols.Last(), dim));
+   }
+
+   Array<HYPRE_BigInt> tdofs(fosls_pfes.Size());
+   Array<HYPRE_BigInt> constraints_tdofs(constraints_pfes.Size());
+   for (int i = 0; i < fosls_pfes.Size(); ++i)
+   {
+      tdofs[i] = fosls_pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << " FOSLS      FE Space " << fosls_spaces[i] << "  for " << fosls_variables[i] << " has " << tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   for (int i = 0; i < constraints_pfes.Size(); ++i)
+   {
+      constraints_tdofs[i] = constraints_pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << " Constraint FE Space " << constraints_spaces[i] << "  for " << constraints_variables[i] << " has " << constraints_tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   if (Mpi::Root())
+   {
+      cout << "Total number of true dofs: " << tdofs.Sum() << endl;
+   }
+
+   // test spaces for F and R
+   fosls_test_fecols.Append(new L2_FECollection(test_order, dim));
+   fosls_test_fecols.Append(new L2_FECollection(test_order, dim));
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(fosls_pfes,fosls_test_fecols);
+   a->SetTestFECollVdim(1,dim); 
+
+   // (∇ × E, F)
+   a->AddTrialIntegrator(new MixedCurlIntegrator(one_cf), nullptr, 0, 0);
+
+   // (i ω μ₀ H, F)
+   a->AddTrialIntegrator(nullptr, new MixedScalarMassIntegrator(omegamu_cf), 1, 0);
+
+   // -i ω ϵ₀ϵ (E, R) = ω ϵ₀ ϵᵢ (E, G) + i (-ω ϵ₀ ϵᵣ E, R)
+   a->AddTrialIntegrator(new VectorFEMassIntegrator(eps0omeg_eps_i),
+                         new VectorFEMassIntegrator(negeps0omeg_eps_r), 0, 1);
+
+   // (∇ × H, R)
+   a->AddTrialIntegrator(new MixedCurlIntegrator(one_cf), nullptr, 1,1);
+   
+   // - ω ϵ₀ (J₁, R)
+   a->AddTrialIntegrator(new VectorMassIntegrator(balancescaled_negomegeps0_cf), nullptr, 2, 1);
+   // - ω ϵ₀ (J₂, R)
+   a->AddTrialIntegrator(new VectorMassIntegrator(balancescaled_negomegeps0_cf), nullptr, 3, 1);
+
+   // test integrators for test norm: ||v||² = ||F||² + ||R||² + ||K₁||² + ||L₁||² + ||K₂||² + ||L₂||² 
+   // (F,δF), F, δF ∈  L²(Ω)
+   a->AddTestIntegrator(new MassIntegrator(one_cf),nullptr, 0, 0);
+   // (R,δR), R, δR ∈ (L²(Ω))²
+   a->AddTestIntegrator(new VectorMassIntegrator(one_cf),nullptr, 1, 1);
+
+   a->Assemble(0);
+
+
+   // Constraint matrix B
+   ParMixedBilinearForm bEK1_r(fosls_pfes[0], constraints_pfes[0]);
+   ParMixedBilinearForm bEK1_i(fosls_pfes[0], constraints_pfes[0]);
+
+   // - c₁ (P B E, K₁)
+   bEK1_r.AddDomainIntegrator(new VectorFEMassIntegrator(*balancescaled_signedcPrbb_cf[0]));
+   bEK1_i.AddDomainIntegrator(new VectorFEMassIntegrator(*balancescaled_signedcPibb_cf[0]));
+
+   bEK1_r.Assemble();
+   bEK1_i.Assemble();
+
+   ParMixedBilinearForm bK2H_r(constraints_pfes[1], fosls_pfes[1]);
+   bK2H_r.Assemble();
+
+   ParMixedBilinearForm bEK2_r(fosls_pfes[0], constraints_pfes[1]);
+   ParMixedBilinearForm bEK2_i(fosls_pfes[0], constraints_pfes[1]);
+   // c₂(PBE, K₂)
+   bEK2_r.AddDomainIntegrator(new VectorFEMassIntegrator(*balancescaled_signedcPrbb_cf[1]));    
+   bEK2_i.AddDomainIntegrator(new VectorFEMassIntegrator(*balancescaled_signedcPibb_cf[1]));    
+
+   bEK2_r.Assemble();
+   bEK2_i.Assemble();
+
+   // (b⋅∇J₁, b⋅∇K₁) + c₁(J₁, K₁) 
+   ParBilinearForm bJ1K1(constraints_pfes[0]);
+   bJ1K1.AddDomainIntegrator(new DirectionalVectorDiffusionIntegrator(scaled_b_cf));
+   bJ1K1.AddDomainIntegrator(new VectorMassIntegrator(*pw_c_coeffs[0]));
+   bJ1K1.Assemble();
+
+   ParBilinearForm bJ2K2(constraints_pfes[1]);
+   bJ2K2.AddDomainIntegrator(new DirectionalVectorDiffusionIntegrator(scaled_b_cf));
+   bJ2K2.AddDomainIntegrator(new VectorMassIntegrator(*pw_c_coeffs[1]));
+   bJ2K2.Assemble();
+
+
+   int fosls_npfes = fosls_pfes.Size();
+   Array<int> fosls_offsets(fosls_npfes+1);  fosls_offsets[0] = 0;
+   Array<int> fosls_toffsets(fosls_npfes+1); fosls_toffsets[0] = 0;
+   for (int i = 0; i<fosls_npfes; i++)
+   {
+      fosls_offsets[i+1] = fosls_pfes[i]->GetVSize();
+      fosls_toffsets[i+1] = fosls_pfes[i]->TrueVSize();
+   }
+   fosls_offsets.PartialSum();
+   fosls_toffsets.PartialSum();
+
+   Array<int> empty;
+   OperatorPtr fosls_Aop;
+
+   a->FormSystemMatrix(empty, fosls_Aop);
+   ComplexOperator * fosls_Ac = fosls_Aop.As<ComplexOperator>();
+
+   OperatorPtr constraintOp_EK1_r, constraintOp_EK1_i;
+   bEK1_r.FormRectangularSystemMatrix(empty, empty, constraintOp_EK1_r);
+   bEK1_i.FormRectangularSystemMatrix(empty, empty, constraintOp_EK1_i);
+
+   OperatorPtr constraintOp_EK2_r, constraintOp_EK2_i;
+   bEK2_r.FormRectangularSystemMatrix(empty, empty, constraintOp_EK2_r);
+   bEK2_i.FormRectangularSystemMatrix(empty, empty, constraintOp_EK2_i);
+
+   OperatorPtr constraintOp_J1K1;
+   bJ1K1.FormSystemMatrix(empty, constraintOp_J1K1);
+   OperatorPtr constraintOp_J2K2;
+   bJ2K2.FormSystemMatrix(empty, constraintOp_J2K2);
+
+
+
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      delete pw_c_coeffs[i];
+      delete c_coeffs[i];
+      delete cPrbb_cf[i]; 
+      delete cPibb_cf[i]; 
+      delete signedcPrbb_cf[i]; 
+      delete signedcPibb_cf[i]; 
+   }
+
+   // Assemble all the system and get the global matrix and right-hand side
+   Array<ParFiniteElementSpace *> all_pfes;
+   all_pfes.Append(fosls_pfes);
+   all_pfes.Append(constraints_pfes);
+
+   int npfes = all_pfes.Size();
+   Array<int> all_offsets(npfes + 1); all_offsets[0] = 0;
+   Array<int> all_toffsets(npfes + 1); all_toffsets[0] = 0;
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      all_offsets[i+1] = all_pfes[i]->GetVSize();
+      all_toffsets[i+1] = all_pfes[i]->GetTrueVSize();
+   }
+   all_offsets.PartialSum();
+   all_toffsets.PartialSum();
+
+
+   // put all the operators into a block operator
+   BlockOperator A_r(all_toffsets);
+   BlockOperator A_i(all_toffsets);
+
+   BlockOperator * BlockAfosls_r = dynamic_cast<BlockOperator *>(&fosls_Ac->real());
+   BlockOperator * BlockAfosls_i = dynamic_cast<BlockOperator *>(&fosls_Ac->imag());
+
+
+   for (int i = 0; i < fosls_pfes.Size(); ++i)
+   {
+      for (int j = 0; j < fosls_pfes.Size(); ++j)
+      {
+         A_r.SetBlock(i, j, &BlockAfosls_r->GetBlock(i, j));
+         A_i.SetBlock(i, j, &BlockAfosls_i->GetBlock(i, j));
+      }
+   }
+
+
+   A_r.SetBlock(fosls_pfes.Size()+0, 0, constraintOp_EK1_r.Ptr());
+   A_i.SetBlock(fosls_pfes.Size()+0, 0, constraintOp_EK1_i.Ptr());
+
+   A_r.SetBlock(fosls_pfes.Size()+1, 0, constraintOp_EK2_r.Ptr());
+   A_i.SetBlock(fosls_pfes.Size()+1, 0, constraintOp_EK2_i.Ptr());
+
+   A_r.SetBlock(fosls_pfes.Size()+0, 2, constraintOp_J1K1.Ptr());
+   A_r.SetBlock(fosls_pfes.Size()+1, 3, constraintOp_J2K2.Ptr());
+
+   // Construct the adjoint operator B̄ᵀ
+   HypreParMatrix * constraintOp_EK1_r_t = constraintOp_EK1_r.As<HypreParMatrix>()->Transpose();
+   HypreParMatrix * constraintOp_EK1_i_t = constraintOp_EK1_i.As<HypreParMatrix>()->Transpose();
+   HypreParMatrix * constraintOp_EK2_r_t = constraintOp_EK2_r.As<HypreParMatrix>()->Transpose();
+   HypreParMatrix * constraintOp_EK2_i_t = constraintOp_EK2_i.As<HypreParMatrix>()->Transpose();
+   HypreParMatrix * constraintOp_J1K1_t = constraintOp_J1K1.As<HypreParMatrix>()->Transpose();
+   HypreParMatrix * constraintOp_J2K2_t = constraintOp_J2K2.As<HypreParMatrix>()->Transpose();
+
+   // We also need to scale the imaginary part of B̄ᵀ by -1;
+   *constraintOp_EK1_i_t *= -1.0;
+   *constraintOp_EK2_i_t *= -1.0;
+
+   A_r.SetBlock(0, fosls_pfes.Size()+0, constraintOp_EK1_r_t);
+   A_i.SetBlock(0, fosls_pfes.Size()+0, constraintOp_EK1_i_t);    
+   A_r.SetBlock(0, fosls_pfes.Size()+1, constraintOp_EK2_r_t);
+   A_i.SetBlock(0, fosls_pfes.Size()+1, constraintOp_EK2_i_t);
+   A_r.SetBlock(2, fosls_pfes.Size()+0, constraintOp_J1K1_t);
+   A_r.SetBlock(3, fosls_pfes.Size()+1, constraintOp_J2K2_t);
+
+   //    // dammies for other coupling terms in B
+   // ParMixedBilinearForm bHK1_r(fosls_pfes[1], constraints_pfes[0]); bHK1_r.Assemble();
+   // ParMixedBilinearForm bHK1_i(fosls_pfes[1], constraints_pfes[0]); bHK1_i.Assemble();
+   // OperatorPtr constraintOp_HK1_r, constraintOp_HK1_i;
+   // bHK1_r.FormRectangularSystemMatrix(empty, empty, constraintOp_HK1_r);
+   // bHK1_i.FormRectangularSystemMatrix(empty, empty, constraintOp_HK1_i);
+   // A_r.SetBlock(fosls_pfes.Size()+0, 1, constraintOp_HK1_r.Ptr());
+   // A_i.SetBlock(fosls_pfes.Size()+0, 1, constraintOp_HK1_i.Ptr());
+   
+   // ParMixedBilinearForm bHK2_r(fosls_pfes[1], constraints_pfes[1]); bHK2_r.Assemble(); 
+   // ParMixedBilinearForm bHK2_i(fosls_pfes[1], constraints_pfes[1]); bHK2_i.Assemble();
+   
+   // ParMixedBilinearForm bK1H_r(constraints_pfes[0], fosls_pfes[1]); bK1H_r.Assemble();
+   // ParMixedBilinearForm bK1H_i(constraints_pfes[0], fosls_pfes[1]); bK1H_i.Assemble();
+   // ParMixedBilinearForm bK2H_r(fosls_pfes[1], constraints_pfes[1]); bK2H_r.Assemble();
+   // ParMixedBilinearForm bK2H_i(fosls_pfes[1], constraints_pfes[0]); bK2H_i.Assemble();
+
+
+
+   // ParBilinearForm bJ1K2(constraints_pfes[0]);
+
+
+
+
+   ComplexOperator * A = new ComplexOperator(&A_r, &A_i, false, false);
+   if (Mpi::Root())
+   {
+      mfem::out << "Complex Operator A finished successfully." << endl;
+   }
+
+
+   socketstream E_out_r;
+   Vector x(2*all_offsets.Last());
+   x = 0.;
+   Array<ParGridFunction *> pgf_r(npfes);
+   Array<ParGridFunction *> pgf_i(npfes);
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i] = new ParGridFunction(all_pfes[i], x, all_offsets[i]);
+      pgf_i[i] = new ParGridFunction(all_pfes[i], x, all_offsets.Last() + all_offsets[i]);
+   }
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+   ParGridFunction E_par_r(&L2_fes);
+   ParGridFunction E_par_i(&L2_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+   std::string output_dir = "ParaView/FOSLS-FEM/" + GetTimestamp();
+
+
+   if (paraview)
+   {
+      if (Mpi::Root()) { WriteParametersToFile(args, output_dir); }
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order
+                         << "_eld_1" ;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("Er",pgf_r[0]);
+      paraview_dc->RegisterField("Ei",pgf_i[0]);
+      paraview_dc->RegisterField("E_par_r",&E_par_r);
+      paraview_dc->RegisterField("E_par_i",&E_par_i);
+      paraview_dc->RegisterField("Hr",pgf_r[1]);
+      paraview_dc->RegisterField("Hi",pgf_i[1]);      
+      paraview_dc->RegisterField("J1r",pgf_r[2]);
+      paraview_dc->RegisterField("J1i",pgf_i[2]);
+      paraview_dc->RegisterField("J2r",pgf_r[3]);
+      paraview_dc->RegisterField("J2i",pgf_i[3]);
+      paraview_dc->RegisterField("Q1r",pgf_r[4]);
+      paraview_dc->RegisterField("Q1i",pgf_i[4]);
+      paraview_dc->RegisterField("Q2r",pgf_r[5]);
+      paraview_dc->RegisterField("Q2i",pgf_i[5]);
+   }
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listJ;
+   Array<int> ess_tdof_listQ;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+
+      // remove internal boundaries
+      for (int i = 0; i<int_bdr_attr.Size(); i++)
+      {
+         ess_bdr[int_bdr_attr[i]-1] = 0;
+      }
+      all_pfes[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += all_toffsets[0];
+      }
+      ess_bdr = 1;
+      for (int i = 0; i<ndiffusionequations;i++)
+      {
+         ess_tdof_listJ.SetSize(0);
+         ess_tdof_listQ.SetSize(0);
+         all_pfes[i+2]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJ);
+         all_pfes[i+4]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listQ);
+         for (int j = 0; j < ess_tdof_listJ.Size(); j++)
+         {
+            ess_tdof_listJ[j] += all_toffsets[i+2];
+         }
+         for (int j = 0; j < ess_tdof_listQ.Size(); j++)
+         {
+            ess_tdof_listQ[j] += all_toffsets[i+4];
+         }
+         ess_tdof_list.Append(ess_tdof_listJ);
+         ess_tdof_list.Append(ess_tdof_listQ);
+      }
+
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+
+   pgf_r[0]->ProjectBdrCoefficientTangent(one_x_cf, one_r_bdr);
+   pgf_r[0]->ProjectBdrCoefficientTangent(negone_x_cf, negone_r_bdr);
+   pgf_i[0]->ProjectBdrCoefficientTangent(one_x_cf, one_i_bdr);
+   pgf_i[0]->ProjectBdrCoefficientTangent(negone_x_cf, negone_i_bdr);
+
+   BlockOperator * P = new BlockOperator(all_offsets, all_toffsets);
+   BlockMatrix * R = new BlockMatrix(all_toffsets, all_offsets);
+   P->owns_blocks = 0;
+   R->owns_blocks = 0;
+
+   for (int i = 0; i < npfes; i++)
+   {
+      HypreParMatrix * P_ = all_pfes[i]->Dof_TrueDof_Matrix();
+      P->SetBlock(i,i,P_);
+      const SparseMatrix * R_ = all_pfes[i]->GetRestrictionMatrix();
+      R->SetBlock(i, i, const_cast<SparseMatrix*>(R_));
+   }
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Build prolongation finished" << endl;
+   }
+
+   int n = P->Width();
+   Vector B(2*n); B = 0.0;
+
+   Vector X(2*n);
+   Vector X_r(X, 0, n);
+   Vector X_i(X, n, n);
+
+   Vector x_r(x, 0, x.Size()/2);
+   Vector x_i(x, x.Size()/2, x.Size()/2);
+
+   R->Mult(x_r, X_r);
+   R->Mult(x_i, X_i);
+
+   ParBlockComplexSystem aa(A);
+   A = aa.EliminateBC(ess_tdof_list, X, B);
+   ess_tdof_list.Print(mfem::out);
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Eliminate BC finished successfully." << endl;
+   }
+
+   bool direct_solve = true;
+
+   if (direct_solve)
+   {
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&A->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&A->imag());
+
+      int nblocks = BlockA_r->NumRowBlocks();
+      Array2D<const HypreParMatrix*> A_r_matrices(nblocks, nblocks);
+      Array2D<const HypreParMatrix*> A_i_matrices(nblocks, nblocks);
+      for (int i = 0; i < nblocks; i++)
+      {
+         for (int j = 0; j < nblocks; j++)
+         {
+            if (BlockA_r->IsZeroBlock(i,j))
+            {
+               A_r_matrices(i,j) = nullptr;
+            }
+            else
+            {
+               A_r_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockA_r->GetBlock(i,j));
+            }
+            if (BlockA_i->IsZeroBlock(i,j))
+            {
+               A_i_matrices(i,j) = nullptr;
+            }
+            else
+            {
+               A_i_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&BlockA_i->GetBlock(i,j));
+            }   
+         }
+      }
+      HypreParMatrix * Ahr = HypreParMatrixFromBlocks(A_r_matrices);
+      HypreParMatrix * Ahi = HypreParMatrixFromBlocks(A_i_matrices);
+
+      ComplexHypreParMatrix * Ahc_hypre =
+         new ComplexHypreParMatrix(Ahr, Ahi,true, true);
+
+      ComplexMUMPSSolver cmumps(MPI_COMM_WORLD);
+      cmumps.SetPrintLevel(1);
+      cmumps.SetOperator(*Ahc_hypre);
+
+      if (Mpi::Root())
+      {
+         mfem::out << "Setup CMUMPS solver finished successfully." << endl;
+      }
+
+      cmumps.Mult(B,X);
+      delete Ahc_hypre;   
+   }
+   else
+   {
+      MFEM_ABORT("to be implemented: iterative solver with preconditioner");
+   }
+
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Solve finished successfully." << endl;
+   }
+
+
+   n = P->Height();
+   int m = P->Width();
+
+   x_r.MakeRef(x, 0, n);
+   x_i.MakeRef(x, n, n);
+
+   X_r.MakeRef(X, 0, m);
+   X_i.MakeRef(X, m, m);
+
+   P->Mult(X_r, x_r);
+   P->Mult(X_i, x_i);
+
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i]->MakeRef(all_pfes[i], x, all_offsets[i]);
+      pgf_i[i]->MakeRef(all_pfes[i], x, all_offsets.Last() + all_offsets[i]);
+   }
+   
+   ParallelECoefficient par_e_r(pgf_r[0]);
+   ParallelECoefficient par_e_i(pgf_i[0]);
+   E_par_r.ProjectCoefficient(par_e_r);
+   E_par_i.ProjectCoefficient(par_e_i);
+   
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, *pgf_r[0],
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((real_t)0);
+      paraview_dc->Save();
+      delete paraview_dc;
+   }
+
+
+
+   return 0;
+
+
+   
+
+
+   // a->RecoverFEMSolution(X, x);
+
+   // for (int i = 0; i < npfes; ++i)
+   // {
+   //    pgf_r[i]->MakeRef(pfes[i], x, offsets[i]);
+   //    pgf_i[i]->MakeRef(pfes[i], x, offsets.Last() + offsets[i]);
+   // }
+   
+   // ParallelECoefficient par_e_r(pgf_r[0]);
+   // ParallelECoefficient par_e_i(pgf_i[0]);
+   // E_par_r.ProjectCoefficient(par_e_r);
+   // E_par_i.ProjectCoefficient(par_e_i);
+   
+   // // // rescale the J solutions
+   // for (int i = 0; i < 2*ndiffusionequations; ++i)
+   // {
+   //    (*pgf_r[2+i]) /= balance_scale;
+   //    (*pgf_i[2+i]) /= balance_scale;
+   // }
+
+   // if (visualization)
+   // {
+   //    const char * keys = nullptr;
+   //    char vishost[] = "localhost";
+   //    int  visport   = 19916;
+   //    common::VisualizeField(E_out_r,vishost, visport, *pgf_r[0],
+   //                           "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   // }
+
+   // if (paraview)
+   // {
+   //    paraview_dc->SetCycle(0);
+   //    paraview_dc->SetTime((real_t)0);
+   //    paraview_dc->Save();
+   //    delete paraview_dc;
+   // }
+
+
+   // delete a;
+   // for (int i = 0; i < trial_fecols.Size(); ++i)
+   // {
+   //    delete trial_fecols[i];
+   //    delete pfes[i];
+   // }
+   // for (int i = 0; i< test_fecols.Size(); ++i)
+   // {
+   //    delete test_fecols[i];
+   // }
+
+   // return 0;
+}

miniapps/dpg/plasma/lh-eld-uw.cpp

@@ -0,0 +1,980 @@
+//                   MFEM UW parallel example
+//
+
+// Electron Landau Damping
+//
+//        ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵᵣ E + i ω²ϵ₀(J₁ + J₂) = 0,   in Ω
+//                   - (b⋅∇)²J₁ + c₁ J₁ - c₁ P b⊗b E = 0,   in Ω     
+//                   - (b⋅∇)²J₂ + c₂ J₂ + c₂ P b⊗b E = 0,   in Ω 
+//                                               E×n = E₀,  on ∂Ω
+//                                               J₁  = 0,   on ∂Ω
+//                                               J₂  = 0,   on ∂Ω
+// First order system:
+//     i ω μ₀ H + ∇ × E                    = 0, in Ω
+// -i ω ϵ₀ ϵᵣ E + ∇ × H - α ω ϵ₀ (J₁ + J₂) = 0, in Ω 
+//            b ⋅ ∇Q₁ + c₁ J₁ - c₁ P B E = 0, in Ω
+//                          Q₁ + b ⋅ ∇J₁ = 0, in Ω 
+//            b ⋅ ∇Q₂ + c₂ J₂ + c₂ P B E = 0, in Ω  
+//                          Q₂ + b ⋅ ∇J₂ = 0, in Ω
+// where B = b ⊗ b. Here α serves as a scaling factor. For α = 0 the system becomes 
+// lower trianular (Maxwell decouples from the diffusion equations).
+
+// Note that in 2D we have the following 2 definitions of the curl.
+// a) for a 2D vector E, ∇ × E:= ∇ ⋅ (R E) where R = [0 1; -1 0]
+// b) for a scalar H, ∇ × H:= R ∇ H
+
+// Define the group variables 
+// then the strong formulation reads:
+// (A u, v) = (i ω μ₀ H, F) + (∇ × E, F) 
+//          - (i ω ϵ₀ ϵᵣ E, W) + (∇ × H, W) - (α ω ϵ₀ (J₁ + J₂), W)
+//          + (b ⋅ ∇Q₁, K₁) + (c₁ J₁, K₁) - (c₁ P B E, K₁) + (Q₁, L₁) + (b ⋅ ∇J₁, L₁)
+//          + (b ⋅ ∇Q₂, K₂) + (c₂ J₂, K₂) + (c₂ P B E, K₂) + (Q₂, L₂) + (b ⋅ ∇J₂, L₂) 
+//
+// and the adjoint operator is defined by:
+// (u, A^⋆ v) = (E, ∇×F + i ω ϵ₀ ϵ⋆ᵣ W - c₁ P̄ B K₁ + c₂ P̄ B K₂)
+//            + (H, -i ω μ₀ F + ∇ × W)
+//            + (J₁, -α ω ϵ₀ W + c₁ K₁ - b ⋅ ∇ L₁)
+//            + (Q₁, - b ⋅ ∇ K₁ + L₁)
+//            + (J₂, -α ω ϵ₀ W + c₂ K₂ - b ⋅ ∇ L₂)
+//            + (Q₂, - b ⋅ ∇ K₂ + L₂)
+// 
+// Define the group trial field and trace variables 
+// u:=(E,H,J₁,Q₁,J₂,Q₂), û:=(Ê, Ĥ, Ĵ₁, Q̂₁, Ĵ₂, Q̂₂) and 
+// the test variable  v:=(F,W,K₁,L₁,K₂,L₂) 
+
+// The ultraweak variational formulation is then given by:
+// Find u ∈ U=((L²(Ω))² × L²(Ω) × (L²(Ω))² × (L²(Ω))² × (L²(Ω))² × (L²(Ω))²),
+// û ∈ Û=( H⁻¹/²(Γₕ) × H¹/²(Γₕ) × (H⁻¹/²(Γₕ))² × (H⁻¹/²(Γₕ))² × (H⁻¹/²(Γₕ))²× (H⁻¹/²(Γₕ))² )
+//  such that ∀ v ∈ V=(H¹(Ω) × H(curl,Ω) × (H¹(Ω))² × (H¹(Ω))² × (H¹(Ω))² × (H¹(Ω))²)
+//  (u, A^⋆ v) + b̂(û, v) = 0,
+// or equivalently
+// 
+// (i ω μ₀ H, F) + (E, ∇×F) + < R Ê, F > = 0,      ∀ F ∈ H¹(Ω)
+// -i ω ϵ₀ ϵᵣ (E, W) + (H, ∇ ×  W) + < Ĥ, W × n > - α ω ϵ₀ (J₁ + J₂, W) = 0,  ∀ W ∈ H(curl,Ω)
+// -(Q, b ⋅ ∇ K₁) + <Q̂₁, K₁> + (c₁ J₁, K₁) - c₁(P B E, K₁) = 0,  ∀ K₁ ∈ (H¹(Ω))²
+//  (Q₁, L₁) - (J₁, b ⋅ ∇L₁) + <Ĵ₁, L₁> = 0,  ∀ L₁ ∈ (H¹(Ω))²
+// -(Q, b⋅ ∇ K₂) + <Q̂₂, K₂> + (c₂ J₂, K₂) + c₂(P B E, K₂) = 0,  ∀ K₂ ∈ (H¹(Ω))²
+//  (Q₂, L₂) - (J₂, b ⋅ ∇L₂) + <Ĵ₂, L₂> = 0,  ∀ L₂ ∈ (H¹(Ω))²
+
+// with the adjoint graph test norm defined by:
+// ‖v‖²_V = ‖A^⋆ v‖² + ‖v‖²
+//        = ‖∇×F + i ω ϵ₀ ϵ⋆ᵣ W - c₁ P̄ B K₁ + c₂ P̄ B K₂‖²
+//        + ‖-i ω μ₀ F + ∇ × W‖²
+//        + ‖-α ω ϵ₀ W + c₁ K₁ - b ⋅ ∇ L₁‖²
+//        + ‖- b ⋅ ∇ K₁ + L₁‖²
+//        + ‖-α ω ϵ₀ W + c₂ K₂ - b ⋅ ∇ L₂‖²
+//        + ‖- b ⋅ ∇ K₂ + L₂‖²
+//        + ‖F‖² + ‖W‖² + ‖K₁‖² + ‖L₁‖² + ‖K₂‖² + ‖L₂‖²
+//
+
+#include "mfem.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../../common/mfem-common.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "utils/lh_utils.hpp"
+#include "../util/utils.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+// Define spaces
+enum TrialSpace
+{
+   E_space     = 0,
+   H_space     = 1,
+   J1_space    = 2,
+   Q1_space    = 3,
+   J2_space    = 4,
+   Q2_space    = 5,
+   hatE_space  = 6,
+   hatH_space  = 7,
+   hatJ1_space = 8,
+   hatQ1_space = 9,
+   hatJ2_space = 10,
+   hatQ2_space = 11
+};
+
+enum TestSpace
+{
+   F_space = 0,
+   W_space = 1,
+   K1_space = 2,
+   L1_space = 3,
+   K2_space = 4,
+   L2_space = 5
+};
+
+inline const char * ToString(TrialSpace ts)
+{
+   switch (ts)
+   {
+      case E_space:     return " L²(Ω)²  for E  ";
+      case H_space:     return " L²(Ω)   for H  ";
+      case J1_space:    return " L²(Ω)²  for J₁ ";
+      case Q1_space:    return " L²(Ω)²  for Q₁ ";
+      case J2_space:    return " L²(Ω)²  for J₂ ";
+      case Q2_space:    return " L²(Ω)²  for Q₂ ";
+      case hatE_space:  return " tr(RT)  for Ê  ";
+      case hatH_space:  return " tr(H¹)  for Ĥ  ";
+      case hatJ1_space: return " tr(RT)² for Ĵ₁ ";
+      case hatQ1_space: return " tr(RT)² for Q̂₁ ";
+      case hatJ2_space: return " tr(RT)² for Ĵ₂ ";
+      case hatQ2_space: return " tr(RT)² for Q̂₂ ";
+      default:          return "Unknown TrialSpace";
+   }
+}
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+   int order = 1;
+   int delta_order = 1;
+   int par_ref_levels = 0;
+   int ser_ref_levels = 0;
+
+   real_t rnum=1.5;
+   real_t mu = 1.257;
+   real_t eps0 = 8.8541878128;
+   real_t alpha = 1.0; // scaling factor for E, Js coupling
+   bool static_cond = false;
+   bool visualization = false;
+   bool paraview = false;
+   bool debug = false;
+   bool mumps_solver = false;
+
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&delta_order, "-do", "--delta-order",
+                  "Finite element order for the test space");                  
+   args.AddOption(&ser_ref_levels, "-sr", "--serial-refinement_levels",
+                  "Number of serial refinement levels.");                  
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&a0, "-a0", "--a0", "P(r) first parameter.");
+   args.AddOption(&a1, "-a1", "--a1", "P(r) second parameter.");
+   args.AddOption(&alpha, "-alpha", "--alpha",
+                  "Scaling factor for E, Js coupling.");
+   args.AddOption(&mumps_solver, "-mumps", "--mumps", "-no-mumps",
+                  "--no-mumps",
+                  "Enable or disable MUMPS solver.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&debug, "-debug", "--debug", "-no-debug",
+                  "--no-debug",
+                  "Enable or disable debug mode (delta = 0.01 and no coupling).");         
+   args.Parse();
+   if (!args.Good())
+   {
+      if (Mpi::Root())
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (Mpi::Root())
+   {
+      args.PrintOptions(cout);
+   }
+
+   // number of diffusion equations
+   int ndiffusionequations = 2; 
+
+   if (!debug) 
+   {
+      delta = 0.0; // disable delta if electron Landau damping is enabled
+      if (Mpi::Root())
+      {
+         cout << "Electron Landau damping enabled, delta set to 0.0." << endl;
+      }
+   }    
+   else
+   {
+      if (Mpi::Root())
+      {
+         alpha = 0.0; 
+         cout << "Setting alpha = 0.0 to disable coupling between E and Js." << endl;
+         cout << "Debug mode enabled, delta = 0.01 and no coupling." << endl;
+      }
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   MFEM_VERIFY(dim == 2, "Dimension != 2 is not supported in this example");
+
+   for (int i = 0; i < ser_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   Array<int> int_bdr_attr;
+   for (int i = 0; i < mesh.GetNBE(); i++)
+   {
+      if (mesh.FaceIsInterior(mesh.GetBdrElementFaceIndex(i)))
+      {
+         int_bdr_attr.Append(mesh.GetBdrAttribute(i));
+      }
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   for (int i = 0; i < par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }  
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+
+   real_t omega = 2.*M_PI*rnum;
+   real_t neg_omega_eps0 = -omega * eps0;
+   real_t omega_eps0 = omega * eps0;
+
+   Vector cvals(ndiffusionequations);
+   Vector csigns(ndiffusionequations);
+   real_t cfactor = 1e-6;
+   cvals(0)  = 25e6;  cvals(1)  = 1e6;
+   csigns(0) = -1.0;  csigns(1) = 1.0;
+   cvals *= cfactor; // scale the coefficients
+
+   // List of coefficients
+   // Trial integrators coefficients
+   DenseMatrix R(dim), Rt(dim);
+   R(0,0) =  0.0;  R(0,1) = 1.0;
+   R(1,0) = -1.0;  R(1,1) = 0.0;
+   R.Transpose(Rt);
+
+   VectorFunctionCoefficient no_scale_bvec_cf(dim,bfunc);// b
+   real_t scaled_cfactor = std::sqrt(cfactor);
+   ScalarVectorProductCoefficient bvec_cf(scaled_cfactor,no_scale_bvec_cf); // scaled b
+   // VectorFunctionCoefficient bvec_cf(dim,bfunc);// b
+   ScalarVectorProductCoefficient neg_bvec_cf(-1.0,bvec_cf); // -b
+   ScalarVectorProductCoefficient alpha_omega_eps0_bvec_cf(eps0*omega*alpha,bvec_cf); // α ω ϵ₀ b
+
+
+   MatrixFunctionCoefficient B_cf(dim, bcrossb);
+   FunctionCoefficient p_r_cf(pfunc_r);
+   FunctionCoefficient p_i_cf(pfunc_i);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   Array<Vector *> cf_arrays(ndiffusionequations);
+   Array<PWConstCoefficient *> c_cf(ndiffusionequations);
+   Array<ProductCoefficient *> neg_c_alpha_omega_eps0_cf(ndiffusionequations);
+   Array<ProductCoefficient *> c2_cf(ndiffusionequations);
+   Array<ProductCoefficient *> signed_c_cf(ndiffusionequations);
+   Array<MatrixCoefficient*> signed_PB_r_cf(ndiffusionequations);
+   Array<MatrixCoefficient*> signed_PB_i_cf(ndiffusionequations);
+   Array<ProductCoefficient*> neg_signed_c_omega_eps_cf(ndiffusionequations);
+   Array<ProductCoefficient*> signed_c_omega_eps_cf(ndiffusionequations);
+   Array<MatrixCoefficient*> signed_PBR_r_cf(ndiffusionequations);
+   Array<MatrixCoefficient*> signed_PBR_i_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> c2_absP2BB_cf(ndiffusionequations);
+   Array<VectorCoefficient*> neg_c_bvec_cf(ndiffusionequations);
+
+   Vector zerovec(nattr); zerovec=0.0;
+
+   // ϵᵣ in plasma (real)
+   MatrixFunctionCoefficient eps_r_temp_cf(dim, epsilon_func_r);
+   // ϵᵣ in plasma(imag)
+   MatrixFunctionCoefficient eps_i_temp_cf(dim, epsilon_func_i);
+
+   coefs_r[nattr-1] = &eps_r_temp_cf;
+   coefs_i[nattr-1] = &eps_i_temp_cf;
+
+   // ωμ₀ 
+   ConstantCoefficient omegamu_cf(mu * omega);
+   // -ωμ₀
+   ConstantCoefficient neg_omegamu_cf(-mu * omega);
+   // ω² μ₀²
+   ConstantCoefficient omega2mu2_cf(mu * mu * omega * omega);  
+   // R 
+   MatrixConstantCoefficient R_cf(R);
+   // Rᵀ
+   MatrixConstantCoefficient Rt_cf(Rt);
+   // ϵᵣ (real)
+   PWMatrixCoefficient eps_r_cf(dim, attr, coefs_r);
+   // ϵᵣ (imag)
+   PWMatrixCoefficient eps_i_cf(dim, attr, coefs_i);
+   // -ω ϵ₀ ϵᵣ (real)
+   ScalarMatrixProductCoefficient neg_omega_eps0_eps_r_cf(neg_omega_eps0, eps_r_cf);
+   // ω ϵ₀ ϵᵣ (imag)
+   ScalarMatrixProductCoefficient omega_eps0_eps_i_cf(omega_eps0, eps_i_cf);
+   // ω ϵ₀ ϵᵣᵢ R 
+   MatrixProductCoefficient omega_eps0_eps_i_R_cf(omega_eps0_eps_i_cf, R_cf);
+   // -ω ϵ₀ ϵᵣᵣ R
+   MatrixProductCoefficient neg_omega_eps0_eps_r_R_cf(neg_omega_eps0_eps_r_cf, R_cf); 
+   // 
+   // ω ϵ₀ Rᵀ ϵᵣᵢ
+   MatrixProductCoefficient omega_eps0_Rt_eps_i_cf(Rt_cf, omega_eps0_eps_i_cf);
+   // ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient omega_eps0_eps_r_cf(omega_eps0, eps_r_cf);
+   // ω ϵ₀ Rᵀ ϵᵣᵣ 
+   MatrixProductCoefficient omega_eps0_Rt_eps_r_cf(Rt_cf, omega_eps0_eps_r_cf);
+   // |ϵᵣ|² = ϵᵣᵣϵᵣᵣ + ϵᵣᵢϵᵣᵢ   
+   MatrixProductCoefficient eps_r_eps_r_cf(eps_r_cf, eps_r_cf);
+   MatrixProductCoefficient eps_i_eps_i_cf(eps_i_cf, eps_i_cf);
+
+   MatrixSumCoefficient abseps2_cf(eps_r_eps_r_cf, eps_i_eps_i_cf);
+   // ω² ϵ₀² 
+   ConstantCoefficient alpha2_omega2_eps02(alpha*alpha*omega * omega * eps0 * eps0);
+   // ω² ϵ₀² |ϵᵣ|²   
+   ScalarMatrixProductCoefficient omega2_eps02_abseps2_cf(omega*omega*eps0*eps0, abseps2_cf);
+
+   // 1
+   ConstantCoefficient one_cf(1.0);
+   // -1
+   ConstantCoefficient neg_one_cf(-1.0);
+   // - ω ϵ₀
+   ConstantCoefficient neg_omega_eps0_cf(neg_omega_eps0);
+   // - α ω ϵ₀
+   ConstantCoefficient neg_alpha_omega_eps0_cf(alpha * neg_omega_eps0); 
+
+   // P B
+   ScalarMatrixProductCoefficient PB_r_cf(p_r_cf, B_cf);
+   ScalarMatrixProductCoefficient PB_i_cf(p_i_cf, B_cf);
+
+   // BB
+   MatrixProductCoefficient BB_cf(B_cf, B_cf);
+   
+   // Pᵣ²
+   ProductCoefficient P2_r_cf(p_r_cf, p_r_cf);
+   // Pᵢ²
+   ProductCoefficient P2_i_cf(p_i_cf, p_i_cf);
+   // |P|²
+   SumCoefficient absP2_cf(P2_r_cf, P2_i_cf);
+   // |P|² B B
+   ScalarMatrixProductCoefficient absP2_BB_cf(absP2_cf, BB_cf);
+   // cᵢ  
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      zerovec[nattr-1] = cvals(i);
+      c_cf[i] = new PWConstCoefficient(zerovec);
+      neg_c_alpha_omega_eps0_cf[i] = new ProductCoefficient(-omega_eps0*alpha, *c_cf[i]);
+      c2_cf[i] = new ProductCoefficient(*c_cf[i], *c_cf[i]);
+      signed_c_cf[i] = new ProductCoefficient(csigns(i), *c_cf[i]);
+      signed_PB_r_cf[i] = new ScalarMatrixProductCoefficient(*signed_c_cf[i], PB_r_cf);
+      signed_PB_i_cf[i] = new ScalarMatrixProductCoefficient(*signed_c_cf[i], PB_i_cf);
+      signed_c_omega_eps_cf[i] = new ProductCoefficient(omega_eps0, *signed_c_cf[i]);
+      neg_signed_c_omega_eps_cf[i] = new ProductCoefficient(neg_omega_eps0, *signed_c_cf[i]);
+      // cᵢ P B R
+      signed_PBR_r_cf[i] = new MatrixProductCoefficient(*signed_PB_r_cf[i], R_cf);
+      signed_PBR_i_cf[i] = new MatrixProductCoefficient(*signed_PB_i_cf[i], R_cf);
+      // cᵢ² |P|² B B
+      c2_absP2BB_cf[i] = new ScalarMatrixProductCoefficient(*c2_cf[i], absP2_BB_cf);
+      neg_c_bvec_cf[i] = new ScalarVectorProductCoefficient(-1.0 * cvals(i), bvec_cf);
+   }   
+
+   ProductCoefficient neg_c1_c2_cf(*signed_c_cf[0], *signed_c_cf[1]);
+   // -c₁ c₂ |P|² B B
+   ScalarMatrixProductCoefficient neg_c1_c2_absP2BB_cf(neg_c1_c2_cf, absP2_BB_cf);
+
+
+   MatrixProductCoefficient PB_r_eps_r_cf(PB_r_cf, eps_r_cf);
+   MatrixProductCoefficient PB_r_eps_i_cf(PB_r_cf, eps_i_cf);
+   MatrixProductCoefficient PB_i_eps_r_cf(PB_i_cf, eps_r_cf);
+   MatrixProductCoefficient PB_i_eps_i_cf(PB_i_cf, eps_i_cf);
+
+
+   Array<FiniteElementCollection *> trial_fecols;
+   Array<ParFiniteElementSpace *> trial_pfes;
+   Array<FiniteElementCollection *> test_fecols;
+
+   // L²(Ω)² space for E
+   trial_fecols.Append(new L2_FECollection(order-1, dim));
+   trial_pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last(), dim));
+   // L²(Ω) space for H
+   trial_fecols.Append(new L2_FECollection(order-1, dim));
+   trial_pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last()));
+   // L²(Ω)² space for J₁, Q₁, J₂, Q₂
+   for (int i = 0; i < 2*ndiffusionequations; ++i)
+   {
+      trial_fecols.Append(new L2_FECollection(order-1, dim));
+      trial_pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last(), dim));
+   }
+   // H⁻¹/²(Γₕ) space for Ê
+   trial_fecols.Append(new RT_Trace_FECollection(order-1, dim));
+   trial_pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last()));
+   // H¹/²(Γₕ) space for Ĥ
+   trial_fecols.Append(new H1_Trace_FECollection(order, dim));
+   trial_pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last()));
+   // H⁻¹/²(Γₕ)² space for Ĵ₁, Q̂₁, Ĵ₂, Q̂₂
+   for (int i = 0; i < 2*ndiffusionequations; ++i)
+   {
+      trial_fecols.Append(new RT_Trace_FECollection(order-1, dim));
+      trial_pfes.Append(new ParFiniteElementSpace(&pmesh, trial_fecols.Last(), dim));
+   }
+
+   Array<HYPRE_BigInt> tdofs(trial_pfes.Size());
+   for (int i = 0; i < trial_pfes.Size(); ++i)
+   {
+      tdofs[i] = trial_pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << "ParFiniteElementSpace " << ToString(TrialSpace(i))  << " has " << tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   if (Mpi::Root())
+   {
+      cout << "Total number of true dofs: " << tdofs.Sum() << endl;
+   }
+
+   // Test spaces   
+   // H¹(Ω) space for F
+   test_fecols.Append(new H1_FECollection(order, dim));
+   // H(curl,Ω) space for W
+   test_fecols.Append(new ND_FECollection(order, dim));
+   // Test spaces for K₁, L₁, K₂, L₂
+   for (int i = 0; i < 2*ndiffusionequations; ++i)
+   {
+      test_fecols.Append(new H1_FECollection(order, dim));
+   }
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_pfes,test_fecols);
+   for (int i = 0; i < 2*ndiffusionequations; ++i)
+   {
+      a->SetTestFECollVdim(i+2,dim);
+   }
+
+   // Trial integrators
+   // i(ω μ₀ H, F)
+   a->AddTrialIntegrator(nullptr,
+                         new MixedScalarMassIntegrator(omegamu_cf),
+                         TrialSpace::H_space,TestSpace::F_space);
+
+   // (E, ∇×F)
+   a->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one_cf)),
+                         nullptr,
+                         TrialSpace::E_space,TestSpace::F_space);
+   
+   // -i ω ϵ₀ ϵᵣ (E, W) = -i(ω ϵ₀ (ϵᵣᵣ+i ϵᵣᵢ)  E, W)   
+   //                   = (ω ϵ₀ ϵᵣᵢ + i (-ωϵ₀ϵᵣᵣ) )  E, W)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(omega_eps0_eps_i_cf)),
+                         new TransposeIntegrator(new VectorFEMassIntegrator(neg_omega_eps0_eps_r_cf)),
+                         TrialSpace::E_space,TestSpace::W_space);
+   
+   // (H, ∇ ×  W)   
+   a->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one_cf)), 
+                         nullptr,
+                         TrialSpace::H_space, TestSpace::W_space);
+   
+
+
+   VectorFunctionCoefficient b_cf(dim, bfunc);
+   ScalarVectorProductCoefficient neg_b_cf(-1.0,b_cf);   
+
+   for (int i = 0; i < ndiffusionequations; i++)
+   {
+      // - α ω ϵ₀ (Jᵢ,W)   
+      a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(neg_alpha_omega_eps0_cf)),
+                            nullptr,
+                            TrialSpace::J1_space + 2*i,TestSpace::W_space);
+      // (Qᵢ, - b ⋅ ∇ Kᵢ)   
+      a->AddTrialIntegrator(new TransposeIntegrator(new DirectionalVectorGradientIntegrator(neg_b_cf)),
+                         nullptr,
+                         TrialSpace::Q1_space + 2*i,TestSpace::K1_space+ 2*i);
+
+      // (cᵢ Jᵢ, Kᵢ)
+      a->AddTrialIntegrator(new VectorMassIntegrator(*c_cf[i]),
+                            nullptr,
+                            TrialSpace::J1_space + 2*i,TestSpace::K1_space + 2*i);       
+                            
+      // (Qᵢ, Lᵢ)
+      a->AddTrialIntegrator(new VectorMassIntegrator(one_cf),
+                            nullptr,
+                            TrialSpace::Q1_space + 2*i,TestSpace::L1_space + 2*i);      
+
+      // (Jᵢ, - b ⋅ ∇Lᵢ)
+      a->AddTrialIntegrator(new TransposeIntegrator(new DirectionalVectorGradientIntegrator(neg_b_cf)),
+                            nullptr,
+                            TrialSpace::J1_space + 2*i,TestSpace::L1_space + 2*i);
+
+      // - c₁(P B E, K₁) = (- c₁ Pᵣ B E, K₁) + i (-c₁ Pᵢ B E, K₁) 
+      //   c₂(P B E, K₂) = (  c₂ Pᵣ B E, K₂) + i ( c₂ Pᵢ B E, K₂) 
+      a->AddTrialIntegrator(new VectorMassIntegrator(*signed_PB_r_cf[i]),
+                            new VectorMassIntegrator(*signed_PB_i_cf[i]),
+                            TrialSpace::E_space,TestSpace::K1_space + 2*i);
+
+      // Trace integrators 
+      // <Q̂ᵢ, Kᵢ>
+      a->AddTrialIntegrator(new VectorTraceIntegrator,nullptr,
+                            TrialSpace::hatQ1_space + 2*i,TestSpace::K1_space + 2*i);
+      // <Ĵᵢ, Lᵢ>
+      a->AddTrialIntegrator(new VectorTraceIntegrator,nullptr,
+                            TrialSpace::hatJ1_space + 2*i,TestSpace::L1_space + 2*i);                      
+
+   }
+
+   // Trace integrators
+   // < R Ê, F > (we include R in the variable)
+   a->AddTrialIntegrator(new TraceIntegrator,
+                         nullptr,
+                         TrialSpace::hatE_space,TestSpace::F_space);
+   //< Ĥ, W × n >                         
+   a->AddTrialIntegrator(new TangentTraceIntegrator,
+                         nullptr,
+                         TrialSpace::hatH_space,TestSpace::W_space);
+
+   // -------------------------------------------------------------------------------   
+   //                         Test integrators 
+   // -------------------------------------------------------------------------------   
+   // (∇×F,∇×δF) = (R ∇ F, R ∇δF) = (Rᵀ R ∇F, ∇δF) = (∇F, ∇δF)
+   a->AddTestIntegrator(new DiffusionIntegrator(one_cf),
+                        nullptr,
+                        TestSpace::F_space, TestSpace::F_space);
+   // i(-ω ϵ₀ ϵᵣ ∇ × F, δW) = i(-ω ϵ₀ (ϵᵣᵣ + i ϵᵣᵢ) ∇ × F, δW)
+   //                       = (ω ϵ₀ ϵᵣᵢ ∇ × F, δW) + i(-ω ϵ₀ ϵᵣᵣ ∇ × F, δW)
+   //                       = (ω ϵ₀ ϵᵣᵢ R ∇ F, δW) + i(-ω ϵ₀ ϵᵣᵣ R ∇ F, δW)
+   a->AddTestIntegrator(new MixedVectorGradientIntegrator(omega_eps0_eps_i_R_cf),
+                        new MixedVectorGradientIntegrator(neg_omega_eps0_eps_r_R_cf),
+                        TestSpace::F_space, TestSpace::W_space);
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      // (-c₁ P B ∇ × F, δK₁) = (-c₁ Pᵣ B ∇ × F, δK₁) + i (-c₁ Pᵢ B ∇ × F, δK₁)
+      // ( c₂ P B ∇ × F, δK₂) = ( c₂ Pᵣ B ∇ × F, δK₂) + i ( c₂ Pᵢ B ∇ × F, δK₂)
+      a->AddTestIntegrator(new MixedCurlIntegrator(*signed_PB_r_cf[i]),
+                           new MixedCurlIntegrator(*signed_PB_i_cf[i]),
+                           TestSpace::F_space, TestSpace::K1_space+ 2*i);
+
+
+
+
+   }                     
+
+   // i(ω ϵ₀ ϵᵣ⋆ W, ∇×δF) = i(ω ϵ₀ (ϵᵣᵣ - i ϵᵣᵢ) W, ∇×δF) (ϵᵣᵣ & ϵᵣᵢ are symmetric)
+   //                     = (ω ϵ₀ ϵᵣᵢ W, ∇×δF) + i(ω ϵ₀ ϵᵣᵣ W, ∇×δF) 
+   //                     = (ω ϵ₀ Rᵀ ϵᵣᵢ W, ∇δF) + i(ω ϵ₀ Rᵀ ϵᵣᵣ W, ∇δF) 
+   a->AddTestIntegrator(new TransposeIntegrator(new MixedVectorGradientIntegrator(omega_eps0_Rt_eps_i_cf)),
+                        new TransposeIntegrator(new MixedVectorGradientIntegrator(omega_eps0_Rt_eps_r_cf)),
+                        TestSpace::W_space, TestSpace::F_space);
+   // (ω² ϵ²₀ ϵᵣϵᵣ⋆ W, δ W) = (ω² ϵ²₀ |ϵᵣ|² W, δW) 
+   a->AddTestIntegrator(new VectorFEMassIntegrator(omega2_eps02_abseps2_cf),
+                        nullptr,
+                        TestSpace::W_space, TestSpace::W_space); 
+   
+   // Pᵢ B ϵᵣᵣ - Pᵣ B ϵᵣᵢ
+   MatrixSumCoefficient PB_i_eps_r_minus_PB_r_eps_i_cf(PB_i_eps_r_cf, PB_r_eps_i_cf, 1.0, -1.0);
+   // Pᵣ B ϵᵣᵣ + Pᵢ B ϵᵣᵢ)
+   MatrixSumCoefficient PB_r_eps_r_plus_PB_i_eps_i_cf(PB_r_eps_r_cf, PB_i_eps_i_cf, 1.0, 1.0);
+   //   ω ϵ₀ c₁ (Pᵢ B ϵᵣᵣ - Pᵣ B ϵᵣᵢ) (negsigned)
+   //  -ω ϵ₀ c₂ (Pᵢ B ϵᵣᵣ - Pᵣ B ϵᵣᵢ) (negsigned) 
+   // - ω ϵ₀ c₁ (Pᵣ B ϵᵣᵣ + Pᵢ B ϵᵣᵢ) (signed)
+   //   ω ϵ₀ c₂ (Pᵣ B ϵᵣᵣ + Pᵢ B ϵᵣᵢ) (signed)
+   Array<MatrixCoefficient*> neg_signed_c_PB_i_eps_r_minus_PB_r_eps_i_cf(ndiffusionequations);
+   Array<MatrixCoefficient*> signed_c_PB_i_eps_r_minus_PB_r_eps_i_cf(ndiffusionequations);
+   Array<MatrixCoefficient*> signed_c_PB_r_eps_r_plus_PB_i_eps_i_cf(ndiffusionequations);
+   Array<MatrixCoefficient*> neg_signed_c_PB_r_eps_r_plus_PB_i_eps_i_cf(ndiffusionequations);
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      neg_signed_c_PB_i_eps_r_minus_PB_r_eps_i_cf[i] = new ScalarMatrixProductCoefficient(*neg_signed_c_omega_eps_cf[i], PB_i_eps_r_minus_PB_r_eps_i_cf);
+      signed_c_PB_r_eps_r_plus_PB_i_eps_i_cf[i] = new ScalarMatrixProductCoefficient(*signed_c_omega_eps_cf[i], PB_r_eps_r_plus_PB_i_eps_i_cf);
+   
+      // i(-ω ϵ₀c₁ P B ϵᵣ⋆ W, δK₁) = ω ϵ₀ c₁ (Pᵢ B ϵᵣᵣ - Pᵣ B ϵᵣᵢ) + i (-ωϵ₀ c₁ (Pᵣ B ϵᵣᵣ + Pᵢ B ϵᵣᵢ))
+      // i( ω ϵ₀c₂ P B ϵᵣ⋆ W, δK₂) = -ω ϵ₀ c₂ (Pᵢ B ϵᵣᵣ - Pᵣ B ϵᵣᵢ) + i (ωϵ₀ c₂ (Pᵣ B ϵᵣᵣ + Pᵢ B ϵᵣᵢ))
+      // TestSpace K1_space and K2_space (2,4)
+      TestSpace tspace = static_cast<TestSpace>(2*i+2);
+      a->AddTestIntegrator(new VectorFEMassIntegrator(*neg_signed_c_PB_i_eps_r_minus_PB_r_eps_i_cf[i]),
+                           new VectorFEMassIntegrator(*signed_c_PB_r_eps_r_plus_PB_i_eps_i_cf[i]),
+                           TestSpace::W_space, tspace);
+   
+      signed_c_PB_i_eps_r_minus_PB_r_eps_i_cf[i] = new ScalarMatrixProductCoefficient(*signed_c_omega_eps_cf[i], PB_i_eps_r_minus_PB_r_eps_i_cf);
+      neg_signed_c_PB_r_eps_r_plus_PB_i_eps_i_cf[i] = new ScalarMatrixProductCoefficient(*neg_signed_c_omega_eps_cf[i], PB_r_eps_r_plus_PB_i_eps_i_cf);
+      // Note that P is scalar and also ϵᵣ B = B ϵᵣ 
+      // i( ω ϵ₀c₁ ϵᵣ P̄ B K₁, δW) = i (K₁,  ω ϵ₀ c₁ P B ϵᵣ⋆ δW) 
+      // i(-ω ϵ₀c₂ ϵᵣ P̄ B K₂, δW) = i (K₂, -ω ϵ₀ c₂ P B ϵᵣ⋆ δW)
+      a->AddTestIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(*signed_c_PB_i_eps_r_minus_PB_r_eps_i_cf[i])),
+                           new TransposeIntegrator(new VectorFEMassIntegrator(*neg_signed_c_PB_r_eps_r_plus_PB_i_eps_i_cf[i])),
+                           tspace, TestSpace::W_space);
+      // (c²₁ P P̄ B B K₁, δK₁)
+      // (c²₂ P P̄ B B K₂, δK₂)
+      a->AddTestIntegrator(new VectorMassIntegrator(*c2_absP2BB_cf[i]),
+                           nullptr,
+                           tspace, tspace);
+
+      // (c₁²K₁, δK₁) 
+      // (c₂²K₂, δK₂)
+      a->AddTestIntegrator(new VectorMassIntegrator(*c2_cf[i]),
+                           nullptr,
+                           tspace, tspace);     
+                           
+      // (-c₁ α ω ϵ₀ W, δK₁) 
+      // (-c₂ α ω ϵ₀ W, δK₂)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(*neg_c_alpha_omega_eps0_cf[i]),
+                           nullptr,
+                           TestSpace::W_space, tspace);
+      // (-c₁ α  ω ϵ₀ K₁, δW) 
+      // (-c₂ α  ω ϵ₀ K₂, δW)                            
+      a->AddTestIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(*neg_c_alpha_omega_eps0_cf[i])),
+                           nullptr,
+                           tspace, TestSpace::W_space);
+
+      // (-c₁ b⋅∇L₁, δK₁) 
+      // (-c₂ b⋅∇L₂, δK₂) 
+      TestSpace Kspace = static_cast<TestSpace>(2*i+2);
+      TestSpace Lspace = static_cast<TestSpace>(2*i+3);
+
+      a->AddTestIntegrator(new DirectionalVectorGradientIntegrator(*neg_c_bvec_cf[i]),
+                           nullptr,
+                           Lspace, Kspace);
+      // (K₁,-c₁ b ⋅ ∇δL₁)
+      // (K₂,-c₂ b ⋅ ∇δL₂)
+      a->AddTestIntegrator(new TransposeIntegrator(new DirectionalVectorGradientIntegrator(*neg_c_bvec_cf[i])),
+                           nullptr,
+                           Kspace, Lspace);
+      // (L₁, δL₁) 
+      // (L₂, δL₂)
+      a->AddTestIntegrator(new VectorMassIntegrator(one_cf),
+                           nullptr,
+                           Lspace, Lspace);   
+      // (b⋅ ∇L₁, b ⋅ ∇δL₁)   
+      // (b⋅ ∇L₂, b ⋅ ∇δL₂)
+      a->AddTestIntegrator(new DirectionalVectorDiffusionIntegrator(bvec_cf),
+                           nullptr, Lspace, Lspace);
+      // (b ⋅ ∇K₁, b ⋅ ∇δK₁)
+      // (b ⋅ ∇K₂, b ⋅ ∇δK₂)       
+      a->AddTestIntegrator(new DirectionalVectorDiffusionIntegrator(bvec_cf),
+                           nullptr, Kspace, Kspace);                    
+
+      //-(b ⋅ ∇K₁, δL₁) 
+      //-(b ⋅ ∇K₂, δL₂) 
+      a->AddTestIntegrator(new DirectionalVectorGradientIntegrator(neg_b_cf),
+                           nullptr, Kspace, Lspace);
+      //(L₁, - b ⋅ ∇δK₁) 
+      //(L₂, - b ⋅ ∇δK₂) 
+      a->AddTestIntegrator(new TransposeIntegrator(new DirectionalVectorGradientIntegrator(neg_b_cf)),
+                           nullptr, Lspace, Kspace);
+
+      // (α ω ϵ₀ b⋅ ∇L₁, δW) 
+      // (α ω ϵ₀ b⋅ ∇L₂, δW)                            
+      a->AddTestIntegrator(new MixedDirectionalVectorGradientIntegrator(alpha_omega_eps0_bvec_cf),
+                           nullptr,
+                           Lspace, TestSpace::W_space);
+      // (W, α ω ϵ₀ b ⋅ ∇δL₁)
+      // (W, α ω ϵ₀ b ⋅ ∇δL₂)
+      a->AddTestIntegrator(new TransposeIntegrator(new MixedDirectionalVectorGradientIntegrator(alpha_omega_eps0_bvec_cf)),
+                           nullptr,
+                           TestSpace::W_space, Lspace);
+
+   }
+
+   // (-c₁ P̄ B K₁, ∇×δF) = (K₁, -c₁ P B ∇×δF) = (K₁, -c₁ Pᵣ B ∇×δF) + i (K₁, -c₁ Pᵢ B ∇×δF)
+   a->AddTestIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(*signed_PB_r_cf[0])),
+                        new TransposeIntegrator(new MixedCurlIntegrator(*signed_PB_i_cf[0])),
+                        TestSpace::K1_space, TestSpace::F_space); 
+   // (c₂ P̄ B K₂, ∇×δF) = (K₂, c₂ P B ∇×δF) = (K₂, c₂ Pᵣ B ∇×δF) + i (K₂, c₂ Pᵢ B ∇×δF)
+   a->AddTestIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(*signed_PB_r_cf[1])),
+                        new TransposeIntegrator(new MixedCurlIntegrator(*signed_PB_i_cf[1])),
+                        TestSpace::K2_space, TestSpace::F_space); 
+
+
+   // (- c₁c₂ P P̄ B B K₁, δK₂)
+   // (- c₁c₂ P P̄ B B K₂, δK₁)
+   a->AddTestIntegrator(new VectorMassIntegrator(neg_c1_c2_absP2BB_cf),
+                        nullptr,
+                        TestSpace::K1_space, TestSpace::K2_space);
+   a->AddTestIntegrator(new VectorMassIntegrator(neg_c1_c2_absP2BB_cf),
+                        nullptr,
+                        TestSpace::K2_space, TestSpace::K1_space);
+
+   // (∇ × W, ∇ × δW)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one_cf),
+                        nullptr,
+                        TestSpace::W_space, TestSpace::W_space);
+   // i(ω μ₀ ∇ × W, δF)
+   a->AddTestIntegrator(nullptr,
+                        new MixedCurlIntegrator(omegamu_cf),
+                        TestSpace::W_space, TestSpace::F_space);
+   // i(-ω μ₀ F, ∇ ×δW)
+   a->AddTestIntegrator(nullptr, 
+                        new TransposeIntegrator(new MixedCurlIntegrator(neg_omegamu_cf)),
+                        TestSpace::F_space, TestSpace::W_space);
+   // (ω² μ₀² F, δF)
+   a->AddTestIntegrator(new MassIntegrator(omega2mu2_cf),
+                        nullptr,
+                        TestSpace::F_space, TestSpace::F_space);
+   // (α² ω² ϵ₀² W, δW) 
+   a->AddTestIntegrator(new VectorFEMassIntegrator(alpha2_omega2_eps02),
+                        nullptr,
+                        TestSpace::W_space,TestSpace::W_space);   
+
+   // (F, δF) 
+   a->AddTestIntegrator(new MassIntegrator(one_cf),
+                        nullptr,
+                        TestSpace::F_space,TestSpace::F_space);
+   // (W, δW) 
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one_cf),
+                        nullptr,
+                        TestSpace::W_space,TestSpace::W_space);
+   // (K₁, δK₁) 
+   a->AddTestIntegrator(new VectorMassIntegrator(one_cf),
+                        nullptr,
+                        TestSpace::K1_space,TestSpace::K1_space);                  
+   // (L₁, δL₁)
+   a->AddTestIntegrator(new VectorMassIntegrator(one_cf),
+                        nullptr,
+                        TestSpace::L1_space,TestSpace::L1_space); 
+   // (K₂, δK₂) 
+   a->AddTestIntegrator(new VectorMassIntegrator(one_cf),
+                        nullptr,
+                        TestSpace::K2_space,TestSpace::K2_space);
+   // (L₂, δL₂)
+   a->AddTestIntegrator(new VectorMassIntegrator(one_cf),
+                        nullptr,
+                        TestSpace::L2_space,TestSpace::L2_space);
+
+   a->Assemble();
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      delete c_cf[i];
+      delete signed_c_cf[i];
+      delete signed_PB_r_cf[i];
+      delete signed_PB_i_cf[i];
+      delete signed_PBR_r_cf[i];
+      delete signed_PBR_i_cf[i];
+   }   
+
+   socketstream E_out_r;
+
+   int npfes = trial_pfes.Size();
+   Array<int> offsets(npfes+1);  offsets[0] = 0;
+   Array<int> toffsets(npfes+1); toffsets[0] = 0;
+   for (int i = 0; i<npfes; i++)
+   {
+      offsets[i+1] = trial_pfes[i]->GetVSize();
+      toffsets[i+1] = trial_pfes[i]->TrueVSize();
+   }
+   offsets.PartialSum();
+   toffsets.PartialSum();
+
+   Vector x(2*offsets.Last());
+   x = 0.;
+   
+   Array<ParGridFunction *> pgf_r(npfes);
+   Array<ParGridFunction *> pgf_i(npfes);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i] = new ParGridFunction(trial_pfes[i], x, offsets[i]);
+      pgf_i[i] = new ParGridFunction(trial_pfes[i], x, offsets.Last() + offsets[i]);
+   }
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+   ParGridFunction E_par_r(&L2_fes);
+   ParGridFunction E_par_i(&L2_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   std::string output_dir = "ParaView/UW/" + GetTimestamp();
+
+   if (paraview)
+   {
+      if (Mpi::Root()) { WriteParametersToFile(args, output_dir); }
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",pgf_r[0]);
+      paraview_dc->RegisterField("E_i",pgf_i[0]);
+      paraview_dc->RegisterField("E_par_r",&E_par_r);
+      paraview_dc->RegisterField("E_par_i",&E_par_i);
+      paraview_dc->RegisterField("H_r",pgf_r[1]);
+      paraview_dc->RegisterField("H_i",pgf_i[1]);      
+      paraview_dc->RegisterField("J_1_r",pgf_r[2]);
+      paraview_dc->RegisterField("J_1_i",pgf_i[2]);
+      paraview_dc->RegisterField("Q_1_r",pgf_r[3]);
+      paraview_dc->RegisterField("Q_1_i",pgf_i[3]);
+      paraview_dc->RegisterField("J_2_r",pgf_r[4]);
+      paraview_dc->RegisterField("J_2_i",pgf_i[4]);
+      paraview_dc->RegisterField("Q_2_r",pgf_r[5]);
+      paraview_dc->RegisterField("Q_2_i",pgf_i[5]);
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listJhat;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+
+      // remove internal boundaries
+      for (int i = 0; i<int_bdr_attr.Size(); i++)
+      {
+         ess_bdr[int_bdr_attr[i]-1] = 0;
+      }
+
+      trial_pfes[2+2*ndiffusionequations]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += toffsets[2+2*ndiffusionequations];
+      }
+      // ess_bdr=1;
+      for (int i = 0; i<ndiffusionequations;i++)
+      {
+         ess_tdof_listJhat.SetSize(0);
+         trial_pfes[2*i+8]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJhat);
+         for (int j = 0; j < ess_tdof_listJhat.Size(); j++)
+         {
+            ess_tdof_listJhat[j] += toffsets[2*i+8];
+         }
+
+         ess_tdof_list.Append(ess_tdof_listJhat);
+      }
+
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+   
+   // rotate the vector
+   // (x,y) -> (y,-x)
+   Vector rot_one_x(dim); rot_one_x = 0.0; rot_one_x(1) = -1.0;
+   Vector rot_negone_x(dim); rot_negone_x = 0.0; rot_negone_x(1) = 1.0;
+   VectorConstantCoefficient rot_one_x_cf(rot_one_x);
+   VectorConstantCoefficient rot_negone_x_cf(rot_negone_x);
+
+   pgf_r[2+2*ndiffusionequations]->ProjectBdrCoefficientNormal(rot_one_x_cf, one_r_bdr);
+   pgf_r[2+2*ndiffusionequations]->ProjectBdrCoefficientNormal(rot_negone_x_cf, negone_r_bdr);
+   pgf_i[2+2*ndiffusionequations]->ProjectBdrCoefficientNormal(rot_one_x_cf, one_i_bdr);
+   pgf_i[2+2*ndiffusionequations]->ProjectBdrCoefficientNormal(rot_negone_x_cf, negone_i_bdr);
+
+   OperatorPtr Ah;
+   Vector X,B;
+   a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+   ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+
+   BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+   int nblocks = BlockA_r->NumRowBlocks();
+
+   {
+      ComplexBlockOperator Ac(*Ahc);
+      Vector Xc(X.Size()); Xc = 0.0;
+      Vector Bc(B.Size());
+      Ac.BlockComplexToComplexBlock(B, Bc);
+   
+      BlockDiagonalPreconditioner Mc(Ac.RowOffsets());
+      for (int i = 0; i < nblocks; ++i)
+      {
+         auto solver = new ComplexMUMPSSolver(MPI_COMM_WORLD);
+         solver->SetPrintLevel(0);
+         solver->SetOperator(Ac.GetBlock(i,i));
+         Mc.SetDiagonalBlock(i, solver);
+      }
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-5);
+      cg.SetMaxIter(500);
+      cg.SetPrintLevel(1);
+      cg.SetPreconditioner(Mc);
+      cg.SetOperator(Ac);
+      cg.Mult(Bc, Xc);
+      Ac.ComplexBlockToBlockComplex(Xc, X);
+   }
+
+   a->RecoverFEMSolution(X, x);
+
+   for (int i = 0; i < npfes; ++i)
+   {
+      pgf_r[i]->MakeRef(trial_pfes[i], x, offsets[i]);
+      pgf_i[i]->MakeRef(trial_pfes[i], x, offsets.Last() + offsets[i]);
+   }
+   
+   ParallelECoefficient par_e_r(pgf_r[0]);
+   ParallelECoefficient par_e_i(pgf_i[0]);
+   E_par_r.ProjectCoefficient(par_e_r);
+   E_par_i.ProjectCoefficient(par_e_i);
+
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, *pgf_r[0],
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((real_t)0);
+      paraview_dc->Save();
+      delete paraview_dc;
+   }
+
+   delete a;
+   for (int i = 0; i < trial_fecols.Size(); ++i)
+   {
+      delete trial_fecols[i];
+      delete trial_pfes[i];
+   }
+   for (int i = 0; i< test_fecols.Size(); ++i)
+   {
+      delete test_fecols[i];
+   }
+
+
+   return 0;
+}   
+

miniapps/dpg/plasma/lh-eld-weak.cpp

@@ -0,0 +1,711 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//                   MFEM Maxwell-Vdiffusion coupling parallel example
+//
+// Compile with: make lh-eld-weak
+//
+// mpirun -np 8 ./lh-eld-weak -o 3 -paraview -pr 0
+// mpirun -np 8 ./lh-eld-weak -o 3 -paraview -pr 1 -sc
+
+// Electron Landau Damping
+
+// Weak Coupling Formulation:
+// Strong formulation:
+//     ∇×(1/μ₀∇×E) - ω² ϵ₀ ϵ E = i ω²ϵ₀(J₁ + J₂),   in Ω
+//                         E×n = E₀,  on ∂Ω
+//             - Δ∥ J₁ + c₁ J₁ =  c₁ P(r) E∥,   in Ω     
+//             - Δ∥ J₂ + c₂ J₂ = -c₂ P(r) E∥,   in Ω 
+//                         J₁  = 0,  on ∂Ω
+//                         J₂  = 0,  on ∂Ω
+//   Find E ∈ H(curl,Ω), J₁  ∈ H¹(Ω), J₂  ∈ H¹(Ω) such that
+//   (1/μ₀ ∇×E, ∇ × F) - ω² ϵ₀ (ϵᵣ E, F) = -i ω²ϵ₀(J₁ + J₂, F),  ∀ F ∈ H(curl,Ω)
+
+// ( (b⋅∇)J₁,(b⋅∇) δJ₁ ) + c₁ (J₁,δJ₁) =  c₁(P(r) b⊗b E, δJ₁),  ∀ δJ₁ ∈ (H¹(Ω))²
+// ( (b⋅∇)J₂,(b⋅∇) δJ₂ ) + c₂ (J₂,δJ₂) = -c₂(P(r) b⊗b E, δJ₂),  ∀ δJ₂ ∈ (H¹(Ω))²
+//                                                    J₁ = J₂ = 0,  on ∂Ω
+// --------------------------------------------------------------------------------
+// |               J₁                |                 J₂           |     RHS      |
+// --------------------------------------------------------------------------------
+// |δJ₁|((b⋅∇)J₁,(b⋅∇)δJ₁)+c₁(J₁,δJ₁)|                              | c₁(P(r)E,δJ₁)|  
+// |   |                             |                              |              |   
+// |δJ₂|                             |((b⋅∇)J₂,(b⋅∇)δJ₂)+ c₂(J₂,δJ₂)|-c₂(P(r)E,δJ₂)|  
+// where (δE,δH,δJ₁,δJ₂) ∈  H¹(Ω) × H(curl,Ω) × (H¹(Ω))² × (H¹(Ω))² 
+
+
+
+#include "mfem.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../util/pcomplexblockform.hpp"
+#include "../../common/mfem-common.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "../util/utils.hpp"
+#include "utils/lh_utils.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+   int order = 1;
+   int par_ref_levels = 0;
+   int ser_ref_levels = 0;
+
+   real_t rnum=1.5;
+   real_t mu = 1.257;
+   real_t eps0 = 8.8541878128;
+   real_t cfactor = 1e-6;
+
+   bool static_cond = false;
+   bool visualization = false;
+   bool paraview = false;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&ser_ref_levels, "-sr", "--serial-refinement_levels",
+                  "Number of serial refinement levels.");                  
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&a0, "-a0", "--a0", "P(r) first parameter.");
+   args.AddOption(&a1, "-a1", "--a1", "P(r) second parameter.");
+   args.AddOption(&delta, "-delta", "--delta", "stability parameter.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (Mpi::Root())
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (Mpi::Root())
+   {
+      args.PrintOptions(cout);
+   }
+
+   // number of diffusion equations
+   int ndiffusionequations = 2;
+
+   Vector cvals(ndiffusionequations);
+   Vector csigns(ndiffusionequations);
+   cvals(0)  = 25e6;  cvals(1)  = 1e6;
+   csigns(0) = 1.0;  csigns(1) = -1.0;
+   cvals *= cfactor; // scale the coefficients
+   real_t omega = 2.*M_PI*rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   MFEM_VERIFY(dim == 2, "Dimension != 2 is not supported in this example");
+
+   for (int i = 0; i < ser_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   Array<int> int_bdr_attr;
+   for (int i = 0; i < mesh.GetNBE(); i++)
+   {
+      if (mesh.FaceIsInterior(mesh.GetBdrElementFaceIndex(i)))
+      {
+         int_bdr_attr.Append(mesh.GetBdrAttribute(i));
+      }
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   for (int i = 0; i < par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }  
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+
+   // Define coefficients
+   ConstantCoefficient muinv(1./mu);
+   ConstantCoefficient one_cf(1.0);
+
+   ConstantCoefficient negomegeps0_cf(-omega*eps0);
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_vcf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   // S(r) 
+   FunctionCoefficient S_cf_r(sfunc_r), S_cf_i(sfunc_i);
+   // P(r) 
+   FunctionCoefficient P_cf_r(pfunc_r), P_cf_i(pfunc_i); 
+
+   VectorFunctionCoefficient b_cf(dim,bfunc);// b
+   ScalarVectorProductCoefficient scaled_b_cf(sqrt(cfactor), b_cf);
+   ConstantCoefficient diff_coeff(cfactor);
+
+   MatrixFunctionCoefficient bb_cf(dim,bcrossb); // b⊗b
+   MatrixSumCoefficient oneminusbb(Mone_cf, bb_cf, 1.0, -1.0); // 1 - b⊗b
+
+   // S(r) (I - b⊗b)
+   ScalarMatrixProductCoefficient Soneminusbb_r(S_cf_r, oneminusbb), Soneminusbb_i(S_cf_i, oneminusbb); 
+
+   // P(r) b⊗b 
+   ScalarMatrixProductCoefficient P_cf_bb_r(P_cf_r, bb_cf), P_cf_bb_i(P_cf_i, bb_cf); 
+
+   // ε = S(r) (I - b⊗b) + P(r) b⊗b 
+   MatrixSumCoefficient eps_r(Soneminusbb_r, P_cf_bb_r, 1.0, 1.0); 
+   MatrixSumCoefficient eps_i(Soneminusbb_i, P_cf_bb_i, 1.0, 1.0); 
+
+   coefs_r[nattr-1] = &eps_r;
+   coefs_i[nattr-1] = &eps_i;
+
+   PWMatrixCoefficient eps_cf_r(dim, attr, coefs_r);
+   PWMatrixCoefficient eps_cf_i(dim, attr, coefs_i);
+
+   ConstantCoefficient eps0omeg(omega * eps0);
+   ConstantCoefficient negeps0omeg(-omega * eps0);
+   ConstantCoefficient negeps0omeg2(-omega * omega * eps0);
+   ConstantCoefficient eps0omeg2(omega * omega * eps0);
+
+   ScalarMatrixProductCoefficient m_cf_r(negeps0omeg2, eps_cf_r);
+   ScalarMatrixProductCoefficient m_cf_i(negeps0omeg2, eps_cf_i);
+
+   // ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient eps0omeg_eps_r(eps0omeg, eps_cf_r);
+   // ω ϵ₀ ϵᵢ
+   ScalarMatrixProductCoefficient eps0omeg_eps_i(eps0omeg, eps_cf_i);
+   // -ω ϵ₀ ϵᵣ 
+   ScalarMatrixProductCoefficient negeps0omeg_eps_r(negeps0omeg, eps_cf_r);
+   // -ω ϵ₀ ϵᵢ
+   ScalarMatrixProductCoefficient negeps0omeg_eps_i(eps0omeg, eps_cf_i);
+
+   // A = [0 1; -1 0]
+   DenseMatrix rot_mat(2);
+   rot_mat(0,0) = 0.; rot_mat(0,1) = 1.;
+   rot_mat(1,0) = -1.; rot_mat(1,1) = 0.;
+   MatrixConstantCoefficient rot(rot_mat);
+
+   // ω ϵ₀ ϵᵣ A
+   MatrixProductCoefficient eps0omeg_eps_r_rot(eps0omeg_eps_r, rot);
+   // ω ϵ₀ ϵᵢ A
+   MatrixProductCoefficient eps0omeg_eps_i_rot(eps0omeg_eps_i, rot);
+   // -ω ϵ₀ ϵᵣ A
+   MatrixProductCoefficient negeps0omeg_eps_r_rot(negeps0omeg_eps_r, rot);
+   // -ω ϵ₀ ϵᵢ A
+   MatrixProductCoefficient negeps0omeg_eps_i_rot(negeps0omeg_eps_i, rot);
+
+   Array<Vector *> c_arrays(ndiffusionequations);
+   Array<PWConstCoefficient *> pw_c_coeffs(ndiffusionequations);
+   Array<MatrixCoefficient *> cPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> cPibb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPrbb_cf(ndiffusionequations);
+   Array<MatrixCoefficient *> signedcPibb_cf(ndiffusionequations);
+
+   Vector temp(nattr); temp=0.0;
+   Array<ConstantCoefficient *> c_coeffs(ndiffusionequations);
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      temp[nattr-1] = cvals(i);
+      // temp = cvals(i);
+      pw_c_coeffs[i] = new PWConstCoefficient(temp);
+      c_coeffs[i] = new ConstantCoefficient(cvals(i));
+      cPrbb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_r);
+      cPibb_cf[i] = new ScalarMatrixProductCoefficient(*pw_c_coeffs[i], P_cf_bb_i);
+      signedcPrbb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPrbb_cf[i]);
+      signedcPibb_cf[i] = new ScalarMatrixProductCoefficient(csigns[i], *cPibb_cf[i]);
+   }
+
+   // Define the spaces
+   Array<FiniteElementCollection *> fem_fecols;
+   Array<ParFiniteElementSpace *> fem_pfes;
+
+
+   ND_FECollection *nd_fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *nd_pfes = new ParFiniteElementSpace(&pmesh, nd_fec);
+
+   // Vector H1 spaces for Jᵢ 
+   for (int i = 0; i < ndiffusionequations; i++)
+   {
+      fem_fecols.Append(new H1_FECollection(order, dim));
+      fem_pfes.Append(new ParFiniteElementSpace(&pmesh, fem_fecols.Last(), dim));
+   }
+
+
+   HYPRE_BigInt nd_tdofs = nd_pfes->GlobalTrueVSize();
+   if (Mpi::Root())
+   {
+      cout << "ND ParFiniteElementSpace has " << nd_tdofs
+           << " true dofs." << endl;
+   }
+   Array<HYPRE_BigInt> fem_tdofs(fem_pfes.Size());
+   for (int i = 0; i < fem_pfes.Size(); ++i)
+   {
+      fem_tdofs[i] = fem_pfes[i]->GlobalTrueVSize();
+      if (Mpi::Root())
+      {
+         cout << "FEM ParFiniteElementSpace " << i << " has " << fem_tdofs[i]
+              << " true dofs." << endl;
+      }
+   }
+   if (Mpi::Root())
+   {
+      cout << "Total number of ND true dofs for Maxwell:     " << nd_tdofs << endl;
+      cout << "Total number of FEM true dofs for VDiffusion: " << fem_tdofs.Sum() << endl;
+   }
+
+   int nfem_pfes = fem_pfes.Size();
+   Array<int> fem_offsets(nfem_pfes+1);  fem_offsets[0] = 0;
+   Array<int> fem_toffsets(nfem_pfes+1); fem_toffsets[0] = 0;
+   for (int i = 0; i<nfem_pfes; i++)
+   {
+      fem_offsets[i+1] = fem_pfes[i]->GetVSize();
+      fem_toffsets[i+1] = fem_pfes[i]->TrueVSize();
+   }
+   fem_offsets.PartialSum();
+   fem_toffsets.PartialSum();
+
+   Vector nd_x(2*nd_pfes->GetVSize());
+   nd_x = 0.;
+   Vector fem_x(2*fem_offsets.Last());
+   fem_x = 0.;
+
+   Vector fem_b(2*fem_offsets.Last());
+   fem_b = 0.;
+
+   ParGridFunction * nd_pgf_r = new ParGridFunction(nd_pfes, nd_x, 0);
+   ParGridFunction * nd_pgf_i = new ParGridFunction(nd_pfes, nd_x, nd_x.Size()/2);
+   Array<ParGridFunction *> fem_pgf_r(nfem_pfes);
+   Array<ParGridFunction *> fem_pgf_i(nfem_pfes);
+
+   Array<ParGridFunction *> old_fem_pgf_r(nfem_pfes);
+   Array<ParGridFunction *> old_fem_pgf_i(nfem_pfes);
+
+   for (int i = 0; i < nfem_pfes; ++i)
+   {
+      fem_pgf_r[i] = new ParGridFunction(fem_pfes[i], fem_x, fem_offsets[i]);
+      fem_pgf_i[i] = new ParGridFunction(fem_pfes[i], fem_x, fem_offsets.Last() + fem_offsets[i]);
+      old_fem_pgf_r[i] = new ParGridFunction(fem_pfes[i]); *old_fem_pgf_r[i] = 0.0;
+      old_fem_pgf_i[i] = new ParGridFunction(fem_pfes[i]); *old_fem_pgf_i[i] = 0.0;
+   }
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+   ParGridFunction E_par_r(&L2_fes);
+   ParGridFunction E_par_i(&L2_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+   std::string output_dir = "ParaView/FEM/WeakCoupling/" + GetTimestamp();
+
+   if (paraview)
+   {
+      if (Mpi::Root()) { WriteParametersToFile(args, output_dir); }
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",nd_pgf_r);
+      paraview_dc->RegisterField("E_i",nd_pgf_i);
+      paraview_dc->RegisterField("E_par_r",&E_par_r);
+      paraview_dc->RegisterField("E_par_i",&E_par_i);
+      paraview_dc->RegisterField("Jh_1_r",fem_pgf_r[0]);
+      paraview_dc->RegisterField("Jh_1_i",fem_pgf_i[0]);
+      paraview_dc->RegisterField("Jh_2_r",fem_pgf_r[1]);
+      paraview_dc->RegisterField("Jh_2_i",fem_pgf_i[1]);
+      paraview_dc->Save();
+   }
+
+
+   
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listJ;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+   
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+   
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }   
+
+
+   int max_fixed_point_iter = 50;
+   // loop through fixed point iterations
+
+   for (int k = 0; k<max_fixed_point_iter; k++)
+   {
+      nd_x = 0.0;
+
+      // reset GridFunctions
+      *nd_pgf_r = 0.0;
+      *nd_pgf_i = 0.0;
+
+      delta = (k == 0) ? 0.01 : 0.01;
+      // delta = 0.01 * pow(0.5, 0.0);
+      delta = 0.01 * pow(0.9, k);
+      ParSesquilinearForm a_nd(nd_pfes);
+
+      // (1/μ₀ ∇×E, ∇ × F)
+      a_nd.AddDomainIntegrator(new CurlCurlIntegrator(muinv), nullptr);
+      // - ω² ϵ₀ (ϵᵣ E, F)
+      a_nd.AddDomainIntegrator(new VectorFEMassIntegrator(m_cf_r),
+                          new VectorFEMassIntegrator(m_cf_i));
+      a_nd.Assemble();
+
+      VectorGridFunctionCoefficient J1_cf_r(fem_pgf_r[0]);
+      VectorGridFunctionCoefficient J1_cf_i(fem_pgf_i[0]);
+      VectorGridFunctionCoefficient J2_cf_r(fem_pgf_r[1]);
+      VectorGridFunctionCoefficient J2_cf_i(fem_pgf_i[1]);
+
+      // -iω²ϵ₀ (Jᵣ + i Jᵢ ,F) = ω² ϵ₀ (Jᵢ - i Jᵣ,F)   
+      ScalarVectorProductCoefficient negomeg2eps0_J1_cf_i(negeps0omeg2, J1_cf_i);
+      ScalarVectorProductCoefficient negomeg2eps0_J2_cf_i(negeps0omeg2, J2_cf_i);
+
+      ScalarVectorProductCoefficient omeg2eps0_J1_cf_i(eps0omeg2, J1_cf_i);
+      ScalarVectorProductCoefficient omeg2eps0_J2_cf_i(eps0omeg2, J2_cf_i);
+
+      ScalarVectorProductCoefficient negomeg2eps0_J1_cf_r(negeps0omeg2, J1_cf_r);
+      ScalarVectorProductCoefficient negomeg2eps0_J2_cf_r(negeps0omeg2, J2_cf_r);
+
+
+      ScalarVectorProductCoefficient omeg2eps0_J1_cf_r(eps0omeg2, J1_cf_r);
+      ScalarVectorProductCoefficient omeg2eps0_J2_cf_r(eps0omeg2, J2_cf_r);
+
+
+      ParComplexLinearForm nd_b(nd_pfes);
+
+      // ω² ϵ₀ (Jᵢ - i Jᵣ,F) = (ω² ϵ₀ Jᵢ, F) + i (-ω² ϵ₀Jᵢ,F)  
+      // nd_b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(omeg2eps0_J1_cf_i),
+      //                          new VectorFEDomainLFIntegrator(negomeg2eps0_J1_cf_r));
+      // nd_b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(omeg2eps0_J2_cf_i),
+      //                          new VectorFEDomainLFIntegrator(negomeg2eps0_J2_cf_r));    
+      
+
+      ScalarVectorProductCoefficient omeg2_eps0_J1_cf_i(eps0*omega*omega, J1_cf_i);
+      ScalarVectorProductCoefficient omeg2_eps0_J2_cf_i(eps0*omega*omega, J2_cf_i);
+
+      ScalarVectorProductCoefficient negomeg2_eps0_J1_cf_r(-eps0*omega*omega, J1_cf_r);
+      ScalarVectorProductCoefficient negomeg2_eps0_J2_cf_r(-eps0*omega*omega, J2_cf_r);
+      
+      nd_b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(omeg2_eps0_J1_cf_i),
+                               new VectorFEDomainLFIntegrator(negomeg2_eps0_J1_cf_r));
+      nd_b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(omeg2_eps0_J2_cf_i),
+                               new VectorFEDomainLFIntegrator(negomeg2_eps0_J2_cf_r));  
+
+      // nd_b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(negomeg2eps0_J1_cf_i), nullptr);
+      // ConstantCoefficient temp(1.0);
+      // nd_b.AddDomainIntegrator(nullptr,new VectorFEDomainLFIntegrator(J1_cf_r));
+      // nd_b.AddDomainIntegrator(nullptr,new VectorFEDomainLFIntegrator(omeg2eps0_J2_cf_r));
+      
+
+      nd_b.Assemble();
+
+      // remove internal boundaries
+      ess_bdr = 1;
+      for (int i = 0; i<int_bdr_attr.Size(); i++)
+      {
+         ess_bdr[int_bdr_attr[i]-1] = 0;
+      }
+
+      nd_pfes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+      // if (k == 0)
+      // {
+      nd_pgf_r->ProjectBdrCoefficientTangent(one_x_cf, one_r_bdr);
+      nd_pgf_r->ProjectBdrCoefficientTangent(negone_x_cf, negone_r_bdr);
+      nd_pgf_i->ProjectBdrCoefficientTangent(one_x_cf, one_i_bdr);
+      nd_pgf_i->ProjectBdrCoefficientTangent(negone_x_cf, negone_i_bdr);
+      // }
+
+      OperatorPtr nd_Ah;
+      Vector nd_X,nd_B;
+      a_nd.FormLinearSystem(ess_tdof_list,nd_x,nd_b,nd_Ah, nd_X,nd_B);
+
+
+      ComplexHypreParMatrix *nd_A = nd_Ah.As<ComplexHypreParMatrix>();
+
+      ComplexMUMPSSolver nd_mumps(MPI_COMM_WORLD);
+      nd_mumps.SetPrintLevel(0);
+      nd_mumps.SetOperator(*nd_A);
+      nd_mumps.Mult(nd_B, nd_X);
+
+      a_nd.RecoverFEMSolution(nd_X, nd_b, nd_x);
+
+      nd_pgf_r->MakeRef(nd_pfes, nd_x, 0);
+      nd_pgf_i->MakeRef(nd_pfes, nd_x, nd_x.Size()/2);
+
+
+      fem_x = 0.0;
+      fem_b = 0.0;
+      for (int i = 0; i < nfem_pfes; ++i)
+      {
+         *fem_pgf_r[i] = 0.0;
+         *fem_pgf_i[i] = 0.0;
+      }
+
+      // Diffusion equations
+      ParComplexBlockForm a_fem(fem_pfes);
+      for (int i = 0; i < ndiffusionequations; i++)
+      {
+         // ((b⋅∇)Jᵢ, (b⋅∇) δJᵢ)
+         a_fem.AddDomainIntegrator(new DirectionalVectorDiffusionIntegrator(scaled_b_cf), 
+                                   nullptr,i,i);
+         // cᵢ(Jᵢ, δJᵢ)
+         a_fem.AddDomainIntegrator(new VectorMassIntegrator(*pw_c_coeffs[i]), 
+                                   nullptr,i,i);
+      }
+
+      // FEM RHS
+      // ±cᵢ(P(r) (b ⊗ b) E, δJᵢ)  
+      real_t * fembdata = fem_b.GetData();
+      ParLinearForm b_J1_r(fem_pfes[0],fembdata);
+      ParLinearForm b_J2_r(fem_pfes[1],fembdata + fem_offsets[1]);
+
+      ParLinearForm b_J1_i(fem_pfes[0],fembdata + fem_offsets.Last());
+      ParLinearForm b_J2_i(fem_pfes[1],fembdata + fem_offsets.Last() + fem_offsets[1]);
+      
+      VectorGridFunctionCoefficient E_r_cf(nd_pgf_r);
+      VectorGridFunctionCoefficient E_i_cf(nd_pgf_i);
+
+      // (a + i b) * (c + i d) = (ac - bd) + i (ad + bc)
+      MatrixVectorProductCoefficient c1_cf_rr(*signedcPrbb_cf[0], E_r_cf);
+      MatrixVectorProductCoefficient c1_cf_ii(*signedcPibb_cf[0], E_i_cf);
+      MatrixVectorProductCoefficient c1_cf_ri(*signedcPrbb_cf[0], E_i_cf);
+      MatrixVectorProductCoefficient c1_cf_ir(*signedcPibb_cf[0], E_r_cf);
+
+      VectorSumCoefficient c1_cf_r(c1_cf_rr, c1_cf_ii, 1.0, -1.0);
+      VectorSumCoefficient c1_cf_i(c1_cf_ri, c1_cf_ir, 1.0,  1.0);
+
+      MatrixVectorProductCoefficient c2_cf_rr(*signedcPrbb_cf[1], E_r_cf);
+      MatrixVectorProductCoefficient c2_cf_ii(*signedcPibb_cf[1], E_i_cf);
+      MatrixVectorProductCoefficient c2_cf_ri(*signedcPrbb_cf[1], E_i_cf);
+      MatrixVectorProductCoefficient c2_cf_ir(*signedcPibb_cf[1], E_r_cf); 
+
+      VectorSumCoefficient c2_cf_r(c2_cf_rr, c2_cf_ii, 1.0, -1.0);
+      VectorSumCoefficient c2_cf_i(c2_cf_ri, c2_cf_ir, 1.0,  1.0);
+
+
+      b_J1_r.AddDomainIntegrator(new VectorDomainLFIntegrator(c1_cf_r));
+      b_J1_i.AddDomainIntegrator(new VectorDomainLFIntegrator(c1_cf_i));
+      b_J2_r.AddDomainIntegrator(new VectorDomainLFIntegrator(c2_cf_r));
+      b_J2_i.AddDomainIntegrator(new VectorDomainLFIntegrator(c2_cf_i));
+      b_J1_r.Assemble();
+      b_J1_i.Assemble();
+      b_J2_r.Assemble();
+      b_J2_i.Assemble();
+
+
+      a_fem.Assemble();
+
+
+      ess_bdr=1;
+      ess_tdof_list.SetSize(0);
+      for (int i = 0; i<ndiffusionequations;i++)
+      {
+         ess_tdof_listJ.SetSize(0);
+         fem_pfes[i]->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJ);
+         for (int j = 0; j < ess_tdof_listJ.Size(); j++)
+         {
+            ess_tdof_listJ[j] += fem_toffsets[i];
+         }
+         ess_tdof_list.Append(ess_tdof_listJ);
+      }
+
+      OperatorPtr fem_Ah;
+      Vector fem_X,fem_B;
+
+      a_fem.FormLinearSystem(ess_tdof_list,fem_x,fem_b,fem_Ah, fem_X,fem_B);
+
+
+      ComplexOperator * fem_Ahc = fem_Ah.As<ComplexOperator>();
+
+      BlockOperator * fem_BlockA_r = dynamic_cast<BlockOperator *>(&fem_Ahc->real());
+      BlockOperator * fem_BlockA_i = dynamic_cast<BlockOperator *>(&fem_Ahc->imag());
+
+      int fem_nblocks = fem_BlockA_r->NumRowBlocks();
+
+      Array2D<const HypreParMatrix*> fem_A_r_matrices(fem_nblocks, fem_nblocks);
+      Array2D<const HypreParMatrix*> fem_A_i_matrices(fem_nblocks, fem_nblocks);
+      for (int i = 0; i < fem_nblocks; i++)
+      {
+         for (int j = 0; j < fem_nblocks; j++)
+         {
+            fem_A_r_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&fem_BlockA_r->GetBlock(i,j));
+            fem_A_i_matrices(i,j) = dynamic_cast<HypreParMatrix*>(&fem_BlockA_i->GetBlock(i,j));
+         }
+      }
+
+      HypreParMatrix * fem_Ahr = HypreParMatrixFromBlocks(fem_A_r_matrices);
+      HypreParMatrix * fem_Ahi = HypreParMatrixFromBlocks(fem_A_i_matrices);
+
+      ComplexHypreParMatrix * fem_Ahc_hypre =
+         new ComplexHypreParMatrix(fem_Ahr, fem_Ahi,false, false);
+
+      if (Mpi::Root())
+      {
+         mfem::out << "FEM Assembly finished successfully." << endl;
+      }   
+
+
+      Array<int> fem_tdof_offsets(2*fem_nblocks+1);
+      fem_tdof_offsets[0] = 0;
+      for (int i=0; i<fem_nblocks; i++)
+      {
+         fem_tdof_offsets[i+1] = fem_A_r_matrices(i,i)->Height();
+         fem_tdof_offsets[fem_nblocks+i+1] = fem_tdof_offsets[i+1];
+      }
+      fem_tdof_offsets.PartialSum();
+
+      BlockDiagonalPreconditioner fem_M(fem_tdof_offsets);
+
+      HypreBoomerAMG * solver_J1 = new HypreBoomerAMG((HypreParMatrix &)
+                                   fem_BlockA_r->GetBlock(0,0));
+      solver_J1->SetPrintLevel(0);
+      solver_J1->SetSystemsOptions(dim);
+      HypreBoomerAMG * solver_J2 = new HypreBoomerAMG((HypreParMatrix &)
+                                   fem_BlockA_r->GetBlock(1,1));
+      solver_J2->SetPrintLevel(0);
+      fem_M.SetDiagonalBlock(0,solver_J1);
+      fem_M.SetDiagonalBlock(1,solver_J2);
+      fem_M.SetDiagonalBlock(fem_nblocks,solver_J1);
+      fem_M.SetDiagonalBlock(fem_nblocks+1,solver_J2);
+
+
+      CGSolver fem_cg(MPI_COMM_WORLD);
+      fem_cg.SetRelTol(1e-16);
+      fem_cg.SetMaxIter(1000);
+      fem_cg.SetPrintLevel(1);
+      fem_cg.SetPreconditioner(fem_M);
+      fem_cg.SetOperator(*fem_Ahc_hypre);
+      fem_cg.Mult(fem_B, fem_X);
+
+      a_fem.RecoverFEMSolution(fem_X, fem_x);
+
+
+      real_t alpha = (k == 0) ? 1.0 : 0.01;
+
+      for (int i = 0; i < nfem_pfes; ++i)
+      {
+         fem_pgf_r[i]->MakeRef(fem_pfes[i], fem_x, fem_offsets[i]);
+         fem_pgf_i[i]->MakeRef(fem_pfes[i], fem_x, fem_offsets.Last() + fem_offsets[i]);
+
+         (*fem_pgf_r[i])*= alpha;
+         (*fem_pgf_i[i])*= alpha;
+         fem_pgf_r[i]->Add(1.0-alpha, *old_fem_pgf_r[i]);
+         fem_pgf_i[i]->Add(1.0-alpha, *old_fem_pgf_i[i]);
+         *old_fem_pgf_r[i] = *fem_pgf_r[i];
+         *old_fem_pgf_i[i] = *fem_pgf_i[i];
+      }
+
+
+      ParallelECoefficient par_e_r(nd_pgf_r);
+      ParallelECoefficient par_e_i(nd_pgf_i);
+      E_par_r.ProjectCoefficient(par_e_r);
+      E_par_i.ProjectCoefficient(par_e_i);      
+
+      if (paraview)
+      {
+         paraview_dc->SetCycle(k);
+         paraview_dc->SetTime((real_t)k);
+         paraview_dc->Save();
+      }
+
+   }
+
+   for (int i = 0; i<ndiffusionequations; i++)
+   {
+      delete pw_c_coeffs[i];
+      delete c_coeffs[i];
+      delete cPrbb_cf[i]; 
+      delete cPibb_cf[i]; 
+      delete signedcPrbb_cf[i]; 
+      delete signedcPibb_cf[i]; 
+   }
+
+
+
+   delete nd_fec;
+   delete nd_pfes;
+   for (int i = 0; i < fem_fecols.Size(); ++i)
+   {
+      delete fem_fecols[i];
+      delete fem_pfes[i];
+   }
+
+   if (paraview_dc) 
+   { 
+      delete paraview_dc; 
+   }
+
+   return 0;
+
+}

miniapps/dpg/plasma/lh-primal-dpg.cpp

@@ -0,0 +1,479 @@
+
+// srun -n 256 ./pmaxwell-primal-tokamak -o 3 -sc -rnum
+// srun -n 448 ./pmaxwell-primal-tokamak -o 4 -sc -rnum 11.0 -paraview
+
+// srun -n 448 ./pmaxwell-primal-tokamak -o 4 -do 0 -sc -paraview (with the new epsilon GridFunction coefficients)
+// Description:
+// This example code demonstrates the use of MFEM to define and solve
+// the "ultraweak" (UW) DPG formulation for the Maxwell problem
+
+//      ∇×(1/μ ∇×E) - ω^2 ϵ₀ ϵ E = 0 ,   in Ω
+//                           E×n = E_0, on ∂Ω
+
+// The primal-DPG formulation is obtained by integration by parts
+// and the introduction of trace unknowns on the mesh skeleton
+
+// in 3D
+// E ∈ H(curl)
+// Ê ∈ H_0^1/2(Ω)(curl, Γ_h)
+//  1/μ (∇×E , ∇×F) + (ω^2 ϵ₀ ϵ , F) + <Ê , F × n> = 0,      ∀ F ∈ H(curl,Ω)
+//                                           Ê × n = E_0     on ∂Ω
+
+#include "mfem.hpp"
+#include "../util/utils.hpp"
+#include "../util/maxwell_utils.hpp"
+#include "utils/lh_utils.hpp"
+#include "../util/pcomplexweakform.hpp"
+#include "../../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "data/LH_hot.msh";
+   int order = 2;
+   int delta_order = 1;
+   int par_ref_levels = 0;
+   int amr_ref_levels = 0;
+
+   // real_t rnum=4.6e9;
+   // real_t mu = 1.257e-6/factor;
+   // real_t epsilon_scale = 8.8541878128e-12*factor;
+   real_t rnum=1.5;
+   real_t mu = 1.257;
+   real_t eps0 = 8.8541878128;
+   bool mumps_solver = false;
+   real_t theta = 0.0;
+
+   bool visualization = false;
+   bool static_cond = false;
+   bool paraview = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement-levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&amr_ref_levels, "-amr", "--parallel-amr-refinement-levels",
+                  "Number of parallel AMR refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+#ifdef MFEM_USE_MUMPS
+   args.AddOption(&mumps_solver, "-mumps", "--mumps-solver", "-no-mumps",
+                  "--no-mumps-solver", "Use the MUMPS Solver.");
+#endif
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+
+   double omega = 2.*M_PI*rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   mesh.RemoveInternalBoundaries();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+
+   for (int i = 0; i<par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+   }
+
+   FiniteElementCollection *E_fec = new ND_FECollection(order,dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh,E_fec);
+
+   // H^-1/2 (curl) space for Ê (in 2D H1 trace, in 3D ND trace)
+   int test_order = order+delta_order;
+   FiniteElementCollection * hatE_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementCollection * F_fec = new ND_FECollection(test_order, dim);
+   ParFiniteElementSpace *hatE_fes = new ParFiniteElementSpace(&pmesh,hatE_fec);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+   trial_fes.Append(E_fes);
+   trial_fes.Append(hatE_fes);
+   test_fec.Append(F_fec);
+
+   int gdofs = 0;
+   for (int i = 0; i<trial_fes.Size(); i++)
+   {
+      gdofs += trial_fes[i]->GlobalTrueVSize();
+   }
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Global number of dofs = " << gdofs << endl;
+   }
+
+   // Define the coefficients 
+   ConstantCoefficient muinv(1./mu);
+   ConstantCoefficient one(1.0);
+
+   int nattr = (pmesh.attributes.Size()) ? pmesh.attributes.Max() : 0;
+   Array<int> attr(nattr);
+   for (int i = 0; i<nattr; i++) { attr[i] = i+1; }
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_vcf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   DenseMatrix Mone(dim); 
+   Mone = 0.0; Mone(0,0) = Mone(1,1) = 1.0;
+   MatrixConstantCoefficient Mone_cf(Mone);
+   DenseMatrix Mzero(dim); Mzero = 0.0;
+   MatrixConstantCoefficient Mzero_cf(Mzero);
+
+   Array<MatrixCoefficient*> coefs_r(nattr);
+   Array<MatrixCoefficient*> coefs_i(nattr);
+   for (int i = 0; i < nattr-1; ++i)
+   {
+      coefs_r[i] = &Mone_cf;
+      coefs_i[i] = &Mzero_cf;
+   }
+
+   // S(r) 
+   FunctionCoefficient S_cf_r(sfunc_r), S_cf_i(sfunc_i);
+   // P(r) 
+   FunctionCoefficient P_cf_r(pfunc_r), P_cf_i(pfunc_i); 
+
+   VectorFunctionCoefficient b_cf(dim,bfunc);// b
+   MatrixFunctionCoefficient bb_cf(dim,bcrossb); // b⊗b
+
+   MatrixSumCoefficient oneminusbb(Mone_cf, bb_cf, 1.0, -1.0); // 1 - b⊗b
+
+   // S(r) (I - b⊗b)
+   ScalarMatrixProductCoefficient Soneminusbb_r(S_cf_r, oneminusbb), Soneminusbb_i(S_cf_i, oneminusbb); 
+
+   // P(r) b⊗b 
+   ScalarMatrixProductCoefficient P_cf_bb_r(P_cf_r, bb_cf), P_cf_bb_i(P_cf_i, bb_cf); 
+
+   // εᵣ = S(r) (I - b⊗b) + P(r) b⊗b 
+   MatrixSumCoefficient eps_r(Soneminusbb_r, P_cf_bb_r, 1.0, 1.0); 
+   MatrixSumCoefficient eps_i(Soneminusbb_i, P_cf_bb_i, 1.0, 1.0); 
+
+   coefs_r[nattr-1] = &eps_r;
+   coefs_i[nattr-1] = &eps_i;
+
+   PWMatrixCoefficient eps_cf_r(dim, attr, coefs_r);
+   PWMatrixCoefficient eps_cf_i(dim, attr, coefs_i);
+
+   ConstantCoefficient negeps0omeg2(-eps0 * omega * omega);
+
+   ScalarMatrixProductCoefficient m_cf_r(negeps0omeg2, eps_cf_r);
+   ScalarMatrixProductCoefficient m_cf_i(negeps0omeg2, eps_cf_i);
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_fes,test_fec);
+
+   // (1/μ₀ ∇ × E,∇ × F)
+   a->AddTrialIntegrator(new CurlCurlIntegrator(muinv), nullptr,0,0);
+
+   // -(ω^2 ϵ₀ ϵ, F)
+   const IntegrationRule *irs[Geometry::NumGeom];
+   int order_quad = 2*order + 2;
+   for (int i = 0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+   const IntegrationRule &ir = IntRules.Get(pmesh.GetElementGeometry(0),
+                                            2*test_order + 2);
+   VectorFEMassIntegrator * integ_r = new VectorFEMassIntegrator(m_cf_r);
+   VectorFEMassIntegrator * integ_i = new VectorFEMassIntegrator(m_cf_i);
+   integ_r->SetIntegrationRule(ir);
+   integ_i->SetIntegrationRule(ir);
+   a->AddTrialIntegrator(integ_r,integ_i,0,0);
+
+   // < n×Ê,F>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,1,0);
+
+   // test integrators
+   // (∇×F ,∇× δF)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,0,0);
+   // (F,δF)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,0,0);
+
+   socketstream E_out_r;
+   socketstream E_theta_out_r;
+
+   ParComplexGridFunction E_gf(E_fes);
+   E_gf.real() = 0.0;
+   E_gf.imag() = 0.0;
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+
+   ParGridFunction E_theta_r(&L2_fes);
+   ParGridFunction E_theta_i(&L2_fes);
+   ParGridFunction E_theta(&L2_fes);
+   E_theta = 0.0;
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+   // Create ParaView directory and file
+   std::string output_dir = "ParaView/PrimalDPG/" + GetTimestamp();
+   if (Mpi::Root())
+   {
+      WriteParametersToFile(args, output_dir);
+   }
+   if (paraview)
+   {
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",&E_gf.real());
+      paraview_dc->RegisterField("E_i",&E_gf.imag());
+      paraview_dc->RegisterField("E_theta_r",&E_theta_r);
+      paraview_dc->RegisterField("E_theta_i",&E_theta_i);
+   }
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+
+   a->Assemble();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+
+      E_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+
+
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = E_fes->GetVSize();
+   offsets[2] = hatE_fes->GetVSize();
+   offsets.PartialSum();
+
+   Vector x(2*offsets.Last());
+   x = 0.;
+   double * xdata = x.GetData();
+
+   E_gf.real().MakeRef(E_fes,&xdata[0]);
+   E_gf.imag().MakeRef(E_fes,&xdata[offsets.Last()]);
+
+
+   E_gf.ProjectBdrCoefficientTangent(one_x_cf,zero_vcf, one_r_bdr);
+   E_gf.ProjectBdrCoefficientTangent(negone_x_cf,zero_vcf, negone_r_bdr);
+   E_gf.ProjectBdrCoefficientTangent(zero_vcf,one_x_cf, one_i_bdr);
+   E_gf.ProjectBdrCoefficientTangent(zero_vcf,negone_x_cf, negone_i_bdr);
+
+   OperatorPtr Ah;
+   Vector X,B;
+   a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+   ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+   BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+   BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+   int num_blocks = BlockA_r->NumRowBlocks();
+   Array<int> tdof_offsets(2*num_blocks+1);
+
+   tdof_offsets[0] = 0;
+   for (int i=0; i<num_blocks; i++)
+   {
+      const int h = BlockA_r->GetBlock(i,i).Height();
+      tdof_offsets[i+1] = h;
+      tdof_offsets[num_blocks+i+1] = h;
+   }
+   tdof_offsets.PartialSum();
+
+   BlockOperator blockA(tdof_offsets);
+   for (int i = 0; i<num_blocks; i++)
+   {
+      for (int j = 0; j<num_blocks; j++)
+      {
+         blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
+         blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
+         blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
+         blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
+      }
+   }
+
+#ifdef MFEM_USE_MUMPS
+   if (mumps_solver)
+   {
+      // Monolithic real part
+      Array2D<const HypreParMatrix * > Ab_r(num_blocks,num_blocks);
+      // Monolithic imag part
+      Array2D<const HypreParMatrix * > Ab_i(num_blocks,num_blocks);
+      for (int i = 0; i<num_blocks; i++)
+      {
+         for (int j = 0; j<num_blocks; j++)
+         {
+            Ab_r(i,j) = &(HypreParMatrix &)BlockA_r->GetBlock(i,j);
+            Ab_i(i,j) = &(HypreParMatrix &)BlockA_i->GetBlock(i,j);
+         }
+      }
+      HypreParMatrix * A_r = HypreParMatrixFromBlocks(Ab_r);
+      HypreParMatrix * A_i = HypreParMatrixFromBlocks(Ab_i);
+
+      ComplexHypreParMatrix Acomplex(A_r, A_i,true,true);
+
+      HypreParMatrix * A = Acomplex.GetSystemMatrix();
+
+      MUMPSSolver mumps(MPI_COMM_WORLD);
+      mumps.SetPrintLevel(0);
+      mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
+      mumps.SetOperator(*A);
+      mumps.Mult(B,X);
+      delete A;
+   }
+#else
+   if (mumps_solver)
+   {
+      MFEM_WARNING("MFEM compiled without mumps. Switching to an iterative solver");
+   }
+   mumps_solver = false;
+#endif
+   if (!mumps_solver)
+   {
+      BlockDiagonalPreconditioner M(tdof_offsets);
+      ParFiniteElementSpace *ams_fes = nullptr;
+      if (static_cond)
+      {
+         ams_fes = new ParFiniteElementSpace(&pmesh,
+                                             E_fes->FEColl()->GetTraceCollection());
+      }
+
+      HypreAMS * solver_E = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(0,0),
+                                          (static_cond) ? ams_fes : E_fes);
+      solver_E->SetPrintLevel(0);
+      HypreBoomerAMG * solver_hatE = new HypreBoomerAMG((HypreParMatrix &)
+                                                         BlockA_r->GetBlock(1,1));
+      solver_hatE->SetPrintLevel(0);
+      solver_hatE->SetRelaxType(88);
+
+      M.SetDiagonalBlock(0,solver_E);
+      M.SetDiagonalBlock(1,solver_hatE);
+      M.SetDiagonalBlock(2,solver_E);
+      M.SetDiagonalBlock(3,solver_hatE);
+
+      if (myid == 0)
+      {
+         std::cout << "PCG iterations" << endl;
+      }
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-8);
+      cg.SetMaxIter(1000);
+      cg.SetPrintLevel(1);
+      cg.SetPreconditioner(M);
+      cg.SetOperator(blockA);
+      cg.Mult(B, X);
+
+      for (int i = 0; i<num_blocks; i++)
+      {
+         delete &M.GetDiagonalBlock(i);
+      }
+
+      //int num_iter = cg.GetNumIterations();
+
+   }
+   a->RecoverFEMSolution(X,x);
+
+   E_gf.real().MakeRef(E_fes,x.GetData());
+   E_gf.imag().MakeRef(E_fes,&x.GetData()[offsets.Last()]);
+
+   AzimuthalECoefficient az_e_r(&E_gf.real());
+   AzimuthalECoefficient az_e_i(&E_gf.imag());
+
+   E_theta_r.ProjectCoefficient(az_e_r);
+   E_theta_i.ProjectCoefficient(az_e_i);
+
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, E_gf.real(),
+                              "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+      common::VisualizeField(E_theta_out_r,vishost, visport, E_theta_r,
+                              "Numerical Electric field (Azimuthal-real)", 501, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime(0.0);
+      paraview_dc->Save();
+   }
+
+   delete a;
+   delete F_fec;
+   delete hatE_fes;
+   delete hatE_fec;
+   delete E_fec;
+   delete E_fes;
+
+   return 0;
+}
+

miniapps/dpg/plasma/makefile

@@ -0,0 +1,145 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/miniapps/dpg/plasma,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+# Include defaults.mk to get XLINKER
+DEFAULTS_MK = $(MFEM_DIR)/config/defaults.mk
+include $(DEFAULTS_MK)
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+DPG_COMPLEX_SEQ_SRC = ../util/utils.cpp ../util/complexweakform.cpp \
+	../util/complexstaticcond.cpp ../util/complexblockform.cpp ../util/preconditioners.cpp
+DPG_COMPLEX_PAR_SRC = $(DPG_COMPLEX_SEQ_SRC) ../util/pcomplexweakform.cpp ../util/pcomplexblockform.cpp ../util/blockcomplexhypremat.cpp  
+DPG_COMPLEX_OBJ = $(DPG_COMPLEX_PAR_SRC:.cpp=.o)
+
+LH_UTILS_SRC = utils/lh_utils.cpp ../util/maxwell_utils.cpp
+LH_UTILS_OBJ = $(LH_UTILS_SRC:.cpp=.o)
+
+LH_PRIMAL_SRC = lh-primal-dpg.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_PRIMAL_OBJ = $(LH_PRIMAL_SRC:.cpp=.o)
+
+LH_ELD_FEM_SRC = lh-eld-fem.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_ELD_FEM_OBJ = $(LH_ELD_FEM_SRC:.cpp=.o)
+
+LH_ELD_FEM_SYMM_SRC = lh-eld-fem-symm.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_ELD_FEM_SYMM_OBJ = $(LH_ELD_FEM_SYMM_SRC:.cpp=.o)
+
+LH_ELD_DPG_SRC = lh-eld-dpg.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_ELD_DPG_OBJ = $(LH_ELD_DPG_SRC:.cpp=.o)
+
+LH_ELD_FOSLS_DPG_SRC = lh-eld-fosls-dpg.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_ELD_FOSLS_DPG_OBJ = $(LH_ELD_FOSLS_DPG_SRC:.cpp=.o)
+
+LH_ELD_FOSLS_FEM_SRC = lh-eld-fosls-fem.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_ELD_FOSLS_FEM_OBJ = $(LH_ELD_FOSLS_FEM_SRC:.cpp=.o)
+
+LH_ELD_DPG_FEM_SRC = lh-eld-dpg-fem.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)	
+LH_ELD_DPG_FEM_OBJ = $(LH_ELD_DPG_FEM_SRC:.cpp=.o)
+
+LH_ELD_WEAK_SRC = lh-eld-weak.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_ELD_WEAK_OBJ = $(LH_ELD_WEAK_SRC:.cpp=.o)
+
+LH_ELD_DPG_FEM_WEAK_SRC = lh-eld-dpg-fem-weak.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_ELD_DPG_FEM_WEAK_OBJ = $(LH_ELD_DPG_FEM_WEAK_SRC:.cpp=.o)
+
+LH_ELD_UW_SRC = lh-eld-uw.cpp $(DPG_COMPLEX_PAR_SRC) $(LH_UTILS_SRC)
+LH_ELD_UW_OBJ = $(LH_ELD_UW_SRC:.cpp=.o)
+
+
+SEQ_MINIAPPS = 
+PAR_MINIAPPS = lh-primal-dpg lh-eld-fem lh-eld-fem-symm lh-eld-dpg  \
+ lh-eld-dpg-fem lh-eld-weak lh-eld-dpg-fem-weak lh-eld-fosls-dpg\
+ lh-eld-fosls-fem lh-eld-uw
+
+ifeq ($(MFEM_USE_MPI),NO)
+   MINIAPPS = $(SEQ_MINIAPPS)
+else
+   MINIAPPS = $(PAR_MINIAPPS) $(SEQ_MINIAPPS)
+endif
+
+COMMON_LIB = -L$(MFEM_BUILD_DIR)/miniapps/common -lmfem-common
+
+# If MFEM_SHARED is set, add the ../common rpath
+COMMON_LIB += $(if $(MFEM_SHARED:YES=),,\
+   $(if $(MFEM_USE_CUDA:YES=),$(CXX_XLINKER),$(CUDA_XLINKER))-rpath,$(abspath\
+   $(MFEM_BUILD_DIR)/miniapps/common))
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all lib-common clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+%.o: %.cpp
+
+%.o: $(SRC)%.cpp $(wildcard $(SRC)%.hpp) $(MFEM_LIB_FILE)\
+ $(CONFIG_MK) | lib-common
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $< -o $@
+
+all: $(MINIAPPS)
+
+lh-primal-dpg: $(LH_PRIMAL_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_PRIMAL_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-fem: $(LH_ELD_FEM_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_FEM_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-fem-symm: $(LH_ELD_FEM_SYMM_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_FEM_SYMM_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-dpg: $(LH_ELD_DPG_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_DPG_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-fosls-dpg: $(LH_ELD_FOSLS_DPG_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_FOSLS_DPG_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-fosls-fem: $(LH_ELD_FOSLS_FEM_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_FOSLS_FEM_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-dpg-fem: $(LH_ELD_DPG_FEM_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_DPG_FEM_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-weak: $(LH_ELD_WEAK_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_WEAK_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-dpg-fem-weak: $(LH_ELD_DPG_FEM_WEAK_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_DPG_FEM_WEAK_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+lh-eld-uw: $(LH_ELD_UW_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(LH_ELD_UW_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+# Rule for building lib-common
+lib-common:
+	$(MAKE) -C $(MFEM_BUILD_DIR)/miniapps/common
+
+MFEM_TESTS = MINIAPPS
+include $(MFEM_TEST_MK)
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(PAR_MINIAPPS) $(SEQ_MINIAPPS)
+	rm -f $(DPG_COMPLEX_OBJ) $(LH_UTILS_OBJ)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -rf ParaView 

miniapps/dpg/plasma/other/makefile

@@ -0,0 +1,121 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../../..
+MFEM_BUILD_DIR ?= ../../../..
+SRC = $(if $(MFEM_DIR:../../../..=),$(MFEM_DIR)/miniapps/dpg/plasma/other,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+# Include defaults.mk to get XLINKER
+DEFAULTS_MK = $(MFEM_DIR)/config/defaults.mk
+include $(DEFAULTS_MK)
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+DPG_COMPLEX_SEQ_SRC = ../../util/utils.cpp ../../util/complexweakform.cpp \
+	../../util/complexstaticcond.cpp ../../util/complexblockform.cpp \
+	../../util/preconditioners.cpp ../../util/maxwell_utils.cpp
+DPG_COMPLEX_PAR_SRC = $(DPG_COMPLEX_SEQ_SRC) ../../util/pcomplexweakform.cpp ../../util/pcomplexblockform.cpp ../../util/blockcomplexhypremat.cpp  
+DPG_COMPLEX_OBJ = $(DPG_COMPLEX_PAR_SRC:.cpp=.o)
+
+MAXWELL_SRC = ../../util/maxwell_utils.cpp
+MAXWELL_OBJ = $(MAXWELL_SRC:.cpp=.o)
+
+PMAXWELL_TOKAMAK_SRC = pmaxwell-tokamak.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_TOKAMAK_OBJ = $(PMAXWELL_TOKAMAK_SRC:.cpp=.o)
+
+PMAXWELL_FEM_TOKAMAK_SRC = pmaxwell-fem-tokamak.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_FEM_TOKAMAK_OBJ = $(PMAXWELL_FEM_TOKAMAK_SRC:.cpp=.o)
+
+PMAXWELL_PRIMAL_TOKAMAK_SRC = pmaxwell-primal-tokamak.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_PRIMAL_TOKAMAK_OBJ = $(PMAXWELL_PRIMAL_TOKAMAK_SRC:.cpp=.o)
+
+PMAXWELL_UW_TOKAMAK_SRC = pmaxwell-uw-tokamak.cpp $(DPG_COMPLEX_PAR_SRC)
+PMAXWELL_UW_TOKAMAK_OBJ = $(PMAXWELL_UW_TOKAMAK_SRC:.cpp=.o)
+
+PMAXWELL_UW_LH_SRC = pmaxwell-uw-lh.cpp $(DPG_COMPLEX_PAR_SRC) $(MAXWELL_SRC)
+PMAXWELL_UW_LH_OBJ = $(PMAXWELL_UW_LH_SRC:.cpp=.o)
+
+PMAXWELL_FEM_LH_SRC = pmaxwell-fem-lh.cpp $(DPG_COMPLEX_PAR_SRC) $(MAXWELL_SRC)
+PMAXWELL_FEM_LH_OBJ = $(PMAXWELL_FEM_LH_SRC:.cpp=.o)
+
+SEQ_MINIAPPS = 
+PAR_MINIAPPS = pmaxwell-tokamak pmaxwell-fem-tokamak \
+ pmaxwell-primal-tokamak pmaxwell-uw-tokamak  \
+ pmaxwell-uw-lh pmaxwell-fem-lh 
+
+ifeq ($(MFEM_USE_MPI),NO)
+   MINIAPPS = $(SEQ_MINIAPPS)
+else
+   MINIAPPS = $(PAR_MINIAPPS) $(SEQ_MINIAPPS)
+endif
+
+COMMON_LIB = -L$(MFEM_BUILD_DIR)/miniapps/common -lmfem-common
+
+# If MFEM_SHARED is set, add the ../common rpath
+COMMON_LIB += $(if $(MFEM_SHARED:YES=),,\
+   $(if $(MFEM_USE_CUDA:YES=),$(CXX_XLINKER),$(CUDA_XLINKER))-rpath,$(abspath\
+   $(MFEM_BUILD_DIR)/miniapps/common))
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all lib-common clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+%.o: %.cpp
+
+%.o: $(SRC)%.cpp $(wildcard $(SRC)%.hpp) $(MFEM_LIB_FILE)\
+ $(CONFIG_MK) | lib-common
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $< -o $@
+
+all: $(MINIAPPS)
+
+pmaxwell-tokamak: $(PMAXWELL_TOKAMAK_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_TOKAMAK_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+pmaxwell-fem-tokamak: $(PMAXWELL_FEM_TOKAMAK_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_FEM_TOKAMAK_OBJ) $(COMMON_LIB) $(MFEM_LIBS)	
+
+pmaxwell-primal-tokamak: $(PMAXWELL_PRIMAL_TOKAMAK_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_PRIMAL_TOKAMAK_OBJ) $(COMMON_LIB) $(MFEM_LIBS)	
+
+pmaxwell-uw-tokamak: $(PMAXWELL_UW_TOKAMAK_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_UW_TOKAMAK_OBJ) $(COMMON_LIB) $(MFEM_LIBS)		
+
+pmaxwell-uw-lh: $(PMAXWELL_UW_LH_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_UW_LH_OBJ) $(COMMON_LIB) $(MFEM_LIBS)		
+
+pmaxwell-fem-lh: $(PMAXWELL_FEM_LH_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PMAXWELL_FEM_LH_OBJ) $(COMMON_LIB) $(MFEM_LIBS)		
+
+# Rule for building lib-common
+lib-common:
+	$(MAKE) -C $(MFEM_BUILD_DIR)/miniapps/common
+
+MFEM_TESTS = MINIAPPS
+include $(MFEM_TEST_MK)
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(PAR_MINIAPPS) $(SEQ_MINIAPPS)
+	rm -f $(DPG_COMPLEX_OBJ) $(MAXWELL_OBJ)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -rf ParaView 

miniapps/dpg/plasma/other/pmaxwell-fem-lh.cpp

@@ -0,0 +1,318 @@
+//                   MFEM Ultraweak DPG Maxwell parallel example
+//
+// the "ultraweak" (UW) DPG formulation for the Maxwell problem
+
+//      ∇×(1/μ ∇×E) - ω² ϵ E = Ĵ ,   in Ω
+//                       E×n = E₀ , on ∂Ω
+
+#include "mfem.hpp"
+#include "../../util/pcomplexweakform.hpp"
+#include "../../util/utils.hpp"
+#include "../../util/maxwell_utils.hpp"
+#include "../../../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+#include <cstring>
+#include <filesystem>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // fine mesh (trianles)
+   // default mesh
+   const char *mesh_file = "data/mesh-tri34K.mesh";
+   // coarse mesh (triangles)
+   // const char *mesh_file = "data/mesh-tri11K.mesh";
+   // coarse mesh (quadrilaterals)
+   // const char *mesh_file = "data/mesh-quad5K.mesh";
+
+   // epsilon tensor
+   const char * eps_r_file = nullptr;
+   const char * eps_i_file = nullptr;
+
+   int order = 2;
+   int par_ref_levels = 0;
+   bool visualization = false;
+   // real_t rnum=4.6e9;
+   // real_t mu = 1.257e-6;
+   // real_t epsilon_scale = 8.8541878128e-12*factor;
+   real_t rnum=4.6;
+   real_t mu = 1.257;
+   real_t epsilon_scale = 8.8541878128;
+
+   bool paraview = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement_levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   if (strcmp(mesh_file, "data/mesh-tri34K.mesh") == 0)
+   {
+      eps_r_file = "data/eps-tri34K_r.gf";
+      eps_i_file = "data/eps-tri34K_i.gf";
+   }
+   else if (strcmp(mesh_file, "data/mesh-tri11K.mesh") == 0)
+   {
+      eps_r_file = "data/eps-tri11K_r.gf";
+      eps_i_file = "data/eps-tri11K_i.gf";
+   }
+   else if (strcmp(mesh_file, "data/mesh-quad5K.mesh") == 0)
+   {
+      eps_r_file = "data/eps-quad5K_r.gf";
+      eps_i_file = "data/eps-quad5K_i.gf";
+   }
+   else
+   {
+      MFEM_ABORT("Unknown mesh file: " + string(mesh_file));
+   }
+
+   real_t omega = 2.*M_PI*rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   Array<int> int_bdr_attr;
+   for (int i = 0; i < mesh.GetNBE(); i++)
+   {
+      if (mesh.FaceIsInterior(mesh.GetBdrElementFaceIndex(i)))
+      {
+         int_bdr_attr.Append(mesh.GetBdrAttribute(i));
+      }
+   }
+   int_bdr_attr.Sort();
+   int_bdr_attr.Unique();
+
+   // mesh.RemoveInternalBoundaries();
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+
+   EpsilonMatrixCoefficient eps_r_cf(eps_r_file,&mesh,&pmesh, epsilon_scale);
+   EpsilonMatrixCoefficient eps_i_cf(eps_i_file,&mesh,&pmesh, epsilon_scale);
+
+   for (int i = 0; i<par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+      eps_r_cf.Update();
+      eps_i_cf.Update();
+   }
+
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh, fec);
+
+   // Bilinear form coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient muinv(1./mu);
+
+   ScalarMatrixProductCoefficient m_cf_r(-omega*omega, eps_r_cf);
+   ScalarMatrixProductCoefficient m_cf_i(-omega*omega, eps_i_cf);
+
+   ParComplexLinearForm *b = new ParComplexLinearForm(E_fes);
+   b->Vector::operator=(0.0);
+
+   ParSesquilinearForm *a = new ParSesquilinearForm(E_fes);
+   a->AddDomainIntegrator(new CurlCurlIntegrator(muinv),nullptr);
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(m_cf_r),
+                          new VectorFEMassIntegrator(m_cf_i));
+
+   socketstream E_out_r;
+   socketstream E_theta_out_r;
+   socketstream E_theta_out_i;
+
+   ParComplexGridFunction E_gf(E_fes);
+   E_gf.real() = 0.0;
+   E_gf.imag() = 0.0;
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+
+   ParGridFunction E_theta_r(&L2_fes);
+   ParGridFunction E_theta_i(&L2_fes);
+   ParGridFunction E_theta(&L2_fes);
+   E_theta = 0.0;
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+   // ParaViewDataCollection * paraview_tdc = nullptr;
+
+   std::string output_dir = "ParaView/FEM/" + GetTimestamp();
+   if (Mpi::Root())
+   {
+      WriteParametersToFile(args, output_dir);
+   }
+
+   if (paraview)
+   {
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",&E_gf.real());
+      paraview_dc->RegisterField("E_i",&E_gf.imag());
+      paraview_dc->RegisterField("E_theta_r",&E_theta_r);
+      paraview_dc->RegisterField("E_theta_i",&E_theta_i);
+
+      // paraview_tdc = new ParaViewDataCollection(mesh_file, &pmesh);
+      // paraview_tdc->SetPrefixPath("ParaViewFEM2D/TimeHarmonic");
+      // paraview_tdc->SetLevelsOfDetail(order);
+      // paraview_tdc->SetCycle(0);
+      // paraview_tdc->SetDataFormat(VTKFormat::BINARY);
+      // paraview_tdc->SetHighOrderOutput(true);
+      // paraview_tdc->SetTime(0.0); // set the time
+      // paraview_tdc->RegisterField("E_theta_t",&E_theta);
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   Array<int> one_r_bdr;
+   Array<int> one_i_bdr;
+   Array<int> negone_r_bdr;
+   Array<int> negone_i_bdr;
+
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      // remove internal boundaries
+      for (int i = 0; i<int_bdr_attr.Size(); i++)
+      {
+         ess_bdr[int_bdr_attr[i]-1] = 0;
+      }
+
+      E_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      one_r_bdr = 0;  one_i_bdr = 0;
+      negone_r_bdr = 0;  negone_i_bdr = 0;
+
+      // attr = 30,2 (real)
+      one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+      // attr = 26,6 (imag)
+      one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+      // attr = 22,10 (real)
+      negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+      // attr = 18,14 (imag)
+      negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+   }
+
+   Vector zero(dim); zero = 0.0;
+   Vector one_x(dim); one_x = 0.0; one_x(0) = 1.0;
+   Vector negone_x(dim); negone_x = 0.0; negone_x(0) = -1.0;
+   VectorConstantCoefficient zero_cf(zero);
+   VectorConstantCoefficient one_x_cf(one_x);
+   VectorConstantCoefficient negone_x_cf(negone_x);
+
+   E_gf.ProjectBdrCoefficientTangent(one_x_cf,zero_cf, one_r_bdr);
+   E_gf.ProjectBdrCoefficientTangent(negone_x_cf,zero_cf, negone_r_bdr);
+   E_gf.ProjectBdrCoefficientTangent(zero_cf,one_x_cf, one_i_bdr);
+   E_gf.ProjectBdrCoefficientTangent(zero_cf,negone_x_cf, negone_i_bdr);
+
+   b->Assemble();
+   a->Assemble();
+
+   OperatorPtr Ah;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, E_gf, *b, Ah, X, B);
+
+#ifdef MFEM_USE_MUMPS
+   HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+   // auto cpardiso = new CPardisoSolver(A->GetComm());
+   auto solver = new MUMPSSolver(MPI_COMM_WORLD);
+   solver->SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
+   solver->SetPrintLevel(1);
+   solver->SetOperator(*A);
+   solver->Mult(B,X);
+   delete A;
+   delete solver;
+#else
+   MFEM_ABORT("MFEM compiled without mumps");
+#endif
+
+   a->RecoverFEMSolution(X, *b, E_gf);
+
+   AzimuthalECoefficient az_e_r(&E_gf.real());
+   AzimuthalECoefficient az_e_i(&E_gf.imag());
+
+   E_theta_r.ProjectCoefficient(az_e_r);
+   E_theta_i.ProjectCoefficient(az_e_i);
+
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, E_gf.real(),
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+      common::VisualizeField(E_theta_out_r,vishost, visport, E_theta_r,
+                             "Numerical Electric field (azimuthal)", 0, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((real_t)0);
+      paraview_dc->Save();
+      delete paraview_dc;
+
+      // int num_frames = 32;
+      // for (int i = 0; i<num_frames; i++)
+      // {
+      //    real_t t = (real_t)(i % num_frames) / num_frames;
+      //    add(cos(real_t(2.0*M_PI)*t), E_theta_r,
+      //        sin(real_t(2.0*M_PI)*t), E_theta_i, E_theta);
+      //    paraview_tdc->SetCycle(i);
+      //    paraview_tdc->SetTime(t);
+      //    paraview_tdc->Save();
+      // }
+      // delete paraview_tdc;
+   }
+
+
+   delete a;
+   delete b;
+   delete E_fes;
+   delete fec;
+
+   return 0;
+
+}

miniapps/dpg/plasma/other/pmaxwell-fem-tokamak.cpp

@@ -0,0 +1,307 @@
+
+// srun -n 256 ./pmaxwell-fem-tokamak -o 3 -sc -rnum
+// srun -n 448 ./pmaxwell-fem-tokamak -o 4 -sc -rnum 11.0 -sigma 2.0 -paraview
+
+// srun -n 448 ./pmaxwell-fem-tokamak -o 4 -paraview
+// Description:
+// This example code demonstrates the use of MFEM to define and solve
+// the standard FEM formulation for the Maxwell problem
+
+//      ∇×(1/μ ∇×E) - (ω^2 ϵ + i ω σ) E = J ,   in Ω
+//                E×n = E_0, on ∂Ω
+
+#include "mfem.hpp"
+#include "../../util/pcomplexweakform.hpp"
+#include "../../../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class EpsilonMatrixCoefficient : public MatrixArrayCoefficient
+{
+private:
+   Mesh * mesh = nullptr;
+   ParMesh * pmesh = nullptr;
+   Array<ParGridFunction * > pgfs;
+   Array<GridFunctionCoefficient * > gf_cfs;
+   GridFunction * vgf = nullptr;
+   int dim;
+public:
+   EpsilonMatrixCoefficient(const char * filename, Mesh * mesh_, ParMesh * pmesh_,
+                            double scale = 1.0)
+      : MatrixArrayCoefficient(mesh_->Dimension()), mesh(mesh_), pmesh(pmesh_),
+        dim(mesh_->Dimension())
+   {
+      std::filebuf fb;
+      fb.open(filename,std::ios::in);
+      std::istream is(&fb);
+      vgf = new GridFunction(mesh,is);
+      fb.close();
+      FiniteElementSpace * vfes = vgf->FESpace();
+      int vdim = vfes->GetVDim();
+      const FiniteElementCollection * fec = vfes->FEColl();
+      FiniteElementSpace * fes = new FiniteElementSpace(mesh, fec);
+      int num_procs = Mpi::WorldSize();
+      int * partitioning = mesh->GeneratePartitioning(num_procs);
+      double *data = vgf->GetData();
+      GridFunction gf;
+      pgfs.SetSize(vdim);
+      gf_cfs.SetSize(vdim);
+      for (int i = 0; i<dim; i++)
+      {
+         for (int j = 0; j<dim; j++)
+         {
+            int k = i*dim+j;
+            // int k = j*dim+i;
+            gf.MakeRef(fes,&data[k*fes->GetVSize()]);
+            pgfs[k] = new ParGridFunction(pmesh,&gf,partitioning);
+            (*pgfs[k])*=scale;
+            gf_cfs[k] = new GridFunctionCoefficient(pgfs[k]);
+            Set(i,j,gf_cfs[k], true);
+         }
+      }
+   }
+   ~EpsilonMatrixCoefficient()
+   {
+      for (int i = 0; i<pgfs.Size(); i++)
+      {
+         delete pgfs[i];
+      }
+      pgfs.DeleteAll();
+   }
+};
+
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "data/mesh_330k.mesh";
+   const char * eps_r_file = "data/eps_r_330k.gf";
+   const char * eps_i_file = "data/eps_i_330k.gf";
+
+   int order = 1;
+   bool visualization = false;
+   double rnum=50.0e6;
+   int sr = 0;
+   int pr = 0;
+   bool paraview = false;
+   double mu = 1.257e-6;
+   double epsilon = 1.0;
+   double epsilon_scale = 8.8541878128e-12;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&sr, "-sref", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pref", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   socketstream E_out_r;
+
+
+   double omega = 2.*M_PI*rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   for (int i = 0; i<sr; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+
+
+   EpsilonMatrixCoefficient eps_r_cf(eps_r_file,&mesh,&pmesh, epsilon_scale);
+   EpsilonMatrixCoefficient eps_i_cf(eps_i_file,&mesh,&pmesh, epsilon_scale);
+
+   mesh.Clear();
+
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh, fec);
+
+   // Bilinear form coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient muinv(1./mu);
+
+   if (myid == 0)
+   {
+      std::cout << "Assembling matrix" << endl;
+   }
+
+   ScalarMatrixProductCoefficient m_cf_r(-omega*omega, eps_r_cf);
+   ScalarMatrixProductCoefficient m_cf_i(-omega*omega, eps_i_cf);
+
+   ParComplexLinearForm *b = new ParComplexLinearForm(E_fes);
+   b->Vector::operator=(0.0);
+
+   ParSesquilinearForm *a = new ParSesquilinearForm(E_fes);
+   a->AddDomainIntegrator(new CurlCurlIntegrator(muinv),nullptr);
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(m_cf_r),
+                          new VectorFEMassIntegrator(m_cf_i));
+
+   ParComplexGridFunction E_gf(E_fes);
+   E_gf.real() = 0.0;
+   E_gf.imag() = 0.0;
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   if (paraview)
+   {
+      paraview_dc = new ParaViewDataCollection(mesh_file, &pmesh);
+      paraview_dc->SetPrefixPath("ParaView");
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",&E_gf.real());
+      paraview_dc->RegisterField("E_i",&E_gf.imag());
+   }
+
+   // internal bdr attributes
+   Array<int> internal_bdr({1, 3, 6, 9, 17, 157, 185, 75, 210, 211,
+                            212, 213, 214, 215, 216, 217, 218, 219,
+                            220, 221, 222, 223, 224, 225, 226, 227,
+                            228, 229, 230, 231, 232, 233, 234, 125});
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   Array<int> one_bdr;
+   Array<int> negone_bdr;
+
+   if (myid == 0)
+   {
+      std::cout << "Attributes" << endl;
+   }
+
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      // need to exclude these attributes
+      for (int i = 0; i<internal_bdr.Size(); i++)
+      {
+         ess_bdr[internal_bdr[i]-1] = 0;
+      }
+      E_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      one_bdr = 0;
+      negone_bdr = 0;
+      one_bdr[234] = 1;
+      negone_bdr[235] = 1;
+   }
+
+   if (myid == 0)
+   {
+      std::cout << "Attributes 2" << endl;
+   }
+
+   Vector z_one(3); z_one = 0.0; z_one(2) = 1.0;
+   Vector zero(3); zero = 0.0;
+   Vector z_negone(3); z_negone = 0.0; z_negone(2) = -1.0;
+   VectorConstantCoefficient z_one_cf(z_one);
+   VectorConstantCoefficient z_negone_cf(z_negone);
+   VectorConstantCoefficient zero_cf(zero);
+
+
+   E_gf.real() = 0.0;
+   E_gf.imag() = 0.0;
+   E_gf.ProjectBdrCoefficientTangent(z_one_cf,zero_cf, one_bdr);
+   E_gf.ProjectBdrCoefficientTangent(z_negone_cf,zero_cf, negone_bdr);
+
+   if (myid == 0)
+   {
+      std::cout << "Assembly started" << endl;
+   }
+   b->Assemble();
+   a->Assemble();
+
+   OperatorPtr Ah;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, E_gf, *b, Ah, X, B);
+
+   if (myid == 0)
+   {
+      std::cout << "Assembly finished" << endl;
+   }
+
+#ifdef MFEM_USE_MUMPS
+   HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+   // auto cpardiso = new CPardisoSolver(A->GetComm());
+   auto solver = new MUMPSSolver(MPI_COMM_WORLD);
+   solver->SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
+   solver->SetPrintLevel(1);
+   solver->SetOperator(*A);
+   solver->Mult(B,X);
+   delete A;
+   delete solver;
+#else
+   MFEM_ABORT("MFEM compiled without mumps");
+#endif
+
+   a->RecoverFEMSolution(X, *b, E_gf);
+
+   if (visualization)
+   {
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, E_gf.real(),
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime((double)0);
+      paraview_dc->Save();
+   }
+
+   if (paraview)
+   {
+      delete paraview_dc;
+   }
+
+   delete a;
+   delete b;
+   delete E_fes;
+   delete fec;
+
+   return 0;
+}
+

miniapps/dpg/plasma/other/pmaxwell-primal-tokamak.cpp

@@ -0,0 +1,495 @@
+
+// srun -n 256 ./pmaxwell-primal-tokamak -o 3 -sc -rnum
+// srun -n 448 ./pmaxwell-primal-tokamak -o 4 -sc -rnum 11.0 -paraview
+
+// srun -n 448 ./pmaxwell-primal-tokamak -o 4 -do 0 -sc -paraview (with the new epsilon GridFunction coefficients)
+// Description:
+// This example code demonstrates the use of MFEM to define and solve
+// the "ultraweak" (UW) DPG formulation for the Maxwell problem
+
+//      ∇×(1/μ ∇×E) - ω^2 ϵ E = Ĵ ,   in Ω
+//                E×n = E_0, on ∂Ω
+
+// The primal-DPG formulation is obtained by integration by parts
+// and the introduction of trace unknowns on the mesh skeleton
+
+// in 3D
+// E ∈ H(curl)
+// Ê ∈ H_0^1/2(Ω)(curl, Γ_h)
+//  1/μ (∇×E , ∇×F) + (ω^2 ϵ , F) + <Ê , F × n> = 0,      ∀ F ∈ H(curl,Ω)
+//                                        Ê × n = E_0     on ∂Ω
+
+#include "mfem.hpp"
+#include "../../util/pcomplexweakform.hpp"
+#include "../../../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class EpsilonMatrixCoefficient : public MatrixArrayCoefficient
+{
+private:
+   Mesh * mesh = nullptr;
+   ParMesh * pmesh = nullptr;
+   Array<ParGridFunction * > pgfs;
+   Array<GridFunctionCoefficient * > gf_cfs;
+   GridFunction * vgf = nullptr;
+   int dim;
+public:
+   EpsilonMatrixCoefficient(const char * filename, Mesh * mesh_, ParMesh * pmesh_,
+                            double scale = 1.0)
+      : MatrixArrayCoefficient(mesh_->Dimension()), mesh(mesh_), pmesh(pmesh_),
+        dim(mesh_->Dimension())
+   {
+      std::filebuf fb;
+      fb.open(filename,std::ios::in);
+      std::istream is(&fb);
+      vgf = new GridFunction(mesh,is);
+      fb.close();
+      FiniteElementSpace * vfes = vgf->FESpace();
+      int vdim = vfes->GetVDim();
+      const FiniteElementCollection * fec = vfes->FEColl();
+      FiniteElementSpace * fes = new FiniteElementSpace(mesh, fec);
+      int num_procs = Mpi::WorldSize();
+      int * partitioning = mesh->GeneratePartitioning(num_procs);
+      double *data = vgf->GetData();
+      GridFunction gf;
+      pgfs.SetSize(vdim);
+      gf_cfs.SetSize(vdim);
+      for (int i = 0; i<dim; i++)
+      {
+         for (int j = 0; j<dim; j++)
+         {
+            int k = i*dim+j;
+            // int k = j*dim+i;
+            gf.MakeRef(fes,&data[k*fes->GetVSize()]);
+            pgfs[k] = new ParGridFunction(pmesh,&gf,partitioning);
+            (*pgfs[k])*=scale;
+            gf_cfs[k] = new GridFunctionCoefficient(pgfs[k]);
+            Set(i,j,gf_cfs[k], true);
+         }
+      }
+   }
+   ~EpsilonMatrixCoefficient()
+   {
+      for (int i = 0; i<pgfs.Size(); i++)
+      {
+         delete pgfs[i];
+      }
+      pgfs.DeleteAll();
+   }
+
+};
+
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "data/mesh_330k.mesh";
+   const char * eps_r_file = "data/eps_r_330k.gf";
+   const char * eps_i_file = "data/eps_i_330k.gf";
+
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = false;
+   // double rnum=50.0;
+   double rnum=50.0e6;
+   int sr = 0;
+   int pr = 0;
+   bool paraview = false;
+   double mu = 1.257e-6;
+   double epsilon = 1.0;
+   double epsilon_scale = 8.8541878128e-12;
+   // double epsilon_scale = 8.8541878128;
+   bool mumps_solver = false;
+
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&sr, "-sref", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pref", "--parallel_ref",
+                  "Number of parallel refinements.");
+#ifdef MFEM_USE_MUMPS
+   args.AddOption(&mumps_solver, "-mumps", "--mumps-solver", "-no-mumps",
+                  "--no-mumps-solver", "Use the MUMPS Solver.");
+#endif
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   double omega = 2.*M_PI*rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   for (int i = 0; i<sr; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+
+   EpsilonMatrixCoefficient eps_r_cf(eps_r_file,&mesh,&pmesh, epsilon_scale);
+   EpsilonMatrixCoefficient eps_i_cf(eps_i_file,&mesh,&pmesh, epsilon_scale);
+   mesh.Clear();
+
+   FiniteElementCollection *E_fec = new ND_FECollection(order,dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh,E_fec);
+
+   // H^-1/2 (curl) space for Ê
+   int test_order = order+delta_order;
+   FiniteElementCollection * hatE_fec = new ND_Trace_FECollection(order,dim);
+   FiniteElementCollection * F_fec = new ND_FECollection(test_order, dim);
+   ParFiniteElementSpace *hatE_fes = new ParFiniteElementSpace(&pmesh,hatE_fec);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+   trial_fes.Append(E_fes);
+   trial_fes.Append(hatE_fes);
+   test_fec.Append(F_fec);
+
+   // Bilinear form coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient invmu_cf(1./mu);
+
+   Vector z_one(3); z_one = 0.0; z_one(2) = 1.0;
+   Vector zero(3); zero = 0.0;
+   Vector z_negone(3); z_negone = 0.0; z_negone(2) = -1.0;
+   VectorConstantCoefficient z_one_cf(z_one);
+   VectorConstantCoefficient z_negone_cf(z_negone);
+   VectorConstantCoefficient zero_cf(zero);
+
+   if (myid == 0)
+   {
+      std::cout << "Assembling matrix" << endl;
+   }
+
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_fes,test_fec);
+
+   // (∇ × E,∇ × F)
+   a->AddTrialIntegrator(new MixedCurlCurlIntegrator(invmu_cf), nullptr,0,0);
+
+   // -(ω^2 ϵ, F)
+   ScalarMatrixProductCoefficient m_cf_r(-omega*omega, eps_r_cf);
+   ScalarMatrixProductCoefficient m_cf_i(-omega*omega, eps_i_cf);
+
+   const IntegrationRule *irs[Geometry::NumGeom];
+   int order_quad = 2*order + 2;
+   for (int i = 0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+   const IntegrationRule &ir = IntRules.Get(pmesh.GetElementGeometry(0),
+                                            2*test_order + 2);
+   VectorFEMassIntegrator * integ_r = new VectorFEMassIntegrator(m_cf_r);
+   VectorFEMassIntegrator * integ_i = new VectorFEMassIntegrator(m_cf_i);
+   integ_r->SetIntegrationRule(ir);
+   integ_i->SetIntegrationRule(ir);
+   a->AddTrialIntegrator(integ_r,integ_i,0,0);
+
+   // < n×Ê,F>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,1,0);
+
+   // test integrators
+   // (∇×F ,∇× δF)
+
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,0,0);
+
+   // (F,δF)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,0,0);
+
+   socketstream E_out_r;
+
+   ParComplexGridFunction E_gf(E_fes);
+   E_gf.real() = 0.0;
+   E_gf.imag() = 0.0;
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   if (paraview)
+   {
+      paraview_dc = new ParaViewDataCollection(mesh_file, &pmesh);
+      paraview_dc->SetPrefixPath("ParaViewPrimalDPG");
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",&E_gf.real());
+      paraview_dc->RegisterField("E_i",&E_gf.imag());
+   }
+
+   // internal bdr attributes
+   Array<int> internal_bdr({1, 3, 6, 9, 17, 157, 185, 75, 210, 211,
+                            212, 213, 214, 215, 216, 217, 218, 219,
+                            220, 221, 222, 223, 224, 225, 226, 227,
+                            228, 229, 230, 231, 232, 233, 234, 125});
+
+   for (int it = 0; it<=pr; it++)
+   {
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      Array<int> one_bdr;
+      Array<int> negone_bdr;
+
+      if (myid == 0)
+      {
+         std::cout << "Attributes" << endl;
+      }
+
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         one_bdr.SetSize(pmesh.bdr_attributes.Max());
+         negone_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         // need to exclude these attributes
+         for (int i = 0; i<internal_bdr.Size(); i++)
+         {
+            ess_bdr[internal_bdr[i]-1] = 0;
+         }
+         E_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+         one_bdr = 0;
+         negone_bdr = 0;
+         one_bdr[234] = 1;
+         negone_bdr[235] = 1;
+      }
+
+      if (myid == 0)
+      {
+         std::cout << "Attributes 2" << endl;
+      }
+
+      Array<int> offsets(3);
+      offsets[0] = 0;
+      offsets[1] = E_fes->GetVSize();
+      offsets[2] = hatE_fes->GetVSize();
+      offsets.PartialSum();
+
+      Vector x(2*offsets.Last());
+      x = 0.;
+      double * xdata = x.GetData();
+
+      E_gf.real().MakeRef(E_fes,&xdata[0]);
+      E_gf.imag().MakeRef(E_fes,&xdata[offsets.Last()]);
+
+      E_gf.ProjectBdrCoefficientTangent(z_one_cf,zero_cf, one_bdr);
+      E_gf.ProjectBdrCoefficientTangent(z_negone_cf,zero_cf, negone_bdr);
+
+      if (myid == 0)
+      {
+         std::cout << "Assembly started" << endl;
+      }
+
+      a->Assemble();
+
+      if (myid == 0)
+      {
+         std::cout << "Assembly finished" << endl;
+      }
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+      int num_blocks = BlockA_r->NumRowBlocks();
+      Array<int> tdof_offsets(2*num_blocks+1);
+
+      tdof_offsets[0] = 0;
+      for (int i=0; i<num_blocks; i++)
+      {
+         tdof_offsets[i+1] = trial_fes[i]->GetTrueVSize();
+         tdof_offsets[num_blocks+i+1] = trial_fes[i]->GetTrueVSize();
+      }
+      tdof_offsets.PartialSum();
+
+      BlockOperator blockA(tdof_offsets);
+      for (int i = 0; i<num_blocks; i++)
+      {
+         for (int j = 0; j<num_blocks; j++)
+         {
+            blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
+            blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
+         }
+      }
+
+
+#ifdef MFEM_USE_MUMPS
+      if (mumps_solver)
+      {
+         // Monolithic real part
+         Array2D <HypreParMatrix * > Ab_r(num_blocks,num_blocks);
+         // Monolithic imag part
+         Array2D <HypreParMatrix * > Ab_i(num_blocks,num_blocks);
+         for (int i = 0; i<num_blocks; i++)
+         {
+            for (int j = 0; j<num_blocks; j++)
+            {
+               Ab_r(i,j) = &(HypreParMatrix &)BlockA_r->GetBlock(i,j);
+               Ab_i(i,j) = &(HypreParMatrix &)BlockA_i->GetBlock(i,j);
+            }
+         }
+         HypreParMatrix * A_r = HypreParMatrixFromBlocks(Ab_r);
+         HypreParMatrix * A_i = HypreParMatrixFromBlocks(Ab_i);
+
+         ComplexHypreParMatrix Acomplex(A_r, A_i,true,true);
+
+         HypreParMatrix * A = Acomplex.GetSystemMatrix();
+
+         MUMPSSolver mumps(MPI_COMM_WORLD);
+         mumps.SetPrintLevel(0);
+         mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
+         mumps.SetOperator(*A);
+         mumps.Mult(B,X);
+         delete A;
+      }
+#else
+      if (mumps_solver)
+      {
+         MFEM_WARNING("MFEM compiled without mumps. Switching to an iterative solver");
+      }
+      mumps_solver = false;
+#endif
+      if (!mumps_solver)
+      {
+         BlockDiagonalPreconditioner M(tdof_offsets);
+
+         HypreAMS * solver_E = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(0,0),
+                                            E_fes);
+         HypreAMS * solver_hatE = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(1,1),
+                                               hatE_fes);
+         solver_E->SetPrintLevel(0);
+         solver_hatE->SetPrintLevel(0);
+
+         M.SetDiagonalBlock(0,solver_E);
+         M.SetDiagonalBlock(1,solver_hatE);
+
+         if (myid == 0)
+         {
+            std::cout << "PCG iterations" << endl;
+         }
+
+
+         double bl2norm = B.Norml2();
+         double xl2norm = X.Norml2();
+         MPI_Allreduce(MPI_IN_PLACE,&bl2norm,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+         MPI_Allreduce(MPI_IN_PLACE,&xl2norm,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+         if (myid == 0)
+         {
+            mfem::out << "X.Norm = " << xl2norm << endl;
+            mfem::out << "B.Norm = " << bl2norm << endl;
+         }
+         CGSolver cg(MPI_COMM_WORLD);
+         cg.SetRelTol(1e-8);
+         cg.SetMaxIter(1000);
+         cg.SetPrintLevel(1);
+         cg.SetPreconditioner(M);
+         cg.SetOperator(blockA);
+         cg.Mult(B, X);
+
+         for (int i = 0; i<num_blocks; i++)
+         {
+            delete &M.GetDiagonalBlock(i);
+         }
+
+         int num_iter = cg.GetNumIterations();
+
+      }
+      a->RecoverFEMSolution(X,x);
+
+      E_gf.real().MakeRef(E_fes,x.GetData());
+      E_gf.imag().MakeRef(E_fes,&x.GetData()[offsets.Last()]);
+
+      int dofs = 0;
+      for (int i = 0; i<trial_fes.Size(); i++)
+      {
+         dofs += trial_fes[i]->GlobalTrueVSize();
+      }
+
+      if (visualization)
+      {
+         const char * keys = (it == 0 && dim == 2) ? "jRcml\n" : nullptr;
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         common::VisualizeField(E_out_r,vishost, visport, E_gf.real(),
+                                "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+      }
+
+      if (paraview)
+      {
+         paraview_dc->SetCycle(it);
+         paraview_dc->SetTime((double)it);
+         paraview_dc->Save();
+      }
+
+      if (it == pr)
+      {
+         break;
+      }
+
+      pmesh.UniformRefinement();
+
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   if (paraview)
+   {
+      delete paraview_dc;
+   }
+
+   delete a;
+   delete F_fec;
+   delete hatE_fes;
+   delete hatE_fec;
+   delete E_fec;
+   delete E_fes;
+
+   return 0;
+}
+

miniapps/dpg/plasma/other/pmaxwell-tokamak.cpp

@@ -0,0 +1,782 @@
+
+// srun -n 256 ./pmaxwell-tokamak -o 3 -sc -rnum
+// srun -n 448 ./pmaxwell-tokamak -o 4 -sc -rnum 11.0 -sigma 2.0 -paraview
+
+// srun -n 448 ./pmaxwell-tokamak -o 4 -do 0 -sc -paraview (with the new epsilon GridFunction coefficients)
+// TODO do > 0 fails with non SPD G matrix
+// Description:
+// This example code demonstrates the use of MFEM to define and solve
+// the "ultraweak" (UW) DPG formulation for the Maxwell problem
+
+//      ∇×(1/μ ∇×E) - (ω^2 ϵ + i ω σ) E = Ĵ ,   in Ω
+//                E×n = E_0, on ∂Ω
+
+// The DPG UW deals with the First Order System
+//        i ω μ H + ∇ × E = 0,   in Ω (Faraday's law)
+//            M E + ∇ × H = J,   in Ω (Ampere's law)
+//            E × n = E_0, on ∂Ω
+// Note: Ĵ = -iωJ
+// where M = -(i ω ϵI + σI)
+
+// The ultraweak-DPG formulation is obtained by integration by parts of both
+// equations and the introduction of trace unknowns on the mesh skeleton
+
+// in 3D
+// E,H ∈ (L^2(Ω))^3
+// Ê ∈ H_0^1/2(Ω)(curl, Γ_h), Ĥ ∈ H^-1/2(curl, Γ_h)
+//  i ω μ (H,F) + (E,∇ × F) + < Ê, F × n > = 0,      ∀ F ∈ H(curl,Ω)
+//      M (E,G) + (H,∇ × G) + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)
+//                                   Ê × n = E_0     on ∂Ω
+// -------------------------------------------------------------------------
+// |   |       E      |      H      |      Ê       |       Ĥ      |  RHS    |
+// -------------------------------------------------------------------------
+// | F |  (E,∇ × F)   | i ω μ (H,F) | < n × Ê, F > |              |         |
+// |   |              |             |              |              |         |
+// | G |    (ME,G)    |  (H,∇ × G)  |              | < n × Ĥ, G > |  (J,G)  |
+// where (F,G) ∈  H(curl,Ω) × H(curl,Ω)
+
+// Here we use the "Adjoint Graph" norm on the test space i.e.,
+// ||(F,G)||^2_V = ||A^*(F,G)||^2 + ||(F,G)||^2 where A is the
+// maxwell operator defined by (1)
+
+
+#include "mfem.hpp"
+#include "../../util/pcomplexweakform.hpp"
+#include "../../../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class EpsilonMatrixCoefficient : public MatrixArrayCoefficient
+{
+private:
+   Mesh * mesh = nullptr;
+   ParMesh * pmesh = nullptr;
+   Array<ParGridFunction * > pgfs;
+   Array<GridFunctionCoefficient * > gf_cfs;
+   GridFunction * vgf = nullptr;
+   int dim;
+public:
+   EpsilonMatrixCoefficient(const char * filename, Mesh * mesh_, ParMesh * pmesh_,
+                            double scale = 1.0)
+      : MatrixArrayCoefficient(mesh_->Dimension()), mesh(mesh_), pmesh(pmesh_),
+        dim(mesh_->Dimension())
+   {
+      std::filebuf fb;
+      fb.open(filename,std::ios::in);
+      std::istream is(&fb);
+      vgf = new GridFunction(mesh,is);
+      fb.close();
+      FiniteElementSpace * vfes = vgf->FESpace();
+      int vdim = vfes->GetVDim();
+      const FiniteElementCollection * fec = vfes->FEColl();
+      FiniteElementSpace * fes = new FiniteElementSpace(mesh, fec);
+      int num_procs = Mpi::WorldSize();
+      int * partitioning = mesh->GeneratePartitioning(num_procs);
+      double *data = vgf->GetData();
+      GridFunction gf;
+      pgfs.SetSize(vdim);
+      gf_cfs.SetSize(vdim);
+      for (int i = 0; i<dim; i++)
+      {
+         for (int j = 0; j<dim; j++)
+         {
+            int k = i*dim+j;
+            // int k = j*dim+i;
+            gf.MakeRef(fes,&data[k*fes->GetVSize()]);
+            pgfs[k] = new ParGridFunction(pmesh,&gf,partitioning);
+            (*pgfs[k])*=scale;
+            gf_cfs[k] = new GridFunctionCoefficient(pgfs[k]);
+            Set(i,j,gf_cfs[k], true);
+         }
+      }
+   }
+   ~EpsilonMatrixCoefficient()
+   {
+      for (int i = 0; i<pgfs.Size(); i++)
+      {
+         delete pgfs[i];
+      }
+      pgfs.DeleteAll();
+   }
+
+};
+
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // const char *mesh_file = "tokamak_100k.msh";
+   // const char *mesh_file = "tokamak_200k.msh";
+   // const char *mesh_file = "meshes/tokamak_100k.msh";
+   // const char *mesh_file = "meshes/tokamak_100k.msh";
+
+   const char *mesh_file = "data/mesh_330k.mesh";
+   const char * eps_r_file = "data/eps_r_330k.gf";
+   const char * eps_i_file = "data/eps_i_330k.gf";
+
+   // const char *mesh_file = "meshes/box.msh";
+
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = false;
+   double rnum=50.0e6;
+   bool static_cond = false;
+   int sr = 0;
+   int pr = 0;
+   bool paraview = false;
+   double factor = 1.0;
+   double mu = 1.257e-6/factor;
+   // double mu = 1.0;
+   double epsilon = 1.0;
+   double sigma = 0.01*factor;
+   double epsilon_scale = 8.8541878128e-12*factor;
+   bool graph_norm = true;
+   bool mumps_solver = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&sigma, "-sigma", "--sigma",
+                  "conductivity");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&sr, "-sref", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pref", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&graph_norm, "-graph", "--graph-norm", "-no-gn",
+                  "--no-graph-norm", "Enable adjoint graph norm.");
+#ifdef MFEM_USE_MUMPS
+   args.AddOption(&mumps_solver, "-mumps", "--mumps-solver", "-no-mumps",
+                  "--no-mumps-solver", "Use the MUMPS Solver.");
+#endif
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   double omega = 2.*M_PI*rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   for (int i = 0; i<sr; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+
+   // Test first with identity matrix coefficient
+   DenseMatrix Id(dim); Id = 0.0;
+   Id(0,0) = 1; Id(0,1) = 0.0; Id(0,2) = 0.0;
+   Id(1,0) = 0.0; Id(1,1) = 1; Id(1,2) = 0.0;
+   Id(2,0) = 0.0; Id(2,1) = 0.0; Id(2,2) = 1;
+   DenseMatrix C(dim); C = 0.0;
+   C(0,0) = 1.0; C(1,1) = 1.0; C(2,2) = 1.0;
+   DenseMatrix zmat(dim); zmat = 0.0;
+   MatrixConstantCoefficient identity_cf(Id);
+
+   // FiniteElementCollection *H1_fec = new H1_FECollection(1,dim);
+   // ParFiniteElementSpace *H1_fes = new ParFiniteElementSpace(&pmesh,H1_fec);
+
+   // Array<ParGridFunction * > pgfs(dim*dim);
+   // Array<GridFunctionCoefficient * > pcfs(dim*dim);
+   // MatrixArrayCoefficient eps_r_cf(dim);
+   // MatrixArrayCoefficient eps_i_cf(dim);
+   // for (int i = 0; i<dim; i++)
+   // {
+   //    for (int j = 0; j<dim; j++)
+   //    {
+   //       int k = i*dim + j;
+   //       pgfs[k] = new ParGridFunction(H1_fes);
+   //       if (i == j)
+   //       {
+   //          (*pgfs[k]) = 1.0;
+   //       }
+   //       else
+   //       {
+   //          (*pgfs[k]) = 0.0;
+   //       }
+   //       pcfs[k] = new GridFunctionCoefficient(pgfs[k]);
+   //       eps_r_cf.Set(i,j,pcfs[k],false);
+   //       eps_i_cf.Set(i,j,pcfs[k], false);
+   //    }
+   // }
+   // MatrixConstantCoefficient eps_r_cf(C);
+
+   // MatrixConstantCoefficient eps_i_cf(zmat);
+
+   DenseMatrix mat_eps_r(dim);
+   mat_eps_r(0,0) = -3.18447132e+02; mat_eps_r(0,1) = -7.73308634e-11;
+   mat_eps_r(0,2) =  2.26832549e+02;
+   mat_eps_r(1,0) = -7.73030845e-11; mat_eps_r(1,1) = -1.26300045e+06;
+   mat_eps_r(1,2) = -7.73308634e-11;
+   mat_eps_r(2,0) = -2.26832549e+02; mat_eps_r(2,1) = -7.73030845e-11;
+   mat_eps_r(2,2) = -3.18447132e+02;
+   DenseMatrix mat_eps_i(dim);
+   mat_eps_i(0,0) =  3.06793840e+02; mat_eps_i(0,1) =  4.06300785e-11;
+   mat_eps_i(0,2) =  7.95645437e+02;
+   mat_eps_i(1,0) =  4.07275170e-11; mat_eps_i(1,1) =  6.64641976e+05;
+   mat_eps_i(1,2) =  4.06300785e-11;
+   mat_eps_i(2,0) = -7.95645437e+02; mat_eps_i(2,1) =  4.07275170e-11;
+   mat_eps_i(2,2) =  3.06793840e+02;
+
+   mat_eps_r *= epsilon_scale;
+   mat_eps_i *= epsilon_scale;
+
+   // MatrixConstantCoefficient eps_r_cf(mat_eps_r);
+   // MatrixConstantCoefficient eps_i_cf(mat_eps_i);
+
+   EpsilonMatrixCoefficient eps_r_cf(eps_r_file,&mesh,&pmesh, epsilon_scale);
+   EpsilonMatrixCoefficient eps_i_cf(eps_i_file,&mesh,&pmesh, epsilon_scale);
+
+   mesh.Clear();
+
+
+   // Matrix Coefficient (M = -i\omega \epsilon - \sigma I);
+   // M = -i * omega * (eps_r + i eps_i) - sigmaI
+   //  = omega eps_i - sigma I + i (-omega eps_r)
+   MatrixSumCoefficient Mr_cf(eps_i_cf,identity_cf,omega,-sigma);
+   ScalarMatrixProductCoefficient Mi_cf(-omega,eps_r_cf);
+
+   // Matrix Coefficient eps_mat = -i\omega \epsilon
+   //                            = omega eps_mat_i + i (-omega eps_mat_r)
+   // ScalarMatrixProductCoefficient mat_eps_r_cf(omega,eps_i_cf);
+   // ScalarMatrixProductCoefficient mat_eps_i_cf(-omega,eps_r_cf);
+
+
+   // Define spaces
+   // L2 space for E
+   FiniteElementCollection *E_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh,E_fec,dim);
+
+   // Vector L2 space for H
+   FiniteElementCollection *H_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *H_fes = new ParFiniteElementSpace(&pmesh,H_fec, dim);
+
+   // H^-1/2 (curl) space for Ê
+   FiniteElementCollection * hatE_fec = nullptr;
+   FiniteElementCollection * hatH_fec = nullptr;
+   FiniteElementCollection * F_fec = nullptr;
+   int test_order = order+delta_order;
+   hatE_fec = new ND_Trace_FECollection(order,dim);
+   hatH_fec = new ND_Trace_FECollection(order,dim);
+   F_fec = new ND_FECollection(test_order, dim);
+
+   ParFiniteElementSpace *hatE_fes = new ParFiniteElementSpace(&pmesh,hatE_fec);
+   ParFiniteElementSpace *hatH_fes = new ParFiniteElementSpace(&pmesh,hatH_fec);
+   FiniteElementCollection * G_fec = new ND_FECollection(test_order, dim);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+   trial_fes.Append(E_fes);
+   trial_fes.Append(H_fes);
+   trial_fes.Append(hatE_fes);
+   trial_fes.Append(hatH_fes);
+   test_fec.Append(F_fec);
+   test_fec.Append(G_fec);
+
+   // Bilinear form coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient eps2omeg2(epsilon*epsilon*omega*omega);
+   ConstantCoefficient mu2omeg2(mu*mu*omega*omega);
+   ConstantCoefficient muomeg(mu*omega);
+   ConstantCoefficient negepsomeg(-epsilon*omega);
+   ConstantCoefficient epsomeg(epsilon*omega);
+   ConstantCoefficient negmuomeg(-mu*omega);
+   ConstantCoefficient sigma_cf(sigma);
+   ConstantCoefficient negsigma_cf(-sigma);
+   ConstantCoefficient sigma2_cf(sigma*sigma);
+
+   Vector z_one(3); z_one = 0.0; z_one(2) = 1.0;
+   Vector zero(3); zero = 0.0;
+   Vector z_negone(3); z_negone = 0.0; z_negone(2) = -1.0;
+   VectorConstantCoefficient z_one_cf(z_one);
+   VectorConstantCoefficient z_negone_cf(z_negone);
+   VectorConstantCoefficient zero_cf(zero);
+
+   if (myid == 0)
+   {
+      std::cout << "Assembling matrix" << endl;
+   }
+
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_fes,test_fec);
+
+   // (E,∇ × F)
+   a->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one)),
+                         nullptr,0,0);
+
+
+   // --------------------------------------------------------------------------
+   // -(i ω ϵ + σ) (E , G) = i (- ω ϵ E, G) - (σ E, G)
+   // a->AddTrialIntegrator(
+   // new TransposeIntegrator(new VectorFEMassIntegrator(negsigma_cf)),
+   // new TransposeIntegrator(new VectorFEMassIntegrator(negepsomeg)),0,1);
+
+   // a->AddTrialIntegrator(
+   // new TransposeIntegrator(new VectorFEMassIntegrator(negsigma_cf)), nullptr,0,1);
+   // a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(mat_eps_r_cf)),
+   //                       new TransposeIntegrator(new VectorFEMassIntegrator(mat_eps_i_cf)), 0,1);
+
+
+   //  (M E , G) = (M_r E, G) + i (M_i E, G)
+   a->AddTrialIntegrator(
+      new TransposeIntegrator(new VectorFEMassIntegrator(Mr_cf)),
+      new TransposeIntegrator(new VectorFEMassIntegrator(Mi_cf)),0,1);
+   // --------------------------------------------------------------------------
+
+
+   //  (H,∇ × G)
+   a->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one)),
+                         nullptr,1,1);
+   // < n×Ĥ ,G>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,3,1);
+   // test integrators
+   // (∇×G ,∇× δG)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,1,1);
+
+   ConstantCoefficient l2weight(1.0);
+
+   // (G,δG)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(l2weight),nullptr,1,1);
+
+   // i ω μ (H, F)
+   a->AddTrialIntegrator(nullptr,
+                         new TransposeIntegrator(new VectorFEMassIntegrator(muomeg)),1,0);
+   // < n×Ê,F>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,2,0);
+   // test integrators
+   // (∇×F,∇×δF)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,0,0);
+   // (F,δF)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,0,0);
+
+   if (graph_norm)
+   {
+      // μ^2 ω^2 (F,δF)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(mu2omeg2),nullptr,0,0);
+      // -i ω μ (F,∇ × δG) = i (F, -ω μ ∇ × δ G)
+      a->AddTestIntegrator(nullptr,new MixedVectorWeakCurlIntegrator(negmuomeg),0,1);
+
+      // --------------------------------------------------------------------------
+      // // -i ω ϵ (∇ × F, δG) - σ (∇ × F, δG)
+      // a->AddTestIntegrator(new MixedVectorCurlIntegrator(negsigma_cf),
+      // new MixedVectorCurlIntegrator(negepsomeg),0,1);
+
+      // a->AddTestIntegrator(new MixedVectorCurlIntegrator(negsigma_cf), nullptr, 0,1);
+
+      // a->AddTestIntegrator(new MixedVectorCurlIntegrator(mat_eps_r_cf),
+      //                      new MixedVectorCurlIntegrator(mat_eps_i_cf),0,1);
+
+      // (M ∇ × F, δG) = (M_r  ∇ × F, δG) + i (M_i  ∇ × F, δG)
+      a->AddTestIntegrator(new MixedVectorCurlIntegrator(Mr_cf),
+                           new MixedVectorCurlIntegrator(Mi_cf),0,1);
+      // --------------------------------------------------------------------------
+
+      // i ω μ (∇ × G,δF)
+      a->AddTestIntegrator(nullptr,new MixedVectorCurlIntegrator(muomeg),1,0);
+   }
+   // --------------------------------------------------------------------------
+   // i ω ϵ (G, ∇ × δF ) - σ (G, ∇ × δF )
+   // a->AddTestIntegrator(new MixedVectorWeakCurlIntegrator(negsigma_cf),
+   // new MixedVectorWeakCurlIntegrator(epsomeg),1,0);
+
+   // // (Eps^* G, ∇ × δF ) - σ (G, ∇ × δF ) = (Eps_r^T G,  ∇ × δF) - i (Eps_i^T G,  ∇ × δF) - σ (G, ∇ × δF )
+   // TransposeMatrixCoefficient mat_eps_rt_cf(mat_eps_r_cf);
+   // TransposeMatrixCoefficient mat_eps_it_cf(mat_eps_i_cf);
+   // ScalarMatrixProductCoefficient neg_mat_eps_it_cf(-1.0,mat_eps_it_cf);
+   // a->AddTestIntegrator(new MixedVectorWeakCurlIntegrator(negsigma_cf), nullptr, 1,0);
+   // a->AddTestIntegrator(new MixedVectorWeakCurlIntegrator(mat_eps_rt_cf),
+   //                      new MixedVectorWeakCurlIntegrator(neg_mat_eps_it_cf), 1,0);
+
+
+
+   // (M^* G, ∇ × δF ) = (Mr^T G,  ∇ × δF) - i (Mi^T G,  ∇ × δF)
+   TransposeMatrixCoefficient Mrt_cf(Mr_cf);
+   TransposeMatrixCoefficient Mit_cf(Mi_cf);
+   ScalarMatrixProductCoefficient negMit_cf(-1.0,Mit_cf);
+   if (graph_norm)
+   {
+      a->AddTestIntegrator(new MixedVectorWeakCurlIntegrator(Mrt_cf),
+                           new MixedVectorWeakCurlIntegrator(negMit_cf),1,0);
+   }
+   // --------------------------------------------------------------------------
+
+   // --------------------------------------------------------------------------
+   // ϵ^2 ω^2 (G,δG)
+   // a->AddTestIntegrator(new VectorFEMassIntegrator(eps2omeg2),nullptr,1,1);
+   // σ^2(G,δG)
+   // a->AddTestIntegrator(new VectorFEMassIntegrator(sigma2_cf),nullptr,1,1);
+
+   // MatrixProductCoefficient ErErt_cf(mat_eps_r_cf,mat_eps_rt_cf);
+   // MatrixProductCoefficient EiEit_cf(mat_eps_i_cf,mat_eps_it_cf);
+   // MatrixProductCoefficient EiErt_cf(mat_eps_i_cf,mat_eps_rt_cf);
+   // MatrixProductCoefficient ErEit_cf(mat_eps_r_cf,mat_eps_it_cf);
+
+   // MatrixSumCoefficient EEr_cf(ErErt_cf,EiEit_cf);
+   // MatrixSumCoefficient EEi_cf(EiErt_cf,ErEit_cf,1.0,-1.0);
+
+   // a->AddTestIntegrator(new VectorFEMassIntegrator(EEr_cf),
+   //                      new VectorFEMassIntegrator(EEi_cf),1,1);
+
+   // M*M^*(G,δG) = (MrMr^t + MiMi^t) + i (MiMr^t - MrMi^t)
+   MatrixProductCoefficient MrMrt_cf(Mr_cf,Mrt_cf);
+   MatrixProductCoefficient MiMit_cf(Mi_cf,Mit_cf);
+   MatrixProductCoefficient MiMrt_cf(Mi_cf,Mrt_cf);
+   MatrixProductCoefficient MrMit_cf(Mr_cf,Mit_cf);
+
+   MatrixSumCoefficient MMr_cf(MrMrt_cf,MiMit_cf);
+   MatrixSumCoefficient MMi_cf(MiMrt_cf,MrMit_cf,1.0,-1.0);
+
+   const IntegrationRule *irs[Geometry::NumGeom];
+   int order_quad = 2*order + 2;
+   for (int i = 0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+   const IntegrationRule &ir = IntRules.Get(pmesh.GetElementGeometry(0),
+                                            2*test_order + 2);
+
+   VectorFEMassIntegrator * integ_r = new VectorFEMassIntegrator(MMr_cf);
+   integ_r->SetIntegrationRule(ir);
+   VectorFEMassIntegrator * integ_i = new VectorFEMassIntegrator(MMi_cf);
+   integ_i->SetIntegrationRule(ir);
+
+   if (graph_norm)
+   {
+      a->AddTestIntegrator(integ_r,integ_i,1,1);
+   }
+   // --------------------------------------------------------------------------
+
+
+   socketstream E_out_r;
+   socketstream H_out_r;
+
+   ParComplexGridFunction E(E_fes);
+   ParComplexGridFunction H(H_fes);
+   E.real() = 0.0;
+   E.imag() = 0.0;
+   H.real() = 0.0;
+   H.imag() = 0.0;
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   if (paraview)
+   {
+      paraview_dc = new ParaViewDataCollection(mesh_file, &pmesh);
+      paraview_dc->SetPrefixPath("ParaViewUWDPG");
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",&E.real());
+      paraview_dc->RegisterField("E_i",&E.imag());
+      paraview_dc->RegisterField("H_r",&H.real());
+      paraview_dc->RegisterField("H_i",&H.imag());
+   }
+
+   // internal bdr attributes
+   Array<int> internal_bdr({1, 3, 6, 9, 17, 157, 185, 75, 210, 211,
+                            212, 213, 214, 215, 216, 217, 218, 219,
+                            220, 221, 222, 223, 224, 225, 226, 227,
+                            228, 229, 230, 231, 232, 233, 234, 125});
+
+   for (int it = 0; it<=pr; it++)
+   {
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      Array<int> one_bdr;
+      Array<int> negone_bdr;
+
+      if (myid == 0)
+      {
+         std::cout << "Attributes" << endl;
+      }
+
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         one_bdr.SetSize(pmesh.bdr_attributes.Max());
+         negone_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         // need to exclude these attributes
+         for (int i = 0; i<internal_bdr.Size(); i++)
+         {
+            ess_bdr[internal_bdr[i]-1] = 0;
+         }
+         hatE_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+         one_bdr = 0;
+         negone_bdr = 0;
+         one_bdr[234] = 1;
+         negone_bdr[235] = 1;
+         // one_bdr[6] = 1;
+         // negone_bdr[7] = 1;
+      }
+
+      if (myid == 0)
+      {
+         std::cout << "Attributes 2" << endl;
+      }
+
+      // Set up bdr conditions
+      // shift the ess_tdofs
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += E_fes->GetTrueVSize() + H_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = E_fes->GetVSize();
+      offsets[2] = H_fes->GetVSize();
+      offsets[3] = hatE_fes->GetVSize();
+      offsets[4] = hatH_fes->GetVSize();
+      offsets.PartialSum();
+
+      Vector x(2*offsets.Last());
+      x = 0.;
+      double * xdata = x.GetData();
+
+      ParComplexGridFunction hatE_gf(hatE_fes);
+      hatE_gf.real().MakeRef(hatE_fes,&xdata[offsets[2]]);
+      hatE_gf.imag().MakeRef(hatE_fes,&xdata[offsets.Last()+ offsets[2]]);
+
+      hatE_gf.ProjectBdrCoefficientTangent(z_one_cf,zero_cf, one_bdr);
+      hatE_gf.ProjectBdrCoefficientTangent(z_negone_cf,zero_cf, negone_bdr);
+
+      if (myid == 0)
+      {
+         std::cout << "Assembly started" << endl;
+      }
+
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      if (myid == 0)
+      {
+         std::cout << "Assembly finished" << endl;
+      }
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+      int num_blocks = BlockA_r->NumRowBlocks();
+      Array<int> tdof_offsets(2*num_blocks+1);
+
+      tdof_offsets[0] = 0;
+      int skip = (static_cond) ? 0 : 2;
+      int k = (static_cond) ? 2 : 0;
+      for (int i=0; i<num_blocks; i++)
+      {
+         tdof_offsets[i+1] = trial_fes[i+k]->GetTrueVSize();
+         tdof_offsets[num_blocks+i+1] = trial_fes[i+k]->GetTrueVSize();
+      }
+      tdof_offsets.PartialSum();
+
+      BlockOperator blockA(tdof_offsets);
+      for (int i = 0; i<num_blocks; i++)
+      {
+         for (int j = 0; j<num_blocks; j++)
+         {
+            blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
+            blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
+         }
+      }
+
+      X = 0.;
+
+#ifdef MFEM_USE_MUMPS
+      if (mumps_solver)
+      {
+         // Monolithic real part
+         Array2D <HypreParMatrix * > Ab_r(num_blocks,num_blocks);
+         // Monolithic imag part
+         Array2D <HypreParMatrix * > Ab_i(num_blocks,num_blocks);
+         for (int i = 0; i<num_blocks; i++)
+         {
+            for (int j = 0; j<num_blocks; j++)
+            {
+               Ab_r(i,j) = &(HypreParMatrix &)BlockA_r->GetBlock(i,j);
+               Ab_i(i,j) = &(HypreParMatrix &)BlockA_i->GetBlock(i,j);
+            }
+         }
+         HypreParMatrix * A_r = HypreParMatrixFromBlocks(Ab_r);
+         HypreParMatrix * A_i = HypreParMatrixFromBlocks(Ab_i);
+
+         ComplexHypreParMatrix Acomplex(A_r, A_i,true,true);
+
+         HypreParMatrix * A = Acomplex.GetSystemMatrix();
+
+         MUMPSSolver mumps(MPI_COMM_WORLD);
+         mumps.SetPrintLevel(0);
+         mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
+         mumps.SetOperator(*A);
+         mumps.Mult(B,X);
+         delete A;
+      }
+#else
+      if (mumps_solver)
+      {
+         MFEM_WARNING("MFEM compiled without mumps. Switching to an iterative solver");
+      }
+      mumps_solver = false;
+#endif
+      if (!mumps_solver)
+      {
+         BlockDiagonalPreconditioner M(tdof_offsets);
+
+         if (!static_cond)
+         {
+            HypreBoomerAMG * solver_E = new HypreBoomerAMG((HypreParMatrix &)
+                                                           BlockA_r->GetBlock(0,0));
+            solver_E->SetPrintLevel(0);
+            solver_E->SetSystemsOptions(dim);
+            HypreBoomerAMG * solver_H = new HypreBoomerAMG((HypreParMatrix &)
+                                                           BlockA_r->GetBlock(1,1));
+            solver_H->SetPrintLevel(0);
+            solver_H->SetSystemsOptions(dim);
+            M.SetDiagonalBlock(0,solver_E);
+            M.SetDiagonalBlock(1,solver_H);
+            M.SetDiagonalBlock(num_blocks,solver_E);
+            M.SetDiagonalBlock(num_blocks+1,solver_H);
+         }
+         HypreAMS * solver_hatE = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(skip,
+                                                                                    skip), hatE_fes);
+         HypreAMS * solver_hatH = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(
+                                                  skip+1,skip+1), hatH_fes);
+         solver_hatE->SetPrintLevel(0);
+         solver_hatH->SetPrintLevel(0);
+
+         M.SetDiagonalBlock(skip,solver_hatE);
+         M.SetDiagonalBlock(skip+1,solver_hatH);
+         M.SetDiagonalBlock(skip+num_blocks,solver_hatE);
+         M.SetDiagonalBlock(skip+num_blocks+1,solver_hatH);
+
+         if (myid == 0)
+         {
+            std::cout << "PCG iterations" << endl;
+         }
+         CGSolver cg(MPI_COMM_WORLD);
+         // GMRESSolver cg(MPI_COMM_WORLD);
+         cg.SetRelTol(1e-6);
+         cg.SetMaxIter(500);
+         cg.SetPrintLevel(1);
+         cg.SetPreconditioner(M);
+         cg.SetOperator(blockA);
+         cg.Mult(B, X);
+
+         for (int i = 0; i<num_blocks; i++)
+         {
+            delete &M.GetDiagonalBlock(i);
+         }
+
+         int num_iter = cg.GetNumIterations();
+      }
+      a->RecoverFEMSolution(X,x);
+
+      E.real().MakeRef(E_fes,x.GetData());
+      E.imag().MakeRef(E_fes,&x.GetData()[offsets.Last()]);
+
+      H.real().MakeRef(H_fes,&x.GetData()[offsets[1]]);
+      H.imag().MakeRef(H_fes,&x.GetData()[offsets.Last()+offsets[1]]);
+
+      int dofs = 0;
+      for (int i = 0; i<trial_fes.Size(); i++)
+      {
+         dofs += trial_fes[i]->GlobalTrueVSize();
+      }
+
+      if (visualization)
+      {
+         const char * keys = (it == 0 && dim == 2) ? "jRcml\n" : nullptr;
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         common::VisualizeField(E_out_r,vishost, visport, E.real(),
+                                "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+         common::VisualizeField(H_out_r,vishost, visport, H.real(),
+                                "Numerical Magnetic field (real part)", 501, 0, 500, 500, keys);
+      }
+
+      if (paraview)
+      {
+         paraview_dc->SetCycle(it);
+         paraview_dc->SetTime((double)it);
+         paraview_dc->Save();
+      }
+
+      if (it == pr)
+      {
+         break;
+      }
+
+      pmesh.UniformRefinement();
+
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   if (paraview)
+   {
+      delete paraview_dc;
+   }
+
+   delete a;
+   delete F_fec;
+   delete G_fec;
+   delete hatH_fes;
+   delete hatH_fec;
+   delete hatE_fes;
+   delete hatE_fec;
+   delete H_fec;
+   delete E_fec;
+   delete H_fes;
+   delete E_fes;
+
+   return 0;
+}
+

miniapps/dpg/plasma/other/pmaxwell-uw-lh.cpp

@@ -0,0 +1,833 @@
+//                   MFEM Ultraweak DPG Maxwell parallel example
+//
+// Description:
+// This example code demonstrates the use of MFEM to define and solve
+// the "ultraweak" (UW) DPG formulation for the Maxwell problem
+
+//      ∇×(1/μ ∇×E) - ω² ϵ E = Ĵ ,   in Ω
+//                       E×n = E₀ , on ∂Ω
+
+// The DPG UW deals with the First Order System
+//  i ω μ H + ∇ × E = 0,   in Ω
+// -i ω ϵ E + ∇ × H = 0,   in Ω
+//            E × n = E_0, on ∂Ω
+
+// The ultraweak-DPG formulation is obtained by integration by parts of both
+// equations and the introduction of trace unknowns on the mesh skeleton
+
+// in 2D
+// E is vector valued and H is scalar.
+//    (∇ × E, F) = (E, ∇ × F) + < n × E , F>
+// or (∇ ⋅ AE , F) = (AE, ∇ F) + < AE ⋅ n, F>
+// where A = [0 1; -1 0];
+
+// E ∈ (L²(Ω))² , H ∈ L²(Ω)
+// Ê ∈ H^-1/2(Γₕ), Ĥ ∈ H^1/2(Γₕ)
+//  i ω μ (H,F) + (E, ∇ × F) + < AÊ, F > = 0,      ∀ F ∈ H¹
+// -i ω ϵ (E,G) + (H,∇ × G)  + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)
+//                                        Ê = E₀      on ∂Ω
+// -------------------------------------------------------------------------
+// |   |       E      |      H      |      Ê       |       Ĥ      |  RHS    |
+// -------------------------------------------------------------------------
+// | F |  (E,∇ × F)   | i ω μ (H,F) |   < Ê, F >   |              |         |
+// |   |              |             |              |              |         |
+// | G | -i ω ϵ (E,G) |  (H,∇ × G)  |              | < Ĥ, G × n > |  (J,G)  |
+// where (F,G) ∈  H¹ × H(curl,Ω)
+
+#include "mfem.hpp"
+#include "../../util/pcomplexweakform.hpp"
+#include "../../util/utils.hpp"
+#include "../../util/maxwell_utils.hpp"
+#include "../../../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+#include <ctime>
+#include <string>
+#include <sstream>
+#include <cstring>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   int num_procs = Mpi::WorldSize();
+   Hypre::Init();
+
+   // fine mesh (trianles)
+   // default mesh
+   const char *mesh_file = "data/mesh-tri34K.mesh";
+   // coarse mesh (triangles)
+   // const char *mesh_file = "data/mesh-tri11K.mesh";
+   // coarse mesh (quadrilaterals)
+   // const char *mesh_file = "data/mesh-quad5K.mesh";
+
+   // epsilon tensor
+   const char * eps_r_file = nullptr;
+   const char * eps_i_file = nullptr;
+
+   int order = 2;
+   int delta_order = 1;
+   int par_ref_levels = 0;
+   int amr_ref_levels = 0;
+   // real_t rnum=4.6e9;
+   // real_t mu = 1.257e-6/factor;
+   // real_t epsilon_scale = 8.8541878128e-12*factor;
+   real_t rnum=4.6;
+   real_t mu = 1.257;
+   real_t epsilon_scale = 8.8541878128;
+
+   //  ∇×(1/μ ∇×E) - ω² ϵ E = Ĵ ,   in Ω
+   //  1/1.257e-6 - 2π*2π*4.6*4.6e18*8.8541878128e-12
+   // 1/1.257e-6 - 2π*2π*4.6*4.6*8.8541878128e6
+   // 1e6/1.257 - 2π*2π*4.6*4.6*8.8541878128e6
+   // 1/1.257 - 2π*2π*4.6*4.6*8.8541878128
+
+   bool visualization = false;
+   bool static_cond = false;
+   bool graph_norm = true;
+   bool mumps_solver = false;
+   real_t theta = 0.0;
+   bool paraview = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&par_ref_levels, "-pr", "--parallel-refinement-levels",
+                  "Number of parallel refinement levels.");
+   args.AddOption(&amr_ref_levels, "-amr", "--parallel-amr-refinement-levels",
+                  "Number of parallel AMR refinement levels.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&graph_norm, "-graph", "--graph-norm", "-no-gn",
+                  "--no-graph-norm", "Enable adjoint graph norm.");
+#ifdef MFEM_USE_MUMPS
+   args.AddOption(&mumps_solver, "-mumps", "--mumps-solver", "-no-mumps",
+                  "--no-mumps-solver", "Use the MUMPS Solver.");
+#endif
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   if (strcmp(mesh_file, "data/mesh-tri34K.mesh") == 0)
+   {
+      eps_r_file = "data/eps-tri34K_r.gf";
+      eps_i_file = "data/eps-tri34K_i.gf";
+   }
+   else if (strcmp(mesh_file, "data/mesh-tri11K.mesh") == 0)
+   {
+      eps_r_file = "data/eps-tri11K_r.gf";
+      eps_i_file = "data/eps-tri11K_i.gf";
+   }
+   else if (strcmp(mesh_file, "data/mesh-quad5K.mesh") == 0)
+   {
+      eps_r_file = "data/eps-quad5K_r.gf";
+      eps_i_file = "data/eps-quad5K_i.gf";
+   }
+   else
+   {
+      MFEM_ABORT("Unknown mesh file: " + string(mesh_file));
+   }
+
+   real_t omega = 2.*M_PI*rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   Array<int> int_bdr_attr;
+   for (int i = 0; i < mesh.GetNBE(); i++)
+   {
+      if (mesh.FaceIsInterior(mesh.GetBdrElementFaceIndex(i)))
+      {
+         int_bdr_attr.Append(mesh.GetBdrAttribute(i));
+      }
+   }
+
+   // mesh.RemoveInternalBoundaries();
+   mesh.EnsureNCMesh(true);
+   int * partitioning = mesh.GeneratePartitioning(num_procs);
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh, partitioning);
+
+   EpsilonMatrixCoefficient eps_r_cf(eps_r_file,&mesh,&pmesh, epsilon_scale);
+   EpsilonMatrixCoefficient eps_i_cf(eps_i_file,&mesh,&pmesh, epsilon_scale);
+   mesh.Clear();
+
+   for (int i = 0; i<par_ref_levels; i++)
+   {
+      pmesh.UniformRefinement();
+      eps_r_cf.Update();
+      eps_i_cf.Update();
+   }
+
+   // eps_r_cf.VisualizeMatrixCoefficient();
+   // eps_i_cf.VisualizeMatrixCoefficient();
+
+   // return 0;
+
+   // Matrix Coefficient (M = -i\omega \epsilon);
+   // M = -i * omega * (eps_r + i eps_i)
+   //  = omega eps_i + i (-omega eps_r)
+   ScalarMatrixProductCoefficient Mr_cf(omega,eps_i_cf);
+   ScalarMatrixProductCoefficient Mi_cf(-omega,eps_r_cf);
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient muomeg(mu*omega);
+   ConstantCoefficient mu2omeg2(mu*mu*omega*omega);
+   ConstantCoefficient negmuomeg(-mu*omega);
+   DenseMatrix rot_mat(2);
+   rot_mat(0,0) = 0.; rot_mat(0,1) = 1.;
+   rot_mat(1,0) = -1.; rot_mat(1,1) = 0.;
+   MatrixConstantCoefficient rot(rot_mat);
+   MatrixProductCoefficient Mrot_r(Mr_cf,rot);
+   MatrixProductCoefficient Mrot_i(Mi_cf,rot);
+   ScalarMatrixProductCoefficient negMrot_i(-1.0,Mrot_i);
+
+   TransposeMatrixCoefficient Mrt_cf(Mr_cf);
+   TransposeMatrixCoefficient Mit_cf(Mi_cf);
+   MatrixProductCoefficient MrMrt_cf(Mr_cf,Mrt_cf);
+   MatrixProductCoefficient MiMit_cf(Mi_cf,Mit_cf);
+   MatrixProductCoefficient MiMrt_cf(Mi_cf,Mrt_cf);
+   MatrixProductCoefficient MrMit_cf(Mr_cf,Mit_cf);
+
+   MatrixSumCoefficient MMr_cf(MrMrt_cf,MiMit_cf);
+   MatrixSumCoefficient MMi_cf(MiMrt_cf,MrMit_cf,1.0,-1.0);
+
+   mesh.Clear();
+
+   // Define spaces
+   enum TrialSpace
+   {
+      E_space     = 0,
+      H_space     = 1,
+      hatE_space  = 2,
+      hatH_space  = 3
+   };
+   enum TestSpace
+   {
+      F_space = 0,
+      G_space = 1
+   };
+
+   int dimc = (dim == 3) ? 3 : 1;
+   int test_order = order+delta_order;
+
+   // Vector L2 L2 space for E
+   FiniteElementCollection *E_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh,E_fec,dim);
+
+   // Vector L2 space for H
+   FiniteElementCollection *H_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *H_fes = new ParFiniteElementSpace(&pmesh,H_fec, dimc);
+
+   // H^-1/2 (curl) space for Ê
+   FiniteElementCollection * hatE_fec = new RT_Trace_FECollection(order-1,dim);
+   FiniteElementCollection * hatH_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementCollection * F_fec = new H1_FECollection(test_order, dim);
+
+   ParFiniteElementSpace *hatE_fes = new ParFiniteElementSpace(&pmesh,hatE_fec);
+   ParFiniteElementSpace *hatH_fes = new ParFiniteElementSpace(&pmesh,hatH_fec);
+   FiniteElementCollection * G_fec = new ND_FECollection(test_order, dim);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+   trial_fes.Append(E_fes);
+   trial_fes.Append(H_fes);
+   trial_fes.Append(hatE_fes);
+   trial_fes.Append(hatH_fes);
+   test_fec.Append(F_fec);
+   test_fec.Append(G_fec);
+
+   int gdofs = 0;
+   for (int i = 0; i<trial_fes.Size(); i++)
+   {
+      gdofs += trial_fes[i]->GlobalTrueVSize();
+   }
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Global number of dofs = " << gdofs << endl;
+   }
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_fes,test_fec);
+   a->StoreMatrices(); // needed for AMR
+
+   const IntegrationRule *irs[Geometry::NumGeom];
+   int order_quad = 2*order + 2;
+   for (int i = 0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+   const IntegrationRule &ir = IntRules.Get(pmesh.GetElementGeometry(0),
+                                            2*test_order + 2);
+
+   // (E,∇ × F)
+   MixedCurlIntegrator * curl_integ = new MixedCurlIntegrator(one);
+   curl_integ->SetIntRule(&ir);
+   a->AddTrialIntegrator(new TransposeIntegrator(curl_integ), nullptr,
+                         TrialSpace::E_space,
+                         TestSpace::F_space);
+
+   //  (M E , G) = (M_r E, G) + i (M_i E, G) = (E, M_rt G) + i (E, Mit G)
+   //            = (M_rt G, E)^T + i (Mit G, E)^T
+   VectorFEMassIntegrator * Mrt_cf_integ = new VectorFEMassIntegrator(Mrt_cf);
+   VectorFEMassIntegrator * Mit_cf_integ = new VectorFEMassIntegrator(Mit_cf);
+   Mrt_cf_integ->SetIntegrationRule(ir);
+   Mit_cf_integ->SetIntegrationRule(ir);
+   a->AddTrialIntegrator(
+      new TransposeIntegrator(Mrt_cf_integ),
+      new TransposeIntegrator(Mit_cf_integ),
+      TrialSpace::E_space, TestSpace::G_space);
+
+   //  (H,∇ × G)
+   MixedCurlIntegrator * curl_integ_H = new MixedCurlIntegrator(one);
+   curl_integ_H->SetIntRule(&ir);
+   a->AddTrialIntegrator(new TransposeIntegrator(curl_integ_H), nullptr,
+                         TrialSpace::H_space, TestSpace::G_space);
+
+   // < n×Ĥ ,G>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,
+                         TrialSpace::hatH_space, TestSpace::G_space);
+
+   // i ω μ (H, F)
+   MixedScalarMassIntegrator * muomeg_integ = new MixedScalarMassIntegrator(
+      muomeg);
+   muomeg_integ->SetIntRule(&ir);
+   a->AddTrialIntegrator(nullptr,muomeg_integ,
+                         TrialSpace::H_space, TestSpace::F_space);
+
+   // < n×Ê,F>
+   a->AddTrialIntegrator(new TraceIntegrator,nullptr,
+                         TrialSpace::hatE_space, TestSpace::F_space);
+
+   // test integrators
+   // (∇×G ,∇× δG)
+   CurlCurlIntegrator * curlcurl_integ = new CurlCurlIntegrator(one);
+   curlcurl_integ->SetIntRule(&ir);
+   a->AddTestIntegrator(curlcurl_integ,nullptr,
+                        TestSpace::G_space,TestSpace::G_space);
+   // (G,δG)
+   VectorFEMassIntegrator * vfemass_integ = new VectorFEMassIntegrator(one);
+   vfemass_integ->SetIntegrationRule(ir);
+   a->AddTestIntegrator(vfemass_integ,nullptr,
+                        TestSpace::G_space,TestSpace::G_space);
+   // (∇F,∇δF)
+   DiffusionIntegrator * diff_integ = new DiffusionIntegrator(one);
+   diff_integ->SetIntRule(&ir);
+   a->AddTestIntegrator(diff_integ,nullptr,
+                        TestSpace::F_space, TestSpace::F_space);
+   // (F,δF)
+   MassIntegrator * mass_integ = new MassIntegrator(one);
+   mass_integ->SetIntRule(&ir);
+   a->AddTestIntegrator(mass_integ,nullptr,
+                        TestSpace::F_space, TestSpace::F_space);
+
+   if (graph_norm)
+   {
+      // μ^2 ω^2 (F,δF)
+      MassIntegrator * mu2omeg2_integ = new MassIntegrator(mu2omeg2);
+      mu2omeg2_integ->SetIntRule(&ir);
+      a->AddTestIntegrator(mu2omeg2_integ,nullptr,
+                           TestSpace::F_space, TestSpace::F_space);
+
+      // -i ω μ (F,∇ × δG) = i (F, -ω μ ∇ × δ G)
+      MixedCurlIntegrator * negmuomeg_integ = new MixedCurlIntegrator(negmuomeg);
+      negmuomeg_integ->SetIntRule(&ir);
+      a->AddTestIntegrator(nullptr, new TransposeIntegrator(negmuomeg_integ),
+                           TestSpace::F_space, TestSpace::G_space);
+
+      // (M ∇ × F, δG) = (M_r ∇ × F, δG) + i (M_i ∇ × F, δG)
+      //               = (M_r A ∇ F, δG) + i (M_i A ∇ F, δG), A = [0 1; -1; 0]
+      MixedVectorGradientIntegrator * Mrot_r_integ = new
+      MixedVectorGradientIntegrator(Mrot_r);
+      MixedVectorGradientIntegrator * Mrot_i_integ = new
+      MixedVectorGradientIntegrator(Mrot_i);
+      Mrot_r_integ->SetIntRule(&ir);
+      Mrot_i_integ->SetIntRule(&ir);
+      a->AddTestIntegrator(Mrot_r_integ, Mrot_i_integ,
+                           TestSpace::F_space, TestSpace::G_space);
+
+      // i ω μ (∇ × G,δF) = i (ω μ ∇ × G, δF )
+      MixedCurlIntegrator * muomeg_integ = new MixedCurlIntegrator(muomeg);
+      muomeg_integ->SetIntRule(&ir);
+      a->AddTestIntegrator(nullptr,muomeg_integ,
+                           TestSpace::G_space, TestSpace::F_space);
+
+      // (M^* G, ∇ × δF ) = (G, Mr A ∇ δF) - i (G, Mi A ∇ δF)
+      MixedVectorGradientIntegrator * Mrot_r_integ2 = new
+      MixedVectorGradientIntegrator(Mrot_r);
+      MixedVectorGradientIntegrator * negMrot_i_integ = new
+      MixedVectorGradientIntegrator(negMrot_i);
+      Mrot_r_integ2->SetIntRule(&ir);
+      negMrot_i_integ->SetIntRule(&ir);
+      a->AddTestIntegrator(new TransposeIntegrator(Mrot_r_integ2),
+                           new TransposeIntegrator(negMrot_i_integ),
+                           TestSpace::G_space, TestSpace::F_space);
+
+      // M*M^*(G,δG) = (MrMr^t + MiMi^t) + i (MiMr^t - MrMi^t)
+      VectorFEMassIntegrator * MMr_integ = new VectorFEMassIntegrator(MMr_cf);
+      VectorFEMassIntegrator * MMi_integ = new VectorFEMassIntegrator(MMi_cf);
+      MMr_integ->SetIntegrationRule(ir);
+      MMi_integ->SetIntegrationRule(ir);
+      a->AddTestIntegrator(MMr_integ, MMi_integ,
+                           TestSpace::G_space,
+                           TestSpace::G_space);
+   }
+
+   socketstream E_out_r;
+   socketstream H_out_r;
+   socketstream E_theta_out_r;
+
+   ParComplexGridFunction E(E_fes);
+   ParComplexGridFunction H(H_fes);
+   E.real() = 0.0;
+   E.imag() = 0.0;
+   H.real() = 0.0;
+   H.imag() = 0.0;
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2_fes(&pmesh, &L2fec);
+
+   ParGridFunction E_theta_r(&L2_fes);
+   ParGridFunction E_theta_i(&L2_fes);
+   ParGridFunction E_theta(&L2_fes);
+   E_theta = 0.0;
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+   ParaViewDataCollection * paraview_tdc = nullptr;
+
+   // Create ParaView directory and file
+   std::string output_dir = "ParaView/UW/2D" + GetTimestamp();
+   if (Mpi::Root())
+   {
+      WriteParametersToFile(args, output_dir);
+   }
+
+   if (paraview)
+   {
+      std::ostringstream paraview_file_name;
+      std::string filename = GetFilename(mesh_file);
+      paraview_file_name << filename
+                         << "_par_ref_" << par_ref_levels
+                         << "_order_" << order;
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath(output_dir);
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",&E.real());
+      paraview_dc->RegisterField("E_i",&E.imag());
+      paraview_dc->RegisterField("H_r",&H.real());
+      paraview_dc->RegisterField("H_i",&H.imag());
+      paraview_dc->RegisterField("E_theta_r",&E_theta_r);
+      paraview_dc->RegisterField("E_theta_i",&E_theta_i);
+
+      std::ostringstream paraview_file_name_th;
+      paraview_file_name_th << filename
+                            << "_par_ref_" << par_ref_levels
+                            << "_order_" << order
+                            << "th";
+      paraview_tdc = new ParaViewDataCollection(paraview_file_name_th.str(), &pmesh);
+      paraview_tdc->SetPrefixPath(output_dir);
+      paraview_tdc->SetLevelsOfDetail(order);
+      paraview_tdc->SetCycle(0);
+      paraview_tdc->SetDataFormat(VTKFormat::BINARY);
+      paraview_tdc->SetHighOrderOutput(true);
+      paraview_tdc->SetTime(0.0); // set the time
+      paraview_tdc->RegisterField("E_theta_t",&E_theta);
+   }
+
+   real_t res0 = 0.;
+   real_t err0 = 0.;
+   int dof0 = 0; // init to suppress gcc warning
+
+   Array<int> elements_to_refine;
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+   for (int it = 0; it<=amr_ref_levels; it++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      Array<int> one_r_bdr;
+      Array<int> one_i_bdr;
+      Array<int> negone_r_bdr;
+      Array<int> negone_i_bdr;
+
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         one_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+         one_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+         negone_r_bdr.SetSize(pmesh.bdr_attributes.Max());
+         negone_i_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+
+         // remove internal boundaries
+         for (int i = 0; i<int_bdr_attr.Size(); i++)
+         {
+            ess_bdr[int_bdr_attr[i]-1] = 0;
+         }
+
+         hatE_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+         one_r_bdr = 0;  one_i_bdr = 0;
+         negone_r_bdr = 0;  negone_i_bdr = 0;
+
+         // attr = 30,2 (real)
+         one_r_bdr[30-1] = 1;  one_r_bdr[2-1] = 1;
+         // attr = 26,6 (imag)
+         one_i_bdr[26-1] = 1;  one_i_bdr[6-1] = 1;
+         // attr = 22,10 (real)
+         negone_r_bdr[22-1] = 1; negone_r_bdr[10-1] = 1;
+         // attr = 18,14 (imag)
+         negone_i_bdr[18-1] = 1; negone_i_bdr[14-1] = 1;
+      }
+
+      // Set up bdr conditions
+      // shift the ess_tdofs
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += E_fes->GetTrueVSize() + H_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = E_fes->GetVSize();
+      offsets[2] = H_fes->GetVSize();
+      offsets[3] = hatE_fes->GetVSize();
+      offsets[4] = hatH_fes->GetVSize();
+      offsets.PartialSum();
+
+      Vector x(2*offsets.Last());
+      x = 0.;
+      real_t * xdata = x.GetData();
+
+      ParComplexGridFunction hatE_gf(hatE_fes);
+      hatE_gf.real().MakeRef(hatE_fes,&xdata[offsets[2]]);
+      hatE_gf.imag().MakeRef(hatE_fes,&xdata[offsets.Last()+ offsets[2]]);
+
+
+      Vector zero(dim); zero = 0.0;
+      VectorConstantCoefficient zero_cf(zero);
+
+      // rotate the vector
+      // (x,y) -> (y,-x)
+      Vector rot_one_x(dim); rot_one_x = 0.0; rot_one_x(1) = -1.0;
+      Vector rot_negone_x(dim); rot_negone_x = 0.0; rot_negone_x(1) = 1.0;
+      VectorConstantCoefficient rot_one_x_cf(rot_one_x);
+      VectorConstantCoefficient rot_negone_x_cf(rot_negone_x);
+
+      hatE_gf.ProjectBdrCoefficientNormal(rot_one_x_cf,zero_cf, one_r_bdr);
+      hatE_gf.ProjectBdrCoefficientNormal(rot_negone_x_cf,zero_cf, negone_r_bdr);
+      hatE_gf.ProjectBdrCoefficientNormal(zero_cf,rot_one_x_cf, one_i_bdr);
+      hatE_gf.ProjectBdrCoefficientNormal(zero_cf,rot_negone_x_cf, negone_i_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+      int num_blocks = BlockA_r->NumRowBlocks();
+      Array<int> tdof_offsets(2*num_blocks+1);
+
+      tdof_offsets[0] = 0;
+      int skip = (static_cond) ? 0 : 2;
+      int k = (static_cond) ? 2 : 0;
+      for (int i=0; i<num_blocks; i++)
+      {
+         tdof_offsets[i+1] = trial_fes[i+k]->GetTrueVSize();
+         tdof_offsets[num_blocks+i+1] = trial_fes[i+k]->GetTrueVSize();
+      }
+      tdof_offsets.PartialSum();
+
+      BlockOperator blockA(tdof_offsets);
+      for (int i = 0; i<num_blocks; i++)
+      {
+         for (int j = 0; j<num_blocks; j++)
+         {
+            blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
+            blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
+         }
+      }
+
+      X = 0.;
+
+#ifdef MFEM_USE_MUMPS
+      if (mumps_solver)
+      {
+         // Monolithic real part
+         Array2D<const HypreParMatrix * > Ab_r(num_blocks,num_blocks);
+         // Monolithic imag part
+         Array2D<const HypreParMatrix * > Ab_i(num_blocks,num_blocks);
+         for (int i = 0; i<num_blocks; i++)
+         {
+            for (int j = 0; j<num_blocks; j++)
+            {
+               Ab_r(i,j) = &(HypreParMatrix &)BlockA_r->GetBlock(i,j);
+               Ab_i(i,j) = &(HypreParMatrix &)BlockA_i->GetBlock(i,j);
+            }
+         }
+         HypreParMatrix * A_r = HypreParMatrixFromBlocks(Ab_r);
+         HypreParMatrix * A_i = HypreParMatrixFromBlocks(Ab_i);
+
+         ComplexHypreParMatrix Acomplex(A_r, A_i,true,true);
+
+         HypreParMatrix * A = Acomplex.GetSystemMatrix();
+
+         MUMPSSolver mumps(MPI_COMM_WORLD);
+         mumps.SetPrintLevel(0);
+         mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
+         mumps.SetOperator(*A);
+         mumps.Mult(B,X);
+         delete A;
+      }
+#else
+      if (mumps_solver)
+      {
+         MFEM_WARNING("MFEM compiled without mumps. Switching to an iterative solver");
+      }
+      mumps_solver = false;
+#endif
+      int num_iter = -1;
+      if (!mumps_solver)
+      {
+         BlockDiagonalPreconditioner M(tdof_offsets);
+         // BlockTriangularSymmetricPreconditioner M(tdof_offsets);
+         // M.SetOperator(blockA);
+         // int nblocks = blockA.NumRowBlocks();
+         // for (int i = 0; i<nblocks; i++)
+         // {
+         //    for (int j = 0; j<nblocks; j++)
+         //    {
+         //       if (i != j)
+         //       {
+         //          M.SetBlock(i,j,&blockA.GetBlock(i,j));
+         //       }
+         //    }
+         // }
+
+
+         if (!static_cond)
+         {
+            HypreBoomerAMG * solver_E = new HypreBoomerAMG((HypreParMatrix &)
+                                                           BlockA_r->GetBlock(0,0));
+            solver_E->SetPrintLevel(0);
+            solver_E->SetSystemsOptions(dim);
+            HypreBoomerAMG * solver_H = new HypreBoomerAMG((HypreParMatrix &)
+                                                           BlockA_r->GetBlock(1,1));
+            solver_H->SetPrintLevel(0);
+            // solver_H->SetSystemsOptions(dim);
+            M.SetDiagonalBlock(0,solver_E);
+            M.SetDiagonalBlock(1,solver_H);
+            M.SetDiagonalBlock(num_blocks,solver_E);
+            M.SetDiagonalBlock(num_blocks+1,solver_H);
+         }
+         HypreAMS * solver_hatE = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(skip,
+                                                                                    skip), hatE_fes);
+         HypreBoomerAMG * solver_hatH = new HypreBoomerAMG((HypreParMatrix &)
+                                                           BlockA_r->GetBlock(
+                                                              skip+1,skip+1));
+         solver_hatE->SetPrintLevel(0);
+         solver_hatH->SetPrintLevel(0);
+         solver_hatH->SetRelaxType(88);
+
+         M.SetDiagonalBlock(skip,solver_hatE);
+         M.SetDiagonalBlock(skip+1,solver_hatH);
+         M.SetDiagonalBlock(skip+num_blocks,solver_hatE);
+         M.SetDiagonalBlock(skip+num_blocks+1,solver_hatH);
+
+         if (myid == 0)
+         {
+            std::cout << "PCG iterations" << endl;
+         }
+         CGSolver cg(MPI_COMM_WORLD);
+         cg.SetRelTol(1e-7);
+         cg.SetMaxIter(1500);
+         cg.SetPrintLevel(3);
+         cg.SetPreconditioner(M);
+         cg.SetOperator(blockA);
+         cg.Mult(B, X);
+
+         for (int i = 0; i<num_blocks; i++)
+         {
+            delete &M.GetDiagonalBlock(i);
+         }
+
+         num_iter = cg.GetNumIterations();
+      }
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+      real_t residual = residuals.Norml2();
+      real_t maxresidual = residuals.Max();
+      real_t globalresidual = residual * residual;
+      MPI_Allreduce(MPI_IN_PLACE, &maxresidual, 1, MPITypeMap<real_t>::mpi_type,
+                    MPI_MAX, MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE, &globalresidual, 1,
+                    MPITypeMap<real_t>::mpi_type, MPI_SUM, MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+      E.real().MakeRef(E_fes,x.GetData());
+      E.imag().MakeRef(E_fes,&x.GetData()[offsets.Last()]);
+
+      H.real().MakeRef(H_fes,&x.GetData()[offsets[1]]);
+      H.imag().MakeRef(H_fes,&x.GetData()[offsets.Last()+offsets[1]]);
+
+      int dofs = 0;
+      for (int i = 0; i<trial_fes.Size(); i++)
+      {
+         dofs += trial_fes[i]->GlobalTrueVSize();
+      }
+
+      real_t rate_res = (it) ? dim*log(res0/globalresidual)/log((
+                                                                   real_t)dof0/dofs) : 0.0;
+
+      res0 = globalresidual;
+      dof0 = dofs;
+
+      if (myid == 0)
+      {
+         std::ios oldState(nullptr);
+         oldState.copyfmt(std::cout);
+         std::cout << std::right << std::setw(5) << it << " | "
+                   << std::setw(10) <<  dof0 << " | "
+                   << std::setprecision(1) << std::fixed
+                   << std::setw(4) <<  2.0*rnum << " π  | "
+                   << std::setprecision(3);
+         std::cout << std::setprecision(3)
+                   << std::setw(10) << std::scientific <<  res0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_res << " | "
+                   << std::setw(6) << std::fixed << num_iter << " | "
+                   << std::endl;
+         std::cout.copyfmt(oldState);
+      }
+
+      AzimuthalECoefficient az_e_r(&E.real());
+      AzimuthalECoefficient az_e_i(&E.imag());
+
+      E_theta_r.ProjectCoefficient(az_e_r);
+      E_theta_i.ProjectCoefficient(az_e_i);
+
+      if (visualization)
+      {
+         const char * keys = nullptr;
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         common::VisualizeField(E_out_r,vishost, visport, E.real(),
+                                "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+         common::VisualizeField(H_out_r,vishost, visport, H.real(),
+                                "Numerical Magnetic field (real part)", 501, 0, 500, 500, keys);
+         common::VisualizeField(E_theta_out_r,vishost, visport, E_theta_r,
+                                "Numerical Electric field (Azimuthal-real)", 501, 0, 500, 500, keys);
+      }
+
+      if (paraview)
+      {
+         paraview_dc->SetCycle(it);
+         paraview_dc->SetTime((real_t)it);
+         paraview_dc->Save();
+      }
+      if (it == amr_ref_levels)
+      {
+         int num_frames = 32;
+         for (int i = 0; i<num_frames; i++)
+         {
+            real_t t = (real_t)(i % num_frames) / num_frames;
+            add(cos(real_t(2.0*M_PI)*t), E_theta_r,
+                sin(real_t(2.0*M_PI)*t), E_theta_i, E_theta);
+            paraview_tdc->SetCycle(i);
+            paraview_tdc->SetTime(t);
+            paraview_tdc->Save();
+         }
+         delete paraview_tdc;
+         break;
+      }
+
+      if (theta > 0.0)
+      {
+         elements_to_refine.SetSize(0);
+         for (int iel = 0; iel<pmesh.GetNE(); iel++)
+         {
+            if (residuals[iel] > theta * maxresidual)
+            {
+               elements_to_refine.Append(iel);
+            }
+         }
+         pmesh.GeneralRefinement(elements_to_refine,1,1);
+      }
+      else
+      {
+         pmesh.UniformRefinement();
+      }
+
+      eps_r_cf.Update();
+      eps_i_cf.Update();
+
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      L2_fes.Update();
+      E_theta_r.Update();
+      E_theta_i.Update();
+      E_theta.Update();
+
+      a->Update();
+
+
+   }
+
+   delete a;
+   delete F_fec;
+   delete G_fec;
+   delete hatH_fes;
+   delete hatH_fec;
+   delete hatE_fes;
+   delete hatE_fec;
+   delete H_fec;
+   delete E_fec;
+   delete H_fes;
+   delete E_fes;
+
+   return 0;
+
+}

miniapps/dpg/plasma/other/pmaxwell-uw-tokamak.cpp

@@ -0,0 +1,676 @@
+
+// srun -n 512 ./pmaxwell-uw-tokamak -o 4 -sc -prob 2 -paraview
+// srun -n 512 ./pmaxwell-uw-tokamak -o 4 -sc -prob 4 -paraview
+
+// Description:
+// This example code demonstrates the use of MFEM to define and solve
+// the "ultraweak" (UW) DPG formulation for the Maxwell problem
+
+//      ∇×(1/μ ∇×E) - (ω^2 ϵ + i ω σ) E = Ĵ ,   in Ω
+//                E×n = E_0, on ∂Ω
+
+// The DPG UW deals with the First Order System
+//        i ω μ H + ∇ × E = 0,   in Ω (Faraday's law)
+//            M E + ∇ × H = J,   in Ω (Ampere's law)
+//            E × n = E_0, on ∂Ω
+// Note: Ĵ = -iωJ
+// where M = -(i ω ϵI + σI)
+
+// The ultraweak-DPG formulation is obtained by integration by parts of both
+// equations and the introduction of trace unknowns on the mesh skeleton
+
+// in 3D
+// E,H ∈ (L^2(Ω))^3
+// Ê ∈ H_0^1/2(Ω)(curl, Γ_h), Ĥ ∈ H^-1/2(curl, Γ_h)
+//  i ω μ (H,F) + (E,∇ × F) + < Ê, F × n > = 0,      ∀ F ∈ H(curl,Ω)
+//      M (E,G) + (H,∇ × G) + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)
+//                                   Ê × n = E_0     on ∂Ω
+// -------------------------------------------------------------------------
+// |   |       E      |      H      |      Ê       |       Ĥ      |  RHS    |
+// -------------------------------------------------------------------------
+// | F |  (E,∇ × F)   | i ω μ (H,F) | < n × Ê, F > |              |         |
+// |   |              |             |              |              |         |
+// | G |    (ME,G)    |  (H,∇ × G)  |              | < n × Ĥ, G > |  (J,G)  |
+// where (F,G) ∈  H(curl,Ω) × H(curl,Ω)
+
+// Here we use the "Adjoint Graph" norm on the test space i.e.,
+// ||(F,G)||^2_V = ||A^*(F,G)||^2 + ||(F,G)||^2 where A is the
+// maxwell operator defined by (1)
+
+
+#include "mfem.hpp"
+#include "../../util/pcomplexweakform.hpp"
+#include "../../util/preconditioners.hpp"
+#include "../../../common/mfem-common.hpp"
+#include "../../util/maxwell_utils.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = false;
+   real_t rnum=5.0;
+   bool static_cond = false;
+   int pr = 0;
+   bool paraview = false;
+   real_t mu = 1.0;
+   real_t epsilon = 1.0;
+   real_t sigma = 0.1;
+   bool graph_norm = true;
+   int prob = 0;
+   bool pmg = false;
+   int pmg_levels = -1;
+   real_t relax_factor = 2.0/3;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&sigma, "-sigma", "--sigma",
+                  "conductivity");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&pr, "-pref", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&prob, "-prob", "--problem",
+                  "Choice of problem: 0: tet mesh eps = 1, 1: hex mesh eps = 1 ,  2: tet-mesh eps from file,", 
+                  "3: hex-mesh eps from file, 4: 400k-tet-mesh eps from file");
+   args.AddOption(&pmg, "-pmg", "--p-refinement-multigrid", "-no-pmg",
+                  "--no-p-refinement-multigrid", "Enable P-Refinement Multigrid.");
+   args.AddOption(&pmg_levels, "-pmgl","--p-refinement-multigrid-levels",
+                  "Number of levels for P-Refinement Multigrid.");
+   args.AddOption(&relax_factor, "-rf", "--relaxation-factor",
+                  "Relaxation factor for the p-multigrid smoother.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&graph_norm, "-graph", "--graph-norm", "-no-gn",
+                  "--no-graph-norm", "Enable adjoint graph norm.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+
+   const char *mesh_file = nullptr;
+   if (prob == 0 || prob == 2)
+   {
+      mesh_file = "meshes/tokamak-tet.mesh";
+      if (prob == 2)
+      {
+         rnum=50.0e6;
+         mu = 1.257e-6;
+         sigma = 0.0;
+      }
+   }
+   else if (prob == 1 || prob == 3)
+   {
+      mesh_file = "meshes/tokamak-hex.mesh";
+      if (prob == 3)
+      {
+         rnum=50.0e6;
+         mu = 1.257e-6;
+         sigma = 0.0;
+      }
+   }
+   else if (prob == 4)
+   {
+      mesh_file = "meshes/tokamak-400K-tet-new.mesh";
+      rnum = 54.0e6;
+      mu = 1.257e-6;
+      sigma = 0.0;
+   }
+   else
+   {
+      MFEM_ABORT("No valid problem choice given");
+   }
+
+
+   real_t omega = 2.*M_PI*rnum;
+   Mesh mesh(mesh_file, 1, 1);
+
+   mesh.RemoveInternalBoundaries();
+
+   int dim = mesh.Dimension();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+
+   DenseMatrix mat_eps_i(dim); mat_eps_i = 0.0;
+   DenseMatrix mat_eps_r(dim); mat_eps_r = 0.0;
+   mat_eps_r(0,0) = epsilon;
+   mat_eps_r(1,1) = epsilon;
+   mat_eps_r(2,2) = epsilon;
+
+   DenseMatrix I(dim); I = 0.0;
+   I(0,0) = 1.0;
+   I(1,1) = 1.0;
+   I(2,2) = 1.0;
+   MatrixConstantCoefficient identity_cf(I);
+
+
+   MatrixCoefficient * eps_r_cf = nullptr;
+   MatrixCoefficient * eps_i_cf = nullptr;
+   if (prob == 0 || prob == 1)
+   {
+      eps_r_cf = new MatrixConstantCoefficient(mat_eps_r);
+      eps_i_cf = new MatrixConstantCoefficient(mat_eps_i);
+   }
+   else
+   {
+      const char * eps_r_file = nullptr;
+      const char * eps_i_file = nullptr;
+      if (prob == 2)
+      {
+         eps_r_file = "data/tet-eps_r.gf";
+         eps_i_file = "data/tet-eps_i.gf";
+      }
+      else if (prob == 3)
+      {
+         eps_r_file = "data/hex-eps_r.gf";
+         eps_i_file = "data/hex-eps_i.gf";
+      }
+      else if (prob == 4)
+      {
+         eps_r_file = "data/tet-400k-eps_r.gf";
+         eps_i_file = "data/tet-400k-eps_i.gf";
+      }
+      real_t epsilon_scale = 8.8541878128e-12;
+      eps_r_cf = new EpsilonMatrixCoefficient(eps_r_file,&mesh,&pmesh, epsilon_scale);
+      eps_i_cf = new EpsilonMatrixCoefficient(eps_i_file,&mesh,&pmesh, epsilon_scale);
+   }
+   mesh.Clear();
+   for (int i = 0; i<pr; i++)
+   {
+      pmesh.UniformRefinement();
+      dynamic_cast<EpsilonMatrixCoefficient *>(eps_r_cf)->Update();
+      dynamic_cast<EpsilonMatrixCoefficient *>(eps_i_cf)->Update();
+   }
+
+
+   auto cf_r = dynamic_cast<EpsilonMatrixCoefficient *>(eps_r_cf);
+   auto cf_i = dynamic_cast<EpsilonMatrixCoefficient *>(eps_i_cf);
+
+   std::ostringstream cf_r_file_name;
+   cf_r_file_name << "epsilor_r" << "_prob_" << prob  << "_order_" << order;
+   std::ostringstream cf_i_file_name;
+   cf_i_file_name << "epsilor_i" << "_prob_" << prob  << "_order_" << order;
+
+
+   VisualizeMatrixArrayCoefficient(*cf_r, &pmesh, order, paraview,
+       cf_r_file_name.str().c_str());
+
+   VisualizeMatrixArrayCoefficient(*cf_i, &pmesh, order, paraview,
+       cf_i_file_name.str().c_str());
+    
+
+   int ne = pmesh.GetNE();
+   MPI_Allreduce(MPI_IN_PLACE,&ne,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
+
+   if (myid == 0)
+   {
+      mfem::out << "Number of elements = " << ne << endl;
+   }
+
+   // Matrix Coefficient (M = -i\omega \epsilon - \sigma I);
+   // M = -i * omega * (eps_r + i eps_i) - sigmaI
+   //  = omega eps_i - sigma I + i (-omega eps_r)
+   MatrixSumCoefficient Mr_cf(*eps_i_cf,identity_cf,omega,-sigma);
+   ScalarMatrixProductCoefficient Mi_cf(-omega,*eps_r_cf);
+
+   // Define spaces
+   // L2 space for E
+   FiniteElementCollection *E_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh,E_fec,dim);
+
+   // Vector L2 space for H
+   FiniteElementCollection *H_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *H_fes = new ParFiniteElementSpace(&pmesh,H_fec, dim);
+
+   // H^-1/2 (curl) space for Ê
+   FiniteElementCollection * hatE_fec = nullptr;
+   FiniteElementCollection * hatH_fec = nullptr;
+   FiniteElementCollection * F_fec = nullptr;
+   int test_order = order+delta_order;
+   hatE_fec = new ND_Trace_FECollection(order,dim);
+   hatH_fec = new ND_Trace_FECollection(order,dim);
+   F_fec = new ND_FECollection(test_order, dim);
+
+   ParFiniteElementSpace *hatE_fes = new ParFiniteElementSpace(&pmesh,hatE_fec);
+   ParFiniteElementSpace *hatH_fes = new ParFiniteElementSpace(&pmesh,hatH_fec);
+   FiniteElementCollection * G_fec = new ND_FECollection(test_order, dim);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+   trial_fes.Append(E_fes);
+   trial_fes.Append(H_fes);
+   trial_fes.Append(hatE_fes);
+   trial_fes.Append(hatH_fes);
+   test_fec.Append(F_fec);
+   test_fec.Append(G_fec);
+
+   int gdofs = 0;
+   for (int i = 0; i<trial_fes.Size(); i++)
+   {
+      gdofs += trial_fes[i]->GlobalTrueVSize();
+   }
+
+   if (myid == 0)
+   {
+      mfem::out << "Global number of dofs = " << gdofs << endl;
+   }
+
+   // Bilinear form coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient eps2omeg2(epsilon*epsilon*omega*omega);
+   ConstantCoefficient mu2omeg2(mu*mu*omega*omega);
+   ConstantCoefficient muomeg(mu*omega);
+   ConstantCoefficient negepsomeg(-epsilon*omega);
+   ConstantCoefficient epsomeg(epsilon*omega);
+   ConstantCoefficient negmuomeg(-mu*omega);
+   ConstantCoefficient sigma_cf(sigma);
+   ConstantCoefficient negsigma_cf(-sigma);
+   ConstantCoefficient sigma2_cf(sigma*sigma);
+
+   Vector z_one(3); z_one = 0.0; z_one(2) = 1.0;
+   Vector zero(3); zero = 0.0;
+   Vector z_negone(3); z_negone = 0.0; z_negone(2) = -1.0;
+   VectorConstantCoefficient z_one_cf(z_one);
+   VectorConstantCoefficient z_negone_cf(z_negone);
+   VectorConstantCoefficient zero_cf(zero);
+
+   if (myid == 0)
+   {
+      std::cout << "Assembling matrix" << endl;
+   }
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_fes,test_fec);
+   const IntegrationRule &ir_test = IntRules.Get(pmesh.GetElementGeometry(0),
+                                           2*test_order + 4);
+   const IntegrationRule &ir_trial = IntRules.Get(pmesh.GetElementGeometry(0),
+                                           order+test_order + 4);
+
+   a->SetTestIntegrationRule(ir_test);
+   // a->SetTrialIntegrationRule(ir_trial);
+   // (E,∇ × F)
+   a->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one)),
+                         nullptr,0,0);
+
+
+   // --------------------------------------------------------------------------
+   //  (M E , G) = (M_r E, G) + i (M_i E, G)
+   a->AddTrialIntegrator(
+      new TransposeIntegrator(new VectorFEMassIntegrator(Mr_cf)),
+      new TransposeIntegrator(new VectorFEMassIntegrator(Mi_cf)),0,1);
+   // --------------------------------------------------------------------------
+
+   //  (H,∇ × G)
+   a->AddTrialIntegrator(new TransposeIntegrator(new MixedCurlIntegrator(one)),
+                         nullptr,1,1);
+   // < n×Ĥ ,G>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,3,1);
+   // test integrators
+   // (∇×G ,∇× δG)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,1,1);
+
+   ConstantCoefficient l2weight(1.0);
+
+   // (G,δG)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(l2weight),nullptr,1,1);
+
+   // i ω μ (H, F)
+   a->AddTrialIntegrator(nullptr,
+                         new TransposeIntegrator(new VectorFEMassIntegrator(muomeg)),1,0);
+   // < n×Ê,F>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,2,0);
+   // test integrators
+   // (∇×F,∇×δF)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,0,0);
+   // (F,δF)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,0,0);
+
+   if (graph_norm)
+   {
+      // μ^2 ω^2 (F,δF)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(mu2omeg2),nullptr,0,0);
+      // -i ω μ (F,∇ × δG) = i (F, -ω μ ∇ × δ G)
+      a->AddTestIntegrator(nullptr,new MixedVectorWeakCurlIntegrator(negmuomeg),0,1);
+
+      // --------------------------------------------------------------------------
+      // (M ∇ × F, δG) = (M_r  ∇ × F, δG) + i (M_i  ∇ × F, δG)
+      a->AddTestIntegrator(new MixedVectorCurlIntegrator(Mr_cf),
+                           new MixedVectorCurlIntegrator(Mi_cf),0,1);
+      // --------------------------------------------------------------------------
+
+      // i ω μ (∇ × G,δF)
+      a->AddTestIntegrator(nullptr,new MixedVectorCurlIntegrator(muomeg),1,0);
+   }
+   // --------------------------------------------------------------------------
+   // (M^* G, ∇ × δF ) = (Mr^T G,  ∇ × δF) - i (Mi^T G,  ∇ × δF)
+   TransposeMatrixCoefficient Mrt_cf(Mr_cf);
+   TransposeMatrixCoefficient Mit_cf(Mi_cf);
+   ScalarMatrixProductCoefficient negMit_cf(-1.0,Mit_cf);
+   if (graph_norm)
+   {
+      a->AddTestIntegrator(new MixedVectorWeakCurlIntegrator(Mrt_cf),
+                           new MixedVectorWeakCurlIntegrator(negMit_cf),1,0);
+   }
+   // --------------------------------------------------------------------------
+
+   // M*M^*(G,δG) = (MrMr^t + MiMi^t) + i (MiMr^t - MrMi^t)
+   MatrixProductCoefficient MrMrt_cf(Mr_cf,Mrt_cf);
+   MatrixProductCoefficient MiMit_cf(Mi_cf,Mit_cf);
+   MatrixProductCoefficient MiMrt_cf(Mi_cf,Mrt_cf);
+   MatrixProductCoefficient MrMit_cf(Mr_cf,Mit_cf);
+
+   MatrixSumCoefficient MMr_cf(MrMrt_cf,MiMit_cf);
+   MatrixSumCoefficient MMi_cf(MiMrt_cf,MrMit_cf,1.0,-1.0);
+
+   const IntegrationRule *irs[Geometry::NumGeom];
+   int order_quad = 2*order + 2;
+   for (int i = 0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+   const IntegrationRule &ir = IntRules.Get(pmesh.GetElementGeometry(0),
+                                            2*test_order + 2);
+
+   if (graph_norm)
+   {
+      a->AddTestIntegrator(new VectorFEMassIntegrator(MMr_cf),
+                           new VectorFEMassIntegrator(MMi_cf),1,1);
+   }
+   // --------------------------------------------------------------------------
+
+
+   socketstream E_out_r;
+   socketstream H_out_r;
+
+   ParComplexGridFunction E(E_fes);
+   ParComplexGridFunction H(H_fes);
+   E.real() = 0.0;
+   E.imag() = 0.0;
+   H.real() = 0.0;
+   H.imag() = 0.0;
+
+   ParGridFunction Et(E_fes);
+   ParGridFunction Ht(H_fes);
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+   ParaViewDataCollection * paraview_dct = nullptr;
+   std::ostringstream paraview_file_name;
+   paraview_file_name << "prob_" << prob
+                      << "_order_" << order
+                      << "_pref_" << pr
+                      << "_sc_" << (int)static_cond;
+
+   std::ostringstream paraview_file_namet;
+   paraview_file_namet << "tim-harmonin-prob_" << prob
+                       << "_order_" << order
+                       << "_pref_" << pr
+                       << "_sc_" << (int)static_cond;
+
+   if (paraview)
+   {
+      paraview_dc = new ParaViewDataCollection(paraview_file_name.str(), &pmesh);
+      paraview_dc->SetPrefixPath("ParaView/UW/3D");
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",&E.real());
+      paraview_dc->RegisterField("E_i",&E.imag());
+      paraview_dc->RegisterField("H_r",&H.real());
+      paraview_dc->RegisterField("H_i",&H.imag());
+
+      // paraview_dct = new ParaViewDataCollection(paraview_file_namet.str(), &pmesh);
+      // paraview_dct->SetPrefixPath("ParaView/UW3D");
+      // paraview_dct->SetLevelsOfDetail(order);
+      // paraview_dct->SetCycle(0);
+      // paraview_dct->SetDataFormat(VTKFormat::BINARY);
+      // paraview_dct->SetHighOrderOutput(true);
+      // paraview_dct->SetTime(0.0); // set the time
+      // paraview_dct->RegisterField("Et",&Et);
+      // paraview_dct->RegisterField("Ht",&Ht);
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   Array<int> one_bdr;
+   Array<int> negone_bdr;
+
+   if (myid == 0)
+   {
+      std::cout << "Setting up boundary attributes" << endl;
+   }
+
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      one_bdr.SetSize(pmesh.bdr_attributes.Max());
+      negone_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      hatE_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      one_bdr = 0;
+      negone_bdr = 0;
+      if (prob == 4)
+      {
+         one_bdr[232-1] = 1;
+         negone_bdr[231-1] = 1;
+      }
+      else
+      {
+         one_bdr[1] = 1;
+         negone_bdr[2] = 1;      
+      }
+   }
+
+
+   if (myid == 0)
+   {
+      std::cout << "Setting up boundary conditions" << endl;
+   }
+   // Set up bdr conditions
+   // shift the ess_tdofs
+   for (int j = 0; j < ess_tdof_list.Size(); j++)
+   {
+      ess_tdof_list[j] += E_fes->GetTrueVSize() + H_fes->GetTrueVSize();
+   }
+
+   Array<int> offsets(5);
+   offsets[0] = 0;
+   offsets[1] = E_fes->GetVSize();
+   offsets[2] = H_fes->GetVSize();
+   offsets[3] = hatE_fes->GetVSize();
+   offsets[4] = hatH_fes->GetVSize();
+   offsets.PartialSum();
+
+   Vector x(2*offsets.Last());
+   x = 0.;
+   real_t * xdata = x.GetData();
+
+   ParComplexGridFunction hatE_gf(hatE_fes);
+   hatE_gf.real().MakeRef(hatE_fes,&xdata[offsets[2]]);
+   hatE_gf.imag().MakeRef(hatE_fes,&xdata[offsets.Last()+ offsets[2]]);
+
+   hatE_gf.ProjectBdrCoefficientTangent(z_one_cf,zero_cf, one_bdr);
+   hatE_gf.ProjectBdrCoefficientTangent(z_negone_cf,zero_cf, negone_bdr);
+
+   if (myid == 0)
+   {
+      std::cout << "Assembly started" << endl;
+   }
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   OperatorPtr Ah;
+   Vector X,B;
+   a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+   if (myid == 0)
+   {
+      std::cout << "Assembly finished" << endl;
+   }
+
+   ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+   BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+
+   Array<ParFiniteElementSpace *> prec_fes;
+   if (static_cond)
+   {
+      a->GetTraceFESpaces(prec_fes);
+   }
+   else
+   {
+      prec_fes = trial_fes;
+   }
+   Solver * cprec = nullptr;
+   if (pmg)
+   {
+#ifdef MFEM_USE_COMPLEX_MUMPS
+      bool mumps_coarse_solver = true;
+#else
+      bool mumps_coarse_solver = false;
+#endif
+      std::vector<Array<int>> ess_bdr_marker(prec_fes.Size());
+      for (int b = 0; b<prec_fes.Size(); b++)
+      {
+         if (pmesh.bdr_attributes.Size())
+         {
+            ess_bdr_marker[b].SetSize(pmesh.bdr_attributes.Max());
+            int ess_block = (static_cond) ? 0 : 2;
+            if (b == ess_block) // hatE
+            {
+               ess_bdr_marker[b] = ess_bdr;
+            }
+            else
+            {
+               ess_bdr_marker[b] = 0;
+            }
+         }
+      }
+      cprec = new ComplexPRefinementMultigrid(prec_fes, ess_bdr_marker, *Ahc, 
+                                              pmg_levels, relax_factor, mumps_coarse_solver);
+   }
+   else
+   {
+      BlockDiagonalPreconditioner * real_prec = new BlockDiagonalPreconditioner(BlockA_r->RowOffsets());
+      real_prec->owns_blocks = 1;
+      for (int i = 0; i<BlockA_r->NumRowBlocks(); i++)
+      {
+         auto prec = MakeFESpaceDefaultSolver(prec_fes[i],0);
+         prec->SetOperator(BlockA_r->GetBlock(i,i));
+         real_prec->SetDiagonalBlock(i,prec);
+      }
+      cprec = new ComplexPreconditioner(real_prec, true);
+   }
+
+   if (myid == 0)
+   {
+      std::cout << "PCG iterations" << endl;
+   }
+   CGSolver cg(MPI_COMM_WORLD);
+   // GMRESSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-6);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   cg.SetOperator(*Ahc);
+   cg.SetPreconditioner(*cprec);
+   cg.Mult(B, X);
+
+   delete cprec;
+
+
+   a->RecoverFEMSolution(X,x);
+
+   E.real().MakeRef(E_fes,x.GetData());
+   E.imag().MakeRef(E_fes,&x.GetData()[offsets.Last()]);
+
+   H.real().MakeRef(H_fes,&x.GetData()[offsets[1]]);
+   H.imag().MakeRef(H_fes,&x.GetData()[offsets.Last()+offsets[1]]);
+
+   if (visualization)
+   {
+      const char * keys = (dim == 2) ? "jRcml\n" : nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      common::VisualizeField(E_out_r,vishost, visport, E.real(),
+                             "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+      common::VisualizeField(H_out_r,vishost, visport, H.real(),
+                             "Numerical Magnetic field (real part)", 501, 0, 500, 500, keys);
+   }
+
+   if (paraview)
+   {
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetTime(0.0);
+      paraview_dc->Save();
+
+      // Et = E.real();
+      // int num_frames = 32;
+      // for (int it = 0; it<num_frames; it++)
+      // {
+      //    real_t t = (real_t)(it % num_frames) / num_frames;
+      //    add(cos(2.0*M_PI*t), E.real(), sin(2.0*M_PI*t), E.imag(), Et);
+      //    paraview_dct->SetCycle(it);
+      //    paraview_dct->SetTime((real_t)it);
+      //    paraview_dct->Save();
+      // }
+
+
+
+   }
+
+   if (paraview)
+   {
+      delete paraview_dc;
+   }
+
+   delete eps_r_cf;
+   delete eps_i_cf;
+   delete a;
+   delete F_fec;
+   delete G_fec;
+   delete hatH_fes;
+   delete hatH_fec;
+   delete hatE_fes;
+   delete hatE_fec;
+   delete H_fec;
+   delete E_fec;
+   delete H_fes;
+   delete E_fes;
+
+   return 0;
+}
+

miniapps/dpg/plasma/utils/lh_utils.cpp

@@ -0,0 +1,96 @@
+#include "lh_utils.hpp"
+
+real_t delta = 0.01; 
+real_t a0 = -1.0;   
+real_t a1 = 5.0;    
+
+void bfunc(const Vector &x, Vector &b)
+{
+   real_t r = std::sqrt(x(0) * x(0) + x(1) * x(1));
+   int dim = x.Size();
+   b.SetSize(dim); b = 0.0;
+   b(0) = -x(1) / r;
+   b(1) =  x(0) / r;
+   if (dim == 3) b(2) = 0.0;
+}
+
+void bcrossb(const Vector &x, DenseMatrix &bb)
+{
+   Vector b;
+   bfunc(x, b);
+   bb.SetSize(b.Size());
+   MultVVt(b, bb);
+}
+
+std::complex<real_t> pfunc(const Vector &x)
+{
+   real_t r = std::sqrt(x(0) * x(0) + x(1) * x(1));
+   return std::complex<real_t>(a0 + a1 *(r-0.9), delta);
+}
+
+std::complex<real_t> sfunc(const Vector &x)
+{
+   return std::complex<real_t>(1.0, delta);
+}
+
+real_t pfunc_r(const Vector &x) { return pfunc(x).real(); }
+real_t pfunc_i(const Vector &x) { return pfunc(x).imag(); }
+
+real_t sfunc_r(const Vector &x) { return sfunc(x).real(); }
+real_t sfunc_i(const Vector &x) { return sfunc(x).imag(); }
+
+
+void epsilon_func_r(const Vector &x, DenseMatrix &eps)
+{
+   std::complex<real_t> p = pfunc(x);
+   std::complex<real_t> s = sfunc(x);
+   DenseMatrix B;
+   bcrossb(x, B);
+   int dim = x.Size();
+   eps.SetSize(dim);
+
+   for (int i = 0; i < dim; i++)
+   {
+      for (int j = 0; j < dim; j++)
+      {
+         if (i == j)
+         {
+            eps(i, j) = (s * (1.0 - B(i, j)) + p * B(i, j)).real();
+         }
+         else
+         {
+            eps(i, j) = ((p - s) * B(i, j)).real();
+         }
+      }
+   }
+}
+
+void epsilon_func_i(const Vector &x, DenseMatrix &eps)
+{
+   int dim = x.Size();
+   eps.SetSize(dim);
+
+   std::complex<real_t> p = pfunc(x);
+   std::complex<real_t> s = sfunc(x);
+   DenseMatrix B;
+   bcrossb(x, B);
+
+   for (int i = 0; i < dim; i++)
+   {
+      for (int j = 0; j < dim; j++)
+      {
+         if (i == j)
+         {
+            eps(i, j) = (s * (1.0 - B(i, j)) + p * B(i, j)).imag();
+         }
+         else
+         {
+            eps(i, j) = ((p - s) * B(i, j)).imag();
+         }
+      }
+   }
+}
+
+
+
+

miniapps/dpg/plasma/utils/lh_utils.hpp

@@ -0,0 +1,21 @@
+#pragma once
+#include "mfem.hpp"
+
+using namespace mfem;
+using namespace std;
+
+extern double delta;
+extern double a0;
+extern double a1;
+
+std::complex<real_t> pfunc(const Vector &x);
+std::complex<real_t> sfunc(const Vector &x);
+
+real_t pfunc_r(const Vector &x);
+real_t pfunc_i(const Vector &x);
+real_t sfunc_r(const Vector &x);
+real_t sfunc_i(const Vector &x);
+void bfunc(const Vector &x, Vector &b);
+void bcrossb(const Vector &x, DenseMatrix &bb);
+void epsilon_func_r(const Vector &x, DenseMatrix &eps);
+void epsilon_func_i(const Vector &x, DenseMatrix &eps);

miniapps/dpg/pmaxwell-coupled-fem.cpp

@@ -0,0 +1,642 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//                   MFEM Ultraweak DPG Maxwell parallel example
+//
+// Compile with: make pmaxwell-coupled
+//
+// mpirun -np 4 ./pmaxwell-coupled -sref 1 -pref 2 -o 2 -rnum 0.5 -m ../../data/ref-cube.mesh -sc
+
+//      ∇×(1/μ ∇×E) - ω² ϵ E + J = F̃ ,   in Ω
+//             -ΔJ + α² J + c² E = G ,   in Ω
+//                       E×n = E₀ , on ∂Ω
+//                         J = J₀ , on ∂Ω
+
+#include "mfem.hpp"
+#include "util/pcomplexweakform.hpp"
+#include "util/pcomplexblockform.hpp"
+#include "../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+void maxwell_solution(const Vector &x, std::vector<complex<double>> &E);
+void maxwell_solution_curl(const Vector &x,
+                           std::vector<complex<double>> &curlE);
+void maxwell_solution_curlcurl(const Vector &x,
+                               std::vector<complex<double>> &curlcurlE);
+void J_solution(const Vector &x,std::vector<complex<double>> &J);
+void J_solution_grad(const Vector &x,
+                     std::vector<std::vector<complex<double>>> &gradJ);
+void J_solution_laplace(const Vector &x,
+                        std::vector<complex<double>> &laplaceJ);
+
+
+void E_exact_r(const Vector &x, Vector & E_r);
+void E_exact_i(const Vector &x, Vector & E_i);
+void H_exact_r(const Vector &x, Vector & H_r);
+void H_exact_i(const Vector &x, Vector & H_i);
+void J_exact_r(const Vector &x, Vector & J_r);
+void J_exact_i(const Vector &x, Vector & J_i);
+
+void curlE_exact_r(const Vector &x, Vector &curlE_r);
+void curlE_exact_i(const Vector &x, Vector &curlE_i);
+void curlH_exact_r(const Vector &x,Vector &curlH_r);
+void curlH_exact_i(const Vector &x,Vector &curlH_i);
+void gradJ_exact_r(const Vector &x, DenseMatrix &gradJ_r);
+void gradJ_exact_i(const Vector &x, DenseMatrix &gradJ_i);
+
+void curlcurlE_exact_r(const Vector &x, Vector & curlcurlE_r);
+void curlcurlE_exact_i(const Vector &x, Vector & curlcurlE_i);
+void LaplaceJ_exact_r(const Vector &x, Vector & d2J_r);
+void LaplaceJ_exact_i(const Vector &x, Vector & d2J_i);
+
+void  rhs1_func_r(const Vector &x, Vector & rhs1_r);
+void  rhs1_func_i(const Vector &x, Vector & rhs1_i);
+void  rhs2_func_r(const Vector &x, Vector & rhs2_r);
+void  rhs2_func_i(const Vector &x, Vector & rhs2_i);
+
+int dim;
+int dimc;
+double omega;
+double mu = 1.0;
+double epsilon = 1.0;
+double alpha = 1.0;
+double c = 1.0;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "../../data/inline-square.mesh";
+   int order = 1;
+   double rnum=1.0;
+   int sr = 0;
+   int pr = 1;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&rnum, "-rnum", "--number-of-wavelengths",
+                  "Number of wavelengths");
+   args.AddOption(&sr, "-sref", "--serial-ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&pr, "-pref", "--parallel-ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+
+   omega = 2.*M_PI*rnum;
+
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+   MFEM_VERIFY(dim > 1, "Dimension = 1 is not supported in this example");
+
+   dimc = (dim == 3) ? 3 : 1;
+
+   for (int i = 0; i<sr; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   enum TrialSpace
+   {
+      E_space     = 0,
+      J_space     = 1
+   };
+
+   // Vector L2 space for E
+   FiniteElementCollection *E_fec = new ND_FECollection(order,dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh,E_fec);
+
+   // Vector H1 space for J
+   FiniteElementCollection *J_fec = new H1_FECollection(order,dim);
+   ParFiniteElementSpace *J_fes = new ParFiniteElementSpace(&pmesh,J_fec, dim);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   trial_fes.Append(E_fes);
+   trial_fes.Append(J_fes);
+
+   ParComplexBlockForm * a = new ParComplexBlockForm(trial_fes);
+
+   // // Bilinear form coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient muinv(1./mu);
+   ConstantCoefficient rec_omega(1.0/omega);
+   ConstantCoefficient muomeg_cf(mu*omega);
+   ConstantCoefficient mu2omeg2_cf(mu*mu*omega*omega);
+   ConstantCoefficient eps2omeg2_cf(epsilon*epsilon*omega*omega);
+   ConstantCoefficient negepsomeg_cf(-epsilon*omega);
+   ConstantCoefficient epsomeg_cf(epsilon*omega);
+   ConstantCoefficient negmuomeg_cf(-mu*omega);
+   ConstantCoefficient c2_cf(c*c);
+   ConstantCoefficient a2_cf(alpha*alpha);
+
+   ConstantCoefficient negepsomeg2_cf(-epsilon*omega*omega);
+
+
+   // (1/μ ∇ × E,∇ × δE)
+   a->AddDomainIntegrator(new CurlCurlIntegrator(muinv), nullptr,
+                          TrialSpace::E_space, TrialSpace::E_space);
+   // (- ω² ϵ E, δE)
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(negepsomeg2_cf), nullptr,
+                          TrialSpace::E_space, TrialSpace::E_space);
+   // //  (J, δE)
+   a->AddDomainIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(one)),
+                          nullptr,
+                          TrialSpace::J_space, TrialSpace::E_space);
+
+   // // (∇ J, ∇ δJ)
+   a->AddDomainIntegrator(new VectorDiffusionIntegrator(one), nullptr,
+                          TrialSpace::J_space, TrialSpace::J_space);
+   // // (α²J,   δJ)
+   a->AddDomainIntegrator(new VectorMassIntegrator(a2_cf), nullptr,
+                          TrialSpace::J_space, TrialSpace::J_space);
+   // // (c²E,   δJ)
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(c2_cf), nullptr,
+                          TrialSpace::E_space, TrialSpace::J_space);
+
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = E_fes->GetVSize();
+   offsets[2] = J_fes->GetVSize();
+   offsets.PartialSum();
+
+   Vector x(2*offsets.Last());
+   x = 0.;
+   Vector b(2*offsets.Last());
+   real_t * bdata = b.GetData();
+
+   ParLinearForm bE_r(E_fes, bdata);
+   ParLinearForm bE_i(E_fes, &bdata[offsets.Last()]);
+   ParLinearForm bJ_r(J_fes, &bdata[offsets[1]]);
+   ParLinearForm bJ_i(J_fes, &bdata[offsets[2]]);
+
+   VectorFunctionCoefficient f_rhs1_r(dim,rhs1_func_r);
+   VectorFunctionCoefficient f_rhs1_i(dim,rhs1_func_i);
+
+   bE_r.AddDomainIntegrator(new VectorFEDomainLFIntegrator(f_rhs1_r));
+   bE_i.AddDomainIntegrator(new VectorFEDomainLFIntegrator(f_rhs1_i));
+
+
+
+   VectorFunctionCoefficient f_rhs2_r(dim,rhs2_func_r);
+   VectorFunctionCoefficient f_rhs2_i(dim,rhs2_func_i);
+
+   bJ_r.AddDomainIntegrator(new VectorDomainLFIntegrator(f_rhs2_r));
+   bJ_i.AddDomainIntegrator(new VectorDomainLFIntegrator(f_rhs2_i));
+   bE_r.Assemble();
+   bE_i.Assemble();
+   bJ_r.Assemble();
+   bJ_i.Assemble();
+
+
+   socketstream E_out_r, E_out_i, Eex_out_r, Eex_out_i;
+   socketstream J_out_r, J_out_i, Jex_out_r, Jex_out_i;
+
+
+   ParGridFunction E_r(E_fes), E_i(E_fes);
+   ParGridFunction J_r(J_fes), J_i(J_fes);
+   ParGridFunction Eex_r(E_fes), Eex_i(E_fes);
+   ParGridFunction Jex_r(J_fes), Jex_i(J_fes);
+
+
+   a->Assemble();
+
+   // cin.get();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listE;
+   Array<int> ess_tdof_listJ;
+   Array<int> ess_bdr;
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      E_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_listE);
+      J_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_listJ);
+   }
+
+   // shift the ess_tdofs
+   for (int j = 0; j < ess_tdof_listJ.Size(); j++)
+   {
+      ess_tdof_listJ[j] += E_fes->GetTrueVSize();
+   }
+   ess_tdof_list.Append(ess_tdof_listE);
+   ess_tdof_list.Append(ess_tdof_listJ);
+
+   ParGridFunction E_gf_r(E_fes, x, offsets[0]); E_gf_r = 0.0;
+   ParGridFunction E_gf_i(E_fes, x, offsets.Last()); E_gf_i = 0.0;
+   VectorFunctionCoefficient Eex_cf_r(dim,E_exact_r);
+   VectorFunctionCoefficient Eex_cf_i(dim,E_exact_i);
+   E_gf_r.ProjectBdrCoefficientTangent(Eex_cf_r, ess_bdr);
+   E_gf_i.ProjectBdrCoefficientTangent(Eex_cf_i, ess_bdr);
+
+   VectorFunctionCoefficient Ecf_r(dim,E_exact_r);
+   VectorFunctionCoefficient Ecf_i(dim,E_exact_i);
+   VectorFunctionCoefficient Jcf_r(dim,J_exact_r);
+   VectorFunctionCoefficient Jcf_i(dim,J_exact_i);
+
+   ParGridFunction J_gf_r(J_fes, x, offsets[1]); J_gf_r = 0.0;
+   ParGridFunction J_gf_i(J_fes, x, offsets.Last() + offsets[1]); J_gf_i = 0.0;
+   J_gf_r.ProjectBdrCoefficient(Jcf_r,ess_bdr);
+   J_gf_i.ProjectBdrCoefficient(Jcf_i,ess_bdr);
+
+   OperatorPtr Ah;
+   Vector X,B;
+   a->FormLinearSystem(ess_tdof_list,x,b, Ah, X,B);
+
+   ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+   BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+   BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+   int nblocks = BlockA_r->NumRowBlocks();
+   Array2D<const HypreParMatrix*> A_r_matrices(nblocks, nblocks);
+   Array2D<const HypreParMatrix*> A_i_matrices(nblocks, nblocks);
+   for (int i = 0; i < nblocks; i++)
+   {
+      for (int j = 0; j < nblocks; j++)
+      {
+         A_r_matrices(i,j) = dynamic_cast<const HypreParMatrix*>(&BlockA_r->GetBlock(i,
+                                                                                     j));
+         A_i_matrices(i,j) = dynamic_cast<const HypreParMatrix*>(&BlockA_i->GetBlock(i,
+                                                                                     j));
+      }
+   }
+   HypreParMatrix * Ahr = HypreParMatrixFromBlocks(A_r_matrices);
+   HypreParMatrix * Ahi = HypreParMatrixFromBlocks(A_i_matrices);
+
+   ComplexHypreParMatrix * Ahc_hypre =
+      new ComplexHypreParMatrix(Ahr, Ahi,false, false);
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Assembly finished successfully." << endl;
+   }
+
+#ifdef MFEM_USE_COMPLEX_MUMPS
+   auto solver = new ComplexMUMPSSolver(MPI_COMM_WORLD);
+   solver->SetPrintLevel(1);
+   solver->SetOperator(*Ahc_hypre);
+   solver->Mult(B,X);
+   delete solver;
+   delete Ahc_hypre;
+#else
+   MFEM_ABORT("MFEM compiled without mumps");
+#endif
+
+   a->RecoverFEMSolution(X, x);
+
+
+   E_r.MakeRef(E_fes,x, 0);
+   E_i.MakeRef(E_fes,x, offsets.Last());
+
+   J_r.MakeRef(J_fes,x, offsets[1]);
+   J_i.MakeRef(J_fes,x, offsets.Last()+offsets[1]);
+
+   if (visualization)
+   {
+      // const char * keys = (it == 0 && dim == 2) ? "jRcml\n" : nullptr;
+      const char * keys = nullptr;
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      VisualizeField(E_out_r,vishost, visport, E_r,
+                     "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+      VisualizeField(J_out_r,vishost, visport, J_r,
+                     "Numerical J field (real part)", 0, 0, 500, 500, keys);
+
+   }
+
+   delete a;
+
+   return 0;
+}
+
+void maxwell_solution(const Vector & X, std::vector<complex<double>> &E)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   E.resize(dim);
+   for (int i = 0; i < dim; ++i)
+   {
+      E[i] = 0.0;
+   }
+   E[0] = exp(zi * omega * (X.Sum()));
+}
+
+void maxwell_solution_curl(const Vector & X,
+                           std::vector<complex<double>> &curlE)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   curlE.resize(dimc);
+   for (int i = 0; i < dimc; ++i)
+   {
+      curlE[i] = 0.0;
+   }
+
+   std::complex<double> pw = exp(zi * omega * (X.Sum()));
+   if (dim == 3)
+   {
+      curlE[0] = 0.0;
+      curlE[1] = zi * omega * pw;
+      curlE[2] = -zi * omega * pw;
+   }
+   else
+   {
+      curlE[0] = -zi * omega * pw;
+   }
+}
+
+void maxwell_solution_curlcurl(const Vector & X,
+                               std::vector<complex<double>> &curlcurlE)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   curlcurlE.resize(dim);
+   for (int i = 0; i < dim; ++i)
+   {
+      curlcurlE[i] = 0.0;;
+   }
+   std::complex<double> pw = exp(zi * omega * (X.Sum()));
+   if (dim == 3)
+   {
+      curlcurlE[0] = 2.0 * omega * omega * pw;
+      curlcurlE[1] = - omega * omega * pw;
+      curlcurlE[2] = - omega * omega * pw;
+   }
+   else
+   {
+      curlcurlE[0] = omega * omega * pw;
+      curlcurlE[1] = -omega * omega * pw;
+   }
+}
+
+void J_solution(const Vector &x,std::vector<complex<double>> &J)
+{
+   J.resize(dim);
+   for (int i = 0; i < dim; ++i)
+   {
+      J[i] = 0.0;
+   }
+   J[0] = x[0] * x[0];
+}
+
+void J_solution_grad(const Vector &x,
+                     std::vector<std::vector<complex<double>>> &gradJ)
+{
+   gradJ.resize(dim);
+   for (int i = 0; i < dim; ++i)
+   {
+      gradJ[i].resize(dim);
+      for (int j = 0; j < dim; ++j)
+      {
+         gradJ[i][j] = 0.0;
+      }
+   }
+   gradJ[0][0] = 2*x[0];
+}
+
+void J_solution_laplace(const Vector &x, std::vector<complex<double>> &laplaceJ)
+{
+   laplaceJ.resize(dim);
+   for (int i = 0; i < dim; ++i)
+   {
+      laplaceJ[i] = 0.0;
+   }
+   laplaceJ[0] = 2.0;
+}
+
+void E_exact_r(const Vector &x, Vector & E_r)
+{
+   std::vector<std::complex<double>> E;
+   maxwell_solution(x,E);
+   E_r.SetSize(E.size());
+   for (unsigned i = 0; i < E.size(); i++)
+   {
+      E_r[i]= E[i].real();
+   }
+}
+
+void E_exact_i(const Vector &x, Vector & E_i)
+{
+   std::vector<std::complex<double>> E;
+   maxwell_solution(x, E);
+   E_i.SetSize(E.size());
+   for (unsigned i = 0; i < E.size(); i++)
+   {
+      E_i[i]= E[i].imag();
+   }
+}
+
+void J_exact_r(const Vector &x, Vector & J_r)
+{
+   std::vector<std::complex<double>> J;
+   J_solution(x,J);
+   J_r.SetSize(J.size());
+   for (unsigned i = 0; i < J.size(); i++)
+   {
+      J_r[i]= J[i].real();
+   }
+}
+
+void J_exact_i(const Vector &x, Vector & J_i)
+{
+   std::vector<std::complex<double>> J;
+   J_solution(x,J);
+   J_i.SetSize(J.size());
+   for (unsigned i = 0; i < J.size(); i++)
+   {
+      J_i[i]= J[i].imag();
+   }
+}
+
+void curlE_exact_r(const Vector &x, Vector &curlE_r)
+{
+   std::vector<std::complex<double>> curlE;
+   maxwell_solution_curl(x, curlE);
+   curlE_r.SetSize(curlE.size());
+   for (unsigned i = 0; i < curlE.size(); i++)
+   {
+      curlE_r[i]= curlE[i].real();
+   }
+}
+
+void curlE_exact_i(const Vector &x, Vector &curlE_i)
+{
+   std::vector<std::complex<double>> curlE;
+   maxwell_solution_curl(x, curlE);
+   curlE_i.SetSize(curlE.size());
+   for (unsigned i = 0; i < curlE.size(); i++)
+   {
+      curlE_i[i]= curlE[i].imag();
+   }
+}
+
+void gradJ_exact_r(const Vector &x, DenseMatrix &gradJ_r)
+{
+   std::vector<std::vector<std::complex<double>>> gradJ;
+   J_solution_grad(x, gradJ);
+   gradJ_r.SetSize(gradJ.size());
+   for (unsigned i = 0; i < gradJ.size(); i++)
+   {
+      for (unsigned j = 0; j < gradJ.size(); j++)
+      {
+         gradJ_r(i,j)= gradJ[i][j].real();
+      }
+   }
+}
+
+void gradJ_exact_i(const Vector &x, DenseMatrix &gradJ_i)
+{
+   std::vector<std::vector<std::complex<double>>> gradJ;
+   J_solution_grad(x, gradJ);
+   gradJ_i.SetSize(gradJ.size());
+   for (unsigned i = 0; i < gradJ.size(); i++)
+   {
+      for (unsigned j = 0; j < gradJ.size(); j++)
+      {
+         gradJ_i(i,j)= gradJ[i][j].imag();
+      }
+   }
+}
+
+void curlcurlE_exact_r(const Vector &x, Vector & curlcurlE_r)
+{
+   std::vector<std::complex<double>> curlcurlE;
+   maxwell_solution_curlcurl(x, curlcurlE);
+   curlcurlE_r.SetSize(curlcurlE.size());
+   for (unsigned i = 0; i < curlcurlE.size(); i++)
+   {
+      curlcurlE_r[i]= curlcurlE[i].real();
+   }
+}
+
+void curlcurlE_exact_i(const Vector &x, Vector & curlcurlE_i)
+{
+   std::vector<std::complex<double>> curlcurlE;
+   maxwell_solution_curlcurl(x, curlcurlE);
+   curlcurlE_i.SetSize(curlcurlE.size());
+   for (unsigned i = 0; i < curlcurlE.size(); i++)
+   {
+      curlcurlE_i[i]= curlcurlE[i].imag();
+   }
+}
+
+
+void LaplaceJ_exact_r(const Vector &x, Vector & d2J_r)
+{
+   std::vector<std::complex<double>> d2J;
+   J_solution_laplace(x, d2J);
+   d2J_r.SetSize(d2J.size());
+   for (unsigned i = 0; i < d2J.size(); i++)
+   {
+      d2J_r[i]= d2J[i].real();
+   }
+}
+
+void LaplaceJ_exact_i(const Vector &x, Vector & d2J_i)
+{
+   std::vector<std::complex<double>> d2J;
+   J_solution_laplace(x, d2J);
+   d2J_i.SetSize(d2J.size());
+   for (unsigned i = 0; i < d2J.size(); i++)
+   {
+      d2J_i[i]= d2J[i].imag();
+   }
+}
+
+// F = ∇×(1/μ ∇×E) - ω² ϵ E + J
+void  rhs1_func_r(const Vector &x, Vector & F_r)
+{
+   Vector E_r, curlcurlE_r, J_r;
+   E_exact_r(x,E_r);
+   curlcurlE_exact_r(x,curlcurlE_r);
+   J_exact_r(x,J_r);
+   F_r.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      F_r(i) = 1.0/mu * curlcurlE_r(i)
+               - omega * omega * epsilon * E_r(i)
+               + J_r(i);
+   }
+}
+
+void  rhs1_func_i(const Vector &x, Vector & F_i)
+{
+   Vector E_i, curlcurlE_i, J_i;
+   E_exact_i(x,E_i);
+   curlcurlE_exact_i(x,curlcurlE_i);
+   J_exact_i(x,J_i);
+   F_i.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      F_i(i) = 1.0/mu * curlcurlE_i(i)
+               - omega * omega * epsilon * E_i(i)
+               + J_i(i);
+   }
+}
+
+
+// G = -ΔJ + α² J + c² E
+// G_r + i G_i = - Δ (J_r + i J_i) + α² (J_r + i J_i) + c² (E_r + i E_i)
+void  rhs2_func_r(const Vector &x, Vector & G_r)
+{
+   // G_r = - Δ J_r + α² J_r + c² E_r
+   Vector E_r, J_r, d2J_r;
+   E_exact_r(x,E_r);
+   J_exact_r(x,J_r);
+   LaplaceJ_exact_r(x,d2J_r);
+   G_r.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      G_r(i) = -d2J_r[i] + alpha*alpha*J_r[i] + c*c * E_r[i];
+   }
+}
+
+void  rhs2_func_i(const Vector &x, Vector & G_i)
+{
+   // G_i = - Δ J_i + α² J_i + c² E_i
+   Vector E_i, J_i, d2J_i;
+   E_exact_i(x,E_i);
+   J_exact_i(x,J_i);
+   LaplaceJ_exact_i(x,d2J_i);
+   G_i.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      G_i(i) = -d2J_i[i] + alpha*alpha*J_i[i] + c*c * E_i[i];
+   }
+}

miniapps/dpg/pmaxwell-primal.cpp

@@ -0,0 +1,595 @@
+//                   MFEM Ultraweak DPG Maxwell parallel example
+//
+// Compile with: make pmaxwell
+//
+// sample run
+// mpirun -np 4 pmaxwell -m ../../data/star.mesh -o 2 -sref 0 -pref 3 -rnum 1.0
+// mpirun -np 4 pmaxwell -m ../../data/inline-quad.mesh -o 3 -sref 0 -pref 3 -rnum 4.8 -sc
+// mpirun -np 4 pmaxwell -m ../../data/inline-hex.mesh -o 2 -sref 0 -pref 1 -rnum 0.8 -sc
+
+// Description:
+// This example code demonstrates the use of MFEM to define and solve
+// the "ultraweak" (UW) DPG formulation for the Maxwell problem
+
+//      ∇×(1/μ ∇×E) - ω² ϵ E = J ,   in Ω
+//                       E×n = E₀ , on ∂Ω
+
+// It solves the following kinds of problems
+// 1) Known exact solutions with error convergence rates
+//    a) A manufactured solution problem where E is a plane wave
+
+#include "mfem.hpp"
+#include "util/pcomplexweakform.hpp"
+#include "../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+void E_exact_r(const Vector &x, Vector & E_r);
+void E_exact_i(const Vector &x, Vector & E_i);
+
+void  rhs_func_r(const Vector &x, Vector & J_r);
+void  rhs_func_i(const Vector &x, Vector & J_i);
+
+void curlE_exact_r(const Vector &x, Vector &curlE_r);
+void curlE_exact_i(const Vector &x, Vector &curlE_i);
+
+void curlcurlE_exact_r(const Vector &x, Vector & curlcurlE_r);
+void curlcurlE_exact_i(const Vector &x, Vector & curlcurlE_i);
+
+void maxwell_solution(const Vector & X,
+                      std::vector<complex<double>> &E);
+
+void maxwell_solution_curl(const Vector & X,
+                           std::vector<complex<double>> &curlE);
+
+void maxwell_solution_curlcurl(const Vector & X,
+                               std::vector<complex<double>> &curlcurlE);
+
+int dim;
+int dimc;
+double omega;
+double mu = 1.0;
+double epsilon = 1.0;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   double rnum=1.0;
+   bool static_cond = false;
+   int sr = 0;
+   int pr = 1;
+   bool visualization = true;
+   bool paraview = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&rnum, "-rnum", "--number-of-wavelengths",
+                  "Number of wavelengths");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&delta_order, "-do", "--delta-order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&sr, "-sref", "--serial-ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&pr, "-pref", "--parallel-ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+
+   omega = 2.*M_PI*rnum;
+
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+   MFEM_VERIFY(dim > 1, "Dimension = 1 is not supported in this example");
+
+   dimc = (dim == 3) ? 3 : 1;
+
+   for (int i = 0; i<sr; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   enum TrialSpace { E_space = 0, hatE_space = 1 };
+   enum TestSpace  { F_space = 0 };
+
+   // H(curl) space for E
+   FiniteElementCollection *E_fec = new ND_FECollection(order,dim);
+   ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh,E_fec);
+
+   // H^-1/2 (curl) space for Ê
+   FiniteElementCollection * hatE_fec = nullptr;
+   int test_order = order+delta_order;
+   if (dim == 2)
+   {
+      hatE_fec = new H1_Trace_FECollection(order,dim);
+   }
+   else
+   {
+      hatE_fec = new ND_Trace_FECollection(order,dim);
+   }
+   ParFiniteElementSpace *hatE_fes = new ParFiniteElementSpace(&pmesh,hatE_fec);
+   FiniteElementCollection * F_fec = new ND_FECollection(test_order, dim);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+   trial_fes.Append(E_fes);
+   trial_fes.Append(hatE_fes);
+   test_fec.Append(F_fec);
+
+   // Bilinear form coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negepsomeg2(-epsilon*omega*omega);
+   ConstantCoefficient muinv(1./mu);
+   // for the 2D case
+
+   ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_fes,test_fec);
+   a->StoreMatrices(); // needed for AMR
+
+   // (∇ × E,∇ × F)
+   a->AddTrialIntegrator(new CurlCurlIntegrator(muinv), nullptr,
+                         TrialSpace::E_space, TestSpace::F_space);
+   // -ω² ϵ (E , F)
+   a->AddTrialIntegrator(new VectorFEMassIntegrator(negepsomeg2), nullptr,
+                         TrialSpace::E_space,TestSpace::F_space);
+   // < n×Ê ,F>
+   if (dim == 3)
+   {
+      a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,
+                            TrialSpace::hatE_space, TestSpace::F_space);
+   }
+   else
+   {
+      a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,
+                            TrialSpace::hatE_space, TestSpace::F_space);
+   }
+   // test integrators
+   // (∇×F ,∇×δF)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,
+                        TestSpace::F_space,TestSpace::F_space);
+   // (F,δF)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,
+                        TestSpace::F_space,TestSpace::F_space);
+
+   // RHS
+   VectorFunctionCoefficient f_rhs_r(dim,rhs_func_r);
+   VectorFunctionCoefficient f_rhs_i(dim,rhs_func_i);
+   a->AddDomainLFIntegrator(new VectorFEDomainLFIntegrator(f_rhs_r),
+                            new VectorFEDomainLFIntegrator(f_rhs_i),
+                            TestSpace::F_space);
+
+   VectorFunctionCoefficient Eex_r(dim,E_exact_r);
+   VectorFunctionCoefficient Eex_i(dim,E_exact_i);
+
+   socketstream E_out_r;
+   if (myid == 0)
+   {
+      std::cout << "\n  Ref |"
+                << "    Dofs    |"
+                << "    ω    |" ;
+      std::cout  << "  H(curl) Error  |"
+                 << "  Rate  |" ;
+      std::cout << "  Residual  |"
+                << "  Rate  |"
+                << " PCG it |" << endl;
+      std::cout << std::string(87,'-')
+                << endl;
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   ParGridFunction E_r, E_i;
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+
+   if (paraview)
+   {
+      paraview_dc = new ParaViewDataCollection("Plane", &pmesh);
+      paraview_dc->SetPrefixPath("ParaViewPrimal/Maxwell");
+      paraview_dc->SetLevelsOfDetail(order);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("E_r",&E_r);
+      paraview_dc->RegisterField("E_i",&E_i);
+   }
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+   for (int it = 0; it<=pr; it++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         E_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      Array<int> offsets(3);
+      offsets[0] = 0;
+      offsets[1] = E_fes->GetVSize();
+      offsets[2] = hatE_fes->GetVSize();
+      offsets.PartialSum();
+
+      Vector x(2*offsets.Last());
+      x = 0.;
+
+      ParGridFunction E_gf_r(E_fes, x, offsets[0]);
+      ParGridFunction E_gf_i(E_fes, x, offsets.Last() + offsets[0]);
+      E_gf_r.ProjectBdrCoefficientTangent(Eex_r, ess_bdr);
+      E_gf_i.ProjectBdrCoefficientTangent(Eex_i, ess_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+      int num_blocks = BlockA_r->NumRowBlocks();
+      Array<int> tdof_offsets(2*num_blocks+1);
+
+      tdof_offsets[0] = 0;
+      for (int i=0; i<num_blocks; i++)
+      {
+         const int h = BlockA_r->GetBlock(i,i).Height();
+         tdof_offsets[i+1] = h;
+         tdof_offsets[num_blocks+i+1] = h;
+      }
+      tdof_offsets.PartialSum();
+
+      BlockOperator blockA(tdof_offsets);
+      for (int i = 0; i<num_blocks; i++)
+      {
+         for (int j = 0; j<num_blocks; j++)
+         {
+            blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
+            blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
+         }
+      }
+
+      X = 0.;
+      BlockDiagonalPreconditioner M(tdof_offsets);
+
+      ParFiniteElementSpace *ams_fes = nullptr;
+      if (static_cond)
+      {
+         ams_fes = new ParFiniteElementSpace(&pmesh,
+                                             E_fes->FEColl()->GetTraceCollection());
+      }
+      HypreAMS * solver_E = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(0,0),
+                                         (static_cond) ? ams_fes : E_fes);
+      solver_E->SetPrintLevel(0);
+
+      HypreSolver * solver_hatE = nullptr;
+
+      if (dim == 2)
+      {
+         solver_hatE = new HypreBoomerAMG((HypreParMatrix &)BlockA_r->GetBlock(1,1));
+         dynamic_cast<HypreBoomerAMG*>(solver_hatE)->SetPrintLevel(0);
+      }
+      else
+      {
+         solver_hatE = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(1,1),
+                                    hatE_fes);
+         dynamic_cast<HypreAMS*>(solver_hatE)->SetPrintLevel(0);
+      }
+
+      M.SetDiagonalBlock(0,solver_E);
+      M.SetDiagonalBlock(1,solver_hatE);
+      M.SetDiagonalBlock(num_blocks,solver_E);
+      M.SetDiagonalBlock(num_blocks+1,solver_hatE);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(10000);
+      cg.SetPrintLevel(0);
+      cg.SetPreconditioner(M);
+      cg.SetOperator(blockA);
+      cg.Mult(B, X);
+      delete ams_fes;
+      for (int i = 0; i<num_blocks; i++)
+      {
+         delete &M.GetDiagonalBlock(i);
+      }
+
+      int num_iter = cg.GetNumIterations();
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      double maxresidual = residuals.Max();
+      double globalresidual = residual * residual;
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+      E_r.MakeRef(E_fes,x, 0);
+      E_i.MakeRef(E_fes,x, offsets.Last());
+
+      int dofs = 0;
+      for (int i = 0; i<trial_fes.Size(); i++)
+      {
+         dofs += trial_fes[i]->GlobalTrueVSize();
+      }
+
+      double HcurlError = 0.0;
+      double rate_err = 0.0;
+
+      VectorFunctionCoefficient curlEex_r(dim,curlE_exact_r);
+      VectorFunctionCoefficient curlEex_i(dim,curlE_exact_i);
+
+      double E_err_r = E_r.ComputeHCurlError(&Eex_r,&curlEex_r);
+      double E_err_i = E_i.ComputeHCurlError(&Eex_i,&curlEex_i);
+
+      HcurlError = sqrt(  E_err_r*E_err_r + E_err_i*E_err_i);
+      rate_err = (it) ? dim*log(err0/HcurlError)/log((double)dof0/dofs) : 0.0;
+      err0 = HcurlError;
+
+      double rate_res =
+         (it) ? dim*log(res0/globalresidual)/log((double)dof0/dofs) : 0.0;
+
+      res0 = globalresidual;
+      dof0 = dofs;
+
+      if (myid == 0)
+      {
+         std::ios oldState(nullptr);
+         oldState.copyfmt(std::cout);
+         std::cout << std::right << std::setw(5) << it << " | "
+                   << std::setw(10) <<  dof0 << " | "
+                   << std::setprecision(1) << std::fixed
+                   << std::setw(4) <<  2.0*rnum << " π  | "
+                   << std::setprecision(3);
+         std::cout << std::setw(15) << std::scientific <<  err0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_err << " | " ;
+         std::cout << std::setprecision(3)
+                   << std::setw(10) << std::scientific <<  res0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_res << " | "
+                   << std::setw(6) << std::fixed << num_iter << " | "
+                   << std::endl;
+         std::cout.copyfmt(oldState);
+      }
+
+      if (visualization)
+      {
+         const char * keys = (it == 0 && dim == 2) ? "jRcml\n" : nullptr;
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         VisualizeField(E_out_r,vishost, visport, E_r,
+                        "Numerical Electric field (real part)", 0, 0, 500, 500, keys);
+      }
+
+      if (paraview)
+      {
+         paraview_dc->SetCycle(it);
+         paraview_dc->SetTime((double)it);
+         paraview_dc->Save();
+      }
+
+      if (it == pr)
+      {
+         break;
+      }
+
+      pmesh.UniformRefinement();
+
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   if (paraview)
+   {
+      delete paraview_dc;
+   }
+
+   delete a;
+   delete F_fec;
+   delete hatE_fes;
+   delete hatE_fec;
+   delete E_fec;
+   delete E_fes;
+
+   return 0;
+}
+
+void E_exact_r(const Vector &x, Vector & E_r)
+{
+   std::vector<std::complex<double>> E;
+   maxwell_solution(x,E);
+   E_r.SetSize(E.size());
+   for (unsigned i = 0; i < E.size(); i++)
+   {
+      E_r[i]= E[i].real();
+   }
+}
+
+void E_exact_i(const Vector &x, Vector & E_i)
+{
+   std::vector<std::complex<double>> E;
+   maxwell_solution(x, E);
+   E_i.SetSize(E.size());
+   for (unsigned i = 0; i < E.size(); i++)
+   {
+      E_i[i]= E[i].imag();
+   }
+}
+
+void curlE_exact_r(const Vector &x, Vector &curlE_r)
+{
+   std::vector<std::complex<double>> curlE;
+   maxwell_solution_curl(x, curlE);
+   curlE_r.SetSize(curlE.size());
+   for (unsigned i = 0; i < curlE.size(); i++)
+   {
+      curlE_r[i]= curlE[i].real();
+   }
+}
+
+void curlE_exact_i(const Vector &x, Vector &curlE_i)
+{
+   std::vector<std::complex<double>> curlE;
+   maxwell_solution_curl(x, curlE);
+   curlE_i.SetSize(curlE.size());
+   for (unsigned i = 0; i < curlE.size(); i++)
+   {
+      curlE_i[i]= curlE[i].imag();
+   }
+}
+
+void curlcurlE_exact_r(const Vector &x, Vector & curlcurlE_r)
+{
+   std::vector<std::complex<double>> curlcurlE;
+   maxwell_solution_curlcurl(x, curlcurlE);
+   curlcurlE_r.SetSize(curlcurlE.size());
+   for (unsigned i = 0; i < curlcurlE.size(); i++)
+   {
+      curlcurlE_r[i]= curlcurlE[i].real();
+   }
+}
+
+void curlcurlE_exact_i(const Vector &x, Vector & curlcurlE_i)
+{
+   std::vector<std::complex<double>> curlcurlE;
+   maxwell_solution_curlcurl(x, curlcurlE);
+   curlcurlE_i.SetSize(curlcurlE.size());
+   for (unsigned i = 0; i < curlcurlE.size(); i++)
+   {
+      curlcurlE_i[i]= curlcurlE[i].imag();
+   }
+}
+
+void  rhs_func_r(const Vector &x, Vector & J_r)
+{
+   Vector E_r, curlcurlE_r;
+   E_exact_r(x,E_r);
+   curlcurlE_exact_r(x,curlcurlE_r);
+   J_r.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      J_r(i) = 1./mu*curlcurlE_r(i) - omega *omega * epsilon * E_r(i);
+   }
+}
+
+void  rhs_func_i(const Vector &x, Vector & J_i)
+{
+   Vector E_i, curlcurlE_i;
+   E_exact_i(x,E_i);
+   curlcurlE_exact_i(x,curlcurlE_i);
+   J_i.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      J_i(i) = 1./mu*curlcurlE_i(i) - omega *omega * epsilon * E_i(i);
+   }
+}
+
+void maxwell_solution(const Vector & X, std::vector<complex<double>> &E)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   E.resize(dim);
+   for (int i = 0; i < dim; ++i)
+   {
+      E[i] = 0.0;
+   }
+   E[0] = exp(zi * omega * (X.Sum()));
+}
+
+void maxwell_solution_curl(const Vector & X,
+                           std::vector<complex<double>> &curlE)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   curlE.resize(dimc);
+   for (int i = 0; i < dimc; ++i)
+   {
+      curlE[i] = 0.0;
+   }
+   std::complex<double> pw = exp(zi * omega * (X.Sum()));
+   if (dim == 3)
+   {
+      curlE[0] = 0.0;
+      curlE[1] = zi * omega * pw;
+      curlE[2] = -zi * omega * pw;
+   }
+   else
+   {
+      curlE[0] = -zi * omega * pw;
+   }
+}
+
+void maxwell_solution_curlcurl(const Vector & X,
+                               std::vector<complex<double>> &curlcurlE)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   curlcurlE.resize(dim);
+   for (int i = 0; i < dim; ++i)
+   {
+      curlcurlE[i] = 0.0;;
+   }
+   std::complex<double> pw = exp(zi * omega * (X.Sum()));
+   if (dim == 3)
+   {
+      curlcurlE[0] = 2.0 * omega * omega * pw;
+      curlcurlE[1] = - omega * omega * pw;
+      curlcurlE[2] = - omega * omega * pw;
+   }
+   else
+   {
+      curlcurlE[0] = omega * omega * pw;
+      curlcurlE[1] = -omega * omega * pw;
+   }
+}

miniapps/dpg/pmaxwell.cpp

@@ -44,7 +44,7 @@
 // The DPG UW deals with the First Order System
 //  i ω μ H + ∇ × E = 0,   in Ω
 // -i ω ϵ E + ∇ × H = J,   in Ω
-//            E × n = E_0, on ∂Ω
+//            E × n = E₀, on ∂Ω
 // Note: Ĵ = -iωJ
 
 // The ultraweak-DPG formulation is obtained by integration by parts of both
@@ -57,7 +57,7 @@
 // where A = [0 1; -1 0];
 
 // E ∈ (L²(Ω))² , H ∈ L²(Ω)
-// Ê ∈ H^-1/2(Ω)(Γₕ), Ĥ ∈ H^1/2(Γₕ)
+// Ê ∈ H^-1/2(Γₕ), Ĥ ∈ H^1/2(Γₕ)
 //  i ω μ (H,F) + (E, ∇ × F) + < AÊ, F > = 0,      ∀ F ∈ H¹
 // -i ω ϵ (E,G) + (H,∇ × G)  + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)
 //                                        Ê = E₀      on ∂Ω
@@ -71,7 +71,7 @@
 
 // in 3D
 // E,H ∈ (L^2(Ω))³
-// Ê ∈ H_0^1/2(Ω)(curl, Γₕ), Ĥ ∈ H^-1/2(curl, Γₕ)
+// Ê ∈ H\_0^1/2(Ω)(curl, Γₕ), Ĥ ∈ H^-1/2(curl, Γₕ)
 //  i ω μ (H,F) + (E,∇ × F) + < Ê, F × n > = 0,      ∀ F ∈ H(curl,Ω)
 // -i ω ϵ (E,G) + (H,∇ × G) + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)
 //                                   Ê × n = E₀      on ∂Ω
@@ -106,7 +106,7 @@
 // in 2D
 // E ∈ (L²(Ω))² , H ∈ L²(Ω)
 // Ê ∈ H^-1/2(Ω)(Γₕ), Ĥ ∈ H^1/2(Γₕ)
-//  i ω μ (α⁻¹ H,F) + (E, ∇ × F) + < AÊ, F > = 0,          ∀ F ∈ H¹
+//  i ω μ (α⁻¹ H,F) + (E, ∇ × F) + < AÊ, F > = 0,         ∀ F ∈ H¹
 // -i ω ϵ (β E,G)   + (H,∇ × G)  + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)
 //                                            Ê = E₀     on ∂Ω
 // ---------------------------------------------------------------------------------
@@ -121,10 +121,10 @@
 //
 // in 3D
 // E,H ∈ (L^2(Ω))³
-// Ê ∈ H_0^1/2(Ω)(curl, Γ_h), Ĥ ∈ H^-1/2(curl, Γₕ)
+// Ê ∈ H_0^1/2(Ω)(curl, Γₕ), Ĥ ∈ H^-1/2(curl, Γₕ)
 //  i ω μ (α⁻¹ H,F) + (E,∇ × F) + < Ê, F × n > = 0,      ∀ F ∈ H(curl,Ω)
-// -i ω ϵ (β E,G)    + (H,∇ × G) + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)
-//                                        Ê × n = E_0     on ∂Ω
+// -i ω ϵ (β E,G)   + (H,∇ × G) + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)
+//                                       Ê × n = E₀      on ∂Ω
 // -------------------------------------------------------------------------------
 // |   |       E      |      H           |      Ê       |       Ĥ      |  RHS    |
 // -------------------------------------------------------------------------------
@@ -138,6 +138,7 @@
 #include "mfem.hpp"
 #include "util/pcomplexweakform.hpp"
 #include "util/pml.hpp"
+#include "util/preconditioners.hpp"
 #include "../common/mfem-common.hpp"
 #include <fstream>
 #include <iostream>
@@ -146,13 +147,13 @@ using namespace std;
 using namespace mfem;
 using namespace mfem::common;
 
+
 void E_exact_r(const Vector &x, Vector & E_r);
 void E_exact_i(const Vector &x, Vector & E_i);
 
 void H_exact_r(const Vector &x, Vector & H_r);
 void H_exact_i(const Vector &x, Vector & H_i);
 
-
 void  rhs_func_r(const Vector &x, Vector & J_r);
 void  rhs_func_i(const Vector &x, Vector & J_i);
 
@@ -221,6 +222,9 @@ int main(int argc, char *argv[])
    int delta_order = 1;
    real_t rnum=1.0;
    real_t theta = 0.0;
+   bool pmg = false;
+   int pmg_levels = -1;
+   real_t relax_factor = 2.0/3;
    bool static_cond = false;
    int iprob = 0;
    int sr = 0;
@@ -255,6 +259,12 @@ int main(int argc, char *argv[])
                   "Number of parallel refinements.");
    args.AddOption(&pr, "-pref", "--parallel-ref",
                   "Number of parallel refinements.");
+   args.AddOption(&pmg, "-pmg", "--p-refinement-multigrid", "-no-pmg",
+                  "--no-p-refinement-multigrid", "Enable P-Refinement Multigrid.");
+   args.AddOption(&pmg_levels, "-pmgl","--p-refinement-multigrid-levels",
+                  "Number of levels for P-Refinement Multigrid.");
+   args.AddOption(&relax_factor, "-rf", "--relaxation-factor",
+                  "Relaxation factor for the p-multigrid smoother.");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -345,7 +355,7 @@ int main(int argc, char *argv[])
       F_space = 0,
       G_space = 1
    };
-   // L2 space for E
+   // Vector L2 L2 space for E
    FiniteElementCollection *E_fec = new L2_FECollection(order-1,dim);
    ParFiniteElementSpace *E_fes = new ParFiniteElementSpace(&pmesh,E_fec,dim);
 
@@ -503,7 +513,16 @@ int main(int argc, char *argv[])
          *negepsomeg_detJ_Jt_J_inv_r_rot, attrPML);
    }
 
+   pmesh.EnsureNodes();
+   int meshorder = pmesh.GetNodalFESpace()->FEColl()->GetOrder();
+   const IntegrationRule &test_ir = IntRules.Get(pmesh.GetElementGeometry(0),
+                                                 2*test_order + 5 + meshorder);
+   const IntegrationRule &trial_ir = IntRules.Get(pmesh.GetElementGeometry(0),
+                                                  order+test_order + 5 + meshorder);
+
    ParComplexDPGWeakForm * a = new ParComplexDPGWeakForm(trial_fes,test_fec);
+   // a->SetTrialIntegrationRule(trial_ir);
+   // a->SetTestIntegrationRule(test_ir);
    a->StoreMatrices(); // needed for AMR
 
    // (E,∇ × F)
@@ -821,89 +840,70 @@ int main(int argc, char *argv[])
       a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
 
       ComplexOperator * Ahc = Ah.As<ComplexOperator>();
-
       BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
-      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
 
-      int num_blocks = BlockA_r->NumRowBlocks();
-      Array<int> tdof_offsets(2*num_blocks+1);
-
-      tdof_offsets[0] = 0;
-      int skip = (static_cond) ? 0 : 2;
-      int k = (static_cond) ? 2 : 0;
-      for (int i=0; i<num_blocks; i++)
+      Array<ParFiniteElementSpace *> prec_fes;
+      if (static_cond)
       {
-         tdof_offsets[i+1] = trial_fes[i+k]->GetTrueVSize();
-         tdof_offsets[num_blocks+i+1] = trial_fes[i+k]->GetTrueVSize();
-      }
-      tdof_offsets.PartialSum();
-
-      BlockOperator blockA(tdof_offsets);
-      for (int i = 0; i<num_blocks; i++)
-      {
-         for (int j = 0; j<num_blocks; j++)
-         {
-            blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
-            blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
-            blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
-            blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
-         }
-      }
-
-      X = 0.;
-      BlockDiagonalPreconditioner M(tdof_offsets);
-
-      if (!static_cond)
-      {
-         HypreBoomerAMG * solver_E = new HypreBoomerAMG((HypreParMatrix &)
-                                                        BlockA_r->GetBlock(0,0));
-         solver_E->SetPrintLevel(0);
-         solver_E->SetSystemsOptions(dim);
-         HypreBoomerAMG * solver_H = new HypreBoomerAMG((HypreParMatrix &)
-                                                        BlockA_r->GetBlock(1,1));
-         solver_H->SetPrintLevel(0);
-         solver_H->SetSystemsOptions(dim);
-         M.SetDiagonalBlock(0,solver_E);
-         M.SetDiagonalBlock(1,solver_H);
-         M.SetDiagonalBlock(num_blocks,solver_E);
-         M.SetDiagonalBlock(num_blocks+1,solver_H);
-      }
-
-      HypreSolver * solver_hatH = nullptr;
-      HypreAMS * solver_hatE = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(skip,
-                                                                                 skip),
-                                            hatE_fes);
-      solver_hatE->SetPrintLevel(0);
-      if (dim == 2)
-      {
-         solver_hatH = new HypreBoomerAMG((HypreParMatrix &)BlockA_r->GetBlock(skip+1,
-                                                                               skip+1));
-         dynamic_cast<HypreBoomerAMG*>(solver_hatH)->SetPrintLevel(0);
+         a->GetTraceFESpaces(prec_fes);
       }
       else
       {
-         solver_hatH = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(skip+1,skip+1),
-                                    hatH_fes);
-         dynamic_cast<HypreAMS*>(solver_hatH)->SetPrintLevel(0);
+         prec_fes = trial_fes;
+      }
+      Solver * cprec = nullptr;
+      if (pmg)
+      {
+#ifdef MFEM_USE_COMPLEX_MUMPS
+         bool mumps_coarse_solver = true;
+#else
+         bool mumps_coarse_solver = false;
+#endif
+         std::vector<Array<int>> ess_bdr_marker(prec_fes.Size());
+         for (int b = 0; b<prec_fes.Size(); b++)
+         {
+            if (pmesh.bdr_attributes.Size())
+            {
+               ess_bdr_marker[b].SetSize(pmesh.bdr_attributes.Max());
+               int ess_block = (static_cond) ? 0 : 2;
+               if (b == ess_block) // hatE
+               {
+                  ess_bdr_marker[b] = ess_bdr;
+               }
+               else
+               {
+                  ess_bdr_marker[b] = 0;
+               }
+            }
+         }
+         cprec = new ComplexPRefinementMultigrid(prec_fes, ess_bdr_marker, *Ahc,
+                                                 pmg_levels, relax_factor, mumps_coarse_solver);
+      }
+      else
+      {
+         BlockDiagonalPreconditioner * real_prec = new BlockDiagonalPreconditioner(
+            BlockA_r->RowOffsets());
+         real_prec->owns_blocks = 1;
+         for (int i = 0; i<BlockA_r->NumRowBlocks(); i++)
+         {
+            auto prec = MakeFESpaceDefaultSolver(prec_fes[i],0);
+            prec->SetOperator(BlockA_r->GetBlock(i,i));
+            real_prec->SetDiagonalBlock(i,prec);
+         }
+         cprec = new ComplexPreconditioner(real_prec, true);
       }
 
-      M.SetDiagonalBlock(skip,solver_hatE);
-      M.SetDiagonalBlock(skip+1,solver_hatH);
-      M.SetDiagonalBlock(skip+num_blocks,solver_hatE);
-      M.SetDiagonalBlock(skip+num_blocks+1,solver_hatH);
-
       CGSolver cg(MPI_COMM_WORLD);
+      // SLISolver cg(MPI_COMM_WORLD);
       cg.SetRelTol(1e-6);
       cg.SetMaxIter(10000);
       cg.SetPrintLevel(0);
-      cg.SetPreconditioner(M);
-      cg.SetOperator(blockA);
+      cg.SetOperator(*Ahc);
+
+      cg.SetPreconditioner(*cprec);
       cg.Mult(B, X);
 
-      for (int i = 0; i<num_blocks; i++)
-      {
-         delete &M.GetDiagonalBlock(i);
-      }
+      delete cprec;
 
       int num_iter = cg.GetNumIterations();
 
@@ -1185,7 +1185,7 @@ void hatE_exact_r(const Vector & x, Vector & hatE_r)
    {
       Vector E_r;
       E_exact_r(x,E_r);
-      hatE_r.SetSize(hatE_r.Size());
+      hatE_r.SetSize(E_r.Size());
       // rotate E_hat
       hatE_r[0] = E_r[1];
       hatE_r[1] = -E_r[0];
@@ -1202,7 +1202,7 @@ void hatE_exact_i(const Vector & x, Vector & hatE_i)
    {
       Vector E_i;
       E_exact_i(x,E_i);
-      hatE_i.SetSize(hatE_i.Size());
+      hatE_i.SetSize(E_i.Size());
       // rotate E_hat
       hatE_i[0] = E_i[1];
       hatE_i[1] = -E_i[0];

miniapps/dpg/pvectordiffusion-primal.cpp

@@ -0,0 +1,390 @@
+//                 MFEM Primal DPG parallel example for diffusion
+//
+// Compile with: make pdiffusion-primal
+//
+// Sample runs
+// mpirun -np 4 pdiffusion-primal -m ../../data/inline-quad.mesh -o 3 -sref 1 -pref 2
+
+//       - Δ u = f,   in Ω
+//           u = u₀, on ∂Ω
+
+
+
+// --------------------------------------
+// |   |     u     |      σ̂    |  RHS   |
+// --------------------------------------
+// | v |  (∇u,∇v)  |  -(σ̂ₙ,v)  |  (f,v) |
+//
+// u ∈ H¹(Ω),   σ̂ₙ ∈ H^-1/2(Τ)
+
+
+#include "mfem.hpp"
+#include "util/pweakform.hpp"
+#include "../common/mfem-common.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+void exact_u(const Vector & X, Vector & u);
+void exact_gradu(const Vector & X, DenseMatrix &du);
+void exact_laplacian_u(const Vector & X, Vector & d2u);
+void f_exact(const Vector & X, Vector & f);
+
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI and HYPRE.
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int sref = 0; // initial uniform mesh refinements
+   int pref = 0; // parallel mesh refinements for AMR
+   bool static_cond = false;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&sref, "-sref", "--num-serial-refinements",
+                  "Number of initial serial uniform refinements");
+   args.AddOption(&pref, "-pref", "--num-parallel-refinements",
+                  "Number of AMR refinements");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   for (int i = 0; i<sref; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+
+   // H1 space for u
+   FiniteElementCollection *u_fec = new H1_FECollection(order,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec,dim);
+
+
+   // H^-1/2 space for σ̂
+   FiniteElementCollection * hatsigma_fec = new RT_Trace_FECollection(order-1,dim);
+   ParFiniteElementSpace *hatsigma_fes = new ParFiniteElementSpace(&pmesh,
+                                                                   hatsigma_fec,dim);
+
+   ParFiniteElementSpace *hatsigma1_fes = new ParFiniteElementSpace(&pmesh,
+                                                                    hatsigma_fec);
+
+   int test_order = order+delta_order;
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+
+   Array<ParFiniteElementSpace * > trial_fes;
+   Array<FiniteElementCollection * > test_fec;
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(hatsigma_fes);
+   test_fec.Append(v_fec);
+
+
+   ConstantCoefficient one(1.0);
+   VectorFunctionCoefficient f(dim,
+                               f_exact); // rhs for the manufactured solution problem
+   VectorFunctionCoefficient uex(dim,exact_u);
+   MatrixFunctionCoefficient graduex(dim,exact_gradu);
+
+   ParDPGWeakForm * a = new ParDPGWeakForm(trial_fes,test_fec);
+   a->SetTestFECollVdim(0,dim);
+   a->StoreMatrices(true); // this is needed for estimation of residual
+
+   // (∇u,∇v)
+   a->AddTrialIntegrator(new VectorDiffusionIntegrator(one),0,0);
+
+   // // -<σ̂,v> (sign is included in σ̂)
+   a->AddTrialIntegrator(new VectorTraceIntegrator,1,0);
+
+   // // (∇v,∇δv)
+   a->AddTestIntegrator(new VectorDiffusionIntegrator(one),0,0);
+   // // (v,δv)
+   a->AddTestIntegrator(new VectorMassIntegrator(one),0,0);
+
+   a->AddDomainLFIntegrator(new VectorDomainLFIntegrator(f),0);
+
+
+   if (myid == 0)
+   {
+      std::cout << "\n  Ref |"
+                << "    Dofs    |"
+                << "  H1 Error  |"
+                << "  Rate  |"
+                << "  Residual  |"
+                << "  Rate  |"
+                << " PCG it |" << endl;
+      std::cout << std::string(72,'-') << endl;
+   }
+
+
+   socketstream u_out, ex_out;
+
+   double err0 = 0.;
+   int dof0=0.;
+   double res0=0.0;
+
+   ParGridFunction u_gf(u_fes);
+   ParGridFunction exact_gf(u_fes);
+   u_gf = 0.0;
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+
+   for (int it = 0; it<=pref; it++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         u_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      Array<int> offsets(3);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = hatsigma_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      u_gf.MakeRef(u_fes,x.GetBlock(0),0);
+      u_gf.ProjectBdrCoefficient(uex,ess_bdr);
+      exact_gf.ProjectCoefficient(uex);
+
+      Vector X,B;
+      OperatorPtr Ah;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockOperator * A = Ah.As<BlockOperator>();
+
+
+
+      // BlockDiagonalPreconditioner M(A->RowOffsets());
+      BlockDiagonalPreconditioner M(A->RowOffsets());
+      M.owns_blocks = 1;
+
+      // HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+      // amg0->SetSystemsOptions(dim);
+#ifdef MFEM_USE_MUMPS
+      MUMPSSolver * amg0 = new MUMPSSolver(MPI_COMM_WORLD);
+      amg0->SetOperator((HypreParMatrix &)A->GetBlock(0,0));
+      MUMPSSolver * prec = new MUMPSSolver(MPI_COMM_WORLD);
+      prec->SetOperator((HypreParMatrix&)A->GetBlock(1,1));
+#else
+      HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+      amg0->SetSystemsOptions(dim);
+      HypreBoomerAMG * prec = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
+      prec->SetSystemsOptions(dim);
+#endif
+      M.SetDiagonalBlock(0,amg0);
+      M.SetDiagonalBlock(1,prec);
+
+      // Array<int> toffsets(4);
+      // toffsets[0] = 0;
+      // toffsets[1] = A->RowOffsets()[1];
+      // toffsets[2] = (A->RowOffsets()[2]-A->RowOffsets()[1])/dim;
+      // toffsets[3] = (A->RowOffsets()[2]-A->RowOffsets()[1])/dim;
+      // toffsets.PartialSum();
+
+      // BlockDiagonalPreconditioner M(toffsets);
+      // M.owns_blocks = 1;
+
+      // HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+      // amg0->SetSystemsOptions(dim);
+
+      // HypreParMatrix & A11 = (HypreParMatrix &)A->GetBlock(1,1);
+      // Array<int> tdofs1(A11.Height()/2);
+      // Array<int> tdofs2(A11.Height()/2);
+      // for (int i = 0; i<tdofs1.Size(); i++)
+      // {
+      //    tdofs1[i] = i;
+      //    tdofs2[i] = tdofs1.Size() + i;
+      // }
+
+      // HypreParMatrix * S1 = GetSubHypreParMatrix(tdofs1,A11);
+      // HypreParMatrix * S2 = GetSubHypreParMatrix(tdofs2,A11);
+      // MUMPSSolver * prec1 = new MUMPSSolver(MPI_COMM_WORLD);
+      // prec1->SetOperator(*S1);
+      // MUMPSSolver * prec2 = new MUMPSSolver(MPI_COMM_WORLD);
+      // prec2->SetOperator(*S2);
+      // HypreSolver * prec1;
+      // HypreSolver * prec2;
+      // if (dim == 2)
+      // {
+      //    // AMS preconditioner for 2D H(div) (trace) space
+      //    prec1 = new HypreAMS(*S1, hatsigma1_fes);
+      //    prec2 = new HypreAMS(*S2, hatsigma1_fes);
+
+      // }
+      // else
+      // {
+      //    // ADS preconditioner for 3D H(div) (trace) space
+      //    prec1 = new HypreADS(*S1, hatsigma1_fes);
+      //    prec2 = new HypreADS(*S2, hatsigma1_fes);
+      // }
+      // M.SetDiagonalBlock(1,prec1);
+      // M.SetDiagonalBlock(2,prec2);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(0);
+      cg.SetPreconditioner(M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+      // Vector residuals;
+
+      double residual = residuals.Norml2();
+
+      double maxresidual = residuals.Max();
+      double globalresidual = residual * residual;
+
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+      u_gf.MakeRef(u_fes,x.GetBlock(0),0);
+
+      int dofs = u_fes->GlobalTrueVSize() + hatsigma_fes->GlobalTrueVSize();
+
+      double u_err = u_gf.ComputeL2Error(uex);
+
+      double rate_err = (it) ? dim*log(err0/u_err)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (it) ? dim*log(res0/globalresidual)/log((
+                                                                   double)dof0/dofs) : 0.0;
+      err0 = u_err;
+      res0 = globalresidual;
+      dof0 = dofs;
+
+      if (myid == 0)
+      {
+         std::ios oldState(nullptr);
+         oldState.copyfmt(std::cout);
+         std::cout << std::right << std::setw(5) << it << " | "
+                   << std::setw(10) <<  dof0 << " | "
+                   << std::setprecision(3)
+                   << std::setw(10) << std::scientific <<  err0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_err << " | "
+                   << std::setprecision(3)
+                   << std::setw(10) << std::scientific <<  res0 << " | "
+                   << std::setprecision(2)
+                   << std::setw(6) << std::fixed << rate_res << " | "
+                   << std::setw(6) << std::fixed << cg.GetNumIterations() << " | "
+                   << std::endl;
+         std::cout.copyfmt(oldState);
+      }
+
+      if (visualization)
+      {
+         const char * keys = (it == 0 && dim == 2) ? "jRcm\n" : nullptr;
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+
+         VisualizeField(u_out,vishost,visport,u_gf,
+                        "Numerical u", 0,0,500,500,keys);
+
+         VisualizeField(ex_out,vishost,visport,exact_gf,
+                        "Exact u", 0,0,500,500,keys);
+      }
+
+      if (it == pref) { break; }
+
+      pmesh.UniformRefinement();
+
+      for (int i=0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      hatsigma1_fes->Update(false);
+      a->Update();
+      exact_gf.Update();
+   }
+
+   delete a;
+   delete v_fec;
+   delete hatsigma_fes;
+   delete hatsigma_fec;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+
+}
+
+
+
+void exact_u(const Vector & X, Vector & u)
+{
+   int size = X.Size();
+   for (int i = 0; i<size; i++)
+   {
+      u(i) = sin(M_PI*X(i));
+   }
+
+}
+
+void exact_gradu(const Vector & X, DenseMatrix & du)
+{
+   du.SetSize(X.Size());
+   for (int i = 0; i<du.Size(); i++)
+   {
+      du(i,i) = M_PI * cos(M_PI * X(i));
+   }
+}
+
+void exact_laplacian_u(const Vector & X, Vector & du)
+{
+   int size = X.Size();
+   du.SetSize(size);
+   for (int i = 0; i<size; i++)
+   {
+      du(i) = -M_PI*M_PI * sin(M_PI*X(i));
+   }
+}
+
+void f_exact(const Vector & X, Vector & f)
+{
+   f.SetSize(X.Size());
+   exact_laplacian_u(X,f);
+   f.Neg();
+}

miniapps/dpg/util/blockcomplexhypremat.cpp

@@ -0,0 +1,120 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "blockcomplexhypremat.hpp"
+
+#ifdef MFEM_USE_MPI
+
+namespace mfem
+{
+
+void ParBlockComplexSystem::FillEssTdofLists(const Array<int> &
+                                             ess_tdof_list)
+{
+   for (int i = 0; i < ess_tdofs.Size(); i++)
+   {
+      delete ess_tdofs[i];
+      ess_tdofs[i] = new Array<int>();
+   }
+   int j;
+   for (int i = 0; i < ess_tdof_list.Size(); i++)
+   {
+      int tdof = ess_tdof_list[i];
+      for (j = 0; j < nblocks; j++)
+      {
+         if (toffsets[j+1] > tdof) { break; }
+      }
+      ess_tdofs[j]->Append(tdof-toffsets[j]);
+   }
+}
+
+ComplexOperator * ParBlockComplexSystem::EliminateBC(const Array<int>
+                                                     ess_tdof_list, Vector &X, Vector & B)
+{
+   FillEssTdofLists(ess_tdof_list);
+   delete op_e_r;
+   delete op_e_i;
+   op_e_r = new BlockOperator(toffsets);
+   op_e_i = new BlockOperator(toffsets);
+   op_e_r->owns_blocks = 1;
+   op_e_i->owns_blocks = 1;
+
+   for (int i = 0; i < nblocks; i++)
+   {
+      for (int j = 0; j < nblocks; j++)
+      {
+         if (op_r->IsZeroBlock(i,j)) { continue; }
+         if (i == j)
+         {
+            auto mat_r = &(HypreParMatrix &)op_r->GetBlock(i,i);
+            op_e_r->SetBlock(i, i, mat_r->EliminateRowsCols(*ess_tdofs[i]));
+            if (!op_i->IsZeroBlock(i,j))
+            {
+               auto mat_i = &(HypreParMatrix &)op_i->GetBlock(i,i);
+               op_e_i->SetBlock(i, i, mat_i->EliminateCols(*ess_tdofs[i]));
+               mat_i->EliminateRows(*ess_tdofs[i]);
+            }
+         }
+         else
+         {
+            auto mat_r = &(HypreParMatrix &)op_r->GetBlock(i,j);
+            op_e_r->SetBlock(i, j, mat_r->EliminateCols(*ess_tdofs[j]));
+            mat_r->EliminateRows(*ess_tdofs[i]);
+            if (!op_i->IsZeroBlock(i,j))
+            {
+               auto mat_i = &(HypreParMatrix &)op_i->GetBlock(i,j);
+               op_e_i->SetBlock(i, j, mat_i->EliminateCols(*ess_tdofs[j]));
+               mat_i->EliminateRows(*ess_tdofs[i]);
+            }
+         }
+      }
+   }
+
+   int n = B.Size()/2;
+   Vector B_r(B, 0, n);
+   Vector B_i(B, n, n);
+
+   Vector X_r(X, 0, n);
+   Vector X_i(X, n, n);
+
+   // eliminate tdof is RHS
+   // B_r -= Ae_r*X_r + Ae_i X_i
+   // B_i -= Ae_i*X_r + Ae_r X_i
+   Vector tmp(B_r.Size());
+   op_e_r->Mult(X_r, tmp); B_r-=tmp;
+   op_e_i->Mult(X_i, tmp); B_r+=tmp;
+
+   op_e_i->Mult(X_r, tmp); B_i-=tmp;
+   op_e_r->Mult(X_i, tmp); B_i-=tmp;
+
+   for (int j = 0; j < nblocks; j++)
+   {
+      if (!ess_tdofs[j]->Size()) { continue; }
+      for (int i = 0; i < ess_tdofs[j]->Size(); i++)
+      {
+         int tdof = (*ess_tdofs[j])[i];
+         int gdof = tdof + toffsets[j];
+         B_r(gdof) = X_r(gdof); // diagonal policy is always one in parallel
+         B_i(gdof) = X_i(gdof); // diagonal policy is always one in parallel
+      }
+   }
+
+   X_r.SetSubVectorComplement(ess_tdof_list, 0.0);
+   X_i.SetSubVectorComplement(ess_tdof_list, 0.0);
+
+   return op;
+}
+
+
+
+} // namespace mfem
+
+#endif

miniapps/dpg/util/blockcomplexhypremat.hpp

@@ -0,0 +1,75 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BLOCKCOMPLEX
+#define MFEM_BLOCKCOMPLEX
+
+#include "mfem.hpp"
+
+#ifdef MFEM_USE_MPI
+
+namespace mfem
+{
+
+class ParBlockComplexSystem
+{
+
+protected:
+
+   /// ess_tdof list for each space
+   Array<Array<int> *> ess_tdofs;
+   Array<int> toffsets;
+   int nblocks;
+
+   /** split ess_tdof_list given in global tdof (for all spaces)
+    to individual lists for each space */
+   void FillEssTdofLists(const Array<int> & ess_tdof_list);
+
+   // Block Prolongation
+   BlockOperator * P = nullptr;
+   // Block Restriction
+   BlockMatrix * R = nullptr;
+
+   ComplexOperator * op = nullptr;
+   BlockOperator * op_r = nullptr;
+   BlockOperator * op_i = nullptr;
+   BlockOperator * op_e_r = nullptr;
+   BlockOperator * op_e_i = nullptr;
+
+public:
+
+   ParBlockComplexSystem() {}
+
+   /// Creates bilinear form associated with FE spaces @a trial_pfes_.
+   ParBlockComplexSystem(ComplexOperator * op_)
+      : op(op_)
+   {
+      op_r = dynamic_cast<BlockOperator *>(&op->real());
+      op_i = dynamic_cast<BlockOperator *>(&op->imag());
+      toffsets = op_r->RowOffsets(); // (assumes square blockoperator)
+      nblocks = toffsets.Size() - 1;
+      ess_tdofs.SetSize(nblocks);
+   }
+
+   ComplexOperator * EliminateBC(const Array<int> ess_tdof_list, Vector &X,
+                                 Vector & B);
+
+   virtual ~ParBlockComplexSystem() {}
+
+
+};
+
+} // namespace mfem
+
+
+#endif // MFEM_USE_MPI
+
+#endif

miniapps/dpg/util/blockstaticcond.cpp

@@ -69,12 +69,14 @@ void BlockStaticCondensation::SetSpaces(Array<FiniteElementSpace*> & fes_)
    nblocks = fes.Size();
    rblocks = 0;
    tr_fes.SetSize(nblocks);
+   tr_fec.SetSize(nblocks);
    mesh = fes[0]->GetMesh();
 
    IsTraceSpace.SetSize(nblocks);
    const FiniteElementCollection * fec;
    for (int i = 0; i < nblocks; i++)
    {
+      tr_fec[i] = nullptr;
       fec = fes[i]->FEColl();
       IsTraceSpace[i] =
          (dynamic_cast<const H1_Trace_FECollection*>(fec) ||
@@ -86,21 +88,24 @@ void BlockStaticCondensation::SetSpaces(Array<FiniteElementSpace*> & fes_)
          pmesh = dynamic_cast<ParMesh *>(mesh);
          tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
                      nullptr : (IsTraceSpace[i]) ? fes[i] :
-                     new ParFiniteElementSpace(pmesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                     new ParFiniteElementSpace(pmesh, tr_fec[i] = fec->GetTraceCollection(),
+                                               fes[i]->GetVDim(),
                                                fes[i]->GetOrdering());
       }
       else
       {
          tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
                      nullptr : (IsTraceSpace[i]) ? fes[i] :
-                     new FiniteElementSpace(mesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                     new FiniteElementSpace(mesh, tr_fec[i] = fec->GetTraceCollection(),
+                                            fes[i]->GetVDim(),
                                             fes[i]->GetOrdering());
       }
 #else
       // skip if it's an L2 space (no trace space to construct)
       tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
                   nullptr : (IsTraceSpace[i]) ? fes[i] :
-                  new FiniteElementSpace(mesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                  new FiniteElementSpace(mesh, tr_fec[i] = fec->GetTraceCollection(),
+                                         fes[i]->GetVDim(),
                                          fes[i]->GetOrdering());
 #endif
       if (tr_fes[i]) { rblocks++; }
@@ -213,39 +218,43 @@ void BlockStaticCondensation::GetReducedElementIndicesAndOffsets(int el,
    {
       int td = 0;
       int ndof;
+      int vdim = fes[i]->GetVDim();
       // if it's an L2 space (bubbles)
       if (!tr_fes[i])
       {
-         ndof = fes[i]->GetVDim()*fes[i]->GetFE(el)->GetDof();
+         ndof = fes[i]->GetFE(el)->GetDof();
          td = 0;
       }
       else if (IsTraceSpace[i])
       {
          for (int iface = 0; iface < numfaces; iface++)
          {
-            td += fes[i]->GetVDim()*fes[i]->GetFaceElement(faces[iface])->GetDof();
+            td += fes[i]->GetFaceElement(faces[iface])->GetDof();
          }
          ndof = td;
       }
       else
       {
          Array<int> trace_dofs;
-         ndof = fes[i]->GetVDim()*fes[i]->GetFE(el)->GetDof();
-         tr_fes[i]->GetElementVDofs(el, trace_dofs);
+         ndof = fes[i]->GetFE(el)->GetDof();
+         tr_fes[i]->GetElementDofs(el, trace_dofs);
          td = trace_dofs.Size(); // number of trace dofs
       }
-      offsets[i+1] = td;
-      tr_dofs.SetSize(td);
-      int_dofs.SetSize(ndof - td);
-      for (int j = 0; j<td; j++)
+      offsets[i+1] = td*vdim;
+      tr_dofs.SetSize(td*vdim);
+      int_dofs.SetSize(vdim*(ndof - td));
+      for (int k = 0; k<vdim; k++)
       {
-         tr_dofs[j] = skip + j;
+         for (int j = 0; j<td; j++)
+         {
+            tr_dofs[j+k*td] = skip + j;
+         }
+         for (int j = 0; j<ndof-td; j++)
+         {
+            int_dofs[j+k*(ndof-td)] = skip + td + j;
+         }
+         skip+=ndof;
       }
-      for (int j = 0; j<ndof-td; j++)
-      {
-         int_dofs[j] = skip + td + j;
-      }
-      skip+=ndof;
 
       trace_ldofs.Append(tr_dofs);
       interior_ldofs.Append(int_dofs);
@@ -976,6 +985,15 @@ BlockStaticCondensation::~BlockStaticCondensation()
       delete lmat[i]; lmat[i] = nullptr;
       delete lvec[i]; lvec[i] = nullptr;
    }
+
+   for (int i = 0; i<tr_fes.Size(); i++)
+   {
+      if (tr_fec[i])
+      {
+         delete tr_fes[i];
+         delete tr_fec[i];
+      }
+   }
 }
 
 } // namespace mfem

miniapps/dpg/util/blockstaticcond.hpp

@@ -40,6 +40,7 @@ class BlockStaticCondensation
    // New set of "reduced" Finite Element Spaces
    // (after static condensation)
    Array<FiniteElementSpace *> tr_fes;
+   Array<FiniteElementCollection *> tr_fec;
 
    Array<int> dof_offsets;
    Array<int> tdof_offsets;
@@ -185,6 +186,11 @@ public:
        full linear system, compute the solution of the full system 'sol'. */
    void ComputeSolution(const Vector &sc_sol, Vector &sol) const;
 
+   void GetTraceFESpaces(Array<FiniteElementSpace *> & trace_fes) const
+   {
+      trace_fes = tr_fes;
+   }
+
 };
 
 }

miniapps/dpg/util/complexblockform.cpp

@@ -0,0 +1,577 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "complexblockform.hpp"
+
+namespace mfem
+{
+
+void ComplexBlockForm::Init()
+{
+   integs_r.SetSize(fes.Size(), fes.Size());
+   integs_i.SetSize(fes.Size(), fes.Size());
+   for (int i = 0; i < integs_r.NumRows(); i++)
+   {
+      for (int j = 0; j < integs_r.NumCols(); j++)
+      {
+         integs_r(i,j) = new Array<BilinearFormIntegrator * >();
+         integs_i(i,j) = new Array<BilinearFormIntegrator * >();
+      }
+   }
+
+   ComputeOffsets();
+
+   mat_r = mat_e_r = NULL;
+   mat_i = mat_e_i = NULL;
+   diag_policy = mfem::Operator::DIAG_ONE;
+   height = dof_offsets[nblocks];
+   width = height;
+
+   initialized = true;
+
+}
+
+void ComplexBlockForm::ComputeOffsets()
+{
+   dof_offsets.SetSize(nblocks+1);
+   tdof_offsets.SetSize(nblocks+1);
+   dof_offsets[0] = 0;
+   tdof_offsets[0] = 0;
+   for (int i =0; i<nblocks; i++)
+   {
+      dof_offsets[i+1] = fes[i]->GetVSize();
+      tdof_offsets[i+1] = fes[i]->GetTrueVSize();
+   }
+   dof_offsets.PartialSum();
+   tdof_offsets.PartialSum();
+}
+
+// Allocate SparseMatrix and RHS
+void ComplexBlockForm::AllocMat()
+{
+   mat_r = new BlockMatrix(dof_offsets);
+   mat_r->owns_blocks = 1;
+   mat_i = new BlockMatrix(dof_offsets);
+   mat_i->owns_blocks = 1;
+
+   for (int i = 0; i < mat_r->NumRowBlocks(); i++)
+   {
+      int h = dof_offsets[i+1] - dof_offsets[i];
+      for (int j = 0; j < mat_r->NumColBlocks(); j++)
+      {
+         int w = dof_offsets[j+1] - dof_offsets[j];
+         mat_r->SetBlock(i,j,new SparseMatrix(h, w));
+         mat_i->SetBlock(i,j,new SparseMatrix(h, w));
+      }
+   }
+}
+
+void ComplexBlockForm::Finalize(int skip_zeros)
+{
+   if (mat_r)
+   {
+      mat_r->Finalize(skip_zeros);
+      mat_i->Finalize(skip_zeros);
+   }
+   if (mat_e_r)
+   {
+      mat_e_r->Finalize(skip_zeros);
+      mat_e_i->Finalize(skip_zeros);
+   }
+}
+
+/// Adds new Domain BF Integrator. Assumes ownership of @a bfi.
+void ComplexBlockForm::AddDomainIntegrator(
+   BilinearFormIntegrator *bfi_r,
+   BilinearFormIntegrator *bfi_i,
+   int n, int m)
+{
+   MFEM_VERIFY(n < fes.Size(),
+               "ComplexBlockFrom::AddDomainIntegrator: fespace row index out of bounds");
+   MFEM_VERIFY(m < fes.Size(),
+               "ComplexBlockFrom::AddDomainIntegrator: fespace col index out of bounds");
+   if (bfi_r) { integs_r(n,m)->Append(bfi_r); }
+   if (bfi_i) { integs_i(n,m)->Append(bfi_i); }
+}
+
+void ComplexBlockForm::BuildProlongation()
+{
+   P = new BlockMatrix(dof_offsets, tdof_offsets);
+   R = new BlockMatrix(tdof_offsets, dof_offsets);
+   P->owns_blocks = 0;
+   R->owns_blocks = 0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      const SparseMatrix *P_ = fes[i]->GetConformingProlongation();
+      if (P_)
+      {
+         const SparseMatrix *R_ = fes[i]->GetRestrictionMatrix();
+         P->SetBlock(i, i, const_cast<SparseMatrix*>(P_));
+         R->SetBlock(i, i, const_cast<SparseMatrix*>(R_));
+      }
+   }
+}
+
+void ComplexBlockForm::ConformingAssemble()
+{
+   Finalize(0);
+   if (!P) { BuildProlongation(); }
+
+   BlockMatrix * Pt = Transpose(*P);
+   BlockMatrix * PtA_r = mfem::Mult(*Pt, *mat_r);
+   BlockMatrix * PtA_i = mfem::Mult(*Pt, *mat_i);
+   mat_r->owns_blocks = 0;
+   mat_i->owns_blocks = 0;
+   for (int i = 0; i < nblocks; i++)
+   {
+      for (int j = 0; j < nblocks; j++)
+      {
+         SparseMatrix * tmp_r = &mat_r->GetBlock(i,j);
+         SparseMatrix * tmp_i = &mat_i->GetBlock(i,j);
+         if (Pt->IsZeroBlock(i, i))
+         {
+            PtA_r->SetBlock(i, j, tmp_r);
+            PtA_i->SetBlock(i, j, tmp_i);
+         }
+         else
+         {
+            delete tmp_r;
+            delete tmp_i;
+         }
+      }
+   }
+   delete mat_r;
+   delete mat_i;
+   if (mat_e_r)
+   {
+      BlockMatrix *PtAe_r = mfem::Mult(*Pt, *mat_e_r);
+      BlockMatrix *PtAe_i = mfem::Mult(*Pt, *mat_e_i);
+      mat_e_r->owns_blocks = 0;
+      mat_e_i->owns_blocks = 0;
+      for (int i = 0; i<nblocks; i++)
+      {
+         for (int j = 0; j<nblocks; j++)
+         {
+            SparseMatrix * tmp_r = &mat_e_r->GetBlock(i, j);
+            SparseMatrix * tmp_i = &mat_e_i->GetBlock(i, j);
+            if (Pt->IsZeroBlock(i, i))
+            {
+               PtAe_r->SetBlock(i, j, tmp_r);
+               PtAe_i->SetBlock(i, j, tmp_i);
+            }
+            else
+            {
+               delete tmp_r;
+               delete tmp_i;
+            }
+         }
+      }
+      delete mat_e_r;
+      delete mat_e_i;
+      mat_e_r = PtAe_r;
+      mat_e_i = PtAe_i;
+   }
+   delete Pt;
+
+   mat_r = mfem::Mult(*PtA_r, *P);
+   mat_i = mfem::Mult(*PtA_i, *P);
+
+   PtA_r->owns_blocks = 0;
+   PtA_i->owns_blocks = 0;
+   for (int i = 0; i < nblocks; i++)
+   {
+      for (int j = 0; j < nblocks; j++)
+      {
+         SparseMatrix * tmp_r = &PtA_r->GetBlock(j, i);
+         SparseMatrix * tmp_i = &PtA_i->GetBlock(j, i);
+         if (P->IsZeroBlock(i, i))
+         {
+            mat_r->SetBlock(j, i, tmp_r);
+            mat_i->SetBlock(j, i, tmp_i);
+         }
+         else
+         {
+            delete tmp_r;
+            delete tmp_i;
+         }
+      }
+   }
+   delete PtA_r;
+   delete PtA_i;
+
+   if (mat_e_r)
+   {
+      BlockMatrix *PtAeP_r = mfem::Mult(*mat_e_r, *P);
+      BlockMatrix *PtAeP_i = mfem::Mult(*mat_e_i, *P);
+      mat_e_r->owns_blocks = 0;
+      mat_e_i->owns_blocks = 0;
+      for (int i = 0; i < nblocks; i++)
+      {
+         for (int j = 0; j < nblocks; j++)
+         {
+            SparseMatrix * tmp_r = &mat_e_r->GetBlock(j, i);
+            SparseMatrix * tmp_i = &mat_e_i->GetBlock(j, i);
+            if (P->IsZeroBlock(i, i))
+            {
+               PtAeP_r->SetBlock(j, i, tmp_r);
+               PtAeP_i->SetBlock(j, i, tmp_i);
+            }
+            else
+            {
+               delete tmp_r;
+               delete tmp_i;
+            }
+         }
+      }
+
+      delete mat_e_r;
+      delete mat_e_i;
+      mat_e_r = PtAeP_r;
+      mat_e_i = PtAeP_i;
+   }
+   height = 2*mat_r->Height();
+   width = 2*mat_r->Width();
+}
+
+/// Assembles the form i.e. sums over all domain integrators.
+void ComplexBlockForm::Assemble(int skip_zeros)
+{
+   ElementTransformation *eltrans;
+   Array<int> faces, ori;
+
+   DofTransformation doftrans_i, doftrans_j;
+   if (mat_r == NULL)
+   {
+      AllocMat();
+   }
+
+   // loop through the elements
+   DenseMatrix A_r, Ae_r;
+   DenseMatrix A_i, Ae_i;
+   Array<int> vdofs;
+
+   // loop through elements
+   for (int iel = 0; iel < mesh -> GetNE(); iel++)
+   {
+      Array<int> offs(fes.Size()+1); offs = 0;
+
+      eltrans = mesh->GetElementTransformation(iel);
+
+      for (int j = 0; j < fes.Size(); j++)
+      {
+         offs[j+1] = fes[j]->GetVDim() * fes[j]->GetFE(iel)->GetDof();
+      }
+      offs.PartialSum();
+
+      A_r.SetSize(offs.Last(),offs.Last()); A_r = 0.0;
+      A_i.SetSize(offs.Last(),offs.Last()); A_i = 0.0;
+
+      for (int j = 0; j < fes.Size(); j++)
+      {
+         const FiniteElement & fe_j = *fes[j]->GetFE(iel);
+         for (int i = 0; i < fes.Size(); i++)
+         {
+            const FiniteElement & fe_i = *fes[i]->GetFE(iel);
+            // real integrators
+            for (int k = 0; k < integs_r(i,j)->Size(); k++)
+            {
+               if (i == j)
+               {
+                  (*integs_r(i,j))[k]->AssembleElementMatrix(fe_i,*eltrans,Ae_r);
+               }
+               else
+               {
+                  (*integs_r(i,j))[k]->AssembleElementMatrix2(fe_i,fe_j,*eltrans,Ae_r);
+               }
+               A_r.AddSubMatrix(offs[j], offs[i], Ae_r);
+            }
+            // imag integrators
+            for (int k = 0; k < integs_i(i,j)->Size(); k++)
+            {
+               if (i == j)
+               {
+                  (*integs_i(i,j))[k]->AssembleElementMatrix(fe_i,*eltrans,Ae_i);
+               }
+               else
+               {
+                  (*integs_i(i,j))[k]->AssembleElementMatrix2(fe_i, fe_j, *eltrans, Ae_i);
+               }
+               A_i.AddSubMatrix(offs[j], offs[i], Ae_i);
+            }
+         }
+      }
+
+      ComplexDenseMatrix A(&A_r, &A_i, false, false);
+
+      // Assembly
+      for (int i = 0; i<fes.Size(); i++)
+      {
+         Array<int> vdofs_i;
+         doftrans_i.SetDofTransformation(nullptr);
+         fes[i]->GetElementVDofs(iel, vdofs_i,doftrans_i);
+         for (int j = 0; j < fes.Size(); j++)
+         {
+            Array<int> vdofs_j;
+            doftrans_j.SetDofTransformation(nullptr);
+            fes[j]->GetElementVDofs(iel, vdofs_j,doftrans_j);
+
+            A.real().GetSubMatrix(offs[i],offs[i+1],
+                                  offs[j],offs[j+1], Ae_r);
+            A.imag().GetSubMatrix(offs[i],offs[i+1],
+                                  offs[j],offs[j+1], Ae_i);
+            TransformDual(doftrans_i, doftrans_j, Ae_r);
+            TransformDual(doftrans_i, doftrans_j, Ae_i);
+            mat_r->GetBlock(i,j).AddSubMatrix(vdofs_i,vdofs_j, Ae_r);
+            mat_i->GetBlock(i,j).AddSubMatrix(vdofs_i,vdofs_j, Ae_i);
+         }
+      }
+   } // end of loop through elements
+}
+
+void ComplexBlockForm::FormLinearSystem(const Array<int>
+                                        &ess_tdof_list,
+                                        Vector &x,
+                                        Vector &b,
+                                        OperatorHandle &A,
+                                        Vector &X,
+                                        Vector &B,
+                                        int copy_interior)
+{
+   FormSystemMatrix(ess_tdof_list, A);
+
+   Vector x_r(x, 0, x.Size()/2);
+   Vector x_i(x, x.Size()/2, x.Size()/2);
+   Vector b_r(b, 0, b.Size()/2);
+   Vector b_i(b, b.Size()/2, b.Size()/2);
+   if (!P)
+   {
+      EliminateVDofsInRHS(ess_tdof_list, x_r,x_i, b_r, b_i);
+      if (!copy_interior)
+      {
+         x_r.SetSubVectorComplement(ess_tdof_list, 0.0);
+         x_i.SetSubVectorComplement(ess_tdof_list, 0.0);
+      }
+      X.MakeRef(x, 0, x.Size());
+      B.MakeRef(b, 0, b.Size());
+   }
+   else // non conforming space
+   {
+      B.SetSize(2*P->Width());
+      Vector B_r(B, 0, P->Width());
+      Vector B_i(B, P->Width(),P->Width());
+
+      P->MultTranspose(b_r, B_r);
+      P->MultTranspose(b_i, B_i);
+      Vector tmp_r,tmp_i;
+      for (int i = 0; i<nblocks; i++)
+      {
+         if (P->IsZeroBlock(i,i))
+         {
+            int offset = tdof_offsets[i];
+            tmp_r.MakeRef(b_r, offset,tdof_offsets[i+1]-tdof_offsets[i]);
+            tmp_i.MakeRef(b_i, offset,tdof_offsets[i+1]-tdof_offsets[i]);
+            B_r.SetVector(tmp_r,offset);
+            B_i.SetVector(tmp_i,offset);
+         }
+      }
+
+      X.SetSize(2*R->Height());
+      Vector X_r(X, 0, X.Size()/2);
+      Vector X_i(X, X.Size()/2, X.Size()/2);
+
+      R->Mult(x_r, X_r);
+      R->Mult(x_i, X_i);
+      for (int i = 0; i<nblocks; i++)
+      {
+         if (R->IsZeroBlock(i,i))
+         {
+            int offset = tdof_offsets[i];
+            tmp_r.MakeRef(x_r, offset, tdof_offsets[i+1]-tdof_offsets[i]);
+            tmp_i.MakeRef(x_i, offset, tdof_offsets[i+1]-tdof_offsets[i]);
+            X_r.SetVector(tmp_r,offset);
+            X_i.SetVector(tmp_i,offset);
+         }
+      }
+
+      EliminateVDofsInRHS(ess_tdof_list, X_r, X_i, B_r, B_i);
+      if (!copy_interior)
+      {
+         X_r.SetSubVectorComplement(ess_tdof_list, 0.0);
+         X_i.SetSubVectorComplement(ess_tdof_list, 0.0);
+      }
+   }
+}
+
+void ComplexBlockForm::FormSystemMatrix(const Array<int>
+                                        &ess_tdof_list,
+                                        OperatorHandle &A)
+{
+   if (!mat_e_r)
+   {
+      bool conforming = true;
+      for (int i = 0; i<nblocks; i++)
+      {
+         const SparseMatrix *P_ = fes[i]->GetConformingProlongation();
+         if (P_)
+         {
+            conforming = false;
+            break;
+         }
+      }
+      if (!conforming) { ConformingAssemble(); }
+      const int remove_zeros = 0;
+      EliminateVDofs(ess_tdof_list, diag_policy);
+      Finalize(remove_zeros);
+   }
+   mat = new ComplexOperator(mat_r,mat_i,false,false);
+   A.Reset(mat,false);
+}
+
+void ComplexBlockForm::EliminateVDofsInRHS(
+   const Array<int> &vdofs, const Vector &x_r, const Vector & x_i,
+   Vector &b_r, Vector & b_i)
+{
+   mat_e_r->AddMult(x_r,b_r,-1.);
+   mat_e_i->AddMult(x_i,b_r,1.);
+   mat_e_r->AddMult(x_i,b_i,-1.);
+   mat_e_i->AddMult(x_r,b_i,-1.);
+   mat_r->PartMult(vdofs,x_r,b_r);
+   mat_r->PartMult(vdofs,x_i,b_i);
+}
+
+void ComplexBlockForm::EliminateVDofs(const Array<int> &vdofs,
+                                      Operator::DiagonalPolicy dpolicy)
+{
+   if (mat_e_r == NULL)
+   {
+      Array<int> offsets;
+
+      offsets.MakeRef( (P) ? tdof_offsets : dof_offsets);
+
+      mat_e_r = new BlockMatrix(offsets);
+      mat_e_r->owns_blocks = 1;
+      mat_e_i = new BlockMatrix(offsets);
+      mat_e_i->owns_blocks = 1;
+      for (int i = 0; i < mat_e_r->NumRowBlocks(); i++)
+      {
+         int h = offsets[i+1] - offsets[i];
+         for (int j = 0; j < mat_e_r->NumColBlocks(); j++)
+         {
+            int w = offsets[j+1] - offsets[j];
+            mat_e_r->SetBlock(i, j, new SparseMatrix(h, w));
+            mat_e_i->SetBlock(i, j, new SparseMatrix(h, w));
+         }
+      }
+   }
+   mat_r->EliminateRowCols(vdofs, mat_e_r, diag_policy);
+   mat_i->EliminateRowCols(vdofs, mat_e_i, Operator::DiagonalPolicy::DIAG_ZERO);
+}
+
+void ComplexBlockForm::RecoverFEMSolution(const Vector &X, Vector &x)
+{
+   if (!P)
+   {
+      x.SyncMemory(X);
+   }
+   else
+   {
+      x.SetSize(2*P->Height());
+      Vector X_r(const_cast<Vector &>(X), 0, X.Size()/2);
+      Vector X_i(const_cast<Vector &>(X), X.Size()/2, X.Size()/2);
+
+      Vector x_r(x, 0, x.Size()/2);
+      Vector x_i(x, x.Size()/2, x.Size()/2);
+
+      P->Mult(X_r, x_r);
+      P->Mult(X_i, x_i);
+
+      Vector tmp_r, tmp_i;
+      for (int i = 0; i<nblocks; i++)
+      {
+         if (P->IsZeroBlock(i,i))
+         {
+            int offset = tdof_offsets[i];
+            tmp_r.MakeRef(X_r, offset, tdof_offsets[i+1]-tdof_offsets[i]);
+            tmp_i.MakeRef(X_i, offset, tdof_offsets[i+1]-tdof_offsets[i]);
+            x_r.SetVector(tmp_r,offset);
+            x_i.SetVector(tmp_i,offset);
+         }
+      }
+   }
+}
+
+void ComplexBlockForm::ReleaseInitMemory()
+{
+   if (initialized)
+   {
+      for (int k = 0; k < integs_r.NumRows(); k++)
+      {
+         for (int l = 0; l < integs_r.NumCols(); l++)
+         {
+            for (int i = 0; i < integs_r(k,l)->Size(); i++)
+            {
+               delete (*integs_r(k,l))[i];
+            }
+            delete integs_r(k,l);
+            for (int i = 0; i < integs_i(k,l)->Size(); i++)
+            {
+               delete (*integs_i(k,l))[i];
+            }
+            delete integs_i(k,l);
+         }
+      }
+      integs_r.DeleteAll();
+      integs_i.DeleteAll();
+   }
+}
+
+void ComplexBlockForm::Update()
+{
+   delete mat_e_r; mat_e_r = nullptr;
+   delete mat_e_i; mat_e_i = nullptr;
+   delete mat; mat = nullptr;
+   delete mat_r; mat_r = nullptr;
+   delete mat_i; mat_i = nullptr;
+
+   if (P)
+   {
+      delete P; P = nullptr;
+      delete R; R = nullptr;
+   }
+
+   ComputeOffsets();
+
+   diag_policy = mfem::Operator::DIAG_ONE;
+   height = dof_offsets[nblocks];
+   width = height;
+
+   initialized = true;
+
+}
+
+ComplexBlockForm::~ComplexBlockForm()
+{
+   delete mat_e_r; mat_e_r = nullptr;
+   delete mat_e_i; mat_e_i = nullptr;
+   delete mat; mat = nullptr;
+   delete mat_r; mat_r = nullptr;
+   delete mat_i; mat_i = nullptr;
+
+   ReleaseInitMemory();
+
+   if (P)
+   {
+      delete P;
+      delete R;
+   }
+}
+
+} // namespace mfem

miniapps/dpg/util/complexblockform.hpp

@@ -0,0 +1,199 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_COMPLEX_BLOCKFORM
+#define MFEM_COMPLEX_BLOCKFORM
+
+#include "mfem.hpp"
+#include "complexstaticcond.hpp"
+
+namespace mfem
+{
+
+class ComplexBlockForm
+{
+
+protected:
+
+   bool initialized = false;
+
+   Mesh * mesh = nullptr;
+   int height, width;
+   int nblocks;
+   Array<int> dof_offsets;
+   Array<int> tdof_offsets;
+
+   /// Block matrix $ M $ to be associated with the real/imag Block bilinear form. Owned.
+   BlockMatrix *mat_r = nullptr;
+   BlockMatrix *mat_i = nullptr;
+   ComplexOperator * mat = nullptr;
+
+   /** @brief Block Matrix $ M_e $ used to store the eliminations
+        from the b.c.  Owned.
+       $ M + M_e = M_{original} $ */
+   BlockMatrix *mat_e_r = nullptr;
+   BlockMatrix *mat_e_i = nullptr;
+
+   /// FE spaces
+   Array<FiniteElementSpace * > fes;
+
+   /// Set of Trial Integrators to be applied for matrix A
+   Array2D<Array<BilinearFormIntegrator * > * > integs_r;
+   Array2D<Array<BilinearFormIntegrator * > * > integs_i;
+
+   /// Block Prolongation
+   BlockMatrix * P = nullptr;
+   /// Block Restriction
+   BlockMatrix * R = nullptr;
+
+   mfem::Operator::DiagonalPolicy diag_policy;
+
+   void Init();
+   void ReleaseInitMemory();
+
+   // Allocate appropriate SparseMatrix and assign it to mat
+   void AllocMat();
+
+   void ConformingAssemble();
+
+   void ComputeOffsets();
+
+   virtual void BuildProlongation();
+
+private:
+
+public:
+
+   ComplexBlockForm()
+   {
+      height = 0;
+      width = 0;
+   }
+
+   /// Creates bilinear form associated with FE spaces @a fes_.
+   ComplexBlockForm(Array<FiniteElementSpace* > & fes_)
+   {
+      SetSpaces(fes_);
+   }
+
+   void SetSpaces(Array<FiniteElementSpace* > & fes_)
+   {
+      fes = fes_;
+      nblocks = fes.Size();
+      mesh = fes[0]->GetMesh();
+      Init();
+   }
+
+   // Get the size of the bilinear form of the ComplexBlockForm
+   int Size() const { return height; }
+
+   // Pre-allocate the internal real and imag BlockMatrix before assembly.
+   void AllocateMatrix() { if (mat_r == nullptr) { AllocMat(); } }
+
+   ///  Finalizes the matrix initialization.
+   void Finalize(int skip_zeros = 1);
+
+   /// Returns a reference to the BlockMatrix:  $ M_r $
+   BlockMatrix &BlockMat_r()
+   {
+      MFEM_VERIFY(mat_r, "mat_r is NULL and can't be dereferenced");
+      return *mat_r;
+   }
+   /// Returns a reference to the BlockMatrix:  $ M_i $
+   BlockMatrix &BlockMat_i()
+   {
+      MFEM_VERIFY(mat_i, "mat_i is NULL and can't be dereferenced");
+      return *mat_i;
+   }
+
+   /// Returns a reference to the BlockMatrix of eliminated b.c.: $ M_e_r $
+   BlockMatrix &BlockMatElim_r()
+   {
+      MFEM_VERIFY(mat_e_r, "mat_e is NULL and can't be dereferenced");
+      return *mat_e_r;
+   }
+
+   /// Returns a reference to the BlockMatrix of eliminated b.c.: $ M_e_i $
+   BlockMatrix &BlockMatElim_i()
+   {
+      MFEM_VERIFY(mat_e_i, "mat_e is NULL and can't be dereferenced");
+      return *mat_e_i;
+   }
+
+   /** Adds new Trial Integrator. Assumes ownership of @a bfi_r and @a bfi_i.
+       @a n and @a m correspond to the trial FESpace and test FEColl
+       respectively */
+   void AddDomainIntegrator(BilinearFormIntegrator *bfi_r,
+                            BilinearFormIntegrator *bfi_i,
+                            int n, int m);
+
+   /// Assembles the form i.e. sums over all integrators.
+   void Assemble(int skip_zeros = 1);
+
+   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
+                                 Vector &x, Vector &b, OperatorHandle & A,
+                                 Vector &X, Vector &B, int copy_interior = 0);
+
+   template <typename OpType>
+   void FormLinearSystem(const Array<int> &ess_tdof_list,
+                         Vector &x, Vector &b, OpType &A,
+                         Vector &X, Vector &B, int copy_interior = 0)
+   {
+      OperatorHandle Ah;
+      FormLinearSystem(ess_tdof_list, x, b, Ah, X, B, copy_interior);
+      OpType *A_ptr = Ah.Is<OpType>();
+      MFEM_VERIFY(A_ptr, "invalid OpType used");
+      A.MakeRef(*A_ptr);
+   }
+
+   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                 OperatorHandle &A);
+
+   template <typename OpType>
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OpType &A)
+   {
+      OperatorHandle Ah;
+      FormSystemMatrix(ess_tdof_list, Ah);
+      OpType *A_ptr = Ah.Is<OpType>();
+      MFEM_VERIFY(A_ptr, "invalid OpType used");
+      A.MakeRef(*A_ptr);
+   }
+
+   void EliminateVDofs(const Array<int> &vdofs,
+                       Operator::DiagonalPolicy dpolicy = Operator::DIAG_ONE);
+
+   void EliminateVDofsInRHS(const Array<int> &vdofs,
+                            const Vector &x_r, const Vector & x_i,
+                            Vector &b_r, Vector & b_i);
+
+   virtual void RecoverFEMSolution(const Vector &X, Vector &x);
+
+   /// Sets diagonal policy used upon construction of the linear system.
+   /** Policies include:
+       - DIAG_ZERO (Set the diagonal values to zero)
+       - DIAG_ONE  (Set the diagonal values to one)
+       - DIAG_KEEP (Keep the diagonal values)
+   */
+   void SetDiagonalPolicy(Operator::DiagonalPolicy policy)
+   {
+      diag_policy = policy;
+   }
+
+   virtual void Update();
+
+   /// Destroys bilinear form.
+   virtual ~ComplexBlockForm();
+
+};
+
+} // namespace mfem
+
+#endif

miniapps/dpg/util/complexstaticcond.cpp

@@ -71,12 +71,14 @@ void ComplexBlockStaticCondensation::SetSpaces(Array<FiniteElementSpace*> &
    nblocks = fes.Size();
    rblocks = 0;
    tr_fes.SetSize(nblocks);
+   tr_fec.SetSize(nblocks);
    mesh = fes[0]->GetMesh();
 
    IsTraceSpace.SetSize(nblocks);
    const FiniteElementCollection * fec;
    for (int i = 0; i < nblocks; i++)
    {
+      tr_fec[i] = nullptr;
       fec = fes[i]->FEColl();
       IsTraceSpace[i] =
          (dynamic_cast<const H1_Trace_FECollection*>(fec) ||
@@ -88,21 +90,24 @@ void ComplexBlockStaticCondensation::SetSpaces(Array<FiniteElementSpace*> &
          pmesh = dynamic_cast<ParMesh *>(mesh);
          tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
                      nullptr : (IsTraceSpace[i]) ? fes[i] :
-                     new ParFiniteElementSpace(pmesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                     new ParFiniteElementSpace(pmesh, tr_fec[i] = fec->GetTraceCollection(),
+                                               fes[i]->GetVDim(),
                                                fes[i]->GetOrdering());
       }
       else
       {
          tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
                      nullptr : (IsTraceSpace[i]) ? fes[i] :
-                     new FiniteElementSpace(mesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                     new FiniteElementSpace(mesh, tr_fec[i] = fec->GetTraceCollection(),
+                                            fes[i]->GetVDim(),
                                             fes[i]->GetOrdering());
       }
 #else
       // skip if it's an L2 space (no trace space to construct)
       tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
                   nullptr : (IsTraceSpace[i]) ? fes[i] :
-                  new FiniteElementSpace(mesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                  new FiniteElementSpace(mesh, tr_fec[i] = fec->GetTraceCollection(),
+                                         fes[i]->GetVDim(),
                                          fes[i]->GetOrdering());
 #endif
       if (tr_fes[i]) { rblocks++; }
@@ -220,40 +225,43 @@ void ComplexBlockStaticCondensation::GetReduceElementIndicesAndOffsets(int el,
    {
       int td = 0;
       int ndof;
+      int vdim = fes[i]->GetVDim();
       // if it's an L2 space (bubbles)
       if (!tr_fes[i])
       {
-         ndof = fes[i]->GetVDim()*fes[i]->GetFE(el)->GetDof();
+         ndof = fes[i]->GetFE(el)->GetDof();
          td = 0;
       }
       else if (IsTraceSpace[i])
       {
          for (int iface = 0; iface < numfaces; iface++)
          {
-            td += fes[i]->GetVDim()*fes[i]->GetFaceElement(faces[iface])->GetDof();
+            td += fes[i]->GetFaceElement(faces[iface])->GetDof();
          }
          ndof = td;
       }
       else
       {
          Array<int> trace_dofs;
-         ndof = fes[i]->GetVDim()*fes[i]->GetFE(el)->GetDof();
-         tr_fes[i]->GetElementVDofs(el, trace_dofs);
+         ndof = fes[i]->GetFE(el)->GetDof();
+         tr_fes[i]->GetElementDofs(el, trace_dofs);
          td = trace_dofs.Size(); // number of trace dofs
       }
-      offsets[i+1] = td;
-      tr_dofs.SetSize(td);
-      int_dofs.SetSize(ndof - td);
-      for (int j = 0; j<td; j++)
+      offsets[i+1] = td*vdim;
+      tr_dofs.SetSize(td*vdim);
+      int_dofs.SetSize(vdim*(ndof - td));
+      for (int k=0; k<vdim; k++)
       {
-         tr_dofs[j] = skip + j;
+         for (int j = 0; j<td; j++)
+         {
+            tr_dofs[j+k*td] = skip + j;
+         }
+         for (int j = 0; j<ndof-td; j++)
+         {
+            int_dofs[j+k*(ndof-td)] = skip + td + j;
+         }
+         skip+=ndof;
       }
-      for (int j = 0; j<ndof-td; j++)
-      {
-         int_dofs[j] = skip + td + j;
-      }
-      skip+=ndof;
-
       trace_ldofs.Append(tr_dofs);
       interior_ldofs.Append(int_dofs);
    }
@@ -1157,6 +1165,16 @@ ComplexBlockStaticCondensation::~ComplexBlockStaticCondensation()
       delete lmat[i]; lmat[i] = nullptr;
       delete lvec[i]; lvec[i] = nullptr;
    }
+
+   for (int i = 0; i<tr_fes.Size(); i++)
+   {
+      if (tr_fec[i])
+      {
+         delete tr_fes[i];
+         delete tr_fec[i];
+      }
+   }
+
 }
 
 }

miniapps/dpg/util/complexstaticcond.hpp

@@ -36,6 +36,7 @@ class ComplexBlockStaticCondensation
    // New set of "reduced" Finite Element Spaces
    // (after static condensation)
    Array<FiniteElementSpace *> tr_fes;
+   Array<FiniteElementCollection *> tr_fec;
 
    Array<int> dof_offsets;
    Array<int> tdof_offsets;
@@ -214,6 +215,11 @@ public:
        full linear system, compute the solution of the full system 'sol'. */
    void ComputeSolution(const Vector &sc_sol, Vector &sol) const;
 
+   void GetTraceFESpaces(Array<FiniteElementSpace *> & trace_fes) const
+   {
+      trace_fes = tr_fes;
+   }
+
 };
 
 }

miniapps/dpg/util/complexweakform.cpp

@@ -314,9 +314,48 @@ void ComplexDPGWeakForm::ConformingAssemble()
    width = 2*mat_r->Width();
 }
 
+void ComplexDPGWeakForm::SetIntegrationRules()
+{
+   if (trial_ir)
+   {
+      for (int i = 0; i < trial_integs_r.NumRows(); i++)
+      {
+         for (int j = 0; j < trial_integs_r.NumCols(); j++)
+         {
+            for (int k = 0; k < trial_integs_r(i,j)->Size(); k++)
+            {
+               (*trial_integs_r(i,j))[k]->SetIntRule(trial_ir);
+            }
+            for (int k = 0; k < trial_integs_i(i,j)->Size(); k++)
+            {
+               (*trial_integs_i(i,j))[k]->SetIntRule(trial_ir);
+            }
+         }
+      }
+   }
+   if (test_ir)
+   {
+      for (int i = 0; i < test_integs_r.NumRows(); i++)
+      {
+         for (int j = 0; j < test_integs_r.NumCols(); j++)
+         {
+            for (int k = 0; k < test_integs_r(i,j)->Size(); k++)
+            {
+               (*test_integs_r(i,j))[k]->SetIntRule(test_ir);
+            }
+            for (int k = 0; k < test_integs_i(i,j)->Size(); k++)
+            {
+               (*test_integs_i(i,j))[k]->SetIntRule(test_ir);
+            }
+         }
+      }
+   }
+}
+
 /// Assembles the form i.e. sums over all domain integrators.
 void ComplexDPGWeakForm::Assemble(int skip_zeros)
 {
+   this->SetIntegrationRules();
    ElementTransformation *eltrans;
    Array<int> faces, ori;
 
@@ -505,8 +544,10 @@ void ComplexDPGWeakForm::Assemble(int skip_zeros)
       }
 
       ComplexCholeskyFactors chol(G_r.GetData(), G_i.GetData());
+
       int h = G_r.Height();
-      chol.Factor(h);
+      bool info = chol.Factor(h);
+      MFEM_VERIFY(info, "Complex Cholesky factorization of G failed");
 
       int w = B_r.Width();
       chol.LSolve(h,w,B_r.GetData(), B_i.GetData());
@@ -919,7 +960,7 @@ void ComplexDPGWeakForm::EnableStaticCondensation()
 Vector & ComplexDPGWeakForm::ComputeResidual(const Vector & x)
 {
    MFEM_VERIFY(store_matrices,
-               "Matrices needed for the residual are not store. Call ComplexDPGWeakForm::StoreMatrices()")
+               "Matrices needed for the residual are not stored. Call ComplexDPGWeakForm::StoreMatrices()")
    // wrap vector in a blockvector
    int n = x.Size()/2;
 
@@ -962,7 +1003,8 @@ Vector & ComplexDPGWeakForm::ComputeResidual(const Vector & x)
          {
             for (int ie = 0; ie < faces.Size(); ie++)
             {
-               trial_offs[j+1] += trial_fes[j]->GetFaceElement(faces[ie])->GetDof();
+               trial_offs[j+1] += trial_fes[j]->GetVDim()*trial_fes[j]->GetFaceElement(
+                                     faces[ie])->GetDof();
             }
          }
          else

miniapps/dpg/util/complexweakform.hpp

@@ -69,6 +69,10 @@ protected:
    Array2D<Array<BilinearFormIntegrator * > * > test_integs_r;
    Array2D<Array<BilinearFormIntegrator * > * > test_integs_i;
 
+   /// Integration rules for the test and trial integrators
+   const IntegrationRule *trial_ir = nullptr;
+   const IntegrationRule *test_ir = nullptr;
+
    /// Set of LinearForm Integrators to be applied.
    Array<Array<LinearFormIntegrator * > * > lfis_r;
    Array<Array<LinearFormIntegrator * > * > lfis_i;
@@ -101,7 +105,8 @@ protected:
    Vector residuals;
 
 private:
-
+   /// Enforces a unique integration rule for all trial/test integrators
+   void SetIntegrationRules();
 public:
 
    ComplexDPGWeakForm()
@@ -198,6 +203,11 @@ public:
                               LinearFormIntegrator *lfi_i,
                               int n);
 
+   /// Sets the same integration rules for all trial integrators
+   void SetTrialIntegrationRule(const IntegrationRule &ir) { trial_ir = &ir; }
+   /// Sets the same integration rules for all test integrators
+   void SetTestIntegrationRule(const IntegrationRule &ir) { test_ir = &ir; }
+
    /// Assembles the form i.e. sums over all integrators.
    void Assemble(int skip_zeros = 1);
 
@@ -273,6 +283,23 @@ public:
 
    Vector & ComputeResidual(const Vector & x);
 
+   void GetTraceFESpaces(Array<FiniteElementSpace *> & trace_fes) const
+   {
+      trace_fes.SetSize(0);
+      Array<FiniteElementSpace *> trace_fes_all;
+      if (static_cond)
+      {
+         static_cond->GetTraceFESpaces(trace_fes_all);
+         for (int i = 0; i < trace_fes_all.Size(); i++)
+         {
+            if (trace_fes_all[i])
+            {
+               trace_fes.Append(trace_fes_all[i]);
+            }
+         }
+      }
+   }
+
    /// Destroys bilinear form.
    virtual ~ComplexDPGWeakForm();
 

miniapps/dpg/util/maxwell_utils.cpp

@@ -0,0 +1,562 @@
+#include "maxwell_utils.hpp"
+
+real_t AzimuthalECoefficient::Eval(ElementTransformation &T,
+                                   const IntegrationPoint &ip)
+{
+   Vector X, E;
+   vgf->GetVectorValue(T,ip,E);
+   T.Transform(ip, X);
+   real_t x = X(0);
+   real_t y = X(1);
+   real_t r = sqrt(x*x + y*y);
+
+   real_t val = -x*E[1] + y*E[0];
+   return val/r;
+}
+
+real_t ParallelECoefficient::Eval(ElementTransformation &T,
+                                  const IntegrationPoint &ip)
+{
+   Vector X, E;
+   vgf->GetVectorValue(T,ip,E);
+   T.Transform(ip, X);
+   Vector b;
+   ComputeB(X, b);
+   return E*b;
+}
+
+EpsilonMatrixCoefficient::EpsilonMatrixCoefficient(const char * filename,
+                                                   Mesh * mesh_, ParMesh * pmesh_,
+                                                   real_t scale)
+   : MatrixArrayCoefficient(mesh_->Dimension()), mesh(mesh_), pmesh(pmesh_),
+     dim(mesh->Dimension())
+{
+   std::filebuf fb;
+   fb.open(filename,std::ios::in);
+   std::istream is(&fb);
+   vgf = new GridFunction(mesh,is);
+   fb.close();
+   FiniteElementSpace * vfes = vgf->FESpace();
+   int vdim = vfes->GetVDim();
+   const FiniteElementCollection * fec = vfes->FEColl();
+   FiniteElementSpace * fes = new FiniteElementSpace(mesh, fec);
+   int * partitioning = mesh->GeneratePartitioning(num_procs);
+   double *data = vgf->GetData();
+   GridFunction gf;
+   pgfs.SetSize(vdim);
+   gf_cfs.SetSize(vdim);
+   sdim = sqrt(vdim);
+   for (int i = 0; i<sdim; i++)
+   {
+      for (int j = 0; j<sdim; j++)
+      {
+         int k = i*sdim+j;
+         gf.MakeRef(fes,&data[k*fes->GetVSize()]);
+         pgfs[k] = new ParGridFunction(pmesh,&gf,partitioning);
+         (*pgfs[k])*=scale;
+         gf_cfs[k] = new GridFunctionCoefficient(pgfs[k]);
+         if (i<dim && j<dim)
+         {
+            Set(i,j,gf_cfs[k], true);
+         }
+      }
+   }
+}
+
+void EpsilonMatrixCoefficient::VisualizeMatrixCoefficient()
+{
+   Array<socketstream *> sol_sock(pgfs.Size());
+   for (int k = 0; k<pgfs.Size(); k++)
+   {
+      if (Mpi::Root()) { mfem::out << "Visualizing component " << k << endl; }
+      char vishost[] = "localhost";
+      int visport = 19916;
+      sol_sock[k] = new socketstream(vishost, visport);
+      sol_sock[k]->precision(8);
+      *sol_sock[k] << "parallel " << num_procs << " " << myid << "\n";
+      int i = k/sdim;
+      int j = k%sdim;
+      *sol_sock[k] << "solution\n" << *pmesh << *pgfs[k]
+                   << "window_title 'Epsilon Matrix Coefficient Component (" << i << "," << j <<
+                   ")'" << flush;
+   }
+}
+
+void EpsilonMatrixCoefficient::Update()
+{
+   pgfs[0]->ParFESpace()->Update();
+   for (int k = 0; k<pgfs.Size(); k++)
+   {
+      pgfs[k]->Update();
+   }
+}
+
+EpsilonMatrixCoefficient::~EpsilonMatrixCoefficient()
+{
+   for (int i = 0; i<pgfs.Size(); i++)
+   {
+      delete pgfs[i];
+   }
+   pgfs.DeleteAll();
+}
+
+DielectricTensorComponentCoefficient::DielectricTensorComponentCoefficient(
+   real_t delta_, real_t a0_, real_t a1_,
+   int row_, int col_,
+   bool use_imag_)
+   : delta(delta_), a0(a0_), a1(a1_), row(row_), col(col_), use_imag(use_imag_) { }
+
+
+real_t DielectricTensorComponentCoefficient::Eval(ElementTransformation &T,
+                                                  const IntegrationPoint &ip)
+{
+   Vector x;
+   T.Transform(ip, x);
+   return use_imag ? ComputeImagPart(x) : ComputeRealPart(x);
+}
+
+real_t DielectricTensorComponentCoefficient::ComputeRealPart(const Vector &x)
+{
+   Vector b;
+   ComputeB(x, b);
+
+   real_t r = std::sqrt(x(0)*x(0) + x(1)*x(1));
+   real_t S = 1.0;
+   real_t P = a0 + a1 * (r - 0.9);
+
+   real_t bb_ij = b(row) * b(col);
+   return S * (row == col) + (P - S) * bb_ij;
+}
+
+real_t DielectricTensorComponentCoefficient::ComputeImagPart(const Vector &x)
+{
+   return (row == col) ? delta : 0.0;
+}
+
+void VisualizeMatrixArrayCoefficient(MatrixArrayCoefficient &mc, ParMesh *pmesh,
+                                     int order, bool paraview, const char *name)
+{
+   MFEM_VERIFY(pmesh != nullptr, "ParMesh pointer must not be null.");
+   int dim = mc.GetVDim();
+
+   if (Mpi::Root())
+   {
+      mfem::out << "Visualizing matrix coefficient with dimension: " << dim << endl;
+      mfem::out << "order = " << order << endl;
+      mfem::out << "pmesh dimension = " << pmesh->Dimension() << endl;
+   }
+   auto fec = new H1_FECollection(order, pmesh->Dimension());
+   auto pfes = new ParFiniteElementSpace(pmesh, fec);
+
+   Array<ParGridFunction *> pgfs(dim * dim);
+
+   Array<GridFunctionCoefficient *> gf_cfs(dim * dim);
+   ParaViewDataCollection * pvdc = nullptr;
+   std::ostringstream label;
+   if (name)
+   {
+      label << name;
+   }
+   else
+   {
+      label << "eps";
+   }
+   if (paraview)
+   {
+      pvdc = new ParaViewDataCollection(label.str(), pmesh);
+      pvdc->SetPrefixPath("ParaView");
+      pvdc->SetLevelsOfDetail(order);
+      pvdc->SetCycle(0);
+      pvdc->SetDataFormat(VTKFormat::BINARY);
+   }
+   for (int i = 0; i < dim; ++i)
+   {
+      for (int j = 0; j < dim; ++j)
+      {
+         Coefficient *c_ij = mc.GetCoeff(i, j);
+         if (!c_ij) { continue; }
+
+         pgfs[i*dim + j] = new ParGridFunction(pfes);
+         *pgfs[i*dim + j] = 0.0;
+         pgfs[i*dim + j]->ProjectCoefficient(*c_ij);
+
+         if (Mpi::Root())
+         {
+            mfem::out << "Projected component (" << i << "," << j << ")" << endl;
+         }
+
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         socketstream sol_sock(vishost, visport);
+
+         sol_sock << "parallel " << pmesh->GetNRanks() << " " << pmesh->GetMyRank() <<
+                  "\n";
+         sol_sock << "solution\n" << * pmesh <<  *pgfs[i*dim + j];
+         sol_sock << "window_title '" << label.str() <<"_" << i << j << "'\n";
+         sol_sock << flush;
+
+         if (paraview)
+         {
+            pvdc->RegisterField(label.str() + std::to_string(i) + std::to_string(j),
+                                pgfs[i*dim + j]);
+         }
+      }
+   }
+   if (paraview)
+   {
+      pvdc->Save();
+      delete pvdc;
+      for (int i = 0; i < dim * dim; ++i)
+      {
+         delete pgfs[i];
+      }
+   }
+   delete pfes;
+   delete fec;
+}
+
+void ComputeB(const Vector &x, Vector &b)
+{
+   real_t x0 = x(0), x1 = x(1);
+   real_t r = std::sqrt(x0 * x0 + x1 * x1);
+   int dim = x.Size();
+   b.SetSize(dim); b = 0.0;
+   b(0) = -x1 / r;
+   b(1) =  x0 / r;
+   if (dim == 3) { b(2) = 0.0; }
+}
+
+void DirectionalVectorDiffusionIntegrator::AssembleElementMatrix(
+   const FiniteElement &el,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   int dof = el.GetDof();
+   int dim = el.GetDim();
+   int vdim = Trans.GetSpaceDim();
+
+   elmat.SetSize(dof * vdim, dof * vdim);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = 2 * el.GetOrder(); // Integration order
+      ir = &IntRules.Get(el.GetGeomType(), order);
+   }
+
+   // Get shape functions and their derivatives
+   DenseMatrix dshape(dof, dim);
+   Vector vec(dim);
+
+   for (int k = 0; k < ir->GetNPoints(); k++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(k);
+      Trans.SetIntPoint(&ip);
+      double w = ip.weight * Trans.Weight();
+      VQ->Eval(vec, Trans, ip);
+      el.CalcPhysDShape(Trans, dshape);
+
+      // Compute (vq·∇)φ for each basis function
+      Vector vq_grad_phi(dof); vq_grad_phi = 0.0;
+      for (int j = 0; j < dof; j++)
+      {
+         for (int d = 0; d < dim; d++)
+         {
+            vq_grad_phi(j) += vec(d) * dshape(j, d);
+         }
+      }
+
+      for (int comp = 0; comp < vdim; comp++)
+      {
+         int offset = comp * dof;
+         for (int j = 0; j < dof; j++)
+         {
+            int jj = j + offset;
+            for (int i = 0; i < dof; i++)
+            {
+               int ii = i + offset;
+               elmat(jj, ii) += w * vq_grad_phi(j)
+                                * vq_grad_phi(i);
+            }
+         }
+      }
+   }
+}
+
+void DirectionalVectorDiffusionIntegrator::AssembleElementMatrix2(
+   const FiniteElement &trial_fe, const FiniteElement &test_fe,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   int trial_dof = trial_fe.GetDof();
+   int test_dof = test_fe.GetDof();
+   int dim = trial_fe.GetDim();
+   int vdim = Trans.GetSpaceDim();
+
+   elmat.SetSize(test_dof * vdim, trial_dof * vdim);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW() - 1;
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+   }
+
+   Vector vec(dim);
+   DenseMatrix trial_dshape(trial_dof, dim);
+   DenseMatrix test_dshape(test_dof, dim);
+
+   for (int k = 0; k < ir->GetNPoints(); k++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(k);
+      Trans.SetIntPoint(&ip);
+      double w = ip.weight * Trans.Weight();
+      VQ->Eval(vec, Trans, ip);
+      trial_fe.CalcPhysDShape(Trans, trial_dshape);
+      test_fe.CalcPhysDShape(Trans, test_dshape);
+
+      // Compute (vq·∇)φ for each basis function
+      Vector vq_grad_phi_trial(trial_dof); vq_grad_phi_trial = 0.0;
+      for (int j = 0; j < trial_dof; j++)
+      {
+         for (int d = 0; d < dim; d++)
+         {
+            vq_grad_phi_trial(j) += vec(d) * trial_dshape(j, d);
+         }
+      }
+
+      Vector vq_grad_phi_test(test_dof); vq_grad_phi_test = 0.0;
+      for (int j = 0; j < test_dof; j++)
+      {
+         for (int d = 0; d < dim; d++)
+         {
+            vq_grad_phi_test(j) += vec(d) * test_dshape(j, d);
+         }
+      }
+
+      for (int trial_comp = 0; trial_comp < vdim; trial_comp++)
+      {
+         int offset_trial = trial_comp * trial_dof;
+         int offset_test = trial_comp * test_dof;
+         for (int j = 0; j < test_dof; j++)
+         {
+            int jj = j + offset_test;
+            for (int i = 0; i < trial_dof; i++)
+            {
+               int ii = i + offset_trial;
+               elmat(jj, ii) += w * vq_grad_phi_test(j) * vq_grad_phi_trial(i);
+            }
+         }
+      }
+   }
+}
+
+
+
+void DirectionalVectorGradientIntegrator::AssembleElementMatrix(
+   const FiniteElement &el,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   int dof = el.GetDof();
+   int sdim = Trans.GetSpaceDim();
+
+   elmat.SetSize(dof * sdim, dof * sdim);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = 2 * el.GetOrder(); // Integration order
+      ir = &IntRules.Get(el.GetGeomType(), order);
+   }
+
+   // Get shape functions and their derivatives
+   DenseMatrix dshape(dof, sdim);
+   Vector shape(dof);
+   Vector vec(sdim);
+   Vector vq_grad_phi(dof);
+   for (int k = 0; k < ir->GetNPoints(); k++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(k);
+      Trans.SetIntPoint(&ip);
+      double w = ip.weight * Trans.Weight();
+      VQ->Eval(vec, Trans, ip);
+
+      // gradient of trial (u) in physical space
+      el.CalcPhysDShape(Trans, dshape);
+
+      // value of test (v) in physical space
+      el.CalcPhysShape(Trans, shape);
+
+      // (vq·∇)φ_i on the trial side
+      vq_grad_phi = 0.0;
+      for (int j = 0; j < dof; j++)
+      {
+         for (int d = 0; d < sdim; d++)
+         {
+            vq_grad_phi(j) += vec(d) * dshape(j, d);
+         }
+      }
+
+      for (int comp = 0; comp < sdim; comp++)
+      {
+         int offset = comp * dof;
+         for (int j = 0; j < dof; j++)
+         {
+            int jj = j + offset;
+            for (int i = 0; i < dof; i++)
+            {
+               int ii = i + offset;
+               elmat(jj, ii) += w * shape(j) * vq_grad_phi(i);
+            }
+         }
+      }
+   }
+}
+
+void DirectionalVectorGradientIntegrator::AssembleElementMatrix2(
+   const FiniteElement &trial_fe, const FiniteElement &test_fe,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   int trial_dof = trial_fe.GetDof();
+   int test_dof = test_fe.GetDof();
+   int dim = trial_fe.GetDim();
+   int sdim = Trans.GetSpaceDim();
+
+   elmat.SetSize(test_dof * sdim, trial_dof * sdim);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+   }
+
+   Vector vec(sdim);
+   DenseMatrix trial_dshape(trial_dof, sdim);
+   Vector test_shape(test_dof);
+   Vector vq_grad_phi_trial(trial_dof);
+
+   for (int k = 0; k < ir->GetNPoints(); k++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(k);
+      Trans.SetIntPoint(&ip);
+      double w = ip.weight * Trans.Weight();
+      VQ->Eval(vec, Trans, ip);
+      // gradient of trial (u) in physical space
+      trial_fe.CalcPhysDShape(Trans, trial_dshape);
+      // value of test (v) in physical space
+      test_fe.CalcPhysShape(Trans, test_shape);
+
+      // (vq·∇)φ_i on the trial side
+      vq_grad_phi_trial = 0.0;
+      for (int i = 0; i < trial_dof; i++)
+      {
+         for (int d = 0; d < dim; d++)
+         {
+            vq_grad_phi_trial(i) += vec(d) * trial_dshape(i, d);
+         }
+      }
+
+      for (int comp = 0; comp < sdim; comp++)
+      {
+         int offset_trial = comp * trial_dof;
+         int offset_test = comp * test_dof;
+         for (int j = 0; j < test_dof; j++)
+         {
+            int jj = j + offset_test;
+            for (int i = 0; i < trial_dof; i++)
+            {
+               int ii = i + offset_trial;
+               elmat(jj, ii) += w * test_shape(j) * vq_grad_phi_trial(i);
+            }
+         }
+      }
+   }
+}
+
+void MixedDirectionalVectorGradientIntegrator::AssembleElementMatrix2(
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   const int trial_dof = trial_fe.GetDof();
+   const int test_dof  = test_fe.GetDof();
+   const int sdim      = Trans.GetSpaceDim();
+
+   MFEM_VERIFY(test_fe.GetMapType() == mfem::FiniteElement::H_CURL ||
+               test_fe.GetMapType() == mfem::FiniteElement::H_DIV,
+               "MixedDirectionalVectorGradientIntegrator requires "
+               "H(curl) or H(div) test space.");
+
+   const int vdim = sdim;
+
+   // Test dofs already correspond to vector basis functions
+   elmat.SetSize(test_dof, trial_dof * vdim);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      const int order =
+         trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+   }
+
+   Vector q(vdim);
+   DenseMatrix trial_dshape(trial_dof, vdim);
+   DenseMatrix test_vshape(test_dof, vdim);
+
+   Vector q_dot_grad_phi(trial_dof);
+
+   for (int k = 0; k < ir->GetNPoints(); k++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(k);
+      Trans.SetIntPoint(&ip);
+
+      const double w = ip.weight * Trans.Weight();
+
+      // q in physical space
+      VQ->Eval(q, Trans, ip);
+
+      // ∇phi_i in physical space
+      trial_fe.CalcPhysDShape(Trans, trial_dshape);
+
+      // vector-valued test shape
+      test_fe.CalcVShape(Trans, test_vshape);
+
+      // (q · ∇) phi_i
+      q_dot_grad_phi = 0.0;
+      for (int i = 0; i < trial_dof; i++)
+      {
+         double val = 0.0;
+         for (int d = 0; d < vdim; d++)
+         {
+            val += q(d) * trial_dshape(i, d);
+         }
+         q_dot_grad_phi(i) = val;
+      }
+
+      // Assemble: v_j · (q·∇phi_i)
+      for (int j = 0; j < test_dof; j++)
+      {
+         for (int comp = 0; comp < vdim; comp++)
+         {
+            const double vj = test_vshape(j, comp);
+            const int offset_trial = comp * trial_dof;
+
+            for (int i = 0; i < trial_dof; i++)
+            {
+               elmat(j, offset_trial + i) +=
+                  w * vj * q_dot_grad_phi(i);
+            }
+         }
+      }
+   }
+}

miniapps/dpg/util/maxwell_utils.hpp

@@ -0,0 +1,135 @@
+#include "mfem.hpp"
+
+using namespace mfem;
+using namespace std;
+
+class AzimuthalECoefficient : public Coefficient
+{
+private:
+   const GridFunction * vgf;
+public:
+   AzimuthalECoefficient(const GridFunction * vgf_)
+      : Coefficient(), vgf(vgf_) {}
+   virtual real_t Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
+class ParallelECoefficient : public Coefficient
+{
+private:
+   const GridFunction * vgf;
+public:
+   ParallelECoefficient(const GridFunction * vgf_)
+      : Coefficient(), vgf(vgf_) {}
+   virtual real_t Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
+class EpsilonMatrixCoefficient : public MatrixArrayCoefficient
+{
+private:
+   Mesh * mesh = nullptr;
+   ParMesh * pmesh = nullptr;
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Array<ParGridFunction * > pgfs;
+   Array<GridFunctionCoefficient * > gf_cfs;
+   GridFunction * vgf = nullptr;
+   int dim;
+   int sdim;
+public:
+   EpsilonMatrixCoefficient(const char * filename, Mesh * mesh_, ParMesh * pmesh_,
+                            real_t scale = 1.0);
+
+   // Visualize the components of the matrix coefficient
+   // in separate GLVis windows for each component
+   void VisualizeMatrixCoefficient();
+   // Update the Gridfunctions after mesh refinement
+   void Update();
+
+   ~EpsilonMatrixCoefficient();
+
+};
+
+class DielectricTensorComponentCoefficient : public Coefficient
+{
+private:
+   real_t delta, a0, a1;
+   int row, col;
+   bool use_imag;
+
+public:
+   /// Constructor
+   /// @param delta_    Imaginary scaling factor
+   /// @param a0_       Constant in P(r)
+   /// @param a1_       Linear term in P(r)
+   /// @param row_      row index of the dielectric tensor
+   /// @param col_      column index of the dielectric tensor
+   /// @param use_imag_ If true, returns imaginary part; otherwise real part
+   DielectricTensorComponentCoefficient(real_t delta_, real_t a0_, real_t a1_,
+                                        int row_, int col_, bool use_imag_ = false);
+   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip);
+
+private:
+   real_t ComputeRealPart(const Vector &x);
+   real_t ComputeImagPart(const Vector &x);
+};
+
+void VisualizeMatrixArrayCoefficient(MatrixArrayCoefficient &mc, ParMesh *pmesh,
+                                     int order, bool paraview = false, const char *name = nullptr);
+
+void ComputeB(const Vector &x, Vector &b);
+
+// Integrator for (vq·∇) u · (vq·∇) v where u,v ∈ (H¹(Ω))ᵈ
+class DirectionalVectorDiffusionIntegrator : public BilinearFormIntegrator
+{
+private:
+   VectorCoefficient *VQ;
+
+public:
+   DirectionalVectorDiffusionIntegrator(VectorCoefficient &q) : VQ(&q) { }
+
+   virtual void AssembleElementMatrix(const FiniteElement &el,
+                                      ElementTransformation &Trans,
+                                      DenseMatrix &elmat);
+   virtual void AssembleElementMatrix2(const FiniteElement &tr_el,
+                                       const FiniteElement &te_el,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+
+};
+
+// Integrator for (vq·∇) u · v where u ∈ (H¹(Ω))ᵈ, v ∈ (H¹(Ω))ᵈ or (L²(Ω))ᵈ
+class DirectionalVectorGradientIntegrator : public BilinearFormIntegrator
+{
+private:
+   VectorCoefficient *VQ;
+
+public:
+   DirectionalVectorGradientIntegrator(VectorCoefficient &q) : VQ(&q) { }
+
+   virtual void AssembleElementMatrix(const FiniteElement &el,
+                                      ElementTransformation &Trans,
+                                      DenseMatrix &elmat);
+   virtual void AssembleElementMatrix2(const FiniteElement &tr_el,
+                                       const FiniteElement &te_el,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+
+};
+
+// Integrator for (vq·∇) u · v where u ∈ (H¹(Ω))ᵈ, v ∈ H(curl) or H(div)
+class MixedDirectionalVectorGradientIntegrator : public BilinearFormIntegrator
+{
+private:
+   VectorCoefficient *VQ;
+
+public:
+   MixedDirectionalVectorGradientIntegrator(VectorCoefficient &q) : VQ(&q) { }
+
+   virtual void AssembleElementMatrix2(const FiniteElement &tr_el,
+                                       const FiniteElement &te_el,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+
+};

miniapps/dpg/util/pcomplexblockform.cpp

@@ -0,0 +1,265 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "pcomplexblockform.hpp"
+
+#ifdef MFEM_USE_MPI
+
+namespace mfem
+{
+
+void ParComplexBlockForm::FillEssTdofLists(const Array<int> &
+                                           ess_tdof_list)
+{
+   int j;
+   for (int i = 0; i < ess_tdof_list.Size(); i++)
+   {
+      int tdof = ess_tdof_list[i];
+      for (j = 0; j < nblocks; j++)
+      {
+         if (tdof_offsets[j+1] > tdof) { break; }
+      }
+      ess_tdofs[j]->Append(tdof-tdof_offsets[j]);
+   }
+}
+
+void ParComplexBlockForm::Assemble(int skip_zeros)
+{
+   ComplexBlockForm::Assemble(skip_zeros);
+}
+
+void ParComplexBlockForm::ParallelAssemble(BlockMatrix *m_r,
+                                           BlockMatrix *m_i)
+{
+   if (!P) { BuildProlongation(); }
+
+   p_mat_r = new BlockOperator(tdof_offsets);
+   p_mat_i = new BlockOperator(tdof_offsets);
+   p_mat_e_r = new BlockOperator(tdof_offsets);
+   p_mat_e_i = new BlockOperator(tdof_offsets);
+   p_mat_r->owns_blocks = 1;
+   p_mat_i->owns_blocks = 1;
+   p_mat_e_r->owns_blocks = 1;
+   p_mat_e_i->owns_blocks = 1;
+   HypreParMatrix * A_r = nullptr;
+   HypreParMatrix * A_i = nullptr;
+   HypreParMatrix * PtAP_r = nullptr;
+   HypreParMatrix * PtAP_i = nullptr;
+   for (int i = 0; i < nblocks; i++)
+   {
+      HypreParMatrix * Pi = (HypreParMatrix*)(&P->GetBlock(i,i));
+      for (int j = 0; j<nblocks; j++)
+      {
+         if (m_r->IsZeroBlock(i,j)) { continue; }
+         if (i == j)
+         {
+            // Make block diagonal square hypre matrix
+            A_r = new HypreParMatrix(pfes[i]->GetComm(), pfes[i]->GlobalVSize(),
+                                     pfes[i]->GetDofOffsets(), &m_r->GetBlock(i,i));
+            PtAP_r = RAP(A_r,Pi);
+            delete A_r;
+            p_mat_e_r->SetBlock(i, i, PtAP_r->EliminateRowsCols(*ess_tdofs[i]));
+
+            A_i = new HypreParMatrix(pfes[i]->GetComm(), pfes[i]->GlobalVSize(),
+                                     pfes[i]->GetDofOffsets(), &m_i->GetBlock(i,i));
+
+            PtAP_i = RAP(A_i,Pi);
+            delete A_i;
+            p_mat_e_i->SetBlock(i, i, PtAP_i->EliminateCols(*ess_tdofs[i]));
+            PtAP_i->EliminateRows(*ess_tdofs[i]);
+         }
+         else
+         {
+            HypreParMatrix * Pj = (HypreParMatrix*)(&P->GetBlock(j,j));
+            A_r = new HypreParMatrix(pfes[i]->GetComm(), pfes[i]->GlobalVSize(),
+                                     pfes[j]->GlobalVSize(), pfes[i]->GetDofOffsets(),
+                                     pfes[j]->GetDofOffsets(), &m_r->GetBlock(i,j));
+            PtAP_r = RAP(Pi,A_r,Pj);
+            delete A_r;
+            p_mat_e_r->SetBlock(i, j, PtAP_r->EliminateCols(*ess_tdofs[j]));
+            PtAP_r->EliminateRows(*ess_tdofs[i]);
+
+            A_i = new HypreParMatrix(pfes[i]->GetComm(), pfes[i]->GlobalVSize(),
+                                     pfes[j]->GlobalVSize(), pfes[i]->GetDofOffsets(),
+                                     pfes[j]->GetDofOffsets(), &m_i->GetBlock(i,j));
+            PtAP_i = RAP(Pi,A_i,Pj);
+            delete A_i;
+            p_mat_e_i->SetBlock(i, j, PtAP_i->EliminateCols(*ess_tdofs[j]));
+            PtAP_i->EliminateRows(*ess_tdofs[i]);
+         }
+         p_mat_r->SetBlock(i, j, PtAP_r);
+         p_mat_i->SetBlock(i, j, PtAP_i);
+      }
+   }
+}
+
+void ParComplexBlockForm::BuildProlongation()
+{
+   P = new BlockOperator(dof_offsets, tdof_offsets);
+   R = new BlockMatrix(tdof_offsets, dof_offsets);
+   P->owns_blocks = 0;
+   R->owns_blocks = 0;
+
+   for (int i = 0; i < nblocks; i++)
+   {
+      HypreParMatrix * P_ = pfes[i]->Dof_TrueDof_Matrix();
+      P->SetBlock(i,i,P_);
+      const SparseMatrix * R_ = pfes[i]->GetRestrictionMatrix();
+      R->SetBlock(i, i, const_cast<SparseMatrix*>(R_));
+   }
+}
+
+void ParComplexBlockForm::FormLinearSystem(const Array<int>
+                                           &ess_tdof_list,
+                                           Vector &x, Vector &b,
+                                           OperatorHandle &A,
+                                           Vector &X, Vector &B,
+                                           int copy_interior)
+{
+   FormSystemMatrix(ess_tdof_list, A);
+
+   int n = P->Width();
+   B.SetSize(2*n);
+   Vector B_r(B, 0, n);
+   Vector B_i(B, n, n);
+
+   Vector b_r(b, 0, b.Size()/2);
+   Vector b_i(b, b.Size()/2, b.Size()/2);
+
+   P->MultTranspose(b_r, B_r);
+   P->MultTranspose(b_i, B_i);
+
+   int m = R->Height();
+   X.SetSize(2*m);
+
+   Vector X_r(X, 0, m);
+   Vector X_i(X, m, m);
+
+   Vector x_r(x, 0, x.Size()/2);
+   Vector x_i(x, x.Size()/2, x.Size()/2);
+
+   R->Mult(x_r, X_r);
+   R->Mult(x_i, X_i);
+
+   // eliminate tdof is RHS
+   // B_r -= Ae_r*X_r + Ae_i X_i
+   // B_i -= Ae_i*X_r + Ae_r X_i
+   Vector tmp(B_r.Size());
+   p_mat_e_r->Mult(X_r, tmp); B_r-=tmp;
+   p_mat_e_i->Mult(X_i, tmp); B_r+=tmp;
+
+   p_mat_e_i->Mult(X_r, tmp); B_i-=tmp;
+   p_mat_e_r->Mult(X_i, tmp); B_i-=tmp;
+
+   for (int j = 0; j < nblocks; j++)
+   {
+      if (!ess_tdofs[j]->Size()) { continue; }
+      for (int i = 0; i < ess_tdofs[j]->Size(); i++)
+      {
+         int tdof = (*ess_tdofs[j])[i];
+         int gdof = tdof + tdof_offsets[j];
+         B_r(gdof) = X_r(gdof); // diagonal policy is always one in parallel
+         B_i(gdof) = X_i(gdof); // diagonal policy is always one in parallel
+      }
+   }
+   if (!copy_interior)
+   {
+      X_r.SetSubVectorComplement(ess_tdof_list, 0.0);
+      X_i.SetSubVectorComplement(ess_tdof_list, 0.0);
+   }
+}
+
+void ParComplexBlockForm::FormSystemMatrix(const Array<int>
+                                           &ess_tdof_list,
+                                           OperatorHandle &A)
+{
+   FillEssTdofLists(ess_tdof_list);
+   if (mat_r)
+   {
+      const int remove_zeros = 0;
+      Finalize(remove_zeros);
+      ParallelAssemble(mat_r, mat_i);
+      delete mat_r;
+      delete mat_i;
+      mat_r = nullptr;
+      mat_i = nullptr;
+      delete mat_e_r;
+      delete mat_e_i;
+      mat_e_r = nullptr;
+      mat_e_i = nullptr;
+   }
+   p_mat = new ComplexOperator(p_mat_r, p_mat_i, false, false);
+   A.Reset(p_mat, false);
+}
+
+void ParComplexBlockForm::RecoverFEMSolution(const Vector &X,
+                                             Vector &x)
+{
+   int n = P->Height();
+   int m = P->Width();
+   x.SetSize(2*n);
+
+   Vector x_r(x,0,n);
+   Vector x_i(x,n,n);
+   Vector X_r(const_cast<Vector&>(X), 0, m);
+   Vector X_i(const_cast<Vector&>(X), m, m);
+
+   P->Mult(X_r, x_r);
+   P->Mult(X_i, x_i);
+}
+
+void ParComplexBlockForm::Update()
+{
+   ComplexBlockForm::Update();
+   delete p_mat_e_r;
+   delete p_mat_e_i;
+   p_mat_e_r = nullptr;
+   p_mat_e_i = nullptr;
+   delete p_mat_r;
+   delete p_mat_i;
+   p_mat_r = nullptr;
+   p_mat_i = nullptr;
+   delete p_mat;
+   p_mat = nullptr;
+   for (int i = 0; i < nblocks; i++)
+   {
+      delete ess_tdofs[i];
+      ess_tdofs[i] = new Array<int>();
+   }
+   delete P;
+   P = nullptr;
+   delete R;
+   R = nullptr;
+}
+
+ParComplexBlockForm::~ParComplexBlockForm()
+{
+   delete p_mat_e_r;
+   delete p_mat_e_i;
+   p_mat_e_r = nullptr;
+   p_mat_e_i = nullptr;
+   delete p_mat_r;
+   delete p_mat_i;
+   p_mat_r = nullptr;
+   p_mat_i = nullptr;
+   delete p_mat;
+   p_mat = nullptr;
+   for (int i = 0; i < nblocks; i++)
+   {
+      delete ess_tdofs[i];
+   }
+   delete P;
+   delete R;
+}
+
+} // namespace mfem
+
+#endif

miniapps/dpg/util/pcomplexblockform.hpp

@@ -0,0 +1,108 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_PCOMPLEX_BLOCKFORM
+#define MFEM_PCOMPLEX_BLOCKFORM
+
+#include "mfem.hpp"
+#include "complexblockform.hpp"
+
+#ifdef MFEM_USE_MPI
+
+namespace mfem
+{
+
+/** @brief Class representing the parallel weak formulation.
+   (Convenient for DPG Equations) */
+class ParComplexBlockForm : public ComplexBlockForm
+{
+
+protected:
+   /// Trial FE spaces
+   Array<ParFiniteElementSpace * > pfes;
+
+   /// ess_tdof list for each space
+   Array<Array<int> *> ess_tdofs;
+
+   /** split ess_tdof_list given in global tdof (for all spaces)
+    to individual lists for each space */
+   void FillEssTdofLists(const Array<int> & ess_tdof_list);
+
+   // Block Prolongation
+   BlockOperator * P = nullptr;
+   // Block Restriction
+   BlockMatrix * R = nullptr;
+
+   ComplexOperator * p_mat = nullptr;
+   BlockOperator * p_mat_r = nullptr;
+   BlockOperator * p_mat_i = nullptr;
+   BlockOperator * p_mat_e_r = nullptr;
+   BlockOperator * p_mat_e_i = nullptr;
+
+   void BuildProlongation();
+
+public:
+
+   ParComplexBlockForm() {}
+
+   /// Creates bilinear form associated with FE spaces @a trial_pfes_.
+   ParComplexBlockForm(Array<ParFiniteElementSpace* > & pfes_)
+      : ComplexBlockForm() { SetParSpaces(pfes_); }
+
+   void SetParSpaces(Array<ParFiniteElementSpace* > & pfes_)
+   {
+      pfes = pfes_;
+      ess_tdofs.SetSize(pfes.Size());
+
+      Array<FiniteElementSpace * > sfes(pfes.Size());
+      for (int i = 0; i<sfes.Size(); i++)
+      {
+         sfes[i] = (FiniteElementSpace *)pfes[i];
+         ess_tdofs[i] = new Array<int>();
+      }
+      SetSpaces(sfes);
+   }
+
+
+   /// Assembles the form i.e. sums over all domain integrators.
+   void Assemble(int skip_zeros = 1);
+
+   /// Returns the matrix assembled on the true dofs, i.e. P^t A P.
+   /** The returned matrix has to be deleted by the caller. */
+   void ParallelAssemble(BlockMatrix *mat_r, BlockMatrix *mat_i);
+
+   void FormLinearSystem(const Array<int> &ess_tdof_list,
+                         Vector &x, Vector &b,
+                         OperatorHandle &A,
+                         Vector &X, Vector &B,
+                         int copy_interior = 0);
+
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
+
+   /** Call this method after solving a linear system constructed using the
+       FormLinearSystem method to recover the solution as a ParGridFunction-size
+       vector in x. Use the same arguments as in the FormLinearSystem call. */
+   virtual void RecoverFEMSolution(const Vector &X, Vector &x);
+
+   virtual void Update();
+
+   /// Destroys bilinear form.
+   virtual ~ParComplexBlockForm();
+
+
+};
+
+} // namespace mfem
+
+
+#endif // MFEM_USE_MPI
+
+#endif

miniapps/dpg/util/pcomplexweakform.hpp

@@ -98,6 +98,17 @@ public:
 
    virtual void Update();
 
+   void GetTraceFESpaces(Array<ParFiniteElementSpace *> & trace_fes) const
+   {
+      Array<FiniteElementSpace *> sr_trace_fes;
+      ComplexDPGWeakForm::GetTraceFESpaces(sr_trace_fes);
+      trace_fes.SetSize(sr_trace_fes.Size());
+      for (int i = 0; i < sr_trace_fes.Size(); i++)
+      {
+         trace_fes[i] = dynamic_cast<ParFiniteElementSpace *>(sr_trace_fes[i]);
+      }
+   }
+
    /// Destroys bilinear form.
    virtual ~ParComplexDPGWeakForm();
 

miniapps/dpg/util/preconditioners.cpp

@@ -0,0 +1,561 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "preconditioners.hpp"
+
+namespace mfem
+{
+
+#ifdef MFEM_USE_MPI
+
+Solver * MakeFESpaceDefaultSolver(
+   const ParFiniteElementSpace * pfespace, int print_level)
+{
+   FiniteElementCollection const &fec = *(pfespace->FEColl());
+   const int vdim = pfespace->GetVDim();
+   const int dim = pfespace->GetParMesh()->Dimension();
+   Solver * prec = nullptr;
+   if (dynamic_cast<const H1_FECollection*>(&fec) ||
+       dynamic_cast<const L2_FECollection*>(&fec) || vdim > 1)
+   {
+      prec = new HypreBoomerAMG();
+      dynamic_cast<HypreBoomerAMG*>(prec)->SetPrintLevel(print_level);
+      if (vdim > 1)
+      {
+         dynamic_cast<HypreBoomerAMG*>(prec)->SetSystemsOptions(vdim);
+      }
+      return prec;
+   }
+   else if (dynamic_cast<const RT_FECollection*>(&fec) && dim == 3)
+   {
+      prec = new HypreADS(const_cast<ParFiniteElementSpace*>(pfespace));
+      dynamic_cast<HypreADS*>(prec)->SetPrintLevel(print_level);
+      return prec;
+   }
+   else if (dynamic_cast<const ND_FECollection*>(&fec) ||
+            dynamic_cast<const RT_FECollection*>(&fec))
+   {
+      prec = new HypreAMS(const_cast<ParFiniteElementSpace*>(pfespace));
+      dynamic_cast<HypreAMS*>(prec)->SetPrintLevel(print_level);
+      return prec;
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported FiniteElementCollection type");
+   }
+   return prec;
+}
+
+
+PRefinementHierarchy::PRefinementHierarchy(const Array<ParFiniteElementSpace*>
+                                           &pfes_,
+                                           const std::vector<Array<int>> & ess_bdr_marker_)
+   : pfes(pfes_), ess_bdr_marker(ess_bdr_marker_), nblocks(pfes.Size())
+{
+   MFEM_VERIFY(nblocks > 0, "Empty pfes.");
+   pmesh = pfes[0]->GetParMesh();
+   MFEM_VERIFY(pmesh, "pfes[0] has null ParMesh.");
+   MFEM_VERIFY(ess_bdr_marker.size() == static_cast<size_t>(nblocks),
+               "ess_bdr_marker size must match nblocks.");
+   int bdr_size = (pmesh->bdr_attributes.Size() > 0) ? pmesh->bdr_attributes.Max()
+                  : 0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      MFEM_VERIFY(ess_bdr_marker[i].Size() == bdr_size,
+                  "ess_bdr_marker[" << i << "] size must match max bdr_attribute in mesh.");
+   }
+}
+
+const ParFiniteElementSpace* PRefinementHierarchy::GetParFESpace(int lev,
+                                                                 int b) const
+{
+   if (lev == maxlevels - 1) { return pfes[b]; }
+   return fes_owned[lev][b].get();
+}
+
+int PRefinementHierarchy::GetFESpaceMinimumOrder(const ParFiniteElementSpace
+                                                 *pfespace)
+const
+{
+   return (dynamic_cast<const L2_FECollection*>(pfespace->FEColl()) ||
+           dynamic_cast<const RT_FECollection*>(pfespace->FEColl())) ? 0 : 1;
+}
+
+void PRefinementHierarchy::BuildSpaceHierarchy(int mgmaxlevels)
+{
+   orders.SetSize(nblocks);
+   Array<int> levels(nblocks);
+   for (int i = 0; i < nblocks; i++)
+   {
+      orders[i] = pfes[i]->FEColl()->GetConstructorOrder();
+      levels[i] = orders[i] - GetFESpaceMinimumOrder(pfes[i]);
+   }
+
+   maxlevels = levels.Min() + 1;
+   if (mgmaxlevels > 0)
+   {
+      maxlevels = std::min(maxlevels, mgmaxlevels);
+   }
+
+   MFEM_VERIFY(maxlevels >= 1, "Invalid maxlevels computed.");
+
+   fec_owned.resize(maxlevels-1);
+   fes_owned.resize(maxlevels-1);
+   T_level.resize(maxlevels-1);
+
+   for (int lev = 0; lev < maxlevels-1; lev++)
+   {
+      fec_owned[lev].resize(nblocks);
+      fes_owned[lev].resize(nblocks);
+      T_level[lev].resize(nblocks);
+   }
+
+   // Build ParFES hierarchy for each block
+   for (int b = 0; b < nblocks; b++)
+   {
+      const FiniteElementCollection *fec_ref = pfes[b]->FEColl();
+      const int vdim = pfes[b]->GetVDim();
+      const Ordering::Type ordering = pfes[b]->GetOrdering();
+
+      for (int lev = 1; lev <= maxlevels - 1; lev++)
+      {
+         const int p = orders[b] - lev;
+
+         auto &fec_ptr = fec_owned[maxlevels - lev - 1][b];
+         auto &fes_ptr = fes_owned[maxlevels - lev - 1][b];
+
+         fec_ptr.reset(fec_ref->Clone(p));
+         fes_ptr = std::make_unique<ParFiniteElementSpace>(pmesh, fec_ptr.get(),
+                                                           vdim, ordering);
+      }
+   }
+
+   // build true dof lists for all levels
+   ess_tdof_list.resize(maxlevels);
+   Array<int> tdof_offsets(nblocks+1);
+   for (int i = 0; i< maxlevels; i++)
+   {
+      tdof_offsets[0] = 0;
+      for (int b = 0; b < nblocks; b++)
+      {
+         tdof_offsets[b+1] = GetParFESpace(i,b)->GetTrueVSize();
+      }
+      tdof_offsets.PartialSum();
+      Array<int> tdof_list;
+      Array<int> block_tdof_list;
+      for (int b = 0; b < nblocks; b++)
+      {
+         block_tdof_list.SetSize(0);
+         GetParFESpace(i,b)->GetEssentialTrueDofs(ess_bdr_marker[b], block_tdof_list);
+         for (int j = 0; j < block_tdof_list.Size(); j++)
+         {
+            block_tdof_list[j] += tdof_offsets[b];
+         }
+         tdof_list.Append(block_tdof_list);
+      }
+      ess_tdof_list[i] = tdof_list;
+   }
+}
+
+BlockOperator *PRefinementHierarchy::BuildProlongation(int lev)
+{
+   MFEM_VERIFY(lev >= 0 &&
+               lev < maxlevels - 1, "Invalid level in BuildProlongation().");
+
+   Array<int> coarse_offsets(nblocks + 1); coarse_offsets[0] = 0;
+   Array<int> fine_offsets(nblocks + 1);   fine_offsets[0]   = 0;
+
+   for (int b = 0; b < nblocks; b++)
+   {
+      coarse_offsets[b+1] = coarse_offsets[b] + GetParFESpace(lev,
+                                                              b)->GetTrueVSize();
+      fine_offsets[b+1]   = fine_offsets[b]   + GetParFESpace(lev+1,
+                                                              b)->GetTrueVSize();
+   }
+
+   BlockOperator *Pblk = new BlockOperator(fine_offsets, coarse_offsets);
+   Pblk->owns_blocks = 0;
+
+   for (int b = 0; b < nblocks; b++)
+   {
+      T_level[lev][b] = std::make_unique<PRefinementTransferOperator>(
+                           *GetParFESpace(lev, b), *GetParFESpace(lev+1, b), true);
+
+      HypreParMatrix *P =
+         dynamic_cast<HypreParMatrix*>(T_level[lev][b]->GetTrueTransferOperator());
+      MFEM_VERIFY(P, "PRefinement transfer returned null.");
+      Pblk->SetBlock(b, b, P);
+   }
+   return Pblk;
+}
+
+
+PRefinementMultigrid::PRefinementMultigrid(const Array<ParFiniteElementSpace*>
+                                           &pfes_,
+                                           const std::vector<Array<int>> & ess_bdr_marker_,
+                                           const BlockOperator &Op_, int mgmaxlevels, real_t smoother_relax_factor,
+                                           bool mumps_coarse_solver)
+   : Multigrid()
+   , hierarchy(pfes_, ess_bdr_marker_)
+   , Op(Op_)
+{
+#ifndef MFEM_USE_MUMPS
+   if (mumps_coarse_solver)
+   {
+      MFEM_WARNING("MUMPS coarse solver requires MFEM built with MUMPS. Switching to default coarse solver.");
+   }
+   mumps_coarse_solver = false;
+#endif
+
+   hierarchy.BuildSpaceHierarchy(mgmaxlevels);
+
+   const int maxlevels = hierarchy.maxlevels;
+   const int nblocks   = hierarchy.nblocks;
+
+   operators.SetSize(maxlevels);
+   ownedOperators.SetSize(maxlevels);
+   smoothers.SetSize(maxlevels);
+   ownedSmoothers.SetSize(maxlevels);
+
+   operators[maxlevels-1] = const_cast<BlockOperator*>(&Op);
+   ownedOperators[maxlevels-1] = false;
+
+   const int nP = std::max(0, maxlevels - 1);
+   prolongations.SetSize(nP);
+   ownedProlongations.SetSize(nP);
+
+   // Build prolongations and Galerkin operators
+   for (int lev = nP - 1; lev >= 0; lev--)
+   {
+      BlockOperator *Pblk = hierarchy.BuildProlongation(lev);
+      prolongations[lev] = new RectangularConstrainedOperator(Pblk,
+                                                              hierarchy.ess_tdof_list[lev], hierarchy.ess_tdof_list[lev+1], true);
+      ownedProlongations[lev] = true;
+
+      BlockOperator *OpLevel = new BlockOperator(Pblk->ColOffsets());
+      OpLevel->owns_blocks = 1;
+
+      BlockOperator *OpFine = dynamic_cast<BlockOperator*>(operators[lev+1]);
+      MFEM_VERIFY(OpFine, "Expected BlockOperator at fine level.");
+
+      for (int i = 0; i < nblocks; i++)
+      {
+         HypreParMatrix *Pi = dynamic_cast<HypreParMatrix*>(&Pblk->GetBlock(i, i));
+         MFEM_VERIFY(Pi, "Expected HypreParMatrix prolongation block.");
+         HypreParMatrix *Pit = Pi->Transpose();
+
+         for (int j = 0; j < nblocks; j++)
+         {
+            if (OpFine->IsZeroBlock(i, j)) { continue; }
+
+            const HypreParMatrix *A_fine =
+               dynamic_cast<const HypreParMatrix*>(&OpFine->GetBlock(i, j));
+            MFEM_VERIFY(A_fine, "Expected HypreParMatrix block.");
+
+            if (i == j)
+            {
+               OpLevel->SetBlock(i, i, RAP(A_fine, Pi));
+            }
+            else
+            {
+               HypreParMatrix *Pj = dynamic_cast<HypreParMatrix*>(&Pblk->GetBlock(j, j));
+               MFEM_VERIFY(Pj, "Expected HypreParMatrix prolongation block.");
+
+               HypreParMatrix *APj  = ParMult(A_fine, Pj, true);
+               HypreParMatrix *PtAP = ParMult(Pit, APj, true);
+               delete APj;
+               OpLevel->SetBlock(i, j, PtAP);
+            }
+         }
+         delete Pit;
+      }
+      operators[lev] = OpLevel;
+      ownedOperators[lev] = true;
+   }
+
+   // Build smoothers
+   for (int lev = 0; lev < operators.Size(); lev++)
+   {
+      auto *cOp = dynamic_cast<BlockOperator*>(operators[lev]);
+      MFEM_VERIFY(cOp, "Expected BlockOperator in operators[].");
+
+      if (lev == 0 && operators.Size() > 1) // coarse
+      {
+#ifdef MFEM_USE_MUMPS
+         if (mumps_coarse_solver)
+         {
+            HypreParMatrix *Acoarse = cOp->GetMonolithicHypreParMatrix();
+            auto *mumps_solver = new MUMPSSolver(MPI_COMM_WORLD);
+            mumps_solver->SetPrintLevel(0);
+            mumps_solver->SetOperator(*Acoarse);
+            delete Acoarse;
+
+            smoothers[lev] = mumps_solver;
+            ownedSmoothers[lev] = true;
+         }
+         else
+#endif
+         {
+            auto *bd = new BlockDiagonalPreconditioner(cOp->RowOffsets());
+            bd->owns_blocks = 1;
+
+            for (int b = 0; b < nblocks; b++)
+            {
+               const HypreParMatrix *Ab =
+                  dynamic_cast<const HypreParMatrix*>(&cOp->GetBlock(b, b));
+               MFEM_VERIFY(Ab, "Expected HypreParMatrix block.");
+
+               auto solver = MakeFESpaceDefaultSolver(hierarchy.GetParFESpace(lev, b), 0);
+               solver->SetOperator(*Ab);
+               bd->SetDiagonalBlock(b, solver);
+            }
+
+            coarse_prec.reset(bd);
+
+            auto *cg = new CGSolver(MPI_COMM_WORLD);
+            cg->SetPrintLevel(-1);
+            cg->SetRelTol(1e-3);
+            cg->SetMaxIter(10);
+            cg->SetOperator(*cOp);
+            cg->SetPreconditioner(*coarse_prec);
+
+            smoothers[lev] = cg;
+            ownedSmoothers[lev] = true;
+         }
+      }
+      else
+      {
+         auto *prec = new SymmetricBlockDiagonalPreconditioner(cOp->RowOffsets(),
+                                                               smoother_relax_factor);
+         prec->owns_blocks = 1;
+
+         for (int b = 0; b < nblocks; b++)
+         {
+            const HypreParMatrix *Ab =
+               dynamic_cast<const HypreParMatrix*>(&cOp->GetBlock(b, b));
+            MFEM_VERIFY(Ab, "Expected HypreParMatrix block.");
+
+            auto solver = MakeFESpaceDefaultSolver(hierarchy.GetParFESpace(lev, b), 0);
+            solver->SetOperator(*Ab);
+            prec->SetDiagonalBlock(b, solver);
+         }
+         smoothers[lev] = prec;
+         ownedSmoothers[lev] = true;
+      }
+   }
+}
+
+ComplexPRefinementMultigrid::ComplexPRefinementMultigrid(
+   const Array<ParFiniteElementSpace*> &pfes_,
+   const std::vector<Array<int>> & ess_bdr_marker,
+   const ComplexOperator &Op_, int mgmaxlevels, real_t smoother_relax_factor,
+   bool mumps_coarse_solver)
+   : Multigrid(), Op(Op_)
+{
+#ifndef MFEM_USE_COMPLEX_MUMPS
+   if (mumps_coarse_solver)
+   {
+      MFEM_WARNING("MUMPS coarse solver requires MFEM built with MUMPS. Switching to default coarse solver.");
+   }
+   mumps_coarse_solver = false;
+#endif
+
+   const auto *Op_r = dynamic_cast<const BlockOperator*>(&Op.real());
+   const auto *Op_i = dynamic_cast<const BlockOperator*>(&Op.imag());
+   MFEM_VERIFY(Op_r, "Expected BlockOperator from ComplexOperator real part.");
+   MFEM_VERIFY(Op_i, "Expected BlockOperator from ComplexOperator imag part.");
+
+   const int nblocks = Op_r->NumRowBlocks();
+   MFEM_VERIFY(nblocks == Op_i->NumRowBlocks(), "Real/imag block counts differ.");
+   hierarchy = std::make_unique<PRefinementHierarchy>(pfes_, ess_bdr_marker);
+
+   hierarchy->BuildSpaceHierarchy(mgmaxlevels);
+
+   const int maxlevels = hierarchy->maxlevels;
+
+   operators.SetSize(maxlevels);
+   ownedOperators.SetSize(maxlevels);
+   smoothers.SetSize(maxlevels);
+   ownedSmoothers.SetSize(maxlevels);
+
+   operators[maxlevels-1] = const_cast<ComplexOperator*>(&Op);
+   ownedOperators[maxlevels-1] = false;
+
+   const int nP = std::max(0, maxlevels - 1);
+   prolongations.SetSize(nP);
+   ownedProlongations.SetSize(nP);
+
+   for (int lev = nP - 1; lev >= 0; lev--)
+   {
+      BlockOperator *Pblk = hierarchy->BuildProlongation(lev);
+
+      auto ConstrOp = new RectangularConstrainedOperator(Pblk,
+                                                         hierarchy->ess_tdof_list[lev],
+                                                         hierarchy->ess_tdof_list[lev+1], true);
+
+      // prolongation as complex (real=Pblk, imag=nullptr)
+      prolongations[lev] = new ComplexOperator(ConstrOp, nullptr, true, true);
+      ownedProlongations[lev] = true;
+
+      auto *OpLevel_r = new BlockOperator(Pblk->ColOffsets());
+      auto *OpLevel_i = new BlockOperator(Pblk->ColOffsets());
+      OpLevel_r->owns_blocks = 1;
+      OpLevel_i->owns_blocks = 1;
+
+      auto *cOp = dynamic_cast<ComplexOperator*>(operators[lev+1]);
+      MFEM_VERIFY(cOp, "Expected ComplexOperator at fine level.");
+
+      auto *cOp_r = dynamic_cast<BlockOperator*>(&cOp->real());
+      auto *cOp_i = dynamic_cast<BlockOperator*>(&cOp->imag());
+      MFEM_VERIFY(cOp_r, "Expected BlockOperator fine real part.");
+      MFEM_VERIFY(cOp_i, "Expected BlockOperator fine imag part.");
+
+      for (int i = 0; i < nblocks; i++)
+      {
+         HypreParMatrix *Pi = dynamic_cast<HypreParMatrix*>(&Pblk->GetBlock(i, i));
+         MFEM_VERIFY(Pi, "Expected HypreParMatrix prolongation block.");
+         HypreParMatrix *Pit = Pi->Transpose();
+
+         for (int j = 0; j < nblocks; j++)
+         {
+            if (!cOp_r->IsZeroBlock(i, j))
+            {
+               const HypreParMatrix *A_fine_r =
+                  dynamic_cast<const HypreParMatrix*>(&cOp_r->GetBlock(i, j));
+               MFEM_VERIFY(A_fine_r, "Expected HypreParMatrix block (real).");
+
+               if (i == j)
+               {
+                  OpLevel_r->SetBlock(i, i, RAP(A_fine_r, Pi));
+               }
+               else
+               {
+                  HypreParMatrix *Pj = dynamic_cast<HypreParMatrix*>(&Pblk->GetBlock(j, j));
+                  MFEM_VERIFY(Pj, "Expected HypreParMatrix prolongation block.");
+
+                  HypreParMatrix *APj  = ParMult(A_fine_r, Pj, true);
+                  HypreParMatrix *PtAP = ParMult(Pit, APj, true);
+                  delete APj;
+
+                  OpLevel_r->SetBlock(i, j, PtAP);
+               }
+            }
+
+            if (!cOp_i->IsZeroBlock(i, j))
+            {
+               const HypreParMatrix *A_fine_i =
+                  dynamic_cast<const HypreParMatrix*>(&cOp_i->GetBlock(i, j));
+               MFEM_VERIFY(A_fine_i, "Expected HypreParMatrix block (imag).");
+
+               if (i == j)
+               {
+                  OpLevel_i->SetBlock(i, i, RAP(A_fine_i, Pi));
+               }
+               else
+               {
+                  HypreParMatrix *Pj = dynamic_cast<HypreParMatrix*>(&Pblk->GetBlock(j, j));
+                  MFEM_VERIFY(Pj, "Expected HypreParMatrix prolongation block.");
+
+                  HypreParMatrix *APj  = ParMult(A_fine_i, Pj, true);
+                  HypreParMatrix *PtAP = ParMult(Pit, APj, true);
+                  delete APj;
+
+                  OpLevel_i->SetBlock(i, j, PtAP);
+               }
+            }
+         }
+
+         delete Pit;
+      }
+
+      auto *OpLevel_c = new ComplexOperator(OpLevel_r, OpLevel_i, true, true);
+      operators[lev] = OpLevel_c;
+      ownedOperators[lev] = true;
+   }
+
+   // smoothers
+   for (int lev = 0; lev < operators.Size(); lev++)
+   {
+      auto *cOp = dynamic_cast<ComplexOperator*>(operators[lev]);
+      MFEM_VERIFY(cOp, "Expected ComplexOperator in operators[].");
+
+      auto *cOp_r = dynamic_cast<BlockOperator*>(&cOp->real());
+      MFEM_VERIFY(cOp_r, "Expected BlockOperator real part in ComplexOperator.");
+
+      if (lev == 0 && operators.Size() > 1)
+      {
+#ifdef MFEM_USE_COMPLEX_MUMPS
+         if (mumps_coarse_solver)
+         {
+            ComplexHypreParMatrix *Ahc = cOp->AsComplexHypreParMatrix();
+            auto *mumps_solver = new ComplexMUMPSSolver(MPI_COMM_WORLD);
+            mumps_solver->SetPrintLevel(0);
+            mumps_solver->SetOperator(*Ahc);
+            smoothers[lev] = mumps_solver;
+            ownedSmoothers[lev] = true;
+            delete Ahc;
+         }
+         else
+#endif
+         {
+            auto *prec_r = new BlockDiagonalPreconditioner(cOp_r->RowOffsets());
+            prec_r->owns_blocks = 1;
+
+            for (int b = 0; b < nblocks; b++)
+            {
+               const HypreParMatrix *Ab =
+                  dynamic_cast<const HypreParMatrix*>(&cOp_r->GetBlock(b, b));
+               MFEM_VERIFY(Ab, "Expected HypreParMatrix block.");
+
+               auto solver = MakeFESpaceDefaultSolver(hierarchy->GetParFESpace(lev, b), 0);
+               solver->SetOperator(*Ab);
+               prec_r->SetDiagonalBlock(b, solver);
+            }
+
+            coarse_prec.reset(new ComplexPreconditioner(prec_r, true));
+
+            auto *cg = new CGSolver(MPI_COMM_WORLD);
+            cg->SetPrintLevel(-1);
+            cg->SetRelTol(1e-3);
+            cg->SetMaxIter(10);
+            cg->SetOperator(*cOp);
+            cg->SetPreconditioner(*coarse_prec);
+
+            smoothers[lev] = cg;
+            ownedSmoothers[lev] = true;
+         }
+      }
+      else
+      {
+         auto *prec_r = new SymmetricBlockDiagonalPreconditioner(cOp_r->RowOffsets(),
+                                                                 smoother_relax_factor);
+         prec_r->owns_blocks = 1;
+
+         for (int b = 0; b < nblocks; b++)
+         {
+            const HypreParMatrix *Ab =
+               dynamic_cast<const HypreParMatrix*>(&cOp_r->GetBlock(b, b));
+            MFEM_VERIFY(Ab, "Expected HypreParMatrix block.");
+
+            auto solver = MakeFESpaceDefaultSolver(hierarchy->GetParFESpace(lev, b), 0);
+            solver->SetOperator(*Ab);
+            prec_r->SetDiagonalBlock(b, solver);
+         }
+
+         smoothers[lev] = new ComplexPreconditioner(prec_r, true);
+         ownedSmoothers[lev] = true;
+      }
+   }
+}
+
+#endif
+
+} // namespace mfem

miniapps/dpg/util/preconditioners.hpp

@@ -0,0 +1,183 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "mfem.hpp"
+
+namespace mfem
+{
+
+// A BlockDiagonalPreconditioner which assumes that all the blocks are symmetric
+// Convenient to use with Multigrid Class in which a MultTranspose is needed
+class SymmetricBlockDiagonalPreconditioner : public BlockDiagonalPreconditioner
+{
+private:
+   real_t c;
+public:
+   SymmetricBlockDiagonalPreconditioner(const Array<int> & offsets,
+                                        real_t c_ = 1.0)
+      : BlockDiagonalPreconditioner(offsets), c(c_) { }
+
+   void Mult(const Vector & x, Vector & y) const override
+   {
+      BlockDiagonalPreconditioner::Mult(x,y);
+      y*=c;
+   }
+
+
+   void MultTranspose (const Vector & x, Vector & y) const override
+   {
+      this->Mult(x,y);
+   }
+};
+
+#ifdef MFEM_USE_MPI
+
+Solver * MakeFESpaceDefaultSolver(
+   const ParFiniteElementSpace * pfespace, int print_level);
+
+
+/// Shared helper class (real/complex) p-refinement multigrid:
+/// - builds FE hierarchy
+/// - builds transfer operators
+///
+class PRefinementHierarchy
+{
+public:
+   Array<int> orders;
+   const Array<ParFiniteElementSpace*> &pfes;
+   std::vector<Array<int>> ess_bdr_marker;
+   std::vector<Array<int>> ess_tdof_list;
+   ParMesh *pmesh = nullptr;
+   int nblocks;
+   int maxlevels = 1;
+
+   // Owned levels: 0..maxlevels-2
+   std::vector<std::vector<std::unique_ptr<FiniteElementCollection>>> fec_owned;
+   std::vector<std::vector<std::unique_ptr<ParFiniteElementSpace>>>   fes_owned;
+
+   // Transfer operators per level and block (owned here)
+   std::vector<std::vector<std::unique_ptr<PRefinementTransferOperator>>> T_level;
+
+   PRefinementHierarchy(const Array<ParFiniteElementSpace*> &pfes_,
+                        const std::vector<Array<int>> & ess_bdr_marker_);
+
+   const ParFiniteElementSpace* GetParFESpace(int lev, int b) const;
+
+   int GetFESpaceMinimumOrder(const ParFiniteElementSpace *pfespace) const;
+
+   /// Computes orders/maxlevels and constructs fec/fes hierarchy and T_level storage.
+   void BuildSpaceHierarchy(int mgmaxlevels = -1);
+
+   /// Builds block-diagonal prolongation for level lev (coarse=lev, fine=lev+1).
+   /// Its diagonal blocks are HypreParMatrix*
+   /// returned by the transfer operators stored in T_level[lev][b].
+   BlockOperator *BuildProlongation(int lev);
+};
+
+class PRefinementMultigrid : public Multigrid
+{
+private:
+   PRefinementHierarchy hierarchy;
+   const BlockOperator &Op;
+
+   std::unique_ptr<Solver> coarse_prec;
+
+public:
+   PRefinementMultigrid(const Array<ParFiniteElementSpace*> &pfes_,
+                        const std::vector<Array<int>> & ess_bdr_marker_,
+                        const BlockOperator &Op_, int mgmaxlevels = -1,
+                        real_t smoother_relax_factor = 2.0/3,
+                        bool mumps_coarse_solver = false);
+
+   ~PRefinementMultigrid() override = default;
+};
+
+
+class ComplexPRefinementMultigrid : public Multigrid
+{
+private:
+   // NOTE: nblocks for the hierarchy is derived from Op.real() at construction time,
+   // so we store hierarchy behind a pointer to avoid a "dummy nblocks" constructor.
+   std::unique_ptr<PRefinementHierarchy> hierarchy;
+
+   const ComplexOperator &Op;
+   std::unique_ptr<Solver> coarse_prec;
+
+public:
+   ComplexPRefinementMultigrid(const Array<ParFiniteElementSpace*> &pfes_,
+                               const std::vector<Array<int>> & ess_bdr_marker,
+                               const ComplexOperator &Op_, int mgmaxlevels = -1,
+                               real_t smoother_relax_factor = 2.0/3,
+                               bool mumps_coarse_solver = false);
+
+   ~ComplexPRefinementMultigrid() override = default;
+};
+
+#endif
+
+// Applies a given real preconditioner to the real and imaginary parts of a complex vector
+class ComplexPreconditioner : public Solver
+{
+private:
+   const Operator *op = nullptr;
+   const Solver * prec = nullptr;
+   bool own_prec = false;
+
+public:
+   ComplexPreconditioner(const Solver * real_prec, bool own = false)
+      : Solver(2*real_prec->Height()), prec(real_prec), own_prec(own) { }
+
+   virtual void Mult(const Vector &x, Vector &y) const override
+   {
+      int n = x.Size()/2;
+      MFEM_VERIFY(x.Size() == 2*n, "Invalid x vector size");
+      MFEM_VERIFY(y.Size() == 2*n, "Invalid y vector size");
+
+      Vector x_r(const_cast<Vector&>(x), 0, n);
+      Vector x_i(const_cast<Vector&>(x), n, n);
+      Vector y_r(y, 0, n);
+      Vector y_i(y, n, n);
+
+      // Apply the preconditioner to the real and imaginary parts separately
+      prec->Mult(x_r, y_r);
+      prec->Mult(x_i, y_i);
+   }
+
+   virtual void MultTranspose(const Vector &x, Vector &y) const override
+   {
+      int n = x.Size()/2;
+      MFEM_VERIFY(x.Size() == 2*n, "Invalid x vector size");
+      MFEM_VERIFY(y.Size() == 2*n, "Invalid y vector size");
+
+      Vector x_r(const_cast<Vector&>(x), 0, n);
+      Vector x_i(const_cast<Vector&>(x), n, n);
+      Vector y_r(y, 0, n);
+      Vector y_i(y, n, n);
+
+      // Apply the preconditioner to the real and imaginary parts separately
+      prec->MultTranspose(x_r, y_r);
+      prec->MultTranspose(x_i, y_i);
+   }
+
+   void SetOperator(const Operator &op_) override
+   {
+      MFEM_VERIFY(dynamic_cast<const ComplexOperator*>(&op_),
+                  "ComplexPreconditioner::SetOperator only accepts ComplexOperator");
+      this->op = &op_;
+   }
+
+   ~ComplexPreconditioner()
+   {
+      if (own_prec) { delete prec; }
+   }
+};
+
+} // namespace mfem

miniapps/dpg/util/pweakform.hpp

@@ -76,7 +76,6 @@ public:
       SetSpaces(trial_sfes,fecol_);
    }
 
-
    /// Assemble the local matrix
    void Assemble(int skip_zeros = 1);
 
@@ -98,6 +97,17 @@ public:
 
    virtual void Update();
 
+   void GetTraceFESpaces(Array<ParFiniteElementSpace *> & trace_fes) const
+   {
+      Array<FiniteElementSpace *> sr_trace_fes;
+      DPGWeakForm::GetTraceFESpaces(sr_trace_fes);
+      trace_fes.SetSize(sr_trace_fes.Size());
+      for (int i = 0; i < sr_trace_fes.Size(); i++)
+      {
+         trace_fes[i] = dynamic_cast<ParFiniteElementSpace *>(sr_trace_fes[i]);
+      }
+   }
+
    /// Destroys bilinear form.
    virtual ~ParDPGWeakForm();
 

miniapps/dpg/util/utils.cpp

@@ -0,0 +1,352 @@
+//
+#include "utils.hpp"
+
+std::string GetTimestamp()
+{
+   std::time_t now = std::time(nullptr);
+   std::tm* local_time = std::localtime(&now);
+   std::ostringstream oss;
+   oss << (local_time->tm_mon + 1) << "-"
+       << local_time->tm_mday << "-"
+       << (local_time->tm_year + 1900) << "_"
+       << local_time->tm_hour << ":"
+       << local_time->tm_min << ":"
+       << local_time->tm_sec;
+   return oss.str();
+}
+
+// Write parsed options to a file in the ParaView directory
+void WriteParametersToFile(const mfem::OptionsParser& args,
+                           const std::string& output_dir)
+{
+   // Ensure directory exists
+   std::string mkdir_command = "mkdir -p " + output_dir;
+   int ret = system(mkdir_command.c_str());
+   if (ret != 0)
+   {
+      std::cerr << "Warning: Failed to create ParaView output directory.\n";
+   }
+
+   std::string filename = output_dir + "/run_parameters.txt";
+
+   std::ofstream param_file(filename);
+   if (param_file.is_open())
+   {
+      param_file << "Simulation Parameters \n";
+      param_file << "------------------------------------\n";
+
+      // Use OptionsParser's Print method to output parameters to the file
+      args.PrintOptions(param_file);
+
+      param_file.close();
+      std::cout << "Parameters saved to " << filename << "\n";
+   }
+   else
+   {
+      std::cerr << "Error: Unable to open file to save parameters.\n";
+   }
+}
+
+void CreateParaViewPath(const char* mesh_file, std::string& output_dir)
+{
+   std::string timestamp = GetTimestamp();
+   std::string paraview_file = timestamp;
+
+   output_dir = output_dir + paraview_file;
+}
+
+std::string GetFilename(const std::string& filePath)
+{
+   // Step 1: Find the last '/' or '\' to isolate the filename
+   size_t lastSlash = filePath.find_last_of("/\\");
+   std::string filename = (lastSlash == std::string::npos) ? filePath :
+                          filePath.substr(lastSlash + 1);
+
+   // Step 2: Find the last '.' to remove the extension
+   size_t lastDot = filename.find_last_of('.');
+   return (lastDot == std::string::npos) ? filename : filename.substr(0, lastDot);
+}
+
+
+
+// Compute offsets for True DOFs across all processors
+void ElementTdofs::ComputeTdofOffsets()
+{
+   tdof_offsets.resize(num_procs);
+   int mytoffset = pfes->GetMyTDofOffset();
+   MPI_Allgather(&mytoffset, 1, MPI_INT, tdof_offsets.data(), 1, MPI_INT, comm);
+}
+
+// Determine which rank owns a given True DOF (tdof)
+int ElementTdofs::GetRank(int tdof)
+{
+   auto up = std::upper_bound(tdof_offsets.begin(), tdof_offsets.end(), tdof);
+   return std::distance(tdof_offsets.begin(), up) - 1;
+}
+
+// Distribute indices using MPI_Alltoallv
+void ElementTdofs::DistributeIndices(Array<int>& indices,
+                                     Array<int>& processors,
+                                     Array<int>& recv_idx)
+{
+   // Step 1: Prepare data to send
+   std::vector<int> send_counts(num_procs, 0);
+   std::map<int, std::vector<int>> send_buffers;
+
+   for (int i = 0; i < indices.Size(); ++i)
+   {
+      send_buffers[processors[i]].push_back(indices[i]);
+   }
+
+   std::vector<int> send_displs(num_procs, 0);
+   std::vector<int> send_data;
+
+   for (int i = 0; i < num_procs; ++i)
+   {
+      send_counts[i] = send_buffers[i].size();
+      send_data.insert(send_data.end(), send_buffers[i].begin(),
+                       send_buffers[i].end());
+   }
+
+   for (int i = 1; i < num_procs; ++i)
+   {
+      send_displs[i] = send_displs[i - 1] + send_counts[i - 1];
+   }
+
+   // Step 2: Gather receive counts
+   std::vector<int> recv_counts(num_procs, 0);
+   MPI_Alltoall(send_counts.data(), 1, MPI_INT, recv_counts.data(), 1, MPI_INT,
+                comm);
+
+   // Step 3: Compute receive displacements
+   std::vector<int> recv_displs(num_procs, 0);
+   int total_recv_size = 0;
+
+   for (int i = 0; i < num_procs; ++i)
+   {
+      recv_displs[i] = total_recv_size;
+      total_recv_size += recv_counts[i];
+   }
+
+   // Step 4: Allocate buffer and perform MPI_Alltoallv
+   recv_idx.SetSize(total_recv_size);
+   MPI_Alltoallv(send_data.data(), send_counts.data(), send_displs.data(), MPI_INT,
+                 recv_idx.GetData(), recv_counts.data(), recv_displs.data(), MPI_INT, comm);
+}
+
+
+ElementTdofs::ElementTdofs(const ParFiniteElementSpace* pfes_)
+   : pfes(pfes_), comm(pfes_->GetComm())
+{
+   MPI_Comm_size(comm, &num_procs);
+   ComputeTdofOffsets();
+}
+
+// Extract and distribute True DOFs based on the element attribute
+Array<int> ElementTdofs::GetTrueDOFs(int element_attribute)
+{
+   Array<int> mydof_list, other_dof_list;
+   Array<int> myltdof, other_ltdof;
+
+   if (boundary)
+   {
+      for (int i = 0; i < pfes->GetNBE(); i++)
+      {
+         if (pfes->GetParMesh()->GetBdrAttribute(i) == element_attribute)
+         {
+            int el,info;
+            pfes->GetParMesh()->GetBdrElementAdjacentElement(i,el,info);
+            Array<int> dofs;
+            pfes->GetElementVDofs(el, dofs);
+
+            for (int j = 0; j < dofs.Size(); j++)
+            {
+               int decoded_dof = ParFiniteElementSpace::DecodeDof(dofs[j]);
+               int ltdof = pfes->GetLocalTDofNumber(decoded_dof);
+               if (ltdof < 0)
+               {
+                  continue; // Skip if this is not a true DOF
+               }
+               int tdof = pfes->GetGlobalTDofNumber(decoded_dof);
+               if (Mpi::WorldRank() == GetRank(tdof))
+               {
+                  mydof_list.Append(tdof);
+                  myltdof.Append(ltdof);
+               }
+               else
+               {
+                  other_dof_list.Append(tdof);
+                  other_ltdof.Append(ltdof);
+               }
+            }
+         }
+      }
+   }
+   else
+   {
+      for (int i = 0; i < pfes->GetNE(); i++)
+      {
+         if (pfes->GetParMesh()->GetAttribute(i) == element_attribute)
+         {
+            Array<int> dofs;
+            pfes->GetElementVDofs(i, dofs);
+
+            for (int j = 0; j < dofs.Size(); j++)
+            {
+               int decoded_dof = ParFiniteElementSpace::DecodeDof(dofs[j]);
+               int ltdof = pfes->GetLocalTDofNumber(decoded_dof);
+               if (ltdof < 0)
+               {
+                  continue; // Skip if this is not a true DOF
+               }
+               int tdof = pfes->GetGlobalTDofNumber(decoded_dof);
+               if (Mpi::WorldRank() == GetRank(tdof))
+               {
+                  mydof_list.Append(tdof);
+                  myltdof.Append(ltdof);
+               }
+               else
+               {
+                  other_dof_list.Append(tdof);
+                  other_ltdof.Append(ltdof);
+               }
+            }
+         }
+      }
+   }
+
+   mydof_list.Sort();
+   mydof_list.Unique();
+   other_dof_list.Sort();
+   other_dof_list.Unique();
+
+   myltdof.Sort();
+   myltdof.Unique();
+   other_ltdof.Sort();
+   other_ltdof.Unique();
+
+   // Assign processors for other DOFs
+   Array<int> other_dof_proc_list(other_dof_list.Size());
+   for (int i = 0; i < other_dof_list.Size(); i++)
+   {
+      other_dof_proc_list[i] = GetRank(other_dof_list[i]);
+   }
+
+   // Communicate DOFs to appropriate processors
+   Array<int> recv_idx;
+   DistributeIndices(other_dof_list, other_dof_proc_list, recv_idx);
+
+   mydof_list.Append(recv_idx);
+   mydof_list.Sort();
+   mydof_list.Unique();
+
+   return mydof_list;
+}
+
+HypreParMatrix * ElementTdofs::GetProlongationMatrix(int element_attribute)
+{
+   Array<int> tdofs = GetTrueDOFs(element_attribute);
+   int h = tdofs.Size();
+   SparseMatrix St(h,pfes->GlobalTrueVSize());
+
+   for (int i = 0; i<h; i++)
+   {
+      int col = tdofs[i];
+      St.Set(i,col,1.0);
+   }
+   St.Finalize();
+   int rows[2];
+   int cols[2];
+   int nrows = St.Height();
+
+   int row_offset;
+   MPI_Scan(&nrows,&row_offset,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
+
+   row_offset-=nrows;
+   rows[0] = row_offset;
+   rows[1] = row_offset+nrows;
+   for (int i = 0; i < 2; i++)
+   {
+      cols[i] = pfes->GetTrueDofOffsets()[i];
+   }
+   int glob_nrows;
+   int glob_ncols = pfes->GlobalTrueVSize();
+   MPI_Allreduce(&nrows, &glob_nrows,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
+
+   HypreParMatrix * Pt = new HypreParMatrix(MPI_COMM_WORLD, nrows, glob_nrows,
+                                            glob_ncols, St.GetI(), St.GetJ(),
+                                            St.GetData(), rows,cols);
+   HypreParMatrix * P = Pt->Transpose();
+   return P;
+}
+
+
+SolverWithFiltering::SolverWithFiltering(MPI_Comm comm_) : Solver()
+{
+   Init(comm_);
+}
+
+void SolverWithFiltering::Init(MPI_Comm comm_)
+{
+   comm=comm_;
+   MPI_Comm_size(comm, &numProcs);
+   MPI_Comm_rank(comm, &myid);
+}
+
+void SolverWithFiltering::SetOperator(const Operator & Op_)
+{
+   Op = &Op_;
+   height = Op->Height();
+   width = Op->Width();
+}
+
+void SolverWithFiltering::SetSubspaceProlongationMap(const Operator & P_)
+{
+   P = &P_;
+   MFEM_VERIFY(P->Height() == Op->Height(),
+               "Prolongation operator height does not match the operator.");
+}
+
+void SolverWithFiltering::SetSolver(const Solver * solver_)
+{
+   solver = solver_;
+   MFEM_VERIFY(solver->Height() == Op->Height(),
+               "Solver height does not match the operator.");
+}
+
+void SolverWithFiltering::SetFilterSolver(const Solver * filter_solver_)
+{
+   filter_solver = filter_solver_;
+   MFEM_VERIFY(filter_solver->Height() == P->Width(),
+               "Filter solver height does not match the subspace dimension.");
+}
+
+void SolverWithFiltering::Mult(const Vector & b, Vector & x) const
+{
+   MFEM_VERIFY(b.Size() == x.Size(), "Inconsistent x and y size");
+   x = 0.0;
+   Vector z(x);
+   solver->Mult(b, z);
+   // 1. Full space correction
+   x+=z;
+   Vector rf(P->Width());
+   Vector xf(P->Width());
+   Vector r(b.Size());
+   // 2. Compute Residual r = b - A x
+   Op->Mult(x,r);
+   r.Neg(); r+=b;
+   // 3. Restrict to subspace
+   P->MultTranspose(r,rf);
+   // 4. Solve on the subspace
+   filter_solver->Mult(rf,xf);
+   // 5. Transfer to fine space
+   P->Mult(xf,z);
+   // 6. Update Correction
+   x+=z;
+   // 7. Compute Residual r = b - A x
+   Op->Mult(x,r);
+   r.Neg(); r+=b;
+   solver->Mult(r, z);
+   // 8. Full space correction
+   x+= z;
+}

miniapps/dpg/util/utils.hpp

@@ -0,0 +1,68 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "mfem.hpp"
+#include <ctime>
+#include <string>
+#include <sstream>
+using namespace mfem;
+
+std::string GetTimestamp();
+void WriteParametersToFile(const mfem::OptionsParser& args,
+                           const std::string& output_dir);
+void CreateParaViewPath(const char* mesh_file, std::string& output_dir);
+
+std::string GetFilename(const std::string& filePath);
+
+class ElementTdofs
+{
+private:
+   const ParFiniteElementSpace* pfes;
+   std::vector<int> tdof_offsets;
+   MPI_Comm comm;
+   int num_procs;
+   bool boundary = false;
+   // Compute offsets for True DOFs across all processors
+   void ComputeTdofOffsets();
+   // Determine which rank owns a given True DOF (tdof)
+   int GetRank(int tdof);
+   // Distribute indices using MPI_Alltoallv
+   void DistributeIndices(Array<int>& indices, Array<int>& processors,
+                          Array<int>& recv_idx);
+public:
+   ElementTdofs(const ParFiniteElementSpace* pfes_);
+   void EnableBoundary() {boundary = true;}
+
+
+   // Extract and distribute True DOFs based on the element attribute
+   Array<int> GetTrueDOFs(int element_attribute);
+   HypreParMatrix * GetProlongationMatrix(int element_attribute);
+};
+
+
+class SolverWithFiltering : public Solver
+{
+private:
+   MPI_Comm comm;
+   int numProcs, myid;
+   const Operator * Op;
+   const Operator * P;
+   const Solver * solver;
+   const Solver * filter_solver;
+   void Init(MPI_Comm comm_);
+public:
+   SolverWithFiltering(MPI_Comm comm_);
+   void SetOperator(const Operator & Op_);
+   void SetSubspaceProlongationMap(const Operator & P_);
+   void SetSolver(const Solver * solver_);
+   void SetFilterSolver(const Solver * filter_solver_);
+   virtual void Mult(const Vector & b, Vector & x) const;
+};

miniapps/dpg/util/weakform.cpp

@@ -752,7 +752,8 @@ Vector & DPGWeakForm::ComputeResidual(const BlockVector & x)
          {
             for (int ie = 0; ie<faces.Size(); ie++)
             {
-               trial_offs[j+1] += trial_fes[j]->GetFaceElement(faces[ie])->GetDof();
+               trial_offs[j+1] += trial_fes[j]->GetVDim()*trial_fes[j]->GetFaceElement(
+                                     faces[ie])->GetDof();
             }
          }
          else

miniapps/dpg/util/weakform.hpp

@@ -290,6 +290,23 @@ public:
    /// Compute DPG residual based error estimator
    Vector & ComputeResidual(const BlockVector & x);
 
+   void GetTraceFESpaces(Array<FiniteElementSpace *> & trace_fes) const
+   {
+      trace_fes.SetSize(0);
+      Array<FiniteElementSpace *> trace_fes_all;
+      if (static_cond)
+      {
+         static_cond->GetTraceFESpaces(trace_fes_all);
+         for (int i = 0; i < trace_fes_all.Size(); i++)
+         {
+            if (trace_fes_all[i])
+            {
+               trace_fes.Append(trace_fes_all[i]);
+            }
+         }
+      }
+   }
+
    virtual ~DPGWeakForm();
 
 };

tests/unit/fem/test_transfer.cpp

@@ -9,8 +9,8 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "unit_tests.hpp"
 #include "mfem.hpp"
+#include "unit_tests.hpp"
 
 using namespace mfem;
 
@@ -75,14 +75,15 @@ void vectorcoeff(const Vector& x, Vector& y)
    }
 }
 
-enum class VecSpace { H1, VectorH1, ND, RT };
+enum class VecSpace { H1, VectorH1nodes, VectorH1vdim, ND, RT };
 
 std::string VecSpaceName(VecSpace vectorspace)
 {
    switch (vectorspace)
    {
       case VecSpace::H1: return "H1";
-      case VecSpace::VectorH1: return "Vector H1";
+      case VecSpace::VectorH1nodes: return "Vector H1 by nodes";
+      case VecSpace::VectorH1vdim: return "Vector H1 by vdim";
       case VecSpace::ND: return "Nedelec";
       case VecSpace::RT: return "Raviart-Thomas";
    }
@@ -91,8 +92,8 @@ std::string VecSpaceName(VecSpace vectorspace)
 
 TEST_CASE("Transfer", "[Transfer]")
 {
-   auto vectorspace = GENERATE(VecSpace::H1, VecSpace::VectorH1, VecSpace::ND,
-                               VecSpace::RT);
+   auto vectorspace = GENERATE(VecSpace::H1, VecSpace::VectorH1nodes,
+                               VecSpace::VectorH1vdim, VecSpace::ND, VecSpace::RT);
    auto geometric = GENERATE(true, false);
    auto simplex = GENERATE(true, false);
    dimension = GENERATE(2, 3);
@@ -124,7 +125,8 @@ TEST_CASE("Transfer", "[Transfer]")
    switch (vectorspace)
    {
       case VecSpace::H1:
-      case VecSpace::VectorH1:
+      case VecSpace::VectorH1nodes:
+      case VecSpace::VectorH1vdim:
          c_fec = new H1_FECollection(order, dimension);
          f_fec = geometric ? c_fec : new H1_FECollection(fineOrder, dimension);
          break;
@@ -144,12 +146,14 @@ TEST_CASE("Transfer", "[Transfer]")
       fineMesh.UniformRefinement();
    }
 
-   const int vdim = (vectorspace == VecSpace::VectorH1) ? dimension : 1;
-
+   const int vdim = (vectorspace == VecSpace::VectorH1nodes
+                     || vectorspace == VecSpace::VectorH1vdim) ? dimension : 1;
+   Ordering::Type ordering = (vectorspace == VecSpace::VectorH1vdim)
+                             ? Ordering::byVDIM : Ordering::byNODES;
    FiniteElementSpace *c_fespace =
-      new FiniteElementSpace(&mesh, c_fec, vdim);
+      new FiniteElementSpace(&mesh, c_fec, vdim, ordering);
    FiniteElementSpace *f_fespace =
-      new FiniteElementSpace(&fineMesh, f_fec, vdim);
+      new FiniteElementSpace(&fineMesh, f_fec, vdim, ordering);
 
    Operator* referenceOperator = nullptr;
 
@@ -217,7 +221,8 @@ TEST_CASE("Transfer", "[Transfer]")
 
 TEST_CASE("Variable Order Transfer", "[Transfer][VariableOrder]")
 {
-   auto vectorspace = GENERATE(VecSpace::H1, VecSpace::VectorH1, VecSpace::ND,
+   auto vectorspace = GENERATE(VecSpace::H1, VecSpace::VectorH1nodes,
+                               VecSpace::VectorH1vdim, VecSpace::ND,
                                VecSpace::RT);
    dimension = GENERATE(2, 3);
 
@@ -244,7 +249,8 @@ TEST_CASE("Variable Order Transfer", "[Transfer][VariableOrder]")
    switch (vectorspace)
    {
       case VecSpace::H1:
-      case VecSpace::VectorH1:
+      case VecSpace::VectorH1nodes:
+      case VecSpace::VectorH1vdim:
          c_fec = new H1_FECollection(order, dimension);
          f_fec = new H1_FECollection(order, dimension);
          break;
@@ -261,12 +267,15 @@ TEST_CASE("Variable Order Transfer", "[Transfer][VariableOrder]")
    mesh.EnsureNCMesh();
    mesh.RandomRefinement(0.5);
 
-   const int vdim = (vectorspace == VecSpace::VectorH1) ? dimension : 1;
+   const int vdim = (vectorspace == VecSpace::VectorH1nodes
+                     || vectorspace == VecSpace::VectorH1vdim) ? dimension : 1;
+   Ordering::Type ordering = (vectorspace == VecSpace::VectorH1vdim)
+                             ? Ordering::byVDIM : Ordering::byNODES;
 
    FiniteElementSpace *c_fespace =
-      new FiniteElementSpace(&mesh, c_fec, vdim);
+      new FiniteElementSpace(&mesh, c_fec, vdim, ordering);
    FiniteElementSpace *f_fespace =
-      new FiniteElementSpace(&mesh, f_fec, vdim);
+      new FiniteElementSpace(&mesh, f_fec, vdim, ordering);
 
    RandomPRefinement(*f_fespace);
 
@@ -322,7 +331,8 @@ TEST_CASE("Variable Order Transfer", "[Transfer][VariableOrder]")
 
 TEST_CASE("Variable Order True Transfer", "[Transfer][VariableOrder]")
 {
-   auto vectorspace = GENERATE(VecSpace::H1, VecSpace::VectorH1);
+   auto vectorspace = GENERATE(VecSpace::H1, VecSpace::VectorH1nodes,
+                               VecSpace::VectorH1vdim);
    dimension = GENERATE(2, 3);
 
    int ne = 2;
@@ -348,12 +358,15 @@ TEST_CASE("Variable Order True Transfer", "[Transfer][VariableOrder]")
    f_fec = new H1_FECollection(order, dimension);
    mesh.EnsureNCMesh();
    mesh.RandomRefinement(0.5);
-   const int vdim = (vectorspace == VecSpace::VectorH1) ? dimension : 1;
+   const int vdim = (vectorspace == VecSpace::VectorH1nodes
+                     || vectorspace == VecSpace::VectorH1vdim) ? dimension : 1;
+   Ordering::Type ordering = (vectorspace == VecSpace::VectorH1vdim)
+                             ? Ordering::byVDIM : Ordering::byNODES;
 
    FiniteElementSpace *c_fespace =
-      new FiniteElementSpace(&mesh, c_fec, vdim);
+      new FiniteElementSpace(&mesh, c_fec, vdim, ordering);
    FiniteElementSpace *f_fespace =
-      new FiniteElementSpace(&mesh, f_fec, vdim);
+      new FiniteElementSpace(&mesh, f_fec, vdim, ordering);
 
    RandomPRefinement(*f_fespace);
 
@@ -460,6 +473,183 @@ TEST_CASE("Restriction Transpose Operator")
    REQUIRE(y3.Normlinf() == MFEM_Approx(0.0));
 }
 
+
+real_t sin_func(const Vector &x)
+{
+   real_t val = sin(M_PI * x.Sum());
+   return val;
+}
+
+void sin_vfunc(const Vector &x, Vector &y)
+{
+   y.SetSize(x.Size());
+   for (int i = 0; i < y.Size(); i++)
+   {
+      y(i) = sin(M_PI * x[i]);
+   }
+}
+
+
+TEST_CASE("Trace PRefinement Serial TrueTransfer", "[Transfer]")
+{
+   auto simplex = GENERATE(true, false);
+   dimension = GENERATE(2, 3);
+   int ne = 4;
+   auto order = GENERATE(1,2,3);
+   auto vectorspace = GENERATE(VecSpace::H1, VecSpace::VectorH1nodes,
+                               VecSpace::VectorH1vdim,
+                               VecSpace::ND,VecSpace::RT);
+   auto assembleP = GENERATE(false, true);
+   auto amr = GENERATE(false, true);
+
+   // Log test case information
+   int total_ne = static_cast<int>(std::pow(ne, dimension));
+   CAPTURE(VecSpaceName(vectorspace),dimension, simplex, total_ne, order,
+           assembleP);
+
+   Mesh mesh;
+   if (dimension == 2)
+   {
+      Element::Type type = simplex ? Element::TRIANGLE : Element::QUADRILATERAL;
+      mesh = Mesh::MakeCartesian2D(ne, ne, type, 1, 1.0, 1.0);
+   }
+   else
+   {
+      Element::Type type = simplex ? Element::TETRAHEDRON : Element::HEXAHEDRON;
+      mesh = Mesh::MakeCartesian3D(ne, ne, ne, type, 1.0, 1.0, 1.0);
+   }
+
+   if (amr) { mesh.RandomRefinement(0.5); }
+
+   FiniteElementCollection *c_fec = nullptr;
+   FiniteElementCollection *f_fec = nullptr;
+
+   FiniteElementCollection *c_trace_fec = nullptr;
+   FiniteElementCollection *f_trace_fec = nullptr;
+
+   switch (vectorspace)
+   {
+      case VecSpace::H1:
+      case VecSpace::VectorH1nodes:
+      case VecSpace::VectorH1vdim:
+         c_fec = new H1_FECollection(order, dimension);
+         f_fec = new H1_FECollection(order+1, dimension);
+         c_trace_fec = new H1_Trace_FECollection(order, dimension);
+         f_trace_fec = new H1_Trace_FECollection(order+1, dimension);
+         break;
+      case VecSpace::ND:
+         c_fec = new ND_FECollection(order, dimension);
+         f_fec = new ND_FECollection(order+1, dimension);
+         c_trace_fec = new ND_Trace_FECollection(order, dimension);
+         f_trace_fec = new ND_Trace_FECollection(order+1, dimension);
+         break;
+      case VecSpace::RT:
+         c_fec = new RT_FECollection(order-1, dimension);
+         f_fec = new RT_FECollection(order, dimension);
+         c_trace_fec = new RT_Trace_FECollection(order-1, dimension);
+         f_trace_fec = new RT_Trace_FECollection(order, dimension);
+         break;
+   }
+
+   const int vdim = (vectorspace == VecSpace::VectorH1nodes
+                     || vectorspace == VecSpace::VectorH1vdim) ? dimension : 1;
+   Ordering::Type ordering = (vectorspace == VecSpace::VectorH1vdim)
+                             ? Ordering::byVDIM : Ordering::byNODES;
+
+   FiniteElementSpace c_fes(&mesh, c_fec, vdim, ordering);
+   FiniteElementSpace f_fes(&mesh, f_fec, vdim, ordering);
+
+   FiniteElementSpace c_trace_fes(&mesh, c_trace_fec, vdim, ordering);
+   FiniteElementSpace f_trace_fes(&mesh, f_trace_fec, vdim, ordering);
+
+   GridFunction x_c(&c_fes); x_c = 0.0;
+   GridFunction x_f(&f_fes); x_f = 0.0;
+   GridFunction x_trace_c(&c_trace_fes); x_trace_c = 0.0;
+   GridFunction x_trace_f(&f_trace_fes); x_trace_f = 0.0;
+
+   if (vectorspace == VecSpace::H1)
+   {
+      FunctionCoefficient cf(sin_func);
+      x_c.ProjectCoefficient(cf);
+      x_trace_c.ProjectTraceCoefficient(cf);
+   }
+   else
+   {
+      VectorFunctionCoefficient vec_cf(dimension, &sin_vfunc);
+      x_c.ProjectCoefficient(vec_cf);
+      switch (vectorspace)
+      {
+         case VecSpace::VectorH1nodes:
+         case VecSpace::VectorH1vdim:
+            x_trace_c.ProjectTraceCoefficient(vec_cf);
+            break;
+         case VecSpace::ND:
+            x_trace_c.ProjectTraceCoefficientTangent(vec_cf);
+            break;
+         case VecSpace::RT:
+            x_trace_c.ProjectTraceCoefficientNormal(vec_cf);
+            break;
+         default:
+            break;
+      }
+   }
+
+   // Generate transfer operators for field and trace spaces
+   PRefinementTransferOperator P(c_fes, f_fes, assembleP);
+   PRefinementTransferOperator P_trace(c_trace_fes, f_trace_fes, assembleP);
+
+   Vector x_c_true(c_fes.GetTrueVSize());
+   Vector x_f_true(f_fes.GetTrueVSize());
+   Vector x_trace_c_true(c_trace_fes.GetTrueVSize());
+   Vector x_trace_f_true(f_trace_fes.GetTrueVSize());
+   x_c.GetTrueDofs(x_c_true);
+   x_trace_c.GetTrueDofs(x_trace_c_true);
+   P.GetTrueTransferOperator()->Mult(x_c_true, x_f_true);
+   P_trace.GetTrueTransferOperator()->Mult(x_trace_c_true, x_trace_f_true);
+
+   x_f.SetFromTrueDofs(x_f_true);
+   x_trace_f.SetFromTrueDofs(x_trace_f_true);
+
+   // zero out interior dofs before and after p-ref to compare with trace
+   Array<int> vdofs;
+   for (int i = 0; i<mesh.GetNE(); i++)
+   {
+      c_fes.GetElementInteriorVDofs(i, vdofs);
+      x_c.SetSubVector(vdofs, 0.0);
+      f_fes.GetElementInteriorVDofs(i, vdofs);
+      x_f.SetSubVector(vdofs, 0.0);
+   }
+
+   // Embed the trace dofs to a field GridFunction for comparison
+   Array<int> face_vdofs, trace_vdofs;
+   Vector values;
+   GridFunction x_embedded_trace_c(&c_fes); x_embedded_trace_c = 0.0;
+   GridFunction x_embedded_trace_f(&f_fes); x_embedded_trace_f = 0.0;
+   for (int i = 0; i<mesh.GetNumFaces(); i++)
+   {
+      c_trace_fes.GetFaceVDofs(i, trace_vdofs);
+      x_trace_c.GetSubVector(trace_vdofs, values);
+      c_fes.GetFaceVDofs(i, face_vdofs);
+      x_embedded_trace_c.SetSubVector(face_vdofs, values);
+
+      f_trace_fes.GetFaceVDofs(i, trace_vdofs);
+      x_trace_f.GetSubVector(trace_vdofs, values);
+      f_fes.GetFaceVDofs(i, face_vdofs);
+      x_embedded_trace_f.SetSubVector(face_vdofs, values);
+   }
+
+   x_embedded_trace_c -= x_c;
+   REQUIRE(x_embedded_trace_c.Norml2() == MFEM_Approx(0.0));
+   x_embedded_trace_f -= x_f;
+   REQUIRE(x_embedded_trace_f.Norml2() == MFEM_Approx(0.0));
+
+   delete f_trace_fec;
+   delete c_trace_fec;
+   delete f_fec;
+   delete c_fec;
+}
+
+
 #ifdef MFEM_USE_MPI
 
 TEST_CASE("Parallel Transfer", "[Transfer][Parallel]")
@@ -584,4 +774,171 @@ TEST_CASE("Parallel Transfer", "[Transfer][Parallel]")
    delete pmesh;
 }
 
+TEST_CASE("Trace PRefinement Parallel TrueTransfer", "[Transfer][Parallel]")
+{
+   auto simplex = GENERATE(true, false);
+   dimension = GENERATE(2, 3);
+   int ne = 4;
+   auto order = GENERATE(1,2,3);
+   auto vectorspace = GENERATE(VecSpace::H1, VecSpace::VectorH1nodes,
+                               VecSpace::VectorH1vdim,
+                               VecSpace::ND,VecSpace::RT);
+   auto assembleP = GENERATE(true, false);
+
+   auto amr = GENERATE(true, false);
+
+   // Log test case information
+   int total_ne = static_cast<int>(std::pow(ne, dimension));
+   CAPTURE(VecSpaceName(vectorspace),dimension, simplex, total_ne, order,
+           assembleP);
+
+   Mesh mesh;
+   if (dimension == 2)
+   {
+      Element::Type type = simplex ? Element::TRIANGLE : Element::QUADRILATERAL;
+      mesh = Mesh::MakeCartesian2D(ne, ne, type, 1, 1.0, 1.0);
+   }
+   else
+   {
+      Element::Type type = simplex ? Element::TETRAHEDRON : Element::HEXAHEDRON;
+      mesh = Mesh::MakeCartesian3D(ne, ne, ne, type, 1.0, 1.0, 1.0);
+   }
+
+   if (amr) { mesh.EnsureNCMesh(true); }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+
+   if (amr) { pmesh.RandomRefinement(0.5); }
+
+   FiniteElementCollection *c_fec = nullptr;
+   FiniteElementCollection *f_fec = nullptr;
+
+   FiniteElementCollection *c_trace_fec = nullptr;
+   FiniteElementCollection *f_trace_fec = nullptr;
+
+   switch (vectorspace)
+   {
+      case VecSpace::H1:
+      case VecSpace::VectorH1nodes:
+      case VecSpace::VectorH1vdim:
+         c_fec = new H1_FECollection(order, dimension);
+         f_fec = new H1_FECollection(order+1, dimension);
+         c_trace_fec = new H1_Trace_FECollection(order, dimension);
+         f_trace_fec = new H1_Trace_FECollection(order+1, dimension);
+         break;
+      case VecSpace::ND:
+         c_fec = new ND_FECollection(order, dimension);
+         f_fec = new ND_FECollection(order+1, dimension);
+         c_trace_fec = new ND_Trace_FECollection(order, dimension);
+         f_trace_fec = new ND_Trace_FECollection(order+1, dimension);
+         break;
+      case VecSpace::RT:
+         c_fec = new RT_FECollection(order-1, dimension);
+         f_fec = new RT_FECollection(order, dimension);
+         c_trace_fec = new RT_Trace_FECollection(order-1, dimension);
+         f_trace_fec = new RT_Trace_FECollection(order, dimension);
+         break;
+   }
+
+   const int vdim = (vectorspace == VecSpace::VectorH1nodes
+                     || vectorspace == VecSpace::VectorH1vdim) ? dimension : 1;
+   Ordering::Type ordering = (vectorspace == VecSpace::VectorH1vdim)
+                             ? Ordering::byVDIM : Ordering::byNODES;
+
+   ParFiniteElementSpace c_fes(&pmesh, c_fec, vdim, ordering);
+   ParFiniteElementSpace f_fes(&pmesh, f_fec, vdim, ordering);
+
+   ParFiniteElementSpace c_trace_fes(&pmesh, c_trace_fec, vdim, ordering);
+   ParFiniteElementSpace f_trace_fes(&pmesh, f_trace_fec, vdim, ordering);
+
+   ParGridFunction x_c(&c_fes); x_c = 0.0;
+   ParGridFunction x_f(&f_fes); x_f = 0.0;
+   ParGridFunction x_trace_c(&c_trace_fes); x_trace_c = 0.0;
+   ParGridFunction x_trace_f(&f_trace_fes); x_trace_f = 0.0;
+
+   if (vectorspace == VecSpace::H1)
+   {
+      FunctionCoefficient cf(sin_func);
+      x_c.ProjectCoefficient(cf);
+      x_trace_c.ProjectTraceCoefficient(cf);
+   }
+   else
+   {
+      VectorFunctionCoefficient vec_cf(dimension, &vectorcoeff);
+      x_c.ProjectCoefficient(vec_cf);
+      switch (vectorspace)
+      {
+         case VecSpace::VectorH1nodes:
+         case VecSpace::VectorH1vdim:
+            x_trace_c.ProjectTraceCoefficient(vec_cf);
+            break;
+         case VecSpace::ND:
+            x_trace_c.ProjectTraceCoefficientTangent(vec_cf);
+            break;
+         case VecSpace::RT:
+            x_trace_c.ProjectTraceCoefficientNormal(vec_cf);
+            break;
+         default:
+            break;
+      }
+   }
+
+   // Generate transfer operators for field and trace spaces
+   PRefinementTransferOperator P(c_fes, f_fes, assembleP);
+   PRefinementTransferOperator P_trace(c_trace_fes, f_trace_fes, assembleP);
+
+   // Apply transfer operators
+   Vector x_c_true(c_fes.GetTrueVSize());
+   Vector x_f_true(f_fes.GetTrueVSize());
+   Vector x_trace_c_true(c_trace_fes.GetTrueVSize());
+   Vector x_trace_f_true(f_trace_fes.GetTrueVSize());
+   x_c.GetTrueDofs(x_c_true);
+   x_trace_c.GetTrueDofs(x_trace_c_true);
+   P.GetTrueTransferOperator()->Mult(x_c_true, x_f_true);
+   P_trace.GetTrueTransferOperator()->Mult(x_trace_c_true, x_trace_f_true);
+
+   x_f.SetFromTrueDofs(x_f_true);
+   x_trace_f.SetFromTrueDofs(x_trace_f_true);
+
+   // zero out interior dofs before and after p-ref to compare with trace
+   Array<int> vdofs;
+   for (int i = 0; i<pmesh.GetNE(); i++)
+   {
+      c_fes.GetElementInteriorVDofs(i, vdofs);
+      x_c.SetSubVector(vdofs, 0.0);
+      f_fes.GetElementInteriorVDofs(i, vdofs);
+      x_f.SetSubVector(vdofs, 0.0);
+   }
+
+   // Embed the trace dofs to a field GridFunction for comparison
+   Array<int> face_vdofs, trace_vdofs;
+   Vector values;
+   ParGridFunction x_embedded_trace_c(&c_fes); x_embedded_trace_c = 0.0;
+   ParGridFunction x_embedded_trace_f(&f_fes); x_embedded_trace_f = 0.0;
+   for (int i = 0; i<pmesh.GetNumFaces(); i++)
+   {
+      c_trace_fes.GetFaceVDofs(i, trace_vdofs);
+      x_trace_c.GetSubVector(trace_vdofs, values);
+      c_fes.GetFaceVDofs(i, face_vdofs);
+      x_embedded_trace_c.SetSubVector(face_vdofs, values);
+
+      f_trace_fes.GetFaceVDofs(i, trace_vdofs);
+      x_trace_f.GetSubVector(trace_vdofs, values);
+      f_fes.GetFaceVDofs(i, face_vdofs);
+      x_embedded_trace_f.SetSubVector(face_vdofs, values);
+   }
+
+   x_embedded_trace_c -= x_c;
+   REQUIRE(x_embedded_trace_c.Norml2() == MFEM_Approx(0.0));
+   x_embedded_trace_f -= x_f;
+   REQUIRE(x_embedded_trace_f.Norml2() == MFEM_Approx(0.0));
+
+   delete f_trace_fec;
+   delete c_trace_fec;
+   delete f_fec;
+   delete c_fec;
+}
+
+
+
 #endif

tests/unit/linalg/test_direct_solvers.cpp

@@ -188,8 +188,6 @@ TEST_CASE("Serial Direct Solvers", "[GPU]")
 
 TEST_CASE("Parallel Direct Solvers", "[Parallel], [GPU]")
 {
-   int rank;
-   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    const int ne = 4;
    for (int dim = 1; dim < 4; ++dim)
    {
@@ -357,3 +355,104 @@ TEST_CASE("Parallel Direct Solvers", "[Parallel], [GPU]")
 }
 
 #endif
+
+
+#ifdef MFEM_USE_COMPLEX_MUMPS
+
+TEST_CASE("ComplexMUMPS Solver", "[Parallel], [GPU]")
+{
+   const int ne = 4;
+   for (int dim = 1; dim < 4; ++dim)
+   {
+      CAPTURE(dim);
+
+      Mesh mesh;
+      if (dim == 1)
+      {
+         mesh = Mesh::MakeCartesian1D(ne, 1.0);
+      }
+      else if (dim == 2)
+      {
+         mesh = Mesh::MakeCartesian2D(
+                   ne, ne, Element::QUADRILATERAL, 1, 1.0, 1.0);
+      }
+      else
+      {
+         mesh = Mesh::MakeCartesian3D(
+                   ne, ne, ne, Element::HEXAHEDRON, 1.0, 1.0, 1.0);
+      }
+
+      ParMesh pmesh(MPI_COMM_WORLD, mesh);
+      mesh.Clear();
+      int order = 3;
+      H1_FECollection fec(order, dim);
+      ParFiniteElementSpace fespace(&pmesh, &fec);
+      Array<int> ess_tdof_list, ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      ConstantCoefficient one(1.0);
+      ConstantCoefficient negone(-1.0);
+      ConstantCoefficient two(2.0);
+
+      ComplexLinearForm b(&fespace);
+      b.AddDomainIntegrator(new DomainLFIntegrator(one), new DomainLFIntegrator(two));
+      b.Assemble();
+
+      ParSesquilinearForm a_r(&fespace);
+      a_r.AddDomainIntegrator(new DiffusionIntegrator(one), nullptr);
+      a_r.AddDomainIntegrator(new MassIntegrator(one), nullptr);
+      a_r.Assemble();
+
+      ParSesquilinearForm a_i(&fespace);
+      a_i.AddDomainIntegrator(nullptr, new DiffusionIntegrator(one));
+      a_i.AddDomainIntegrator(nullptr, new MassIntegrator(one));
+      a_i.Assemble();
+
+      ParSesquilinearForm a_c(&fespace);
+      a_c.AddDomainIntegrator(new DiffusionIntegrator(one), new MassIntegrator(two));
+      a_c.AddDomainIntegrator(new MassIntegrator(negone),nullptr);
+      a_c.Assemble();
+
+      ParComplexGridFunction x_c(&fespace);
+      ParComplexGridFunction x_r(&fespace);
+      ParComplexGridFunction x_i(&fespace);
+      x_c = 0.0; x_r = 0.0; x_i = 0.0;
+
+      OperatorPtr Ac, Ar, Ai;
+      Vector Bc, Br, Bi, Xr, Xc, Xi;
+      a_c.FormLinearSystem(ess_tdof_list, x_c, b, Ac, Xc, Bc);
+      a_r.FormLinearSystem(ess_tdof_list, x_r, b, Ar, Xr, Br);
+      a_i.FormLinearSystem(ess_tdof_list, x_i, b, Ai, Xi, Bi);
+
+      ComplexHypreParMatrix *Ahc = Ac.As<ComplexHypreParMatrix>();
+      ComplexHypreParMatrix *Ahr = Ar.As<ComplexHypreParMatrix>();
+      ComplexHypreParMatrix *Ahi = Ai.As<ComplexHypreParMatrix>();
+
+      ComplexMUMPSSolver cmumps(MPI_COMM_WORLD);
+      cmumps.SetPrintLevel(0);
+
+      cmumps.SetOperator(*Ahc);
+      cmumps.Mult(Bc, Xc);
+
+      cmumps.SetOperator(*Ahr);
+      cmumps.Mult(Br, Xr);
+
+      cmumps.SetOperator(*Ahi);
+      cmumps.Mult(Bi, Xi);
+
+      Vector Yc(Xc.Size()), Yr(Xr.Size()), Yi(Xi.Size());
+      Ahc->Mult(Xc, Yc); Ahr->Mult(Xr, Yr); Ahi->Mult(Xi, Yi);
+      Yc -= Bc; Yr -= Br; Yi -= Bi;
+      REQUIRE(Yc.Norml2() < 1.e-12);
+      REQUIRE(Yr.Norml2() < 1.e-12);
+      REQUIRE(Yi.Norml2() < 1.e-12);
+   }
+
+} // Test case "ComplexMUMPS Solver"
+
+#endif // MFEM_USE_COMPLEX_MUMPS