starting ST with artificial absorption

examples/ex25.cpp

@@ -154,6 +154,10 @@ prob_type prob;
 
 int main(int argc, char *argv[])
 {
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_SELF, &num_procs);
+   MPI_Comm_rank(MPI_COMM_SELF, &myid);
    // 1. Parse command-line options.
    const char *mesh_file = nullptr;
    int order = 1;
@@ -423,10 +427,34 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_SUITESPARSE
    if (!pa && umf_solver)
    {
+      chrono.Clear();
+      chrono.Start();
       ComplexUMFPackSolver csolver(*A.As<ComplexSparseMatrix>());
       csolver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
       csolver.SetPrintLevel(1);
       csolver.Mult(B, X);
+      chrono.Stop();
+      cout << "UMFPACK for ComplexSparseMatrix = " << chrono.RealTime() << endl;
+   }
+   {
+      chrono.Clear();
+      chrono.Start();
+      HYPRE_Int rowstarts[2]; rowstarts[0] = 0;
+      rowstarts[1] = fespace->GetTrueVSize();
+      HypreParMatrix * HypreMat_r =
+         new HypreParMatrix(MPI_COMM_SELF,rowstarts[1],rowstarts,
+                            &(*A.As<ComplexSparseMatrix>()).real());
+      HypreParMatrix * HypreMat_i =
+         new HypreParMatrix(MPI_COMM_SELF,rowstarts[1],rowstarts,
+                            &(*A.As<ComplexSparseMatrix>()).imag());
+      ComplexHypreParMatrix * HypreMat =
+         new ComplexHypreParMatrix(HypreMat_r,HypreMat_i,true,true);
+      ComplexMUMPSSolver csolver;
+      csolver.SetOperator(*HypreMat);
+      csolver.Mult(B, X);
+      delete HypreMat;
+      chrono.Stop();
+      cout << "MUMPS  for ComplexSparseMatrix = " << chrono.RealTime() << endl;
    }
 #endif
    // 14a. Set up the Bilinear form a(.,.) for the preconditioner
@@ -614,6 +642,7 @@ int main(int argc, char *argv[])
    delete fespace;
    delete fec;
    delete mesh;
+   MPI_Finalize();
    return 0;
 }
 

examples/ex25p.cpp

@@ -266,6 +266,10 @@ int main(int argc, char *argv[])
    }
 
    Mesh * mesh = new Mesh(mesh_file, 1, 1);
+
+   // Mesh * mesh = new Mesh(1,1,16,Element::HEXAHEDRON,true,1.0,1.0,16.0);
+
+
    dim = mesh->Dimension();
 
    // Angular frequency
@@ -278,7 +282,7 @@ int main(int argc, char *argv[])
    switch (prob)
    {
       case disc:
-         length = 0.2;
+         length = 0.3;
          break;
       case lshape:
          length(0, 0) = 0.1;
@@ -291,6 +295,7 @@ int main(int argc, char *argv[])
          break;
       case beam:
          length(0, 1) = 2.0;
+         // length(2, 1) = 2.0;
          break;
       default:
          length = 0.25;
@@ -463,8 +468,11 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_SUPERLU
    if (!pa && slu_solver)
    {
+      StopWatch chrono;
       // Transform to monolithic HypreParMatrix
       HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+      chrono.Clear();
+      chrono.Start();
       SuperLURowLocMatrix SA(*A);
       SuperLUSolver superlu(MPI_COMM_WORLD);
       superlu.SetPrintStatistics(false);
@@ -473,19 +481,50 @@ int main(int argc, char *argv[])
       superlu.SetOperator(SA);
       superlu.Mult(B, X);
       delete A;
+      chrono.Stop();
+      if (myid == 0)
+      {
+         cout << "Superlu for monolithic HyperMat = " << chrono.RealTime() << endl;
+      }
    }
 #endif
 #ifdef MFEM_USE_MUMPS
    if (!pa && mumps_solver)
    {
+      StopWatch chrono;
       HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+      chrono.Clear();
+      chrono.Start();
       MUMPSSolver mumps;
       mumps.SetPrintLevel(0);
       mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
       mumps.SetOperator(*A);
       mumps.Mult(B,X);
+      chrono.Stop();
+      if (myid == 0)
+      {
+         cout << "MUMPS for monolithic HyperMat = " << chrono.RealTime() << endl;
+      }
       delete A;
    }
+   if (mumps_solver)
+   {
+      StopWatch chrono;
+      chrono.Clear();
+      chrono.Start();
+      ComplexMUMPSSolver cmumps;
+      cmumps.SetPrintLevel(0);
+      cmumps.SetOperator(*Ah);
+      cmumps.Mult(B,X);
+      chrono.Stop();
+      if (myid == 0)
+      {
+         cout << "MUMPS for ComplexHyperMat     = " << chrono.RealTime() << endl;
+      }
+   }
+
+
+
 #endif
    // 16a. Set up the parallel Bilinear form a(.,.) for the preconditioner
    //
@@ -801,6 +840,7 @@ void maxwell_solution(const Vector &x, vector<complex<double>> &E)
          {
             double k10 = sqrt(k * k - M_PI * M_PI);
             E[1] = -zi * k / M_PI * sin(M_PI*x(2))*exp(zi * k10 * x(0));
+            // E[1] = -zi * k / M_PI * sin(M_PI*x(0))*exp(zi * k10 * x(2));
          }
          else if (dim == 2)
          {

examples/maxwell-solver/DST/DST.cpp

@@ -0,0 +1,701 @@
+//Diagonal Source Transfer Preconditioner
+
+#include "DST.hpp"
+
+DST::DST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+         double omega_, Coefficient * ws_,  int nrlayers_ , int nx_, int ny_, int nz_)
+   : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), 
+     bf(bf_), Pmllength(Pmllength_), omega(omega_), ws(ws_), nrlayers(nrlayers_)
+{
+
+   // Indentify problem ... Helmholtz or Maxwell
+   int prob_kind = bf->FESpace()->FEColl()->GetContType();
+
+   Mesh * mesh = bf->FESpace()->GetMesh();
+   dim = mesh->Dimension();
+   int partition_kind = 2;
+   nx=nx_; ny=ny_;  nz=nz_;
+   ovlpnrlayers = nrlayers+1;
+   part = new MeshPartition(mesh, partition_kind,nx,ny,nz, ovlpnrlayers);
+   nx = part->nxyz[0]; ny = part->nxyz[1];  nz = part->nxyz[2];
+   nrpatch = part->nrpatch;
+
+   // partition_kind = 1;
+   // MeshPartition * part1 = new MeshPartition(mesh, partition_kind,nx,ny,nz);
+   // SaveMeshPartition(part1->patch_mesh, "output/mesh3x3.", "output/sol3x3.");
+   // SaveMeshPartition(part->patch_mesh, "output/mesh3x3.", "output/sol3x3.");
+   swp = new Sweep(dim);
+
+   StopWatch chrono;
+   chrono.Clear();
+   chrono.Start();
+   dmap  = new DofMap(bf->FESpace(),part);
+   chrono.Stop();
+   cout << "Computing subdomain to global maps: " 
+        << chrono.RealTime() <<" s" << endl; 
+
+
+   chrono.Clear();
+   chrono.Start();
+   NeighborMap = new NeighborDofMaps(part,bf->FESpace(),dmap,ovlpnrlayers); 
+   chrono.Stop();
+   cout << "Computing subdomain to neighbor maps: " 
+        << chrono.RealTime() <<" s" << endl; 
+
+   chrono.Clear();
+   chrono.Start();
+   MarkOverlapElements();
+   MarkOverlapDofs();
+   chrono.Stop();
+   cout << "Computing subdomain overlap dofs: " 
+        << chrono.RealTime() <<" s" << endl; 
+
+   // Set up the local patch problems
+   sqf.SetSize(nrpatch);
+   Optr.SetSize(nrpatch);
+   PmlMat.SetSize(nrpatch);
+   PmlMatInv.SetSize(nrpatch);
+   f_orig.SetSize(nrpatch);
+   f_transf.resize(nrpatch);
+   cout << "nrsubdomain = " << nrpatch << endl;
+
+   chrono.Clear();
+   chrono.Start();
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      // cout << "Setting up patch ip = " << ip << endl;
+      if (prob_kind == 0)
+      {
+         SetHelmholtzPmlSystemMatrix(ip);
+      }
+      else if (prob_kind == 1)
+      {
+         SetMaxwellPmlSystemMatrix(ip);
+      }
+      PmlMat[ip] = Optr[ip]->As<ComplexSparseMatrix>();
+
+      // cout << "Factorizing patch ip = " << ip << endl;
+
+      PmlMatInv[ip] = new ComplexUMFPackSolver;
+      PmlMatInv[ip]->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
+
+      int ndofs = dmap->Dof2GlobalDof[ip].Size();
+      f_orig[ip] = new Vector(ndofs); 
+      f_transf[ip].SetSize(swp->nsweeps);
+      for (int i=0;i<swp->nsweeps; i++)
+      {
+         f_transf[ip][i] = new Vector(ndofs);
+      }
+   }
+   chrono.Stop();
+   cout << "Computing and factoring subdomain matrices: " 
+        << chrono.RealTime() <<" s" << endl; 
+
+   zaux.SetSize(2*bf->FESpace()->GetTrueVSize());
+
+   // char vishost[] = "localhost";
+   // int visport = 19916;
+   // socketstream mesh_sock1(vishost, visport);
+   // mesh_sock1.precision(8);
+   // mesh_sock1 << "mesh\n"
+   //           << *part->patch_mesh[0] << "window_title 'Subdomain'" << flush;
+
+}
+
+void DST::Mult(const Vector &r, Vector &z) const
+{
+   char vishost[] = "localhost";
+   int visport = 19916;
+   socketstream sol_sock(vishost, visport);
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      *f_orig[ip] = 0.0;
+      for (int i=0;i<swp->nsweeps; i++)
+      {
+         *f_transf[ip][i] = 0.0;
+      }
+   }
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      Array<int> * Dof2GlobalDof = &dmap->Dof2GlobalDof[ip];
+      r.GetSubVector(*Dof2GlobalDof,*f_orig[ip]);
+
+      int i,j,k;
+      Getijk(ip,i,j,k);
+      Array<int> ijk(dim);
+      ijk[0] = i;
+      ijk[1] = j;
+      if (dim == 3) ijk[2] = k;
+      Array2D<int> direct(dim,2); direct = 0;
+      for (int d=0;d<dim; d++)
+      {
+         if (ijk[d] > 0) direct[d][0] = 1; 
+         if (ijk[d] < part->nxyz[d]-1) direct[d][1] = 1; 
+      }
+      GetChiRes(*f_orig[ip],ip,direct);
+   }
+
+   z = 0.0; 
+   int nsteps;
+   switch(dim)
+   {
+      case 1: nsteps = nx; break;
+      case 2: nsteps = nx+ny-1; break;
+      default: nsteps = nx+ny+nz-2; break;
+   }
+   int nsweeps = swp->nsweeps;
+
+   for (int l=0; l<nsweeps; l++)
+   {  
+      for (int s = 0; s<nsteps; s++)
+      {
+         Array2D<int> subdomains;
+         GetStepSubdomains(l,s,subdomains);
+
+         int nsubdomains = subdomains.NumRows();
+         for (int sb=0; sb< nsubdomains; sb++)
+         {
+            Array<int> ijk(dim);
+            for (int d=0; d<dim; d++) ijk[d] = subdomains[sb][d]; 
+            int ip = GetPatchId(ijk);
+
+            Array<int> * Dof2GlobalDof = &dmap->Dof2GlobalDof[ip];
+            int ndofs = Dof2GlobalDof->Size();
+
+            Vector sol_local(ndofs); 
+            Vector res_local(ndofs); res_local = 0.0;
+            if (l==0) res_local += *f_orig[ip];
+            res_local += *f_transf[ip][l];
+            if (res_local.Norml2() < 1e-8) continue;
+            PmlMatInv[ip]->Mult(res_local, sol_local);
+            TransferSources(l,ip, sol_local);
+            z.AddElementVector(*Dof2GlobalDof, sol_local);
+         }
+         PlotSolution(z,sol_sock,0,false);
+         cin.get();
+      }
+   }
+}
+
+
+void DST::Getijk(int ip, int & i, int & j, int & k) const
+{
+   k = ip/(nx*ny);
+   j = (ip-k*nx*ny)/nx;
+   i = (ip-k*nx*ny)%nx;
+}
+
+int DST::GetPatchId(const Array<int> & ijk) const
+{
+   int d=ijk.Size();
+   int z = (d==2)? 0 : ijk[2];
+   return part->subdomains(ijk[0],ijk[1],z);
+}
+
+
+void DST::TransferSources(int s, int ip0, Vector & sol0) const
+{
+//   Find all neighbors of patch ip0
+   int i0, j0, k0;
+   Getijk(ip0, i0,j0,k0);
+   Array<int> directions(dim);   
+   for (int i=-1; i<2; i++)
+   {
+      int i1 = i0 + i;
+      if (i1 <0 || i1>=nx) continue;
+      directions[0] = i;
+      for (int j=-1; j<2; j++)
+      {
+         int j1 = j0 + j;
+         if (j1 <0 || j1>=ny) continue;
+         directions[1] = j;
+         int kbeg = (dim == 2) ? 0 : -1;
+         int kend = (dim == 2) ? 1 :  2;
+         for (int k=kbeg; k<kend; k++)
+         {
+            int k1 = k0 + k;
+            if (k1 <0 || k1>=nz) continue;
+            if (dim == 3) directions[2] = k;
+
+            if (i==0 && j==0 && k==0) continue;
+
+            int l = GetSweepToTransfer(s,directions);
+            if (l == -1) continue;
+
+            Vector raux;
+            int ip1 = SourceTransfer(sol0,directions,ip0,raux);
+            *f_transf[ip1][l]-=raux;
+         }
+      }  
+   }
+}
+
+void DST::SetHelmholtzPmlSystemMatrix(int ip)
+{
+   Mesh * mesh = part->patch_mesh[ip];
+   double h = part->MeshSize;
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+
+   int i,j,k;
+   Getijk(ip,i,j,k);
+   if (i == 0 )    length[0][0] = Pmllength[0][0];
+   if (i == nx-1 ) length[0][1] = Pmllength[0][1];
+   if (dim  > 1)
+   {
+      if (j == 0 )    length[1][0] = Pmllength[1][0];
+      if (j == ny-1 ) length[1][1] = Pmllength[1][1];
+   }
+   if (dim  == 3)
+   {
+      if (k == 0 )    length[2][0] = Pmllength[2][0];
+      if (k == nz-1 ) length[2][1] = Pmllength[2][1];
+   }
+   
+   CartesianPML pml(mesh, length);
+   pml.SetOmega(omega);
+
+   Array <int> ess_tdof_list;
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      dmap->fespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+   ProductCoefficient c2_re(c2_re0, *ws);
+   ProductCoefficient c2_im(c2_im0, *ws);
+   sqf[ip] = new SesquilinearForm (dmap->fespaces[ip],bf->GetConvention());
+
+   sqf[ip]->AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   sqf[ip]->AddDomainIntegrator(new MassIntegrator(c2_re),
+                         new MassIntegrator(c2_im));
+   sqf[ip]->Assemble();
+
+   Optr[ip] = new OperatorPtr;
+   sqf[ip]->FormSystemMatrix(ess_tdof_list,*Optr[ip]);
+}
+
+void DST::SetMaxwellPmlSystemMatrix(int ip)
+{
+   Mesh * mesh = part->patch_mesh[ip];
+   double h = part->MeshSize;
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+
+   int i,j,k;
+   Getijk(ip,i,j,k);
+   if (i == 0 )    length[0][0] = Pmllength[0][0];
+   if (i == nx-1 ) length[0][1] = Pmllength[0][1];
+   if (dim  > 1)
+   {
+      if (j == 0 )    length[1][0] = Pmllength[1][0];
+      if (j == ny-1 ) length[1][1] = Pmllength[1][1];
+   }
+   if (dim  == 3)
+   {
+      if (k == 0 )    length[2][0] = Pmllength[2][0];
+      if (k == nz-1 ) length[2][1] = Pmllength[2][1];
+   }
+   
+   CartesianPML pml(mesh, length);
+   pml.SetOmega(omega);
+   Array <int> ess_tdof_list;
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      dmap->fespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ConstantCoefficient omeg(-pow(omega, 2));
+   int cdim = (dim == 2) ? 1 : dim;
+
+   PmlMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, &pml);
+   PmlMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, &pml);
+
+   PmlMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,&pml);
+   ScalarMatrixProductCoefficient c2_Re0(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im0(omeg,pml_c2_Im);
+   ScalarMatrixProductCoefficient c2_Re(*ws,c2_Re0);
+   ScalarMatrixProductCoefficient c2_Im(*ws,c2_Im0);
+
+   sqf[ip] = new SesquilinearForm(dmap->fespaces[ip],bf->GetConvention());
+
+   sqf[ip]->AddDomainIntegrator(new CurlCurlIntegrator(pml_c1_Re),
+                         new CurlCurlIntegrator(pml_c1_Im));
+   sqf[ip]->AddDomainIntegrator(new VectorFEMassIntegrator(c2_Re),
+                         new VectorFEMassIntegrator(c2_Im));
+   sqf[ip]->Assemble();
+
+   Optr[ip] = new OperatorPtr;
+   sqf[ip]->FormSystemMatrix(ess_tdof_list,*Optr[ip]);
+}
+
+int DST::SourceTransfer(const Vector & Psi0, Array<int> direction, int ip0, Vector & Psi1) const
+{
+   int i0,j0,k0;
+   Getijk(ip0,i0,j0,k0);
+
+   int i1 = i0+direction[0];   
+   int j1 = j0+direction[1];   
+   int k1;
+   if (dim==3) k1 = k0+direction[2];   
+   Array<int> ijk(dim); ijk[0]=i1; ijk[1]=j1; 
+   if (dim == 3 ) ijk[2]=k1;
+   int ip1 = GetPatchId(ijk);
+
+   // Array<int> * Dof2GlobalDof0 = &dmap->Dof2GlobalDof[ip0];
+   Array<int> * Dof2GlobalDof1 = &dmap->Dof2GlobalDof[ip1];
+   // zaux.SetSubVector(*Dof2GlobalDof1,0.0);
+   // zaux.SetSubVector(*Dof2GlobalDof0,Psi0);
+   Psi1.SetSize(Dof2GlobalDof1->Size());
+   Vector zloc(Psi1.Size()); zloc = 0.0;
+   // zaux.GetSubVector(*Dof2GlobalDof1,zloc);
+
+
+   Array<int> test_list0;
+   Array<int> test_list1;
+   Array<int>direction1(dim);
+
+   for (int i = 0; i<dim; i++) direction1[i] = -direction[i];
+
+   NeighborMap->GetNeighborDofMap(ip0,direction,test_list0);
+   NeighborMap->GetNeighborDofMap(ip1,direction1,test_list1);
+
+   Vector test1(zloc.Size()); test1 = 0.0;
+   for (int i = 0; i<test_list0.Size(); i++)
+   {
+      // pick up input possition
+      int j = test_list0[i];
+      // destination
+      int k = test_list1[i];
+      zloc[k] = Psi0[j];
+   }
+
+   PmlMat[ip1]->Mult(zloc,Psi1);
+
+   Array2D<int> direct(dim,2); direct = 0;
+   for (int d = 0; d<dim; d++)
+   {
+      if (direction[d]==1) direct[d][0] = 1;
+      if (direction[d]==-1) direct[d][1] = 1;
+   }
+
+   GetChiRes(Psi1,ip1,direct);
+   return ip1;
+}
+
+void DST::GetChiRes(Vector & res, int ip, Array2D<int> direct) const
+{
+   for (int d=0; d<dim; d++)
+   {
+      // negative direction
+      if (direct[d][0]==1) res.SetSubVector(NovlpDofs[ip][d],0.0);
+      // possitive direction
+      if (direct[d][1]==1) res.SetSubVector(NovlpDofs[ip][d+dim],0.0);
+   }
+}
+
+
+void DST::GetStepSubdomains(const int sweep, const int step, Array2D<int> & subdomains) const
+{
+   Array<int> aux;
+   switch(dim)
+   {
+      case 2: 
+      for (int i=nx-1;i>=0; i--)
+      {  
+         int j;
+         switch (sweep)
+         {
+            case 0:  j = step-i;         break;
+            case 1:  j = step-nx+i+1;    break;
+            case 2:  j = nx+i-step-1;    break;
+            default: j = nx+ny-i-step-2; break;
+         }
+         if (j<0 || j>=ny) continue;
+         aux.Append(i); aux.Append(j);
+      }
+      break; 
+      default:
+      for (int i=nx-1;i>=0; i--)
+      {  
+         for (int j=ny-1;j>=0; j--)
+         {
+            int k;
+            switch (sweep)
+            {
+               case 0:  k = step-i-j;            break;
+               case 1:  k = step-nx+i+1-j;       break;
+               case 2:  k = step-ny+j+1-i;       break;
+               case 3:  k = step-nx-ny+i+j+2;    break;
+               case 4:  k = i+j+nz-1-step;       break;
+               case 5:  k = nx+nz-i+j-step-2;    break;
+               case 6:  k = ny+nz+i-j-step-2;    break;
+               default: k = nx+ny+nz-i-j-step-3; break;
+            }
+            if (k<0 || k>=nz) continue;   
+            aux.Append(i); aux.Append(j); aux.Append(k); 
+         }
+      }
+      break;
+   }
+
+   int nrows = aux.Size()/dim;
+   int ncols = dim;
+
+   subdomains.SetSize(nrows,ncols);
+   for (int r=0;r<nrows; r++)
+   {
+      for (int c=0; c<ncols; c++)
+      {
+         int k = r*ncols + c;
+         subdomains[r][c] = aux[k];
+      }
+   }
+}
+
+
+int DST::GetSweepToTransfer(const int s, Array<int> directions) const
+{
+   int l1=-1;
+   int nsweeps = swp->nsweeps;
+   Array<int> sweep0;   
+   swp->GetSweep(s,sweep0);
+
+
+   switch (dim)
+   {
+      case 2:
+      for (int l=s; l<nsweeps; l++)
+      {
+         // Rule 1: the transfer source direction has to be similar with 
+         // the sweep direction
+         Array<int> sweep1;   
+         swp->GetSweep(l,sweep1);
+         int ddot = 0;
+         for (int d=0; d<dim; d++) ddot+= sweep1[d] * directions[d];
+         if (ddot <= 0) continue;
+
+         // Rule 2: The horizontal or vertical transfer source cannot be used
+         // Case of horizontal or vertical transfer source 
+         // (it can't be both 0 cause it's skipped)
+         if (directions[0]==0 || directions[1] == 0) 
+         {
+            if (sweep0[0] == -sweep1[0] && sweep0[1] == -sweep1[1]) continue;
+         }
+         l1 = l;
+         break;
+      }
+      break;
+      default:
+      for (int l=s; l<nsweeps; l++)
+      {
+         // Rule 1: (similar directions) the transfer source direction has to be similar with 
+         // the sweep direction
+         Array<int> sweep1;   
+         swp->GetSweep(l,sweep1);
+         int ddot = 0;
+         bool similar = true;
+         for (int d=0; d<dim; d++) 
+         {
+            if (sweep1[d] * directions[d] < 0) similar = false;
+            ddot+= sweep1[d] * directions[d];
+         }
+         if (!similar || ddot<=0) continue; // not similar
+
+         // Rule 2: (oposite directions) the transfer source direction has to be similar with 
+         // the sweep direction
+         // 
+         // check any of the projections onto the planes
+         // (xy, xz, yz)
+
+         if ( (directions[0]==0 && directions[1] != 0) || 
+              (directions[0]!=0 && directions[1] == 0) ||
+              (directions[0]==0 && directions[2] != 0) || 
+              (directions[0]!=0 && directions[2] == 0) ||
+              (directions[2]==0 && directions[1] != 0) || 
+              (directions[2]!=0 && directions[1] == 0) ) 
+         {
+            if (sweep0[0] == -sweep1[0] && 
+                sweep0[1] == -sweep1[1] && 
+                sweep0[2] == -sweep1[2]) continue;
+         }
+
+         l1 = l;
+         break;
+      }
+      break;
+   }
+
+   return l1;   
+}
+
+
+DST::~DST()
+{
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      for (int i=0;i<swp->nsweeps; i++)
+      {
+         delete f_transf[ip][i];
+      }
+      delete f_orig[ip];
+      delete PmlMatInv[ip];
+      delete Optr[ip];
+      delete sqf[ip];
+      // delete PmlMat[ip];
+   }
+   delete dmap;
+   delete part; 
+}
+
+void DST::PlotSolution(Vector & sol, socketstream & sol_sock, int ip, 
+                  bool localdomain) const
+{
+   FiniteElementSpace * fes;
+   if (!localdomain)
+   {
+      fes = bf->FESpace();
+   }
+   else
+   {
+      fes = dmap->fespaces[ip];
+   }
+   Mesh * mesh = fes->GetMesh();
+   GridFunction gf(fes);
+   double * data = sol.GetData();
+   gf.SetData(data);
+   
+   string keys;
+   // if (ip == 0) 
+   keys = "keys mrRljc\n";
+   sol_sock << "solution\n" << *mesh << gf << keys << "valuerange -0.05 0.05 \n"  << flush;
+   // sol_sock << "solution\n" << *mesh << gf << keys << flush;
+}
+
+
+   void DST::MarkOverlapElements()
+   {
+      // cout<< "Compute Overlap Elements (in each possible direction) " << endl;
+      
+      // Lists of elements
+      // x,y,z = +/- 1 ovlp 
+      NovlpElems.resize(nrpatch);
+
+      for (int ip = 0; ip<nrpatch; ip++)
+      {
+         int i,j,k;
+         Getijk(ip,i,j,k);
+         int ijk[dim]; ijk[0] = i; ijk[1]=j;
+         if (dim==3) ijk[2] = k;
+         int nxyz[dim]; nxyz[0] = nx; nxyz[1]=ny; nxyz[2]=nz;
+
+         FiniteElementSpace * fes = dmap->fespaces[ip];
+         Mesh * mesh = fes->GetMesh();
+         NovlpElems[ip].resize(2*dim);
+
+         Vector pmin, pmax;
+         mesh->GetBoundingBox(pmin,pmax);
+         double h = part->MeshSize;
+         // Loop through elements
+         for (int iel=0; iel<mesh->GetNE(); iel++)
+         {
+            // Get element center
+            Vector center(dim);
+            int geom = mesh->GetElementBaseGeometry(iel);
+            ElementTransformation * tr = mesh->GetElementTransformation(iel);
+            tr->Transform(Geometries.GetCenter(geom),center);
+
+            // Assign elements to the appropriate lists
+            for (int d=0;d<dim; d++)
+            {
+               if (ijk[d]>0)
+               {
+                  if (center[d] >= pmin[d]+h*ovlpnrlayers) 
+                  {
+                     NovlpElems[ip][d].Append(iel);
+                  }
+               }
+               else
+               {
+                  NovlpElems[ip][d].Append(iel);
+               }
+               
+               if (ijk[d]<nxyz[d]-1)
+               {
+                  if (center[d] <= pmax[d]-h*ovlpnrlayers) 
+                  {
+                     NovlpElems[ip][dim+d].Append(iel);
+                  }
+               }
+               else
+               {
+                  NovlpElems[ip][dim+d].Append(iel);
+               }
+            }
+         }
+      }   
+   }
+
+
+
+void DST::MarkOverlapDofs()
+{
+   // cout<< "Compute Overlap dofs (in each possible direction) " << endl;
+   NovlpDofs.resize(nrpatch);
+   for (int ip = 0; ip<nrpatch; ip++)
+   {
+      FiniteElementSpace * fes = dmap->fespaces[ip];
+      // Loop through the marked elements
+      NovlpDofs[ip].resize(2*dim);
+      
+      int n = fes->GetTrueVSize();
+      Array<int> marker(n);
+      for (int d=0;d<2*dim; d++)
+      {
+         marker = 0;
+         int m = 0;
+         int melems = NovlpElems[ip][d].Size(); 
+         for (int iel=0; iel<melems; iel++)
+         {
+            Array<int> ElemDofs;
+            int el =  NovlpElems[ip][d][iel];
+            fes->GetElementDofs(el,ElemDofs);
+            int ndof = ElemDofs.Size();
+            for (int i = 0; i<ndof; ++i)
+            {
+               int eldof = ElemDofs[i];
+               int tdof = (eldof >= 0) ? eldof : abs(eldof) - 1; 
+               if (marker[tdof] == 1) continue;
+               marker[tdof] = 1;
+               m++;
+            }
+         }
+         int k = n-m;
+         NovlpDofs[ip][d].SetSize(2*k);
+         int l = 0;
+         for (int i = 0; i<n; i++)
+         {
+            if (marker[i]==0) 
+            {
+               NovlpDofs[ip][d][l] = i;  // real dofs
+               NovlpDofs[ip][d][l+k] = i+n;  // imag dofs
+               l++;
+            }
+         }
+      }
+   }
+}

examples/maxwell-solver/DST/DST.hpp

@@ -0,0 +1,64 @@
+#pragma once
+#include "../common/Utilities.hpp"
+#include "../common/PML.hpp"
+using namespace std;
+using namespace mfem;
+
+
+
+class DST : public Solver//
+{
+private:
+   // Constructor inputs
+   SesquilinearForm *bf=nullptr;
+   Array2D<double> Pmllength;
+   double omega = 0.5;
+   Coefficient * ws;
+   int nrlayers;
+   // 
+   int nrpatch;
+   int dim;
+   int nx, ny, nz;
+   int ovlpnrlayers;
+   MeshPartition * part=nullptr;
+   DofMap * dmap = nullptr;
+   // Auxiliary global vector for transfers
+   std::vector<std::vector<Array<int>>> NovlpElems;
+   std::vector<std::vector<Array<int>>> NovlpDofs;
+   NeighborDofMaps * NeighborMap = nullptr;
+   Array< SesquilinearForm * > sqf;
+   Array< OperatorPtr * > Optr;
+   Array<ComplexSparseMatrix *> PmlMat;
+   Array<ComplexUMFPackSolver *> PmlMatInv;
+   Sweep * swp=nullptr;
+   mutable Array<Vector *> f_orig;
+   mutable std::vector<Array<Vector * >> f_transf;
+   mutable Vector zaux;
+
+
+
+   void MarkOverlapElements();
+   void MarkOverlapDofs();
+   void ComputeOverlapDofMaps();
+   void Getijk(int ip, int & i, int & j, int & k ) const;
+   int GetPatchId(const Array<int> & ijk) const;
+  
+   void SetHelmholtzPmlSystemMatrix(int ip);
+   void SetMaxwellPmlSystemMatrix(int ip);
+
+   void GetChiRes(Vector & res, int ip, Array2D<int> direct) const;                    
+   void GetStepSubdomains(const int sweep, const int step, Array2D<int> & subdomains) const;
+   void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+   int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+   int GetSweepToTransfer(const int s, Array<int> directions) const;
+   void PlotSolution(Vector & sol, socketstream & sol_sock, int ip,bool localdomain) const;
+  
+public:
+   DST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+       double omega_, Coefficient * ws_, int nrlayers_, int nx_=2, int ny_=2, int nz_=2);
+   virtual void SetOperator(const Operator &op) {}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~DST();
+};
+
+

examples/maxwell-solver/DST2D/AdditiveST2D.cpp

@@ -0,0 +1,545 @@
+//Additive Source Transfer Preconditioner
+
+#include "AdditiveST2D.hpp"
+
+
+AdditiveST2D::AdditiveST2D(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+         double omega_, Coefficient * ws_,  int nrlayers_)
+   : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), 
+     bf(bf_), Pmllength(Pmllength_), omega(omega_), ws(ws_), nrlayers(nrlayers_)
+{
+   Mesh * mesh = bf->FESpace()->GetMesh();
+   dim = mesh->Dimension();
+
+   int partition_kind;
+
+   // 1. Ovelapping partition with overlap = 2h 
+   partition_kind = 2; // Non Overlapping partition 
+   int nx=2;
+   int ny=2; 
+   int nz=1;
+   ovlpnrlayers = nrlayers+2;
+   povlp = new MeshPartition(mesh, partition_kind,nx,ny,nz, ovlpnrlayers);
+
+   partition_kind = 1;
+   novlp = new MeshPartition(mesh, partition_kind,nx,ny,nz);
+
+   nxyz[0] = povlp->nxyz[0];
+   nxyz[1] = povlp->nxyz[1];
+   nxyz[2] = povlp->nxyz[2];
+   nrpatch = povlp->nrpatch;
+   subdomains = povlp->subdomains;
+
+
+   ovlp_prob  = new DofMap(bf,povlp); 
+   nvlp_prob  = new DofMap(bf,novlp); 
+   PmlMat.SetSize(nrpatch);
+   PmlMatInv.SetSize(nrpatch);
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      PmlMat[ip] = GetPmlSystemMatrix(ip);
+      PmlMatInv[ip] = new KLUSolver;
+      PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
+   }
+
+   int nsteps = nx + ny - 1;
+
+   f_orig.SetSize(nrpatch);
+   usol.SetSize(nrpatch);
+   f_s.SetSize(nrpatch);
+   f_diag.SetSize(nrpatch);
+   // Construct a simple map used for directions of transfer
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      int n = 2*ovlp_prob->fespaces[ip]->GetTrueVSize(); // (x 2 for complex ) 
+      f_orig[ip] = new Vector(n); *f_orig[ip] = 0.0;
+      usol[ip] = new Vector(n); *usol[ip] = 0.0;
+      f_s[ip].SetSize(nsteps);
+      f_diag[ip].SetSize(nsteps);
+      for (int i=0;i<nsteps; i++)
+      {
+         f_s[ip][i] = new Vector(n); 
+         f_diag[ip][i] = new Vector(n);
+      }
+   }
+}
+
+void AdditiveST2D::Mult(const Vector &r, Vector &z) const
+{
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      *f_orig[ip] = 0.0;
+      *usol[ip] = 0.0;
+      for (int i=0;i< f_s[ip].Size(); i++)
+      {
+         *f_s[ip][i] = 0.0;
+         *f_diag[ip][i] = 0.0;
+      }
+   }
+   socketstream res_sock(vishost, visport);
+   Vector res(r);
+   PlotSolution(res,res_sock,0,false); 
+   // for (int ip=0; ip<nrpatch; ip++)
+   for (int ip=nrpatch-1; ip>=0; ip--)
+   {
+      Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+      r.GetSubVector(*Dof2GlobalDof,*f_orig[ip]);
+
+      // make sure that f_ij is compactly supported in \Omega_ij (non overlapping)
+      int i,j,k;
+      Getijk(ip,i,j,k);
+      Array<int> directions(2); directions = 0;
+      int nx = nxyz[0];
+      int ny = nxyz[1];
+      if (i+1<nx) directions[0] = 1;
+      if (j+1<ny) directions[1] = 1;
+      Vector faux(f_orig[ip]->Size());
+      GetChiRes(*f_orig[ip],faux,ip,directions,ovlpnrlayers);
+      directions = 0.0;
+      if (i>0) directions[0] = -1;
+      if (j>0) directions[1] = -1;
+      *f_orig[ip] = 0.0;
+      GetChiRes(faux,*f_orig[ip],ip,directions,ovlpnrlayers);
+      // Array<int> * nDof2GlobalDof = &nvlp_prob->Dof2GlobalDof[ip];
+      // Vector faux(nDof2GlobalDof->Size());
+      // r.GetSubVector(*nDof2GlobalDof,faux);
+      // res = 0.0;
+      // res.SetSubVector(*nDof2GlobalDof,faux);
+      // res.GetSubVector(*Dof2GlobalDof,*f_orig[ip]);
+   }
+
+   z = 0.0; 
+   Vector znew(z);
+
+   // --------------------------------------------
+   //       Sweep in the direction (1,1)
+   // --------------------------------------------
+   int nx = nxyz[0];
+   int ny = nxyz[1];
+   int nsteps = (nx + ny - 1);
+
+   for (int s = 0; s<nsteps; s++)
+   {
+      for (int ip = 0; ip<nrpatch; ip++)
+      {
+         Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+         int ndofs = Dof2GlobalDof->Size();
+
+         Vector sol_local(ndofs); sol_local = 0.0;
+         Vector res_local(ndofs); res_local = 0.0;
+         if (s==0) 
+         {
+            res_local = *f_orig[ip];
+         }
+         else if (s == 1)
+         {
+            res_local = *f_s[ip][s-1];
+         }
+         else
+         {
+            res_local = *f_s[ip][s-1];
+            res_local += *f_diag[ip][s-2];
+         }
+         // cout << "reslocal norm = " << res_local.Norml2() << endl;
+         if (res_local.Norml2() < 1e-12) continue;
+         PmlMatInv[ip]->Mult(res_local, sol_local);
+         AdditiveTransferSources(s, ip, sol_local);
+         *usol[ip] += sol_local;
+
+         // Array<int>directions(2); directions = 0; 
+         // int i,j,k;
+         // Getijk(ip,i,j,k);
+         // if (i+1<nx) directions[0] = 1;
+         // if (j+1<ny) directions[1] = 1;
+         // Vector cfsol_local;
+         // GetCutOffSolution(sol_local,cfsol_local,ip,directions,ovlpnrlayers,true);
+         // sol_local = cfsol_local;
+         // directions = 0.0;
+         // if (i>0) directions[0] = -1;
+         // if (j>0) directions[1] = -1;
+         // GetCutOffSolution(sol_local,cfsol_local,ip,directions,ovlpnrlayers,true);
+         // znew = 0.0;
+         // znew.SetSubVector(*Dof2GlobalDof, cfsol_local);
+         // z+=znew;
+      }
+      // socketstream sol1_sock(vishost, visport);
+      // PlotSolution(z,sol1_sock,0,false); cin.get();
+   }
+   for (int ip = 0; ip<nrpatch; ip++)
+   {
+      Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+
+      Array<int>directions(2); directions = 0; 
+      int i,j,k;
+      Getijk(ip,i,j,k);
+      if (i+1<nx) directions[0] = 1;
+      if (j+1<ny) directions[1] = 1;
+      Vector cfsol_local;
+      GetCutOffSolution(*usol[ip],cfsol_local,ip,directions,ovlpnrlayers,true);
+      *usol[ip] = cfsol_local;
+      directions = 0.0;
+      if (i>0) directions[0] = -1;
+      if (j>0) directions[1] = -1;
+      GetCutOffSolution(*usol[ip],cfsol_local,ip,directions,ovlpnrlayers,true);
+      znew = 0.0;
+      znew.SetSubVector(*Dof2GlobalDof, cfsol_local);
+      z+=znew;
+   }
+}
+
+
+void AdditiveST2D::GetCutOffSolution(const Vector & sol, Vector & cfsol, 
+                  int ip, Array<int> directions, int nlayers, bool local) const
+{
+
+   // int d = directions.Size();
+   // int directx = directions[0]; // 1,0,-1
+   // int directy = directions[1]; // 1,0,-1
+   // int directz;
+   // if (d ==3) directz = directions[2];
+
+   Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+   
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+   double h = GetUniformMeshElementSize(povlp->patch_mesh[ip]);
+
+   int i, j, k;
+   Getijk(ip,i,j,k);
+   // int nx = nxyz[0];
+   // int ny = nxyz[1];
+   if (directions[0]==1) pmax[0] -= h*nrlayers; 
+   if (directions[1]==1) pmax[1] -= h*nrlayers; 
+
+   if (directions[0]==-1) pmin[0] += h*nrlayers; 
+   if (directions[1]==-1) pmin[1] += h*nrlayers; 
+
+   Array2D<double> pmlh(dim,2); pmlh = 0.0;
+   
+   if (directions[0]==1)
+   {
+      pmlh[0][1] = h*(nlayers-nrlayers-1);
+   }
+   if (directions[0]==-1)
+   {
+      pmlh[0][0] = h*(nlayers-nrlayers-1);
+   }
+   if (directions[1]==1)
+   {
+      pmlh[1][1] = h*(nlayers-nrlayers-1);
+   }
+   if (directions[1]==-1)
+   {
+      pmlh[1][0] = h*(nlayers-nrlayers-1);
+   }
+
+   CutOffFnCoefficient cf(CutOffFncn, pmin, pmax, pmlh);
+   double * data = sol.GetData();
+   FiniteElementSpace * fes;
+   if (!local)
+   {
+      fes = bf->FESpace();
+   }
+   else
+   {
+      fes = ovlp_prob->fespaces[ip];
+   }
+   int n = fes->GetTrueVSize();
+   GridFunction solgf_re(fes, data);
+   GridFunction solgf_im(fes, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fes);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   cfsol.SetSize(sol.Size());
+   cfsol = gf;
+}
+
+
+AdditiveST2D::~AdditiveST2D()
+{
+}
+
+
+void AdditiveST2D::Getijk(int ip, int & i, int & j, int & k) const
+{
+   k = ip/(nxyz[0]*nxyz[1]);
+   j = (ip-k*nxyz[0]*nxyz[1])/nxyz[0];
+   i = (ip-k*nxyz[0]*nxyz[1])%nxyz[0];
+}
+
+int AdditiveST2D::GetPatchId(const Array<int> & ijk) const
+{
+   int d=ijk.Size();
+   int z = (d==2)? 0 : ijk[2];
+   return subdomains(ijk[0],ijk[1],z);
+}
+
+
+
+void AdditiveST2D::AdditiveTransferSources(int s, int ip0, Vector & sol0) const
+{
+ // Find all neighbors of patch ip0
+   int nx = nxyz[0];
+   int ny = nxyz[1];
+   int i0, j0, k0;
+   Getijk(ip0, i0,j0,k0);
+   for (int i=-1; i<2; i++)
+   {
+      int i1 = i0 + i;
+      if (i1 <0 || i1>=nx) continue;
+      for (int j=-1; j<2; j++)
+      {
+         if (i==0 && j==0) continue;
+
+         int j1 = j0 + j;
+         if (j1 <0 || j1>=ny) continue;
+         Array<int> ij1(2); ij1[0] = i1; ij1[1]=j1;
+         int ip1 = GetPatchId(ij1);
+
+         Array<int> directions(2);
+         directions[0] = i;
+         directions[1] = j;
+         Vector cfsol0;
+         GetCutOffSolution(sol0,cfsol0,ip0,directions,ovlpnrlayers,true);
+
+         Vector raux;
+         SourceTransfer(cfsol0,directions,ip0,raux);
+         if (abs(i)+abs(j) == 2)
+         {
+            *f_diag[ip1][s]+=raux;
+         }
+         else
+         {
+            *f_s[ip1][s]+=raux;
+         }
+      }
+   }
+}
+
+SparseMatrix * AdditiveST2D::GetPmlSystemMatrix(int ip)
+{
+   double h = GetUniformMeshElementSize(povlp->patch_mesh[ip]);
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+
+   int i,j,k;
+   int nx = nxyz[0];
+   int ny = nxyz[1];
+   Getijk(ip,i,j,k);
+   if (i == 0 ) length[0][0] = Pmllength[0][0];
+   if (j == 0 ) length[1][0] = Pmllength[1][0];
+   if (i == nx-1 ) length[0][1] = Pmllength[0][1];
+   if (j == ny-1 ) length[1][1] = Pmllength[1][1];
+
+   CartesianPML pml(povlp->patch_mesh[ip], length);
+   pml.SetOmega(omega);
+
+   Array <int> ess_tdof_list;
+   if (povlp->patch_mesh[ip]->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(povlp->patch_mesh[ip]->bdr_attributes.Max());
+      ess_bdr = 1;
+      ovlp_prob->fespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+   ProductCoefficient c2_re(c2_re0, *ws);
+   ProductCoefficient c2_im(c2_im0, *ws);
+   SesquilinearForm a(ovlp_prob->fespaces[ip],ComplexOperator::HERMITIAN);
+
+   a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   a.AddDomainIntegrator(new MassIntegrator(c2_re),
+                         new MassIntegrator(c2_im));
+   a.Assemble();
+
+   OperatorPtr Alocal;
+   a.FormSystemMatrix(ess_tdof_list,Alocal);
+   ComplexSparseMatrix * AZ_ext = Alocal.As<ComplexSparseMatrix>();
+   SparseMatrix * Mat = AZ_ext->GetSystemMatrix();
+   Mat->Threshold(1e-13);
+   return Mat;
+}
+
+void AdditiveST2D::PlotSolution(Vector & sol, socketstream & sol_sock, int ip, 
+                  bool localdomain) const
+{
+   FiniteElementSpace * fes;
+   if (!localdomain)
+   {
+      fes = bf->FESpace();
+   }
+   else
+   {
+      fes = ovlp_prob->fespaces[ip];
+   }
+   Mesh * mesh = fes->GetMesh();
+   GridFunction gf(fes);
+   double * data = sol.GetData();
+   gf.SetData(data);
+   
+   string keys;
+   keys = "keys mrRljc\n";
+   sol_sock << "solution\n" << *mesh << gf << keys << flush;
+}
+
+void AdditiveST2D::PlotMesh(socketstream & mesh_sock, int ip) const
+{
+   FiniteElementSpace * fes = ovlp_prob->fespaces[ip];
+   Mesh * mesh = fes->GetMesh();
+   mesh_sock << "mesh\n" << *mesh << flush;
+}
+
+void AdditiveST2D::SaveSolution(Vector & sol, int ip, bool localdomain) const
+{
+   FiniteElementSpace * fes;
+   if (!localdomain)
+   {
+      fes = bf->FESpace();
+   }
+   else
+   {
+      // fes = ovlp_prob->fespaces[ip];
+      fes = nvlp_prob->fespaces[ip];
+   }
+   Mesh * mesh = fes->GetMesh();
+   int n = fes->GetTrueVSize();
+   GridFunction gf_re(fes);
+   GridFunction gf_im(fes);
+   double * data = sol.GetData();
+   gf_re.SetData(data);
+   gf_im.SetData(&data[n]);
+   cout << "saving mesh no " << ip << endl;
+   string mfilename = "output/mesh_nvlp.";
+   ostringstream mesh_name;
+   mesh_name << mfilename << setfill('0') << setw(6) << ip;
+   ofstream mesh_ofs(mesh_name.str().c_str());
+   mesh_ofs.precision(8);
+   mesh->Print(mesh_ofs);
+
+   string sfilename_re = "output/sol_nvlp.";
+
+   ostringstream solre_name;
+   solre_name << sfilename_re << setfill('0') << setw(6) << ip;
+   ofstream solre_ofs(solre_name.str().c_str());
+   gf_re.Save(solre_ofs);
+
+}
+
+int AdditiveST2D::SourceTransfer(const Vector & Psi0, Array<int> direction, int ip0, Vector & Psi1) const
+{
+   int i0,j0,k0;
+   Getijk(ip0,i0,j0,k0);
+
+   int i1 = i0+direction[0];   
+   int j1 = j0+direction[1];   
+   Array<int> ij(2); ij[0]=i1; ij[1]=j1;
+   int ip1 = GetPatchId(ij);
+
+   MFEM_VERIFY(i1 < nxyz[0] && i1>=0, "SourceTransfer: i1 out of bounds");
+   MFEM_VERIFY(j1 < nxyz[1] && j1>=0, "SourceTransfer: j1 out of bounds");
+
+   Array<int> * Dof2GlobalDof0 = &ovlp_prob->Dof2GlobalDof[ip0];
+   Array<int> * Dof2GlobalDof1 = &ovlp_prob->Dof2GlobalDof[ip1];
+   Psi1.SetSize(Dof2GlobalDof1->Size()); Psi1=0.0;
+   Vector r(2*bf->FESpace()->GetTrueVSize());
+   r = 0.0;
+   r.SetSubVector(*Dof2GlobalDof0,Psi0);
+   Vector zloc(Psi1.Size()); zloc = 0.0;
+   r.GetSubVector(*Dof2GlobalDof1,zloc);
+
+   Vector Psi(Dof2GlobalDof1->Size()); Psi=0.0;
+
+   PmlMat[ip1]->Mult(zloc,Psi);
+   Psi *=-1.0;
+
+   Array<int> direct(2); direct = 0;
+   direct[0] = -direction[0];
+   direct[1] = -direction[1];
+   GetChiRes(Psi, Psi1,ip1,direct, ovlpnrlayers);
+
+   return ip1;
+}
+
+
+
+
+void AdditiveST2D::GetChiRes(const Vector & res, Vector & cfres, 
+                    int ip, Array<int> directions, int nlayers) const
+{
+   // int l,k;
+   // int d = directions.Size();
+   // int directx = directions[0]; // 1,0,-1
+   // int directy = directions[1]; // 1,0,-1
+   // int directz;
+   // if (d ==3) directz = directions[2];
+
+   Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+   double h = GetUniformMeshElementSize(mesh);
+   
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+   Array2D<double> pmlh(dim,2); pmlh = 0.0;
+   int i,j,k;
+   Getijk(ip,i,j,k);
+   if (directions[0]==-1)
+   {
+      pmlh[0][0] = h;
+      pmin[0] += h*(nlayers-1);
+   }
+   if (directions[0]==1)
+   {
+      pmlh[0][1] = h;
+      pmax[0] -= h*(nlayers-1);
+
+   }
+   if (directions[1]==-1)
+   {
+      pmlh[1][0] = h;
+      pmin[1] += h*(nlayers-1);
+   }
+   if (directions[1]==1)
+   {
+      pmlh[1][1] = h;
+      pmax[1] -= h*(nlayers-1);
+   }
+   CutOffFnCoefficient cf(ChiFncn, pmin, pmax, pmlh);
+
+   double * data = res.GetData();
+
+   FiniteElementSpace * fespace;
+   fespace = ovlp_prob->fespaces[ip];
+   
+   int n = fespace->GetTrueVSize();
+
+   GridFunction solgf_re(fespace, data);
+   GridFunction solgf_im(fespace, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fespace);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   cfres.SetSize(res.Size());
+   cfres = gf;
+}

examples/maxwell-solver/DST2D/AdditiveST2D.hpp

@@ -0,0 +1,56 @@
+#pragma once
+#include "../common/Utilities.hpp"
+#include "../common/PML.hpp"
+using namespace std;
+using namespace mfem;
+
+class AdditiveST2D : public Solver//
+{
+private:
+   int nrpatch;
+   int dim;
+   MeshPartition * povlp=nullptr;
+   MeshPartition * novlp=nullptr;
+   int ovlpnrlayers;
+   int nxyz[3];
+   const Operator * A=nullptr;
+   DofMap * ovlp_prob = nullptr;
+   DofMap * nvlp_prob = nullptr;
+   Array<SparseMatrix *> PmlMat;
+   Array<KLUSolver *> PmlMatInv;
+   Array3D<int> subdomains;
+   mutable Array<Vector *> f_orig;
+   mutable Array<Vector *> usol;
+   mutable Array<Array<Vector * >> f_s;
+   mutable Array<Array<Vector * >> f_diag;
+
+   SesquilinearForm *bf=nullptr;
+   Array2D<double> Pmllength;
+   double omega = 0.5;
+   Coefficient * ws;
+   int nrlayers;
+
+
+   SparseMatrix * GetPmlSystemMatrix(int ip);
+  
+   void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+                          int ip, Array<int> directions, int nlayers, bool local=false) const;
+   void GetChiRes(const Vector & res, Vector & cfres,
+                  int ip, Array<int> directions, int nlayers) const;  
+   
+   void AdditiveTransferSources(int step, int ip, Vector & sol_ext) const;
+   int GetPatchId(const Array<int> & ijk) const;
+   void Getijk(int ip, int & i, int & j, int & k ) const;
+   int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+   void PlotSolution(Vector & sol, socketstream & sol_sock, int ip,bool localdomain) const;
+   void SaveSolution(Vector & sol, int ip,bool localdomain) const;
+   void PlotMesh(socketstream & mesh_sock, int ip) const;
+public:
+   AdditiveST2D(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+       double omega_, Coefficient * ws_, int nrlayers_);
+   virtual void SetOperator(const Operator &op) {A = &op;}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~AdditiveST2D();
+};
+
+

examples/maxwell-solver/DST2D/DST2D.cpp

@@ -0,0 +1,699 @@
+//Diagonal Source Transfer Preconditioner
+
+#include "DST2D.hpp"
+
+DST2D::DST2D(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+         double omega_, Coefficient * ws_,  int nrlayers_)
+   : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), 
+     bf(bf_), Pmllength(Pmllength_), omega(omega_), ws(ws_), nrlayers(nrlayers_)
+{
+   Mesh * mesh = bf->FESpace()->GetMesh();
+   dim = mesh->Dimension();
+
+   int partition_kind;
+
+   partition_kind = 2; 
+   int nx=2;
+   int ny=2; 
+   int nz=1;
+   ovlpnrlayers = nrlayers+1;
+   povlp = new MeshPartition(mesh, partition_kind,nx,ny,nz, ovlpnrlayers);
+
+   partition_kind = 1;
+   novlp = new MeshPartition(mesh, partition_kind,nx,ny,nz);
+
+   nxyz[0] = povlp->nxyz[0];
+   nxyz[1] = povlp->nxyz[1];
+   nxyz[2] = povlp->nxyz[2];
+   nrpatch = povlp->nrpatch;
+   subdomains = povlp->subdomains;
+
+   ovlp_prob  = new DofMap(bf->FESpace(),povlp); 
+   nvlp_prob  = new DofMap(bf->FESpace(),novlp); 
+   PmlMat.SetSize(nrpatch);
+   PmlMatInv.SetSize(nrpatch);
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      PmlMat[ip] = GetPmlSystemMatrix(ip);
+      PmlMatInv[ip] = new KLUSolver;
+      PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
+   }
+   nsweeps = pow(2,dim);
+   sweeps.SetSize(nsweeps,dim);
+   // 2D
+   sweeps(0,0) =  1; sweeps(0,1) = 1;
+   sweeps(1,0) = -1; sweeps(1,1) = 1;
+   sweeps(2,0) =  1; sweeps(2,1) =-1;
+   sweeps(3,0) = -1; sweeps(3,1) =-1;
+
+   // Set up src arrays size
+   f_orig.SetSize(nrpatch);
+   f_transf.SetSize(nrpatch);
+   // Construct a simple map used for directions of transfer
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      int n = 2*ovlp_prob->fespaces[ip]->GetTrueVSize(); // (x 2 for complex ) 
+      f_orig[ip] = new Vector(n); *f_orig[ip] = 0.0;
+      f_transf[ip].SetSize(nsweeps);
+      for (int i=0;i<nsweeps; i++)
+      {
+         f_transf[ip][i] = new Vector(n);
+      }
+   }
+}
+
+void DST2D::Mult(const Vector &r, Vector &z) const
+{
+   // char vishost[] = "localhost";
+   // int  visport   = 19916;
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      *f_orig[ip] = 0.0;
+      for (int i=0;i<nsweeps; i++)
+      {
+         *f_transf[ip][i] = 0.0;
+      }
+   }
+
+   // socketstream res_sock(vishost, visport);
+   // Vector res(r);
+   // PlotSolution(res,res_sock,0,false); 
+
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+      
+      r.GetSubVector(*Dof2GlobalDof,*f_orig[ip]);
+
+      // make sure that f_ij is compactly supported in \Omega_ij (non overlapping)
+      int i,j,k;
+      Getijk(ip,i,j,k);
+      int nx = nxyz[0];
+      int ny = nxyz[1];
+      // Array<int> directions(2); directions = 0;
+      // if (i+1<nx) directions[0] = 1;
+      // if (j+1<ny) directions[1] = 1;
+      // Vector faux(*f_orig[ip]);
+      // GetChiRes(*f_orig[ip],faux,ip,directions,ovlpnrlayers);
+      // directions = 0.0;
+      // if (i>0) directions[0] = -1;
+      // if (j>0) directions[1] = -1;
+      // *f_orig[ip] = 0.0;
+      // GetChiRes(faux,*f_orig[ip],ip,directions,ovlpnrlayers);
+
+      Array2D<int> direct(dim,2); direct = 0;
+      if (i>0) direct[0][0] = 1;
+      if (i+1<nx) direct[0][1] = 1;
+      if (j>0) direct[1][0] = 1;
+      if (j+1<ny) direct[1][1] = 1;
+
+      Vector faux(*f_orig[ip]);
+      *f_orig[ip] = 0.0;
+      GetChiRes(faux,*f_orig[ip],ip,direct,ovlpnrlayers);
+
+   }
+
+   z = 0.0; 
+   Vector znew(z);
+
+   // --------------------------------------------
+   //       Sweep in the direction (1,1)
+   // --------------------------------------------
+   int nx = nxyz[0];
+   int ny = nxyz[1];
+
+   int nsteps = nx + ny - 1;
+
+   for (int l=0; l<nsweeps; l++)
+   {
+      for (int s = 0; s<nsteps; s++)
+      {
+         for (int i=nx-1;i>=0; i--)
+         {
+            int j;
+            switch (l)
+            {
+               case 0:  j = s-i;         break;
+               case 1:  j = s-nx+i+1;    break;
+               case 2:  j = nx+i-s-1;    break;
+               default: j = nx+ny-i-s-2; break;
+            }
+            if (j<0 || j>=ny) continue;
+
+            Array<int> ij(2); ij[0] = i; ij[1]=j;
+            int ip = GetPatchId(ij);
+
+            // Solve the PML problem in patch ip with all sources
+            // Original and all transfered 
+            Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+            int ndofs = Dof2GlobalDof->Size();
+
+            Vector sol_local(ndofs); sol_local = 0.0;
+            Vector res_local(ndofs); res_local = 0.0;
+            if (l==0) res_local += *f_orig[ip];
+            res_local += *f_transf[ip][l];
+            if (res_local.Norml2() < 1e-12) continue;
+            PmlMatInv[ip]->Mult(res_local, sol_local);
+            TransferSources(l,ip, sol_local);
+            Array2D<int> direct(dim,2); direct = 0;
+            if (i>0) direct[0][0] = 1;
+            if (i+1<nx) direct[0][1] = 1;
+            if (j>0) direct[1][0] = 1;
+            if (j+1<ny) direct[1][1] = 1;
+            Vector cfsol_local;
+            GetCutOffSolution(sol_local,cfsol_local,ip,direct,ovlpnrlayers,true);
+            
+            znew = 0.0;
+            znew.SetSubVector(*Dof2GlobalDof, cfsol_local);
+            z+=znew;
+         }
+      }
+   }
+}
+
+
+void DST2D::GetCutOffSolution(const Vector & sol, Vector & cfsol, 
+                  int ip, Array<int> directions, int nlayers, bool local) const
+{
+
+   // int d = directions.Size();
+   // int directx = directions[0]; // 1,0,-1
+   // int directy = directions[1]; // 1,0,-1
+   // int directz;
+   // if (d ==3) directz = directions[2];
+
+   Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+   
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+   double h = GetUniformMeshElementSize(povlp->patch_mesh[ip]);
+
+   int i, j, k;
+   Getijk(ip,i,j,k);
+   if (directions[0]==1) pmax[0] -= h*nrlayers; 
+   if (directions[1]==1) pmax[1] -= h*nrlayers; 
+
+   if (directions[0]==-1) pmin[0] += h*nrlayers; 
+   if (directions[1]==-1) pmin[1] += h*nrlayers; 
+
+   Array2D<double> pmlh(dim,2); pmlh = 0.0;
+   
+   if (directions[0]==1)
+   {
+      pmlh[0][1] = h*(nlayers-nrlayers-1);
+   }
+   if (directions[0]==-1)
+   {
+      pmlh[0][0] = h*(nlayers-nrlayers-1);
+   }
+   if (directions[1]==1)
+   {
+      pmlh[1][1] = h*(nlayers-nrlayers-1);
+   }
+   if (directions[1]==-1)
+   {
+      pmlh[1][0] = h*(nlayers-nrlayers-1);
+   }
+
+   CutOffFnCoefficient cf(CutOffFncn, pmin, pmax, pmlh);
+   double * data = sol.GetData();
+   FiniteElementSpace * fes;
+   if (!local)
+   {
+      fes = bf->FESpace();
+   }
+   else
+   {
+      fes = ovlp_prob->fespaces[ip];
+   }
+   int n = fes->GetTrueVSize();
+   GridFunction solgf_re(fes, data);
+   GridFunction solgf_im(fes, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fes);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   cfsol.SetSize(sol.Size());
+   cfsol = gf;
+}
+
+void DST2D::GetCutOffSolution(const Vector & sol, Vector & cfsol, 
+                  int ip, Array2D<int> direct, int nlayers, bool local) const
+{
+
+
+   Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+   
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+   double h = GetUniformMeshElementSize(povlp->patch_mesh[ip]);
+
+
+   int i, j, k;
+   Getijk(ip,i,j,k);
+
+   Array2D<double> pmlh(dim,2); pmlh = 0.0;
+   for (int i=0; i<dim; i++)
+   {
+      if (direct[i][0]==1) pmin[i] += h*nrlayers; 
+      if (direct[i][1]==1) pmax[i] -= h*nrlayers; 
+      for (int j=0; j<2; j++)
+      {
+         if (direct[i][j]==1)
+         {
+            pmlh[i][j] = h*(nlayers-nrlayers-1);
+         }
+      }  
+   }
+
+   CutOffFnCoefficient cf(CutOffFncn, pmin, pmax, pmlh);
+   double * data = sol.GetData();
+   FiniteElementSpace * fes;
+   if (!local)
+   {
+      fes = bf->FESpace();
+   }
+   else
+   {
+      fes = ovlp_prob->fespaces[ip];
+   }
+   int n = fes->GetTrueVSize();
+   GridFunction solgf_re(fes, data);
+   GridFunction solgf_im(fes, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fes);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   cfsol.SetSize(sol.Size());
+   cfsol = gf;
+}
+
+DST2D::~DST2D()
+{
+}
+
+
+void DST2D::Getijk(int ip, int & i, int & j, int & k) const
+{
+   k = ip/(nxyz[0]*nxyz[1]);
+   j = (ip-k*nxyz[0]*nxyz[1])/nxyz[0];
+   i = (ip-k*nxyz[0]*nxyz[1])%nxyz[0];
+}
+
+int DST2D::GetPatchId(const Array<int> & ijk) const
+{
+   int d=ijk.Size();
+   int z = (d==2)? 0 : ijk[2];
+   return subdomains(ijk[0],ijk[1],z);
+}
+
+
+void DST2D::TransferSources(int sweep, int ip0, Vector & sol0) const
+{
+ // Find all neighbors of patch ip0
+   int nx = nxyz[0];
+   int ny = nxyz[1];
+   int i0, j0, k0;
+   Getijk(ip0, i0,j0,k0);
+   int is = sweeps(sweep,0);
+   int js = sweeps(sweep,1);
+   for (int i=-1; i<2; i++)
+   {
+      int i1 = i0 + i;
+      if (i1 <0 || i1>=nx) continue;
+      for (int j=-1; j<2; j++)
+      {
+         if (i==0 && j==0) continue;
+
+         int j1 = j0 + j;
+         if (j1 <0 || j1>=ny) continue;
+         Array<int> ij1(2); ij1[0] = i1; ij1[1]=j1;
+         int ip1 = GetPatchId(ij1);
+
+         for (int l=sweep; l<nsweeps; l++)
+         {
+            // Conditions on sweeps
+            // Rule 1: the transfer source direction has to be similar with 
+            // the sweep direction
+            int il = sweeps(l,0); 
+            int jl = sweeps(l,1);
+            int ddot = il*i + jl * j;
+            if (ddot <= 0) continue;
+
+            // Rule 2: The horizontal or vertical transfer source cannot be used
+            // in a later sweep that with opposite directions
+
+            if (i==0 || j == 0) // Case of horizontal or vertical transfer source
+            {
+               // skip if the two sweeps have opposite direction
+               if (is == -il && js == -jl) continue;
+            }
+
+            Array2D<int> direct(dim,2); direct = 0;
+            if (i==-1) direct[0][0] = 1;
+            if (i==1) direct[0][1] = 1;
+            if (j==-1) direct[1][0] = 1;
+            if (j==1) direct[1][1] = 1;
+            Vector cfsol0;
+            GetCutOffSolution(sol0,cfsol0,ip0,direct,ovlpnrlayers,true);
+
+
+            Array<int> directions(2);
+            directions[0] = i;
+            directions[1] = j;
+            Vector raux;
+            SourceTransfer(cfsol0,directions,ip0,raux);
+            // SourceTransfer1(cfsol0,directions,ip0,raux);
+            *f_transf[ip1][l]+=raux;
+            break;
+         }
+      }  
+   }
+}
+
+SparseMatrix * DST2D::GetPmlSystemMatrix(int ip)
+{
+   double h = GetUniformMeshElementSize(povlp->patch_mesh[ip]);
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+
+   int i,j,k;
+   int nx = nxyz[0];
+   int ny = nxyz[1];
+   Getijk(ip,i,j,k);
+   if (i == 0 ) length[0][0] = Pmllength[0][0];
+   if (j == 0 ) length[1][0] = Pmllength[1][0];
+   if (i == nx-1 ) length[0][1] = Pmllength[0][1];
+   if (j == ny-1 ) length[1][1] = Pmllength[1][1];
+
+   CartesianPML pml(povlp->patch_mesh[ip], length);
+   pml.SetOmega(omega);
+
+   Array <int> ess_tdof_list;
+   if (povlp->patch_mesh[ip]->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(povlp->patch_mesh[ip]->bdr_attributes.Max());
+      ess_bdr = 1;
+      ovlp_prob->fespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+   ProductCoefficient c2_re(c2_re0, *ws);
+   ProductCoefficient c2_im(c2_im0, *ws);
+   SesquilinearForm a(ovlp_prob->fespaces[ip],ComplexOperator::HERMITIAN);
+
+   a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   a.AddDomainIntegrator(new MassIntegrator(c2_re),
+                         new MassIntegrator(c2_im));
+   a.Assemble();
+
+   OperatorPtr Alocal;
+   a.FormSystemMatrix(ess_tdof_list,Alocal);
+   ComplexSparseMatrix * AZ_ext = Alocal.As<ComplexSparseMatrix>();
+   SparseMatrix * Mat = AZ_ext->GetSystemMatrix();
+   Mat->Threshold(1e-13);
+   return Mat;
+}
+
+void DST2D::PlotSolution(Vector & sol, socketstream & sol_sock, int ip, 
+                  bool localdomain) const
+{
+   FiniteElementSpace * fes;
+   if (!localdomain)
+   {
+      fes = bf->FESpace();
+   }
+   else
+   {
+      fes = ovlp_prob->fespaces[ip];
+   }
+   Mesh * mesh = fes->GetMesh();
+   GridFunction gf(fes);
+   double * data = sol.GetData();
+   gf.SetData(data);
+   
+   string keys;
+   // if (ip == 0) 
+   keys = "keys mrRljc\n";
+   // sol_sock << "solution\n" << *mesh << gf << keys << "valuerange -0.1 0.1 \n"  << flush;
+   sol_sock << "solution\n" << *mesh << gf << keys << flush;
+}
+
+void DST2D::PlotMesh(socketstream & mesh_sock, int ip) const
+{
+   FiniteElementSpace * fes = ovlp_prob->fespaces[ip];
+   Mesh * mesh = fes->GetMesh();
+   mesh_sock << "mesh\n" << *mesh << flush;
+}
+
+void DST2D::SaveSolution(Vector & sol, int ip, bool localdomain) const
+{
+   FiniteElementSpace * fes;
+   if (!localdomain)
+   {
+      fes = bf->FESpace();
+   }
+   else
+   {
+      // fes = ovlp_prob->fespaces[ip];
+      fes = nvlp_prob->fespaces[ip];
+   }
+   Mesh * mesh = fes->GetMesh();
+   int n = fes->GetTrueVSize();
+   GridFunction gf_re(fes);
+   GridFunction gf_im(fes);
+   double * data = sol.GetData();
+   gf_re.SetData(data);
+   gf_im.SetData(&data[n]);
+   cout << "saving mesh no " << ip << endl;
+   // string mfilename = "output/globalmesh.";
+   string mfilename = "output/mesh_nvlp.";
+   ostringstream mesh_name;
+   mesh_name << mfilename << setfill('0') << setw(6) << ip;
+   ofstream mesh_ofs(mesh_name.str().c_str());
+   mesh_ofs.precision(8);
+   mesh->Print(mesh_ofs);
+
+   // string sfilename_re = "output/sol_re.";
+   // string sfilename_im = "output/sol_im.";
+   string sfilename_re = "output/sol_nvlp.";
+   // string sfilename_im = "output/sol_im.";
+
+   ostringstream solre_name;
+   solre_name << sfilename_re << setfill('0') << setw(6) << ip;
+   ofstream solre_ofs(solre_name.str().c_str());
+   gf_re.Save(solre_ofs);
+
+   // ostringstream solim_name;
+   // solim_name << sfilename_im << setfill('0') << setw(6) << ip;
+   // ofstream solim_ofs(solim_name.str().c_str());
+   // gf_im.Save(solim_ofs);
+}
+
+void DST2D::SourceTransfer(const Vector & Psi0, Array<int> direction, int ip0, Vector & Psi1) const
+{
+   int i0,j0,k0;
+   Getijk(ip0,i0,j0,k0);
+
+   int i1 = i0+direction[0];   
+   int j1 = j0+direction[1];   
+   Array<int> ij(2); ij[0]=i1; ij[1]=j1;
+   int ip1 = GetPatchId(ij);
+
+   MFEM_VERIFY(i1 < nxyz[0] && i1>=0, "SourceTransfer: i1 out of bounds");
+   MFEM_VERIFY(j1 < nxyz[1] && j1>=0, "SourceTransfer: j1 out of bounds");
+
+   Array<int> * Dof2GlobalDof0 = &ovlp_prob->Dof2GlobalDof[ip0];
+   Array<int> * Dof2GlobalDof1 = &ovlp_prob->Dof2GlobalDof[ip1];
+   Psi1.SetSize(Dof2GlobalDof1->Size()); Psi1=0.0;
+   Vector r(2*bf->FESpace()->GetTrueVSize());
+   r = 0.0;
+   r.SetSubVector(*Dof2GlobalDof0,Psi0);
+   Vector zloc(Psi1.Size()); zloc = 0.0;
+   r.GetSubVector(*Dof2GlobalDof1,zloc);
+
+   Vector Psi(Dof2GlobalDof1->Size()); Psi=0.0;
+
+   PmlMat[ip1]->Mult(zloc,Psi);
+   Psi *=-1.0;
+
+   Array2D<int> direct(dim,2); direct = 0;
+   if (direction[0]==1) direct[0][0] = 1;
+   if (direction[0]==-1) direct[0][1] = 1;
+   if (direction[1]==1) direct[1][0] = 1;
+   if (direction[1]==-1) direct[1][1] = 1;
+
+   GetChiRes(Psi, Psi1,ip1,direct, ovlpnrlayers);
+
+
+}
+
+void DST2D::GetChiRes(const Vector & res, Vector & cfres, 
+                    int ip, Array<int> directions, int nlayers) const
+{
+
+   Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+   double h = GetUniformMeshElementSize(mesh);
+   
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+   Array2D<double> pmlh(dim,2); pmlh = 0.0;
+   int i,j,k;
+   Getijk(ip,i,j,k);
+   if (directions[0]==-1)
+   {
+      pmlh[0][0] = h;
+      pmin[0] += h*(nlayers-1);
+   }
+   if (directions[0]==1)
+   {
+      pmlh[0][1] = h;
+      pmax[0] -= h*(nlayers-1);
+
+   }
+   if (directions[1]==-1)
+   {
+      pmlh[1][0] = h;
+      pmin[1] += h*(nlayers-1);
+   }
+   if (directions[1]==1)
+   {
+      pmlh[1][1] = h;
+      pmax[1] -= h*(nlayers-1);
+   }
+   CutOffFnCoefficient cf(ChiFncn, pmin, pmax, pmlh);
+
+   double * data = res.GetData();
+
+   FiniteElementSpace * fespace;
+   fespace = ovlp_prob->fespaces[ip];
+   
+   int n = fespace->GetTrueVSize();
+
+   GridFunction solgf_re(fespace, data);
+   GridFunction solgf_im(fespace, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fespace);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   cfres.SetSize(res.Size());
+   cfres = gf;
+}
+
+
+void DST2D::GetChiRes(const Vector & res, Vector & cfres, 
+                    int ip, Array2D<int> direct, int nlayers) const
+{
+
+   Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+   double h = GetUniformMeshElementSize(mesh);
+   
+   Array2D<double> pmlh(dim,2); pmlh = 0.0;
+
+   for (int i=0; i<dim; i++)
+   {
+      if (direct[i][0]==1) pmin[i] += h*(nlayers-1); 
+      if (direct[i][1]==1) pmax[i] -= h*(nlayers-1); 
+      for (int j=0; j<2; j++)
+      {
+         if (direct[i][j]==1)
+         {
+            pmlh[i][j] = h;
+         }
+      }  
+   }
+
+   CutOffFnCoefficient cf(ChiFncn, pmin, pmax, pmlh);
+
+   double * data = res.GetData();
+
+   FiniteElementSpace * fespace;
+   fespace = ovlp_prob->fespaces[ip];
+   
+   int n = fespace->GetTrueVSize();
+
+   GridFunction solgf_re(fespace, data);
+   GridFunction solgf_im(fespace, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fespace);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   cfres.SetSize(res.Size());
+   cfres = gf;
+}
+
+
+void DST2D::SourceTransfer1(const Vector & Psi0, Array<int> direction, int ip0, Vector & Psi1) const
+{
+   int i0,j0,k0;
+   Getijk(ip0,i0,j0,k0);
+
+   int i1 = i0+direction[0];   
+   int j1 = j0+direction[1];   
+   Array<int> ij(2); ij[0]=i1; ij[1]=j1;
+   int ip1 = GetPatchId(ij);
+
+   MFEM_VERIFY(i1 < nxyz[0] && i1>=0, "SourceTransfer: i1 out of bounds");
+   MFEM_VERIFY(j1 < nxyz[1] && j1>=0, "SourceTransfer: j1 out of bounds");
+
+   Vector Psi(Psi0.Size());
+   PmlMat[ip0]->Mult(Psi0,Psi);
+   Psi *=-1.0;
+
+   Array<int> * Dof2GlobalDof0 = &ovlp_prob->Dof2GlobalDof[ip0];
+   Array<int> * Dof2GlobalDof1 = &ovlp_prob->Dof2GlobalDof[ip1];
+   Vector r(2*bf->FESpace()->GetTrueVSize());
+   r = 0.0;
+   r.SetSubVector(*Dof2GlobalDof0,Psi);
+   Psi1.SetSize(Dof2GlobalDof1->Size()); Psi1=0.0;
+   r.GetSubVector(*Dof2GlobalDof1,Psi1);
+
+   // Array<int> direct(2); direct = 0;
+   // direct[0] = -direction[0];
+   // direct[1] = -direction[1];
+   Psi = Psi1;
+
+   Array2D<int> direct(dim,2); direct = 0;
+   if (direction[0]==1) direct[0][0] = 1;
+   if (direction[0]==-1) direct[0][1] = 1;
+   if (direction[1]==1) direct[1][0] = 1;
+   if (direction[1]==-1) direct[1][1] = 1;
+
+   GetChiRes(Psi, Psi1,ip1,direct, ovlpnrlayers);
+
+}
+

examples/maxwell-solver/DST2D/DST2D.hpp

@@ -0,0 +1,65 @@
+#pragma once
+#include "../common/Utilities.hpp"
+#include "../common/PML.hpp"
+using namespace std;
+using namespace mfem;
+
+class DST2D : public Solver//
+{
+private:
+   int nrpatch;
+   int dim;
+   MeshPartition * povlp=nullptr;
+   MeshPartition * novlp=nullptr;
+   int ovlpnrlayers;
+   int nxyz[3];
+   const Operator * A=nullptr;
+   DofMap * ovlp_prob = nullptr;
+   DofMap * nvlp_prob = nullptr;
+   Array<SparseMatrix *> PmlMat;
+   Array<KLUSolver *> PmlMatInv;
+   Array3D<int> subdomains;
+   mutable Array<Vector *> f_orig;
+   int nsweeps;
+   Array2D<int> sweeps;
+   mutable Array<Array<Vector * >> f_transf;
+
+   SesquilinearForm *bf=nullptr;
+   Array2D<double> Pmllength;
+   double omega = 0.5;
+   Coefficient * ws;
+   int nrlayers;
+
+
+   SparseMatrix * GetPmlSystemMatrix(int ip);
+  
+   void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+                          int ip, Array<int> directions, int nlayers, bool local=false) const;
+   void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+                          int ip, Array2D<int> directions, int nlayers, bool local=false) const;                          
+   void GetChiRes(const Vector & res, Vector & cfres,
+                  int ip, Array<int> directions, int nlayers) const;  
+   void GetChiRes(const Vector & res, Vector & cfres,
+                  int ip, Array2D<int> directions, int nlayers) const;  
+   void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+   int GetPatchId(const Array<int> & ijk) const;
+   void Getijk(int ip, int & i, int & j, int & k ) const;
+   void SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+   void SourceTransfer1(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+   void PlotSolution(Vector & sol, socketstream & sol_sock, int ip,bool localdomain) const;
+   void SaveSolution(Vector & sol, int ip,bool localdomain) const;
+   void PlotMesh(socketstream & mesh_sock, int ip) const;
+  
+   // void SourceTransfer1(const Vector & Psi0, Array<int> direction, int ip0, Vector & Psi1) const;
+   // void SetSubMeshesAttributes();
+   // void GetRestrCoeffAttr(const Array<int> & directions, Array<int> & attr) const;
+   // double GetSolOvlpNorm(const Vector & sol, const Array<int> & directions, int ip) const;
+public:
+   DST2D(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+       double omega_, Coefficient * ws_, int nrlayers_);
+   virtual void SetOperator(const Operator &op) {A = &op;}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~DST2D();
+};
+
+

examples/maxwell-solver/DST2D/MeshPartition2D.cpp

@@ -0,0 +1,707 @@
+// #include "MeshPartition2D.hpp"
+
+// double GetUniformMeshElementSize(Mesh * mesh)
+// {
+//    int dim = mesh->Dimension();
+//    int nrelem = mesh->GetNE();
+
+//    DenseMatrix J(dim);
+//    double hmin, hmax;
+//    hmin = infinity();
+//    hmax = -infinity();
+//    Vector attr(nrelem);
+//    for (int iel=0; iel<nrelem; ++iel)
+//    {
+//       int geom = mesh->GetElementBaseGeometry(iel);
+//       ElementTransformation *T = mesh->GetElementTransformation(iel);
+//       T->SetIntPoint(&Geometries.GetCenter(geom));
+//       Geometries.JacToPerfJac(geom, T->Jacobian(), J);
+//       attr(iel) = J.Det();
+//       attr(iel) = pow(abs(attr(iel)), 1.0/double(dim));
+//       hmin = min(hmin, attr(iel));
+//       hmax = max(hmax, attr(iel));
+//    }
+//    MFEM_VERIFY(abs(hmin-hmax) < 1e-12, "Case not supported yet")
+
+//    return hmax;
+// }
+
+// Mesh * ExtendMesh(Mesh * mesh, const Array<int> & directions)
+// {
+//     // extrute on one dimension
+//    // flag = 1 +x, -1 -x, 2 +y, -2 +y , 3 +z, -3, -z
+
+//    // copy the original mesh;
+//    Mesh * mesh_orig = new Mesh(*mesh);
+//    if (!directions.Size()) return mesh_orig;
+   
+//    int dim = mesh_orig->Dimension();
+
+//    Mesh * mesh_ext=nullptr;
+
+//    for (int j=0; j<directions.Size(); j++)
+//    {
+//       int d = directions[j];
+//       MFEM_VERIFY(abs(d)<= dim, "Cannot Extend in dimension " << d << ". Dim = " << dim << endl);
+
+//       Vector pmin;
+//       Vector pmax;
+//       mesh_orig->GetBoundingBox(pmin,pmax);
+//       double h = GetUniformMeshElementSize(mesh_orig);
+//       double val;
+//       // find the vertices on the specific boundary
+//       switch (d)
+//       {
+//       case 1:
+//          val = pmax[0];
+//          break;
+//       case -1:
+//          val = pmin[0];
+//          h = -h;
+//          break;    
+//       case 2:
+//          val = pmax[1];
+//          break;
+//       case -2:
+//          val = pmin[1];
+//          h = -h;
+//          break;   
+//       case 3:
+//          val = pmax[2];
+//          break;
+//       case -3:
+//          val = pmin[2];
+//          h = -h;
+//          break;      
+//       }
+//       int k = 0;
+//       for (int i = 0; i<mesh_orig->GetNV(); ++i)
+//       {
+//          double * coords = mesh_orig->GetVertex(i);
+//          switch (abs(d))
+//          {
+//          case 1:
+//             if (coords[0] == val) k++;
+//             break;
+//          case 2:
+//             if (coords[1] == val) k++;
+//             break;   
+//          case 3:
+//             if (coords[2] == val) k++;
+//             break;      
+//          }
+//       }
+//       int nrvertices = mesh_orig->GetNV() + k;
+//       int nrelements = mesh_orig->GetNE() + pow(pow(k,1.0/(dim-1))-1.0,dim-1);
+
+//       mesh_ext = new Mesh(dim, nrvertices, nrelements);
+
+//       // Add existing vertices
+//       Array<int> vmap(mesh_orig->GetNV()); vmap = 0;
+//       k = mesh_orig->GetNV();
+//       for (int i=0; i<mesh_orig->GetNV(); ++i)
+//       {
+//          double * vert = mesh_orig->GetVertex(i);
+//          mesh_ext->AddVertex(vert);
+//          switch (abs(d))
+//          {
+//          case 1:
+//             if (vert[0] == val) 
+//             {
+//                vmap[i] = k;
+//                k++;
+//             }
+//             break;
+//          case 2:
+//             if (vert[1] == val)
+//             {
+//                vmap[i] = k;
+//                k++;
+//             }
+//             break;   
+//          case 3:
+//             if (vert[2] == val)
+//             {
+//                vmap[i] = k;
+//                k++;
+//             }
+//             break;      
+//          }
+//       } 
+//       // Add existing elements
+//       for (int i=0; i<mesh_orig->GetNE(); ++i)
+//       {
+//          Array<int>ind;
+//          mesh_orig->GetElementVertices(i,ind);
+//          if (dim == 2)
+//          {
+//             mesh_ext->AddQuad(ind);
+//          }
+//          else if (dim == 3)
+//          {
+//             mesh_ext->AddHex(ind);
+//          }
+//       } 
+//       // Add new vertices
+//       k = mesh_orig->GetNV();
+//       for (int i=0; i<mesh_orig->GetNV(); ++i)
+//       {
+//          double * vert = mesh_orig->GetVertex(i);
+//          switch (abs(d))
+//          {
+//          case 1:
+//             if (vert[0] == val) 
+//             {
+//                double coords[dim];
+//                coords[0] = vert[0] + h;
+//                coords[1] = vert[1];
+//                if (dim == 3) coords[2] = vert[2];
+//                mesh_ext->AddVertex(coords);
+//             }
+//             break;
+//          case 2:
+//             if (vert[1] == val)
+//             {
+//                double coords[dim];
+//                coords[0] = vert[0];
+//                coords[1] = vert[1] + h;
+//                if (dim == 3) coords[2] = vert[2];
+//                mesh_ext->AddVertex(coords);
+//             }
+//             break;   
+//          case 3:
+//             if (vert[2] == val)
+//             {
+//                double coords[dim];
+//                coords[0] = vert[0];
+//                coords[1] = vert[1];
+//                coords[2] = vert[2] + h;
+//                mesh_ext->AddVertex(coords);
+//             }
+//             break;      
+//          }
+//       }    
+//       // loop through boundary elements and extend in the given direction
+//       for (int i=0; i<mesh_orig->GetNBE(); ++i) 
+//       {
+//          Array<int> vertices;
+//          mesh_orig->GetBdrElementVertices(i,vertices);
+//          if (dim == 2)
+//          {
+//             int ind[4];
+//             if (vmap[vertices[0]] && vmap[vertices[1]])
+//             {
+//                ind[0] = vmap[vertices[0]];
+//                ind[1] = vmap[vertices[1]];
+//                ind[2] = vertices[1];
+//                ind[3] = vertices[0];
+//                mesh_ext->AddQuad(ind);
+//             }
+//          }
+//          else if (dim == 3)
+//          {
+//             int ind[8];
+//             if (vmap[vertices[0]] && vmap[vertices[1]] && vmap[vertices[2]] && vmap[vertices[3]])
+//             {
+//                ind[0] = vmap[vertices[0]];
+//                ind[1] = vmap[vertices[1]];
+//                ind[2] = vmap[vertices[2]];
+//                ind[3] = vmap[vertices[3]];
+//                ind[4] = vertices[0];
+//                ind[5] = vertices[1];
+//                ind[6] = vertices[2];
+//                ind[7] = vertices[3];
+//                mesh_ext->AddHex(ind);
+//             }
+//          }   
+//       }
+//       mesh_ext->FinalizeTopology();
+
+//       if (j<directions.Size()-1)
+//       {
+//          delete mesh_orig;
+//          mesh_orig = mesh_ext;
+//       }
+//    }
+//    delete mesh_orig;
+//    return mesh_ext;
+// }
+
+// // constructor
+   
+// OverlappingCartesianMeshPartition::OverlappingCartesianMeshPartition(Mesh *mesh_,int & nx,int & ny,int & nz) : mesh(mesh_)
+// {  // default overlap size is 2 elements 
+//    int dim = mesh->Dimension();
+//    int n = pow(mesh->GetNE(), 1.0/(double)dim);
+//    if (nx > n) 
+//    {
+//       nx = n;
+//       MFEM_WARNING("Changed partition in the x direction to nx = " << n << endl);
+//    }
+//    if (ny > n) 
+//    {
+//       ny = n;
+//       MFEM_WARNING("Changed partition in the y direction to ny = " << n << endl);
+//    }
+//    if (nz > n) 
+//    {
+//       nz = n;
+//       MFEM_WARNING("Changed partition in the z direction to nz = " << n << endl);
+//    } 
+//    if (dim == 2) nz = 1;
+//    subdomains.SetSize(nx,ny,nz);
+//    nxyz[0] = nx; nxyz[1]=ny; nxyz[2] = nz;
+//    nrpatch = nx*ny*nz;
+//    Vector pmin, pmax;
+//    mesh->GetBoundingBox(pmin, pmax);
+//    double h = GetUniformMeshElementSize(mesh);
+
+//    element_map.resize(nrpatch);
+
+//    double ppt[dim];
+//    Vector pt(ppt, dim);
+//    int nrelem = mesh->GetNE();
+
+//    for (int el = 0; el < nrelem; el++)
+//    {
+//       mesh->GetElementTransformation(el)->Transform(
+//          Geometries.GetCenter(mesh->GetElementBaseGeometry(el)), pt);
+//       // Given the center coordinates determine the patches that this element contributes to
+//       Array<int> idx0(dim);
+//       Array<int> idx1(dim);
+//       Array<int> idx2(dim);
+//       vector<Array<int>> idx(3);
+//       if (dim == 2) idx[2].Append(0);
+
+//       for (int i = 0; i<dim; i++)
+//       {
+//          idx0[i]  = (int)floor(nxyz[i]*((pt(i) - pmin[i])/(pmax[i] - pmin[i])));
+//          idx1[i] = (int)floor(nxyz[i]*((pt(i)+h - pmin[i])/(pmax[i] - pmin[i])));
+//          idx2[i] = (int)floor(nxyz[i]*((pt(i)-h - pmin[i])/(pmax[i] - pmin[i])));
+
+//          if (idx0[i] < 0) idx0[i] = 0;
+//          if (idx0[i] >= nxyz[i]) idx0[i] = nxyz[i]-1;
+         
+//          if (idx1[i] < 0) idx1[i] = 0;
+//          if (idx1[i] >= nxyz[i]) idx1[i] = nxyz[i]-1;
+
+//          if (idx2[i] < 0) idx2[i] = 0;
+//          if (idx2[i] >= nxyz[i]) idx2[i] = nxyz[i]-1;
+//          // convenient to put in one list
+//          idx[i].Append(idx0[i]);
+//          if (idx1[i] != idx0[i]) idx[i].Append(idx1[i]);
+//          if (idx2[i] != idx0[i] && idx2[i] != idx1[i]) idx[i].Append(idx2[i]);
+//       }
+//       // Now loop through all the combinations according to the idx above
+//       // in case of dim = 2 then kk = 0
+//       for (int k=0; k<idx[2].Size(); k++)
+//       {
+//          int kk = idx[2][k];
+//          for (int j=0; j<idx[1].Size(); j++)
+//          {
+//             int jj = idx[1][j];
+//             for (int i=0; i<idx[0].Size(); i++)
+//             {
+//                int ii = idx[0][i];
+//                int ip = kk*nxyz[0]*nxyz[1] + jj*nxyz[0]+ii;
+//                element_map[ip].Append(el);
+//             }
+//          }
+//       }
+//    }
+//    for (int k = 0; k<nz; k++)
+//    {
+//       for (int j = 0; j<ny; j++)
+//       {
+//          for (int i = 0; i<nx; i++)
+//          {
+//             subdomains(i,j,k) = k*ny*nx + j*nx + i;
+//          }
+//       }   
+//    }
+// }
+
+
+// OverlappingCartesianMeshPartition::OverlappingCartesianMeshPartition(Mesh *mesh_,int & nx,int & ny,int & nz, int ovlp_nlayers) : mesh(mesh_)
+// {  // default overlap size is 2 elements 
+//    int dim = mesh->Dimension();
+//    int n = pow(mesh->GetNE(), 1.0/(double)dim);
+//    if (nx > n) 
+//    {
+//       nx = n;
+//       MFEM_WARNING("Changed partition in the x direction to nx = " << n << endl);
+//    }
+//    if (ny > n) 
+//    {
+//       ny = n;
+//       MFEM_WARNING("Changed partition in the y direction to ny = " << n << endl);
+//    }
+//    if (nz > n) 
+//    {
+//       nz = n;
+//       MFEM_WARNING("Changed partition in the z direction to nz = " << n << endl);
+//    } 
+//    if (dim == 2) nz = 1;
+//    subdomains.SetSize(nx,ny,nz);
+//    nxyz[0] = nx; nxyz[1]=ny; nxyz[2] = nz;
+//    nrpatch = nx*ny*nz;
+//    Vector pmin, pmax;
+//    mesh->GetBoundingBox(pmin, pmax);
+//    double h = GetUniformMeshElementSize(mesh);
+//    cout << "h = " << h << endl;
+
+//    // Check that ovlp_size does not exit subdomain size
+//    MFEM_VERIFY((pmax[0]-pmin[0])/nx >= h*ovlp_nlayers, 
+//                 "Check ovlp size in partition"); 
+//                 cout << "pmax[0]-pmin[0])/nx = " << (pmax[0]-pmin[0])/nx << endl;
+//                 cout << "ovlp_nlayers = " << ovlp_nlayers << endl;
+//                 cout << "h*ovlp_nlayers = " << h*ovlp_nlayers << endl;
+//    MFEM_VERIFY((pmax[1]-pmin[1])/ny >= h*ovlp_nlayers, 
+//                "Check ovlp size in partition"); 
+//    if (dim == 3)
+//    {
+//       MFEM_VERIFY((pmax[2]-pmin[2])/nz >= h*ovlp_nlayers, 
+//                "Check ovlp size in partition"); 
+//    }
+//    element_map.resize(nrpatch);
+
+//    double ppt[dim];
+//    Vector pt(ppt, dim);
+//    int nrelem = mesh->GetNE();
+
+//    for (int el = 0; el < nrelem; el++)
+//    {
+//       mesh->GetElementTransformation(el)->Transform(
+//          Geometries.GetCenter(mesh->GetElementBaseGeometry(el)), pt);
+//       // Given the center coordinates determine the patches that this element contributes to
+//       Array<int> idx0(dim);
+//       Array<int> idx1(dim);
+//       Array<int> idx2(dim);
+//       vector<Array<int>> idx(3);
+//       if (dim == 2) idx[2].Append(0);
+
+//       for (int i = 0; i<dim; i++)
+//       {
+//          idx0[i]  = (int)floor(nxyz[i]*((pt(i) - pmin[i])/(pmax[i] - pmin[i])));
+//          idx1[i] = (int)floor(nxyz[i]*((pt(i)+ovlp_nlayers*h - pmin[i])/(pmax[i] - pmin[i])));
+//          idx2[i] = (int)floor(nxyz[i]*((pt(i)-ovlp_nlayers*h - pmin[i])/(pmax[i] - pmin[i])));
+
+//          if (idx0[i] < 0) idx0[i] = 0;
+//          if (idx0[i] >= nxyz[i]) idx0[i] = nxyz[i]-1;
+         
+//          if (idx1[i] < 0) idx1[i] = 0;
+//          if (idx1[i] >= nxyz[i]) idx1[i] = nxyz[i]-1;
+
+//          if (idx2[i] < 0) idx2[i] = 0;
+//          if (idx2[i] >= nxyz[i]) idx2[i] = nxyz[i]-1;
+//          // convenient to put in one list
+//          idx[i].Append(idx0[i]);
+//          if (idx1[i] != idx0[i]) idx[i].Append(idx1[i]);
+//          if (idx2[i] != idx0[i] && idx2[i] != idx1[i]) idx[i].Append(idx2[i]);
+//       }
+//       // Now loop through all the combinations according to the idx above
+//       // in case of dim = 2 then kk = 0
+//       for (int k=0; k<idx[2].Size(); k++)
+//       {
+//          int kk = idx[2][k];
+//          for (int j=0; j<idx[1].Size(); j++)
+//          {
+//             int jj = idx[1][j];
+//             for (int i=0; i<idx[0].Size(); i++)
+//             {
+//                int ii = idx[0][i];
+//                int ip = kk*nxyz[0]*nxyz[1] + jj*nxyz[0]+ii;
+//                element_map[ip].Append(el);
+//             }
+//          }
+//       }
+//    }
+//    for (int k = 0; k<nz; k++)
+//    {
+//       for (int j = 0; j<ny; j++)
+//       {
+//          for (int i = 0; i<nx; i++)
+//          {
+//             subdomains(i,j,k) = k*ny*nx + j*nx + i;
+//          }
+//       }   
+//    }
+// }
+
+// // constructor
+// CartesianMeshPartition::CartesianMeshPartition(Mesh *mesh_,int & nx, int & ny, int & nz) : mesh(mesh_)
+// {
+//    int dim = mesh->Dimension();
+//    nxyz[0] = nx;
+//    nxyz[1] = ny;
+//    nxyz[2] = nz;
+//    nrpatch = nx*ny*nz;
+//    subdomains.SetSize(nx,ny,nz);
+//    Vector pmin, pmax;
+//    mesh->GetBoundingBox(pmin, pmax);
+
+//    int nrelem = mesh->GetNE();
+//    int partitioning[nrelem];
+
+//    // determine the partitioning using the centers of the elements
+//    double ppt[dim];
+//    Vector pt(ppt, dim);
+//    for (int el = 0; el < nrelem; el++)
+//    {
+//       mesh->GetElementTransformation(el)->Transform(
+//          Geometries.GetCenter(mesh->GetElementBaseGeometry(el)), pt);
+//       int part = 0;
+//       for (int i = dim-1; i >= 0; i--)
+//       {
+//          int idx = (int)floor(nxyz[i]*((pt(i) - pmin[i])/(pmax[i] - pmin[i])));
+//          if (idx < 0)
+//          {
+//             idx = 0;
+//          }
+//          if (idx >= nxyz[i])
+//          {
+//             idx = nxyz[i]-1;
+//          }
+//          part = part * nxyz[i] + idx;
+//       }
+//       partitioning[el] = part;
+//    }
+
+//    element_map.resize(nrpatch);
+//    for (int iel = 0; iel < nrelem; iel++)
+//    {
+//       int ip = partitioning[iel];
+//       element_map[ip].Append(iel);
+//    }
+//    for (int k = 0; k<nz; k++)
+//    {
+//       for (int j = 0; j<ny; j++)
+//       {
+//          for (int i = 0; i<nx; i++)
+//          {
+//             subdomains(i,j,k) = k*ny*nx + j*nx + i;
+//          }
+//       }   
+//    }
+// }
+
+// STPOverlappingCartesianMeshPartition::STPOverlappingCartesianMeshPartition(Mesh *mesh_) : mesh(mesh_)
+// {
+//    int dim = mesh->Dimension();
+//    nx = 5;
+//    ny = 1;
+//    nz = 1;
+//    int nxyz[3] = {nx,ny,nz};
+//    // nrpatch = nx*ny*nz;
+
+//    Vector pmin, pmax;
+//    mesh->GetBoundingBox(pmin, pmax);
+
+//    int nrelem = mesh->GetNE();
+//    int partitioning[nrelem];
+
+//    // determine the partitioning using the centers of the elements
+//    double ppt[dim];
+//    Vector pt(ppt, dim);
+//    for (int el = 0; el < nrelem; el++)
+//    {
+//       mesh->GetElementTransformation(el)->Transform(
+//          Geometries.GetCenter(mesh->GetElementBaseGeometry(el)), pt);
+//       int part = 0;
+//       for (int i = dim-1; i >= 0; i--)
+//       {
+//          int idx = (int)floor(nxyz[i]*((pt(i) - pmin[i])/(pmax[i] - pmin[i])));
+//          if (idx < 0)
+//          {
+//             idx = 0;
+//          }
+//          if (idx >= nxyz[i])
+//          {
+//             idx = nxyz[i]-1;
+//          }
+//          part = part * nxyz[i] + idx;
+//       }
+//       partitioning[el] = part;
+//    }
+
+//    std::vector<Array<int>> elem_map;
+//    int npatch = nx*ny*nz;
+//    elem_map.resize(npatch);
+//    for (int iel = 0; iel < nrelem; iel++)
+//    {
+//       int ip = partitioning[iel];
+//       elem_map[ip].Append(iel);
+//    }
+//    // Append the next subdomain to the previous
+//    nrpatch = nx*ny*nz-1;
+//    element_map.resize(nrpatch);
+//    for (int ip = 0; ip<nrpatch; ip++)
+//    {
+//       element_map[ip].Append(elem_map[ip]);
+//       element_map[ip].Append(elem_map[ip+1]);
+//    }
+// }
+
+// MeshPartition::MeshPartition(Mesh* mesh_, int part,int nx, int ny, int nz, int nrlayers): mesh(mesh_)
+// {
+//    partition_kind = part;
+//    if (part == 1)
+//    {
+//       cout << "Non Overlapping Cartesian Partition " << endl;
+//       CartesianMeshPartition partition(mesh,nx, ny, nz);
+//       element_map = partition.element_map;
+//       // subdomains = partition.subdomains;
+//    }
+//    // else if (part == 3 || part == 4)
+//    else if (part == 2)
+//    {
+//       cout << "Overlapping Cartesian Partition " << endl;
+//       OverlappingCartesianMeshPartition partition(mesh,nx, ny, nz,nrlayers);
+//       element_map = partition.element_map;
+//       subdomains = partition.subdomains;
+//       nxyz[0] = partition.nxyz[0];
+//       nxyz[1] = partition.nxyz[1];
+//       nxyz[2] = partition.nxyz[2];
+//    }
+//    else if (part == 3 || part == 4)
+//    // else if (part == 2)
+//    {
+//       cout << "STP Overlapping Cartesian Partition " << endl;
+//       STPOverlappingCartesianMeshPartition partition(mesh);
+//       element_map = partition.element_map;
+//    }
+//    else
+//    {
+//       MFEM_ABORT("Overlapping Vertex based partition not supprorted")
+//    }
+
+//    nrpatch = element_map.size();
+
+//    int dim = mesh->Dimension();
+
+//    patch_mesh.SetSize(nrpatch);
+//    for (int ip = 0; ip<nrpatch; ++ip)
+//    {
+//       int patch_nrelems = element_map[ip].Size();
+//       element_map[ip].SetSize(patch_nrelems);
+//       Array<int> patch_vertices;
+//       for (int iel=0; iel<patch_nrelems; ++iel)
+//       {
+//          // get the vertices list for the element
+//          Array<int> elem_vertices;
+//          int iel_idx = element_map[ip][iel];
+//          mesh->GetElementVertices(iel_idx,elem_vertices);
+//          patch_vertices.Append(elem_vertices);
+//       }
+//       patch_vertices.Sort();
+//       patch_vertices.Unique();
+//       int patch_nrvertices = patch_vertices.Size();
+
+//       // create the mesh
+//       patch_mesh[ip] = new Mesh(dim,patch_nrvertices,patch_nrelems);
+//       // Add the vertices
+//       for (int iv = 0; iv<patch_nrvertices; ++iv)
+//       {
+//          int vert_idx = patch_vertices[iv];
+//          patch_mesh[ip]->AddVertex(mesh->GetVertex(vert_idx));
+//       }
+
+//       // Add the elements (for now search through all the vertices in the patch is needed)
+//       for (int iel=0; iel<patch_nrelems; ++iel)
+//       {
+//          // get the vertices list for the element
+//          Array<int> elem_vertices;
+//          int iel_idx = element_map[ip][iel];
+//          mesh->GetElementVertices(iel_idx,elem_vertices);
+//          int nrvert = elem_vertices.Size();
+//          int ind[nrvert];
+//          for (int iv = 0; iv<nrvert; ++iv)
+//          {
+//             ind[iv] = patch_vertices.FindSorted(elem_vertices[iv]);
+//          }
+//          mfem::Element::Type elem_type = mesh->GetElementType(element_map[ip][iel]);
+
+//          AddElementToMesh(patch_mesh[ip],elem_type,ind);
+
+//       }
+//       patch_mesh[ip]->FinalizeTopology();
+//    }
+// }
+
+// void MeshPartition::AddElementToMesh(Mesh * mesh,mfem::Element::Type elem_type,
+//                                      int * ind)
+// {
+//    switch (elem_type)
+//    {
+//       case Element::QUADRILATERAL:
+//          mesh->AddQuad(ind);
+//          break;
+//       case Element::TRIANGLE :
+//          mesh->AddTri(ind);
+//          break;
+//       case Element::HEXAHEDRON :
+//          mesh->AddHex(ind);
+//          break;
+//       case Element::TETRAHEDRON :
+//          mesh->AddTet(ind);
+//          break;
+//       default:
+//          MFEM_ABORT("Unknown element type");
+//          break;
+//    }
+// }
+
+// void MeshPartition::PrintElementMap()
+// {
+//    mfem::out << "Element map" << endl;
+//    for (int ip = 0; ip<nrpatch; ++ip)
+//    {
+//       mfem::out << "Patch No: " << ip;
+//       mfem::out << ", element map: " ;
+//       element_map[ip].Print(cout,element_map[ip].Size());
+//    }
+// }
+
+// void SaveMeshPartition(Array<Mesh *> meshes, string mfilename, string sfilename)
+// {
+//    int nrmeshes = meshes.Size();
+//    for (int ip = 0; ip<nrmeshes; ++ip)
+//    {
+//       cout << "saving mesh no " << ip << endl;
+//       ostringstream mesh_name;
+//       mesh_name << mfilename << setfill('0') << setw(6) << ip;
+//       ofstream mesh_ofs(mesh_name.str().c_str());
+//       mesh_ofs.precision(8);
+//       meshes[ip]->Print(mesh_ofs);
+//       L2_FECollection L2fec(1,meshes[ip]->Dimension());
+//       FiniteElementSpace L2fes(meshes[ip], &L2fec);
+//       GridFunction x(&L2fes); 
+      
+//       ConstantCoefficient alpha((double)ip);
+//       x.ProjectCoefficient(alpha);
+//       ostringstream sol_name;
+//       sol_name << sfilename << setfill('0') << setw(6) << ip;
+//       ofstream sol_ofs(sol_name.str().c_str());
+//       x.Save(sol_ofs);
+//    }
+// }
+
+// void SaveMesh(Mesh * mesh, string mfilename)
+// {
+//    cout << "saving global mesh " << endl;
+//    ostringstream mesh_name;
+//    mesh_name << mfilename;
+//    ofstream mesh_ofs(mesh_name.str().c_str());
+//    mesh_ofs.precision(8);
+//    mesh->Print(mesh_ofs);
+// }
+
+// MeshPartition::~MeshPartition()
+// {
+//    for (int ip = 0; ip<nrpatch; ++ip)
+//    {
+//       delete patch_mesh[ip];
+//       patch_mesh[ip] = nullptr;
+//    }
+//    patch_mesh.DeleteAll();
+// }

examples/maxwell-solver/DST2D/MeshPartition2D.hpp

@@ -0,0 +1,74 @@
+// #pragma once
+// #include "mfem.hpp"
+// #include <fstream>
+// #include <iostream>
+// using namespace std;
+// using namespace mfem;
+
+// double GetUniformMeshElementSize(Mesh * mesh);
+// Mesh * ExtendMesh(Mesh * mesh, const Array<int> & directions);
+
+// class CartesianMeshPartition 
+// {
+// private:
+//    Mesh *mesh=nullptr;
+// public:
+//    int nrpatch;
+//    int nxyz[3];
+//    std::vector<Array<int>> element_map;
+//    Array3D<int>subdomains;
+//    // constructor
+//    CartesianMeshPartition(Mesh * mesh_,int & nx, int & ny, int & nz);
+//    ~CartesianMeshPartition() {};
+// };
+
+// class OverlappingCartesianMeshPartition 
+// {
+// private:
+//    Mesh *mesh=nullptr;
+// public:
+//    int nrpatch;
+//    int nxyz[3];
+//    std::vector<Array<int>> element_map;
+//    Array3D<int> subdomains;
+//    // constructor
+//    OverlappingCartesianMeshPartition(Mesh * mesh_,int & nx, int & ny, int & nz);
+//    OverlappingCartesianMeshPartition(Mesh * mesh_,int & nx, int & ny, int & nz, int ovlp_nlayers);
+//    ~OverlappingCartesianMeshPartition() {};
+// };
+
+// class STPOverlappingCartesianMeshPartition // Special layered partition for STP
+// {
+// private:
+//    Mesh *mesh=nullptr;
+// public:
+//    int nrpatch;
+//    int nx, ny, nz;
+//    std::vector<Array<int>> element_map;
+//    // constructor
+//    STPOverlappingCartesianMeshPartition(Mesh * mesh_);
+//    ~STPOverlappingCartesianMeshPartition() {};
+// };
+
+// class MeshPartition
+// {
+// private:
+//    Mesh *mesh=nullptr;
+//    void AddElementToMesh(Mesh * mesh,mfem::Element::Type elem_type,int * ind);
+//    void GetNumVertices(int type, mfem::Element::Type & elem_type, int & nrvert);
+//    void PrintElementMap();
+// public:
+//    int nrpatch;
+//    std::vector<Array<int>> element_map;
+//    Array3D<int> subdomains;
+//    Array<Mesh *> patch_mesh;
+//    int partition_kind;
+//    int nxyz[3];
+//    // constructor
+//    MeshPartition(Mesh * mesh_, int part, int mx=1, int my=1, int mz=1, int ovl_nlayers=0);
+//    ~MeshPartition();
+// };
+
+// void SaveMeshPartition(Array<Mesh * > meshes, 
+//                        string mfilename="output/mesh.",
+//                        string sfilename="output/sol.");

examples/maxwell-solver/DST2D/PML2D.cpp

@@ -0,0 +1,170 @@
+// #include "PML2D.hpp"
+
+
+// CartesianPML::CartesianPML(Mesh *mesh_, Array2D<double> length_)
+//    : mesh(mesh_), length(length_)
+// {
+//    dim = mesh->Dimension();
+//    SetBoundaries();
+// }
+
+// void CartesianPML::SetBoundaries()
+// {
+//    comp_dom_bdr.SetSize(dim, 2);
+//    dom_bdr.SetSize(dim, 2);
+//    // initialize with any vertex
+//    for (int i = 0; i < dim; i++)
+//    {
+//       dom_bdr(i, 0) = mesh->GetVertex(0)[i];
+//       dom_bdr(i, 1) = mesh->GetVertex(0)[i];
+//    }
+
+//    for (int i = 0; i < mesh->GetNBE(); i++)
+//    {
+//       Array<int> bdr_vertices;
+//       mesh->GetBdrElementVertices(i, bdr_vertices);
+//       for (int j = 0; j < bdr_vertices.Size(); j++)
+//       {
+//          for (int k = 0; k < dim; k++)
+//          {
+//             dom_bdr(k, 0) = min(dom_bdr(k, 0), mesh->GetVertex(bdr_vertices[j])[k]);
+//             dom_bdr(k, 1) = max(dom_bdr(k, 1), mesh->GetVertex(bdr_vertices[j])[k]);
+//          }
+//       }
+//    }
+
+//    for (int i = 0; i < dim; i++)
+//    {
+//       comp_dom_bdr(i, 0) = dom_bdr(i, 0) + length(i, 0);
+//       comp_dom_bdr(i, 1) = dom_bdr(i, 1) - length(i, 1);
+//    }
+// }
+
+// void CartesianPML::SetAttributes(Mesh *mesh_)
+// {
+//    int nrelem = mesh_->GetNE();
+//    elems.SetSize(nrelem);
+
+//    for (int i = 0; i < nrelem; ++i)
+//    {
+//       elems[i] = 1;
+//       bool in_pml = false;
+//       Element *el = mesh_->GetElement(i);
+//       Array<int> vertices;
+//       // Initialize Attribute
+//       el->SetAttribute(1);
+//       el->GetVertices(vertices);
+//       int nrvert = vertices.Size();
+//       // Check if any vertex is in the pml
+//       for (int iv = 0; iv < nrvert; ++iv)
+//       {
+//          int vert_idx = vertices[iv];
+//          double *coords = mesh_->GetVertex(vert_idx);
+//          for (int comp = 0; comp < dim; ++comp)
+//          {
+//             if (coords[comp] > comp_dom_bdr(comp, 1) ||
+//                 coords[comp] < comp_dom_bdr(comp, 0))
+//             {
+//                in_pml = true;
+//                break;
+//             }
+//          }
+//       }
+//       if (in_pml)
+//       {
+//          elems[i] = 0;
+//          el->SetAttribute(2);
+//       }
+//    }
+//    mesh_->SetAttributes();
+// }
+
+// void CartesianPML::StretchFunction(const Vector &x,
+//                                    vector<complex<double>> &dxs, double omega)
+// {
+//    complex<double> zi = complex<double>(0., 1.);
+
+//    double n = 2.0;
+//    double c = 5.0;
+//    double coeff;
+//    // Stretch in each direction independently
+//    for (int i = 0; i < dim; ++i)
+//    {
+//       dxs[i] = 1.0;
+//       if (x(i) >= comp_dom_bdr(i, 1))
+//       {
+//          coeff = n * c / omega / pow(length(i, 1), n);
+//          dxs[i] = 1.0 + zi * coeff * abs(pow(x(i) - comp_dom_bdr(i, 1), n - 1.0));
+//       }
+//       if (x(i) <= comp_dom_bdr(i, 0))
+//       {
+//          coeff = n * c / omega / pow(length(i, 0), n);
+//          dxs[i] = 1.0 + zi * coeff * abs(pow(x(i) - comp_dom_bdr(i, 0), n - 1.0));
+//       }
+//    }
+// }
+
+
+// double pml_detJ_Re(const Vector & x, CartesianPML * pml)
+// {
+//    int dim = pml->dim;
+//    double omega = pml->omega;
+//    std::vector<std::complex<double>> dxs(dim);
+//    complex<double> det(1.0,0.0);
+//    pml->StretchFunction(x, dxs, omega);
+//    for (int i=0; i<dim; ++i) det *= dxs[i];
+//    return det.real();
+// }
+
+// double pml_detJ_Im(const Vector & x, CartesianPML * pml)
+// {
+//    int dim = pml->dim;
+//    double omega = pml->omega;
+//    std::vector<std::complex<double>> dxs(dim);
+//    complex<double> det(1.0,0.0);
+//    pml->StretchFunction(x, dxs, omega);
+//    for (int i=0; i<dim; ++i) det *= dxs[i];
+//    return det.imag();
+// }
+
+// void pml_detJ_JT_J_inv_Re(const Vector & x, CartesianPML * pml , DenseMatrix & M)
+// {
+//    int dim = pml->dim;
+//    double omega = pml->omega;
+//    std::vector<std::complex<double>> dxs(dim);
+//    complex<double> det(1.0,0.0);
+//    pml->StretchFunction(x, dxs, omega);
+
+//    for (int i = 0; i<dim; ++i)
+//    {
+//       det *= dxs[i];
+//    }
+
+//    M=0.0;
+//    for (int i = 0; i<dim; ++i)
+//    {
+//       M(i,i) = (det / pow(dxs[i],2)).real();
+//    }
+// }
+
+// void pml_detJ_JT_J_inv_Im(const Vector & x, CartesianPML * pml , DenseMatrix & M)
+// {
+//    int dim = pml->dim;
+//    double omega = pml->omega;
+
+//    std::vector<std::complex<double>> dxs(dim);
+//    complex<double> det = 1.0;
+//    pml->StretchFunction(x, dxs, omega);
+
+//    for (int i = 0; i<dim; ++i)
+//    {
+//       det *= dxs[i];
+//    }
+
+//    M=0.0;
+//    for (int i = 0; i<dim; ++i)
+//    {
+//       M(i,i) = (det / pow(dxs[i],2)).imag();
+//    }
+// }
+

examples/maxwell-solver/DST2D/PML2D.hpp

@@ -0,0 +1,101 @@
+// #pragma once
+// #include "mfem.hpp"
+// #include <fstream>
+// #include <iostream>
+
+// using namespace std;
+// using namespace mfem;
+
+// // Class for setting up a simple Cartesian PML region
+// class CartesianPML
+// {
+// private:
+//    Mesh *mesh;
+
+//    // Length of the PML Region in each direction
+//    Array2D<double> length;
+
+//    // Computational Domain Boundary
+//    Array2D<double> comp_dom_bdr;
+
+//    // Domain Boundary
+//    Array2D<double> dom_bdr;
+
+//    // Integer Array identifying elements in the pml
+//    // 0: in the pml, 1: not in the pml
+//    Array<int> elems;
+
+//    // Compute Domain and Computational Domain Boundaries
+//    void SetBoundaries();
+
+// public:
+//    // Constructor
+//    CartesianPML(Mesh *mesh_,Array2D<double> length_);
+
+//    int dim;
+//    double omega;
+//    // Return Computational Domain Boundary
+//    Array2D<double> GetCompDomainBdr() {return comp_dom_bdr;}
+
+//    // Return Domain Boundary
+//    Array2D<double> GetDomainBdr() {return dom_bdr;}
+
+//    // Return Marker list for elements
+//    Array<int> * GetMarkedPMLElements() {return &elems;}
+
+//    // Mark element in the PML region
+//    void SetAttributes(Mesh *mesh_);
+
+//    void SetOmega(double omega_) {omega = omega_;}
+
+//    // PML complex stretching function
+//    void StretchFunction(const Vector &x, vector<complex<double>> &dxs, double omega);
+// };
+
+
+// class PmlCoefficient : public Coefficient
+// {
+// private:
+//    CartesianPML * pml = nullptr;
+//    double (*Function)(const Vector &, CartesianPML * );
+// public:
+//    PmlCoefficient(double (*F)(const Vector &, CartesianPML *), CartesianPML * pml_)
+//       : pml(pml_), Function(F)
+//    {}
+//    virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+//    {
+//       double x[3];
+//       Vector transip(x, 3);
+//       T.Transform(ip, transip);
+//       return ((*Function)(transip, pml));
+//    }
+// };
+
+
+// // This includes scalar coefficients
+// class PmlMatrixCoefficient : public MatrixCoefficient
+// {
+// private:
+//    CartesianPML * pml = nullptr;
+//    void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
+// public:
+//    PmlMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+//                                           DenseMatrix &),
+//                         CartesianPML * pml_)
+//       : MatrixCoefficient(dim), pml(pml_), Function(F)
+//    {}
+//    virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+//                      const IntegrationPoint &ip)
+//    {
+//       double x[3];
+//       Vector transip(x, 3);
+//       T.Transform(ip, transip);
+//       K.SetSize(height, width);
+//       (*Function)(transip, pml, K);
+//    }
+// };
+
+// double pml_detJ_Re(const Vector & x, CartesianPML * pml);
+// double pml_detJ_Im(const Vector & x, CartesianPML * pml);
+// void pml_detJ_JT_J_inv_Re(const Vector & x, CartesianPML * pml , DenseMatrix & M);
+// void pml_detJ_JT_J_inv_Im(const Vector & x, CartesianPML * pml , DenseMatrix & M);

examples/maxwell-solver/DST2D/Utilities2D.cpp

@@ -0,0 +1,382 @@
+// #include "Utilities2D.hpp"
+
+// double CutOffFncn(const Vector &x, const Vector & pmin, const Vector & pmax, const Array2D<double> & h_)
+// {
+//    int dim = pmin.Size();
+//    Vector h0(dim);
+//    Vector h1(dim);
+//    for (int i=0; i<dim; i++)
+//    {
+//       h0(i) = h_[i][0];
+//       h1(i) = h_[i][1];
+//    }
+//    Vector x0(dim);
+//    Vector x1(dim);
+//    x0 = pmin; x0+=h0;
+//    x1 = pmax; x1-=h1;
+
+//    double f = 1.0;
+//    for (int i = 0; i<dim; i++)
+//    {
+//       double val = 1.0;
+//       if( x(i) >= pmax(i) || x(i) <= pmin(i))
+//       {
+//          val = 0.0;
+//       }  
+//       else if (x(i) < pmax(i) && x(i) >= x1(i))
+//       {
+//          if(h1(i) != 0.0)
+//             // val = (x(i)-pmax(i))/(x1(i)-pmax(i)); 
+//             val = pow((x(i)-pmax(i))/(x1(i)-pmax(i)),1.0); 
+//       }
+//       else if (x(i) > pmin(i) && x(i) <= x0(i))
+//       {
+//          if (h0(i) != 0.0)
+//             // val = (x(i)-pmin(i))/(x0(i)-pmin(i)); 
+//             val = pow((x(i)-pmin(i))/(x0(i)-pmin(i)),1.0); 
+//       }
+
+//       if (h0(i) == 0 && x(i) <= x1(i))
+//       {
+//          val = 1.0;
+//       }
+//       if (h1(i) == 0 && x(i) >= x0(i))
+//       {
+//          val = 1.0;
+//       }
+//       f *= val;
+//    }
+//    return f;
+// }
+
+// double ChiFncn(const Vector &x, const Vector & pmin, const Vector & pmax, const Array2D<double> & h_)
+// {
+//    int dim = pmin.Size();
+//    Vector h0(dim);
+//    Vector h1(dim);
+//    for (int i=0; i<dim; i++)
+//    {
+//       h0(i) = h_[i][0];
+//       h1(i) = h_[i][1];
+//    }
+//    Vector x0(dim);
+//    Vector x1(dim);
+//    x0 = pmin; x0+=h0;
+//    x1 = pmax; x1-=h1;
+
+//    double f = 1.0;
+//    for (int i = 0; i<dim; i++)
+//    {
+//       double val = 1.0;
+//       if( x(i) >= pmax(i) || x(i) <= pmin(i))
+//       {
+//          val = 0.0;
+//       }  
+//       else if (x(i) < pmax(i) && x(i) >= x1(i))
+//       {
+//          if(h1(i) != 0.0)
+//             // val = (x(i)-pmax(i))/(x1(i)-pmax(i)); 
+//             // This function has to be changed to smth more reasonable
+//             val = pow((x(i)-pmax(i))/(x1(i)-pmax(i)),100.0); 
+//       }
+//       else if (x(i) > pmin(i) && x(i) <= x0(i))
+//       {
+//          if (h0(i) != 0.0)
+//             // val = (x(i)-pmin(i))/(x0(i)-pmin(i)); 
+//             val = pow((x(i)-pmin(i))/(x0(i)-pmin(i)),100.0); 
+//       }
+
+//       if (h0(i) == 0 && x(i) <= x1(i))
+//       {
+//          val = 1.0;
+//       }
+//       if (h1(i) == 0 && x(i) >= x0(i))
+//       {
+//          val = 1.0;
+//       }
+//       f *= val;
+//    }
+//    return f;
+// }
+
+
+// DofMap::DofMap(SesquilinearForm * bf_ , MeshPartition * partition_) 
+//                : bf(bf_), partition(partition_)
+// {
+//    // int partition_kind = partition->partition_kind;
+//    // MFEM_VERIFY(partition_kind == 1, "Check Partition kind");
+//    fespace = bf->FESpace();
+//    // Mesh * mesh = fespace->GetMesh();
+//    const FiniteElementCollection * fec = fespace->FEColl();
+//    nrpatch = partition->nrpatch;
+
+//    fespaces.SetSize(nrpatch);
+
+//    Dof2GlobalDof.resize(nrpatch);
+
+//    for (int ip=0; ip<nrpatch; ++ip)
+//    {
+//       // create finite element spaces for each patch 
+//       fespaces[ip] = new FiniteElementSpace(partition->patch_mesh[ip],fec);
+
+//       // construct the patch tdof to global tdof map
+//       int nrdof = fespaces[ip]->GetTrueVSize();
+//       Dof2GlobalDof[ip].SetSize(2*nrdof);
+
+//       // loop through the elements in the patch
+//       for (int iel = 0; iel<partition->element_map[ip].Size(); ++iel)
+//       {
+//          // index in the global mesh
+//          int iel_idx = partition->element_map[ip][iel];
+//          // get the dofs of this element
+//          Array<int> ElemDofs;
+//          Array<int> GlobalElemDofs;
+//          fespaces[ip]->GetElementDofs(iel,ElemDofs);
+//          fespace->GetElementDofs(iel_idx,GlobalElemDofs);
+//          // the sizes have to match
+//          MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+//                      "Size inconsistency");
+//          // loop through the dofs and take into account the signs;
+//          int ndof = ElemDofs.Size();
+//          for (int i = 0; i<ndof; ++i)
+//          {
+//             int pdof_ = ElemDofs[i];
+//             int gdof_ = GlobalElemDofs[i];
+//             int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+//             int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+//             Dof2GlobalDof[ip][pdof] = gdof;
+//             Dof2GlobalDof[ip][pdof+nrdof] = gdof+fespace->GetTrueVSize();
+//          }
+//       }
+//    }
+// }
+
+// DofMap::DofMap(SesquilinearForm * bf_ , MeshPartition * partition_, int nrlayers) 
+//                : bf(bf_), partition(partition_)
+// {
+
+//    nx = partition->nxyz[0];
+//    ny = partition->nxyz[1];
+//    nz = partition->nxyz[2];
+
+//    int partition_kind = partition->partition_kind;
+//    fespace = bf->FESpace();
+//    // Mesh * mesh = fespace->GetMesh();
+//    const FiniteElementCollection * fec = fespace->FEColl();
+//    nrpatch = partition->nrpatch;
+
+//    fespaces.SetSize(nrpatch);
+//    PmlMeshes.SetSize(nrpatch);
+//    // Extend patch meshes to include pml
+
+//    for  (int ip = 0; ip<nrpatch; ip++)
+//    {
+//       int k = ip/(nx*ny);
+//       int j = (ip-k*nx*ny)/nx;
+//       int i = (ip-k*nx*ny)%nx;
+      
+//       Array<int> directions;
+//       if (i > 0)
+//       {
+//          for (int i=0; i<nrlayers; i++)
+//          {
+//             directions.Append(-1);
+//          }
+//       }
+//       if (j > 0)
+//       {
+//          for (int i=0; i<nrlayers; i++)
+//          {
+//             directions.Append(-2);
+//          }
+//       }
+//       if (k > 0)
+//       {
+//          for (int i=0; i<nrlayers; i++)
+//          {
+//             directions.Append(-3);
+//          }
+//       }
+//       if (i < nx-1)
+//       {
+//          for (int i=0; i<nrlayers; i++)
+//          {
+//             if (partition_kind == 3 || partition_kind == 2) directions.Append(1);
+//          }
+//       }
+//       if (j < ny-1)
+//       {
+//          for (int i=0; i<nrlayers; i++)
+//          {
+//             if (partition_kind == 3 || partition_kind == 2) directions.Append(2);
+//          }
+//       }
+//       if (k < nz-1)
+//       {
+//          for (int i=0; i<nrlayers; i++)
+//          {
+//             if (partition_kind == 3 || partition_kind == 2) directions.Append(1);
+//          }
+//       }
+//       PmlMeshes[ip] = ExtendMesh(partition->patch_mesh[ip],directions);
+//    }
+
+//    // Save PML_meshes
+//    string meshpath;
+//    string solpath;
+//    if (partition_kind == 3 || partition_kind == 2) 
+//    {
+//       meshpath = "output/mesh_ovlp_pml.";
+//       solpath = "output/sol_ovlp_pml.";
+//    }
+//    else if (partition_kind == 4)
+//    {
+//       meshpath = "output/mesh_novlp_pml.";
+//       solpath = "output/sol_novlp_pml.";
+//    }
+//    else
+//    {
+//       MFEM_ABORT("This partition kind not supported yet");
+//    }
+   
+//    // SaveMeshPartition(PmlMeshes, meshpath, solpath);
+
+//    PmlFespaces.SetSize(nrpatch);
+//    Dof2GlobalDof.resize(nrpatch);
+//    Dof2PmlDof.resize(nrpatch);
+
+//    for (int ip=0; ip<nrpatch; ++ip)
+//    {
+//       // create finite element spaces for each patch 
+//       fespaces[ip] = new FiniteElementSpace(partition->patch_mesh[ip],fec);
+//       PmlFespaces[ip] = new FiniteElementSpace(PmlMeshes[ip],fec);
+
+//       // construct the patch tdof to global tdof map
+//       int nrdof = fespaces[ip]->GetTrueVSize();
+//       Dof2GlobalDof[ip].SetSize(2*nrdof);
+//       Dof2PmlDof[ip].SetSize(2*nrdof);
+
+//       // build dof maps between patch and extended patch
+//       // loop through the patch elements and constract the dof map
+//       // The same elements in the extended mesh have the same ordering (but not the dofs)
+
+//       // loop through the elements in the patch
+//       for (int iel = 0; iel<partition->element_map[ip].Size(); ++iel)
+//       {
+//          // index in the global mesh
+//          int iel_idx = partition->element_map[ip][iel];
+//          // get the dofs of this element
+//          Array<int> ElemDofs;
+//          Array<int> PmlElemDofs;
+//          Array<int> GlobalElemDofs;
+//          fespaces[ip]->GetElementDofs(iel,ElemDofs);
+//          PmlFespaces[ip]->GetElementDofs(iel,PmlElemDofs);
+//          fespace->GetElementDofs(iel_idx,GlobalElemDofs);
+//          // the sizes have to match
+//          MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+//                      "Size inconsistency");
+//          MFEM_VERIFY(ElemDofs.Size() == PmlElemDofs.Size(),
+//                      "Size inconsistency");            
+//          // loop through the dofs and take into account the signs;
+//          int ndof = ElemDofs.Size();
+//          for (int i = 0; i<ndof; ++i)
+//          {
+//             int pdof_ = ElemDofs[i];
+//             int gdof_ = GlobalElemDofs[i];
+//             int pmldof_ = PmlElemDofs[i];
+//             int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+//             int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+//             int pmldof = (pmldof_ >= 0) ? pmldof_ : abs(pmldof_) - 1;
+
+//             Dof2GlobalDof[ip][pdof] = gdof;
+//             Dof2GlobalDof[ip][pdof+nrdof] = gdof+fespace->GetTrueVSize();
+//             Dof2PmlDof[ip][pdof] = pmldof;
+//             Dof2PmlDof[ip][pdof+nrdof] = pmldof+PmlFespaces[ip]->GetTrueVSize();
+//          }
+//       }
+//    }
+// }
+
+
+// LocalDofMap::LocalDofMap(const FiniteElementCollection * fec_, MeshPartition * part1_, 
+//                MeshPartition * part2_):fec(fec_), part1(part1_), part2(part2_)
+// {
+//    // Each overlapping patch has 2 non-overlapping subdomains
+//    // Thre are n non-overlapping and and n-1 overlapping subdomains
+//    int nrpatch = part2->nrpatch;
+//    MFEM_VERIFY(part1->nrpatch-1 == part2->nrpatch, "Check number of subdomains");
+
+//    cout << "Constructing local dof maps" << endl; 
+//    map1.resize(nrpatch);
+//    map2.resize(nrpatch);
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       // Get the 3 meshes involved
+//       Mesh * mesh = part2->patch_mesh[ip];
+//       Mesh * mesh1 = part1->patch_mesh[ip];
+//       Mesh * mesh2 = part1->patch_mesh[ip+1];
+
+//       // Define the fespaces
+//       FiniteElementSpace fespace(mesh, fec);
+//       FiniteElementSpace fespace1(mesh1, fec);
+//       FiniteElementSpace fespace2(mesh2, fec);
+
+//       int ndof1 = fespace1.GetTrueVSize();
+//       int ndof2 = fespace2.GetTrueVSize();
+
+//       map1[ip].SetSize(2*ndof1); // times 2 because it's complex
+//       map2[ip].SetSize(2*ndof2); // times 2 because it's complex
+
+//       // loop through the elements in the patches
+//       // map 1 is constructed by the first half of elements
+//       // map 2 is constructed by the second half of elements
+
+//       for (int iel = 0; iel<part1->element_map[ip].Size(); ++iel)
+//       {
+//          // index in the overlapping mesh
+//          int iel_idx = iel;
+//          Array<int> ElemDofs;
+//          Array<int> GlobalElemDofs;
+//          fespace1.GetElementDofs(iel,ElemDofs);
+//          fespace.GetElementDofs(iel_idx,GlobalElemDofs);
+//          // the sizes have to match
+//          MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+//                      "Size inconsistency");
+//          // loop through the dofs and take into account the signs;
+//          int ndof = ElemDofs.Size();
+//          for (int i = 0; i<ndof; ++i)
+//          {
+//             int pdof_ = ElemDofs[i];
+//             int gdof_ = GlobalElemDofs[i];
+//             int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+//             int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+//             map1[ip][pdof] = gdof;
+//             map1[ip][pdof+ndof1] = gdof+fespace.GetTrueVSize();
+//          }
+//       }
+//       for (int iel = 0; iel<part1->element_map[ip+1].Size(); ++iel)
+//       {
+//          // index in the overlapping mesh
+//          int k = part1->element_map[ip].Size();
+//          int iel_idx = iel+k;
+//          Array<int> ElemDofs;
+//          Array<int> GlobalElemDofs;
+//          fespace2.GetElementDofs(iel,ElemDofs);
+//          fespace.GetElementDofs(iel_idx,GlobalElemDofs);
+//          // the sizes have to match
+//          MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+//                      "Size inconsistency");
+//          // loop through the dofs and take into account the signs;
+//          int ndof = ElemDofs.Size();
+//          for (int i = 0; i<ndof; ++i)
+//          {
+//             int pdof_ = ElemDofs[i];
+//             int gdof_ = GlobalElemDofs[i];
+//             int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+//             int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+//             map2[ip][pdof] = gdof;
+//             map2[ip][pdof+ndof2] = gdof+fespace.GetTrueVSize();
+//          }
+//       }
+//    }
+// }

examples/maxwell-solver/DST2D/Utilities2D.hpp

@@ -0,0 +1,103 @@
+// #pragma once
+// #include "MeshPartition2D.hpp"
+
+// struct hash_pair {
+//    template <class T1, class T2>
+//    size_t operator()(const pair<T1, T2>& p) const{
+//       auto hash1 = hash<T1>{}(p.first);
+//       auto hash2 = hash<T2>{}(p.second);
+//       return hash1 ^ hash2;
+//    }
+// };
+
+// struct UniqueIndexGenerator
+// {
+//    int counter = 0;
+//    std::unordered_map<pair<int,int>,int, hash_pair> idx;
+//    int Get(int i, int j)
+//    {
+//       pair<int,int> p1(i,j);
+//       std::unordered_map<pair<int,int>,int, hash_pair>::iterator f = idx.find(p1);
+//       if (f == idx.end())
+//       {
+//          idx[p1] = counter;
+//          return counter++;
+//       }
+//       else
+//       {
+//          return (*f).second;
+//       }
+//    }
+//    void Reset()
+//    {
+//       counter = 0;
+//       idx.clear();
+//    }
+// };
+
+
+
+// // Function coefficient that takes the boundingbox of the mesh as an input
+// class CutOffFnCoefficient : public Coefficient
+// {
+// private:
+//    double (*Function)(const Vector &, const Vector &, const Vector &, const Array2D<double> &);
+//    Vector pmin, pmax;
+//    Array2D<double> h; // specify the with of the cutoff function (h in each direction)
+   
+
+// public:
+//    CutOffFnCoefficient(double (*F)(const Vector &, const Vector &, const Vector &, const Array2D<double> &), 
+//                              const Vector & pmin_, const Vector & pmax_, Array2D<double> & h_)
+//       : Function(F), pmin(pmin_), pmax(pmax_), h(h_)
+//    {}
+//    virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+//    {
+//       double x[3];
+//       Vector transip(x, 3);
+//       T.Transform(ip, transip);
+//       return ((*Function)(transip, pmin, pmax, h));
+//    }
+// };
+
+// double CutOffFncn(const Vector &x, const Vector & pmin, 
+// const Vector & pmax, const Array2D<double> & h_);
+// double ChiFncn(const Vector &x, const Vector & pmin, 
+// const Vector & pmax, const Array2D<double> & h_);
+
+// class DofMap // Constructs dof maps for a given partition
+// {
+//    FiniteElementSpace *fespace=nullptr;
+//    SesquilinearForm * bf=nullptr;
+//    MeshPartition * partition=nullptr;
+// public:
+//    int nrpatch, nx, ny, nz;
+//    vector<Array<int>> Dof2GlobalDof;
+//    vector<Array<int>> Dof2PmlDof;
+//    Array<Mesh *> PmlMeshes;
+//    Array<FiniteElementSpace *> fespaces;
+//    Array<FiniteElementSpace *> PmlFespaces;
+//    // constructor
+//    // Non PML contructor dof map
+//    DofMap(SesquilinearForm * bf_, MeshPartition * partition_);
+//    // PML
+//    DofMap(SesquilinearForm * bf_ , MeshPartition * partition_, int nrlayers);
+//    ~DofMap();
+// };
+
+
+// class LocalDofMap // Constructs dof mapbetween two partitions
+// {
+//    const FiniteElementCollection *fec=nullptr;
+//    MeshPartition * part1=nullptr;
+//    MeshPartition * part2=nullptr;
+// public:
+//    int nrpatch, nx, ny, nz;
+//    vector<Array<int>> map1;
+//    vector<Array<int>> map2;
+//    // constructor
+//    LocalDofMap(const FiniteElementCollection * fec_, MeshPartition * part1_, 
+//                MeshPartition * part2_);
+//    ~LocalDofMap();
+// };
+

examples/maxwell-solver/DST2D/back/DSTnew.cpp

@@ -0,0 +1,668 @@
+// //Diagonal Source Transfer Preconditioner
+
+// #include "DST.hpp"
+
+
+// DST::DST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+//          double omega_, Coefficient * ws_,  int nrlayers_)
+//    : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), 
+//      bf(bf_), Pmllength(Pmllength_), omega(omega_), ws(ws_), nrlayers(nrlayers_)
+// {
+//    Mesh * mesh = bf->FESpace()->GetMesh();
+//    dim = mesh->Dimension();
+
+//    // ----------------- Step 1 --------------------
+//    // Introduce 2 layered partitios of the domain 
+//    // 
+//    int partition_kind;
+
+//    // 1. Ovelapping partition with overlap = 2h 
+//    partition_kind = 2; // Non Overlapping partition 
+//    int nx=4;
+//    int ny=1; 
+//    int nz=1;
+
+//    povlp = new MeshPartition(mesh, partition_kind,nx,ny,nz, nrlayers);
+//    nxyz[0] = povlp->nxyz[0];
+//    nxyz[1] = povlp->nxyz[1];
+//    nxyz[2] = povlp->nxyz[2];
+//    nrpatch = povlp->nrpatch;
+//    subdomains = povlp->subdomains;
+
+//    //
+//    // ----------------- Step 1a -------------------
+//    // Save the partition for visualization
+//    // SaveMeshPartition(povlp->patch_mesh, "output/mesh_ovlp.", "output/sol_ovlp.");
+
+//    ovlp_prob  = new DofMap(bf,povlp); 
+//    PmlMat.SetSize(nrpatch);
+//    PmlMatInv.SetSize(nrpatch);
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       PmlMat[ip] = GetPmlSystemMatrix(ip);
+//       PmlMatInv[ip] = new KLUSolver;
+//       PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
+//    }
+//    nsweeps = pow(2,dim);
+//    sweeps.SetSize(nsweeps,dim);
+//    // 2D
+//    sweeps(0,0) =  1; sweeps(0,1) = 1;
+//    sweeps(1,0) = -1; sweeps(1,1) = 1;
+//    sweeps(2,0) =  1; sweeps(2,1) =-1;
+//    sweeps(3,0) = -1; sweeps(3,1) =-1;
+
+//    // Set up src arrays size
+//    f_orig.SetSize(nrpatch);
+//    f_transf.SetSize(nrpatch);
+//    // Construct a simple map used for directions of transfer
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       int n = 2*ovlp_prob->fespaces[ip]->GetTrueVSize(); // (x 2 for complex ) 
+//       f_orig[ip] = new Vector(n); *f_orig[ip] = 0.0;
+//       f_transf[ip].SetSize(nsweeps);
+//       for (int i=0;i<nsweeps; i++)
+//       {
+//          f_transf[ip][i] = new Vector(n);
+//       }
+//    }
+// }
+
+
+
+// void DST::Mult(const Vector &r, Vector &z) const
+// {
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       *f_orig[ip] = 0.0;
+//       for (int i=0;i<nsweeps; i++)
+//       {
+//          *f_transf[ip][i] = 0.0;
+//       }
+//    }
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+//       r.GetSubVector(*Dof2GlobalDof,*f_orig[ip]);
+//    }
+
+//    char vishost[] = "localhost";
+//    int  visport   = 19916;
+//    z = 0.0; 
+//    Vector znew(z);
+//    Vector z1(z);
+//    Vector z2(z);
+
+//    // --------------------------------------------
+//    //       Sweep in the direction (1,1)
+//    // --------------------------------------------
+//    int nx = nxyz[0];
+//    int ny = nxyz[1];
+
+//    int nsteps = nx + ny - 1;
+
+//    for (int l=0; l<1; l++)
+//    {
+//       for (int s = 0; s<nsteps; s++)
+//       {
+//          // the patches involved are the ones such that
+//          // i+j = s
+//          // cout << "Step no: " << s << endl;
+//          for (int i=0;i<nx; i++)
+//          {
+//             int j;
+//             switch (l)
+//             {
+//                case 0:  j = s-i;         break;
+//                case 1:  j = s-nx+i+1;    break;
+//                case 2:  j = nx+i-s-1;    break;
+//                default: j = nx+ny-i-s-2; break;
+//             }
+//             if (j<0 || j>=ny) continue;
+//             // cout << "Patch no: (" << i <<"," << j << ")" << endl; 
+
+//             // find patch id
+//             Array<int> ij(2); ij[0] = i; ij[1]=j;
+//             int ip = GetPatchId(ij);
+//             // cout << "ip = " << ip << endl;
+
+//             // Solve the PML problem in patch ip with all sources
+//             // Original and all transfered (maybe some of them)
+//             Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+//             int ndofs = Dof2GlobalDof->Size();
+
+//             Vector sol_local(ndofs); sol_local = 0.0;
+//             Vector res_local(ndofs); res_local = 0.0;
+//             if (l==0) res_local += *f_orig[ip];
+//             // res_local += *f_orig[ip];
+//             res_local += *f_transf[ip][l];
+//             // Extend by zero to the PML mesh
+//             // if (res_local.Norml2() < 1e-11) continue;
+//             PmlMatInv[ip]->Mult(res_local, sol_local);
+
+//             TransferSources(l,ip, sol_local);
+
+//             // cut off the ip solution to all possible directions
+//             Array<int>directions(2); directions = 0; 
+            
+//             if (i+1<nx) directions[0] = 1;
+//             if (j+1<ny) directions[1] = 1;
+//             Vector cfsol_local;
+//             GetCutOffSolution(sol_local,cfsol_local,ip,directions,nrlayers,true);
+//             sol_local = cfsol_local;
+//             directions = 0.0;
+//             if (i>0) directions[0] = -1;
+//             if (j>0) directions[1] = -1;
+//             GetCutOffSolution(sol_local,cfsol_local,ip,directions,nrlayers,true);
+//             znew = 0.0;
+//             // znew.SetSubVector(*Dof2GlobalDof, cfsol_local);
+//             znew.SetSubVector(*Dof2GlobalDof, sol_local);
+//             z+=znew;
+//          }
+//          socketstream zsock(vishost, visport);
+//          PlotSolution(z,zsock,0); cin.get();
+//       }
+//    }
+// }
+
+
+// void DST::GetCutOffSolution(const Vector & sol, Vector & cfsol, 
+//                   int ip, Array<int> directions, int nlayers, bool local) const
+// {
+
+//    int d = directions.Size();
+//    int directx = directions[0]; // 1,0,-1
+//    int directy = directions[1]; // 1,0,-1
+//    int directz;
+//    if (d ==3) directz = directions[2];
+
+//    Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+   
+//    Vector pmin, pmax;
+//    mesh->GetBoundingBox(pmin, pmax);
+//    double h = GetUniformMeshElementSize(povlp->patch_mesh[ip]);
+//    Array2D<double> pmlh(dim,2); pmlh = 0.0;
+   
+//    if (directions[0]==1)
+//    {
+//       pmlh[0][1] = h*nlayers;
+//    }
+//    if (directions[0]==-1)
+//    {
+//       pmlh[0][0] = h*nlayers;
+//    }
+//    if (directions[1]==1)
+//    {
+//       pmlh[1][1] = h*nlayers;
+//    }
+//    if (directions[1]==-1)
+//    {
+//       pmlh[1][0] = h*nlayers;
+//    }
+
+//    CutOffFnCoefficient cf(CutOffFncn, pmin, pmax, pmlh);
+
+//    double * data = sol.GetData();
+
+//    FiniteElementSpace * fes;
+//    if (!local)
+//    {
+//       fes = bf->FESpace();
+//    }
+//    else
+//    {
+//       fes = ovlp_prob->fespaces[ip];
+//    }
+   
+//    int n = fes->GetTrueVSize();
+
+//    GridFunction solgf_re(fes, data);
+//    GridFunction solgf_im(fes, &data[n]);
+
+
+//    GridFunctionCoefficient coeff1_re(&solgf_re);
+//    GridFunctionCoefficient coeff1_im(&solgf_im);
+
+//    ProductCoefficient prod_re(coeff1_re, cf);
+//    ProductCoefficient prod_im(coeff1_im, cf);
+
+//    ComplexGridFunction gf(fes);
+//    gf.ProjectCoefficient(prod_re,prod_im);
+
+//    cfsol.SetSize(sol.Size());
+//    cfsol = gf;
+// }
+
+// void DST::GetChiRes(const Vector & res, Vector & cfres, 
+//                        int ip, Array<int> directions, int nlayers) const
+// {
+//    // int l,k;
+//    int d = directions.Size();
+//    int directx = directions[0]; // 1,0,-1
+//    int directy = directions[1]; // 1,0,-1
+//    int directz;
+//    if (d ==3) directz = directions[2];
+
+//    Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+//    double h = GetUniformMeshElementSize(mesh);
+   
+//    Vector pmin, pmax;
+//    mesh->GetBoundingBox(pmin, pmax);
+
+//    Array2D<double> pmlh(dim,2); pmlh = 0.0;
+   
+//    if (directions[0]==1)
+//    {
+//       pmlh[0][1] = h*nlayers;
+//    }
+//    if (directions[0]==-1)
+//    {
+//       pmlh[0][0] = h*nlayers;
+//    }
+//    if (directions[1]==1)
+//    {
+//       pmlh[1][1] = h*nlayers;
+//    }
+//    if (directions[1]==-1)
+//    {
+//       pmlh[1][0] = h*nlayers;
+//    }
+
+//    CutOffFnCoefficient cf(ChiFncn, pmin, pmax, pmlh);
+
+//    double * data = res.GetData();
+
+//    FiniteElementSpace * fespace;
+//    fespace = ovlp_prob->fespaces[ip];
+   
+//    int n = fespace->GetTrueVSize();
+
+//    GridFunction solgf_re(fespace, data);
+//    GridFunction solgf_im(fespace, &data[n]);
+
+//    GridFunctionCoefficient coeff1_re(&solgf_re);
+//    GridFunctionCoefficient coeff1_im(&solgf_im);
+
+//    ProductCoefficient prod_re(coeff1_re, cf);
+//    ProductCoefficient prod_im(coeff1_im, cf);
+
+//    ComplexGridFunction gf(fespace);
+//    gf.ProjectCoefficient(prod_re,prod_im);
+
+//    cfres.SetSize(res.Size());
+//    cfres = gf;
+// }
+
+
+// DST::~DST()
+// {
+// }
+
+
+// void DST::Getijk(int ip, int & i, int & j, int & k) const
+// {
+//    k = ip/(nxyz[0]*nxyz[1]);
+//    j = (ip-k*nxyz[0]*nxyz[1])/nxyz[0];
+//    i = (ip-k*nxyz[0]*nxyz[1])%nxyz[0];
+// }
+
+// int DST::GetPatchId(const Array<int> & ijk) const
+// {
+//    int d=ijk.Size();
+//    int z = (dim==2)? 0 : ijk[2];
+//    return subdomains(ijk[0],ijk[1],z);
+// }
+
+
+// void DST::TransferSources(int sweep, int ip0, Vector & sol0) const
+// {
+//  // Find all neighbors of patch ip0
+//    int nx = nxyz[0];
+//    int ny = nxyz[1];
+//    int i0, j0, k0;
+//    Getijk(ip0, i0,j0,k0);
+//    // cout << "Transfer to : " << endl;
+//    // loop through possible directions
+//    for (int i=-1; i<2; i++)
+//    {
+//       int i1 = i0 + i;
+//       if (i1 <0 || i1>=nx) continue;
+//       for (int j=-1; j<2; j++)
+//       {
+//          if (i==0 && j==0) continue;
+//          int j1 = j0 + j;
+//          if (j1 <0 || j1>=ny) continue;
+//          // cout << "(" << i1 << "," << j1 <<"), ";
+//          // Find ip 1
+//          Array<int> ij1(2); ij1[0] = i1; ij1[1]=j1;
+//          int ip1 = GetPatchId(ij1);
+//          // cout << "ip1 = " << ip1;
+//          // cout << " in the direction of (" << i <<", " <<j <<")" << endl;
+//          Array<int> directions(2);
+//          directions[0] = i;
+//          directions[1] = j;
+//          Vector cfsol0;
+//          GetCutOffSolution(sol0,cfsol0,ip0,directions,nrlayers,true);
+         
+//          // Transfer solution to ip1;
+//          Array<int> * Dof2GlobalDof0 = &ovlp_prob->Dof2GlobalDof[ip0];
+//          Array<int> * Dof2GlobalDof1 = &ovlp_prob->Dof2GlobalDof[ip1];
+
+//          Vector znew(2*bf->FESpace()->GetTrueVSize());
+//          znew = 0.0;
+//          znew.SetSubVector(*Dof2GlobalDof0,sol0);
+
+//          Vector sol1(Dof2GlobalDof1->Size()); sol1 = 0.0;
+//          Vector res1(Dof2GlobalDof1->Size()); res1 = 0.0;
+//          znew.GetSubVector(*Dof2GlobalDof1,sol1);
+//          PmlMat[ip1]->Mult(sol1,res1);
+//          res1 *=-1.0;
+
+//          // remove the source in the pml restrict to the non overlapping subdomain
+
+//          Array<int> direct(2); direct = 0;
+//          if (i1>0) direct[0] = -1; 
+//          if (j1>0) direct[1] = -1;
+//          Vector cfraux(res1.Size()); cfraux = 0.0; 
+//          GetChiRes(res1, cfraux,ip1,direct, nrlayers);
+//          direct = 0;
+//          if (i1+1<nx) direct[0] = 1; 
+//          if (j1+1<ny) direct[1] = 1; 
+//          GetChiRes(cfraux, res1,ip1,direct, nrlayers);
+
+
+//          // Find the minumum sweep number that to transfer the source that 
+//          // satisfies the two rules
+//          for (int l=sweep; l<nsweeps; l++)
+//          {
+//             // Conditions on sweeps
+//             // Rule 1: the transfer source direction has to be similar with 
+//             // the sweep direction
+//             int is = sweeps(l,0); 
+//             int js = sweeps(l,1);
+//             int ddot = is*i + js * j;
+//             // cout << "(i,j) = (" << i <<"," <<j <<")" << endl;
+//             // cout << "(is,js) = (" << is <<"," <<js <<")" << endl;
+//             // cout << "ip0 , ip1 = " << ip0 << ", " << ip1 << endl;
+//             if (ddot <= 0) continue;
+
+//             // Rule 2: The horizontal or vertical transfer source cannot be used
+//             // in a later sweep that with opposite directions
+
+//             if (i==0 || j == 0) // Case of horizontal or vertical transfer source
+//             {
+//                int il = sweeps(l,0);
+//                int jl = sweeps(l,1);
+//                // skip if the two sweeps have opposite direction
+//                if (is == -il && js == -jl) continue;
+//             }
+//             // cout << "Passing ip0 = " << ip0 << " to ip1 = " << ip1 
+//                //   << " to sweep no l = " << l << endl;  
+//             MFEM_VERIFY(f_transf[ip1][l]->Size()==res1.Size(), 
+//                         "Transfer Sources: inconsistent size");
+//             *f_transf[ip1][l]+=res1;
+//             break;
+//          }
+//       }  
+//    }
+// }
+
+
+
+
+// SparseMatrix * DST::GetPmlSystemMatrix(int ip)
+// {
+//    double h = GetUniformMeshElementSize(povlp->patch_mesh[ip]);
+//    Array2D<double> length(dim,2);
+//    length = h*(nrlayers);
+
+//    CartesianPML pml(povlp->patch_mesh[ip], length);
+//    pml.SetOmega(omega);
+
+//    Array <int> ess_tdof_list;
+//    if (povlp->patch_mesh[ip]->bdr_attributes.Size())
+//    {
+//       Array<int> ess_bdr(povlp->patch_mesh[ip]->bdr_attributes.Max());
+//       ess_bdr = 1;
+//       ovlp_prob->fespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+//    }
+
+//    ConstantCoefficient one(1.0);
+//    ConstantCoefficient sigma(-pow(omega, 2));
+//    PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+//    PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+//    PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+//    PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+//    ProductCoefficient c2_re0(sigma, detJ_re);
+//    ProductCoefficient c2_im0(sigma, detJ_im);
+//    ProductCoefficient c2_re(c2_re0, *ws);
+//    ProductCoefficient c2_im(c2_im0, *ws);
+//    SesquilinearForm a(ovlp_prob->fespaces[ip],ComplexOperator::HERMITIAN);
+
+//    a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+//                          new DiffusionIntegrator(c1_im));
+//    a.AddDomainIntegrator(new MassIntegrator(c2_re),
+//                          new MassIntegrator(c2_im));
+//    a.Assemble();
+
+//    OperatorPtr Alocal;
+//    a.FormSystemMatrix(ess_tdof_list,Alocal);
+//    ComplexSparseMatrix * AZ_ext = Alocal.As<ComplexSparseMatrix>();
+//    SparseMatrix * Mat = AZ_ext->GetSystemMatrix();
+//    Mat->Threshold(0.0);
+//    return Mat;
+// }
+
+// void DST::PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const
+// {
+//    FiniteElementSpace * fespace = bf->FESpace();
+//    Mesh * mesh = fespace->GetMesh();
+//    GridFunction gf(fespace);
+//    double * data = sol.GetData();
+//    gf.SetData(data);
+   
+//    string keys;
+//    if (ip == 0) keys = "keys mrRljc\n";
+//    // sol_sock << "solution\n" << *mesh << gf << keys << "valuerange -0.1 0.1 \n"  << flush;
+//    sol_sock << "solution\n" << *mesh << gf << keys << flush;
+// }
+
+
+// // void DST::SetSubMeshesAttributes()
+// // {
+// //    // For each subdomain there are 2 associated meshes. 
+// //    // The one from non-overlapping partitioning and the one from 
+// //    // an overlapping one. We need to mark the elements according to the 
+// //    // following diagram
+// //    //        _______________________________________
+// //    //       |       |                       |       |
+// //    //       |   3   |          4            |   5   |
+// //    //       |_______|_______________________|_______|
+// //    //       |       |                       |       |
+// //    //       |       |                       |       |
+// //    //       |       |                       |       |
+// //    //       |   2   |          9            |   6   |
+// //    //       |       |                       |       |
+// //    //       |       |                       |       |
+// //    //       |       |                       |       |
+// //    //       |_______|_______________________|_______|
+// //    //       |       |                       |       |
+// //    //       |   1   |          8            |   7   |
+// //    //       |_______|_______________________|_______|
+
+// //    for (int ip=0; ip<nrpatch; ip++)
+// //    {
+// //       Mesh * mesh1 = nvlp_prob->fespaces[ip]->GetMesh();
+// //       Vector pmin, pmax;
+// //       mesh1->GetBoundingBox(pmin,pmax);
+// //       Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+// //       int dim=mesh->Dimension();
+// //       for (int iel=0; iel<mesh->GetNE(); iel++)
+// //       {
+// //          Vector center(dim);
+// //          int geom = mesh->GetElementBaseGeometry(iel);
+// //          ElementTransformation * tr = mesh->GetElementTransformation(iel);
+// //          tr->Transform(Geometries.GetCenter(geom), center);
+// //          int attr = 9;
+// //          if (center[0] < pmin[0])
+// //          {
+// //             if (center[1] < pmin[1])
+// //             {
+// //                attr = 1;
+// //             }
+// //             else if (center[1] > pmax[1])
+// //             {
+// //                attr = 3;
+// //             }
+// //             else
+// //             {
+// //                attr = 2;
+// //             }
+// //          }
+// //          else if (center[0] < pmax[0])
+// //          {
+// //             if (center[1] < pmin[1])
+// //             {
+// //                attr = 8;
+// //             }
+// //             else if (center[1] > pmax[1])
+// //             {
+// //                attr = 4;
+// //             }
+// //          }
+// //          else
+// //          {
+// //             if (center[1] < pmin[1])
+// //             {
+// //                attr = 7;
+// //             }
+// //             else if (center[1] > pmax[1])
+// //             {
+// //                attr = 5;
+// //             }
+// //             else
+// //             {
+// //                attr = 6;
+// //             }
+            
+// //          }
+// //          mesh->SetAttribute(iel,attr);
+// //       }
+// //       mesh->SetAttributes();
+// //    }
+// // }
+
+// // void DST::GetRestrCoeffAttr(const Array<int> & directions, Array<int> & attr) const
+// // {
+// //    attr.SetSize(9); attr = 1;
+// //    // Set the attributes of the restricted coeff
+// //    if (directions[0] == 1)
+// //    {
+// //       if (directions[1] == 0)
+// //       {
+// //          attr[0] = 0;
+// //          attr[1] = 0;
+// //          attr[2] = 0;
+// //       }
+// //       else if (directions[1] == 1)
+// //       {
+// //          attr[0] = 0;
+// //          attr[1] = 0;
+// //          attr[2] = 0;
+// //          attr[7] = 0;
+// //          attr[6] = 0;
+// //       }
+// //       else if (directions[1] == -1)
+// //       {
+// //          attr[0] = 0;
+// //          attr[1] = 0;
+// //          attr[2] = 0;
+// //          attr[3] = 0;
+// //          attr[4] = 0;
+// //       }
+// //    }
+// //    else if (directions[0] == 0)
+// //    {
+// //       if (directions[1] == 1)
+// //       {
+// //          attr[0] = 0;
+// //          attr[7] = 0;
+// //          attr[6] = 0;
+// //       }
+// //       else if (directions[1] == -1)
+// //       {
+// //          attr[2] = 0;
+// //          attr[3] = 0;
+// //          attr[4] = 0;
+// //       }
+// //    }
+// //    if (directions[0] == -1)
+// //    {
+// //       if (directions[1] == 0)
+// //       {
+// //          attr[4] = 0;
+// //          attr[5] = 0;
+// //          attr[6] = 0;
+// //       }
+// //       else if (directions[1] == 1)
+// //       {
+// //          attr[0] = 0;
+// //          attr[7] = 0;
+// //          attr[6] = 0;
+// //          attr[5] = 0;
+// //          attr[4] = 0;
+// //       }
+// //       else if (directions[1] == -1)
+// //       {
+// //          attr[2] = 0;
+// //          attr[3] = 0;
+// //          attr[4] = 0;
+// //          attr[5] = 0;
+// //          attr[6] = 0;
+// //       }
+// //    }
+// // }
+
+// // double DST::GetSolOvlpNorm(const Vector & sol, 
+// //                      const Array<int> & directions, int ip) const
+// // {
+// //    FiniteElementSpace * fes = ovlp_prob->fespaces[ip];
+// //    Mesh * mesh = fes->GetMesh();
+   
+// //    int n = fes->GetTrueVSize();
+// //    GridFunction gf_re(fes);
+// //    GridFunction gf_im(fes);
+// //    double * data = sol.GetData();
+// //    gf_re.SetData(data);
+// //    gf_im.SetData(&data[0]);
+
+// //    Array<int> elems(mesh->GetNE()); elems = 0;
+// //    // Find the elements in the ovlp
+// //    Array<int> attr;
+// //    Array<int> direct(2); 
+// //    direct[0] = -directions[0];
+// //    direct[1] = -directions[1];
+// //    GetRestrCoeffAttr(direct,attr);
+// //    for (int iel = 0; iel<mesh->GetNE(); iel++)
+// //    {
+      
+// //       int i = mesh->GetAttribute(iel);
+// //       if (attr[i-1] == 0)
+// //       {
+// //          // elems.Append(iel);
+// //          elems[iel] = 1;
+// //       }
+// //    }
+
+// //    ConstantCoefficient zero(0.0);
+// //    GridFunction error(fes);
+
+// //    gf_re.ComputeElementL2Errors(zero, error);
+
+// //    double norm = 0.0;
+// //    for (int iel = 0; iel<mesh->GetNE(); iel++)
+// //    {
+// //       if (elems[iel] == 1) norm+=error[iel];
+// //    }
+
+// //    return norm;
+// // }

examples/maxwell-solver/DST2D/back/DSTnew.hpp

@@ -0,0 +1,52 @@
+// #pragma once
+// #include "Utilities.hpp"
+// #include "PML.hpp"
+// using namespace std;
+// using namespace mfem;
+
+// class DST : public Solver//
+// {
+// private:
+//    int nrpatch;
+//    int dim;
+//    SesquilinearForm *bf=nullptr;
+//    MeshPartition * povlp=nullptr;
+//    double omega = 0.5;
+//    Coefficient * ws;
+//    int nrlayers;
+//    int nxyz[3];
+//    const Operator * A=nullptr;
+//    DofMap * ovlp_prob = nullptr;
+//    Array<SparseMatrix *> PmlMat;
+//    Array<KLUSolver *> PmlMatInv;
+//    Array2D<double> Pmllength;
+//    Array3D<int> subdomains;
+//    mutable Array<Vector *> f_orig;
+//    int ntransf_directions;
+//    int nsweeps;
+//    Array2D<int> sweeps;
+//    Array<int> dirx;
+//    Array<int> diry;
+//    Array<int> dirz;
+//    mutable Array<Array<Vector * >> f_transf;
+//    Array<Array<Vector * >> usol;
+
+//    SparseMatrix * GetPmlSystemMatrix(int ip);
+//    void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+//    void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+//                           int ip, Array<int> directions, int nlayers, bool local=false) const;
+//    void GetChiRes(const Vector & res, Vector & cfres,
+//                int ip, Array<int> directions, int nlayers) const;                             
+//    void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+//    int GetPatchId(const Array<int> & ijk) const;
+//    void Getijk(int ip, int & i, int & j, int & k ) const;
+//    int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+// public:
+//    DST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+//        double omega_, Coefficient * ws_, int nrlayers_);
+//    virtual void SetOperator(const Operator &op) {A = &op;}
+//    virtual void Mult(const Vector &r, Vector &z) const;
+//    virtual ~DST();
+// };
+
+

examples/maxwell-solver/DST2D/back/DiagST.hpp

@@ -0,0 +1,122 @@
+// // #pragma once
+// // #include "Utilities.hpp"
+// // #include "PML.hpp"
+// // using namespace std;
+// // using namespace mfem;
+
+// // class DiagST : public Solver//
+// // {
+// // private:
+// //    int nrpatch;
+// //    int dim;
+// //    SesquilinearForm *bf=nullptr;
+// //    MeshPartition * povlp=nullptr;
+// //    double omega = 0.5;
+// //    Coefficient * ws;
+// //    int nrlayers;
+// //    int ovlpnrlayers;
+// //    int nxyz[3];
+// //    const Operator * A=nullptr;
+// //    Vector B;
+// //    DofMap * ovlp_prob = nullptr;
+// //    Array<SparseMatrix *> PmlMat;
+// //    Array<KLUSolver *> PmlMatInv;
+// //    Array2D<double> Pmllength;
+// //    Array3D<int> subdomains;
+// //    mutable Array<Vector *> f_orig;
+// //    int ntransf_directions;
+// //    int nsweeps;
+// //    Array2D<int> sweeps;
+// //    Array<int> dirx;
+// //    Array<int> diry;
+// //    Array<int> dirz;
+// //    mutable Array<Array<Vector * >> f_transf;
+// //    Array<Array<Vector * >> usol;
+
+// //    SparseMatrix * GetPmlSystemMatrix(int ip);
+// //    void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+// //    // void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+// //                         //   int ip, Array<int> directions, bool local=false) const;
+// //    void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+// //                         int ip, Array<int> directions, int ovlpnlayers, bool local=false) const;                       
+// //    void GetChiRes(const Vector & res, Vector & cfres,
+// //                   int ip, Array<int> directions, int nlayers) const;                       
+// //    void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+// //    int  GetDirectionId(const Array<int> & ijk) const;
+// //    void GetDirectionijk(int id, Array<int> & ijk) const;
+// //    void ConstructDirectionsMap();
+// //    int GetPatchId(const Array<int> & ijk) const;
+// //    void Getijk(int ip, int & i, int & j, int & k ) const;
+// //    int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+// // public:
+// //    DiagST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+// //        double omega_, Coefficient * ws_, int nrlayers_);
+// //    void SetLoadVector(Vector load) { B = load;}
+// //    virtual void SetOperator(const Operator &op) {A = &op;}
+// //    virtual void Mult(const Vector &r, Vector &z) const;
+// //    virtual ~DiagST();
+// // };
+
+// #pragma once
+// #include "Utilities.hpp"
+// #include "PML.hpp"
+// using namespace std;
+// using namespace mfem;
+
+// class DiagST : public Solver//
+// {
+// private:
+//    int nrpatch;
+//    int dim;
+//    SesquilinearForm *bf=nullptr;
+//    MeshPartition * povlp=nullptr;
+//    MeshPartition * novlp=nullptr;
+//    double omega = 0.5;
+//    Coefficient * ws;
+//    int nrlayers;
+//    int ovlpnrlayers;
+//    int nxyz[3];
+//    const Operator * A=nullptr;
+//    Vector B;
+//    DofMap * ovlp_prob = nullptr;
+//    DofMap * nvlp_prob = nullptr;
+//    Array<SparseMatrix *> PmlMat;
+//    Array<KLUSolver *> PmlMatInv;
+//    Array2D<double> Pmllength;
+//    Array3D<int> subdomains;
+//    mutable Array<Vector *> f_orig;
+//    int ntransf_directions;
+//    int nsweeps;
+//    Array2D<int> sweeps;
+//    Array<int> dirx;
+//    Array<int> diry;
+//    Array<int> dirz;
+//    mutable Array<Array<Vector * >> f_transf;
+//    Array<Vector * > usol;
+
+//    SparseMatrix * GetPmlSystemMatrix(int ip);
+//    void PlotSolution(Vector & sol, socketstream & sol_sock, int ip, bool localdomain = false, bool pmldomain = false) const;
+//    // void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+//                         //   int ip, Array<int> directions, bool local=false) const;
+//    void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+//                         int ip, Array<int> directions, int ovlpnlayers, bool local=false) const;                       
+//    void GetChiRes(const Vector & res, Vector & cfres,
+//                   int ip, Array<int> directions, int nlayers) const;                       
+//    void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+//    int  GetDirectionId(const Array<int> & ijk) const;
+//    void GetDirectionijk(int id, Array<int> & ijk) const;
+//    void ConstructDirectionsMap();
+//    int GetPatchId(const Array<int> & ijk) const;
+//    void Getijk(int ip, int & i, int & j, int & k ) const;
+//    int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+// public:
+//    DiagST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+//        double omega_, Coefficient * ws_, int nrlayers_);
+//    void SetLoadVector(Vector load) { B = load;}
+//    virtual void SetOperator(const Operator &op) {A = &op;}
+//    virtual void Mult(const Vector &r, Vector &z) const;
+//    virtual ~DiagST();
+// };
+
+
+

examples/maxwell-solver/DST2D/back/DiagST1.hpp

@@ -0,0 +1,120 @@
+// // #pragma once
+// // #include "Utilities.hpp"
+// // #include "PML.hpp"
+// // using namespace std;
+// // using namespace mfem;
+
+// // class DiagST : public Solver//
+// // {
+// // private:
+// //    int nrpatch;
+// //    int dim;
+// //    SesquilinearForm *bf=nullptr;
+// //    MeshPartition * povlp=nullptr;
+// //    double omega = 0.5;
+// //    Coefficient * ws;
+// //    int nrlayers;
+// //    int ovlpnrlayers;
+// //    int nxyz[3];
+// //    const Operator * A=nullptr;
+// //    Vector B;
+// //    DofMap * ovlp_prob = nullptr;
+// //    Array<SparseMatrix *> PmlMat;
+// //    Array<KLUSolver *> PmlMatInv;
+// //    Array2D<double> Pmllength;
+// //    Array3D<int> subdomains;
+// //    mutable Array<Vector *> f_orig;
+// //    int ntransf_directions;
+// //    int nsweeps;
+// //    Array2D<int> sweeps;
+// //    Array<int> dirx;
+// //    Array<int> diry;
+// //    Array<int> dirz;
+// //    mutable Array<Array<Vector * >> f_transf;
+// //    Array<Array<Vector * >> usol;
+
+// //    SparseMatrix * GetPmlSystemMatrix(int ip);
+// //    void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+// //    // void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+// //                         //   int ip, Array<int> directions, bool local=false) const;
+// //    void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+// //                         int ip, Array<int> directions, int ovlpnlayers, bool local=false) const;                       
+// //    void GetChiRes(const Vector & res, Vector & cfres,
+// //                   int ip, Array<int> directions, int nlayers) const;                       
+// //    void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+// //    int  GetDirectionId(const Array<int> & ijk) const;
+// //    void GetDirectionijk(int id, Array<int> & ijk) const;
+// //    void ConstructDirectionsMap();
+// //    int GetPatchId(const Array<int> & ijk) const;
+// //    void Getijk(int ip, int & i, int & j, int & k ) const;
+// //    int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+// // public:
+// //    DiagST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+// //        double omega_, Coefficient * ws_, int nrlayers_);
+// //    void SetLoadVector(Vector load) { B = load;}
+// //    virtual void SetOperator(const Operator &op) {A = &op;}
+// //    virtual void Mult(const Vector &r, Vector &z) const;
+// //    virtual ~DiagST();
+// // };
+
+// #pragma once
+// #include "Utilities.hpp"
+// #include "PML.hpp"
+// using namespace std;
+// using namespace mfem;
+
+// class DiagST : public Solver//
+// {
+// private:
+//    int nrpatch;
+//    int dim;
+//    SesquilinearForm *bf=nullptr;
+//    MeshPartition * povlp=nullptr;
+//    double omega = 0.5;
+//    Coefficient * ws;
+//    int nrlayers;
+//    int ovlpnrlayers;
+//    int nxyz[3];
+//    const Operator * A=nullptr;
+//    Vector B;
+//    DofMap * ovlp_prob = nullptr;
+//    Array<SparseMatrix *> PmlMat;
+//    Array<KLUSolver *> PmlMatInv;
+//    Array2D<double> Pmllength;
+//    Array3D<int> subdomains;
+//    mutable Array<Vector *> f_orig;
+//    int ntransf_directions;
+//    int nsweeps;
+//    Array2D<int> sweeps;
+//    Array<int> dirx;
+//    Array<int> diry;
+//    Array<int> dirz;
+//    mutable Array<Array<Vector * >> f_transf;
+//    Array<Array<Vector * >> usol;
+
+//    SparseMatrix * GetPmlSystemMatrix(int ip);
+//    void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+//    // void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+//                         //   int ip, Array<int> directions, bool local=false) const;
+//    void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+//                         int ip, Array<int> directions, int ovlpnlayers, bool local=false) const;                       
+//    void GetChiRes(const Vector & res, Vector & cfres,
+//                   int ip, Array<int> directions, int nlayers) const;                       
+//    void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+//    int  GetDirectionId(const Array<int> & ijk) const;
+//    void GetDirectionijk(int id, Array<int> & ijk) const;
+//    void ConstructDirectionsMap();
+//    int GetPatchId(const Array<int> & ijk) const;
+//    void Getijk(int ip, int & i, int & j, int & k ) const;
+//    int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+// public:
+//    DiagST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+//        double omega_, Coefficient * ws_, int nrlayers_);
+//    void SetLoadVector(Vector load) { B = load;}
+//    virtual void SetOperator(const Operator &op) {A = &op;}
+//    virtual void Mult(const Vector &r, Vector &z) const;
+//    virtual ~DiagST();
+// };
+
+
+

examples/maxwell-solver/DST2D/back/DiagST2.cpp

@@ -0,0 +1,696 @@
+// //Diagonal Source Transfer Preconditioner
+
+// #include "DiagST.hpp"
+
+// DiagST::DiagST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+//          double omega_, Coefficient * ws_,  int nrlayers_)
+//    : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), 
+//      bf(bf_), Pmllength(Pmllength_), omega(omega_), ws(ws_), nrlayers(nrlayers_)
+// {
+//    Mesh * mesh = bf->FESpace()->GetMesh();
+//    dim = mesh->Dimension();
+
+//    // ----------------- Step 1 --------------------
+//    // Introduce 2 layered partitios of the domain 
+//    // 
+//    int partition_kind;
+
+//    // 1. Ovelapping partition with overlap = 2h 
+//    partition_kind = 2; // Non Overlapping partition 
+//    int nx=3;
+//    int ny=3; 
+//    int nz=1;
+//    povlp = new MeshPartition(mesh, partition_kind,nx,ny,nz);
+//    nxyz[0] = povlp->nxyz[0];
+//    nxyz[1] = povlp->nxyz[1];
+//    nxyz[2] = povlp->nxyz[2];
+//    nrpatch = povlp->nrpatch;
+//    // cout<< "nrpatch = " << nrpatch << endl;
+//    // cout << "nx = " << nx << endl;
+//    // cout << "ny = " << ny << endl;
+//    // cout << "nz = " << nz << endl;
+//    subdomains = povlp->subdomains;
+//    // for (int k = 0; k<nxyz[2]; k++)
+//    // {
+//    //    for (int j = 0; j<nxyz[1]; j++)
+//    //    {
+//    //       for (int i = 0; i<nxyz[0]; i++)
+//    //       {
+//    //          Array<int> ijk(3);
+//    //          ijk[0]=i;
+//    //          ijk[1]=j;
+//    //          ijk[2]=k;
+//    //          // cout << "("<<i<<","<<j<<","<<k<<") = " << povlp->subdomains(i,j,k) << endl;
+//    //          cout << "("<<i<<","<<j<<","<<k<<") = " << GetPatchId(ijk) << endl;
+//    //       }
+//    //    }   
+//    // }
+
+//    // for (int ip = 0; ip<nrpatch; ip++)
+//    // {
+//    //    int i, j, k;
+//    //    Getijk(ip, i,j,k);
+//    //    cout << "ip = " << ip << ": ("<<i<<","<<j<<","<<k<<")"<< endl;
+//    // }
+
+   
+
+//    //
+//    // ----------------- Step 1a -------------------
+//    // Save the partition for visualization
+//    // SaveMeshPartition(povlp->patch_mesh, "output/mesh_ovlp.", "output/sol_ovlp.");
+
+//    // // // ------------------Step 2 --------------------
+//    // // // Construct the dof maps from subdomains to global (for the extended and not)
+//    ovlp_prob  = new DofMap(bf,povlp,nrlayers); 
+
+//    // ------------------Step 3 --------------------
+//    // Assemble the PML Problem matrices and factor them
+//    PmlMat.SetSize(nrpatch);
+//    PmlMatInv.SetSize(nrpatch);
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       PmlMat[ip] = GetPmlSystemMatrix(ip);
+//       PmlMatInv[ip] = new KLUSolver;
+//       PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
+//    }
+
+//    // Set up src arrays size
+//    f_orig.SetSize(nrpatch);
+//    f_transf.SetSize(nrpatch);
+
+
+//    // Construct a simple map used for directions of transfer
+//    ConstructDirectionsMap();
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       int n = 2*ovlp_prob->fespaces[ip]->GetTrueVSize(); // (x 2 for complex ) 
+//       f_orig[ip] = new Vector(n); *f_orig[ip] = 0.0;
+//       f_transf[ip].SetSize(ntransf_directions);
+//       for (int i=0;i<ntransf_directions; i++)
+//       {
+//          f_transf[ip][i] = new Vector(n); *f_transf[ip][i] = 0.0;
+//       }
+//    }
+// }
+
+// SparseMatrix * DiagST::GetPmlSystemMatrix(int ip)
+// {
+//    double h = GetUniformMeshElementSize(ovlp_prob->PmlMeshes[ip]);
+//    Array2D<double> length(dim,2);
+//    length = h*(nrlayers);
+
+//    CartesianPML pml(ovlp_prob->PmlMeshes[ip], length);
+//    pml.SetOmega(omega);
+
+//    Array <int> ess_tdof_list;
+//    if (ovlp_prob->PmlMeshes[ip]->bdr_attributes.Size())
+//    {
+//       Array<int> ess_bdr(ovlp_prob->PmlMeshes[ip]->bdr_attributes.Max());
+//       ess_bdr = 1;
+//       ovlp_prob->PmlFespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+//    }
+
+//    ConstantCoefficient one(1.0);
+//    ConstantCoefficient sigma(-pow(omega, 2));
+
+//    PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+//    PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+//    PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+//    PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+//    ProductCoefficient c2_re0(sigma, detJ_re);
+//    ProductCoefficient c2_im0(sigma, detJ_im);
+
+//    ProductCoefficient c2_re(c2_re0, *ws);
+//    ProductCoefficient c2_im(c2_im0, *ws);
+
+//    SesquilinearForm a(ovlp_prob->PmlFespaces[ip],ComplexOperator::HERMITIAN);
+
+//    a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+//                          new DiffusionIntegrator(c1_im));
+//    a.AddDomainIntegrator(new MassIntegrator(c2_re),
+//                          new MassIntegrator(c2_im));
+//    a.Assemble();
+
+//    OperatorPtr Alocal;
+//    a.FormSystemMatrix(ess_tdof_list,Alocal);
+//    ComplexSparseMatrix * AZ_ext = Alocal.As<ComplexSparseMatrix>();
+//    SparseMatrix * Mat = AZ_ext->GetSystemMatrix();
+//    Mat->Threshold(0.0);
+//    return Mat;
+// }
+
+
+// void DiagST::Mult(const Vector &r, Vector &z) const
+// {
+//    // Step 0
+//    // Restrict original sources to the patches
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       *f_orig[ip] = 0.0;
+//       for (int i=0;i<ntransf_directions; i++)
+//       {
+//          *f_transf[ip][i] = 0.0;
+//       }
+//    }
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+//       r.GetSubVector(*Dof2GlobalDof,*f_orig[ip]);
+//    }
+
+//    char vishost[] = "localhost";
+//    int  visport   = 19916;
+//    z = 0.0; 
+//    Vector rnew(r);
+//    Vector znew(z);
+//    znew = 0.0;
+   
+//    // in 2D there are a total of 4 sweeps
+//    // with nx + ny - 1 serial steps each
+//    // --------------------------------------------
+//    //       Sweep in the direction (1,1)
+//    // --------------------------------------------
+//    int nx = nxyz[0];
+//    int ny = nxyz[1];
+
+//    int nsteps = nx + ny - 1;
+//    // loop through the steps
+//    Array<int> sweep_direction(2); sweep_direction = 1;
+//    for (int s = 0; s<nsteps; s++)
+//    {
+//       // the patches involved are the ones such that
+//       // i+j = s
+//       // cout << "Step no: " << s << endl;
+//       for (int i=0;i<nx; i++)
+//       {
+//          int j = s-i;
+//          if (j<0 || j>=ny) continue;
+//          // cout << "Patch no: (" << i <<"," << j << ")" << endl; 
+
+//          // find patch id
+//          Array<int> ij(2); ij[0] = i; ij[1]=j;
+//          int ip = GetPatchId(ij);
+//          // cout << "ip = " << ip << endl;
+
+//          // Solve the PML problem in patch ip with all sources
+//          // Original and all transfered (maybe some of them)
+//          Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+//          Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip];
+//          int ndofs = Dof2GlobalDof->Size();
+//          Vector sol_local(ndofs);
+//          Vector res_local(ndofs);
+//          res_local = *f_orig[ip];
+
+//          // RULE 3.1 (form Leng & Ju paper)
+//          for (int nd=0; nd<ntransf_directions; nd++)
+//          {
+//             // only the transfer sourcers in the similar direction
+//             // of the sweep should be used
+//             Array<int> ijk(2);
+//             GetDirectionijk(nd,ijk);
+//             ijk[0]*=-1; ijk[1]*=-1;
+//             if (sweep_direction[0]*ijk[0] + sweep_direction[1]*ijk[1] > 0)
+//             {
+
+//              // INSTEAD OF MULTIPLE COPIES FOR EACH DIRECTION 
+//              // USE MULTIPLE COPIES FOR EACH SWEEP FOR EACH SUBDOMAIN
+//              // i.e, each subdomain will have 4 different transfer sources 
+//              // which you accumulate as you go.     
+
+
+//                res_local += *f_transf[ip][nd];
+//             }
+//          }
+//          // Extend by zero to the PML mesh
+//          int nrdof_ext = PmlMat[ip]->Height();
+      
+//          Vector res_ext(nrdof_ext); res_ext = 0.0;
+//          Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+
+//          res_ext.SetSubVector(*Dof2PmlDof,res_local);
+//          PmlMatInv[ip]->Mult(res_ext, sol_ext);
+
+//          // Multiply with the cutoff functions, find the new sources and 
+//          // and propagate to all neighboring subdomains 
+//          // (possible 8 in 2D, 26 in 3D)
+//          TransferSources(ip, sol_ext);
+//          Vector cfsol_ext(sol_ext.Size());
+//          // cut off the ip solution to all possible directions
+//          Array<int>directions(2); directions = 0; 
+//          if (i+1<nx) directions[0] = 1;
+//          if (j+1<ny) directions[1] = 1;
+//          GetCutOffSolution(sol_ext,cfsol_ext,ip,directions,true);
+//          // directions = 0;
+//          // if (i-1>=0) directions[0] = -1;
+//          // if (j-1>=0) directions[1] = -1;
+//          // sol_ext = cfsol_ext;
+//          // GetCutOffSolution(sol_ext,cfsol_ext,ip,directions,true);
+//          cfsol_ext.GetSubVector(*Dof2PmlDof, sol_local);
+//          znew = 0.0;
+//          znew.SetSubVector(*Dof2GlobalDof, sol_local);
+//          z+=znew;
+//       }
+//       // socketstream zsock(vishost, visport);
+//       // PlotSolution(z,zsock,0);
+//       // cin.get();
+//    }
+   
+
+// }
+
+// void DiagST::PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const
+// {
+//    FiniteElementSpace * fespace = bf->FESpace();
+//    Mesh * mesh = fespace->GetMesh();
+//    GridFunction gf(fespace);
+//    double * data = sol.GetData();
+//    // gf.SetData(&data[fespace->GetTrueVSize()]);
+//    gf.SetData(data);
+   
+//    string keys;
+//    if (ip == 0) keys = "keys mrRljc\n";
+//    sol_sock << "solution\n" << *mesh << gf << keys << "valuerange -0.1 0.1 \n"  << flush;
+// }
+
+// void DiagST::GetCutOffSolution(const Vector & sol, Vector & cfsol, 
+//                                int ip0, Array<int> directions, bool local) const
+// {
+//    // int l,k;
+//    int d = directions.Size();
+//    int directx = directions[0]; // 1,0,-1
+//    int directy = directions[1]; // 1,0,-1
+//    int directz;
+//    if (d ==3) directz = directions[2];
+
+//    // cout << "ip0 = " << ip0 << endl; 
+
+//    int i0, j0, k0;
+//    Getijk(ip0,i0, j0, k0);
+//    // cout << "(i0,j0) = " << "(" <<i0 <<","<<j0<<")" << endl;
+
+//    // 2D for now...
+//    // Find the id of the neighboring patch
+//    int i1 = i0 + directx;
+//    int j1 = j0 + directy;
+//    MFEM_VERIFY(i1 < nxyz[0] && i1>=0, "GetCutOffSolution: i1 out of bounds");
+//    MFEM_VERIFY(j1 < nxyz[1] && j1>=0, "GetCutOffSolution: j1 out of bounds");
+   
+//    Array<int> ijk(d);
+//    ijk[0] = i1;
+//    ijk[1] = j1;
+//    int ip1 = GetPatchId(ijk);
+
+//    // cout << "ip1 = " << ip1 << endl;
+//    // cout << "(i1,j1) = " << "(" << i1 <<","<<j1<<")" << endl;
+
+//    Mesh * mesh0 = ovlp_prob->fespaces[ip0]->GetMesh();
+//    Mesh * mesh1 = ovlp_prob->fespaces[ip1]->GetMesh();
+   
+//    Vector pmin0, pmax0;
+//    Vector pmin1, pmax1;
+//    mesh0->GetBoundingBox(pmin0, pmax0);
+//    mesh1->GetBoundingBox(pmin1, pmax1);
+
+//    Array2D<double> h(dim,2); h = 0.0;
+   
+//    if (directions[0]==1)
+//    {
+//       h[0][1] = pmax0[0] - pmin1[0];
+//    }
+//    if (directions[0]==-1)
+//    {
+//       h[0][0] = pmax1[0] - pmin0[0];
+//    }
+//    if (directions[1]==1)
+//    {
+//       h[1][1] = pmax0[1] - pmin1[1];
+//    }
+//    if (directions[1]==-1)
+//    {
+//       h[1][0] = pmax1[1] - pmin0[1];
+//    }
+
+//    CutOffFnCoefficient cf(CutOffFncn, pmin0, pmax0, h);
+
+//    double * data = sol.GetData();
+
+//    FiniteElementSpace * fespace;
+//    if (!local)
+//    {
+//       fespace = bf->FESpace();
+//    }
+//    else
+//    {
+//       fespace = ovlp_prob->PmlFespaces[ip0];
+//    }
+   
+//    int n = fespace->GetTrueVSize();
+//    // GridFunction cutF(fespace);
+//    // cutF.ProjectCoefficient(cf);
+//    // char vishost[] = "localhost";
+//    // int  visport   = 19916;
+
+//    // socketstream sub_sock1(vishost, visport);
+//    // sub_sock1 << "solution\n" << *fespace->GetMesh() << cutF << flush;
+//    // cin.get();
+
+
+//    GridFunction solgf_re(fespace, data);
+//    GridFunction solgf_im(fespace, &data[n]);
+
+//    // socketstream sub_sock(vishost, visport);
+//    // sub_sock << "solution\n" << *fespace->GetMesh() << solgf_re << flush;
+//    // cin.get();
+
+
+//    GridFunctionCoefficient coeff1_re(&solgf_re);
+//    GridFunctionCoefficient coeff1_im(&solgf_im);
+
+//    ProductCoefficient prod_re(coeff1_re, cf);
+//    ProductCoefficient prod_im(coeff1_im, cf);
+
+//    ComplexGridFunction gf(fespace);
+//    gf.ProjectCoefficient(prod_re,prod_im);
+
+//    cfsol.SetSize(sol.Size());
+//    cfsol = gf;
+//    // socketstream sub_sock2(vishost, visport);
+//    // sub_sock2 << "solution\n" << *fespace->GetMesh() << gf.real() << flush;
+//    // cin.get();
+// }
+
+// DiagST::~DiagST()
+// {
+//    for (int ip = 0; ip<nrpatch; ++ip)
+//    {
+//       delete PmlMatInv[ip];
+//       delete PmlMat[ip];
+//    }
+//    PmlMat.DeleteAll();
+//    PmlMatInv.DeleteAll();
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       delete f_orig[ip];
+//       for (int i=0;i<ntransf_directions; i++)
+//       {
+//          delete f_transf[ip][i];
+//       }
+//    }
+// }
+
+// void DiagST::Getijk(int ip, int & i, int & j, int & k) const
+// {
+//    k = ip/(nxyz[0]*nxyz[1]);
+//    j = (ip-k*nxyz[0]*nxyz[1])/nxyz[0];
+//    i = (ip-k*nxyz[0]*nxyz[1])%nxyz[0];
+// }
+
+// int DiagST::GetPatchId(const Array<int> & ijk) const
+// {
+//    int d=ijk.Size();
+//    if (d==2)
+//    {
+//       return subdomains(ijk[0],ijk[1],0);
+//    }
+//    else
+//    {
+//       return subdomains(ijk[0],ijk[1],ijk[2]);
+//    }
+// }
+
+// int DiagST::SourceTransfer(const Vector & Psi0, Array<int> direction, int ip0, Vector & Psi1) const
+// {
+//    // For now 2D problems only
+//    // Directions
+//    // direction (1,1)
+//    int i0,j0,k0;
+//    Getijk(ip0,i0,j0,k0);
+
+//    int i1 = i0+direction[0];   
+//    int j1 = j0+direction[1];   
+//    Array<int> ij(2); ij[0]=i1; ij[1]=j1;
+//    int ip1 = GetPatchId(ij);
+
+//    MFEM_VERIFY(i1 < nxyz[0] && i1>=0, "SourceTransfer: i1 out of bounds");
+//    MFEM_VERIFY(j1 < nxyz[1] && j1>=0, "SourceTransfer: j1 out of bounds");
+
+//    Array<int> * Dof2GlobalDof0 = &ovlp_prob->Dof2GlobalDof[ip0];
+//    Array<int> * Dof2GlobalDof1 = &ovlp_prob->Dof2GlobalDof[ip1];
+//    Psi1.SetSize(Dof2GlobalDof1->Size()); Psi1=0.0;
+//    Vector r(2*bf->FESpace()->GetTrueVSize());
+//    r = 0.0;
+//    r.SetSubVector(*Dof2GlobalDof0,Psi0);
+//    r.GetSubVector(*Dof2GlobalDof1,Psi1);
+//    return ip1;
+// }
+
+// void DiagST::ConstructDirectionsMap()
+// {
+//    // total of 8 possible directions of transfer (2D)
+//    // form left            ( 1 ,  0)
+//    // form left-above      ( 1 , -1)
+//    // form left-below      ( 1 ,  1)
+//    // form right           (-1 ,  0)
+//    // form right-below     (-1 ,  1)
+//    // form right-above     (-1 , -1)
+//    // form above           ( 0 , -1)
+//    // form below           ( 0 ,  1)
+//    ntransf_directions = pow(3,dim);
+
+//    dirx.SetSize(ntransf_directions);
+//    diry.SetSize(ntransf_directions);
+//    int n=3;
+//    Array<int> ijk(dim);
+//    if (dim==2)
+//    {
+//       for (int i=-1; i<=1; i++) // directions x
+//       {
+//          for (int j=-1; j<=1; j++) // directions y
+//          {
+//             ijk[0]=i;
+//             ijk[1]=j;
+//             int k=GetDirectionId(ijk);
+//             dirx[k]=i;
+//             diry[k]=j;
+//          }
+//       }
+//    }
+//    else if (dim==3)
+//    {
+//       dirz.SetSize(ntransf_directions);
+//       for (int i=-1; i<=1; i++) // directions x
+//       {
+//          for (int j=-1; j<=1; j++) // directions y
+//          {
+//             for (int k=-1; k<=1; k++) // directions zß
+//             {
+//                ijk[0]=i;
+//                ijk[1]=j;
+//                ijk[2]=k;
+//                int l=GetDirectionId(ijk);
+//                dirx[l]=i;
+//                diry[l]=j;
+//                dirz[l]=k;
+//             }
+//          }
+//       }
+//    }
+
+//    // cout << "dirx = " << endl;
+//    // dirx.Print(cout,ntransf_directions);
+//    // cout << "diry = " << endl;
+//    // diry.Print(cout,ntransf_directions);
+
+//    if (dim==2)
+//    {
+//       for (int id=0; id<9; id++)
+//       {
+//          GetDirectionijk(id,ijk);
+//          // cout << "for id = " << id << ": (" <<ijk[0] << ", " << ijk[1] << ")" << endl;
+//       }
+//    }
+//    else
+//    {
+//       cout << "dirz = " << endl;
+//       dirz.Print(cout,ntransf_directions);
+//       for (int id=0; id<27; id++)
+//       {
+//          GetDirectionijk(id,ijk);
+//          // cout << "for id = " << id << ": (" <<ijk[0] << ", " <<ijk[1] << ", " << ijk[2] << ")" << endl;
+//       }
+//    }
+// }
+
+// int DiagST::GetDirectionId(const Array<int> & ijk) const
+// {
+//    int d = ijk.Size();
+//    int n=3;
+//    if (d==2)
+//    {
+//       return (ijk[0]+1)*n+(ijk[1]+1);
+//    }
+//    else
+//    {
+//       return (ijk[0]+1)*n*n+(ijk[1]+1)*n+ijk[2]+1;
+//    }
+// }
+
+// void DiagST::GetDirectionijk(int id, Array<int> & ijk) const
+// {
+//    int d = ijk.Size();
+//    int n=3;
+//    if (d==2)
+//    {
+//       ijk[0]=id/n - 1;
+//       ijk[1]=id%n - 1;
+//    }
+//    else
+//    {
+//       ijk[0]=id/(n*n)-1;
+//       ijk[1]=(id-(ijk[0]+1)*n*n)/n - 1;
+//       ijk[2]=(id-(ijk[0]+1)*n*n)%n - 1;
+//    }
+//    // cout << "ijk = " ; ijk.Print();
+// }
+
+
+
+
+// void DiagST::TransferSources(int ip0, Vector & sol_ext) const
+// {
+//    // Find all neighbors of patch ip
+//    int nx = nxyz[0];
+//    int ny = nxyz[1];
+//    int i0, j0, k0;
+//    Getijk(ip0, i0,j0,k0);
+//    // cout << "Transfer to : " << endl;
+//    // loop through possible directions
+//    for (int i=-1; i<2; i++)
+//    {
+//       int i1 = i0 + i;
+//       if (i1 <0 || i1>=nx) continue;
+//       for (int j=-1; j<2; j++)
+//       {
+//          int j1 = j0 + j;
+//          if (j1 <0 || j1>=ny) continue;
+//          // cout << "(" << i1 << "," << j1 <<"), ";
+//          // Find ip 1
+//          Array<int> ij1(2); ij1[0] = i1; ij1[1]=j1;
+//          int ip1 = GetPatchId(ij1);
+//          // cout << "ip1 = " << ip1;
+//          // cout << " in the direction of (" << i <<", " <<j <<")" << endl;
+//          Array<int> directions(2);
+//          directions[0] = i;
+//          directions[1] = j;
+//          Vector cfsol_ext;
+//          Vector res_ext(sol_ext.Size());
+//          GetCutOffSolution(sol_ext,cfsol_ext,ip0,directions,true);
+//          // sol_ext = cfsol_ext;
+//          // Calculate source to be transfered
+//          PmlMat[ip0]->Mult(cfsol_ext, res_ext); res_ext*= -1.0;
+//          Array<int> *Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip0];
+//          Vector res_local(Dof2PmlDof->Size()); res_local = 0.0;
+//          res_ext.GetSubVector(*Dof2PmlDof,res_local);
+//          // Find the direction id to store the transfered source
+//          Array<int> dij(2); dij[0] = -i; dij[1] = -j; 
+//          int did = GetDirectionId(dij);
+//          int jp1 = SourceTransfer(res_local,directions,ip0,*f_transf[ip1][did]);
+//          MFEM_VERIFY(ip1 == jp1, "Check SourceTransfer patch id");
+//       }  
+//    }
+// }
+
+
+
+
+
+
+// // void DiagST::GetCutOffSolution(Vector & sol, int ip, int direction, bool local) const
+// // {
+// //    int l,k;
+// //    k=(direction == 1)? ip: ip-1;
+// //    l=(direction == 1)? ip+1: ip;
+
+// //    Mesh * mesh1 = ovlp_prob->fespaces[k]->GetMesh();
+// //    Mesh * mesh2 = ovlp_prob->fespaces[l]->GetMesh();
+   
+// //    Vector pmin1, pmax1;
+// //    Vector pmin2, pmax2;
+// //    mesh1->GetBoundingBox(pmin1, pmax1);
+// //    mesh2->GetBoundingBox(pmin2, pmax2);
+
+// //    Array2D<double> h(dim,2); h = 0.0;
+   
+// //    Vector pmin, pmax;
+// //    if (direction == 1)
+// //    {
+// //       h[0][1] = pmax1[0] - pmin2[0];
+// //       CutOffFnCoefficient cf(CutOffFncn, pmin1, pmax1, h);
+// //       pmin = pmin1;
+// //       pmax = pmax1;
+// //    }
+// //    else if (direction == -1)
+// //    {
+// //       h[0][0] = pmax1[0] - pmin2[0];
+// //       pmin = pmin2;
+// //       pmax = pmax2;
+// //    }
+// //    CutOffFnCoefficient cf(CutOffFncn, pmin, pmax, h);
+
+// //    double * data = sol.GetData();
+
+// //    FiniteElementSpace * fespace;
+// //    if (!local)
+// //    {
+// //       fespace = bf->FESpace();
+// //    }
+// //    else
+// //    {
+// //       if (direction == 1)
+// //       {
+// //          fespace = ovlp_prob->PmlFespaces[k];
+// //       }
+// //       else
+// //       {
+// //          fespace = ovlp_prob->PmlFespaces[l];
+// //       }
+// //    }
+   
+// //    int n = fespace->GetTrueVSize();
+// //    GridFunction cutF(fespace);
+// //    cutF.ProjectCoefficient(cf);
+// //    // char vishost[] = "localhost";
+// //    // int  visport   = 19916;
+
+   
+
+// //    // socketstream sub_sock1(vishost, visport);
+// //    // sub_sock1 << "solution\n" << *fespace->GetMesh() << cutF << flush;
+// //    // cin.get();
+
+
+// //    GridFunction solgf_re(fespace, data);
+
+// //    // socketstream sub_sock(vishost, visport);
+// //    // sub_sock << "solution\n" << *fespace->GetMesh() << solgf_re << flush;
+// //    // cin.get();
+
+// //    GridFunction solgf_im(fespace, &data[n]);
+
+// //    GridFunctionCoefficient coeff1_re(&solgf_re);
+// //    GridFunctionCoefficient coeff1_im(&solgf_im);
+
+// //    ProductCoefficient prod_re(coeff1_re, cf);
+// //    ProductCoefficient prod_im(coeff1_im, cf);
+
+// //    ComplexGridFunction gf(fespace);
+// //    gf.ProjectCoefficient(prod_re,prod_im);
+
+// //    sol = gf;
+// //    // socketstream sub_sock2(vishost, visport);
+// //    // sub_sock2 << "solution\n" << *fespace->GetMesh() << gf.real() << flush;
+// //    // cin.get();
+// // }

examples/maxwell-solver/DST2D/back/DiagST2.hpp

@@ -0,0 +1,53 @@
+// #pragma once
+// #include "Utilities.hpp"
+// #include "PML.hpp"
+// using namespace std;
+// using namespace mfem;
+
+// class DiagST : public Solver//
+// {
+// private:
+//    int nrpatch;
+//    int dim;
+//    SesquilinearForm *bf=nullptr;
+//    MeshPartition * povlp=nullptr;
+//    double omega = 0.5;
+//    Coefficient * ws;
+//    int nrlayers;
+//    int nxyz[3];
+//    const Operator * A=nullptr;
+//    Vector B;
+//    DofMap * ovlp_prob = nullptr;
+//    Array<SparseMatrix *> PmlMat;
+//    Array<KLUSolver *> PmlMatInv;
+//    Array2D<double> Pmllength;
+//    Array3D<int> subdomains;
+//    mutable Array<Vector *> f_orig;
+//    int ntransf_directions;
+//    UniqueIndexGenerator gen;
+//    Array<int> dirx;
+//    Array<int> diry;
+//    Array<int> dirz;
+//    mutable Array<Array<Vector * >> f_transf;
+
+//    SparseMatrix * GetPmlSystemMatrix(int ip);
+//    void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+//    void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+//                           int ip, Array<int> directions, bool local=false) const;
+//    void TransferSources(int ip, Vector & sol_ext) const;
+//    int  GetDirectionId(const Array<int> & ijk) const;
+//    void GetDirectionijk(int id, Array<int> & ijk) const;
+//    void ConstructDirectionsMap();
+//    int GetPatchId(const Array<int> & ijk) const;
+//    void Getijk(int ip, int & i, int & j, int & k ) const;
+//    int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+// public:
+//    DiagST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+//        double omega_, Coefficient * ws_, int nrlayers_);
+//    void SetLoadVector(Vector load) { B = load;}
+//    virtual void SetOperator(const Operator &op) {A = &op;}
+//    virtual void Mult(const Vector &r, Vector &z) const;
+//    virtual ~DiagST();
+// };
+
+

examples/maxwell-solver/DST2D/back/DiagSTnew.hpp

@@ -0,0 +1,120 @@
+// #pragma once
+// #include "Utilities.hpp"
+// #include "PML.hpp"
+// using namespace std;
+// using namespace mfem;
+
+// class DiagST : public Solver//
+// {
+// private:
+//    int nrpatch;
+//    int dim;
+//    SesquilinearForm *bf=nullptr;
+//    MeshPartition * povlp=nullptr;
+//    double omega = 0.5;
+//    Coefficient * ws;
+//    int nrlayers;
+//    int ovlpnrlayers;
+//    int nxyz[3];
+//    const Operator * A=nullptr;
+//    Vector B;
+//    DofMap * ovlp_prob = nullptr;
+//    Array<SparseMatrix *> PmlMat;
+//    Array<KLUSolver *> PmlMatInv;
+//    Array2D<double> Pmllength;
+//    Array3D<int> subdomains;
+//    mutable Array<Vector *> f_orig;
+//    int ntransf_directions;
+//    int nsweeps;
+//    Array2D<int> sweeps;
+//    Array<int> dirx;
+//    Array<int> diry;
+//    Array<int> dirz;
+//    mutable Array<Array<Vector * >> f_transf;
+//    Array<Array<Vector * >> usol;
+
+//    SparseMatrix * GetPmlSystemMatrix(int ip);
+//    void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+//    // void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+//                         //   int ip, Array<int> directions, bool local=false) const;
+//    void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+//                         int ip, Array<int> directions, int ovlpnlayers, bool local=false) const;                       
+//    void GetChiRes(const Vector & res, Vector & cfres,
+//                   int ip, Array<int> directions, int nlayers) const;                       
+//    void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+//    int  GetDirectionId(const Array<int> & ijk) const;
+//    void GetDirectionijk(int id, Array<int> & ijk) const;
+//    void ConstructDirectionsMap();
+//    int GetPatchId(const Array<int> & ijk) const;
+//    void Getijk(int ip, int & i, int & j, int & k ) const;
+//    int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+// public:
+//    DiagST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+//        double omega_, Coefficient * ws_, int nrlayers_);
+//    void SetLoadVector(Vector load) { B = load;}
+//    virtual void SetOperator(const Operator &op) {A = &op;}
+//    virtual void Mult(const Vector &r, Vector &z) const;
+//    virtual ~DiagST();
+// };
+
+#pragma once
+#include "Utilities.hpp"
+#include "PML.hpp"
+using namespace std;
+using namespace mfem;
+
+class DiagST : public Solver//
+{
+private:
+   int nrpatch;
+   int dim;
+   SesquilinearForm *bf=nullptr;
+   MeshPartition * povlp=nullptr;
+   double omega = 0.5;
+   Coefficient * ws;
+   int nrlayers;
+   int ovlpnrlayers;
+   int nxyz[3];
+   const Operator * A=nullptr;
+   Vector B;
+   DofMap * ovlp_prob = nullptr;
+   Array<SparseMatrix *> PmlMat;
+   Array<KLUSolver *> PmlMatInv;
+   Array2D<double> Pmllength;
+   Array3D<int> subdomains;
+   mutable Array<Vector *> f_orig;
+   int ntransf_directions;
+   int nsweeps;
+   Array2D<int> sweeps;
+   Array<int> dirx;
+   Array<int> diry;
+   Array<int> dirz;
+   mutable Array<Array<Vector * >> f_transf;
+   Array<Array<Vector * >> usol;
+
+   SparseMatrix * GetPmlSystemMatrix(int ip);
+   void PlotSolution(Vector & sol, socketstream & sol_sock, int ip, bool localdomain = false, bool pmldomain = false) const;
+   // void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+                        //   int ip, Array<int> directions, bool local=false) const;
+   void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+                        int ip, Array<int> directions, int ovlpnlayers, bool local=false) const;                       
+   void GetChiRes(const Vector & res, Vector & cfres,
+                  int ip, Array<int> directions, int nlayers) const;                       
+   void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+   int  GetDirectionId(const Array<int> & ijk) const;
+   void GetDirectionijk(int id, Array<int> & ijk) const;
+   void ConstructDirectionsMap();
+   int GetPatchId(const Array<int> & ijk) const;
+   void Getijk(int ip, int & i, int & j, int & k ) const;
+   int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+public:
+   DiagST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+       double omega_, Coefficient * ws_, int nrlayers_);
+   void SetLoadVector(Vector load) { B = load;}
+   virtual void SetOperator(const Operator &op) {A = &op;}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~DiagST();
+};
+
+
+

examples/maxwell-solver/DST2D/back/ex1_testL2f.cpp

@@ -0,0 +1,224 @@
+// //                                MFEM Example 1
+// //
+// // Compile with: make ex1
+// //
+
+// #include "mfem.hpp"
+// #include <fstream>
+// #include <iostream>
+
+// using namespace std;
+// using namespace mfem;
+
+// void SetElemAttr(Mesh * mesh);
+// double SolExact(const Vector & x);
+// double ChiExact(const Vector & x);
+// double BumpFncn(const Vector & x);
+
+// int main(int argc, char *argv[])
+// {
+//    // 1. Parse command-line options.
+//    const char *mesh_file = "../data/star.mesh";
+//    int order = 1;
+//    bool visualization = true;
+
+//    OptionsParser args(argc, argv);
+//    args.AddOption(&mesh_file, "-m", "--mesh",
+//                   "Mesh file to use.");
+//    args.AddOption(&order, "-o", "--order",
+//                   "Finite element order (polynomial degree) or -1 for"
+//                   " isoparametric space.");
+//    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+//                   "--no-visualization",
+//                   "Enable or disable GLVis visualization.");
+//    args.Parse();
+//    if (!args.Good())
+//    {
+//       args.PrintUsage(cout);
+//       return 1;
+//    }
+//    args.PrintOptions(cout);
+
+//    Mesh *mesh = new Mesh(mesh_file, 1, 1);
+//    int dim = mesh->Dimension();
+
+//    int ref_levels = 4;
+//    for (int l = 0; l < ref_levels; l++)
+//    {
+//       mesh->UniformRefinement();
+//    }
+
+//    // SetElemAttr(mesh);
+
+//    // Array<int> attr;
+//    // if (mesh->attributes.Size())
+//    // {
+//    //    attr.SetSize(mesh->attributes.Max());
+//    //    attr = 0;   attr[1] = 1;
+//    // }
+
+
+//    FiniteElementCollection *fec = new H1_FECollection(order, dim);
+//    FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+//    cout << "Number of finite element unknowns: "
+//         << fespace->GetTrueVSize() << endl;
+
+//    Array<int> ess_tdof_list;
+
+//    // mesh->bdr_attributes.Print();
+//    // if (mesh->bdr_attributes.Size())
+//    // {
+//    //    Array<int> ess_bdr(mesh->bdr_attributes.Max());
+//    //    ess_bdr = 0;
+//    //    // ess_bdr[3] = 1;
+//    //    // ess_bdr[1] = 1;
+//    //    fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+//    // }
+
+
+//    // LinearForm *b = new LinearForm(fespace);
+//    // ConstantCoefficient one(1.0);
+//    // RestrictedCoefficient restr(one,attr);
+
+
+//    // b->AddDomainIntegrator(new DomainLFIntegrator(restr));
+//    // b->AddDomainIntegrator(new DomainLFIntegrator(one));
+//    // b->Assemble();
+
+//    // GridFunction x(fespace);
+//    // FunctionCoefficient chi(ChiExact);
+//    // x.ProjectCoefficient(chi);
+//    // x = 0.0;
+
+//    // BilinearForm *a = new BilinearForm(fespace);
+//    // // a->AddDomainIntegrator(new DiffusionIntegrator(one));
+//    // ConstantCoefficient epsilon(0.000001);
+//    // a->AddDomainIntegrator(new DiffusionIntegrator(epsilon));
+//    // a->AddDomainIntegrator(new MassIntegrator(one));
+//    // a->Assemble();
+
+//    // OperatorPtr A;
+//    // Vector B, X;
+//    // a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+//    // cout << "Size of linear system: " << A->Height() << endl;
+
+
+//    // UMFPackSolver umf_solver;
+//    // umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+//    // umf_solver.SetOperator(*A);
+//    // umf_solver.Mult(B, X);
+
+
+//    // a->RecoverFEMSolution(X, *b, x);
+
+
+//    GridFunction bump(fespace);
+//    FunctionCoefficient c1(BumpFncn);
+//    bump.ProjectCoefficient(c1);
+
+
+//    // GridFunction uex(fespace);
+//    // FunctionCoefficient u_ex(SolExact);
+//    // uex.ProjectCoefficient(u_ex);
+
+//    // int order_quad = max(2, 2 * order + 1);
+//    // const IntegrationRule *irs[Geometry::NumGeom];
+//    // for (int i = 0; i < Geometry::NumGeom; ++i)
+//    // {
+//    //    irs[i] = &(IntRules.Get(i, order_quad));
+//    // }
+//    // double l2error = x.ComputeL2Error(u_ex, irs);
+//    // cout << "l2error = "<< l2error << endl;
+//    // 14. Send the solution by socket to a GLVis server.
+//    if (visualization)
+//    {
+//       char vishost[] = "localhost";
+//       int  visport   = 19916;
+//       socketstream sol_sock(vishost, visport);
+//       sol_sock.precision(8);
+//       // sol_sock << "solution\n" << *mesh << x << flush;
+//       sol_sock << "solution\n" << *mesh << bump << flush;
+
+//       // socketstream ex_sock(vishost, visport);
+//       // ex_sock.precision(8);
+//       // ex_sock << "solution\n" << *mesh << uex << flush;
+
+//       // GridFunction err(uex);
+//       // err-= x;
+//       // socketstream diff_sock(vishost, visport);
+//       // diff_sock.precision(8);
+//       // diff_sock << "solution\n" << *mesh << err << flush;
+
+//    }
+
+//    // 15. Free the used memory.
+//    // delete a;
+//    // delete b;
+//    delete fespace;
+//    delete fec;
+//    delete mesh;
+
+//    return 0;
+// }
+
+
+// void SetElemAttr(Mesh * mesh)
+// {
+//    int dim=mesh->Dimension();
+//    double h = 1.0/sqrt(mesh->GetNE());
+//    for (int iel=0; iel<mesh->GetNE(); iel++)
+//    {
+//       Vector center(dim);
+//       int geom = mesh->GetElementBaseGeometry(iel);
+//       ElementTransformation * tr = mesh->GetElementTransformation(iel);
+//       tr->Transform(Geometries.GetCenter(geom), center);
+//       int attr = (center[0] < 15*h) ? 1 : 2;
+//       mesh->SetAttribute(iel,attr);
+//    }
+//    mesh->SetAttributes();
+// }
+
+// double SolExact(const Vector & x)
+// {
+//    double u;
+//    if (x(0) < 0.5) 
+//    {
+//       u = x(0)/8.0;
+//    }
+//    else
+//    {
+//       u = - x(0)*x(0)/2.0 + 5.0 * x(0) / 8.0 - 1.0/8.0;
+//    }
+
+//    return u;
+// }
+
+// double ChiExact(const Vector & x)
+// {
+//    double u;
+//    if (x(0) == 0.0) 
+//    {
+//       u = 0.0;
+//    }
+//    else
+//    {
+//       u = 0.0;
+//    }
+
+//    return u;
+// }
+
+// double BumpFncn(const Vector & x)
+// {
+//    double u;
+//    if (x(0) == 0.0) 
+//    {
+//       u = 0.0;
+//    }
+//    else
+//    {
+//       u = exp(- 0.01/(1.0-pow(x(0)-1.0,2)));
+//    }
+
+//    return u;
+// }

examples/maxwell-solver/DST2D/back/helmholtz1.cpp

@@ -0,0 +1,468 @@
+// //
+// // Compile with: make helmholtz
+// //
+// // Sample runs:  helmholtz -m ../data/one-hex.mesh
+// //               helmholtz -m ../data/fichera.mesh
+// //               helmholtz -m ../data/fichera-mixed.mesh
+// //
+// // Description:  This example code demonstrates the use of MFEM to define a
+// //               simple finite element discretization of the Helmholtz problem
+// //               -Delta p - omega^2 p = 1 with impedance boundary condition.
+// //
+// #include "mfem.hpp"
+// #include <fstream>
+// #include <iostream>
+// // #include "DiagST.hpp"
+// #include "DST.hpp"
+
+// using namespace std;
+// using namespace mfem;
+
+// // Exact solution and r.h.s., see below for implementation.
+// double f_exact_Re(const Vector &x);
+// double f_exact_Im(const Vector &x);
+
+// double wavespeed(const Vector &x);
+
+
+// int dim;
+// double omega;
+// int sol = 1;
+// bool pml = false;
+// double length = 1.0;
+// double pml_length = 0.25;
+// bool scatter = false;
+// Array2D<double>comp_bdr;
+
+// #ifndef MFEM_USE_SUPERLU
+// #error This example requires that MFEM is built with MFEM_USE_PETSC=YES
+// #endif
+
+// int main(int argc, char *argv[])
+// {
+
+//    // 2. Parse command-line options.
+//    // geometry file
+//    const char *mesh_file = "../../data/one-hex.mesh";
+//    // finite element order of approximation
+//    int order = 1;
+//    // static condensation flag
+//    bool static_cond = false;
+//    bool visualization = 1;
+//    // number of wavelengths
+//    double k = 0.5;
+//    // number of mg levels
+//    int ref = 1;
+//    // dimension
+//    int nd = 2;
+
+//    // optional command line inputs
+//    OptionsParser args(argc, argv);
+//    args.AddOption(&mesh_file, "-m", "--mesh",
+//                   "Mesh file to use.");
+//    args.AddOption(&order, "-o", "--order",
+//                   "Finite element order (polynomial degree) or -1 for"
+//                   " isoparametric space.");
+//    args.AddOption(&nd, "-nd", "--dim","Problem space dimension");
+//    args.AddOption(&sol, "-sol", "--exact",
+//                   "Exact solution flag - 0:polynomial, 1: plane wave, -1: unknown exact");
+//    args.AddOption(&k, "-k", "--wavelengths",
+//                   "Number of wavelengths.");
+//    args.AddOption(&pml, "-pml", "--pml", "-no-pml",
+//                   "--no-pml", "Enable PML.");
+//    args.AddOption(&pml_length, "-pml_length", "--pml_length",
+//                   "Length of the PML region in each direction");
+//    args.AddOption(&length, "-length", "--length",
+//                   "length of the domainin in each direction.");
+//    args.AddOption(&ref, "-ref", "--ref",
+//                   "Number of Refinements.");
+//    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+//                   "--no-static-condensation", "Enable static condensation.");
+//    args.AddOption(&scatter, "-scat", "--scattering-prob", "-no-scat",
+//                   "--no-scattering", "Solve a scattering problem");
+//    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+//                   "--no-visualization",
+//                   "Enable or disable GLVis visualization.");
+//    args.Parse();
+//    // check if the inputs are correct
+//    if (!args.Good())
+//    {
+// 		args.PrintUsage(cout);
+// 		return 1;
+//    }
+// 	args.PrintOptions(cout);
+//    // Angular frequency
+//    omega = 2.0 * M_PI * k;
+
+//    // 3. Read the mesh from the given mesh file.
+//    Mesh *mesh;
+
+//    if (nd == 2)
+//    {
+//       // mesh = new Mesh(mesh_file,1,1);
+//       mesh = new Mesh(4, 4, Element::QUADRILATERAL, true, length, length, false);
+//    }
+//    else
+//    {
+//       mesh = new Mesh(1, 1, 1, Element::HEXAHEDRON, true, length, length, length,false);
+//    }
+
+//    // 3. Executing uniform h-refinement
+//    for (int i = 0; i < ref; i++ )
+//    {
+//       mesh->UniformRefinement();
+//    }
+//    dim = mesh->Dimension();
+
+//    double hl = GetUniformMeshElementSize(mesh);
+//    Vector pmin, pmax;
+//    mesh->GetBoundingBox(pmin,pmax);
+//    double domain_length = pmax[0] - pmin[0];
+//    double pml_thickness = 0.125/domain_length;
+//    // int nrlayers = pml_thickness/hl;
+//    int nrlayers = 2;
+//    Array<int> directions;
+   
+//    for (int i = 0; i<nrlayers; i++)
+//    {
+//       for (int comp=0; comp<dim; ++comp)
+//       {
+//          // directions.Append(comp+1);
+//          // directions.Append(-comp-1);
+//       }
+//    }
+//    // Find uniform h size of the original mesh
+//    cout << "pml layers = " << nrlayers << endl;
+//    cout << "pml length = " << hl*nrlayers << endl;
+//    Mesh *mesh_ext = ExtendMesh(mesh,directions);
+
+
+//    Array2D<double> lengths(dim,2);
+//    lengths = hl*nrlayers;
+//    // lengths[0][1] = 0.0;
+//    // lengths[1][1] = 0.0;
+//    // lengths[1][0] = 0.0;
+//    // lengths[0][0] = 0.0;
+//    CartesianPML pml(mesh_ext,lengths);
+//    pml.SetOmega(omega);
+//    comp_bdr.SetSize(dim,2);
+//    comp_bdr = pml.GetCompDomainBdr(); 
+
+//    // 6. Define a finite element space on the mesh.
+//    FiniteElementCollection *fec = new H1_FECollection(order, dim);
+//    FiniteElementSpace *fespace = new FiniteElementSpace(mesh_ext, fec);
+
+//    // 6. Set up the linear form (Real and Imaginary part)
+//    FunctionCoefficient f_Re(f_exact_Re);
+//    FunctionCoefficient f_Im(f_exact_Im);
+
+//    // ParLinearForm *b_Re(new ParLinearForm);
+//    ComplexLinearForm b(fespace, ComplexOperator::HERMITIAN);
+//    b.AddDomainIntegrator(new DomainLFIntegrator(f_Re),
+//                          new DomainLFIntegrator(f_Im));
+//    b.real().Vector::operator=(0.0);
+//    b.imag().Vector::operator=(0.0);
+//    b.Assemble();
+
+//    // 7. Set up the bilinear form (Real and Imaginary part)
+//    ConstantCoefficient one(1.0);
+//    ConstantCoefficient sigma(-pow(omega, 2));
+
+//    FunctionCoefficient ws(wavespeed);
+
+//    PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+//    PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+//    PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+//    PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+//    ProductCoefficient c2_re0(sigma, detJ_re);
+//    ProductCoefficient c2_im0(sigma, detJ_im);
+
+//    ProductCoefficient c2_re(c2_re0, ws);
+//    ProductCoefficient c2_im(c2_im0, ws);
+
+
+//    SesquilinearForm a(fespace,ComplexOperator::HERMITIAN);
+
+//    a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+//                          new DiffusionIntegrator(c1_im));
+//    a.AddDomainIntegrator(new MassIntegrator(c2_re),new MassIntegrator(c2_im));
+
+//    a.Assemble();
+//    a.Finalize();
+
+//    Array<int> ess_tdof_list;
+//    Array<int> ess_bdr(mesh_ext->bdr_attributes.Max());
+//    ess_bdr = 1;
+//    fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+//    // Solution grid function
+//    ComplexGridFunction p_gf(fespace);
+//    OperatorHandle Ah;
+//    Vector X, B;
+
+//    a.FormLinearSystem(ess_tdof_list, p_gf, b, Ah, X, B);
+
+//    ComplexSparseMatrix * AZ = Ah.As<ComplexSparseMatrix>();
+//    SparseMatrix * A = AZ->GetSystemMatrix();
+
+
+//    cout << "Size of fine grid system: "
+//          << A->Height() << " x " << A->Width() << endl;
+
+
+//    DST S(&a,lengths, omega, &ws, nrlayers);
+//    // DiagST S(&a,lengths, omega, &ws, nrlayers);
+// 	S.SetOperator(*A);
+//    // S.SetLoadVector(B);
+   
+
+
+
+//    X = 0.0;
+// 	GMRESSolver gmres;
+// 	gmres.SetPreconditioner(S);
+// 	gmres.SetOperator(*A);
+// 	gmres.SetRelTol(1e-8);
+// 	gmres.SetMaxIter(50);
+// 	gmres.SetPrintLevel(1);
+// 	gmres.Mult(B, X);
+
+
+
+
+//    int n= 20;
+//    X = 0.0;
+//    Vector z(X.Size()); z = 0.0;
+//    Vector r(B);
+//    Vector ztemp(r.Size());
+//    Vector Ax(X.Size());
+//    double tol = 1e-8;
+//    cout << endl;
+   
+//    for (int i = 0; i<n; i++)
+//    {
+//       A->Mult(X,Ax); Ax *=-1.0;
+//       r = b; r+=Ax;
+//       cout << "   ST Solver   Iteration :   " << i <<"  || r || = " <<  r.Norml2() << endl;
+//       if (r.Norml2() < tol) 
+//       {
+//          cout << "Convergence in " << i+1 << " iterations" << endl;
+//          break;
+//       }
+//       S.Mult(r,z); 
+//       X += z;
+
+//       // X1-=z;
+
+//          // p_gf = 0.0;
+//          // a.RecoverFEMSolution(X,B,p_gf);
+//          // char vishost[] = "localhost";
+//          // int  visport   = 19916;
+//          // string keys;
+//          // if (dim ==2 )
+//          // {
+//          //    keys = "keys mrRljc\n";
+//          // }
+//          // else
+//          // {
+//          //    keys = "keys mc\n";
+//          // }
+//          // socketstream sol1_sock_re(vishost, visport);
+//          // sol1_sock_re.precision(8);
+//          // sol1_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+//          //             "window_title 'Numerical Pressure (real part)' "
+//          //             << keys << flush;
+//          // cin.get();
+//    }
+
+//    KLUSolver klu(*A);
+//    Vector X1(X.Size());
+//    klu.Mult(B,X1);
+//    X1-= X;
+
+
+//    if (visualization)
+//    {
+//       char vishost[] = "localhost";
+//       int  visport   = 19916;
+//       string keys;
+//       if (dim ==2 )
+//       {
+//          keys = "keys mrRljc\n";
+//       }
+//       else
+//       {
+//          keys = "keys mc\n";
+//       }
+//       socketstream st_sock_re(vishost, visport);
+//       st_sock_re.precision(8);
+
+//       a.RecoverFEMSolution(X,B,p_gf);
+//       st_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+//                   "window_title 'Numerical Pressure (real part from KLU)' "
+//                   << keys << flush;
+//       a.RecoverFEMSolution(X1,B,p_gf);
+//       socketstream sol_sock_re(vishost, visport);
+//       sol_sock_re.precision(8);
+//       sol_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+//                   "window_title 'Numerical Pressure (real part from KLU)' "
+//                   << keys << flush;
+//                   // << keys << "valuerange -0.1 0.1 \n" << flush;
+//       // socketstream diff_sock_re(vishost, visport);
+//       // diff_sock_re.precision(8);
+//       // diff_sock_re << "solution\n" << *mesh_ext << p_gf1.real() <<
+//       //             "window_title 'Numerical Pressure (real part from KLU)' "
+//       //             << keys << flush;
+
+                                  
+//    }
+//    delete fespace;
+//    delete fec;
+// 	delete mesh_ext;
+// 	delete mesh;
+//    return 0;
+// }
+
+
+// //calculate RHS from exact solution f = - \Delta u
+// double f_exact_Re(const Vector &x)
+// {
+//    double f_re = 0.0;
+//    double x0 = length/2.0;
+//    double x1 = length/2.0;
+//    double x2 = length/2.0;
+//    // x0 = 0.59;
+//    // x0 = 0.19;
+//    x0 = 0.5;
+//    // x1 = 0.768;
+//    // x1 = 0.168;
+//    x1 = 0.5;
+//    double alpha,beta;
+//    // double n = 5.0*omega/M_PI;
+//    double n = 4.0*omega/M_PI;
+//    // double n = 1.0;
+//    // double coeff = pow(n,2)/M_PI;
+//    beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+//    if (dim == 3) { beta += pow(x2-x(2),2); }
+//    // alpha = -pow(n,2) * beta;
+//    // double coeff = pow(n,2)/M_PI;
+//    double coeff = 16.0*omega*omega/M_PI/M_PI/M_PI;
+//    alpha = -pow(n,2) * beta;
+//    f_re = coeff*exp(alpha);
+
+//    x0 = 0.85;
+//    x1 = 0.15;
+//    beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+//    if (dim == 3) { beta += pow(x2-x(2),2); }
+//    alpha = -pow(n,2) * beta;
+//    // f_re += coeff*exp(alpha);
+
+//    // x0 = 0.5;
+//    // x1 = 0.8;
+//    // beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+//    // if (dim == 3) { beta += pow(x2-x(2),2); }
+//    // alpha = -pow(n,2) * beta;
+//    // f_re += coeff*exp(alpha);
+
+//    bool in_pml = false;
+//    for (int i = 0; i<dim; i++)
+//    {
+//       if (x(i)<=comp_bdr(i,0) || x(i)>=comp_bdr(i,1))
+//       {
+//          in_pml = true;
+//          break;
+//       }
+//    }
+//    if (in_pml) f_re = 0.0;
+
+//    return f_re;
+
+// }
+// double f_exact_Im(const Vector &x)
+// {
+//    double f_im;
+//    f_im = 0.0;
+//    return f_im;
+// }
+
+// double wavespeed(const Vector &x)
+// {
+//    double ws;
+//    // if (x(0) <= 0.25)
+//    // {
+//    //    ws = 1.0;
+//    // }
+//    // else if(x(0)<=0.5)
+//    // {
+//    //    ws = 1.0;
+//    // }
+//    // else if(x(0)<=0.75)
+//    // {
+//    //    ws = 0.75;
+//    //    // ws = 0.5;
+//    // }
+//    // else 
+//    // {
+//    //    ws = 0.75;
+//    //    // ws = 1.0;
+//    // }
+//    // if (x(1) <= 1.0/3.0)
+//    // {
+//    //    ws = 2.0;
+//    // }
+//    // else if(x(1)<=2.0/3.0)
+//    // {
+//    //    ws = 1.0;
+//    // }
+//    // else 
+//    // {
+//    //    // ws = 0.75;
+//    //    ws = 0.25;
+//    // }
+
+//    // if (x(0) <= 0.33)
+//    // {
+//    //    ws = 1.0;
+//    // }
+//    // else if(x(0)<=0.66)
+//    // {
+//    //    ws = -0.65 + 5.0*x(0);
+//    // }
+//    // else 
+//    // {
+//    //    ws = 2.65;
+//    //    // ws = 0.5;
+//    // }
+
+
+//    ws = 1.0;
+//    return ws;
+// }
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

examples/maxwell-solver/DST2D/helmholtz.cpp

@@ -0,0 +1,500 @@
+//
+// Compile with: make helmholtz
+//
+// Sample runs:  helmholtz -m ../data/one-hex.mesh
+//               helmholtz -m ../data/fichera.mesh
+//               helmholtz -m ../data/fichera-mixed.mesh
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Helmholtz problem
+//               -Delta p - omega^2 p = 1 with impedance boundary condition.
+//
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "DST2D.hpp"
+#include "AdditiveST2D.hpp"
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution and r.h.s., see below for implementation.
+double f_exact_Re(const Vector &x);
+double f_exact_Im(const Vector &x);
+
+double wavespeed(const Vector &x);
+
+
+int dim;
+double omega;
+int sol = 1;
+bool pml = false;
+double length = 1.0;
+double pml_length = 0.25;
+bool scatter = false;
+Array2D<double>comp_bdr;
+
+#ifndef MFEM_USE_SUPERLU
+#error This example requires that MFEM is built with MFEM_USE_PETSC=YES
+#endif
+
+int main(int argc, char *argv[])
+{
+
+   // 2. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../../data/one-hex.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // static condensation flag
+   bool static_cond = false;
+   bool visualization = 1;
+   // number of wavelengths
+   double k = 0.5;
+   // number of mg levels
+   int ref = 1;
+   // dimension
+   int nd = 2;
+
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nd, "-nd", "--dim","Problem space dimension");
+   args.AddOption(&sol, "-sol", "--exact",
+                  "Exact solution flag - 0:polynomial, 1: plane wave, -1: unknown exact");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&pml, "-pml", "--pml", "-no-pml",
+                  "--no-pml", "Enable PML.");
+   args.AddOption(&pml_length, "-pml_length", "--pml_length",
+                  "Length of the PML region in each direction");
+   args.AddOption(&length, "-length", "--length",
+                  "length of the domainin in each direction.");
+   args.AddOption(&ref, "-ref", "--ref",
+                  "Number of Refinements.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&scatter, "-scat", "--scattering-prob", "-no-scat",
+                  "--no-scattering", "Solve a scattering problem");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+		args.PrintUsage(cout);
+		return 1;
+   }
+	args.PrintOptions(cout);
+   // Angular frequency
+   omega = 2.0 * M_PI * k;
+
+   // 3. Read the mesh from the given mesh file.
+   Mesh *mesh;
+
+   if (nd == 2)
+   {
+      // mesh = new Mesh(mesh_file,1,1);
+      mesh = new Mesh(4, 4, Element::QUADRILATERAL, true, length, length, false);
+   }
+   else
+   {
+      mesh = new Mesh(1, 1, 1, Element::HEXAHEDRON, true, length, length, length,false);
+   }
+
+   // 3. Executing uniform h-refinement
+   for (int i = 0; i < ref; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+   dim = mesh->Dimension();
+
+   double hl = GetUniformMeshElementSize(mesh);
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin,pmax);
+   // double domain_length = pmax[0] - pmin[0];
+   // double pml_thickness = 0.125/domain_length;
+   // int nrlayers = pml_thickness/hl;
+   int nrlayers = 4;
+   Array<int> directions;
+   
+   for (int i = 0; i<nrlayers; i++)
+   {
+      for (int comp=0; comp<dim; ++comp)
+      {
+         directions.Append(comp+1);
+         directions.Append(-comp-1);
+      }
+   }
+   // Find uniform h size of the original mesh
+   cout << "pml layers = " << nrlayers << endl;
+   cout << "pml length = " << hl*nrlayers << endl;
+   Mesh *mesh_ext = ExtendMesh(mesh,directions);
+
+
+   Array2D<double> lengths(dim,2);
+   lengths = hl*nrlayers;
+   // lengths[0][1] = 0.0;
+   // lengths[1][1] = 0.0;
+   // lengths[1][0] = 0.0;
+   // lengths[0][0] = 0.0;
+   CartesianPML pml(mesh_ext,lengths);
+   pml.SetOmega(omega);
+   comp_bdr.SetSize(dim,2);
+   comp_bdr = pml.GetCompDomainBdr(); 
+
+   // 6. Define a finite element space on the mesh.
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh_ext, fec);
+
+   // 6. Set up the linear form (Real and Imaginary part)
+   FunctionCoefficient f_Re(f_exact_Re);
+   FunctionCoefficient f_Im(f_exact_Im);
+
+   // ParLinearForm *b_Re(new ParLinearForm);
+   ComplexLinearForm b(fespace, ComplexOperator::HERMITIAN);
+   b.AddDomainIntegrator(new DomainLFIntegrator(f_Re),
+                         new DomainLFIntegrator(f_Im));
+   b.real().Vector::operator=(0.0);
+   b.imag().Vector::operator=(0.0);
+   b.Assemble();
+
+   // 7. Set up the bilinear form (Real and Imaginary part)
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+   FunctionCoefficient ws(wavespeed);
+
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+
+   ProductCoefficient c2_re(c2_re0, ws);
+   ProductCoefficient c2_im(c2_im0, ws);
+
+
+   SesquilinearForm a(fespace,ComplexOperator::HERMITIAN);
+
+   a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   a.AddDomainIntegrator(new MassIntegrator(c2_re),new MassIntegrator(c2_im));
+
+   a.Assemble();
+   a.Finalize();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(mesh_ext->bdr_attributes.Max());
+   ess_bdr = 1;
+   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   // Solution grid function
+   ComplexGridFunction p_gf(fespace);
+   OperatorHandle Ah;
+   Vector X, B;
+
+   a.FormLinearSystem(ess_tdof_list, p_gf, b, Ah, X, B);
+
+   ComplexSparseMatrix * AZ = Ah.As<ComplexSparseMatrix>();
+   SparseMatrix * A = AZ->GetSystemMatrix();
+
+
+   cout << "Size of fine grid system: "
+         << A->Height() << " x " << A->Width() << endl;
+
+
+   DST2D S1(&a,lengths, omega, &ws, nrlayers);
+   // AdditiveST S2(&a,lengths, omega, &ws, nrlayers);
+   
+
+   StopWatch chrono;
+
+   // chrono.Clear();
+   // chrono.Start();
+   X = 0.0;
+	GMRESSolver gmres;
+	// gmres.iterative_mode = true;
+   gmres.SetPreconditioner(S1);
+	gmres.SetOperator(*A);
+	gmres.SetRelTol(1e-10);
+	gmres.SetMaxIter(50);
+	gmres.SetPrintLevel(1);
+	gmres.Mult(B, X);
+
+   // X = 0.0;
+	// gmres.SetPreconditioner(S2);
+	// gmres.Mult(B, X);
+
+   // chrono.Stop();
+   // cout << "GMRES time: " << chrono.RealTime() << endl; 
+
+   X = 0.0;
+   SLISolver sli;
+   sli.iterative_mode = true;
+   sli.SetPreconditioner(S1);
+   sli.SetOperator(*A);
+   sli.SetRelTol(1e-10);
+   sli.SetMaxIter(50);
+   sli.SetPrintLevel(1);
+   sli.Mult(B,X);
+
+   // int n= 200;
+   // X = 0.0;
+   // Vector z(X.Size()); z = 0.0;
+   // Vector r(B);
+   // Vector ztemp(r.Size());
+   // Vector Ax(X.Size());
+   // double tol = 1e-10;
+   // cout << endl;
+   // chrono.Clear();
+   // chrono.Start();
+   // for (int i = 0; i<n; i++)
+   // {
+   //    A->Mult(X,Ax); Ax *=-1.0;
+   //    r = b; r+=Ax;
+   //    cout << "   ST Solver   Iteration :   " << i <<"  || r || = " <<  r.Norml2() << endl;
+   //    if (r.Norml2() < tol) 
+   //    {
+   //       cout << "Convergence in " << i << " iterations" << endl;
+   //       break;
+   //    }
+   //    S1.Mult(r,z); 
+   //    X += z;
+
+   //    // X1-=z;
+   //    // p_gf = 0.0;
+   //    // a.RecoverFEMSolution(X,B,p_gf);
+   //    //    char vishost[] = "localhost";
+   //    //    int  visport   = 19916;
+   //    //    string keys;
+   //    //    if (dim ==2 )
+   //    //    {
+   //    //       keys = "keys mrRljc\n";
+   //    //    }
+   //    //    else
+   //    //    {
+   //    //       keys = "keys mc\n";
+   //    //    }
+   //    //    socketstream sol1_sock_re(vishost, visport);
+   //    //    sol1_sock_re.precision(8);
+   //    //    sol1_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+   //    //                "window_title 'Numerical Pressure (real part)' "
+   //    //                << keys << flush;
+   //    //    cin.get();
+   // }
+
+   // chrono.Stop();
+   // cout << "Solver time: " << chrono.RealTime() << endl; 
+
+   a.RecoverFEMSolution(X,B,p_gf);
+
+   KLUSolver klu(*A);
+   Vector X1(X.Size());
+   klu.Mult(B,X1);
+   X1-= X;
+
+   ComplexGridFunction error_gf(fespace);
+
+   a.RecoverFEMSolution(X1,B,error_gf);
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      string keys;
+      if (dim ==2 )
+      {
+         keys = "keys mrRljc\n";
+      }
+      else
+      {
+         keys = "keys mc\n";
+      }
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re.precision(8);
+      sol_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+                  "window_title 'Numerical Pressure (real part from DST)' "
+                  // << keys << flush;
+                  << keys << "valuerange -0.08 0.08 \n" << flush;
+      socketstream err_sock_re(vishost, visport);
+      err_sock_re.precision(8);
+      err_sock_re << "solution\n" << *mesh_ext << error_gf.real() <<
+                  "window_title 'Numerical Pressure (real part from KLU)' "
+                  << keys << flush;
+   }
+   delete fespace;
+   delete fec;
+	delete mesh_ext;
+	delete mesh;
+   return 0;
+}
+
+
+//calculate RHS from exact solution f = - \Delta u
+double f_exact_Re(const Vector &x)
+{
+   double f_re = 0.0;
+   double x0 = length/2.0;
+   double x1 = length/2.0;
+   double x2 = length/2.0;
+   // x0 = 0.59;
+   // x0 = 0.19;
+   x0 = 0.5;
+   // x1 = 0.768;
+   // x1 = 0.168;
+   x1 = 0.5;
+   double alpha,beta;
+   // double n = 5.0*omega/M_PI;
+   double n = 4.0*omega/M_PI;
+   // double n = 1.0;
+   // double coeff = pow(n,2)/M_PI;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   // alpha = -pow(n,2) * beta;
+   // double coeff = pow(n,2)/M_PI;
+   double coeff = 16.0*omega*omega/M_PI/M_PI/M_PI;
+   alpha = -pow(n,2) * beta;
+   f_re = coeff*exp(alpha);
+
+   x0 = 0.85;
+   x1 = 0.15;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   alpha = -pow(n,2) * beta;
+   // f_re += coeff*exp(alpha);
+
+   x0 = 0.8;
+   x1 = 0.4;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   alpha = -pow(n,2) * beta;
+   // f_re += coeff*exp(alpha);
+
+   bool in_pml = false;
+   for (int i = 0; i<dim; i++)
+   {
+      if (x(i)<=comp_bdr(i,0) || x(i)>=comp_bdr(i,1))
+      {
+         in_pml = true;
+         break;
+      }
+   }
+   if (in_pml) f_re = 0.0;
+
+   return f_re;
+
+}
+double f_exact_Im(const Vector &x)
+{
+   double f_im;
+   f_im = 0.0;
+   return f_im;
+}
+
+double wavespeed(const Vector &x)
+{
+   double ws;
+   // if (x(0) <= 0.25)
+   // {
+   //    ws = 1.0;
+   // }
+   // else if(x(0)<=0.5)
+   // {
+   //    ws = 1.0;
+   // }
+   // else if(x(0)<=0.75)
+   // {
+   //    ws = 0.75;
+   //    // ws = 0.5;
+   // }
+   // else 
+   // {
+   //    ws = 0.75;
+   //    // ws = 1.0;
+   // }
+   // if (x(1) <= 1.0/3.0)
+   // {
+   //    ws = 2.0;
+   // }
+   // else if(x(1)<=2.0/3.0)
+   // {
+   //    ws = 1.0;
+   // }
+   // else 
+   // {
+   //    // ws = 0.75;
+   //    ws = 0.25;
+   // }
+
+   // if (x(0) <= 0.33)
+   // {
+   //    ws = 1.0;
+   // }
+   // else if(x(0)<=0.66)
+   // {
+   //    ws = -0.65 + 5.0*x(0);
+   // }
+   // else 
+   // {
+   //    ws = 2.65;
+   //    // ws = 0.5;
+   // }
+
+   // if (x(0) <= x(1) && x(1) >= 1.0-x(0))
+   // {
+   //    ws = 1.0;
+   // }
+   // else if (x(0) > x(1) && x(1) >= 1.0-x(0))
+   // {
+   //    ws = 3.0;
+   // }
+   // else if (x(0) <= x(1) && x(1) < 1.0-x(0))
+   // {
+   //    ws = 2.0;
+   // }
+   // else
+   // {
+   //    ws = 4.0;
+   // }
+   
+
+
+   ws = 1.0;
+   return ws;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

examples/maxwell-solver/DST2D/makefile

@@ -0,0 +1,61 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/maxwell-solver/DST2D,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = helmholtz
+PAR_EXAMPLES = 
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean
+.PRECIOUS: %.o
+
+COMMON_O= ../common/PML.o ../common/MeshPartition.o ../common/Utilities.o DST2D.o AdditiveST2D.o
+
+# Remove built-in rules
+%: %.cpp
+%.o: %.cpp
+
+all: $(EXAMPLES)
+
+# Rules for building the EXAMPLES
+
+%: $(SRC)%.cpp $(COMMON_O) $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(COMMON_O) $(MFEM_LIBS)
+
+# Rules for compiling miniapp dependencies
+$(COMMON_O) $($(EXAMPLES)): \
+%.o: $(SRC)%.cpp $(SRC)%.hpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $(<) -o $(@)
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+	rm output/*
+
+

examples/maxwell-solver/FOSLS.cpp

@@ -0,0 +1,475 @@
+#include "FOSLS.hpp"
+
+
+ComplexMaxwellFOSLS::ComplexMaxwellFOSLS(ParFiniteElementSpace * fes_) : fes(fes_)
+{ };
+
+void ComplexMaxwellFOSLS::SetLoadData(Array<VectorFunctionCoefficient *> & loads_)
+{
+   loads = loads_;
+}
+void ComplexMaxwellFOSLS::SetEssentialData(Array<VectorFunctionCoefficient *> &  ess_data_)
+{
+   ess_data = ess_data_;
+}
+
+void ComplexMaxwellFOSLS::GetFOSLSLinearSystem(Array2D<HypreParMatrix *> & A_, 
+                                               BlockVector & X_,
+                                               BlockVector & Rhs_)
+{
+   if (A.NumCols() == 0)
+   {
+      FormSystem(true);
+   }
+   A_ = A;
+   X_ = X;
+   Rhs_ = Rhs;
+}
+
+void ComplexMaxwellFOSLS::GetFOSLSMatrix(Array2D<HypreParMatrix *> & A_)
+{
+   if (A.NumCols() == 0)
+   {
+      FormSystem(false);
+   }
+   A_ = A;
+}
+
+void ComplexMaxwellFOSLS::FormSystem(bool system)
+{
+   // HYPRE_Int size = fes->GlobalTrueVSize();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   pmesh = fes->GetParMesh();
+   if (pmesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   VectorFunctionCoefficient * E_ex_re = ess_data[0];
+   // VectorFunctionCoefficient * H_ex_re = ess_data[1];
+   VectorFunctionCoefficient * E_ex_im = ess_data[2];
+   // VectorFunctionCoefficient * H_ex_im = ess_data[3];
+
+   // VectorFunctionCoefficient * f_ex_re = loads[0];
+   VectorFunctionCoefficient * g_ex_re = loads[1];
+   // VectorFunctionCoefficient * f_ex_im = loads[2];
+   VectorFunctionCoefficient * g_ex_im = loads[3];
+
+   int n = fes->GetVSize();
+   int N = fes->GetTrueVSize();
+   block_offsets.SetSize(5);
+   block_offsets[0] = 0;
+   block_offsets[1] = n;
+   block_offsets[2] = n;
+   block_offsets[3] = n;
+   block_offsets[4] = n;
+   block_offsets.PartialSum();
+
+   block_trueOffsets.SetSize(5);
+   block_trueOffsets[0] = 0;
+   block_trueOffsets[1] = N;
+   block_trueOffsets[2] = N;
+   block_trueOffsets[3] = N;
+   block_trueOffsets[4] = N;
+   block_trueOffsets.PartialSum();
+
+   ParGridFunction E_gf_re, E_gf_im, H_gf_re, H_gf_im;
+
+   if(system)
+   {
+      x.Update(block_offsets);
+      rhs.Update(block_offsets);
+      X.Update(block_trueOffsets);
+      Rhs.Update(block_trueOffsets);
+      x = 0.0;  rhs = 0.0; X = 0.0;  Rhs = 0.0;
+
+      E_gf_re.MakeRef(fes,x.GetBlock(0)); E_gf_re = 0.0;
+      H_gf_re.MakeRef(fes,x.GetBlock(1)); H_gf_re = 0.0;
+      E_gf_im.MakeRef(fes,x.GetBlock(2)); E_gf_im = 0.0;
+      H_gf_im.MakeRef(fes,x.GetBlock(3)); H_gf_im = 0.0;
+
+      E_gf_re.ProjectCoefficient(*E_ex_re);
+      E_gf_im.ProjectCoefficient(*E_ex_im);
+   }
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient negomeg(-omega);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega * omega);
+   ScalarVectorProductCoefficient wJi(omeg,*g_ex_im);
+   ScalarVectorProductCoefficient negJr(negone,*g_ex_re);
+   ScalarVectorProductCoefficient negwJr(negomeg,*g_ex_re);
+   ScalarVectorProductCoefficient negJi(negone,*g_ex_im);
+
+   ParLinearForm b0, b1, b2, b3;
+
+   if(system)
+   {
+      b0.Update(fes,rhs.GetBlock(0),0);
+      b1.Update(fes,rhs.GetBlock(1),0);
+      b2.Update(fes,rhs.GetBlock(2),0);
+      b3.Update(fes,rhs.GetBlock(3),0);
+      b0.AddDomainIntegrator(new VectorFEDomainLFIntegrator(wJi));
+      b1.AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(negJr));
+      b2.AddDomainIntegrator(new VectorFEDomainLFIntegrator(negwJr));
+      b3.AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(negJi));
+      b0.Assemble();
+      b1.Assemble();
+      b2.Assemble();
+      b3.Assemble();
+   }
+   A.SetSize(4,4); 
+   for (int i = 0; i<4; i++)
+   {
+      for (int j = 0; j<4; j++)
+      {
+         A[i][j] = nullptr;
+      }
+   }
+
+   ParBilinearForm a00(fes);
+   a00.AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a00.Assemble();
+   if (system)
+   {
+      a00.EliminateEssentialBC(ess_bdr,x.GetBlock(0),rhs.GetBlock(0),mfem::Operator::DIAG_ONE);
+   }
+   else
+   {
+      a00.EliminateEssentialBC(ess_bdr);
+   }
+   a00.Finalize();
+   A[0][0] = a00.ParallelAssemble();
+
+   ParMixedBilinearForm a03(fes,fes);
+   a03.AddDomainIntegrator(new MixedVectorCurlIntegrator(negomeg));
+   a03.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(negomeg));
+   a03.Assemble();
+   a03.EliminateTestDofs(ess_bdr);
+   a03.Finalize();
+   A[0][3] = a03.ParallelAssemble();
+
+   ParBilinearForm a11(fes);
+   a11.AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a11.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a11.Assemble();
+   a11.Finalize();
+   A[1][1] = a11.ParallelAssemble();
+
+   ParMixedBilinearForm a21(fes,fes);
+   a21.AddDomainIntegrator(new MixedVectorCurlIntegrator(omeg));
+   a21.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(omeg));
+   a21.Assemble();
+   a21.EliminateTestDofs(ess_bdr);
+   a21.Finalize();
+   A[2][1] = a21.ParallelAssemble();
+
+   if (system)
+   {
+      ParMixedBilinearForm a12(fes,fes);
+      a12.AddDomainIntegrator(new MixedVectorCurlIntegrator(omeg));
+      a12.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(omeg));
+      a12.Assemble();
+      a12.EliminateTrialDofs(ess_bdr,x.GetBlock(2),rhs.GetBlock(1));
+      a12.Finalize();
+      A[1][2] = a12.ParallelAssemble();
+   }
+   else
+   {
+      A[1][2] = A[2][1]->Transpose();
+   }
+
+
+   ParBilinearForm a22(fes);
+   a22.AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a22.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a22.Assemble();
+   if (system)
+   {
+      a22.EliminateEssentialBC(ess_bdr,x.GetBlock(2),rhs.GetBlock(2),mfem::Operator::DIAG_ONE);
+   }
+   else
+   {
+      a22.EliminateEssentialBC(ess_bdr);
+   }
+   a22.Finalize();
+   A[2][2] = a22.ParallelAssemble();
+
+   if (system)
+   {
+      ParMixedBilinearForm a30(fes,fes);
+      a30.AddDomainIntegrator(new MixedVectorCurlIntegrator(negomeg));
+      a30.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(negomeg));
+      a30.Assemble();
+      a30.EliminateTrialDofs(ess_bdr,x.GetBlock(0),rhs.GetBlock(3));
+      a30.Finalize();
+      A[3][0] = a30.ParallelAssemble();
+   }
+   else
+   {
+      A[3][0] = A[0][3]->Transpose();
+   }
+
+   ParBilinearForm a33(fes);
+   a33.AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a33.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a33.Assemble();
+   a33.Finalize();
+   A[3][3] = a33.ParallelAssemble();
+
+   if (system)
+   {
+      for (int i = 0; i<4; i++)
+      {
+         fes->GetRestrictionMatrix()->Mult(x.GetBlock(i), X.GetBlock(i));
+         fes->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(i),Rhs.GetBlock(i));
+      }
+   }
+};
+
+HelmholtzFOSLS::HelmholtzFOSLS(Array<ParFiniteElementSpace * > & fes_, 
+bool definite_, bool complex_) : fes(fes_), definite(definite_), complex(complex_)
+{ 
+   n = complex ? 2 : 1;
+   Init();
+};
+
+void HelmholtzFOSLS::Init()
+{
+   f.SetSize(n);
+   Q.SetSize(n);
+   p_ex_coeff.SetSize(n);
+   u_ex_coeff.SetSize(n);
+   for (int i = 0; i<n; i++)
+   {
+      f[i] = nullptr;
+      Q[i] = nullptr;
+      p_ex_coeff[i] = nullptr;
+      u_ex_coeff[i] = nullptr;
+   }
+}
+
+void HelmholtzFOSLS::SetLoadData(Array<FunctionCoefficient * > & f_)
+{
+   f = f_;
+}
+void HelmholtzFOSLS::SetLoadData(Array<VectorFunctionCoefficient * > & Q_)
+{
+   Q = Q_;
+}
+
+void HelmholtzFOSLS::SetEssentialData(Array<FunctionCoefficient * > & p_ex_coeff_)
+{
+   p_ex_coeff = p_ex_coeff_;
+}
+void HelmholtzFOSLS::SetEssentialData(Array<VectorFunctionCoefficient * > & u_ex_coeff_)
+{
+   u_ex_coeff = u_ex_coeff_;
+}
+
+void HelmholtzFOSLS::GetFOSLSLinearSystem(Array2D<HypreParMatrix *> & A_, 
+                                          BlockVector & X_,
+                                          BlockVector & Rhs_)
+{
+   if (A.NumCols() == 0)
+   {
+      FormSystem(true);
+   }
+   A_ = A;
+   X_ = X;
+   Rhs_ = Rhs;
+}
+
+void HelmholtzFOSLS::GetFOSLSMatrix(Array2D<HypreParMatrix *> & A_)
+{
+   if (A.NumCols() == 0)
+   {
+      FormSystem(false);
+   }
+   A_ = A;
+}
+
+void HelmholtzFOSLS::FormSystem(bool system)
+{
+   // HYPRE_Int size = fes[0]->GlobalTrueVSize() + fes[1]->GlobalTrueVSize();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   pmesh = fes[0]->GetParMesh();
+   if (pmesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+   int blksize = complex ? 5 : 3;
+   block_offsets.SetSize(blksize);
+   block_offsets[0] = 0;
+   block_offsets[1] = fes[0]->GetVSize();
+   block_offsets[2] = fes[1]->GetVSize();
+   if (complex)
+   {
+      block_offsets[3] = fes[0]->GetVSize();
+      block_offsets[4] = fes[1]->GetVSize();
+   }
+   block_offsets.PartialSum();
+
+   block_trueOffsets.SetSize(blksize);
+   block_trueOffsets[0] = 0;
+   block_trueOffsets[1] = fes[0]->GetTrueVSize();
+   block_trueOffsets[2] = fes[1]->GetTrueVSize();
+   if (complex)
+   {
+      block_trueOffsets[3] = fes[0]->GetTrueVSize();
+      block_trueOffsets[4] = fes[1]->GetTrueVSize();
+   }
+   block_trueOffsets.PartialSum();
+
+   ParGridFunction p_gf_re, u_gf_re, p_gf_im, u_gf_im;
+
+   if(system)
+   {
+      x.Update(block_offsets);
+      rhs.Update(block_offsets);
+      X.Update(block_trueOffsets);
+      Rhs.Update(block_trueOffsets);
+      x = 0.0;  rhs = 0.0; X = 0.0;  Rhs = 0.0;
+
+      p_gf_re.MakeRef(fes[0],x.GetBlock(0)); p_gf_re = 0.0;
+      u_gf_re.MakeRef(fes[1],x.GetBlock(1)); u_gf_re = 0.0;
+
+      if (complex)
+      {
+         p_gf_im.MakeRef(fes[0],x.GetBlock(2)); p_gf_im = 0.0;
+         u_gf_im.MakeRef(fes[1],x.GetBlock(3)); u_gf_im = 0.0;
+      }
+
+      if (p_ex_coeff[0])
+      {
+         p_gf_re.ProjectCoefficient(*p_ex_coeff[0]);
+      }
+      if (u_ex_coeff[0])
+      {
+         u_gf_re.ProjectCoefficient(*u_ex_coeff[0]);
+      }
+      if (complex)
+      {
+         if (p_ex_coeff[1])
+         {
+            p_gf_im.ProjectCoefficient(*p_ex_coeff[1]);
+         }
+         if (u_ex_coeff[1])
+         {
+            u_gf_im.ProjectCoefficient(*u_ex_coeff[1]);
+         }
+      }
+   }
+
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negomeg(-omega);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega * omega);
+   ProductCoefficient omega_f(omeg,*f[0]);
+   ProductCoefficient neg_f(negone,*f[0]);
+
+   ParLinearForm b0, b1;
+
+   if(system)
+   {
+      b0.Update(fes[0],rhs.GetBlock(0),0);
+      b1.Update(fes[1],rhs.GetBlock(1),0);
+      b0.AddDomainIntegrator(new DomainLFIntegrator(omega_f));
+      if (definite)
+      {
+         b1.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(neg_f));
+      }
+      else
+      {
+         b1.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(*f[0]));
+      }
+      b0.Assemble();
+      b1.Assemble();
+   }
+   A.SetSize(2,2); 
+   for (int i = 0; i<2; i++)
+   {
+      for (int j = 0; j<2; j++)
+      {
+         A[i][j] = nullptr;
+      }
+   }
+
+   ParBilinearForm a00(fes[0]);
+   a00.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a00.AddDomainIntegrator(new MassIntegrator(omeg2));
+   a00.Assemble();
+   if (system)
+   {
+      a00.EliminateEssentialBC(ess_bdr,x.GetBlock(0),rhs.GetBlock(0),mfem::Operator::DIAG_ONE);
+   }
+   else
+   {
+      a00.EliminateEssentialBC(ess_bdr,mfem::Operator::DIAG_ONE);
+   }
+   a00.Finalize();
+   A[0][0] = a00.ParallelAssemble();
+
+   ParMixedBilinearForm a01(fes[1],fes[0]);
+   if (definite)
+   {
+      a01.AddDomainIntegrator(new MixedScalarDivergenceIntegrator(negomeg));
+   }
+   else
+   {
+      a01.AddDomainIntegrator(new MixedScalarDivergenceIntegrator(omeg));
+   }
+   a01.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg));
+   a01.Assemble();
+   a01.EliminateTestDofs(ess_bdr);
+   a01.Finalize();
+   A[0][1] = a01.ParallelAssemble();
+
+   if (system)
+   {
+      ParMixedBilinearForm a10(fes[0],fes[1]);
+      if (definite)
+      {
+         a10.AddDomainIntegrator(new MixedScalarWeakGradientIntegrator(omeg));
+      }
+      else
+      {
+         a10.AddDomainIntegrator(new MixedScalarWeakGradientIntegrator(negomeg));
+      }
+      a10.AddDomainIntegrator(new MixedVectorGradientIntegrator(negomeg));
+      a10.Assemble();
+      a10.EliminateTrialDofs(ess_bdr,x.GetBlock(0),rhs.GetBlock(1));
+      a10.Finalize();
+      A[1][0] = a10.ParallelAssemble();
+   }
+   else
+   {
+      A[1][0] = A[0][1]->Transpose();
+   }
+
+   ParBilinearForm a11(fes[1]);
+   a11.AddDomainIntegrator(new DivDivIntegrator(one));
+   a11.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a11.Assemble();
+   a11.Finalize();
+   A[1][1] = a11.ParallelAssemble();
+
+
+   if (system)
+   {
+      fes[0]->GetRestrictionMatrix()->Mult(x.GetBlock(0), X.GetBlock(0));
+      fes[1]->GetRestrictionMatrix()->Mult(x.GetBlock(1), X.GetBlock(1));
+      fes[0]->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(0),Rhs.GetBlock(0));
+      fes[1]->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(1),Rhs.GetBlock(1));
+   }
+}

examples/maxwell-solver/FOSLS.hpp

@@ -0,0 +1,77 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+using namespace std;
+using namespace mfem;
+
+
+class ComplexMaxwellFOSLS 
+{
+private:
+   ParFiniteElementSpace * fes = nullptr;
+   ParMesh * pmesh = nullptr;
+   double omega = 1.0;
+   Array<VectorFunctionCoefficient *> loads;
+   Array<VectorFunctionCoefficient *> ess_data;
+   Array2D<HypreParMatrix *> A;
+   BlockVector x,rhs;
+   BlockVector X,Rhs;
+   Array<int> block_offsets;
+   Array<int> block_trueOffsets;
+
+   void FormSystem(bool system = true);
+public:
+   ComplexMaxwellFOSLS(ParFiniteElementSpace * fes_);
+   void SetOmega(double omega_) { omega = omega_; }
+   void SetLoadData(Array<VectorFunctionCoefficient *> & loads_);
+   void SetEssentialData(Array<VectorFunctionCoefficient *> & ess_data_);
+   void GetFOSLSLinearSystem(Array2D<HypreParMatrix *> & A_, 
+                             BlockVector & X_,
+                             BlockVector & Rhs_);
+   void GetFOSLSMatrix(Array2D<HypreParMatrix *> & A_);
+};
+
+
+   // -------------------------------------------------------------------
+   // |   |            p             |           u           |   RHS    | 
+   // -------------------------------------------------------------------
+   // | q | (gradp,gradq) + w^2(p,q) | w(divu,q)-w(u, gradq) |  w(f,q)  |
+   // |   |                          |                       |          |
+   // | v | w(p,divv) - w(gradp,v)   | (divu,divv) + w^2(u,v)| (f,divv) |
+
+class HelmholtzFOSLS 
+{
+private:
+   Array<ParFiniteElementSpace * > fes;
+   bool definite;
+   bool complex;
+   ParMesh * pmesh = nullptr;
+   double omega = 1.0;
+   int n; // 
+   Array<FunctionCoefficient * > f;
+   Array<VectorFunctionCoefficient * > Q;
+   Array<FunctionCoefficient * > p_ex_coeff;
+   Array<VectorFunctionCoefficient * > u_ex_coeff;
+   Array2D<HypreParMatrix *> A;
+   BlockVector x,rhs;
+   BlockVector X,Rhs;
+   Array<int> block_offsets;
+   Array<int> block_trueOffsets;
+
+   void FormSystem(bool system = true);
+   void Init();
+public:
+   HelmholtzFOSLS(Array<ParFiniteElementSpace * > & fes_, 
+                  bool definite_ = false,
+                  bool complex_ = false);
+   void SetOmega(double omega_) { omega = omega_; }
+   void SetLoadData(Array<FunctionCoefficient * > & f_);
+   void SetLoadData(Array<VectorFunctionCoefficient * > & Q_);
+   void SetEssentialData(Array<FunctionCoefficient * > & p_ex_coeff_);
+   void SetEssentialData(Array<VectorFunctionCoefficient * > & u_ex_coeff_);
+   void GetFOSLSLinearSystem(Array2D<HypreParMatrix *> & A_, 
+                             BlockVector & X_,
+                             BlockVector & Rhs_);
+   void GetFOSLSMatrix(Array2D<HypreParMatrix *> & A_);
+};

examples/maxwell-solver/FOSLS2D_maxwell.cpp

@@ -0,0 +1,310 @@
+// Example run: ./FOSLS2D_maxwell -ref 4  -o 3 -sol 1 -k 3.0
+
+//  ∇ × E - ω H = 0
+// -ω E + ∇ × H = J
+
+// --------------------------------------------------------------------------
+// |   |            E             |             H              |     RHS    | 
+// --------------------------------------------------------------------------
+// | F | (∇ × E,∇ × F)+ ω^2 (E,F) | - ω (∇ × H,F) - ω (H,curF) | - ω (J,F)  |
+// |   |                          |                            |            |
+// | G |-ω (E,∇ × G)-ω (∇ × E,G)  | (∇ × H,∇ × G)+ ω^2(H,G)    | (J,∇ × G)  |
+
+// for E in H1 (scalar) we have ∇ × E = [0 1;-1 0] ∇ E
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// Define exact solution
+double E_exact(const Vector &x);
+void H_exact(const Vector &x, Vector &H);
+double frhs(const Vector &x);
+void fvrhs(const Vector &x, Vector &f);
+void get_maxwell_solution(const Vector &x, double & E, Vector & curlE, double & curl2E);
+
+int dim;
+double omega;
+int isol = 0;
+
+int main(int argc, char *argv[])
+{
+   StopWatch chrono;
+
+   // 1. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../data/star.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // visualization flag
+   bool visualization = 1;
+   int ref = 1;
+   // number of wavelengths
+   double k = 0.6;
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&ref, "-ref", "--init-refinements",
+                  "Number of initial mesh refinements");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&isol, "-sol", "--solution",
+                  "Exact Solution: 0) Polynomial, 1) Sinusoidal.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   omega = 2.0 * M_PI * k;
+
+   // Mesh mesh(1, 1, Element::QUADRILATERAL, true, 1.0, 1.0, false);
+   Mesh mesh(mesh_file, 1, 1);
+
+   dim = mesh.Dimension();
+   if (dim == 3) {MFEM_ABORT("This is 2D Maxwell")};
+
+   for (int i = 0; i < ref; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   H1_FECollection H1fec(order,dim);
+   FiniteElementSpace H1fes(&mesh, &H1fec);
+
+   ND_FECollection NDfec(order, dim);
+   FiniteElementSpace NDfes(&mesh, &NDfec);
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      // Essential BC on E. Nothing on H
+      H1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+   Array<int> block_offsets(3);
+   block_offsets[0] = 0;
+   block_offsets[1] = H1fes.GetVSize();
+   block_offsets[2] = NDfes.GetVSize();
+   block_offsets.PartialSum();
+
+   BlockVector x(block_offsets), b(block_offsets);
+   x = 0.0;
+   b = 0.0;
+
+   FunctionCoefficient Eex(E_exact);
+   VectorFunctionCoefficient Hex(dim, H_exact);
+   GridFunction E_gf;
+   GridFunction H_gf;
+   E_gf.MakeRef(&H1fes, x.GetBlock(0));
+   E_gf.ProjectBdrCoefficient(Eex,ess_bdr);
+   H_gf.MakeRef(&NDfes, x.GetBlock(1));
+
+   FunctionCoefficient f(frhs);
+   ProductCoefficient f_E(-omega, f);
+   VectorFunctionCoefficient f_H(1,fvrhs);
+   LinearForm b_E;
+   b_E.Update(&H1fes, b.GetBlock(0), 0);
+   b_E.AddDomainIntegrator(new DomainLFIntegrator(f_E));
+   b_E.Assemble();
+
+   LinearForm b_H;
+   b_H.Update(&NDfes, b.GetBlock(1), 0);
+   b_H.AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(f_H));
+   b_H.Assemble();
+
+   // 7. Bilinear form a(.,.) on the finite element space
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient omeg2(pow(omega, 2));
+   ConstantCoefficient negomega(-(omega));
+   DenseMatrix mat(2);
+   mat(0,0) = 0.; mat(0,1) = 1.;
+   mat(1,0) = -1.; mat(1,1) = 0.;
+   MatrixConstantCoefficient rot(mat);
+
+   BilinearForm a_EE(&H1fes);
+   a_EE.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a_EE.AddDomainIntegrator(new MassIntegrator(omeg2));
+   a_EE.Assemble();
+   a_EE.EliminateEssentialBC(ess_bdr, x.GetBlock(0), b.GetBlock(0));
+   a_EE.Finalize();
+   SparseMatrix &A_EE = a_EE.SpMat();
+
+   ScalarMatrixProductCoefficient c1(-omega, rot);
+   MixedBilinearForm a_EH(&H1fes,&NDfes);
+   // - omega (rot grad E, G) - (omega E, curl G)
+   a_EH.AddDomainIntegrator(new MixedVectorGradientIntegrator(c1));
+   a_EH.AddDomainIntegrator(new MixedScalarWeakCurlIntegrator(negomega));
+   a_EH.Assemble();
+   a_EH.EliminateTrialDofs(ess_bdr, x.GetBlock(0), b.GetBlock(1));
+   a_EH.Finalize();
+   SparseMatrix &A_EH = a_EH.SpMat();
+   SparseMatrix * A_HE = Transpose(A_EH);
+
+   BilinearForm a_HH(&NDfes);
+   a_HH.AddDomainIntegrator(new CurlCurlIntegrator(one)); // one is the coeff
+   a_HH.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2)); // one is the coeff
+   a_HH.Assemble();
+   a_HH.Finalize();
+   SparseMatrix &A_HH = a_HH.SpMat();
+
+   BlockMatrix LS_Maxwellop(block_offsets);
+   LS_Maxwellop.SetBlock(0, 0, &A_EE);
+   LS_Maxwellop.SetBlock(0, 1, A_HE);
+   LS_Maxwellop.SetBlock(1, 0, &A_EH);
+   LS_Maxwellop.SetBlock(1, 1, &A_HH);
+
+   UMFPackSolver invE;
+   invE.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   invE.SetOperator(LS_Maxwellop.GetBlock(0,0));
+
+   UMFPackSolver invH;
+   invH.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   invH.SetOperator(LS_Maxwellop.GetBlock(1,1));
+
+   BlockDiagonalPreconditioner prec(block_offsets);
+   prec.SetDiagonalBlock(0, &invE);
+   prec.SetDiagonalBlock(1, &invH);
+
+   int maxit(5000);
+   double rtol(1.e-16);
+   double atol(0.0);
+
+   CGSolver pcg;
+   pcg.SetAbsTol(atol);
+   pcg.SetRelTol(rtol);
+   pcg.SetMaxIter(maxit);
+   pcg.SetOperator(LS_Maxwellop);
+   pcg.SetPreconditioner(prec);
+   pcg.SetPrintLevel(3);
+   pcg.Mult(b, x);
+
+   int order_quad = max(2, 2 * order + 1);
+   const IntegrationRule *irs[Geometry::NumGeom];
+   for (int i = 0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+
+   double Error_E = E_gf.ComputeL2Error(Eex, irs);
+   double Error_H = H_gf.ComputeL2Error(Hex, irs);
+   
+   cout << "|| E_h - E || = " << Error_E << "\n";
+   cout << "|| H_h - H || = " << Error_H << "\n";
+   cout << "Total error = " << sqrt(Error_H*Error_H+Error_E*Error_E) << "\n";
+
+   GridFunction E_exgf(&H1fes);
+   E_exgf.ProjectCoefficient(Eex);
+
+   GridFunction H_exgf(&NDfes);
+   H_exgf.ProjectCoefficient(Hex);
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int visport = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      socketstream ex_sock(vishost, visport);
+      ex_sock.precision(8);
+      socketstream sol_sockH(vishost, visport);
+      sol_sockH.precision(8);
+      socketstream ex_sockH(vishost, visport);
+      ex_sockH.precision(8);
+      sol_sock << "solution\n"
+               << mesh << E_gf << "window_title 'Numerical E'" << "keys rRljc\n"
+               << flush;
+      ex_sock << "solution\n"
+               << mesh << E_exgf << "window_title 'Exact E'" << "keys rRljc\n"
+               << flush;
+      sol_sockH << "solution\n"
+               << mesh << H_gf << "window_title 'Numerical H'" << "keys rRljc\n"
+               << flush;
+      ex_sockH << "solution\n"
+               << mesh << H_exgf << "window_title 'Exact H'" << "keys rRljc\n"
+               << flush;        
+   }
+   delete A_HE;
+   return 0;
+}
+
+double E_exact(const Vector &x)
+{
+   double E, curl2E;
+   Vector curlE(2);
+   get_maxwell_solution(x, E, curlE, curl2E);
+   return E;  //Scalar
+}
+
+//define exact solution
+void H_exact(const Vector &x, Vector &H)
+{
+   double E, curl2E;
+   Vector curlE(2);
+   get_maxwell_solution(x, E, curlE, curl2E);
+   H[0] = curlE[0]/omega;
+   H[1] = curlE[1]/omega;
+}
+
+double frhs(const Vector &x)
+{
+   double E, curl2E;
+   Vector curlE(2);
+   get_maxwell_solution(x, E, curlE, curl2E);
+
+   // - omega E + curl H = f
+   // - omega E + curl (curl E) / omega = f
+   double f = - omega * E + curl2E / omega;
+   return f;
+}
+
+void fvrhs(const Vector &x, Vector &f)
+{
+   double E, curl2E;
+   Vector curlE(2);
+   get_maxwell_solution(x, E, curlE, curl2E);
+   f[0] = - omega * E + curl2E / omega;
+}
+
+
+void get_maxwell_solution(const Vector &X, double & E, Vector & curlE, double & curl2E)
+{
+   double x = X[0];
+   double y = X[1];
+   double Ex, Ey, Exx, Eyy;
+   if (isol == 0) // polynomial
+   {
+      E = x * (1.0 - x) * y * (1.0 - y);
+      Ex = (1.0 - 2.0 * x) * y * (1.0 - y);
+      Ey = x * (1.0 - x) * (1.0 - 2.0 * y);
+
+      Exx = -2.0 * y * (1.0 - y);
+      Eyy = -2.0 * x * (1.0 - x);
+   }
+   else 
+   {
+      double s = omega * (y+x);
+      E = cos(s);
+      Ex = -omega * sin(s);  
+      Ey = Ex;
+      Exx = - omega * omega * E;  
+      Eyy = Exx;  
+   }
+   curlE[0] = Ey;
+   curlE[1] = -Ex;
+   curl2E = -Exx - Eyy;
+}

examples/maxwell-solver/LS-LOR-results.txt

@@ -0,0 +1,42 @@
+./LS-helmholtzp_lor  -o 4  -m ../../data/inline-quad.mesh
+
+
+
+
+omega/2pi |   dof_H1  | dof_RT  |    H1_err    |    Hdiv_err  |   Exact LOR   |  Inexact LOR  |   AMG/AMS ho  |
+--------------------------------------------------------------------------------------------------------------   
+5         |   1089    | 2112    | 7.5044e-02   |  6.8128e-02  | 47 (0.551404) | 180 (0.16153) | 155 (0.914495)
+10        |   4225    | 8320    | 7.2461e-02   |  6.6677e-02  | 64 (2.23832)  | 197 (0.931944)| 164 (0.918901)
+20        |   16641   | 33024   | 7.1488e-02   |  6.6131e-02  | 79 (9.42402)  | 292 (4.14552) | 246 (11.3769)
+40        |   66049   | 131584  | 6.9699e-02   |  6.5160e-02  | 87 (39.8902)  | 380 (25.3352) | 251 (48.977)
+80        |  263169   | 525312  | 6.7949e-02   |  6.4224e-02  | 98 (179.461)  | 401 (114.649) | 316 (271.633)
+160       |  1050625  | 2099200 | 6.7005e-02   |  6.3724e-02  | 96 (1791.43)  | 377 (569.546) | 299 (1277.63) 
+
+
+omega/2pi |   dof_H1  |   H1_err |   Exact LOR  |
+--------------------------------------------------------------------------------------------------------------   
+5         |    1089   | 4.2337e-02 |  16 (0.176056)
+10        |    4225   | 4.2329e-02 |  30 (1.02578)
+20        |   16641   | 4.2328e-02 |  209 (16.7772)
+40        |   66049   | 4.2327e-02 | >2000 (600.249)
+80        |  263169   |     *      | >2000 (2818.08) 
+160       | 1050625   |     *      |   
+
+
+srun -n 4 ./LS_maxwellp -rnum 2.0 -o 3 -sr 2 -pr 0 -no-vis -m ../data/inline-hex.mesh -solution 1
+omega/2pi |   dof (x4) |  L2 err rel | ranks |  AMG/AMS ho  |
+--------------------------------------------------------------------------------------------------------------   
+2         |   45000    |  0.0129377  |    4  |  143 (31.2249)|
+4         |   345744   |  0.0345918  |   32  |  262 (108.072)| 
+8         |  2709792   |  0.0547445  |  256  |  354 (244.849)| 
+16        |
+
+// Complex indefinite Helmholtz
+./LS_complex_helmholtzp -o 5 -solution 1 -sr 0 -rnum 4.0
+omega/2pi |   dof_H1  | dof_RT  |    H1_err    |    Hdiv_err  |   Exact LOR   |  Inexact LOR  |   AMG/AMS ho  |
+--------------------------------------------------------------------------------------------------------------   
+4         |   1681    | 3280    | 1.0107e-03   |  8.8146e-04  |  71 (2.2144)  | 426 (1.35841) | 329 (3.65064) 
+8         |   6561    | 12960   | 1.4215e-03   |  1.2339e-03  | 128 (13.258)  | 625 (8.7082)  | 481 (24.1927)
+16        |   25921   | 51520   | 2.3111e-02   |  9.9057e-03  | 239 (88.5967) |1045 (57.6934) | 724 (149.518)
+32        |   103041  | 205440  | 1.4054e-02   |  1.4147e-02  | 286 (408.915) |1153 (279.299) | 860 (748.056)
+64        |   410881  | 820480  | 7.9836e-01   |  2.8242e-02  | 574 (3819.05) |2798 (2860.15) | 1974 (6881.4)

examples/maxwell-solver/LS-helmholtzp.cpp

@@ -0,0 +1,378 @@
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+void gradp_exact(const Vector &x, Vector &gradu);
+double divu_exact(const Vector &x);
+double d2_exact(const Vector &x);
+
+int dim;
+double omega;
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = 1;
+   int sr = 1;
+   int pr = 1;
+   double rnum=1.0;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&sr, "-sr", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pr", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");                  
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+
+
+   omega = 2.0 * M_PI * rnum;
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution.
+   for (int i = 0; i < sr; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   // 6. Define a parallel finite element space on the parallel mesh.
+   FiniteElementCollection *H1fec = new H1_FECollection(order,dim); 
+   ParFiniteElementSpace *H1fespace = new ParFiniteElementSpace(pmesh, H1fec);
+
+   FiniteElementCollection *RTfec = new RT_FECollection(order,dim); 
+   ParFiniteElementSpace *RTfespace = new ParFiniteElementSpace(pmesh, RTfec);
+
+
+   // -------------------------------------------------------------------
+   // |   |            p             |           u           |   RHS    | 
+   // -------------------------------------------------------------------
+   // | q | (gradp,gradq) + w^2(p,q) | w(divu,q)-w(u, gradq) |  w(f,q)  |
+   // |   |                          |                       |          |
+   // | v | w(p,divv) - w(gradp,v)   | (divu,divv) + w^2(u,v)| (f,divv) |
+
+
+   // omega(f,q) 
+   ParLinearForm b_q(H1fespace);
+   ConstantCoefficient omeg(omega);
+   FunctionCoefficient f_rhs(rhs_func);
+   ProductCoefficient omega_f(omeg,f_rhs);
+   b_q.AddDomainIntegrator(new DomainLFIntegrator(omega_f));
+   // (f, div v)
+   ParLinearForm b_v(RTfespace);
+#ifdef DEFINITE
+   ConstantCoefficient negone(-1.0);
+   ProductCoefficient neg_f(negone,f_rhs);
+   b_v.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(neg_f));
+#else    
+   b_v.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(f_rhs));
+#endif
+
+   ParBilinearForm a_qp(H1fespace);
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negomeg(-omega);
+   ConstantCoefficient omeg2(omega*omega);
+   // (grad p, grad q) + \omega^2 (p,q)
+   a_qp.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a_qp.AddDomainIntegrator(new MassIntegrator(omeg2));
+
+   ParMixedBilinearForm a_qu(RTfespace, H1fespace);
+#ifdef DEFINITE
+   // -w(divu,q)
+   a_qu.AddDomainIntegrator(new MixedScalarDivergenceIntegrator(negomeg));
+#else   
+   // w(divu,q)
+   a_qu.AddDomainIntegrator(new MixedScalarDivergenceIntegrator(omeg));
+#endif
+   // -w(u, gradq)
+   a_qu.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg));
+   // w(p,divv) - w(gradp,v)
+   ParMixedBilinearForm a_vp(H1fespace, RTfespace);
+#ifdef DEFINITE
+   // -w(p,divv)
+   a_vp.AddDomainIntegrator(new MixedScalarWeakGradientIntegrator(omeg));
+#else
+   // w(p,divv)
+   a_vp.AddDomainIntegrator(new MixedScalarWeakGradientIntegrator(negomeg));
+#endif
+   // - w(gradp,v)
+   a_vp.AddDomainIntegrator(new MixedVectorGradientIntegrator(negomeg));
+
+   ParBilinearForm a_vu(RTfespace);
+   a_vu.AddDomainIntegrator(new DivDivIntegrator(one));
+   a_vu.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+
+
+   ConvergenceStudy ratesH1;
+   ConvergenceStudy ratesRT;
+   FunctionCoefficient p_ex(p_exact);
+   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
+   VectorFunctionCoefficient u_ex(dim,u_exact);
+   FunctionCoefficient divu_ex(divu_exact);
+   ParGridFunction p_gf, u_gf;
+
+   for (int l = 0; l <= pr; l++)
+   {
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh->bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+         ess_bdr = 1;
+         H1fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      Array<int> block_offsets(3);
+      block_offsets[0] = 0;
+      block_offsets[1] = H1fespace->GetVSize();
+      block_offsets[2] = RTfespace->GetVSize();
+      block_offsets.PartialSum();
+
+      Array<int> block_trueOffsets(3);
+      block_trueOffsets[0] = 0;
+      block_trueOffsets[1] = H1fespace->TrueVSize();
+      block_trueOffsets[2] = RTfespace->TrueVSize();
+      block_trueOffsets.PartialSum();
+
+      BlockVector x(block_offsets), rhs(block_offsets);
+      BlockVector trueX(block_trueOffsets), trueRhs(block_trueOffsets);
+      x = 0.0;  rhs = 0.0;
+      trueX = 0.0;  trueRhs = 0.0;
+
+      p_gf.MakeRef(H1fespace,x.GetBlock(0));
+      p_gf.ProjectBdrCoefficient(p_ex,ess_bdr);
+
+      u_gf.MakeRef(RTfespace,x.GetBlock(1));
+      u_gf = 0.0;
+
+      b_q.Update(H1fespace,rhs.GetBlock(0),0);
+      b_q.Assemble();
+
+      b_v.Update(RTfespace,rhs.GetBlock(1),0);
+      b_v.Assemble();
+
+
+      a_qp.Assemble();
+      a_qp.EliminateEssentialBC(ess_bdr,x.GetBlock(0),rhs.GetBlock(0));
+      a_qp.Finalize();
+      HypreParMatrix * A_qp = a_qp.ParallelAssemble();
+
+      a_qu.Assemble();
+      a_qu.EliminateTestDofs(ess_bdr);
+      a_qu.Finalize();
+      HypreParMatrix * A_qu = a_qu.ParallelAssemble();
+
+
+      a_vp.Assemble();
+      a_vp.EliminateTrialDofs(ess_bdr,x.GetBlock(0),rhs.GetBlock(1));
+      a_vp.Finalize();
+      HypreParMatrix * A_vp = a_vp.ParallelAssemble();
+
+      a_vu.Assemble();
+      a_vu.Finalize();
+      HypreParMatrix * A_vu = a_vu.ParallelAssemble();
+
+
+
+      H1fespace->GetRestrictionMatrix()->Mult(x.GetBlock(0), trueX.GetBlock(0));
+      H1fespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(0),trueRhs.GetBlock(0));
+
+      RTfespace->GetRestrictionMatrix()->Mult(x.GetBlock(1), trueX.GetBlock(1));
+      RTfespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(1),trueRhs.GetBlock(1));
+
+
+      Array2D<HypreParMatrix *> Ah(2,2);
+      Ah[0][0] = A_qp; 
+      Ah[0][1] = A_qu;
+      Ah[1][0] = A_vp;
+      Ah[1][1] = A_vu;
+      HypreParMatrix * A = HypreParMatrixFromBlocks(Ah);
+
+      HypreBoomerAMG amg_p(*A_qp);
+      amg_p.SetPrintLevel(0);
+
+      Solver *prec = nullptr;
+      if (dim == 2) 
+      {
+         prec = new HypreAMS(*A_vu,RTfespace);
+         dynamic_cast<HypreAMS *>(prec)->SetPrintLevel(0);
+      }
+      else
+      {
+         prec = new HypreADS(*A_vu,RTfespace);
+         dynamic_cast<HypreADS *>(prec)->SetPrintLevel(0);
+      }
+
+      BlockDiagonalPreconditioner M(block_trueOffsets);
+      // BlockDiagonalMultiplicativePreconditioner M(block_trueOffsets);
+      // M.SetOperator(*A);
+      M.SetDiagonalBlock(0,&amg_p);
+      ScaledOperator S(prec,1.0);
+      M.SetDiagonalBlock(1,&S);
+
+      StopWatch chrono;
+      chrono.Clear();
+      chrono.Start();
+      // GMRESSolver cg(MPI_COMM_WORLD);
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-6);
+      // cg.SetAbsTol(1e-6);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(1);
+      cg.SetPreconditioner(M);
+      cg.SetOperator(*A);
+      cg.Mult(trueRhs, trueX);
+      delete prec;
+      chrono.Stop();
+      cout << "PCG time " << chrono.RealTime() << endl;
+
+      chrono.Clear();
+      chrono.Start();
+      MUMPSSolver mumps;
+      mumps.SetPrintLevel(0);
+      mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
+      mumps.SetOperator(*A);
+      Vector trueY(trueX.Size());
+      mumps.Mult(trueRhs,trueY);
+      chrono.Stop();
+      cout << "MUMPS time " << chrono.RealTime() << endl;
+
+
+
+
+      delete A;
+      delete A_vu;
+      delete A_qp;
+      delete A_vp;
+      delete A_qu;
+
+      p_gf = 0.0;
+      u_gf = 0.0;
+      p_gf.Distribute(&(trueX.GetBlock(0)));
+      u_gf.Distribute(&(trueX.GetBlock(1)));
+
+      ratesH1.AddH1GridFunction(&p_gf,&p_ex,&gradp_ex);
+      ratesRT.AddHdivGridFunction(&u_gf,&u_ex,&divu_ex);
+
+      if (l==pr) break;
+
+      pmesh->UniformRefinement();
+      H1fespace->Update();
+      RTfespace->Update();
+      a_qp.Update();
+      a_qu.Update();
+      a_vp.Update();
+      a_vu.Update();
+      b_q.Update();
+      b_v.Update();
+      p_gf.Update();
+      u_gf.Update();
+   }
+   ratesH1.Print(true);
+   ratesRT.Print(true);
+
+   // 10. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << p_gf <<
+               "window_title 'Numerical Pressure (real part)' "
+               << flush;
+   }
+
+   // // 11. Free the used memory.
+   delete H1fespace;
+   delete RTfespace;
+   delete H1fec;
+   delete RTfec;
+   delete pmesh;
+   MPI_Finalize();
+   return 0;
+}
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return divu + omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   return sin(omega*x.Sum());
+}
+
+void gradp_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   grad = omega * cos(omega * x.Sum());
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   gradp_exact(x,u);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   return d2_exact(x)/omega;
+}
+
+double d2_exact(const Vector &x)
+{
+   return -dim * omega * omega * sin(omega*x.Sum());
+}

examples/maxwell-solver/LS-helmholtzp_lor.cpp

@@ -0,0 +1,310 @@
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "FOSLS.hpp"
+#include "lor.hpp"
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+
+
+
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+void gradp_exact(const Vector &x, Vector &gradu);
+double divu_exact(const Vector &x);
+double d2_exact(const Vector &x);
+
+#ifdef DEFINITE   
+   bool definite = true;
+#else
+   bool definite = false;
+#endif
+int dim;
+double omega;
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = 1;
+   int sr = 1;
+   int pr = 1;
+   double rnum=1.0;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&sr, "-sr", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pr", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");                  
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   omega = 2.0 * M_PI * rnum;
+
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution.
+   for (int i = 0; i < sr; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   int btype = BasisType::GaussLobatto;
+   ParMesh pmesh_lor(pmesh, order, btype);
+
+   unique_ptr<FiniteElementCollection> H1fec_ho, H1fec_lor;
+   unique_ptr<FiniteElementCollection> RTfec_ho, RTfec_lor;
+
+   H1fec_ho.reset(new H1_FECollection(order, dim));
+   H1fec_lor.reset(new H1_FECollection(1, dim));
+   RTfec_ho.reset(new RT_FECollection(order-1, dim, BasisType::GaussLobatto, BasisType::Integrated));
+   RTfec_lor.reset(new RT_FECollection(0, dim, BasisType::GaussLobatto, BasisType::Integrated));
+     
+   ParFiniteElementSpace H1fes_ho(pmesh, H1fec_ho.get());
+   ParFiniteElementSpace H1fes_lor(&pmesh_lor, H1fec_lor.get());
+   ParFiniteElementSpace RTfes_ho(pmesh, RTfec_ho.get());
+   ParFiniteElementSpace RTfes_lor(&pmesh_lor, RTfec_lor.get());
+
+   Array<int> block_trueOffsets(3);
+   block_trueOffsets[0] = 0;
+   block_trueOffsets[1] = H1fes_ho.TrueVSize();
+   block_trueOffsets[2] = RTfes_ho.TrueVSize();
+   block_trueOffsets.PartialSum();
+
+   BlockVector trueX(block_trueOffsets), trueRhs(block_trueOffsets);
+   trueX = 0.0;  trueRhs = 0.0;
+
+   FunctionCoefficient p_ex(p_exact);
+   FunctionCoefficient f_rhs(rhs_func);
+   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
+   VectorFunctionCoefficient u_ex(dim,u_exact);
+   FunctionCoefficient divu_ex(divu_exact);
+
+   Vector trueY(trueX);
+   Vector trueZ(trueX);
+
+   Array<ParFiniteElementSpace *> fes_ho(2);
+   fes_ho[0] = &H1fes_ho;
+   fes_ho[1] = &RTfes_ho;
+
+   HelmholtzFOSLS ho_system(fes_ho,definite);
+
+   ho_system.SetOmega(omega);
+   Array<FunctionCoefficient * > F_rhs(1);
+   F_rhs[0] = &f_rhs;
+   ho_system.SetLoadData(F_rhs);
+
+   Array<FunctionCoefficient * > P_ex(1);
+   P_ex[0] = &p_ex;
+   ho_system.SetEssentialData(P_ex);
+
+   Array<ParFiniteElementSpace *> fes_lor(2);
+   fes_lor[0] = &H1fes_lor;
+   fes_lor[1] = &RTfes_lor;
+   HelmholtzFOSLS lor_system(fes_lor,definite);
+   lor_system.SetOmega(omega);
+
+   Array2D<HypreParMatrix *> Ah_ho(2,2);
+   ho_system.GetFOSLSLinearSystem(Ah_ho,trueX,trueRhs);
+
+   Array2D<HypreParMatrix *> Ah_lor(2,2);
+   lor_system.GetFOSLSMatrix(Ah_lor);
+   HypreParMatrix * A_ho = HypreParMatrixFromBlocks(Ah_ho);
+   HypreParMatrix * A_lor = HypreParMatrixFromBlocks(Ah_lor);
+
+   HypreBoomerAMG * amg_p = new HypreBoomerAMG(*Ah_ho[0][0]);
+   amg_p->SetPrintLevel(0);
+   HypreBoomerAMG * amg_lor_p = new HypreBoomerAMG(*Ah_lor[0][0]);
+   amg_lor_p->SetPrintLevel(0);
+
+   Solver *prec = nullptr;
+   Solver *prec_lor = nullptr;
+   if (dim == 2) 
+   {
+      prec = new HypreAMS(*Ah_ho[1][1],&RTfes_ho);
+      dynamic_cast<HypreAMS *>(prec)->SetPrintLevel(0);
+      prec_lor = new HypreAMS(*Ah_lor[1][1],&RTfes_lor);
+      dynamic_cast<HypreAMS *>(prec_lor)->SetPrintLevel(0);
+   }
+   else
+   {
+      prec = new HypreADS(*Ah_ho[1][1],&RTfes_ho);
+      dynamic_cast<HypreADS *>(prec)->SetPrintLevel(0);
+      prec_lor = new HypreADS(*Ah_lor[1][1],&RTfes_lor);
+      dynamic_cast<HypreADS *>(prec_lor)->SetPrintLevel(0);
+   }
+
+   BlockDiagonalPreconditioner M(block_trueOffsets);
+   BlockDiagonalPreconditioner M_lor2(block_trueOffsets);
+
+   FiniteElement::MapType t = FiniteElement::H_DIV;
+   Array<int> perm = ComputeVectorFE_LORPermutation(RTfes_ho, RTfes_lor, t);
+   
+   RealLORSolver M_lor(*A_lor, perm);
+   M.SetDiagonalBlock(0,amg_p);
+   ScaledOperator S(prec,1.0);
+   M.SetDiagonalBlock(1,&S);
+
+   M_lor2.SetDiagonalBlock(0,amg_lor_p);
+   ScaledOperator S_lor(prec_lor,1.0);
+   M_lor2.SetDiagonalBlock(1,&S_lor);
+
+   RealLORSolver M_lor_inexact(*A_lor, perm, false, &M_lor2);
+
+   StopWatch chrono;
+   chrono.Clear();
+   chrono.Start();
+   // GMRESSolver cg(MPI_COMM_WORLD);
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-6);
+   // cg.SetAbsTol(1e-6);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetOperator(*A_ho);
+   // cg.SetPreconditioner(M);
+   cg.SetPreconditioner(M_lor);
+   cg.Mult(trueRhs, trueX);
+   chrono.Stop();
+   cout << "LOR exact - PCG time " << chrono.RealTime() << endl;
+
+   chrono.Clear();
+   chrono.Start();
+   cg.SetPreconditioner(M_lor_inexact);
+   cg.Mult(trueRhs, trueY);
+
+   chrono.Stop();
+   cout << "LOR inexact PCG time " << chrono.RealTime() << endl;
+
+   chrono.Clear();
+   chrono.Start();
+   cg.SetPreconditioner(M);
+   cg.Mult(trueRhs, trueZ);
+   
+
+   chrono.Stop();
+   cout << "AMG/AMS PCG time " << chrono.RealTime() << endl;
+
+   for (int i = 0; i<2; i++)
+   {
+      for (int j = 0; j<2; j++)
+      {
+         delete Ah_ho[i][j];
+         delete Ah_lor[i][j];
+      }
+   }
+
+   ParGridFunction p_gf(&H1fes_ho);
+   ParGridFunction u_gf(&RTfes_ho);
+   ParGridFunction p_zero(&H1fes_ho);
+   ParGridFunction u_zero(&RTfes_ho);
+   p_gf = 0.0; p_zero = 0.0;
+   u_gf = 0.0; u_zero = 0.0;
+   p_gf.Distribute(&(trueX.GetBlock(0)));
+   u_gf.Distribute(&(trueX.GetBlock(1)));
+
+   double H1_error = p_gf.ComputeH1Error(&p_ex,&gradp_ex);
+   double H1_norm  = p_zero.ComputeH1Error(&p_ex,&gradp_ex);
+   double Hdiv_error = u_gf.ComputeHDivError(&u_ex,&divu_ex);
+   double Hdiv_norm = u_zero.ComputeHDivError(&u_ex,&divu_ex);
+
+
+   if (myid == 0)
+   {
+      cout << "H1 rel error     = " << H1_error/H1_norm << endl;
+      cout << "H(div) rel error = " << Hdiv_error/Hdiv_norm << endl;
+   }
+
+   // 10. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << p_gf <<
+               "window_title 'Numerical Pressure (real part)' "
+               << flush;
+   }
+
+   // // 11. Free the used memory.
+   delete amg_lor_p;
+   delete amg_p;
+   delete prec;
+   delete prec_lor;
+   delete pmesh;
+   MPI_Finalize();
+   return 0;
+}
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return divu + omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   return sin(omega*x.Sum());
+}
+
+void gradp_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   grad = omega * cos(omega * x.Sum());
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   gradp_exact(x,u);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   return d2_exact(x)/omega;
+}
+
+double d2_exact(const Vector &x)
+{
+   return -dim * omega * omega * sin(omega*x.Sum());
+}

examples/maxwell-solver/LS_complex_helmholtzp.cpp

@@ -0,0 +1,734 @@
+
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "FOSLS.hpp"
+#include "lor.hpp"
+
+using namespace std;
+using namespace mfem;
+
+int dim;
+double omega;
+int exact = 0;
+
+void helmholtz_solution(const Vector &x, complex<double> & sol, 
+                        std::vector<complex<double>> & grad,
+                        complex<double> & grad2);
+
+double p_exact_re(const Vector &x);
+void u_exact_re(const Vector &x, Vector &u);
+double p_exact_im(const Vector &x);
+void u_exact_im(const Vector &x, Vector &u);
+void gradp_exact_re(const Vector &x, Vector &gradu);
+double divu_exact_re(const Vector &x);
+void gradp_exact_im(const Vector &x, Vector &gradu);
+double divu_exact_im(const Vector &x);
+
+
+void f_exact_re(const Vector &x, Vector &f);
+double g_exact_re(const Vector &x);
+void f_exact_im(const Vector &x, Vector &f);
+double g_exact_im(const Vector &x);
+void plotfield(socketstream &,ParMesh * pmesh,const ParGridFunction & , string &);
+
+// ----------------------------------------------------------------------
+// |   |            p             |             u           |    RHS    | 
+// ----------------------------------------------------------------------
+// | q | (grad p,grad q)+w^2(p,q) |-iw(div u,q)+iw(u,grad q)| -iw(f,q)  |
+// |   |                          |                         |           |
+// | v | iw(p,div v)-iw(grad p,v) | (div u,div v)+w^2(u,v)  | (f,div v) |
+// ----------------------------------------------------------------------
+int main(int argc, char *argv[])
+{
+    // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = 1;
+   int sr = 1;
+   int pr = 1;
+   double rnum=1.0;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&sr, "-sr", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pr", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");                  
+   args.AddOption(&exact, "-solution", "--exact_solution",
+                  "Exact solution : 0-polynomial, 1-plane wave");                  
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   omega = 2.0 * M_PI * rnum;
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+
+   dim = mesh->Dimension();
+
+   for (int i = 0; i < sr; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   for (int i = 0; i < pr; i++ )
+   {
+      pmesh->UniformRefinement();
+   }
+
+   int btype = BasisType::GaussLobatto;
+   ParMesh pmesh_lor(pmesh, order, btype);
+
+   unique_ptr<FiniteElementCollection> H1fec_ho, H1fec_lor;
+   unique_ptr<FiniteElementCollection> RTfec_ho, RTfec_lor;
+
+   H1fec_ho.reset(new H1_FECollection(order, dim));
+   H1fec_lor.reset(new H1_FECollection(1, dim));
+   RTfec_ho.reset(new RT_FECollection(order-1, dim, BasisType::GaussLobatto, BasisType::Integrated));
+   RTfec_lor.reset(new RT_FECollection(0, dim, BasisType::GaussLobatto, BasisType::Integrated));
+   
+   ParFiniteElementSpace H1fes_ho(pmesh, H1fec_ho.get());
+   ParFiniteElementSpace H1fes_lor(&pmesh_lor, H1fec_lor.get());
+   ParFiniteElementSpace RTfes_ho(pmesh, RTfec_ho.get());
+   ParFiniteElementSpace RTfes_lor(&pmesh_lor, RTfec_lor.get());
+
+   HYPRE_Int H1size = H1fes_ho.GlobalTrueVSize();
+   HYPRE_Int RTsize = RTfes_ho.GlobalTrueVSize();
+
+   if (myid == 0)
+   {
+      cout << "Number of H1 True Dofs = " << H1size << endl;
+      cout << "Number of RT True Dofs = " << RTsize << endl;
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (pmesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      H1fes_ho.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   FunctionCoefficient p_ex_re(p_exact_re);
+   VectorFunctionCoefficient u_ex_re(dim,u_exact_re);
+   FunctionCoefficient p_ex_im(p_exact_im);
+   VectorFunctionCoefficient u_ex_im(dim,u_exact_im);
+
+   VectorFunctionCoefficient f_ex_re(dim,f_exact_re);
+   FunctionCoefficient g_ex_re(g_exact_re);
+   VectorFunctionCoefficient f_ex_im(dim,f_exact_im);
+   FunctionCoefficient g_ex_im(g_exact_im);
+
+   int n0 = H1fes_ho.GetVSize();
+   int N0 = H1fes_ho.GetTrueVSize();
+   int n1 = RTfes_ho.GetVSize();
+   int N1 = RTfes_ho.GetTrueVSize();
+   Array<int> block_offsets(5);
+   block_offsets[0] = 0;
+   block_offsets[1] = n0;
+   block_offsets[2] = n1;
+   block_offsets[3] = n0;
+   block_offsets[4] = n1;
+   block_offsets.PartialSum();
+
+   Array<int> block_trueOffsets(5);
+   block_trueOffsets[0] = 0;
+   block_trueOffsets[1] = N0;
+   block_trueOffsets[2] = N1;
+   block_trueOffsets[3] = N0;
+   block_trueOffsets[4] = N1;
+   block_trueOffsets.PartialSum();
+
+   BlockVector x(block_offsets), rhs(block_offsets);
+   BlockVector X(block_trueOffsets), Rhs(block_trueOffsets);
+   x = 0.0;  rhs = 0.0; X = 0.0;  Rhs = 0.0;
+
+   ParGridFunction p_gf_re, p_gf_im, u_gf_re, u_gf_im;
+
+   p_gf_re.MakeRef(&H1fes_ho,x.GetBlock(0)); p_gf_re = 0.0;
+   u_gf_re.MakeRef(&RTfes_ho,x.GetBlock(1)); u_gf_re = 0.0;
+   p_gf_im.MakeRef(&H1fes_ho,x.GetBlock(2)); p_gf_im = 0.0;
+   u_gf_im.MakeRef(&RTfes_ho,x.GetBlock(3)); u_gf_im = 0.0;
+
+   // E_gf_re.ProjectBdrCoefficientTangent(E_ex_re,ess_bdr);
+   // E_gf_im.ProjectBdrCoefficientTangent(E_ex_im,ess_bdr);
+   p_gf_re.ProjectCoefficient(p_ex_re);
+   p_gf_im.ProjectCoefficient(p_ex_im);
+
+// ----------------------------------------------------------------------
+// |   |            p             |             u           |    RHS    | 
+// ----------------------------------------------------------------------
+// | q | (grad p,grad q)+w^2(p,q) |-iw(div u,q)+iw(u,grad q)| -iw(g,q)  |
+// |   |                          |                         |           |
+// | v | iw(p,div v)-iw(grad p,v) | (div u,div v)+w^2(u,v)  | (g,div v) |
+// ----------------------------------------------------------------------
+
+   // for convinience we convert the above 2 x 2 blocks to 4 x 4 in order
+   // to accomodate complex valued operators
+
+   // A = (grad p,grad q)+w^2(p,q)
+   // B = (div u,div v)+w^2(u,v)
+   // C = -w(div u,q) + w(u,grad q)
+   // D = w(p,div v)-w(grad p,v)
+   // b0 = w(g_im,q)
+   // b1 = (g_re,div v)
+   // b2 = -w(g_re,q)
+   // b3 = (g_im,div v)
+
+   // | A   0   0  -C |  | p_re |     | b0 |
+   // | 0   B  -D   0 |  | u_re |  =  | b1 |
+   // | 0   C   A   0 |  | p_im |     | b2 |
+   // | D   0   0   B |  | u_im |     | b3 |
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient negomeg(-omega);
+   ConstantCoefficient omeg2(omega * omega);
+
+   ProductCoefficient wgi(omeg,g_ex_im);
+   ProductCoefficient negwgr(negomeg,g_ex_re);
+
+   ParLinearForm b0, b1, b2, b3;
+   b0.Update(&H1fes_ho,rhs.GetBlock(0),0);
+   b1.Update(&RTfes_ho,rhs.GetBlock(1),0);
+   b2.Update(&H1fes_ho,rhs.GetBlock(2),0);
+   b3.Update(&RTfes_ho,rhs.GetBlock(3),0);
+
+
+   b0.AddDomainIntegrator(new DomainLFIntegrator(wgi));
+   b1.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(g_ex_re));
+   b2.AddDomainIntegrator(new DomainLFIntegrator(negwgr));
+   b3.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(g_ex_im));
+   
+   b0.Assemble();
+   b1.Assemble();
+   b2.Assemble();
+   b3.Assemble();
+
+   Array2D<HypreParMatrix *> Ah(4,4); 
+   for (int i = 0; i<4; i++)
+   {
+      for (int j = 0; j<4; j++)
+      {
+         Ah[i][j] = nullptr;
+      }
+   }
+
+   // A00 = (grad p,grad q)+w^2(p,q)
+   ParBilinearForm a00(&H1fes_ho);
+   a00.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a00.AddDomainIntegrator(new MassIntegrator(omeg2));
+   a00.Assemble();
+   a00.EliminateEssentialBC(ess_bdr,x.GetBlock(0),rhs.GetBlock(0),mfem::Operator::DIAG_ONE);
+   a00.Finalize();
+   Ah[0][0] = a00.ParallelAssemble();
+
+   // -C = w(div u,q) - w(u,grad q)
+   ParMixedBilinearForm a03(&RTfes_ho,&H1fes_ho);
+   // w(divu,q)
+   a03.AddDomainIntegrator(new MixedScalarDivergenceIntegrator(omeg));
+   // -w(u, gradq)
+   a03.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg));
+   a03.Assemble();
+   a03.EliminateTestDofs(ess_bdr);
+   a03.Finalize();
+   Ah[0][3] = a03.ParallelAssemble();
+
+   // A11 = (div u,div v)+w^2(u,v)
+   ParBilinearForm a11(&RTfes_ho);
+   a11.AddDomainIntegrator(new DivDivIntegrator(one));
+   a11.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a11.Assemble();
+   a11.Finalize();
+   Ah[1][1] = a11.ParallelAssemble();
+
+   // A12 = -w(p,div v)+w(grad p,v)
+   ParMixedBilinearForm a12(&H1fes_ho,&RTfes_ho);
+   // -w(p,divv)
+   a12.AddDomainIntegrator(new MixedScalarWeakGradientIntegrator(omeg));
+   // w(grad p,v)
+   a12.AddDomainIntegrator(new MixedVectorGradientIntegrator(omeg));
+   a12.Assemble();
+   a12.EliminateTrialDofs(ess_bdr,x.GetBlock(2),rhs.GetBlock(1));
+   a12.Finalize();
+   Ah[1][2] = a12.ParallelAssemble();
+
+   // A21 = -w(div u,q) + w(u,grad q)
+   ParMixedBilinearForm a21(&RTfes_ho,&H1fes_ho);
+   // -w(div u,q) 
+   a21.AddDomainIntegrator(new MixedScalarDivergenceIntegrator(negomeg));
+   // w(u,grad q)
+   a21.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(negomeg));
+   a21.Assemble();
+   a21.EliminateTestDofs(ess_bdr);
+   a21.Finalize();
+   Ah[2][1] = a21.ParallelAssemble();
+
+   // A22 = (grad p,grad q)+w^2(p,q)
+   ParBilinearForm a22(&H1fes_ho);
+   a22.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a22.AddDomainIntegrator(new MassIntegrator(omeg2));
+   a22.Assemble();
+   a22.EliminateEssentialBC(ess_bdr,x.GetBlock(2),rhs.GetBlock(2),mfem::Operator::DIAG_ONE);
+   a22.Finalize();
+   Ah[2][2] = a22.ParallelAssemble();
+
+
+   // A30 = w(p,div v)-w(grad p,v)
+   ParMixedBilinearForm a30(&H1fes_ho,&RTfes_ho);
+   // w(p,div v)
+   a30.AddDomainIntegrator(new MixedScalarWeakGradientIntegrator(negomeg));
+   // -w(grad p,v)
+   a30.AddDomainIntegrator(new MixedVectorGradientIntegrator(negomeg));
+   a30.Assemble();
+   a30.EliminateTrialDofs(ess_bdr,x.GetBlock(0),rhs.GetBlock(3));
+   a30.Finalize();
+   Ah[3][0] = a30.ParallelAssemble();
+
+   ParBilinearForm a33(&RTfes_ho);
+   a33.AddDomainIntegrator(new DivDivIntegrator(one));
+   a33.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a33.Assemble();
+   a33.Finalize();
+   Ah[3][3] = a33.ParallelAssemble();
+
+   for (int i = 0; i<2; i++)
+   {
+      H1fes_ho.GetRestrictionMatrix()->Mult(x.GetBlock(2*i), X.GetBlock(2*i));
+      H1fes_ho.GetProlongationMatrix()->MultTranspose(rhs.GetBlock(2*i),Rhs.GetBlock(2*i));
+      RTfes_ho.GetRestrictionMatrix()->Mult(x.GetBlock(2*i+1), X.GetBlock(2*i+1));
+      RTfes_ho.GetProlongationMatrix()->MultTranspose(rhs.GetBlock(2*i+1),Rhs.GetBlock(2*i+1));
+   }
+
+   HypreParMatrix * A = HypreParMatrixFromBlocks(Ah);
+
+
+// -----------------------------------------------------
+//        L O R    P R E C O N D I T I O N E R 
+// -----------------------------------------------------
+   Array2D<HypreParMatrix *> Ah_lor(4,4); 
+   for (int i = 0; i<4; i++)
+   {
+      for (int j = 0; j<4; j++)
+      {
+         Ah_lor[i][j] = nullptr;
+      }
+   }
+
+   ParBilinearForm a00_lor(&H1fes_lor);
+   a00_lor.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a00_lor.AddDomainIntegrator(new MassIntegrator(omeg2));
+   a00_lor.Assemble();
+   a00_lor.EliminateEssentialBC(ess_bdr,mfem::Operator::DIAG_ONE);
+   a00_lor.Finalize();
+   Ah_lor[0][0] = a00_lor.ParallelAssemble();
+
+   ParMixedBilinearForm a03_lor(&RTfes_lor,&H1fes_lor);
+   a03_lor.AddDomainIntegrator(new MixedScalarDivergenceIntegrator(omeg));
+   a03_lor.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg));
+   a03_lor.Assemble();
+   a03_lor.EliminateTestDofs(ess_bdr);
+   a03_lor.Finalize();
+   Ah_lor[0][3] = a03_lor.ParallelAssemble();
+   Ah_lor[3][0] = Ah_lor[0][3]->Transpose();
+
+   ParBilinearForm a11_lor(&RTfes_lor);
+   a11_lor.AddDomainIntegrator(new DivDivIntegrator(one));
+   a11_lor.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a11_lor.Assemble();
+   a11_lor.Finalize();
+   Ah_lor[1][1] = a11_lor.ParallelAssemble();
+
+
+   ParMixedBilinearForm a21_lor(&RTfes_lor,&H1fes_lor);
+   a21_lor.AddDomainIntegrator(new MixedScalarDivergenceIntegrator(negomeg));
+   a21_lor.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(negomeg));
+   a21_lor.Assemble();
+   a21_lor.EliminateTestDofs(ess_bdr);
+   a21_lor.Finalize();
+   Ah_lor[2][1] = a21_lor.ParallelAssemble();
+   Ah_lor[1][2] = Ah_lor[2][1]->Transpose();
+
+   ParBilinearForm a22_lor(&H1fes_lor);
+   a22_lor.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a22_lor.AddDomainIntegrator(new MassIntegrator(omeg2));
+   a22_lor.Assemble();
+   a22_lor.EliminateEssentialBC(ess_bdr,mfem::Operator::DIAG_ONE);
+   a22_lor.Finalize();
+   Ah_lor[2][2] = a22_lor.ParallelAssemble();
+
+   ParBilinearForm a33_lor(&RTfes_lor);
+   a33_lor.AddDomainIntegrator(new DivDivIntegrator(one));
+   a33_lor.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a33_lor.Assemble();
+   a33_lor.Finalize();
+   Ah_lor[3][3] = a33_lor.ParallelAssemble();
+
+   HypreParMatrix * A_lor = HypreParMatrixFromBlocks(Ah_lor);
+
+
+// -----------------------------------------------------
+// -----------------------------------------------------
+
+   FiniteElement::MapType t = FiniteElement::H_DIV;
+   Array<int> perm = ComputeVectorFE_LORPermutation(RTfes_ho, RTfes_lor, t);
+
+   HypreBoomerAMG * amg_p0 = new HypreBoomerAMG(*Ah[0][0]);
+   amg_p0->SetPrintLevel(0);
+   HypreBoomerAMG * amg_lor_p0 = new HypreBoomerAMG(*Ah_lor[0][0]);
+   amg_lor_p0->SetPrintLevel(0);
+   HypreBoomerAMG * amg_p2 = new HypreBoomerAMG(*Ah[2][2]);
+   amg_p2->SetPrintLevel(0);
+   HypreBoomerAMG * amg_lor_p2 = new HypreBoomerAMG(*Ah_lor[2][2]);
+   amg_lor_p2->SetPrintLevel(0);
+
+   Solver *prec1 = nullptr;
+   Solver *prec3 = nullptr;
+   Solver *prec1_lor = nullptr;
+   Solver *prec3_lor = nullptr;
+   if (dim == 2) 
+   {
+      prec1 = new HypreAMS(*Ah[1][1],&RTfes_ho);
+      dynamic_cast<HypreAMS *>(prec1)->SetPrintLevel(0);
+      prec3 = new HypreAMS(*Ah[3][3],&RTfes_ho);
+      dynamic_cast<HypreAMS *>(prec3)->SetPrintLevel(0);
+      prec1_lor = new HypreAMS(*Ah_lor[1][1],&RTfes_lor);
+      dynamic_cast<HypreAMS *>(prec1_lor)->SetPrintLevel(0);
+      prec3_lor = new HypreAMS(*Ah_lor[3][3],&RTfes_lor);
+      dynamic_cast<HypreAMS *>(prec3_lor)->SetPrintLevel(0);
+   }
+   else
+   {
+      prec1 = new HypreADS(*Ah[1][1],&RTfes_ho);
+      dynamic_cast<HypreADS *>(prec1)->SetPrintLevel(0);
+      prec3 = new HypreADS(*Ah[3][3],&RTfes_ho);
+      dynamic_cast<HypreADS *>(prec3)->SetPrintLevel(0);
+      prec1_lor = new HypreADS(*Ah_lor[1][1],&RTfes_lor);
+      dynamic_cast<HypreADS *>(prec1_lor)->SetPrintLevel(0);
+      prec3_lor = new HypreADS(*Ah_lor[3][3],&RTfes_lor);
+      dynamic_cast<HypreADS *>(prec3_lor)->SetPrintLevel(0);
+   }
+   // 1st preconditioner: Exact LOR with direct solver
+   ComplexLORSolver M_lor_exact(*A_lor, perm);
+
+   // 2nd preconditioner: AMG/AMS on the high order system
+   BlockDiagonalPreconditioner M(block_trueOffsets);
+   M.SetDiagonalBlock(0,amg_p0);
+   M.SetDiagonalBlock(1,prec1);
+   M.SetDiagonalBlock(2,amg_p2);
+   M.SetDiagonalBlock(3,prec3);
+
+   // 3rd preconditioner: AMG/AMS on the LOR system
+   BlockDiagonalPreconditioner M_lor2(block_trueOffsets);
+   M_lor2.SetDiagonalBlock(0,amg_lor_p0);
+   M_lor2.SetDiagonalBlock(1,prec1_lor);
+   M_lor2.SetDiagonalBlock(2,amg_lor_p2);
+   M_lor2.SetDiagonalBlock(3,prec3_lor);
+
+   ComplexLORSolver M_lor(*A_lor, perm,false,&M_lor2);
+
+   Vector Y(X), Z(X);
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-6);
+   cg.SetMaxIter(5000);
+   cg.SetPrintLevel(3);
+   cg.SetOperator(*A);
+   StopWatch chrono;
+   chrono.Clear();
+   chrono.Start();
+   cg.SetPreconditioner(M_lor_exact);
+   cg.Mult(Rhs, X);
+   chrono.Stop();
+   cout << "PCG Exact LOR time = " << chrono.RealTime() << endl;
+
+   chrono.Clear();
+   chrono.Start();
+   cg.SetPreconditioner(M_lor);
+   cg.Mult(Rhs, Y);
+   chrono.Stop();
+   cout << "PCG AMG/AMS LOR time = " << chrono.RealTime() << endl;
+
+   chrono.Clear();
+   chrono.Start();
+   cg.SetPreconditioner(M);
+   cg.Mult(Rhs, Z);
+   chrono.Stop();
+   cout << "PCG AMG/AMS HO time = " << chrono.RealTime() << endl;
+   {
+      MUMPSSolver mumps;
+      mumps.SetPrintLevel(0);
+      mumps.SetOperator(*A);
+      mumps.Mult(Rhs,X);
+   }
+
+
+   p_gf_re = 0.0;
+   p_gf_im = 0.0;
+   u_gf_re = 0.0;
+   u_gf_im = 0.0;
+
+   p_gf_re.Distribute(&(X.GetBlock(0)));
+   u_gf_re.Distribute(&(X.GetBlock(1)));
+   p_gf_im.Distribute(&(X.GetBlock(2)));
+   u_gf_im.Distribute(&(X.GetBlock(3)));
+
+   ConvergenceStudy ratesH1;
+   ConvergenceStudy ratesRT;
+
+   VectorFunctionCoefficient gradp_ex(dim,gradp_exact_re);
+   FunctionCoefficient divu_ex(divu_exact_re);
+
+   ratesH1.AddH1GridFunction(&p_gf_re,&p_ex_re,&gradp_ex);
+   ratesRT.AddHdivGridFunction(&u_gf_re,&u_ex_re,&divu_ex);
+
+   ratesH1.Print(true);
+   ratesRT.Print(true);
+
+   // 10. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << p_gf_re <<
+               "window_title 'Numerical Pressure (real part)' "
+               << flush;
+      socketstream sol_sockex(vishost, visport);
+      ParGridFunction p_ex(&H1fes_ho);
+      p_ex.ProjectCoefficient(p_ex_re);
+      sol_sockex << "parallel " << num_procs << " " << myid << "\n";
+      sol_sockex.precision(8);
+      sol_sockex << "solution\n" << *pmesh << p_ex <<
+               "window_title 'Exact Pressure (real part)' "
+               << flush;         
+   }
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+
+
+void helmholtz_solution(const Vector &X, complex<double> &sol, 
+                                         std::vector<complex<double>> &grad,
+                                         complex<double> &grad2)
+{
+   double x = X(0), y = X(1);
+   double z;
+   if (dim == 3 ) z = X(2);
+
+   complex<double> zi(0,1);
+   if (exact == 0)
+   {
+      if (dim == 2)
+      {
+         sol = x*(1.0-x) * y*(1.0-y);
+
+         grad[0] = (1.0 - 2*x) * y*(1.0 - y);
+         grad[1] = (1.0 - 2*y) * x*(1.0 - x);
+         grad2 = -2 * y*(1.0 - y) - 2 * x*(1.0 - x);
+      }
+      else
+      {
+         sol = x*(1.0-x) * y*(1.0-y) * z*(1.0-z);
+         grad[0] = (1.0 - 2*x) * y*(1.0 - y) * z*(1.0-z);
+         grad[1] = (1.0 - 2*y) * x*(1.0 - x) * z*(1.0-z);
+         grad[2] = (1.0 - 2*z) * x*(1.0 - x) * y*(1.0-y);
+         grad2 = -2 * y*(1.0 - y) * z*(1.0-z)  
+                 -2 * x*(1.0 - x) * z*(1.0-z) 
+                 -2 * x*(1.0 - x) * y*(1.0-y);
+      }
+   }
+   else
+   {
+      complex<double> alpha;
+      if (dim == 2)
+      {
+         alpha = zi * omega / sqrt(2);
+         sol = exp(alpha*(x+y));
+         grad[0] = alpha * sol;
+         grad[1] = alpha * sol;
+         grad2 = 2.0*alpha*alpha*sol;
+      }
+      else
+      {
+         alpha = zi * omega / sqrt(3);
+         sol = exp(alpha*(x+y+z));
+         grad[0] = alpha * sol;
+         grad[1] = alpha * sol;
+         grad[2] = alpha * sol;
+         grad2 = 3.0*alpha*alpha*sol;
+      }
+   }
+}
+
+double p_exact_re(const Vector &x)
+{
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+   return sol.real();
+}
+
+double p_exact_im(const Vector &x)
+{
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+   return sol.imag();
+}
+
+void gradp_exact_re(const Vector &x, Vector &gradp)
+{
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+   for (int i=0; i<dim; i++)
+   {
+      gradp[i] = grad[i].real();
+   }
+}
+void gradp_exact_im(const Vector &x, Vector &gradp)
+{
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+   for (int i=0; i<dim; i++)
+   {
+      gradp[i] = grad[i].real();
+   }
+}
+
+
+void u_exact_re(const Vector &x, Vector &u)
+{
+   complex<double> zi(0,1);
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+   // u = i grad p / w 
+   for (int i=0; i<dim; i++)
+   {
+      u[i] = (zi * grad[i]/omega).real();
+   }
+}
+
+void u_exact_im(const Vector &x, Vector &u)
+{
+   complex<double> zi(0,1);
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+   // u = i grad p / w 
+   for (int i=0; i<dim; i++)
+   {
+      u[i] = (zi * grad[i]/omega).imag();
+   }
+}
+
+double divu_exact_re(const Vector &x)
+{
+   complex<double> zi(0,1);
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+   return (zi/omega * grad2).real();
+
+}
+double divu_exact_im(const Vector &x)
+{
+   complex<double> zi(0,1);
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+   return (zi/omega * grad2).imag();
+}
+
+void f_exact_re(const Vector &x, Vector &f)
+{
+   f = 0.0;
+}
+
+void f_exact_im(const Vector &x, Vector &f)
+{
+   f = 0.0;
+}
+
+double g_exact_re(const Vector &x)
+{
+   // f = i omega p + div u  
+   // f = i / omega *( omega * omega p + grad2) 
+   complex<double> zi(0,1);
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+
+   return (zi / omega *(omega * omega * sol + grad2)).real();
+}
+
+double g_exact_im(const Vector &x)
+{
+   // f = i omega p + div u  
+   // f = i / omega *( omega * omega p + grad2) 
+   complex<double> zi(0,1);
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+
+   return (zi / omega *(omega * omega * sol + grad2)).imag();
+}
+
+void plotfield(socketstream & socket, ParMesh * pmesh, const ParGridFunction & pgf, string & title )
+{
+   int num_procs, myid;
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+   ostringstream oss;
+   oss << title;
+   socket << "parallel " << num_procs << " " << myid << "\n";
+   socket.precision(8);
+   socket << "solution\n" << *pmesh << pgf 
+          << "window_title '" << oss.str() << "'" << flush;
+}

examples/maxwell-solver/LS_maxwell_lor.cpp

@@ -0,0 +1,419 @@
+
+
+//                                MFEM Example multigrid-grid Cycle
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "FOSLS.hpp"
+
+using namespace std;
+using namespace mfem;
+
+void maxwell_solution(const Vector &x, std::vector<complex<double>> & sol, 
+                                       std::vector<complex<double>> & curl,
+                                       std::vector<complex<double>> & curl2);
+
+void E_exact_re(const Vector &x, Vector &E);
+void H_exact_re(const Vector &x, Vector &H);
+void E_exact_im(const Vector &x, Vector &E);
+void H_exact_im(const Vector &x, Vector &H);
+
+void f_exact_re(const Vector &x, Vector &f);
+void g_exact_re(const Vector &x, Vector &g);
+void f_exact_im(const Vector &x, Vector &f);
+void g_exact_im(const Vector &x, Vector &g);
+void plotfield(socketstream &,ParMesh * pmesh,const ParGridFunction & , string &);
+
+int dim;
+double omega;
+int exact = 0;
+
+   // ----------------------------------------------------------------------
+   // |   |            E             |             H          |     RHS    | 
+   // ----------------------------------------------------------------------
+   // | F | (curlE,curlF)+w^2(E,F)   | iw(curlH,F)+iw(H,curF) | -iw(J,F)   |
+   // |   |                          |                        |            |
+   // | G |-iw(E,curlG)-iw(curlE,G)  | (curlH,curlG)+w^2(H,G) | -(J,curlG) |
+
+int main(int argc, char *argv[])
+{
+    // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/inline-hex.mesh";
+   int order = 1;
+   bool visualization = 1;
+   int sr = 1;
+   int pr = 1;
+   double rnum=1.0;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&sr, "-sr", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pr", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");                  
+   args.AddOption(&exact, "-solution", "--exact_solution",
+                  "Exact solution : 0-polynomial, 1-plane wave");                  
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   omega = 2.0 * M_PI * rnum;
+   // omega = rnum;
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+
+   dim = mesh->Dimension();
+
+   MFEM_VERIFY(dim == 3, "only 3D problems supported by this formulation");
+
+   for (int i = 0; i < sr; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   for (int i = 0; i < pr; i++ )
+   {
+      pmesh->UniformRefinement();
+   }
+
+   FiniteElementCollection *fec = new ND_FECollection(order,dim); 
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+
+   if (myid == 0)
+   {
+      cout << "Number of True Dofs = " << size << endl;
+   }
+
+   VectorFunctionCoefficient E_ex_re(dim,E_exact_re);
+   VectorFunctionCoefficient H_ex_re(dim,H_exact_re);
+   VectorFunctionCoefficient E_ex_im(dim,E_exact_im);
+   VectorFunctionCoefficient H_ex_im(dim,H_exact_im);
+   VectorFunctionCoefficient f_ex_re(dim,f_exact_re);
+   VectorFunctionCoefficient g_ex_re(dim,g_exact_re);
+   VectorFunctionCoefficient f_ex_im(dim,f_exact_im);
+   VectorFunctionCoefficient g_ex_im(dim,g_exact_im);
+
+   int n = fespace->GetVSize();
+   int N = fespace->GetTrueVSize();
+   Array<int> block_offsets(5);
+   block_offsets = n; 
+   block_offsets[0] = 0;
+   block_offsets.PartialSum();
+
+   Array<int> block_trueOffsets(5);
+   block_trueOffsets = N;
+   block_trueOffsets[0] = 0;
+   block_trueOffsets.PartialSum();
+
+   BlockVector X(block_trueOffsets), Rhs(block_trueOffsets);
+   X = 0.0;  Rhs = 0.0;
+
+   ComplexMaxwellFOSLS fosls(fespace);
+   fosls.SetOmega(omega);
+   Array<VectorFunctionCoefficient * > ess_data(4);
+   ess_data[0] = &E_ex_re;
+   ess_data[1] = &H_ex_re;
+   ess_data[2] = &E_ex_im;
+   ess_data[3] = &H_ex_im;
+   fosls.SetEssentialData(ess_data);
+   Array<VectorFunctionCoefficient * > loads(4);
+   loads[0] = &f_ex_re;
+   loads[1] = &g_ex_re;
+   loads[2] = &f_ex_im;
+   loads[3] = &g_ex_im;
+   fosls.SetLoadData(loads);
+
+   Array2D<HypreParMatrix *> Ah;
+   fosls.GetFOSLSLinearSystem(Ah,X,Rhs);
+
+   HypreParMatrix * A = HypreParMatrixFromBlocks(Ah);
+   
+   HypreAMS ams0(*Ah[0][0],fespace);
+   HypreAMS ams1(*Ah[1][1],fespace);
+
+   BlockDiagonalPreconditioner prec(block_trueOffsets);
+   prec.SetDiagonalBlock(0,&ams0);
+   prec.SetDiagonalBlock(1,&ams1);
+   prec.SetDiagonalBlock(2,&ams0);
+   prec.SetDiagonalBlock(3,&ams1);
+
+   StopWatch chrono;
+   chrono.Clear();
+   chrono.Start();
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-6);
+   // cg.SetAbsTol(1e-6);
+   cg.SetMaxIter(400);
+   cg.SetPrintLevel(1);
+   cg.SetOperator(*A);
+   cg.SetPreconditioner(prec);
+   cg.Mult(Rhs, X);
+   chrono.Stop();
+   double t1 = chrono.RealTime();
+   if (myid == 0)
+   {
+      cout << "PCG time = " << t1 << endl;
+   }
+   
+   // {
+   //    MUMPSSolver mumps;
+   //    mumps.SetPrintLevel(0);
+   //    mumps.SetOperator(*A);
+   //    mumps.Mult(Rhs,X);
+   // }
+   ParGridFunction E_gf_re(fespace);
+   ParGridFunction H_gf_re(fespace);
+   ParGridFunction E_gf_im(fespace);
+   ParGridFunction H_gf_im(fespace);
+   E_gf_re = 0.0;
+   E_gf_im = 0.0;
+   H_gf_re = 0.0;
+   H_gf_im = 0.0;
+
+   E_gf_re.Distribute(&(X.GetBlock(0)));
+   H_gf_re.Distribute(&(X.GetBlock(1)));
+   E_gf_im.Distribute(&(X.GetBlock(2)));
+   H_gf_im.Distribute(&(X.GetBlock(3)));
+
+   double E_re_L2_Error = E_gf_re.ComputeL2Error(E_ex_re);
+   double E_im_L2_Error = E_gf_im.ComputeL2Error(E_ex_im);
+   double H_re_L2_Error = H_gf_re.ComputeL2Error(H_ex_re);
+   double H_im_L2_Error = H_gf_im.ComputeL2Error(H_ex_im);
+
+   ParGridFunction zero(fespace);
+   zero = 0.0;
+   double E_re_L2_norm = zero.ComputeL2Error(E_ex_re);
+   double E_im_L2_norm = zero.ComputeL2Error(E_ex_im);
+   double H_re_L2_norm = zero.ComputeL2Error(H_ex_re);
+   double H_im_L2_norm = zero.ComputeL2Error(H_ex_im);
+   if (myid == 0)
+   {
+      cout << "E_re L2 Error = " << E_re_L2_Error/E_re_L2_norm << endl;
+      cout << "E_im L2 Error = " << E_im_L2_Error/E_im_L2_norm << endl;
+      cout << "H_re L2 Error = " << H_re_L2_Error/H_re_L2_norm << endl;
+      cout << "H_im L2 Error = " << H_im_L2_Error/H_im_L2_norm << endl;
+   }   
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock0(vishost, visport);
+      socketstream sol_sock1(vishost, visport);
+      socketstream sol_sock2(vishost, visport);
+      socketstream sol_sock3(vishost, visport);
+      string str0 = "E_re";
+      plotfield(sol_sock0,pmesh, E_gf_re,str0);
+      string str1 = "E_im";
+      plotfield(sol_sock1,pmesh,E_gf_im,str1);
+      string str2 = "H_re";
+      plotfield(sol_sock2,pmesh,H_gf_re,str2);
+      string str3 = "H_im";
+      plotfield(sol_sock3,pmesh,H_gf_im,str3);
+
+      ParGridFunction E_exact_re(fespace);
+      ParGridFunction E_exact_im(fespace);
+      ParGridFunction H_exact_re(fespace);
+      ParGridFunction H_exact_im(fespace);
+      E_exact_re.ProjectCoefficient(E_ex_re);
+      E_exact_im.ProjectCoefficient(E_ex_im);
+      H_exact_re.ProjectCoefficient(H_ex_re);
+      H_exact_im.ProjectCoefficient(H_ex_im);
+
+      socketstream sol_sock_ex0(vishost, visport);
+      socketstream sol_sock_ex1(vishost, visport);
+      socketstream sol_sock_ex2(vishost, visport);
+      socketstream sol_sock_ex3(vishost, visport);
+      str0 = "E_exact_re";
+      plotfield(sol_sock_ex0,pmesh,E_exact_re,str0);
+      str1 = "E_exact_im";
+      plotfield(sol_sock_ex1,pmesh,E_exact_im,str1);
+      str2 = "H_exact_re";
+      plotfield(sol_sock_ex2,pmesh,H_exact_re,str2);
+      str3 = "H_exact_im";
+      plotfield(sol_sock_ex3,pmesh,H_exact_im,str3);
+   }
+
+   MPI_Finalize();
+   return 0;
+}
+
+
+void maxwell_solution(const Vector &X, std::vector<complex<double>> &sol, 
+                                       std::vector<complex<double>> &curl,
+                                       std::vector<complex<double>> &curl2)
+{
+   double x = X(0), y = X(1), z = X(2);
+
+   complex<double> zi(0,1);
+   if (exact == 0)
+   {
+      sol[0] = y*(1.0-y)*z*(1.0-z) + zi * 2.0;
+      sol[1] = y*x*(1.0-x)*z*(1.0-z)+ zi * 2.0;
+      sol[2] = x*(1.0-x)*y*(1.0-y) + zi * 2.0;
+
+      curl[0] = (1.0-x)*x*(y*(2.0*z-3.0)+1.0);
+      curl[1] = 2.0*(1.0-y)*y*(x-z);
+      curl[2] = (z-1.0)*z*(y*(2*x-3)+1.0);
+
+      curl2[0] = (2.0*x-3.0)*(z-1.0)*z-2.0*y*y+2*y;
+      curl2[1] = -2.0*y*(x*x-x+(z-1.0)*z);
+      curl2[2] = 2*(x*(1.5-z)+x*x*(z-1.5)-y*y+y);
+   }
+   else
+   {
+      complex<double> alpha = zi * omega / sqrt(3);
+      sol[0] = exp(alpha*(x+y+z));
+      sol[1] = 0.0;
+      sol[2] = 0.0;
+
+      curl[0] = 0.0;
+      curl[1] = alpha * sol[0];
+      curl[2] = -alpha * sol[0];
+
+      curl2[0] = -2.0 * alpha * alpha * sol[0];
+      curl2[1] = alpha * alpha * sol[0];
+      curl2[2] = curl2[1];
+   }
+
+
+   // sol[0] = 1.0 + 2.0*zi;
+   // sol[1] = 1.0 + 2.0*zi;
+   // sol[2] = 1.0 + 2.0*zi;
+   // curl[0] = 0.0;
+   // curl[1] =0.0;
+   // curl[2] =0.0;
+   // curl2[0] =0.0;
+   // curl2[1] =0.0;
+   // curl2[2] =0.0;  
+
+}
+
+void E_exact_re(const Vector &x, Vector &E)
+{
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      E(i) = sol[i].real();
+   }
+}
+void H_exact_re(const Vector &x, Vector &H)
+{
+   complex<double> zi(0,1);
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   // H = i curlE / w 
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      H[i] = (zi * curl[i]/omega).real();
+   }
+}
+void E_exact_im(const Vector &x, Vector &E)
+{
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      E(i) = sol[i].imag();
+   }
+}
+void H_exact_im(const Vector &x, Vector &H)
+{
+   complex<double> zi(0,1);
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   // H = i curlE / w 
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      H[i] = (zi * curl[i]/omega).imag();
+   }
+}
+
+void f_exact_re(const Vector &x, Vector &f)
+{
+   f = 0.0;
+}
+void g_exact_re(const Vector &x, Vector &g)
+{
+   // J = i omega E - curl H 
+   // J = - i / omega (curl curl E - omega * omega E)
+   complex<double> zi(0,1);
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      g(i) = (-zi / omega *(curl2[i] - omega * omega * sol[i])).real();
+   }
+}
+void f_exact_im(const Vector &x, Vector &f)
+{
+   f = 0.0;
+}
+void g_exact_im(const Vector &x, Vector &g)
+{
+   // J = i omega E - curl H 
+   // J = - i / omega (curl curl E - omega * omega E)
+   complex<double> zi(0,1);
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      g(i) = (-zi / omega *(curl2[i] - omega * omega * sol[i])).imag();
+   }
+}
+
+
+void plotfield(socketstream & socket, ParMesh * pmesh, const ParGridFunction & pgf, string & title )
+{
+   int num_procs, myid;
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+   ostringstream oss;
+   oss << title;
+   socket << "parallel " << num_procs << " " << myid << "\n";
+   socket.precision(8);
+   socket << "solution\n" << *pmesh << pgf 
+          << "window_title '" << oss.str() << "'" << flush;
+}

examples/maxwell-solver/LS_maxwellp.cpp

@@ -0,0 +1,568 @@
+
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "FOSLS.hpp"
+
+using namespace std;
+using namespace mfem;
+
+void maxwell_solution(const Vector &x, std::vector<complex<double>> & sol, 
+                                       std::vector<complex<double>> & curl,
+                                       std::vector<complex<double>> & curl2);
+
+void E_exact_re(const Vector &x, Vector &E);
+void H_exact_re(const Vector &x, Vector &H);
+void E_exact_im(const Vector &x, Vector &E);
+void H_exact_im(const Vector &x, Vector &H);
+
+void f_exact_re(const Vector &x, Vector &f);
+void g_exact_re(const Vector &x, Vector &g);
+void f_exact_im(const Vector &x, Vector &f);
+void g_exact_im(const Vector &x, Vector &g);
+void plotfield(socketstream &,ParMesh * pmesh,const ParGridFunction & , string &);
+
+int dim;
+double omega;
+int exact = 0;
+
+   // ----------------------------------------------------------------------
+   // |   |            E             |             H          |     RHS    | 
+   // ----------------------------------------------------------------------
+   // | F | (curlE,curlF)+w^2(E,F)   | iw(curlH,F)+iw(H,curF) | -iw(J,F)   |
+   // |   |                          |                        |            |
+   // | G |-iw(E,curlG)-iw(curlE,G)  | (curlH,curlG)+w^2(H,G) | -(J,curlG) |
+
+int main(int argc, char *argv[])
+{
+    // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/inline-hex.mesh";
+   int order = 1;
+   bool visualization = 1;
+   int sr = 1;
+   int pr = 1;
+   double rnum=1.0;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&sr, "-sr", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pr", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");                  
+   args.AddOption(&exact, "-solution", "--exact_solution",
+                  "Exact solution : 0-polynomial, 1-plane wave");                  
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   omega = 2.0 * M_PI * rnum;
+   // omega = rnum;
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+
+   dim = mesh->Dimension();
+
+   MFEM_VERIFY(dim == 3, "only 3D problems supported by this formulation");
+
+   for (int i = 0; i < sr; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   for (int i = 0; i < pr; i++ )
+   {
+      pmesh->UniformRefinement();
+   }
+
+   FiniteElementCollection *fec = new ND_FECollection(order,dim); 
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+
+   if (myid == 0)
+   {
+      cout << "Number of True Dofs = " << size << endl;
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (pmesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+
+   VectorFunctionCoefficient E_ex_re(dim,E_exact_re);
+   VectorFunctionCoefficient H_ex_re(dim,H_exact_re);
+   VectorFunctionCoefficient E_ex_im(dim,E_exact_im);
+   VectorFunctionCoefficient H_ex_im(dim,H_exact_im);
+
+   VectorFunctionCoefficient f_ex_re(dim,f_exact_re);
+   VectorFunctionCoefficient g_ex_re(dim,g_exact_re);
+   VectorFunctionCoefficient f_ex_im(dim,f_exact_im);
+   VectorFunctionCoefficient g_ex_im(dim,g_exact_im);
+
+   int n = fespace->GetVSize();
+   int N = fespace->GetTrueVSize();
+   Array<int> block_offsets(5);
+   block_offsets[0] = 0;
+   block_offsets[1] = n;
+   block_offsets[2] = n;
+   block_offsets[3] = n;
+   block_offsets[4] = n;
+   block_offsets.PartialSum();
+
+   Array<int> block_trueOffsets(5);
+   block_trueOffsets[0] = 0;
+   block_trueOffsets[1] = N;
+   block_trueOffsets[2] = N;
+   block_trueOffsets[3] = N;
+   block_trueOffsets[4] = N;
+   block_trueOffsets.PartialSum();
+
+   BlockVector x(block_offsets), rhs(block_offsets);
+   BlockVector X(block_trueOffsets), Rhs(block_trueOffsets);
+   x = 0.0;  rhs = 0.0; X = 0.0;  Rhs = 0.0;
+
+   ParGridFunction E_gf_re, E_gf_im, H_gf_re, H_gf_im;
+
+   E_gf_re.MakeRef(fespace,x.GetBlock(0)); E_gf_re = 0.0;
+   H_gf_re.MakeRef(fespace,x.GetBlock(1)); H_gf_re = 0.0;
+   E_gf_im.MakeRef(fespace,x.GetBlock(2)); E_gf_im = 0.0;
+   H_gf_im.MakeRef(fespace,x.GetBlock(3)); H_gf_im = 0.0;
+
+   // E_gf_re.ProjectBdrCoefficientTangent(E_ex_re,ess_bdr);
+   // E_gf_im.ProjectBdrCoefficientTangent(E_ex_im,ess_bdr);
+   E_gf_re.ProjectCoefficient(E_ex_re);
+   E_gf_im.ProjectCoefficient(E_ex_im);
+
+   // ----------------------------------------------------------------------
+   // |   |            E             |             H          |     RHS    | 
+   // ----------------------------------------------------------------------
+   // | F | (curlE,curlF)+w^2(E,F)   | iw(curlH,F)+iw(H,curF) | -iw(J,F)   |
+   // |   |                          |                        |            |
+   // | G |-iw(E,curlG)-iw(curlE,G)  | (curlH,curlG)+w^2(H,G) | -(J,curlG) |
+   // ----------------------------------------------------------------------
+
+   // for convinience we convert the above 2 x 2 blocks to 4 x 4 in order
+   // to accomodate complex valued operators
+
+   // A = (curlE,curlF)+w^2(E,F)
+   // B = w(curlH,F)+w(H,curF)
+   // b0 = w(J_im,F)
+   // b1 = -(J_re,curlG)
+   // b2 = -w(J_re,G)
+   // b3 = -(J_im,G)
+
+   // | A   0   0  -B |  | E_re |     | b0 |
+   // | 0   A   B   0 |  | H_re |  =  | b1 |
+   // | 0   B   A   0 |  | E_Im |     | b2 |
+   // |-B   0   0   A |  | H_im |     | b3 |
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient negomeg(-omega);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega * omega);
+
+
+   ScalarVectorProductCoefficient wJi(omeg,g_ex_im);
+   ScalarVectorProductCoefficient negJr(negone,g_ex_re);
+   ScalarVectorProductCoefficient negwJr(negomeg,g_ex_re);
+   ScalarVectorProductCoefficient negJi(negone,g_ex_im);
+
+
+   ParLinearForm b0(fespace);
+   ParLinearForm b1(fespace);
+   ParLinearForm b2(fespace);
+   ParLinearForm b3(fespace);
+   b0.Update(fespace,rhs.GetBlock(0),0);
+   b1.Update(fespace,rhs.GetBlock(1),0);
+   b2.Update(fespace,rhs.GetBlock(2),0);
+   b3.Update(fespace,rhs.GetBlock(3),0);
+   
+   b0.AddDomainIntegrator(new VectorFEDomainLFIntegrator(wJi));
+   b1.AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(negJr));
+   b2.AddDomainIntegrator(new VectorFEDomainLFIntegrator(negwJr));
+   b3.AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(negJi));
+   
+   b0.Assemble();
+   b1.Assemble();
+   b2.Assemble();
+   b3.Assemble();
+
+   Array2D<HypreParMatrix *> Ah(4,4); 
+   for (int i = 0; i<4; i++)
+   {
+      for (int j = 0; j<4; j++)
+      {
+         Ah[i][j] = nullptr;
+      }
+   }
+
+
+   ParBilinearForm a00(fespace);
+   a00.AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a00.Assemble();
+   a00.EliminateEssentialBC(ess_bdr,x.GetBlock(0),rhs.GetBlock(0),mfem::Operator::DIAG_ONE);
+   a00.Finalize();
+   Ah[0][0] = a00.ParallelAssemble();
+
+   ParMixedBilinearForm a03(fespace,fespace);
+   a03.AddDomainIntegrator(new MixedVectorCurlIntegrator(negomeg));
+   a03.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(negomeg));
+   a03.Assemble();
+   a03.EliminateTestDofs(ess_bdr);
+   a03.Finalize();
+   Ah[0][3] = a03.ParallelAssemble();
+
+   ParBilinearForm a11(fespace);
+   a11.AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a11.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a11.Assemble();
+   a11.Finalize();
+   Ah[1][1] = a11.ParallelAssemble();
+
+   ParMixedBilinearForm a12(fespace,fespace);
+   a12.AddDomainIntegrator(new MixedVectorCurlIntegrator(omeg));
+   a12.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(omeg));
+   a12.Assemble();
+   a12.EliminateTrialDofs(ess_bdr,x.GetBlock(2),rhs.GetBlock(1));
+   a12.Finalize();
+   Ah[1][2] = a12.ParallelAssemble();
+
+   ParMixedBilinearForm a21(fespace,fespace);
+   a21.AddDomainIntegrator(new MixedVectorCurlIntegrator(omeg));
+   a21.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(omeg));
+   a21.Assemble();
+   a21.EliminateTestDofs(ess_bdr);
+   a21.Finalize();
+   Ah[2][1] = a21.ParallelAssemble();
+   // Ah[2][1] = Ah[1][2]->Transpose();
+   // (*Ah[2][1]) *=-1.0;
+
+   ParBilinearForm a22(fespace);
+   a22.AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a22.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a22.Assemble();
+   a22.EliminateEssentialBC(ess_bdr,x.GetBlock(2),rhs.GetBlock(2),mfem::Operator::DIAG_ONE);
+   a22.Finalize();
+   Ah[2][2] = a22.ParallelAssemble();
+
+   ParMixedBilinearForm a30(fespace,fespace);
+   a30.AddDomainIntegrator(new MixedVectorCurlIntegrator(negomeg));
+   a30.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(negomeg));
+   a30.Assemble();
+   a30.EliminateTrialDofs(ess_bdr,x.GetBlock(0),rhs.GetBlock(3));
+   a30.Finalize();
+   Ah[3][0] = a30.ParallelAssemble();
+   // Ah[3][0] = Ah[0][3]->Transpose();
+   // (*Ah[3][0])*=-1.0;
+
+   ParBilinearForm a33(fespace);
+   a33.AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a33.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a33.Assemble();
+   a33.Finalize();
+   Ah[3][3] = a33.ParallelAssemble();
+   // Ah[3][3] = Ah[2][2];
+
+   // HypreParMatrix * diff = new HypreParMatrix(*Ah[0][3]);
+   // *diff += *Ah[3][0];
+
+
+   for (int i = 0; i<4; i++)
+   {
+      fespace->GetRestrictionMatrix()->Mult(x.GetBlock(i), X.GetBlock(i));
+      fespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(i),Rhs.GetBlock(i));
+   }
+
+   HypreParMatrix * A = HypreParMatrixFromBlocks(Ah);
+   
+   HypreAMS ams0(*Ah[0][0],fespace);
+   HypreAMS ams1(*Ah[1][1],fespace);
+
+   BlockDiagonalPreconditioner prec(block_trueOffsets);
+   prec.SetDiagonalBlock(0,&ams0);
+   prec.SetDiagonalBlock(1,&ams1);
+   prec.SetDiagonalBlock(2,&ams0);
+   prec.SetDiagonalBlock(3,&ams1);
+
+   StopWatch chrono;
+   chrono.Clear();
+   chrono.Start();
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-6);
+   // cg.SetAbsTol(1e-6);
+   cg.SetMaxIter(400);
+   cg.SetPrintLevel(1);
+   cg.SetOperator(*A);
+   cg.SetPreconditioner(prec);
+   cg.Mult(Rhs, X);
+   chrono.Stop();
+   double t1 = chrono.RealTime();
+   if (myid == 0)
+   {
+      cout << "PCG time = " << t1 << endl;
+   }
+   
+   // {
+   //    MUMPSSolver mumps;
+   //    mumps.SetPrintLevel(0);
+   //    mumps.SetOperator(*A);
+   //    mumps.Mult(Rhs,X);
+   // }
+   E_gf_re = 0.0;
+   E_gf_im = 0.0;
+   H_gf_re = 0.0;
+   H_gf_im = 0.0;
+
+   E_gf_re.Distribute(&(X.GetBlock(0)));
+   H_gf_re.Distribute(&(X.GetBlock(1)));
+   E_gf_im.Distribute(&(X.GetBlock(2)));
+   H_gf_im.Distribute(&(X.GetBlock(3)));
+
+   double E_re_L2_Error = E_gf_re.ComputeL2Error(E_ex_re);
+   double E_im_L2_Error = E_gf_im.ComputeL2Error(E_ex_im);
+   double H_re_L2_Error = H_gf_re.ComputeL2Error(H_ex_re);
+   double H_im_L2_Error = H_gf_im.ComputeL2Error(H_ex_im);
+
+   ParGridFunction zero(fespace);
+   zero = 0.0;
+   double E_re_L2_norm = zero.ComputeL2Error(E_ex_re);
+   double E_im_L2_norm = zero.ComputeL2Error(E_ex_im);
+   double H_re_L2_norm = zero.ComputeL2Error(H_ex_re);
+   double H_im_L2_norm = zero.ComputeL2Error(H_ex_im);
+   if (myid == 0)
+   {
+      cout << "E_re L2 Error = " << E_re_L2_Error/E_re_L2_norm << endl;
+      cout << "E_im L2 Error = " << E_im_L2_Error/E_im_L2_norm << endl;
+      cout << "H_re L2 Error = " << H_re_L2_Error/H_re_L2_norm << endl;
+      cout << "H_im L2 Error = " << H_im_L2_Error/H_im_L2_norm << endl;
+   }   
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock0(vishost, visport);
+      socketstream sol_sock1(vishost, visport);
+      socketstream sol_sock2(vishost, visport);
+      socketstream sol_sock3(vishost, visport);
+      string str0 = "E_re";
+      plotfield(sol_sock0,pmesh, E_gf_re,str0);
+      string str1 = "E_im";
+      plotfield(sol_sock1,pmesh,E_gf_im,str1);
+      string str2 = "H_re";
+      plotfield(sol_sock2,pmesh,H_gf_re,str2);
+      string str3 = "H_im";
+      plotfield(sol_sock3,pmesh,H_gf_im,str3);
+
+      ParGridFunction E_exact_re(fespace);
+      ParGridFunction E_exact_im(fespace);
+      ParGridFunction H_exact_re(fespace);
+      ParGridFunction H_exact_im(fespace);
+      E_exact_re.ProjectCoefficient(E_ex_re);
+      E_exact_im.ProjectCoefficient(E_ex_im);
+      H_exact_re.ProjectCoefficient(H_ex_re);
+      H_exact_im.ProjectCoefficient(H_ex_im);
+
+      socketstream sol_sock_ex0(vishost, visport);
+      socketstream sol_sock_ex1(vishost, visport);
+      socketstream sol_sock_ex2(vishost, visport);
+      socketstream sol_sock_ex3(vishost, visport);
+      str0 = "E_exact_re";
+      plotfield(sol_sock_ex0,pmesh,E_exact_re,str0);
+      str1 = "E_exact_im";
+      plotfield(sol_sock_ex1,pmesh,E_exact_im,str1);
+      str2 = "H_exact_re";
+      plotfield(sol_sock_ex2,pmesh,H_exact_re,str2);
+      str3 = "H_exact_im";
+      plotfield(sol_sock_ex3,pmesh,H_exact_im,str3);
+   }
+
+   MPI_Finalize();
+   return 0;
+}
+
+
+void maxwell_solution(const Vector &X, std::vector<complex<double>> &sol, 
+                                       std::vector<complex<double>> &curl,
+                                       std::vector<complex<double>> &curl2)
+{
+   double x = X(0), y = X(1), z = X(2);
+
+   complex<double> zi(0,1);
+   if (exact == 0)
+   {
+      sol[0] = y*(1.0-y)*z*(1.0-z) + zi * 2.0;
+      sol[1] = y*x*(1.0-x)*z*(1.0-z)+ zi * 2.0;
+      sol[2] = x*(1.0-x)*y*(1.0-y) + zi * 2.0;
+
+      curl[0] = (1.0-x)*x*(y*(2.0*z-3.0)+1.0);
+      curl[1] = 2.0*(1.0-y)*y*(x-z);
+      curl[2] = (z-1.0)*z*(y*(2*x-3)+1.0);
+
+      curl2[0] = (2.0*x-3.0)*(z-1.0)*z-2.0*y*y+2*y;
+      curl2[1] = -2.0*y*(x*x-x+(z-1.0)*z);
+      curl2[2] = 2*(x*(1.5-z)+x*x*(z-1.5)-y*y+y);
+   }
+   else
+   {
+      complex<double> alpha = zi * omega / sqrt(3);
+      sol[0] = exp(alpha*(x+y+z));
+      sol[1] = 0.0;
+      sol[2] = 0.0;
+
+      curl[0] = 0.0;
+      curl[1] = alpha * sol[0];
+      curl[2] = -alpha * sol[0];
+
+      curl2[0] = -2.0 * alpha * alpha * sol[0];
+      curl2[1] = alpha * alpha * sol[0];
+      curl2[2] = curl2[1];
+   }
+
+
+   // sol[0] = 1.0 + 2.0*zi;
+   // sol[1] = 1.0 + 2.0*zi;
+   // sol[2] = 1.0 + 2.0*zi;
+   // curl[0] = 0.0;
+   // curl[1] =0.0;
+   // curl[2] =0.0;
+   // curl2[0] =0.0;
+   // curl2[1] =0.0;
+   // curl2[2] =0.0;  
+
+}
+
+void E_exact_re(const Vector &x, Vector &E)
+{
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      E(i) = sol[i].real();
+   }
+}
+void H_exact_re(const Vector &x, Vector &H)
+{
+   complex<double> zi(0,1);
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   // H = i curlE / w 
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      H[i] = (zi * curl[i]/omega).real();
+   }
+}
+void E_exact_im(const Vector &x, Vector &E)
+{
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      E(i) = sol[i].imag();
+   }
+}
+void H_exact_im(const Vector &x, Vector &H)
+{
+   complex<double> zi(0,1);
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   // H = i curlE / w 
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      H[i] = (zi * curl[i]/omega).imag();
+   }
+}
+
+void f_exact_re(const Vector &x, Vector &f)
+{
+   f = 0.0;
+}
+void g_exact_re(const Vector &x, Vector &g)
+{
+   // J = i omega E - curl H 
+   // J = - i / omega (curl curl E - omega * omega E)
+   complex<double> zi(0,1);
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      g(i) = (-zi / omega *(curl2[i] - omega * omega * sol[i])).real();
+   }
+}
+void f_exact_im(const Vector &x, Vector &f)
+{
+   f = 0.0;
+}
+void g_exact_im(const Vector &x, Vector &g)
+{
+   // J = i omega E - curl H 
+   // J = - i / omega (curl curl E - omega * omega E)
+   complex<double> zi(0,1);
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      g(i) = (-zi / omega *(curl2[i] - omega * omega * sol[i])).imag();
+   }
+}
+
+
+void plotfield(socketstream & socket, ParMesh * pmesh, const ParGridFunction & pgf, string & title )
+{
+   int num_procs, myid;
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+   ostringstream oss;
+   oss << title;
+   socket << "parallel " << num_procs << " " << myid << "\n";
+   socket.precision(8);
+   socket << "solution\n" << *pmesh << pgf 
+          << "window_title '" << oss.str() << "'" << flush;
+}

examples/maxwell-solver/ParDST/DofMapsDST.cpp

@@ -0,0 +1,960 @@
+#include "DofMapsDST.hpp"
+
+double testcoeff(const Vector & x)
+{
+   return sin(3*M_PI*(x.Sum()));
+}
+
+int get_rank(int tdof, std::vector<int> & tdof_offsets)
+{
+   int size = tdof_offsets.size();
+   if (size == 1) { return 0; }
+   std::vector<int>::iterator up;
+   up=std::upper_bound(tdof_offsets.begin(), tdof_offsets.end(),tdof); // 
+   return std::distance(tdof_offsets.begin(),up)-1;
+}
+
+void ComputeTdofOffsets(const MPI_Comm & comm, const ParFiniteElementSpace * pfes, 
+                        std::vector<int> & tdof_offsets)
+{
+   int num_procs;
+   MPI_Comm_size(comm, &num_procs);
+   tdof_offsets.resize(num_procs);
+   int mytoffset = pfes->GetMyTDofOffset();
+   MPI_Allgather(&mytoffset,1,MPI_INT,&tdof_offsets[0],1,MPI_INT,comm);
+}
+
+void GetSubdomainijk(int ip, const Array<int> nxyz, Array<int> & ijk)
+{
+   ijk.SetSize(3);
+   ijk[2] = ip/(nxyz[0]*nxyz[1]);
+   ijk[1] = (ip-ijk[2]*nxyz[0]*nxyz[1])/nxyz[0];
+   ijk[0] = (ip-ijk[2]*nxyz[0]*nxyz[1])%nxyz[0];
+}
+void GetDirectionijk(int id, Array<int> & ijk)
+{
+   ijk.SetSize(3);
+   int n = 3;
+   ijk[2] = id/(n*n) - 1;
+   ijk[1] = (id-(ijk[2]+1)*n*n)/n - 1;
+   ijk[0] = (id-(ijk[2]+1)*n*n)%n - 1;
+}
+
+int GetSubdomainId(const Array<int> nxyz, Array<int> & ijk)
+{
+   int dim=ijk.Size();
+   int k = (dim==2)? 0 : ijk[2];
+   return k*nxyz[1]*nxyz[0] + ijk[1]*nxyz[0] + ijk[0];
+}
+
+int GetDirectionId(const Array<int> & ijk)
+{
+   int n = 3;
+   int dim = ijk.Size();
+   int k = (dim == 2) ? -1 : ijk[2];
+   return (k+1)*n*n + (ijk[1]+1)*n + ijk[0]+1; 
+}
+
+void DofMaps::Init()
+{
+   comm = pfes->GetComm();
+   MPI_Comm_size(comm, &num_procs);
+   MPI_Comm_rank(comm, &myid);
+
+   dim = pfes->GetParMesh()->Dimension();
+   ComputeTdofOffsets(comm, pfes, tdof_offsets);
+   myelemoffset = part->myelem_offset;
+   mytoffset = pfes->GetMyTDofOffset();
+   subdomain_rank = part->subdomain_rank;
+   nrsubdomains = part->nrsubdomains;
+   nxyz.SetSize(3);
+   for (int i = 0; i<3; i++) { nxyz[i] = part->nxyz[i]; }
+
+   //compute sign factors for tdofs
+   int lsize = pfes->GetVSize();
+   int tsize = pfes->GetTrueVSize();
+   tdof_sign.SetSize(tsize); 
+   for (int i = 0; i<lsize; i++)
+   {
+      int j = pfes->GetGlobalTDofNumber(i);
+      if (j<mytoffset || j>=mytoffset+tsize) continue;
+      tdof_sign[j-mytoffset] = pfes->GetDofSign(i);
+   }
+}
+
+DofMaps::DofMaps(ParFiniteElementSpace *pfes_, ParMeshPartition * part_, bool CompFlag_)
+: pfes(pfes_), part(part_), CompFlag(CompFlag_)
+{
+   Init();
+   Setup();
+}
+
+void DofMaps::Setup()
+{
+   // Setup the local FiniteElementSpaces
+   const FiniteElementCollection * fec = pfes->FEColl();
+   fes.SetSize(nrsubdomains);
+   for (int i = 0; i<nrsubdomains; i++)
+   {
+      fes[i] = nullptr; // initialize with null on all procs
+      if (myid == subdomain_rank[i])
+      {
+         fes[i] = new FiniteElementSpace(part->subdomain_mesh[i],fec);
+      }
+   }
+   // cout << "Computing Overlap Tdofs" << endl;
+   SubdomainToSubdomainMapsSetup();
+   // TestSubdomainToSubdomainMaps();
+
+   SubdomainToGlobalMapsSetup();
+   // TestSubdomainToGlobalMaps();
+}
+
+void DofMaps::SubdomainToSubdomainMapsSetup()
+{
+   ComputeOvlpElems();
+   ComputeOvlpTdofs();
+}
+
+void DofMaps::AddElementToOvlpLists(int l, int iel, 
+                            const Array<bool> & neg, const Array<bool> & pos)
+{
+   int kbeg = (dim == 2) ? 0 : -1;
+   int kend = (dim == 2) ? 0 :  1;
+   Array<int> dijk(3);
+   for (int k = kbeg; k<=kend; k++)
+   {
+      if (dim == 3)
+      {
+         if (k == -1 && !neg[2]) continue;   
+         if (k ==  1 && !pos[2]) continue;   
+      }   
+
+      for (int j = -1; j<=1; j++)
+      {
+         if (j== -1 && !neg[1]) continue;   
+         if (j==  1 && !pos[1]) continue;   
+         for (int i = -1; i<=1; i++)
+         {
+            // cases to skip 
+            if (i==-1 && !neg[0]) continue;   
+            if (i== 1 && !pos[0]) continue;   
+
+            if (i==0 && j==0 && k == 0) continue;
+            dijk[0] = i; dijk[1] = j; dijk[2] = (dim==2)?-1 : k;  
+            int DirId = GetDirectionId(dijk);
+            OvlpElems[l][DirId].Append(iel);
+         }
+      }
+   }
+}
+
+void DofMaps::ComputeOvlpElems()
+{
+   // first compute the element in the overlaps
+   OvlpElems.resize(nrsubdomains);
+   int nlayers = 2*part->OvlpNlayers; 
+   // loop through subdomains
+   for (int l = 0; l<nrsubdomains; l++)
+   {
+      if (myid == subdomain_rank[l])
+      {
+         Array<int> ijk;
+         GetSubdomainijk(l,nxyz,ijk);
+         Mesh * mesh = part->subdomain_mesh[l];
+         OvlpElems[l].resize(pow(3,dim)); 
+         Vector pmin, pmax;
+         mesh->GetBoundingBox(pmin,pmax);
+         double h = part->MeshSize;
+         // loop through the elements in the mesh and assign them to the 
+         // appropriate lists of overlaps
+         for (int iel=0; iel< mesh->GetNE(); iel++)
+         {
+            // Get element center
+            Vector center(dim);
+            int geom = mesh->GetElementBaseGeometry(iel);
+            ElementTransformation * tr = mesh->GetElementTransformation(iel);
+            tr->Transform(Geometries.GetCenter(geom),center);
+
+            Array<bool> pos(dim); pos = false;
+            Array<bool> neg(dim); neg = false;
+            // loop through dimensions   
+            for (int d=0;d<dim; d++)
+            {
+               if (ijk[d]>0 && center[d] < pmin[d]+h*nlayers)
+               {
+                  neg[d] = true;
+               }
+
+               if (ijk[d]<nxyz[d]-1 && center[d] > pmax[d]-h*nlayers) 
+               {
+                  pos[d] = true;
+               }
+            }
+            // Add the element to the appropriate lists
+            AddElementToOvlpLists(l,iel,neg,pos);        
+         }   
+      }
+   }
+}
+
+void DofMaps::ComputeOvlpTdofs()
+{
+   OvlpTDofs.resize(nrsubdomains);
+   int nrneighbors = pow(3,dim); // including its self
+
+   // loop through subdomains
+   for (int l = 0; l<nrsubdomains; l++)
+   {
+      if (myid != subdomain_rank[l]) continue;
+      int ntdofs = fes[l]->GetTrueVSize();
+      Array<int> tdof_marker(ntdofs); 
+      OvlpTDofs[l].resize(nrneighbors);
+      // loop through neighboring directions/neighbors
+      for (int d=0; d<nrneighbors; d++)
+      {
+         tdof_marker = 0;
+         Array<int> tdoflist;
+         // Get the direction
+         Array<int> dijk;
+         GetDirectionijk(l,dijk);
+         int nel = OvlpElems[l][d].Size();
+         Array<int>Elems = OvlpElems[l][d];
+         for (int iel = 0; iel<nel; ++iel)
+         {
+            int jel = Elems[iel];   
+            Array<int> ElemDofs;
+
+            fes[l]->GetElementDofs(jel,ElemDofs);
+            int ndof = ElemDofs.Size();
+            for (int i = 0; i<ndof; ++i)
+            {
+               int dof_ = ElemDofs[i];
+               int dof = (dof_ >= 0) ? dof_ : abs(dof_) - 1;
+               if (!tdof_marker[dof])
+               {
+                  tdoflist.Append(dof); // dofs of ip0 in ovlp
+                  tdof_marker[dof] = 1;
+               }
+            }
+         }
+         OvlpTDofs[l][d] = tdoflist; 
+         if (CompFlag)
+         {
+            for (int i=0; i<tdoflist.Size(); i++)
+            {
+               tdoflist[i] += fes[l]->GetTrueVSize();
+            }
+            OvlpTDofs[l][d].Append(tdoflist);
+         }
+      }
+   }
+}
+
+void DofMaps::PrintOvlpTdofs()
+{
+   int nrneighbors = pow(3,dim); // including its self
+   if (myid == 0)
+   {
+      for (int i = 0; i<nrsubdomains; i++)
+      {
+         if (myid != subdomain_rank[i]) continue;
+         Array<int> ijk;
+         GetSubdomainijk(i,nxyz,ijk);
+         cout << "subdomain = " ; ijk.Print();
+         cout << "myid = " << myid << endl;
+         cout << "ip   = " << i << endl;
+         for (int d = 0; d<nrneighbors; d++)
+         {
+            Array<int> dijk;
+            GetDirectionijk(d,dijk);
+            cout << "direction = " ; dijk.Print();
+
+            if (OvlpTDofs[i][d].Size())
+            {
+               cout << "OvlpTdofs = " ; 
+               OvlpTDofs[i][d].Print(cout,OvlpTDofs[i][d].Size() );
+            }
+         }
+      }
+   }
+}
+
+void DofMaps::TransferToNeighbors(const Array<int> & SubdomainIds, const Array<Vector *> & x,
+std::vector<std::vector<Vector * >> & OvlpSol)
+{
+   // 2D for now....
+   MFEM_VERIFY(SubdomainIds.Size() == x.Size(), "TransferToNeighbors: Size inconsistency");
+   int nrsendIds = SubdomainIds.Size();
+   int nrneighbors = pow(3,dim);
+   MPI_Request *recv_requests = new MPI_Request[nrsendIds*nrneighbors];
+   MPI_Request *send_requests = new MPI_Request[nrsendIds*nrneighbors];
+   MPI_Status  *recv_statuses = new MPI_Status[nrsendIds*nrneighbors];
+   MPI_Status  *send_statuses = new MPI_Status[nrsendIds*nrneighbors];
+   Array<Vector * > send_buffer(nrsendIds*nrneighbors);
+   Array<Vector * > recv_buffer(nrsendIds*nrneighbors);
+   int send_counter = 0;
+   int recv_counter = 0;
+   for (int is = 0; is<nrsendIds; is++)
+   {
+      int i0 = SubdomainIds[is];
+      Array<int> ijk;
+      GetSubdomainijk(i0,nxyz,ijk);
+      for (int d=0;d<nrneighbors; d++)
+      {
+         Array<int>directions;
+         GetDirectionijk(d,directions);
+
+         if (dim == 2 && directions[0] == 0 && directions[1] == 0) continue;
+         if (dim == 3 && directions[0] == 0 
+                      && directions[1] == 0 
+                      && directions[2] == 0) continue;
+         int i = ijk[0] + directions[0];
+         if (i<0 || i>=nxyz[0]) continue;
+         int j = ijk[1] + directions[1];
+         if (j<0 || j>=nxyz[1]) continue;
+         int k = (dim ==3 ) ? ijk[2] + directions[2] : 0;
+         if (k<0 || k>=nxyz[2]) continue;
+         Array<int>ijk1(3);
+         ijk1[0] = i;
+         ijk1[1] = j;
+         ijk1[2] = k;
+         int i1 = GetSubdomainId(nxyz,ijk1);
+         if (myid == subdomain_rank[i0])
+         {
+            Array<int> tdofs0 = OvlpTDofs[i0][d]; // map of dofs in the overlap
+            send_buffer[send_counter] = new Vector(tdofs0.Size());
+            x[is]->GetSubVector(tdofs0,*send_buffer[send_counter]);
+            // Destination rank
+            int dest = subdomain_rank[i1];
+            int tag = i0 * nrneighbors + d;
+
+            int count = tdofs0.Size();
+            MPI_Isend(send_buffer[send_counter]->GetData(),count,MPI_DOUBLE,dest,
+                      tag,comm,&send_requests[send_counter]);
+            send_counter++;
+
+         }
+         if (myid == subdomain_rank[i1])
+         {
+            Array<int> direction1(3); direction1 = -1;
+            for (int dd=0;dd<dim;dd++) 
+            { 
+               direction1[dd] = -directions[dd];
+            }
+            int d1 = GetDirectionId(direction1);
+
+            int count = OvlpTDofs[i1][d1].Size();
+            recv_buffer[recv_counter] = new Vector(count); 
+            int src = subdomain_rank[i0];
+            int tag = i0 * nrneighbors + d;
+            MPI_Irecv(recv_buffer[recv_counter]->GetData(), count,MPI_DOUBLE,src,
+                      tag,comm, &recv_requests[recv_counter]);
+            recv_counter++;
+         }
+      }   
+   }
+   MPI_Waitall(send_counter, send_requests, send_statuses);
+   MPI_Waitall(recv_counter, recv_requests, recv_statuses);
+
+   delete [] send_statuses;
+   delete [] send_requests;
+   delete [] recv_statuses;
+   delete [] recv_requests;
+
+   for (int i = 0; i<send_counter; i++)
+   {
+      delete send_buffer[i];
+   }
+   send_buffer.DeleteAll();
+
+
+   // Extract the transfered solutions
+   recv_counter = 0;
+   for (int is = 0; is<nrsendIds; is++)
+   {
+      int i0 = SubdomainIds[is];
+      Array<int> ijk;
+      GetSubdomainijk(i0,nxyz,ijk);
+      for (int d=0;d<nrneighbors; d++)
+      {
+         Array<int>directions;
+         GetDirectionijk(d,directions);
+         if (dim == 2 && directions[0] == 0 && directions[1] == 0) continue;
+         if (dim == 3 && directions[0] == 0 
+                      && directions[1] == 0 
+                      && directions[2] == 0) continue;
+         int i = ijk[0] + directions[0];
+         if (i<0 || i>=nxyz[0]) continue;
+         int j = ijk[1] + directions[1];
+         if (j<0 || j>=nxyz[1]) continue;
+         int k = (dim ==3 ) ? ijk[2] + directions[2] : 0;
+         if (k<0 || k>=nxyz[2]) continue;
+
+         Array<int>ijk1(3);
+         ijk1[0] = i;
+         ijk1[1] = j;
+         ijk1[2] = k;
+         int i1 = GetSubdomainId(nxyz,ijk1);
+         if (myid == subdomain_rank[i1])
+         {
+            Array<int> direction1(3); direction1 = -1;
+            for (int d=0;d<dim;d++) 
+            { 
+               direction1[d] = -directions[d];
+            }
+            int d1 = GetDirectionId(direction1);
+            Array<int> tdofs1 = OvlpTDofs[i1][d1];
+            if (!OvlpSol[i1][d1])
+            {
+               OvlpSol[i1][d1] = new Vector(2*fes[i1]->GetTrueVSize());
+            }
+            *OvlpSol[i1][d1] = 0.0;
+            OvlpSol[i1][d1]->SetSubVector(tdofs1,*recv_buffer[recv_counter]);
+            recv_counter++;
+         }
+      }   
+   }
+   for (int i = 0; i<recv_counter; i++)
+   {
+      delete recv_buffer[i];
+   }
+   recv_buffer.DeleteAll();
+}
+
+void DofMaps::TestSubdomainToSubdomainMaps()
+{
+      // testing inter-subdomain communication
+   FunctionCoefficient c1(testcoeff);
+   int nrsub = nrsubdomains;
+   Array<int> subdomain_ids(nrsub);
+   Array<Vector*> x(nrsub);
+   for (int i = 0; i<nrsub; i++)
+   {
+      x[i] = nullptr;
+      subdomain_ids[i] = i;
+      if (fes[i])
+      {
+         ComplexGridFunction gf(fes[i]);
+         gf = 0.0;
+         gf.ProjectCoefficient(c1,c1);
+         x[i] = new Vector(2*fes[i]->GetTrueVSize());
+         *x[i] = gf;
+      }
+   }
+
+   std::vector<std::vector<Vector * >> OvlpSol;
+
+   OvlpSol.resize(nrsubdomains);
+   int nrneighbors = pow(3,dim);
+   for (int ip = 0; ip<nrsubdomains; ip++)
+   {
+      if (myid == subdomain_rank[ip])
+      {
+         OvlpSol[ip].resize(nrneighbors);
+      }
+   }
+
+
+   TransferToNeighbors(subdomain_ids,x,OvlpSol);
+
+   string keys = "keys amrRljc\n";
+   for (int i0 = 0 ; i0< nrsubdomains; i0++)
+   {
+      if (fes[i0])
+      {
+         ComplexGridFunction gf0(fes[i0]);
+         for (int d = 0; d<nrneighbors; d++)
+         {
+            if(OvlpSol[i0][d])
+            {
+               Array<int>dijk;
+               GetDirectionijk(d,dijk);
+               Array<int>ijk;
+               GetSubdomainijk(i0,nxyz,ijk);
+               ostringstream oss;
+               oss << "myid: " << myid 
+                     << ", subdomain: (" << ijk[0] << "," << ijk[1] <<")"
+                     << ", direction: (" << dijk[0] << "," << dijk[1] <<")";
+
+               gf0 = 0.0;
+               gf0.real().SetVector(*OvlpSol[i0][d],0);
+               gf0.imag().SetVector(*OvlpSol[i0][d],fes[i0]->GetTrueVSize());
+               char vishost[] = "localhost";
+               int  visport   = 19916;
+               socketstream sol_sock(vishost, visport);
+               sol_sock.precision(8);
+               sol_sock << "solution\n" << *(part->subdomain_mesh[i0]) << gf0.real() 
+               << keys
+               << "window_title '" << oss.str() << "'" << flush;
+            }
+         }
+      }
+   }
+   for (int i = 0; i<nrsub; i++)
+   {
+      delete x[i];
+   }
+}
+
+void DofMaps::SubdomainToGlobalMapsSetup()
+{
+   // workspace for MPI_AlltoAll
+   send_count.SetSize(num_procs);  send_count = 0;
+   send_displ.SetSize(num_procs);  send_displ = 0; 
+   recv_count.SetSize(num_procs);  recv_count = 0;
+   recv_displ.SetSize(num_procs);  recv_displ = 0;
+
+   // 1. Communicate to the subdomain rank the list of tdofs 
+   // a. Compute send count
+   for (int ip = 0; ip<nrsubdomains; ++ip)
+   {
+      // avoid any communication if on subdomain rank
+      int nel = part->local_element_map[ip].Size();
+
+      for (int iel = 0; iel<nel; iel++)
+      {
+         int elem_idx = part->local_element_map[ip][iel] - myelemoffset;
+         // int ndofs = local_tdofs[ip].Size();
+         int ndofs = pfes->GetFE(elem_idx)->GetDof();
+
+         send_count[subdomain_rank[ip]] += 2 + ndofs; 
+      }
+   }
+   // b. Compute receive count
+   MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,comm);
+   for (int k=0; k<num_procs-1; k++)
+   {
+      send_displ[k+1] = send_displ[k] + send_count[k];
+      recv_displ[k+1] = recv_displ[k] + recv_count[k];
+   }
+   sbuff_size = send_count.Sum();
+   rbuff_size = recv_count.Sum();
+   // c. Allocate and fill the send buffer
+   Array<int> sendbuf(sbuff_size);  sendbuf = 0;
+   Array<int> soffs(num_procs); soffs = 0;
+   for (int ip = 0; ip<nrsubdomains; ++ip)
+   {
+      int nel = part->local_element_map[ip].Size();
+      for (int iel = 0; iel<nel; iel++)
+      {
+         int elem_idx = part->local_element_map[ip][iel] - myelemoffset;
+         Array<int>ElemDofs;
+         pfes->GetElementDofs(elem_idx,ElemDofs);
+         int ndofs = ElemDofs.Size();
+
+         int j = send_displ[subdomain_rank[ip]] + soffs[subdomain_rank[ip]];
+         sendbuf[j] = ip;
+         sendbuf[j+1] = ndofs;
+
+         for (int k = 0; k < ndofs ; ++k)
+         {
+            int edof_ = ElemDofs[k];
+            int edof = (edof_ >= 0) ? edof_ : abs(edof_) - 1;
+            sendbuf[j+2+k] = pfes->GetGlobalTDofNumber(edof);
+         }
+         soffs[subdomain_rank[ip]] +=  2 + ndofs;
+      }
+   }
+
+   // d. Communication
+   Array<int> recvbuf(rbuff_size);
+   MPI_Alltoallv(sendbuf, send_count, send_displ, MPI_INT, recvbuf,
+                 recv_count, recv_displ, MPI_INT, comm);
+
+   // 3. Extract from recv_buffer
+   std::vector<Array<int>> global_tdofs(nrsubdomains);
+   int k=0;
+   while (k<rbuff_size)
+   {
+      int ip = recvbuf[k++]; 
+      int ndofs = recvbuf[k++];
+      for (int i = 0; i < ndofs; ++i)
+      {
+         global_tdofs[ip].Append(recvbuf[i+k]); 
+      }
+      k += ndofs;
+   }
+
+   SubdomainGTrueDofs.resize(nrsubdomains);
+   // 4. Construct SubdomainTdof to Global mesh tdof maps
+   for (int ip=0; ip<nrsubdomains; ++ip)
+   {
+      if (myid != subdomain_rank[ip]) continue;
+      int nrdof = fes[ip]->GetTrueVSize();
+
+      SubdomainGTrueDofs[ip].SetSize(nrdof);
+      int nel = part->element_map[ip].Size();
+      int k = 0;
+      for (int iel = 0; iel<nel; ++iel)
+      {
+         Array<int> elem_dofs;
+         fes[ip]->GetElementDofs(iel,elem_dofs);
+         int ndof = elem_dofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int edof_ = elem_dofs[i];
+            int edof = (edof_ >= 0) ? edof_ : abs(edof_) - 1;
+            // rearranging dofs from serial fespace to pfes ordering
+            SubdomainGTrueDofs[ip][edof] = global_tdofs[ip][k++];
+         }
+      }
+   }
+
+   // 5. Communicate SubdomainGTrueDofs to participating ranks 
+   send_count = 0;  send_displ = 0;
+   recv_count = 0;  recv_displ = 0;
+
+   for (int ip = 0; ip < nrsubdomains; ++ip)
+   {
+      if (myid != subdomain_rank[ip]) continue;
+      int ndofs = SubdomainGTrueDofs[ip].Size();
+      for (int i = 0; i<ndofs; ++i)
+      {
+         int tdof = SubdomainGTrueDofs[ip][i];
+         int rank = get_rank(tdof,tdof_offsets);
+         if (rank == subdomain_rank[ip]) continue; // <--------------
+         send_count[rank] += 2; // 1 for the dof and 1 for the ip that goes to 
+      }
+   }
+
+   // communicate so that recv_count is constructed
+   MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,comm);
+   //
+   for (int k=0; k<num_procs-1; k++)
+   {
+      send_displ[k+1] = send_displ[k] + send_count[k];
+      recv_displ[k+1] = recv_displ[k] + recv_count[k];
+   }
+   sbuff_size = send_count.Sum();
+   rbuff_size = recv_count.Sum();
+
+   sendbuf.SetSize(sbuff_size);  
+   sendbuf = 0; soffs = 0;
+
+   for (int ip = 0; ip < nrsubdomains; ip++)
+   {
+      if (myid != subdomain_rank[ip]) continue;
+      int ndofs = SubdomainGTrueDofs[ip].Size();
+      // loop through dofs
+      for (int i = 0; i<ndofs; ++i)
+      {
+         int tdof = SubdomainGTrueDofs[ip][i];
+         int irank = get_rank(tdof,tdof_offsets);
+         if (irank == subdomain_rank[ip]) continue; // <--------------
+         int j = send_displ[irank] + soffs[irank];
+         sendbuf[j] = ip;
+         sendbuf[j+1] = SubdomainGTrueDofs[ip][i];
+         soffs[irank] +=  2 ;
+      }
+   }
+
+   recvbuf.SetSize(rbuff_size);
+   MPI_Alltoallv(sendbuf, send_count, send_displ, MPI_INT, recvbuf,
+                 recv_count, recv_displ, MPI_INT, comm);
+
+   // List of tdofs owned by the processor for subdomains not owned
+   SubdomainLTrueDofs.resize(nrsubdomains); 
+   for (int k=0; k<rbuff_size/2; k++)
+   {
+      int ip = recvbuf[2*k];
+      int tdof = recvbuf[2*k+1];
+      SubdomainLTrueDofs[ip].Append(tdof);
+   }    
+}
+
+// Restriction of global residual to subdomain residuals
+void DofMaps::GlobalToSubdomains(const Vector & y, Array<Vector*> & x)
+{
+   send_count = 0; send_displ = 0;
+   recv_count = 0; recv_displ = 0;
+
+   // Compute send_counts
+   int m = (CompFlag) ? 2 : 1 ;
+   for (int ip = 0; ip < nrsubdomains; ip++)
+   {
+      if (myid == subdomain_rank[ip]) continue;  // <---------------
+      int ndofs = SubdomainLTrueDofs[ip].Size();
+      send_count[subdomain_rank[ip]] += m * ndofs;
+   }
+
+    // communicate so that recv_count is constructed
+   MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,comm);
+
+   for (int k=0; k<num_procs-1; k++)
+   {
+      send_displ[k+1] = send_displ[k] + send_count[k];
+      recv_displ[k+1] = recv_displ[k] + recv_count[k];
+   }
+   sbuff_size = send_count.Sum();
+   rbuff_size = recv_count.Sum();
+
+   Array<double> sendbuf(sbuff_size); sendbuf = 0;
+   Array<int> soffs(num_procs); soffs = 0;
+
+   for (int ip = 0; ip < nrsubdomains; ip++)
+   {
+      if (myid == subdomain_rank[ip]) continue;  // <---------------
+      int ndofs = SubdomainLTrueDofs[ip].Size();
+      for (int i = 0; i<ndofs; i++)
+      {
+         int tdof = SubdomainLTrueDofs[ip][i];
+         int j = send_displ[subdomain_rank[ip]] + soffs[subdomain_rank[ip]];
+         soffs[subdomain_rank[ip]] +=m;
+         int k = tdof - mytoffset;
+         // sendbuf[j] = y[k];
+         sendbuf[j] = tdof_sign[k]*y[k];
+         if (CompFlag)
+         {  // if complex valued 
+            int tsize = pfes->GetTrueVSize();
+            // sendbuf[j+1] = y[k+tsize];
+            sendbuf[j+1] = tdof_sign[k]*y[k+tsize];
+         }
+      }
+   }
+
+   // communication
+   Array<double> recvbuf(rbuff_size);
+   MPI_Alltoallv(sendbuf, send_count, send_displ, MPI_DOUBLE, recvbuf,
+                 recv_count, recv_displ, MPI_DOUBLE, comm);
+   Array<int> roffs(num_procs);
+   roffs = 0;
+   // Now each process will construct the res vector
+   x.SetSize(nrsubdomains);
+   for (int ip = 0; ip < nrsubdomains; ip++)
+   {
+      if (myid != subdomain_rank[ip]) continue;
+      int ndof = SubdomainGTrueDofs[ip].Size();
+      if (!x[ip]) x[ip] = new Vector(m*ndof); 
+      *x[ip] = 0.0;
+      // extract the data from receiv buffer
+      for (int i=0; i<ndof; i++)
+      {
+         // pick up the tdof and find its rank
+         int tdof = SubdomainGTrueDofs[ip][i];
+         int tdof_rank = get_rank(tdof,tdof_offsets);
+         if (tdof_rank != subdomain_rank[ip])  // <---------------
+         {
+            int k = recv_displ[tdof_rank] + roffs[tdof_rank];
+            roffs[tdof_rank] += m;
+            (*x[ip])[i] = recvbuf[k];
+            if (CompFlag)
+            {  
+               (*x[ip])[i+ndof] = recvbuf[k+1];
+            }
+         }
+         else
+         {
+            int k = tdof - mytoffset;
+            // (*x[ip])[i] = y[k];
+            (*x[ip])[i] = tdof_sign[k]*y[k];
+            if (CompFlag)
+            {  
+               int gtsize = pfes->GetTrueVSize();
+               (*x[ip])[i+ndof] = tdof_sign[k]*y[k+gtsize];
+            }
+         }
+      }
+   }
+}
+
+// Prolongation of subdomain solutions to the global solution
+void DofMaps::SubdomainsToGlobal(const Array<Vector*> & x, Vector & y)
+{
+   send_count = 0; send_displ = 0;
+   recv_count = 0; recv_displ = 0;
+
+   // Compute send_counts
+   int m = (CompFlag) ? 2 : 1 ;
+   for (int ip = 0; ip < nrsubdomains; ip++)
+   {
+      if (myid != subdomain_rank[ip]) continue;
+      int ndofs = SubdomainGTrueDofs[ip].Size();
+      for (int i=0; i<ndofs; i++)
+      {
+         // pick up the tdof and find its rank
+         int tdof = SubdomainGTrueDofs[ip][i];
+         int tdof_rank = get_rank(tdof,tdof_offsets);
+         if (tdof_rank == subdomain_rank[ip]) continue;
+         send_count[tdof_rank] +=m;
+      }
+   }
+
+   MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,comm);
+
+   for (int k=0; k<num_procs-1; k++)
+   {
+      send_displ[k+1] = send_displ[k] + send_count[k];
+      recv_displ[k+1] = recv_displ[k] + recv_count[k];
+   }
+   sbuff_size = send_count.Sum();
+   rbuff_size = recv_count.Sum();
+
+   Array<double> sendbuf(sbuff_size); sendbuf = 0;
+   Array<int> soffs(num_procs); soffs = 0;
+
+   for (int ip = 0; ip < nrsubdomains; ip++)
+   {
+      if (myid != subdomain_rank[ip]) continue;
+      int ndofs = SubdomainGTrueDofs[ip].Size();
+      // loop through dofs
+      for (int i=0; i<ndofs; i++)
+      {
+         //  pick up the dof and find its tdof_rank
+         int tdof = SubdomainGTrueDofs[ip][i];
+         int tdof_rank = get_rank(tdof,tdof_offsets);
+         // offset
+         if (tdof_rank == subdomain_rank[ip]) continue;
+         int k = send_displ[tdof_rank] + soffs[tdof_rank];
+         soffs[tdof_rank] +=m;
+         sendbuf[k] = (*x[ip])[i];
+         if (CompFlag)
+         {
+            sendbuf[k+1] = (*x[ip])[i+ndofs];
+         }
+      }
+   }
+
+   Array<double> recvbuf(rbuff_size);
+   Array<int> roffs(num_procs); roffs = 0;
+   MPI_Alltoallv(sendbuf, send_count, send_displ, MPI_DOUBLE, recvbuf,
+                 recv_count, recv_displ, MPI_DOUBLE, comm);
+
+   for (int ip = 0; ip < nrsubdomains; ip++)
+   {
+      if (myid == subdomain_rank[ip]) 
+      {
+         int ndofs = SubdomainGTrueDofs[ip].Size();
+         for (int i = 0; i<ndofs; i++)
+         {
+            int tdof = SubdomainGTrueDofs[ip][i];
+            int k = tdof - mytoffset;
+            if (k<0 || k>=pfes->GetTrueVSize()) continue;
+            y[k] += tdof_sign[k] * (*x[ip])[i];
+            if (CompFlag)
+            {  
+               int gtsize = pfes->GetTrueVSize();
+               y[k+gtsize] += tdof_sign[k]*(*x[ip])[i+ndofs];
+            }
+         }
+      }
+      else
+      {
+         int ndofs = SubdomainLTrueDofs[ip].Size();
+         for (int i = 0; i<ndofs; i++)
+         {
+            int tdof = SubdomainLTrueDofs[ip][i];
+            int k = tdof - mytoffset;
+            int j = recv_displ[subdomain_rank[ip]] + roffs[subdomain_rank[ip]];
+            roffs[subdomain_rank[ip]] +=m;
+            y[k] += tdof_sign[k] * recvbuf[j];
+            if (CompFlag)
+            {
+               int tsize = pfes->GetTrueVSize();
+               y[k+tsize] += tdof_sign[k]*recvbuf[j+1];
+            }
+         }
+      }
+   }              
+}
+
+void DofMaps::TestSubdomainToGlobalMaps()
+{
+   cout << "Testing Subdomain To Global Maps" << endl;
+   FunctionCoefficient c1(testcoeff);
+   Array<Vector*> x(nrsubdomains);
+   Vector y(pfes->GetTrueVSize()); y = 0.0;
+   for (int i = 0 ; i<nrsubdomains; i++)
+   {
+      if (myid != subdomain_rank[i]) continue;
+      x[i] = new Vector(fes[i]->GetTrueVSize());
+      GridFunction gf(fes[i]);
+      gf = 0.0;
+
+      if (i==3) gf.ProjectCoefficient(c1);
+      *x[i] = gf;
+   }
+
+   SubdomainsToGlobal(x,y);
+
+   // cout << "1: myid = " <<  myid << ", y = "; y.Print();
+
+   string keys = (dim==2) ? "keys amrRljc\n": "keys m\n";
+   ParGridFunction pgf(pfes);
+
+   const Operator &P = *pfes->GetProlongationMatrix();
+   P.Mult(y, pgf);
+   
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock(vishost, visport);
+   sol_sock.precision(8);
+   sol_sock << "parallel " << num_procs << " " << myid << "\n"
+               << "solution\n" << *pfes->GetParMesh() << pgf 
+               << keys << flush;
+
+   ParGridFunction pgf1(pfes);
+   pgf1.ProjectCoefficient(c1);
+   Vector y1(pfes->GetTrueVSize());
+   const SparseMatrix * R = pfes->GetRestrictionMatrix();
+
+   R->Mult(pgf1,y1);
+   // P.MultTranspose(pgf1,y1);
+   Array<Vector*> x1;
+   GlobalToSubdomains(y1,x1);
+
+
+   // for (int i = 0 ; i<nrsubdomains; i++)
+   // {
+   //    if (myid != subdomain_rank[i]) continue;
+   //    ostringstream mesh_name;
+   //    mesh_name << "output/mesh." << setfill('0') << setw(6) << i;
+   //    ofstream mesh_ofs(mesh_name.str().c_str());
+   //    mesh_ofs.precision(8);
+   //    fes[i]->GetMesh()->Print(mesh_ofs);
+   //    GridFunction gf(fes[i]);
+   //    gf = x1[i];
+   //    ostringstream gf_name;
+   //    gf_name << "output/gf." << setfill('0') << setw(6) << i;
+   //    ofstream gf_ofs(gf_name.str().c_str());
+   //    gf_ofs.precision(8);
+   //    gf.Save(gf_ofs);
+   // }
+
+
+
+   int nrsub = nrsubdomains;
+   for (int i = 0 ; i<nrsub; i++)
+   {
+      if (myid == subdomain_rank[i])
+      {
+         socketstream sol_sock1(vishost, visport);
+         sol_sock1.precision(8);
+         sol_sock1 << "parallel " << nrsub << " " << i << "\n";
+         GridFunction gf(fes[i]);
+         GridFunction gf1(fes[i]);
+         gf1.ProjectCoefficient(c1);
+         gf = *x1[i];
+         gf1-=gf;
+         cout << "ip, Diff norm = " <<i<<", " << gf1.Norml2() << endl;
+         sol_sock1 << "solution\n" << *fes[i]->GetMesh() << gf
+                   << keys << flush;
+      }
+      MPI_Barrier(MPI_COMM_WORLD);
+   }
+
+   socketstream gf_sock(vishost, visport);
+   gf_sock.precision(8);
+   gf_sock << "parallel " << num_procs << " " << myid << "\n"
+           << "solution\n" << *pfes->GetParMesh() << pgf1 
+           << keys << flush;
+}
+
+
+DofMaps::~DofMaps()
+{
+   for (int i = 0; i<nrsubdomains; i++)
+   {
+      delete fes[i];
+   }
+}

examples/maxwell-solver/ParDST/DofMapsDST.hpp

@@ -0,0 +1,110 @@
+#pragma once
+#include "../common/Utilities.hpp"
+#include "../common/PML.hpp"
+#include "../DST/DST.hpp"
+using namespace std;
+using namespace mfem;
+
+double testcoeff(const Vector & x);
+int get_rank(int tdof, std::vector<int> & tdof_offsets);
+
+
+void ComputeTdofOffsets(const MPI_Comm & comm, const ParFiniteElementSpace * pfes, 
+                        std::vector<int> & tdof_offsets);
+
+void GetSubdomainijk(int ip, const Array<int> nxyz, Array<int> & ijk);
+void GetDirectionijk(int id, Array<int> & ijk);
+int GetSubdomainId(const Array<int> nxyz, Array<int> & ijk);
+int GetDirectionId(const Array<int> & ijk);
+
+
+// class handling two types of dof maps
+// 1. Subdomain truedofs ---> Global truedofs
+// 2. Subdomain truedofs ---> Neighbor truedofs 
+class DofMaps
+{
+private:
+   // The FE space of the problem (H1/Hcurl)
+   ParFiniteElementSpace *pfes = nullptr;
+
+   // The given partition of the parmesh
+   ParMeshPartition *part = nullptr;
+   // partition in x-y-z
+   Array<int> nxyz;
+
+   // MPI parameters
+   MPI_Comm comm = MPI_COMM_WORLD;
+   int num_procs, myid;
+
+   // true dof offset and element offset of the processor
+   vector<int> tdof_offsets;
+   int mytoffset;
+   int myelemoffset;
+   
+   int dim;
+   // Total number of subdomains
+   int nrsubdomains;
+   
+   // Array specifying the subdomain rank
+   Array<int> subdomain_rank;
+
+   // Complex flag
+   bool CompFlag;
+
+   // sign factors
+   Array<int> tdof_sign;
+   // Initializing mpi and helper parameters
+   void Init();
+
+   // 1. Setting up the subdomains FE spaces
+   // 2. Setting up the subdomains-to-subdomains maps
+   // 3. Setting up the subdomain-to-global maps
+   void Setup();
+
+   // -----------------------------------------------
+   //           Subdomain to Subdomain maps
+   // -----------------------------------------------
+   std::vector<std::vector<Array<int>>> OvlpElems;
+   void AddElementToOvlpLists(int l, int iel, 
+                            const Array<bool> & neg, 
+                            const Array<bool> & pos);
+   std::vector<std::vector<Array<int>>> OvlpTDofs;
+   void SubdomainToSubdomainMapsSetup();
+   void ComputeOvlpElems();
+   void ComputeOvlpTdofs();
+   void PrintOvlpTdofs();
+
+   // -----------------------------------------------
+   //           Subdomain to Global maps
+   // -----------------------------------------------
+   std::vector<Array<int>> SubdomainGTrueDofs; // Subdomain Tdofs to Global Tdofs
+   std::vector<Array<int>> SubdomainLTrueDofs; // Subdomain Tdofs to Local (on rank) Tdofs
+
+   Array<int> send_count, send_displ;  
+   Array<int> recv_count, recv_displ;  
+   int sbuff_size = 0;
+   int rbuff_size = 0;
+   void SubdomainToGlobalMapsSetup();
+
+   // Testing
+   void TestSubdomainToGlobalMaps();
+   void TestSubdomainToSubdomainMaps();
+
+public:
+   // constructor
+
+   // FiniteElementSpaces of the subdomains
+   Array<FiniteElementSpace *> fes;
+
+   DofMaps(ParFiniteElementSpace *fespace_, ParMeshPartition * part_, bool CompFlag_ = false);
+   ~DofMaps();
+   // Transfering from subdomains SubdomainIds to all their neighbors
+   void TransferToNeighbors(const Array<int> & SubdomainIds, const Array<Vector *> & x, 
+   std::vector<std::vector<Vector * >> & OvlpSol);
+
+   // Prolongation of subdomain solutions to the global solution
+   void SubdomainsToGlobal(const Array<Vector*> & x, Vector & y);
+   // Restriction of global residual to subdomain residuals
+   // bool comp: true for complex valued problems
+   void GlobalToSubdomains(const Vector & y, Array<Vector*> & x);
+};

examples/maxwell-solver/ParDST/ParDST.cpp

@@ -0,0 +1,941 @@
+//Parallel Diagonal Source Transfer Preconditioner
+
+#include "ParDST.hpp"
+
+ParDST::ParDST(ParSesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+         double omega_, Coefficient * Q_,  
+         int nrlayers_ , int nx_, int ny_, int nz_, 
+         BCType bc_type_, Coefficient * LossCoeff_)
+   : Solver(2*bf_->ParFESpace()->GetTrueVSize(), 2*bf_->ParFESpace()->GetTrueVSize()), 
+     bf(bf_), Pmllength(Pmllength_), omega(omega_), 
+     Q(Q_), nrlayers(nrlayers_), bc_type(bc_type_), LossCoeff(LossCoeff_)
+{
+   nx = nx_; ny = ny_; nz = nz_;
+   Init();
+}
+ParDST::ParDST(ParSesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+         double omega_, MatrixCoefficient * MQ_,  
+         int nrlayers_ , int nx_, int ny_, int nz_, 
+         BCType bc_type_, Coefficient * LossCoeff_)
+   : Solver(2*bf_->ParFESpace()->GetTrueVSize(), 2*bf_->ParFESpace()->GetTrueVSize()), 
+     bf(bf_), Pmllength(Pmllength_), omega(omega_), 
+     MQ(MQ_), nrlayers(nrlayers_), bc_type(bc_type_), LossCoeff(LossCoeff_)
+{
+   nx = nx_; ny = ny_; nz = nz_;
+   Init();
+}
+
+void ParDST::Init()
+{
+   pfes = bf->ParFESpace();
+   fec = pfes->FEColl();
+ 
+   comm = pfes->GetComm();
+   MPI_Comm_size(comm, &num_procs);
+   MPI_Comm_rank(comm, &myid);
+
+   //1. Indentify problem ... Helmholtz or Maxwell
+   prob_kind = fec->GetContType();
+   if (myid == 0)
+   {
+      cout << " 1. Indentify problem to be solved ... " << endl;
+      if (prob_kind == 0) cout << "    Helmholtz" << endl;
+      if (prob_kind == 1) cout << "    Maxwell" << endl; 
+   }
+
+   //2. Create the parallel mesh partition
+   pmesh = pfes->GetParMesh();
+   dim = pmesh->Dimension();
+   if (myid == 0)
+   {
+      cout << "\n 2. Generating ParMesh partitioning ... " << endl;
+   }
+   ovlpnrlayers = nrlayers+1;
+   part = new ParMeshPartition(pmesh,nx,ny,nz,ovlpnrlayers);
+   nxyz.SetSize(3);
+   nxyz[0] = nx = part->nxyz[0]; 
+   nxyz[1] = ny = part->nxyz[1];  
+   nxyz[2] = nz = part->nxyz[2];
+
+   nrsubdomains = part->nrsubdomains;
+   SubdomainRank = part->subdomain_rank;
+
+   for (int ip = 0; ip<nrsubdomains; ip++)
+   {
+      if (myid == SubdomainRank[ip])
+      {
+         RankSubdomains.Append(ip);
+      }
+   }
+
+   cout << " myid: " << myid 
+        << ", nrsubdomains: " << RankSubdomains.Size() << endl;
+
+   MPI_Barrier(comm);
+
+   if (myid == 0)
+   {
+      cout << "    Done ! " << endl;
+   }
+   //3. Setup info for sweeps
+   if (myid == 0)
+   {
+      cout << "\n 3. Computing sweeps info ..." << endl; 
+   }
+   sweeps = new Sweep(dim);
+   if (myid == 0)
+   {
+      cout << "    Done ! " << endl;
+   }
+   //4. Create LocalToGlobal maps 
+   //   (local GridFunctions/Vector to Global ParGridFunction/Vector) 
+   if (myid == 0)
+   {
+      cout << "\n 4. Computing true dofs maps ..." << endl; 
+   }
+
+   // if (myid == SubdomainRank[0])
+   // {
+   //    cout << "myid = " << myid << endl;
+   //    char vishost[] = "localhost";
+   //    int visport = 19916;
+   //    socketstream mesh_sock1(vishost, visport);
+   //    mesh_sock1.precision(8);
+   //    mesh_sock1 << "mesh\n"
+   //               << *part->subdomain_mesh[0] << "window_title 'Subdomain'" << flush;
+   //    part->subdomain_mesh[0]->Print();
+   
+   // }
+   bool comp = true;
+
+   dmaps = new DofMaps(pfes,part, comp);
+   
+   if (myid == 0)
+   {
+      cout << "    Done ! " << endl;
+   }
+   // 4. Setting up the local problems 
+   if (myid == 0)
+   {
+      cout << "\n 5. Setting up the subdomain problems ..." << endl; 
+   }
+
+   SetupSubdomainProblems();
+   
+   if (myid == 0)
+   {
+      cout << "    Done ! " << endl;
+   }
+   if (myid == 0)
+   {
+      cout << "\n 6. Mark subdomain overlap truedofs ..." << endl; 
+   }
+   MarkSubdomainOverlapDofs(comp);
+   if (myid == 0)
+   {
+      cout << "    Done ! " << endl;
+   }
+}
+
+void ParDST::Mult(const Vector &r, Vector &z) const
+{
+   // Initialize transfered residuals to 0.0;
+   for (int ip=0; ip<nrsubdomains; ip++)
+   {
+      if (myid != SubdomainRank[ip]) continue;
+      for (int i=0;i<sweeps->nsweeps; i++)
+      {
+         *f_transf[ip][i] = 0.0;
+      }
+   }
+
+   // restrict given residual to subdomains
+   dmaps->GlobalToSubdomains(r,f_orig);
+
+   for (int ip=0; ip<nrsubdomains; ip++)
+   {
+      if (myid != SubdomainRank[ip]) continue;
+      Array<int> ijk(3);
+      GetSubdomainijk(ip,nxyz,ijk);
+      Array2D<int> direct(dim,2); direct = 0;
+      for (int d=0;d<dim; d++)
+      {
+         if (ijk[d] > 0) direct[d][0] = 1; 
+         if (ijk[d] < part->nxyz[d]-1) direct[d][1] = 1; 
+      }
+      GetChiRes(*f_orig[ip],ip,direct);
+   }
+
+   z = 0.0; 
+   int nsweeps = sweeps->nsweeps;
+   // 1. Loop through sweeps
+   if (dim == 3 && nz == 1) { nsweeps = 4; } // x-y partition only;
+   for (int l=0; l<nsweeps; l++)
+   {  
+      // cout << "sweep = " << l << endl;
+      int nsteps = GetSweepNumSteps(l);
+      // 2. loop through diagonals/steps of each sweep   
+      for (int s = 0; s<nsteps; s++)
+      {
+         // cout << "step = " << s << endl;
+         Array2D<int> subdomains;
+         GetStepSubdomains(l,s,subdomains);
+         // cout << "subdomains = " << endl;
+         // subdomains.Print(cout, subdomains.NumCols());
+         // cin.get();
+
+         int nsubdomains = subdomains.NumRows();
+
+         // 3. Loop through the subdomains on the diagonal
+         Array<int> subdomain_ids;
+         for (int sb=0; sb < nsubdomains; sb++)
+         {
+            Array<int> ijk(dim); ijk = 0;
+            for (int d=0; d<dim; d++) ijk[d] = subdomains[sb][d]; 
+            int ip = GetSubdomainId(nxyz,ijk);
+            subdomain_ids.Append(ip);
+            if (myid != SubdomainRank[ip]) continue;
+
+            int n = dmaps->fes[ip]->GetTrueVSize();
+            Vector res_local(2*n); res_local = 0.0;
+
+            if (l==0)  { res_local += *f_orig[ip]; }
+            res_local += *f_transf[ip][l];
+            if (res_local.Norml2() < 1e-12) 
+            {
+               *subdomain_sol[ip] = 0.0;
+               continue;
+            }
+            
+            // char vishost[] = "localhost";
+            // int visport = 19916;
+
+            // socketstream res_sock(vishost, visport);
+            // PlotLocal(res_local,res_sock,ip);
+
+            PmlMatInv[ip]->Mult(res_local, *subdomain_sol[ip]);
+            // GetSubdomainijk(ip,nxyz,ijk);
+            // Array2D<int> direct(dim,2); direct = 0;
+            // for (int d=0;d<dim; d++)
+            // {
+            //    if (ijk[d] > 0) direct[d][0] = 1; 
+            //    if (ijk[d] < part->nxyz[d]-1) direct[d][1] = 1; 
+            // }
+            // cout << "direct = " ; direct.Print();
+            // GetChiRes(*subdomain_sol[ip],ip,direct);
+
+            // socketstream sol_sock1(vishost, visport);
+            // PlotLocal(*subdomain_sol[ip],sol_sock1,ip);
+            // cout << "ip = " << ip << endl;
+            // cin.get();
+         }
+         // 4. Transfer solutions to neighbors so that the subdomain
+         // residuals are updated
+         TransferSources(l,subdomain_ids);
+      }
+      // 5. Update the global solution 
+      dmaps->SubdomainsToGlobal(subdomain_sol,z);
+      // char vishost[] = "localhost";
+      // int visport = 19916;
+      // socketstream sol_sock1(vishost, visport);
+      // PlotGlobal(z,sol_sock1);
+      // cin.get();
+   }
+   
+}
+
+void ParDST::SetupSubdomainProblems()
+{
+   sqf.SetSize(nrsubdomains);
+   Optr.SetSize(nrsubdomains);
+   PmlMat.SetSize(nrsubdomains);
+   PmlMatInv.SetSize(nrsubdomains);
+   f_orig.SetSize(nrsubdomains);
+   f_transf.resize(nrsubdomains);
+   subdomain_sol.SetSize(nrsubdomains);
+   for (int ip=0; ip<nrsubdomains; ip++)
+   {
+      sqf[ip] = nullptr;
+      f_orig[ip] = nullptr;
+      subdomain_sol[ip] = nullptr;
+      PmlMat[ip] = nullptr;
+      PmlMatInv[ip] = nullptr;
+      Optr[ip] = nullptr;
+
+      if (myid != SubdomainRank[ip]) continue;
+      subdomain_sol[ip] = new Vector(2*dmaps->fes[ip]->GetTrueVSize());
+      if (prob_kind == 0)
+      {
+         SetHelmholtzPmlSystemMatrix(ip);
+      }
+      else if (prob_kind == 1)
+      {
+         SetMaxwellPmlSystemMatrix(ip);
+      }
+      PmlMat[ip] = Optr[ip]->As<ComplexSparseMatrix>();
+
+      PmlMatInv[ip] = new ComplexUMFPackSolver;
+      PmlMatInv[ip]->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
+
+      // HYPRE_Int rowstarts[2]; rowstarts[0] = 0;
+      // rowstarts[1] = dmaps->fes[ip]->GetTrueVSize();
+      // HypreParMatrix * HypreMat_r =
+      //    new HypreParMatrix(MPI_COMM_SELF,rowstarts[1],rowstarts,
+      //                       &(PmlMat[ip]->real()));
+      // HypreParMatrix * HypreMat_i =
+      //    new HypreParMatrix(MPI_COMM_SELF,rowstarts[1],rowstarts,
+      //                       &(PmlMat[ip]->imag()));
+      // ComplexHypreParMatrix * HypreMat =
+      //    new ComplexHypreParMatrix(HypreMat_r,HypreMat_i,true,true);
+      // PmlMatInv[ip] = new ComplexMUMPSSolver;
+      // PmlMatInv[ip]->SetOperator(*HypreMat);
+      // delete HypreMat;
+      int ndofs = dmaps->fes[ip]->GetTrueVSize();
+      f_transf[ip].SetSize(sweeps->nsweeps);
+      for (int i=0;i<sweeps->nsweeps; i++)
+      {
+         f_transf[ip][i] = new Vector(2*ndofs);
+      }
+   }
+}
+
+
+
+void ParDST::SetHelmholtzPmlSystemMatrix(int ip)
+{
+   MFEM_VERIFY(part->subdomain_mesh[ip], "Null mesh pointer");
+   Mesh * mesh = part->subdomain_mesh[ip];
+   double h = part->MeshSize;
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+
+   Array<int> ijk;
+   GetSubdomainijk(ip,nxyz,ijk);
+   int i = ijk[0];
+   int j = ijk[1];
+   int k = ijk[2];
+
+   if (i == 0 )    length[0][0] = Pmllength[0][0];
+   if (i == nx-1 ) length[0][1] = Pmllength[0][1];
+   if (dim  > 1)
+   {
+      if (j == 0 )    length[1][0] = Pmllength[1][0];
+      if (j == ny-1 ) length[1][1] = Pmllength[1][1];
+   }
+   if (dim  == 3)
+   {
+      if (k == 0 )    length[2][0] = Pmllength[2][0];
+      if (k == nz-1 ) length[2][1] = Pmllength[2][1];
+   }
+   
+   CartesianPML pml(mesh, length);
+   pml.SetOmega(omega);
+
+   Array <int> ess_tdof_list;
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = (bc_type == BCType::DIRICHLET) ? 1 : 0;
+      dmaps->fes[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+   ProductCoefficient c2_re(c2_re0, *Q);
+   ProductCoefficient c2_im(c2_im0, *Q);
+   sqf[ip] = new SesquilinearForm (dmaps->fes[ip],bf->GetConvention());
+
+   sqf[ip]->AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   sqf[ip]->AddDomainIntegrator(new MassIntegrator(c2_re),
+                         new MassIntegrator(c2_im));
+   sqf[ip]->Assemble();
+
+   Optr[ip] = new OperatorPtr;
+   sqf[ip]->FormSystemMatrix(ess_tdof_list,*Optr[ip]);
+}
+
+void ParDST::SetMaxwellPmlSystemMatrix(int ip)
+{
+   MFEM_VERIFY(part->subdomain_mesh[ip], "Null mesh pointer");
+   Mesh * mesh = part->subdomain_mesh[ip];
+   double h = part->MeshSize;
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+
+   Array<int> ijk;
+   GetSubdomainijk(ip,nxyz,ijk);
+   int i = ijk[0];
+   int j = ijk[1];
+   int k = ijk[2];
+
+   if (i == 0 )    length[0][0] = Pmllength[0][0];
+   if (i == nx-1 ) length[0][1] = Pmllength[0][1];
+   if (dim  > 1)
+   {
+      if (j == 0 )    length[1][0] = Pmllength[1][0];
+      if (j == ny-1 ) length[1][1] = Pmllength[1][1];
+   }
+   if (dim  == 3)
+   {
+      if (k == 0 )    length[2][0] = Pmllength[2][0];
+      if (k == nz-1 ) length[2][1] = Pmllength[2][1];
+   }
+   
+   CartesianPML pml(mesh, length);
+   pml.SetOmega(omega);
+   pml.SetAttributes(mesh);
+
+   Array <int> ess_tdof_list;
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = (bc_type == BCType::DIRICHLET) ? 1 : 0;
+      dmaps->fes[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   Array<int> attr;
+   Array<int> attrPML;
+   if (mesh->attributes.Size())
+   {
+      attr.SetSize(mesh->attributes.Max());
+      attrPML.SetSize(mesh->attributes.Max());
+      attr = 0;   attr[0] = 1;
+      attrPML = 0;
+      if (mesh->attributes.Max() > 1)
+      {
+         attrPML[1] = 1;
+      }
+   }
+
+   // Integrators inside the computational domain (excluding the PML region)
+   double mu = 1.0;
+   double epsilon = 1.0;
+   ConstantCoefficient muinv(1.0/mu);
+   ConstantCoefficient omeg(-pow(omega, 2)* epsilon);
+   RestrictedCoefficient * restr_loss = nullptr;
+
+   RestrictedCoefficient restr_muinv(muinv,attr);
+   RestrictedCoefficient restr_omeg(omeg,attr);
+
+   sqf[ip] = new SesquilinearForm(dmaps->fes[ip],bf->GetConvention());
+   sqf[ip]->SetDiagonalPolicy(mfem::Matrix::DIAG_ONE);
+
+   sqf[ip]->AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv),NULL);
+   sqf[ip]->AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);
+
+   if (LossCoeff) 
+   {
+      restr_loss = new RestrictedCoefficient(*LossCoeff,attr);
+      // sqf[ip]->AddDomainIntegrator(NULL, new VectorFEMassIntegrator(*restr_loss));      
+      sqf[ip]->AddDomainIntegrator(NULL, new VectorFEMassIntegrator(*LossCoeff));      
+   }
+
+
+   int cdim = (dim == 2) ? 1 : dim;
+
+   PmlMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, &pml);
+   PmlMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, &pml);
+   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+
+
+   PmlMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,&pml);
+   ScalarMatrixProductCoefficient c2_Re0(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im0(omeg,pml_c2_Im);
+
+   MatrixCoefficient * c2_Re=nullptr;
+   MatrixCoefficient * c2_Im=nullptr;
+
+   if (Q)
+   {
+      c2_Re = new ScalarMatrixProductCoefficient(*Q,c2_Re0);
+      c2_Im = new ScalarMatrixProductCoefficient(*Q,c2_Im0);
+   }
+   else if (MQ)
+   {
+      c2_Re = new MatrixMatrixProductCoefficient(c2_Re0,*MQ);
+      c2_Im = new MatrixMatrixProductCoefficient(c2_Im0,*MQ);
+   }
+   
+   MatrixRestrictedCoefficient restr_c2_Re(*c2_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c2_Im(*c2_Im,attrPML);
+
+
+   sqf[ip]->AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
+                                new CurlCurlIntegrator(restr_c1_Im));
+   sqf[ip]->AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
+                                new VectorFEMassIntegrator(restr_c2_Im));
+   sqf[ip]->Assemble();
+   Optr[ip] = new OperatorPtr;
+   sqf[ip]->FormSystemMatrix(ess_tdof_list,*Optr[ip]);
+   delete c2_Re;
+   delete c2_Im;
+   if (LossCoeff)  delete restr_loss;
+}
+
+
+void ParDST::MarkSubdomainOverlapDofs(const bool comp)
+{
+   // First mark the elements
+         // cout<< "Compute Overlap Elements (in each possible direction) " << endl;
+      // Lists of elements
+      // x,y,z = +/- 1 ovlp 
+   NovlpElems.resize(nrsubdomains);
+
+   for (int ip = 0; ip<nrsubdomains; ip++)
+   {
+      if (myid != SubdomainRank[ip]) continue;
+      Array<int> ijk;
+      GetSubdomainijk(ip,nxyz,ijk);
+
+      Mesh * mesh = dmaps->fes[ip]->GetMesh();
+      NovlpElems[ip].resize(2*dim);
+
+      Vector pmin, pmax;
+      mesh->GetBoundingBox(pmin,pmax);
+      double h = part->MeshSize;
+      // Loop through elements
+      for (int iel=0; iel<mesh->GetNE(); iel++)
+      {
+         // Get element center
+         Vector center(dim);
+         int geom = mesh->GetElementBaseGeometry(iel);
+         ElementTransformation * tr = mesh->GetElementTransformation(iel);
+         tr->Transform(Geometries.GetCenter(geom),center);
+
+         // Assign elements to the appropriate lists
+         for (int d=0;d<dim; d++)
+         {
+            if (ijk[d]>0)
+            {
+               if (center[d] >= pmin[d]+h*ovlpnrlayers) 
+               {
+                  NovlpElems[ip][d].Append(iel);
+               }
+            }
+            else
+            {
+               NovlpElems[ip][d].Append(iel);
+            }
+               
+            if (ijk[d]<nxyz[d]-1)
+            {
+               if (center[d] <= pmax[d]-h*ovlpnrlayers) 
+               {
+                  NovlpElems[ip][dim+d].Append(iel);
+               }
+            }
+            else
+            {
+               NovlpElems[ip][dim+d].Append(iel);
+            }
+         }
+      }
+   }
+
+      // mark dofs
+   NovlpDofs.resize(nrsubdomains);
+   int mm = (comp) ? 2 : 1; // complex or real valued
+   for (int ip = 0; ip<nrsubdomains; ip++)
+   {
+      if (myid != SubdomainRank[ip]) continue;
+      FiniteElementSpace * fes = dmaps->fes[ip];
+      // Loop through the marked elements
+      NovlpDofs[ip].resize(2*dim);
+      int n = fes->GetTrueVSize();
+      Array<int> marker(n);
+      for (int d=0;d<2*dim; d++)
+      {
+         marker = 0;
+         int m = 0;
+         int melems = NovlpElems[ip][d].Size(); 
+         for (int iel=0; iel<melems; iel++)
+         {
+            Array<int> ElemDofs;
+            int el =  NovlpElems[ip][d][iel];
+            fes->GetElementDofs(el,ElemDofs);
+            int ndof = ElemDofs.Size();
+            for (int i = 0; i<ndof; ++i)
+            {
+               int eldof = ElemDofs[i];
+               int tdof = (eldof >= 0) ? eldof : abs(eldof) - 1; 
+               if (marker[tdof] == 1) continue;
+               marker[tdof] = 1;
+               m++;
+            }
+         }
+         int k = mm*(n-m);
+         NovlpDofs[ip][d].SetSize(k);
+         int l = 0;
+         for (int i = 0; i<n; i++)
+         {
+            if (marker[i]==0) 
+            {
+               NovlpDofs[ip][d][l] = i;  // real dofs
+               if (comp) 
+               {
+                  NovlpDofs[ip][d][l+k/2] = i+fes->GetTrueVSize();
+               }
+               l++;
+            }
+         }
+      }
+   }
+}
+
+void ParDST::GetChiRes(Vector & res, int ip, Array2D<int> direct) const
+{
+   for (int d=0; d<dim; d++)
+   {
+      // negative direction
+      if (direct[d][0]==1) res.SetSubVector(NovlpDofs[ip][d],0.0);
+      // possitive direction
+      if (direct[d][1]==1) res.SetSubVector(NovlpDofs[ip][d+dim],0.0);
+   }
+}
+
+
+
+void ParDST::PlotLocal(Vector & sol, socketstream & sol_sock, int ip) const
+{
+   FiniteElementSpace * fes = dmaps->fes[ip];
+   Mesh * mesh = fes->GetMesh();
+   GridFunction gf(fes);
+   double * data = sol.GetData();
+   gf.SetData(data);
+   
+   string keys;
+   keys = "keys mrRljc\n";
+   sol_sock << "solution\n" << *mesh << gf << keys << flush;
+}
+
+void ParDST::PlotGlobal(Vector & sol, socketstream & sol_sock) const
+{
+   ParMesh * pmesh = pfes->GetParMesh();
+   ParGridFunction pgf(pfes);
+   double * data = sol.GetData();
+   pgf.SetData(data);
+   string keys;
+   keys = "keys mrRljc\n";
+   sol_sock << "solution\n" << *pmesh << pgf << keys << flush;
+}
+
+
+double ParDST::GetSweepNumSteps(const int sweep) const
+{
+   int nsteps;
+   switch(dim)
+   {
+      case 1: nsteps = nx; break;
+      case 2: nsteps = nx+ny-1; break;
+      default: nsteps = nx+ny+nz-2; break;
+   }
+   return nsteps;
+}
+
+void ParDST::GetStepSubdomains(const int sweep, const int step, Array2D<int> & subdomains) const
+{
+   Array<int> aux;
+
+   switch(dim)
+   {
+      case 2: 
+      for (int i=nx-1;i>=0; i--)
+      {  
+         int j;
+         switch (sweep)
+         {
+            case 0:  j = step-i;         break;
+            case 1:  j = step-nx+i+1;    break;
+            case 2:  j = nx+i-step-1;    break;
+            default: j = nx+ny-i-step-2; break;
+         }
+         if (j<0 || j>=ny) continue;
+         aux.Append(i); aux.Append(j);
+      }
+      break; 
+      default:
+      for (int i=nx-1;i>=0; i--)
+      {  
+         for (int j=ny-1;j>=0; j--)
+         {
+            int k;
+            switch (sweep)
+            {
+               case 0:  k = step-i-j;            break;
+               case 1:  k = step-nx+i+1-j;       break;
+               case 2:  k = step-ny+j+1-i;       break;
+               case 3:  k = step-nx-ny+i+j+2;    break;
+               case 4:  k = i+j+nz-1-step;       break;
+               case 5:  k = nx+nz-i+j-step-2;    break;
+               case 6:  k = ny+nz+i-j-step-2;    break;
+               default: k = nx+ny+nz-i-j-step-3; break;
+            }
+            if (k<0 || k>=nz) continue;   
+            aux.Append(i); aux.Append(j); aux.Append(k); 
+         }
+      }
+      break;
+   }
+
+   int nrows = aux.Size()/dim;
+   int ncols = dim;
+
+   subdomains.SetSize(nrows,ncols);
+   for (int r=0;r<nrows; r++)
+   {
+      for (int c=0; c<ncols; c++)
+      {
+         int k = r*ncols + c;
+         subdomains[r][c] = aux[k];
+      }
+   }
+}
+
+
+void ParDST::TransferSources(int sweep, const Array<int> & subdomain_ids) const
+{
+   OvlpSol.resize(nrsubdomains);
+   int nrneighbors = pow(3,dim);
+   for (int ip = 0; ip<nrsubdomains; ip++)
+   {
+      if (myid == SubdomainRank[ip])
+      {
+         OvlpSol[ip].resize(nrneighbors);
+      }
+   }
+   int m = subdomain_ids.Size();
+   Array<Vector *> x(m);
+   for (int i = 0; i<m; i++)
+   {
+      x[i] = nullptr;
+      int ip = subdomain_ids[i];
+      if (myid != SubdomainRank[ip]) continue;
+      x[i] = new Vector(subdomain_sol[ip]->GetData(),subdomain_sol[ip]->Size());
+   }
+   dmaps->TransferToNeighbors(subdomain_ids,x,OvlpSol);
+   for (int i = 0; i<m; i++)
+   {
+      delete x[i]; x[i] = nullptr;
+   }
+   // Update residuals
+//   Find all neighbors of patch ip0
+   for (int is = 0; is<m; is++)
+   {
+      int ip0 = subdomain_ids[is];
+      Array<int> ijk;
+      Array<int> ijk1(3);
+      GetSubdomainijk(ip0,nxyz,ijk);
+      // cout << "Subdomain to transfer its sources: " << "(" <<ijk[0] << "," << ijk[1] << ")" <<endl;
+      Array<int> directions(3);   
+      for (int i=-1; i<2; i++)
+      {
+         int i1 = ijk[0] + i;
+         if (i1 <0 || i1>=nx) continue;
+         directions[0] = i;
+         ijk1[0] = i1;
+         for (int j=-1; j<2; j++)
+         {
+            int j1 = ijk[1] + j;
+            if (j1 <0 || j1>=ny) continue;
+            directions[1] = j;
+            ijk1[1] = j1;
+            int kbeg = (dim == 2) ? 0 : -1;
+            int kend = (dim == 2) ? 1 :  2;
+            for (int k=kbeg; k<kend; k++)
+            {
+               int k1 = ijk[2] + k;
+               if (k1 <0 || k1>=nz) continue;
+               directions[2] = (dim == 3) ? k : -1 ;
+               if (i==0 && j==0 && k==0) continue;
+
+               int l = GetSweepToTransfer(sweep,directions);
+               // cout << "in the direction " ; directions.Print(); 
+               // cout << "sweep of transfer = " << l << endl;
+               if (l == -1) continue;
+               ijk1[2] = k1;
+               int ip1 = GetSubdomainId(nxyz,ijk1);
+
+               if (myid != SubdomainRank[ip1]) continue;
+               Array<int>directions1(3); directions1 = -1;
+               for (int i = 0; i<dim; i++) directions1[i] = -directions[i];
+               int dir = GetDirectionId(directions1);
+               int n = dmaps->fes[ip1]->GetTrueVSize();
+               Vector res(2*n);
+               PmlMat[ip1]->Mult(*OvlpSol[ip1][dir],res);
+
+               Array2D<int> direct(dim,2); direct = 0;
+               for (int d = 0; d<dim; d++)
+               {
+                  if (directions[d]==1)  direct[d][0] = 1;
+                  if (directions[d]==-1) direct[d][1] = 1;
+               }   
+               GetChiRes(res,ip1,direct);
+               *f_transf[ip1][l] -= res;
+            }
+         }  
+      }
+      // cin.get();
+   }
+
+   for (int ip = 0; ip<nrsubdomains; ip++)
+   {
+      if (myid == SubdomainRank[ip])
+      {
+         for (int i = 0; i<nrneighbors; i++)
+         {
+            if (OvlpSol[ip][i])
+            {
+               delete OvlpSol[ip][i];
+            }
+         }
+         OvlpSol[ip].clear();
+      }
+   }
+}
+
+int ParDST::GetSweepToTransfer(const int s, Array<int> directions) const
+{
+   int l1=-1;
+   int nsweeps = sweeps->nsweeps;
+   Array<int> sweep0;   
+   sweeps->GetSweep(s,sweep0);
+   switch (dim)
+   {
+      case 2:
+      for (int l=s; l<nsweeps; l++)
+      {
+         // Rule 1: the transfer source direction has to be similar with 
+         // the sweep direction
+         Array<int> sweep1;   
+         sweeps->GetSweep(l,sweep1);
+         int ddot = 0;
+         for (int d=0; d<dim; d++) ddot+= sweep1[d] * directions[d];
+         if (ddot <= 0) continue;
+
+         // Rule 2: The horizontal or vertical transfer source cannot be used
+         // Case of horizontal or vertical transfer source 
+         // (it can't be both 0 cause it's skipped)
+         if (directions[0]==0 || directions[1] == 0) 
+         {
+            if (sweep0[0] == -sweep1[0] && sweep0[1] == -sweep1[1]) continue;
+         }
+         l1 = l;
+         break;
+      }
+      break;
+      default:
+      for (int l=s; l<nsweeps; l++)
+      {
+         // Rule 1: (similar directions) the transfer source direction has to be similar with 
+         // the sweep direction
+         Array<int> sweep1;   
+         sweeps->GetSweep(l,sweep1);
+         int ddot = 0;
+         bool similar = true;
+         for (int d=0; d<dim; d++) 
+         {
+            if (sweep1[d] * directions[d] < 0) similar = false;
+            ddot+= sweep1[d] * directions[d];
+         }
+         if (!similar || ddot<=0) continue; // not similar
+
+         // Rule 2: (oposite directions) the transfer source direction has to be similar with 
+         // the sweep direction
+         // 
+         // check any of the projections onto the planes
+         // (xy, xz, yz)
+
+         if ( (directions[0]==0 && directions[1] != 0) || 
+              (directions[0]!=0 && directions[1] == 0) ||
+              (directions[0]==0 && directions[2] != 0) || 
+              (directions[0]!=0 && directions[2] == 0) ||
+              (directions[2]==0 && directions[1] != 0) || 
+              (directions[2]!=0 && directions[1] == 0) ) 
+         {
+            if (sweep0[0] == -sweep1[0] && 
+                sweep0[1] == -sweep1[1] && 
+                sweep0[2] == -sweep1[2]) continue;
+         }
+         l1 = l;
+         break;
+      }
+      break;
+   }
+
+   return l1;   
+}
+
+void ParDST::CorrectOrientation(int ip,Vector &x) const
+{
+   FiniteElementSpace * fespace = dmaps->fes[ip];
+   Mesh * mesh = fespace->GetMesh();
+   int nrelems = mesh->GetNE();
+   // GridFunction test; 
+   // test.SetFromTrueDofs(x)
+   Array<int> signs(fespace->GetTrueVSize()); signs = 0;
+   for (int iel=0; iel<nrelems; iel++)
+   {
+      Array<int> ElemDofs;
+      fespace->GetElementDofs(iel,ElemDofs);
+      int ndofs = ElemDofs.Size();
+      ElemDofs.Print();
+      for (int i = 0; i< ndofs; i++)
+      {
+         int pdof_ = ElemDofs[i];
+         if (pdof_ < 0) 
+         {
+            signs[abs(pdof_)-1] += 1.0 ; 
+         }
+         else
+         {
+            signs[pdof_] -= 1.0 ; 
+         }
+      }
+   }
+
+   cout << "signs = " ; signs.Print();
+   for (int i = 0; i<fespace->GetTrueVSize(); i++)
+   {
+      if (signs[i]<0) 
+      {
+         x(i) *= -1.0;
+         x(i+fespace->GetTrueVSize()) *= -1.0;
+      }
+   }
+}
+
+
+ParDST::~ParDST()
+{
+
+   for (int ip=0; ip<nrsubdomains; ip++)
+   {
+      delete Optr[ip];
+      delete subdomain_sol[ip];
+      delete PmlMatInv[ip];
+      delete sqf[ip];
+      if (myid != SubdomainRank[ip]) continue;
+      for (int i=0;i<sweeps->nsweeps; i++)
+      {
+         delete f_transf[ip][i];
+      }
+      delete f_orig[ip];
+   }
+   f_orig.DeleteAll();
+   delete dmaps;
+   delete sweeps;
+   delete part;
+
+}

examples/maxwell-solver/ParDST/ParDST.hpp

@@ -0,0 +1,82 @@
+#pragma once
+#include "../common/Utilities.hpp"
+#include "../common/PML.hpp"
+#include "../DST/DST.hpp"
+#include "DofMapsDST.hpp"
+using namespace std;
+using namespace mfem;
+
+
+class ParDST : public Solver//
+{
+public:
+   enum BCType
+   {
+       NEUMANN,
+       DIRICHLET
+   };
+   ParDST(ParSesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+       double omega_, Coefficient * Q_, int nrlayers_, 
+       int nx_=2, int ny_=2, int nz_=2, 
+       BCType bc_type_ = BCType::DIRICHLET, Coefficient * LossCoeff_ = nullptr);
+   ParDST(ParSesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+       double omega_, MatrixCoefficient * MQ_, int nrlayers_, int nx_=2, int ny_=2, int nz_=2, 
+       BCType bc_type_ = BCType::DIRICHLET, Coefficient * LossCoeff_ = nullptr);
+   virtual void SetOperator(const Operator &op) {}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~ParDST();
+private:
+   MPI_Comm comm = MPI_COMM_WORLD;
+   int num_procs, myid;
+   // Constructor inputs
+   int prob_kind;
+   ParSesquilinearForm *bf=nullptr;
+   ParFiniteElementSpace * pfes = nullptr;
+   ParMesh * pmesh = nullptr;
+   ParMeshPartition * part = nullptr;
+   Array<int> SubdomainRank;
+   Array<int> RankSubdomains;
+   const FiniteElementCollection * fec = nullptr;
+   Array2D<double> Pmllength;
+   int dim = 2;
+   double omega = 0.5;
+   Coefficient * Q=nullptr;
+   MatrixCoefficient * MQ=nullptr;
+   int nrlayers;
+   BCType bc_type = BCType::DIRICHLET;
+   Coefficient * LossCoeff=nullptr;
+   
+   int ovlpnrlayers;
+   int nrsubdomains = 0;
+   int nx,ny,nz;
+   Array<int> nxyz;
+   Sweep * sweeps = nullptr;
+   DofMaps * dmaps = nullptr;
+   Array< SesquilinearForm * > sqf;
+   Array< OperatorPtr * > Optr;
+   Array<ComplexSparseMatrix *> PmlMat;
+   Array<ComplexUMFPackSolver *> PmlMatInv;
+//    Array<ComplexMUMPSSolver *> PmlMatInv;
+   mutable Array<Vector *> f_orig;
+   mutable std::vector<Array<Vector * >> f_transf;
+   mutable Array<Vector * > subdomain_sol;
+   mutable std::vector<std::vector<Vector * >> OvlpSol;
+   void SetupSubdomainProblems();
+   std::vector<std::vector<Array<int>>> NovlpElems;
+   std::vector<std::vector<Array<int>>> NovlpDofs;
+   void MarkSubdomainOverlapDofs(const bool comp = false);
+   void SetHelmholtzPmlSystemMatrix(int ip);   
+   void SetMaxwellPmlSystemMatrix(int ip);
+   void GetChiRes(Vector & res, int ip, Array2D<int> direct) const;
+   void PlotLocal(Vector & sol, socketstream & sol_sock, int ip) const;
+   void PlotGlobal(Vector & sol, socketstream & sol_sock) const;
+   double GetSweepNumSteps(const int sweep) const;
+   void GetStepSubdomains(const int sweep, const int step, Array2D<int> & subdomains) const;
+   void TransferSources(int sweep, const Array<int> & subdomain_ids) const;
+   int GetSweepToTransfer(const int s, Array<int> directions) const;
+   void CorrectOrientation(int ip, Vector & x) const;
+   void Init();
+
+};
+
+

examples/maxwell-solver/ST/PST.cpp

@@ -0,0 +1,795 @@
+// Pure Source Transfer Preconditioner
+#include "PST.hpp"
+
+PSTP::PSTP(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+         double omega_, Coefficient * ws_,  int nrlayers_)
+   : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), 
+     bf(bf_), Pmllength(Pmllength_), omega(omega_), ws(ws_), nrlayers(nrlayers_)
+{
+   Mesh * mesh = bf->FESpace()->GetMesh();
+   dim = mesh->Dimension();
+
+   // ----------------- Step 1 --------------------
+   // Introduce 2 layered partitios of the domain 
+   // 
+   int partition_kind;
+   // 1. Non ovelapping 
+   partition_kind = 1; // Non Ovelapping partition 
+   pnovlp = new MeshPartition(mesh, partition_kind);
+
+   // 2. Overlapping to the right
+   partition_kind = 3; // Overlapping partition for the full space
+   povlp = new MeshPartition(mesh, partition_kind);
+
+   nrpatch = povlp->nrpatch;
+   MFEM_VERIFY(povlp->nrpatch+1 == pnovlp->nrpatch,"Check nrpatch");
+
+
+   lmap = new LocalDofMap(bf->FESpace()->FEColl(),pnovlp,povlp);
+
+   // Given the two partitions create a dof map between the non-ovelapping 
+   // subdomain dofs and the overlapping ones
+
+
+   //
+   // ----------------- Step 1a -------------------
+   // Save the partition for visualization
+   // SaveMeshPartition(povlp->patch_mesh, "output/mesh_ovlp.", "output/sol_ovlp.");
+   // SaveMeshPartition(pnovlp->patch_mesh, "output/mesh_novlp.", "output/sol_novlp.");
+
+   // ------------------Step 2 --------------------
+   // Construct the dof maps from subdomains to global (for the extended and not)
+   // The non ovelapping is extended on the left by pml (halfspace problem)
+   // The overlapping is extended left and right by pml (unbounded domain problem)
+   novlp_prob = new DofMap(bf,pnovlp);
+   ovlp_prob  = new DofMap(bf,povlp,nrlayers); 
+
+   // Given 
+
+   // ------------------Step 3 --------------------
+   // Assemble the PML Problem matrices and factor them
+   PmlMat.SetSize(nrpatch);
+   PmlMatInv.SetSize(nrpatch);
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      PmlMat[ip] = GetPmlSystemMatrix(ip);
+      PmlMatInv[ip] = new KLUSolver;
+      PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
+   }
+}
+
+SparseMatrix * PSTP::GetPmlSystemMatrix(int ip)
+{
+   double h = GetUniformMeshElementSize(ovlp_prob->PmlMeshes[ip]);
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+   if (ip == nrpatch-1 || ip == 0) 
+   {
+      length[0][0] = Pmllength[0][0];
+      length[0][1] = Pmllength[0][1];
+   }
+   length[1][0] = Pmllength[1][0];
+   length[1][1] = Pmllength[1][1];
+
+   CartesianPML pml(ovlp_prob->PmlMeshes[ip], length);
+   pml.SetOmega(omega);
+
+   Array <int> ess_tdof_list;
+   if (ovlp_prob->PmlMeshes[ip]->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(ovlp_prob->PmlMeshes[ip]->bdr_attributes.Max());
+      ess_bdr = 1;
+      ovlp_prob->PmlFespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+
+   ProductCoefficient c2_re(c2_re0, *ws);
+   ProductCoefficient c2_im(c2_im0, *ws);
+
+   SesquilinearForm a(ovlp_prob->PmlFespaces[ip],ComplexOperator::HERMITIAN);
+
+   a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   a.AddDomainIntegrator(new MassIntegrator(c2_re),
+                         new MassIntegrator(c2_im));
+   a.Assemble();
+
+   OperatorPtr Alocal;
+   a.FormSystemMatrix(ess_tdof_list,Alocal);
+   ComplexSparseMatrix * AZ_ext = Alocal.As<ComplexSparseMatrix>();
+   SparseMatrix * Mat = AZ_ext->GetSystemMatrix();
+   Mat->Threshold(0.0);
+   return Mat;
+}
+
+void PSTP::Mult(const Vector &r, Vector &z) const
+{
+   z = 0.0; 
+   res.SetSize(nrpatch);
+   Vector rnew(r);
+   Vector znew(z);
+   Vector z1(z);
+   Vector z2(z);
+   Vector raux(znew.Size());
+   Vector res_local, sol_local;
+   znew = 0.0;
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   Array<Vector> fsol(nrpatch+1);
+   Array<Vector> bsol(nrpatch+1);
+
+   // source transfer algorithm
+   for (int ip = 0; ip < nrpatch; ip++)
+   {
+      // cout << "ip = " << ip << endl;
+      Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+      Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip];
+      int ndofs = Dof2GlobalDof->Size();
+      res_local.SetSize(ndofs);
+      sol_local.SetSize(ndofs);
+
+      rnew.GetSubVector(*Dof2GlobalDof, res_local);
+
+      int nrdof_ext = PmlMat[ip]->Height();
+         
+      Vector res_ext(nrdof_ext); res_ext = 0.0;
+      Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+
+      res_ext.SetSubVector(*Dof2PmlDof,res_local.GetData());
+      PmlMatInv[ip]->Mult(res_ext, sol_ext);
+
+      sol_ext.GetSubVector(*Dof2PmlDof,sol_local);
+
+      znew = 0.0;
+      znew.SetSubVector(*Dof2GlobalDof,sol_local);
+
+      // cout << "ip+1 = " << ip+1 << endl;
+      Array<int> * nDof2GlobalDof = &novlp_prob->Dof2GlobalDof[ip+1];
+      fsol[ip+1].SetSize(nDof2GlobalDof->Size());
+      znew.GetSubVector(*nDof2GlobalDof,fsol[ip+1]);
+
+      socketstream subsol_sock(vishost, visport);
+      // PlotSolution(znew, subsol_sock,ip); cin.get();
+
+      // z.AddElementVector(*Dof2GlobalDof,sol_local);
+      int direction = 1;
+      if (ip <nrpatch-1) GetCutOffSolution(znew, ip, direction);
+      
+
+      if (ip != 0) z1+=znew;
+      // PlotSolution(z, subsol_sock,1); cin.get();
+
+      A->Mult(znew, raux);
+      rnew -= raux;
+      // PlotSolution(rnew, subsol_sock,ip); cin.get();
+
+   }
+
+   // socketstream subsol1_sock(vishost, visport);
+   // PlotSolution(z1, subsol1_sock,0); 
+
+   rnew = r;
+   for (int ip = nrpatch-1; ip >=0; ip--)
+   {
+      Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+      Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip];
+      int ndofs = Dof2GlobalDof->Size();
+      res_local.SetSize(ndofs);
+      sol_local.SetSize(ndofs);
+
+      rnew.GetSubVector(*Dof2GlobalDof, res_local);
+
+      //-----------------------------------------------
+      // Extend by zero to the extended mesh
+      int nrdof_ext = PmlMat[ip]->Height();
+         
+      Vector res_ext(nrdof_ext); res_ext = 0.0;
+      Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+
+      res_ext.SetSubVector(*Dof2PmlDof,res_local.GetData());
+      PmlMatInv[ip]->Mult(res_ext, sol_ext);
+
+      sol_ext.GetSubVector(*Dof2PmlDof,sol_local);
+
+      znew = 0.0;
+      znew.SetSubVector(*Dof2GlobalDof,sol_local);
+
+
+      Array<int> * nDof2GlobalDof = &novlp_prob->Dof2GlobalDof[ip];
+      bsol[ip].SetSize(nDof2GlobalDof->Size());
+      znew.GetSubVector(*nDof2GlobalDof,bsol[ip]);
+      // cout << "ip = " << ip << endl;
+
+      // PlotSolution(znew, subsol_sock,ip); cin.get();
+
+      // z.AddElementVector(*Dof2GlobalDof,sol_local);
+
+
+      int direction = -1;
+      if (ip>0) GetCutOffSolution(znew, ip-1, direction);
+
+      if (ip != nrpatch-1) z2+=znew;
+      // PlotSolution(z, subsol_sock,1); cin.get();
+
+      A->Mult(znew, raux);
+      rnew -= raux;
+      // PlotSolution(rnew, subsol_sock,ip); cin.get();
+
+   }
+   // socketstream subsol2_sock(vishost, visport);
+   // PlotSolution(z2, subsol2_sock,0); cin.get();
+
+
+   // construct solution z by z1 and z2
+   // vizualize solutions 
+   // Forward solutions
+   // socketstream subsol3_sock(vishost, visport);
+   for (int ip = 0; ip<nrpatch; ip++)
+   {
+      // cout << "ip = " << ip << endl;
+      znew = 0.0;
+      Array<int> * nDof2GlobalDof = &novlp_prob->Dof2GlobalDof[ip+1];
+      znew.SetSubVector(*nDof2GlobalDof,fsol[ip+1]);
+      // PlotSolution(znew, subsol3_sock,0); cin.get();
+   }
+
+      // Backward solutions
+   // socketstream subsol4_sock(vishost, visport);
+   for (int ip = 0; ip<nrpatch; ip++)
+   {
+      znew = 0.0;
+      Array<int> * nDof2GlobalDof = &novlp_prob->Dof2GlobalDof[ip];
+      znew.SetSubVector(*nDof2GlobalDof,bsol[ip]);
+      // PlotSolution(znew, subsol4_sock,0); cin.get();
+   }
+
+
+   Array<Vector> gsol(nrpatch+1);
+   // socketstream subsol5_sock(vishost, visport);
+
+   for (int ip = 0; ip<=nrpatch; ip++)
+   {
+      if (ip == 0) 
+      {
+         gsol[ip].SetSize(bsol[ip].Size());
+         gsol[ip] = bsol[ip];
+      }
+      else if (ip == nrpatch)
+      {
+         gsol[ip].SetSize(fsol[ip].Size());
+         gsol[ip] = fsol[ip];
+      }
+      else
+      {
+         gsol[ip].SetSize(fsol[ip].Size());
+         gsol[ip] = 0.0;
+         gsol[ip] += bsol[ip];
+         gsol[ip] += fsol[ip];
+      }
+      
+      znew = 0.0;
+      Array<int> * nDof2GlobalDof = &novlp_prob->Dof2GlobalDof[ip];
+      znew.SetSubVector(*nDof2GlobalDof,gsol[ip]);
+      // PlotSolution(znew, subsol5_sock,0); cin.get();
+      z.SetSubVector(*nDof2GlobalDof,gsol[ip]);
+   }
+
+
+   // required for visualization
+   // char vishost[] = "localhost";
+   // int  visport   = 19916;
+   // socketstream subsol_sock(vishost, visport);
+   // socketstream subsol1_sock(vishost, visport);
+   // socketstream subsol2_sock(vishost, visport);
+   // socketstream subsol3_sock(vishost, visport);
+
+   // // Initialize correction
+   // z = 0.0; 
+   // Vector fpml;
+   // Vector zpml;
+   // Vector z1(z);
+   // Vector res(z);
+
+   // // Construct the sources in each non-overlapping subdomain by restricting 
+   // // the global source
+   // Array<Vector> fn(nrpatch+1);
+   // Array<Vector> ftransf(nrpatch+1);
+   // for (int ip=0; ip<=nrpatch; ip++)
+   // {
+   //    Array<int> *Dof2GDof = &novlp_prob->Dof2GlobalDof[ip];
+   //    fn[ip].SetSize(Dof2GDof->Size());
+   //    ftransf[ip].SetSize(Dof2GDof->Size());
+   //    r.GetSubVector(*Dof2GDof,fn[ip]);
+   // }
+
+   // // source transfer algorithm 1 (forward sweep)
+   // Vector f;
+   // for (int ip = 0; ip < nrpatch; ip++)
+   // {
+   //    // construct the source in the overlapping PML problem
+   //    if (ip == 0) ftransf[ip] = fn[ip];
+
+   //    int ndof = ovlp_prob->Dof2GlobalDof[ip].Size();
+   //    f.SetSize(ndof); f = 0.0;
+   //    f.SetSubVector(lmap->map1[ip],ftransf[ip]);
+   //    f.SetSubVector(lmap->map2[ip],fn[ip+1]);
+
+   //    // Extend to the pml problem and solve for the local pml solution
+   //    Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip];
+   //    int ndof_pml = PmlMat[ip]->Height();
+   //    fpml.SetSize(ndof_pml); fpml=0.0;
+   //    zpml.SetSize(ndof_pml); zpml=0.0;
+   //    fpml.SetSubVector(*Dof2PmlDof,f);
+   //    // Solve the pml problem
+   //    PmlMatInv[ip]->Mult(fpml, zpml);
+   //    // PlotLocalSolution(zpml,subsol_sock,ip); cin.get();
+
+   //    //--------------------------------------------------
+   //    // Save the solution to the global solution
+   //     // restrict to non-pml problem
+   //    Vector sol(ndof); 
+   //    zpml.GetSubVector(*Dof2PmlDof, sol);
+   //    // restrict to the non-ovlp subdomain
+   //    // z1.AddElementVector(ovlp_prob->Dof2GlobalDof[ip],sol);
+
+   //    int m = lmap->map2[ip].Size();
+   //    Vector soll(m);
+   //    sol.GetSubVector(lmap->map2[ip],soll);
+   //    // prolong to the global solution
+   //    z.SetSubVector(novlp_prob->Dof2GlobalDof[ip+1],soll);
+
+   //    // PlotSolution(z,subsol1_sock,0); 
+   //    // PlotSolution(z1,subsol2_sock,0); 
+
+   //    //--------------------------------------------------
+
+   //    if (ip == nrpatch-1) continue;
+
+   //    int direction = 1;
+   //    GetCutOffSol(zpml, ip, direction);
+   //    // PlotLocalSolution(zpml,subsol3_sock,ip); cin.get();
+
+   //    // Calculate source to be trasfered to the pml mesh
+   //    Vector respml(zpml.Size());
+   //    PmlMat[ip]->Mult(zpml,respml);
+
+   //    // PlotLocalSolution(respml,subsol_sock,ip); cin.get();
+   //    // restrict to non-pml problem
+   //    Vector res(ndof); 
+   //    respml.GetSubVector(*Dof2PmlDof, res);
+   //    // source to be transfered
+   //    res.GetSubVector(lmap->map2[ip],ftransf[ip+1]);
+
+   //    // restrict to nonpml problem
+   //    // restrict to non-pml problem
+   //    // Vector sol1(ndof); 
+   //    // zpml.GetSubVector(*Dof2PmlDof, sol1);
+   //    // // prolong to global sol
+   //    // z1 = 0.0;
+   //    // Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+   //    // z1.SetSubVector(*Dof2GlobalDof, sol1);
+   //    // // calculate new source
+   //    // A->Mult(z1,res);
+
+   //    // //restrict to subdomain ip+1
+   //    // Array<int> * nDof2GlobalDof = &novlp_prob->Dof2GlobalDof[ip+1];
+   //    // res.GetSubVector(*nDof2GlobalDof,ftransf[ip+1]);
+   // }
+
+   // // source transfer algorithm 2 (backward sweep)
+   // for (int ip = nrpatch-1; ip >= 0; ip--)
+   // {
+   //    // construct the source in the overlapping PML problem
+   //    if (ip == nrpatch-1) ftransf[ip+1] = fn[ip+1];
+
+   //    int ndof = ovlp_prob->Dof2GlobalDof[ip].Size();
+   //    f.SetSize(ndof); f = 0.0;
+   //    f.SetSubVector(lmap->map1[ip],fn[ip]);
+   //    f.SetSubVector(lmap->map2[ip],ftransf[ip+1]);
+
+   //    // Extend to the pml problem and solve for the local pml solution
+   //    Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip];
+   //    int ndof_pml = PmlMat[ip]->Height();
+   //    fpml.SetSize(ndof_pml); fpml=0.0;
+   //    zpml.SetSize(ndof_pml); zpml=0.0;
+   //    fpml.SetSubVector(*Dof2PmlDof,f);
+   //    // Solve the pml problem
+   //    PmlMatInv[ip]->Mult(fpml, zpml);
+   //    PlotLocalSolution(zpml,subsol_sock,ip); cin.get();
+
+   //    //--------------------------------------------------
+   //    // Save the solution to the global solution
+   //    // restrict to non-pml problem
+   //    Vector sol(ndof); 
+   //    zpml.GetSubVector(*Dof2PmlDof, sol);
+   //    // restrict to the non-ovlp subdomain
+   //    int m = lmap->map1[ip].Size();
+   //    Vector soll(m);
+   //    sol.GetSubVector(lmap->map1[ip],soll);
+   //    // prolong to the global solution
+   //    z.AddElementVector(novlp_prob->Dof2GlobalDof[ip],soll);
+
+   //    // PlotSolution(z,subsol_sock,0); cin.get();
+
+   //    //--------------------------------------------------
+
+   //    if (ip == 0) continue;
+
+   //    int direction = -1;
+   //    GetCutOffSol(zpml, ip-1, direction);
+   //    PlotLocalSolution(zpml,subsol_sock,ip); cin.get();
+
+   //    // Calculate source to be trasfered to the pml mesh
+   //    Vector respml(zpml.Size());
+   //    PmlMat[ip]->Mult(zpml,respml);
+
+   //    // PlotLocalSolution(respml,subsol_sock,ip); cin.get();
+   //    // restrict to non-pml problem
+   //    Vector res(ndof); 
+   //    respml.GetSubVector(*Dof2PmlDof, res);
+   //    // source to be transfered
+   //    res.GetSubVector(lmap->map2[ip],ftransf[ip]);
+   // }
+
+
+   // PlotSolution(z,subsol_sock,0); cin.get();
+
+
+   // res.SetSize(nrpatch);
+   // Vector rnew(r);
+   // Vector rnew2(r);
+   // Vector znew(z);
+   // Vector znew1(z);
+   // Vector znew2(z);
+   // Vector raux(znew.Size());
+   // Vector res_local, sol_local;
+   // znew = 0.0;
+   // znew1= 0.0;
+   // znew2= 0.0;
+
+   // char vishost[] = "localhost";
+   // int  visport   = 19916;
+   // socketstream subsol_sock(vishost, visport);
+
+   // std::vector<Vector*> zloc;
+   // zloc.resize(nrpatch+1);
+
+   // // allocate memory and initialize
+   // for (int ip = 0; ip <= nrpatch; ip++)
+   // {
+   //    int n = novlp_prob->Dof2GlobalDof[ip].Size();
+   //    zloc[ip] = new Vector(n); *zloc[ip]=0.0;
+   // }
+
+
+   // // source transfer algorithm 1 (forward sweep)
+   // for (int ip = 0; ip < nrpatch; ip++)
+   // {
+   //    Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+   //    Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip];
+   //    int ndofs = Dof2GlobalDof->Size();
+   //    res_local.SetSize(ndofs);
+   //    sol_local.SetSize(ndofs);
+
+   //    rnew.GetSubVector(*Dof2GlobalDof, res_local);
+
+   //    //-----------------------------------------------
+   //    // Extend by zero to the extended mesh
+   //    int nrdof_ext = PmlMat[ip]->Height();
+   //    Vector res_ext(nrdof_ext); res_ext = 0.0;
+   //    Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+   //    res_ext.SetSubVector(*Dof2PmlDof,res_local.GetData());
+   //    PmlMatInv[ip]->Mult(res_ext, sol_ext);
+   //    sol_ext.GetSubVector(*Dof2PmlDof,sol_local);
+   //    znew = 0.0;
+   //    znew.SetSubVector(*Dof2GlobalDof,sol_local);
+
+   //    Array<int> * Dof2GDof = &novlp_prob->Dof2GlobalDof[ip+1];
+   //    int n = Dof2GDof->Size();
+   //    Vector nsol(n);
+   //    znew.GetSubVector(*Dof2GDof, nsol);
+   //    *zloc[ip+1] += nsol;
+      
+   //    int direction = 1;
+      // if (ip < nrpatch-1) GetCutOffSolution(znew, ip, direction);
+
+   //    A->Mult(znew, raux);
+   //    rnew -= raux;
+   //    znew1 += znew;
+
+
+   // }
+
+   // PlotSolution(znew1, subsol_sock,0); cin.get();
+
+   // // source transfer algorithm 2 (backward sweep)
+   // for (int ip = nrpatch-1; ip >=0; ip--)
+   // {
+   //    Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+   //    Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip];
+   //    int ndofs = Dof2GlobalDof->Size();
+   //    res_local.SetSize(ndofs);
+   //    sol_local.SetSize(ndofs);
+   //    rnew2.GetSubVector(*Dof2GlobalDof, res_local);
+
+   //    //-----------------------------------------------
+   //    // Extend by zero to the extended mesh
+   //    int nrdof_ext = PmlMat[ip]->Height();
+   //    Vector res_ext(nrdof_ext); res_ext = 0.0;
+   //    Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+   //    res_ext.SetSubVector(*Dof2PmlDof,res_local.GetData());
+   //    PmlMatInv[ip]->Mult(res_ext, sol_ext);
+   //    sol_ext.GetSubVector(*Dof2PmlDof,sol_local);
+   //    znew = 0.0;
+   //    znew.SetSubVector(*Dof2GlobalDof,sol_local);
+
+
+
+   //    Array<int> * Dof2GDof = &novlp_prob->Dof2GlobalDof[ip];
+   //    int n = Dof2GDof->Size();
+   //    Vector nsol(n);
+   //    znew.GetSubVector(*Dof2GDof, nsol);
+   //    *zloc[ip] += nsol;
+      
+   //    int direction = -1;
+   //    if (ip > 0) GetCutOffSolution(znew, ip-1, direction);
+
+   //    A->Mult(znew, raux);
+   //    rnew2 -= raux;
+   //    znew2 += znew;
+   // }
+
+   // // PlotSolution(znew2, subsol_sock,0); cin.get();
+
+   // // propagate to global dofs 
+   // z = 0.0;
+   // for (int ip = 0; ip <= nrpatch; ip++)
+   // {
+   //    Array<int> Dof2GDof = novlp_prob->Dof2GlobalDof[ip];
+   //    z.AddElementVector(Dof2GDof,*zloc[ip]);
+   // }
+
+   // PlotSolution(z, subsol_sock,0); cin.get();
+
+
+
+}
+
+void PSTP::PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const
+{
+   FiniteElementSpace * fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   GridFunction gf(fespace);
+   double * data = sol.GetData();
+   gf.SetData(data);
+   
+   string keys = "keys z\n";
+   if (ip ==0) keys = "keys rRljc\n";
+   sol_sock << "solution\n" << *mesh << gf << flush;
+   
+}
+
+
+void PSTP::PlotLocalSolution(Vector & sol, socketstream & sol_sock, int ip) const
+{
+   FiniteElementSpace * fespace = ovlp_prob->PmlFespaces[ip];
+   Mesh * mesh = fespace->GetMesh();
+   GridFunction gf(fespace);
+   double * data = sol.GetData();
+   gf.SetData(data);
+   
+   string keys = "keys z\n";
+   if (ip ==0) keys = "keys rRljc\n";
+   sol_sock << "solution\n" << *mesh << gf << flush;
+   
+}
+
+void PSTP::GetCutOffSolution(Vector & sol, int ip, int direction) const
+{
+
+   int l,k;
+
+   l=(direction == 1)? ip+1: ip;
+   k=(direction == 1)? ip: ip+1;
+
+   Mesh * mesh1 = ovlp_prob->fespaces[l]->GetMesh();
+   Mesh * mesh2 = ovlp_prob->fespaces[k]->GetMesh();
+   
+   Vector pmin1, pmax1;
+   Vector pmin2, pmax2;
+   mesh1->GetBoundingBox(pmin1, pmax1);
+   mesh2->GetBoundingBox(pmin2, pmax2);
+
+   Array2D<double> h(dim,2);
+   
+   h[0][0] = pmin2[0] - pmin1[0];
+   h[0][1] = pmax2[0] - pmin1[0];
+   h[1][0] = pmin2[1] - pmin1[1];
+   h[1][1] = pmax2[1] - pmax1[1];
+
+   if (direction == 1)
+   {
+      h[0][0] = 0.0;
+   }
+   else if (direction == -1)
+   {
+      h[0][1] = 0.0;
+   }
+   CutOffFnCoefficient cf(CutOffFncn, pmin2, pmax2, h);
+   double * data = sol.GetData();
+
+   FiniteElementSpace * fespace = bf->FESpace();
+   int n = fespace->GetTrueVSize();
+
+   GridFunction solgf_re(fespace, data);
+   GridFunction solgf_im(fespace, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fespace);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   sol = gf;
+}
+
+PSTP::~PSTP()
+{
+   for (int ip = 0; ip<nrpatch; ++ip)
+   {
+      delete PmlMatInv[ip];
+      delete PmlMat[ip];
+   }
+   PmlMat.DeleteAll();
+   PmlMatInv.DeleteAll();
+}
+
+void PSTP::GetCutOffSol(Vector & sol, int ip, int direction) const
+{
+   int l,k;
+   l=(direction == 1)? ip+1: ip;
+   k=(direction == 1)? ip: ip+1;
+
+   Mesh * mesh1 = ovlp_prob->fespaces[l]->GetMesh();
+   Mesh * mesh2 = ovlp_prob->fespaces[k]->GetMesh();
+   
+   Vector pmin1, pmax1;
+   Vector pmin2, pmax2;
+   mesh1->GetBoundingBox(pmin1, pmax1);
+   mesh2->GetBoundingBox(pmin2, pmax2);
+
+   Array2D<double> h(dim,2);
+   
+   h[0][0] = pmin2[0] - pmin1[0];
+   h[0][1] = pmax2[0] - pmin1[0];
+   h[1][0] = pmin2[1] - pmin1[1];
+   h[1][1] = pmax2[1] - pmax1[1];
+
+   if (direction == 1)
+   {
+      h[0][0] = 0.0;
+   }
+   else if (direction == -1)
+   {
+      h[0][1] = 0.0;
+   }
+   CutOffFnCoefficient cf(CutOffFncn, pmin2, pmax2, h);
+   double * data = sol.GetData();
+
+   int m = (direction == 1) ? ip : ip+1;
+   FiniteElementSpace * fespace = ovlp_prob->PmlFespaces[m];
+   int n = fespace->GetTrueVSize();
+
+   GridFunction solgf_re(fespace, data);
+   GridFunction solgf_im(fespace, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fespace);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   sol = gf;
+}
+
+
+
+LocalDofMap::LocalDofMap(const FiniteElementCollection * fec_, MeshPartition * part1_, 
+               MeshPartition * part2_):fec(fec_), part1(part1_), part2(part2_)
+{
+   // Each overlapping patch has 2 non-overlapping subdomains
+   // Thre are n non-overlapping and and n-1 overlapping subdomains
+   int nrpatch = part2->nrpatch;
+   MFEM_VERIFY(part1->nrpatch-1 == part2->nrpatch, "Check number of subdomains");
+
+   cout << "Constructing local dof maps" << endl; 
+   map1.resize(nrpatch);
+   map2.resize(nrpatch);
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      // Get the 3 meshes involved
+      Mesh * mesh = part2->patch_mesh[ip];
+      Mesh * mesh1 = part1->patch_mesh[ip];
+      Mesh * mesh2 = part1->patch_mesh[ip+1];
+
+      // Define the fespaces
+      FiniteElementSpace fespace(mesh, fec);
+      FiniteElementSpace fespace1(mesh1, fec);
+      FiniteElementSpace fespace2(mesh2, fec);
+
+      int ndof1 = fespace1.GetTrueVSize();
+      int ndof2 = fespace2.GetTrueVSize();
+
+      map1[ip].SetSize(2*ndof1); // times 2 because it's complex
+      map2[ip].SetSize(2*ndof2); // times 2 because it's complex
+
+      // loop through the elements in the patches
+      // map 1 is constructed by the first half of elements
+      // map 2 is constructed by the second half of elements
+
+      for (int iel = 0; iel<part1->element_map[ip].Size(); ++iel)
+      {
+         // index in the overlapping mesh
+         int iel_idx = iel;
+         Array<int> ElemDofs;
+         Array<int> GlobalElemDofs;
+         fespace1.GetElementDofs(iel,ElemDofs);
+         fespace.GetElementDofs(iel_idx,GlobalElemDofs);
+         // the sizes have to match
+         MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+                     "Size inconsistency");
+         // loop through the dofs and take into account the signs;
+         int ndof = ElemDofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = ElemDofs[i];
+            int gdof_ = GlobalElemDofs[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            map1[ip][pdof] = gdof;
+            map1[ip][pdof+ndof1] = gdof+fespace.GetTrueVSize();
+         }
+      }
+      for (int iel = 0; iel<part1->element_map[ip+1].Size(); ++iel)
+      {
+         // index in the overlapping mesh
+         int k = part1->element_map[ip].Size();
+         int iel_idx = iel+k;
+         Array<int> ElemDofs;
+         Array<int> GlobalElemDofs;
+         fespace2.GetElementDofs(iel,ElemDofs);
+         fespace.GetElementDofs(iel_idx,GlobalElemDofs);
+         // the sizes have to match
+         MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+                     "Size inconsistency");
+         // loop through the dofs and take into account the signs;
+         int ndof = ElemDofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = ElemDofs[i];
+            int gdof_ = GlobalElemDofs[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            map2[ip][pdof] = gdof;
+            map2[ip][pdof+ndof2] = gdof+fespace.GetTrueVSize();
+         }
+      }
+   }
+}

examples/maxwell-solver/ST/PST.hpp

@@ -0,0 +1,62 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "ST.hpp"
+using namespace std;
+using namespace mfem;
+
+
+class LocalDofMap // Constructs dof mapbetween two partitions
+{
+   const FiniteElementCollection *fec=nullptr;
+   MeshPartition * part1=nullptr;
+   MeshPartition * part2=nullptr;
+public:
+   int nrpatch, nx, ny, nz;
+   vector<Array<int>> map1;
+   vector<Array<int>> map2;
+   // constructor
+   LocalDofMap(const FiniteElementCollection * fec_, MeshPartition * part1_, 
+               MeshPartition * part2_);
+   ~LocalDofMap();
+};
+
+
+class PSTP : public Solver//
+{
+private:
+   int nrpatch;
+   int dim;
+   SesquilinearForm *bf=nullptr;
+   MeshPartition * povlp;
+   MeshPartition * pnovlp;
+   double omega = 0.5;
+   Coefficient * ws;
+   int nrlayers;
+   const Operator * A;
+   Vector B;
+   DofMap * ovlp_prob = nullptr;
+   DofMap * novlp_prob = nullptr;
+   LocalDofMap * lmap=nullptr;
+   Array<SparseMatrix *> PmlMat;
+   Array<KLUSolver *> PmlMatInv;
+   Array2D<double> Pmllength;
+   mutable Array<Vector * > res;
+
+   SparseMatrix * GetPmlSystemMatrix(int ip);
+   void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+   void PlotLocalSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+   void GetCutOffSolution(Vector & sol, int ip, int direction) const;
+   void GetCutOffSol(Vector & sol, int ip, int direction) const;
+
+public:
+   PSTP(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+       double omega_, Coefficient * ws_, int nrlayers_);
+   void SetLoadVector(Vector load) { B = load;}
+   virtual void SetOperator(const Operator &op) {A = &op;}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~PSTP();
+};
+
+

examples/maxwell-solver/ST/ST.cpp

@@ -0,0 +1,617 @@
+//Source Transfer Preconditioner
+
+#include "ST.hpp"
+
+DofMap::DofMap(SesquilinearForm * bf_ , MeshPartition * partition_) 
+               : bf(bf_), partition(partition_)
+{
+   int partition_kind = partition->partition_kind;
+   MFEM_VERIFY(partition_kind == 1, "Check Partition kind");
+   fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   const FiniteElementCollection * fec = fespace->FEColl();
+   nrpatch = partition->nrpatch;
+
+   fespaces.SetSize(nrpatch);
+
+   Dof2GlobalDof.resize(nrpatch);
+
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // create finite element spaces for each patch 
+      fespaces[ip] = new FiniteElementSpace(partition->patch_mesh[ip],fec);
+
+      // construct the patch tdof to global tdof map
+      int nrdof = fespaces[ip]->GetTrueVSize();
+      Dof2GlobalDof[ip].SetSize(2*nrdof);
+
+      // loop through the elements in the patch
+      for (int iel = 0; iel<partition->element_map[ip].Size(); ++iel)
+      {
+         // index in the global mesh
+         int iel_idx = partition->element_map[ip][iel];
+         // get the dofs of this element
+         Array<int> ElemDofs;
+         Array<int> GlobalElemDofs;
+         fespaces[ip]->GetElementDofs(iel,ElemDofs);
+         fespace->GetElementDofs(iel_idx,GlobalElemDofs);
+         // the sizes have to match
+         MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+                     "Size inconsistency");
+         // loop through the dofs and take into account the signs;
+         int ndof = ElemDofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = ElemDofs[i];
+            int gdof_ = GlobalElemDofs[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            Dof2GlobalDof[ip][pdof] = gdof;
+            Dof2GlobalDof[ip][pdof+nrdof] = gdof+fespace->GetTrueVSize();
+         }
+      }
+   }
+}
+
+
+DofMap::DofMap(SesquilinearForm * bf_ , MeshPartition * partition_, int nrlayers) 
+               : bf(bf_), partition(partition_)
+{
+   int partition_kind = partition->partition_kind;
+   fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   const FiniteElementCollection * fec = fespace->FEColl();
+   nrpatch = partition->nrpatch;
+
+   fespaces.SetSize(nrpatch);
+   PmlMeshes.SetSize(nrpatch);
+   // Extend patch meshes to include pml
+   for  (int ip = 0; ip<nrpatch; ip++)
+   {
+      Array<int> directions;
+      if (ip > 0)
+      {
+         for (int i=0; i<nrlayers; i++)
+         {
+            directions.Append(-1);
+         }
+      }
+      if (ip < nrpatch-1)
+      {
+         for (int i=0; i<nrlayers; i++)
+         {
+            if (partition_kind == 3) directions.Append(1);
+         }
+      }
+      PmlMeshes[ip] = ExtendMesh(partition->patch_mesh[ip],directions);
+   }
+
+   // Save PML_meshes
+   string meshpath;
+   string solpath;
+   if (partition_kind == 3) 
+   {
+      meshpath = "output/mesh_ovlp_pml.";
+      solpath = "output/sol_ovlp_pml.";
+   }
+   else if (partition_kind == 4)
+   {
+      meshpath = "output/mesh_novlp_pml.";
+      solpath = "output/sol_novlp_pml.";
+   }
+   else
+   {
+      MFEM_ABORT("This partition kind not supported yet");
+   }
+   
+   // SaveMeshPartition(PmlMeshes, meshpath, solpath);
+
+   PmlFespaces.SetSize(nrpatch);
+   Dof2GlobalDof.resize(nrpatch);
+   Dof2PmlDof.resize(nrpatch);
+
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // create finite element spaces for each patch 
+      fespaces[ip] = new FiniteElementSpace(partition->patch_mesh[ip],fec);
+      PmlFespaces[ip] = new FiniteElementSpace(PmlMeshes[ip],fec);
+
+      // construct the patch tdof to global tdof map
+      int nrdof = fespaces[ip]->GetTrueVSize();
+      Dof2GlobalDof[ip].SetSize(2*nrdof);
+      Dof2PmlDof[ip].SetSize(2*nrdof);
+
+      // build dof maps between patch and extended patch
+      // loop through the patch elements and constract the dof map
+      // The same elements in the extended mesh have the same ordering (but not the dofs)
+
+      // loop through the elements in the patch
+      for (int iel = 0; iel<partition->element_map[ip].Size(); ++iel)
+      {
+         // index in the global mesh
+         int iel_idx = partition->element_map[ip][iel];
+         // get the dofs of this element
+         Array<int> ElemDofs;
+         Array<int> PmlElemDofs;
+         Array<int> GlobalElemDofs;
+         fespaces[ip]->GetElementDofs(iel,ElemDofs);
+         PmlFespaces[ip]->GetElementDofs(iel,PmlElemDofs);
+         fespace->GetElementDofs(iel_idx,GlobalElemDofs);
+         // the sizes have to match
+         MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+                     "Size inconsistency");
+         MFEM_VERIFY(ElemDofs.Size() == PmlElemDofs.Size(),
+                     "Size inconsistency");            
+         // loop through the dofs and take into account the signs;
+         int ndof = ElemDofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = ElemDofs[i];
+            int gdof_ = GlobalElemDofs[i];
+            int pmldof_ = PmlElemDofs[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            int pmldof = (pmldof_ >= 0) ? pmldof_ : abs(pmldof_) - 1;
+
+            Dof2GlobalDof[ip][pdof] = gdof;
+            Dof2GlobalDof[ip][pdof+nrdof] = gdof+fespace->GetTrueVSize();
+            Dof2PmlDof[ip][pdof] = pmldof;
+            Dof2PmlDof[ip][pdof+nrdof] = pmldof+PmlFespaces[ip]->GetTrueVSize();
+         }
+      }
+   }
+}
+
+
+STP::STP(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+         double omega_, Coefficient * ws_,  int nrlayers_)
+   : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), 
+     bf(bf_), Pmllength(Pmllength_), omega(omega_), ws(ws_), nrlayers(nrlayers_)
+{
+   Mesh * mesh = bf->FESpace()->GetMesh();
+   dim = mesh->Dimension();
+
+   // ----------------- Step 1 --------------------
+   // Introduce 2 layered partitios of the domain 
+   // 
+   int partition_kind;
+   // 1. Non ovelapping 
+   partition_kind = 4; // Ovelapping partition for the halfspace problem
+   pnovlp = new MeshPartition(mesh, partition_kind);
+
+   // 2. Overlapping to the right
+   partition_kind = 3; // Ovelapping partition for the full space
+   povlp = new MeshPartition(mesh, partition_kind);
+   nrpatch = pnovlp->nrpatch;
+   //
+   // ----------------- Step 1a -------------------
+   // Save the partition for visualization
+   // SaveMeshPartition(povlp->patch_mesh, "output/mesh_ovlp.", "output/sol_ovlp.");
+   // SaveMeshPartition(pnovlp->patch_mesh, "output/mesh_novlp.", "output/sol_novlp.");
+
+   // ------------------Step 2 --------------------
+   // Construct the dof maps from subdomains to global (for the extended and not)
+   // The non ovelapping is extended on the left by pml (halfspace problem)
+   // The overlapping is extended left and right by pml (unbounded domain problem)
+   novlp_prob = new DofMap(bf,pnovlp,nrlayers);
+   ovlp_prob  = new DofMap(bf,povlp,nrlayers); 
+
+   // ------------------Step 3 --------------------
+   // Assemble the PML Problem matrices and factor them
+   PmlMat.SetSize(nrpatch);
+   PmlMatInv.SetSize(nrpatch);
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      PmlMat[ip] = GetPmlSystemMatrix(ip);
+      PmlMatInv[ip] = new KLUSolver;
+      PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
+   }
+
+   HalfSpaceMat.SetSize(nrpatch);
+   HalfSpaceMatInv.SetSize(nrpatch);
+   HalfSpaceForms.SetSize(nrpatch);
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      HalfSpaceMat[ip] = GetHalfSpaceSystemMatrix(ip);
+      HalfSpaceMatInv[ip] = new KLUSolver;
+      HalfSpaceMatInv[ip]->SetOperator(*HalfSpaceMat[ip]);
+   }
+}
+
+SparseMatrix * STP::GetPmlSystemMatrix(int ip)
+{
+   double h = GetUniformMeshElementSize(ovlp_prob->PmlMeshes[ip]);
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+   if (ip == nrpatch-1 || ip == 0) 
+   {
+      length[0][0] = Pmllength[0][0];
+      length[0][1] = Pmllength[0][1];
+   }
+   length[1][0] = Pmllength[1][0];
+   length[1][1] = Pmllength[1][1];
+
+   CartesianPML pml(ovlp_prob->PmlMeshes[ip], length);
+   pml.SetOmega(omega);
+
+   Array <int> ess_tdof_list;
+   if (ovlp_prob->PmlMeshes[ip]->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(ovlp_prob->PmlMeshes[ip]->bdr_attributes.Max());
+      ess_bdr = 1;
+      ovlp_prob->PmlFespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+
+   ProductCoefficient c2_re(c2_re0, *ws);
+   ProductCoefficient c2_im(c2_im0, *ws);
+
+   SesquilinearForm a(ovlp_prob->PmlFespaces[ip],ComplexOperator::HERMITIAN);
+
+   a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   a.AddDomainIntegrator(new MassIntegrator(c2_re),
+                         new MassIntegrator(c2_im));
+   a.Assemble();
+
+   OperatorPtr Alocal;
+   a.FormSystemMatrix(ess_tdof_list,Alocal);
+   ComplexSparseMatrix * AZ_ext = Alocal.As<ComplexSparseMatrix>();
+   SparseMatrix * Mat = AZ_ext->GetSystemMatrix();
+   Mat->Threshold(0.0);
+   return Mat;
+}
+
+SparseMatrix * STP::GetHalfSpaceSystemMatrix(int ip)
+{
+   double h = GetUniformMeshElementSize(novlp_prob->PmlMeshes[ip]);
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+   if (ip == nrpatch-1 || ip == 0) 
+   {
+      length[0][0] = Pmllength[0][0];
+   }
+   length[1][0] = Pmllength[1][0];
+   length[1][1] = Pmllength[1][1];
+   length[0][1] = 0.0;
+
+   CartesianPML pml(novlp_prob->PmlMeshes[ip], length);
+   pml.SetOmega(omega);
+
+   Array <int> ess_tdof_list;
+   if (novlp_prob->PmlMeshes[ip]->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(ovlp_prob->PmlMeshes[ip]->bdr_attributes.Max());
+      ess_bdr = 1;
+      novlp_prob->PmlFespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+
+   ProductCoefficient c2_re(c2_re0, *ws);
+   ProductCoefficient c2_im(c2_im0, *ws);
+
+   HalfSpaceForms[ip] = new SesquilinearForm(novlp_prob->PmlFespaces[ip],
+                        ComplexOperator::HERMITIAN);
+   HalfSpaceForms[ip]->AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   HalfSpaceForms[ip]->AddDomainIntegrator(new MassIntegrator(c2_re),
+                         new MassIntegrator(c2_im));
+   HalfSpaceForms[ip]->Assemble();
+
+   OperatorPtr Alocal;
+   HalfSpaceForms[ip]->FormSystemMatrix(ess_tdof_list, Alocal);
+
+   ComplexSparseMatrix * AZ_ext = Alocal.As<ComplexSparseMatrix>();
+   SparseMatrix * Mat = AZ_ext->GetSystemMatrix();
+   Mat->Threshold(0.0);
+   return Mat;
+}
+
+void STP::SolveHalfSpaceLinearSystem(int ip, Vector &x, Vector & load) const
+{
+   Array <int> ess_tdof_list;
+   if (novlp_prob->PmlMeshes[ip]->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(ovlp_prob->PmlMeshes[ip]->bdr_attributes.Max());
+      ess_bdr = 1;
+      novlp_prob->PmlFespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   OperatorHandle Ah;
+   Vector X,Modload;
+
+   HalfSpaceForms[ip]->FormLinearSystem(ess_tdof_list,x,load,
+   Ah,X,Modload);
+   HalfSpaceMatInv[ip]->Mult(Modload,X);
+   HalfSpaceForms[ip]->RecoverFEMSolution(X,Modload,x);
+}
+
+
+void STP::Mult(const Vector &r, Vector &z) const
+{
+   z = 0.0; 
+   res.SetSize(nrpatch);
+   Vector rnew(r);
+   Vector znew(z);
+   Vector z1(z);
+   Vector raux(znew.Size());
+   Vector res_local, sol_local;
+   znew = 0.0;
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   // socketstream subsol_sock1(vishost, visport);
+   // socketstream subsol_sock(vishost, visport);
+
+   // source transfer algorithm
+   for (int ip = 0; ip < nrpatch; ip++)
+   {
+      Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+      Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip];
+      int ndofs = Dof2GlobalDof->Size();
+      res_local.SetSize(ndofs);
+      sol_local.SetSize(ndofs);
+
+      rnew.GetSubVector(*Dof2GlobalDof, res_local);
+
+      // store residuals for the non overlapping partition
+      Array<int> * nDof2GlobalDof;
+      if (ip == nrpatch-1 )
+      {
+         nDof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+      }
+      else
+      {
+         nDof2GlobalDof = &novlp_prob->Dof2GlobalDof[ip];
+      }
+      int mdofs = nDof2GlobalDof->Size();
+      res[ip] = new Vector(mdofs);
+      rnew.GetSubVector(*nDof2GlobalDof, *res[ip]);
+      if (ip == nrpatch-1) continue;
+
+      //-----------------------------------------------
+      // Extend by zero to the extended mesh
+      int nrdof_ext = PmlMat[ip]->Height();
+         
+      Vector res_ext(nrdof_ext); res_ext = 0.0;
+      Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+
+      res_ext.SetSubVector(*Dof2PmlDof,res_local.GetData());
+      PmlMatInv[ip]->Mult(res_ext, sol_ext);
+
+      sol_ext.GetSubVector(*Dof2PmlDof,sol_local);
+
+      znew = 0.0;
+      znew.SetSubVector(*Dof2GlobalDof,sol_local);
+
+      // PlotSolution(znew, subsol_sock,ip); cin.get();
+
+      // z.AddElementVector(*Dof2GlobalDof,sol_local);
+      int direction = 1;
+      GetCutOffSolution(znew, ip, direction);
+
+      z1+=znew;
+      // PlotSolution(z, subsol_sock,1); cin.get();
+
+      A->Mult(znew, raux);
+      rnew -= raux;
+      // PlotSolution(rnew, subsol_sock,ip); cin.get();
+
+   }
+
+   // solution stage
+   // First solve the nrpatch-1 problem (last subdomain)
+   // extend residual to all around pml
+   int nrdof_ext = PmlMat[nrpatch-1]->Height();
+   Vector res_ext(nrdof_ext); res_ext = 0.0;
+   Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+   Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[nrpatch-1];
+   Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[nrpatch-1];
+   res_ext.SetSubVector(*Dof2PmlDof,*res[nrpatch-1]);
+   PmlMatInv[nrpatch-1]->Mult(res_ext, sol_ext);
+   int ndofs = Dof2GlobalDof->Size();
+   sol_local.SetSize(ndofs);
+   sol_ext.GetSubVector(*Dof2PmlDof,sol_local);
+   znew = 0.0;
+   znew.SetSubVector(*Dof2GlobalDof,sol_local);
+   z.SetSubVector(*Dof2GlobalDof,sol_local);
+   z1+=znew;
+
+
+   // z = z1;
+
+   // PlotSolution(z1, subsol_sock1,0); cin.get();
+
+   // backward sweep for half space problems
+
+   Vector z_loc(z.Size());
+   for (int ip = nrpatch-2; ip >= 0; ip--)
+   {
+      // Get solution from previous layer
+      Array<int> * Dof2GlobalDof = &novlp_prob->Dof2GlobalDof[ip];
+      Array<int> * Dof2PmlDof = &novlp_prob->Dof2PmlDof[ip];
+      int ndof = Dof2GlobalDof->Size();
+      Vector sol_loc(ndof);      
+      znew.GetSubVector(* Dof2GlobalDof, sol_loc);
+
+      // extend by zero to the halfspace pml problem
+      FiniteElementSpace * subfespace = novlp_prob->PmlFespaces[ip];
+      int mdof = 2*subfespace->GetTrueVSize();
+      Vector sol_pml(mdof); sol_pml = 0.0;
+      sol_pml.SetSubVector(* Dof2PmlDof, sol_loc);
+      Mesh * submesh = subfespace->GetMesh();
+
+      // Set to zero the non boundary dofs
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr(submesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      subfespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      int n = ess_tdof_list.Size();
+      for (int i=0; i<n; i++)
+      {
+         ess_tdof_list.Append(ess_tdof_list[i]+mdof/2);
+      }
+      sol_pml.SetSubVectorComplement(ess_tdof_list,0.0);
+      
+      // Set up the halfspace problem
+      // extend the residual by zero to pml region
+      Vector pmlres(sol_pml.Size()); pmlres = 0.0;
+      pmlres.SetSubVector(* Dof2PmlDof,*res[ip]);
+      SolveHalfSpaceLinearSystem(ip, sol_pml, pmlres);
+
+      sol_loc = 0.0;
+      sol_pml.GetSubVector(* Dof2PmlDof, sol_loc);
+
+      z_loc = 0.0;
+      z_loc.SetSubVector(* Dof2GlobalDof, sol_loc);
+      znew = z_loc;
+      z.SetSubVector(* Dof2GlobalDof, sol_loc);
+   }
+      // PlotSolution(z, subsol_sock,1); cin.get();
+
+}
+
+void STP::PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const
+{
+   FiniteElementSpace * fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   GridFunction gf(fespace);
+   double * data = sol.GetData();
+   gf.SetData(data);
+   
+   string keys;
+   if (ip == 0) keys = "keys mrRljc\n";
+   sol_sock << "solution\n" << *mesh << gf << keys << flush;
+}
+
+void STP::GetCutOffSolution(Vector & sol, int ip, int direction) const
+{
+   int l,k;
+   l=(direction == 1)? ip+1: ip;
+   k=(direction == 1)? ip: ip+1;
+
+   Mesh * mesh1 = ovlp_prob->fespaces[l]->GetMesh();
+   Mesh * mesh2 = ovlp_prob->fespaces[k]->GetMesh();
+   
+   Vector pmin1, pmax1;
+   Vector pmin2, pmax2;
+   mesh1->GetBoundingBox(pmin1, pmax1);
+   mesh2->GetBoundingBox(pmin2, pmax2);
+
+   Array2D<double> h(dim,2);
+   
+   h[0][0] = pmin2[0] - pmin1[0];
+   h[0][1] = pmax2[0] - pmin1[0];
+   h[1][0] = pmin2[1] - pmin1[1];
+   h[1][1] = pmax2[1] - pmax1[1];
+
+   if (direction == 1)
+   {
+      h[0][0] = 0.0;
+   }
+   else if (direction == -1)
+   {
+      h[0][1] = 0.0;
+   }
+   CutOffFnCoefficient cf(CutOffFncn, pmin2, pmax2, h);
+   double * data = sol.GetData();
+
+   FiniteElementSpace * fespace = bf->FESpace();
+   int n = fespace->GetTrueVSize();
+
+   GridFunction solgf_re(fespace, data);
+   GridFunction solgf_im(fespace, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fespace);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   sol = gf;
+}
+
+STP::~STP()
+{
+   for (int ip = 0; ip<nrpatch; ++ip)
+   {
+      delete HalfSpaceForms[ip];
+      delete HalfSpaceMat[ip];
+      delete HalfSpaceMatInv[ip];
+      delete PmlMatInv[ip];
+      delete PmlMat[ip];
+   }
+   HalfSpaceForms.DeleteAll();
+   HalfSpaceMat.DeleteAll();
+   HalfSpaceMatInv.DeleteAll();
+   PmlMat.DeleteAll();
+   PmlMatInv.DeleteAll();
+}
+
+
+double CutOffFncn(const Vector &x, const Vector & pmin, const Vector & pmax, const Array2D<double> & h_)
+{
+   int dim = pmin.Size();
+   Vector h0(dim);
+   Vector h1(dim);
+   for (int i=0; i<dim; i++)
+   {
+      h0(i) = h_[i][0];
+      h1(i) = h_[i][1];
+   }
+   Vector x0(dim);
+   x0 = pmax; x0-=h1;
+   Vector x1(dim);
+   x1 = pmin; x1+=h0;
+
+   double f = 1.0;
+
+   for (int i = 0; i<dim; i++)
+   {
+      double val = 1.0;
+      if( x(i) > pmax(i) || x(i) < pmin(i))
+      {
+         val = 0.0;
+      }  
+      else if (x(i) <= pmax(i) && x(i) >= x0(i))
+      {
+         if(x0(i)-pmax(i) != 0.0)
+            val = (x(i)-pmax(i))/(x0(i)-pmax(i)); 
+      }
+      else if (x(i) >= pmin(i) && x(i) <= x1(i))
+      {
+         if (x1(i)-pmin(i) != 0.0)
+            val = (x(i)-pmin(i))/(x1(i)-pmin(i)); 
+      }
+      else
+      {
+         val = 1.0;
+      }
+      f *= val;
+   }
+   return f;
+}

examples/maxwell-solver/ST/ST.hpp

@@ -0,0 +1,96 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "complex_additive_schwarz.hpp"
+using namespace std;
+using namespace mfem;
+
+
+
+// Function coefficient that takes the boundingbox of the mesh as an input
+class CutOffFnCoefficient : public Coefficient
+{
+private:
+   double (*Function)(const Vector &, const Vector &, const Vector &, const Array2D<double> &);
+   Vector pmin, pmax;
+   Array2D<double> h; // specify the with of the cutoff function (h in each direction)
+   
+
+public:
+   CutOffFnCoefficient(double (*F)(const Vector &, const Vector &, const Vector &, const Array2D<double> &), 
+                             const Vector & pmin_, const Vector & pmax_, Array2D<double> & h_)
+      : Function(F), pmin(pmin_), pmax(pmax_), h(h_)
+   {}
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+   {
+      double x[3];
+      Vector transip(x, 3);
+      T.Transform(ip, transip);
+      return ((*Function)(transip, pmin, pmax, h));
+   }
+};
+
+double CutOffFncn(const Vector &x, const Vector &pmax, const Vector &pmin, const Array2D<double> & h_);
+
+
+class DofMap // Constructs dof maps for a given partition
+{
+   FiniteElementSpace *fespace=nullptr;
+   SesquilinearForm * bf=nullptr;
+   MeshPartition * partition=nullptr;
+public:
+   int nrpatch, nx, ny, nz;
+   vector<Array<int>> Dof2GlobalDof;
+   vector<Array<int>> Dof2PmlDof;
+   Array<Mesh *> PmlMeshes;
+   Array<FiniteElementSpace *> fespaces;
+   Array<FiniteElementSpace *> PmlFespaces;
+   // constructor
+   // Non PML contructor dof map
+   DofMap(SesquilinearForm * bf_, MeshPartition * partition_);
+   // PML
+   DofMap(SesquilinearForm * bf_ , MeshPartition * partition_, int nrlayers);
+   ~DofMap();
+};
+
+
+class STP : public Solver//
+{
+private:
+   int nrpatch;
+   int dim;
+   SesquilinearForm *bf=nullptr;
+   MeshPartition * povlp;
+   MeshPartition * pnovlp;
+   double omega = 0.5;
+   Coefficient * ws;
+   int nrlayers;
+   const Operator * A=nullptr;
+   Vector B;
+   DofMap * ovlp_prob = nullptr;
+   DofMap * novlp_prob = nullptr;
+   Array<SesquilinearForm *> HalfSpaceForms;
+   Array<SparseMatrix *> PmlMat;
+   Array<SparseMatrix *> HalfSpaceMat;
+   Array<KLUSolver *> PmlMatInv;
+   Array<KLUSolver *> HalfSpaceMatInv;
+   Array2D<double> Pmllength;
+   mutable Array<Vector * > res;
+
+   SparseMatrix * GetPmlSystemMatrix(int ip);
+   SparseMatrix * GetHalfSpaceSystemMatrix(int ip);
+   void SolveHalfSpaceLinearSystem(int ip, Vector & x, Vector & load) const;
+   void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+   void GetCutOffSolution(Vector & sol, int ip, int direction) const;
+
+public:
+   STP(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+       double omega_, Coefficient * ws_, int nrlayers_);
+   void SetLoadVector(Vector load) { B = load;}
+   virtual void SetOperator(const Operator &op) {A = &op;}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~STP();
+};
+
+

examples/maxwell-solver/ST/SourceTransfer.cpp

@@ -0,0 +1,473 @@
+#include "SourceTransfer.hpp"
+
+STPmlPatchAssembly::STPmlPatchAssembly(SesquilinearForm * bf_, Array<int> & ess_tdofs, 
+                                   double omega_, int nrlayers_, int part) 
+                                   : bf(bf_), omega(omega_), nrlayers(nrlayers_)
+{
+   fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   int dim = mesh->Dimension();
+   const FiniteElementCollection *fec = fespace->FEColl();
+
+   p = new MeshPartition(mesh, part);
+   nx = p->nx;
+   ny = p->ny;
+   nz = p->nz;
+   // SaveMeshPartition(p->patch_mesh);
+   nrpatch = p->nrpatch;
+   patch_fespaces.SetSize(nrpatch);
+   patch_meshes_ext.SetSize(nrpatch);
+   patch_fespaces_ext.SetSize(nrpatch);
+   dof2extdof_map.resize(nrpatch);
+   patch_dof_map.resize(nrpatch);
+   patch_mat.SetSize(nrpatch);
+   patch_mat_ext.SetSize(nrpatch);
+   patch_mat_inv.SetSize(nrpatch);
+   patch_mat_inv_ext.SetSize(nrpatch);
+   ess_tdof_list.resize(nrpatch);
+   ess_tdof_list_ext.resize(nrpatch);
+
+   // construct extended meshes for the pml
+
+   int ip = -1;
+   for (int kz = 0; kz<nz; kz++)
+   {
+      for (int ky = 0; ky<ny; ky++)
+      {
+         for (int kx = 0; kx<nx; kx++)
+         {
+            ip++;
+            Array<int> ext_directions;
+            for (int j=0; j<nrlayers; ++j)// one more layer of extension (epsilon layer)
+            {
+               for (int comp=0; comp<dim; ++comp)
+               {
+                  if (comp == 0 && kx != 0)
+                  {
+                     ext_directions.Append(-comp-1);
+                  }
+                  if (comp == 0 && kx != nx-1)
+                  {
+                     ext_directions.Append(comp+1);
+                  }
+                  if (comp == 1 && ky != 0)
+                  {
+                     ext_directions.Append(-comp-1);
+                  }
+                  if (comp == 1 && ky != ny-1)
+                  {
+                     ext_directions.Append(comp+1);
+                  }
+                  if (comp == 2 && kz != 0)
+                  {
+                     // ext_directions.Append(-comp-1);
+                  }
+                  if (comp == 2 && kz != nz-1)
+                  {
+                     // ext_directions.Append(comp+1);
+                  }
+               }
+            }
+            if (ip < nrpatch-1)
+            {
+               // ext_directions.Append(1);
+               // ext_directions.Append(1);
+               // ext_directions.Append(1);
+               // ext_directions.Append(1);
+            }
+            patch_meshes_ext[ip] = ExtendMesh(p->patch_mesh[ip],ext_directions);
+         }
+      }
+   }
+   // SaveMeshPartition(patch_meshes_ext, "output/ext_mesh.", "output/ext_sol.");
+   // // cout << p->patch_mesh[0]->GetNE() << endl;
+
+
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // create finite element spaces for each patch // This might be avoided
+      patch_fespaces[ip] = new FiniteElementSpace(p->patch_mesh[ip],fec);
+      // create finite element spaces on the extented (PML) meshes
+      patch_fespaces_ext[ip] = new FiniteElementSpace(patch_meshes_ext[ip],fec);
+
+      // construct the patch tdof to global tdof map
+      int nrdof = patch_fespaces[ip]->GetTrueVSize();
+      patch_dof_map[ip].SetSize(2*nrdof);
+      dof2extdof_map[ip].SetSize(2*nrdof);
+
+      // build dof maps between patch and extended patch
+      // loop through the patch elements and constract the dof map
+      // The same elements in the extended mesh have the same ordering (but not the dofs)
+
+      // loop through the elements in the patch
+      for (int iel = 0; iel<p->element_map[ip].Size(); ++iel)
+      {
+         // index in the global mesh
+         int iel_idx = p->element_map[ip][iel];
+         // get the dofs of this element
+         Array<int> patch_elem_dofs;
+         Array<int> patch_elem_dofs_ext;
+         Array<int> global_elem_dofs;
+         patch_fespaces[ip]->GetElementDofs(iel,patch_elem_dofs);
+         patch_fespaces_ext[ip]->GetElementDofs(iel,patch_elem_dofs_ext);
+         fespace->GetElementDofs(iel_idx,global_elem_dofs);
+         // the sizes have to match
+         MFEM_VERIFY(patch_elem_dofs.Size() == global_elem_dofs.Size(),
+                     "Size inconsistency");
+         MFEM_VERIFY(patch_elem_dofs.Size() == patch_elem_dofs_ext.Size(),
+                     "Size inconsistency");            
+         // loop through the dofs and take into account the signs;
+         int ndof = patch_elem_dofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = patch_elem_dofs[i];
+            int gdof_ = global_elem_dofs[i];
+            int extdof_ = patch_elem_dofs_ext[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            int extdof = (extdof_ >= 0) ? extdof_ : abs(extdof_) - 1;
+            patch_dof_map[ip][pdof] = gdof;
+            patch_dof_map[ip][pdof+nrdof] = gdof+fespace->GetTrueVSize();
+            dof2extdof_map[ip][pdof] = extdof;
+            dof2extdof_map[ip][pdof+nrdof] = extdof+patch_fespaces_ext[ip]->GetTrueVSize();
+         }
+      }
+      // // Define the patch bilinear form and apply boundary conditions (only the LHS)
+      // Array <int> ess_temp_list;
+      // if (p->patch_mesh[ip]->bdr_attributes.Size())
+      // {
+      //    Array<int> ess_bdr(p->patch_mesh[ip]->bdr_attributes.Max());
+      //    ess_bdr = 0;
+      //    patch_fespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_temp_list);
+      // }
+
+      Array <int> ess_list_ext;
+      if (patch_meshes_ext[ip]->bdr_attributes.Size())
+      {
+         Array<int> ess_bdr(patch_meshes_ext[ip]->bdr_attributes.Max());
+         ess_bdr = 1;
+         patch_fespaces_ext[ip]->GetEssentialTrueDofs(ess_bdr, ess_list_ext);
+      }
+      ess_tdof_list_ext[ip] = ess_list_ext;
+
+      // // Adjust the essential tdof list for each patch
+      // for (int i=0; i<ess_temp_list.Size(); i++)
+      // {
+      //    int ldof = ess_temp_list[i];
+      //    int tdof = patch_dof_map[ip][ldof];
+      //    // check the kind of this tdof
+      //    if (!global_tdofs[tdof]) ess_tdof_list[ip].Append(ldof);
+      // }
+
+      // SesquilinearForm a(patch_fespaces[ip], &bf->real(), &bf->imag());
+      
+      //-----------------PML FORMULATION----------------------------
+      Array2D<double> length(dim,2);
+      double h = GetUniformMeshElementSize(patch_meshes_ext[ip]);
+      length = h*(nrlayers);
+
+      if (ip < nrpatch-1)
+      {
+         // length(0,1) = h*(nrlayers+4);
+      }
+      // if (ip != 0) 
+      // {
+      //    length(0,0) = 0.0;
+      // }
+      // // length = h * nrlayers;
+      // // if (ip != 0)
+      // // {
+      // //    length(0,0) = 0.0;
+      // //    length(1,0) = 0.0;
+      // // }
+      // // length(0,1) = h * nrlayers;
+      // // length(1,1) = h * nrlayers;
+      // // if (ip == 1 || ip == 2 || ip == 3) length(1,0) = h * nrlayers;
+      // // if (ip == 4 || ip == 8 || ip == 12) length(0,0) = h * nrlayers;
+
+
+      CartesianPML pml(patch_meshes_ext[ip], length);
+      pml.SetOmega(omega);
+
+      ConstantCoefficient one(1.0);
+      ConstantCoefficient sigma(-pow(omega, 2));
+
+      PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+      PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+      PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+      PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+      ProductCoefficient c2_re(sigma, detJ_re);
+      ProductCoefficient c2_im(sigma, detJ_im);
+
+      SesquilinearForm a_ext(patch_fespaces_ext[ip],ComplexOperator::HERMITIAN);
+
+      a_ext.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                                new DiffusionIntegrator(c1_im));
+      a_ext.AddDomainIntegrator(new MassIntegrator(c2_re),
+                                new MassIntegrator(c2_im));
+
+      //------------------------------------------------------------
+
+      // a.Assemble();
+      a_ext.Assemble();
+      // OperatorPtr Alocal;
+      // a.FormSystemMatrix(ess_tdof_list[ip],Alocal);
+      // ComplexSparseMatrix * AZ = Alocal.As<ComplexSparseMatrix>();
+      // patch_mat[ip] = AZ->GetSystemMatrix();
+      // patch_mat[ip]->Threshold(0.0);
+      // // Save the inverse
+      // patch_mat_inv[ip] = new KLUSolver;
+      // patch_mat_inv[ip]->SetOperator(*patch_mat[ip]);
+
+
+      OperatorPtr Alocal_ext;
+      a_ext.FormSystemMatrix(ess_list_ext,Alocal_ext);
+      ComplexSparseMatrix * AZ_ext = Alocal_ext.As<ComplexSparseMatrix>();
+      patch_mat_ext[ip] = AZ_ext->GetSystemMatrix();
+      patch_mat_ext[ip]->Threshold(0.0);
+      patch_mat_inv_ext[ip] = new KLUSolver;
+      patch_mat_inv_ext[ip]->SetOperator(*patch_mat_ext[ip]);
+
+
+      // delete patch_fespaces[ip];
+      // delete patch_fespaces_ext[ip];
+   }
+   // delete p;
+}
+
+STPmlPatchAssembly::~STPmlPatchAssembly()
+{
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // delete patch_fespaces[ip]; patch_fespaces[ip]=nullptr;
+      delete patch_fespaces[ip];
+      delete patch_fespaces_ext[ip];
+      delete patch_meshes_ext[ip];
+      patch_meshes_ext[ip]=nullptr;
+      // delete patch_mat_inv[ip];
+      delete patch_mat_inv_ext[ip];
+      // patch_mat_inv[ip]=nullptr;
+      patch_mat_inv_ext[ip]=nullptr;
+      // delete patch_mat[ip];
+      delete patch_mat_ext[ip];
+      // patch_mat[ip]=nullptr;
+      patch_mat_ext[ip]=nullptr;
+   }
+   // patch_fespaces.DeleteAll();
+   patch_meshes_ext.DeleteAll();
+   patch_mat_ext.DeleteAll();
+   // patch_mat.DeleteAll();
+   // patch_mat_inv.DeleteAll();
+   // patch_mat_inv.DeleteAll();
+   // delete p;
+}
+
+
+
+
+void SourceTransferPrecond::GetCutOffSolution(Vector & sol, int ip) const
+{
+
+   Mesh * mesh = p->patch_fespaces[ip]->GetMesh();
+   int n = p->patch_fespaces[ip]->GetTrueVSize();
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+   int dim = mesh->Dimension();
+   double hl = GetUniformMeshElementSize(mesh);
+   Array2D<double> h(dim,2);
+   h[0][0] = 0.0;
+   h[0][1] = hl;
+   h[1][0] = 0.0;
+   h[1][1] = 0.0;
+   CutOffFunctionCoefficient cf(CutOffFn, pmin, pmax, h);
+
+   double * data = sol.GetData();
+
+   GridFunction solgf_re(p->patch_fespaces[ip], data);
+   GridFunction solgf_im(p->patch_fespaces[ip], &data[n]);
+
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(p->patch_fespaces[ip]);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   sol = gf;
+}
+
+
+
+
+
+
+SourceTransferPrecond::SourceTransferPrecond(SesquilinearForm * bf_, Array<int> & ess_tdofs, double omega_, int nrlayers_, int i)
+   : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), bf(bf_), omega(omega_), nrlayers(nrlayers_),
+     part(i)
+{
+   p = new STPmlPatchAssembly(bf_, ess_tdofs, omega, nrlayers, part);
+   nrpatch = p->nrpatch;
+}
+
+void SourceTransferPrecond::Mult(const Vector &r, Vector &z) const
+{
+   z = 0.0;
+   Vector rnew(r);
+   Vector znew(z);
+   Vector raux(znew.Size());
+   Vector res_local, sol_local;
+   Array<int> visit(znew.Size());
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+
+   // zero out sources from other subdomains
+   // save the first subdomain
+   // rnew = 0.0;
+   // Array<int> * dof_map0 = &p->patch_dof_map[0];
+   // int ndofs = dof_map0->Size();
+   // res_local.SetSize(ndofs);
+   // r.GetSubVector(*dof_map0, res_local);
+   // rnew.SetSubVector(*dof_map0,res_local.GetData());
+
+
+   // socketstream sol_sock(vishost, visport);
+   // sol_sock.precision(8);
+   // socketstream res_sock(vishost, visport);
+   // res_sock.precision(8);
+   // cout << "nrpatch = " << nrpatch << endl; 
+   for (int iter = 0; iter < maxit; iter++)
+   {
+      znew = 0.0;
+      visit = 0;
+      for (int ip = 0; ip < nrpatch; ip++)
+      {
+         // cout << "ip = " << ip << endl;
+         Array<int> * dof_map = &p->patch_dof_map[ip];
+         int ndofs = dof_map->Size();
+         res_local.SetSize(ndofs);
+         sol_local.SetSize(ndofs);
+
+         rnew.GetSubVector(*dof_map, res_local);
+
+         //-----------------------------------------------
+         // Extend by zero to the extended mesh
+         int nrdof_ext = p->patch_mat_ext[ip]->Height();
+         
+         Vector res_ext(nrdof_ext); res_ext = 0.0;
+         Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+
+         res_ext.SetSubVector(p->dof2extdof_map[ip],res_local.GetData());
+
+         p->patch_mat_inv_ext[ip]->Mult(res_ext, sol_ext);
+
+
+
+         sol_ext.GetSubVector(p->dof2extdof_map[ip],sol_local);
+
+         // Smooth the solution before transfer
+         // if (ip < nrpatch-1) GetCutOffSolution(sol_local, ip);
+
+         if (type == 1) znew = 0.0;
+         znew.AddElementVector(*dof_map,sol_local);
+         // zero out the contributions to the dofs which are already updated
+         // for (int i = 0; i<ndofs; i++)
+         // {
+         //    int j = (*dof_map)[i];
+         //    if (visit[j])
+         //    {
+         //       znew(j) = 0.0;
+         //    }
+         //    else
+         //    {
+         //       visit[j] = 1;
+         //    }
+         // }
+         if (type == 1)
+         {
+            z.Add(theta, znew);
+            A->Mult(znew, raux);
+            rnew -= raux;
+         }
+         // PlotSolution(z, sol_sock, ip); cin.get();
+         // PlotSolution(rnew, res_sock, ip); cin.get();
+      }
+      if (type == 0)
+      {
+         z.Add(theta, znew);
+         A->Mult(znew, raux);
+         rnew -= raux;
+      }
+      // Update residual
+      if (iter + 1 < maxit)
+      {
+         A->Mult(znew, raux);
+         rnew -= raux;
+      }
+   }
+   // PlotSolution(rnew, sol_sock, 0); cin.get();
+}
+
+
+void SourceTransferPrecond::PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const
+{
+   FiniteElementSpace * fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   ComplexGridFunction gf(fespace);
+   bf->RecoverFEMSolution(sol,B,gf);
+   
+   string keys;
+   if (ip == 0) keys = "keys mrRljc\n";
+   sol_sock << "solution\n" << *mesh << gf.imag() << keys << flush;
+}
+
+SourceTransferPrecond::~SourceTransferPrecond(){ }
+
+
+
+double CutOffFn(const Vector &x, const Vector & pmin, const Vector & pmax, const Array2D<double> & h_)
+{
+   int dim = pmin.Size();
+   Vector h0(dim);
+   Vector h1(dim);
+   for (int i=0; i<dim; i++)
+   {
+      h0(i) = h_[i][0];
+      h1(i) = h_[i][1];
+   }
+   Vector x0(dim);
+   x0 = pmax; x0-=h1;
+   Vector x1(dim);
+   x1 = pmin; x1+=h0;
+
+   double f = 1.0;
+
+   for (int i = 0; i<dim; i++)
+   {
+      double val = 1.0;
+      if( x(i) > pmax(i) || x(i) < pmin(i))
+      {
+         val = 0.0;
+      }  
+      else if (x(i) <= pmax(i) && x(i) >= x0(i))
+      {
+         if(x0(i)-pmax(i) != 0.0)
+            val = (x(i)-pmax(i))/(x0(i)-pmax(i)); 
+      }
+      else if (x(i) >= pmin(i) && x(i) <= x1(i))
+      {
+         if (x1(i)-pmin(i) != 0.0)
+            val = (x(i)-pmin(i))/(x1(i)-pmin(i)); 
+      }
+      else
+      {
+         val = 1.0;
+      }
+      f *= val;
+   }
+   return f;
+}

examples/maxwell-solver/ST/SourceTransfer.hpp

@@ -0,0 +1,92 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "complex_additive_schwarz.hpp"
+using namespace std;
+using namespace mfem;
+
+class STPmlPatchAssembly
+{
+   FiniteElementSpace *fespace=nullptr;
+   SesquilinearForm *bf=nullptr;
+   double omega = 0.5;
+   int nrlayers = 4;
+public:
+   int nrpatch, nx, ny, nz;
+   MeshPartition * p;
+   Array<FiniteElementSpace *> patch_fespaces;
+   Array<FiniteElementSpace *> patch_fespaces_ext;
+   Array<Mesh *> patch_meshes_ext;
+   std::vector<Array<int>> patch_dof_map;
+   std::vector<Array<int>> complex_patch_dof_map;
+   std::vector<Array<int>> dof2extdof_map;
+   Array<SparseMatrix *> patch_mat;
+   Array<SparseMatrix *> patch_mat_ext;
+   Array<KLUSolver * > patch_mat_inv_ext;
+   Array<KLUSolver * > patch_mat_inv;
+   std::vector<Array<int>> ess_tdof_list;
+   std::vector<Array<int>> ess_tdof_list_ext;
+
+   // constructor
+   STPmlPatchAssembly(SesquilinearForm * bf_, Array<int> & ess_tdofs, double omega_, int nrlayers_, int part);
+
+   ~STPmlPatchAssembly();
+};
+
+
+class SourceTransferPrecond : public Solver//
+{
+private:
+   int nrpatch;
+   int maxit = 1;
+   SesquilinearForm *bf=nullptr;
+   int type = 0;
+   double theta = 0.5;
+   double omega = 0.5;
+   int nrlayers;
+   int part;
+   STPmlPatchAssembly * p;
+   const Operator * A;
+   Vector B;
+   void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+   void GetCutOffSolution(Vector & sol, int ip) const;
+
+   
+public:
+   SourceTransferPrecond(SesquilinearForm * bf_, Array<int> & ess_tdofs, double omega_, int nrlayers_, int i = 0);
+   void SetNumSmoothSteps(const int iter) { maxit = iter;}
+   void SetLoadVector(Vector load) { B = load;}
+   void SetSmoothType(int itype) { type = itype;}
+   void SetDumpingParam(const double & dump_param) {theta = dump_param;}
+   void SetOmega(const double & omega_) {omega = omega_;}
+   virtual void SetOperator(const Operator &op) {A = &op;}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~SourceTransferPrecond();
+};
+
+
+// Function coefficient that takes the boundingbox of the mesh as an input
+class CutOffFunctionCoefficient : public Coefficient
+{
+private:
+   double (*Function)(const Vector &, const Vector &, const Vector &, const Array2D<double> &);
+   Vector pmin, pmax;
+   Array2D<double> h; // specify the with of the cutoff function (h in each direction)
+   
+
+public:
+   CutOffFunctionCoefficient(double (*F)(const Vector &, const Vector &, const Vector &, const Array2D<double> &), 
+                             const Vector & pmin_, const Vector & pmax_, Array2D<double> & h_)
+      : Function(F), pmin(pmin_), pmax(pmax_), h(h_)
+   {}
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+   {
+      double x[3];
+      Vector transip(x, 3);
+      T.Transform(ip, transip);
+      return ((*Function)(transip, pmin, pmax, h));
+   }
+};
+
+double CutOffFn(const Vector &x, const Vector &pmax, const Vector &pmin, const Array2D<double> & h_);

examples/maxwell-solver/ST/additive_schwarz.cpp

@@ -0,0 +1,823 @@
+#include "additive_schwarz.hpp"
+
+
+// constructor
+OverlappingCartesianMeshPartition::OverlappingCartesianMeshPartition(Mesh *mesh_) : mesh(mesh_)
+{  // default overlap size is 2 elements 
+   int dim = mesh->Dimension();
+   int n = pow(mesh->GetNE(), 1.0/(double)dim);
+   nx = 16;
+   ny = 1;
+   nz = 1;
+   if (nx > n) 
+   {
+      nx = n;
+      MFEM_WARNING("Changed partition in the x direction to nx = " << n << endl);
+   }
+   if (ny > n) 
+   {
+      ny = n;
+      MFEM_WARNING("Changed partition in the y direction to ny = " << n << endl);
+   }
+   if (nz > n) 
+   {
+      nz = n;
+      MFEM_WARNING("Changed partition in the z direction to nz = " << n << endl);
+   } 
+   if (dim == 2) nz = 1;
+   int nxyz[3] = {nx,ny,nz};
+   nrpatch = nx*ny*nz;
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+   double h = GetUniformMeshElementSize(mesh);
+
+   element_map.resize(nrpatch);
+
+   double ppt[dim];
+   Vector pt(ppt, dim);
+   int nrelem = mesh->GetNE();
+
+   for (int el = 0; el < nrelem; el++)
+   {
+      mesh->GetElementTransformation(el)->Transform(
+         Geometries.GetCenter(mesh->GetElementBaseGeometry(el)), pt);
+      // Given the center coordinates determine the patches that this element contributes to
+      Array<int> idx0(dim);
+      Array<int> idx1(dim);
+      Array<int> idx2(dim);
+      vector<Array<int>> idx(3);
+      if (dim == 2) idx[2].Append(0);
+
+      for (int i = 0; i<dim; i++)
+      {
+         idx0[i]  = (int)floor(nxyz[i]*((pt(i) - pmin[i])/(pmax[i] - pmin[i])));
+         idx1[i] = (int)floor(nxyz[i]*((pt(i)-2*h - pmin[i])/(pmax[i] - pmin[i])));
+         idx2[i] = (int)floor(nxyz[i]*((pt(i)-h - pmin[i])/(pmax[i] - pmin[i])));
+
+         if (idx0[i] < 0) idx0[i] = 0;
+         if (idx0[i] >= nxyz[i]) idx0[i] = nxyz[i]-1;
+         
+         if (idx1[i] < 0) idx1[i] = 0;
+         if (idx1[i] >= nxyz[i]) idx1[i] = nxyz[i]-1;
+
+         if (idx2[i] < 0) idx2[i] = 0;
+         if (idx2[i] >= nxyz[i]) idx2[i] = nxyz[i]-1;
+         // convenient to put in one list
+         idx[i].Append(idx0[i]);
+         if (idx1[i] != idx0[i]) idx[i].Append(idx1[i]);
+         if (idx2[i] != idx0[i] && idx2[i] != idx1[i]) idx[i].Append(idx2[i]);
+      }
+      // Now loop through all the combinations according to the idx above
+      // in case of dim = 2 then kk = 0
+      for (int k=0; k<idx[2].Size(); k++)
+      {
+         int kk = idx[2][k];
+         for (int j=0; j<idx[1].Size(); j++)
+         {
+            int jj = idx[1][j];
+            for (int i=0; i<idx[0].Size(); i++)
+            {
+               int ii = idx[0][i];
+               int ip = kk*nxyz[0]*nxyz[1] + jj*nxyz[0]+ii;
+               element_map[ip].Append(el);
+            }
+         }
+      }
+   }
+}
+
+
+// constructor
+CartesianMeshPartition::CartesianMeshPartition(Mesh *mesh_) : mesh(mesh_)
+{
+   int dim = mesh->Dimension();
+   nx = 5;
+   ny = 1;
+   nz = 1;
+   int nxyz[3] = {nx,ny,nz};
+   nrpatch = nx*ny*nz;
+
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+
+   int nrelem = mesh->GetNE();
+   int partitioning[nrelem];
+
+   // determine the partitioning using the centers of the elements
+   double ppt[dim];
+   Vector pt(ppt, dim);
+   for (int el = 0; el < nrelem; el++)
+   {
+      mesh->GetElementTransformation(el)->Transform(
+         Geometries.GetCenter(mesh->GetElementBaseGeometry(el)), pt);
+      int part = 0;
+      for (int i = dim-1; i >= 0; i--)
+      {
+         int idx = (int)floor(nxyz[i]*((pt(i) - pmin[i])/(pmax[i] - pmin[i])));
+         if (idx < 0)
+         {
+            idx = 0;
+         }
+         if (idx >= nxyz[i])
+         {
+            idx = nxyz[i]-1;
+         }
+         part = part * nxyz[i] + idx;
+      }
+      partitioning[el] = part;
+   }
+
+   element_map.resize(nrpatch);
+   for (int iel = 0; iel < nrelem; iel++)
+   {
+      int ip = partitioning[iel];
+      element_map[ip].Append(iel);
+   }
+}
+
+
+STPOverlappingCartesianMeshPartition::STPOverlappingCartesianMeshPartition(Mesh *mesh_) : mesh(mesh_)
+{
+   int dim = mesh->Dimension();
+   nx = 9;
+   ny = 1;
+   nz = 1;
+   int nxyz[3] = {nx,ny,nz};
+   // nrpatch = nx*ny*nz;
+
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+
+   int nrelem = mesh->GetNE();
+   int partitioning[nrelem];
+
+   // determine the partitioning using the centers of the elements
+   double ppt[dim];
+   Vector pt(ppt, dim);
+   for (int el = 0; el < nrelem; el++)
+   {
+      mesh->GetElementTransformation(el)->Transform(
+         Geometries.GetCenter(mesh->GetElementBaseGeometry(el)), pt);
+      int part = 0;
+      for (int i = dim-1; i >= 0; i--)
+      {
+         int idx = (int)floor(nxyz[i]*((pt(i) - pmin[i])/(pmax[i] - pmin[i])));
+         if (idx < 0)
+         {
+            idx = 0;
+         }
+         if (idx >= nxyz[i])
+         {
+            idx = nxyz[i]-1;
+         }
+         part = part * nxyz[i] + idx;
+      }
+      partitioning[el] = part;
+   }
+   
+   // element_map.resize(nrpatch);
+   // for (int iel = 0; iel < nrelem; iel++)
+   // {
+   //    int ip = partitioning[iel];
+   //    element_map[ip].Append(iel);
+   // }
+   // // Append the next subdomain to the previous
+   // for (int ip = 0; ip<nrpatch-1; ip++)
+   // {
+   //    element_map[ip].Append(element_map[ip+1]);
+   // }
+
+
+   std::vector<Array<int>> elem_map;
+   int npatch = nx*ny*nz;
+   elem_map.resize(npatch);
+   for (int iel = 0; iel < nrelem; iel++)
+   {
+      int ip = partitioning[iel];
+      elem_map[ip].Append(iel);
+   }
+   // Append the next subdomain to the previous
+   nrpatch = nx*ny*nz-1;
+   element_map.resize(nrpatch);
+   for (int ip = 0; ip<nrpatch; ip++)
+   {
+      element_map[ip].Append(elem_map[ip]);
+      element_map[ip].Append(elem_map[ip+1]);
+   }
+
+
+
+}
+
+// constructor
+VertexMeshPartition::VertexMeshPartition(Mesh *mesh_) : mesh(mesh_)
+{
+   nrpatch = mesh->GetNV();
+   element_map.resize(nrpatch);
+   //every element will contribute to the the patches of its vertices
+   // loop through the elements
+   int nrelems = mesh->GetNE();
+   for (int iel=0; iel<nrelems; ++iel)
+   {
+      // get element vertex index
+      Array<int> vertices;
+      mesh->GetElementVertices(iel,vertices);
+      int nrvert = vertices.Size();
+      // fill in the element contribution lists
+      for (int iv = 0; iv< nrvert; ++iv)
+      {
+         int ip = vertices[iv];
+         element_map[ip].Append(iel);
+      }
+   }
+}
+
+
+MeshPartition::MeshPartition(Mesh* mesh_, int part): mesh(mesh_)
+{
+   partition_kind = part;
+   if (part == 1)
+   {
+      cout << "Non Overlapping Cartesian Partition " << endl;
+      CartesianMeshPartition partition(mesh);
+      element_map = partition.element_map;
+      nx = partition.nx;
+      ny = partition.ny;
+      nz = partition.nz;
+   }
+   // else if (part == 3 || part == 4)
+   else if (part == 2)
+   {
+      cout << "Overlapping Cartesian Partition " << endl;
+      OverlappingCartesianMeshPartition partition(mesh);
+      element_map = partition.element_map;
+      nx = partition.nx;
+      ny = partition.ny;
+      nz = partition.nz;
+   }
+   else if (part == 3 || part == 4)
+   // else if (part == 2)
+   {
+      cout << "STP Overlapping Cartesian Partition " << endl;
+      STPOverlappingCartesianMeshPartition partition(mesh);
+      element_map = partition.element_map;
+      nx = partition.nx;
+      ny = partition.ny;
+      nz = partition.nz;
+   }
+   else
+   {
+      cout << "Overlapping Vertex based partition " << endl;
+      VertexMeshPartition partition(mesh);
+      element_map = partition.element_map;
+      partition_kind = 0;
+   }
+
+   nrpatch = element_map.size();
+   int dim = mesh->Dimension();
+
+   patch_mesh.SetSize(nrpatch);
+   for (int ip = 0; ip<nrpatch; ++ip)
+   {
+      int patch_nrelems = element_map[ip].Size();
+      element_map[ip].SetSize(patch_nrelems);
+      // need to ensure that a vertex is not added more than once
+      // and that the ordering of vertices is known for when the element is added
+      // create a list of for this patch including possible repetitions
+      // loop through elements in the patch
+      Array<int> patch_vertices;
+      for (int iel=0; iel<patch_nrelems; ++iel)
+      {
+         // get the vertices list for the element
+         Array<int> elem_vertices;
+         int iel_idx = element_map[ip][iel];
+         mesh->GetElementVertices(iel_idx,elem_vertices);
+         patch_vertices.Append(elem_vertices);
+      }
+      patch_vertices.Sort();
+      patch_vertices.Unique();
+      int patch_nrvertices = patch_vertices.Size();
+
+      // create the mesh
+      patch_mesh[ip] = new Mesh(dim,patch_nrvertices,patch_nrelems);
+      // Add the vertices
+      for (int iv = 0; iv<patch_nrvertices; ++iv)
+      {
+         int vert_idx = patch_vertices[iv];
+         patch_mesh[ip]->AddVertex(mesh->GetVertex(vert_idx));
+      }
+
+      // Add the elements (for now search through all the vertices in the patch is needed)
+      for (int iel=0; iel<patch_nrelems; ++iel)
+      {
+         // get the vertices list for the element
+         Array<int> elem_vertices;
+         int iel_idx = element_map[ip][iel];
+         mesh->GetElementVertices(iel_idx,elem_vertices);
+         int nrvert = elem_vertices.Size();
+         int ind[nrvert];
+         for (int iv = 0; iv<nrvert; ++iv)
+         {
+            ind[iv] = patch_vertices.FindSorted(elem_vertices[iv]);
+         }
+         mfem::Element::Type elem_type = mesh->GetElementType(element_map[ip][iel]);
+
+         AddElementToMesh(patch_mesh[ip],elem_type,ind);
+
+      }
+      patch_mesh[ip]->FinalizeTopology();
+   }
+}
+
+
+void MeshPartition::AddElementToMesh(Mesh * mesh,mfem::Element::Type elem_type,
+                                     int * ind)
+{
+   switch (elem_type)
+   {
+      case Element::QUADRILATERAL:
+         mesh->AddQuad(ind);
+         break;
+      case Element::TRIANGLE :
+         mesh->AddTri(ind);
+         break;
+      case Element::HEXAHEDRON :
+         mesh->AddHex(ind);
+         break;
+      case Element::TETRAHEDRON :
+         mesh->AddTet(ind);
+         break;
+      default:
+         MFEM_ABORT("Unknown element type");
+         break;
+   }
+}
+
+void MeshPartition::PrintElementMap()
+{
+   mfem::out << "Element map" << endl;
+   for (int ip = 0; ip<nrpatch; ++ip)
+   {
+      mfem::out << "Patch No: " << ip;
+      mfem::out << ", element map: " ;
+      element_map[ip].Print(cout,element_map[ip].Size());
+   }
+}
+
+void SaveMeshPartition(Array<Mesh *> meshes, string mfilename, string sfilename)
+{
+   int nrmeshes = meshes.Size();
+   for (int ip = 0; ip<nrmeshes; ++ip)
+   {
+      cout << "saving mesh no " << ip << endl;
+      ostringstream mesh_name;
+      mesh_name << mfilename << setfill('0') << setw(6) << ip;
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      meshes[ip]->Print(mesh_ofs);
+      L2_FECollection L2fec(1,meshes[ip]->Dimension());
+      FiniteElementSpace L2fes(meshes[ip], &L2fec);
+      GridFunction x(&L2fes); 
+      
+      ConstantCoefficient alpha((double)ip);
+      x.ProjectCoefficient(alpha);
+      ostringstream sol_name;
+      sol_name << sfilename << setfill('0') << setw(6) << ip;
+      ofstream sol_ofs(sol_name.str().c_str());
+      x.Save(sol_ofs);
+   }
+}
+MeshPartition::~MeshPartition()
+{
+   for (int ip = 0; ip<nrpatch; ++ip)
+   {
+      delete patch_mesh[ip];
+      patch_mesh[ip] = nullptr;
+   }
+   patch_mesh.DeleteAll();
+}
+
+// constructor
+PatchAssembly::PatchAssembly(BilinearForm *bf_, Array<int> & ess_tdofs, int part) : bf(bf_)
+{
+   fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   const FiniteElementCollection *fec = fespace->FEColl();
+
+   // list of dofs to distiguish between interior/boundary and essential
+   Array<int> global_tdofs(fespace->GetTrueVSize());
+   Array<int> bdr_tdofs(fespace->GetTrueVSize());
+   global_tdofs = 0;
+   // Mark boundary dofs and ess_dofs
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, bdr_tdofs);
+   }
+
+   // mark boundary dofs
+   for (int i = 0; i<bdr_tdofs.Size(); i++) global_tdofs[bdr_tdofs[i]] = 1;
+   // overwrite flag for essential dofs
+   for (int i = 0; i<ess_tdofs.Size(); i++) global_tdofs[ess_tdofs[i]] = 0;
+
+   MeshPartition * p = new MeshPartition(mesh, part);
+   // SaveMeshPartition(p->patch_mesh);
+   nrpatch = p->nrpatch;
+   patch_fespaces.SetSize(nrpatch);
+   patch_dof_map.resize(nrpatch);
+   patch_mat.SetSize(nrpatch);
+   patch_mat_inv.SetSize(nrpatch);
+   ess_tdof_list.resize(nrpatch);
+   ess_int_tdofs.resize(nrpatch);
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // create finite element spaces for each patch
+      patch_fespaces[ip] = new FiniteElementSpace(p->patch_mesh[ip],fec);
+      // construct the patch tdof to global tdof map
+      int nrdof = patch_fespaces[ip]->GetTrueVSize();
+      patch_dof_map[ip].SetSize(nrdof);
+      // loop through the elements in the patch
+      for (int iel = 0; iel<p->element_map[ip].Size(); ++iel)
+      {
+         // index in the global mesh
+         int iel_idx = p->element_map[ip][iel];
+         // get the dofs of this element
+         Array<int> patch_elem_dofs;
+         Array<int> global_elem_dofs;
+         patch_fespaces[ip]->GetElementDofs(iel,patch_elem_dofs);
+         fespace->GetElementDofs(iel_idx,global_elem_dofs);
+         // the sizes have to match
+         MFEM_VERIFY(patch_elem_dofs.Size() == global_elem_dofs.Size(),
+                     "Size inconsistency");
+         // loop through the dofs and take into account the signs;
+         int ndof = patch_elem_dofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = patch_elem_dofs[i];
+            int gdof_ = global_elem_dofs[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            patch_dof_map[ip][pdof] = gdof;
+         }
+      }
+      // Define the patch bilinear form and apply boundary conditions (only the LHS)
+      Array <int> ess_temp_list;
+      if (p->patch_mesh[ip]->bdr_attributes.Size())
+      {
+         Array<int> ess_bdr(p->patch_mesh[ip]->bdr_attributes.Max());
+         ess_bdr = 1;
+         patch_fespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_temp_list);
+      }
+
+      // Adjust the essential tdof list for each patch
+      for (int i=0; i<ess_temp_list.Size(); i++)
+      {
+         int ldof = ess_temp_list[i];
+         int tdof = patch_dof_map[ip][ldof];
+         // check the kind of this tdof
+         if (!global_tdofs[tdof]) ess_tdof_list[ip].Append(ldof);
+      }
+
+      BilinearForm a(patch_fespaces[ip], bf);
+      a.Assemble();
+      OperatorPtr Alocal;
+      a.FormSystemMatrix(ess_tdof_list[ip],Alocal);
+      delete patch_fespaces[ip];
+      patch_mat[ip] = new SparseMatrix((SparseMatrix&)(*Alocal));
+      patch_mat[ip]->Threshold(0.0);
+      // Save the inverse
+      patch_mat_inv[ip] = new KLUSolver;
+      patch_mat_inv[ip]->SetOperator(*patch_mat[ip]);
+   }
+   delete p;
+}
+
+void PatchAssembly::print_patch_dof_map()
+{
+   mfem::out << "Patch dof map" << endl;
+   for (int ip = 0; ip<nrpatch; ++ip)
+   {
+      mfem::out << "Patch No: " << ip;
+      mfem::out << ", dof map: " ;
+      patch_dof_map[ip].Print(cout,patch_dof_map[ip].Size());
+   }
+}
+
+PatchAssembly::~PatchAssembly()
+{
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // delete patch_fespaces[ip]; patch_fespaces[ip]=nullptr;
+      delete patch_mat_inv[ip];
+      patch_mat_inv[ip]=nullptr;
+      delete patch_mat[ip];
+      patch_mat[ip]=nullptr;
+   }
+   patch_fespaces.DeleteAll();
+   patch_mat.DeleteAll();
+   patch_mat_inv.DeleteAll();
+}
+
+AddSchwarz::AddSchwarz(BilinearForm * bf_, Array<int> & global_ess_tdof_list, int i)
+   : Solver(bf_->FESpace()->GetTrueVSize(), bf_->FESpace()->GetTrueVSize()),
+     part(i)
+{
+   p = new PatchAssembly(bf_, global_ess_tdof_list,  part);
+   nrpatch = p->nrpatch;
+}
+
+void AddSchwarz::Mult(const Vector &r, Vector &z) const
+{
+   z = 0.0;
+   Vector rnew(r);
+   Vector znew(z);
+   Vector raux(znew.Size());
+   Vector res_local, sol_local;
+   for (int iter = 0; iter < maxit; iter++)
+   {
+      znew = 0.0;
+      for (int ip = 0; ip < nrpatch; ip++)
+      {
+         Array<int> * dof_map = &p->patch_dof_map[ip];
+         int ndofs = dof_map->Size();
+         res_local.SetSize(ndofs);
+         sol_local.SetSize(ndofs);
+
+         rnew.GetSubVector(*dof_map, res_local);
+         Array<int> ess_bdr_indices = p->ess_tdof_list[ip];
+         // for the overlapping case
+         // zero out the entries corresponding to the ess_bdr
+         p->patch_mat_inv[ip]->Mult(res_local, sol_local);
+         if (!part) { sol_local.SetSubVector(ess_bdr_indices,0.0); }
+         znew.AddElementVector(*dof_map,sol_local);
+      }
+      // Relaxation parameter
+      znew *= theta;
+      z += znew;
+      // Update residual
+      if (iter + 1 < maxit)
+      {
+         A->Mult(znew, raux);
+         rnew -= raux;
+      }
+   }
+}
+
+AddSchwarz::~AddSchwarz()
+{
+   delete p;
+}
+
+
+
+double GetUniformMeshElementSize(Mesh * mesh)
+{
+   int dim = mesh->Dimension();
+   int nrelem = mesh->GetNE();
+
+   DenseMatrix J(dim);
+   double hmin, hmax;
+   hmin = infinity();
+   hmax = -infinity();
+   Vector attr(nrelem);
+
+   for (int iel=0; iel<nrelem; ++iel)
+   {
+      int geom = mesh->GetElementBaseGeometry(iel);
+      ElementTransformation *T = mesh->GetElementTransformation(iel);
+      T->SetIntPoint(&Geometries.GetCenter(geom));
+      Geometries.JacToPerfJac(geom, T->Jacobian(), J);
+      attr(iel) = J.Det();
+      attr(iel) = pow(abs(attr(iel)), 1.0/double(dim));
+      hmin = min(hmin, attr(iel));
+      hmax = max(hmax, attr(iel));
+   }
+
+   MFEM_VERIFY(abs(hmin-hmax) < 1e-12, "Case not supported yet")
+
+   return hmax;
+}
+
+
+
+Mesh * ExtendMesh(Mesh * mesh, const Array<int> & directions)
+{
+    // extrute on one dimension
+   // flag = 1 +x, -1 -x, 2 +y, -2 +y , 3 +z, -3, -z
+
+   // copy the original mesh;
+   Mesh * mesh_orig = new Mesh(*mesh);
+   if (!directions.Size()) return mesh_orig;
+   
+   int dim = mesh_orig->Dimension();
+
+   Mesh * mesh_ext=nullptr;
+
+   for (int j=0; j<directions.Size(); j++)
+   {
+      int d = directions[j];
+      MFEM_VERIFY(abs(d)<= dim, "Cannot Extend in dimension " << d << ". Dim = " << dim << endl);
+
+      Vector pmin;
+      Vector pmax;
+      mesh_orig->GetBoundingBox(pmin,pmax);
+
+      // DenseMatrix J(dim);
+      // double hmin, hmax;
+      // hmin = infinity();
+      // hmax = -infinity();
+      // Vector attr(nrelem);
+      // // element size
+
+      // for (int iel=0; iel<nrelem; ++iel)
+      // {
+      //    int geom = mesh_orig->GetElementBaseGeometry(iel);
+      //    ElementTransformation *T = mesh_orig->GetElementTransformation(iel);
+      //    T->SetIntPoint(&Geometries.GetCenter(geom));
+      //    Geometries.JacToPerfJac(geom, T->Jacobian(), J);
+      //    attr(iel) = J.Det();
+      //    attr(iel) = pow(abs(attr(iel)), 1.0/double(dim));
+      //    hmin = min(hmin, attr(iel));
+      //    hmax = max(hmax, attr(iel));
+      // }
+      // MFEM_VERIFY(hmin==hmax, "Case not supported yet")
+      double h = GetUniformMeshElementSize(mesh_orig);
+      double val;
+      // find the vertices on the specific boundary
+      switch (d)
+      {
+      case 1:
+         val = pmax[0];
+         break;
+      case -1:
+         val = pmin[0];
+         h = -h;
+         break;    
+      case 2:
+         val = pmax[1];
+         break;
+      case -2:
+         val = pmin[1];
+         h = -h;
+         break;   
+      case 3:
+         val = pmax[2];
+         break;
+      case -3:
+         val = pmin[2];
+         h = -h;
+         break;      
+      }
+      int k = 0;
+      for (int i = 0; i<mesh_orig->GetNV(); ++i)
+      {
+         double * coords = mesh_orig->GetVertex(i);
+         switch (abs(d))
+         {
+         case 1:
+            if (coords[0] == val) k++;
+            break;
+         case 2:
+            if (coords[1] == val) k++;
+            break;   
+         case 3:
+            if (coords[2] == val) k++;
+            break;      
+         }
+      }
+      int nrvertices = mesh_orig->GetNV() + k;
+      int nrelements = mesh_orig->GetNE() + pow(pow(k,1.0/(dim-1))-1.0,dim-1);
+
+      mesh_ext = new Mesh(dim, nrvertices, nrelements);
+
+      // Add existing vertices
+      Array<int> vmap(mesh_orig->GetNV()); vmap = 0;
+      k = mesh_orig->GetNV();
+      for (int i=0; i<mesh_orig->GetNV(); ++i)
+      {
+         double * vert = mesh_orig->GetVertex(i);
+         mesh_ext->AddVertex(vert);
+         switch (abs(d))
+         {
+         case 1:
+            if (vert[0] == val) 
+            {
+               vmap[i] = k;
+               k++;
+            }
+            break;
+         case 2:
+            if (vert[1] == val)
+            {
+               vmap[i] = k;
+               k++;
+            }
+            break;   
+         case 3:
+            if (vert[2] == val)
+            {
+               vmap[i] = k;
+               k++;
+            }
+            break;      
+         }
+      } 
+      // Add existing elements
+      for (int i=0; i<mesh_orig->GetNE(); ++i)
+      {
+         Array<int>ind;
+         mesh_orig->GetElementVertices(i,ind);
+         if (dim == 2)
+         {
+            mesh_ext->AddQuad(ind);
+         }
+         else if (dim == 3)
+         {
+            mesh_ext->AddHex(ind);
+         }
+      } 
+      // Add new vertices
+      k = mesh_orig->GetNV();
+      for (int i=0; i<mesh_orig->GetNV(); ++i)
+      {
+         double * vert = mesh_orig->GetVertex(i);
+         switch (abs(d))
+         {
+         case 1:
+            if (vert[0] == val) 
+            {
+               double coords[dim];
+               coords[0] = vert[0] + h;
+               coords[1] = vert[1];
+               if (dim == 3) coords[2] = vert[2];
+               mesh_ext->AddVertex(coords);
+            }
+            break;
+         case 2:
+            if (vert[1] == val)
+            {
+               double coords[dim];
+               coords[0] = vert[0];
+               coords[1] = vert[1] + h;
+               if (dim == 3) coords[2] = vert[2];
+               mesh_ext->AddVertex(coords);
+            }
+            break;   
+         case 3:
+            if (vert[2] == val)
+            {
+               double coords[dim];
+               coords[0] = vert[0];
+               coords[1] = vert[1];
+               coords[2] = vert[2] + h;
+               mesh_ext->AddVertex(coords);
+            }
+            break;      
+         }
+      }    
+      // loop through boundary elements and extend in the given direction
+      for (int i=0; i<mesh_orig->GetNBE(); ++i) 
+      {
+         Array<int> vertices;
+         mesh_orig->GetBdrElementVertices(i,vertices);
+         if (dim == 2)
+         {
+            int ind[4];
+            if (vmap[vertices[0]] && vmap[vertices[1]])
+            {
+               ind[0] = vmap[vertices[0]];
+               ind[1] = vmap[vertices[1]];
+               ind[2] = vertices[1];
+               ind[3] = vertices[0];
+               mesh_ext->AddQuad(ind);
+            }
+         }
+         else if (dim == 3)
+         {
+            int ind[8];
+            if (vmap[vertices[0]] && vmap[vertices[1]] && vmap[vertices[2]] && vmap[vertices[3]])
+            {
+               ind[0] = vmap[vertices[0]];
+               ind[1] = vmap[vertices[1]];
+               ind[2] = vmap[vertices[2]];
+               ind[3] = vmap[vertices[3]];
+               ind[4] = vertices[0];
+               ind[5] = vertices[1];
+               ind[6] = vertices[2];
+               ind[7] = vertices[3];
+               mesh_ext->AddHex(ind);
+            }
+         }   
+      }
+      mesh_ext->FinalizeTopology();
+
+      if (j<directions.Size()-1)
+      {
+         delete mesh_orig;
+         mesh_orig = mesh_ext;
+      }
+   }
+   delete mesh_orig;
+   return mesh_ext;
+}

examples/maxwell-solver/ST/additive_schwarz.hpp

@@ -0,0 +1,133 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+using namespace std;
+using namespace mfem;
+
+
+class OverlappingCartesianMeshPartition 
+{
+private:
+   Mesh *mesh=nullptr;
+public:
+   int nrpatch;
+   int nx, ny, nz;
+   std::vector<Array<int>> element_map;
+   // constructor
+   OverlappingCartesianMeshPartition(Mesh * mesh_);
+   ~OverlappingCartesianMeshPartition() {};
+};
+
+class STPOverlappingCartesianMeshPartition // Special layered partition for STP
+{
+private:
+   Mesh *mesh=nullptr;
+public:
+   int nrpatch;
+   int nx, ny, nz;
+   std::vector<Array<int>> element_map;
+   // constructor
+   STPOverlappingCartesianMeshPartition(Mesh * mesh_);
+   ~STPOverlappingCartesianMeshPartition() {};
+};
+
+
+class CartesianMeshPartition // for now every vertex defines a patch
+{
+private:
+   Mesh *mesh=nullptr;
+public:
+   int nrpatch;
+   int nx, ny, nz;
+   std::vector<Array<int>> element_map;
+   // constructor
+   CartesianMeshPartition(Mesh * mesh_);
+   ~CartesianMeshPartition() {};
+};
+
+class VertexMeshPartition // for now every vertex defines a patch
+{
+private:
+   Mesh *mesh=nullptr;
+public:
+   int nrpatch;
+   // map local (patch) element to global (original mesh) element
+   std::vector<Array<int>> element_map; 
+   // constructor
+   VertexMeshPartition(Mesh * mesh_);
+   ~VertexMeshPartition() {};
+};
+
+class MeshPartition
+{
+private:
+   Mesh *mesh=nullptr;
+   void AddElementToMesh(Mesh * mesh,mfem::Element::Type elem_type,int * ind);
+   void GetNumVertices(int type, mfem::Element::Type & elem_type, int & nrvert);
+   void PrintElementMap();
+public:
+   int nrpatch;
+   int nx, ny, nz;
+   std::vector<Array<int>> element_map;
+   Array<Mesh *> patch_mesh;
+   int partition_kind;
+   // constructor
+   MeshPartition(Mesh * mesh_, int part);
+   ~MeshPartition();
+};
+
+void SaveMeshPartition(Array<Mesh * > meshes, 
+                       string mfilename="output/mesh.",
+                       string sfilename="output/sol.");
+
+
+class PatchAssembly // for now every vertex defines a patch
+{
+   FiniteElementSpace *fespace=nullptr;
+   BilinearForm *bf=nullptr;
+   void print_patch_dof_map();
+public:
+   int nrpatch;
+   Array<FiniteElementSpace *> patch_fespaces;
+   std::vector<Array<int>> patch_dof_map;
+   Array<SparseMatrix *> patch_mat;
+   Array<KLUSolver * > patch_mat_inv;
+   std::vector<Array<int>> ess_tdof_list;
+   std::vector<Array<int>> ess_int_tdofs;
+
+   // constructor
+   PatchAssembly(BilinearForm * bf_, Array<int> & ess_tdofs, int part);
+   ~PatchAssembly();
+};
+
+class AddSchwarz : public Solver//
+{
+private:
+   int nrpatch;
+   int maxit = 1;
+   int part;
+   double theta = 0.5;
+   PatchAssembly * p;
+   const Operator * A;
+public:
+   AddSchwarz(BilinearForm * bf_, Array<int> & ess_tdofs, int i = 0);
+   void SetNumSmoothSteps(const int iter)
+   {
+      maxit = iter;
+   }
+   void SetDumpingParam(const double dump_param)
+   {
+      theta = dump_param;
+   }
+   virtual void SetOperator(const Operator &op)
+   {
+      A = &op;
+   }
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~AddSchwarz();
+};
+
+Mesh * ExtendMesh(Mesh * mesh, const Array<int> & directions);
+
+double GetUniformMeshElementSize(Mesh * mesh);

examples/maxwell-solver/ST/additive_schwarzp.hpp

@@ -0,0 +1,182 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include <unordered_map>
+
+using namespace std;
+using namespace mfem;
+
+
+struct UniqueIndexGenerator
+{
+   int counter = 0;
+   std::unordered_map<int,int> idx;
+   int Get(int i)
+   {
+      std::unordered_map<int,int>::iterator f = idx.find(i);
+      if (f == idx.end())
+      {
+         idx[i] = counter;
+         return counter++;
+      }
+      else
+      {
+         return (*f).second;
+      }
+   }
+   void Reset()
+   {
+      counter = 0;
+      idx.clear();
+   }
+};
+
+
+class CartesianParMeshPartition // for now every vertex defines a patch
+{
+private:
+   ParMesh *pmesh=nullptr;
+public:
+   int nrpatch;
+   Array<int> patch_rank;
+   std::vector<Array<int>> local_element_map;
+   // constructor
+   CartesianParMeshPartition(ParMesh * pmesh_);
+   ~CartesianParMeshPartition() {};
+};
+
+class VertexParMeshPartition
+{
+private:
+   ParMesh *pmesh=nullptr;
+public:
+   int nrpatch;
+   Array<int> patch_rank;
+   std::vector<Array<int>> local_element_map;
+   // constructor
+   VertexParMeshPartition(ParMesh * pmesh_);
+   ~VertexParMeshPartition() {};
+};
+
+class ParMeshPartition
+{
+private:
+   MPI_Comm comm;
+   ParMesh *pmesh=nullptr;
+   void AddElementToMesh(Mesh * mesh,mfem::Element::Type elem_type,int * ind);
+   void GetNumVertices(int type, mfem::Element::Type & elem_type, int & nrvert);
+   void SaveMeshPartition();
+public:
+   int nrpatch;
+   int myelem_offset = 0;
+   Array<int> patch_rank;
+   std::vector<Array<int>> element_map;
+   std::vector<Array<int>> local_element_map;
+   Array<Mesh *> patch_mesh;
+   // constructor
+   ParMeshPartition(ParMesh * pmesh_, int part);
+   ~ParMeshPartition();
+};
+
+class ParPatchDofInfo
+{
+public:
+   MPI_Comm comm = MPI_COMM_WORLD;
+   int nrpatch;
+   Array<int> patch_rank;
+   vector<Array<int>> PatchGlobalTrueDofs; // list of all the true dofs in a patch
+   vector<Array<int>> PatchTrueDofs; // list of only
+   Array<FiniteElementSpace *> patch_fespaces;
+   std::vector<Array<int>> patch_dof_map;
+   ParMeshPartition * p;
+   // constructor
+   ParPatchDofInfo(ParFiniteElementSpace *fespace, int part);
+   // void Print();
+   ~ParPatchDofInfo();
+};
+
+
+
+class ParPatchAssembly // for now every vertex defines a patch
+{
+private:
+   std::vector<int> tdof_offsets;
+   ParBilinearForm *bf=nullptr;
+   void compute_trueoffsets();
+   void AssemblePatchMatrices(ParPatchDofInfo * p);
+   void print_patch_dof_map() {};
+public:
+   MPI_Comm comm;
+   int nrpatch;
+   ParFiniteElementSpace *fespace=nullptr;
+   Array<int> patch_rank;
+   std::vector<Array<int>> patch_true_dofs;
+   std::vector<Array<int>> patch_local_dofs;
+
+   Array<SparseMatrix *> patch_mat;
+   Array<BilinearForm * > patch_bilinear_forms;
+   Array<KLUSolver * > patch_mat_inv;
+   std::vector<Array<int>> ess_tdof_list;
+
+   // constructor
+   ParPatchAssembly(ParBilinearForm * bf_, int part);
+   int get_rank(int tdof);
+   ~ParPatchAssembly();
+};
+
+
+class ParPatchRestriction
+{
+private:
+   MPI_Comm comm;
+   int num_procs, myid;
+   Array<int> patch_rank;
+   ParPatchAssembly * P;
+   int nrpatch;
+   Array<int> send_count;
+   Array<int> send_displ;
+   Array<int> recv_count;
+   Array<int> recv_displ;
+   int sbuff_size, rbuff_size;
+public:
+   ParPatchRestriction(ParPatchAssembly * P_);
+   // void Mult(const Vector & r , Array<BlockVector *> & res);
+   void Mult(const Vector & r , std::vector<Vector  > & res);
+   // void MultTranspose(const Array<BlockVector*> & sol, Vector & z);
+   void MultTranspose(const std::vector<Vector  > & sol, Vector & z);
+   virtual ~ParPatchRestriction() {}
+};
+
+
+class ParAddSchwarz : public Solver//
+{
+private:
+   MPI_Comm comm;
+   int nrpatch;
+   int part;
+   int maxit = 1;
+   double theta = 0.5;
+   ParPatchAssembly * p;
+   const Operator * A;
+   ParPatchRestriction * R;
+public:
+   ParAddSchwarz(ParBilinearForm * bf_, int i = 0);
+
+   void SetNumSmoothSteps(const int iter)
+   {
+      maxit = iter;
+   }
+   void SetDumpingParam(const double dump_param)
+   {
+      theta = dump_param;
+   }
+   virtual void SetOperator(const Operator &op)
+   {
+      A = &op;
+   }
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~ParAddSchwarz();
+};
+
+

examples/maxwell-solver/ST/complex_additive_schwarz.cpp

@@ -0,0 +1,348 @@
+#include "complex_additive_schwarz.hpp"
+
+ComplexPatchAssembly::ComplexPatchAssembly(SesquilinearForm * bf_, Array<int> & ess_tdofs, int part) : bf(bf_)
+{
+   fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   int dim = mesh->Dimension();
+   const FiniteElementCollection *fec = fespace->FEColl();
+
+   // list of dofs to distiguish between interior/boundary and essential
+   Array<int> global_tdofs(fespace->GetTrueVSize());
+   Array<int> bdr_tdofs(fespace->GetTrueVSize());
+   global_tdofs = 0;
+   // Mark boundary dofs and ess_dofs
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, bdr_tdofs);
+   }
+
+   // mark boundary dofs
+   for (int i = 0; i<bdr_tdofs.Size(); i++) global_tdofs[bdr_tdofs[i]] = 1;
+   // overwrite flag for essential dofs
+   for (int i = 0; i<ess_tdofs.Size(); i++) global_tdofs[ess_tdofs[i]] = 0;
+
+
+   MeshPartition * p = new MeshPartition(mesh, part);
+   nx = p->nx;
+   ny = p->ny;
+   nz = p->nz;
+   // SaveMeshPartition(p->patch_mesh);
+   nrpatch = p->nrpatch;
+   patch_fespaces.SetSize(nrpatch);
+   patch_meshes_ext.SetSize(nrpatch);
+   patch_fespaces_ext.SetSize(nrpatch);
+   dof2extdof_map.resize(nrpatch);
+   patch_dof_map.resize(nrpatch);
+   patch_mat.SetSize(nrpatch);
+   patch_mat_ext.SetSize(nrpatch);
+   patch_mat_inv.SetSize(nrpatch);
+   patch_mat_inv_ext.SetSize(nrpatch);
+   ess_tdof_list.resize(nrpatch);
+
+   // construct extended meshes
+   int ip = -1;
+   int nrlayers = 0;
+   if (!part)
+   {
+      for (int ip = 0; ip<nrpatch; ip++)
+      {
+         patch_meshes_ext[ip] = new Mesh(*p->patch_mesh[ip]);
+      }
+   }
+   else
+   {   
+      for (int kz = 0; kz<nz; kz++)
+      {
+         for (int ky = 0; ky<ny; ky++)
+         {
+            for (int kx = 0; kx<nx; kx++)
+            {
+               ip++;
+               Array<int> ext_directions;
+               for (int j=0; j<nrlayers; ++j)
+               {
+                  for (int comp=0; comp<dim; ++comp)
+                  {
+                     if (comp == 0 && kx != 0)
+                     {
+                        ext_directions.Append(-comp-1);
+                     }
+                     if (comp == 0 && kx != nx-1)
+                     {
+                        ext_directions.Append(comp+1);
+                     }
+                     if (comp == 1 && ky != 0)
+                     {
+                        ext_directions.Append(-comp-1);
+                     }
+                     if (comp == 1 && ky != ny-1)
+                     {
+                        ext_directions.Append(comp+1);
+                     }
+                     if (comp == 2 && kz != 0)
+                     {
+                        ext_directions.Append(-comp-1);
+                     }
+                     if (comp == 2 && kz != ny-1)
+                     {
+                        ext_directions.Append(comp+1);
+                     }
+                  }
+               }
+               patch_meshes_ext[ip] = ExtendMesh(p->patch_mesh[ip],ext_directions);
+            }
+         }
+      }
+   }
+
+   
+   // SaveMeshPartition(patch_meshes_ext, "output/ext_mesh.", "output/ext_sol.");
+
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // create finite element spaces for each patch // This might be avoided
+      patch_fespaces[ip] = new FiniteElementSpace(p->patch_mesh[ip],fec);
+      // create finite element spaces on the extented (PML) meshes
+      patch_fespaces_ext[ip] = new FiniteElementSpace(patch_meshes_ext[ip],fec);
+
+      // construct the patch tdof to global tdof map
+      int nrdof = patch_fespaces[ip]->GetTrueVSize();
+      patch_dof_map[ip].SetSize(2*nrdof);
+      dof2extdof_map[ip].SetSize(2*nrdof);
+
+      // build dof maps between patch and extended patch
+      //loop through the patch elements and constract the dof map
+      // The same elements in the extended mesh have the same ordering (but not the dofs)
+
+      // loop through the elements in the patch
+      for (int iel = 0; iel<p->element_map[ip].Size(); ++iel)
+      {
+         // index in the global mesh
+         int iel_idx = p->element_map[ip][iel];
+         // get the dofs of this element
+         Array<int> patch_elem_dofs;
+         Array<int> patch_elem_dofs_ext;
+         Array<int> global_elem_dofs;
+         patch_fespaces[ip]->GetElementDofs(iel,patch_elem_dofs);
+         patch_fespaces_ext[ip]->GetElementDofs(iel,patch_elem_dofs_ext);
+         fespace->GetElementDofs(iel_idx,global_elem_dofs);
+         // the sizes have to match
+         MFEM_VERIFY(patch_elem_dofs.Size() == global_elem_dofs.Size(),
+                     "Size inconsistency");
+         MFEM_VERIFY(patch_elem_dofs.Size() == patch_elem_dofs_ext.Size(),
+                     "Size inconsistency");            
+         // loop through the dofs and take into account the signs;
+         int ndof = patch_elem_dofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = patch_elem_dofs[i];
+            int gdof_ = global_elem_dofs[i];
+            int extdof_ = patch_elem_dofs_ext[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            int extdof = (extdof_ >= 0) ? extdof_ : abs(extdof_) - 1;
+            patch_dof_map[ip][pdof] = gdof;
+            patch_dof_map[ip][pdof+nrdof] = gdof+fespace->GetTrueVSize();
+            dof2extdof_map[ip][pdof] = extdof;
+            dof2extdof_map[ip][pdof+nrdof] = extdof+patch_fespaces_ext[ip]->GetTrueVSize();
+         }
+      }
+      // Define the patch bilinear form and apply boundary conditions (only the LHS)
+      Array <int> ess_temp_list;
+      if (p->patch_mesh[ip]->bdr_attributes.Size())
+      {
+         Array<int> ess_bdr(p->patch_mesh[ip]->bdr_attributes.Max());
+         ess_bdr = 0;
+         patch_fespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_temp_list);
+      }
+
+      Array <int> ess_list_ext;
+      if (patch_meshes_ext[ip]->bdr_attributes.Size())
+      {
+         Array<int> ess_bdr(patch_meshes_ext[ip]->bdr_attributes.Max());
+         ess_bdr = 0;
+         patch_fespaces_ext[ip]->GetEssentialTrueDofs(ess_bdr, ess_list_ext);
+      }
+
+      // Adjust the essential tdof list for each patch
+      for (int i=0; i<ess_temp_list.Size(); i++)
+      {
+         int ldof = ess_temp_list[i];
+         int tdof = patch_dof_map[ip][ldof];
+         // check the kind of this tdof
+         if (!global_tdofs[tdof]) ess_tdof_list[ip].Append(ldof);
+      }
+
+      SesquilinearForm a(patch_fespaces[ip], &bf->real(), &bf->imag());
+      SesquilinearForm a_ext(patch_fespaces_ext[ip], &bf->real(), &bf->imag());
+      
+      a.Assemble();
+      a_ext.Assemble();
+      OperatorPtr Alocal;
+      a.FormSystemMatrix(ess_tdof_list[ip],Alocal);
+      ComplexSparseMatrix * AZ = Alocal.As<ComplexSparseMatrix>();
+      patch_mat[ip] = AZ->GetSystemMatrix();
+      patch_mat[ip]->Threshold(0.0);
+      // Save the inverse
+      patch_mat_inv[ip] = new KLUSolver;
+      patch_mat_inv[ip]->SetOperator(*patch_mat[ip]);
+
+
+      OperatorPtr Alocal_ext;
+      a_ext.FormSystemMatrix(ess_list_ext,Alocal_ext);
+      ComplexSparseMatrix * AZ_ext = Alocal_ext.As<ComplexSparseMatrix>();
+      patch_mat_ext[ip] = AZ_ext->GetSystemMatrix();
+      patch_mat_ext[ip]->Threshold(0.0);
+      patch_mat_inv_ext[ip] = new KLUSolver;
+      patch_mat_inv_ext[ip]->SetOperator(*patch_mat_ext[ip]);
+
+
+      delete patch_fespaces[ip];
+      delete patch_fespaces_ext[ip];
+   }
+   delete p;
+}
+
+ComplexPatchAssembly::~ComplexPatchAssembly()
+{
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // delete patch_fespaces[ip]; patch_fespaces[ip]=nullptr;
+      delete patch_meshes_ext[ip];
+      patch_meshes_ext[ip]=nullptr;
+      delete patch_mat_inv[ip];
+      patch_mat_inv[ip]=nullptr;
+      delete patch_mat[ip];
+      patch_mat[ip]=nullptr;
+   }
+   patch_fespaces.DeleteAll();
+   patch_meshes_ext.DeleteAll();
+   patch_mat.DeleteAll();
+   patch_mat_inv.DeleteAll();
+}
+
+
+
+ComplexAddSchwarz::ComplexAddSchwarz(SesquilinearForm * bf_, Array<int> & ess_tdofs, int i)
+   : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), bf(bf_),
+     part(i)
+{
+   p = new ComplexPatchAssembly(bf_, ess_tdofs, part);
+   nrpatch = p->nrpatch;
+}
+
+void  ComplexAddSchwarz::Mult(const Vector &r, Vector &z) const
+{
+   z = 0.0;
+   Vector rnew(r);
+   Vector znew(z);
+   Vector raux(znew.Size());
+   Vector res_local, sol_local;
+   Array<int> visit(znew.Size());
+   // char vishost[] = "localhost";
+   // int  visport   = 19916;
+
+   // socketstream sol_sock(vishost, visport);
+   // sol_sock.precision(8);
+   for (int iter = 0; iter < maxit; iter++)
+   {
+      znew = 0.0;
+      visit = 0;
+      for (int ip = 0; ip < nrpatch; ip++)
+      {
+         Array<int> * dof_map = &p->patch_dof_map[ip];
+         int ndofs = dof_map->Size();
+         res_local.SetSize(ndofs);
+         sol_local.SetSize(ndofs);
+
+         rnew.GetSubVector(*dof_map, res_local);
+
+         //-----------------------------------------------
+         // Extend by zero to the extended mesh
+         int nrdof_ext = p->patch_mat_ext[ip]->Height();
+         
+         Vector res_ext(nrdof_ext); res_ext = 0.0;
+         Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+
+         res_ext.SetSubVector(p->dof2extdof_map[ip],res_local.GetData());
+
+         p->patch_mat_inv_ext[ip]->Mult(res_ext, sol_ext);
+
+         sol_ext.GetSubVector(p->dof2extdof_map[ip],sol_local);
+
+
+         //-----------------------------------------------
+
+         // p->patch_mat_inv[ip]->Mult(res_local, sol_local);
+
+         // for the overlapping case
+         // zero out the entries corresponding to the ess_bdr
+         Array<int> ess_bdr_indices_re = p->ess_tdof_list[ip]; // real part
+         Array<int> ess_bdr_indices(2*ess_bdr_indices_re.Size()); //imag part
+
+         for (int i = 0; i< ess_bdr_indices_re.Size(); i++)
+         {
+            ess_bdr_indices[i] = ess_bdr_indices_re[i];
+            ess_bdr_indices[i+ess_bdr_indices_re.Size()] = ess_bdr_indices_re[i]+ndofs/2;
+         }
+         if (!part) 
+         { 
+            sol_local.SetSubVector(ess_bdr_indices,0.0); 
+         }
+         if (type == 1) znew = 0.0;
+
+         znew.AddElementVector(*dof_map,sol_local);
+         // zero out the contributions to the dofs which are already updated
+         if (type == 1)
+         {
+            for (int i = 0; i<ndofs; i++)
+            {
+               int j = (*dof_map)[i];
+               if (visit[j])
+               {
+                  znew(j) = 0.0;
+               }
+               else
+               {
+                  visit[j] = 1;
+               }
+            }
+            z.Add(theta, znew);
+            A->Mult(znew, raux);
+            rnew -= raux;
+         }
+         // PlotSolution(z, sol_sock, ip); cin.get();
+      }
+      if (type == 0)
+      {
+         z.Add(theta, znew);
+         A->Mult(znew, raux);
+         rnew -= raux;
+      }
+      // Update residual
+      if (iter + 1 < maxit)
+      {
+         A->Mult(znew, raux);
+         rnew -= raux;
+      }
+   }
+   // PlotSolution(z, sol_sock, 0); cin.get();
+}
+
+
+void ComplexAddSchwarz::PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const
+{
+   FiniteElementSpace * fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   ComplexGridFunction gf(fespace);
+   bf->RecoverFEMSolution(sol,B,gf);
+   
+   string keys;
+   if (ip == 0) keys = "keys mrRljc\n";
+   sol_sock << "solution\n" << *mesh << gf.real() << keys << flush;
+}
+
+ComplexAddSchwarz::~ComplexAddSchwarz(){ delete p;}

examples/maxwell-solver/ST/complex_additive_schwarz.hpp

@@ -0,0 +1,58 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "additive_schwarz.hpp"
+#include "pml.hpp"
+using namespace std;
+using namespace mfem;
+
+class ComplexPatchAssembly
+{
+   FiniteElementSpace *fespace=nullptr;
+   SesquilinearForm *bf=nullptr;
+public:
+   int nrpatch, nx, ny, nz;
+   Array<FiniteElementSpace *> patch_fespaces;
+   Array<FiniteElementSpace *> patch_fespaces_ext;
+   Array<Mesh *> patch_meshes_ext;
+   std::vector<Array<int>> patch_dof_map;
+   std::vector<Array<int>> complex_patch_dof_map;
+   std::vector<Array<int>> dof2extdof_map;
+   Array<SparseMatrix *> patch_mat;
+   Array<SparseMatrix *> patch_mat_ext;
+   Array<KLUSolver * > patch_mat_inv_ext;
+   Array<KLUSolver * > patch_mat_inv;
+   std::vector<Array<int>> ess_tdof_list;
+
+   // constructor
+   ComplexPatchAssembly(SesquilinearForm * bf_, Array<int> & ess_tdofs, int part);
+   ~ComplexPatchAssembly();
+};
+
+
+class ComplexAddSchwarz : public Solver//
+{
+private:
+   int nrpatch;
+   int maxit = 1;
+   SesquilinearForm *bf=nullptr;
+   int part;
+   int type = 0;
+   double theta = 0.5;
+   ComplexPatchAssembly * p;
+   const Operator * A;
+   Vector B;
+   void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+
+   
+public:
+   ComplexAddSchwarz(SesquilinearForm * bf_, Array<int> & ess_tdofs, int i = 0);
+   void SetNumSmoothSteps(const int iter) { maxit = iter;}
+   void SetLoadVector(Vector load) { B = load;}
+   void SetSmoothType(int itype) { type = itype;}
+   void SetDumpingParam(const double dump_param) {theta = dump_param;}
+   virtual void SetOperator(const Operator &op) {A = &op;}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~ComplexAddSchwarz();
+};

examples/maxwell-solver/ST/complex_additive_schwarzp.cpp

@@ -0,0 +1,88 @@
+#include "complex_additive_schwarzp.hpp"
+
+// constructor
+ComplexParPatchAssembly::ComplexParPatchAssembly(ParSesquilinearForm * bf_) :
+   bf(bf_)
+{
+
+
+
+}
+
+
+void ComplexParPatchAssembly::AssemblePatchMatrices(ParPatchDofInfo * p)
+{
+   // patch_mat.SetSize(nrpatch);
+   // patch_bilinear_forms.SetSize(nrpatch);
+   // patch_mat_inv.SetSize(nrpatch);
+   // ess_tdof_list.resize(nrpatch);
+   // for (int ip=0; ip<nrpatch; ++ip)
+   // {
+   //    patch_bilinear_forms[ip] = nullptr;
+   //    patch_mat_inv[ip] = nullptr;
+   //    patch_mat[ip] = nullptr;
+   //    if (p->p->patch_mesh[ip])
+   //    {
+   //       // Define the patch bilinear form and apply boundary conditions (only the LHS)
+   //       FiniteElementSpace * patch_fespace = p->patch_fespaces[ip];
+   //       Mesh * patch_mesh = p->p->patch_mesh[ip];
+   //       if (patch_mesh->bdr_attributes.Size())
+   //       {
+   //          Array<int> ess_bdr(patch_mesh->bdr_attributes.Max());
+   //          ess_bdr = 1;
+   //          patch_fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list[ip]);
+   //       }
+   //       patch_bilinear_forms[ip] = new SesquilinearForm(patch_fespace, bf);
+   //       patch_bilinear_forms[ip]->Assemble();
+   //       OperatorPtr Alocal;
+   //       // need to add the method FormSystemMatrix to complex_fem
+   //       // patch_bilinear_forms[ip]->FormSystemMatrix(ess_tdof_list[ip],Alocal);
+   //       patch_mat[ip] = &(SparseMatrix&)(*Alocal);
+   //       patch_mat[ip]->Threshold(0.0);
+   //       // Save the inverse
+   //       patch_mat_inv[ip] = new KLUSolver;
+   //       patch_mat_inv[ip]->SetOperator(*patch_mat[ip]);
+   //    }
+   // }
+}
+
+
+ComplexParPatchAssembly::~ComplexParPatchAssembly() {};
+
+
+ComplexParPatchRestriction::ComplexParPatchRestriction(ComplexParPatchAssembly *
+                                                       P_)
+{}
+
+void ComplexParPatchRestriction::Mult(const Vector & r ,
+                                      std::vector<Vector > & res)
+{}
+
+
+void ComplexParPatchRestriction::MultTranspose(const std::vector<Vector > & sol,
+                                               Vector & z)
+{}
+
+
+ComplexParAddSchwarz::ComplexParAddSchwarz(ParSesquilinearForm * pbf_)
+   : Solver(2*pbf_->ParFESpace()->GetTrueVSize(),
+            2*pbf_->ParFESpace()->GetTrueVSize())
+{
+   // cout << "Testing ComplexParAddSchwarz" << endl;
+   // comm = pbf_->ParFESpace()->GetComm();
+   // p = new ComplexParPatchAssembly(pbf_);
+   // nrpatch = p->nrpatch;
+   // R = new ComplexParPatchRestriction(p);
+}
+
+void ComplexParAddSchwarz::Mult(const Vector &r, Vector &z) const
+{}
+
+ComplexParAddSchwarz::~ComplexParAddSchwarz()
+{
+   // delete p;
+   // delete R;
+}
+
+
+

examples/maxwell-solver/ST/complex_additive_schwarzp.hpp

@@ -0,0 +1,87 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "additive_schwarzp.hpp"
+using namespace std;
+using namespace mfem;
+
+
+
+class ComplexParPatchAssembly
+{
+   // std::vector<int> tdof_offsets;
+   ParSesquilinearForm * bf=nullptr;
+   void compute_trueoffsets();
+   void AssemblePatchMatrices(ParPatchDofInfo * p);
+public:
+   // MPI_Comm comm;
+   // int nrpatch;
+   // ParFiniteElementSpace *fespace=nullptr;
+   // Array<int> patch_rank;
+   // std::vector<Array<int>> patch_true_dofs;
+   // std::vector<Array<int>> patch_local_dofs;
+   // Array<SparseMatrix *> patch_mat;
+   // Array<SesquilinearForm * > patch_bilinear_forms;
+   // Array<KLUSolver * > patch_mat_inv;
+   // std::vector<Array<int>> ess_tdof_list;
+
+   // constructor
+   ComplexParPatchAssembly(ParSesquilinearForm * bf_);
+   int get_rank(int tdof);
+   ~ComplexParPatchAssembly();
+};
+
+
+class ComplexParPatchRestriction
+{
+private:
+   // MPI_Comm comm;
+   // int num_procs, myid;
+   // Array<int> patch_rank;
+   // ParPatchAssembly * P;
+   // int nrpatch;
+   // Array<int> send_count;
+   // Array<int> send_displ;
+   // Array<int> recv_count;
+   // Array<int> recv_displ;
+   // int sbuff_size, rbuff_size;
+public:
+   ComplexParPatchRestriction(ComplexParPatchAssembly * P_);
+   void Mult(const Vector & r , std::vector<Vector > & res);
+   void MultTranspose(const std::vector<Vector > & sol, Vector & z);
+   virtual ~ComplexParPatchRestriction() {}
+};
+
+
+class ComplexParAddSchwarz : public Solver//
+{
+private:
+   // MPI_Comm comm;
+   // int nrpatch;
+   // int maxit = 1;
+   // double theta = 0.5;
+   // FiniteElementSpace *fespace=nullptr;
+   // ComplexParPatchAssembly * p;
+   // const Operator * A;
+   // ParSesquilinearForm * pbf;
+   // ComplexParPatchRestriction * R;
+public:
+   ComplexParAddSchwarz(ParSesquilinearForm * pbf_);
+   void SetNumSmoothSteps(const int iter)
+   {
+      // maxit = iter;
+   }
+   void SetDumpingParam(const double dump_param)
+   {
+      // theta = dump_param;
+   }
+   virtual void SetOperator(const Operator &op)
+   {
+      // A = &op;
+   }
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~ComplexParAddSchwarz();
+};
+
+

examples/maxwell-solver/ST/example1.cpp

@@ -0,0 +1,150 @@
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "additive_schwarz.hpp"
+#include "schwarz.hpp"
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/star.mesh";
+   // const char *mesh_file = "../../../data/beam-quad.mesh";
+   int order = 1;
+   int ref_levels = 1;
+   bool visualization = true;
+   StopWatch chrono;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&ref_levels, "-ref", "--ref_levels",
+                  "Number of uniform h-refinements");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+
+   Mesh *mesh;
+   // mesh = new Mesh(mesh_file, 1, 1);
+   mesh = new Mesh(1, 1, Element::QUADRILATERAL, true, 1, 1, false);
+   int dim = mesh->Dimension();
+
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   // FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   FiniteElementSpace * fespace = new FiniteElementSpace(mesh, fec);
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (mesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
+   //    the basis functions in the finite element fespace.
+   LinearForm *b = new LinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   // 8. Define the solution vector x as a finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   GridFunction x(fespace);
+   x = 1.0;
+
+   // 9. Set up the bilinear form a(.,.) on the finite element space
+   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //    domain integrator.
+   BilinearForm *a = new BilinearForm(fespace);
+   a->SetDiagonalPolicy(mfem::Matrix::DIAG_ONE);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 10. Assemble the bilinear form and the corresponding linear system,
+   //     applying any necessary transformations such as: eliminating boundary
+   //     conditions, applying conforming constraints for non-conforming AMR,
+   //     static condensation, etc.
+   a->Assemble();
+
+   OperatorPtr A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   cout << "Size of linear system: " << A->Height() << endl;
+
+   AddSchwarz * prec = new AddSchwarz(a,ess_tdof_list, 0);
+   prec->SetOperator((SparseMatrix&)(*A));
+   prec->SetNumSmoothSteps(1);
+   prec->SetDumpingParam(0.5);
+
+   SchwarzSmoother * prec2 = new SchwarzSmoother(mesh,0,fespace,&(SparseMatrix&)(*A),ess_bdr);
+   prec2->SetNumSmoothSteps(1);
+   prec2->SetDumpingParam(0.5);
+
+
+   int maxit = 2000;
+   double rtol = 1e-8;
+   double atol = 1e-8;
+   Vector X0(X);
+   CGSolver pcg;
+   pcg.iterative_mode = false;
+   pcg.SetPrintLevel(1);
+   pcg.SetMaxIter(maxit);
+   pcg.SetRelTol(rtol);
+   pcg.SetAbsTol(atol);
+   pcg.SetPreconditioner(*prec);
+   pcg.SetOperator((SparseMatrix&)(*A));
+   pcg.Mult(B, X0);
+
+   X0 = X;
+   pcg.SetPreconditioner(*prec2);
+   pcg.Mult(B, X0);
+
+   // 12. Recover the solution as a finite element grid function.
+   a->RecoverFEMSolution(X0, *b, x);
+
+   // 14. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream mesh_sock(vishost, visport);
+      mesh_sock.precision(8);
+      mesh_sock << "mesh\n" << *mesh << flush;
+
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *mesh << x << "keys rRjmc" << flush;
+   }
+
+   // 15. Free the used memory.
+   delete prec;
+   delete a;
+   delete b;
+   delete fespace;
+   delete fec;
+   delete mesh;
+
+   return 0;
+}

examples/maxwell-solver/ST/example1p.cpp

@@ -0,0 +1,177 @@
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "additive_schwarzp.hpp"
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   StopWatch chrono;
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../../data/star.mesh";
+   int order = 1;
+   int ref_levels = 1;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&ref_levels, "-ref", "--ref_levels",
+                  "Number of uniform h-refinements");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+
+   // Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   Mesh * mesh = new Mesh(1, 1, Element::QUADRILATERAL, true, 1, 1, false);
+   // Mesh * mesh = new Mesh(1, 1,1, Element::HEXAHEDRON, true, 1, 1, 1, false);
+   int dim = mesh->Dimension();
+
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   // FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ParLinearForm *b = new ParLinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   ParGridFunction x(fespace);
+   x = 0.0;
+
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   chrono.Clear();
+   chrono.Start();
+   a->Assemble();
+
+   HypreParMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+   chrono.Stop();
+   if (myid == 0) { cout << "Form Linear System time: " << chrono.RealTime() << endl; }
+
+   // Array<int>elem_vertices;
+   // for (int iel = 0; iel<pmesh->GetNE(); iel++)
+   // {
+   //    pmesh->GetElementVertices(iel,elem_vertices);
+   //    cout << "myid, iel: " << myid <<", " << iel << ", " ; elem_vertices.Print(cout,10);
+   // }
+
+   Array<double> times(4);
+
+   chrono.Clear();
+   chrono.Start();
+   ParAddSchwarz *prec = new ParAddSchwarz(a,0);
+   prec->SetOperator(A);
+   prec->SetNumSmoothSteps(1);
+   prec->SetDumpingParam(0.5);
+
+   chrono.Stop();
+   times[0] = chrono.RealTime();
+
+   int maxit = 200;
+   double rtol = 1e-8;
+   double atol = 1e-8;
+   X = 0.0;
+   CGSolver pcg(MPI_COMM_WORLD);
+   pcg.SetPrintLevel(1);
+   pcg.SetMaxIter(maxit);
+   pcg.SetRelTol(rtol);
+   pcg.SetAbsTol(atol);
+   pcg.SetPreconditioner(*prec);
+   pcg.SetOperator(A);
+
+   chrono.Clear();
+   chrono.Start();
+   pcg.Mult(B, X);
+   chrono.Stop();
+   times[1] = chrono.RealTime();
+   delete prec;
+
+   if (myid == 0)
+   {
+      cout << "prec construction time: " << times[0] << endl;
+      cout << "PCG solution time:      " << times[1] << endl;
+   }
+
+   a->RecoverFEMSolution(X, *b, x);
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      // socketstream mesh_sock(vishost, visport);
+      // mesh_sock << "parallel " << num_procs << " " << myid << "\n";
+      // mesh_sock.precision(8);
+      // mesh_sock << "mesh\n" << *pmesh  << "keys n/n" << flush;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh  << x <<"keys " << flush;
+   }
+
+   // // 17. Free the used memory.
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0)
+   {
+      delete fec;
+   }
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}

examples/maxwell-solver/ST/helmholtz.cpp

@@ -0,0 +1,306 @@
+//
+// Compile with: make helmholtz
+//
+// Sample runs:  helmholtz -m ../data/one-hex.mesh
+//               helmholtz -m ../data/fichera.mesh
+//               helmholtz -m ../data/fichera-mixed.mesh
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Helmholtz problem
+//               -Delta p - omega^2 p = 1 with impedance boundary condition.
+//
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "complex_additive_schwarz.hpp"
+#include "schwarz.hpp"
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution and r.h.s., see below for implementation.
+double f_exact_Re(const Vector &x);
+double f_exact_Im(const Vector &x);
+
+int dim;
+double omega;
+int sol = 1;
+bool pml = false;
+double length = 1.0;
+double pml_length = 0.25;
+bool scatter = false;
+
+#ifndef MFEM_USE_SUPERLU
+#error This example requires that MFEM is built with MFEM_USE_PETSC=YES
+#endif
+
+int main(int argc, char *argv[])
+{
+
+   // 2. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../../data/one-hex.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // static condensation flag
+   bool static_cond = false;
+   bool visualization = 1;
+   // number of wavelengths
+   double k = 0.5;
+   // number of mg levels
+   int ref = 1;
+   // dimension
+   int nd = 2;
+
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nd, "-nd", "--dim","Problem space dimension");
+   args.AddOption(&sol, "-sol", "--exact",
+                  "Exact solution flag - 0:polynomial, 1: plane wave, -1: unknown exact");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&pml, "-pml", "--pml", "-no-pml",
+                  "--no-pml", "Enable PML.");
+   args.AddOption(&pml_length, "-pml_length", "--pml_length",
+                  "Length of the PML region in each direction");
+   args.AddOption(&length, "-length", "--length",
+                  "length of the domainin in each direction.");
+   args.AddOption(&ref, "-ref", "--ref",
+                  "Number of Refinements.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&scatter, "-scat", "--scattering-prob", "-no-scat",
+                  "--no-scattering", "Solve a scattering problem");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+		args.PrintUsage(cout);
+		return 1;
+   }
+	args.PrintOptions(cout);
+   // Angular frequency
+   omega = 2.0 * M_PI * k;
+
+   // 3. Read the mesh from the given mesh file.
+   Mesh *mesh;
+
+   if (nd == 2)
+   {
+      // mesh = new Mesh(mesh_file,1,1);
+      mesh = new Mesh(1, 1, Element::QUADRILATERAL, true, length, length, false);
+   }
+   else
+   {
+      mesh = new Mesh(1, 1, 1, Element::HEXAHEDRON, true, length, length, length,false);
+   }
+
+   // 3. Executing uniform h-refinement
+   for (int i = 0; i < ref; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+   dim = mesh->Dimension();
+
+
+   // 6. Define a finite element space on the mesh.
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+
+   // 6. Set up the linear form (Real and Imaginary part)
+   FunctionCoefficient f_Re(f_exact_Re);
+   FunctionCoefficient f_Im(f_exact_Im);
+
+   // ParLinearForm *b_Re(new ParLinearForm);
+   ComplexLinearForm b(fespace, ComplexOperator::HERMITIAN);
+   b.AddDomainIntegrator(new DomainLFIntegrator(f_Re),
+                         new DomainLFIntegrator(f_Im));
+   b.real().Vector::operator=(0.0);
+   b.imag().Vector::operator=(0.0);
+   b.Assemble();
+
+   // 7. Set up the bilinear form (Real and Imaginary part)
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+   SesquilinearForm a(fespace,ComplexOperator::HERMITIAN);
+   ConstantCoefficient impedance(omega);
+
+
+   Array<int> bdr_attr(mesh->bdr_attributes.Max());
+   bdr_attr = 1;
+   RestrictedCoefficient imp_rest(impedance,bdr_attr);
+   a.AddDomainIntegrator(new DiffusionIntegrator(one),NULL);
+   a.AddDomainIntegrator(new MassIntegrator(sigma),NULL);
+   a.AddBoundaryIntegrator(NULL,new BoundaryMassIntegrator(imp_rest));
+   a.Assemble();
+   a.Finalize();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(mesh->bdr_attributes.Max());
+   ess_bdr = 0;
+   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   // Solution grid function
+   ComplexGridFunction p_gf(fespace);
+
+   OperatorHandle Ah;
+   Vector X, B;
+
+   a.FormLinearSystem(ess_tdof_list, p_gf, b, Ah, X, B);
+
+   ComplexSparseMatrix * AZ = Ah.As<ComplexSparseMatrix>();
+   SparseMatrix * A = AZ->GetSystemMatrix();
+
+
+      cout << "Size of fine grid system: "
+           << A->Height() << " x " << A->Width() << endl;
+
+
+
+
+   ComplexAddSchwarz S(&a,ess_tdof_list, 1);
+	S.SetOperator(*A);
+   S.SetSmoothType(0);
+   S.SetLoadVector(B);
+   // S.SetNumSmoothSteps(7);
+   S.SetDumpingParam(1.0);
+
+	BlkSchwarzSmoother * BlkS = new BlkSchwarzSmoother(mesh,0,fespace,A);
+
+	X = 0.0;
+	GMRESSolver gmres;
+	gmres.SetPreconditioner(*BlkS);
+	gmres.SetOperator(*A);
+	gmres.SetRelTol(1e-4);
+	gmres.SetMaxIter(500);
+	gmres.SetPrintLevel(1);
+	gmres.Mult(B, X);
+
+
+	X = 0.0;
+	gmres.SetPreconditioner(S);
+	gmres.Mult(B, X);
+
+   KLUSolver klu(*A);
+   klu.Mult(B,X);
+
+
+   a.RecoverFEMSolution(X,B,p_gf);
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      string keys;
+      if (dim ==2 )
+      {
+         keys = "keys mrRljc\n";
+      }
+      else
+      {
+         keys = "keys mc\n";
+      }
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re.precision(8);
+      sol_sock_re << "solution\n" << *mesh << p_gf.real() <<
+                  "window_title 'Numerical Pressure (real part): (KLU solver)' "
+                  << keys << flush;
+   }
+
+   delete fespace;
+   delete fec;
+	delete mesh;
+   return 0;
+}
+
+
+//calculate RHS from exact solution f = - \Delta u
+double f_exact_Re(const Vector &x)
+{
+   double f_re = 0.0;
+   double x0 = length/2.0;
+   double x1 = length/2.0;
+   double x2 = length/2.0;
+   x0 = 0.1;
+   x1 = 0.1;
+   double alpha,beta;
+   double n = 5.0 * omega/M_PI;
+   double coeff = pow(n,2)/M_PI;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   alpha = -pow(n,2) * beta;
+   f_re = coeff*exp(alpha);
+
+   // x0 = 0.9;
+   // x1 = 0.9;
+   // n = 5.0 * omega/M_PI;
+   // coeff = pow(n,2)/M_PI;
+   // beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   // if (dim == 3) { beta += pow(x2-x(2),2); }
+   // alpha = -pow(n,2) * beta;
+   // f_re += coeff*exp(alpha);
+   
+   // x0 = 0.9;
+   // x1 = 0.1;
+   // n = 5.0 * omega/M_PI;
+   // coeff = pow(n,2)/M_PI;
+   // beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   // if (dim == 3) { beta += pow(x2-x(2),2); }
+   // alpha = -pow(n,2) * beta;
+   // f_re += coeff*exp(alpha);
+
+   // x0 = 0.1;
+   // x1 = 0.9;
+   // n = 5.0 * omega/M_PI;
+   // coeff = pow(n,2)/M_PI;
+   // beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   // if (dim == 3) { beta += pow(x2-x(2),2); }
+   // alpha = -pow(n,2) * beta;
+   // f_re += coeff*exp(alpha);
+
+   return f_re;
+}
+double f_exact_Im(const Vector &x)
+{
+   double f_im;
+   f_im = 0.0;
+   return f_im;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

examples/maxwell-solver/ST/helmholtz_pml.cpp

@@ -0,0 +1,368 @@
+//
+// Compile with: make helmholtz
+//
+// Sample runs:  helmholtz -m ../data/one-hex.mesh
+//               helmholtz -m ../data/fichera.mesh
+//               helmholtz -m ../data/fichera-mixed.mesh
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Helmholtz problem
+//               -Delta p - omega^2 p = 1 with impedance boundary condition.
+//
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "pml.hpp"
+#include "LSweepsPrecond.hpp"
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution and r.h.s., see below for implementation.
+double f_exact_Re(const Vector &x);
+double f_exact_Im(const Vector &x);
+
+
+
+int dim;
+double omega;
+int sol = 1;
+bool pml = false;
+double length = 1.0;
+double pml_length = 0.25;
+bool scatter = false;
+Array2D<double>comp_bdr;
+
+
+#ifndef MFEM_USE_SUPERLU
+#error This example requires that MFEM is built with MFEM_USE_PETSC=YES
+#endif
+
+int main(int argc, char *argv[])
+{
+
+   // 2. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../../data/one-hex.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // static condensation flag
+   bool static_cond = false;
+   bool visualization = 1;
+   // number of wavelengths
+   double k = 0.5;
+   // number of mg levels
+   int ref = 1;
+   // dimension
+   int nd = 2;
+
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nd, "-nd", "--dim","Problem space dimension");
+   args.AddOption(&sol, "-sol", "--exact",
+                  "Exact solution flag - 0:polynomial, 1: plane wave, -1: unknown exact");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&pml, "-pml", "--pml", "-no-pml",
+                  "--no-pml", "Enable PML.");
+   args.AddOption(&pml_length, "-pml_length", "--pml_length",
+                  "Length of the PML region in each direction");
+   args.AddOption(&length, "-length", "--length",
+                  "length of the domainin in each direction.");
+   args.AddOption(&ref, "-ref", "--ref",
+                  "Number of Refinements.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&scatter, "-scat", "--scattering-prob", "-no-scat",
+                  "--no-scattering", "Solve a scattering problem");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+		args.PrintUsage(cout);
+		return 1;
+   }
+	args.PrintOptions(cout);
+   // Angular frequency
+   omega = 2.0 * M_PI * k;
+
+   // 3. Read the mesh from the given mesh file.
+   Mesh *mesh;
+
+   if (nd == 2)
+   {
+      // mesh = new Mesh(mesh_file,1,1);
+      mesh = new Mesh(1, 1, Element::QUADRILATERAL, true, length, length, false);
+   }
+   else
+   {
+      mesh = new Mesh(1, 1, 1, Element::HEXAHEDRON, true, length, length, length,false);
+   }
+
+   // 3. Executing uniform h-refinement
+   for (int i = 0; i < ref; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+   dim = mesh->Dimension();
+
+   Array<int> directions;
+   int nrlayers = 4;
+   
+   for (int i = 0; i<nrlayers; i++)
+   {
+      for (int comp=0; comp<dim; ++comp)
+      {
+         directions.Append(comp+1);
+         directions.Append(-comp-1);
+      }
+   }
+   // Find uniform h size of the original mesh
+   double h = GetUniformMeshElementSize(mesh);
+   cout << "pml length = " << h*nrlayers << endl;
+   Mesh *mesh_ext = ExtendMesh(mesh,directions);
+
+   Array2D<double> lengths(dim,2);
+   lengths = h*nrlayers;
+   CartesianPML pml(mesh_ext,lengths);
+   pml.SetOmega(omega);
+   comp_bdr.SetSize(dim,2);
+   comp_bdr = pml.GetCompDomainBdr(); 
+
+   // 6. Define a finite element space on the mesh.
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh_ext, fec);
+
+   // 6. Set up the linear form (Real and Imaginary part)
+   FunctionCoefficient f_Re(f_exact_Re);
+   FunctionCoefficient f_Im(f_exact_Im);
+
+   // ParLinearForm *b_Re(new ParLinearForm);
+   ComplexLinearForm b(fespace, ComplexOperator::HERMITIAN);
+   b.AddDomainIntegrator(new DomainLFIntegrator(f_Re),
+                         new DomainLFIntegrator(f_Im));
+   b.real().Vector::operator=(0.0);
+   b.imag().Vector::operator=(0.0);
+   b.Assemble();
+
+   // 7. Set up the bilinear form (Real and Imaginary part)
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+
+
+
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+   ProductCoefficient c2_re(sigma, detJ_re);
+   ProductCoefficient c2_im(sigma, detJ_im);
+
+   SesquilinearForm a(fespace,ComplexOperator::HERMITIAN);
+
+   a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   a.AddDomainIntegrator(new MassIntegrator(c2_re),new MassIntegrator(c2_im));
+
+   a.Assemble();
+   a.Finalize();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(mesh_ext->bdr_attributes.Max());
+   ess_bdr = 1;
+   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   // Solution grid function
+   ComplexGridFunction p_gf(fespace);
+
+   OperatorHandle Ah;
+   Vector X, B;
+
+   a.FormLinearSystem(ess_tdof_list, p_gf, b, Ah, X, B);
+
+   ComplexSparseMatrix * AZ = Ah.As<ComplexSparseMatrix>();
+   SparseMatrix * A = AZ->GetSystemMatrix();
+
+
+   cout << "Size of fine grid system: "
+         << A->Height() << " x " << A->Width() << endl;
+
+
+   LSweepsPrecond S(&a,ess_tdof_list, omega,nrlayers, 1);
+	S.SetOperator(*A);
+   S.SetSmoothType(1);
+   S.SetLoadVector(B);
+   S.SetDumpingParam(1.0);
+   
+   // X = 0.0;
+	// GMRESSolver gmres;
+	// gmres.SetPreconditioner(S);
+	// gmres.SetOperator(*A);
+	// gmres.SetRelTol(1e-8);
+	// gmres.SetMaxIter(500);
+	// gmres.SetPrintLevel(1);
+	// gmres.Mult(B, X);
+
+
+
+
+
+   X = 0.0;
+   Vector z(X.Size()); z = 0.0;
+   Vector r(B);
+   // r = B;
+   Vector ztemp(r.Size());
+
+   int n= 1;
+
+   Vector Ax(X.Size());
+   for (int i = 0; i<n; i++)
+   {
+      A->Mult(X,Ax); Ax *=-1.0;
+      r = b; r+=Ax;
+      // A->AddMult(X,r,-1.0); //r = r-Ax
+      cout << "residual norm =" << r.Norml2() << endl;
+      // S.Mult(r,z); 
+      S.Mult(r,z); 
+      cout << "correction norm =" << z.Norml2() << endl;
+
+      X += z;
+
+      cout << "solution norm =" << X.Norml2() << endl;
+
+      p_gf = 0.0;
+      a.RecoverFEMSolution(X,B,p_gf);
+
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         string keys;
+         if (dim ==2 )
+         {
+            keys = "keys mrRljc\n";
+         }
+         else
+         {
+            keys = "keys mc\n";
+         }
+         socketstream sol_sock_re(vishost, visport);
+         sol_sock_re.precision(8);
+         sol_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+                     "window_title 'Numerical Pressure (real part)' "
+                     << keys << flush;
+      cout << "Iteration " << i << endl;
+      cin.get();
+   }
+
+
+
+   KLUSolver klu(*A);
+   klu.Mult(B,X);
+   ComplexGridFunction p_gf1(fespace);
+
+   a.RecoverFEMSolution(X,B,p_gf1);
+
+   p_gf1 -= p_gf;
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      string keys;
+      if (dim ==2 )
+      {
+         keys = "keys mrRljc\n";
+      }
+      else
+      {
+         keys = "keys mc\n";
+      }
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re.precision(8);
+      sol_sock_re << "solution\n" << *mesh_ext << p_gf1.real() <<
+                  "window_title 'Numerical Pressure (real part from KLU)' "
+                  << keys << flush;
+   }
+
+   delete fespace;
+   delete fec;
+	delete mesh_ext;
+	delete mesh;
+   return 0;
+}
+
+
+//calculate RHS from exact solution f = - \Delta u
+double f_exact_Re(const Vector &x)
+{
+   double f_re = 0.0;
+   double x0 = length/2.0;
+   double x1 = length/2.0;
+   double x2 = length/2.0;
+   x0 = 0.0;
+   x1 = 0.0;
+   double alpha,beta;
+   double n = 5.0 * omega/M_PI;
+   double coeff = pow(n,2)/M_PI;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   alpha = -pow(n,2) * beta;
+   f_re = coeff*exp(alpha);
+
+   bool in_pml = false;
+   for (int i = 0; i<dim; i++)
+   {
+      if (x(i)<=comp_bdr(i,0) || x(i)>=comp_bdr(i,1))
+      {
+         in_pml = true;
+         break;
+      }
+   }
+   if (in_pml) f_re = 0.0;
+
+   return f_re;
+
+}
+double f_exact_Im(const Vector &x)
+{
+   double f_im;
+   f_im = 0.0;
+   return f_im;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

examples/maxwell-solver/ST/helmholtz_pml_ST.cpp

@@ -0,0 +1,448 @@
+//
+// Compile with: make helmholtz
+//
+// Sample runs:  helmholtz -m ../data/one-hex.mesh
+//               helmholtz -m ../data/fichera.mesh
+//               helmholtz -m ../data/fichera-mixed.mesh
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Helmholtz problem
+//               -Delta p - omega^2 p = 1 with impedance boundary condition.
+//
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "pml.hpp"
+// #include "PST.hpp"
+#include "ST.hpp"
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution and r.h.s., see below for implementation.
+double f_exact_Re(const Vector &x);
+double f_exact_Im(const Vector &x);
+
+double wavespeed(const Vector &x);
+
+
+int dim;
+double omega;
+int sol = 1;
+bool pml = false;
+double length = 1.0;
+double pml_length = 0.25;
+bool scatter = false;
+Array2D<double>comp_bdr;
+
+#ifndef MFEM_USE_SUPERLU
+#error This example requires that MFEM is built with MFEM_USE_PETSC=YES
+#endif
+
+int main(int argc, char *argv[])
+{
+
+   // 2. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../../data/one-hex.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // static condensation flag
+   bool static_cond = false;
+   bool visualization = 1;
+   // number of wavelengths
+   double k = 0.5;
+   // number of mg levels
+   int ref = 1;
+   // dimension
+   int nd = 2;
+
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nd, "-nd", "--dim","Problem space dimension");
+   args.AddOption(&sol, "-sol", "--exact",
+                  "Exact solution flag - 0:polynomial, 1: plane wave, -1: unknown exact");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&pml, "-pml", "--pml", "-no-pml",
+                  "--no-pml", "Enable PML.");
+   args.AddOption(&pml_length, "-pml_length", "--pml_length",
+                  "Length of the PML region in each direction");
+   args.AddOption(&length, "-length", "--length",
+                  "length of the domainin in each direction.");
+   args.AddOption(&ref, "-ref", "--ref",
+                  "Number of Refinements.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&scatter, "-scat", "--scattering-prob", "-no-scat",
+                  "--no-scattering", "Solve a scattering problem");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+		args.PrintUsage(cout);
+		return 1;
+   }
+	args.PrintOptions(cout);
+   // Angular frequency
+   omega = 2.0 * M_PI * k;
+
+   // 3. Read the mesh from the given mesh file.
+   Mesh *mesh;
+
+   if (nd == 2)
+   {
+      // mesh = new Mesh(mesh_file,1,1);
+      mesh = new Mesh(1, 1, Element::QUADRILATERAL, true, length, length, false);
+   }
+   else
+   {
+      mesh = new Mesh(1, 1, 1, Element::HEXAHEDRON, true, length, length, length,false);
+   }
+
+   // 3. Executing uniform h-refinement
+   for (int i = 0; i < ref; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+   dim = mesh->Dimension();
+
+   double hl = GetUniformMeshElementSize(mesh);
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin,pmax);
+   double domain_length = pmax[0] - pmin[0];
+   double pml_thickness = 0.25/domain_length;
+   int nrlayers = pml_thickness/hl;
+   // int nrlayers = 4;
+   Array<int> directions;
+   
+   for (int i = 0; i<nrlayers; i++)
+   {
+      for (int comp=0; comp<dim; ++comp)
+      {
+         directions.Append(comp+1);
+         directions.Append(-comp-1);
+      }
+   }
+   // Find uniform h size of the original mesh
+   cout << "pml layers = " << nrlayers << endl;
+   cout << "pml length = " << hl*nrlayers << endl;
+   Mesh *mesh_ext = ExtendMesh(mesh,directions);
+
+
+   Array2D<double> lengths(dim,2);
+   lengths = hl*nrlayers;
+   // lengths[0][1] = 0.0;
+   // lengths[1][1] = 0.0;
+   // lengths[1][0] = 0.0;
+   // lengths[0][0] = 0.0;
+   CartesianPML pml(mesh_ext,lengths);
+   pml.SetOmega(omega);
+   comp_bdr.SetSize(dim,2);
+   comp_bdr = pml.GetCompDomainBdr(); 
+
+   // 6. Define a finite element space on the mesh.
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh_ext, fec);
+
+   // 6. Set up the linear form (Real and Imaginary part)
+   FunctionCoefficient f_Re(f_exact_Re);
+   FunctionCoefficient f_Im(f_exact_Im);
+
+   // ParLinearForm *b_Re(new ParLinearForm);
+   ComplexLinearForm b(fespace, ComplexOperator::HERMITIAN);
+   b.AddDomainIntegrator(new DomainLFIntegrator(f_Re),
+                         new DomainLFIntegrator(f_Im));
+   b.real().Vector::operator=(0.0);
+   b.imag().Vector::operator=(0.0);
+   b.Assemble();
+
+   // 7. Set up the bilinear form (Real and Imaginary part)
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+   FunctionCoefficient ws(wavespeed);
+
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+
+   ProductCoefficient c2_re(c2_re0, ws);
+   ProductCoefficient c2_im(c2_im0, ws);
+
+
+   SesquilinearForm a(fespace,ComplexOperator::HERMITIAN);
+
+   a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   a.AddDomainIntegrator(new MassIntegrator(c2_re),new MassIntegrator(c2_im));
+
+   a.Assemble();
+   a.Finalize();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(mesh_ext->bdr_attributes.Max());
+   ess_bdr = 1;
+   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   // Solution grid function
+   ComplexGridFunction p_gf(fespace);
+
+   OperatorHandle Ah;
+   Vector X, B;
+
+   a.FormLinearSystem(ess_tdof_list, p_gf, b, Ah, X, B);
+
+   ComplexSparseMatrix * AZ = Ah.As<ComplexSparseMatrix>();
+   SparseMatrix * A = AZ->GetSystemMatrix();
+
+
+   cout << "Size of fine grid system: "
+         << A->Height() << " x " << A->Width() << endl;
+
+
+   // PSTP S(&a,lengths, omega, &ws, nrlayers);
+   STP S(&a,lengths, omega, &ws, nrlayers);
+	S.SetOperator(*A);
+   // S.SetLoadVector(B);
+   
+
+
+
+   X = 0.0;
+	GMRESSolver gmres;
+	gmres.SetPreconditioner(S);
+	gmres.SetOperator(*A);
+	gmres.SetRelTol(1e-8);
+	gmres.SetMaxIter(50);
+	gmres.SetPrintLevel(1);
+	gmres.Mult(B, X);
+
+
+
+
+   int n= 50;
+   X = 0.0;
+   Vector z(X.Size()); z = 0.0;
+   Vector r(B);
+   Vector ztemp(r.Size());
+   Vector Ax(X.Size());
+   double tol = 1e-8;
+   cout << endl;
+   
+   for (int i = 0; i<n; i++)
+   {
+      A->Mult(X,Ax); Ax *=-1.0;
+      r = b; r+=Ax;
+      cout << "   ST Solver   Iteration :   " << i <<"  || r || = " <<  r.Norml2() << endl;
+      if (r.Norml2() < tol) 
+      {
+         // cout << "Convergence in " << i+1 << " iterations" << endl;
+         break;
+      }
+      S.Mult(r,z); 
+      X += z;
+      // p_gf = 0.0;
+      // a.RecoverFEMSolution(X,B,p_gf);
+      //    char vishost[] = "localhost";
+      //    int  visport   = 19916;
+      //    string keys;
+      //    if (dim ==2 )
+      //    {
+      //       keys = "keys mrRljc\n";
+      //    }
+      //    else
+      //    {
+      //       keys = "keys mc\n";
+      //    }
+      //    socketstream sol1_sock_re(vishost, visport);
+      //    sol1_sock_re.precision(8);
+      //    sol1_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+      //                "window_title 'Numerical Pressure (real part)' "
+      //                << keys << flush;
+   }
+
+   // KLUSolver klu(*A);
+   // klu.Mult(B,X);
+   a.RecoverFEMSolution(X,B,p_gf);
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      string keys;
+      if (dim ==2 )
+      {
+         keys = "keys mrRljc\n";
+      }
+      else
+      {
+         keys = "keys mc\n";
+      }
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re.precision(8);
+      sol_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+                  "window_title 'Numerical Pressure (real part from KLU)' "
+                  << keys << flush;
+      // socketstream diff_sock_re(vishost, visport);
+      // diff_sock_re.precision(8);
+      // diff_sock_re << "solution\n" << *mesh_ext << p_gf1.real() <<
+      //             "window_title 'Numerical Pressure (real part from KLU)' "
+      //             << keys << flush;
+
+                                  
+   }
+   delete fespace;
+   delete fec;
+	delete mesh_ext;
+	delete mesh;
+   return 0;
+}
+
+
+//calculate RHS from exact solution f = - \Delta u
+double f_exact_Re(const Vector &x)
+{
+   double f_re = 0.0;
+   double x0 = length/2.0;
+   double x1 = length/2.0;
+   double x2 = length/2.0;
+   x0 = 0.1;
+   x1 = 0.5;
+   double alpha,beta;
+   double n = 5.0*omega/M_PI;
+   // double n = 1.0;
+   double coeff = pow(n,2)/M_PI;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   alpha = -pow(n,2) * beta;
+   f_re = coeff*exp(alpha);
+
+   // x0 = 0.9;
+   // x1 = 0.5;
+   // beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   // if (dim == 3) { beta += pow(x2-x(2),2); }
+   // alpha = -pow(n,2) * beta;
+   // f_re += coeff*exp(alpha);
+
+   // x0 = 0.5;
+   // x1 = 0.8;
+   // beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   // if (dim == 3) { beta += pow(x2-x(2),2); }
+   // alpha = -pow(n,2) * beta;
+   // f_re += coeff*exp(alpha);
+
+   bool in_pml = false;
+   for (int i = 0; i<dim; i++)
+   {
+      if (x(i)<=comp_bdr(i,0) || x(i)>=comp_bdr(i,1))
+      {
+         in_pml = true;
+         break;
+      }
+   }
+   if (in_pml) f_re = 0.0;
+
+   return f_re;
+
+}
+double f_exact_Im(const Vector &x)
+{
+   double f_im;
+   f_im = 0.0;
+   return f_im;
+}
+
+double wavespeed(const Vector &x)
+{
+   double ws;
+   // if (x(0) <= 0.25)
+   // {
+   //    ws = 1.0;
+   // }
+   // else if(x(0)<=0.5)
+   // {
+   //    ws = 1.0;
+   // }
+   // else if(x(0)<=0.75)
+   // {
+   //    ws = 0.75;
+   //    // ws = 0.5;
+   // }
+   // else 
+   // {
+   //    ws = 0.75;
+   //    // ws = 1.0;
+   // }
+   // if (x(1) <= 1.0/3.0)
+   // {
+   //    ws = 2.0;
+   // }
+   // else if(x(1)<=2.0/3.0)
+   // {
+   //    ws = 1.0;
+   // }
+   // else 
+   // {
+   //    // ws = 0.75;
+   //    ws = 0.25;
+   // }
+
+   // if (x(0) <= 0.33)
+   // {
+   //    ws = 1.0;
+   // }
+   // else if(x(0)<=0.66)
+   // {
+   //    ws = -0.65 + 5.0*x(0);
+   // }
+   // else 
+   // {
+   //    ws = 2.65;
+   //    // ws = 0.5;
+   // }
+
+
+   ws = 1.0;
+   return ws;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

examples/maxwell-solver/ST/helmholtzp.cpp

@@ -0,0 +1,317 @@
+//
+// Compile with: make helmholtz
+//
+// Sample runs:  helmholtz -m ../data/one-hex.mesh
+//               helmholtz -m ../data/fichera.mesh
+//               helmholtz -m ../data/fichera-mixed.mesh
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Helmholtz problem
+//               -Delta p - omega^2 p = 1 with impedance boundary condition.
+//
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "complex_additive_schwarzp.hpp"
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution and r.h.s., see below for implementation.
+double f_exact_Re(const Vector &x);
+double f_exact_Im(const Vector &x);
+
+int dim;
+double omega;
+int sol = 1;
+bool pml = false;
+double length = 1.0;
+double pml_length = 0.25;
+bool scatter = false;
+
+#ifndef MFEM_USE_SUPERLU
+#error This example requires that MFEM is built with MFEM_USE_PETSC=YES
+#endif
+
+int main(int argc, char *argv[])
+{
+
+   // 1. Initialise MPI
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);                    // Initialise MPI
+   MPI_Comm_size(MPI_COMM_WORLD,
+                 &num_procs); //total number of processors available
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);      // Determine process identifier
+
+   //-----------------------------------------------------------------------------
+
+   // 2. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../../data/one-hex.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // static condensation flag
+   bool static_cond = false;
+   bool visualization = 1;
+   // number of wavelengths
+   double k = 0.5;
+   // number of mg levels
+   int ref = 1;
+   // number of initial ref
+   int initref = 1;
+   // dimension
+   int nd = 2;
+
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nd, "-nd", "--dim","Problem space dimension");
+   args.AddOption(&sol, "-sol", "--exact",
+                  "Exact solution flag - 0:polynomial, 1: plane wave, -1: unknown exact");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&pml, "-pml", "--pml", "-no-pml",
+                  "--no-pml", "Enable PML.");
+   args.AddOption(&pml_length, "-pml_length", "--pml_length",
+                  "Length of the PML region in each direction");
+   args.AddOption(&length, "-length", "--length",
+                  "length of the domainin in each direction.");
+   args.AddOption(&ref, "-ref", "--ref",
+                  "Number of Refinements.");
+   args.AddOption(&initref, "-initref", "--initref",
+                  "Number of initial refinements.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&scatter, "-scat", "--scattering-prob", "-no-scat",
+                  "--no-scattering", "Solve a scattering problem");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+   // Angular frequency
+   omega = 2.0 * M_PI * k;
+
+   // 3. Read the mesh from the given mesh file.
+   Mesh *mesh;
+
+   if (nd == 2)
+   {
+      mesh = new Mesh(1, 1, Element::QUADRILATERAL, true, length, length, false);
+   }
+   else
+   {
+      mesh = new Mesh(1, 1, 1, Element::HEXAHEDRON, true, length, length, length,
+                      false);
+   }
+
+   // 3. Executing uniform h-refinement
+   for (int i = 0; i < initref; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+   dim = mesh->Dimension();
+
+   // 5. Define a parallel mesh and delete the serial mesh.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   // ----------------------------------------------------------------------------
+
+   for (int i = 0; i < ref; i++)
+   {
+      pmesh->UniformRefinement();
+   }
+
+   // 6. Define a finite element space on the mesh.
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+
+   // 6. Set up the linear form (Real and Imaginary part)
+   FunctionCoefficient f_Re(f_exact_Re);
+   FunctionCoefficient f_Im(f_exact_Im);
+
+   // ParLinearForm *b_Re(new ParLinearForm);
+   ParComplexLinearForm b(fespace, ComplexOperator::HERMITIAN);
+   b.AddDomainIntegrator(new DomainLFIntegrator(f_Re),
+                         new DomainLFIntegrator(f_Im));
+   b.real().Vector::operator=(0.0);
+   b.imag().Vector::operator=(0.0);
+   b.Assemble();
+
+   // 7. Set up the bilinear form (Real and Imaginary part)
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+   ParSesquilinearForm * a = new ParSesquilinearForm(fespace,
+                                                     ComplexOperator::HERMITIAN);
+   ConstantCoefficient impedance(omega);
+
+
+   Array<int> bdr_attr(pmesh->bdr_attributes.Max());
+   bdr_attr = 1;
+   RestrictedCoefficient imp_rest(impedance,bdr_attr);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one),NULL);
+   a->AddDomainIntegrator(new MassIntegrator(sigma),NULL);
+   a->AddBoundaryIntegrator(NULL,new BoundaryMassIntegrator(imp_rest));
+   a->Assemble();
+   a->Finalize();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+   ess_bdr = 0;
+   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   // Solution grid function
+   ParComplexGridFunction p_gf(fespace);
+   ParComplexGridFunction p_gf_ex(fespace);
+
+   OperatorHandle Ah;
+   Vector X, B;
+
+   a->FormLinearSystem(ess_tdof_list, p_gf, b, Ah, X, B);
+
+   ComplexHypreParMatrix * AZ = Ah.As<ComplexHypreParMatrix>();
+   HypreParMatrix * A = AZ->GetSystemMatrix();
+
+
+   if (myid == 0)
+   {
+      cout << "Size of fine grid system: "
+           << A->GetGlobalNumRows() << " x " << A->GetGlobalNumCols() << endl;
+   }
+
+   SuperLURowLocMatrix * Arow = new SuperLURowLocMatrix(*A);
+   SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
+   superlu->SetPrintStatistics(false);
+   superlu->SetSymmetricPattern(true);
+   superlu->SetColumnPermutation(superlu::PARMETIS);
+   superlu->SetOperator(*Arow);
+   superlu->Mult(B,X);
+
+   a->RecoverFEMSolution(X,B,p_gf);
+
+
+
+   ComplexParAddSchwarz * test = new ComplexParAddSchwarz(a);
+   delete test;
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      string keys;
+      if (dim ==2 )
+      {
+         keys = "keys mrRljc\n";
+      }
+      else
+      {
+         keys = "keys mc\n";
+      }
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_re.precision(8);
+      sol_sock_re << "solution\n" << *pmesh << p_gf.real() <<
+                  "window_title 'Numerical Pressure (real part)' "
+                  << keys << flush;
+   }
+
+   delete a;
+   delete fespace;
+   delete fec;
+   delete pmesh;
+   MPI_Finalize();
+   return 0;
+}
+
+
+//calculate RHS from exact solution f = - \Delta u
+double f_exact_Re(const Vector &x)
+{
+   double f_re = 0.0;
+   double x0 = length/2.0;
+   double x1 = length/2.0;
+   double x2 = length/2.0;
+   double alpha,beta;
+   double n = 5.0 * omega/M_PI;
+   double coeff = pow(n,2)/M_PI;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   alpha = -pow(n,2) * beta;
+   f_re = coeff*exp(alpha);
+   return f_re;
+}
+double f_exact_Im(const Vector &x)
+{
+   double f_im;
+   f_im = 0.0;
+   return f_im;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+// int ndofs = nodes->FESpace()->GetNDofs();
+//    Vector xcoords(ndofs), ycoords(ndofs), zcoords(ndofs);
+
+//    for (int comp = 0; comp < nodes->FESpace()->GetVDim(); comp++)
+//    {
+//       for (int i = 0; i < ndofs; i++)
+//       {
+//          if (comp == 0)
+//          {
+//             xcoords(i) = *nodes[nodes->FESpace()->DofToVDof(i, comp)];
+//          }
+//          else if (comp == 1)
+//          {
+//             ycoords(i) = *nodes[nodes->FESpace()->DofToVDof(i, comp)];
+//          }
+//          else if (comp == 2)
+//          {
+//             zcoords(i) = *nodes[nodes->FESpace()->DofToVDof(i, comp)];
+//          }
+//       }
+//    }

examples/maxwell-solver/ST/makefile

@@ -0,0 +1,63 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/maxwell-solver,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = example1 helmholtz helmholtz_pml helmholtz_pml_ST mesh_partition
+PAR_EXAMPLES = example1p helmholtzp
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean
+.PRECIOUS: %.o
+
+COMMON_O= schwarz.o complex_additive_schwarz.o \
+		    complex_additive_schwarzp.o additive_schwarz.o \
+			 additive_schwarzp.o pml.o PST.o ST.o
+
+# Remove built-in rules
+%: %.cpp
+%.o: %.cpp
+
+all: $(EXAMPLES)
+
+# Rules for building the EXAMPLES
+
+%: $(SRC)%.cpp $(COMMON_O) $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(COMMON_O) $(MFEM_LIBS)
+
+# Rules for compiling miniapp dependencies
+$(COMMON_O) $($(EXAMPLES)): \
+%.o: $(SRC)%.cpp $(SRC)%.hpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $(<) -o $(@)
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+	rm output/*
+
+

examples/maxwell-solver/ST/mesh_add_layer.cpp

@@ -0,0 +1,308 @@
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
+   //    the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   // Mesh *mesh = new Mesh(4,4, Element::QUADRILATERAL, true, 1.0, 1.0, false);
+
+   int dim = mesh->Dimension();
+
+   // 14. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "mesh\n" << *mesh << 
+       "window_title 'Original Mesh' " << flush;
+   }
+
+   // Extend the mesh by n layers
+   // This is assuming uniform quad/hex mesh (for now)
+   
+   
+
+   // extrute on one dimension
+   // d = 1 +x, -1 -x, 2 +y, -2 +y , 3 +z, -3, -z
+
+   // copy the original mesh;
+   Mesh * mesh_orig = new Mesh(*mesh);
+   Mesh * mesh_ext = nullptr;
+
+   Array<int> directions(6);
+   directions[0] = 1;
+   directions[1] = -1;
+   directions[2] = 2;
+   directions[3] = -2;
+   directions[4] = 2;
+   directions[5] = -1;
+
+   for (int j=0; j<directions.Size(); j++)
+   {
+      int d = directions[j];
+      int nrelem = mesh_orig->GetNE();
+
+
+      Vector pmin;
+      Vector pmax;
+      mesh_orig->GetBoundingBox(pmin,pmax);
+
+      DenseMatrix J(dim);
+      double hmin, hmax;
+      hmin = infinity();
+      hmax = -infinity();
+      Vector attr(nrelem);
+      // element size
+
+      for (int iel=0; iel<nrelem; ++iel)
+      {
+         int geom = mesh_orig->GetElementBaseGeometry(iel);
+         ElementTransformation *T = mesh_orig->GetElementTransformation(iel);
+         T->SetIntPoint(&Geometries.GetCenter(geom));
+         Geometries.JacToPerfJac(geom, T->Jacobian(), J);
+         attr(iel) = J.Det();
+         if (attr(iel) < 0.0)
+         {
+            attr(iel) = -pow(-attr(iel), 1.0/double(dim));
+         }
+         else
+         {
+            attr(iel) = pow(attr(iel), 1.0/double(dim));
+         }
+         hmin = min(hmin, attr(iel));
+         hmax = max(hmax, attr(iel));
+      }
+      MFEM_VERIFY(hmin==hmax, "Case not supported yet")
+
+      double val;
+      // find the vertices on the specific boundary
+      switch (d)
+      {
+      case 1:
+         val = pmax[0];
+         break;
+      case -1:
+         val = pmin[0];
+         hmax = -hmax;
+         break;    
+      case 2:
+         val = pmax[1];
+         break;
+      case -2:
+         val = pmin[1];
+         hmax = -hmax;
+         break;   
+      case 3:
+         val = pmax[2];
+         break;
+      case -3:
+         val = pmin[2];
+         hmax = -hmax;
+         break;      
+      }
+      int k = 0;
+      for (int i = 0; i<mesh_orig->GetNV(); ++i)
+      {
+         double * coords = mesh_orig->GetVertex(i);
+         switch (abs(d))
+         {
+         case 1:
+            if (coords[0] == val) k++;
+            break;
+         case 2:
+            if (coords[1] == val) k++;
+            break;   
+         case 3:
+            if (coords[2] == val) k++;
+            break;      
+         }
+      }
+      int nrvertices = mesh_orig->GetNV() + k;
+      int nrelements = mesh_orig->GetNE() + pow(pow(k,1.0/(dim-1))-1.0,dim-1);
+
+      mesh_ext = new Mesh(dim, nrvertices, nrelements);
+
+      // Add existing vertices
+      Array<int> vmap(mesh_orig->GetNV()); vmap = 0;
+      k = mesh_orig->GetNV();
+      for (int i=0; i<mesh_orig->GetNV(); ++i)
+      {
+         double * vert = mesh_orig->GetVertex(i);
+         mesh_ext->AddVertex(vert);
+         switch (abs(d))
+         {
+         case 1:
+            if (vert[0] == val) 
+            {
+               vmap[i] = k;
+               k++;
+            }
+            break;
+         case 2:
+            if (vert[1] == val)
+            {
+               vmap[i] = k;
+               k++;
+            }
+            break;   
+         case 3:
+            if (vert[2] == val)
+            {
+               vmap[i] = k;
+               k++;
+            }
+            break;      
+         }
+      } 
+      // Add existing elements
+      for (int i=0; i<mesh_orig->GetNE(); ++i)
+      {
+         Array<int>ind;
+         mesh_orig->GetElementVertices(i,ind);
+         if (dim == 2)
+         {
+            mesh_ext->AddQuad(ind);
+         }
+         else if (dim == 3)
+         {
+            mesh_ext->AddHex(ind);
+         }
+      } 
+      // Add new vertices
+      k = mesh_orig->GetNV();
+      for (int i=0; i<mesh_orig->GetNV(); ++i)
+      {
+         double * vert = mesh_orig->GetVertex(i);
+         switch (abs(d))
+         {
+         case 1:
+            if (vert[0] == val) 
+            {
+               double coords[dim];
+               coords[0] = vert[0] + hmax;
+               coords[1] = vert[1];
+               if (dim == 3) coords[2] = vert[2];
+               mesh_ext->AddVertex(coords);
+            }
+            break;
+         case 2:
+            if (vert[1] == val)
+            {
+               double coords[dim];
+               coords[0] = vert[0];
+               coords[1] = vert[1] + hmax;
+               if (dim == 3) coords[2] = vert[2];
+               mesh_ext->AddVertex(coords);
+            }
+            break;   
+         case 3:
+            if (vert[2] == val)
+            {
+               double coords[dim];
+               coords[0] = vert[0];
+               coords[1] = vert[1];
+               coords[2] = vert[2] + hmax;
+               mesh_ext->AddVertex(coords);
+            }
+            break;      
+         }
+      }    
+      // loop through boundary elements and extend in the given direction
+      for (int i=0; i<mesh_orig->GetNBE(); ++i) 
+      {
+         Array<int> vertices;
+         mesh_orig->GetBdrElementVertices(i,vertices);
+         if (dim == 2)
+         {
+            int ind[4];
+            if (vmap[vertices[0]] && vmap[vertices[1]])
+            {
+               ind[0] = vmap[vertices[0]];
+               ind[1] = vmap[vertices[1]];
+               ind[2] = vertices[1];
+               ind[3] = vertices[0];
+               mesh_ext->AddQuad(ind);
+            }
+         }
+         else if (dim == 3)
+         {
+            int ind[8];
+            if (vmap[vertices[0]] && vmap[vertices[1]] && vmap[vertices[2]] && vmap[vertices[3]])
+            {
+               ind[0] = vmap[vertices[0]];
+               ind[1] = vmap[vertices[1]];
+               ind[2] = vmap[vertices[2]];
+               ind[3] = vmap[vertices[3]];
+               ind[4] = vertices[0];
+               ind[5] = vertices[1];
+               ind[6] = vertices[2];
+               ind[7] = vertices[3];
+               mesh_ext->AddHex(ind);
+            }
+         }   
+      }
+      mesh_ext->FinalizeTopology();
+
+      if (visualization)
+      {
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         socketstream mesh_sock(vishost, visport);
+         mesh_sock.precision(8);
+         mesh_sock << "mesh\n" << *mesh_ext <<
+         "window_title 'New Mesh' " << flush;
+      }
+
+      if (j<directions.Size())
+      {
+         delete mesh_orig;
+         mesh_orig = mesh_ext;
+      }
+   }
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream mesh_sock(vishost, visport);
+      mesh_sock.precision(8);
+      mesh_sock << "mesh\n" << *mesh_ext <<
+      "window_title 'New Mesh' " << flush;
+   }
+
+   // 15. Free the used memory.
+   delete mesh;
+   return 0;
+}

examples/maxwell-solver/ST/mesh_extension.cpp

@@ -0,0 +1,306 @@
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+Mesh * ExtendMesh(Mesh * mesh, const Array<int> & directions);
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
+   //    the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   // Mesh *mesh = new Mesh(4,4, Element::QUADRILATERAL, true, 1.0, 1.0, false);
+
+   // 14. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "mesh\n" << *mesh << 
+       "window_title 'Original Mesh' " <<  "keys anm" << flush;
+   }
+
+
+   Array<int> directions;
+   // directions.Append(2);
+   // directions.Append(2);
+   // directions.Append(1);
+   // directions.Append(-1);
+   // directions.Append(2);
+   // directions.Append(-2);
+   // directions.Append(-1);
+
+   directions.Append(1);
+   directions.Append(-1);
+   directions.Append(2);
+   directions.Append(-2);
+   directions.Append(3);
+   directions.Append(-3);
+   // directions.Append(-1);
+
+   Mesh * mesh_ext = ExtendMesh(mesh,directions);
+  
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream mesh_sock(vishost, visport);
+      mesh_sock.precision(8);
+      mesh_sock << "mesh\n" << *mesh_ext <<
+      "window_title 'New Mesh' " << "keys anm" << flush;
+   }
+
+   // 15. Free the used memory.
+   delete mesh;
+   return 0;
+}
+
+Mesh * ExtendMesh(Mesh * mesh, const Array<int> & directions)
+{
+    // extrute on one dimension
+   // flag = 1 +x, -1 -x, 2 +y, -2 +y , 3 +z, -3, -z
+
+   // copy the original mesh;
+   Mesh * mesh_orig = new Mesh(*mesh);
+   int dim = mesh_orig->Dimension();
+
+//    Mesh * mesh_ext;
+
+//    for (int j=0; j<directions.Size(); j++)
+//    {
+//       int d = directions[j];
+//       MFEM_VERIFY(abs(d)<= dim, "Cannot Extend in dimension " << d << ". Dim = " << dim << endl);
+//       int nrelem = mesh_orig->GetNE();
+
+
+//       Vector pmin;
+//       Vector pmax;
+//       mesh_orig->GetBoundingBox(pmin,pmax);
+
+//       DenseMatrix J(dim);
+//       double hmin, hmax;
+//       hmin = infinity();
+//       hmax = -infinity();
+//       Vector attr(nrelem);
+//       // element size
+
+//       for (int iel=0; iel<nrelem; ++iel)
+//       {
+//          int geom = mesh_orig->GetElementBaseGeometry(iel);
+//          ElementTransformation *T = mesh_orig->GetElementTransformation(iel);
+//          T->SetIntPoint(&Geometries.GetCenter(geom));
+//          Geometries.JacToPerfJac(geom, T->Jacobian(), J);
+//          attr(iel) = J.Det();
+//          attr(iel) = pow(abs(attr(iel)), 1.0/double(dim));
+//          hmin = min(hmin, attr(iel));
+//          hmax = max(hmax, attr(iel));
+//       }
+//       MFEM_VERIFY(hmin==hmax, "Case not supported yet")
+
+//       double val;
+//       // find the vertices on the specific boundary
+//       switch (d)
+//       {
+//       case 1:
+//          val = pmax[0];
+//          break;
+//       case -1:
+//          val = pmin[0];
+//          hmax = -hmax;
+//          break;    
+//       case 2:
+//          val = pmax[1];
+//          break;
+//       case -2:
+//          val = pmin[1];
+//          hmax = -hmax;
+//          break;   
+//       case 3:
+//          val = pmax[2];
+//          break;
+//       case -3:
+//          val = pmin[2];
+//          hmax = -hmax;
+//          break;      
+//       }
+//       int k = 0;
+//       for (int i = 0; i<mesh_orig->GetNV(); ++i)
+//       {
+//          double * coords = mesh_orig->GetVertex(i);
+//          switch (abs(d))
+//          {
+//          case 1:
+//             if (coords[0] == val) k++;
+//             break;
+//          case 2:
+//             if (coords[1] == val) k++;
+//             break;   
+//          case 3:
+//             if (coords[2] == val) k++;
+//             break;      
+//          }
+//       }
+//       int nrvertices = mesh_orig->GetNV() + k;
+//       int nrelements = mesh_orig->GetNE() + pow(pow(k,1.0/(dim-1))-1.0,dim-1);
+
+//       mesh_ext = new Mesh(dim, nrvertices, nrelements);
+
+//       // Add existing vertices
+//       Array<int> vmap(mesh_orig->GetNV()); vmap = 0;
+//       k = mesh_orig->GetNV();
+//       for (int i=0; i<mesh_orig->GetNV(); ++i)
+//       {
+//          double * vert = mesh_orig->GetVertex(i);
+//          mesh_ext->AddVertex(vert);
+//          switch (abs(d))
+//          {
+//          case 1:
+//             if (vert[0] == val) 
+//             {
+//                vmap[i] = k;
+//                k++;
+//             }
+//             break;
+//          case 2:
+//             if (vert[1] == val)
+//             {
+//                vmap[i] = k;
+//                k++;
+//             }
+//             break;   
+//          case 3:
+//             if (vert[2] == val)
+//             {
+//                vmap[i] = k;
+//                k++;
+//             }
+//             break;      
+//          }
+//       } 
+//       // Add existing elements
+//       for (int i=0; i<mesh_orig->GetNE(); ++i)
+//       {
+//          Array<int>ind;
+//          mesh_orig->GetElementVertices(i,ind);
+//          if (dim == 2)
+//          {
+//             mesh_ext->AddQuad(ind);
+//          }
+//          else if (dim == 3)
+//          {
+//             mesh_ext->AddHex(ind);
+//          }
+//       } 
+//       // Add new vertices
+//       k = mesh_orig->GetNV();
+//       for (int i=0; i<mesh_orig->GetNV(); ++i)
+//       {
+//          double * vert = mesh_orig->GetVertex(i);
+//          switch (abs(d))
+//          {
+//          case 1:
+//             if (vert[0] == val) 
+//             {
+//                double coords[dim];
+//                coords[0] = vert[0] + hmax;
+//                coords[1] = vert[1];
+//                if (dim == 3) coords[2] = vert[2];
+//                mesh_ext->AddVertex(coords);
+//             }
+//             break;
+//          case 2:
+//             if (vert[1] == val)
+//             {
+//                double coords[dim];
+//                coords[0] = vert[0];
+//                coords[1] = vert[1] + hmax;
+//                if (dim == 3) coords[2] = vert[2];
+//                mesh_ext->AddVertex(coords);
+//             }
+//             break;   
+//          case 3:
+//             if (vert[2] == val)
+//             {
+//                double coords[dim];
+//                coords[0] = vert[0];
+//                coords[1] = vert[1];
+//                coords[2] = vert[2] + hmax;
+//                mesh_ext->AddVertex(coords);
+//             }
+//             break;      
+//          }
+//       }    
+//       // loop through boundary elements and extend in the given direction
+//       for (int i=0; i<mesh_orig->GetNBE(); ++i) 
+//       {
+//          Array<int> vertices;
+//          mesh_orig->GetBdrElementVertices(i,vertices);
+//          if (dim == 2)
+//          {
+//             int ind[4];
+//             if (vmap[vertices[0]] && vmap[vertices[1]])
+//             {
+//                ind[0] = vmap[vertices[0]];
+//                ind[1] = vmap[vertices[1]];
+//                ind[2] = vertices[1];
+//                ind[3] = vertices[0];
+//                mesh_ext->AddQuad(ind);
+//             }
+//          }
+//          else if (dim == 3)
+//          {
+//             int ind[8];
+//             if (vmap[vertices[0]] && vmap[vertices[1]] && vmap[vertices[2]] && vmap[vertices[3]])
+//             {
+//                ind[0] = vmap[vertices[0]];
+//                ind[1] = vmap[vertices[1]];
+//                ind[2] = vmap[vertices[2]];
+//                ind[3] = vmap[vertices[3]];
+//                ind[4] = vertices[0];
+//                ind[5] = vertices[1];
+//                ind[6] = vertices[2];
+//                ind[7] = vertices[3];
+//                mesh_ext->AddHex(ind);
+//             }
+//          }   
+//       }
+//       mesh_ext->FinalizeTopology();
+
+//       if (j<directions.Size()-1)
+//       {
+//          delete mesh_orig;
+//          mesh_orig = mesh_ext;
+//       }
+//    }
+//    delete mesh_orig;
+//    return mesh_ext;
+// }

examples/maxwell-solver/ST/pml.cpp

@@ -0,0 +1,170 @@
+#include "pml.hpp"
+
+
+CartesianPML::CartesianPML(Mesh *mesh_, Array2D<double> length_)
+   : mesh(mesh_), length(length_)
+{
+   dim = mesh->Dimension();
+   SetBoundaries();
+}
+
+void CartesianPML::SetBoundaries()
+{
+   comp_dom_bdr.SetSize(dim, 2);
+   dom_bdr.SetSize(dim, 2);
+   // initialize with any vertex
+   for (int i = 0; i < dim; i++)
+   {
+      dom_bdr(i, 0) = mesh->GetVertex(0)[i];
+      dom_bdr(i, 1) = mesh->GetVertex(0)[i];
+   }
+
+   for (int i = 0; i < mesh->GetNBE(); i++)
+   {
+      Array<int> bdr_vertices;
+      mesh->GetBdrElementVertices(i, bdr_vertices);
+      for (int j = 0; j < bdr_vertices.Size(); j++)
+      {
+         for (int k = 0; k < dim; k++)
+         {
+            dom_bdr(k, 0) = min(dom_bdr(k, 0), mesh->GetVertex(bdr_vertices[j])[k]);
+            dom_bdr(k, 1) = max(dom_bdr(k, 1), mesh->GetVertex(bdr_vertices[j])[k]);
+         }
+      }
+   }
+
+   for (int i = 0; i < dim; i++)
+   {
+      comp_dom_bdr(i, 0) = dom_bdr(i, 0) + length(i, 0);
+      comp_dom_bdr(i, 1) = dom_bdr(i, 1) - length(i, 1);
+   }
+}
+
+void CartesianPML::SetAttributes(Mesh *mesh_)
+{
+   int nrelem = mesh_->GetNE();
+   elems.SetSize(nrelem);
+
+   for (int i = 0; i < nrelem; ++i)
+   {
+      elems[i] = 1;
+      bool in_pml = false;
+      Element *el = mesh_->GetElement(i);
+      Array<int> vertices;
+      // Initialize Attribute
+      el->SetAttribute(1);
+      el->GetVertices(vertices);
+      int nrvert = vertices.Size();
+      // Check if any vertex is in the pml
+      for (int iv = 0; iv < nrvert; ++iv)
+      {
+         int vert_idx = vertices[iv];
+         double *coords = mesh_->GetVertex(vert_idx);
+         for (int comp = 0; comp < dim; ++comp)
+         {
+            if (coords[comp] > comp_dom_bdr(comp, 1) ||
+                coords[comp] < comp_dom_bdr(comp, 0))
+            {
+               in_pml = true;
+               break;
+            }
+         }
+      }
+      if (in_pml)
+      {
+         elems[i] = 0;
+         el->SetAttribute(2);
+      }
+   }
+   mesh_->SetAttributes();
+}
+
+void CartesianPML::StretchFunction(const Vector &x,
+                                   vector<complex<double>> &dxs, double omega)
+{
+   complex<double> zi = complex<double>(0., 1.);
+
+   double n = 2.0;
+   double c = 5.0;
+   double coeff;
+   // Stretch in each direction independently
+   for (int i = 0; i < dim; ++i)
+   {
+      dxs[i] = 1.0;
+      if (x(i) >= comp_dom_bdr(i, 1))
+      {
+         coeff = n * c / omega / pow(length(i, 1), n);
+         dxs[i] = 1.0 + zi * coeff * abs(pow(x(i) - comp_dom_bdr(i, 1), n - 1.0));
+      }
+      if (x(i) <= comp_dom_bdr(i, 0))
+      {
+         coeff = n * c / omega / pow(length(i, 0), n);
+         dxs[i] = 1.0 + zi * coeff * abs(pow(x(i) - comp_dom_bdr(i, 0), n - 1.0));
+      }
+   }
+}
+
+
+double pml_detJ_Re(const Vector & x, CartesianPML * pml)
+{
+   int dim = pml->dim;
+   double omega = pml->omega;
+   std::vector<std::complex<double>> dxs(dim);
+   complex<double> det(1.0,0.0);
+   pml->StretchFunction(x, dxs, omega);
+   for (int i=0; i<dim; ++i) det *= dxs[i];
+   return det.real();
+}
+
+double pml_detJ_Im(const Vector & x, CartesianPML * pml)
+{
+   int dim = pml->dim;
+   double omega = pml->omega;
+   std::vector<std::complex<double>> dxs(dim);
+   complex<double> det(1.0,0.0);
+   pml->StretchFunction(x, dxs, omega);
+   for (int i=0; i<dim; ++i) det *= dxs[i];
+   return det.imag();
+}
+
+void pml_detJ_JT_J_inv_Re(const Vector & x, CartesianPML * pml , DenseMatrix & M)
+{
+   int dim = pml->dim;
+   double omega = pml->omega;
+   std::vector<std::complex<double>> dxs(dim);
+   complex<double> det(1.0,0.0);
+   pml->StretchFunction(x, dxs, omega);
+
+   for (int i = 0; i<dim; ++i)
+   {
+      det *= dxs[i];
+   }
+
+   M=0.0;
+   for (int i = 0; i<dim; ++i)
+   {
+      M(i,i) = (det / pow(dxs[i],2)).real();
+   }
+}
+
+void pml_detJ_JT_J_inv_Im(const Vector & x, CartesianPML * pml , DenseMatrix & M)
+{
+   int dim = pml->dim;
+   double omega = pml->omega;
+
+   std::vector<std::complex<double>> dxs(dim);
+   complex<double> det = 1.0;
+   pml->StretchFunction(x, dxs, omega);
+
+   for (int i = 0; i<dim; ++i)
+   {
+      det *= dxs[i];
+   }
+
+   M=0.0;
+   for (int i = 0; i<dim; ++i)
+   {
+      M(i,i) = (det / pow(dxs[i],2)).imag();
+   }
+}
+

examples/maxwell-solver/ST/pml.hpp

@@ -0,0 +1,101 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// Class for setting up a simple Cartesian PML region
+class CartesianPML
+{
+private:
+   Mesh *mesh;
+
+   // Length of the PML Region in each direction
+   Array2D<double> length;
+
+   // Computational Domain Boundary
+   Array2D<double> comp_dom_bdr;
+
+   // Domain Boundary
+   Array2D<double> dom_bdr;
+
+   // Integer Array identifying elements in the pml
+   // 0: in the pml, 1: not in the pml
+   Array<int> elems;
+
+   // Compute Domain and Computational Domain Boundaries
+   void SetBoundaries();
+
+public:
+   // Constructor
+   CartesianPML(Mesh *mesh_,Array2D<double> length_);
+
+   int dim;
+   double omega;
+   // Return Computational Domain Boundary
+   Array2D<double> GetCompDomainBdr() {return comp_dom_bdr;}
+
+   // Return Domain Boundary
+   Array2D<double> GetDomainBdr() {return dom_bdr;}
+
+   // Return Marker list for elements
+   Array<int> * GetMarkedPMLElements() {return &elems;}
+
+   // Mark element in the PML region
+   void SetAttributes(Mesh *mesh_);
+
+   void SetOmega(double omega_) {omega = omega_;}
+
+   // PML complex stretching function
+   void StretchFunction(const Vector &x, vector<complex<double>> &dxs, double omega);
+};
+
+
+class PmlCoefficient : public Coefficient
+{
+private:
+   CartesianPML * pml = nullptr;
+   double (*Function)(const Vector &, CartesianPML * );
+public:
+   PmlCoefficient(double (*F)(const Vector &, CartesianPML *), CartesianPML * pml_)
+      : pml(pml_), Function(F)
+   {}
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+   {
+      double x[3];
+      Vector transip(x, 3);
+      T.Transform(ip, transip);
+      return ((*Function)(transip, pml));
+   }
+};
+
+
+// This includes scalar coefficients
+class PmlMatrixCoefficient : public MatrixCoefficient
+{
+private:
+   CartesianPML * pml = nullptr;
+   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
+public:
+   PmlMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                          DenseMatrix &),
+                        CartesianPML * pml_)
+      : MatrixCoefficient(dim), pml(pml_), Function(F)
+   {}
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip)
+   {
+      double x[3];
+      Vector transip(x, 3);
+      T.Transform(ip, transip);
+      K.SetSize(height, width);
+      (*Function)(transip, pml, K);
+   }
+};
+
+double pml_detJ_Re(const Vector & x, CartesianPML * pml);
+double pml_detJ_Im(const Vector & x, CartesianPML * pml);
+void pml_detJ_JT_J_inv_Re(const Vector & x, CartesianPML * pml , DenseMatrix & M);
+void pml_detJ_JT_J_inv_Im(const Vector & x, CartesianPML * pml , DenseMatrix & M);

examples/maxwell-solver/ST/schwarz.cpp

@@ -0,0 +1,510 @@
+
+#include "mfem.hpp"
+#include "schwarz.hpp"
+
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+
+void print(std::vector<int> const &input)
+{
+	for (int i = 0; i < (int)input.size(); i++) {
+		std::cout << input.at(i) << ' ';
+	}
+}
+
+// constructor
+patch_nod_info::patch_nod_info(Mesh *mesh_, int ref_levels_)
+    : mesh(mesh_), ref_levels(ref_levels_)
+{
+   /* The patches are defined by all the "active" vertices of the coarse mesh
+   We define a low order H1 fespace and perform refinements so that we can get
+   the H1 prolongation operator recursively. This way we can easily find  
+   all the patches that the fine mesh vertices contribute to. After the vertices 
+   are done the edges, faces and elements can be found easily because they
+   contribute to the same patches as their vertices. */
+
+   // Number of patches
+   nrpatch = mesh->GetNV();
+   int dim = mesh->Dimension();
+   FiniteElementCollection *fec = new H1_FECollection(1, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+
+   // First we need to construct a list of non-essential coarse grid vertices
+
+   // SparseMatrix *Pr = nullptr;
+   //initialize Pr with the Identity
+   Vector ones(fespace->GetTrueVSize());
+   ones = 1.0;
+   SparseMatrix * Pr = new SparseMatrix(ones);
+   // 4. Refine the mesh
+   for (int i = 0; i < ref_levels; i++)
+   {
+      const FiniteElementSpace cfespace(*fespace);
+      mesh->UniformRefinement();
+      // Update fespace
+      fespace->Update();
+      OperatorHandle Tr(Operator::MFEM_SPARSEMAT);
+      fespace->GetTransferOperator(cfespace, Tr);
+      Tr.SetOperatorOwner(false);
+      SparseMatrix *P;
+      Tr.Get(P);
+      if (!Pr)
+      {
+         Pr = P;
+      }
+      else
+      {
+         Pr = Mult(*P, *Pr);
+      }
+   }
+   // if there is no refinement the prolongation is the identity
+   Pr->Threshold(0.0);
+   int nvert = mesh->GetNV();
+   vertex_contr.resize(nvert);
+   for (int iv = 0; iv < nvert; iv++)
+   {
+      int nz = Pr->RowSize(iv);
+      vertex_contr[iv].SetSize(nz);
+      int *col = Pr->GetRowColumns(iv);
+      for (int i = 0; i < nz; i++)
+      {
+         vertex_contr[iv][i] = col[i];
+      }
+   }
+
+   delete Pr;
+
+   Array<int> edge_vertices;
+   int nedge = mesh->GetNEdges();
+   edge_contr.resize(nedge);
+   for (int ie = 0; ie < nedge; ie++)
+   {
+      mesh->GetEdgeVertices(ie, edge_vertices);
+      int nv = edge_vertices.Size(); // always 2 but ok
+      // The edge will contribute to the same patches as its vertices
+      for (int iv = 0; iv < nv; iv++)
+      {
+         int ivert = edge_vertices[iv];
+         edge_contr[ie].Append(vertex_contr[ivert]);
+      }
+      edge_contr[ie].Sort();
+      edge_contr[ie].Unique();
+   }
+
+   Array<int> face_vertices;
+   int nface = mesh->GetNFaces();
+   face_contr.resize(nface);
+   for (int ifc = 0; ifc < nface; ifc++)
+   {
+      mesh->GetFaceVertices(ifc, face_vertices);
+      int nv = face_vertices.Size();
+      // The face will contribute to the same patches as its vertices
+      for (int iv = 0; iv < nv; iv++)
+      {
+         int ivert = face_vertices[iv];
+         face_contr[ifc].Append(vertex_contr[ivert]);
+      }
+      face_contr[ifc].Sort();
+      face_contr[ifc].Unique();
+   }
+
+   Array<int> elem_vertices;
+   int nelem = mesh->GetNE();
+   elem_contr.resize(nelem);
+   for (int iel = 0; iel < nelem; iel++)
+   {
+      mesh->GetElementVertices(iel, elem_vertices);
+      int nv = elem_vertices.Size();
+      // The element will contribute to the same patches as its vertices
+      for (int iv = 0; iv < nv; iv++)
+      {
+         int ivert = elem_vertices[iv];
+         elem_contr[iel].Append(vertex_contr[ivert]);
+      }
+      elem_contr[iel].Sort();
+      elem_contr[iel].Unique();
+   }
+   delete fespace;
+   delete fec;
+}
+// Constructor of patch local problems
+patch_assembly::patch_assembly(Mesh *cmesh_, int ref_levels_, FiniteElementSpace *fespace)
+    : cmesh(*cmesh_), ref_levels(ref_levels_)
+{
+   patch_nod_info *patches = new patch_nod_info(&cmesh, ref_levels);
+
+   nrpatch = patches->nrpatch;
+   Pid.SetSize(nrpatch);
+   patch_dof_map.SetSize(nrpatch);
+   // Build a sparse matrix out of this map to extract the patch submatrix
+   Array<int> dofoffset(nrpatch);
+   dofoffset = 0;
+   int height = fespace->GetVSize();
+   // allocation of sparse matrices.
+   for (int i = 0; i < nrpatch; i++)
+   {
+      Pid[i] = new SparseMatrix(height);
+   }
+   // Now the filling of the matrices with vertex,edge,face,interior dofs
+   Mesh *mesh = fespace->GetMesh();
+   int nrvert = mesh->GetNV();
+   int nredge = mesh->GetNEdges();
+   int nrface = mesh->GetNFaces();
+   int nrelem = mesh->GetNE();
+   // First the vertices
+   for (int i = 0; i < nrvert; i++)
+   {
+      int np = patches->vertex_contr[i].Size();
+      Array<int> vertex_dofs;
+      fespace->GetVertexDofs(i, vertex_dofs);
+      int nv = vertex_dofs.Size();
+
+      for (int j = 0; j < np; j++)
+      {
+         int k = patches->vertex_contr[i][j];
+         for (int l = 0; l < nv; l++)
+         {
+            int m = vertex_dofs[l];
+            Pid[k]->Set(m, dofoffset[k], 1.0);
+            dofoffset[k]++;
+         }
+      }
+   }
+   // Edges
+   for (int i = 0; i < nredge; i++)
+   {
+      int np = patches->edge_contr[i].Size();
+      Array<int> edge_dofs;
+      fespace->GetEdgeInteriorDofs(i, edge_dofs);
+      int ne = edge_dofs.Size();
+      for (int j = 0; j < np; j++)
+      {
+         int k = patches->edge_contr[i][j];
+         for (int l = 0; l < ne; l++)
+         {
+            int m = edge_dofs[l];
+            Pid[k]->Set(m, dofoffset[k], 1.0);
+            dofoffset[k]++;
+         }
+      }
+   }
+   // Faces
+   for (int i = 0; i < nrface; i++)
+   {
+      int np = patches->face_contr[i].Size();
+      Array<int> face_dofs;
+      fespace->GetFaceInteriorDofs(i, face_dofs);
+      int nfc = face_dofs.Size();
+      for (int j = 0; j < np; j++)
+      {
+         int k = patches->face_contr[i][j];
+         for (int l = 0; l < nfc; l++)
+         {
+            int m = face_dofs[l];
+            Pid[k]->Set(m, dofoffset[k], 1.0);
+            dofoffset[k]++;
+         }
+      }
+   }
+
+   // The following can be skipped in case of static condensation
+   // Elements
+   for (int i = 0; i < nrelem; i++)
+   {
+      int np = patches->elem_contr[i].Size();
+      Array<int> elem_dofs;
+      fespace->GetElementInteriorDofs(i, elem_dofs);
+      int nel = elem_dofs.Size();
+      for (int j = 0; j < np; j++)
+      {
+         int k = patches->elem_contr[i][j];
+         for (int l = 0; l < nel; l++)
+         {
+            int m = elem_dofs[l];
+            Pid[k]->Set(m, dofoffset[k], 1.0);
+            dofoffset[k]++;
+         }
+      }
+   }
+
+   for (int i = 0; i < nrpatch; i++)
+   {
+      Pid[i]->SetWidth(dofoffset[i]);
+      Pid[i]->Finalize();
+
+      patch_dof_map[i].SetSize(Pid[i]->Width());
+      // copy from sparse matrix to a simple injection map
+      // use the traspose
+      SparseMatrix * temp = Transpose(*Pid[i]);
+      // Extract row by row of the transpose
+      for (int k =0; k<temp->Height(); ++k)
+      {
+         int * col = temp->GetRowColumns(k);
+         patch_dof_map[i][k] = col[0];
+      }
+      delete temp;
+   }
+   delete patches;
+}
+
+patch_assembly:: ~patch_assembly()
+{
+   for (int i=0; i<nrpatch; i++)
+   {
+      delete Pid[i];
+   }
+   Pid.DeleteAll();
+}
+
+// constructor
+SchwarzSmoother::SchwarzSmoother(Mesh *cmesh_, int ref_levels_, FiniteElementSpace *fespace_, SparseMatrix *A_, Array<int> ess_bdr)
+    : Solver(A_->Height(), A_->Width()), A(A_)
+{
+   P = new patch_assembly(cmesh_, ref_levels_, fespace_);
+
+   ess_bdr = 0;
+   GetNonEssentialPatches(cmesh_, ess_bdr, patch_ids);
+
+   // nrpatch = P->nrpatch;
+   nrpatch = patch_ids.size();
+   A_local.SetSize(nrpatch);
+   invA_local.SetSize(nrpatch);
+
+   for (int i = 0; i < nrpatch; i++)
+   {
+      int k = patch_ids[i];
+      SparseMatrix *Pr = P->Pid[k];
+      // construct the local problems. Factor the patch matrices
+      A_local[i] = RAP(*Pr, *A, *Pr);
+      // if (i == 0) A_local[i]->PrintMatlab(cout);
+      invA_local[i] = new KLUSolver;
+      // invA_local[i] = new UMFPackSolver;
+      // invA_local[i]->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      invA_local[i]->SetOperator(*A_local[i]);
+   }
+}
+
+void SchwarzSmoother::GetNonEssentialPatches(Mesh *cmesh, const Array<int> &ess_bdr, vector<int> &patch_ids)
+{
+   Array<int> ess_vertices;
+   Array<int> bdr_vertices;
+
+   for (int i = 0; i < cmesh->GetNBE(); i++)
+   {
+      int bdr = cmesh->GetBdrAttribute(i);
+      //check if it's essential;
+      if (ess_bdr[bdr - 1] == 1)
+      {
+         cmesh->GetBdrElementVertices(i, bdr_vertices);
+         ess_vertices.Append(bdr_vertices);
+      }
+   }
+   ess_vertices.Sort();
+   ess_vertices.Unique();
+
+   int nrpatch = cmesh->GetNV() - ess_vertices.Size();
+   patch_ids.resize(nrpatch);
+
+   if (ess_vertices.Size() > 0)
+   {
+      int m = 0;
+      int l = 0;
+      for (int i = 0; i < cmesh->GetNV(); i++)
+      {
+         if (m<ess_vertices.Size() && i == ess_vertices[m])
+         {
+            m++;
+         }
+         else
+         {
+            patch_ids[l] = i;
+            l++;
+         }
+      }
+   }
+   else
+   {
+      for (int i = 0; i < cmesh->GetNV(); i++)
+      {
+         patch_ids[i] = i;
+      }
+   }
+}
+
+void SchwarzSmoother::Mult(const Vector &r, Vector &z) const
+{
+   // Apply the smoother patch on the restriction of the residual
+   z = 0.0;
+   Vector rnew(r);
+   Vector znew(z);
+   Vector raux(znew.Size());
+   Vector res_local, sol_local;
+   switch (sType)
+   {
+   case Schwarz::SmootherType::ADDITIVE:
+   {
+      for (int iter = 0; iter < maxit; iter++)
+      {
+         znew = 0.0;
+         for (int i = 0; i < nrpatch; i++)
+         {
+            int k = patch_ids[i];
+            Array<int> * dof_map = &P->patch_dof_map[k];
+            // SparseMatrix *Pr = P->Pid[k];
+            // res_local.SetSize(Pr->NumCols());
+            // sol_local.SetSize(Pr->NumCols());
+            // Pr->MultTranspose(rnew, res_local[i]);
+            int ndofs = dof_map->Size();
+            res_local.SetSize(ndofs);
+            sol_local.SetSize(ndofs);
+            rnew.GetSubVector(*dof_map, res_local);
+
+            invA_local[i]->Mult(res_local, sol_local);
+            znew.AddElementVector(*dof_map,sol_local);
+            // Pr->Mult(sol_local[i], zaux[i]);
+            // znew += zaux[i];
+         }
+         // Relaxation parameter
+         znew *= theta;
+         z += znew;
+
+         //Update residual
+         if (iter + 1 < maxit)
+         {
+            A->Mult(znew, raux);
+            rnew -= raux;
+         }
+      }
+   }
+   break;
+   case Schwarz::SmootherType::MULTIPLICATIVE:
+   {
+      //   TODO
+   }
+   break;
+   case Schwarz::SmootherType::SYM_MULTIPLICATIVE:
+   {
+      //   TODO
+   }
+   break;
+   }
+}
+
+SchwarzSmoother:: ~SchwarzSmoother() 
+{
+   delete P;
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      delete A_local[ip]; 
+      delete invA_local[ip]; 
+   }
+   A_local.DeleteAll();
+   invA_local.DeleteAll();
+}
+
+
+
+BlkSchwarzSmoother::BlkSchwarzSmoother(Mesh *cmesh_, int ref_levels_, FiniteElementSpace* fespace_, SparseMatrix *A_) 
+: Solver(A_->Height(), A_->Width()), A(A_)
+{
+   P = new patch_assembly(cmesh_, ref_levels_, fespace_);
+
+   nrpatch = cmesh_->GetNV();
+	cout << "nrpatch = " << nrpatch << endl;
+   patch_ids.resize(nrpatch);
+   for (int i=0; i<nrpatch; i++) {patch_ids[i]=i;}
+
+   nrpatch = patch_ids.size();
+   A_local.SetSize(nrpatch);
+   invA_local.SetSize(nrpatch);
+
+   for (int i = 0; i < nrpatch; i++)
+   {
+      int k = patch_ids[i];
+      SparseMatrix *Pr  = P->Pid[k];
+      Array<int> offsets_i(3);
+      Array<int> offsets_j(3);
+      offsets_i[0] = 0; 
+      offsets_i[1] = Pr->Height();
+      offsets_i[2] = Pr->Height();
+      offsets_i.PartialSum();
+      offsets_j[0] = 0; 
+      offsets_j[1] = Pr->Width();
+      offsets_j[2] = Pr->Width();
+      offsets_j.PartialSum();
+
+      BlockMatrix * BlockPr = new BlockMatrix(offsets_i,offsets_j);
+      BlockPr->SetBlock(0,0,Pr);
+      BlockPr->SetBlock(1,1,Pr);
+      // Fake blocks
+      SparseMatrix * fakemat = new SparseMatrix(Pr->Height(),Pr->Width()); fakemat->Finalize();
+      BlockPr->SetBlock(0,1,fakemat);
+      BlockPr->SetBlock(1,0,fakemat);
+
+      SparseMatrix *Bpr = BlockPr->CreateMonolithic();
+      A_local[i] = RAP(*Bpr, *A, *Bpr);
+      invA_local[i] = new UMFPackSolver;
+      invA_local[i]->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      invA_local[i]->SetOperator(*A_local[i]);
+   }
+}
+
+
+void BlkSchwarzSmoother::Mult(const Vector &r, Vector &z) const
+{
+   // Apply the smoother patch on the restriction of the residual
+   Array<Vector> res_local(nrpatch);
+   Array<Vector> sol_local(nrpatch);
+   Array<Vector> zaux(nrpatch);
+   z = 0.0;
+   Vector rnew(r);
+   Vector znew(z);
+
+   for (int iter = 0; iter < maxit; iter++)
+   {
+      znew = 0.0;
+      for (int i = 0; i < nrpatch; i++)
+      {
+         int k = patch_ids[i];
+         SparseMatrix *Pr = P->Pid[k];
+         Array<int> offsets_i(3);
+         Array<int> offsets_j(3);
+         offsets_i[0] = 0; 
+         offsets_i[1] = Pr->Height();
+         offsets_i[2] = Pr->Height();
+         offsets_i.PartialSum();
+         offsets_j[0] = 0; 
+         offsets_j[1] = Pr->Width();
+         offsets_j[2] = Pr->Width();
+         offsets_j.PartialSum();
+         BlockMatrix * BlockPr = new BlockMatrix(offsets_i,offsets_j);
+         BlockPr->SetBlock(0,0,Pr);
+         BlockPr->SetBlock(1,1,Pr);
+         SparseMatrix * fakemat = new SparseMatrix(Pr->Height(),Pr->Width()); fakemat->Finalize();
+         BlockPr->SetBlock(0,1,fakemat);
+         BlockPr->SetBlock(1,0,fakemat);
+         SparseMatrix *Bpr = BlockPr->CreateMonolithic();
+         res_local[i].SetSize(Bpr->NumCols());
+         sol_local[i].SetSize(Bpr->NumCols());
+         Bpr->MultTranspose(rnew, res_local[i]);
+
+         invA_local[i]->Mult(res_local[i], sol_local[i]);
+         zaux[i].SetSize(r.Size());
+         zaux[i] = 0.0;
+         Bpr->Mult(sol_local[i], zaux[i]);
+         znew += zaux[i];
+      }
+
+      // Relaxation parameter
+      znew *= theta;
+      z += znew;
+      //Update residual
+      Vector raux(znew.Size());
+      A->Mult(znew, raux);
+      rnew -= raux;
+   }
+}

examples/maxwell-solver/ST/schwarz.hpp

@@ -0,0 +1,89 @@
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+namespace Schwarz
+{
+   enum SmootherType{ADDITIVE, MULTIPLICATIVE, SYM_MULTIPLICATIVE};
+}
+
+struct patch_nod_info 
+{
+   int nrpatch;
+   vector<Array<int>> vertex_contr;
+   vector<Array<int>> edge_contr;
+   vector<Array<int>> face_contr;
+   vector<Array<int>> elem_contr;
+
+   // constructor
+   patch_nod_info(Mesh * mesh_, int ref_levels_);
+private:
+   Mesh *mesh=nullptr;
+   int ref_levels=0;;
+};
+
+struct patch_assembly 
+{
+   int nrpatch;
+   Mesh cmesh;
+   int ref_levels;
+   Array<SparseMatrix *> Pid; 
+   Array<Array<int>> patch_dof_map;
+   // constructor
+   patch_assembly(Mesh * cmesh_, int ref_levels_,FiniteElementSpace *fespace);
+   ~patch_assembly();
+};
+
+class SchwarzSmoother : virtual public Solver {
+private:
+   int nrpatch;
+   /// The linear system matrix
+   SparseMatrix * A;
+   patch_assembly * P;
+   Array<SparseMatrix  *> A_local;
+   // Array<UMFPackSolver *> invA_local;
+   Array<KLUSolver *> invA_local;
+   Array<int>vert_dofs;
+   Schwarz::SmootherType sType=Schwarz::SmootherType::ADDITIVE; 
+   vector<int> patch_ids;
+   int maxit = 1;
+   double theta = 0.5;
+public:
+   SchwarzSmoother(Mesh * cmesh_, int ref_levels_, FiniteElementSpace *fespace,SparseMatrix *A_, Array<int> ess_bdr);
+
+   void SetType(const Schwarz::SmootherType Type) {sType = Type;}
+   void SetNumSmoothSteps(const int iter) {maxit = iter;}
+   void SetDumpingParam(const double dump_param) {theta = dump_param;}
+   virtual void SetOperator(const Operator &op) {}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   void GetNonEssentialPatches(Mesh * cmesh, const Array<int> &ess_bdr, vector <int> & patch_ids);
+   virtual ~SchwarzSmoother();
+};
+
+
+class BlkSchwarzSmoother : public Solver {
+private:
+   int nrpatch;
+   /// The linear system matrix
+   SparseMatrix * A;
+   patch_assembly * P;
+   Array<SparseMatrix  *> A_local;
+   Array<UMFPackSolver *> invA_local;
+   Array<int>vert_dofs;
+   Schwarz::SmootherType sType=Schwarz::SmootherType::ADDITIVE; 
+   vector<int> patch_ids;
+   int maxit = 1;
+   double theta = 0.5;
+public:
+   BlkSchwarzSmoother(Mesh *cmesh_, int ref_levels_, FiniteElementSpace *fespace_, SparseMatrix *A_);
+   void SetType(const Schwarz::SmootherType Type) {sType = Type;}
+   void SetNumSmoothSteps(const int iter) {maxit = iter;}
+   void SetDumpingParam(const double dump_param) {theta = dump_param;}
+   virtual void SetOperator(const Operator &op) {}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   // void GetNonEssentialPatches(Mesh * cmesh, const Array<int> &ess_bdr, vector <int> & patch_ids);
+   virtual ~BlkSchwarzSmoother() {}
+};

examples/maxwell-solver/ToroidST/DofMaps.cpp

@@ -0,0 +1,341 @@
+
+#include "DofMaps.hpp"
+#include "MeshPart.hpp"
+
+
+void E_exact(const Vector &x, Vector &E)
+{
+   double kappa = 1.0;
+   int dim = x.Size();
+   if (dim == 3)
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(2));
+      E(2) = sin(kappa * x(0));
+   }
+   else
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(0));
+      if (x.Size() == 3) { E(2) = 0.0; }
+   }
+}
+
+
+void FindPtsGetCommonElements(Mesh & mesh0, Mesh & mesh1, 
+                              Array<int> & elems0, Array<int> & elems1)
+{
+   int dim = mesh0.Dimension();
+   const int ne0 = mesh0.GetNE();
+   Vector centers(ne0*dim);
+   elems0.SetSize(0);
+   elems1.SetSize(0);
+   for (int i = 0; i < ne0; i++)
+   {
+      Vector center(dim);
+      mesh0.GetElementCenter(i,center);
+      for (int d=0; d<dim; d++)
+      {
+         centers[ne0*d + i] = center[d];
+      }
+   }
+   // Evaluate mesh 1 grid function.
+   FindPointsGSLIB finder;
+   finder.Setup(mesh1);
+   finder.FindPoints(centers);
+   Array<int> elem_map = finder.GetElem();
+   Array<int> code = finder.GetCode();
+   finder.FreeData();
+
+   for (int i = 0; i<code.Size(); i++)
+   {
+      if (!code[i]) 
+      {  // element is found
+         elems0.Append(i);
+         elems1.Append(elem_map[i]);
+      }
+   }
+}                              
+
+// Assuming there are no dublicated indices in the lists
+void GetCommonIndices(const Array<int> & list0, const Array<int> & list1, Array<int> & idx0, Array<int> & idx1)
+{
+   unordered_map<int, int> map0, map1;
+   int i = 0, j = 0;
+   for (auto k : list0) map0[k] = i++;
+   for (auto k : list1) map1[k] = j++;
+   
+   for (auto k : map0)
+   {
+      if (map1.find(k.first) != map1.end())
+      {
+         idx0.Append(k.second);
+         idx1.Append(map1[k.first]);
+      }
+   }
+}
+
+// dofs0 the fes0 indices (Domain)
+// dofs1 the fes1 indices (Range)
+void GetDofMaps(const FiniteElementSpace &fes0, const FiniteElementSpace &fes1, 
+               Array<int> & dofs0, Array<int> & dofs1,
+               const Array<int> * elems0_, const Array<int> * elems1_)
+{
+   Array<int> elems0, elems1;
+   if (!elems0_ || !elems1_)
+   {  // construct the element lists using gslib
+      FindPtsGetCommonElements(*fes0.GetMesh(), *fes1.GetMesh(), elems0, elems1);
+   }
+   else
+   {
+      GetCommonIndices(*elems0_, *elems1_, elems0, elems1);   
+   }
+
+   // construct dof maps fes0->fes1 (possibly not a subspace)
+   int nel = elems0.Size();
+   MFEM_VERIFY(elems1.Size() == nel, "Inconsistent number of elements");
+   Array<int> dof_marker(fes0.GetTrueVSize()); dof_marker = 0;
+   for (int ie = 0; ie<nel; ie++)
+   {
+      int iel0 = elems0[ie];
+      int iel1 = elems1[ie];
+      Array<int> ElemDofs0;
+      Array<int> ElemDofs1;
+      fes0.GetElementDofs(iel0,ElemDofs0);
+      fes1.GetElementDofs(iel1,ElemDofs1);
+      int ndof = ElemDofs0.Size();
+      for (int i = 0; i<ndof; i++)
+      {
+         int dof0_ = ElemDofs0[i];
+         int dof1_ = ElemDofs1[i];
+         int dof0 = (dof0_ >= 0) ? dof0_ : - dof0_ - 1;
+         int dof1 = (dof1_ >= 0) ? dof1_ : - dof1_ - 1;
+         if (dof_marker[dof0]) continue;
+         dofs0.Append(dof0);
+         dofs1.Append(dof1);
+         dof_marker[dof0] = 1;
+      }
+   }
+}
+
+
+void PartitionFE(const FiniteElementSpace * fes, int nrsubmeshes, double ovlp, 
+                 Array<FiniteElementSpace*> & fespaces, 
+                 Array<Array<int> * > & ElemMaps,
+                 Array<Array<int> * > & DofMaps0, Array<Array<int> * > & DofMaps1,
+                 Array<Array<int> * > & OvlpMaps0, Array<Array<int> * > & OvlpMaps1)
+{
+   Mesh * mesh = fes->GetMesh();
+   Array<Mesh *> meshes;
+
+   PartitionMesh(mesh,nrsubmeshes,ovlp,meshes,ElemMaps);
+
+   // DofMaps from subdomains to global mesh
+
+   const FiniteElementCollection * fec = fes->FEColl();
+   Array<int> GlobalElems(mesh->GetNE());
+   for (int i = 0; i<GlobalElems.Size(); i++) GlobalElems[i] = i;
+
+   fespaces.SetSize(nrsubmeshes);
+   DofMaps0.SetSize(nrsubmeshes);
+   DofMaps1.SetSize(nrsubmeshes);
+   for (int i = 0; i<nrsubmeshes; i++)
+   {
+      fespaces[i] = new FiniteElementSpace(meshes[i],fec);
+      cout << " fespace size " << fespaces[i]->GetTrueVSize() << endl;
+      DofMaps0[i] = new Array<int>();
+      DofMaps1[i] = new Array<int>();
+      GetDofMaps(*fespaces[i],*fes,*DofMaps0[i], *DofMaps1[i], ElemMaps[i], &GlobalElems);
+   }
+   int nroverlaps = nrsubmeshes-1;
+   OvlpMaps0.SetSize(nroverlaps);
+   OvlpMaps1.SetSize(nroverlaps);
+   for (int i = 0; i<nroverlaps; i++)
+   {
+      OvlpMaps0[i] = new Array<int>();
+      OvlpMaps1[i] = new Array<int>();
+      GetDofMaps(*fespaces[i],*fespaces[i+1],*OvlpMaps0[i], *OvlpMaps1[i], 
+                  ElemMaps[i], ElemMaps[i+1]);
+   }
+}
+
+void GetElements(Mesh &mesh, double ovlp, int direction, Array<int> & elems)
+{
+   double amin, amax;
+   GetMeshAngleRange(&mesh, amin, amax);
+   int dim = mesh.Dimension();
+   // loop through elements
+   int ne = mesh.GetNE();
+   for (int i=0; i<ne; i++)
+   {
+      Vector center(dim);
+      mesh.GetElementCenter(i,center);
+      double thetad = GetPointAngle(center);
+
+      switch (direction)
+      {
+      case -1:
+         if (thetad >= amin + ovlp)  
+         {
+            elems.Append(i); 
+         }
+         break;
+      case  1: 
+         if (thetad <= amax - ovlp)
+         {
+
+            elems.Append(i);
+         }   
+         break;
+      default:
+         if (thetad >= amin + ovlp && thetad <= amax - ovlp) 
+         {
+            elems.Append(i); 
+         }
+         break;
+      }
+   }
+}
+
+void GetRestrictionDofs(FiniteElementSpace &fes, int direction, double ovlp, Array<int> & rdofs)
+{
+   Array<int> elems;
+   GetElements(*fes.GetMesh(),ovlp,direction,elems);
+   int ne = elems.Size();
+   int tsize = fes.GetTrueVSize();
+   Array<int> tdof_marker(tsize); tdof_marker = 0;
+   for (int i=0; i<ne; i++)
+   {
+      int ie = elems[i];
+      Array<int> elem_dofs;
+      fes.GetElementDofs(ie,elem_dofs);
+      for (auto x : elem_dofs)
+      {
+         int tdof = (x>=0) ? x : -1 - x;
+         tdof_marker[tdof] = 1;
+      }
+   }
+   int n = tdof_marker.Sum();
+   rdofs.SetSize(n);
+   int k=0;
+   for (int i=0; i<tsize; i++)
+   {
+      if (tdof_marker[i])
+      {
+         rdofs[k++] = i;
+      }
+   }
+}
+
+void RestrictDofs(const Array<int> & rdofs, int tsize, Vector & x)
+{
+   int n = rdofs.Size();
+   Array<int> dofs(2*n);
+   for (int i =0; i<n; i++)
+   {
+      dofs[i] = rdofs[i];
+      dofs[n+i] = rdofs[i]+tsize;
+   }
+   x.SetSubVectorComplement(dofs,0.0);
+}
+
+
+void MapDofs(const Array<int> & dmap0, const Array<int> & dmap1,
+             const Vector &gf0, Vector &gf1)
+{
+   int tsize0 = gf0.Size()/2;
+   int tsize1 = gf1.Size()/2;
+   for (int i = 0; i< dmap0.Size(); i++)
+   {
+      int j = dmap0[i];   
+      int k = dmap1[i];   
+      gf1[k] = gf0[j];
+      gf1[k+tsize1] = gf0[j+tsize0];
+   }
+}
+
+void AddMapDofs(const Array<int> & dmap0, const Array<int> & dmap1,
+             const Vector &gf0, Vector &gf1)
+{
+   int tsize0 = gf0.Size()/2;
+   int tsize1 = gf1.Size()/2;
+   for (int i = 0; i< dmap0.Size(); i++)
+   {
+      int j = dmap0[i];   
+      int k = dmap1[i];   
+      gf1[k] += gf0[j];
+      gf1[k+tsize1] += gf0[j+tsize0];
+   }
+}
+
+void DofMapTests(FiniteElementSpace &fes0, FiniteElementSpace &fes1,
+                 const Array<int> & dmap0, const Array<int> & dmap1)
+{
+
+   Mesh * mesh0=fes0.GetMesh();
+   Mesh * mesh1=fes1.GetMesh();
+   ComplexGridFunction gf0(&fes0);
+   ComplexGridFunction gf1(&fes1); gf1 = 0.0;
+   int dim = mesh0->Dimension();
+   // Vector vone(dim); vone = 1.0;
+   // VectorConstantCoefficient one(vone);
+   VectorFunctionCoefficient cf(dim,E_exact);
+   gf0.ProjectCoefficient(cf,cf);
+
+   MapDofs(dmap0,dmap1,gf0,gf1);
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+
+   // socketstream sol_sock0(vishost, visport);
+   // sol_sock0.precision(8);
+   // sol_sock0 << "solution\n" << *mesh0 << gf0.real() 
+   //          //  << "valuerange -1 1 \n"
+   //           << "window_title ' gf_0 ' " << flush;                     
+
+   // socketstream sol_sock1(vishost, visport);
+   // sol_sock1.precision(8);
+   // sol_sock1 << "solution\n" << *mesh1 << gf1.real() 
+   //          //  << "valuerange -1 1 \n"
+   //           << "window_title ' gf_1 ' " << flush;  
+   int n = 2;
+   {
+      socketstream solsock(vishost, visport);
+      solsock.precision(8);
+      solsock << "parallel " << n << " " << 0 << "\n";
+      solsock << "solution\n" << *mesh0 << gf0.real() << flush;     
+   }
+   {
+      socketstream solsock(vishost, visport);
+      solsock.precision(8);
+      solsock << "parallel " << n << " " << 1 << "\n";
+      solsock << "solution\n" << *mesh1 << gf1.real()  << flush;     
+   }
+}
+
+
+void DofMapOvlpTest(FiniteElementSpace &fes, const Array<int> & dmap)
+{
+   Mesh * mesh = fes.GetMesh();
+   int dim = mesh->Dimension();
+   int tsize = fes.GetTrueVSize();
+   ComplexGridFunction gf(&fes);
+   VectorFunctionCoefficient cf(dim,E_exact);
+   gf.ProjectCoefficient(cf,cf);
+   
+   RestrictDofs(dmap,tsize,gf);
+
+   string keys = "keys mac\n" ;
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+
+   {
+      socketstream solsock_re(vishost, visport);
+      solsock_re.precision(8);
+      solsock_re << "solution\n" << *mesh << gf.real() << keys << flush;     
+      socketstream solsock_im(vishost, visport);
+      solsock_im.precision(8);
+      solsock_im << "solution\n" << *mesh << gf.imag() << keys << flush; 
+   }
+}           

examples/maxwell-solver/ToroidST/DofMaps.hpp

@@ -0,0 +1,57 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void E_exact(const Vector &x, Vector &E);
+
+
+void FindPtsGetCommonElements(Mesh & mesh0, Mesh & mesh1, 
+                              Array<int> & elems0, Array<int> & elems1);
+
+void GetCommonIndices(const Array<int> & list0, const Array<int> & list1, Array<int> & idx0, Array<int> & idx1);
+
+// Given two FiniteElementSpaces and an ElementMap compute
+// the dof map between fes0 and fes1
+void GetDofMaps(const FiniteElementSpace &fes0, const FiniteElementSpace &fes1, 
+                Array<int> & dofs0, Array<int> & dofs1,  
+                const Array<int> * elems0_ = nullptr, const Array<int> * elems1_ = nullptr);
+
+// Partition the given mesh to nrsubmeshes with overlap given by ovlp
+// ElemMaps: For each subdomain the element indices of the global mesh
+// Dofmap0[i]: dof indices of fes in the shared region with subdomain i
+// Dofmap1[i]: dof indices of fespaces[i] in the shared region with fes
+// OvlpMaps0 : dof indices of fespaces[i] in overlapping region i
+// OvlpMaps1 : dof indices of fespaces[i+1] in overlapping region i
+void PartitionFE(const FiniteElementSpace * fes, int nrsubmeshes, double ovlp, 
+                 Array<FiniteElementSpace*> & fespaces, 
+                 Array<Array<int> * > & ElemMaps,
+                 Array<Array<int> * > & DofMaps0,
+                 Array<Array<int> * > & DofMaps1,
+                 Array<Array<int> * > & OvlpMaps0, 
+                 Array<Array<int> * > & OvlpMaps1);
+
+
+void GetRestrictionDofs(FiniteElementSpace &fes, int direction, double ovlp, Array<int> & rdofs);
+void RestrictDofs(const Array<int> & rdofs, int tsize, Vector & x);
+
+// direction:  1 left (anti-clockwise)
+//            -1 right (clockwise)
+//             0 both the above
+// ovlp     :  given in degrees         
+void GetElements(Mesh &mesh, double ovlp, int direction, Array<int> & elems);
+
+void MapDofs(const Array<int> & dmap0, const Array<int> & dmap1,
+             const Vector &gf0, Vector &gf1);
+
+void AddMapDofs(const Array<int> & dmap0, const Array<int> & dmap1,
+                const Vector &gf0, Vector &gf1);
+
+void DofMapTests(FiniteElementSpace &fes0, FiniteElementSpace &fes1,
+                 const Array<int> & dmap0, const Array<int> & dmap1);                 
+
+void DofMapOvlpTest(FiniteElementSpace &fes, const Array<int> & dmap);
+                       

examples/maxwell-solver/ToroidST/MeshPart.cpp

@@ -0,0 +1,216 @@
+
+#include "MeshPart.hpp"
+
+double GetPointAngle(const Vector & pt)
+{
+   double x = pt(0);
+   double y = pt(1);
+   x = (abs(x)<1e-12) ? 0.0 : x;
+   y = (abs(y)<1e-12) ? 0.0 : y;
+   double theta = (x == 0) ? M_PI/2.0 : atan(y/x);
+   int k = (x<=0.0) ? 1 : ((y<0.0) ? 2 : 0.0);
+   theta += k*M_PI;
+   return theta * 180.0/M_PI;
+}
+
+void GetMeshAngleRange(Mesh * mesh, double & amin, double & amax)
+{
+   amin = infinity();
+   amax = -infinity();
+   int nbe = mesh->GetNBE();
+   int dim = mesh->Dimension();
+
+   for (int i = 0; i < nbe; ++i)
+   {
+      Vector center(dim);
+      int geom = mesh->GetBdrElementBaseGeometry(i);
+      ElementTransformation * T = mesh->GetBdrElementTransformation(i);
+      T->Transform(Geometries.GetCenter(geom),center);
+      double thetad = GetPointAngle(center);
+      amin = min(amin,thetad);
+      amax = max(amax,thetad);
+   }
+}
+
+int get_angle_range(double angle, Array<double> angles)
+{
+   auto it = std::upper_bound(angles.begin(), angles.end(), angle);
+   return std::distance(angles.begin(),it)-1;
+}
+
+void SetMeshAttributes(Mesh * mesh, int subdivisions, double ovlp)
+{
+   Array<double> angles(2*subdivisions);
+
+   double amin, amax; 
+   GetMeshAngleRange(mesh,amin,amax);
+   angles[0] = amin;
+
+   double length = (amax-amin)/subdivisions;
+   double range;
+   for (int i = 1; i<subdivisions; i++)
+   {
+      range = i*length;
+      angles[2*i-1] = range-ovlp;
+      angles[2*i] = range+ovlp;
+   }
+   angles[2* subdivisions-1] = amax;
+
+   int ne = mesh->GetNE();
+   int dim = mesh->Dimension();
+   // set element attributes
+   for (int i = 0; i < ne; ++i)
+   {
+      Element *el = mesh->GetElement(i);
+      // roughly the element center
+      Vector center(dim);
+      mesh->GetElementCenter(i,center);
+      double thetad = GetPointAngle(center);
+      // Find the angle relative to (0,0,z)
+      int attr = get_angle_range(thetad, angles) + 1;
+      el->SetAttribute(attr);
+   }
+   mesh->SetAttributes();
+   cout << "Max attributes " << mesh->attributes.Max() << endl;
+   cout << "angles = " ; angles.Print(cout, 2*subdivisions);
+   if (!angles.IsSorted()) 
+      MFEM_WARNING("Check mesh partitioning angles ");
+}
+
+// remove/leave elements with attributes given by attr
+Mesh * GetPartMesh(const Mesh * mesh0, const Array<int> & attr_, Array<int> & elem_map,
+ bool complement)
+{
+   Array<int> bdr_attr;
+   int max_attr     = mesh0->attributes.Max();
+   int min_attr     = mesh0->attributes.Min();
+
+   Array<int> attr;
+   
+   Array<int> all_attr(max_attr); all_attr = 0;
+   for (int i = 0; i<attr_.Size(); i++)
+   {
+      all_attr[attr_[i]-1] = 1;
+   }
+   for (int i = min_attr; i<=max_attr; i++)
+   {
+      
+      if (complement && all_attr[i-1]==0) attr.Append(i);
+      if (!complement && all_attr[i-1]==1) attr.Append(i);
+   }
+
+
+   int max_bdr_attr = mesh0->bdr_attributes.Max();
+
+   bdr_attr.SetSize(attr.Size());
+   for (int i=0; i<attr.Size(); i++)
+   {
+      bdr_attr[i] = max_bdr_attr + attr[i];
+   }
+
+   Array<int> marker(max_attr);
+   Array<int> attr_inv(max_attr);
+   marker = 0;
+   attr_inv = 0;
+   for (int i=0; i<attr.Size(); i++)
+   {
+      marker[attr[i]-1] = 1;
+      attr_inv[attr[i]-1] = i;
+   }
+
+   // Count the number of elements in the final mesh
+   int num_elements = 0;
+   for (int e=0; e<mesh0->GetNE(); e++)
+   {
+      int elem_attr = mesh0->GetElement(e)->GetAttribute();
+      if (!marker[elem_attr-1]) { num_elements++; }
+   }
+
+   Mesh * mesh = new Mesh(mesh0->Dimension(), mesh0->GetNV(), num_elements);
+   // Copy vertices
+   for (int v=0; v<mesh0->GetNV(); v++)
+   {
+      mesh->AddVertex(mesh0->GetVertex(v));
+   }
+
+   // Copy elements
+   elem_map.SetSize(num_elements);
+   int k = 0;
+   for (int e=0; e<mesh0->GetNE(); e++)
+   {
+      const Element * el = mesh0->GetElement(e);
+      
+      int elem_attr = el->GetAttribute();
+      if (!marker[elem_attr-1])
+      {
+         Element * nel = mesh->NewElement(el->GetGeometryType());
+         nel->SetAttribute(elem_attr);
+         nel->SetVertices(el->GetVertices());
+         mesh->AddElement(nel);
+         elem_map[k++] = e;
+      }
+   }
+
+   mesh->FinalizeTopology();
+   mesh->RemoveUnusedVertices();
+
+   const GridFunction * nodes0 = mesh0->GetNodes();
+
+   int order = nodes0->FESpace()->GetOrder(0);
+   if (order > 1)
+   {
+      mesh->SetCurvature(order, false, 3, Ordering::byVDIM);
+   }
+
+   GridFunction * nodes = mesh->GetNodes();
+   int nel = mesh0->GetNE();
+   // copy nodes
+   int jel = 0;
+   for (int iel = 0; iel< nel; iel++)
+   {
+      int elem_attr = mesh0->GetElement(iel)->GetAttribute();
+      if (!marker[elem_attr-1]) 
+      {
+         Array<int> vdofs0,vdofs;
+         nodes0->FESpace()->GetElementVDofs(iel,vdofs0);
+         Vector x;
+         nodes0->GetSubVector(vdofs0,x);
+         nodes->FESpace()->GetElementVDofs(jel++,vdofs);
+         nodes->SetSubVector(vdofs,x);
+      }
+   }
+   return mesh;
+}
+
+// Partition mesh to nrsubmeshes (equally spaced in the azimuthal direction)
+void PartitionMesh(Mesh * mesh, int nrsubmeshes, double ovlp, 
+                   Array<Mesh*> & SubMeshes, Array<Array<int> *> & elems)
+{
+   cout << "Partitioning the global Mesh" << endl;
+
+   SetMeshAttributes(mesh,nrsubmeshes,ovlp);
+   int maxattr = mesh->attributes.Max();
+   // Produce the subdomains
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   SubMeshes.SetSize(nrsubmeshes);
+   elems.SetSize(nrsubmeshes);
+   for (int i = 0; i<nrsubmeshes; i++)
+   {
+      cout << "mesh " << i << endl;
+      Array<int> attr;
+      for (int j = 0; j<3; j++)
+      {
+         if (2*i+j >0 && 2*i+j <= maxattr) attr.Append(2*i+j);
+      }
+      Array<int> elem_map;
+      // attr.Print();
+      elems[i] = new Array<int>(0);
+      SubMeshes[i] = GetPartMesh(mesh,attr,*elems[i],true);
+      // socketstream mesh_sock(vishost, visport);
+      // mesh_sock << "parallel " << nrsubmeshes <<  " " << i << "\n";
+      // mesh_sock.precision(8);
+      // mesh_sock << "mesh\n" << *SubMeshes[i] << flush;
+      // cout << "nrelemes = " << mesh1->GetNE() << endl;
+   }
+}

examples/maxwell-solver/ToroidST/MeshPart.hpp

@@ -0,0 +1,25 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+double GetPointAngle(const Vector & pt);
+
+void GetMeshAngleRange(Mesh * mesh, double & amin, double & amax);
+
+int get_angle_range(double angle, Array<double> angles);
+
+void SetMeshAttributes(Mesh * mesh, int subdivisions, double ovlp);
+
+// Partition mesh according to Attributes
+// @input: mesh0 : the mesh to get trimmed
+//         attr  : Attributes to remove or leave depending on the complement flag
+Mesh * GetPartMesh(const Mesh * mesh0, const Array<int> & attr, 
+                    Array<int> & elem_map, bool complement = false);
+
+// Partition mesh to nrsubmeshes (equally spaced in the azimuthal direction)
+void PartitionMesh(Mesh * mesh, int nrsubmeshes, double ovlp, 
+                   Array<Mesh*> & SubMeshes, Array<Array<int> *> & elems);                    

examples/maxwell-solver/ToroidST/ST_bend.cpp

@@ -0,0 +1,434 @@
+
+
+// sample runs: ./ST_bend -ref 2 -o 2 -f 0.6
+//              ./ST_bend -ref 3 -o 2 -f 1.2  (6 iterations)
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "ToroidST.hpp"
+
+using namespace std;
+using namespace mfem;
+
+void maxwell_solution(const Vector &x, vector<complex<double>> &E);
+void maxwell_curl(const Vector &x, vector<complex<double>> &curlE);
+
+void E_bdr_data_Re(const Vector &x, Vector &E);
+void E_bdr_data_Im(const Vector &x, Vector &E);
+
+
+Array2D<double> comp_domain_bdr;
+Array2D<double> domain_bdr;
+double mu = 1.0;
+double epsilon = 1.0;
+double omega;
+int dim;
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_SELF, &num_procs);
+   MPI_Comm_rank(MPI_COMM_SELF, &myid);
+   // 1. Parse command-line options.
+   const char *mesh_file = "meshes/toroid3_4_2.mesh";
+
+   int order = 1;
+   int ref_levels = 1;
+   double freq = 0.6;
+   bool herm_conv = true;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-ref", "--refinements",
+                  "Number of refinements");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&freq, "-f", "--frequency",
+                  "Frequency (in Hz).");
+   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
+                  "--no-hermitian", "Use convention for Hermitian operators.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+
+   // 2. Setup the mesh
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+
+   Mesh * mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+   mesh->RemoveInternalBoundaries();
+
+   cout << "Initial number of elements = " << mesh->GetNE() << endl;
+
+
+   for (int iter = 0; iter<ref_levels; iter++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // Angular frequency
+   omega = 2.0 * M_PI * freq;
+
+   ToroidPML tpml(mesh);
+   Vector zlim, rlim, alim;
+   tpml.GetDomainBdrs(zlim,rlim,alim);
+   Vector zpml_thickness(2); zpml_thickness = 0.0;
+   Vector rpml_thickness(2); rpml_thickness = 0.0;
+   Vector apml_thickness(2); apml_thickness = 0.0; 
+   bool zstretch = false;
+   bool astretch = false;
+   bool rstretch = false;
+   apml_thickness[1] = 20.0;
+   astretch = true;
+   tpml.SetPmlAxes(zstretch,rstretch,astretch);
+   tpml.SetPmlWidth(zpml_thickness,rpml_thickness,apml_thickness);
+   tpml.SetOmega(omega); 
+
+
+
+   ComplexOperator::Convention conv =
+      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
+
+
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+
+   int size = fespace->GetTrueVSize();
+   cout << "Number of finite element unknowns: " << size << endl;
+   tpml.SetAttributes(mesh); 
+
+
+   ComplexGridFunction x(fespace);
+   x = 0.0;
+   VectorFunctionCoefficient E_Re(dim, E_bdr_data_Re);
+   VectorFunctionCoefficient E_Im(dim, E_bdr_data_Im);
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (mesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   ComplexLinearForm b(fespace, conv);
+   b.Vector::operator=(0.0);
+   b.Assemble();
+
+   x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);
+
+   Array<int> attr;
+   Array<int> attrPML;
+   if (mesh->attributes.Size())
+   {
+      attr.SetSize(mesh->attributes.Max());
+      attrPML.SetSize(mesh->attributes.Max());
+      attr = 0;   attr[0] = 1;
+      attrPML = 0;
+      if (mesh->attributes.Max() > 1)
+      {
+         attrPML[1] = 1;
+      }
+   }
+
+   ConstantCoefficient muinv(1.0/mu);
+   ConstantCoefficient omeg(-pow(omega, 2) * epsilon);
+   RestrictedCoefficient restr_muinv(muinv,attr);
+   RestrictedCoefficient restr_omeg(omeg,attr);
+
+   // Integrators inside the computational domain (excluding the PML region)
+   SesquilinearForm a(fespace, conv);
+   a.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv),NULL);
+   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);
+
+   int cdim = (dim == 2) ? 1 : dim;
+
+   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, &tpml);
+   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, &tpml);
+   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+
+   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,&tpml);
+   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,&tpml);
+   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+
+   // Integrators inside the PML region
+   a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
+                        new CurlCurlIntegrator(restr_c1_Im));
+   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
+                        new VectorFEMassIntegrator(restr_c2_Im));
+
+   a.Assemble(0);
+
+   OperatorPtr A;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   Vector Y(X);
+
+
+
+
+   // SparseMatrix * SpMat = (*A.As<ComplexSparseMatrix>()).GetSystemMatrix();
+   // // SpMat->Threshold(0.0);
+   // // SpMat->PrintMatlab(cout);
+   // // cin.get();
+   // HYPRE_Int global_size = SpMat->Height();
+   // HYPRE_Int row_starts[2]; row_starts[0] = 0; row_starts[1] = global_size;
+   // HypreParMatrix * HypreMat = new HypreParMatrix(MPI_COMM_SELF,global_size,row_starts,SpMat);
+   // {
+   //    MUMPSSolver mumps;
+   //    mumps.SetOperator(*HypreMat);
+   //    mumps.Mult(B,X);
+   // }
+   // cout << "X norm = " << X.Norml2() << endl;
+
+
+   // double overlap = 7; // in degrees;
+   // double ovlerlap = 0.5; // in degrees;
+   // int nrmeshes = 9;
+
+   // Array<Array<int> *> ElemMaps, DofMaps0, DofMaps1, OvlpMaps0, OvlpMaps1;
+   // Array<FiniteElementSpace *> fespaces;
+   // PartitionFE(fespace,nrmeshes,overlap,fespaces, 
+   //             ElemMaps,
+   //             DofMaps0, DofMaps1,
+   //             OvlpMaps0, OvlpMaps1);
+
+     // Test local to global dof Maps
+   // for (int i = 0; i<nrmeshes; i++)
+   // {
+   //    DofMapTests(*fespaces[i],*fespace,*DofMaps0[i], *DofMaps1[i]);
+   //    // DofMapTests(*fespace,*fespaces[i], *DofMaps1[i], *DofMaps0[i]);
+   //    cin.get();
+   // }
+
+   // for (int i = 0; i<nrmeshes-1; i++)
+   // {
+   //    // DofMapTests(*fespaces[i],*fespaces[i+1],*OvlpMaps0[i], *OvlpMaps1[i]);
+   //    // DofMapTests(*fespaces[i+1],*fespaces[i],*OvlpMaps1[i], *OvlpMaps0[i]);
+   //    Array<int> rdofs;
+   //    RestrictDofs(*fespaces[i],0,overlap,rdofs);
+   //    DofMapOvlpTest(*fespaces[i],rdofs);
+   //    cin.get();
+   // }
+   // a.RecoverFEMSolution(X, b, x);
+
+
+   int nrsubdomains = 5;
+   ToroidST * STSolver = new ToroidST(&a,apml_thickness,omega,nrsubdomains);
+   STSolver->Mult(B,Y);
+
+   GMRESSolver gmres;
+	// gmres.iterative_mode = true;
+   gmres.SetPreconditioner(*STSolver);
+	gmres.SetOperator(*A);
+	gmres.SetRelTol(1e-8);
+	gmres.SetMaxIter(100);
+	gmres.SetPrintLevel(1);
+	gmres.Mult(B, Y);
+   delete STSolver;
+
+   cout << "Y norm = " << Y.Norml2() << endl;
+
+
+   // cin.get();
+
+   a.RecoverFEMSolution(Y, b, x);
+   // a.RecoverFEMSolution(X, b, x);
+
+
+
+
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      // Define visualization keys for GLVis (see GLVis documentation)
+      string keys;
+      keys = (dim == 3) ? "keys macF\n" : keys = "keys amrRljcUUuu\n";
+
+      char vishost[] = "localhost";
+      int visport = 19916;
+
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re.precision(8);
+      sol_sock_re << "solution\n"
+                  << *mesh << x.real() << keys
+                  << "window_title 'Solution real part'" << flush;
+
+      socketstream sol_sock_im(vishost, visport);
+      sol_sock_im.precision(8);
+      sol_sock_im << "solution\n"
+                  << *mesh << x.imag() << keys
+                  << "window_title 'Solution imag part'" << flush;
+
+      GridFunction x_t(fespace);
+      x_t = x.real();
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n"
+               << *mesh << x_t << keys << "autoscale off\n"
+               << "window_title 'Harmonic Solution (t = 0.0 T)'"
+               << "pause\n" << flush;
+      cout << "GLVis visualization paused."
+           << " Press space (in the GLVis window) to resume it.\n";
+      int num_frames = 16;
+      int i = 0;
+      while (sol_sock)
+      {
+         double t = (double)(i % num_frames) / num_frames;
+         ostringstream oss;
+         oss << "Harmonic Solution (t = " << t << " T)";
+
+         add(cos(2.0 * M_PI * t), x.real(),
+             sin(2.0 * M_PI * t), x.imag(), x_t);
+         sol_sock << "solution\n"
+                  << *mesh << x_t
+                  << "window_title '" << oss.str() << "'" << flush;
+         i++;
+      }
+   }
+
+   // 17. Free the used memory.
+   // delete pml;
+   delete fespace;
+   delete fec;
+   delete mesh;
+   MPI_Finalize();
+   return 0;
+}
+
+
+void E_bdr_data_Re(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   if (abs(x(1))<1e-12 && x(0)>0) 
+   {
+      vector<complex<double>> Eval(E.Size());
+      maxwell_solution(x, Eval);
+      for (int i = 0; i < dim; ++i)
+      {
+         E[i] = Eval[i].real();
+      }
+   }
+}
+
+// Define bdr_data solution
+void E_bdr_data_Im(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   if (abs(x(1))<1e-12 && x(0)>0) 
+   {
+      vector<complex<double>> Eval(E.Size());
+      maxwell_solution(x, Eval);
+      for (int i = 0; i < dim; ++i)
+      {
+         E[i] = Eval[i].imag();
+      }
+   }
+}
+
+void E_exact_Re(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   vector<complex<double>> Eval(E.Size());
+   maxwell_solution(x, Eval);
+   for (int i = 0; i < dim; ++i)
+   {
+      E[i] = Eval[i].real();
+   }
+}
+
+void E_exact_Im(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   vector<complex<double>> Eval(E.Size());
+   maxwell_solution(x, Eval);
+   for (int i = 0; i < dim; ++i)
+   {
+      E[i] = Eval[i].imag();
+   }
+}
+
+void maxwell_solution(const Vector &x, vector<complex<double>> &E)
+{
+   complex<double> zi = complex<double>(0., 1.);
+
+   double k = omega * sqrt(epsilon * mu);
+   // T_10 mode
+   double k10 = sqrt(k * k - M_PI * M_PI);
+   E[2] = -zi * k / M_PI * sin(M_PI*(x(0)))*exp(zi * k10 * x(1)); 
+}
+
+
+
+
+   // double ovlerlap = 7.5; // in degrees;
+   // // double ovlerlap = 0.5; // in degrees;
+   // int nrmeshes = 9;
+
+   // Array<Array<int> *> ElemMaps, DofMaps0, DofMaps1, OvlpMaps0, OvlpMaps1;
+   // Array<FiniteElementSpace *> fespaces;
+   // PartitionFE(fespace,nrmeshes,ovlerlap,fespaces, 
+   //             ElemMaps,
+   //             DofMaps0, DofMaps1,
+   //             OvlpMaps0, OvlpMaps1);
+
+
+   // Test local to global dof Maps
+   // for (int i = 0; i<nrmeshes; i++)
+   // {
+   //    DofMapTests(*fespaces[i],*fespace,*DofMaps0[i], *DofMaps1[i]);
+   //    // DofMapTests(*fespace,*fespaces[i], *DofMaps1[i], *DofMaps0[i]);
+   //    cin.get();
+   // }
+
+   // for (int i = 0; i<nrmeshes-1; i++)
+   // {
+   //    // DofMapTests(*fespaces[i],*fespaces[i+1],*OvlpMaps0[i], *OvlpMaps1[i]);
+   //    DofMapTests(*fespaces[i+1],*fespaces[i],*OvlpMaps1[i], *OvlpMaps0[i]);
+   //    cin.get();
+   // }
+   
+
+   // if (visualization)
+   // {
+   //    // GLVis server to visualize to
+   //    char vishost[] = "localhost";
+   //    int  visport   = 19916;
+
+   //    socketstream mesh0_sock(vishost, visport);
+   //    mesh0_sock.precision(8);
+   //    mesh0_sock << "mesh\n" << *mesh << flush;
+
+   //    socketstream mesh1_sock(vishost, visport);
+   //    mesh1_sock.precision(8);
+   //    mesh1_sock << "mesh\n" << *mesh1 << flush;
+
+   //    socketstream mesh2_sock(vishost, visport);
+   //    mesh2_sock.precision(8);
+   //    mesh2_sock << "mesh\n" << *mesh2 << flush;
+   // }
+
+
+   // mesh = mesh1;
+   // return 0;

examples/maxwell-solver/ToroidST/ToroidST.cpp

@@ -0,0 +1,303 @@
+
+#include "ToroidST.hpp"
+
+
+void ToroidST::SetupSubdomainProblems()
+{
+   // Sesquilinear forms and Operator
+   sqf.SetSize(nrsubdomains);
+   Optr.SetSize(nrsubdomains);
+   // Subdomain Matrix and its LU factorization
+   PmlMat.SetSize(nrsubdomains);
+   PmlMatInv.SetSize(nrsubdomains);
+   // Right hand sides
+   f_orig.SetSize(nrsubdomains);
+   forward_transf.SetSize(nrsubdomains);
+   backward_transf.SetSize(nrsubdomains);
+
+   for (int ip=0; ip<nrsubdomains; ip++)
+   {
+      cout << "Ip = " << ip << endl;
+      SetMaxwellPmlSystemMatrix(ip);
+      PmlMat[ip] = Optr[ip]->As<ComplexSparseMatrix>();
+      // PmlMat[ip]->PrintMatlab(cout);
+      PmlMatInv[ip] = new ComplexUMFPackSolver;
+      PmlMatInv[ip]->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      cout << "ComplexUMFPack: size = " << PmlMat[ip]->Height() << endl;
+      PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
+      int ndofs = fespaces[ip]->GetTrueVSize();
+      f_orig[ip] = new Vector(2*ndofs);
+      forward_transf[ip] = new Vector(2*ndofs);
+      backward_transf[ip] = new Vector(2*ndofs);
+   }
+}
+
+void ToroidST::SetMaxwellPmlSystemMatrix(int ip)
+{
+   Mesh * mesh = fespaces[ip]->GetMesh();
+   // Mesh * mesh = fes->GetMesh();
+   MFEM_VERIFY(mesh, "Null mesh pointer");
+   int dim = mesh->Dimension();
+   ToroidPML tpml(mesh);
+   Vector zlim, rlim, alim;
+   tpml.GetDomainBdrs(zlim,rlim,alim);
+   Vector zpml(2); zpml = 0.0;
+   Vector rpml(2); rpml = 0.0;
+   Vector apml(2); apml = 0.0; 
+   bool zstretch = false;
+   bool astretch = true;
+   bool rstretch = false;
+   apml = aPmlThickness[1]; // just for this test (toroid waveguide)
+   if (ip == 0) 
+   {
+      apml[0] = aPmlThickness[0];
+   }
+   if (ip == nrsubdomains-1)
+   {
+      apml[1] = aPmlThickness[1];
+   }
+   tpml.SetPmlAxes(zstretch,rstretch,astretch);
+   tpml.SetPmlWidth(zpml,rpml,apml);
+   tpml.SetOmega(omega); 
+
+
+
+   ComplexOperator::Convention conv = bf->GetConvention();
+   tpml.SetAttributes(mesh);
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (mesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+   fespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   // fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   Array<int> attr;
+   Array<int> attrPML;
+   if (mesh->attributes.Size())
+   {
+      attr.SetSize(mesh->attributes.Max());
+      attrPML.SetSize(mesh->attributes.Max());
+      attr = 0;   attr[0] = 1;
+      attrPML = 0;
+      if (mesh->attributes.Max() > 1)
+      {
+         attrPML[1] = 1;
+      }
+   }
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient omeg(-pow(omega, 2));
+   RestrictedCoefficient restr_one(one,attr);
+   RestrictedCoefficient restr_omeg(omeg,attr);
+
+     // Integrators inside the computational domain (excluding the PML region)
+   sqf[ip] = new SesquilinearForm(fespaces[ip], conv);
+   // sqf[ip] = new SesquilinearForm(fes, conv);
+   sqf[ip]->AddDomainIntegrator(new CurlCurlIntegrator(restr_one),NULL);
+   sqf[ip]->AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);
+
+   PMLMatrixCoefficient pml_c1_Re(dim,detJ_inv_JT_J_Re, &tpml);
+   PMLMatrixCoefficient pml_c1_Im(dim,detJ_inv_JT_J_Im, &tpml);
+   ScalarMatrixProductCoefficient c1_Re(one,pml_c1_Re);
+   ScalarMatrixProductCoefficient c1_Im(one,pml_c1_Im);
+   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+
+   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,&tpml);
+   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,&tpml);
+   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+
+   // Integrators inside the PML region
+   sqf[ip]->AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
+                        new CurlCurlIntegrator(restr_c1_Im));
+   sqf[ip]->AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
+                        new VectorFEMassIntegrator(restr_c2_Im));
+   sqf[ip]->Assemble(0);
+
+   Optr[ip] = new OperatorPtr;
+   sqf[ip]->FormSystemMatrix(ess_tdof_list,*Optr[ip]);
+   // SparseMatrix * SpMat = (*Optr[ip]->As<ComplexSparseMatrix>()).GetSystemMatrix();
+   // SpMat->PrintMatlab(cout);
+}
+
+
+
+
+
+ToroidST::ToroidST(SesquilinearForm * bf_, const Vector & aPmlThickness_, 
+       double omega_, int nrsubdomains_)
+: bf(bf_), aPmlThickness(aPmlThickness_), omega(omega_), nrsubdomains(nrsubdomains_)       
+{
+   fes = bf->FESpace();
+   cout << "In ToroidST" << endl;
+
+   // overlap = 2.5;
+   overlap = 1.25;
+   ovlp = overlap + aPmlThickness[1]; // for now
+   //-------------------------------------------------------
+   // Step 0: Generate Mesh and FiniteElementSpace Partition
+   // ------------------------------------------------------
+   Array<Array<int> *> ElemMaps;
+   PartitionFE(fes,nrsubdomains,ovlp,fespaces, ElemMaps, 
+               DofMaps0, DofMaps1, OvlpMaps0, OvlpMaps1);
+   for (int i = 0; i<nrsubdomains; i++) delete ElemMaps[i];
+
+   //-------------------------------------------------------
+   // Step 1: Setup local Maxwell Problems PML
+   // ------------------------------------------------------
+   cout << "Setting up local problems " << endl;
+   SetupSubdomainProblems();
+   cout << "Done "<< endl;
+
+
+   // Test local to global dof Maps
+   // cout << "Testing local to global maps " << endl;
+   // for (int i = 0; i<nrsubdomains; i++)
+   // {
+   //    DofMapTests(*fespaces[i],*fes,*DofMaps0[i], *DofMaps1[i]);
+   //    // DofMapTests(*fes,*fespaces[i], *DofMaps1[i], *DofMaps0[i]);
+   // }
+
+   // cout << "Testing local to neighbor maps " << endl;
+   // for (int i = 0; i<nrsubdomains-1; i++)
+   // {
+   //    DofMapTests(*fespaces[i],*fespaces[i+1],*OvlpMaps0[i], *OvlpMaps1[i]);
+   //    DofMapTests(*fespaces[i+1],*fespaces[i],*OvlpMaps1[i], *OvlpMaps0[i]);
+   // }
+
+   // cout << "Testing local to overlap maps " << endl;
+   // for (int i = 0; i<nrsubdomains; i++)
+   // {
+      // Array<int> rdofs;
+      // GetRestrictionDofs(*fespaces[i],1,ovlp,rdofs);
+      // GetRestrictionDofs(*fespaces[i],-1,ovlp,rdofs);
+      // DofMapOvlpTest(*fespaces[i],rdofs);
+   // }
+}
+
+
+void ToroidST::Mult(const Vector & r, Vector & z) const 
+{
+   cout << "ToroidST::Mult " << endl;
+   cout << "r norm = " << r.Norml2() << endl;
+
+   z = 0.0;
+   // Step 0;
+   // Initialize transfered residuals to 0.0 and 
+   // restrict Source to subdomains
+   for (int ip=0; ip<nrsubdomains; ip++)
+   {
+      *forward_transf[ip] = 0.0;
+      *backward_transf[ip] = 0.0;
+      MapDofs(*DofMaps1[ip], *DofMaps0[ip],r,*f_orig[ip]);
+      // cout << "0:f_orig[ip] norm = " << f_orig[ip]->Norml2() << endl; 
+      // cout << "ovlp = " << ovlp << endl;
+      int direction = 0;
+      if (ip == 0) direction = 1;
+      if (ip == nrsubdomains-1) direction = -1;
+      if (nrsubdomains == 1) continue;
+      Array<int> rdofs;
+      GetRestrictionDofs(*fespaces[ip],direction,ovlp,rdofs);
+      // cout << "direction = " << direction << endl;
+      // DofMapOvlpTest(*fespaces[ip],rdofs);
+      // cin.get();
+      // rdofs.Print(cout, 20);
+      RestrictDofs(rdofs,f_orig[ip]->Size()/2,*f_orig[ip]);
+      // cout << "1:f_orig[ip] norm = " << f_orig[ip]->Norml2() << endl; 
+      // cin.get();
+   }
+
+   // Step 1; "forward sweep"
+   for (int ip=0; ip<nrsubdomains; ip++)
+   {
+      int n = fespaces[ip]->GetTrueVSize();
+      Vector res(2*n); res = 0.0;
+      res += *f_orig[ip];
+      res += *forward_transf[ip];
+      Vector sol(2*n);
+      PmlMatInv[ip]->Mult(res,sol);
+      // accumulate for the global correction;
+      // AddMapDofs(*DofMaps1[ip],*DofMaps0[ip],sol,z);
+      // Transfer source to (forward) neighbor
+      int sweep = 1;
+      SourceTransfer(ip,sol, sweep);
+      // cout << "res norm = " << res.Norml2() << endl;
+      // cout << "sol norm = " << sol.Norml2() << endl;
+      AddMapDofs(*DofMaps0[ip],*DofMaps1[ip],sol,z);
+      // cout << "z norm = " << z.Norml2() << endl;
+   }
+   // Step 2: "Backward Sweep"
+   for (int ip=nrsubdomains-1; ip>=0; ip--)
+   {
+      int n = fespaces[ip]->GetTrueVSize();
+      Vector res(2*n); res = 0.0;
+      res += *backward_transf[ip];
+      Vector sol(2*n);
+      PmlMatInv[ip]->Mult(res,sol);
+      int sweep = -1;
+      SourceTransfer(ip,sol,sweep);
+      AddMapDofs(*DofMaps0[ip],*DofMaps1[ip],sol,z);
+   }
+}
+
+void ToroidST::SourceTransfer(int ip, const Vector & sol, int sweep) const
+{
+   // Transfer to ip+1 and ip-1
+   int ip0 = ip-1;
+   int ip1 = ip+1;
+
+   // sweep : 1  - forward
+   // sweep : -1 - forward
+   // direction : 0  - both
+
+   if (ip0 >= 0)
+   {  // map sol from ip to ip0
+      int n = fespaces[ip0]->GetTrueVSize();
+      Vector sol0(2*n); sol0 = 0.0;
+      Vector Psi0(2*n);
+      MapDofs(*OvlpMaps1[ip0], *OvlpMaps0[ip0],sol,sol0);
+      PmlMat[ip0]->Mult(sol0,Psi0);
+      int direction = 1;
+      Array<int> rdofs;
+      GetRestrictionDofs(*fespaces[ip0],direction,ovlp,rdofs);
+      RestrictDofs(rdofs,backward_transf[ip0]->Size()/2,Psi0);
+      *backward_transf[ip0]-= Psi0;
+   }
+
+   if (sweep == 1)
+   {
+      if (ip1 <= nrsubdomains-1)
+      {
+         int n = fespaces[ip1]->GetTrueVSize();
+         Vector sol1(2*n); sol1 = 0.0;
+         Vector Psi1(2*n);
+         MapDofs(*OvlpMaps0[ip], *OvlpMaps1[ip],sol,sol1);
+         PmlMat[ip1]->Mult(sol1,Psi1);
+         int direction = -1;
+         Array<int> rdofs;
+         GetRestrictionDofs(*fespaces[ip1],direction,ovlp,rdofs);
+         RestrictDofs(rdofs,forward_transf[ip1]->Size()/2,Psi1);
+         *forward_transf[ip1]-= Psi1;
+      } 
+   }
+
+
+}
+
+
+ToroidST::~ToroidST()
+{
+   for (int i = 0; i<nrsubdomains-1; i++)
+   {
+      delete DofMaps0[i];
+      delete DofMaps1[i];
+      delete OvlpMaps0[i];
+      delete OvlpMaps1[i];
+   }
+   delete DofMaps0[nrsubdomains-1];     
+   delete DofMaps1[nrsubdomains-1];     
+}

examples/maxwell-solver/ToroidST/ToroidST.hpp

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "../common/PML.hpp"
+#include "DofMaps.hpp"
+using namespace std;
+using namespace mfem;
+class ToroidST : public Solver//
+{
+private:
+   SesquilinearForm *bf=nullptr;
+   FiniteElementSpace * fes = nullptr;
+   Mesh * mesh = nullptr;
+   double omega;
+   int nrsubdomains;
+   Vector aPmlThickness; 
+   double overlap,ovlp; 
+   Array<FiniteElementSpace *> fespaces;
+   Array<Array<int> *> DofMaps0, DofMaps1, OvlpMaps0, OvlpMaps1;
+
+
+   Array< SesquilinearForm * > sqf;
+   Array< OperatorPtr * > Optr;
+   Array<ComplexSparseMatrix *> PmlMat;
+   Array<ComplexUMFPackSolver *> PmlMatInv;
+   mutable Array<Vector *> f_orig;
+   mutable Array<Vector *> forward_transf;
+   mutable Array<Vector *> backward_transf;
+   void SetupSubdomainProblems();
+   void SetMaxwellPmlSystemMatrix(int ip);
+   // sweep 1: forward
+   // sweep -1: backward
+   void SourceTransfer(int ip, const Vector & sol, int sweep) const;
+public:
+   ToroidST(SesquilinearForm * bf_, const Vector & aPmlThickness_, 
+       double omega_, int nrsubdomains_ = 2);
+   virtual void SetOperator(const Operator &op) {}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~ToroidST();
+};

examples/maxwell-solver/ToroidST/bend-waveguide.cpp

@@ -0,0 +1,877 @@
+
+
+// sample runs: ./bend-waveguide -prob 2 -ref 2 -o 2 -f 0.6
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "../common/PML.hpp"
+#include "DofMaps.hpp"
+
+using namespace std;
+using namespace mfem;
+
+void maxwell_solution(const Vector &x, vector<complex<double>> &E);
+void maxwell_curl(const Vector &x, vector<complex<double>> &curlE);
+
+int prob_kind=0;
+double L;
+double ylim;
+
+// Class for returning the PML coefficients of the bilinear form
+class PMLDiagMatrixCoefficient : public VectorCoefficient
+{
+private:
+   ToroidPML * pml = nullptr;
+   void (*Function)(const Vector &, ToroidPML * , Vector &);
+public:
+   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, ToroidPML *,
+                                              Vector &),
+                            ToroidPML * pml_)
+      : VectorCoefficient(dim), pml(pml_), Function(F)
+   {}
+
+   using VectorCoefficient::Eval;
+
+   virtual void Eval(Vector &K, ElementTransformation &T,
+                     const IntegrationPoint &ip)
+   {
+      double x[3];
+      Vector transip(x, 3);
+      T.Transform(ip, transip);
+      K.SetSize(vdim);
+      (*Function)(transip, pml, K);
+   }
+};
+
+class PMLMatrixCoefficient : public MatrixCoefficient
+{
+private:
+   ToroidPML * pml = nullptr;
+   void (*Function)(const Vector &, ToroidPML * , DenseMatrix &);
+public:
+   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, ToroidPML *,
+                                              DenseMatrix &),
+                            ToroidPML * pml_)
+      : MatrixCoefficient(dim), pml(pml_), Function(F)
+   {}
+
+   using MatrixCoefficient::Eval;
+
+   virtual void Eval(DenseMatrix &M, ElementTransformation &T,
+                     const IntegrationPoint &ip)
+   {
+      double x[3];
+      Vector transip(x, 3);
+      T.Transform(ip, transip);
+      M.SetSize(height,width);
+      (*Function)(transip, pml, M);
+   }
+};
+
+
+void E_bdr_data_Re(const Vector &x, Vector &E);
+void E_bdr_data_Im(const Vector &x, Vector &E);
+
+void E_exact_Re(const Vector &x, Vector &E);
+void E_exact_Im(const Vector &x, Vector &E);
+
+void E_exact_Curl_Re(const Vector &x, Vector &E);
+void E_exact_Curl_Im(const Vector &x, Vector &E);
+
+void source(const Vector &x, Vector & f);
+
+// Functions for computing the necessary coefficients after PML stretching.
+// J is the Jacobian matrix of the stretching function
+void detJ_JT_J_inv_Re(const Vector &x, ToroidPML * pml, Vector &D);
+void detJ_JT_J_inv_Im(const Vector &x, ToroidPML * pml, Vector &D);
+void detJ_inv_JT_J_Re(const Vector &x, ToroidPML * pml, Vector &D);
+void detJ_inv_JT_J_Im(const Vector &x, ToroidPML * pml, Vector &D);
+
+
+void detJ_JT_J_inv_Re(const Vector &x, ToroidPML * pml, DenseMatrix & M);
+void detJ_JT_J_inv_Im(const Vector &x, ToroidPML * pml, DenseMatrix & M);
+void detJ_inv_JT_J_Re(const Vector &x, ToroidPML * pml, DenseMatrix & M);
+void detJ_inv_JT_J_Im(const Vector &x, ToroidPML * pml, DenseMatrix & M);
+
+
+
+Array2D<double> comp_domain_bdr;
+Array2D<double> domain_bdr;
+
+double mu = 1.0;
+double epsilon = 1.0;
+double omega;
+int dim;
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_SELF, &num_procs);
+   MPI_Comm_rank(MPI_COMM_SELF, &myid);
+   // 1. Parse command-line options.
+   // const char *mesh_file = "torus1_4.mesh";
+   // const char *mesh_file = "waveguide-bend2.mesh";
+   const char *mesh_file = "meshes/waveguide-bend.mesh";
+
+   int order = 1;
+   int ref_levels = 1;
+   double freq = 5.0;
+   bool herm_conv = true;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&prob_kind, "-prob", "--problem-kind",
+                  "Problem/mesh choice");                  
+   args.AddOption(&ref_levels, "-ref", "--refinements",
+                  "Number of refinements");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&freq, "-f", "--frequency",
+                  "Frequency (in Hz).");
+   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
+                  "--no-hermitian", "Use convention for Hermitian operators.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+
+   // 2. Setup the mesh
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+
+   switch (prob_kind)
+   {
+   case 0: 
+   {
+      mesh_file = "meshes/waveguide-bend.mesh"; 
+      L = -2.;
+      ylim = -3;
+   }
+   break;
+   case 1: 
+   {
+      mesh_file = "meshes/waveguide-bend2.mesh";
+      L = -5.;
+      ylim = 0.0;
+   }
+   break;
+   case 2: mesh_file = "meshes/toroid3_4_2.mesh"; break;
+   // case 3: mesh_file = "toroid-hex-o3-s0_r.mesh"; break;
+   // case 3: mesh_file = "../../data/square-disc.mesh"; break;
+   case 3: mesh_file = "meshes/annulus-quad-o3.mesh"; break;
+   // case 3: mesh_file = "cylinder.mesh"; break;
+   default:
+      MFEM_ABORT("Not a valid problem choice ");
+      break;
+   }
+
+   Mesh * mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+
+   mesh->RemoveInternalBoundaries();
+
+   mesh->UniformRefinement();
+   mesh->UniformRefinement();
+
+   
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+
+   double ovlerlap = 7.5; // in degrees;
+   // double ovlerlap = 0.5; // in degrees;
+   int nrmeshes = 9;
+
+   Array<Array<int> *> ElemMaps, DofMaps0, DofMaps1, OvlpMaps0, OvlpMaps1;
+   Array<FiniteElementSpace *> fespaces;
+   PartitionFE(fespace,nrmeshes,ovlerlap,fespaces, 
+               ElemMaps,
+               DofMaps0, DofMaps1,
+               OvlpMaps0, OvlpMaps1);
+
+
+   // Test local to global dof Maps
+   // for (int i = 0; i<nrmeshes; i++)
+   // {
+   //    DofMapTests(*fespaces[i],*fespace,*DofMaps0[i], *DofMaps1[i]);
+   //    // DofMapTests(*fespace,*fespaces[i], *DofMaps1[i], *DofMaps0[i]);
+   //    cin.get();
+   // }
+
+   for (int i = 0; i<nrmeshes-1; i++)
+   {
+      // DofMapTests(*fespaces[i],*fespaces[i+1],*OvlpMaps0[i], *OvlpMaps1[i]);
+      DofMapTests(*fespaces[i+1],*fespaces[i],*OvlpMaps1[i], *OvlpMaps0[i]);
+      cin.get();
+   }
+   
+
+   // if (visualization)
+   // {
+   //    // GLVis server to visualize to
+   //    char vishost[] = "localhost";
+   //    int  visport   = 19916;
+
+   //    socketstream mesh0_sock(vishost, visport);
+   //    mesh0_sock.precision(8);
+   //    mesh0_sock << "mesh\n" << *mesh << flush;
+
+   //    socketstream mesh1_sock(vishost, visport);
+   //    mesh1_sock.precision(8);
+   //    mesh1_sock << "mesh\n" << *mesh1 << flush;
+
+   //    socketstream mesh2_sock(vishost, visport);
+   //    mesh2_sock.precision(8);
+   //    mesh2_sock << "mesh\n" << *mesh2 << flush;
+   // }
+
+
+   // mesh = mesh1;
+   return 0;
+
+   // Angular frequency
+   omega = 2.0 * M_PI * freq;
+
+   ToroidPML tpml(mesh);
+   Vector zlim, rlim, alim;
+   tpml.GetDomainBdrs(zlim,rlim,alim);
+   Vector zpml_thickness(2); zpml_thickness = 0.0;
+   Vector rpml_thickness(2); rpml_thickness = 0.0;
+   Vector apml_thickness(2); apml_thickness = 0.0; 
+   bool zstretch = false;
+   bool astretch = false;
+   bool rstretch = false;
+   switch (prob_kind)
+   {
+      case 0: break;
+      case 1: break;
+      case 2: 
+      {
+         apml_thickness[1] = 45.0; 
+         astretch = true;
+      }
+      break;// degrees 
+      case 3: 
+      {
+         rpml_thickness[1] = 0.3; 
+         rstretch = true;
+      }
+      break;
+      default: break;
+   }
+   
+   tpml.SetPmlAxes(zstretch,rstretch,astretch);
+   tpml.SetPmlWidth(zpml_thickness,rpml_thickness,apml_thickness);
+   tpml.SetOmega(omega); 
+
+   ComplexOperator::Convention conv =
+      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
+
+
+
+   ComplexGridFunction x(fespace);
+   x = 0.0;
+   VectorFunctionCoefficient E_Re(dim, E_bdr_data_Re);
+   VectorFunctionCoefficient E_Im(dim, E_bdr_data_Im);
+
+   ConvergenceStudy rates_r;
+   ConvergenceStudy rates_i;
+
+   for (int iter = 0; iter<ref_levels; iter++)
+   {
+      int size = fespace->GetTrueVSize();
+      cout << "Number of finite element unknowns: " << size << endl;
+      tpml.SetAttributes(mesh); 
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (mesh->bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(mesh->bdr_attributes.Max());
+         ess_bdr = 1;
+      }
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+      VectorFunctionCoefficient f(dim, source);
+      ComplexLinearForm b(fespace, conv);
+      // b.AddDomainIntegrator(NULL, new VectorFEDomainLFIntegrator(f));
+      b.Vector::operator=(0.0);
+      b.Assemble();
+
+      x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);
+
+      Array<int> attr;
+      Array<int> attrPML;
+      if (mesh->attributes.Size())
+      {
+         attr.SetSize(mesh->attributes.Max());
+         attrPML.SetSize(mesh->attributes.Max());
+         attr = 0;   attr[0] = 1;
+         attrPML = 0;
+         if (mesh->attributes.Max() > 1)
+         {
+            attrPML[1] = 1;
+         }
+      }
+
+      ConstantCoefficient muinv(1.0/mu);
+      ConstantCoefficient omeg(-pow(omega, 2) * epsilon);
+      RestrictedCoefficient restr_muinv(muinv,attr);
+      RestrictedCoefficient restr_omeg(omeg,attr);
+
+      // Integrators inside the computational domain (excluding the PML region)
+      SesquilinearForm a(fespace, conv);
+      a.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv),NULL);
+      a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);
+
+      int cdim = (dim == 2) ? 1 : dim;
+
+      PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, &tpml);
+      PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, &tpml);
+      ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+      ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+      MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+      MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+
+      PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,&tpml);
+      PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,&tpml);
+      ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+      ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+      MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+      MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+
+      // Integrators inside the PML region
+      a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
+                           new CurlCurlIntegrator(restr_c1_Im));
+      a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
+                           new VectorFEMassIntegrator(restr_c2_Im));
+
+      a.Assemble(0);
+
+      OperatorPtr A;
+      Vector B, X;
+      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+      SparseMatrix * SpMat = (*A.As<ComplexSparseMatrix>()).GetSystemMatrix();
+      HYPRE_Int global_size = SpMat->Height();
+      HYPRE_Int row_starts[2]; row_starts[0] = 0; row_starts[1] = global_size;
+      HypreParMatrix * HypreMat = new HypreParMatrix(MPI_COMM_SELF,global_size,row_starts,SpMat);
+      {
+         MUMPSSolver mumps;
+         mumps.SetOperator(*HypreMat);
+         mumps.Mult(B,X);
+      }
+
+      a.RecoverFEMSolution(X, b, x);
+
+      if (prob_kind == 3)
+      {
+         rates_r.SetElementList(tpml.GetMarkedPMLElements());
+         rates_i.SetElementList(tpml.GetMarkedPMLElements());
+
+         VectorFunctionCoefficient E_ex_Re(dim, E_exact_Re);
+         VectorFunctionCoefficient E_ex_Im(dim, E_exact_Im);
+         VectorFunctionCoefficient E_Curl_Re(cdim, E_exact_Curl_Re);
+         VectorFunctionCoefficient E_Curl_Im(cdim, E_exact_Curl_Im);
+
+         rates_r.AddHcurlGridFunction(&x.real(),&E_ex_Re,&E_Curl_Re);
+         rates_i.AddHcurlGridFunction(&x.imag(),&E_ex_Im,&E_Curl_Im);
+      }
+
+      if (iter == ref_levels) break;
+      mesh->UniformRefinement();
+      fespace->Update();
+      x.Update();
+   }
+
+   if (prob_kind == 3)
+   {
+      rates_r.Print(false);      
+      rates_i.Print(false);     
+   }
+   
+
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      // Define visualization keys for GLVis (see GLVis documentation)
+      string keys;
+      keys = (dim == 3) ? "keys macF\n" : keys = "keys amrRljcUUuu\n";
+
+      char vishost[] = "localhost";
+      int visport = 19916;
+
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re.precision(8);
+      sol_sock_re << "solution\n"
+                  << *mesh << x.real() << keys
+                  << "window_title 'Solution real part'" << flush;
+
+      socketstream sol_sock_im(vishost, visport);
+      sol_sock_im.precision(8);
+      sol_sock_im << "solution\n"
+                  << *mesh << x.imag() << keys
+                  << "window_title 'Solution imag part'" << flush;
+
+      GridFunction x_t(fespace);
+      x_t = x.real();
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n"
+               << *mesh << x_t << keys << "autoscale off\n"
+               << "window_title 'Harmonic Solution (t = 0.0 T)'"
+               << "pause\n" << flush;
+      cout << "GLVis visualization paused."
+           << " Press space (in the GLVis window) to resume it.\n";
+      int num_frames = 16;
+      int i = 0;
+      while (sol_sock)
+      {
+         double t = (double)(i % num_frames) / num_frames;
+         ostringstream oss;
+         oss << "Harmonic Solution (t = " << t << " T)";
+
+         add(cos(2.0 * M_PI * t), x.real(),
+             sin(2.0 * M_PI * t), x.imag(), x_t);
+         sol_sock << "solution\n"
+                  << *mesh << x_t
+                  << "window_title '" << oss.str() << "'" << flush;
+         i++;
+      }
+   }
+
+   // 17. Free the used memory.
+   // delete pml;
+   delete fespace;
+   delete fec;
+   delete mesh;
+
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+void source(const Vector &x, Vector &f)
+{
+   Vector center(dim);
+   double r = 0.0;
+   center = 0.5;
+   center(2) = 0.15;
+
+   for (int i = 0; i < dim; ++i)
+   {
+      r += pow(x[i] - center[i], 2.);
+   }
+   double n = 5.0 * omega * sqrt(epsilon * mu) / M_PI;
+   double coeff = pow(n, 2) / M_PI;
+   double alpha = -pow(n, 2) * r;
+   f = 0.0;
+   f[0] = coeff * exp(alpha);
+}
+
+void E_bdr_data_Re(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   if (prob_kind == 2)
+   {
+      if (abs(x(1))<1e-12 && x(0)>0) 
+      {
+         vector<complex<double>> Eval(E.Size());
+         maxwell_solution(x, Eval);
+         for (int i = 0; i < dim; ++i)
+         {
+            E[i] = Eval[i].real();
+         }
+      }
+   }
+   else if (prob_kind == 3)
+   {
+      double r = sqrt(x(0)*x(0) + x(1)*x(1));
+      // check if in pml
+
+      // if (abs(r-1.0)<1e-10) 
+      // if (r < 0.3) // not in pml
+      // if (x(0) <0.8 && x(0)>0.2 && x(1) < 0.8 && x(1) >0.2 )
+      // if (x(0) <0.3 && x(0)>-0.3 && x(1) < 0.3 && x(1) >-0.3 )
+      if (r < 0.3 )
+      {
+         vector<complex<double>> Eval(E.Size());
+         maxwell_solution(x, Eval);
+         for (int i = 0; i < dim; ++i)
+         {
+            E[i] = Eval[i].real();
+         }
+      }
+   }
+   else
+   {
+      if (x(1) == ylim) 
+      {
+         vector<complex<double>> Eval(E.Size());
+         maxwell_solution(x, Eval);
+         for (int i = 0; i < dim; ++i)
+         {
+            E[i] = Eval[i].real();
+         }
+      }
+   }
+}
+
+// Define bdr_data solution
+void E_bdr_data_Im(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   if (prob_kind == 2)
+   {
+      if (abs(x(1))<1e-12 && x(0)>0) 
+      {
+         vector<complex<double>> Eval(E.Size());
+         maxwell_solution(x, Eval);
+         for (int i = 0; i < dim; ++i)
+         {
+            E[i] = Eval[i].imag();
+         }
+      }
+   }
+   else if (prob_kind == 3)
+   {
+      double r = sqrt(x(0)*x(0) + x(1)*x(1));
+      // if (abs(r-1.0)<1e-10) 
+      // if (r < 0.3) // not in pml
+      // if (x(0) < 0.5) // not in pml
+      // if (x(0) <0.8 && x(0)>0.2 && x(1) < 0.8 && x(1) >0.2 )
+      // if (x(0) <0.3 && x(0)>-0.3 && x(1) < 0.3 && x(1) >-0.3 )
+      if (r < 0.3 )
+      {
+         vector<complex<double>> Eval(E.Size());
+         maxwell_solution(x, Eval);
+         for (int i = 0; i < dim; ++i)
+         {
+            E[i] = Eval[i].imag();
+         }
+      }
+   }
+   else
+   {
+      if (x(1) == ylim) 
+      {
+         vector<complex<double>> Eval(E.Size());
+         maxwell_solution(x, Eval);
+         for (int i = 0; i < dim; ++i)
+         {
+            E[i] = Eval[i].imag();
+         }
+      }
+   }
+}
+
+void E_exact_Re(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   vector<complex<double>> Eval(E.Size());
+   maxwell_solution(x, Eval);
+   for (int i = 0; i < dim; ++i)
+   {
+      E[i] = Eval[i].real();
+   }
+}
+
+void E_exact_Im(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   vector<complex<double>> Eval(E.Size());
+   maxwell_solution(x, Eval);
+   for (int i = 0; i < dim; ++i)
+   {
+      E[i] = Eval[i].imag();
+   }
+}
+
+void maxwell_solution(const Vector &x, vector<complex<double>> &E)
+{
+   complex<double> zi = complex<double>(0., 1.);
+
+   if (prob_kind == 2)
+   {  // for a straight waveguide
+      double k = omega * sqrt(epsilon * mu);
+      // T_10 mode
+      double k10 = sqrt(k * k - M_PI * M_PI);
+      E[2] = -zi * k / M_PI * sin(M_PI*(x(0)))*exp(zi * k10 * x(1)); 
+   }
+   else
+   {
+      double k = omega * sqrt(epsilon * mu);
+      Vector shift(dim);
+      shift = 0.0;
+      double x0 = x(0) + shift(0);
+      double x1 = x(1) + shift(1);
+      double r = sqrt(x0 * x0 + x1 * x1);
+      double beta = k * r;
+
+      // Bessel functions
+      complex<double> H0, H0_r, H0_rr;
+      complex<double> H1; 
+      complex<double> H2;
+      H0 = jn(0,beta) + zi * yn(0,beta);
+      H1 = jn(1,beta) + zi * yn(1,beta);
+      H2 = jn(2,beta) + zi * yn(2,beta);
+      // H3 = jn(3,beta) + zi * yn(3,beta);
+
+      H0_r = - k * H1;
+      H0_rr = - k * k * (1.0/beta * H1 - H2); 
+
+      // First derivatives
+      double r_x = x0 / r;
+      double r_y = x1 / r;
+      double r_xy = -(r_x / r) * r_y;
+      double r_xx = (1.0 / r) * (1.0 - r_x * r_x);
+
+      complex<double> val, val_xx, val_xy;
+      val = 0.25 * zi * H0;
+      val_xx = 0.25 * zi * (r_xx * H0_r + r_x * r_x * H0_rr);
+      val_xy = 0.25 * zi * (r_xy * H0_r + r_x * r_y * H0_rr);
+      E[0] = zi / k * (k * k * val + val_xx);
+      E[1] = zi / k * val_xy;
+   }
+}
+
+void E_exact_Curl_Re(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   vector<complex<double>> Eval(E.Size());
+   maxwell_curl(x, Eval);
+   for (int i = 0; i < E.Size(); ++i)
+   {
+      E[i] = Eval[i].real();
+   }
+}
+
+void E_exact_Curl_Im(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   vector<complex<double>> Eval(E.Size());
+   maxwell_curl(x, Eval);
+   for (int i = 0; i < E.Size(); ++i)
+   {
+      E[i] = Eval[i].imag();
+   }
+}
+
+
+void maxwell_curl(const Vector &x, vector<complex<double>> &curlE)
+{
+   complex<double> zi = complex<double>(0., 1.);
+
+   double k = omega * sqrt(epsilon * mu);
+   Vector shift(dim);
+   shift = 0.0;
+   double x0 = x(0) + shift(0);
+   double x1 = x(1) + shift(1);
+   double r = sqrt(x0 * x0 + x1 * x1);
+   double beta = k * r;
+
+   // Bessel functions
+   complex<double> H0_r;
+   complex<double> H1;
+   // complex<double> H2, H2_r;
+   // complex<double> H3;
+   // H0 = jn(0,beta) + zi * yn(0,beta);
+   H1 = jn(1,beta) + zi * yn(1,beta);
+   // H2 = jn(2,beta) + zi * yn(2,beta);
+   // H3 = jn(3,beta) + zi * yn(3,beta);
+
+   H0_r   = - k * H1;
+   // H1_r   =   k * (1.0/beta * H1 - H2);
+   // H2_r   = - k * (2.0/beta * H2 - H3); 
+   // H0_rr  = - k * H1_r;
+   // H1_rr  = k * k * (- 2.0 /(beta * beta) * H1 + 1.0/beta * H1_r - H2_r); 
+   // H0_rrr = - k * H1_rr;
+
+   // First derivatives
+   // double r_x = x0 / r;
+   double r_y = x1 / r;
+   // double r_xy = -(r_x / r) * r_y;
+   // double r_yx = r_xy;
+   // double r_yy = (1.0 / r) * (1.0 - r_y * r_y);
+   // double r_xx = (1.0 / r) * (1.0 - r_x * r_x);
+   // double r_xxx = r_x * (r_x * r_x - 2. * r_xx * r - 1.0) /(r * r);
+   // double r_xyy = (r_x * r_y * r_y - r * r_xy * r_y - r * r_x * r_yy)/(r * r);
+
+   complex<double> val_y;
+   // val = 0.25 * zi * H0;
+   val_y = 0.25 * zi * H0_r * r_y;
+   // val_xx = 0.25 * zi * (r_xx * H0_r + r_x * r_x * H0_rr);
+   // val_xy = 0.25 * zi * (r_xy * H0_r + r_x * r_y * H0_rr);
+   curlE[0] = zi / k * (- k * k * val_y);
+}
+
+
+void detJ_JT_J_inv_Re(const Vector &x, ToroidPML * pml, Vector &D)
+{
+   // vector<complex<double>> dxs(dim);
+   // complex<double> det(1.0, 0.0);
+   // pml->StretchFunction(x, dxs,omega);
+   ComplexDenseMatrix J(dim);
+   pml->StretchFunction(x,J,omega);
+   complex<double> det = J.Det();
+
+   // for (int i = 0; i < dim; ++i)
+   // {
+   //    det *= dxs[i];
+   // }
+
+   for (int i = 0; i < dim; ++i)
+   {
+      D(i) = (det / pow(J(i,i), 2)).real();
+   }
+}
+
+void detJ_JT_J_inv_Im(const Vector &x, ToroidPML * pml, Vector &D)
+{
+   // vector<complex<double>> dxs(dim);
+   // complex<double> det = 1.0;
+   // pml->StretchFunction(x, dxs,omega);
+   ComplexDenseMatrix J(dim);
+   pml->StretchFunction(x,J,omega);
+   complex<double> det = J.Det();
+
+   // for (int i = 0; i < dim; ++i)
+   // {
+   //    det *= dxs[i];
+   // }
+
+   for (int i = 0; i < dim; ++i)
+   {
+      D(i) = (det / pow(J(i,i), 2)).imag();
+   }
+}
+
+void detJ_inv_JT_J_Re(const Vector &x, ToroidPML * pml, Vector &D)
+{
+   // vector<complex<double>> dxs(dim);
+   // complex<double> det(1.0, 0.0);
+   // pml->StretchFunction(x, dxs,omega);
+   ComplexDenseMatrix J(dim);
+   pml->StretchFunction(x,J,omega);
+   complex<double> det = J.Det();
+   // for (int i = 0; i < dim; ++i)
+   // {
+   //    det *= dxs[i];
+   // }
+   // in the 2D case the coefficient is scalar 1/det(J)
+   if (dim == 2)
+   {
+      D = (1.0 / det).real();
+   }
+   else
+   {
+      for (int i = 0; i < dim; ++i)
+      {
+         D(i) = (pow(J(i,i), 2) / det).real();
+      }
+   }
+}
+
+void detJ_inv_JT_J_Im(const Vector &x, ToroidPML * pml, Vector &D)
+{
+   // vector<complex<double>> dxs(dim);
+   // complex<double> det = 1.0;
+   // pml->StretchFunction(x, dxs,omega);
+   ComplexDenseMatrix J(dim);
+   pml->StretchFunction(x,J,omega);
+   complex<double> det = J.Det();
+   // for (int i = 0; i < dim; ++i)
+   // {
+   //    det *= dxs[i];
+   // }
+
+   if (dim == 2)
+   {
+      D = (1.0 / det).imag();
+   }
+   else
+   {
+      for (int i = 0; i < dim; ++i)
+      {
+         D(i) = (pow(J(i,i), 2) / det).imag();
+      }
+   }
+}
+
+
+//-----------------------------------------------------------------
+
+void detJ_JT_J_inv_Re(const Vector &x, ToroidPML * pml, DenseMatrix & M)
+{
+   ComplexDenseMatrix J(dim);
+   pml->StretchFunction(x,J,omega);
+   complex<double> det = J.Det();
+   ComplexDenseMatrix JtJ(dim);
+   MultAtB(J,J,JtJ);
+   ComplexDenseMatrixInverse InvJtJ(JtJ);
+   InvJtJ *=det;
+   InvJtJ.GetReal(M);
+}
+
+void detJ_JT_J_inv_Im(const Vector &x, ToroidPML * pml, DenseMatrix & M)
+{
+   ComplexDenseMatrix J(dim);
+   pml->StretchFunction(x,J,omega);
+   complex<double> det = J.Det();
+   ComplexDenseMatrix JtJ(dim);
+   MultAtB(J,J,JtJ);
+   ComplexDenseMatrixInverse InvJtJ(JtJ);
+   InvJtJ *=det;
+   InvJtJ.GetImag(M);
+}
+
+void detJ_inv_JT_J_Re(const Vector &x, ToroidPML * pml, DenseMatrix & M)
+{
+   ComplexDenseMatrix J(dim);
+   pml->StretchFunction(x,J,omega);
+   complex<double> det = J.Det();
+   if (dim == 2)
+   {
+      M = (1.0 / det).real();
+   }
+   else
+   {
+      ComplexDenseMatrix JtJ(dim);
+      MultAtB(J,J,JtJ);
+      JtJ *= 1.0/det;
+      JtJ.GetReal(M);
+   }
+}
+
+void detJ_inv_JT_J_Im(const Vector &x, ToroidPML * pml, DenseMatrix & M)
+{
+   ComplexDenseMatrix J(dim);
+   pml->StretchFunction(x,J,omega);
+   complex<double> det = J.Det();
+   if (dim == 2)
+   {
+      M = (1.0 / det).imag();
+   }
+   else
+   {
+      ComplexDenseMatrix JtJ(dim);
+      MultAtB(J,J,JtJ);
+      JtJ *= 1.0/det;
+      JtJ.GetImag(M);
+   }
+}

examples/maxwell-solver/ToroidST/makefile

@@ -0,0 +1,66 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/maxwell-solver/ToroidST,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = waveguide bend-waveguide ST_bend
+PAR_EXAMPLES = 
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean
+.PRECIOUS: %.o
+
+COMMON_O= MeshPart.o ../common/PML.o ../common/complex_linalg.o DofMaps.o \
+			 ToroidST.o
+
+# Remove built-in rules
+%: %.cpp
+%.o: %.cpp
+
+all: $(EXAMPLES)
+
+# Rules for building the EXAMPLES
+
+%: $(SRC)%.cpp $(COMMON_O) $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(COMMON_O) $(MFEM_LIBS)
+
+# Rules for compiling miniapp dependencies
+$(COMMON_O) $($(EXAMPLES)): \
+%.o: $(SRC)%.cpp $(SRC)%.hpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $(<) -o $(@)
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -f DST/*.o
+	rm -f ParDST/*.o
+	rm -f common/*.o
+	rm -f DST2D/*.o
+	rm -rf *.dSYM *.TVD.*breakpoints
+	rm output/*
+
+

examples/maxwell-solver/common/MeshPartition.hpp

@@ -0,0 +1,151 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+using namespace std;
+using namespace mfem;
+
+
+struct UniqueIndexGenerator
+{
+   int counter = 0;
+   std::unordered_map<int,int> idx;
+   int Get(int i)
+   {
+      std::unordered_map<int,int>::iterator f = idx.find(i);
+      if (f == idx.end())
+      {
+         idx[i] = counter;
+         return counter++;
+      }
+      else
+      {
+         return (*f).second;
+      }
+   }
+   void Reset()
+   {
+      counter = 0;
+      idx.clear();
+   }
+};
+
+
+double GetUniformMeshElementSize(Mesh * mesh);
+Mesh * ExtendMesh(Mesh * mesh, const Array<int> & directions);
+
+class CartesianMeshPartition 
+{
+private:
+   Mesh *mesh=nullptr;
+public:
+   int nrpatch;
+   int nxyz[3];
+   double MeshSize;
+   std::vector<Array<int>> element_map;
+   Array3D<int>subdomains;
+   // constructor
+   CartesianMeshPartition(Mesh * mesh_,int & nx, int & ny, int & nz);
+   ~CartesianMeshPartition() {};
+};
+
+class OverlappingCartesianMeshPartition 
+{
+private:
+   Mesh *mesh=nullptr;
+public:
+   int nrpatch;
+   double MeshSize;
+   int nxyz[3];
+   std::vector<Array<int>> element_map;
+   Array3D<int> subdomains;
+   // constructor
+   OverlappingCartesianMeshPartition(Mesh * mesh_,int & nx, int & ny, int & nz);
+   OverlappingCartesianMeshPartition(Mesh * mesh_,int & nx, int & ny, int & nz, int ovlp_nlayers);
+   ~OverlappingCartesianMeshPartition() {};
+};
+
+class STPOverlappingCartesianMeshPartition // Special layered partition for STP
+{
+private:
+   Mesh *mesh=nullptr;
+public:
+   int nrpatch;
+   int nx, ny, nz;
+   std::vector<Array<int>> element_map;
+   // constructor
+   STPOverlappingCartesianMeshPartition(Mesh * mesh_);
+   ~STPOverlappingCartesianMeshPartition() {};
+};
+
+class MeshPartition
+{
+private:
+   Mesh *mesh=nullptr;
+   void AddElementToMesh(Mesh * mesh,mfem::Element::Type elem_type,int * ind);
+   void GetNumVertices(int type, mfem::Element::Type & elem_type, int & nrvert);
+   void PrintElementMap();
+public:
+   int nrpatch;
+   double MeshSize;
+   std::vector<Array<int>> element_map;
+   Array3D<int> subdomains;
+   Array<Mesh *> patch_mesh;
+   int partition_kind;
+   int nxyz[3];
+   // constructor
+   MeshPartition(Mesh * mesh_, int part, int mx=1, int my=1, int mz=1, int ovl_nlayers=0);
+   ~MeshPartition();
+};
+
+void SaveMeshPartition(Array<Mesh * > meshes, 
+                       string mfilename="output/mesh.",
+                       string sfilename="output/sol.");
+
+
+#ifdef MFEM_USE_MPI
+
+class CartesianParMeshPartition 
+{
+private:
+   ParMesh *pmesh=nullptr;
+public:
+   int nrsubdomains;
+   int nxyz[3];
+   double MeshSize;
+   std::vector<Array<int>> local_element_map;
+   Array<int> subdomain_rank;
+   Array3D<int>subdomains;
+   // constructor
+   CartesianParMeshPartition(ParMesh * pmesh_,int & nx, int & ny, int & nz, 
+                             int ovlp_nlayers);
+   ~CartesianParMeshPartition() {};
+};
+
+class ParMeshPartition
+{
+private:
+   MPI_Comm comm;
+   ParMesh *pmesh=nullptr;
+   void AddElementToMesh(Mesh * mesh,mfem::Element::Type elem_type,int * ind);
+   void GetNumVertices(int type, mfem::Element::Type & elem_type, int & nrvert);
+   void PrintElementMap();
+public:
+   int nrsubdomains;
+   int OvlpNlayers;
+   int myelem_offset = 0;
+   double MeshSize;
+   std::vector<Array<int>> element_map;
+   std::vector<Array<int>> local_element_map;
+   Array3D<int> subdomains;
+   Array<Mesh *> subdomain_mesh;
+   Array<int> subdomain_rank;
+   int partition_kind;
+   int nxyz[3];
+   // constructor
+   ParMeshPartition(ParMesh * pmesh_, int mx=1, int my=1, int mz=1, int ovl_nlayers=0);
+   void SaveMeshPartition();
+   ~ParMeshPartition();
+};
+
+#endif

examples/maxwell-solver/common/PML.cpp

@@ -0,0 +1,639 @@
+#include "PML.hpp"
+
+CartesianPML::CartesianPML(Mesh *mesh_, Array2D<double> length_)
+   : mesh(mesh_), length(length_)
+{
+   dim = mesh->Dimension();
+   SetBoundaries();
+}
+
+void CartesianPML::SetBoundaries()
+{
+   comp_dom_bdr.SetSize(dim, 2);
+   dom_bdr.SetSize(dim, 2);
+   // initialize
+   for (int i = 0; i < dim; i++)
+   {
+      dom_bdr(i, 0) = infinity();
+      dom_bdr(i, 1) = -infinity();
+   }
+
+   for (int i = 0; i < mesh->GetNBE(); i++)
+   {
+      Array<int> bdr_vertices;
+      mesh->GetBdrElementVertices(i, bdr_vertices);
+      for (int j = 0; j < bdr_vertices.Size(); j++)
+      {
+         for (int k = 0; k < dim; k++)
+         {
+            dom_bdr(k, 0) = min(dom_bdr(k, 0), mesh->GetVertex(bdr_vertices[j])[k]);
+            dom_bdr(k, 1) = max(dom_bdr(k, 1), mesh->GetVertex(bdr_vertices[j])[k]);
+         }
+      }
+   }
+
+#ifdef MFEM_USE_MPI
+   ParMesh * pmesh = dynamic_cast<ParMesh *>(mesh);
+   if (pmesh)
+   {
+      for (int d=0; d<dim; d++)
+      {
+         MPI_Allreduce(MPI_IN_PLACE,&dom_bdr(d,0),1,MPI_DOUBLE,MPI_MIN,pmesh->GetComm());
+         MPI_Allreduce(MPI_IN_PLACE,&dom_bdr(d,1),1,MPI_DOUBLE,MPI_MAX,pmesh->GetComm());
+      }
+   }
+#endif
+
+   for (int i = 0; i < dim; i++)
+   {
+      comp_dom_bdr(i, 0) = dom_bdr(i, 0) + length(i, 0);
+      comp_dom_bdr(i, 1) = dom_bdr(i, 1) - length(i, 1);
+   }
+}
+
+void CartesianPML::SetAttributes(Mesh *mesh_)
+{
+   int nrelem = mesh_->GetNE();
+   elems.SetSize(nrelem);
+
+   for (int i = 0; i < nrelem; ++i)
+   {
+      elems[i] = 1;
+      bool in_pml = false;
+      Element *el = mesh_->GetElement(i);
+      Array<int> vertices;
+      // Initialize Attribute
+      el->SetAttribute(1);
+      el->GetVertices(vertices);
+      int nrvert = vertices.Size();
+      // Check if any vertex is in the pml
+      for (int iv = 0; iv < nrvert; ++iv)
+      {
+         int vert_idx = vertices[iv];
+         double *coords = mesh_->GetVertex(vert_idx);
+         for (int comp = 0; comp < dim; ++comp)
+         {
+            if (coords[comp] > comp_dom_bdr(comp, 1) ||
+                coords[comp] < comp_dom_bdr(comp, 0))
+            {
+               in_pml = true;
+               break;
+            }
+         }
+      }
+      if (in_pml)
+      {
+         elems[i] = 0;
+         el->SetAttribute(2);
+      }
+   }
+   mesh_->SetAttributes();
+}
+
+void CartesianPML::StretchFunction(const Vector &x,
+                                   vector<complex<double>> &dxs, double omega)
+{
+   complex<double> zi = complex<double>(0., 1.);
+
+   double n = 2.0;
+   double c = 10.0;
+   // double c = log(omega);
+   double coeff;
+   // Stretch in each direction independently
+   for (int i = 0; i < dim; ++i)
+   {
+      dxs[i] = 1.0;
+      if (x(i) >= comp_dom_bdr(i, 1))
+      {
+         coeff = n * c / omega / pow(length(i, 1), n);
+         dxs[i] = 1.0 + zi * coeff * abs(pow(x(i) - comp_dom_bdr(i, 1), n - 1.0));
+      }
+      if (x(i) <= comp_dom_bdr(i, 0))
+      {
+         coeff = n * c / omega / pow(length(i, 0), n);
+         dxs[i] = 1.0 + zi * coeff * abs(pow(x(i) - comp_dom_bdr(i, 0), n - 1.0));
+      }
+   }
+}
+
+ToroidPML::ToroidPML(Mesh *mesh_)
+   : mesh(mesh_)
+{
+   dim = mesh->Dimension();
+   zlim.SetSize(2);
+   rlim.SetSize(2);
+   alim.SetSize(2);
+   zpml_thickness.SetSize(2);
+   rpml_thickness.SetSize(2);
+   apml_thickness.SetSize(2);
+   SetBoundaries();
+}
+
+void ToroidPML::SetBoundaries()
+{
+   mesh->EnsureNodes();
+   int nrnodes = mesh->GetNodalFESpace()->GetTrueVSize()/dim;
+   double zmin = infinity();
+   double zmax = -infinity();
+   double rmin = infinity();
+   double rmax = -infinity();
+   double amin = infinity(); // in degrees
+   double amax = -infinity(); // in degrees
+   for (int i = 0; i<nrnodes; i++)
+   {
+      Vector coord(dim);
+      mesh->GetNode(i,coord);
+      for (int d = 0; d<dim; d++)
+      {
+         if (abs(coord[d])<1e-13) coord[d] = 0.0;
+      }
+      // Find r and a for this point
+      double x = coord[0];
+      double y = coord[1];
+      double z = 0.0;
+      if (dim == 3) z = coord[2];
+      double a = GetAngle(x,y);
+      double r = sqrt(x*x + y*y);
+      
+      zmin = min(zmin,z);
+      zmax = max(zmax,z);
+      rmin = min(rmin,r);
+      rmax = max(rmax,r);
+      amin = min(amin,a);
+      amax = max(amax,a);
+   }
+
+   zlim[0] = zmin;
+   zlim[1] = zmax;
+   rlim[0] = rmin;
+   rlim[1] = rmax;
+   alim[0] = amin;
+   alim[1] = amax;
+
+
+#ifdef MFEM_USE_MPI
+   ParMesh * pmesh = dynamic_cast<ParMesh *>(mesh);
+   if (pmesh)
+   {
+      MPI_Allreduce(MPI_IN_PLACE,&zlim[0],1,MPI_DOUBLE,MPI_MIN,pmesh->GetComm());
+      MPI_Allreduce(MPI_IN_PLACE,&zlim[1],1,MPI_DOUBLE,MPI_MAX,pmesh->GetComm());
+      MPI_Allreduce(MPI_IN_PLACE,&rlim[0],1,MPI_DOUBLE,MPI_MIN,pmesh->GetComm());
+      MPI_Allreduce(MPI_IN_PLACE,&rlim[1],1,MPI_DOUBLE,MPI_MAX,pmesh->GetComm());
+      MPI_Allreduce(MPI_IN_PLACE,&alim[0],1,MPI_DOUBLE,MPI_MIN,pmesh->GetComm());
+      MPI_Allreduce(MPI_IN_PLACE,&alim[1],1,MPI_DOUBLE,MPI_MAX,pmesh->GetComm());
+   }
+#endif
+}
+
+void ToroidPML::SetAttributes(Mesh *mesh_)
+{
+   int nrelem = mesh_->GetNE();
+   elems.SetSize(nrelem);
+
+   // Loop through the elements and identify which of them are in the PML
+   for (int i = 0; i < nrelem; ++i)
+   {
+      // initialize with 1
+      elems[i] = 1;
+      Element *el = mesh_->GetElement(i);
+      // Initialize attribute
+      el->SetAttribute(1);
+
+      Array<int> vertices;
+      el->GetVertices(vertices);
+      int nrvert = vertices.Size();
+      // Check if any vertex is in the pml
+      bool in_pml = false;
+      for (int iv = 0; iv < nrvert; ++iv)
+      {
+         int vert_idx = vertices[iv];
+         double *coords = mesh_->GetVertex(vert_idx);
+         double x = coords[0];
+         double y = coords[1];
+         double a = GetAngle(x,y);
+         double r = sqrt(x*x + y*y);
+
+         if (astretch)
+         {
+            if ( (a <= alim[0]+apml_thickness[0]) ||
+                 (a >= alim[1]-apml_thickness[1]) )
+            {
+               in_pml = true;
+               break;  
+            }
+         }
+
+         if (rstretch)
+         {
+            if ( (r <= rlim[0]+rpml_thickness[0]) ||
+                 (r >= rlim[1]-rpml_thickness[1]) )
+            {
+               in_pml = true;
+               break;  
+            }
+         }
+      }
+      if (in_pml)
+      {
+         elems[i] = 0;
+         el->SetAttribute(2);
+      }
+
+      // Vector center;
+      // mesh_->GetElementCenter(i,center);
+      // double x = center[0];
+      // double y = center[1];
+      // double a = GetAngle(x,y);
+      // double r = sqrt(x*x + y*y);
+      // // check upper and lower bound
+      // if (astretch)
+      // {
+      //    if ( (a <= alim[0]+apml_thickness[0]) ||
+      //      (a >= alim[1]-apml_thickness[1]) )
+      //    {
+      //       elems[i] = 0;
+      //       el->SetAttribute(2);
+      //    }
+      // }
+      // if (rstretch)
+      // {
+      //    if ( (r <= rlim[0]+rpml_thickness[0]) ||
+      //         (r >= rlim[1]-rpml_thickness[1]) )
+      //    {
+      //       elems[i] = 0;
+      //       el->SetAttribute(2);
+      //    }
+      // }
+   }
+   mesh_->SetAttributes();
+}
+
+
+double ToroidPML::GetAngle(const double x, const double y)
+{
+   // Find r and a for this point
+   double arad;
+   if (x == 0.0)
+   {
+      arad = (y > 0.0)? M_PI/2.0 : 3.0 * M_PI/2.0;
+   }
+   else
+   {
+      arad = atan(y/x);
+      int k = 0;
+      if (x<0)
+      {
+         k = 1;
+      }
+      else if (y<0)
+      {
+         k = 2;
+      }
+      arad += k*M_PI;
+   }
+   return arad * 180.0/M_PI;
+}
+
+// void ToroidPML::StretchFunction(const Vector &X,
+//                                    vector<complex<double>> &dxs, double omega)
+void ToroidPML::StretchFunction(const Vector &X, ComplexDenseMatrix & J, double omega)
+
+{
+   complex<double> zi = complex<double>(0., 1.);
+
+   double n = 2.0;
+   double c = 5.0;
+   // double c = log(omega);
+   // Stretch in the azimuthal direction
+   double x = X[0];
+   double y = X[1];
+   if (abs(x) < 1e-12) x = 0.0;
+   if (abs(y) < 1e-12) y = 0.0;
+   double a = GetAngle(x,y);
+   double r = sqrt(x*x + y*y);
+   // dxs[0] = 1.0;
+   // dxs[1] = 1.0;
+   J = 0.0;
+   J(0,0) = 1.0;
+   J(1,1) = 1.0;
+   if (dim == 3) J(2,2) = 1.0;
+
+   if (astretch)
+   {
+      double th = a * M_PI/180.0;
+      double thl, thL, thH;
+      bool in_pml = false;
+      // negative direction 
+      if (a <= alim[0]+apml_thickness[0])
+      {
+         in_pml = true;
+         thL = alim[1] * M_PI/180.0;
+         thH = apml_thickness[1] * M_PI/180.0;
+         thl = thL + thH;
+      }
+      // positive direction 
+      if (a >= alim[1]-apml_thickness[1])
+      {
+         in_pml = true;
+         thL = alim[1] * M_PI/180.0;
+         thH = apml_thickness[1] * M_PI/180.0;
+         thl = thL - thH;
+      }
+      // double c1 = min(20.0*M_PI/180.0,thH);
+      if (in_pml)
+      {
+         double c1 = thH;
+         double coeff = n * c / omega / pow(c1,n);
+         double f_th = pow(th - thl,n-1);
+         double th_x = - y / (r * r);
+         double th_y =   x / (r * r);
+
+         J(0,0) = 1.0 + zi * coeff * abs(f_th * th_x);
+         J(0,1) = zi * f_th * th_y;
+         J(1,0) = zi * f_th * th_x;
+         J(1,1) = 1.0 + zi * coeff * abs(f_th * th_y);
+      }
+   }
+   // Stretch in the radial direction
+   if (rstretch)
+   {  // negative
+      double rl, rL, rH;
+      bool in_pml = false;
+      if (r <= rlim[0]+rpml_thickness[0])
+      {
+         in_pml = true;
+         rL = rlim[0];     
+         rH = rpml_thickness[0];
+         rl = rL + rH;
+      }
+      // positive direction 
+      if (r >= rlim[1]-rpml_thickness[1])
+      {
+         in_pml = true;
+         rL = rlim[1];     
+         rH = rpml_thickness[1];
+         rl = rL - rH;
+      }
+
+      if (in_pml)
+      {
+         double coeff = n * c / omega / pow (rH,n);
+         double f_r =  pow(r-rl,n-1.0);
+         double r_x = x / r;
+         double r_y = y / r;
+
+         J(0,0) = 1.0 + zi * coeff * abs(f_r*r_x);
+         // J(0,1) = zi * f_r * r_y;
+         // J(1,0) = zi * f_r * r_x;
+         J(1,1) = 1.0 + zi * coeff * abs(f_r*r_y);
+      }
+   }
+}
+
+double pml_detJ_Re(const Vector & x, CartesianPML * pml)
+{
+   int dim = pml->dim;
+   double omega = pml->omega;
+   std::vector<std::complex<double>> dxs(dim);
+   complex<double> det(1.0,0.0);
+   pml->StretchFunction(x, dxs, omega);
+   for (int i=0; i<dim; ++i) det *= dxs[i];
+   return det.real();
+}
+
+double pml_detJ_Im(const Vector & x, CartesianPML * pml)
+{
+   int dim = pml->dim;
+   double omega = pml->omega;
+   std::vector<std::complex<double>> dxs(dim);
+   complex<double> det(1.0,0.0);
+   pml->StretchFunction(x, dxs, omega);
+   for (int i=0; i<dim; ++i) det *= dxs[i];
+   return det.imag();
+}
+
+void pml_detJ_JT_J_inv_Re(const Vector & x, CartesianPML * pml , DenseMatrix & M)
+{
+   int dim = pml->dim;
+   double omega = pml->omega;
+   std::vector<std::complex<double>> dxs(dim);
+   complex<double> det(1.0,0.0);
+   pml->StretchFunction(x, dxs, omega);
+
+   for (int i = 0; i<dim; ++i)
+   {
+      det *= dxs[i];
+   }
+
+   M=0.0;
+   for (int i = 0; i<dim; ++i)
+   {
+      M(i,i) = (det / pow(dxs[i],2)).real();
+   }
+}
+
+void pml_detJ_JT_J_inv_Im(const Vector & x, CartesianPML * pml , DenseMatrix & M)
+{
+   int dim = pml->dim;
+   double omega = pml->omega;
+
+   std::vector<std::complex<double>> dxs(dim);
+   complex<double> det = 1.0;
+   pml->StretchFunction(x, dxs, omega);
+
+   for (int i = 0; i<dim; ++i)
+   {
+      det *= dxs[i];
+   }
+
+   M=0.0;
+   for (int i = 0; i<dim; ++i)
+   {
+      M(i,i) = (det / pow(dxs[i],2)).imag();
+   }
+}
+
+
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+{
+   int dim = pml->dim;
+   double omega = pml->omega;
+   vector<complex<double>> dxs(dim);
+   complex<double> det(1.0, 0.0);
+   pml->StretchFunction(x, dxs, omega);
+
+   for (int i = 0; i < dim; ++i)
+   {
+      det *= dxs[i];
+   }
+
+   M = 0.0;
+   for (int i = 0; i < dim; ++i)
+   {
+      M(i, i) = (det / pow(dxs[i], 2)).real();
+   }
+}
+
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+{
+   int dim = pml->dim;
+   double omega = pml->omega;
+   vector<complex<double>> dxs(dim);
+   complex<double> det = 1.0;
+   pml->StretchFunction(x, dxs, omega);
+
+   for (int i = 0; i < dim; ++i)
+   {
+      det *= dxs[i];
+   }
+
+   M = 0.0;
+   for (int i = 0; i < dim; ++i)
+   {
+      M(i, i) = (det / pow(dxs[i], 2)).imag();
+   }
+}
+
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+{
+   int dim = pml->dim;
+   double omega = pml->omega;
+   vector<complex<double>> dxs(dim);
+   complex<double> det = 1.0;
+   pml->StretchFunction(x, dxs, omega);
+
+   for (int i = 0; i < dim; ++i)
+   {
+      det *= dxs[i];
+   }
+
+   M = 0.0;
+   for (int i = 0; i < dim; ++i)
+   {
+      M(i, i) = abs(det / pow(dxs[i], 2));
+   }
+}
+
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+{
+   int dim = pml->dim;
+   double omega = pml->omega;
+   vector<complex<double>> dxs(dim);
+   complex<double> det(1.0, 0.0);
+   pml->StretchFunction(x, dxs, omega);
+
+   for (int i = 0; i < dim; ++i)
+   {
+      det *= dxs[i];
+   }
+
+   // in the 2D case the coefficient is scalar 1/det(J)
+   if (dim == 2)
+   {
+      M = (1.0 / det).real();
+   }
+   else
+   {
+      M = 0.0;
+      for (int i = 0; i < dim; ++i)
+      {
+         M(i, i) = (pow(dxs[i], 2) / det).real();
+      }
+   }
+}
+
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+{
+   int dim = pml->dim;
+   double omega = pml->omega;
+   vector<complex<double>> dxs(dim);
+   complex<double> det = 1.0;
+   pml->StretchFunction(x, dxs, omega);
+
+   for (int i = 0; i < dim; ++i)
+   {
+      det *= dxs[i];
+   }
+
+   if (dim == 2)
+   {
+      M = (1.0 / det).imag();
+   }
+   else
+   {
+      M = 0.0;
+      for (int i = 0; i < dim; ++i)
+      {
+         M(i, i) = (pow(dxs[i], 2) / det).imag();
+      }
+   }
+}
+
+
+void detJ_JT_J_inv_Re(const Vector &x, ToroidPML * pml, DenseMatrix & M)
+{
+   int dim = pml->dim;
+   double omega = pml->omega;
+   ComplexDenseMatrix J(dim);
+   pml->StretchFunction(x,J,omega);
+   complex<double> det = J.Det();
+   ComplexDenseMatrix JtJ(dim);
+   MultAtB(J,J,JtJ);
+   ComplexDenseMatrixInverse InvJtJ(JtJ);
+   InvJtJ *=det;
+   InvJtJ.GetReal(M);
+}
+
+void detJ_JT_J_inv_Im(const Vector &x, ToroidPML * pml, DenseMatrix & M)
+{
+   int dim = pml->dim;
+   double omega = pml->omega;
+   ComplexDenseMatrix J(dim);
+   pml->StretchFunction(x,J,omega);
+   complex<double> det = J.Det();
+   ComplexDenseMatrix JtJ(dim);
+   MultAtB(J,J,JtJ);
+   ComplexDenseMatrixInverse InvJtJ(JtJ);
+   InvJtJ *=det;
+   InvJtJ.GetImag(M);
+}
+
+void detJ_inv_JT_J_Re(const Vector &x, ToroidPML * pml, DenseMatrix & M)
+{
+   int dim = pml->dim;
+   double omega = pml->omega;
+   ComplexDenseMatrix J(dim);
+   pml->StretchFunction(x,J,omega);
+   complex<double> det = J.Det();
+   if (dim == 2)
+   {
+      M = (1.0 / det).real();
+   }
+   else
+   {
+      ComplexDenseMatrix JtJ(dim);
+      MultAtB(J,J,JtJ);
+      JtJ *= 1.0/det;
+      JtJ.GetReal(M);
+   }
+}
+
+void detJ_inv_JT_J_Im(const Vector &x, ToroidPML * pml, DenseMatrix & M)
+{
+   int dim = pml->dim;
+   double omega = pml->omega;
+   ComplexDenseMatrix J(dim);
+   pml->StretchFunction(x,J,omega);
+   complex<double> det = J.Det();
+   if (dim == 2)
+   {
+      M = (1.0 / det).imag();
+   }
+   else
+   {
+      ComplexDenseMatrix JtJ(dim);
+      MultAtB(J,J,JtJ);
+      JtJ *= 1.0/det;
+      JtJ.GetImag(M);
+   }
+}

examples/maxwell-solver/common/PML.hpp

@@ -0,0 +1,211 @@
+#pragma once
+#include "mfem.hpp"
+#include "complex_linalg.hpp"
+#include <fstream>
+#include <iostream>
+using namespace std;
+using namespace mfem;
+
+
+// Class for setting up a simple Cartesian PML region
+class CartesianPML
+{
+private:
+   Mesh *mesh;
+
+   // Length of the PML Region in each direction
+   Array2D<double> length;
+
+   // Computational Domain Boundary
+   Array2D<double> comp_dom_bdr;
+
+   // Domain Boundary
+   Array2D<double> dom_bdr;
+
+   // Integer Array identifying elements in the pml
+   // 0: in the pml, 1: not in the pml
+   Array<int> elems;
+
+   // Compute Domain and Computational Domain Boundaries
+   void SetBoundaries();
+
+public:
+   // Constructor
+   CartesianPML(Mesh *mesh_,Array2D<double> length_);
+
+   int dim;
+   double omega;
+   // Return Computational Domain Boundary
+   Array2D<double> GetCompDomainBdr() {return comp_dom_bdr;}
+
+   // Return Domain Boundary
+   Array2D<double> GetDomainBdr() {return dom_bdr;}
+
+   // Return Marker list for elements
+   Array<int> * GetMarkedPMLElements() {return &elems;}
+
+   // Mark element in the PML region
+   void SetAttributes(Mesh *mesh_);
+
+   void SetOmega(double omega_) {omega = omega_;}
+
+   // PML complex stretching function
+   void StretchFunction(const Vector &x, vector<complex<double>> &dxs, double omega);
+};
+
+class ToroidPML
+{
+private:
+   Mesh *mesh;
+
+   Vector zlim, zpml_thickness; // range in axial direction
+   Vector rlim, rpml_thickness; // range in radial direction
+   Vector alim, apml_thickness; // range in azimuthal direction
+
+   // Integer Array identifying elements in the pml
+   // 0: in the pml, 1: not in the pml
+   Array<int> elems;
+
+   double GetAngle(const double x, const double y);
+
+   // Compute Domain and Computational Domain Boundaries
+   void SetBoundaries();
+
+   bool zstretch = false;
+   bool rstretch = false;
+   bool astretch = false;
+
+public:
+   // Constructor
+   ToroidPML(Mesh *mesh_);
+
+   int dim;
+   double omega;
+   // Return Computational Domain Boundary
+
+   // Return Domain Boundary
+   void GetDomainBdrs(Vector & zlim_, Vector & rlim_, Vector & alim_)
+   {
+      zlim_.SetSize(2); zlim_ = zlim;
+      rlim_.SetSize(2); rlim_ = rlim;
+      alim_.SetSize(2); alim_ = alim;
+   }
+
+   void SetPmlWidth(const Vector & zpml, const Vector & rpml, const Vector & apml)
+   {
+      MFEM_VERIFY(zpml.Size() == 2 , "Check zpml size");
+      MFEM_VERIFY(rpml.Size() == 2 , "Check rpml size");
+      MFEM_VERIFY(apml.Size() == 2 , "Check apml size");
+      zpml_thickness = zpml;
+      rpml_thickness = rpml;
+      apml_thickness = apml;
+   }
+
+   void SetPmlAxes(const bool zstretch_, 
+                   const bool rstretch_, 
+                   const bool astretch_ )
+   {
+      zstretch = zstretch_;
+      rstretch = rstretch_;
+      astretch = astretch_;
+   }
+
+   // // Return Marker list for elements
+   Array<int> * GetMarkedPMLElements() {return &elems;}
+
+   // Mark element in the PML region
+   void SetAttributes(Mesh *mesh_);
+
+   void SetOmega(double omega_) {omega = omega_;}
+
+   // PML complex stretching function
+   // void StretchFunction(const Vector &X, vector<complex<double>> &dxs, double omega);
+   void StretchFunction(const Vector &X, ComplexDenseMatrix & J, double omega);
+};
+
+class PmlCoefficient : public Coefficient
+{
+private:
+   CartesianPML * pml = nullptr;
+   double (*Function)(const Vector &, CartesianPML * );
+public:
+   PmlCoefficient(double (*F)(const Vector &, CartesianPML *), CartesianPML * pml_)
+      : pml(pml_), Function(F)
+   {}
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+   {
+      double x[3];
+      Vector transip(x, 3);
+      T.Transform(ip, transip);
+      return ((*Function)(transip, pml));
+   }
+};
+
+
+// This includes scalar coefficients
+class PmlMatrixCoefficient : public MatrixCoefficient
+{
+private:
+   CartesianPML * pml = nullptr;
+   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
+public:
+   PmlMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                          DenseMatrix &),
+                        CartesianPML * pml_)
+      : MatrixCoefficient(dim), pml(pml_), Function(F)
+   {}
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip)
+   {
+      double x[3];
+      Vector transip(x, 3);
+      T.Transform(ip, transip);
+      K.SetSize(height, width);
+      (*Function)(transip, pml, K);
+   }
+};
+
+class PMLMatrixCoefficient : public MatrixCoefficient
+{
+private:
+   ToroidPML * pml = nullptr;
+   void (*Function)(const Vector &, ToroidPML * , DenseMatrix &);
+public:
+   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, ToroidPML *,
+                                              DenseMatrix &),
+                            ToroidPML * pml_)
+      : MatrixCoefficient(dim), pml(pml_), Function(F)
+   {}
+
+   using MatrixCoefficient::Eval;
+
+   virtual void Eval(DenseMatrix &M, ElementTransformation &T,
+                     const IntegrationPoint &ip)
+   {
+      double x[3];
+      Vector transip(x, 3);
+      T.Transform(ip, transip);
+      M.SetSize(height,width);
+      (*Function)(transip, pml, M);
+   }
+};
+
+// Helmholtz pml Functions
+double pml_detJ_Re(const Vector & x, CartesianPML * pml);
+double pml_detJ_Im(const Vector & x, CartesianPML * pml);
+void pml_detJ_JT_J_inv_Re(const Vector & x, CartesianPML * pml , DenseMatrix & M);
+void pml_detJ_JT_J_inv_Im(const Vector & x, CartesianPML * pml , DenseMatrix & M);
+
+// Maxwell Pml functions
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+
+// Functions for computing the necessary coefficients after PML stretching.
+// J is the Jacobian matrix of the stretching function
+void detJ_JT_J_inv_Re(const Vector &x, ToroidPML * pml, DenseMatrix & M);
+void detJ_JT_J_inv_Im(const Vector &x, ToroidPML * pml, DenseMatrix & M);
+void detJ_inv_JT_J_Re(const Vector &x, ToroidPML * pml, DenseMatrix & M);
+void detJ_inv_JT_J_Im(const Vector &x, ToroidPML * pml, DenseMatrix & M);

examples/maxwell-solver/common/Utilities.cpp

@@ -0,0 +1,619 @@
+#include "Utilities.hpp"
+
+Sweep::Sweep(int dim_) : dim(dim_)
+{
+   nsweeps = pow(2,dim);
+   sweeps.resize(nsweeps);
+
+   for (int is = 0; is<nsweeps; is++)
+   {
+      sweeps[is].SetSize(dim);
+   }
+
+   switch(dim)
+   {
+      case 1: 
+         sweeps[0][0] =  1; 
+         sweeps[1][0] = -1; 
+         break;
+      case 2: 
+         sweeps[0][0] =  1; sweeps[0][1] =  1; 
+         sweeps[1][0] = -1; sweeps[1][1] =  1; 
+         sweeps[2][0] =  1; sweeps[2][1] = -1; 
+         sweeps[3][0] = -1; sweeps[3][1] = -1;
+         break;
+      default:
+         sweeps[0][0] =  1; sweeps[0][1] =  1; sweeps[0][2] =  1; 
+         sweeps[1][0] = -1; sweeps[1][1] =  1; sweeps[1][2] =  1; 
+         sweeps[2][0] =  1; sweeps[2][1] = -1; sweeps[2][2] =  1; 
+         sweeps[3][0] = -1; sweeps[3][1] = -1; sweeps[3][2] =  1;
+         sweeps[4][0] =  1; sweeps[4][1] =  1; sweeps[4][2] = -1; 
+         sweeps[5][0] = -1; sweeps[5][1] =  1; sweeps[5][2] = -1; 
+         sweeps[6][0] =  1; sweeps[6][1] = -1; sweeps[6][2] = -1; 
+         sweeps[7][0] = -1; sweeps[7][1] = -1; sweeps[7][2] = -1;
+         break;
+   }
+}
+
+Sweep::~Sweep()
+{
+   for (int i = 0; i<nsweeps; i++)
+   {
+      sweeps[i].DeleteAll();
+   }
+}
+
+
+double CutOffFncn(const Vector &x, const Vector & pmin, const Vector & pmax, const Array2D<double> & h_)
+{
+   int dim = pmin.Size();
+   Vector h0(dim);
+   Vector h1(dim);
+   for (int i=0; i<dim; i++)
+   {
+      h0(i) = h_[i][0];
+      h1(i) = h_[i][1];
+   }
+   Vector x0(dim);
+   Vector x1(dim);
+   x0 = pmin; x0+=h0;
+   x1 = pmax; x1-=h1;
+
+   double f = 1.0;
+   for (int i = 0; i<dim; i++)
+   {
+      double val = 1.0;
+      if( x(i) >= pmax(i) || x(i) <= pmin(i))
+      {
+         val = 0.0;
+      }  
+      else if (x(i) < pmax(i) && x(i) >= x1(i))
+      {
+         if(h1(i) != 0.0)
+            // val = (x(i)-pmax(i))/(x1(i)-pmax(i)); 
+            val = pow((x(i)-pmax(i))/(x1(i)-pmax(i)),1.0); 
+      }
+      else if (x(i) > pmin(i) && x(i) <= x0(i))
+      {
+         if (h0(i) != 0.0)
+            // val = (x(i)-pmin(i))/(x0(i)-pmin(i)); 
+            val = pow((x(i)-pmin(i))/(x0(i)-pmin(i)),1.0); 
+      }
+
+      if (h0(i) == 0 && x(i) <= x1(i))
+      {
+         val = 1.0;
+      }
+      if (h1(i) == 0 && x(i) >= x0(i))
+      {
+         val = 1.0;
+      }
+      f *= val;
+   }
+   return f;
+}
+
+double ChiFncn(const Vector &x, const Vector & pmin, const Vector & pmax, const Array2D<double> & h_)
+{
+   int dim = pmin.Size();
+   Vector h0(dim);
+   Vector h1(dim);
+   for (int i=0; i<dim; i++)
+   {
+      h0(i) = h_[i][0];
+      h1(i) = h_[i][1];
+   }
+   Vector x0(dim);
+   Vector x1(dim);
+   x0 = pmin; x0+=h0;
+   x1 = pmax; x1-=h1;
+
+   double f = 1.0;
+   for (int i = 0; i<dim; i++)
+   {
+      double val = 1.0;
+      if( x(i) >= pmax(i) || x(i) <= pmin(i))
+      {
+         val = 0.0;
+      }  
+      else if (x(i) < pmax(i) && x(i) >= x1(i))
+      {
+         if(h1(i) != 0.0)
+            val = (x(i)-pmax(i))/(x1(i)-pmax(i)); 
+            // This function has to be changed to smth more reasonable
+            // val = pow((x(i)-pmax(i))/(x1(i)-pmax(i)),100.0); 
+      }
+      else if (x(i) > pmin(i) && x(i) <= x0(i))
+      {
+         if (h0(i) != 0.0)
+            val = (x(i)-pmin(i))/(x0(i)-pmin(i)); 
+            // val = pow((x(i)-pmin(i))/(x0(i)-pmin(i)),100.0); 
+      }
+
+      if (h0(i) == 0 && x(i) <= x1(i))
+      {
+         val = 1.0;
+      }
+      if (h1(i) == 0 && x(i) >= x0(i))
+      {
+         val = 1.0;
+      }
+      f *= val;
+   }
+   return f;
+}
+
+
+DofMap::DofMap(FiniteElementSpace * fes , MeshPartition * partition) 
+{
+   const FiniteElementCollection * fec = fes->FEColl();
+   nrpatch = partition->nrpatch;
+
+   fespaces.SetSize(nrpatch);
+
+   Dof2GlobalDof.resize(nrpatch);
+
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // create finite element spaces for each patch 
+      fespaces[ip] = new FiniteElementSpace(partition->patch_mesh[ip],fec);
+
+      // construct the patch tdof to global tdof map
+      int nrdof = fespaces[ip]->GetTrueVSize();
+      Dof2GlobalDof[ip].SetSize(2*nrdof);
+
+      // loop through the elements in the patch
+      for (int iel = 0; iel<partition->element_map[ip].Size(); ++iel)
+      {
+         // index in the global mesh
+         int iel_idx = partition->element_map[ip][iel];
+         // get the dofs of this element
+         Array<int> ElemDofs;
+         Array<int> GlobalElemDofs;
+         fespaces[ip]->GetElementDofs(iel,ElemDofs);
+         fes->GetElementDofs(iel_idx,GlobalElemDofs);
+         // the sizes have to match
+         MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+                     "Size inconsistency");
+         // loop through the dofs and take into account the signs;
+         int ndof = ElemDofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = ElemDofs[i];
+            int gdof_ = GlobalElemDofs[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            Dof2GlobalDof[ip][pdof] = gdof;
+            Dof2GlobalDof[ip][pdof+nrdof] = gdof+fes->GetTrueVSize();
+         }
+      }
+   }
+}
+
+DofMap::DofMap(FiniteElementSpace * fes , MeshPartition * partition, int nrlayers) 
+{
+
+   nx = partition->nxyz[0];
+   ny = partition->nxyz[1];
+   nz = partition->nxyz[2];
+
+   int partition_kind = partition->partition_kind;
+   // Mesh * mesh = fespace->GetMesh();
+   const FiniteElementCollection * fec = fes->FEColl();
+   nrpatch = partition->nrpatch;
+
+   fespaces.SetSize(nrpatch);
+   PmlMeshes.SetSize(nrpatch);
+   // Extend patch meshes to include pml
+
+   for  (int ip = 0; ip<nrpatch; ip++)
+   {
+      int k = ip/(nx*ny);
+      int j = (ip-k*nx*ny)/nx;
+      int i = (ip-k*nx*ny)%nx;
+      
+      Array<int> directions;
+      if (i > 0)
+      {
+         for (int i=0; i<nrlayers; i++)
+         {
+            directions.Append(-1);
+         }
+      }
+      if (j > 0)
+      {
+         for (int i=0; i<nrlayers; i++)
+         {
+            directions.Append(-2);
+         }
+      }
+      if (k > 0)
+      {
+         for (int i=0; i<nrlayers; i++)
+         {
+            directions.Append(-3);
+         }
+      }
+      if (i < nx-1)
+      {
+         for (int i=0; i<nrlayers; i++)
+         {
+            if (partition_kind == 3 || partition_kind == 2) directions.Append(1);
+         }
+      }
+      if (j < ny-1)
+      {
+         for (int i=0; i<nrlayers; i++)
+         {
+            if (partition_kind == 3 || partition_kind == 2) directions.Append(2);
+         }
+      }
+      if (k < nz-1)
+      {
+         for (int i=0; i<nrlayers; i++)
+         {
+            if (partition_kind == 3 || partition_kind == 2) directions.Append(1);
+         }
+      }
+      PmlMeshes[ip] = ExtendMesh(partition->patch_mesh[ip],directions);
+   }
+
+   // Save PML_meshes
+   string meshpath;
+   string solpath;
+   if (partition_kind == 3 || partition_kind == 2) 
+   {
+      meshpath = "output/mesh_ovlp_pml.";
+      solpath = "output/sol_ovlp_pml.";
+   }
+   else if (partition_kind == 4)
+   {
+      meshpath = "output/mesh_novlp_pml.";
+      solpath = "output/sol_novlp_pml.";
+   }
+   else
+   {
+      MFEM_ABORT("This partition kind not supported yet");
+   }
+   
+   // SaveMeshPartition(PmlMeshes, meshpath, solpath);
+
+   PmlFespaces.SetSize(nrpatch);
+   Dof2GlobalDof.resize(nrpatch);
+   Dof2PmlDof.resize(nrpatch);
+
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // create finite element spaces for each patch 
+      fespaces[ip] = new FiniteElementSpace(partition->patch_mesh[ip],fec);
+      PmlFespaces[ip] = new FiniteElementSpace(PmlMeshes[ip],fec);
+
+      // construct the patch tdof to global tdof map
+      int nrdof = fespaces[ip]->GetTrueVSize();
+      Dof2GlobalDof[ip].SetSize(2*nrdof);
+      Dof2PmlDof[ip].SetSize(2*nrdof);
+
+      // build dof maps between patch and extended patch
+      // loop through the patch elements and constract the dof map
+      // The same elements in the extended mesh have the same ordering (but not the dofs)
+
+      // loop through the elements in the patch
+      for (int iel = 0; iel<partition->element_map[ip].Size(); ++iel)
+      {
+         // index in the global mesh
+         int iel_idx = partition->element_map[ip][iel];
+         // get the dofs of this element
+         Array<int> ElemDofs;
+         Array<int> PmlElemDofs;
+         Array<int> GlobalElemDofs;
+         fespaces[ip]->GetElementDofs(iel,ElemDofs);
+         PmlFespaces[ip]->GetElementDofs(iel,PmlElemDofs);
+         fes->GetElementDofs(iel_idx,GlobalElemDofs);
+         // the sizes have to match
+         MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+                     "Size inconsistency");
+         MFEM_VERIFY(ElemDofs.Size() == PmlElemDofs.Size(),
+                     "Size inconsistency");            
+         // loop through the dofs and take into account the signs;
+         int ndof = ElemDofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = ElemDofs[i];
+            int gdof_ = GlobalElemDofs[i];
+            int pmldof_ = PmlElemDofs[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            int pmldof = (pmldof_ >= 0) ? pmldof_ : abs(pmldof_) - 1;
+
+            Dof2GlobalDof[ip][pdof] = gdof;
+            Dof2GlobalDof[ip][pdof+nrdof] = gdof+fes->GetTrueVSize();
+            Dof2PmlDof[ip][pdof] = pmldof;
+            Dof2PmlDof[ip][pdof+nrdof] = pmldof+PmlFespaces[ip]->GetTrueVSize();
+         }
+      }
+   }
+}
+
+
+LocalDofMap::LocalDofMap(const FiniteElementCollection * fec_, MeshPartition * part1_, 
+               MeshPartition * part2_):fec(fec_), part1(part1_), part2(part2_)
+{
+   // Each overlapping patch has 2 non-overlapping subdomains
+   // Thre are n non-overlapping and and n-1 overlapping subdomains
+   int nrpatch = part2->nrpatch;
+   MFEM_VERIFY(part1->nrpatch-1 == part2->nrpatch, "Check number of subdomains");
+
+   cout << "Constructing local dof maps" << endl; 
+   map1.resize(nrpatch);
+   map2.resize(nrpatch);
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      // Get the 3 meshes involved
+      Mesh * mesh = part2->patch_mesh[ip];
+      Mesh * mesh1 = part1->patch_mesh[ip];
+      Mesh * mesh2 = part1->patch_mesh[ip+1];
+
+      // Define the fespaces
+      FiniteElementSpace fespace(mesh, fec);
+      FiniteElementSpace fespace1(mesh1, fec);
+      FiniteElementSpace fespace2(mesh2, fec);
+
+      int ndof1 = fespace1.GetTrueVSize();
+      int ndof2 = fespace2.GetTrueVSize();
+
+      map1[ip].SetSize(2*ndof1); // times 2 because it's complex
+      map2[ip].SetSize(2*ndof2); // times 2 because it's complex
+
+      // loop through the elements in the patches
+      // map 1 is constructed by the first half of elements
+      // map 2 is constructed by the second half of elements
+
+      for (int iel = 0; iel<part1->element_map[ip].Size(); ++iel)
+      {
+         // index in the overlapping mesh
+         int iel_idx = iel;
+         Array<int> ElemDofs;
+         Array<int> GlobalElemDofs;
+         fespace1.GetElementDofs(iel,ElemDofs);
+         fespace.GetElementDofs(iel_idx,GlobalElemDofs);
+         // the sizes have to match
+         MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+                     "Size inconsistency");
+         // loop through the dofs and take into account the signs;
+         int ndof = ElemDofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = ElemDofs[i];
+            int gdof_ = GlobalElemDofs[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            map1[ip][pdof] = gdof;
+            map1[ip][pdof+ndof1] = gdof+fespace.GetTrueVSize();
+         }
+      }
+      for (int iel = 0; iel<part1->element_map[ip+1].Size(); ++iel)
+      {
+         // index in the overlapping mesh
+         int k = part1->element_map[ip].Size();
+         int iel_idx = iel+k;
+         Array<int> ElemDofs;
+         Array<int> GlobalElemDofs;
+         fespace2.GetElementDofs(iel,ElemDofs);
+         fespace.GetElementDofs(iel_idx,GlobalElemDofs);
+         // the sizes have to match
+         MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+                     "Size inconsistency");
+         // loop through the dofs and take into account the signs;
+         int ndof = ElemDofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = ElemDofs[i];
+            int gdof_ = GlobalElemDofs[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            map2[ip][pdof] = gdof;
+            map2[ip][pdof+ndof2] = gdof+fespace.GetTrueVSize();
+         }
+      }
+   }
+};
+
+
+NeighborDofMaps::NeighborDofMaps(MeshPartition * part_, FiniteElementSpace * fes_,
+                                DofMap * dmap_, 
+                                int ovlp_layers_) : part(part_), fes(fes_), 
+                                dmap(dmap_),
+                                ovlp_layers(ovlp_layers_)
+{
+
+   nrsubdomains = part->nrpatch;
+   nxyz.SetSize(3);
+   mesh = fes->GetMesh();
+   dim = mesh->Dimension();
+   for (int d=0; d<3; d++)  nxyz[d] = part->nxyz[d];
+   MarkOvlpElements();
+   ComputeNeighborDofMaps();
+}
+
+void NeighborDofMaps::MarkOvlpElements()
+{
+   // Lists of elements
+   // x,y,z = +/- 1 ovlp 
+   OvlpElems.resize(nrsubdomains);
+
+   for (int ip = 0; ip<nrsubdomains; ip++)
+   {
+      int i0,j0,k0;
+      Getijk(ip,i0,j0,k0);
+      int ijk[dim]; ijk[0] = i0; ijk[1]=j0;
+      if (dim==3) ijk[2] = k0;
+
+      FiniteElementSpace * sub_fes = dmap->fespaces[ip];
+      Mesh * sub_mesh = sub_fes->GetMesh();
+      // OvlpElems[ip].resize(2*dim);
+      OvlpElems[ip].resize(pow(3,dim)); 
+
+      Vector pmin, pmax;
+      sub_mesh->GetBoundingBox(pmin,pmax);
+      double h = part->MeshSize;
+      // Loop through elements
+      for (int iel=0; iel<sub_mesh->GetNE(); iel++)
+      {
+         // Get element center
+         Vector center(dim);
+         int geom = sub_mesh->GetElementBaseGeometry(iel);
+         ElementTransformation * tr = sub_mesh->GetElementTransformation(iel);
+         tr->Transform(Geometries.GetCenter(geom),center);
+
+         // loop through dimensions   
+         Array<bool> pos(dim); pos = 0;
+         Array<bool> neg(dim); neg = 0;
+
+         for (int d=0;d<dim; d++)
+         {
+            if (ijk[d]>0 && center[d] < pmin[d]+2.0*h*ovlp_layers)
+            {
+               neg[d] = true;
+            }
+
+            if (ijk[d]<nxyz[d]-1 && center[d] > pmax[d]-2.0*h*ovlp_layers) 
+            {
+               pos[d] = true;
+            }
+         }
+         SetElementToOverlap(ip,iel,neg,pos);        
+      }   
+   }
+}
+
+void NeighborDofMaps::ComputeNeighborDofMaps()
+{
+   OvlpDofMaps.resize(nrsubdomains);
+
+   // Array<UniqueIndexGen * > Gen(nrsubdomains);
+   // // construct unique number generator for the elements of a patch
+   // for (int ip = 0; ip<nrsubdomains; ip++)
+   // {
+   //    Gen[ip] = new UniqueIndexGen;
+   //    // register the elements
+   //    int nel = part->element_map[ip].Size();
+   //    for (int iel=0; iel<nel; iel++)
+   //    {
+   //       int iel_idx = part->element_map[ip][iel];
+   //       Gen[ip]->Set(iel_idx);
+   //    }
+   // }
+
+   // construct dof maps
+   int nrneighbors = pow(3,dim); // including its self
+
+   for (int ip0 = 0; ip0<nrsubdomains; ip0++)
+   {
+      OvlpDofMaps[ip0].resize(nrneighbors);
+
+      FiniteElementSpace * fes0 = dmap->fespaces[ip0];
+      int tdofs0 = fes0->GetTrueVSize();
+      Array<int> marker0(tdofs0); marker0 = 0;
+      int i0, j0, k0;
+      Array<int> ijk(dim);
+      Getijk(ip0, i0,j0,k0);
+
+      int kbeg = (dim == 2) ? 0 : -1;
+      int kend = (dim == 2) ? 1 :  2;
+      for (int k=kbeg; k<kend; k++)
+      {
+         int k1 = k0 + k;
+         if (k1 <0 || k1>=nxyz[2]) continue;
+         int kk = (dim == 2) ? -1 : k;
+         for (int j=-1; j<2; j++)
+         {
+            int j1 = j0 + j;
+            if (j1 <0 || j1>=nxyz[1]) continue;
+            for (int i=-1; i<2; i++)
+            {
+               int i1 = i0 + i;
+               if (i1 <0 || i1>=nxyz[0]) continue;
+     
+               Array<int> ip0list;  marker0 = 0;
+               int directionId = GetDirectionId(i,j,kk);
+
+               Array<int> Elems = OvlpElems[ip0][directionId];
+               int nel = Elems.Size();
+
+               for (int iel = 0; iel<nel; ++iel)
+               {
+                  int iel0 = Elems[iel];   
+                  Array<int> ElemDofs0;
+
+                  fes0->GetElementDofs(iel0,ElemDofs0);
+                  int ndof = ElemDofs0.Size();
+                  // since the elements are added to the subdomain meshes 
+                  // in the same ordered fashion (as they come from the
+                  // original mesh) then the ordering of elements in each 
+                  // subdomain is the same. Hence the dof ovlp lists
+                  // can be computed for each subdomain independendly
+                  for (int l = 0; l<ndof; ++l)
+                  {
+                     int dof0_ = ElemDofs0[l];
+                     int dof0 = (dof0_ >= 0) ? dof0_ : abs(dof0_) - 1;
+                     if (!marker0[dof0])
+                     {
+                        ip0list.Append(dof0); // dofs of ip0 in ovlp
+                        marker0[dof0] = 1;
+                     }
+                  }
+               }
+               
+               OvlpDofMaps[ip0][directionId].Append(ip0list);
+               int tsize = fes0->GetTrueVSize();
+               // Imaginary part
+               for (int l=0;l<ip0list.Size(); l++) { ip0list[l] += tsize; }
+               OvlpDofMaps[ip0][directionId].Append(ip0list);
+            }
+         }
+      }
+   }
+}
+
+
+void NeighborDofMaps::GetNeighborDofMap(const int ip,
+                                        const Array<int> & directions, 
+                                        Array<int> & dofmap)
+{
+   int k = (dim == 2) ? -1 : directions[2];
+   int directionid = GetDirectionId(directions[0],directions[1],k);
+   dofmap = OvlpDofMaps[ip][directionid];
+}
+
+
+void NeighborDofMaps::SetElementToOverlap(int ip, int iel,
+                        const Array<bool> & neg, 
+                        const Array<bool> & pos)
+{
+   int kbeg = (dim == 2) ? 0 : -1;
+   int kend = (dim == 2) ? 0 :  1;
+   for (int k = kbeg; k<=kend; k++)
+   {
+      if (dim == 3)
+      {
+         if (k == -1 && !neg[2]) continue;   
+         if (k ==  1 && !pos[2]) continue;   
+      }   
+      for (int j = -1; j<=1; j++)
+      {
+         if (j== -1 && !neg[1]) continue;   
+         if (j==  1 && !pos[1]) continue;   
+         for (int i = -1; i<=1; i++)
+         {
+            // cases to skip 
+            if (i==-1 && !neg[0]) continue;   
+            if (i== 1 && !pos[0]) continue;   
+
+            if (i==0 && j==0 && k == 0) continue;
+            int kk  = (dim==2)?-1 : k;  
+            int DirId = GetDirectionId(i,j,kk);
+            OvlpElems[ip][DirId].Append(iel);
+         }
+      }
+   }
+}

examples/maxwell-solver/common/Utilities.hpp

@@ -0,0 +1,178 @@
+#pragma once
+#include "MeshPartition.hpp"
+
+struct UniqueIndexGen
+{
+   int counter = 0;
+   std::unordered_map<int,int> idx;
+
+   void Set(int i)
+   {
+      std::unordered_map<int,int>::iterator f = idx.find(i);
+      if (f == idx.end())
+      {
+         idx[i] = counter;
+         counter++;
+      }
+   }
+
+   int Get(int i)
+   {
+      std::unordered_map<int,int>::iterator f = idx.find(i);
+      if (f == idx.end())
+      {
+         return -1;
+      }
+      else
+      {
+         return (*f).second;
+      }
+   }
+   void Reset()
+   {
+      counter = 0;
+      idx.clear();
+   }
+};
+
+
+struct Sweep
+{
+private:
+   int dim;
+   std::vector<Array<int>> sweeps;
+public:   
+   int nsweeps;
+   Sweep(int dim_);
+   ~Sweep();
+   void GetSweep(const int i, Array<int> & sweep)
+   {
+      MFEM_VERIFY(i<nsweeps, "Sweep number out of bounds");
+      sweep.SetSize(dim);
+      sweep = sweeps[i];
+   }
+
+
+};
+
+
+
+// Function coefficient that takes the bounding box of the mesh as an input
+class CutOffFnCoefficient : public Coefficient
+{
+private:
+   double (*Function)(const Vector &, const Vector &, const Vector &, const Array2D<double> &);
+   Vector pmin, pmax;
+   Array2D<double> h; // specify the with of the cutoff function (h in each direction)
+   
+
+public:
+   CutOffFnCoefficient(double (*F)(const Vector &, const Vector &, const Vector &, const Array2D<double> &), 
+                             const Vector & pmin_, const Vector & pmax_, Array2D<double> & h_)
+      : Function(F), pmin(pmin_), pmax(pmax_), h(h_)
+   {}
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+   {
+      double x[3];
+      Vector transip(x, 3);
+      T.Transform(ip, transip);
+      return ((*Function)(transip, pmin, pmax, h));
+   }
+};
+
+double CutOffFncn(const Vector &x, const Vector & pmin, 
+const Vector & pmax, const Array2D<double> & h_);
+double ChiFncn(const Vector &x, const Vector & pmin, 
+const Vector & pmax, const Array2D<double> & h_);
+
+class DofMap // Constructs dof maps for a given partition
+{
+public:
+   int nrpatch, nx, ny, nz;
+   vector<Array<int>> Dof2GlobalDof;
+   vector<Array<int>> Dof2PmlDof;
+   Array<Mesh *> PmlMeshes;
+   Array<FiniteElementSpace *> fespaces;
+   Array<FiniteElementSpace *> PmlFespaces;
+   // constructor
+   // Non PML constructor dof map
+   DofMap(FiniteElementSpace * fes, MeshPartition * partition);
+   // PML
+   DofMap(FiniteElementSpace * fes , MeshPartition * partition, int nrlayers);
+   ~DofMap(){};
+};
+
+
+
+class LocalDofMap // Constructs dof mapbetween two partitions
+{
+   const FiniteElementCollection *fec=nullptr;
+   MeshPartition * part1=nullptr;
+   MeshPartition * part2=nullptr;
+public:
+   int nrpatch, nx, ny, nz;
+   vector<Array<int>> map1;
+   vector<Array<int>> map2;
+   // constructor
+   LocalDofMap(const FiniteElementCollection * fec_, MeshPartition * part1_, 
+               MeshPartition * part2_);
+   ~LocalDofMap();
+};
+
+
+struct NeighborDofMaps
+{
+private:
+   int dim;
+   MeshPartition * part = nullptr;
+   FiniteElementSpace * fes = nullptr;
+   Mesh * mesh = nullptr;
+   std::vector<std::vector<Array<int>>> OvlpElems;
+   std::vector<std::vector<Array<int>>> OvlpDofMaps;
+
+   DofMap * dmap = nullptr;
+   int nrsubdomains = 0;
+   int ovlp_layers = 0;
+   Array<int> nxyz;
+   void SetElementToOverlap(int ip, int iel, 
+                           const Array<bool> & neg, 
+                           const Array<bool> & pos);
+
+   void MarkOvlpElements();
+   void ComputeNeighborDofMaps();
+
+   void Getijk(int ip, int & i, int & j, int & k) const
+   {
+      k = ip/(nxyz[0]*nxyz[1]);
+      j = (ip-k*nxyz[0]*nxyz[1])/nxyz[0];
+      i = (ip-k*nxyz[0]*nxyz[1])%nxyz[0];
+   }
+
+   int GetPatchId(const Array<int> & ijk) const
+   {
+      int d=ijk.Size();
+      int z = (d==2)? 0 : ijk[2];
+      return part->subdomains(ijk[0],ijk[1],z);
+   }
+      int GetDirectionId(int i, int j, int k=-1) 
+   {
+      int n = 3;
+      return (k+1)*n*n + (j+1)*n + i+1; 
+   }
+   void GetDirections(const int id, int & i, int & j, int & k) 
+   {
+      int n = 3;
+      k = id/(n*n) - 1;
+      j = (id-(k+1)*n*n)/n - 1;
+      i = (id-(k+1)*n*n)%n - 1;
+   }
+
+public:   
+   NeighborDofMaps(MeshPartition * part_, 
+                   FiniteElementSpace * fes_, 
+                   DofMap * dmap_,
+                   int ovlp_layers_);
+
+   void GetNeighborDofMap(const int ip, const Array<int> & directions,
+                          Array<int> & dofmap);
+};

examples/maxwell-solver/common/complex_linalg.cpp

@@ -0,0 +1,358 @@
+
+#include "../../../linalg/kernels.hpp"
+#include "complex_linalg.hpp"
+
+
+ComplexDenseMatrix::ComplexDenseMatrix(){}
+
+ComplexDenseMatrix::ComplexDenseMatrix(int s) 
+{
+   MFEM_ASSERT(s >= 0, "invalid ComplexDenseMatrix size: " << s);
+   height = s;
+   width  = s;
+   if (s > 0)
+   {
+      data = new complex<double>[s*s];
+      *this = 0.0; // init with zeroes
+   }
+}
+
+
+ComplexDenseMatrix::ComplexDenseMatrix(int m, int n) 
+{
+   MFEM_VERIFY(m >= 0 && n >= 0,
+               "invalid DenseMatrix size: " << m << " x " << n);
+   const int s = m*n;
+   height = m;
+   width  = n;
+   if (s > 0)
+   {
+      data = new complex<double>[s];
+      *this = 0.0; // init with zeroes
+   }
+}
+
+void ComplexDenseMatrix::SetSize(int h, int w)
+{
+   MFEM_VERIFY(h >= 0 && w >= 0,
+               "invalid ComplexDenseMatrix size: " << h << " x " << w);
+   if (Height() == h && Width() == w)
+   {
+      return;
+   }
+   height = h;
+   width = w;
+   const int hw = h*w;
+   delete data;
+   data = new complex<double>[hw];
+   *this = 0.0; // init with zeroes
+}
+
+ComplexDenseMatrix &ComplexDenseMatrix::operator=(double c)
+{
+   const int s = Height()*Width();
+   for (int i = 0; i < s; i++)
+   {
+      data[i] = c;
+   }
+   return *this;
+}
+
+ComplexDenseMatrix &ComplexDenseMatrix::operator=(complex<double> c)
+{
+   const int s = Height()*Width();
+   for (int i = 0; i < s; i++)
+   {
+      data[i] = c;
+   }
+   return *this;
+}
+
+
+std::complex<double> ComplexDenseMatrix::Det() const
+{
+   MFEM_ASSERT(Height() == Width() && Height() > 0,
+               "The matrix must be square and "
+               << "sized larger than zero to compute the determinant."
+               << "  Height() = " << Height()
+               << ", Width() = " << Width());
+
+   switch (Height())
+   {
+      case 1:
+         return data[0];
+
+      case 2:
+         return data[0] * data[3] - data[1] * data[2];
+
+      case 3:
+      {
+         const complex<double> *d = data;
+         return
+            d[0] * (d[4] * d[8] - d[5] * d[7]) +
+            d[3] * (d[2] * d[7] - d[1] * d[8]) +
+            d[6] * (d[1] * d[5] - d[2] * d[4]);
+      }
+      default:
+      {
+         MFEM_ABORT("dim>3 not supported yet");         
+         return 0;
+      }
+   }
+}
+
+DenseMatrix * ComplexDenseMatrix::real() const
+{
+   DenseMatrix * Ar = new DenseMatrix(height,width);
+   double * data = Ar->Data();
+   complex<double> * zdata = this->data;
+   for (int s = 0; s<height*width; s++)
+   {
+         data[s] = zdata[s].real();
+   }
+   return Ar;
+}
+DenseMatrix * ComplexDenseMatrix::imag() const
+{
+   DenseMatrix * Ai = new DenseMatrix(height,width);
+   double * data = Ai->Data();
+   complex<double> * zdata = this->data;
+   for (int s = 0; s<height*width; s++)
+   {
+         data[s] = zdata[s].imag();
+   }
+   return Ai;
+}
+
+void ComplexDenseMatrix::GetReal(DenseMatrix & Ar) 
+{
+   MFEM_ASSERT(Ar.Height() == height && Ar.Width() == width, "Incompatible dimensions");
+   double * data = Ar.Data();
+   complex<double> * zdata = this->data;
+   for (int s = 0; s<height*width; s++)
+   {
+         data[s] = zdata[s].real();
+   }
+}
+
+void ComplexDenseMatrix::GetImag(DenseMatrix & Ai) 
+{
+   double * data = Ai.Data();
+   complex<double> * zdata = this->data;
+   for (int s = 0; s<height*width; s++)
+   {
+         data[s] = zdata[s].imag();
+   }
+}
+
+ComplexDenseMatrix &ComplexDenseMatrix::operator=(const ComplexDenseMatrix &m)
+{
+   SetSize(m.height, m.width);
+
+   const int hw = height * width;
+   for (int i = 0; i < hw; i++)
+   {
+      data[i] = m.data[i];
+   }
+   return *this;
+}
+
+ComplexDenseMatrix &ComplexDenseMatrix::operator+=(const complex<double> *m)
+{
+   const int hw = Height()*Width();
+   for (int i = 0; i < hw; i++)
+   {
+      data[i] += m[i];
+   }
+   return *this;
+}
+
+ComplexDenseMatrix &ComplexDenseMatrix::operator+=(const ComplexDenseMatrix &m)
+{
+   MFEM_ASSERT(Height() == m.Height() && Width() == m.Width(),
+               "incompatible matrix sizes.");
+   return *this += m.GetData();
+}
+
+ComplexDenseMatrix &ComplexDenseMatrix::operator-=(const ComplexDenseMatrix &m)
+{
+   int s = Height()*Width();
+   complex<double> * mdata = m.GetData();
+   for (int i = 0; i < s; i++)
+   {
+      data[i] -= mdata[s];
+   }
+   return *this;
+}
+
+ComplexDenseMatrix &ComplexDenseMatrix::operator*=(complex<double> c)
+{
+   int s = Height()*Width();
+   for (int i = 0; i < s; i++)
+   {
+      data[i] *= c;
+   }
+   return *this;
+}
+
+
+void ComplexDenseMatrix::Print(std::ostream &out, int width_) const
+{
+   // save current output flags
+   ios::fmtflags old_flags = out.flags();
+   // output flags = scientific + show sign
+   out << setiosflags(ios::scientific | ios::showpos);
+   for (int i = 0; i < height; i++)
+   {
+      out << "[row " << i << "]\n";
+      for (int j = 0; j < width; j++)
+      {
+         out << (*this)(i,j);
+         if (j+1 == width || (j+1) % width_ == 0)
+         {
+            out << '\n';
+         }
+         else
+         {
+            out << ' ';
+         }
+      }
+   }
+   // reset output flags to original values
+   out.flags(old_flags);
+}
+
+void ComplexDenseMatrix::PrintMatlab(std::ostream &out) const
+{
+   // save current output flags
+   // ios::fmtflags old_flags = out.flags();
+   // output flags = scientific + show sign
+   // out << setiosflags(ios::scientific | ios::showpos);
+   for (int i = 0; i < height; i++)
+   {
+      for (int j = 0; j < width; j++)
+      {
+         out << (*this)(i,j);
+         out << ' ';
+      }
+      out << "\n";
+   }
+   // reset output flags to original values
+   // out.flags(old_flags);
+}
+
+ComplexDenseMatrixInverse::ComplexDenseMatrixInverse(const ComplexDenseMatrix & A) : ComplexDenseMatrix(A.Height())
+{
+   MFEM_VERIFY(A.Height() == A.Width(), "The matrix is not square");
+   MFEM_VERIFY(A.Height() < 4, "dim > 3 is not supported yet");
+
+   std::complex<double> detA = A.Det();
+   MFEM_VERIFY(abs(A.Det())>1e-14, "The given matrix is singular");
+
+   std::complex<double> * d = this->Data();
+   std::complex<double> *dA = A.GetData();
+   switch (A.Height())
+   {
+   case 1:
+      d[0] = 1.0/dA[0];
+      break;
+   case 2:
+      d[0] =  1.0/detA * dA[3];
+      d[1] = -1.0/detA * dA[1];
+      d[2] = -1.0/detA * dA[2];
+      d[3] =  1.0/detA * dA[0];
+      break;   
+   case 3:
+      d[0] =   1.0/detA*(dA[4]*dA[8] - dA[5]*dA[7]);
+      d[1] =  -1.0/detA*(dA[1]*dA[8] - dA[2]*dA[7]);
+      d[2] =   1.0/detA*(dA[1]*dA[5] - dA[2]*dA[4]);
+      d[3] =  -1.0/detA*(dA[3]*dA[8] - dA[5]*dA[6]);
+      d[4] =   1.0/detA*(dA[0]*dA[8] - dA[2]*dA[6]);
+      d[5] =  -1.0/detA*(dA[0]*dA[5] - dA[2]*dA[3]);
+      d[6] =   1.0/detA*(dA[3]*dA[7] - dA[4]*dA[6]);
+      d[7] =  -1.0/detA*(dA[0]*dA[7] - dA[1]*dA[6]);
+      d[8] =   1.0/detA*(dA[0]*dA[4] - dA[1]*dA[3]);
+      break;      
+   default:
+      // Should be unreachable 
+      break;
+   }
+}
+
+
+
+/// Matrix matrix multiplication.  A = B * C.
+void Mult(const ComplexDenseMatrix &b, const ComplexDenseMatrix &c, ComplexDenseMatrix &a)
+{
+   MFEM_ASSERT(a.Height() == b.Height() && a.Width() == c.Width() &&
+               b.Width() == c.Height(), "incompatible dimensions");
+
+   const int ah = a.Height();
+   const int aw = a.Width();
+   const int bw = b.Width();
+   complex<double> *ad = a.Data();
+   const complex<double> *bd = b.Data();
+   const complex<double> *cd = c.Data();
+   kernels::Mult(ah,aw,bw,bd,cd,ad);               
+}
+
+/// Multiply the transpose of a matrix A with a matrix B:   At*B
+void MultAtB(const ComplexDenseMatrix &A, const ComplexDenseMatrix &B, ComplexDenseMatrix &AtB)
+{
+   MFEM_ASSERT(A.Width() == AtB.Height() && B.Width() == AtB.Width() &&
+               A.Height() == B.Height(), "incompatible dimensions");
+   const int ah = A.Height();
+   const int aw = A.Width();
+   const int bw = B.Width();
+   const complex<double> *ad = A.Data();
+   const complex<double> *bd = B.Data();
+   complex<double> *cd = AtB.Data();
+
+   for (int j = 0; j < bw; j++)
+   {
+      const complex<double> *ap = ad;
+      for (int i = 0; i < aw; i++)
+      {
+         complex<double> d = 0.0;
+         for (int k = 0; k < ah; k++)
+         {
+            d += ap[k] * bd[k];
+         }
+         *(cd++) = d;
+         ap += ah;
+      }
+      bd += ah;
+   }            
+}
+
+/// Multiply the conjugate transpose of a matrix A with a matrix B:   At*B
+void MultAhB(const ComplexDenseMatrix &A, const ComplexDenseMatrix &B, ComplexDenseMatrix &AtB)
+{
+   MFEM_ASSERT(A.Width() == AtB.Height() && B.Width() == AtB.Width() &&
+               A.Height() == B.Height(), "incompatible dimensions");
+      MFEM_ASSERT(A.Width() == AtB.Height() && B.Width() == AtB.Width() &&
+               A.Height() == B.Height(), "incompatible dimensions");
+   const int ah = A.Height();
+   const int aw = A.Width();
+   const int bw = B.Width();
+   const complex<double> *ad = A.Data();
+   const complex<double> *bd = B.Data();
+   complex<double> *cd = AtB.Data();
+
+   for (int j = 0; j < bw; j++)
+   {
+      const complex<double> *ap = ad;
+      for (int i = 0; i < aw; i++)
+      {
+         complex<double> d = 0.0;
+         for (int k = 0; k < ah; k++)
+         {
+            d += conj(ap[k]) * bd[k];
+         }
+         *(cd++) = d;
+         ap += ah;
+      }
+      bd += ah;
+   }                    
+
+}

examples/maxwell-solver/common/complex_linalg.hpp

@@ -0,0 +1,98 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class ComplexDenseMatrix 
+{
+private:
+   std::complex<double> * data = nullptr;
+   int height = 0;
+   int width  = 0;
+public:
+   ComplexDenseMatrix();
+   
+   /// Creates square matrix of size s.
+   explicit ComplexDenseMatrix(int s);
+
+   /// Creates rectangular matrix of size m x n.
+   ComplexDenseMatrix(int m, int n);
+
+   /// Change the size of the DenseMatrix to s x s.
+   void SetSize(int s) { SetSize(s, s); }
+
+   /// Change the size of the DenseMatrix to h x w.
+   void SetSize(int h, int w);
+
+   /// Returns the matrix data array.
+   inline complex<double> *Data() const
+   { return const_cast<complex<double>*>((const complex<double>*)data);}
+
+   /// Returns the matrix data array.
+   inline complex<double> *GetData() const { return Data(); }
+
+   /// Returns reference to a_{ij}.
+   inline complex<double> &operator()(int i, int j);
+   inline const complex<double> &operator()(int i, int j) const;
+
+   inline int Height() const { return height; }
+   inline int Width() const { return width; }
+
+   /// Sets the matrix elements equal to constant c
+   ComplexDenseMatrix &operator=(std::complex<double> c);
+   ComplexDenseMatrix &operator=(double c);
+
+   /// Sets the matrix size and elements equal to those of m
+   ComplexDenseMatrix &operator=(const ComplexDenseMatrix &m);
+   ComplexDenseMatrix &operator+=(const complex<double> *m);
+   ComplexDenseMatrix &operator+=(const ComplexDenseMatrix &m);
+   ComplexDenseMatrix &operator-=(const ComplexDenseMatrix &m);
+   ComplexDenseMatrix &operator*=(complex<double> c);
+
+   /// Calculates the determinant of the matrix
+   /// (for 2x2, 3x3)
+   std::complex<double> Det() const;
+
+   virtual void Print(std::ostream &out = mfem::out, int width_ = 4) const;
+   virtual void PrintMatlab(std::ostream &out = mfem::out) const;
+
+   DenseMatrix * real() const;
+   DenseMatrix * imag() const;
+
+   void GetReal(DenseMatrix & Ar);
+   void GetImag(DenseMatrix & Ai);
+};
+
+inline complex<double> &ComplexDenseMatrix::operator()(int i, int j)
+{
+   MFEM_VERIFY(data && i >= 0 && i < height && j >= 0 && j < width, "");
+   // return data[i*width+j];
+   return data[j*height+i];
+}
+
+inline const complex<double> &ComplexDenseMatrix::operator()(int i, int j) const
+{
+   MFEM_VERIFY(data && i >= 0 && i < height && j >= 0 && j < width, "");
+   // return data[i*width+j];
+   return data[j*height+i];
+}
+
+
+class ComplexDenseMatrixInverse : public ComplexDenseMatrix
+{
+private:
+public:
+   ComplexDenseMatrixInverse(const ComplexDenseMatrix & );   
+};
+
+/// Matrix matrix multiplication.  A = B * C.
+void Mult(const ComplexDenseMatrix &b, const ComplexDenseMatrix &c, ComplexDenseMatrix &a);
+
+/// Multiply the transpose of a matrix A with a matrix B:   At*B
+void MultAtB(const ComplexDenseMatrix &A, const ComplexDenseMatrix &B, ComplexDenseMatrix &AtB);
+
+/// Multiply the conjugate transpose of a matrix A with a matrix B:   At*B
+void MultAhB(const ComplexDenseMatrix &A, const ComplexDenseMatrix &B, ComplexDenseMatrix &AtB);

examples/maxwell-solver/ex22p-DST.cpp

@@ -0,0 +1,456 @@
+//                       MFEM Example 22 - Parallel Version
+//
+// Compile with: make ex22p
+//
+//               mpirun -np 4 ex22p -m ../../data/inline-quad.mesh -o 3 
+//               mpirun -np 4 ex22p -m ../../data/inline-hex.mesh -o 2
+//               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
+//
+// Description:  This example code demonstrates the use of MFEM to define and
+//               solve simple complex-valued linear systems. It implements three
+//               variants of a damped harmonic oscillator:
+//
+//                A vector H(Curl) field
+//                Curl(a Curl u) - omega^2 b u + i omega c u = 0
+//
+//               In each case the field is driven by a forced oscillation, with
+//               angular frequency omega, imposed at the boundary or a portion
+//               of the boundary.
+//
+//               In electromagnetics the coefficients are typically named the
+//               permeability, mu = 1/a, permittivity, epsilon = b, and
+//               conductivity, sigma = c. The user can specify these constants
+//               using either set of names.
+//
+//               The example also demonstrates how to display a time-varying
+//               solution as a sequence of fields sent to a single GLVis socket.
+//
+//               We recommend viewing examples 1, 3 and 4 before viewing this
+//               example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "ParDST/ParDST.hpp"
+
+using namespace std;
+using namespace mfem;
+
+static double mu_ = 1.0;
+static double epsilon_ = 1.0;
+static double sigma_ = 20.0;
+static double omega_ = 10.0;
+
+double wavespeed(const Vector &x)
+{
+   double ws;
+   ws = 1.0;
+   return ws;
+}
+
+double u0_real_exact(const Vector &);
+double u0_imag_exact(const Vector &);
+
+void u1_real_exact(const Vector &, Vector &);
+void u1_imag_exact(const Vector &, Vector &);
+
+void u2_real_exact(const Vector &, Vector &);
+void u2_imag_exact(const Vector &, Vector &);
+
+bool check_for_inline_mesh(const char * mesh_file);
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/inline-quad.mesh";
+   int ser_ref_levels = 1;
+   int par_ref_levels = 1;
+   int order = 1;
+   double freq = -1.0;
+   double a_coef = 0.0;
+   bool visualization = 1;
+   bool herm_conv = true;
+   bool exact_sol = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&a_coef, "-a", "--stiffness-coef",
+                  "Stiffness coefficient (spring constant or 1/mu).");
+   args.AddOption(&epsilon_, "-b", "--mass-coef",
+                  "Mass coefficient (or epsilon).");
+   args.AddOption(&sigma_, "-c", "--damping-coef",
+                  "Damping coefficient (or sigma).");
+   args.AddOption(&mu_, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon_, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&sigma_, "-sigma", "--conductivity",
+                  "Conductivity (or damping constant).");
+   args.AddOption(&freq, "-f", "--frequency",
+                  "Frequency (in Hz).");
+   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
+                  "--no-hermitian", "Use convention for Hermitian operators.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   if ( a_coef != 0.0 )
+   {
+      mu_ = 1.0 / a_coef;
+   }
+   if ( freq > 0.0 )
+   {
+      omega_ = 2.0 * M_PI * freq;
+   }
+
+   exact_sol = check_for_inline_mesh(mesh_file);
+   if (myid == 0 && exact_sol)
+   {
+      cout << "Identified a mesh with known exact solution" << endl;
+   }
+
+   ComplexOperator::Convention conv =
+      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors. We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 5. Refine the serial mesh on all processors to increase the resolution.
+   for (int l = 0; l < ser_ref_levels; l++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   for (int l = 0; l < par_ref_levels; l++)
+   {
+      pmesh->UniformRefinement();
+   }
+
+   FiniteElementCollection *fec = new ND_FECollection(order, dim); 
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 8. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    based on the type of mesh and the problem type.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (pmesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 9. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system.
+   ParComplexLinearForm b(fespace, conv);
+   b.Vector::operator=(0.0);
+
+   // 10. Define the solution vector u as a parallel complex finite element grid
+   //     function corresponding to fespace. Initialize u with initial guess of
+   //     1+0i or the exact solution if it is known.
+   ParComplexGridFunction u(fespace);
+   ParComplexGridFunction * u_exact = NULL;
+   if (exact_sol) { u_exact = new ParComplexGridFunction(fespace); }
+
+   FunctionCoefficient u0_r(u0_real_exact);
+   FunctionCoefficient u0_i(u0_imag_exact);
+   VectorFunctionCoefficient u1_r(dim, u1_real_exact);
+   VectorFunctionCoefficient u1_i(dim, u1_imag_exact);
+   VectorFunctionCoefficient u2_r(dim, u2_real_exact);
+   VectorFunctionCoefficient u2_i(dim, u2_imag_exact);
+
+   ConstantCoefficient zeroCoef(0.0);
+   ConstantCoefficient oneCoef(1.0);
+
+   Vector zeroVec(dim); zeroVec = 0.0;
+   Vector  oneVec(dim);  oneVec = 0.0; oneVec[0] = 1.0;
+   VectorConstantCoefficient zeroVecCoef(zeroVec);
+   VectorConstantCoefficient oneVecCoef(oneVec);
+
+   if (exact_sol)
+   {
+      u.ProjectBdrCoefficientTangent(u1_r, u1_i, ess_bdr);
+      u_exact->ProjectCoefficient(u1_r, u1_i);
+   }
+   else
+   {
+      u.ProjectBdrCoefficientTangent(oneVecCoef, zeroVecCoef, ess_bdr);
+   }
+
+   if (visualization && exact_sol)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock_r(vishost, visport);
+      sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_r.precision(8);
+      sol_sock_r << "solution\n" << *pmesh << u_exact->real()
+                 << "window_title 'Exact: Real Part'" << flush;
+      MPI_Barrier(MPI_COMM_WORLD); // try to prevent streams from mixing
+
+      socketstream sol_sock_i(vishost, visport);
+      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_i.precision(8);
+      sol_sock_i << "solution\n" << *pmesh << u_exact->imag()
+                 << "window_title 'Exact: Imaginary Part'" << flush;
+      MPI_Barrier(MPI_COMM_WORLD); // try to prevent streams from mixing
+
+   }
+
+   // 11. Set up the parallel sesquilinear form a(.,.) on the finite element
+   //     space corresponding to the damped harmonic oscillator operator of the
+   //     appropriate type:
+   //
+   //     A vector H(Curl) field
+   //     Curl(a Curl) - omega^2 b + i omega c
+   //
+   ConstantCoefficient stiffnessCoef(1.0/mu_);
+   ConstantCoefficient massCoef(-omega_ * omega_ * epsilon_);
+   ConstantCoefficient lossCoef(omega_ * sigma_);
+   ConstantCoefficient negMassCoef(omega_ * omega_ * epsilon_);
+
+   ParSesquilinearForm *a = new ParSesquilinearForm(fespace, conv);
+   a->AddDomainIntegrator(new CurlCurlIntegrator(stiffnessCoef),
+                           NULL);
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef),
+                          new VectorFEMassIntegrator(lossCoef));
+   a->Assemble();
+
+   OperatorHandle Ah;
+   Vector B, U;
+
+   a->FormLinearSystem(ess_tdof_list, u, b, Ah, U, B);
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: "
+           << 2 * fespace->GlobalTrueVSize() << endl << endl;
+   }
+
+   int nrlayers = 5;
+   Array2D<double> lengths(dim,2);
+   lengths = 0.0;
+
+   FunctionCoefficient ws(wavespeed);
+   int nx = 1;
+   int ny = 4;
+   int nz = 4;
+   ParDST * S = new ParDST(a,lengths, omega_, &ws, nrlayers, nx, ny, nz, &lossCoef);
+   // X = 0.0;
+	GMRESSolver gmres(MPI_COMM_WORLD);
+	gmres.iterative_mode = true;
+   gmres.SetPreconditioner(*S);
+	gmres.SetOperator(*Ah);
+	gmres.SetRelTol(1e-8);
+	gmres.SetMaxIter(100);
+	gmres.SetPrintLevel(1);
+	gmres.Mult(B, U);
+   delete S;
+
+
+   // HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+   // MUMPSSolver mumps;
+   // mumps.SetPrintLevel(0);
+   // mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
+   // mumps.SetOperator(*A);
+   // mumps.Mult(B,U);
+
+   a->RecoverFEMSolution(U, b, u);
+
+   if (exact_sol)
+   {
+      double err_r = -1.0;
+      double err_i = -1.0;
+
+      err_r = u.real().ComputeL2Error(u1_r);
+      err_i = u.imag().ComputeL2Error(u1_i);
+      if ( myid == 0 )
+      {
+         cout << endl;
+         cout << "|| Re (u_h - u) ||_{L^2} = " << err_r << endl;
+         cout << "|| Im (u_h - u) ||_{L^2} = " << err_i << endl;
+         cout << endl;
+      }
+   }
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock_r(vishost, visport);
+      sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_r.precision(8);
+      sol_sock_r << "solution\n" << *pmesh << u.real()
+                 << "window_title 'Solution: Real Part'" << flush;
+      MPI_Barrier(MPI_COMM_WORLD); // try to prevent streams from mixing
+
+      socketstream sol_sock_i(vishost, visport);
+      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_i.precision(8);
+      sol_sock_i << "solution\n" << *pmesh << u.imag()
+                 << "window_title 'Solution: Imaginary Part'" << flush;
+      MPI_Barrier(MPI_COMM_WORLD); // try to prevent streams from mixing
+
+   }
+   if (visualization && exact_sol)
+   {
+      *u_exact -= u;
+
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock_r(vishost, visport);
+      sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_r.precision(8);
+      sol_sock_r << "solution\n" << *pmesh << u_exact->real()
+                 << "window_title 'Error: Real Part'" << flush;
+      MPI_Barrier(MPI_COMM_WORLD); // try to prevent streams from mixing
+
+      socketstream sol_sock_i(vishost, visport);
+      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_i.precision(8);
+      sol_sock_i << "solution\n" << *pmesh << u_exact->imag()
+                 << "window_title 'Error: Imaginary Part'" << flush;
+      MPI_Barrier(MPI_COMM_WORLD); // try to prevent streams from mixing
+
+   }
+   if (visualization)
+   {
+      ParGridFunction u_t(fespace);
+      u_t = u.real();
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << u_t 
+               << "valuerange -0.5 0.5 \n" 
+               << "autoscale off \n"  
+               << "window_title 'Harmonic Solution (t = 0.0 T)'"
+               << "pause\n" << flush;
+
+      if (myid == 0)
+         cout << "GLVis visualization paused."
+              << " Press space (in the GLVis window) to resume it.\n";
+      int num_frames = 32;
+      int i = 0;
+      while (sol_sock)
+      {
+         double t = (double)(i % num_frames) / num_frames;
+         ostringstream oss;
+         oss << "Harmonic Solution (t = " << t << " T)";
+
+         add(cos( 2.0 * M_PI * t), u.real(),
+             sin(-2.0 * M_PI * t), u.imag(), u_t);
+         sol_sock << "parallel " << num_procs << " " << myid << "\n";
+         sol_sock << "solution\n" << *pmesh << u_t
+                  << "valuerange -0.5 0.5 \n" 
+                  << "autoscale off \n"  
+                  << "window_title '" << oss.str() << "'" << flush;
+         i++;
+      }
+   }
+
+   // 17. Free the used memory.
+   delete a;
+   delete u_exact;
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+bool check_for_inline_mesh(const char * mesh_file)
+{
+   string file(mesh_file);
+   size_t p0 = file.find_last_of("/");
+   string s0 = file.substr((p0==string::npos)?0:(p0+1),7);
+   return s0 == "inline-";
+}
+
+complex<double> u0_exact(const Vector &x)
+{
+   int dim = x.Size();
+   complex<double> i(0.0, 1.0);
+   complex<double> alpha = (epsilon_ * omega_ - i * sigma_);
+   complex<double> kappa = std::sqrt(mu_ * omega_* alpha);
+   return std::exp(-i * kappa * x[dim - 1]);
+}
+
+double u0_real_exact(const Vector &x)
+{
+   return u0_exact(x).real();
+}
+
+double u0_imag_exact(const Vector &x)
+{
+   return u0_exact(x).imag();
+}
+
+void u1_real_exact(const Vector &x, Vector &v)
+{
+   int dim = x.Size();
+   v.SetSize(dim); v = 0.0; v[0] = u0_real_exact(x);
+}
+
+void u1_imag_exact(const Vector &x, Vector &v)
+{
+   int dim = x.Size();
+   v.SetSize(dim); v = 0.0; v[0] = u0_imag_exact(x);
+}
+
+void u2_real_exact(const Vector &x, Vector &v)
+{
+   int dim = x.Size();
+   v.SetSize(dim); v = 0.0; v[dim-1] = u0_real_exact(x);
+}
+
+void u2_imag_exact(const Vector &x, Vector &v)
+{
+   int dim = x.Size();
+   v.SetSize(dim); v = 0.0; v[dim-1] = u0_imag_exact(x);
+}

examples/maxwell-solver/helmholtz.cpp

@@ -0,0 +1,534 @@
+//
+// Compile with: make helmholtz
+//
+// Sample runs:  helmholtz -m ../data/one-hex.mesh
+//               helmholtz -m ../data/fichera.mesh
+//               helmholtz -m ../data/fichera-mixed.mesh
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Helmholtz problem
+//               -Delta p - omega^2 p = 1 with impedance boundary condition.
+//
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "DST/DST.hpp"
+#include "DST2D/DST2D.hpp"
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution and r.h.s., see below for implementation.
+double f_exact_Re(const Vector &x);
+double f_exact_Im(const Vector &x);
+
+double wavespeed(const Vector &x);
+
+
+int dim;
+double omega;
+int sol = 1;
+double length = 1.0;
+double pml_length = 0.25;
+Array2D<double>comp_bdr;
+
+#ifndef MFEM_USE_SUPERLU
+#error This example requires that MFEM is built with MFEM_USE_PETSC=YES
+#endif
+
+int main(int argc, char *argv[])
+{
+
+   // 2. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../../data/one-hex.mesh";
+   // finite element order of approximation
+   int order = 1;
+   bool visualization = 1;
+   // number of wavelengths
+   double k = 0.5;
+   // number of mg levels
+   int ref = 1;
+   // dimension
+   int nd = 2;
+
+   int nx=2;
+   int ny=2;
+   int nz=2;
+   bool herm_conv = true;
+
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nd, "-nd", "--dim","Problem space dimension");
+   args.AddOption(&nx, "-nx", "--nx","Number of subdomains in x direction");
+   args.AddOption(&ny, "-ny", "--ny","Number of subdomains in y direction");
+   args.AddOption(&nz, "-nz", "--nz","Number of subdomains in z direction");
+   args.AddOption(&sol, "-sol", "--exact",
+                  "Exact solution flag - 0:polynomial, 1: plane wave, -1: unknown exact");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&pml_length, "-pml_length", "--pml_length",
+                  "Length of the PML region in each direction");
+   args.AddOption(&length, "-length", "--length",
+                  "length of the domain in each direction.");
+   args.AddOption(&ref, "-ref", "--ref",
+                  "Number of Refinements.");
+   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
+                  "--no-hermitian", "Use convention for Hermitian operators.");                  
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+		args.PrintUsage(cout);
+		return 1;
+   }
+	args.PrintOptions(cout);
+   // Angular frequency
+   omega = 2.0 * M_PI * k;
+
+   // 3. Read the mesh from the given mesh file.
+   Mesh *mesh;
+
+   if (nd == 2)
+   {
+      // mesh = new Mesh(mesh_file,1,1);
+      mesh = new Mesh(1, 1, Element::QUADRILATERAL, true, length, length, false);
+   }
+   else
+   {
+      mesh = new Mesh(1, 1, 1, Element::HEXAHEDRON, true, length, length, length,false);
+   }
+
+   // 3. Executing uniform h-refinement
+   for (int i = 0; i < ref; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+   dim = mesh->Dimension();
+
+   double hl = GetUniformMeshElementSize(mesh);
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin,pmax);
+   // double domain_length = pmax[0] - pmin[0];
+   // double pml_thickness = 0.125/domain_length;
+   // int nrlayers = pml_thickness/hl;
+   int nrlayers = 7;
+   Array<int> directions;
+   
+   for (int i = 0; i<nrlayers; i++)
+   {
+      for (int comp=0; comp<dim; ++comp)
+      {
+         // directions.Append(comp+1);
+         // directions.Append(-comp-1);
+      }
+   }
+
+   // Find uniform h size of the original mesh
+   // cout << "pml layers = " << nrlayers << endl;
+   // cout << "pml length = " << hl*nrlayers << endl;
+   Mesh *mesh_ext = ExtendMesh(mesh,directions);
+
+
+   // if (visualization)
+   // {
+   //    char vishost[] = "localhost";
+   //    int  visport   = 19916;
+   //    socketstream mesh_sock(vishost, visport);
+   //    mesh_sock.precision(8);
+   //    mesh_sock << "mesh\n" << *mesh_ext << flush;
+   // }
+
+   // char vishost[] = "localhost";
+   // int  visport   = 19916;
+   // socketstream mesh_sock(vishost, visport);
+   // mesh_sock.precision(8);
+   // mesh_sock << "mesh\n" << *mesh_ext << flush;
+
+   // cin.get();
+
+   Array2D<double> lengths(dim,2);
+   lengths = hl*nrlayers;
+   // lengths[0][1] = 0.0;
+   // lengths[1][1] = 0.0;
+   // lengths[1][0] = 0.0;
+   // lengths[0][0] = 0.0;
+   CartesianPML pml(mesh_ext,lengths);
+   pml.SetOmega(omega);
+   comp_bdr.SetSize(dim,2);
+   comp_bdr = pml.GetCompDomainBdr(); 
+
+   // 6. Define a finite element space on the mesh.
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh_ext, fec);
+
+   // 6. Set up the linear form (Real and Imaginary part)
+   FunctionCoefficient f_Re(f_exact_Re);
+   FunctionCoefficient f_Im(f_exact_Im);
+
+
+   // 8. Setup Complex Operator convention
+   ComplexOperator::Convention conv =
+      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
+
+   // ParLinearForm *b_Re(new ParLinearForm);
+   ComplexLinearForm b(fespace, conv);
+   b.AddDomainIntegrator(new DomainLFIntegrator(f_Re),
+                         new DomainLFIntegrator(f_Im));
+   b.real().Vector::operator=(0.0);
+   b.imag().Vector::operator=(0.0);
+   b.Assemble();
+
+   // 7. Set up the bilinear form (Real and Imaginary part)
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+   FunctionCoefficient ws(wavespeed);
+
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+
+   ProductCoefficient c2_re(c2_re0, ws);
+   ProductCoefficient c2_im(c2_im0, ws);
+
+   SesquilinearForm a(fespace,conv);
+
+   a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   a.AddDomainIntegrator(new MassIntegrator(c2_re),new MassIntegrator(c2_im));
+
+   a.Assemble();
+   a.Finalize();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(mesh_ext->bdr_attributes.Max());
+   ess_bdr = 1;
+   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   // Solution grid function
+   ComplexGridFunction p_gf(fespace); p_gf = 0.0;
+   OperatorHandle Ah;
+   Vector X, B;
+
+   a.FormLinearSystem(ess_tdof_list, p_gf, b, Ah, X, B);
+
+   ComplexSparseMatrix * AZ = Ah.As<ComplexSparseMatrix>();
+   SparseMatrix * A = AZ->GetSystemMatrix();
+
+   cout << "Size of fine grid system: "
+         << A->Height() << " x " << A->Width() << endl;
+
+   // StopWatch chrono;
+   // chrono.Clear();
+   // chrono.Start();
+   // DST S(&a,lengths, omega, &ws, nrlayers, nx, ny, nz);
+   // chrono.Stop();
+   // cout << "Construction time: " << chrono.RealTime() << endl; 
+
+
+   // chrono.Clear();
+   // chrono.Start();
+   // X = 0.0;
+	// GMRESSolver gmres;
+	// // gmres.iterative_mode = true;
+   // gmres.SetPreconditioner(S);
+	// gmres.SetOperator(*AZ);
+	// gmres.SetRelTol(1e-6);
+	// gmres.SetMaxIter(20);
+	// gmres.SetPrintLevel(1);
+	// gmres.Mult(B, X);
+
+
+   // DST2D S2D(&a,lengths, omega, &ws, nrlayers);
+   // X = 0.0;
+	// gmres.SetPreconditioner(S2D);
+	// gmres.Mult(B, X);
+
+   // chrono.Stop();
+   // cout << "GMRES time: " << chrono.RealTime() << endl; 
+
+   // X = 0.0;
+   // SLISolver sli;
+   // sli.iterative_mode = true;
+   // sli.SetPreconditioner(S);
+   // sli.SetOperator(*A);
+   // sli.SetRelTol(1e-6);
+   // sli.SetMaxIter(50);
+   // sli.SetPrintLevel(1);
+   // sli.Mult(B,X);
+
+   // int n= 200;
+   // X = 0.0;
+   // Vector z(X.Size()); z = 0.0;
+   // Vector r(B);
+   // Vector ztemp(r.Size());
+   // Vector Ax(X.Size());
+   // double tol = 1e-10;
+   // cout << endl;
+   // chrono.Clear();
+   // chrono.Start();
+   // for (int i = 0; i<n; i++)
+   // {
+   //    A->Mult(X,Ax); Ax *=-1.0;
+   //    r = b; r+=Ax;
+   //    cout << "   ST Solver   Iteration :   " << i <<"  || r || = " <<  r.Norml2() << endl;
+   //    if (r.Norml2() < tol) 
+   //    {
+   //       cout << "Convergence in " << i << " iterations" << endl;
+   //       break;
+   //    }
+   //    S1.Mult(r,z); 
+   //    X += z;
+
+   //    // X1-=z;
+   //    // p_gf = 0.0;
+   //    // a.RecoverFEMSolution(X,B,p_gf);
+   //    //    char vishost[] = "localhost";
+   //    //    int  visport   = 19916;
+   //    //    string keys;
+   //    //    if (dim ==2 )
+   //    //    {
+   //    //       keys = "keys mrRljc\n";
+   //    //    }
+   //    //    else
+   //    //    {
+   //    //       keys = "keys mc\n";
+   //    //    }
+   //    //    socketstream sol1_sock_re(vishost, visport);
+   //    //    sol1_sock_re.precision(8);
+   //    //    sol1_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+   //    //                "window_title 'Numerical Pressure (real part)' "
+   //    //                << keys << flush;
+   //    //    cin.get();
+   // }
+
+   // chrono.Stop();
+   // cout << "Solver time: " << chrono.RealTime() << endl; 
+
+   // a.RecoverFEMSolution(X,B,p_gf);
+
+   // chrono.Clear();
+   // chrono.Start();
+   ComplexUMFPackSolver csolver;
+   csolver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   csolver.SetOperator(*AZ);
+   Vector X1(X.Size());
+   csolver.Mult(B,X);
+   // chrono.Stop();
+   // cout << "UMFPack time: " << chrono.RealTime() << endl; 
+   // X1-= X;
+
+   // ComplexGridFunction error_gf(fespace);
+
+   a.RecoverFEMSolution(X,B,p_gf);
+   // a.RecoverFEMSolution(X1,B,error_gf);
+
+
+   // cout << "error l2 norm = " << error_gf.Norml2() << endl;
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      string keys;
+      if (dim ==2 )
+      {
+         keys = "keys mrRljc\n";
+      }
+      else
+      {
+         keys = "keys mc\n";
+      }
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re.precision(8);
+      sol_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+                  "window_title 'Numerical Pressure (real part from DST)' "
+                  // << keys << flush;
+                  << keys << "valuerange -0.08 0.08 \n" << flush;
+      // socketstream err_sock_re(vishost, visport);
+      // err_sock_re.precision(8);
+      // err_sock_re << "solution\n" << *mesh_ext << error_gf.real() <<
+      //             "window_title 'Difference (real part from UMFPACK)' "
+      //             << keys << flush;
+   }
+   delete fespace;
+   delete fec;
+	delete mesh_ext;
+	delete mesh;
+   return 0;
+}
+
+
+//calculate RHS from exact solution f = - \Delta u
+double f_exact_Re(const Vector &x)
+{
+   double f_re = 0.0;
+   double x0 = length/2.0;
+   double x1 = length/2.0;
+   double x2 = length/2.0;
+   // x0 = 0.59;
+   // x0 = 0.19;
+   x0 = 0.1;
+   // x1 = 0.768;
+   // x1 = 0.168;
+   x1 = 0.35;
+   x2 = 0.25;
+   double alpha,beta;
+   // double n = 5.0*omega/M_PI;
+   double n = 4.0*omega/M_PI;
+   // double n = 1.0;
+   // double coeff = pow(n,2)/M_PI;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   // alpha = -pow(n,2) * beta;
+   // double coeff = pow(n,2)/M_PI;
+   double coeff = 16.0*omega*omega/M_PI/M_PI/M_PI;
+   alpha = -pow(n,2) * beta;
+   f_re = coeff*exp(alpha);
+
+   // x0 = 0.85;
+   // x1 = 0.15;
+   // beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   // // if (dim == 3) { beta += pow(x2-x(2),2); }
+   // alpha = -pow(n,2) * beta;
+   // f_re += coeff*exp(alpha);
+
+   x0 = 0.8;
+   x1 = 0.7;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   alpha = -pow(n,2) * beta;
+   f_re += coeff*exp(alpha);
+
+   bool in_pml = false;
+   for (int i = 0; i<dim; i++)
+   {
+      if (x(i)<=comp_bdr(i,0) || x(i)>=comp_bdr(i,1))
+      {
+         in_pml = true;
+         break;
+      }
+   }
+   if (in_pml) f_re = 0.0;
+
+   return f_re;
+
+}
+double f_exact_Im(const Vector &x)
+{
+   double f_im;
+   f_im = 0.0;
+   return f_im;
+}
+
+double wavespeed(const Vector &x)
+{
+   double ws;
+   // if (x(0) <= 0.25)
+   // {
+   //    ws = 1.0;
+   // }
+   // else if(x(0)<=0.5)
+   // {
+   //    ws = 1.0;
+   // }
+   // else if(x(0)<=0.75)
+   // {
+   //    ws = 0.75;
+   //    // ws = 0.5;
+   // }
+   // else 
+   // {
+   //    ws = 0.75;
+   //    // ws = 1.0;
+   // }
+   // if (x(1) <= 1.0/3.0)
+   // {
+   //    ws = 2.0;
+   // }
+   // else if(x(1)<=2.0/3.0)
+   // {
+   //    ws = 1.0;
+   // }
+   // else 
+   // {
+   //    // ws = 0.75;
+   //    ws = 0.25;
+   // }
+
+   // if (x(0) <= 0.33)
+   // {
+   //    ws = 1.0;
+   // }
+   // else if(x(0)<=0.66)
+   // {
+   //    ws = -0.65 + 5.0*x(0);
+   // }
+   // else 
+   // {
+   //    ws = 2.65;
+   //    // ws = 0.5;
+   // }
+
+   if (x(0) <= x(1) && x(1) >= 1.0-x(0))
+   {
+      ws = 1.0;
+   }
+   else if (x(0) > x(1) && x(1) >= 1.0-x(0))
+   {
+      ws = 3.0;
+   }
+   else if (x(0) <= x(1) && x(1) < 1.0-x(0))
+   {
+      ws = 2.0;
+   }
+   else
+   {
+      ws = 4.0;
+   }
+   
+
+
+   // ws = 1.0;
+   return ws;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

examples/maxwell-solver/helmholtzp.cpp

@@ -0,0 +1,654 @@
+//
+// Compile with: make helmholtz
+//
+// Sample runs:  helmholtz -m ../data/one-hex.mesh
+//               helmholtz -m ../data/fichera.mesh
+//               helmholtz -m ../data/fichera-mixed.mesh
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Helmholtz problem
+//               -Delta p - omega^2 p = 1 with impedance boundary condition.
+//
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "ParDST/ParDST.hpp"
+#include "common/PML.hpp"
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution and r.h.s., see below for implementation.
+double f_exact_Re(const Vector &x);
+double f_exact_Im(const Vector &x);
+
+double wavespeed(const Vector &x);
+
+double funccoeff_re(const Vector & x);
+double funccoeff_im(const Vector & x);
+
+
+int dim;
+double omega;
+int sol = 1;
+double length = 1.0;
+double pml_length = 0.25;
+Array2D<double>comp_bdr;
+
+// #ifndef MFEM_USE_SUPERLU
+// #error This example requires that MFEM is built with MFEM_USE_PETSC=YES
+// #endif
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+   // 2. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../../data/one-hex.mesh";
+   // finite element order of approximation
+   int order = 1;
+   int bc_type = 1;
+   bool visualization = 1;
+   // number of wavelengths
+   double k = 0.5;
+   // number of serial refinements
+   int ser_ref_levels = 1;
+   // number of parallel refinements
+   int par_ref_levels = 2;
+   // dimension
+   int nd = 2;
+   int nx=2;
+   int ny=2;
+   int nz=2;
+   bool herm_conv = true;
+
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nd, "-nd", "--dim","Problem space dimension");
+   args.AddOption(&nx, "-nx", "--nx","Number of subdomains in x direction");
+   args.AddOption(&ny, "-ny", "--ny","Number of subdomains in y direction");
+   args.AddOption(&nz, "-nz", "--nz","Number of subdomains in z direction");
+   args.AddOption(&sol, "-sol", "--exact",
+                  "Exact solution flag - 0:polynomial, 1: plane wave, -1: unknown exact");
+   args.AddOption(&bc_type, "-bct", "--bc-type",
+                  "BC type - 0:Neumann, 1: Dirichlet");                  
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&pml_length, "-pml_length", "--pml_length",
+                  "Length of the PML region in each direction");
+   args.AddOption(&length, "-length", "--length",
+                  "length of the domain in each direction.");
+   args.AddOption(&ser_ref_levels, "-sr", "--ser_ref_levels",
+                  "Number of Serial Refinements.");
+   args.AddOption(&par_ref_levels, "-pr", "--par_ref_levels",
+                  "Number of Parallel Refinements.");                  
+   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
+                  "--no-hermitian", "Use convention for Hermitian operators.");                  
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+   // Angular frequency
+   omega = 2.0 * M_PI * k;
+
+   // 3. Read the mesh from the given mesh file.
+   Mesh *mesh;
+
+   if (nd == 2)
+   {
+      // mesh = new Mesh(mesh_file,1,1);
+      mesh = new Mesh(1, 1, Element::QUADRILATERAL, true, length, length, false);
+   }
+   else
+   {
+      mesh = new Mesh(1, 1, 1, Element::HEXAHEDRON, true, length, length, length,false);
+   }
+
+   // 3. Executing uniform h-refinement
+   dim = mesh->Dimension();
+   for (int i = 0; i < ser_ref_levels; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+
+
+
+   // 4. Define a parallel mesh by a partitioning of the serial mesh.
+   // ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   int nprocs;
+   int nprocsx;
+   int nprocsy;
+   int nprocsz;
+   if (dim == 2)
+   {
+      nprocs = sqrt(num_procs); 
+      // nprocsx = nprocs;
+      // nprocsy = nprocs;
+      nprocsx = 1;
+      nprocsy = num_procs;
+      nprocsz = 1;
+   }    
+   else
+   {
+      nprocs = cbrt(num_procs); 
+      // nprocsx = nprocs;
+      // nprocsy = nprocs;
+      // nprocsz = nprocs;
+      nprocsx = 1;
+      if (nz != 1)
+      {
+         nprocsy = sqrt(num_procs);
+         nprocsz = nprocsy;
+      }
+      else
+      {
+         nprocsy = num_procs;
+         nprocsz = 1;
+      }
+   }
+   // MFEM_VERIFY(nprocs*nprocs == num_procs, "Check MPI partitioning");
+   // int nxyz[3] = {num_procs,1,1};
+   // int nxyz[3] = {nprocs,nprocs,1};
+   // int nxyz[3] = {1,num_procs,1};
+   
+   int nxyz[3] = {nprocsx,nprocsy,nprocsz};
+   // int nxyz[3] = {num_procs,1,1};
+   int * part = mesh->CartesianPartitioning(nxyz);
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD,*mesh,part);
+   // ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD,*mesh);
+   delete [] part;
+   delete mesh;
+
+   for (int l = 0; l < par_ref_levels; l++)
+   {
+      pmesh->UniformRefinement();
+   }
+
+
+   double hl = GetUniformMeshElementSize(pmesh);
+   int nrlayers = 4;
+
+   Array2D<double> lengths(dim,2);
+   lengths = hl*nrlayers;
+   // lengths[0][1] = 0.0;
+   // lengths[1][1] = 0.0;
+   // lengths[1][0] = 0.0;
+   // lengths[0][0] = 0.0;
+   // CartesianPML pml(mesh,lengths);
+   CartesianPML pml(pmesh,lengths);
+   pml.SetOmega(omega);
+   comp_bdr.SetSize(dim,2);
+   comp_bdr = pml.GetCompDomainBdr(); 
+
+   int basis = BasisType::GetType('G');
+
+   // 6. Define a finite element space on the mesh.
+   FiniteElementCollection *fec = new H1_FECollection(order, dim,basis);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+   // 6. Set up the linear form (Real and Imaginary part)
+   FunctionCoefficient f_Re(f_exact_Re);
+   FunctionCoefficient f_Im(f_exact_Im);
+
+   // 8. Setup Complex Operator convention
+   ComplexOperator::Convention conv =
+      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
+
+   // ParLinearForm *b_Re(new ParLinearForm);
+   ParComplexLinearForm b(fespace, conv);
+   b.AddDomainIntegrator(new DomainLFIntegrator(f_Re),
+                         new DomainLFIntegrator(f_Im));
+   b.real().Vector::operator=(0.0);
+   b.imag().Vector::operator=(0.0);
+   b.Assemble();
+
+   // 7. Set up the bilinear form (Real and Imaginary part)
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+   FunctionCoefficient ws(wavespeed);
+
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+
+   ProductCoefficient c2_re(c2_re0, ws);
+   ProductCoefficient c2_im(c2_im0, ws);
+
+   ParSesquilinearForm a(fespace,conv);
+   a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   a.AddDomainIntegrator(new MassIntegrator(c2_re),
+                         new MassIntegrator(c2_im));
+   a.Assemble();
+   a.Finalize();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+   ess_bdr = bc_type;
+   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   // Solution grid function
+   ParComplexGridFunction p_gf(fespace); p_gf = 0.0;
+   OperatorHandle Ah;
+   Vector X, B;
+
+   a.FormLinearSystem(ess_tdof_list, p_gf, b, Ah, X, B);
+
+   // // lor preconditioner
+   // ParMesh *pmesh_lor = NULL;
+   // FiniteElementCollection *fec_lor = NULL;
+   // ParFiniteElementSpace *fespace_lor = NULL;
+   // int basis_lor = basis;
+   // cout << order << endl;
+   // pmesh_lor = new ParMesh(pmesh, order, basis_lor);
+   
+   // CartesianPML pml_lor(pmesh_lor,lengths);
+   // pml_lor.SetOmega(omega);
+   
+   
+   // fec_lor = new H1_FECollection(1, dim);
+   // fespace_lor = new ParFiniteElementSpace(pmesh_lor, fec_lor);
+   // ParSesquilinearForm a_lor(fespace_lor,conv);
+
+   // PmlMatrixCoefficient c1_re_lor(dim,pml_detJ_JT_J_inv_Re,&pml_lor);
+   // PmlMatrixCoefficient c1_im_lor(dim,pml_detJ_JT_J_inv_Im,&pml_lor);
+
+   // PmlCoefficient detJ_re_lor(pml_detJ_Re,&pml_lor);
+   // PmlCoefficient detJ_im_lor(pml_detJ_Im,&pml_lor);
+
+   // ProductCoefficient c2_re0_lor(sigma, detJ_re_lor);
+   // ProductCoefficient c2_im0_lor(sigma, detJ_im_lor);
+
+   // ProductCoefficient c2_re_lor(c2_re0_lor, ws);
+   // ProductCoefficient c2_im_lor(c2_im0_lor, ws);
+
+
+   // a_lor.AddDomainIntegrator(new DiffusionIntegrator(c1_re_lor),
+                           //   new DiffusionIntegrator(c1_im_lor));
+   // a_lor.AddDomainIntegrator(new MassIntegrator(c2_re_lor),
+                           //   new MassIntegrator(c2_im_lor));
+   // a_lor.Assemble();
+   // a_lor.Finalize();
+
+   // Solution grid function
+   // OperatorHandle Ah_lor;
+   // a_lor.FormSystemMatrix(ess_tdof_list, Ah_lor);
+
+   // ComplexMUMPSSolver prec;
+
+   // StopWatch chrono;
+   // chrono.Clear();
+   // chrono.Start();
+   // prec.SetOperator(*Ah.As<ComplexHypreParMatrix>());
+   // prec.SetOperator(*Ah_lor.As<ComplexHypreParMatrix>());
+   // chrono.Stop();
+   // cout << " myid: " << myid 
+      //   << ", lor time: " << chrono.RealTime() << endl;
+
+   {
+      StopWatch chrono;
+      chrono.Clear();
+      chrono.Start();
+      ParDST::BCType bct = (bc_type == 1)? ParDST::BCType::DIRICHLET : ParDST::BCType::NEUMANN;
+      ParDST S(&a,lengths,omega, &ws,nrlayers,nx,ny,nz, bct);
+      // ParDST Slor(&a_lor,lengths,omega, &ws,nrlayers,nx,ny,nz);
+      chrono.Stop();
+      double t1 = chrono.RealTime();
+
+      chrono.Clear();
+      chrono.Start();
+      // X = 0.0;
+      GMRESSolver gmres(MPI_COMM_WORLD);
+      // gmres.SetPreconditioner(Slor);
+      gmres.SetOperator(*Ah);
+      gmres.SetPreconditioner(S);
+      // gmres.SetPreconditioner(prec);
+      gmres.SetRelTol(1e-12);
+      gmres.SetMaxIter(200);
+      gmres.SetPrintLevel(1);
+      gmres.Mult(B, X);
+      chrono.Stop();
+
+      double t2 = chrono.RealTime();
+      
+      MPI_Barrier(MPI_COMM_WORLD);
+
+
+      cout << " myid: " << myid 
+           << ", setup time: " << t1
+           << ", solution time: " << t2 << endl; 
+
+
+      // chrono.Clear();
+      // chrono.Start();
+      // X = 0.0;
+      // SLISolver sli(MPI_COMM_WORLD);
+      // sli.iterative_mode = true;
+      // sli.SetPreconditioner(S);
+      // sli.SetOperator(*Ah);
+      // sli.SetRelTol(1e-6);
+      // sli.SetMaxIter(20);
+      // sli.SetPrintLevel(1);
+      // sli.Mult(B,X);
+
+
+      // chrono.Stop();
+      // double t3 = chrono.RealTime();
+
+      // cout << " myid: " << myid 
+      //      << ", SLI solution time: " << t3 << endl; 
+
+         //    cout << " myid: " << myid 
+         //   << ", setup time: " << t1
+         //   << ", SLI solution time: " << t3 << endl; 
+         //   << ", solution time: " << t2 << endl; 
+
+      a.RecoverFEMSolution(X,B,p_gf);
+      if (visualization)
+      {
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         string keys;
+         if (dim ==2 )
+         {
+            keys = "keys mrRljc\n";
+         }
+         else
+         {
+            keys = "keys mc\n";
+         }
+         // socketstream mesh_sock(vishost, visport);
+         // mesh_sock.precision(8);
+         // mesh_sock << "parallel " << num_procs << " " << myid << "\n"
+         //             << "mesh\n" << *pmesh  << flush;
+         socketstream sol_sock_re(vishost, visport);
+         sol_sock_re.precision(8);
+         sol_sock_re << "parallel " << num_procs << " " << myid << "\n"
+                     << "solution\n" << *pmesh << p_gf.real() << keys 
+                     << "window_title 'Numerical Pressure: Real Part' " << flush;                     
+
+         socketstream sol_sock_im(vishost, visport);
+         sol_sock_im.precision(8);
+         sol_sock_im << "parallel " << num_procs << " " << myid << "\n"
+                     << "solution\n" << *pmesh << p_gf.imag() << keys 
+                     << "window_title 'Numerical Pressure: Imag Part' " << flush;     
+
+      int num_frames = 16;
+      GridFunction x_t(fespace);
+      x_t = p_gf.real();
+      // ParaViewDataCollection * pd = new ParaViewDataCollection("helmholtz_var_ws16", pmesh);
+      // pd->SetPrefixPath("ParaView");
+      // pd->RegisterField("solution", &x_t);
+      // pd->SetLevelsOfDetail(order);
+      // pd->SetDataFormat(VTKFormat::BINARY);
+      // pd->SetHighOrderOutput(true);
+      // pd->SetCycle(0);
+      // pd->SetTime(0.0);
+      // pd->Save();
+
+
+      // while (sol_sock)
+      // {
+      for (int i = 1; i<num_frames; i++)
+      {   
+         double t = (double)(i % num_frames) / num_frames;
+         // ostringstream oss;
+         // oss << "Harmonic Solution (t = " << t << " T)";
+
+         add(cos(2.0 * M_PI * t), p_gf.real(),
+             sin(2.0 * M_PI * t), p_gf.imag(), x_t);
+         // sol_sock << "solution\n"
+         //          << *mesh << x_t
+         //          << "window_title '" << oss.str() << "'" << flush;
+         // i++;
+
+         // pd->SetCycle(i);
+         // pd->SetTime((double)i);
+         // pd->Save();
+      }               
+
+      }
+   }
+
+
+
+
+//    // solve
+// {
+//    HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+//    SuperLURowLocMatrix SA(*A);
+//    SuperLUSolver superlu(MPI_COMM_WORLD);
+//    superlu.SetPrintStatistics(false);
+//    superlu.SetSymmetricPattern(false);
+//    superlu.SetColumnPermutation(superlu::PARMETIS);
+//    superlu.SetOperator(SA);
+//    superlu.Mult(B, X);
+//    delete A;
+// }
+//    a.RecoverFEMSolution(X,B,p_gf);
+
+//    if (visualization)
+//    {
+//       char vishost[] = "localhost";
+//       int  visport   = 19916;
+//       string keys;
+//       if (dim ==2 )
+//       {
+//          keys = "keys mrRljc\n";
+//       }
+//       else
+//       {
+//          keys = "keys mc\n";
+//       }
+//       socketstream sol_sock_re(vishost, visport);
+//       sol_sock_re.precision(8);
+//       sol_sock_re << "parallel " << num_procs << " " << myid << "\n"
+//                   << "solution\n" << *pmesh << p_gf.real() <<
+//                   "window_title 'Numerical Pressure' "
+//                   // << keys << "valuerange -0.08 0.08 \n" << flush;
+//                   << keys << flush;
+//    }
+   delete fespace;
+   delete fec;
+	delete pmesh;
+   MPI_Finalize();
+   return 0;
+}
+
+
+double f_exact_Re(const Vector &x)
+{
+   
+   // int nrsources = (dim == 2) ? 4 : 8;
+   int nrsources = 1;
+   Vector x0(nrsources);
+   Vector y0(nrsources);
+   Vector z0(nrsources);
+   // x0(0) = 0.25; y0(0) = 0.25; z0(0) = 0.25;
+   x0(0) = 0.5; y0(0) = 0.45; z0(0) = 0.25;
+   // x0(1) = 0.75; y0(1) = 0.25; z0(1) = 0.25;
+   // x0(2) = 0.25; y0(2) = 0.75; z0(2) = 0.25;
+   // x0(3) = 0.75; y0(3) = 0.75; z0(3) = 0.25;
+   if (dim == 3)
+   {
+      // x0(4) = 0.25; y0(4) = 0.25; z0(4) = 0.75;
+      // x0(5) = 0.75; y0(5) = 0.25; z0(5) = 0.75;
+      // x0(6) = 0.25; y0(6) = 0.75; z0(6) = 0.75;
+      // x0(7) = 0.75; y0(7) = 0.75; z0(7) = 0.75;
+   }
+  
+   double n = 4.0*omega/M_PI;
+   double coeff = 16.0*omega*omega/M_PI/M_PI/M_PI;
+
+   double f_re = 0.0;
+   // for (int i = 0; i<1; i++)
+   for (int i = 0; i<nrsources; i++)
+   {
+      double beta = pow(x0(i)-x(0),2) + pow(y0(i)-x(1),2);
+      if (dim == 3) { beta += pow(z0(i)-x(2),2); }
+      double alpha = -pow(n,2) * beta;
+      f_re += coeff*exp(alpha);
+   }
+
+   bool in_pml = false;
+   for (int i = 0; i<dim; i++)
+   {
+      if (x(i)<=comp_bdr(i,0) || x(i)>=comp_bdr(i,1))
+      {
+         in_pml = true;
+         break;
+      }
+   }
+   if (in_pml) f_re = 0.0;
+
+   return f_re;
+
+}
+double f_exact_Im(const Vector &x)
+{
+   double f_im;
+   f_im = 0.0;
+   return f_im;
+}
+
+double wavespeed(const Vector &x)
+{
+   double ws;
+   ws = 1.0;
+   // if (x(0) <= 0.25)
+   // {
+   //    ws = 1.0;
+   // }
+   // else if(x(0)<=0.5)
+   // {
+   //    ws = 1.0;
+   // }
+   // else if(x(0)<=0.75)
+   // {
+   //    ws = 0.75;
+   //    // ws = 0.5;
+   // }
+   // else 
+   // {
+   //    ws = 0.75;
+   //    // ws = 1.0;
+   // }
+
+   if (x(0) <= 0.33)
+   {
+      ws = 1.0;
+   }
+   else if(x(0)<=0.66)
+   {
+      ws = -0.65 + 5.0*x(0);
+   }
+   else 
+   {
+      ws = 2.65;
+      // ws = 0.5;
+   }
+
+
+   if (x(0) <= x(1) && x(1) >= 1.0-x(0))
+   {
+      ws = 5.0;
+   }
+   else if (x(0) > x(1) && x(1) >= 1.0-x(0))
+   {
+      ws = 3.0;
+   }
+   else if (x(0) <= x(1) && x(1) < 1.0-x(0))
+   {
+      ws = 2.0;
+   }
+   else
+   {
+      ws = 4.0;
+   }
+
+   // if (x(1) <= 1.0/3.0)
+   // {
+   //    ws = 2.0;
+   // }
+   // else if(x(1)<=2.0/3.0)
+   // {
+   //    ws = 1.0;
+   // }
+   // else 
+   // {
+   //    // ws = 0.75;
+   //    ws = 0.25;
+   // }
+
+
+   ws = 1.0;
+   return ws;
+}
+
+
+double funccoeff_re(const Vector & x)
+{
+   return sin(3*M_PI*(x.Sum()));
+}
+
+double funccoeff_im(const Vector & x)
+{
+   return cos(10*M_PI*(x.Sum()));
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

examples/maxwell-solver/helmholtzp_real.cpp

@@ -0,0 +1,246 @@
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+using namespace std;
+using namespace mfem;
+
+double p_exact(const Vector &x);
+double rhs_func(const Vector &x);
+void gradp_exact(const Vector &x, Vector &gradu);
+double d2_exact(const Vector &x);
+
+
+int dim;
+double omega;
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = 1;
+   int sr = 1;
+   int pr = 1;
+   double rnum=1.0;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&sr, "-sr", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pr", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");                  
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   omega = 2.0 * M_PI * rnum;
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution.
+   for (int i = 0; i < sr; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   int btype = BasisType::GaussLobatto;
+   ParMesh pmesh_lor(pmesh, order, btype);
+
+   // 6. Define a parallel finite element space on the parallel mesh.
+   FiniteElementCollection *fec = new H1_FECollection(order,dim); 
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+
+   FiniteElementCollection *fec_lor = new H1_FECollection(1,dim); 
+   ParFiniteElementSpace *fespace_lor = new ParFiniteElementSpace(&pmesh_lor, fec_lor);
+
+   // (f,q)
+   ParLinearForm b(fespace);
+   FunctionCoefficient f_rhs(rhs_func);
+   b.AddDomainIntegrator(new DomainLFIntegrator(f_rhs));
+
+
+   ParBilinearForm a(fespace);
+   ParBilinearForm a_lor(fespace_lor);
+   ParBilinearForm aprec(fespace);
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient omeg(-omega*omega);
+   ConstantCoefficient posomeg(omega*omega);
+   // (grad u, grad v) - \omega^2 (u,v)
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a.AddDomainIntegrator(new MassIntegrator(omeg));
+
+   a_lor.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a_lor.AddDomainIntegrator(new MassIntegrator(omeg));
+
+   aprec.AddDomainIntegrator(new DiffusionIntegrator(one));
+   aprec.AddDomainIntegrator(new MassIntegrator(posomeg));
+
+   
+   ParGridFunction x(fespace);
+   x = 0.0;
+   FunctionCoefficient p_ex(p_exact);
+   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
+
+   // 9. Perform successive parallel refinements, compute the L2 error and the
+   //    corresponding rate of convergence.
+   ConvergenceStudy rates;
+   for (int l = 0; l <= pr; l++)
+   {
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh->bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+         ess_bdr = 1;
+         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      b.Assemble();
+      a.Assemble();
+      a_lor.Assemble();
+      x.ProjectBdrCoefficient(p_ex,ess_bdr);      
+
+      OperatorPtr A;
+      Vector B, X;
+      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+      
+      // OperatorPtr M;
+      // Array<int> ess_tdof_list1;
+      // ess_tdof_list1 = ess_tdof_list;
+
+      a_lor.EliminateEssentialBC(ess_bdr,mfem::Matrix::DIAG_ONE);
+      a_lor.Finalize();
+      HypreParMatrix * A_lor = a_lor.ParallelAssemble();
+
+      aprec.Assemble();
+      aprec.EliminateEssentialBC(ess_bdr,mfem::Matrix::DIAG_ONE);
+      aprec.Finalize();
+      // aprec.FormSystemMatrix(ess_tdof_list,M);
+      HypreParMatrix * M = aprec.ParallelAssemble();
+      MUMPSSolver mumps_prec;
+      mumps_prec.SetPrintLevel(0);
+      mumps_prec.SetOperator(*M);
+
+
+      // MUMPSSolver mumps_lor;
+      // mumps_lor.SetOperator(*A_lor);
+      // mumps_lor.SetPrintLevel(0);
+
+
+      // HypreBoomerAMG amg(*M);
+      // amg.SetPrintLevel(0);
+
+      StopWatch chrono;
+      chrono.Clear();
+      chrono.Start();
+      GMRESSolver gmres(MPI_COMM_WORLD);
+      gmres.SetRelTol(1e-6);
+      gmres.SetAbsTol(0.0);
+      gmres.SetMaxIter(2000);
+      gmres.SetPrintLevel(1);
+      gmres.SetOperator(*A);
+      gmres.SetPreconditioner(mumps_prec);
+      // gmres.SetPreconditioner(mumps_lor);
+      // gmres.SetPreconditioner(amg);
+      gmres.Mult(B, X);
+      chrono.Stop();
+      cout << "LOR exact - GMRES time " << chrono.RealTime() << endl;
+
+      // MUMPSSolver mumps;
+      // mumps.SetPrintLevel(0);
+      // mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
+      // mumps.SetOperator(*A);
+      // mumps.Mult(B,X);
+
+      a.RecoverFEMSolution(X, b, x);
+
+      rates.AddH1GridFunction(&x,&p_ex,&gradp_ex);
+
+      if (l==pr) break;
+
+      pmesh->UniformRefinement();
+      fespace->Update();
+      a.Update();
+      aprec.Update();
+      b.Update();
+      x.Update();
+   }
+   rates.Print(true);
+
+   // 10. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x <<
+               "window_title 'Numerical Pressure (real part)' "
+               << flush;
+   }
+
+   // 11. Free the used memory.
+   delete fespace;
+   delete fec;
+   delete pmesh;
+   MPI_Finalize();
+   return 0;
+}
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double d2p = d2_exact(x);
+   return -d2p - omega * omega * p;
+}
+
+double p_exact(const Vector &x)
+{
+   return sin(omega*x.Sum());
+}
+
+void gradp_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   grad = omega * cos(omega * x.Sum());
+}
+
+double d2_exact(const Vector &x)
+{
+   return -dim * omega * omega * sin(omega*x.Sum());
+}

examples/maxwell-solver/lor.cpp

@@ -0,0 +1,214 @@
+#include "lor.hpp"
+
+
+const Array<int> &GetDofMap(FiniteElementSpace &fes, int i)
+{
+   const FiniteElement *fe = fes.GetFE(i);
+   auto tfe = dynamic_cast<const TensorBasisElement*>(fe);
+   MFEM_ASSERT(tfe != NULL, "");
+   return tfe->GetDofMap();
+}
+
+Array<int> ComputeVectorFE_LORPermutation(
+   FiniteElementSpace &fes_ho,
+   FiniteElementSpace &fes_lor,
+   FiniteElement::MapType type)
+{
+   // Given an index `i` of a LOR dof, `perm[i]` is the index of the
+   // corresponding HO dof.
+   Array<int> perm(fes_lor.GetVSize());
+   Array<int> vdof_ho, vdof_lor;
+
+   Mesh &mesh_lor = *fes_lor.GetMesh();
+   int dim = mesh_lor.Dimension();
+   const CoarseFineTransformations &cf_tr = mesh_lor.GetRefinementTransforms();
+   for (int ilor=0; ilor<mesh_lor.GetNE(); ++ilor)
+   {
+      int iho = cf_tr.embeddings[ilor].parent;
+      int lor_index = cf_tr.embeddings[ilor].matrix;
+
+      int p = fes_ho.GetOrder(iho);
+      int p1 = p+1;
+      int ndof_per_dim = (dim == 2) ? p*p1 :
+                         type == FiniteElement::H_CURL ? p*p1*p1 : p*p*p1;
+
+      fes_ho.GetElementVDofs(iho, vdof_ho);
+      fes_lor.GetElementVDofs(ilor, vdof_lor);
+
+      const Array<int> &dofmap_ho = GetDofMap(fes_ho, iho);
+      const Array<int> &dofmap_lor = GetDofMap(fes_lor, ilor);
+
+      int off_x = lor_index % p;
+      int off_y = (lor_index / p) % p;
+      int off_z = (lor_index / p) / p;
+
+      auto absdof = [](int i) { return i < 0 ? -1-i : i; };
+
+      auto set_perm = [&](int off_lor, int off_ho, int n1, int n2)
+      {
+         for (int i1=0; i1<2; ++i1)
+         {
+            int m = (dim == 2 || type == FiniteElement::H_DIV) ? 1 : 2;
+            for (int i2=0; i2<m; ++i2)
+            {
+               int i;
+               i = dofmap_lor[off_lor + i1 + i2*2];
+               int s1 = i < 0 ? -1 : 1;
+               int idof_lor = vdof_lor[absdof(i)];
+               i = dofmap_ho[off_ho + i1*n1 + i2*n2];
+               int s2 = i < 0 ? -1 : 1;
+               int idof_ho = vdof_ho[absdof(i)];
+               int s3 = idof_lor < 0 ? -1 : 1;
+               int s4 = idof_ho < 0 ? -1 : 1;
+               int s = s1*s2*s3*s4;
+               i = absdof(idof_ho);
+               perm[absdof(idof_lor)] = s < 0 ? -1-absdof(i) : absdof(i);
+            }
+         }
+      };
+
+      int offset;
+
+      if (type == FiniteElement::H_CURL)
+      {
+         // x
+         offset = off_x + off_y*p + off_z*p*p1;
+         set_perm(0, offset, p, p*p1);
+         // y
+         offset = ndof_per_dim + off_x + off_y*(p1) + off_z*p1*p;
+         set_perm(dim == 2 ? 2 : 4, offset, 1, p*p1);
+         // z
+         if (dim == 3)
+         {
+            offset = 2*ndof_per_dim + off_x + off_y*p1 + off_z*p1*p1;
+            set_perm(8, offset, 1, p+1);
+         }
+      }
+      else
+      {
+         // x
+         offset = off_x + off_y*p1 + off_z*p*p1;
+         set_perm(0, offset, 1, 0);
+         // y
+         offset = ndof_per_dim + off_x + off_y*p + off_z*p1*p;
+         set_perm(2, offset, p, 0);
+         // z
+         if (dim == 3)
+         {
+            offset = 2*ndof_per_dim + off_x + off_y*p + off_z*p*p;
+            set_perm(4, offset, p*p, 0);
+         }
+      }
+   }
+
+   return perm;
+}
+
+
+RealLORSolver::RealLORSolver(HypreParMatrix & A, const Array<int> p_, bool exact, Solver * prec)
+   : Solver(A.Height()), p(p_)
+{
+   if (exact)
+   {
+      solv = new MUMPSSolver;
+      dynamic_cast<MUMPSSolver*>(solv)->SetOperator(A);
+   }
+   else
+   {
+      solv = prec;
+   }
+   int n = A.Height();
+   n2 = p.Size();
+   n1 = n - n2;
+   perm.SetSize(n);
+   for (int i = 0; i<n1; i++) { perm[i] = i; }
+   for (int i = 0; i<n2; i++) { perm[i+n1] = p[i]; }
+}
+
+void RealLORSolver::Mult(const Vector &b, Vector &x) const
+{
+   Vector bp(b.Size());
+   Vector xp(x.Size());
+   
+   for (int i=0; i<n1; ++i) 
+   { 
+      bp[i] = b[i];
+   }
+   
+   for (int i=n1; i<n1+n2; ++i) 
+   { 
+      int m = perm[i] < 0 ? n1-1-perm[i] : n1+perm[i];
+      bp[i] = perm[i] < 0 ? -b[m] : b[m]; 
+   }
+
+   solv->Mult(bp, xp);
+
+   for (int i=0; i<n1; ++i)
+   {
+      x[i] = xp[i];
+   }
+   for (int i=n1; i<x.Size(); ++i)
+   {
+      int pi = perm[i];
+      int s = pi < 0 ? -1 : 1;
+      int n = pi < 0 ? n1-1-pi : n1 + pi;
+      x[n] = s*xp[i];
+   }
+}
+
+ComplexLORSolver::ComplexLORSolver(HypreParMatrix & A, const Array<int> p_, bool exact, Solver * prec)
+   : Solver(A.Height()), p(p_)
+{
+   if (exact)
+   {
+      solv = new MUMPSSolver;
+      dynamic_cast<MUMPSSolver*>(solv)->SetOperator(A);
+   }
+   else
+   {
+      solv = prec;
+   }
+   int n = A.Height()/2;
+   n2 = p.Size();
+   n1 = n - n2;
+   perm.SetSize(n);
+   for (int i = 0; i<n1; i++) { perm[i] = i; }
+   for (int i = 0; i<n2; i++) { perm[i+n1] = p[i]; }
+}
+
+void ComplexLORSolver::Mult(const Vector &b, Vector &x) const
+{
+   Vector bp(b.Size());
+   Vector xp(x.Size());
+   
+   for (int i=0; i<n1; ++i) 
+   { 
+      bp[i] = b[i];
+      bp[n1+n2+i] = b[n1+n2+i];
+   }
+   
+   for (int i=n1; i<n1+n2; ++i) 
+   { 
+      int m = perm[i] < 0 ? n1-1-perm[i] : n1+perm[i];
+      bp[i] = perm[i] < 0 ? -b[m] : b[m]; 
+      bp[n1+n2+i] = perm[i] < 0 ? -b[n1+n2+m] : b[n1+n2+m]; 
+   }
+
+   solv->Mult(bp, xp);
+
+   for (int i=0; i<n1; ++i)
+   {
+      x[i] = xp[i];
+      x[n1+n2+i] = xp[n1+n2+i];
+   }
+
+   for (int i=n1; i<n1+n2; ++i)
+   {
+      int pi = perm[i];
+      int s = pi < 0 ? -1 : 1;
+      int n = pi < 0 ? n1-1-pi : n1 + pi;
+      x[n] = s*xp[i];
+      x[n+n1+n2] = s*xp[i+n1+n2];
+   }
+
+}

examples/maxwell-solver/lor.hpp

@@ -0,0 +1,45 @@
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+using namespace std;
+using namespace mfem;
+
+const Array<int> &GetDofMap(FiniteElementSpace &fes, int i);
+Array<int> ComputeVectorFE_LORPermutation(FiniteElementSpace &fes_ho,
+                                          FiniteElementSpace &fes_lor,
+                                          FiniteElement::MapType type);
+
+class RealLORSolver : public Solver
+{
+
+private:
+   int n1;
+   int n2;
+   Array<int> perm;
+   Array<int> p;
+   Solver *solv=nullptr;
+public:
+   RealLORSolver(HypreParMatrix & A, const Array<int> p_, 
+                         bool exact = true, Solver * prec = nullptr);
+void SetOperator(const Operator&) { }
+
+void Mult(const Vector &b, Vector &x) const;
+};
+
+class ComplexLORSolver : public Solver
+{
+
+private:
+   int n1;
+   int n2;
+   Array<int> perm;
+   Array<int> p;
+   Solver *solv=nullptr;
+public:
+   ComplexLORSolver(HypreParMatrix & A, const Array<int> p_, 
+                    bool exact = true, Solver * prec = nullptr);
+void SetOperator(const Operator&) { }
+
+void Mult(const Vector &b, Vector &x) const;
+};

examples/maxwell-solver/makefile

@@ -0,0 +1,67 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/maxwell-solver/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = helmholtz maxwell maxwell-impedanceBC annulus pml_torus torus_generate_ovlp_partitioning
+PAR_EXAMPLES = helmholtzp maxwellp
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean
+.PRECIOUS: %.o
+
+COMMON_O= common/PML.o common/MeshPartition.o \
+			 common/Utilities.o common/complex_linalg.o\
+			 DST/DST.o ParDST/ParDST.o ParDST/DofMapsDST.o FOSLS.o lor.o
+
+# Remove built-in rules
+%: %.cpp
+%.o: %.cpp
+
+all: $(EXAMPLES)
+
+# Rules for building the EXAMPLES
+
+%: $(SRC)%.cpp $(COMMON_O) $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(COMMON_O) $(MFEM_LIBS)
+
+# Rules for compiling miniapp dependencies
+$(COMMON_O) $($(EXAMPLES)): \
+%.o: $(SRC)%.cpp $(SRC)%.hpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $(<) -o $(@)
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -f DST/*.o
+	rm -f ParDST/*.o
+	rm -f common/*.o
+	rm -f DST2D/*.o
+	rm -rf *.dSYM *.TVD.*breakpoints
+	rm output/*
+
+

examples/maxwell-solver/maxwell-annulus.cpp

@@ -0,0 +1,418 @@
+
+
+// sample runs: ./maxwell-annulus -ref 2 -o 2 -f 0.6
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void maxwell_solution(const Vector &x, vector<complex<double>> &E);
+void maxwell_curl(const Vector &x, vector<complex<double>> &curlE);
+
+void E_bdr_data_Re(const Vector &x, Vector &E);
+void E_bdr_data_Im(const Vector &x, Vector &E);
+
+void E_exact_Re(const Vector &x, Vector &E);
+void E_exact_Im(const Vector &x, Vector &E);
+
+void E_exact_Curl_Re(const Vector &x, Vector &E);
+void E_exact_Curl_Im(const Vector &x, Vector &E);
+
+void source(const Vector &x, Vector & f);
+double sigma_func(const Vector &x);
+
+double mu = 1.0;
+double epsilon = 1.0;
+double omega;
+int dim;
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_SELF, &num_procs);
+   MPI_Comm_rank(MPI_COMM_SELF, &myid);
+   // 1. Parse command-line options.
+   // const char *mesh_file = "torus1_4.mesh";
+   // const char *mesh_file = "waveguide-bend2.mesh";
+   const char *mesh_file = "meshes/annulus-quad-o3.mesh";
+
+   int order = 1;
+   int ref_levels = 1;
+   double freq = 5.0;
+   bool herm_conv = true;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-ref", "--refinements",
+                  "Number of refinements");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&freq, "-f", "--frequency",
+                  "Frequency (in Hz).");
+   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
+                  "--no-hermitian", "Use convention for Hermitian operators.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+
+   // 2. Setup the mesh
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   Mesh * mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+   mesh->RemoveInternalBoundaries();
+
+   mesh->UniformRefinement();
+
+   // Angular frequency
+   omega = 2.0 * M_PI * freq;
+
+   ComplexOperator::Convention conv =
+      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
+
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+
+   ComplexGridFunction x(fespace);
+   x = 0.0;
+   VectorFunctionCoefficient E_Re(dim, E_bdr_data_Re);
+   VectorFunctionCoefficient E_Im(dim, E_bdr_data_Im);
+
+
+   H1_FECollection H1fec(order, dim);
+   FiniteElementSpace H1fes(mesh, &H1fec);
+
+   GridFunction bump(&H1fes);
+   FunctionCoefficient bump_coeff(sigma_func);
+   bump.ProjectCoefficient(bump_coeff);
+   {
+      char vishost[] = "localhost";
+      int visport = 19916;
+
+      socketstream sol_sock_sigma(vishost, visport);
+      sol_sock_sigma.precision(8);
+      sol_sock_sigma << "solution\n"
+                  << *mesh << bump 
+                  << "window_title 'bump function'" << flush;
+   }
+
+
+   for (int iter = 0; iter<ref_levels; iter++)
+   {
+
+      int size = fespace->GetTrueVSize();
+      cout << "Number of finite element unknowns: " << size << endl;
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (mesh->bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(mesh->bdr_attributes.Max());
+         ess_bdr = 1;
+      }
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+      ComplexLinearForm b(fespace, conv);
+      b.Vector::operator=(0.0);
+      b.Assemble();
+
+      x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);
+
+
+      ConstantCoefficient muinv(1.0/mu);
+      ConstantCoefficient omeg(-pow(omega, 2) * epsilon);
+      ConstantCoefficient sigma(-pow(omega, 2) * epsilon);
+      ProductCoefficient c1(sigma,bump_coeff);
+      ConstantCoefficient sigma1(-omega * epsilon);
+
+      // Integrators inside the computational domain (excluding the PML region)
+      SesquilinearForm a(fespace, conv);
+      a.AddDomainIntegrator(new CurlCurlIntegrator(muinv),NULL);
+      a.AddDomainIntegrator(new VectorFEMassIntegrator(omeg),NULL);
+
+      a.AddDomainIntegrator(NULL,new VectorFEMassIntegrator(c1));
+      a.Assemble(0);
+
+
+      SesquilinearForm prec(fespace, conv);
+      prec.AddDomainIntegrator(new CurlCurlIntegrator(muinv),NULL);
+      prec.AddDomainIntegrator(new VectorFEMassIntegrator(omeg),NULL);
+
+
+      prec.AddDomainIntegrator(NULL,new VectorFEMassIntegrator(c1));
+      prec.Assemble(0);
+
+      OperatorPtr A;
+      Vector B, X;
+      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+      OperatorPtr pA;
+      prec.FormSystemMatrix(ess_tdof_list, pA);
+
+
+      SparseMatrix * SpMat = (*pA.As<ComplexSparseMatrix>()).GetSystemMatrix();
+      HYPRE_Int global_size = SpMat->Height();
+      HYPRE_Int row_starts[2]; row_starts[0] = 0; row_starts[1] = global_size;
+      HypreParMatrix * HypreMat = new HypreParMatrix(MPI_COMM_SELF,global_size,row_starts,SpMat);
+      {
+         MUMPSSolver mumps;
+         mumps.SetOperator(*HypreMat);
+         mumps.Mult(B,X);
+
+         GMRESSolver gmres(MPI_COMM_WORLD);
+         gmres.SetRelTol(1e-12);
+         gmres.SetMaxIter(2000);
+         gmres.SetPrintLevel(1);
+         gmres.SetOperator(*A);
+         gmres.SetPreconditioner(mumps); 
+         gmres.Mult(B, X);
+      }
+
+      a.RecoverFEMSolution(X, b, x);
+
+      int cdim = (dim == 2) ? 1 : dim;
+
+      if (iter == ref_levels) break;
+      mesh->UniformRefinement();
+      fespace->Update();
+      x.Update();
+   }
+
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      // Define visualization keys for GLVis (see GLVis documentation)
+      string keys;
+      keys = (dim == 3) ? "keys macF\n" : keys = "keys amrRljcUUuuu\n";
+
+      char vishost[] = "localhost";
+      int visport = 19916;
+
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re.precision(8);
+      sol_sock_re << "solution\n"
+                  << *mesh << x.real() << keys
+                  << "window_title 'Solution real part'" << flush;
+
+      socketstream sol_sock_im(vishost, visport);
+      sol_sock_im.precision(8);
+      sol_sock_im << "solution\n"
+                  << *mesh << x.imag() << keys
+                  << "window_title 'Solution imag part'" << flush;
+      {
+         GridFunction x_t(fespace);
+         x_t = x.real();
+         socketstream sol_sock(vishost, visport);
+         sol_sock.precision(8);
+         sol_sock << "solution\n"
+                  << *mesh << x_t << keys << "autoscale off\n"
+                  << "window_title 'Harmonic Solution (t = 0.0 T)'"
+                  << "pause\n" << flush;
+
+         cout << "GLVis visualization paused."
+               << " Press space (in the GLVis window) to resume it.\n";
+      int num_frames = 16;
+
+         while (sol_sock)
+         {
+            for (int i = 1; i<num_frames; i++)
+            {   
+               double t = (double)(i % num_frames) / num_frames;
+               ostringstream oss;
+               oss << "Harmonic Solution (t = " << t << " T)";
+
+               add(cos(2.0 * M_PI * t), x.real(),
+                  sin(2.0 * M_PI * t), x.imag(), x_t);
+               sol_sock << "solution\n"
+                        << *mesh << x_t 
+                        << "window_title '" << oss.str() << "'" << flush;
+            }
+         }
+      }
+   }
+
+   // 17. Free the used memory.
+   delete fespace;
+   delete fec;
+   delete mesh;
+
+   return 0;
+}
+
+double sigma_func(const Vector &x)
+{
+   double r = x.Norml2();
+   double val = 0.0;
+   if (r < 0.3)
+   {
+      val = 1.0;
+   }
+   else
+   {
+      r*=1.5;
+      if (r<1)
+      {
+         // r*=.5;
+         double d = r*r;
+         double factor = 0.1;
+         val = exp(factor) * exp(-factor/(1.-d));
+      }
+   }
+
+   return 1.-val;
+}
+
+
+void E_bdr_data_Re(const Vector &x, Vector &E)
+{
+   // vector<complex<double>> Eval(E.Size());
+   // maxwell_solution(x, Eval);
+   // for (int i = 0; i < dim; ++i)
+   // {
+   //    E[i] = Eval[i].real();
+   // }
+   E_exact_Re(x,E);
+}
+
+// Define bdr_data solution
+void E_bdr_data_Im(const Vector &x, Vector &E)
+{
+   // double r = x.Norml2();
+   // vector<complex<double>> Eval(E.Size());
+   // maxwell_solution(x, Eval);
+   // for (int i = 0; i < dim; ++i)
+   // {
+   //    E[i] = Eval[i].imag();
+   // }
+   E_exact_Im(x,E);
+}
+
+void E_exact_Re(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   if (x.Norml2() < 0.3 )
+   {
+      vector<complex<double>> Eval(E.Size());
+      maxwell_solution(x, Eval);
+      for (int i = 0; i < dim; ++i)
+      {
+         E[i] = Eval[i].real();
+      }
+   }
+}
+
+void E_exact_Im(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   if (x.Norml2() < 0.3 )
+   {
+      vector<complex<double>> Eval(E.Size());
+      maxwell_solution(x, Eval);
+      for (int i = 0; i < dim; ++i)
+      {
+         E[i] = Eval[i].imag();
+      }
+   }
+}
+
+void maxwell_solution(const Vector &x, vector<complex<double>> &E)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   double k = omega * sqrt(epsilon * mu);
+   Vector shift(dim);
+   shift = 0.0;
+   double x0 = x(0) + shift(0);
+   double x1 = x(1) + shift(1);
+   double r = sqrt(x0 * x0 + x1 * x1);
+   double beta = k * r;
+
+   // Bessel functions
+   complex<double> H0, H0_r, H0_rr;
+   complex<double> H1; 
+   complex<double> H2;
+   H0 = jn(0,beta) + zi * yn(0,beta);
+   H1 = jn(1,beta) + zi * yn(1,beta);
+   H2 = jn(2,beta) + zi * yn(2,beta);
+   // H3 = jn(3,beta) + zi * yn(3,beta);
+
+   H0_r = - k * H1;
+   H0_rr = - k * k * (1.0/beta * H1 - H2); 
+
+   // First derivatives
+   double r_x = x0 / r;
+   double r_y = x1 / r;
+   double r_xy = -(r_x / r) * r_y;
+   double r_xx = (1.0 / r) * (1.0 - r_x * r_x);
+
+   complex<double> val, val_xx, val_xy ;
+   val = 0.25 * zi * H0;
+   val_xx = 0.25 * zi * (r_xx * H0_r + r_x * r_x * H0_rr);
+   val_xy = 0.25 * zi * (r_xy * H0_r + r_x * r_y * H0_rr);
+   E[0] = zi / k * (k * k * val + val_xx);
+   E[1] = zi / k * val_xy;
+}
+
+void E_exact_Curl_Re(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   vector<complex<double>> Eval(E.Size());
+   maxwell_curl(x, Eval);
+   for (int i = 0; i < E.Size(); ++i)
+   {
+      E[i] = Eval[i].real();
+   }
+}
+
+void E_exact_Curl_Im(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   vector<complex<double>> Eval(E.Size());
+   maxwell_curl(x, Eval);
+   for (int i = 0; i < E.Size(); ++i)
+   {
+      E[i] = Eval[i].imag();
+   }
+}
+
+
+void maxwell_curl(const Vector &x, vector<complex<double>> &curlE)
+{
+   complex<double> zi = complex<double>(0., 1.);
+
+   double k = omega * sqrt(epsilon * mu);
+   Vector shift(dim);
+   shift = 0.0;
+   double x0 = x(0) + shift(0);
+   double x1 = x(1) + shift(1);
+   double r = sqrt(x0 * x0 + x1 * x1);
+   double beta = k * r;
+
+   // Bessel functions
+   complex<double> H0_r;
+   complex<double> H1;
+   H1 = jn(1,beta) + zi * yn(1,beta);
+   H0_r   = - k * H1;
+
+   double r_y = x1 / r;
+   complex<double> val_y;
+   val_y = 0.25 * zi * H0_r * r_y;
+   curlE[0] = zi / k * (- k * k * val_y);
+}

examples/maxwell-solver/maxwell-impedanceBC.cpp

@@ -0,0 +1,385 @@
+
+//
+// Compile with: make maxwell-impedanceBC
+//
+//               maxwell-impedanceBC -o 2 -f 1.6 -ref 2 -prob 0 -m ../../data/beam-hex.mesh
+//               maxwell-impedanceBC -o 2 -f 1.6 -ref 2 -prob 0 -m ../../data/beam-tet.mesh
+//
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace mfem;
+using namespace std;
+
+void E_exact_Re(const Vector &x, Vector &E);
+void E_exact_Im(const Vector &x, Vector &E);
+void Curl_exact_Re(const Vector &x, Vector &Curl);
+void Curl_exact_Im(const Vector &x, Vector &Curl);
+void maxwell_solution(const Vector &x, vector<complex<double>> &Eval);
+void maxwell_curl(const Vector &x, vector<complex<double>> &Curl);
+void maxwell_curlcurl(const Vector &x, vector<complex<double>> &CurlCurl);
+void f_exact_Re(const Vector &x, Vector &E);
+void f_exact_Im(const Vector &x, Vector &E);
+
+double omega;
+int prob_kind = 0;
+int dim;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/beam-hex.mesh";
+   // const char *mesh_file = "../../data/beam-tet.mesh";
+   int order = 2;
+   int ref_levels = 2;
+   double freq = 1.6;
+   bool herm_conv = true;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&prob_kind, "-prob", "--problem_kind",
+                  "Choice of problem");                  
+   args.AddOption(&ref_levels, "-ref", "--refinements",
+                  "Number of refinements");
+   args.AddOption(&freq, "-f", "--frequency",
+                  "Frequency (in Hz).");
+   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
+                  "--no-hermitian", "Use convention for Hermitian operators.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+
+   Mesh * mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // Angular frequency
+   omega = 2.0 * M_PI * freq;
+   if (prob_kind == 0)  MFEM_VERIFY (omega > M_PI * M_PI, "increase fequency");
+
+   mesh->ReorientTetMesh();
+
+   // 7. Define a finite element space on the mesh. Here we use the Nedelec
+   //    finite elements of the specified order.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+
+   ComplexOperator::Convention conv =
+   herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
+
+   Array<int> ess_bdr;
+   Array<int> imp_bdr;
+   cout << mesh->bdr_attributes.Max() << endl;
+   if (mesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh->bdr_attributes.Max());
+      imp_bdr.SetSize(mesh->bdr_attributes.Max());
+      ess_bdr = 0;
+      imp_bdr = 1;
+   }
+
+   // required coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient omeg(-pow(omega, 2));
+   ConstantCoefficient om(omega);
+   VectorFunctionCoefficient E_Re(dim, E_exact_Re);
+   VectorFunctionCoefficient E_Im(dim, E_exact_Im);
+   VectorFunctionCoefficient Curl_Re(dim, Curl_exact_Re);
+   VectorFunctionCoefficient Curl_Im(dim, Curl_exact_Im);
+   VectorFunctionCoefficient f_Re(dim,f_exact_Re);
+   VectorFunctionCoefficient f_Im(dim,f_exact_Im);
+   // For - <n x curl E> 
+   ScalarVectorProductCoefficient c1_Re(-1.0,Curl_Re);                        
+   ScalarVectorProductCoefficient c1_Im(-1.0,Curl_Im);
+   // For  i omega (n x n x E)
+   ScalarVectorProductCoefficient c2_Re(-omega,E_Im);                        
+   ScalarVectorProductCoefficient c2_Im(omega,E_Re);
+
+
+   // Weak form with impedance condition
+   // n x curl E + i omega (n x n x E) = G on \partial \Omega 
+   // (curlE, curlH) - omega^2 (E,H) + < n x curlE, H> = (F, H)
+   // (curlE, curlH) - omega^2 (E,H) - i omega <n x n x E, H> = (F,H) - <G,H>
+
+   ComplexLinearForm b(fespace, conv);
+   b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(f_Re), 
+                         new VectorFEDomainLFIntegrator(f_Im));
+   b.AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(c1_Re), 
+                           new VectorFEBoundaryTangentLFIntegrator(c1_Im), imp_bdr);
+   b.AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(c2_Re), 
+                           new VectorFEDomainLFIntegrator(c2_Im), imp_bdr);   
+
+   SesquilinearForm a(fespace, conv);
+   a.AddDomainIntegrator(new CurlCurlIntegrator(one), NULL);
+   a.AddDomainIntegrator(new VectorFEMassIntegrator(omeg),NULL);
+   a.AddBoundaryIntegrator(NULL, new VectorFEMassIntegrator(om),imp_bdr);
+
+   ComplexGridFunction x(fespace);
+   x = 0.0;
+   x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);
+
+
+   ConvergenceStudy rates_re, rates_im;
+
+   for (int l = 0; l<=ref_levels; l++)
+   {
+      int size = fespace->GetTrueVSize();
+      cout << "Number of finite element unknowns: " << size << endl;
+   
+      b.Assemble();
+      a.Assemble();
+
+      OperatorPtr A;
+      Vector B, X;
+      Array<int> ess_tdof_list;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+      // 14. Solve using a direct solver
+      UMFPackSolver csolver(*A.As<ComplexSparseMatrix>()->GetSystemMatrix());
+      csolver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      csolver.SetPrintLevel(1);
+      csolver.Mult(B, X);
+
+      a.RecoverFEMSolution(X, b, x);
+
+      rates_re.AddHcurlGridFunction(&x.real(),&E_Re,&Curl_Re);
+      rates_im.AddHcurlGridFunction(&x.imag(),&E_Im,&Curl_Im);
+
+      if (l==ref_levels) break;
+
+      mesh->UniformRefinement();
+      mesh->ReorientTetMesh();
+      fespace->Update();
+      a.Update();
+      b.Update();
+      x.Update();
+   }
+
+   rates_re.Print();
+   rates_im.Print();
+
+
+   ComplexGridFunction x_ex(fespace);
+   x_ex.ProjectCoefficient(E_Re, E_Im);
+
+   if (visualization)
+   {
+
+      char vishost[] = "localhost";
+      int visport = 19916;
+
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re.precision(8);
+      sol_sock_re << "solution\n"
+                  << *mesh << x.real() 
+                  << "window_title 'Solution real part'" << flush;
+
+      socketstream sol_sock_im(vishost, visport);
+      sol_sock_im.precision(8);
+      sol_sock_im << "solution\n"
+                  << *mesh << x.imag()
+                  << "window_title 'Solution imag part'" << flush;
+
+      socketstream sol_sock_re_ex(vishost, visport);
+      sol_sock_re_ex.precision(8);
+      sol_sock_re_ex << "solution\n"
+                  << *mesh << x_ex.real() 
+                  << "window_title 'Exact real part'" << flush;
+
+      socketstream sol_sock_im_ex(vishost, visport);
+      sol_sock_im_ex.precision(8);
+      sol_sock_im_ex << "solution\n"
+                  << *mesh << x_ex.imag()
+                  << "window_title 'Exact imag part'" << flush;            
+
+   }
+   // 18. Free the used memory.
+   delete fespace;
+   delete fec;
+   delete mesh;
+}
+
+
+void maxwell_solution(const Vector &x, vector<complex<double>> &E)
+{
+   // Initialize
+   int dim = x.Size();
+   for (int i = 0; i < dim; ++i)
+   {
+      E[i] = 0.0;
+   }
+   if (prob_kind == 0)
+   {
+      complex<double> zi = complex<double>(0., 1.);
+      double k10 = sqrt(omega * omega - M_PI * M_PI);
+      E[1] = -zi * omega / M_PI * sin(M_PI*x(2))*exp(zi * k10 * x(0));
+   }
+   else if (prob_kind == 1)
+   {
+      E[0] = x(0)*x(1);
+      E[1] = x(1)*x(2);
+      E[2] = x(2)*x(0);
+   }
+   else
+   {
+      E[0] = x(0)*x(1)*x(2)*x(2);
+      E[1] = x(1)*x(2)*x(0)*x(0)*x(0);
+      E[2] = x(2)*x(0)*x(1);
+   }
+   
+
+
+}
+
+void maxwell_curl(const Vector &x, vector<complex<double>> &Curl)
+{
+   // Initialize
+   int dim = x.Size();
+   for (int i = 0; i < dim; ++i)
+   {
+      Curl[i] = 0.0;
+   }
+   if (prob_kind == 0)
+   {
+      complex<double> zi = complex<double>(0., 1.);
+      double k10 = sqrt(omega * omega - M_PI * M_PI);
+      Curl[0] = zi * omega * cos(M_PI*x(2)) * exp(zi*k10*x(0));
+      Curl[1] = 0.0;
+      Curl[2] = omega * k10 / M_PI * sin(M_PI * x(2)) * exp(zi*k10*x(0));
+   }
+   else if (prob_kind == 1)
+   {
+      Curl[0] = -x(1);
+      Curl[1] = -x(2);
+      Curl[2] = -x(0);
+   }
+   else
+   {
+      Curl[0] = x(0)*x(2) - x(0)*x(0)*x(0)*x(1);
+      Curl[1] = (2.0*x(0)-1.0)*x(1)*x(2);
+      Curl[2] = x(0)*x(2)*(3.0*x(0)*x(1)-x(2));
+   }
+   
+}
+
+void maxwell_curlcurl(const Vector &x, vector<complex<double>> &CurlCurl)
+{
+   // Initialize
+   int dim = x.Size();
+   for (int i = 0; i < dim; ++i)
+   {
+      CurlCurl[i] = 0.0;
+   }
+
+   if (prob_kind == 0)
+   {
+      complex<double> zi = complex<double>(0., 1.);
+      double k10 = sqrt(omega * omega - M_PI * M_PI);
+   // complex<double> f = -zi * omega / M_PI * sin(M_PI*x(2))*exp(zi * k10 * x(0));
+   // complex<double> f_x = omega * k10 /M_PI * sin(M_PI*x(2))*exp(zi * k10 * x(0));
+      complex<double> f_xx = zi * omega * k10 * k10 /M_PI * sin(M_PI*x(2))*exp(zi * k10 * x(0));
+      complex<double> f_xy = 0.0;
+   // complex<double> f_z = -zi * omega * cos(M_PI*x(2))*exp(zi * k10 * x(0));
+      complex<double> f_zy = 0.0;
+      complex<double> f_zz = zi * omega * M_PI * sin(M_PI*x(2))*exp(zi * k10 * x(0));
+      CurlCurl[0] = f_xy;
+      CurlCurl[1] = -f_zz - f_xx;
+      CurlCurl[2] = f_zy;
+   }
+   else if (prob_kind == 1)
+   {
+      CurlCurl[0] = 1.0;
+      CurlCurl[1] = 1.0;
+      CurlCurl[2] = 1.0;
+   }
+   else
+   {
+      CurlCurl[0] = 3.0*x(0)*x(0)*x(2) - 2*x(0)*x(1) + x(1);
+      CurlCurl[1] = -6*x(0)*x(1)*x(2) + x(0) + x(2)*x(2);
+      CurlCurl[2] = x(0)*x(0)*x(0) + 2.0*x(1)*x(2);
+   }
+}
+
+void E_exact_Re(const Vector &x, Vector &E)
+{
+   int dim = x.Size();
+   vector<complex<double>> Eval(E.Size());
+   maxwell_solution(x, Eval);
+   for (int i = 0; i < dim; ++i)
+   {
+      E[i] = Eval[i].real();
+   }
+}
+
+void E_exact_Im(const Vector &x, Vector &E)
+{
+   int dim = x.Size();
+   vector<complex<double>> Eval(E.Size());
+   maxwell_solution(x, Eval);
+   for (int i = 0; i < dim; ++i)
+   {
+      E[i] = Eval[i].imag();
+   }
+}
+
+void Curl_exact_Re(const Vector &x, Vector &Curl)
+{
+   int dim = x.Size();
+   vector<complex<double>> Eval(Curl.Size());
+   maxwell_curl(x, Eval);
+   for (int i = 0; i < dim; ++i)
+   {
+      Curl[i] = Eval[i].real();
+   }
+}
+
+void Curl_exact_Im(const Vector &x, Vector &Curl)
+{
+   int dim = x.Size();
+   vector<complex<double>> Eval(Curl.Size());
+   maxwell_curl(x, Eval);
+   for (int i = 0; i < dim; ++i)
+   {
+      Curl[i] = Eval[i].imag();
+   }
+}
+
+void f_exact_Re(const Vector &x, Vector &E)
+{
+   int dim = x.Size();
+   vector<complex<double>> Eval(E.Size());
+   vector<complex<double>> CurlCurl(E.Size());
+   maxwell_solution(x, Eval);
+   maxwell_curlcurl(x, CurlCurl);
+   for (int i = 0; i < dim; ++i)
+   {
+      E[i] = (CurlCurl[i] - omega * omega * Eval[i]).real();
+   }
+}
+
+void f_exact_Im(const Vector &x, Vector &E)
+{
+   int dim = x.Size();
+   vector<complex<double>> Eval(E.Size());
+   vector<complex<double>> CurlCurl(E.Size());
+   maxwell_solution(x, Eval);
+   maxwell_curlcurl(x, CurlCurl);
+   for (int i = 0; i < dim; ++i)
+   {
+      E[i] = (CurlCurl[i] - omega * omega * Eval[i]).imag();
+   }
+}

examples/maxwell-solver/maxwell.cpp

@@ -0,0 +1,339 @@
+//
+// Compile with: make maxwell
+//
+//               maxwell -o 2 -f 8.0 -ref 3 -prob 4 -m ../data/inline-quad.mesh
+//
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "DST/DST.hpp"
+
+using namespace std;
+using namespace mfem;
+
+void source_re(const Vector &x, Vector & f);
+void source_im(const Vector &x, Vector & f);
+double wavespeed(const Vector &x);
+
+
+
+double mu = 1.0;
+double epsilon = 1.0;
+double omega;
+int dim;
+double length = 1.0;
+Array2D<double> comp_bdr;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   int ref_levels = 3;
+   double freq = 5.0;
+   bool herm_conv = true;
+   bool visualization = 1;
+   int nd=2;
+   int nx=2;
+   int ny=2;
+   int nz=2;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&nd, "-nd", "--dim","Problem space dimension");
+   args.AddOption(&nx, "-nx", "--nx","Number of subdomains in x direction");
+   args.AddOption(&ny, "-ny", "--ny","Number of subdomains in y direction");
+   args.AddOption(&nz, "-nz", "--nz","Number of subdomains in z direction");               
+   args.AddOption(&ref_levels, "-ref", "--refinements",
+                  "Number of refinements");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&freq, "-f", "--frequency",
+                  "Frequency (in Hz).");
+   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
+                  "--no-hermitian", "Use convention for Hermitian operators.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+
+
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+
+   Mesh *mesh;
+
+   if (nd == 2)
+   {
+      mesh = new Mesh(4, 4, Element::QUADRILATERAL, true, length, length, false);
+   }
+   else
+   {
+      mesh = new Mesh(1, 1, 1, Element::HEXAHEDRON, true, length, length, length,false);
+   }
+
+   dim = mesh->Dimension();
+
+   // Angular frequency
+   omega = 2.0 * M_PI * freq;
+
+   // 4. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // char vishost[] = "localhost";
+   // int visport = 19916;
+
+   // socketstream mesh_sock(vishost, visport);
+   // mesh_sock.precision(8);
+   // mesh_sock << "mesh\n"
+   //           << *mesh << "window_title 'Global mesh'" << flush;
+
+   // Setup PML length
+   int nrlayers = 2;
+   double hl = GetUniformMeshElementSize(mesh);
+   Array2D<double> lengths(dim, 2); 
+   lengths = hl*nrlayers;
+
+   CartesianPML * pml = new CartesianPML(mesh,lengths);
+   pml->SetOmega(omega);
+   comp_bdr.SetSize(dim,2);
+   comp_bdr = pml->GetCompDomainBdr(); 
+
+
+   // 6. Define a finite element space on the mesh. Here we use the Nedelec
+   //    finite elements of the specified order.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   int size = fespace->GetTrueVSize();
+   cout << "Number of finite element unknowns: " << size << endl;
+
+   // 7. Determine the list of true essential boundary dofs. In this example,
+   //    the boundary conditions are defined based on the specific mesh and the
+   //    problem type.
+   Array<int> ess_tdof_list;
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 8. Setup Complex Operator convention
+   ComplexOperator::Convention conv =
+      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
+
+   // 9. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system.
+   VectorFunctionCoefficient f_re(dim, source_re);
+   VectorFunctionCoefficient f_im(dim, source_re);
+   ComplexLinearForm b(fespace, conv);
+   b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(f_re),
+                         new VectorFEDomainLFIntegrator(f_im));
+   b.Vector::operator=(0.0);
+   b.Assemble();
+
+   // 10. Define the solution vector x as a complex finite element grid function
+   //     corresponding to fespace.
+   ComplexGridFunction x(fespace);
+   x = 0.0;
+   // 11. Set up the sesquilinear form a(.,.)
+   //
+   //       1/mu (1/det(J) J^T J Curl E, Curl F)
+   //        - omega^2 * epsilon (det(J) * (J^T J)^-1 * E, F)
+   //
+   FunctionCoefficient ws(wavespeed);
+   ConstantCoefficient omeg(-pow(omega, 2));
+   int cdim = (dim == 2) ? 1 : dim;
+   PmlMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PmlMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+
+   PmlMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PmlMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarMatrixProductCoefficient c2_Re0(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im0(omeg,pml_c2_Im);
+   ScalarMatrixProductCoefficient c2_Re(ws,c2_Re0);
+   ScalarMatrixProductCoefficient c2_Im(ws,c2_Im0);
+
+   SesquilinearForm a(fespace, conv);
+   a.AddDomainIntegrator(new CurlCurlIntegrator(pml_c1_Re),
+                         new CurlCurlIntegrator(pml_c1_Im));
+   a.AddDomainIntegrator(new VectorFEMassIntegrator(c2_Re),
+                         new VectorFEMassIntegrator(c2_Im));
+
+   a.Assemble(0);
+
+   OperatorHandle Ah;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);
+
+   ComplexSparseMatrix * Ac = Ah.As<ComplexSparseMatrix>();
+   StopWatch chrono;
+   // chrono.Clear();
+   // chrono.Start();
+   // {
+   //    ComplexUMFPackSolver csolver;
+   //    csolver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   //    csolver.SetOperator(*Ac);
+   //    // csolver.SetPrintLevel(2);
+   //    csolver.Mult(B,X);
+   // }
+   // chrono.Stop();
+   // cout << "Time 1 = " << chrono.RealTime() << endl;
+
+
+   chrono.Clear();
+   chrono.Start();
+   DST S(&a,lengths, omega, &ws, nrlayers, nx, ny, nz);
+   chrono.Stop();
+   cout << "Time 2 = " << chrono.RealTime() << endl;
+   chrono.Clear();
+   chrono.Start();
+   X = 0.0;
+	GMRESSolver gmres;
+	// gmres.iterative_mode = true;
+   gmres.SetPreconditioner(S);
+	gmres.SetOperator(*Ac);
+	gmres.SetRelTol(1e-8);
+	gmres.SetMaxIter(50);
+	gmres.SetPrintLevel(1);
+	gmres.Mult(B, X);
+   chrono.Stop();
+   cout << "Time 3 = " << chrono.RealTime() << endl;
+   // 14. Solve using a direct or an iterative solver
+   // Vector Y(X);
+
+   // chrono.Stop();
+   // cout << "Time 3 = " << chrono.RealTime() << endl;
+
+   // cout << endl;
+
+   // cout << "X norm = " << X.Norml2() << endl;
+   // cout << "Y norm = " << Y.Norml2() << endl;
+   // Y-=X;
+   // cout << "diff norm = " << Y.Norml2() << endl;
+
+
+
+   a.RecoverFEMSolution(X, b, x);
+
+   // 17. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      // Define visualization keys for GLVis (see GLVis documentation)
+      string keys;
+      keys = (dim == 3) ? "keys acF\n" : keys = "keys amrRljcUUuu\n";
+
+      char vishost[] = "localhost";
+      int visport = 19916;
+
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re.precision(8);
+      sol_sock_re << "solution\n"
+                  << *mesh << x.real() << keys
+                  << "window_title 'Solution real part'" << flush;
+
+      socketstream sol_sock_im(vishost, visport);
+      sol_sock_im.precision(8);
+      sol_sock_im << "solution\n"
+                  << *mesh << x.imag() << keys
+                  << "window_title 'Solution imag part'" << flush;
+
+      GridFunction x_t(fespace);
+      x_t = x.real();
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n"
+               << *mesh << x_t << keys << "autoscale off\n"
+               << "window_title 'Harmonic Solution (t = 0.0 T)'"
+               << "pause\n" << flush;
+      cout << "GLVis visualization paused."
+           << " Press space (in the GLVis window) to resume it.\n";
+      int num_frames = 32;
+      int i = 0;
+      while (sol_sock)
+      {
+         double t = (double)(i % num_frames) / num_frames;
+         ostringstream oss;
+         oss << "Harmonic Solution (t = " << t << " T)";
+
+         add(cos(2.0 * M_PI * t), x.real(),
+             sin(2.0 * M_PI * t), x.imag(), x_t);
+         sol_sock << "solution\n"
+                  << *mesh << x_t
+                  << "window_title '" << oss.str() << "'" << flush;
+         i++;
+      }
+   }
+
+   // 18. Free the used memory.
+   delete pml;
+   delete fespace;
+   delete fec;
+   delete mesh;
+   return 0;
+}
+
+void source_re(const Vector &x, Vector &f)
+{
+   f = 0.0;
+   double x0 = length/2.0;
+   double x1 = length/2.0;
+   double x2 = length/2.0;
+   x0 = 0.45;
+   x1 = 0.35;
+   x2 = 0.25;
+   double alpha,beta;
+   double n = 4.0*omega/M_PI;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   double coeff = 16.0*omega*omega/M_PI/M_PI/M_PI;
+   alpha = -pow(n,2) * beta;
+   f[0] = coeff*exp(alpha);
+   // f[1] = coeff*exp(alpha);
+
+
+
+   x0 = 0.8;
+   x1 = 0.8;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   alpha = -pow(n,2) * beta;
+   // f[0] += coeff*exp(alpha);
+
+
+   bool in_pml = false;
+   for (int i = 0; i<dim; i++)
+   {
+      if (x(i)<=comp_bdr(i,0) || x(i)>=comp_bdr(i,1))
+      {
+         in_pml = true;
+         break;
+      }
+   }
+   if (in_pml) f = 0.0;
+}
+
+void source_im(const Vector &x, Vector &f)
+{
+   f = 0.0;
+}
+
+double wavespeed(const Vector &x)
+{
+   double ws;
+   ws = 1.0;
+   return ws;
+}

examples/maxwell-solver/maxwellp.cpp

@@ -0,0 +1,614 @@
+//
+// Compile with: make maxwellp
+//
+//               mpirun -np 4 ./maxwellp -o 2 -f 8.0 -sr 3 -m ../../data/inline-quad.mesh
+//
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "ParDST/ParDST.hpp"
+#include "common/PML.hpp"
+
+using namespace std;
+using namespace mfem;
+  
+void source_re(const Vector &x, Vector & f);
+void source_im(const Vector &x, Vector & f);
+void exact_re(const Vector & x, Vector & E);
+void exact_im(const Vector & x, Vector & E);
+void maxwell_solution(const Vector & x, double E[], double curl2E[]);
+double wavespeed(const Vector &x);
+void Mwavespeed(const Vector & x, DenseMatrix & M);
+
+void ess_data_func(const Vector & x, Vector & E);
+
+
+double mu = 1.0;
+double epsilon = 1.0;
+double omega;
+int dim;
+double length = 1.0;
+double sigma_ = 0.0;
+
+Array2D<double> comp_bdr;
+Array2D<double> domain_bdr;
+bool exact_known = false;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   // number of serial refinements
+   int ser_ref_levels = 1;
+   // number of parallel refinements
+   int par_ref_levels = 2;
+   double freq = 5.0;
+   int bc_type = 1;
+   bool herm_conv = true;
+   bool visualization = 1;
+   int nd=2;
+   int nx=2;
+   int ny=2;
+   int nz=2;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&nd, "-nd", "--dim","Problem space dimension");
+   args.AddOption(&nx, "-nx", "--nx","Number of subdomains in x direction");
+   args.AddOption(&ny, "-ny", "--ny","Number of subdomains in y direction");
+   args.AddOption(&nz, "-nz", "--nz","Number of subdomains in z direction");               
+   args.AddOption(&ser_ref_levels, "-sr", "--ser_ref_levels",
+                  "Number of Serial Refinements.");
+   args.AddOption(&par_ref_levels, "-pr", "--par_ref_levels",
+                  "Number of Parallel Refinements."); 
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&sigma_, "-sigma", "--damping-coef",
+                  "Damping coefficient (or sigma).");          
+   args.AddOption(&bc_type, "-bct", "--bc-type",
+                  "BC type - 0:Neumann, 1: Dirichlet");                        
+   args.AddOption(&freq, "-f", "--frequency",
+                  "Frequency (in Hz).");
+   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
+                  "--no-hermitian", "Use convention for Hermitian operators.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // Angular frequency
+   omega = 2.0 * M_PI * freq;
+
+   Mesh *mesh;
+
+
+   int nel = 1;
+   if (nd == 2)
+   {
+      mesh = new Mesh(nel, nel, Element::QUADRILATERAL, true, length, length, false);
+   }
+   else
+   {
+      mesh = new Mesh(nel, nel, nel, Element::HEXAHEDRON, true, length, length, length,false);
+   }
+
+   dim = mesh->Dimension();
+
+   // 4. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ser_ref_levels; l++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 4. Define a parallel mesh by a partitioning of the serial mesh.
+   // ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   int nprocs;
+   int nprocsx;
+   int nprocsy;
+   int nprocsz;
+   if (dim == 2)
+   {
+      nprocs = sqrt(num_procs); 
+      nprocsx = nprocs;
+      nprocsy = nprocs;
+      nprocsz = 1;
+   }    
+   else
+   {
+      nprocs = cbrt(num_procs); 
+      nprocsx = nprocs;
+      nprocsy = nprocs;
+      nprocsz = nprocs;
+   }
+   int nxyz[3] = {nprocsx,nprocsy,nprocsz};
+   int * part = mesh->CartesianPartitioning(nxyz);
+   // ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD,*mesh,part);
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD,*mesh);
+   delete [] part;
+
+   
+   delete mesh;
+
+   for (int l = 0; l < par_ref_levels; l++)
+   {
+      pmesh->UniformRefinement();
+   }
+
+   //    char vishost[] = "localhost";
+   // int visport = 19916;
+   // socketstream mesh_sock1(vishost, visport);
+   // mesh_sock1.precision(8);
+   // mesh_sock1 << "parallel " << num_procs << " " << myid << "\n"
+   //            << "mesh\n"
+   //            << *pmesh << "window_title 'Global mesh'" << flush;
+
+   double hl = GetUniformMeshElementSize(pmesh);
+   int nrlayers = 3;
+   Array2D<double> lengths(dim,2);
+   lengths = hl*nrlayers;
+   // lengths[0][1] = 0.0;
+   // lengths[1][1] = 0.0;
+   // lengths[1][0] = 0.0;
+   // lengths[0][0] = 0.0;
+   if (exact_known) lengths = 0.0;
+   // CartesianPML pml(mesh,lengths);
+   CartesianPML pml(pmesh,lengths);
+   pml.SetAttributes(pmesh);
+   pml.SetOmega(omega);
+   comp_bdr.SetSize(dim,2);
+   comp_bdr = pml.GetCompDomainBdr(); 
+
+
+   // 6. Define a finite element space on the mesh. Here we use the Nedelec
+   //    finite elements of the specified order.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true essential boundary dofs. In this example,
+   //    the boundary conditions are defined based on the specific mesh and the
+   //    problem type.
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = bc_type;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+
+   Array<int> attr;
+   Array<int> attrPML;
+   if (pmesh->attributes.Size())
+   {
+      attr.SetSize(pmesh->attributes.Max());
+      attrPML.SetSize(pmesh->attributes.Max());
+      attr = 0;   attr[0] = 1;
+      attrPML = 0;
+      if (pmesh->attributes.Max() > 1)
+      {
+         attrPML[1] = 1;
+      }
+   }
+
+   // 8. Setup Complex Operator convention
+   ComplexOperator::Convention conv =
+      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
+
+   // 9. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system.
+   VectorFunctionCoefficient f_re(dim, source_re);
+   VectorFunctionCoefficient f_im(dim, source_re);
+   ParComplexLinearForm b(fespace, conv);
+   b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(f_re),
+                         new VectorFEDomainLFIntegrator(f_im));
+   b.Vector::operator=(0.0);
+   b.Assemble();
+
+   // 10. Define the solution vector x as a complex finite element grid function
+   //     corresponding to fespace.
+   ParComplexGridFunction x(fespace);
+   x = 0.0;
+   // VectorFunctionCoefficient done(dim,ess_data_func);
+   // x.ProjectCoefficient(done,done);
+   VectorFunctionCoefficient E_re(dim,exact_re);
+   VectorFunctionCoefficient E_im(dim,exact_re);
+   if (exact_known)
+   {
+      x.ProjectCoefficient(E_re,E_re);
+   }
+   // 11. Set up the sesquilinear form a(.,.)
+   //
+   //       1/mu (1/det(J) J^T J Curl E, Curl F)
+   //        - omega^2 * epsilon (det(J) * (J^T J)^-1 * E, F)
+   //
+   FunctionCoefficient ws(wavespeed);
+
+   // MatrixFunctionCoefficient Mws(dim,Mwavespeed);
+
+   // DenseMatrix M(dim); M = 0.0;
+   // M(0,0) = -pow(omega, 2);
+   // M(1,1) = -pow(omega, 2);
+   // M(2,2) = -pow(omega, 2);
+   // MatrixConstantCoefficient Momeg(M);
+   MatrixFunctionCoefficient eps_func(dim,Mwavespeed);
+
+   ConstantCoefficient muinv(1.0/mu);
+   ConstantCoefficient omeg(-pow(omega, 2) * epsilon);
+   ConstantCoefficient lossCoef(-omega * sigma_);
+   RestrictedCoefficient restr_loss(lossCoef,attr);
+   RestrictedCoefficient restr_muinv(muinv,attr);
+   RestrictedCoefficient restr_omeg(omeg,attr);
+
+   // Integrators inside the computational domain (excluding the PML region)
+   ParSesquilinearForm a(fespace, conv);
+   a.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv),NULL);
+   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);
+   a.AddDomainIntegrator(NULL, new VectorFEMassIntegrator(lossCoef));                         
+   // a.AddDomainIntegrator(NULL, new VectorFEMassIntegrator(restr_loss));                         
+
+
+
+   // int cdim = (dim == 2) ? 1 : dim;
+   // PmlMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, &pml);
+   // PmlMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, &pml);
+
+   // PmlMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,&pml);
+   // PmlMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,&pml);
+   // ScalarMatrixProductCoefficient c2_Re0(omeg,pml_c2_Re);
+   // ScalarMatrixProductCoefficient c2_Im0(omeg,pml_c2_Im);
+
+   // MatrixMatrixProductCoefficient c2_Re(c2_Re0,eps_func);
+   // MatrixMatrixProductCoefficient c2_Im(c2_Im0,eps_func);
+
+   int cdim = (dim == 2) ? 1 : dim;
+   PmlMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, &pml);
+   PmlMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, &pml);
+   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+
+   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+
+   PmlMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,&pml);
+   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+
+   // Integrators inside the PML region
+   a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
+                         new CurlCurlIntegrator(restr_c1_Im));
+   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
+                         new VectorFEMassIntegrator(restr_c2_Im));
+
+
+   a.Assemble(0);
+
+   OperatorHandle Ah;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);
+
+   ComplexSparseMatrix * Ac = Ah.As<ComplexSparseMatrix>();
+   StopWatch chrono;
+
+
+   chrono.Clear();
+   chrono.Start();
+
+   ParDST::BCType bct = (bc_type == 1)? ParDST::BCType::DIRICHLET : ParDST::BCType::NEUMANN;
+   ParDST * S = new ParDST(&a,lengths, omega, &ws, nrlayers, nx, ny, nz, bct, &lossCoef);
+   chrono.Stop();
+   double t1 = chrono.RealTime();
+
+   chrono.Clear();
+   chrono.Start();
+   // X = 0.0;
+	GMRESSolver gmres(MPI_COMM_WORLD);
+	// gmres.iterative_mode = true;
+   gmres.SetPreconditioner(*S);
+	gmres.SetOperator(*Ac);
+	gmres.SetRelTol(1e-8);
+	gmres.SetMaxIter(100);
+	gmres.SetPrintLevel(1);
+	gmres.Mult(B, X);
+   delete S;
+   chrono.Stop();
+   double t2 = chrono.RealTime();
+
+   MPI_Barrier(MPI_COMM_WORLD);
+
+
+   cout << " myid: " << myid 
+         << ", setup time: " << t1
+         << ", solution time: " << t2 << endl; 
+
+   // {
+   //    HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+   //    SuperLURowLocMatrix SA(*A);
+   //    SuperLUSolver superlu(MPI_COMM_WORLD);
+   //    superlu.SetPrintStatistics(false);
+   //    superlu.SetSymmetricPattern(false);
+   //    superlu.SetColumnPermutation(superlu::PARMETIS);
+   //    superlu.SetOperator(SA);
+   //    superlu.Mult(B, X);
+   //    delete A;
+   // }
+
+   a.RecoverFEMSolution(X, b, x);
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      string keys;
+      if (dim ==2 )
+      {
+         keys = "keys mrRljcUUuuu\n";
+      }
+      else
+      {
+         keys = "keys mc\n";
+      }
+      // socketstream mesh_sock(vishost, visport);
+      // mesh_sock.precision(8);
+      // mesh_sock << "parallel " << num_procs << " " << myid << "\n"
+      //             << "mesh\n" << *pmesh  << flush;
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re.precision(8);
+      sol_sock_re << "parallel " << num_procs << " " << myid << "\n"
+                  << "solution\n" << *pmesh << x.real() << keys 
+                  << "window_title 'E: Real Part' " << flush;                     
+
+      socketstream sol_sock_im(vishost, visport);
+      sol_sock_im.precision(8);
+      sol_sock_im << "parallel " << num_procs << " " << myid << "\n"
+                  << "solution\n" << *pmesh << x.imag() << keys 
+                  << "window_title 'E: Imag Part' " << flush;   
+
+
+      {
+         ParGridFunction x_t(fespace);
+         x_t = x.real();
+
+         socketstream sol_sock(vishost, visport);
+         sol_sock.precision(8);
+         sol_sock << "parallel " << num_procs << " " << myid << "\n"
+                  << "solution\n" << *pmesh << x_t << keys << "autoscale off\n"
+                  << "window_title 'Harmonic Solution (t = 0.0 T)'"
+                  << "pause\n" << flush;
+
+         if (myid == 0)
+         {
+            cout << "GLVis visualization paused."
+                 << " Press space (in the GLVis window) to resume it.\n";
+         }
+
+         int num_frames = 32;
+         int i = 0;
+         while (sol_sock)
+         {
+            double t = (double)(i % num_frames) / num_frames;
+            ostringstream oss;
+            oss << "Harmonic Solution (t = " << t << " T)";
+
+            add(cos(2.0*M_PI*t), x.real(), sin(2.0*M_PI*t), x.imag(), x_t);
+            sol_sock << "parallel " << num_procs << " " << myid << "\n";
+            sol_sock << "solution\n" << *pmesh << x_t
+                     << "window_title '" << oss.str() << "'" << flush;
+            i++;
+         }
+      }
+   }
+
+   // 18. Free the used memory.
+   delete fespace;
+   delete fec;
+   delete pmesh;
+   MPI_Finalize();
+   return 0;
+}
+
+void source_re(const Vector &x, Vector &f)
+{
+   f = 0.0;
+   if (exact_known)
+   {
+      double E[3], curl2E[3];
+      maxwell_solution(x, E, curl2E);
+      // curl ( curl E) +/- omega^2 E = f
+      double coeff = -omega * omega;
+      f(0) = curl2E[0] + coeff * E[0];
+      f(1) = curl2E[1] + coeff * E[1];
+      if (dim == 2)
+      {
+         if (x.Size() == 3) {f(2)=0.0;}
+      }
+      else
+      {
+         f(2) = curl2E[2] + coeff * E[2];
+      }
+   }
+   else
+   {
+      int nrsources = (dim == 2) ? 4 : 8;
+      Vector x0(nrsources);
+      Vector y0(nrsources);
+      Vector z0(nrsources);
+      x0(0) = 0.25; y0(0) = 0.25; z0(0) = 0.25;
+      x0(1) = 0.75; y0(1) = 0.25; z0(1) = 0.25;
+      x0(2) = 0.25; y0(2) = 0.75; z0(2) = 0.25;
+      x0(3) = 0.75; y0(3) = 0.75; z0(3) = 0.25;
+      if (dim == 3)
+      {
+         x0(4) = 0.25; y0(4) = 0.25; z0(4) = 0.75;
+         x0(5) = 0.75; y0(5) = 0.25; z0(5) = 0.75;
+         x0(6) = 0.25; y0(6) = 0.75; z0(6) = 0.75;
+         x0(7) = 0.75; y0(7) = 0.75; z0(7) = 0.75;
+      }
+  
+      double n = 4.0*omega/M_PI;
+      double coeff = 16.0*omega*omega/M_PI/M_PI/M_PI;
+
+      // for (int i = 0; i<nrsources; i++)
+      x0(0) = 0.5; y0(0) = 0.5; 
+      for (int i = 0; i<1; i++)
+      {
+         double beta = pow(x0(i)-x(0),2) + pow(y0(i)-x(1),2);
+         if (dim == 3) { beta += pow(z0(i)-x(2),2); }
+         double alpha = -pow(n,2) * beta;
+         f[0] += coeff*exp(alpha);
+      }
+
+      bool in_pml = false;
+      for (int i = 0; i<dim; i++)
+      {
+         if (x(i)<=comp_bdr(i,0) || x(i)>=comp_bdr(i,1))
+         {
+            in_pml = true;
+            break;
+         }
+      }
+      if (in_pml) f = 0.0;
+   }
+}
+
+void source_im(const Vector &x, Vector &f)
+{
+   f = 0.0;
+}
+
+double wavespeed(const Vector &x)
+{
+   double ws;
+   ws = 1.0;
+   return ws;
+}
+
+void Mwavespeed(const Vector & x, DenseMatrix & M)
+{
+   M = 0.0;
+   M(0,0) = 1.0;
+   M(1,1) = 1.0;
+   // M(2,2) = 4.0*x(0)-1.0;
+   if (dim == 3) M(2,2) = 1.0;
+}
+
+
+void exact_re(const Vector & x, Vector & E)
+{
+   double curl2E[3];
+   maxwell_solution(x, E, curl2E);
+}
+void exact_im(const Vector & x, Vector & E)
+{
+   // double curl2E[3];
+   // maxwell_solution(x, E, curl2E);
+   E = 0.0;
+}
+void maxwell_solution(const Vector & x, double E[], double curl2E[])
+{
+   // point source
+   if (dim == 2)
+   {
+      // shift to avoid singularity
+      double x0 = x(0) + 0.1;
+      double x1 = x(1) + 0.1;
+      //
+      double r = sqrt(x0 * x0 + x1 * x1);
+
+      E[0] = cos(omega * r);
+      E[1] = 0.0;
+
+      double r_x = x0 / r;
+      double r_y = x1 / r;
+      double r_xy = -(r_x / r) * r_y;
+      double r_yx = r_xy;
+      double r_yy = (1.0 / r) * (1.0 - r_y * r_y);
+
+      curl2E[0] = omega * ((r_yy ) * sin(omega * r) + (omega * r_y * r_y) * cos(omega * r));
+      curl2E[1] = -omega * (r_yx * sin(omega * r) + omega * r_y * r_x * cos(omega * r));
+      curl2E[2] = 0.0;
+   }
+   else
+   {
+   // shift to avoid singularity
+      double x0 = x(0) + 0.1;
+      double x1 = x(1) + 0.1;
+      double x2 = x(2) + 0.1;
+      //
+      double r = sqrt(x0 * x0 + x1 * x1 + x2 * x2);
+
+      E[0] = cos(omega * r);
+      E[1] = 0.0;
+      E[2] = 0.0;
+
+      double r_x = x0 / r;
+      double r_y = x1 / r;
+      double r_z = x2 / r;
+      double r_xy = -(r_x / r) * r_y;
+      double r_xz = -(r_x / r) * r_z;
+      double r_yx = r_xy;
+      double r_yy = (1.0 / r) * (1.0 - r_y * r_y);
+      double r_zx = r_xz;
+      double r_zz = (1.0 / r) * (1.0 - r_z * r_z);
+
+      curl2E[0] = omega * ((r_yy + r_zz) * sin(omega * r) +
+                           (omega * r_y * r_y + omega * r_z * r_z) * cos(omega * r));
+      curl2E[1] = -omega * (r_yx * sin(omega * r) + omega * r_y * r_x * cos(omega * r));
+      curl2E[2] = -omega * (r_zx * sin(omega * r) + omega * r_z * r_x * cos(omega * r));
+   }
+}
+
+
+void ess_data_func(const Vector & x, Vector & E)
+{
+   E = 0.0;
+   // if (x(0)==0.0) E[0] = sin(x(0)+x(1));
+   if (x(1)==0.0) E[0] = sin(x(0)+x(1));
+
+
+   bool in_pml = false;
+   for (int i = 0; i<dim; i++)
+   {
+      if (x(i)<comp_bdr(i,0) || x(i)>comp_bdr(i,1))
+      {
+         in_pml = true;
+         break;
+      }
+   }
+   if (in_pml) E = 0.0;
+
+}

examples/maxwell-solver/pml_torus.cpp

@@ -0,0 +1,840 @@
+
+
+// sample runs: ./pml_torus -prob 2 -ref 2 -o 2 -f 0.6
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "common/PML.hpp"
+
+using namespace std;
+using namespace mfem;
+
+void maxwell_solution(const Vector &x, vector<complex<double>> &E);
+void maxwell_curl(const Vector &x, vector<complex<double>> &curlE);
+
+int prob_kind=0;
+double L;
+double ylim;
+
+// Class for returning the PML coefficients of the bilinear form
+class PMLDiagMatrixCoefficient : public VectorCoefficient
+{
+private:
+   ToroidPML * pml = nullptr;
+   void (*Function)(const Vector &, ToroidPML * , Vector &);
+public:
+   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, ToroidPML *,
+                                              Vector &),
+                            ToroidPML * pml_)
+      : VectorCoefficient(dim), pml(pml_), Function(F)
+   {}
+
+   using VectorCoefficient::Eval;
+
+   virtual void Eval(Vector &K, ElementTransformation &T,
+                     const IntegrationPoint &ip)
+   {
+      double x[3];
+      Vector transip(x, 3);
+      T.Transform(ip, transip);
+      K.SetSize(vdim);
+      (*Function)(transip, pml, K);
+   }
+};
+
+// class PMLMatrixCoefficient : public MatrixCoefficient
+// {
+// private:
+//    ToroidPML * pml = nullptr;
+//    void (*Function)(const Vector &, ToroidPML * , DenseMatrix &);
+// public:
+//    PMLMatrixCoefficient(int dim, void(*F)(const Vector &, ToroidPML *,
+//                                               DenseMatrix &),
+//                             ToroidPML * pml_)
+//       : MatrixCoefficient(dim), pml(pml_), Function(F)
+//    {}
+
+//    using MatrixCoefficient::Eval;
+
+//    virtual void Eval(DenseMatrix &M, ElementTransformation &T,
+//                      const IntegrationPoint &ip)
+//    {
+//       double x[3];
+//       Vector transip(x, 3);
+//       T.Transform(ip, transip);
+//       M.SetSize(height,width);
+//       (*Function)(transip, pml, M);
+//    }
+// };
+
+
+void E_bdr_data_Re(const Vector &x, Vector &E);
+void E_bdr_data_Im(const Vector &x, Vector &E);
+
+void E_exact_Re(const Vector &x, Vector &E);
+void E_exact_Im(const Vector &x, Vector &E);
+
+void E_exact_Curl_Re(const Vector &x, Vector &E);
+void E_exact_Curl_Im(const Vector &x, Vector &E);
+
+void source(const Vector &x, Vector & f);
+
+// Functions for computing the necessary coefficients after PML stretching.
+// J is the Jacobian matrix of the stretching function
+void detJ_JT_J_inv_Re(const Vector &x, ToroidPML * pml, Vector &D);
+void detJ_JT_J_inv_Im(const Vector &x, ToroidPML * pml, Vector &D);
+void detJ_inv_JT_J_Re(const Vector &x, ToroidPML * pml, Vector &D);
+void detJ_inv_JT_J_Im(const Vector &x, ToroidPML * pml, Vector &D);
+
+
+// void detJ_JT_J_inv_Re(const Vector &x, ToroidPML * pml, DenseMatrix & M);
+// void detJ_JT_J_inv_Im(const Vector &x, ToroidPML * pml, DenseMatrix & M);
+// void detJ_inv_JT_J_Re(const Vector &x, ToroidPML * pml, DenseMatrix & M);
+// void detJ_inv_JT_J_Im(const Vector &x, ToroidPML * pml, DenseMatrix & M);
+
+
+
+Array2D<double> comp_domain_bdr;
+Array2D<double> domain_bdr;
+
+double mu = 1.0;
+double epsilon = 1.0;
+double omega;
+int dim;
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_SELF, &num_procs);
+   MPI_Comm_rank(MPI_COMM_SELF, &myid);
+   // 1. Parse command-line options.
+   // const char *mesh_file = "torus1_4.mesh";
+   // const char *mesh_file = "waveguide-bend2.mesh";
+   const char *mesh_file = "meshes/waveguide-bend.mesh";
+
+   int order = 1;
+   int ref_levels = 1;
+   double freq = 5.0;
+   bool herm_conv = true;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&prob_kind, "-prob", "--problem-kind",
+                  "Problem/mesh choice");                  
+   args.AddOption(&ref_levels, "-ref", "--refinements",
+                  "Number of refinements");
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&freq, "-f", "--frequency",
+                  "Frequency (in Hz).");
+   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
+                  "--no-hermitian", "Use convention for Hermitian operators.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+
+   // 2. Setup the mesh
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+
+   switch (prob_kind)
+   {
+   case 0: 
+   {
+      mesh_file = "meshes/waveguide-bend.mesh"; 
+      L = -2.;
+      ylim = -3;
+   }
+   break;
+   case 1: 
+   {
+      mesh_file = "meshes/waveguide-bend2.mesh";
+      L = -5.;
+      ylim = 0.0;
+   }
+   break;
+   case 2: mesh_file = "meshes/toroid3_4_2.mesh"; break;
+   // case 3: mesh_file = "toroid-hex-o3-s0_r.mesh"; break;
+   // case 3: mesh_file = "../../data/square-disc.mesh"; break;
+   case 3: mesh_file = "meshes/annulus-quad-o3.mesh"; break;
+   // case 3: mesh_file = "cylinder.mesh"; break;
+   default:
+      MFEM_ABORT("Not a valid problem choice ");
+      break;
+   }
+
+   Mesh * mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+
+   mesh->RemoveInternalBoundaries();
+
+   // Angular frequency
+   omega = 2.0 * M_PI * freq;
+
+   ToroidPML tpml(mesh);
+   Vector zlim, rlim, alim;
+   tpml.GetDomainBdrs(zlim,rlim,alim);
+   Vector zpml_thickness(2); zpml_thickness = 0.0;
+   Vector rpml_thickness(2); rpml_thickness = 0.0;
+   Vector apml_thickness(2); apml_thickness = 0.0; 
+   bool zstretch = false;
+   bool astretch = false;
+   bool rstretch = false;
+   switch (prob_kind)
+   {
+      case 0: break;
+      case 1: break;
+      case 2: 
+      {
+         apml_thickness[1] = 45.0; 
+         astretch = true;
+      }
+      break;// degrees 
+      case 3: 
+      {
+         rpml_thickness[1] = 0.5; 
+         rstretch = true;
+      }
+      break;
+      default: break;
+   }
+   
+   tpml.SetPmlAxes(zstretch,rstretch,astretch);
+   tpml.SetPmlWidth(zpml_thickness,rpml_thickness,apml_thickness);
+   tpml.SetOmega(omega); 
+
+   ComplexOperator::Convention conv =
+      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
+
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+
+   ComplexGridFunction x(fespace);
+   x = 0.0;
+   VectorFunctionCoefficient E_Re(dim, E_bdr_data_Re);
+   VectorFunctionCoefficient E_Im(dim, E_bdr_data_Im);
+
+   ConvergenceStudy rates_r;
+   ConvergenceStudy rates_i;
+
+   for (int iter = 0; iter<ref_levels; iter++)
+   {
+      int size = fespace->GetTrueVSize();
+      cout << "Number of finite element unknowns: " << size << endl;
+      tpml.SetAttributes(mesh); 
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (mesh->bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(mesh->bdr_attributes.Max());
+         ess_bdr = 1;
+      }
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+      VectorFunctionCoefficient f(dim, source);
+      ComplexLinearForm b(fespace, conv);
+      // b.AddDomainIntegrator(NULL, new VectorFEDomainLFIntegrator(f));
+      b.Vector::operator=(0.0);
+      b.Assemble();
+
+      x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);
+
+      Array<int> attr;
+      Array<int> attrPML;
+      if (mesh->attributes.Size())
+      {
+         attr.SetSize(mesh->attributes.Max());
+         attrPML.SetSize(mesh->attributes.Max());
+         attr = 0;   attr[0] = 1;
+         attrPML = 0;
+         if (mesh->attributes.Max() > 1)
+         {
+            attrPML[1] = 1;
+         }
+      }
+
+      ConstantCoefficient muinv(1.0/mu);
+      ConstantCoefficient omeg(-pow(omega, 2) * epsilon);
+      RestrictedCoefficient restr_muinv(muinv,attr);
+      RestrictedCoefficient restr_omeg(omeg,attr);
+
+      // Integrators inside the computational domain (excluding the PML region)
+      SesquilinearForm a(fespace, conv);
+      a.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv),NULL);
+      a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);
+
+      int cdim = (dim == 2) ? 1 : dim;
+
+      PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, &tpml);
+      PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, &tpml);
+      ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+      ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+      MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+      MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+
+      PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,&tpml);
+      PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,&tpml);
+      ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+      ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+      MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+      MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+
+      // Integrators inside the PML region
+      a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
+                           new CurlCurlIntegrator(restr_c1_Im));
+      a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
+                           new VectorFEMassIntegrator(restr_c2_Im));
+
+      a.Assemble(0);
+
+      OperatorPtr A;
+      Vector B, X;
+      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+      SparseMatrix * SpMat = (*A.As<ComplexSparseMatrix>()).GetSystemMatrix();
+      HYPRE_Int global_size = SpMat->Height();
+      HYPRE_Int row_starts[2]; row_starts[0] = 0; row_starts[1] = global_size;
+      HypreParMatrix * HypreMat = new HypreParMatrix(MPI_COMM_SELF,global_size,row_starts,SpMat);
+      {
+         MUMPSSolver mumps;
+         mumps.SetOperator(*HypreMat);
+         mumps.Mult(B,X);
+      }
+
+      a.RecoverFEMSolution(X, b, x);
+
+      if (prob_kind == 3)
+      {
+         rates_r.SetElementList(tpml.GetMarkedPMLElements());
+         rates_i.SetElementList(tpml.GetMarkedPMLElements());
+
+         VectorFunctionCoefficient E_ex_Re(dim, E_exact_Re);
+         VectorFunctionCoefficient E_ex_Im(dim, E_exact_Im);
+         VectorFunctionCoefficient E_Curl_Re(cdim, E_exact_Curl_Re);
+         VectorFunctionCoefficient E_Curl_Im(cdim, E_exact_Curl_Im);
+
+         rates_r.AddHcurlGridFunction(&x.real(),&E_ex_Re,&E_Curl_Re);
+         rates_i.AddHcurlGridFunction(&x.imag(),&E_ex_Im,&E_Curl_Im);
+      }
+
+      if (iter == ref_levels) break;
+      mesh->UniformRefinement();
+      fespace->Update();
+      x.Update();
+   }
+
+   if (prob_kind == 3)
+   {
+      rates_r.Print(false);      
+      rates_i.Print(false);     
+   }
+   
+
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      // Define visualization keys for GLVis (see GLVis documentation)
+      string keys;
+      keys = (dim == 3) ? "keys macF\n" : keys = "keys amrRljcUUuuu\n";
+
+      char vishost[] = "localhost";
+      int visport = 19916;
+
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re.precision(8);
+      sol_sock_re << "solution\n"
+                  << *mesh << x.real() << keys
+                  << "window_title 'Solution real part'" << flush;
+
+      socketstream sol_sock_im(vishost, visport);
+      sol_sock_im.precision(8);
+      sol_sock_im << "solution\n"
+                  << *mesh << x.imag() << keys
+                  << "window_title 'Solution imag part'" << flush;
+      {
+         GridFunction x_t(fespace);
+         x_t = x.real();
+         socketstream sol_sock(vishost, visport);
+         sol_sock.precision(8);
+         sol_sock << "solution\n"
+                  << *mesh << x_t << keys << "autoscale off\n"
+                  << "window_title 'Harmonic Solution (t = 0.0 T)'"
+                  << "pause\n" << flush;
+
+         cout << "GLVis visualization paused."
+               << " Press space (in the GLVis window) to resume it.\n";
+      int num_frames = 16;
+      // int i = 0;
+
+      // ParaViewDataCollection * pd = new ParaViewDataCollection("PML_circle16", mesh);
+      // pd->SetPrefixPath("ParaView");
+      // pd->RegisterField("solution", &x_t);
+      // pd->SetLevelsOfDetail(order);
+      // pd->SetDataFormat(VTKFormat::BINARY);
+      // pd->SetHighOrderOutput(true);
+      // pd->SetCycle(0);
+      // pd->SetTime(0.0);
+      // pd->Save();
+
+
+         while (sol_sock)
+         {
+            for (int i = 1; i<num_frames; i++)
+            {   
+               double t = (double)(i % num_frames) / num_frames;
+               ostringstream oss;
+               oss << "Harmonic Solution (t = " << t << " T)";
+
+               add(cos(2.0 * M_PI * t), x.real(),
+                  sin(2.0 * M_PI * t), x.imag(), x_t);
+               sol_sock << "solution\n"
+                        << *mesh << x_t 
+                        << "window_title '" << oss.str() << "'" << flush;
+               // i++;
+            // pd->SetCycle(i);
+            // pd->SetTime((double)i);
+            // pd->Save();
+            }
+         }
+      }
+   }
+
+   // 17. Free the used memory.
+   // delete pml;
+   delete fespace;
+   delete fec;
+   delete mesh;
+
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+void source(const Vector &x, Vector &f)
+{
+   Vector center(dim);
+   double r = 0.0;
+   center = 0.5;
+   center(2) = 0.15;
+
+   for (int i = 0; i < dim; ++i)
+   {
+      r += pow(x[i] - center[i], 2.);
+   }
+   double n = 5.0 * omega * sqrt(epsilon * mu) / M_PI;
+   double coeff = pow(n, 2) / M_PI;
+   double alpha = -pow(n, 2) * r;
+   f = 0.0;
+   f[0] = coeff * exp(alpha);
+}
+
+void E_bdr_data_Re(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   if (prob_kind == 2)
+   {
+      if (abs(x(1))<1e-12 && x(0)>0) 
+      {
+         vector<complex<double>> Eval(E.Size());
+         maxwell_solution(x, Eval);
+         for (int i = 0; i < dim; ++i)
+         {
+            E[i] = Eval[i].real();
+         }
+      }
+   }
+   else if (prob_kind == 3)
+   {
+      double r = sqrt(x(0)*x(0) + x(1)*x(1));
+      // check if in pml
+
+      // if (abs(r-1.0)<1e-10) 
+      // if (r < 0.3) // not in pml
+      // if (x(0) <0.8 && x(0)>0.2 && x(1) < 0.8 && x(1) >0.2 )
+      // if (x(0) <0.3 && x(0)>-0.3 && x(1) < 0.3 && x(1) >-0.3 )
+      if (r < 0.3 )
+      {
+         vector<complex<double>> Eval(E.Size());
+         maxwell_solution(x, Eval);
+         for (int i = 0; i < dim; ++i)
+         {
+            E[i] = Eval[i].real();
+         }
+      }
+   }
+   else
+   {
+      if (x(1) == ylim) 
+      {
+         vector<complex<double>> Eval(E.Size());
+         maxwell_solution(x, Eval);
+         for (int i = 0; i < dim; ++i)
+         {
+            E[i] = Eval[i].real();
+         }
+      }
+   }
+}
+
+// Define bdr_data solution
+void E_bdr_data_Im(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   if (prob_kind == 2)
+   {
+      if (abs(x(1))<1e-12 && x(0)>0) 
+      {
+         vector<complex<double>> Eval(E.Size());
+         maxwell_solution(x, Eval);
+         for (int i = 0; i < dim; ++i)
+         {
+            E[i] = Eval[i].imag();
+         }
+      }
+   }
+   else if (prob_kind == 3)
+   {
+      double r = sqrt(x(0)*x(0) + x(1)*x(1));
+      // if (abs(r-1.0)<1e-10) 
+      // if (r < 0.3) // not in pml
+      // if (x(0) < 0.5) // not in pml
+      // if (x(0) <0.8 && x(0)>0.2 && x(1) < 0.8 && x(1) >0.2 )
+      // if (x(0) <0.3 && x(0)>-0.3 && x(1) < 0.3 && x(1) >-0.3 )
+      if (r < 0.3 )
+      {
+         vector<complex<double>> Eval(E.Size());
+         maxwell_solution(x, Eval);
+         for (int i = 0; i < dim; ++i)
+         {
+            E[i] = Eval[i].imag();
+         }
+      }
+   }
+   else
+   {
+      if (x(1) == ylim) 
+      {
+         vector<complex<double>> Eval(E.Size());
+         maxwell_solution(x, Eval);
+         for (int i = 0; i < dim; ++i)
+         {
+            E[i] = Eval[i].imag();
+         }
+      }
+   }
+}
+
+void E_exact_Re(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   vector<complex<double>> Eval(E.Size());
+   maxwell_solution(x, Eval);
+   for (int i = 0; i < dim; ++i)
+   {
+      E[i] = Eval[i].real();
+   }
+}
+
+void E_exact_Im(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   vector<complex<double>> Eval(E.Size());
+   maxwell_solution(x, Eval);
+   for (int i = 0; i < dim; ++i)
+   {
+      E[i] = Eval[i].imag();
+   }
+}
+
+void maxwell_solution(const Vector &x, vector<complex<double>> &E)
+{
+   complex<double> zi = complex<double>(0., 1.);
+
+   if (prob_kind == 2)
+   {  // for a straight waveguide
+      double k = omega * sqrt(epsilon * mu);
+      // T_10 mode
+      double k10 = sqrt(k * k - M_PI * M_PI);
+      E[2] = -zi * k / M_PI * sin(M_PI*(x(0)))*exp(zi * k10 * x(1)); 
+   }
+   else
+   {
+      double k = omega * sqrt(epsilon * mu);
+      Vector shift(dim);
+      shift = 0.0;
+      double x0 = x(0) + shift(0);
+      double x1 = x(1) + shift(1);
+      double r = sqrt(x0 * x0 + x1 * x1);
+      double beta = k * r;
+
+      // Bessel functions
+      complex<double> H0, H0_r, H0_rr, H0_rrr;
+      complex<double> H1, H1_r, H1_rr;
+      complex<double> H2, H2_r;
+      complex<double> H3;
+      H0 = jn(0,beta) + zi * yn(0,beta);
+      H1 = jn(1,beta) + zi * yn(1,beta);
+      H2 = jn(2,beta) + zi * yn(2,beta);
+      // H3 = jn(3,beta) + zi * yn(3,beta);
+
+      H0_r = - k * H1;
+      H0_rr = - k * k * (1.0/beta * H1 - H2); 
+
+      // First derivatives
+      double r_x = x0 / r;
+      double r_y = x1 / r;
+      double r_xy = -(r_x / r) * r_y;
+      double r_xx = (1.0 / r) * (1.0 - r_x * r_x);
+
+      complex<double> val, val_x, val_xx, val_xxx, val_xy, val_xyy;
+      val = 0.25 * zi * H0;
+      val_xx = 0.25 * zi * (r_xx * H0_r + r_x * r_x * H0_rr);
+      val_xy = 0.25 * zi * (r_xy * H0_r + r_x * r_y * H0_rr);
+      E[0] = zi / k * (k * k * val + val_xx);
+      E[1] = zi / k * val_xy;
+   }
+}
+
+void E_exact_Curl_Re(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   vector<complex<double>> Eval(E.Size());
+   maxwell_curl(x, Eval);
+   for (int i = 0; i < E.Size(); ++i)
+   {
+      E[i] = Eval[i].real();
+   }
+}
+
+void E_exact_Curl_Im(const Vector &x, Vector &E)
+{
+   E = 0.0;
+   vector<complex<double>> Eval(E.Size());
+   maxwell_curl(x, Eval);
+   for (int i = 0; i < E.Size(); ++i)
+   {
+      E[i] = Eval[i].imag();
+   }
+}
+
+
+void maxwell_curl(const Vector &x, vector<complex<double>> &curlE)
+{
+   complex<double> zi = complex<double>(0., 1.);
+
+   double k = omega * sqrt(epsilon * mu);
+   Vector shift(dim);
+   shift = 0.0;
+   double x0 = x(0) + shift(0);
+   double x1 = x(1) + shift(1);
+   double r = sqrt(x0 * x0 + x1 * x1);
+   double beta = k * r;
+
+   // Bessel functions
+   complex<double> H0_r;
+   complex<double> H1;
+   // complex<double> H2, H2_r;
+   // complex<double> H3;
+   // H0 = jn(0,beta) + zi * yn(0,beta);
+   H1 = jn(1,beta) + zi * yn(1,beta);
+   // H2 = jn(2,beta) + zi * yn(2,beta);
+   // H3 = jn(3,beta) + zi * yn(3,beta);
+
+   H0_r   = - k * H1;
+   // H1_r   =   k * (1.0/beta * H1 - H2);
+   // H2_r   = - k * (2.0/beta * H2 - H3); 
+   // H0_rr  = - k * H1_r;
+   // H1_rr  = k * k * (- 2.0 /(beta * beta) * H1 + 1.0/beta * H1_r - H2_r); 
+   // H0_rrr = - k * H1_rr;
+
+   // First derivatives
+   // double r_x = x0 / r;
+   double r_y = x1 / r;
+   // double r_xy = -(r_x / r) * r_y;
+   // double r_yx = r_xy;
+   // double r_yy = (1.0 / r) * (1.0 - r_y * r_y);
+   // double r_xx = (1.0 / r) * (1.0 - r_x * r_x);
+   // double r_xxx = r_x * (r_x * r_x - 2. * r_xx * r - 1.0) /(r * r);
+   // double r_xyy = (r_x * r_y * r_y - r * r_xy * r_y - r * r_x * r_yy)/(r * r);
+
+   complex<double> val_y;
+   // val = 0.25 * zi * H0;
+   val_y = 0.25 * zi * H0_r * r_y;
+   // val_xx = 0.25 * zi * (r_xx * H0_r + r_x * r_x * H0_rr);
+   // val_xy = 0.25 * zi * (r_xy * H0_r + r_x * r_y * H0_rr);
+   curlE[0] = zi / k * (- k * k * val_y);
+}
+
+
+void detJ_JT_J_inv_Re(const Vector &x, ToroidPML * pml, Vector &D)
+{
+   // vector<complex<double>> dxs(dim);
+   // complex<double> det(1.0, 0.0);
+   // pml->StretchFunction(x, dxs,omega);
+   ComplexDenseMatrix J(dim);
+   pml->StretchFunction(x,J,omega);
+   complex<double> det = J.Det();
+
+   // for (int i = 0; i < dim; ++i)
+   // {
+   //    det *= dxs[i];
+   // }
+
+   for (int i = 0; i < dim; ++i)
+   {
+      D(i) = (det / pow(J(i,i), 2)).real();
+   }
+}
+
+void detJ_JT_J_inv_Im(const Vector &x, ToroidPML * pml, Vector &D)
+{
+   // vector<complex<double>> dxs(dim);
+   // complex<double> det = 1.0;
+   // pml->StretchFunction(x, dxs,omega);
+   ComplexDenseMatrix J(dim);
+   pml->StretchFunction(x,J,omega);
+   complex<double> det = J.Det();
+
+   // for (int i = 0; i < dim; ++i)
+   // {
+   //    det *= dxs[i];
+   // }
+
+   for (int i = 0; i < dim; ++i)
+   {
+      D(i) = (det / pow(J(i,i), 2)).imag();
+   }
+}
+
+void detJ_inv_JT_J_Re(const Vector &x, ToroidPML * pml, Vector &D)
+{
+   // vector<complex<double>> dxs(dim);
+   // complex<double> det(1.0, 0.0);
+   // pml->StretchFunction(x, dxs,omega);
+   ComplexDenseMatrix J(dim);
+   pml->StretchFunction(x,J,omega);
+   complex<double> det = J.Det();
+   // for (int i = 0; i < dim; ++i)
+   // {
+   //    det *= dxs[i];
+   // }
+   // in the 2D case the coefficient is scalar 1/det(J)
+   if (dim == 2)
+   {
+      D = (1.0 / det).real();
+   }
+   else
+   {
+      for (int i = 0; i < dim; ++i)
+      {
+         D(i) = (pow(J(i,i), 2) / det).real();
+      }
+   }
+}
+
+void detJ_inv_JT_J_Im(const Vector &x, ToroidPML * pml, Vector &D)
+{
+   // vector<complex<double>> dxs(dim);
+   // complex<double> det = 1.0;
+   // pml->StretchFunction(x, dxs,omega);
+   ComplexDenseMatrix J(dim);
+   pml->StretchFunction(x,J,omega);
+   complex<double> det = J.Det();
+   // for (int i = 0; i < dim; ++i)
+   // {
+   //    det *= dxs[i];
+   // }
+
+   if (dim == 2)
+   {
+      D = (1.0 / det).imag();
+   }
+   else
+   {
+      for (int i = 0; i < dim; ++i)
+      {
+         D(i) = (pow(J(i,i), 2) / det).imag();
+      }
+   }
+}
+
+
+//-----------------------------------------------------------------
+
+// void detJ_JT_J_inv_Re(const Vector &x, ToroidPML * pml, DenseMatrix & M)
+// {
+//    ComplexDenseMatrix J(dim);
+//    pml->StretchFunction(x,J,omega);
+//    complex<double> det = J.Det();
+//    ComplexDenseMatrix JtJ(dim);
+//    MultAtB(J,J,JtJ);
+//    ComplexDenseMatrixInverse InvJtJ(JtJ);
+//    InvJtJ *=det;
+//    InvJtJ.GetReal(M);
+// }
+
+// void detJ_JT_J_inv_Im(const Vector &x, ToroidPML * pml, DenseMatrix & M)
+// {
+//    ComplexDenseMatrix J(dim);
+//    pml->StretchFunction(x,J,omega);
+//    complex<double> det = J.Det();
+//    ComplexDenseMatrix JtJ(dim);
+//    MultAtB(J,J,JtJ);
+//    ComplexDenseMatrixInverse InvJtJ(JtJ);
+//    InvJtJ *=det;
+//    InvJtJ.GetImag(M);
+// }
+
+// void detJ_inv_JT_J_Re(const Vector &x, ToroidPML * pml, DenseMatrix & M)
+// {
+//    ComplexDenseMatrix J(dim);
+//    pml->StretchFunction(x,J,omega);
+//    complex<double> det = J.Det();
+//    if (dim == 2)
+//    {
+//       M = (1.0 / det).real();
+//    }
+//    else
+//    {
+//       ComplexDenseMatrix JtJ(dim);
+//       MultAtB(J,J,JtJ);
+//       JtJ *= 1.0/det;
+//       JtJ.GetReal(M);
+//    }
+// }
+
+// void detJ_inv_JT_J_Im(const Vector &x, ToroidPML * pml, DenseMatrix & M)
+// {
+//    ComplexDenseMatrix J(dim);
+//    pml->StretchFunction(x,J,omega);
+//    complex<double> det = J.Det();
+//    if (dim == 2)
+//    {
+//       M = (1.0 / det).imag();
+//    }
+//    else
+//    {
+//       ComplexDenseMatrix JtJ(dim);
+//       MultAtB(J,J,JtJ);
+//       JtJ *= 1.0/det;
+//       JtJ.GetImag(M);
+//    }
+// }

examples/maxwell-solver/torus_generate_ovlp_partitioning.cpp

@@ -0,0 +1,135 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+// Compile with: make subdomainmap
+//
+// Sample runs:
+//   subdomainmap -m1 global.mesh -m2 local.mesh
+
+#include "mfem.hpp"
+#include <fstream>
+
+using namespace mfem;
+using namespace std;
+
+double funccoeff(const Vector & x);
+int get_angle_range(double angle, Array<double> angles);
+
+
+int main (int argc, char *argv[])
+{
+   // Set the method's default parameters.
+   const char *mesh_file = "TokamakMeshes/torus.mesh";
+   // const char *tar_mesh_file = "torus1_4.mesh";
+   const char *tar_mesh_file = "torus2_4.mesh";
+   int order          = 3; // unused
+
+   // Parse command-line options.
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file for the starting solution.");
+   args.AddOption(&order, "-o", "--order",
+                  "Order of the interpolated solution.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // Input meshes.
+   Mesh mesh(mesh_file, 1, 1, false);
+   // mesh.UniformRefinement();
+   // mesh.UniformRefinement();
+   const int dim = mesh.Dimension();
+   int ne1 = mesh.GetNE();
+   int subdivisions = 4;
+   Array<double> angles(subdivisions+1);
+   angles[0] = 0.0;
+   double length = 360/subdivisions;
+   double range;
+   for (int i = 1; i<=subdivisions; i++)
+   {
+      range = i*length;
+      angles[i] = range;
+   }
+   
+   // set element attributes
+   for (int i = 0; i < ne1; ++i)
+   {
+      Element *el = mesh.GetElement(i);
+      // roughly the element center
+      Vector center(dim);
+      mesh.GetElementCenter(i,center);
+      // center.Print();
+      double x = center[0];
+      double y = center[1];
+      double theta = atan(y/x);
+      int k = 0;
+      
+      if (x<0)
+      {
+         k = 1;
+      }
+      else if (y<0)
+      {
+         k = 2;
+      }
+      theta += k*M_PI;
+
+      double thetad = theta * 180.0/M_PI;
+
+      // Find the angle relative to (0,0,z)
+      int attr = get_angle_range(thetad, angles) + 1;
+      el->SetAttribute(attr);
+   }
+   mesh.SetAttributes();
+   ofstream mesh_ofs("mesh1.mesh");
+   mesh_ofs.precision(8);
+   mesh.Print(mesh_ofs);
+
+   // char vishost[] = "localhost";
+   // int  visport   = 19916;
+   // string keys;
+   // if (dim ==2 )
+   // {
+   //    keys = "keys mrRljc\n";
+   // }
+   // else
+   // {
+   //    keys = "keys mc\n";
+   // }
+   // socketstream sol_sock1(vishost, visport);
+   // sol_sock1.precision(8);
+   // sol_sock1 << "solution\n" << mesh_1 << gf1 << keys 
+   //           << "window_title ' ' " << flush;                     
+
+   // socketstream sol_sock2(vishost, visport);
+   // sol_sock2.precision(8);
+   // sol_sock2 << "solution\n" << mesh_2 << gf2 << keys 
+   //           << "window_title ' ' " << flush;  
+
+   return 0;
+}
+
+
+double funccoeff(const Vector & x)
+{
+   return sin(3*M_PI*(x.Sum()));
+}
+
+int get_angle_range(double angle, Array<double> angles)
+{
+   auto it = std::upper_bound(angles.begin(), angles.end(), angle);
+   return std::distance(angles.begin(),it)-1;
+   
+}

examples/maxwell-solver/waveguide.cpp

@@ -0,0 +1,324 @@
+//
+// Compile with: make maxwellp
+//
+//               mpirun ./maxwellp -o 3 -f 8.0 -sr 2 -pr 2 -m ../../data/inline-quad.mesh -nx 4 -ny 4
+//
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "ParDST/ParDST.hpp"
+#include "common/PML.hpp"
+
+using namespace std;
+using namespace mfem;
+  
+void maxwell_solution(const Vector &x, vector<complex<double>> &Eval);
+void ess_data_func_re(const Vector & x, Vector & E);
+void ess_data_func_im(const Vector & x, Vector & E);
+
+double mu = 1.0;
+double epsilon = 1.0;
+double omega;
+int dim;
+double length = 1.0;
+Array2D<double> comp_domain_bdr;
+Array2D<double> domain_bdr;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+   int order = 1;
+   // number of serial refinements
+   int ser_ref_levels = 1;
+   // number of parallel refinements
+   int par_ref_levels = 2;
+   double freq = 5.0;
+   bool herm_conv = true;
+   bool visualization = 1;
+   int nd=2;
+   int nx=2;
+   int ny=2;
+   int nz=2;
+   OptionsParser args(argc, argv);
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&nd, "-nd", "--dim",
+                  "Problem space dimension");                  
+   args.AddOption(&nx, "-nx", "--nx","Number of subdomains in x direction");
+   args.AddOption(&ny, "-ny", "--ny","Number of subdomains in y direction");
+   args.AddOption(&nz, "-nz", "--nz","Number of subdomains in z direction");     
+   args.AddOption(&ser_ref_levels, "-sr", "--ser_ref_levels",
+                  "Number of Serial Refinements.");
+   args.AddOption(&par_ref_levels, "-pr", "--par_ref_levels",
+                  "Number of Parallel Refinements."); 
+   args.AddOption(&freq, "-f", "--frequency",
+                  "Frequency (in Hz).");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // Angular frequency
+   omega = 2.0 * M_PI * freq;
+
+   Mesh *mesh;
+
+   int nel = 1;
+   int nelx = 8;
+   double lengthx = 8*length;
+   if (nd == 3)
+   {
+      mesh = new Mesh(nelx, nel, nel, Element::HEXAHEDRON, true, lengthx, length, length,false);
+   }
+   else
+   {
+      mesh = new Mesh(nelx, nel, Element::QUADRILATERAL, true, lengthx, length,false);
+   }
+   dim = mesh->Dimension();
+   // 4. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ser_ref_levels; l++) { mesh->UniformRefinement(); }
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD,*mesh);
+   delete mesh;
+   for (int l = 0; l < par_ref_levels; l++) {pmesh->UniformRefinement(); }
+
+
+   double hl = GetUniformMeshElementSize(pmesh);
+   int nrlayers = 4;
+   Array2D<double> lengths(dim,2);
+   lengths = 0.0;
+   // lengths = hl*nrlayers;
+   lengths(0, 1) = hl*nrlayers;
+   CartesianPML pml(pmesh,lengths);
+   pml.SetOmega(omega);
+   comp_domain_bdr.SetSize(dim,2);
+   comp_domain_bdr = pml.GetCompDomainBdr(); 
+
+   // 6. Define a finite element space on the mesh. Here we use the Nedelec
+   //    finite elements of the specified order.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+
+   if (pmesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 9. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system.
+   ParComplexLinearForm b(fespace);
+   b.Vector::operator=(0.0);
+   b.Assemble();
+
+   // 10. Define the solution vector x as a complex finite element grid function
+   //     corresponding to fespace.
+   ParComplexGridFunction x(fespace);
+   x = 0.0;
+   VectorFunctionCoefficient E_re(dim,ess_data_func_re);
+   VectorFunctionCoefficient E_im(dim,ess_data_func_im);
+   x.ProjectBdrCoefficientTangent(E_re, E_im, ess_bdr);
+   // 11. Set up the sesquilinear form a(.,.)
+   //
+   //       1/mu (1/det(J) J^T J Curl E, Curl F)
+   //        - omega^2 * epsilon (det(J) * (J^T J)^-1 * E, F)
+   //
+   ConstantCoefficient omeg(-pow(omega, 2));
+   int cdim = (dim == 2) ? 1 : dim;
+   PmlMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, &pml);
+   PmlMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, &pml);
+
+   PmlMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,&pml);
+   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+
+   ParSesquilinearForm a(fespace);
+   a.AddDomainIntegrator(new CurlCurlIntegrator(pml_c1_Re),
+                         new CurlCurlIntegrator(pml_c1_Im));
+   a.AddDomainIntegrator(new VectorFEMassIntegrator(c2_Re),
+                         new VectorFEMassIntegrator(c2_Im));
+   a.Assemble(0);
+
+   OperatorPtr A;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   ConstantCoefficient one(1.0);
+   ParDST * S = new ParDST(&a,lengths, omega, &one, nrlayers, nx, ny, nz);
+   X = 0.0;
+	GMRESSolver gmres(MPI_COMM_WORLD);
+   gmres.SetPreconditioner(*S);
+	gmres.SetOperator(*A);
+	gmres.SetRelTol(1e-8);
+	gmres.SetMaxIter(50);
+	gmres.SetPrintLevel(1);
+	gmres.Mult(B, X);
+   delete S;
+
+   // {
+   //    ComplexMUMPSSolver mumps;
+   //    mumps.SetOperator(*A.As<ComplexHypreParMatrix>());
+   //    mumps.Mult(B,X);
+   // }
+
+   a.RecoverFEMSolution(X, b, x);
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      string keys;
+      // keys = "keys mc\n";
+      keys = "keys macFFiYYYYYYYYYYYYYYYYYY\n";
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re.precision(8);
+      sol_sock_re << "parallel " << num_procs << " " << myid << "\n"
+                  << "solution\n" << *pmesh << x.real() << keys 
+                  << "window_title 'E: Real Part' " << flush;                     
+
+      socketstream sol_sock_im(vishost, visport);
+      sol_sock_im.precision(8);
+      sol_sock_im << "parallel " << num_procs << " " << myid << "\n"
+                  << "solution\n" << *pmesh << x.imag() << keys 
+                  << "window_title 'E: Imag Part' " << flush;   
+      {
+         ParGridFunction x_t(fespace);
+         x_t = x.real();
+
+         socketstream sol_sock(vishost, visport);
+         sol_sock.precision(8);
+         sol_sock << "parallel " << num_procs << " " << myid << "\n"
+                  << "solution\n" << *pmesh << x_t << keys << "autoscale off\n"
+                  << "window_title 'Harmonic Solution (t = 0.0 T)'"
+                  << "pause\n" << flush;
+
+         if (myid == 0)
+         {
+            cout << "GLVis visualization paused."
+                 << " Press space (in the GLVis window) to resume it.\n";
+         }
+
+         int num_frames = 32;
+         int i = 0;
+         while (sol_sock)
+         {
+            double t = (double)(i % num_frames) / num_frames;
+            ostringstream oss;
+            oss << "Harmonic Solution (t = " << t << " T)";
+
+            add(cos(2.0*M_PI*t), x.real(), sin(2.0*M_PI*t), x.imag(), x_t);
+            sol_sock << "parallel " << num_procs << " " << myid << "\n";
+            sol_sock << "solution\n" << *pmesh << x_t
+                     << "window_title '" << oss.str() << "'" << flush;
+            i++;
+         }
+      }
+   }
+
+   // 18. Free the used memory.
+   delete fespace;
+   delete fec;
+   delete pmesh;
+   MPI_Finalize();
+   return 0;
+}
+
+void maxwell_solution(const Vector &x, vector<complex<double>> &E)
+{
+   complex<double> zi = complex<double>(0., 1.);
+   if (dim == 3)
+   {
+      double k10 = sqrt(omega * omega - M_PI * M_PI);
+      E[1] = -zi * omega / M_PI * sin(M_PI*x(2))*exp(zi * k10 * x(0));
+   }
+   else
+   {
+      E[1] = -zi * omega / M_PI * exp(zi * omega * x(0));
+   }
+   // E[1] = -zi * omega / M_PI * sin(M_PI*x(0))*exp(zi * k10 * x(2));
+   E[0] = 0.0;
+   if (dim == 3) E[2] = 0.0;
+}
+
+
+void ess_data_func_re(const Vector & x, Vector & E)
+{
+   E = 0.0;
+   bool in_pml = false;
+
+   for (int i = 0; i < dim; ++i)
+   {
+      // check if in PML
+      if (x(i) - comp_domain_bdr(i, 0) < 0.0 ||
+          x(i) - comp_domain_bdr(i, 1) > 0.0)
+      {
+         in_pml = true;
+         break;
+      }
+   }
+   if (!in_pml)
+   {
+      vector<complex<double>> Eval(E.Size());
+      maxwell_solution(x, Eval);
+      for (int i = 0; i < dim; ++i)
+      {
+         E[i] = Eval[i].real();
+      }
+   }
+
+}
+
+void ess_data_func_im(const Vector & x, Vector & E)
+{
+   E = 0.0;
+   bool in_pml = false;
+
+   for (int i = 0; i < dim; ++i)
+   {
+      // check if in PML
+      if (x(i) - comp_domain_bdr(i, 0) < 0.0 ||
+          x(i) - comp_domain_bdr(i, 1) > 0.0)
+      {
+         in_pml = true;
+         break;
+      }
+   }
+   if (!in_pml)
+   {
+      vector<complex<double>> Eval(E.Size());
+      maxwell_solution(x, Eval);
+      for (int i = 0; i < dim; ++i)
+      {
+         E[i] = Eval[i].imag();
+      }
+   }
+}

fem/coefficient.cpp

@@ -1040,6 +1040,29 @@ void MatrixVectorProductCoefficient::Eval(Vector &V, ElementTransformation &T,
    ma.Mult(vb, V);
 }
 
+
+MatrixMatrixProductCoefficient::MatrixMatrixProductCoefficient(
+   MatrixCoefficient &A,
+   MatrixCoefficient &B)
+   : MatrixCoefficient(A.GetHeight(), A.GetWidth()),
+     a(&A), b(&B),
+     ma(A.GetHeight(), A.GetWidth()),
+     mb(B.GetHeight(), B.GetWidth())
+{
+   MFEM_ASSERT(A.GetWidth() == B.GetHeight(),
+               "MatrixMatrixProductCoefficient:  "
+               "Arguments must have the same dimensions.");
+}
+
+void MatrixMatrixProductCoefficient::Eval(DenseMatrix &M,
+                                          ElementTransformation &T,
+                                          const IntegrationPoint &ip)
+{
+   a->Eval(ma, T, ip);
+   b->Eval(mb, T, ip);
+   Mult(ma, mb, M);
+}
+
 void IdentityMatrixCoefficient::Eval(DenseMatrix &M, ElementTransformation &T,
                                      const IntegrationPoint &ip)
 {