starting ST with artificial absorption

Add hipSPARSE support

.gitlab/configs/lassen-config.yml

@@ -45,5 +45,5 @@ variables:
     - echo ${MFEM_DATA_DIR}
     - echo ${SPEC}
     # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
-    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - lalloc 1 -W 30 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
   needs: [setup]

.gitlab/lassen-build-and-test.yml

@@ -19,6 +19,11 @@ opt_mpi_cuda_xl_16_1_1_8:
     SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
   extends: .build_and_test_on_lassen
 
+opt_mpi_cuda_hypre_cuda_xl:
+  variables:
+    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
+  extends: .build_and_test_on_lassen
+
 # Jobs report
 report_job_success:
   stage: report

.gitlab/quartz-build-and-test.yml

@@ -59,7 +59,7 @@ opt_par_gcc_6_1_0_sundials:
 
 opt_par_gcc_6_1_0_petsc:
   variables:
-    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps"
+    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps~superlu-dist"
   extends: .build_and_test_on_quartz
 
 opt_par_gcc_6_1_0_pumi:

.gitlab/scripts/baseline

@@ -29,9 +29,9 @@ mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
 
 # run
 if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
-  srun --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
-  srun --nodes=1 -t 60 -p mi60 ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "lassen" ]]; then
   lalloc 1 -q pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 else

CHANGELOG

@@ -10,10 +10,24 @@
 
 Version 4.3.1 (development)
 ===========================
-- Added support for automatic differentiation. Users can select between 
-  native implementation and external library implementation at the 
-  configuration phase. A parallel and two serial examples are implemented
-  in the autodiff miniapp directory.
+
+- Add hipSPARSE support for sparse mat-vec multiplications.
+
+- Added support for using the HYPRE library built with HIP support. Similar to
+  the HYPRE + CUDA support added earlier, most of the MFEM examples and miniapps
+  work transparently with HYPRE + HIP builds. This includes the BoomerAMG, AMS,
+  and ADS solvers.
+
+- More explicit and consistent formating of the output of iterative solvers
+  with the new IterativeSolver::PrintLevel options. See linalg/solvers.hpp.
+
+- Added support for automatic differentiation. Users can select between native
+  implementation and external library implementation during configuration. One
+  parallel and two serial examples are implemented in the miniapps/autodiff/
+  directory.
+
+- GridFunctionCoefficient (and the related vector, gradient, divergence, and
+  curl classes) now work properly with LORDiscretization and LORSolver.
 
 - Added support for mesh preprocessing to resolve fine scale problem data
   before simulation. This feature uses adaptive mesh refinement to control the
@@ -71,13 +85,16 @@ Version 4.3.1 (development)
 
 - Added initial TMOP-based capabilities for surface fitting and tangential
   relaxation in the mesh-optimizer and pmesh-optimizer miniapps.
- 
+
 - Added ParMesh Adjaceny Set (adjset) creation support to the Conduit Mesh
   Blueprint MFEM wrapper functions in ConduitDataCollection.
 
 - `HypreParVector` and `Vector` now support move semantics, and the copy
   constructor for `HypreParVector` now copies the local vector data.
 
+- The HPC versions of ex1 and ex1p (in miniapps/performance) now support
+  runtime selection of either 2D or 3D meshes.
+
 
 Version 4.3, released on July 29, 2021
 ======================================

CMakeLists.txt

@@ -16,9 +16,6 @@ set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
 
 # Require C++11 and disable compiler-specific extensions
 set(CMAKE_CXX_STANDARD 11)
-if (MFEM_USE_GINKGO)
-  set(CMAKE_CXX_STANDARD 14)
-endif()
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
@@ -84,6 +81,9 @@ if (MFEM_USE_STRUMPACK)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
+if (MFEM_USE_GINKGO AND ("${CMAKE_CXX_STANDARD}" LESS "14"))
+  set(CMAKE_CXX_STANDARD 14)
+endif()
 
 # Include xSDK default CMake file.
 include("${CMAKE_CURRENT_SOURCE_DIR}/config/XSDKDefaults.cmake")
@@ -180,6 +180,19 @@ else()
   set(MFEM_DEBUG OFF)
 endif()
 
+# AMD HIP
+if (MFEM_USE_HIP)
+  if (HIP_ARCH)
+    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
+    set(GPU_TARGETS "${HIP_ARCH}" CACHE STRING "HIP targets to compile for")
+  endif()
+  if (ROCM_PATH)
+    list(INSERT CMAKE_PREFIX_PATH 0 ${ROCM_PATH})
+  endif()
+  find_package(HIP REQUIRED)
+  find_package(HIPSPARSE REQUIRED)
+endif()
+
 # MPI -> hypre; PETSc (optional)
 if (MFEM_USE_MPI)
   find_package(MPI REQUIRED)
@@ -354,7 +367,7 @@ endif()
 if (MFEM_USE_PUMI)
   # If PUMI_DIR was specified, only link to that directory,
   # i.e. don't link to another installation in /usr/lib by mistake
-  find_package(SCOREC 2.1.0 REQUIRED OPTIONAL_COMPONENTS gmi_sim
+  find_package(SCOREC 2.2.6 REQUIRED OPTIONAL_COMPONENTS gmi_sim
     CONFIG PATHS ${PUMI_DIR} NO_DEFAULT_PATH)
   if (SCOREC_FOUND)
     # Define a header file with the MFEM_USE_SIMMETRIX preprocessor variable
@@ -403,18 +416,6 @@ if (MFEM_USE_CALIPER)
   find_package(Caliper REQUIRED)
 endif()
 
-# AMD HIP
-if (MFEM_USE_HIP)
-  find_package(HIP REQUIRED)
-  if (HIP_ARCH)
-    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
-    list(APPEND HIP_HIPCC_FLAGS "--amdgpu-target=${HIP_ARCH}")
-    if (MFEM_USE_GINKGO)
-      list(APPEND HIP_HIPCC_FLAGS "-std=c++14")
-    endif()
-  endif()
-endif()
-
 # ADIOS2 for parallel I/O
 if (MFEM_USE_ADIOS2)
    find_package(ADIOS2 REQUIRED)
@@ -454,10 +455,11 @@ endif()
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
-    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG MPI_CXX)
+set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS
+    PETSC SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
+    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
+    ADIOS2 CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
+    MPI_CXX HIP HIPSPARSE)
 
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -500,8 +502,6 @@ endforeach()
 
 if (MFEM_USE_CUDA)
   set_source_files_properties(${SOURCES} PROPERTIES LANGUAGE CUDA)
-elseif(MFEM_USE_HIP)
-  set_source_files_properties(${SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
 endif()
 
 add_subdirectory(config)

CODE_OF_CONDUCT.md

@@ -0,0 +1,133 @@
+# MFEM Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at mfem@llnl.gov.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident. Anyone involved in the reported behavior will recuse
+themselves from the investigation and decision making about the resolution of
+the complaint.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTING.md

@@ -21,6 +21,9 @@ documentation; new examples and miniapps; HPC performance improvements; etc.
 MFEM is distributed under the terms of the BSD-3 license. All new contributions
 must be made under this license.
 
+Note also that MFEM has a [Code of Conduct](CODE_OF_CONDUCT.md). By participating
+in the MFEM community, you agree to abide by its rules.
+
 If you plan on contributing to MFEM, consider reviewing the
 [issue tracker](https://github.com/mfem/mfem/issues) first to check if a thread
 already exists for your desired feature or the bug you ran into. Use a pull

INSTALL

@@ -37,7 +37,7 @@ as CUDA, HIP, OCCA, OpenMP and RAJA.
   https://developer.nvidia.com/cuda-toolkit
 
 - HIP support requires an AMD GPU and an installation of the ROCm software stack
-  https://rocm.github.io/ROCmInstall.html#installing-from-amd-rocm-repositories
+  https://rocmdocs.amd.com
 
 - OCCA support requires the OCCA library
   https://libocca.org
@@ -58,7 +58,8 @@ following package managers:
 
 - Spack, https://github.com/spack/spack
 - OpenHPC, http://openhpc.community
-- Conda-forge, https://conda-forge.org (pre-built binaries linked with OpenMPI/MPICH, hypre, and METIS)
+- Conda-forge, https://conda-forge.org (pre-built binaries linked with
+  OpenMPI/MPICH, hypre, and METIS)
 - Homebrew/Science, https://github.com/Homebrew/homebrew-science (deprecated)
 
 We also recommend downloading and building the MFEM-based GLVis visualization
@@ -78,7 +79,7 @@ Parallel build:
 
 CUDA build:
    make cuda -j 4
-   (build for a specific compute capability: 'make cuda -j 4 CUDA_ARCH=sm_30')
+   (build for a specific compute capability: 'make cuda -j 4 CUDA_ARCH=sm_70')
 
 HIP build:
    make hip -j 4
@@ -566,7 +567,7 @@ The specific libraries and their options are:
   Options: HYPRE_OPT, HYPRE_LIB.
   Versions: HYPRE >= 2.10.0b  (HYPRE built without CUDA)
             HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
-            HYPRE >= 2.22.1   (HYPRE built with CUDA)
+            HYPRE >= 2.22.1   (HYPRE built with CUDA or HIP)
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
   MFEM_USE_METIS_5 = YES (default is to use METIS 4).
@@ -639,7 +640,8 @@ The specific libraries and their options are:
   C++ compiler that supports the C++-14 standard. For additional requirements
   and dependencies of specific modules, see the Ginkgo webpage below.
   URL: https://ginkgo-project.github.io
-  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or Debug).
+  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or
+           Debug).
   Versions: Ginkgo >= 1.4.0.
 
 - AmgX (optional), used when MFEM_USE_AMGX = YES.
@@ -704,7 +706,7 @@ The specific libraries and their options are:
   URL: https://scorec.rpi.edu/pumi
        https://github.com/SCOREC/core
   Options: PUMI_OPT, PUMI_LIB.
-  Versions: PUMI == 2.2.3.
+  Versions: PUMI >= 2.2.6.
 
 - HiOp (optional), used when MFEM_USE_HIOP = YES.
   URL: https://github.com/LLNL/hiop
@@ -737,7 +739,7 @@ The specific libraries and their options are:
   Versions: CUDA >= 10.1.168.
 
 - HIP (optional), used when MFEM_USE_HIP = YES.
-  URL: https://rocm.github.io/ROCmInstall.html
+  URL: https://rocmdocs.amd.com
   Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB.
 
 - OCCA (optional), used when MFEM_USE_OCCA = YES.
@@ -924,6 +926,7 @@ MFEM_USE_HIOP
 MFEM_USE_CODIPACK
 MFEM_USE_ADFORWARD
 MFEM_USE_CUDA
+MFEM_USE_HIP
 MFEM_USE_OCCA
 MFEM_USE_CEED
 MFEM_USE_RAJA

README

@@ -12,6 +12,9 @@ to enable high-performance scalable finite element discretization research and
 application development on a wide variety of platforms, ranging from laptops to
 supercomputers.
 
+We welcome contributions and feedback from the community. Please see the file
+CONTRIBUTING.md for additional details about our development process.
+
 * For building instructions, see the file INSTALL, or type "make help".
 
 * Copyright and licensing information can be found in files LICENSE and NOTICE.
@@ -19,9 +22,6 @@ supercomputers.
 * The best starting point for new users interested in MFEM's features is to
   review the examples and miniapps at https://mfem.org/examples.
 
-* Developers interested in contributing to the library, should read the
-  instructions and documentation in the CONTRIBUTING.md file.
-
 Conceptually, MFEM can be viewed as a finite element toolbox that provides the
 building blocks for developing finite element algorithms in a manner similar to
 that of MATLAB for linear algebra methods. In particular, MFEM provides support

config/cmake/modules/FindHIP.cmake

@@ -1,692 +0,0 @@
-###############################################################################
-# FindHIP.cmake
-###############################################################################
-include(CheckCXXCompilerFlag)
-###############################################################################
-# SET: Variable defaults
-###############################################################################
-# User defined flags
-set(HIP_HIPCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HIPCC")
-set(HIP_HCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HCC")
-set(HIP_CLANG_FLAGS "" CACHE STRING "Semicolon delimited flags for CLANG")
-set(HIP_NVCC_FLAGS "" CACHE STRING "Semicolon delimted flags for NVCC")
-mark_as_advanced(HIP_HIPCC_FLAGS HIP_HCC_FLAGS HIP_CLANG_FLAGS HIP_NVCC_FLAGS)
-
-set(_hip_configuration_types ${CMAKE_CONFIGURATION_TYPES} ${CMAKE_BUILD_TYPE} Debug MinSizeRel Release RelWithDebInfo)
-list(REMOVE_DUPLICATES _hip_configuration_types)
-foreach(config ${_hip_configuration_types})
-    string(TOUPPER ${config} config_upper)
-    set(HIP_HIPCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HIPCC")
-    set(HIP_HCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HCC")
-    set(HIP_CLANG_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for CLANG")
-    set(HIP_NVCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for NVCC")
-    mark_as_advanced(HIP_HIPCC_FLAGS_${config_upper} HIP_HCC_FLAGS_${config_upper} HIP_CLANG_FLAGS_${config_upper} HIP_NVCC_FLAGS_${config_upper})
-endforeach()
-option(HIP_HOST_COMPILATION_CPP "Host code compilation mode" ON)
-option(HIP_VERBOSE_BUILD "Print out the commands run while compiling the HIP source file.  With the Makefile generator this defaults to VERBOSE variable specified on the command line, but can be forced on with this option." OFF)
-mark_as_advanced(HIP_HOST_COMPILATION_CPP)
-
-###############################################################################
-# FIND: HIP and associated helper binaries
-###############################################################################
-
-get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_DIR}/../" REALPATH)
-
-# HIP is supported on Linux only
-if(UNIX AND NOT APPLE AND NOT CYGWIN)
-    # Search for HIP installation
-    if(NOT HIP_ROOT_DIR)
-        # Search in user specified path first
-        find_path(
-            HIP_ROOT_DIR
-            NAMES bin/hipconfig
-            PATHS
-            "$ENV{ROCM_PATH}/hip"
-            ENV HIP_PATH
-            ${_IMPORT_PREFIX}
-            /opt/rocm/hip
-            DOC "HIP installed location"
-            NO_DEFAULT_PATH
-            )
-        if(NOT EXISTS ${HIP_ROOT_DIR})
-            if(HIP_FIND_REQUIRED)
-                message(FATAL_ERROR "Specify HIP_ROOT_DIR")
-            elseif(NOT HIP_FIND_QUIETLY)
-                message("HIP_ROOT_DIR not found or specified")
-            endif()
-        endif()
-        # And push it back to the cache
-        set(HIP_ROOT_DIR ${HIP_ROOT_DIR} CACHE PATH "HIP installed location" FORCE)
-    endif()
-
-    # Find HIPCC executable
-    find_program(
-        HIP_HIPCC_EXECUTABLE
-        NAMES hipcc
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCC_EXECUTABLE)
-        # Now search in default paths
-        find_program(HIP_HIPCC_EXECUTABLE hipcc)
-    endif()
-    mark_as_advanced(HIP_HIPCC_EXECUTABLE)
-
-    # Find HIPCONFIG executable
-    find_program(
-        HIP_HIPCONFIG_EXECUTABLE
-        NAMES hipconfig
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCONFIG_EXECUTABLE)
-        # Now search in default paths
-        find_program(HIP_HIPCONFIG_EXECUTABLE hipconfig)
-    endif()
-    mark_as_advanced(HIP_HIPCONFIG_EXECUTABLE)
-
-    # Find HIPCC_CMAKE_LINKER_HELPER executable
-    find_program(
-        HIP_HIPCC_CMAKE_LINKER_HELPER
-        NAMES hipcc_cmake_linker_helper
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCC_CMAKE_LINKER_HELPER)
-        # Now search in default paths
-        find_program(HIP_HIPCC_CMAKE_LINKER_HELPER hipcc_cmake_linker_helper)
-    endif()
-    mark_as_advanced(HIP_HIPCC_CMAKE_LINKER_HELPER)
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_VERSION)
-        # Compute the version
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --version
-            OUTPUT_VARIABLE _hip_version
-            ERROR_VARIABLE _hip_error
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            ERROR_STRIP_TRAILING_WHITESPACE
-            )
-        if(NOT _hip_error)
-            set(HIP_VERSION ${_hip_version} CACHE STRING "Version of HIP as computed from hipcc")
-        else()
-            set(HIP_VERSION "0.0.0" CACHE STRING "Version of HIP as computed by FindHIP()")
-        endif()
-        mark_as_advanced(HIP_VERSION)
-    endif()
-    if(HIP_VERSION)
-        string(REPLACE "." ";" _hip_version_list "${HIP_VERSION}")
-        list(GET _hip_version_list 0 HIP_VERSION_MAJOR)
-        list(GET _hip_version_list 1 HIP_VERSION_MINOR)
-        list(GET _hip_version_list 2 HIP_VERSION_PATCH)
-        set(HIP_VERSION_STRING "${HIP_VERSION}")
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_PLATFORM)
-        # Compute the platform
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform
-            OUTPUT_VARIABLE _hip_platform
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_PLATFORM ${_hip_platform} CACHE STRING "HIP platform as computed by hipconfig")
-        mark_as_advanced(HIP_PLATFORM)
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_COMPILER)
-        # Compute the compiler
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --compiler
-            OUTPUT_VARIABLE _hip_compiler
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_COMPILER ${_hip_compiler} CACHE STRING "HIP compiler as computed by hipconfig")
-        mark_as_advanced(HIP_COMPILER)
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_RUNTIME)
-        # Compute the runtime
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --runtime
-            OUTPUT_VARIABLE _hip_runtime
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_RUNTIME ${_hip_runtime} CACHE STRING "HIP runtime as computed by hipconfig")
-        mark_as_advanced(HIP_RUNTIME)
-    endif()
-endif()
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(
-    HIP
-    REQUIRED_VARS
-    HIP_ROOT_DIR
-    HIP_HIPCC_EXECUTABLE
-    HIP_HIPCONFIG_EXECUTABLE
-    HIP_PLATFORM
-    HIP_COMPILER
-    HIP_RUNTIME
-    VERSION_VAR HIP_VERSION
-    )
-
-###############################################################################
-# Set HIP CMAKE Flags
-###############################################################################
-# Copy the invocation styles from CXX to HIP
-set(CMAKE_HIP_ARCHIVE_CREATE ${CMAKE_CXX_ARCHIVE_CREATE})
-set(CMAKE_HIP_ARCHIVE_APPEND ${CMAKE_CXX_ARCHIVE_APPEND})
-set(CMAKE_HIP_ARCHIVE_FINISH ${CMAKE_CXX_ARCHIVE_FINISH})
-set(CMAKE_SHARED_LIBRARY_SONAME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG})
-set(CMAKE_SHARED_LIBRARY_CREATE_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CXX_FLAGS})
-#set(CMAKE_SHARED_LIBRARY_LINK_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG})
-set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG_SEP ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP})
-set(CMAKE_SHARED_LIBRARY_LINK_STATIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_STATIC_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_CXX_FLAGS})
-
-set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "")
-set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "")
-
-if("${HIP_COMPILER}" STREQUAL "nvcc")
-    # Set the CMake Flags to use the nvcc Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-elseif("${HIP_COMPILER}" STREQUAL "hcc")
-    # Set the CMake Flags to use the hcc Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-elseif("${HIP_COMPILER}" STREQUAL "clang")
-    #Number of parallel jobs by default is 1
-    if(NOT DEFINED HIP_CLANG_NUM_PARALLEL_JOBS)
-      set(HIP_CLANG_NUM_PARALLEL_JOBS 1)
-    endif()
-    #Add support for parallel build and link
-    if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
-      check_cxx_compiler_flag("-parallel-jobs=1" HIP_CLANG_SUPPORTS_PARALLEL_JOBS)
-    endif()
-    if(HIP_CLANG_NUM_PARALLEL_JOBS GREATER 1)
-      if(${HIP_CLANG_SUPPORTS_PARALLEL_JOBS})
-        set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "-Wno-format-nonliteral -parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}")
-        set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "-parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}")
-      else()
-        message("clang compiler doesn't support parallel jobs")
-      endif()
-    endif()
-
-    # Set the CMake Flags to use the HIP-Clang Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-
-    if("${HIP_RUNTIME}" STREQUAL "rocclr")
-      if(TARGET host)
-        message(STATUS "host interface - found")
-        set(HIP_HOST_INTERFACE host)
-      endif()
-    endif()
-endif()
-
-###############################################################################
-# MACRO: Locate helper files
-###############################################################################
-macro(HIP_FIND_HELPER_FILE _name _extension)
-    set(_hip_full_name "${_name}.${_extension}")
-    get_filename_component(CMAKE_CURRENT_LIST_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
-    set(HIP_${_name} "${CMAKE_CURRENT_LIST_DIR}/FindHIP/${_hip_full_name}")
-    if(NOT EXISTS "${HIP_${_name}}")
-        set(error_message "${_hip_full_name} not found in ${CMAKE_CURRENT_LIST_DIR}/FindHIP")
-        if(HIP_FIND_REQUIRED)
-            message(FATAL_ERROR "${error_message}")
-        else()
-            if(NOT HIP_FIND_QUIETLY)
-                message(STATUS "${error_message}")
-            endif()
-        endif()
-    endif()
-    # Set this variable as internal, so the user isn't bugged with it.
-    set(HIP_${_name} ${HIP_${_name}} CACHE INTERNAL "Location of ${_full_name}" FORCE)
-endmacro()
-
-###############################################################################
-hip_find_helper_file(run_make2cmake cmake)
-hip_find_helper_file(run_hipcc cmake)
-###############################################################################
-
-###############################################################################
-# MACRO: Reset compiler flags
-###############################################################################
-macro(HIP_RESET_FLAGS)
-    unset(HIP_HIPCC_FLAGS)
-    unset(HIP_HCC_FLAGS)
-    unset(HIP_CLANG_FLAGS)
-    unset(HIP_NVCC_FLAGS)
-    foreach(config ${_hip_configuration_types})
-        string(TOUPPER ${config} config_upper)
-        unset(HIP_HIPCC_FLAGS_${config_upper})
-        unset(HIP_HCC_FLAGS_${config_upper})
-        unset(HIP_CLANG_FLAGS_${config_upper})
-        unset(HIP_NVCC_FLAGS_${config_upper})
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Separate the options from the sources
-###############################################################################
-macro(HIP_GET_SOURCES_AND_OPTIONS _sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options)
-    set(${_sources})
-    set(${_cmake_options})
-    set(${_hipcc_options})
-    set(${_hcc_options})
-    set(${_clang_options})
-    set(${_nvcc_options})
-    set(_hipcc_found_options FALSE)
-    set(_hcc_found_options FALSE)
-    set(_clang_found_options FALSE)
-    set(_nvcc_found_options FALSE)
-    foreach(arg ${ARGN})
-        if("x${arg}" STREQUAL "xHIPCC_OPTIONS")
-            set(_hipcc_found_options TRUE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xHCC_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options TRUE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xCLANG_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options TRUE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xNVCC_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options TRUE)
-        elseif(
-                "x${arg}" STREQUAL "xEXCLUDE_FROM_ALL" OR
-                "x${arg}" STREQUAL "xSTATIC" OR
-                "x${arg}" STREQUAL "xSHARED" OR
-                "x${arg}" STREQUAL "xMODULE"
-                )
-            list(APPEND ${_cmake_options} ${arg})
-        else()
-            if(_hipcc_found_options)
-                list(APPEND ${_hipcc_options} ${arg})
-            elseif(_hcc_found_options)
-                list(APPEND ${_hcc_options} ${arg})
-            elseif(_clang_found_options)
-                list(APPEND ${_clang_options} ${arg})
-            elseif(_nvcc_found_options)
-                list(APPEND ${_nvcc_options} ${arg})
-            else()
-                # Assume this is a file
-                list(APPEND ${_sources} ${arg})
-            endif()
-        endif()
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Add include directories to pass to the hipcc command
-###############################################################################
-set(HIP_HIPCC_INCLUDE_ARGS_USER "")
-macro(HIP_INCLUDE_DIRECTORIES)
-    foreach(dir ${ARGN})
-        list(APPEND HIP_HIPCC_INCLUDE_ARGS_USER $<$<BOOL:${dir}>:-I${dir}>)
-    endforeach()
-endmacro()
-
-###############################################################################
-# FUNCTION: Helper to avoid clashes of files with the same basename but different paths
-###############################################################################
-function(HIP_COMPUTE_BUILD_PATH path build_path)
-    # Convert to cmake style paths
-    file(TO_CMAKE_PATH "${path}" bpath)
-    if(IS_ABSOLUTE "${bpath}")
-        string(FIND "${bpath}" "${CMAKE_CURRENT_BINARY_DIR}" _binary_dir_pos)
-        if(_binary_dir_pos EQUAL 0)
-            file(RELATIVE_PATH bpath "${CMAKE_CURRENT_BINARY_DIR}" "${bpath}")
-        else()
-            file(RELATIVE_PATH bpath "${CMAKE_CURRENT_SOURCE_DIR}" "${bpath}")
-        endif()
-    endif()
-
-    # Remove leading /
-    string(REGEX REPLACE "^[/]+" "" bpath "${bpath}")
-    # Avoid absolute paths by removing ':'
-    string(REPLACE ":" "_" bpath "${bpath}")
-    # Avoid relative paths that go up the tree
-    string(REPLACE "../" "__/" bpath "${bpath}")
-    # Avoid spaces
-    string(REPLACE " " "_" bpath "${bpath}")
-    # Strip off the filename
-    get_filename_component(bpath "${bpath}" PATH)
-
-    set(${build_path} "${bpath}" PARENT_SCOPE)
-endfunction()
-
-###############################################################################
-# MACRO: Parse OPTIONS from ARGN & set variables prefixed by _option_prefix
-###############################################################################
-macro(HIP_PARSE_HIPCC_OPTIONS _option_prefix)
-    set(_hip_found_config)
-    foreach(arg ${ARGN})
-        # Determine if we are dealing with a per-configuration flag
-        foreach(config ${_hip_configuration_types})
-            string(TOUPPER ${config} config_upper)
-            if(arg STREQUAL "${config_upper}")
-                set(_hip_found_config _${arg})
-                # Clear arg to prevent it from being processed anymore
-                set(arg)
-            endif()
-        endforeach()
-        if(arg)
-            list(APPEND ${_option_prefix}${_hip_found_config} "${arg}")
-        endif()
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Try and include dependency file if it exists
-###############################################################################
-macro(HIP_INCLUDE_HIPCC_DEPENDENCIES dependency_file)
-    set(HIP_HIPCC_DEPEND)
-    set(HIP_HIPCC_DEPEND_REGENERATE FALSE)
-
-    # Create the dependency file if it doesn't exist
-    if(NOT EXISTS ${dependency_file})
-        file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
-    endif()
-    # Include the dependency file
-    include(${dependency_file})
-
-    # Verify the existence of all the included files
-    if(HIP_HIPCC_DEPEND)
-        foreach(f ${HIP_HIPCC_DEPEND})
-            if(NOT EXISTS ${f})
-                # If they aren't there, regenerate the file again
-                set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
-            endif()
-        endforeach()
-    else()
-        # No dependencies, so regenerate the file
-        set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
-    endif()
-
-    # Regenerate the dependency file if needed
-    if(HIP_HIPCC_DEPEND_REGENERATE)
-        set(HIP_HIPCC_DEPEND ${dependency_file})
-        file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
-    endif()
-endmacro()
-
-###############################################################################
-# MACRO: Prepare cmake commands for the target
-###############################################################################
-macro(HIP_PREPARE_TARGET_COMMANDS _target _format _generated_files _source_files)
-    set(_hip_flags "")
-    string(TOUPPER "${CMAKE_BUILD_TYPE}" _hip_build_configuration)
-    if(HIP_HOST_COMPILATION_CPP)
-        set(HIP_C_OR_CXX CXX)
-    else()
-        set(HIP_C_OR_CXX C)
-    endif()
-    set(generated_extension ${CMAKE_${HIP_C_OR_CXX}_OUTPUT_EXTENSION})
-
-    # Initialize list of includes with those specified by the user. Append with
-    # ones specified to cmake directly.
-    set(HIP_HIPCC_INCLUDE_ARGS ${HIP_HIPCC_INCLUDE_ARGS_USER})
-
-    # Add the include directories
-    set(include_directories_generator "$<TARGET_PROPERTY:${_target},INCLUDE_DIRECTORIES>")
-    list(APPEND HIP_HIPCC_INCLUDE_ARGS "$<$<BOOL:${include_directories_generator}>:-I$<JOIN:${include_directories_generator}, -I>>")
-
-    get_directory_property(_hip_include_directories INCLUDE_DIRECTORIES)
-    list(REMOVE_DUPLICATES _hip_include_directories)
-    if(_hip_include_directories)
-        foreach(dir ${_hip_include_directories})
-            list(APPEND HIP_HIPCC_INCLUDE_ARGS $<$<BOOL:${dir}>:-I${dir}>)
-        endforeach()
-    endif()
-
-    HIP_GET_SOURCES_AND_OPTIONS(_hip_sources _hip_cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_HIPCC_FLAGS ${_hipcc_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_HCC_FLAGS ${_hcc_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_CLANG_FLAGS ${_clang_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_NVCC_FLAGS ${_nvcc_options})
-
-    # Add the compile definitions
-    set(compile_definition_generator "$<TARGET_PROPERTY:${_target},COMPILE_DEFINITIONS>")
-    list(APPEND HIP_HIPCC_FLAGS "$<$<BOOL:${compile_definition_generator}>:-D$<JOIN:${compile_definition_generator}, -D>>")
-
-    # Check if we are building shared library.
-    set(_hip_build_shared_libs FALSE)
-    list(FIND _hip_cmake_options SHARED _hip_found_SHARED)
-    list(FIND _hip_cmake_options MODULE _hip_found_MODULE)
-    if(_hip_found_SHARED GREATER -1 OR _hip_found_MODULE GREATER -1)
-        set(_hip_build_shared_libs TRUE)
-    endif()
-    list(FIND _hip_cmake_options STATIC _hip_found_STATIC)
-    if(_hip_found_STATIC GREATER -1)
-        set(_hip_build_shared_libs FALSE)
-    endif()
-
-    # If we are building a shared library, add extra flags to HIP_HIPCC_FLAGS
-    if(_hip_build_shared_libs)
-        list(APPEND HIP_HCC_FLAGS "-fPIC")
-        list(APPEND HIP_CLANG_FLAGS "-fPIC")
-        list(APPEND HIP_NVCC_FLAGS "--shared -Xcompiler '-fPIC'")
-    endif()
-
-    # Set host compiler
-    set(HIP_HOST_COMPILER "${CMAKE_${HIP_C_OR_CXX}_COMPILER}")
-
-    # Set compiler flags
-    set(_HIP_HOST_FLAGS "set(CMAKE_HOST_FLAGS ${CMAKE_${HIP_C_OR_CXX}_FLAGS})")
-    set(_HIP_HIPCC_FLAGS "set(HIP_HIPCC_FLAGS ${HIP_HIPCC_FLAGS})")
-    set(_HIP_HCC_FLAGS "set(HIP_HCC_FLAGS ${HIP_HCC_FLAGS})")
-    set(_HIP_CLANG_FLAGS "set(HIP_CLANG_FLAGS ${HIP_CLANG_FLAGS})")
-    set(_HIP_NVCC_FLAGS "set(HIP_NVCC_FLAGS ${HIP_NVCC_FLAGS})")
-    foreach(config ${_hip_configuration_types})
-        string(TOUPPER ${config} config_upper)
-        set(_HIP_HOST_FLAGS "${_HIP_HOST_FLAGS}\nset(CMAKE_HOST_FLAGS_${config_upper} ${CMAKE_${HIP_C_OR_CXX}_FLAGS_${config_upper}})")
-        set(_HIP_HIPCC_FLAGS "${_HIP_HIPCC_FLAGS}\nset(HIP_HIPCC_FLAGS_${config_upper} ${HIP_HIPCC_FLAGS_${config_upper}})")
-        set(_HIP_HCC_FLAGS "${_HIP_HCC_FLAGS}\nset(HIP_HCC_FLAGS_${config_upper} ${HIP_HCC_FLAGS_${config_upper}})")
-        set(_HIP_CLANG_FLAGS "${_HIP_CLANG_FLAGS}\nset(HIP_CLANG_FLAGS_${config_upper} ${HIP_CLANG_FLAGS_${config_upper}})")
-        set(_HIP_NVCC_FLAGS "${_HIP_NVCC_FLAGS}\nset(HIP_NVCC_FLAGS_${config_upper} ${HIP_NVCC_FLAGS_${config_upper}})")
-    endforeach()
-
-    # Reset the output variable
-    set(_hip_generated_files "")
-    set(_hip_source_files "")
-
-    # Iterate over all arguments and create custom commands for all source files
-    foreach(file ${ARGN})
-        # Ignore any file marked as a HEADER_FILE_ONLY
-        get_source_file_property(_is_header ${file} HEADER_FILE_ONLY)
-        # Allow per source file overrides of the format. Also allows compiling non .cu files.
-        get_source_file_property(_hip_source_format ${file} HIP_SOURCE_PROPERTY_FORMAT)
-        if((${file} MATCHES "\\.cu$" OR _hip_source_format) AND NOT _is_header)
-            set(host_flag FALSE)
-        else()
-            set(host_flag TRUE)
-        endif()
-
-        if(NOT host_flag)
-            # Determine output directory
-            HIP_COMPUTE_BUILD_PATH("${file}" hip_build_path)
-            set(hip_compile_output_dir "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${_target}.dir/${hip_build_path}")
-
-            get_filename_component(basename ${file} NAME)
-            set(generated_file_path "${hip_compile_output_dir}/${CMAKE_CFG_INTDIR}")
-            set(generated_file_basename "${_target}_generated_${basename}${generated_extension}")
-
-            # Set file names
-            set(generated_file "${generated_file_path}/${generated_file_basename}")
-            set(cmake_dependency_file "${hip_compile_output_dir}/${generated_file_basename}.depend")
-            set(custom_target_script_pregen "${hip_compile_output_dir}/${generated_file_basename}.cmake.pre-gen")
-            set(custom_target_script "${hip_compile_output_dir}/${generated_file_basename}.cmake")
-
-            # Set properties for object files
-            set_source_files_properties("${generated_file}"
-                PROPERTIES
-                EXTERNAL_OBJECT true # This is an object file not to be compiled, but only be linked
-                )
-
-            # Don't add CMAKE_CURRENT_SOURCE_DIR if the path is already an absolute path
-            get_filename_component(file_path "${file}" PATH)
-            if(IS_ABSOLUTE "${file_path}")
-                set(source_file "${file}")
-            else()
-                set(source_file "${CMAKE_CURRENT_SOURCE_DIR}/${file}")
-            endif()
-
-            # Bring in the dependencies
-            HIP_INCLUDE_HIPCC_DEPENDENCIES(${cmake_dependency_file})
-
-            # Configure the build script
-            configure_file("${HIP_run_hipcc}" "${custom_target_script_pregen}" @ONLY)
-            file(GENERATE
-                OUTPUT "${custom_target_script}"
-                INPUT "${custom_target_script_pregen}"
-                )
-            set(main_dep DEPENDS ${source_file})
-            if(CMAKE_GENERATOR MATCHES "Makefiles")
-                set(verbose_output "$(VERBOSE)")
-            elseif(HIP_VERBOSE_BUILD)
-                set(verbose_output ON)
-            else()
-                set(verbose_output OFF)
-            endif()
-
-            # Create up the comment string
-            file(RELATIVE_PATH generated_file_relative_path "${CMAKE_BINARY_DIR}" "${generated_file}")
-            set(hip_build_comment_string "Building HIPCC object ${generated_file_relative_path}")
-
-            # Build the generated file and dependency file
-            add_custom_command(
-                OUTPUT ${generated_file}
-                # These output files depend on the source_file and the contents of cmake_dependency_file
-                ${main_dep}
-                DEPENDS ${HIP_HIPCC_DEPEND}
-                DEPENDS ${custom_target_script}
-                # Make sure the output directory exists before trying to write to it.
-                COMMAND ${CMAKE_COMMAND} -E make_directory "${generated_file_path}"
-                COMMAND ${CMAKE_COMMAND} ARGS
-                -D verbose:BOOL=${verbose_output}
-                -D build_configuration:STRING=${_hip_build_configuration}
-                -D "generated_file:STRING=${generated_file}"
-                -P "${custom_target_script}"
-                WORKING_DIRECTORY "${hip_compile_output_dir}"
-                COMMENT "${hip_build_comment_string}"
-                )
-
-            # Make sure the build system knows the file is generated
-            set_source_files_properties(${generated_file} PROPERTIES GENERATED TRUE)
-            list(APPEND _hip_generated_files ${generated_file})
-            list(APPEND _hip_source_files ${file})
-        endif()
-    endforeach()
-
-    # Set the return parameter
-    set(${_generated_files} ${_hip_generated_files})
-    set(${_source_files} ${_hip_source_files})
-endmacro()
-
-###############################################################################
-# HIP_ADD_EXECUTABLE
-###############################################################################
-macro(HIP_ADD_EXECUTABLE hip_target)
-    # Separate the sources from the options
-    HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} CLANG_OPTIONS ${_clang_options} NVCC_OPTIONS ${_nvcc_options})
-    if(_source_files)
-        list(REMOVE_ITEM _sources ${_source_files})
-    endif()
-    if("${HIP_COMPILER}" STREQUAL "hcc")
-        if("x${HCC_HOME}" STREQUAL "x")
-            if (DEFINED ENV{ROCM_PATH})
-                set(HCC_HOME "$ENV{ROCM_PATH}/hcc")
-            elseif(DEFINED ENV{HIP_PATH})
-                set(HCC_HOME "$ENV{HIP_PATH}/../hcc")
-            else()
-                set(HCC_HOME "/opt/rocm/hcc")
-            endif()
-        endif()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    elseif("${HIP_COMPILER}" STREQUAL "clang")
-        if("x${HIP_CLANG_PATH}" STREQUAL "x")
-            if(DEFINED ENV{HIP_CLANG_PATH})
-                set(HIP_CLANG_PATH $ENV{HIP_CLANG_PATH})
-            elseif(DEFINED ENV{ROCM_PATH})
-                set(HIP_CLANG_PATH "$ENV{ROCM_PATH}/llvm/bin")
-            elseif(DEFINED ENV{HIP_PATH})
-                set(HIP_CLANG_PATH "$ENV{HIP_PATH}/../llvm/bin")
-            else()
-                set(HIP_CLANG_PATH "/opt/rocm/llvm/bin")
-            endif()
-        endif()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    else()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    endif()
-    if ("${_sources}" STREQUAL "")
-        add_executable(${hip_target} ${_cmake_options} ${_generated_files} "")
-    else()
-        add_executable(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
-    endif()
-    set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE HIP)
-    # Link with host
-    if (HIP_HOST_INTERFACE)
-        # hip rt should be rocclr, compiler should be clang
-        target_link_libraries(${hip_target} ${HIP_HOST_INTERFACE})
-    endif()
-endmacro()
-
-###############################################################################
-# HIP_ADD_LIBRARY
-###############################################################################
-macro(HIP_ADD_LIBRARY hip_target)
-    # Separate the sources from the options
-    HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} ${_cmake_options} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} CLANG_OPTIONS ${_clang_options} NVCC_OPTIONS ${_nvcc_options})
-    if(_source_files)
-        list(REMOVE_ITEM _sources ${_source_files})
-    endif()
-    if ("${_sources}" STREQUAL "")
-        add_library(${hip_target} ${_cmake_options} ${_generated_files} "")
-    else()
-        add_library(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
-    endif()
-    set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE ${HIP_C_OR_CXX})
-    # Link with host
-    if (HIP_HOST_INTERFACE)
-        # hip rt should be rocclr, compiler should be clang
-        target_link_libraries(${hip_target} ${HIP_HOST_INTERFACE})
-    endif()
-endmacro()
-
-# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHIP/run_hipcc.cmake

@@ -1,182 +0,0 @@
-###############################################################################
-# Runs commands using HIPCC
-###############################################################################
-
-###############################################################################
-# This file runs the hipcc commands to produce the desired output file
-# along with the dependency file needed by CMake to compute dependencies.
-#
-# Input variables:
-#
-# verbose:BOOL=<>               OFF: Be as quiet as possible (default)
-#                               ON : Describe each step
-# build_configuration:STRING=<> Build configuration. Defaults to Debug.
-# generated_file:STRING=<>      File to generate. Mandatory argument.
-
-if(NOT build_configuration)
-    set(build_configuration Debug)
-endif()
-if(NOT generated_file)
-    message(FATAL_ERROR "You must specify generated_file on the command line")
-endif()
-
-# Set these up as variables to make reading the generated file easier
-set(HIP_HIPCC_EXECUTABLE "@HIP_HIPCC_EXECUTABLE@") # path
-set(HIP_HIPCONFIG_EXECUTABLE "@HIP_HIPCONFIG_EXECUTABLE@") #path
-set(HIP_HOST_COMPILER "@HIP_HOST_COMPILER@") # path
-set(CMAKE_COMMAND "@CMAKE_COMMAND@") # path
-set(HIP_run_make2cmake "@HIP_run_make2cmake@") # path
-set(HCC_HOME "@HCC_HOME@") #path
-set(HIP_CLANG_PATH "@HIP_CLANG_PATH@") #path
-set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "@HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS@")
-
-@HIP_HOST_FLAGS@
-@_HIP_HIPCC_FLAGS@
-@_HIP_HCC_FLAGS@
-@_HIP_CLANG_FLAGS@
-@_HIP_NVCC_FLAGS@
-#Needed to bring the HIP_HIPCC_INCLUDE_ARGS variable in scope
-set(HIP_HIPCC_INCLUDE_ARGS @HIP_HIPCC_INCLUDE_ARGS@) # list
-
-set(cmake_dependency_file "@cmake_dependency_file@") # path
-set(source_file "@source_file@") # path
-set(host_flag "@host_flag@") # bool
-
-# Determine compiler and compiler flags
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform OUTPUT_VARIABLE HIP_PLATFORM OUTPUT_STRIP_TRAILING_WHITESPACE)
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --compiler OUTPUT_VARIABLE HIP_COMPILER OUTPUT_STRIP_TRAILING_WHITESPACE)
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --runtime OUTPUT_VARIABLE HIP_RUNTIME OUTPUT_STRIP_TRAILING_WHITESPACE)
-if(NOT host_flag)
-    set(__CC ${HIP_HIPCC_EXECUTABLE})
-    if("${HIP_PLATFORM}" STREQUAL "amd")
-        if("${HIP_COMPILER}" STREQUAL "hcc")
-            if(NOT "x${HCC_HOME}" STREQUAL "x")
-                set(ENV{HCC_HOME} ${HCC_HOME})
-            endif()
-            set(__CC_FLAGS ${HIP_HIPCC_FLAGS} ${HIP_HCC_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_HCC_FLAGS_${build_configuration}})
-        elseif("${HIP_COMPILER}" STREQUAL "clang")
-            if(NOT "x${HIP_CLANG_PATH}" STREQUAL "x")
-                set(ENV{HIP_CLANG_PATH} ${HIP_CLANG_PATH})
-            endif()
-            # Temporarily include HIP_HCC_FLAGS for HIP-Clang for PyTorch builds
-            set(__CC_FLAGS ${HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS} ${HIP_HIPCC_FLAGS} ${HIP_HCC_FLAGS} ${HIP_CLANG_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_HCC_FLAGS_${build_configuration}} ${HIP_CLANG_FLAGS_${build_configuration}})
-        endif()
-    else()
-        set(__CC_FLAGS ${HIP_HIPCC_FLAGS} ${HIP_NVCC_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_NVCC_FLAGS_${build_configuration}})
-    endif()
-else()
-    set(__CC ${HIP_HOST_COMPILER})
-    set(__CC_FLAGS ${CMAKE_HOST_FLAGS} ${CMAKE_HOST_FLAGS_${build_configuration}})
-endif()
-set(__CC_INCLUDES ${HIP_HIPCC_INCLUDE_ARGS})
-
-# hip_execute_process - Executes a command with optional command echo and status message.
-#   status     - Status message to print if verbose is true
-#   command    - COMMAND argument from the usual execute_process argument structure
-#   ARGN       - Remaining arguments are the command with arguments
-#   HIP_result - Return value from running the command
-macro(hip_execute_process status command)
-    set(_command ${command})
-    if(NOT "x${_command}" STREQUAL "xCOMMAND")
-        message(FATAL_ERROR "Malformed call to hip_execute_process.  Missing COMMAND as second argument. (command = ${command})")
-    endif()
-    if(verbose)
-        execute_process(COMMAND "${CMAKE_COMMAND}" -E echo -- ${status})
-        # Build command string to print
-        set(hip_execute_process_string)
-        foreach(arg ${ARGN})
-            # Escape quotes if any
-            string(REPLACE "\"" "\\\"" arg ${arg})
-            # Surround args with spaces with quotes
-            if(arg MATCHES " ")
-                list(APPEND hip_execute_process_string "\"${arg}\"")
-            else()
-                list(APPEND hip_execute_process_string ${arg})
-            endif()
-        endforeach()
-        # Echo the command
-        execute_process(COMMAND ${CMAKE_COMMAND} -E echo ${hip_execute_process_string})
-    endif()
-    # Run the command
-    execute_process(COMMAND ${ARGN} RESULT_VARIABLE HIP_result)
-endmacro()
-
-# Delete the target file
-hip_execute_process(
-    "Removing ${generated_file}"
-    COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
-    )
-
-# Generate the dependency file
-hip_execute_process(
-    "Generating dependency file: ${cmake_dependency_file}.pre"
-    COMMAND "${__CC}"
-    -M
-    "${source_file}"
-    -o "${cmake_dependency_file}.pre"
-    ${__CC_FLAGS}
-    ${__CC_INCLUDES}
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Generate the cmake readable dependency file to a temp file
-hip_execute_process(
-    "Generating temporary cmake readable file: ${cmake_dependency_file}.tmp"
-    COMMAND "${CMAKE_COMMAND}"
-    -D "input_file:FILEPATH=${cmake_dependency_file}.pre"
-    -D "output_file:FILEPATH=${cmake_dependency_file}.tmp"
-    -D "verbose=${verbose}"
-    -P "${HIP_run_make2cmake}"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Copy the file if it is different
-hip_execute_process(
-    "Copy if different ${cmake_dependency_file}.tmp to ${cmake_dependency_file}"
-    COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${cmake_dependency_file}.tmp" "${cmake_dependency_file}"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Delete the temporary file
-hip_execute_process(
-    "Removing ${cmake_dependency_file}.tmp and ${cmake_dependency_file}.pre"
-    COMMAND "${CMAKE_COMMAND}" -E remove "${cmake_dependency_file}.tmp" "${cmake_dependency_file}.pre"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Generate the output file
-hip_execute_process(
-    "Generating ${generated_file}"
-    COMMAND "${__CC}"
-    -c
-    "${source_file}"
-    -o "${generated_file}"
-    ${__CC_FLAGS}
-    ${__CC_INCLUDES}
-    )
-
-if(HIP_result)
-    # Make sure that we delete the output file
-    hip_execute_process(
-        "Removing ${generated_file}"
-        COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
-        )
-    message(FATAL_ERROR "Error generating file ${generated_file}")
-else()
-    if(verbose)
-        message("Generated ${generated_file} successfully.")
-    endif()
-endif()
-# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHIP/run_make2cmake.cmake

@@ -1,50 +0,0 @@
-###############################################################################
-# Computes dependencies using HIPCC
-###############################################################################
-
-###############################################################################
-# This file converts dependency files generated using hipcc to a format that
-# cmake can understand.
-
-# Input variables:
-#
-# input_file:STRING=<> Dependency file to parse. Required argument
-# output_file:STRING=<> Output file to generate. Required argument
-
-if(NOT input_file OR NOT output_file)
-    message(FATAL_ERROR "You must specify input_file and output_file on the command line")
-endif()
-
-file(READ ${input_file} depend_text)
-
-if (NOT "${depend_text}" STREQUAL "")
-    string(REPLACE " /" "\n/" depend_text ${depend_text})
-    string(REGEX REPLACE "^.*:" "" depend_text ${depend_text})
-    string(REGEX REPLACE "[ \\\\]*\n" ";" depend_text ${depend_text})
-
-    set(dependency_list "")
-
-    foreach(file ${depend_text})
-        string(REGEX REPLACE "^ +" "" file ${file})
-        if(NOT EXISTS "${file}")
-            message(WARNING " Removing non-existent dependency file: ${file}")
-            set(file "")
-        endif()
-
-        if(NOT IS_DIRECTORY "${file}")
-            get_filename_component(file_absolute "${file}" ABSOLUTE)
-            list(APPEND dependency_list "${file_absolute}")
-        endif()
-    endforeach()
-endif()
-
-# Remove the duplicate entries and sort them.
-list(REMOVE_DUPLICATES dependency_list)
-list(SORT dependency_list)
-
-foreach(file ${dependency_list})
-    set(hip_hipcc_depend "${hip_hipcc_depend} \"${file}\"\n")
-endforeach()
-
-file(WRITE ${output_file} "# Generated by: FindHIP.cmake. Do not edit.\nSET(HIP_HIPCC_DEPEND\n ${hip_hipcc_depend})\n\n")
-# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHYPRE.cmake

@@ -14,10 +14,33 @@
 #   - HYPRE_LIBRARIES
 #   - HYPRE_INCLUDE_DIRS
 #   - HYPRE_VERSION
+#   - HYPRE_USING_HIP (internal)
+
+if (HYPRE_FOUND)
+  if (HYPRE_USING_HIP)
+    find_package(rocsparse REQUIRED)
+    find_package(rocrand REQUIRED)
+  endif()
+  return()
+endif()
 
 include(MfemCmakeUtilities)
 mfem_find_package(HYPRE HYPRE HYPRE_DIR "include" "HYPRE.h" "lib" "HYPRE"
-  "Paths to headers required by HYPRE." "Libraries required by HYPRE.")
+  "Paths to headers required by HYPRE." "Libraries required by HYPRE."
+  CHECK_BUILD HYPRE_USING_HIP FALSE
+  "
+#undef HYPRE_USING_HIP
+#include <HYPRE_config.h>
+
+#ifndef HYPRE_USING_HIP
+#error HYPRE is built without HIP.
+#endif
+
+int main()
+{
+   return 0;
+}
+")
 
 if (HYPRE_FOUND AND (NOT HYPRE_VERSION))
   try_run(HYPRE_VERSION_RUN_RESULT HYPRE_VERSION_COMPILE_RESULT
@@ -33,3 +56,12 @@ if (HYPRE_FOUND AND (NOT HYPRE_VERSION))
     message(FATAL_ERROR "Unable to determine HYPRE version.")
   endif()
 endif()
+
+if (HYPRE_FOUND AND HYPRE_USING_HIP)
+  find_package(rocsparse REQUIRED)
+  find_package(rocrand REQUIRED)
+  list(APPEND HYPRE_LIBRARIES ${rocsparse_LIBRARIES} ${rocrand_LIBRARIES})
+  set(HYPRE_LIBRARIES ${HYPRE_LIBRARIES} CACHE STRING
+      "HYPRE libraries + dependencies." FORCE)
+  message(STATUS "Updated HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}")
+endif()

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -46,8 +46,7 @@ endfunction()
 # Wrapper for add_executable that calls the HIP wrapper if applicable
 macro(mfem_add_executable NAME)
   if (MFEM_USE_HIP)
-    hip_add_executable(${NAME} ${ARGN})
-    set_target_properties(${NAME} PROPERTIES LINKER_LANGUAGE CXX)
+    add_executable(${NAME} ${ARGN})
   else()
     add_executable(${NAME} ${ARGN})
   endif()
@@ -56,7 +55,7 @@ endmacro()
 # Wrapper for add_library that calls the HIP wrapper if applicable
 macro(mfem_add_library NAME)
   if (MFEM_USE_HIP)
-    hip_add_library(${NAME} ${ARGN})
+    add_library(${NAME} ${ARGN})
   else()
     add_library(${NAME} ${ARGN})
   endif()
@@ -92,8 +91,6 @@ macro(add_mfem_examples EXE_SRCS)
     # If CUDA is enabled, tag source files to be compiled with nvcc.
     if (MFEM_USE_CUDA)
       set_source_files_properties(${SRC_FILE} PROPERTIES LANGUAGE CUDA)
-    elseif(MFEM_USE_HIP)
-      set_source_files_properties(${SRC_FILE} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
     endif()
 
     get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
@@ -159,8 +156,6 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
       endforeach()
       set(EXTRA_OPTIONS_LIST ${LIST_})
     endif()
-  elseif(MFEM_USE_HIP)
-    set_source_files_properties(${MAIN_LIST} ${EXTRA_SOURCES_LIST} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
   endif()
 
   # Actually add the executable
@@ -534,12 +529,15 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
             if (NOT ImportConfig)
               set(ImportConfig RELEASE)
             endif()
+            set(ImportConfigSuffix "_${ImportConfig}")
             get_target_property(ImpConfigs ${TargetName} IMPORTED_CONFIGURATIONS)
             list(FIND ImpConfigs ${ImportConfig} _Index)
-            if (_Index EQUAL -1)
-              message(FATAL_ERROR " *** ${ReqPack}: configuration "
-                "${ImportConfig} not found. Set ${ReqPack}_IMPORT_CONFIG "
-                "from the list: ${ImpConfigs}.")
+            if ((_Index EQUAL -1) OR ("${ImportConfig}" STREQUAL "NO_CONFIG"))
+              set(ImportConfig "NO_CONFIG")
+              set(ImportConfigSuffix "")
+              # message(FATAL_ERROR " *** ${ReqPack}: configuration "
+              #   "${ImportConfig} not found. Set ${ReqPack}_IMPORT_CONFIG "
+              #   "from the list: ${ImpConfigs}.")
             endif()
           endif()
           # Set _Pack_LIBS
@@ -551,8 +549,8 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
             endif()
           else()
             # Set _Pack_LIBS from the target properties for ImportConfig
-            foreach (_prop IMPORTED_LOCATION_${ImportConfig}
-                IMPORTED_LINK_INTERFACE_LIBRARIES_${ImportConfig})
+            foreach (_prop IMPORTED_LOCATION${ImportConfigSuffix}
+                IMPORTED_LINK_INTERFACE_LIBRARIES${ImportConfigSuffix})
               get_target_property(_value ${TargetName} ${_prop})
               if (_value)
                 list(APPEND _Pack_LIBS ${_value})
@@ -564,7 +562,7 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
             endif()
           endif()
           # Set _Pack_INCS
-          foreach (_prop INCLUDE_DIRECTORIES)
+          foreach (_prop INCLUDE_DIRECTORIES INTERFACE_INCLUDE_DIRECTORIES)
             get_target_property(_value ${TargetName} ${_prop})
             if (_value)
               list(APPEND _Pack_INCS ${_value})
@@ -742,6 +740,133 @@ function(mfem_find_library Name Prefix Lib LibDoc CheckVar CheckSrc)
 endfunction(mfem_find_library)
 
 
+#
+# Extract compile and link options needed by the given target.
+#
+function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
+
+  if (NOT TARGET ${Target})
+    return()
+  endif()
+
+  # CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG -> '-Wl,-rpath,'
+  set(shared_link_flag ${CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG})
+  if (NOT shared_link_flag)
+    set(shared_link_flag "-Wl,-rpath,")
+  endif()
+
+  set(tgt "${Target}")
+  unset(CompileOpts)
+  unset(LinkOpts)
+  get_target_property(IsImported ${tgt} IMPORTED)
+  # message(STATUS "${tgt}[IMPORTED]: ${IsImported}")
+  # Generally, the possible target types are: STATIC_LIBRARY, MODULE_LIBRARY,
+  # SHARED_LIBRARY, INTERFACE_LIBRARY, EXECUTABLE.
+  get_target_property(type ${tgt} TYPE)
+  # message(STATUS "${tgt}[TYPE]: ${type}")
+  unset(ImportConfig)
+  get_target_property(ImportConfigs ${tgt} IMPORTED_CONFIGURATIONS)
+  if (ImportConfigs)
+    list(GET ImportConfigs 0 ImportConfig)
+  endif()
+  if (NOT ImportConfig)
+    set(ImportConfig RELEASE)
+  endif()
+  # message(STATUS "${tgt}[ImportConfig]: ${ImportConfig}")
+  # List all properties with: cmake --help-property-list
+  get_target_property(Defs ${tgt} INTERFACE_COMPILE_DEFINITIONS)
+  if (Defs)
+    list(REMOVE_DUPLICATES Defs)
+    foreach(Def ${Defs})
+      list(APPEND CompileOpts "-D${Def}")
+    endforeach()
+  endif()
+  get_target_property(Opts ${tgt} INTERFACE_COMPILE_OPTIONS)
+  if (Opts)
+    foreach(Opt ${Opts})
+      list(APPEND CompileOpts "${Opt}")
+    endforeach()
+  endif()
+  get_target_property(Dirs ${tgt} INTERFACE_INCLUDE_DIRECTORIES)
+  if (Dirs)
+    list(REMOVE_DUPLICATES Dirs)
+    foreach(Dir ${Dirs})
+      list(APPEND CompileOpts "-I\"${Dir}\"")
+    endforeach()
+  endif()
+  get_target_property(SysDirs ${tgt} INTERFACE_SYSTEM_INCLUDE_DIRECTORIES)
+  if (SysDirs)
+    list(REMOVE_DUPLICATES SysDirs)
+    foreach(SysDir ${SysDirs})
+      list(APPEND CompileOpts "-isystem \"${SysDir}\"")
+    endforeach()
+  endif()
+  if ("${type}" STREQUAL "STATIC_LIBRARY")
+    get_target_property(Location ${tgt} LOCATION)
+    if (Location)
+      list(APPEND LinkOpts "\"${Location}\"")
+    else()
+      message(STATUS " *** Warning: [${tgt}] LOCATION not defined!")
+    endif()
+  elseif ("${type}" STREQUAL "SHARED_LIBRARY")
+    get_target_property(Location ${tgt} LOCATION)
+    if (Location)
+      get_filename_component(Dir ${Location} DIRECTORY)
+      get_filename_component(NameWE ${Location} NAME_WE)
+      string(REGEX REPLACE "^lib" "" LibName ${NameWE})
+      list(APPEND LinkOpts
+        "-L\"${Dir}\""
+        "${shared_link_flag}\"${Dir}\""
+        "-l${LibName}")
+    else()
+      message(STATUS " *** Warning: [${tgt}] LOCATION not defined!")
+    endif()
+  elseif ("${type}" STREQUAL "INTERFACE_LIBRARY")
+    get_target_property(Libs ${tgt} INTERFACE_LINK_LIBRARIES)
+    if (Libs)
+      foreach(Lib ${Libs})
+        if (NOT (TARGET ${Lib}))
+          list(APPEND LinkOpts "${Lib}")
+        else()
+          mfem_get_target_options(${Lib} COpts LOpts)
+          list(APPEND CompileOpts ${COpts})
+          list(APPEND LinkOpts ${LOpts})
+        endif()
+      endforeach()
+    endif()
+    # Other properties we may need to handle:
+    # INTERFACE_LINK_DEPENDS
+    # INTERFACE_LINK_DIRECTORIES
+    # INTERFACE_LINK_OPTIONS
+  else()
+    message(STATUS " *** Warning: [${tgt}] uses target type '${type}'"
+      " which is not supported!")
+  endif()
+
+  # Other potentially relevant properties:
+  # - For all target types:
+  # IMPORTED_LIBNAME
+  # IMPORTED_LIBNAME_${ImportConfig}
+  # INTERFACE_AUTOUIC_OPTIONS
+  # INTERFACE_COMPILE_FEATURES
+  # INTERFACE_POSITION_INDEPENDENT_CODE
+  # INTERFACE_SOURCES
+  # INTERFACE_SYSTEM_INCLUDE_DIRECTORIES)
+  # - For non-"INTERFACE_LIBRARY" target types only:
+  # IMPORTED_LOCATION
+  # IMPORTED_LOCATION_${ImportConfig}
+  # IMPORTED_LINK_INTERFACE_LIBRARIES
+  # IMPORTED_LINK_INTERFACE_LIBRARIES_${ImportConfig}
+  # LINK_FLAGS
+  # LINK_FLAGS_${ImportConfig}
+  # LOCATION_${ImportConfig})
+
+  set(${CompileOptsVar} "${CompileOpts}" PARENT_SCOPE)
+  set(${LinkOptsVar} "${LinkOpts}" PARENT_SCOPE)
+
+endfunction(mfem_get_target_options)
+
+
 #
 #   Function that creates 'config.mk' from 'config.mk.in' for the both the
 #   build- and the install-locations and define install rules for 'config.mk'
@@ -760,13 +885,15 @@ function(mfem_export_mk_files)
   # Convert Boolean vars to YES/NO without writing the values to cache
   set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
       MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
-      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
+      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP
       MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
-      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX
-      MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
-      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI
-      MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD
-      MFEM_USE_ADIOS2 MFEM_USE_BENCHMARK MFEM_USE_PARELAG)
+      MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS MFEM_USE_STRUMPACK
+      MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_GNUTLS MFEM_USE_NETCDF
+      MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS
+      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA
+      MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER
+      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO
+      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG)
   foreach(var ${CONFIG_MK_BOOL_VARS})
     if (${var})
       set(${var} YES)
@@ -868,8 +995,18 @@ function(mfem_export_mk_files)
     get_filename_component(suffix ${lib} EXT)
     # handle interfaces (e.g., SCOREC::apf)
     if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*")
-    elseif (NOT "${lib}" MATCHES "SCOREC::.*" AND "${lib}" MATCHES ".*::.*")
-      message(FATAL_ERROR "***** interface lib found ... exiting *****")
+    elseif (TARGET "${lib}")
+      mfem_get_target_options(${lib} CompileOpts LinkOpts)
+      # Removing duplicates may lead to issues:
+      # list(REMOVE_DUPLICATES CompileOpts)
+      # list(REMOVE_DUPLICATES LinkOpts)
+      string(REPLACE ";" " " COpts "${CompileOpts}")
+      string(REPLACE ";" " " LOpts "${LinkOpts}")
+      # message(STATUS "${lib}[COpts]: '${COpts}'")
+      # message(STATUS "${lib}[LOpts]: '${LOpts}'")
+      set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} ${COpts}")
+      set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${LOpts}")
+      # message(FATAL_ERROR "***** interface lib found ... exiting *****")
       # handle static and shared libs
     elseif ("${suffix}" STREQUAL "${CMAKE_SHARED_LIBRARY_SUFFIX}")
       get_filename_component(dir ${lib} DIRECTORY)

config/config.mk.in

@@ -91,7 +91,7 @@ MFEM_MPIEXEC_NP = @MFEM_MPIEXEC_NP@
 MFEM_MPI_NP     = @MFEM_MPI_NP@
 
 # The NVCC compiler cannot link with -x=cu
-MFEM_LINK_FLAGS := $(filter-out -x=cu, $(MFEM_FLAGS))
+MFEM_LINK_FLAGS := $(filter-out -x=cu -xhip, $(MFEM_FLAGS))
 
 # Optional extra configuration
 @MFEM_CONFIG_EXTRA@

config/defaults.cmake

@@ -50,6 +50,7 @@ option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
 option(MFEM_USE_CUDA "Enable CUDA" OFF)
+option(MFEM_USE_HIP "Enable HIP" OFF)
 option(MFEM_USE_OCCA "Enable OCCA" OFF)
 option(MFEM_USE_RAJA "Enable RAJA" OFF)
 option(MFEM_USE_CEED "Enable CEED" OFF)
@@ -107,6 +108,7 @@ if (MFEM_USE_CUDA)
    set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
        "Libraries that HYPRE depends on.")
 endif()
+# HIP dependency for HYPRE is handled in FindHYPRE.cmake.
 
 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")
 
@@ -226,6 +228,11 @@ set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")
 
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
+# If RAJA is built with external CAMP:
+# set(RAJA_REQUIRED_PACKAGES "camp"
+#     CACHE STRING "Packages that RAJA depends on.")
+# set(camp_DIR "${MFEM_DIR}/../camp/lib/cmake/camp"
+#     CACHE PATH "Path to CAMP CMake files.")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
 set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")
@@ -245,7 +252,8 @@ set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")
 set(LAPACK_LIBRARIES "" CACHE STRING "The LAPACK library.")
 
-set(CODIPACK_INCLUDE_DIRS "${MFEM_DIR}/../CoDiPack/inlude" CACHE STRING "Path to CoDiPack headers.")
+set(CODIPACK_INCLUDE_DIRS "${MFEM_DIR}/../CoDiPack/include" CACHE STRING
+    "Path to CoDiPack headers.")
 set(CODIPACK_LIBRARIES "")
 
 # Some useful variables:

config/defaults.mk

@@ -172,6 +172,20 @@ ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
    MPI_LIB = -L$(MPI_DIR)/lib $(XLINKER)-rpath,$(MPI_DIR)/lib -lmpi
 endif
 
+# ROCM/HIP directory such that ROCM/HIP libraries like rocsparse and rocrand are
+# found in $(HIP_DIR)/lib, usually as links. Typically, this directoory is of
+# the form /opt/rocm-X.Y.Z which is called ROCM_PATH by hipconfig.
+ifeq ($(MFEM_USE_HIP),YES)
+   HIP_DIR := $(patsubst %/,%,$(dir $(shell which $(HIP_CXX))))
+   HIP_DIR := $(patsubst %/,%,$(dir $(HIP_DIR)))
+   ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*))
+      HIP_DIR := $(shell hipconfig --rocmpath 2> /dev/null)
+      ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*))
+         $(error Unable to determine HIP_DIR. Please set it manually.)
+      endif
+   endif
+endif
+
 # Compile and link options for zlib.
 ZLIB_DIR =
 ZLIB_OPT = $(if $(ZLIB_DIR),-I$(ZLIB_DIR)/include)
@@ -189,6 +203,11 @@ ifeq (YES,$(MFEM_USE_CUDA))
    # This is only necessary when hypre is built with cuda:
    HYPRE_LIB += -lcusparse -lcurand
 endif
+ifeq (YES,$(MFEM_USE_HIP))
+   # This is only necessary when hypre is built with hip:
+   HYPRE_LIB += -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib\
+ -lrocsparse -lrocrand
+endif
 
 # METIS library configuration
 ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
@@ -430,9 +449,9 @@ GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs
 CUDA_OPT =
 CUDA_LIB = -lcusparse
 
-# HIP library configuration (currently not needed)
+# HIP library configuration
 HIP_OPT =
-HIP_LIB =
+HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse
 
 # OCCA library configuration
 OCCA_DIR = @MFEM_DIR@/../occa

config/test.mk

@@ -58,14 +58,14 @@ ifneq (,$(filter test%,$(MAKECMDGOALS)))
    MAKEFLAGS += -k
 endif
 # Test runs of the examples/miniapps with parameters - check exit code:
-# 0 means success, 255 means the test was skipped, anything else means error
+# 0 means success, 242 means the test was skipped, anything else means error
 mfem-test = \
    printf "   $(3) [$(2) $(1) ... ]: "; \
    $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) $(if $(5),,-no-vis )$(4) \
      > $(1).stderr 2>&1); \
    err="$$3"; \
    if [ "$$3" = 0 ]; then $(PRINT_OK); \
-   else if [ "$$3" = 255 ]; then $(PRINT_SKIP); err=0; \
+   else if [ "$$3" = 242 ]; then $(PRINT_SKIP); err=0; \
    else $(PRINT_FAILED); cat $(1).stderr; fi; fi; \
    rm -f $(1).stderr; exit $$err
 
@@ -76,7 +76,7 @@ mfem-test-file = \
    $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) -no-vis > $(1).stderr 2>&1); \
    err="$$3"; \
    if [ "$$3" = 0 ] && [ -e $(4) ]; then $(PRINT_OK); \
-   else if [ "$$3" = 255 ] && [ -e $(4) ]; then $(PRINT_SKIP); err=0; \
+   else if [ "$$3" = 242 ] && [ -e $(4) ]; then $(PRINT_SKIP); err=0; \
    else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; fi; \
    rm -f $(1).stderr; exit $$err
 

examples/ex10.cpp

@@ -149,7 +149,7 @@ void InitialDeformation(const Vector &x, Vector &y);
 
 void InitialVelocity(const Vector &x, Vector &v);
 
-void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
                GridFunction *field, const char *field_name = NULL,
                bool init_vis = false);
 
@@ -376,10 +376,10 @@ int main(int argc, char *argv[])
 }
 
 
-void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
                GridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!out)
+   if (!os)
    {
       return;
    }
@@ -389,24 +389,25 @@ void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
 
    mesh->SwapNodes(nodes, owns_nodes);
 
-   out << "solution\n" << *mesh << *field;
+   os << "solution\n" << *mesh << *field;
 
    mesh->SwapNodes(nodes, owns_nodes);
 
    if (init_vis)
    {
-      out << "window_size 800 800\n";
-      out << "window_title '" << field_name << "'\n";
+      os << "window_size 800 800\n";
+      os << "window_title '" << field_name << "'\n";
       if (mesh->SpaceDimension() == 2)
       {
-         out << "view 0 0\n"; // view from top
-         out << "keys jl\n";  // turn off perspective and light
+         os << "view 0 0\n"; // view from top
+         os << "keys jl\n";  // turn off perspective and light
       }
-      out << "keys cm\n";         // show colorbar and mesh
-      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
-      out << "pause\n";
+      os << "keys cm\n";         // show colorbar and mesh
+      // update value-range; keep mesh-extents fixed
+      os << "autoscale value\n";
+      os << "pause\n";
    }
-   out << flush;
+   os << flush;
 }
 
 

examples/ex10p.cpp

@@ -154,7 +154,8 @@ void InitialDeformation(const Vector &x, Vector &y);
 
 void InitialVelocity(const Vector &x, Vector &v);
 
-void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
+void visualize(ostream &os, ParMesh *mesh,
+               ParGridFunction *deformed_nodes,
                ParGridFunction *field, const char *field_name = NULL,
                bool init_vis = false);
 
@@ -438,10 +439,11 @@ int main(int argc, char *argv[])
    return 0;
 }
 
-void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
+void visualize(ostream &os, ParMesh *mesh,
+               ParGridFunction *deformed_nodes,
                ParGridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!out)
+   if (!os)
    {
       return;
    }
@@ -451,25 +453,27 @@ void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
 
    mesh->SwapNodes(nodes, owns_nodes);
 
-   out << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() << "\n";
-   out << "solution\n" << *mesh << *field;
+   os << "parallel " << mesh->GetNRanks()
+      << " " << mesh->GetMyRank() << "\n";
+   os << "solution\n" << *mesh << *field;
 
    mesh->SwapNodes(nodes, owns_nodes);
 
    if (init_vis)
    {
-      out << "window_size 800 800\n";
-      out << "window_title '" << field_name << "'\n";
+      os << "window_size 800 800\n";
+      os << "window_title '" << field_name << "'\n";
       if (mesh->SpaceDimension() == 2)
       {
-         out << "view 0 0\n"; // view from top
-         out << "keys jl\n";  // turn off perspective and light
+         os << "view 0 0\n"; // view from top
+         os << "keys jl\n";  // turn off perspective and light
       }
-      out << "keys cm\n";         // show colorbar and mesh
-      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
-      out << "pause\n";
+      os << "keys cm\n";         // show colorbar and mesh
+      // update value-range; keep mesh-extents fixed
+      os << "autoscale value\n";
+      os << "pause\n";
    }
-   out << flush;
+   os << flush;
 }
 
 

examples/ex18.hpp

@@ -32,7 +32,7 @@ private:
    mutable DenseTensor flux;
    mutable Vector z;
 
-   void GetFlux(const DenseMatrix &state, DenseTensor &flux) const;
+   void GetFlux(const DenseMatrix &state_, DenseTensor &flux_) const;
 
 public:
    FE_Evolution(FiniteElementSpace &vfes_,
@@ -256,26 +256,26 @@ inline double ComputeMaxCharSpeed(const Vector &state, const int dim)
 }
 
 // Compute the flux at solution nodes.
-void FE_Evolution::GetFlux(const DenseMatrix &x, DenseTensor &flux) const
+void FE_Evolution::GetFlux(const DenseMatrix &x_, DenseTensor &flux_) const
 {
-   const int dof = flux.SizeI();
-   const int dim = flux.SizeJ();
+   const int flux_dof = flux_.SizeI();
+   const int flux_dim = flux_.SizeJ();
 
-   for (int i = 0; i < dof; i++)
+   for (int i = 0; i < flux_dof; i++)
    {
-      for (int k = 0; k < num_equation; k++) { state(k) = x(i, k); }
-      ComputeFlux(state, dim, f);
+      for (int k = 0; k < num_equation; k++) { state(k) = x_(i, k); }
+      ComputeFlux(state, flux_dim, f);
 
-      for (int d = 0; d < dim; d++)
+      for (int d = 0; d < flux_dim; d++)
       {
          for (int k = 0; k < num_equation; k++)
          {
-            flux(i, d, k) = f(k, d);
+            flux_(i, d, k) = f(k, d);
          }
       }
 
       // Update max char speed
-      const double mcs = ComputeMaxCharSpeed(state, dim);
+      const double mcs = ComputeMaxCharSpeed(state, flux_dim);
       if (mcs > max_char_speed) { max_char_speed = mcs; }
    }
 }

examples/ex19.cpp

@@ -171,7 +171,7 @@ public:
 };
 
 // Visualization driver
-void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
                GridFunction *field, const char *field_name = NULL,
                bool init_vis = false);
 
@@ -542,10 +542,10 @@ RubberOperator::~RubberOperator()
 
 
 // Inline visualization
-void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
                GridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!out)
+   if (!os)
    {
       return;
    }
@@ -555,23 +555,25 @@ void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
 
    mesh->SwapNodes(nodes, owns_nodes);
 
-   out << "solution\n" << *mesh << *field;
+   os << "solution\n" << *mesh << *field;
 
    mesh->SwapNodes(nodes, owns_nodes);
 
    if (init_vis)
    {
-      out << "window_size 800 800\n";
-      out << "window_title '" << field_name << "'\n";
+      os << "window_size 800 800\n";
+      os << "window_title '" << field_name << "'\n";
       if (mesh->SpaceDimension() == 2)
       {
-         out << "view 0 0\n"; // view from top
-         out << "keys jlA\n"; // turn off perspective and light, +anti-aliasing
+         os << "view 0 0\n"; // view from top
+         // turn off perspective and light, +anti-aliasing
+         os << "keys jlA\n";
       }
-      out << "keys cmA\n";        // show colorbar and mesh, +anti-aliasing
-      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
+      os << "keys cmA\n"; // show colorbar and mesh, +anti-aliasing
+      // update value-range; keep mesh-extents fixed
+      os << "autoscale value\n";
    }
-   out << flush;
+   os << flush;
 }
 
 void ReferenceConfiguration(const Vector &x, Vector &y)

examples/ex19p.cpp

@@ -185,7 +185,8 @@ public:
 };
 
 // Visualization driver
-void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
+void visualize(ostream &os, ParMesh *mesh,
+               ParGridFunction *deformed_nodes,
                ParGridFunction *field, const char *field_name = NULL,
                bool init_vis = false);
 
@@ -196,10 +197,10 @@ void InitialDeformation(const Vector &x, Vector &y);
 
 int main(int argc, char *argv[])
 {
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
-        << "is NOT supported with the CUDA version of hypre.\n\n";
-   return 255;
+        << "is NOT supported with the GPU version of hypre.\n\n";
+   return 242;
 #endif
 
    // 1. Initialize MPI
@@ -486,8 +487,8 @@ void JacobianPreconditioner::SetOperator(const Operator &op)
 
       if (!spaces[0]->GetParMesh()->Nonconforming())
       {
-#ifndef HYPRE_USING_CUDA
-         // Not available yet when hypre is built with CUDA
+#if !defined(HYPRE_USING_GPU)
+         // Not available yet when hypre is built with GPU support
          stiff_prec_amg->SetElasticityOptions(spaces[0]);
 #endif
       }
@@ -617,10 +618,11 @@ RubberOperator::~RubberOperator()
 
 
 // Inline visualization
-void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
+void visualize(ostream &os, ParMesh *mesh,
+               ParGridFunction *deformed_nodes,
                ParGridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!out)
+   if (!os)
    {
       return;
    }
@@ -630,24 +632,27 @@ void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
 
    mesh->SwapNodes(nodes, owns_nodes);
 
-   out << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() << "\n";
-   out << "solution\n" << *mesh << *field;
+   os << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() <<
+      "\n";
+   os << "solution\n" << *mesh << *field;
 
    mesh->SwapNodes(nodes, owns_nodes);
 
    if (init_vis)
    {
-      out << "window_size 800 800\n";
-      out << "window_title '" << field_name << "'\n";
+      os << "window_size 800 800\n";
+      os << "window_title '" << field_name << "'\n";
       if (mesh->SpaceDimension() == 2)
       {
-         out << "view 0 0\n"; // view from top
-         out << "keys jlA\n"; // turn off perspective and light, +anti-aliasing
+         os << "view 0 0\n"; // view from top
+         // turn off perspective and light, +anti-aliasing
+         os << "keys jlA\n";
       }
-      out << "keys cmA\n";        // show colorbar and mesh, +anti-aliasing
-      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
+      os << "keys cmA\n"; // show colorbar and mesh, +anti-aliasing
+      // update value-range; keep mesh-extents fixed
+      os << "autoscale value\n";
    }
-   out << flush;
+   os << flush;
 }
 
 void ReferenceConfiguration(const Vector &x, Vector &y)

examples/ex1_es.cpp

@@ -1,266 +0,0 @@
-//                                MFEM Example 1
-//
-// Compile with: make ex1
-//
-// Sample runs:  ex1 -m ../data/square-disc.mesh
-//               ex1 -m ../data/star.mesh
-//               ex1 -m ../data/escher.mesh
-//               ex1 -m ../data/fichera.mesh
-//               ex1 -m ../data/square-disc-p2.vtk -o 2
-//               ex1 -m ../data/square-disc-p3.mesh -o 3
-//               ex1 -m ../data/square-disc-nurbs.mesh -o -1
-//               ex1 -m ../data/disc-nurbs.mesh -o -1
-//               ex1 -m ../data/pipe-nurbs.mesh -o -1
-//               ex1 -m ../data/star-surf.mesh
-//               ex1 -m ../data/square-disc-surf.mesh
-//               ex1 -m ../data/inline-segment.mesh
-//               ex1 -m ../data/amr-quad.mesh
-//               ex1 -m ../data/amr-hex.mesh
-//               ex1 -m ../data/fichera-amr.mesh
-//               ex1 -m ../data/mobius-strip.mesh
-//               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "./star-set.mesh";
-   int order = 1;
-   int rs = -1;
-   int ra = 0;
-   int bt = EntitySets::INVALID;
-   const char *bs = "Origin";
-   bool static_cond = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&rs, "-rs", "--refine-serial",
-                  "Number of serial refinement levels");
-   args.AddOption(&ra, "-ra", "--refine-adaptive",
-                  "Number of adaptive refinement levels");
-   args.AddOption(&bt, "-bt", "--bc-entity-type",
-                  "");
-   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
-                  "");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 3. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 50,000
-   //    elements.
-   {
-      int ref_levels = ( rs >= 0 ) ? rs :
-                       (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-
-   if ( ra > 0 )
-   {
-      cout << "calling EnsureNCMesh" << endl;
-      mesh->EnsureNCMesh();
-      cout << "back from EnsureNCMesh" << endl;
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   cout << "Calling RandomRefinement " << ra << " times." << endl;
-   for (int l = 0; l < ra; l++)
-   {
-      mesh->RandomRefinement(0.2);
-   }
-   cout << "Done with refinement" << endl;
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   if ( mesh->ncmesh )
-   {
-      mesh->ncmesh->PrintStats(cout);
-
-      ofstream ofsV("vp.out");
-      ofstream ofsE("ce.out");
-      mesh->ncmesh->PrintVertexParents(ofsV);
-      mesh->ncmesh->PrintCoarseElements(ofsE);
-   }
-
-   // 4. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order. If order < 1, we
-   //    instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (mesh->GetNodes())
-   {
-      fec = mesh->GetNodes()->OwnFEC();
-      cout << "Using isoparametric FEs: " << fec->Name() << endl;
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-   cout << "Number of finite element unknowns: "
-        << fespace->GetTrueVSize() << endl;
-
-   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if ( bt == EntitySets::INVALID )
-   {
-      if (mesh->bdr_attributes.Size())
-      {
-         Array<int> ess_bdr(mesh->bdr_attributes.Max());
-         ess_bdr = 1;
-         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   else
-   {
-      fespace->GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                    ess_tdof_list);
-   }
-
-   cout << "Number of Dirichlet dofs: " << ess_tdof_list.Size() << endl;
-
-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   LinearForm *b = new LinearForm(fespace);
-   ConstantCoefficient one(1.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->Assemble();
-
-   // 7. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(fespace);
-   x = 0.0;
-
-   // 8. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   BilinearForm *a = new BilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 9. Assemble the bilinear form and the corresponding linear system,
-   //    applying any necessary transformations such as: eliminating boundary
-   //    conditions, applying conforming constraints for non-conforming AMR,
-   //    static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   SparseMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   cout << "Size of linear system: " << A.Height() << endl;
-
-#ifndef MFEM_USE_SUITESPARSE
-   // 10. Define a simple symmetric Gauss-Seidel preconditioner and use it to
-   //     solve the system A X = B with PCG.
-   GSSmoother M(A);
-   PCG(A, M, B, X, 1, 200, 1e-12, 0.0);
-#else
-   // 10. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-   UMFPackSolver umf_solver;
-   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-   umf_solver.SetOperator(A);
-   umf_solver.Mult(B, X);
-#endif
-
-   // 11. Recover the solution as a finite element grid function.
-   a->RecoverFEMSolution(X, *b, x);
-
-   // 12. Save the refined mesh and the solution. This output can be viewed later
-   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh->Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-   // 13. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x << flush;
-   }
-
-   // 14. Free the used memory.
-   delete a;
-   delete b;
-   delete fespace;
-   if (order > 0) { delete fec; }
-   delete mesh;
-
-   return 0;
-}

examples/ex1p_es.cpp

@@ -1,388 +0,0 @@
-//                       MFEM Example 1 - Parallel Version
-//
-// Compile with: make ex1p
-//
-// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
-//               mpirun -np 4 ex1p -m ../data/star.mesh
-//               mpirun -np 4 ex1p -m ../data/escher.mesh
-//               mpirun -np 4 ex1p -m ../data/fichera.mesh
-//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
-//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
-//
-//               The following are examples of using EntitySets to define
-//               homogeneous Dirichlet boundary condition.  These examples
-//               require a modified mesh file and a specialized version of
-//               example 1 called "ex1p_es".
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh -bt 0 -bs Origin
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh -bt 1 -bs Axes
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
-//                                    -bt 1 -bs "Negative Axes"
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
-//                                    -bt 2 -bs "Interior Corner"
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
-//                                    -bt 2 -bs "Exterior Corner"
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
-//                                    -bt 3 -bs "Interior Corner"
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
-//                                    -bt 3 -bs "Exterior Corner"
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh -bt 3 -bs "Steps"
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "./star-set.mesh";
-   int order = 1;
-   int rs = -1;
-   int rp = 2;
-   int ra = 0;
-   int bt = EntitySets::INVALID;
-   const char *bs = "Origin";
-   bool static_cond = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&rs, "-rs", "--refine-serial",
-                  "Number of serial refinement levels");
-   args.AddOption(&rp, "-rp", "--refine-parallel",
-                  "Number of parallel refinement levels");
-   args.AddOption(&ra, "-ra", "--refine-adaptive",
-                  "Number of adaptive refinement levels");
-   args.AddOption(&bt, "-bt", "--bc-entity-type",
-                  "");
-   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
-                  "");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 10,000 elements.
-   {
-      int ref_levels = ( rs >= 0 ) ? rs :
-                       (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in serial..."; }
-         mesh->UniformRefinement();
-      }
-      MPI_Barrier(MPI_COMM_WORLD);
-      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     At this point we have a serial mesh containing an EntitySets
-     object which stores the current node/edge/face/element indices
-     for each entity in each set.  This data is duplicated on each MPI
-     rank.
-    */
-   if ( ra > 0 )
-   {
-      cout << "calling EnsureNCMesh" << endl;
-      mesh->EnsureNCMesh();
-      cout << "back from EnsureNCMesh" << endl;
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     We now have an NCEntitySets object which stores the node indices
-     describing each enity in each node/edge/face set and the element
-     indices for the elements in each element set.  This data is
-     duplicated on each MPI rank.
-    */
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   cout << "creating ParMesh from serial mesh" << endl;
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   cout << "done creating ParMesh from serial mesh" << endl;
-   delete mesh;
-   if ( pmesh->pent_sets )
-   {
-      cout << "pmesh->pent_sets is non NULL" << endl;
-      pmesh->pent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "pmesh->pent_sets is NULL" << endl;
-   }
-   /*
-     We now have a ParEntitySets object which marshals the data stored
-     in EntitySets objects.  The data has now been pruned so that each
-     rank only contains indices of local entities.
-
-     The NCEntitySets object remains unchanged...
-
-     If we have an NC mesh a different path is taken and the
-     EntitySets are ignored.
-
-     1) ParNCMesh is created from NCMesh
-       a) Creates a ParNCEntitySets object from ncmesh (every rank contains
-          information to find every entity)
-     2) ParNCMesh is pruned which involves renumbering elements and vertices
-     3) ParMesh is initialized from ParNCMesh
-     4) ParNCMesh::OnMeshUpdated is called
-     5) Mesh::GenerateNCFaceInfo is called
-    */
-   {
-      int par_ref_levels = rp;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in parallel..."; }
-         pmesh->UniformRefinement();
-      }
-      MPI_Barrier(MPI_COMM_WORLD);
-      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
-   }
-   /*
-     RandomRefinement will end up calling
-     ParMesh::NonconformingRefinement which will create a new ParMesh
-     object using the ParNCMesh object and then call
-     ParMesh::OnMeshUpdated on this new mesh.
-    */
-
-   for (int l = 0; l < ra; l++)
-   {
-      pmesh->RandomRefinement(0.2);
-   }
-   if ( ra > 0 )
-   {
-      if ( pmesh->pent_sets )
-      {
-         cout << "pmesh->pent_sets is non NULL post random refinement" << endl;
-         pmesh->pent_sets->PrintSetInfo(cout);
-      }
-      else
-      {
-         cout << "pmesh->pent_sets is NULL post random refinement" << endl;
-      }
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (pmesh->GetNodes())
-   {
-      fec = pmesh->GetNodes()->OwnFEC();
-      if (myid == 0)
-      {
-         cout << "Using isoparametric FEs: " << fec->Name() << endl;
-      }
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if ( bt == EntitySets::INVALID )
-   {
-      if (pmesh->bdr_attributes.Size())
-      {
-         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-         ess_bdr = 1;
-         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   else
-   {
-      fespace->GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                    ess_tdof_list);
-   }
-   for (int i=0; i<num_procs; i++)
-   {
-      if (myid == i)
-      {
-         cout << "Number of Dirichlet dofs on proc " << i << ": "
-              << ess_tdof_list.Size() << endl;
-      }
-   }
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm *b = new ParLinearForm(fespace);
-   ConstantCoefficient one(1.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   ParGridFunction x(fespace);
-   x = 0.0;
-
-   // 10. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   HypreParMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   if (myid == 0)
-   {
-      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
-   }
-
-   // 12. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
-   //     preconditioner from hypre.
-   HypreSolver *amg = new HypreBoomerAMG(A);
-   HyprePCG *pcg = new HyprePCG(A);
-   pcg->SetTol(1e-12);
-   pcg->SetMaxIter(200);
-   pcg->SetPrintLevel(2);
-   pcg->SetPreconditioner(*amg);
-   pcg->Mult(B, X);
-
-   // 13. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-
-   // 14. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 15. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 16. Free the used memory.
-   delete pcg;
-   delete amg;
-   delete a;
-   delete b;
-   delete fespace;
-   if (order > 0) { delete fec; }
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}

examples/ex25.cpp

@@ -154,6 +154,10 @@ prob_type prob;
 
 int main(int argc, char *argv[])
 {
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_SELF, &num_procs);
+   MPI_Comm_rank(MPI_COMM_SELF, &myid);
    // 1. Parse command-line options.
    const char *mesh_file = nullptr;
    int order = 1;
@@ -423,10 +427,34 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_SUITESPARSE
    if (!pa && umf_solver)
    {
+      chrono.Clear();
+      chrono.Start();
       ComplexUMFPackSolver csolver(*A.As<ComplexSparseMatrix>());
       csolver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
       csolver.SetPrintLevel(1);
       csolver.Mult(B, X);
+      chrono.Stop();
+      cout << "UMFPACK for ComplexSparseMatrix = " << chrono.RealTime() << endl;
+   }
+   {
+      chrono.Clear();
+      chrono.Start();
+      HYPRE_Int rowstarts[2]; rowstarts[0] = 0;
+      rowstarts[1] = fespace->GetTrueVSize();
+      HypreParMatrix * HypreMat_r =
+         new HypreParMatrix(MPI_COMM_SELF,rowstarts[1],rowstarts,
+                            &(*A.As<ComplexSparseMatrix>()).real());
+      HypreParMatrix * HypreMat_i =
+         new HypreParMatrix(MPI_COMM_SELF,rowstarts[1],rowstarts,
+                            &(*A.As<ComplexSparseMatrix>()).imag());
+      ComplexHypreParMatrix * HypreMat =
+         new ComplexHypreParMatrix(HypreMat_r,HypreMat_i,true,true);
+      ComplexMUMPSSolver csolver;
+      csolver.SetOperator(*HypreMat);
+      csolver.Mult(B, X);
+      delete HypreMat;
+      chrono.Stop();
+      cout << "MUMPS  for ComplexSparseMatrix = " << chrono.RealTime() << endl;
    }
 #endif
    // 14a. Set up the Bilinear form a(.,.) for the preconditioner
@@ -614,6 +642,7 @@ int main(int argc, char *argv[])
    delete fespace;
    delete fec;
    delete mesh;
+   MPI_Finalize();
    return 0;
 }
 

examples/ex25p.cpp

@@ -266,6 +266,10 @@ int main(int argc, char *argv[])
    }
 
    Mesh * mesh = new Mesh(mesh_file, 1, 1);
+
+   // Mesh * mesh = new Mesh(1,1,16,Element::HEXAHEDRON,true,1.0,1.0,16.0);
+
+
    dim = mesh->Dimension();
 
    // Angular frequency
@@ -278,7 +282,7 @@ int main(int argc, char *argv[])
    switch (prob)
    {
       case disc:
-         length = 0.2;
+         length = 0.3;
          break;
       case lshape:
          length(0, 0) = 0.1;
@@ -291,6 +295,7 @@ int main(int argc, char *argv[])
          break;
       case beam:
          length(0, 1) = 2.0;
+         // length(2, 1) = 2.0;
          break;
       default:
          length = 0.25;
@@ -463,8 +468,11 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_SUPERLU
    if (!pa && slu_solver)
    {
+      StopWatch chrono;
       // Transform to monolithic HypreParMatrix
       HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+      chrono.Clear();
+      chrono.Start();
       SuperLURowLocMatrix SA(*A);
       SuperLUSolver superlu(MPI_COMM_WORLD);
       superlu.SetPrintStatistics(false);
@@ -473,19 +481,50 @@ int main(int argc, char *argv[])
       superlu.SetOperator(SA);
       superlu.Mult(B, X);
       delete A;
+      chrono.Stop();
+      if (myid == 0)
+      {
+         cout << "Superlu for monolithic HyperMat = " << chrono.RealTime() << endl;
+      }
    }
 #endif
 #ifdef MFEM_USE_MUMPS
    if (!pa && mumps_solver)
    {
+      StopWatch chrono;
       HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+      chrono.Clear();
+      chrono.Start();
       MUMPSSolver mumps;
       mumps.SetPrintLevel(0);
       mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
       mumps.SetOperator(*A);
       mumps.Mult(B,X);
+      chrono.Stop();
+      if (myid == 0)
+      {
+         cout << "MUMPS for monolithic HyperMat = " << chrono.RealTime() << endl;
+      }
       delete A;
    }
+   if (mumps_solver)
+   {
+      StopWatch chrono;
+      chrono.Clear();
+      chrono.Start();
+      ComplexMUMPSSolver cmumps;
+      cmumps.SetPrintLevel(0);
+      cmumps.SetOperator(*Ah);
+      cmumps.Mult(B,X);
+      chrono.Stop();
+      if (myid == 0)
+      {
+         cout << "MUMPS for ComplexHyperMat     = " << chrono.RealTime() << endl;
+      }
+   }
+
+
+
 #endif
    // 16a. Set up the parallel Bilinear form a(.,.) for the preconditioner
    //
@@ -801,6 +840,7 @@ void maxwell_solution(const Vector &x, vector<complex<double>> &E)
          {
             double k10 = sqrt(k * k - M_PI * M_PI);
             E[1] = -zi * k / M_PI * sin(M_PI*x(2))*exp(zi * k10 * x(0));
+            // E[1] = -zi * k / M_PI * sin(M_PI*x(0))*exp(zi * k10 * x(2));
          }
          else if (dim == 2)
          {

examples/ex27.cpp

@@ -75,7 +75,7 @@ Mesh * GenerateSerialMesh(int ref);
 // alpha*n.Grad(sol) + beta*sol - gamma over the same boundary.
 double IntegrateBC(const GridFunction &sol, const Array<int> &bdr_marker,
                    double alpha, double beta, double gamma,
-                   double &err);
+                   double &error);
 
 int main(int argc, char *argv[])
 {
@@ -302,26 +302,26 @@ int main(int argc, char *argv[])
    {
       // Integrate the solution on the Dirichlet boundary and compare to the
       // expected value.
-      double err, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, err);
+      double error, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, error);
 
       bool hom_dbc = (dbc_val == 0.0);
-      err /=  hom_dbc ? 1.0 : fabs(dbc_val);
+      error /=  hom_dbc ? 1.0 : fabs(dbc_val);
       mfem::out << "Average of solution on Gamma_dbc:\t"
                 << avg << ", \t"
                 << (hom_dbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and compare
       // to the expected value.
-      double err, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, err);
+      double error, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, error);
 
       bool hom_nbc = (nbc_val == 0.0);
-      err /=  hom_nbc ? 1.0 : fabs(nbc_val);
+      error /=  hom_nbc ? 1.0 : fabs(nbc_val);
       mfem::out << "Average of n.Grad(u) on Gamma_nbc:\t"
                 << avg << ", \t"
                 << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) on the homogeneous Neumann boundary and compare to
@@ -330,25 +330,26 @@ int main(int argc, char *argv[])
       nbc0_bdr = 0;
       nbc0_bdr[3] = 1;
 
-      double err, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, err);
+      double error, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, error);
 
       bool hom_nbc = true;
       mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
                 << avg << ", \t"
                 << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) + a * u on the Robin boundary and compare to the
       // expected value.
-      double err, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, err);
+      double error;
+      double avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, error);
 
       bool hom_rbc = (rbc_b_val == 0.0);
-      err /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
+      error /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
       mfem::out << "Average of n.Grad(u)+a*u on Gamma_rbc:\t"
                 << avg << ", \t"
                 << (hom_rbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
 
    // 14. Save the refined mesh and the solution. This output can be viewed
@@ -637,11 +638,11 @@ Mesh * GenerateSerialMesh(int ref)
 
 double IntegrateBC(const GridFunction &x, const Array<int> &bdr,
                    double alpha, double beta, double gamma,
-                   double &err)
+                   double &error)
 {
    double nrm = 0.0;
    double avg = 0.0;
-   err = 0.0;
+   error = 0.0;
 
    const bool a_is_zero = alpha == 0.0;
    const bool b_is_zero = beta == 0.0;
@@ -705,20 +706,20 @@ double IntegrateBC(const GridFunction &x, const Array<int> &bdr,
 
          // Integrate |alpha * n.Grad(x) + beta * x - gamma|^2
          val -= gamma;
-         err += (val*val) * ip.weight * face_weight;
+         error += (val*val) * ip.weight * face_weight;
       }
    }
 
    // Normalize by the length of the boundary
    if (std::abs(nrm) > 0.0)
    {
-      err /= nrm;
+      error /= nrm;
       avg /= nrm;
    }
 
    // Compute l2 norm of the error in the boundary condition (negative
-   // quadrature weights may produce negative 'err')
-   err = (err >= 0.0) ? sqrt(err) : -sqrt(-err);
+   // quadrature weights may produce negative 'error')
+   error = (error >= 0.0) ? sqrt(error) : -sqrt(-error);
 
    // Return the average value of alpha * n.Grad(x) + beta * x
    return avg;

examples/ex27p.cpp

@@ -75,7 +75,7 @@ Mesh * GenerateSerialMesh(int ref);
 // alpha*n.Grad(sol) + beta*sol - gamma over the same boundary.
 double IntegrateBC(const ParGridFunction &sol, const Array<int> &bdr_marker,
                    double alpha, double beta, double gamma,
-                   double &err);
+                   double &error);
 
 int main(int argc, char *argv[])
 {
@@ -321,26 +321,26 @@ int main(int argc, char *argv[])
    {
       // Integrate the solution on the Dirichlet boundary and compare to the
       // expected value.
-      double err, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, err);
+      double error, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, error);
 
       bool hom_dbc = (dbc_val == 0.0);
-      err /=  hom_dbc ? 1.0 : fabs(dbc_val);
+      error /=  hom_dbc ? 1.0 : fabs(dbc_val);
       mfem::out << "Average of solution on Gamma_dbc:\t"
                 << avg << ", \t"
                 << (hom_dbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and compare
       // to the expected value.
-      double err, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, err);
+      double error, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, error);
 
       bool hom_nbc = (nbc_val == 0.0);
-      err /=  hom_nbc ? 1.0 : fabs(nbc_val);
+      error /=  hom_nbc ? 1.0 : fabs(nbc_val);
       mfem::out << "Average of n.Grad(u) on Gamma_nbc:\t"
                 << avg << ", \t"
                 << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) on the homogeneous Neumann boundary and compare to
@@ -349,25 +349,26 @@ int main(int argc, char *argv[])
       nbc0_bdr = 0;
       nbc0_bdr[3] = 1;
 
-      double err, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, err);
+      double error, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, error);
 
       bool hom_nbc = true;
       mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
                 << avg << ", \t"
                 << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) + a * u on the Robin boundary and compare to the
       // expected value.
-      double err, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, err);
+      double error, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val,
+                                      error);
 
       bool hom_rbc = (rbc_b_val == 0.0);
-      err /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
+      error /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
       mfem::out << "Average of n.Grad(u)+a*u on Gamma_rbc:\t"
                 << avg << ", \t"
                 << (hom_rbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
 
    // 15. Save the refined mesh and the solution in parallel. This output can be
@@ -667,11 +668,11 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
    double loc_vals[3];
    double &nrm = loc_vals[0];
    double &avg = loc_vals[1];
-   double &err = loc_vals[2];
+   double &error = loc_vals[2];
 
    nrm = 0.0;
    avg = 0.0;
-   err = 0.0;
+   error = 0.0;
 
    const bool a_is_zero = alpha == 0.0;
    const bool b_is_zero = beta == 0.0;
@@ -735,7 +736,7 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
 
          // Integrate |alpha * n.Grad(x) + beta * x - gamma|^2
          val -= gamma;
-         err += (val*val) * ip.weight * face_weight;
+         error += (val*val) * ip.weight * face_weight;
       }
    }
 
@@ -754,7 +755,7 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
    }
 
    // Compute l2 norm of the error in the boundary condition (negative
-   // quadrature weights may produce negative 'err')
+   // quadrature weights may produce negative 'error')
    glb_err = (glb_err >= 0.0) ? sqrt(glb_err) : -sqrt(-glb_err);
 
    // Return the average value of alpha * n.Grad(x) + beta * x

examples/ex28p.cpp

@@ -81,10 +81,10 @@ Mesh * build_trapezoid_mesh(double offset)
 
 int main(int argc, char *argv[])
 {
-#ifdef HYPRE_USING_CUDA
+#ifdef HYPRE_USING_GPU
    cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
-        << "is NOT supported with the CUDA version of hypre.\n\n";
-   return 255;
+        << "is NOT supported with the GPU version of hypre.\n\n";
+   return 242;
 #endif
 
    // 1. Initialize MPI.

examples/ex29.cpp

@@ -167,9 +167,9 @@ int main(int argc, char *argv[])
 
    // 13. Compute error in the solution and its flux
    FunctionCoefficient uCoef(uExact);
-   double err = x.ComputeL2Error(uCoef);
+   double error = x.ComputeL2Error(uCoef);
 
-   cout << "|u - u_h|_2 = " << err << endl;
+   cout << "|u - u_h|_2 = " << error << endl;
 
    FiniteElementSpace flux_fespace(mesh, &fec, 3);
    GridFunction flux(&flux_fespace);

examples/ex29p.cpp

@@ -197,9 +197,9 @@ int main(int argc, char *argv[])
 
    // 15. Compute error in the solution and its flux
    FunctionCoefficient uCoef(uExact);
-   double err = x.ComputeL2Error(uCoef);
+   double error = x.ComputeL2Error(uCoef);
 
-   if (myid == 0) { cout << "|u - u_h|_2 = " << err << endl; }
+   if (myid == 0) { cout << "|u - u_h|_2 = " << error << endl; }
 
    ParFiniteElementSpace flux_fespace(&pmesh, &fec, 3);
    ParGridFunction flux(&flux_fespace);

examples/ex3p.cpp

@@ -255,10 +255,10 @@ int main(int argc, char *argv[])
 
    // 15. Compute and print the L^2 norm of the error.
    {
-      double err = x.ComputeL2Error(E);
+      double error = x.ComputeL2Error(E);
       if (myid == 0)
       {
-         cout << "\n|| E_h - E ||_{L^2} = " << err << '\n' << endl;
+         cout << "\n|| E_h - E ||_{L^2} = " << error << '\n' << endl;
       }
    }
 

examples/ex3p_es.cpp

@@ -1,411 +0,0 @@
-//                       MFEM Example 3 - Parallel Version
-//
-// Compile with: make ex3p
-//
-// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
-//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
-//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/escher.mesh
-//               mpirun -np 4 ex3p -m ../data/fichera.mesh
-//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
-//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
-//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
-//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
-//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
-//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
-//
-// Description:  This example code solves a simple electromagnetic diffusion
-//               problem corresponding to the second order definite Maxwell
-//               equation curl curl E + E = f with boundary condition
-//               E x n = <given tangential field>. Here, we use a given exact
-//               solution E and compute the corresponding r.h.s. f.
-//               We discretize with Nedelec finite elements in 2D or 3D.
-//
-//               The example demonstrates the use of H(curl) finite element
-//               spaces with the curl-curl and the (vector finite element) mass
-//               bilinear form, as well as the computation of discretization
-//               error when the exact solution is known. Static condensation is
-//               also illustrated.
-//
-//               We recommend viewing examples 1-2 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Exact solution, E, and r.h.s., f. See below for implementation.
-//void E_exact(const Vector &, Vector &);
-//void f_exact(const Vector &, Vector &);
-//double freq = 1.0, kappa;
-void f_const(const Vector &, Vector &);
-
-int dim;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../data/beam-tet.mesh";
-   int order = 1;
-   int rs = -1;
-   int rp = 2;
-   int ra = 0;
-   int bt = EntitySets::INVALID;
-   const char *bs = "Origin";
-   bool static_cond = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   /*
-   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
-                  " solution.");
-   */
-   args.AddOption(&rs, "-rs", "--refine-serial",
-                  "Number of serial refinement levels");
-   args.AddOption(&rp, "-rp", "--refine-parallel",
-                  "Number of parallel refinement levels");
-   args.AddOption(&ra, "-ra", "--refine-adaptive",
-                  "Number of adaptive refinement levels");
-   args.AddOption(&bt, "-bt", "--bc-entity-type",
-                  "");
-   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
-                  "");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-   // kappa = freq * M_PI;
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels = ( rs >= 0 ) ? rs :
-                       (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in serial..."; }
-         mesh->UniformRefinement();
-      }
-      MPI_Barrier(MPI_COMM_WORLD);
-      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     At this point we have a serial mesh containing an EntitySets
-     object which stores the current node/edge/face/element indices
-     for each entity in each set.  This data is duplicated on each MPI
-     rank.
-    */
-   if ( ra > 0 )
-   {
-      cout << "calling EnsureNCMesh" << endl;
-      mesh->EnsureNCMesh();
-      cout << "back from EnsureNCMesh" << endl;
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     We now have an NCEntitySets object which stores the node indices
-     describing each enity in each node/edge/face set and the element
-     indices for the elements in each element set.  This data is
-     duplicated on each MPI rank.
-    */
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   if ( pmesh->pent_sets )
-   {
-      cout << "pmesh->pent_sets is non NULL" << endl;
-      pmesh->pent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "pmesh->pent_sets is NULL" << endl;
-   }
-   {
-      int par_ref_levels = rp;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in parallel..."; }
-         pmesh->UniformRefinement();
-      }
-      MPI_Barrier(MPI_COMM_WORLD);
-      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
-   }
-   pmesh->ReorientTetMesh();
-   pmesh->ent_sets->PrintSetInfo(cout);
-
-   for (int l = 0; l < ra; l++)
-   {
-      pmesh->RandomRefinement(0.2);
-   }
-   if ( ra > 0 )
-   {
-      if ( pmesh->pent_sets )
-      {
-         cout << "pmesh->pent_sets is non NULL post random refinement" << endl;
-         pmesh->pent_sets->PrintSetInfo(cout);
-      }
-      else
-      {
-         cout << "pmesh->pent_sets is NULL post random refinement" << endl;
-      }
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use the Nedelec finite elements of the specified order.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if ( bt == EntitySets::INVALID )
-   {
-      if (pmesh->bdr_attributes.Size())
-      {
-         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-         ess_bdr = 1;
-         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   else
-   {
-      fespace->GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                    ess_tdof_list);
-   }
-   if (myid == 0)
-   {
-      cout << "Number of Dirichlet dofs: " << ess_tdof_list.Size() << endl;
-   }
-
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
-   //    basis functions in the finite element fespace.
-   VectorFunctionCoefficient f(sdim, f_const);
-   ParLinearForm *b = new ParLinearForm(fespace);
-   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x by projecting the exact
-   //    solution. Note that only values from the boundary edges will be used
-   //    when eliminating the non-homogeneous boundary condition to modify the
-   //    r.h.s. vector b.
-   ParGridFunction x(fespace);
-   // VectorFunctionCoefficient E(sdim, E_exact);
-   // x.ProjectCoefficient(E);
-   x = 0.0;
-
-   // 10. Set up the parallel bilinear form corresponding to the EM diffusion
-   //     operator curl muinv curl + sigma I, by adding the curl-curl and the
-   //     mass domain integrators.
-   Coefficient *muinv = new ConstantCoefficient(1.0);
-   Coefficient *sigma = new ConstantCoefficient(1.0);
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
-   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   HypreParMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   if (myid == 0)
-   {
-      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
-   }
-
-   // 12. Define and apply a parallel PCG solver for AX=B with the AMS
-   //     preconditioner from hypre.
-   ParFiniteElementSpace *prec_fespace =
-      (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
-   HypreSolver *ams = new HypreAMS(A, prec_fespace);
-   HyprePCG *pcg = new HyprePCG(A);
-   pcg->SetTol(1e-12);
-   pcg->SetMaxIter(500);
-   pcg->SetPrintLevel(2);
-   pcg->SetPreconditioner(*ams);
-   pcg->Mult(B, X);
-
-   // 13. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-
-   /*
-   // 14. Compute and print the L^2 norm of the error.
-   {
-      double err = x.ComputeL2Error(E);
-      if (myid == 0)
-      {
-         cout << "\n|| E_h - E ||_{L^2} = " << err << '\n' << endl;
-      }
-   }
-   */
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   delete pcg;
-   delete ams;
-   delete a;
-   delete sigma;
-   delete muinv;
-   delete b;
-   delete fespace;
-   delete fec;
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}
-
-/*
-void E_exact(const Vector &x, Vector &E)
-{
-   if (dim == 3)
-   {
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(2));
-      E(2) = sin(kappa * x(0));
-   }
-   else
-   {
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(0));
-      if (x.Size() == 3) { E(2) = 0.0; }
-   }
-}
-
-void f_exact(const Vector &x, Vector &f)
-{
-   if (dim == 3)
-   {
-      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (1. + kappa * kappa) * sin(kappa * x(2));
-      f(2) = (1. + kappa * kappa) * sin(kappa * x(0));
-   }
-   else
-   {
-      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
-      if (x.Size() == 3) { f(2) = 0.0; }
-   }
-}
-*/
-void f_const(const Vector &x, Vector &f)
-{
-   if (dim == 3)
-   {
-      f(0) = 1.0;
-      f(1) = 1.0;
-      f(2) = 1.0;
-   }
-   else
-   {
-      f(0) = 1.0;
-      f(1) = 1.0;
-      if (x.Size() == 3) { f(2) = 0.0; }
-   }
-}

examples/ex4p.cpp

@@ -256,10 +256,10 @@ int main(int argc, char *argv[])
 
    // 15. Compute and print the L^2 norm of the error.
    {
-      double err = x.ComputeL2Error(F);
+      double error = x.ComputeL2Error(F);
       if (myid == 0)
       {
-         cout << "\n|| F_h - F ||_{L^2} = " << err << '\n' << endl;
+         cout << "\n|| F_h - F ||_{L^2} = " << error << '\n' << endl;
       }
    }
 

examples/ex4p_es.cpp

@@ -1,438 +0,0 @@
-//                       MFEM Example 4 - Parallel Version
-//
-// Compile with: make ex4p
-//
-// Sample runs:  mpirun -np 4 ex4p -m ../data/square-disc.mesh
-//               mpirun -np 4 ex4p -m ../data/star.mesh
-//               mpirun -np 4 ex4p -m ../data/beam-tet.mesh
-//               mpirun -np 4 ex4p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex4p -m ../data/escher.mesh -o 2 -sc
-//               mpirun -np 4 ex4p -m ../data/fichera.mesh -o 2 -hb
-//               mpirun -np 4 ex4p -m ../data/fichera-q2.vtk
-//               mpirun -np 4 ex4p -m ../data/fichera-q3.mesh -o 2 -sc
-//               mpirun -np 4 ex4p -m ../data/square-disc-nurbs.mesh -o 3
-//               mpirun -np 4 ex4p -m ../data/beam-hex-nurbs.mesh -o 3
-//               mpirun -np 4 ex4p -m ../data/periodic-square.mesh -no-bc
-//               mpirun -np 4 ex4p -m ../data/periodic-cube.mesh -no-bc
-//               mpirun -np 4 ex4p -m ../data/amr-quad.mesh
-//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
-//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
-//               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
-//
-// Description:  This example code solves a simple 2D/3D H(div) diffusion
-//               problem corresponding to the second order definite equation
-//               -grad(alpha div F) + beta F = f with boundary condition F dot n
-//               = <given normal field>. Here, we use a given exact solution F
-//               and compute the corresponding r.h.s. f.  We discretize with
-//               Raviart-Thomas finite elements.
-//
-//               The example demonstrates the use of H(div) finite element
-//               spaces with the grad-div and H(div) vector finite element mass
-//               bilinear form, as well as the computation of discretization
-//               error when the exact solution is known. Bilinear form
-//               hybridization and static condensation are also illustrated.
-//
-//               We recommend viewing examples 1-3 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Exact solution, F, and r.h.s., f. See below for implementation.
-//void F_exact(const Vector &, Vector &);
-//void f_exact(const Vector &, Vector &);
-//double freq = 1.0, kappa;
-void f_const(const Vector &, Vector &);
-
-int dim;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-   int rs = -1;
-   int rp = 2;
-   int ra = 0;
-   int bt = EntitySets::INVALID;
-   const char *bs = "Origin";
-   bool set_bc = true;
-   bool static_cond = false;
-   bool hybridization = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
-                  "Impose or not essential boundary conditions.");
-   args.AddOption(&rs, "-rs", "--refine-serial",
-                  "Number of serial refinement levels");
-   args.AddOption(&rp, "-rp", "--refine-parallel",
-                  "Number of parallel refinement levels");
-   args.AddOption(&ra, "-ra", "--refine-adaptive",
-                  "Number of adaptive refinement levels");
-   args.AddOption(&bt, "-bt", "--bc-entity-type",
-                  "");
-   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
-                  "");
-   // args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
-   //                 " solution.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&hybridization, "-hb", "--hybridization", "-no-hb",
-                  "--no-hybridization", "Enable hybridization.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-   // kappa = freq * M_PI;
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume, as well as periodic meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels = ( rs >= 0 ) ? rs :
-                       (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in serial..."; }
-         mesh->UniformRefinement();
-      }
-      MPI_Barrier(MPI_COMM_WORLD);
-      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     At this point we have a serial mesh containing an EntitySets
-     object which stores the current node/edge/face/element indices
-     for each entity in each set.  This data is duplicated on each MPI
-     rank.
-    */
-   if ( ra > 0 )
-   {
-      cout << "calling EnsureNCMesh" << endl;
-      mesh->EnsureNCMesh();
-      cout << "back from EnsureNCMesh" << endl;
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     We now have an NCEntitySets object which stores the node indices
-     describing each enity in each node/edge/face set and the element
-     indices for the elements in each element set.  This data is
-     duplicated on each MPI rank.
-    */
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them (this is needed in the ADS solver below).
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   if ( pmesh->pent_sets )
-   {
-      cout << "pmesh->pent_sets is non NULL" << endl;
-      pmesh->pent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "pmesh->pent_sets is NULL" << endl;
-   }
-   {
-      int par_ref_levels = rp;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in parallel..."; }
-         pmesh->UniformRefinement();
-      }
-   }
-   pmesh->ReorientTetMesh();
-
-   for (int l = 0; l < ra; l++)
-   {
-      pmesh->RandomRefinement(0.2);
-   }
-   if ( ra > 0 )
-   {
-      if ( pmesh->pent_sets )
-      {
-         cout << "pmesh->pent_sets is non NULL post random refinement" << endl;
-         pmesh->pent_sets->PrintSetInfo(cout);
-      }
-      else
-      {
-         cout << "pmesh->pent_sets is NULL post random refinement" << endl;
-      }
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use the Raviart-Thomas finite elements of the specified order.
-   FiniteElementCollection *fec = new RT_FECollection(order-1, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if ( bt == EntitySets::INVALID )
-   {
-      if (pmesh->bdr_attributes.Size())
-      {
-         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-         ess_bdr = set_bc ? 1 : 0;
-         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   else
-   {
-      fespace->GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                    ess_tdof_list);
-   }
-   if (myid == 0)
-   {
-      cout << "Number of Dirichlet dofs: " << ess_tdof_list.Size() << endl;
-   }
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
-   //    basis functions in the finite element fespace.
-   VectorFunctionCoefficient f(sdim, f_const);
-   ParLinearForm *b = new ParLinearForm(fespace);
-   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x by projecting the exact
-   //    solution. Note that only values from the boundary faces will be used
-   //    when eliminating the non-homogeneous boundary condition to modify the
-   //    r.h.s. vector b.
-   ParGridFunction x(fespace);
-   // VectorFunctionCoefficient F(sdim, F_exact);
-   // x.ProjectCoefficient(F);
-   x = 0.0;
-
-   // 10. Set up the parallel bilinear form corresponding to the H(div)
-   //     diffusion operator grad alpha div + beta I, by adding the div-div and
-   //     the mass domain integrators.
-   Coefficient *alpha = new ConstantCoefficient(1.0);
-   Coefficient *beta  = new ConstantCoefficient(1.0);
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DivDivIntegrator(*alpha));
-   a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation,
-   //     hybridization, etc.
-   FiniteElementCollection *hfec = NULL;
-   ParFiniteElementSpace *hfes = NULL;
-   if (static_cond)
-   {
-      a->EnableStaticCondensation();
-   }
-   else if (hybridization)
-   {
-      hfec = new DG_Interface_FECollection(order-1, dim);
-      hfes = new ParFiniteElementSpace(pmesh, hfec);
-      a->EnableHybridization(hfes, new NormalTraceJumpIntegrator(),
-                             ess_tdof_list);
-   }
-   a->Assemble();
-
-   HypreParMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   HYPRE_Int glob_size = A.GetGlobalNumRows();
-   if (myid == 0)
-   {
-      cout << "Size of linear system: " << glob_size << endl;
-   }
-
-   // 12. Define and apply a parallel PCG solver for A X = B with the 2D AMS or
-   //     the 3D ADS preconditioners from hypre. If using hybridization, the
-   //     system is preconditioned with hypre's BoomerAMG.
-   HypreSolver *prec = NULL;
-   CGSolver *pcg = new CGSolver(A.GetComm());
-   pcg->SetOperator(A);
-   pcg->SetRelTol(1e-12);
-   pcg->SetMaxIter(500);
-   pcg->SetPrintLevel(1);
-   if (hybridization) { prec = new HypreBoomerAMG(A); }
-   else
-   {
-      ParFiniteElementSpace *prec_fespace =
-         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
-      if (dim == 2)   { prec = new HypreAMS(A, prec_fespace); }
-      else            { prec = new HypreADS(A, prec_fespace); }
-   }
-   pcg->SetPreconditioner(*prec);
-   pcg->Mult(B, X);
-
-   // 13. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-   /*
-   // 14. Compute and print the L^2 norm of the error.
-   {
-      double err = x.ComputeL2Error(F);
-      if (myid == 0)
-      {
-         cout << "\n|| F_h - F ||_{L^2} = " << err << '\n' << endl;
-      }
-   }
-   */
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   delete pcg;
-   delete prec;
-   delete hfes;
-   delete hfec;
-   delete a;
-   delete alpha;
-   delete beta;
-   delete b;
-   delete fespace;
-   delete fec;
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}
-
-/*
-// The exact solution (for non-surface meshes)
-void F_exact(const Vector &p, Vector &F)
-{
-   int dim = p.Size();
-
-   double x = p(0);
-   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0;
-
-   F(0) = cos(kappa*x)*sin(kappa*y);
-   F(1) = cos(kappa*y)*sin(kappa*x);
-   if (dim == 3)
-   {
-      F(2) = 0.0;
-   }
-}
-
-// The right hand side
-void f_exact(const Vector &p, Vector &f)
-{
-   int dim = p.Size();
-
-   double x = p(0);
-   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0;
-
-   double temp = 1 + 2*kappa*kappa;
-
-   f(0) = temp*cos(kappa*x)*sin(kappa*y);
-   f(1) = temp*cos(kappa*y)*sin(kappa*x);
-   if (dim == 3)
-   {
-      f(2) = 0;
-   }
-}
-*/
-void f_const(const Vector &x, Vector &f)
-{
-   if (dim == 3)
-   {
-      f(0) = 1.0;
-      f(1) = 1.0;
-      f(2) = 1.0;
-   }
-   else
-   {
-      f(0) = 1.0;
-      f(1) = 1.0;
-      if (x.Size() == 3) { f(2) = 0.0; }
-   }
-}

examples/ex6p_es.cpp

@@ -1,325 +0,0 @@
-//                       MFEM Example 6 - Parallel Version
-//
-// Compile with: make ex6p
-//
-// Sample runs:  mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 1
-//               mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/square-disc-nurbs.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/star.mesh -o 3
-//               mpirun -np 4 ex6p -m ../data/escher.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/fichera.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/disc-nurbs.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/ball-nurbs.mesh
-//               mpirun -np 4 ex6p -m ../data/pipe-nurbs.mesh
-//               mpirun -np 4 ex6p -m ../data/star-surf.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/square-disc-surf.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/amr-quad.mesh
-//
-// Description:  This is a version of Example 1 with a simple adaptive mesh
-//               refinement loop. The problem being solved is again the Laplace
-//               equation -Delta u = 1 with homogeneous Dirichlet boundary
-//               conditions. The problem is solved on a sequence of meshes which
-//               are locally refined in a conforming (triangles, tetrahedrons)
-//               or non-conforming (quadrilaterals, hexahedra) manner according
-//               to a simple ZZ error estimator.
-//
-//               The example demonstrates MFEM's capability to work with both
-//               conforming and nonconforming refinements, in 2D and 3D, on
-//               linear, curved and surface meshes. Interpolation of functions
-//               from coarse to fine meshes, as well as persistent GLVis
-//               visualization are also illustrated.
-//
-//               We recommend viewing Example 1 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-static int max_dofs = 100000;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "./star-set.mesh";
-   int order = 1;
-   int bt = EntitySets::INVALID;
-   const char *bs = "";
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&max_dofs, "-md", "--max-dofs",
-                  "Maximum number of degrees of freedom.");
-   args.AddOption(&bt, "-bt", "--bc-entity-type",
-                  "");
-   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
-                  "");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution.
-   //    Also project a NURBS mesh to a piecewise-quadratic curved mesh. Make
-   //    sure that the mesh is non-conforming.
-   if (mesh->NURBSext)
-   {
-      mesh->UniformRefinement();
-      mesh->SetCurvature(2);
-   }
-   mesh->EnsureNCMesh();
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-
-   // 5. Define a parallel mesh by partitioning the serial mesh.
-   //    Once the parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   if ( pmesh.pent_sets )
-   {
-      cout << "pmesh->pent_sets is non NULL" << endl;
-      pmesh.pent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "pmesh->pent_sets is NULL" << endl;
-   }
-
-   // 6. Define a finite element space on the mesh. The polynomial order is
-   //    one (linear) by default, but this can be changed on the command line.
-   H1_FECollection fec(order, dim);
-   ParFiniteElementSpace fespace(&pmesh, &fec);
-
-   Array<int> ess_tdof_list;
-   if ( bt == EntitySets::INVALID )
-   {
-      if (pmesh.bdr_attributes.Size())
-      {
-         Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-         ess_bdr = 1;
-         fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   else
-   {
-      fespace.GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                   ess_tdof_list);
-   }
-
-   // 7. As in Example 1p, we set up bilinear and linear forms corresponding to
-   //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
-   //    problem yet, this will be done in the main loop.
-   ParBilinearForm a(&fespace);
-   ParLinearForm b(&fespace);
-
-   ConstantCoefficient one(1.0);
-
-   BilinearFormIntegrator *integ = new DiffusionIntegrator(one);
-   a.AddDomainIntegrator(integ);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-
-   // 8. The solution vector x and the associated finite element grid function
-   //    will be maintained over the AMR iterations. We initialize it to zero.
-   ParGridFunction x(&fespace);
-   x = 0;
-
-   // 9. Connect to GLVis.
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-
-   socketstream sout;
-   if (visualization)
-   {
-      sout.open(vishost, visport);
-      if (!sout)
-      {
-         if (myid == 0)
-         {
-            cout << "Unable to connect to GLVis server at "
-                 << vishost << ':' << visport << endl;
-            cout << "GLVis visualization disabled.\n";
-         }
-         visualization = false;
-      }
-
-      sout.precision(8);
-   }
-
-   // 10. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
-   //     with L2 projection in the smoothing step to better handle hanging
-   //     nodes and parallel partitioning. We need to supply a space for the
-   //     discontinuous flux (L2) and a space for the smoothed flux (H(div) is
-   //     used here).
-   L2_FECollection flux_fec(order, dim);
-   ParFiniteElementSpace flux_fes(&pmesh, &flux_fec, sdim);
-   RT_FECollection smooth_flux_fec(order-1, dim);
-   ParFiniteElementSpace smooth_flux_fes(&pmesh, &smooth_flux_fec);
-   // Another possible option for the smoothed flux space:
-   // H1_FECollection smooth_flux_fec(order, dim);
-   // ParFiniteElementSpace smooth_flux_fes(&pmesh, &smooth_flux_fec, dim);
-   L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fes, smooth_flux_fes);
-
-   // 11. A refiner selects and refines elements based on a refinement strategy.
-   //     The strategy here is to refine elements with errors larger than a
-   //     fraction of the maximum element error. Other strategies are possible.
-   //     The refiner will call the given error estimator.
-   ThresholdRefiner refiner(estimator);
-   refiner.SetTotalErrorFraction(0.7);
-
-   // 12. The main AMR loop. In each iteration we solve the problem on the
-   //     current mesh, visualize the solution, and refine the mesh.
-   // const int max_dofs = 100000;
-   for (int it = 0; ; it++)
-   {
-      HYPRE_Int global_dofs = fespace.GlobalTrueVSize();
-      if (myid == 0)
-      {
-         cout << "\nAMR iteration " << it << endl;
-         cout << "Number of unknowns: " << global_dofs << endl;
-      }
-
-      // 13. Assemble the stiffness matrix and the right-hand side. Note that
-      //     MFEM doesn't care at this point that the mesh is nonconforming
-      //     and parallel. The FE space is considered 'cut' along hanging
-      //     edges/faces, and also across processor boundaries.
-      a.Assemble();
-      b.Assemble();
-
-      // 14. Create the parallel linear system: eliminate boundary conditions,
-      //     constrain hanging nodes and nodes across processor boundaries.
-      //     The system will be solved for true (unconstrained/unique) DOFs only.
-      // Array<int> ess_tdof_list;
-      if ( bt == EntitySets::INVALID )
-      {
-         if (pmesh.bdr_attributes.Size())
-         {
-            Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-            ess_bdr = 1;
-            fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-         }
-      }
-      else
-      {
-         fespace.GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                      ess_tdof_list);
-      }
-
-      HypreParMatrix A;
-      Vector B, X;
-      const int copy_interior = 1;
-      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
-
-      // 15. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
-      //     preconditioner from hypre.
-      HypreBoomerAMG amg;
-      amg.SetPrintLevel(0);
-      CGSolver pcg(A.GetComm());
-      pcg.SetPreconditioner(amg);
-      pcg.SetOperator(A);
-      pcg.SetRelTol(1e-6);
-      pcg.SetMaxIter(200);
-      pcg.SetPrintLevel(3); // print the first and the last iterations only
-      pcg.Mult(B, X);
-
-      // 16. Extract the parallel grid function corresponding to the finite element
-      //     approximation X. This is the local solution on each processor.
-      a.RecoverFEMSolution(X, b, x);
-
-      // 17. Send the solution by socket to a GLVis server.
-      if (visualization)
-      {
-         sout << "parallel " << num_procs << " " << myid << "\n";
-         sout << "solution\n" << pmesh << x << flush;
-      }
-
-      if (global_dofs > max_dofs)
-      {
-         if (myid == 0)
-         {
-            cout << "Reached the maximum number of dofs. Stop." << endl;
-         }
-         break;
-      }
-
-      // 18. Call the refiner to modify the mesh. The refiner calls the error
-      //     estimator to obtain element errors, then it selects elements to be
-      //     refined and finally it modifies the mesh. The Stop() method can be
-      //     used to determine if a stopping criterion was met.
-      refiner.Apply(pmesh);
-      if (refiner.Stop())
-      {
-         if (myid == 0)
-         {
-            cout << "Stopping criterion satisfied. Stop." << endl;
-         }
-         break;
-      }
-
-      // 19. Update the finite element space (recalculate the number of DOFs,
-      //     etc.) and create a grid function update matrix. Apply the matrix
-      //     to any GridFunctions over the space. In this case, the update
-      //     matrix is an interpolation matrix so the updated GridFunction will
-      //     still represent the same function as before refinement.
-      fespace.Update();
-      x.Update();
-
-      // 20. Load balance the mesh, and update the space and solution. Currently
-      //     available only for nonconforming meshes.
-      if (pmesh.Nonconforming())
-      {
-         pmesh.Rebalance();
-
-         // Update the space and the GridFunction. This time the update matrix
-         // redistributes the GridFunction among the processors.
-         fespace.Update();
-         x.Update();
-      }
-
-      // 21. Inform also the bilinear and linear forms that the space has
-      //     changed.
-      a.Update();
-      b.Update();
-   }
-
-   MPI_Finalize();
-   return 0;
-}

examples/ex7p.cpp

@@ -282,10 +282,10 @@ int main(int argc, char *argv[])
    delete b;
 
    // 12. Compute and print the L^2 norm of the error.
-   double err = x.ComputeL2Error(sol_coef);
+   double error = x.ComputeL2Error(sol_coef);
    if (myid == 0)
    {
-      cout << "\nL2 norm of error: " << err << endl;
+      cout << "\nL2 norm of error: " << error << endl;
    }
 
    // 13. Save the refined mesh and the solution. This output can be viewed

examples/fichera-mixed-set.mesh

@@ -1,162 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-elements
-14
-1 4 13 15 21 25
-1 4 12 13 15 21
-1 4 13 21 22 25
-1 4 15 24 21 25
-1 4 13 15 25 16
-1 5  0  1  4  3  9 10 13 12
-1 5  8  9 12 11 17 18 21 20
-1 5  2  3  6  5 11 12 15 14
-1 6  3  4  6 12 13 15
-1 6  4  7  6 13 16 15
-1 6 12 13 21  9 10 18
-1 6 13 22 21 10 19 18
-1 6 11 14 20 12 15 21
-1 6 15 21 24 14 20 23
-
-boundary
-30
- 1 3  5  6  3  2
- 2 2  3  6  4
- 2 2  4  6  7
- 3 3  3  4  1  0
- 4 3 11 12  9  8
- 5 3  2  3 12 11
- 6 3  0  1 10  9
- 7 2  9 10 18
- 7 2 10 19 18
- 8 3  8  9 18 17
- 9 3  1  4 13 10
-10 3  4  7 16 13
-11 2 13 16 25
-11 2 13 25 22
-12 3 10 13 22 19
-13 3  7  6 15 16
-14 3  6  5 14 15
-15 3 15 14 23 24
-16 2 16 15 25
-16 2 15 24 25
-17 3  5  2 11 14
-18 3  3  0  9 12
-19 3 11  8 17 20
-20 2 11 20 14
-20 2 14 20 23
-21 3 17 18 21 20
-22 3 18 19 22 21
-23 2 21 22 25
-23 2 21 25 24
-24 3 20 21 24 23
-
-vertices
-26
-3
-0 -1 -1
-1 -1 -1
--1 0 -1
-0 0 -1
-1 0 -1
--1 1 -1
-0 1 -1
-1 1 -1
--1 -1 0
-0 -1 0
-1 -1 0
--1 0 0
-0 0 0
-1 0 0
--1 1 0
-0 1 0
-1 1 0
--1 -1 1
-0 -1 1
-1 -1 1
--1 0 1
-0 0 1
-1 0 1
--1 1 1
-0 1 1
-1 1 1
-
-MFEM sets v1.0
-
-vertex_sets
-1
-
-Origin
-1
-12
-
-edge_sets
-2
-
-Axes
-3
-12 13
-12 15
-12 21
-
-Negative Axes
-3
-12 9
-12 11
-12 3
-
-face_sets
-2
-
-Interior Corner
-3
-3 11 12  9 8
-3  2  3 12 11
-3  3  0  9 12
-
-Exterior Corner
-15
-2 13 16 25
-2 13 25 22
-2 16 15 25
-2 15 24 25
-2 21 22 25
-2 21 25 24
-3 10 13 22 19
-3 4 7 16 13
-3 1 4 13 10
-3 7 6 15 16
-3 6 5 14 15
-3 15 14 23 24
-3 20 21 24 23
-3 18 19 22 21
-3 17 18 21 20
-
-element_sets
-3
-
-Interior Corner
-3
-5 6 7
-
-Exterior Corner
-5
-0 1 2 3 4
-
-Steps
-3
-6 8 9

examples/fichera-set.mesh

@@ -1,145 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-#
-
-dimension
-3
-
-elements
-7
-1 5 0 1 4 3 9 10 13 12
-1 5 3 4 7 6 12 13 16 15
-1 5 2 3 6 5 11 12 15 14
-1 5 8 9 12 11 17 18 21 20
-1 5 9 10 13 12 18 19 22 21
-1 5 12 13 16 15 21 22 25 24
-1 5 11 12 15 14 20 21 24 23
-
-boundary
-24
-1 3 5 6 3 2
-2 3 6 7 4 3
-3 3 3 4 1 0
-4 3 11 12 9 8
-5 3 2 3 12 11
-6 3 0 1 10 9
-7 3 9 10 19 18
-8 3 8 9 18 17
-9 3 1 4 13 10
-10 3 4 7 16 13
-11 3 13 16 25 22
-12 3 10 13 22 19
-13 3 7 6 15 16
-14 3 6 5 14 15
-15 3 15 14 23 24
-16 3 16 15 24 25
-17 3 5 2 11 14
-18 3 3 0 9 12
-19 3 11 8 17 20
-20 3 14 11 20 23
-21 3 17 18 21 20
-22 3 18 19 22 21
-23 3 21 22 25 24
-24 3 20 21 24 23
-
-vertices
-26
-3
-0 -1 -1
-1 -1 -1
--1 0 -1
-0 0 -1
-1 0 -1
--1 1 -1
-0 1 -1
-1 1 -1
--1 -1 0
-0 -1 0
-1 -1 0
--1 0 0
-0 0 0
-1 0 0
--1 1 0
-0 1 0
-1 1 0
--1 -1 1
-0 -1 1
-1 -1 1
--1 0 1
-0 0 1
-1 0 1
--1 1 1
-0 1 1
-1 1 1
-
-MFEM sets v1.0
-
-vertex_sets
-1
-
-Origin
-1
-12
-
-edge_sets
-2
-
-Axes
-3
-12 13
-12 15
-12 21
-
-Negative Axes
-3
-12 9
-12 11
-12 3
-
-face_sets
-2
-
-Interior Corner
-3
-3 11 12  9 8
-3  2  3 12 11
-3  3  0  9 12
-
-Exterior Corner
-12
-3 13 16 25 22
-3 16 15 24 25
-3 21 22 25 24
-3 10 13 22 19
-3 4 7 16 13
-3 1 4 13 10
-3 7 6 15 16
-3 6 5 14 15
-3 15 14 23 24
-3 20 21 24 23
-3 18 19 22 21
-3 17 18 21 20
-
-element_sets
-3
-
-Interior Corner
-3
-0 2 3
-
-Exterior Corner
-1
-5
-
-Steps
-2
-1 3

examples/maxwell-solver/DST/DST.cpp

@@ -0,0 +1,701 @@
+//Diagonal Source Transfer Preconditioner
+
+#include "DST.hpp"
+
+DST::DST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+         double omega_, Coefficient * ws_,  int nrlayers_ , int nx_, int ny_, int nz_)
+   : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), 
+     bf(bf_), Pmllength(Pmllength_), omega(omega_), ws(ws_), nrlayers(nrlayers_)
+{
+
+   // Indentify problem ... Helmholtz or Maxwell
+   int prob_kind = bf->FESpace()->FEColl()->GetContType();
+
+   Mesh * mesh = bf->FESpace()->GetMesh();
+   dim = mesh->Dimension();
+   int partition_kind = 2;
+   nx=nx_; ny=ny_;  nz=nz_;
+   ovlpnrlayers = nrlayers+1;
+   part = new MeshPartition(mesh, partition_kind,nx,ny,nz, ovlpnrlayers);
+   nx = part->nxyz[0]; ny = part->nxyz[1];  nz = part->nxyz[2];
+   nrpatch = part->nrpatch;
+
+   // partition_kind = 1;
+   // MeshPartition * part1 = new MeshPartition(mesh, partition_kind,nx,ny,nz);
+   // SaveMeshPartition(part1->patch_mesh, "output/mesh3x3.", "output/sol3x3.");
+   // SaveMeshPartition(part->patch_mesh, "output/mesh3x3.", "output/sol3x3.");
+   swp = new Sweep(dim);
+
+   StopWatch chrono;
+   chrono.Clear();
+   chrono.Start();
+   dmap  = new DofMap(bf->FESpace(),part);
+   chrono.Stop();
+   cout << "Computing subdomain to global maps: " 
+        << chrono.RealTime() <<" s" << endl; 
+
+
+   chrono.Clear();
+   chrono.Start();
+   NeighborMap = new NeighborDofMaps(part,bf->FESpace(),dmap,ovlpnrlayers); 
+   chrono.Stop();
+   cout << "Computing subdomain to neighbor maps: " 
+        << chrono.RealTime() <<" s" << endl; 
+
+   chrono.Clear();
+   chrono.Start();
+   MarkOverlapElements();
+   MarkOverlapDofs();
+   chrono.Stop();
+   cout << "Computing subdomain overlap dofs: " 
+        << chrono.RealTime() <<" s" << endl; 
+
+   // Set up the local patch problems
+   sqf.SetSize(nrpatch);
+   Optr.SetSize(nrpatch);
+   PmlMat.SetSize(nrpatch);
+   PmlMatInv.SetSize(nrpatch);
+   f_orig.SetSize(nrpatch);
+   f_transf.resize(nrpatch);
+   cout << "nrsubdomain = " << nrpatch << endl;
+
+   chrono.Clear();
+   chrono.Start();
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      // cout << "Setting up patch ip = " << ip << endl;
+      if (prob_kind == 0)
+      {
+         SetHelmholtzPmlSystemMatrix(ip);
+      }
+      else if (prob_kind == 1)
+      {
+         SetMaxwellPmlSystemMatrix(ip);
+      }
+      PmlMat[ip] = Optr[ip]->As<ComplexSparseMatrix>();
+
+      // cout << "Factorizing patch ip = " << ip << endl;
+
+      PmlMatInv[ip] = new ComplexUMFPackSolver;
+      PmlMatInv[ip]->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
+
+      int ndofs = dmap->Dof2GlobalDof[ip].Size();
+      f_orig[ip] = new Vector(ndofs); 
+      f_transf[ip].SetSize(swp->nsweeps);
+      for (int i=0;i<swp->nsweeps; i++)
+      {
+         f_transf[ip][i] = new Vector(ndofs);
+      }
+   }
+   chrono.Stop();
+   cout << "Computing and factoring subdomain matrices: " 
+        << chrono.RealTime() <<" s" << endl; 
+
+   zaux.SetSize(2*bf->FESpace()->GetTrueVSize());
+
+   // char vishost[] = "localhost";
+   // int visport = 19916;
+   // socketstream mesh_sock1(vishost, visport);
+   // mesh_sock1.precision(8);
+   // mesh_sock1 << "mesh\n"
+   //           << *part->patch_mesh[0] << "window_title 'Subdomain'" << flush;
+
+}
+
+void DST::Mult(const Vector &r, Vector &z) const
+{
+   char vishost[] = "localhost";
+   int visport = 19916;
+   socketstream sol_sock(vishost, visport);
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      *f_orig[ip] = 0.0;
+      for (int i=0;i<swp->nsweeps; i++)
+      {
+         *f_transf[ip][i] = 0.0;
+      }
+   }
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      Array<int> * Dof2GlobalDof = &dmap->Dof2GlobalDof[ip];
+      r.GetSubVector(*Dof2GlobalDof,*f_orig[ip]);
+
+      int i,j,k;
+      Getijk(ip,i,j,k);
+      Array<int> ijk(dim);
+      ijk[0] = i;
+      ijk[1] = j;
+      if (dim == 3) ijk[2] = k;
+      Array2D<int> direct(dim,2); direct = 0;
+      for (int d=0;d<dim; d++)
+      {
+         if (ijk[d] > 0) direct[d][0] = 1; 
+         if (ijk[d] < part->nxyz[d]-1) direct[d][1] = 1; 
+      }
+      GetChiRes(*f_orig[ip],ip,direct);
+   }
+
+   z = 0.0; 
+   int nsteps;
+   switch(dim)
+   {
+      case 1: nsteps = nx; break;
+      case 2: nsteps = nx+ny-1; break;
+      default: nsteps = nx+ny+nz-2; break;
+   }
+   int nsweeps = swp->nsweeps;
+
+   for (int l=0; l<nsweeps; l++)
+   {  
+      for (int s = 0; s<nsteps; s++)
+      {
+         Array2D<int> subdomains;
+         GetStepSubdomains(l,s,subdomains);
+
+         int nsubdomains = subdomains.NumRows();
+         for (int sb=0; sb< nsubdomains; sb++)
+         {
+            Array<int> ijk(dim);
+            for (int d=0; d<dim; d++) ijk[d] = subdomains[sb][d]; 
+            int ip = GetPatchId(ijk);
+
+            Array<int> * Dof2GlobalDof = &dmap->Dof2GlobalDof[ip];
+            int ndofs = Dof2GlobalDof->Size();
+
+            Vector sol_local(ndofs); 
+            Vector res_local(ndofs); res_local = 0.0;
+            if (l==0) res_local += *f_orig[ip];
+            res_local += *f_transf[ip][l];
+            if (res_local.Norml2() < 1e-8) continue;
+            PmlMatInv[ip]->Mult(res_local, sol_local);
+            TransferSources(l,ip, sol_local);
+            z.AddElementVector(*Dof2GlobalDof, sol_local);
+         }
+         PlotSolution(z,sol_sock,0,false);
+         cin.get();
+      }
+   }
+}
+
+
+void DST::Getijk(int ip, int & i, int & j, int & k) const
+{
+   k = ip/(nx*ny);
+   j = (ip-k*nx*ny)/nx;
+   i = (ip-k*nx*ny)%nx;
+}
+
+int DST::GetPatchId(const Array<int> & ijk) const
+{
+   int d=ijk.Size();
+   int z = (d==2)? 0 : ijk[2];
+   return part->subdomains(ijk[0],ijk[1],z);
+}
+
+
+void DST::TransferSources(int s, int ip0, Vector & sol0) const
+{
+//   Find all neighbors of patch ip0
+   int i0, j0, k0;
+   Getijk(ip0, i0,j0,k0);
+   Array<int> directions(dim);   
+   for (int i=-1; i<2; i++)
+   {
+      int i1 = i0 + i;
+      if (i1 <0 || i1>=nx) continue;
+      directions[0] = i;
+      for (int j=-1; j<2; j++)
+      {
+         int j1 = j0 + j;
+         if (j1 <0 || j1>=ny) continue;
+         directions[1] = j;
+         int kbeg = (dim == 2) ? 0 : -1;
+         int kend = (dim == 2) ? 1 :  2;
+         for (int k=kbeg; k<kend; k++)
+         {
+            int k1 = k0 + k;
+            if (k1 <0 || k1>=nz) continue;
+            if (dim == 3) directions[2] = k;
+
+            if (i==0 && j==0 && k==0) continue;
+
+            int l = GetSweepToTransfer(s,directions);
+            if (l == -1) continue;
+
+            Vector raux;
+            int ip1 = SourceTransfer(sol0,directions,ip0,raux);
+            *f_transf[ip1][l]-=raux;
+         }
+      }  
+   }
+}
+
+void DST::SetHelmholtzPmlSystemMatrix(int ip)
+{
+   Mesh * mesh = part->patch_mesh[ip];
+   double h = part->MeshSize;
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+
+   int i,j,k;
+   Getijk(ip,i,j,k);
+   if (i == 0 )    length[0][0] = Pmllength[0][0];
+   if (i == nx-1 ) length[0][1] = Pmllength[0][1];
+   if (dim  > 1)
+   {
+      if (j == 0 )    length[1][0] = Pmllength[1][0];
+      if (j == ny-1 ) length[1][1] = Pmllength[1][1];
+   }
+   if (dim  == 3)
+   {
+      if (k == 0 )    length[2][0] = Pmllength[2][0];
+      if (k == nz-1 ) length[2][1] = Pmllength[2][1];
+   }
+   
+   CartesianPML pml(mesh, length);
+   pml.SetOmega(omega);
+
+   Array <int> ess_tdof_list;
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      dmap->fespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+   ProductCoefficient c2_re(c2_re0, *ws);
+   ProductCoefficient c2_im(c2_im0, *ws);
+   sqf[ip] = new SesquilinearForm (dmap->fespaces[ip],bf->GetConvention());
+
+   sqf[ip]->AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   sqf[ip]->AddDomainIntegrator(new MassIntegrator(c2_re),
+                         new MassIntegrator(c2_im));
+   sqf[ip]->Assemble();
+
+   Optr[ip] = new OperatorPtr;
+   sqf[ip]->FormSystemMatrix(ess_tdof_list,*Optr[ip]);
+}
+
+void DST::SetMaxwellPmlSystemMatrix(int ip)
+{
+   Mesh * mesh = part->patch_mesh[ip];
+   double h = part->MeshSize;
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+
+   int i,j,k;
+   Getijk(ip,i,j,k);
+   if (i == 0 )    length[0][0] = Pmllength[0][0];
+   if (i == nx-1 ) length[0][1] = Pmllength[0][1];
+   if (dim  > 1)
+   {
+      if (j == 0 )    length[1][0] = Pmllength[1][0];
+      if (j == ny-1 ) length[1][1] = Pmllength[1][1];
+   }
+   if (dim  == 3)
+   {
+      if (k == 0 )    length[2][0] = Pmllength[2][0];
+      if (k == nz-1 ) length[2][1] = Pmllength[2][1];
+   }
+   
+   CartesianPML pml(mesh, length);
+   pml.SetOmega(omega);
+   Array <int> ess_tdof_list;
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      dmap->fespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ConstantCoefficient omeg(-pow(omega, 2));
+   int cdim = (dim == 2) ? 1 : dim;
+
+   PmlMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, &pml);
+   PmlMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, &pml);
+
+   PmlMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,&pml);
+   ScalarMatrixProductCoefficient c2_Re0(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im0(omeg,pml_c2_Im);
+   ScalarMatrixProductCoefficient c2_Re(*ws,c2_Re0);
+   ScalarMatrixProductCoefficient c2_Im(*ws,c2_Im0);
+
+   sqf[ip] = new SesquilinearForm(dmap->fespaces[ip],bf->GetConvention());
+
+   sqf[ip]->AddDomainIntegrator(new CurlCurlIntegrator(pml_c1_Re),
+                         new CurlCurlIntegrator(pml_c1_Im));
+   sqf[ip]->AddDomainIntegrator(new VectorFEMassIntegrator(c2_Re),
+                         new VectorFEMassIntegrator(c2_Im));
+   sqf[ip]->Assemble();
+
+   Optr[ip] = new OperatorPtr;
+   sqf[ip]->FormSystemMatrix(ess_tdof_list,*Optr[ip]);
+}
+
+int DST::SourceTransfer(const Vector & Psi0, Array<int> direction, int ip0, Vector & Psi1) const
+{
+   int i0,j0,k0;
+   Getijk(ip0,i0,j0,k0);
+
+   int i1 = i0+direction[0];   
+   int j1 = j0+direction[1];   
+   int k1;
+   if (dim==3) k1 = k0+direction[2];   
+   Array<int> ijk(dim); ijk[0]=i1; ijk[1]=j1; 
+   if (dim == 3 ) ijk[2]=k1;
+   int ip1 = GetPatchId(ijk);
+
+   // Array<int> * Dof2GlobalDof0 = &dmap->Dof2GlobalDof[ip0];
+   Array<int> * Dof2GlobalDof1 = &dmap->Dof2GlobalDof[ip1];
+   // zaux.SetSubVector(*Dof2GlobalDof1,0.0);
+   // zaux.SetSubVector(*Dof2GlobalDof0,Psi0);
+   Psi1.SetSize(Dof2GlobalDof1->Size());
+   Vector zloc(Psi1.Size()); zloc = 0.0;
+   // zaux.GetSubVector(*Dof2GlobalDof1,zloc);
+
+
+   Array<int> test_list0;
+   Array<int> test_list1;
+   Array<int>direction1(dim);
+
+   for (int i = 0; i<dim; i++) direction1[i] = -direction[i];
+
+   NeighborMap->GetNeighborDofMap(ip0,direction,test_list0);
+   NeighborMap->GetNeighborDofMap(ip1,direction1,test_list1);
+
+   Vector test1(zloc.Size()); test1 = 0.0;
+   for (int i = 0; i<test_list0.Size(); i++)
+   {
+      // pick up input possition
+      int j = test_list0[i];
+      // destination
+      int k = test_list1[i];
+      zloc[k] = Psi0[j];
+   }
+
+   PmlMat[ip1]->Mult(zloc,Psi1);
+
+   Array2D<int> direct(dim,2); direct = 0;
+   for (int d = 0; d<dim; d++)
+   {
+      if (direction[d]==1) direct[d][0] = 1;
+      if (direction[d]==-1) direct[d][1] = 1;
+   }
+
+   GetChiRes(Psi1,ip1,direct);
+   return ip1;
+}
+
+void DST::GetChiRes(Vector & res, int ip, Array2D<int> direct) const
+{
+   for (int d=0; d<dim; d++)
+   {
+      // negative direction
+      if (direct[d][0]==1) res.SetSubVector(NovlpDofs[ip][d],0.0);
+      // possitive direction
+      if (direct[d][1]==1) res.SetSubVector(NovlpDofs[ip][d+dim],0.0);
+   }
+}
+
+
+void DST::GetStepSubdomains(const int sweep, const int step, Array2D<int> & subdomains) const
+{
+   Array<int> aux;
+   switch(dim)
+   {
+      case 2: 
+      for (int i=nx-1;i>=0; i--)
+      {  
+         int j;
+         switch (sweep)
+         {
+            case 0:  j = step-i;         break;
+            case 1:  j = step-nx+i+1;    break;
+            case 2:  j = nx+i-step-1;    break;
+            default: j = nx+ny-i-step-2; break;
+         }
+         if (j<0 || j>=ny) continue;
+         aux.Append(i); aux.Append(j);
+      }
+      break; 
+      default:
+      for (int i=nx-1;i>=0; i--)
+      {  
+         for (int j=ny-1;j>=0; j--)
+         {
+            int k;
+            switch (sweep)
+            {
+               case 0:  k = step-i-j;            break;
+               case 1:  k = step-nx+i+1-j;       break;
+               case 2:  k = step-ny+j+1-i;       break;
+               case 3:  k = step-nx-ny+i+j+2;    break;
+               case 4:  k = i+j+nz-1-step;       break;
+               case 5:  k = nx+nz-i+j-step-2;    break;
+               case 6:  k = ny+nz+i-j-step-2;    break;
+               default: k = nx+ny+nz-i-j-step-3; break;
+            }
+            if (k<0 || k>=nz) continue;   
+            aux.Append(i); aux.Append(j); aux.Append(k); 
+         }
+      }
+      break;
+   }
+
+   int nrows = aux.Size()/dim;
+   int ncols = dim;
+
+   subdomains.SetSize(nrows,ncols);
+   for (int r=0;r<nrows; r++)
+   {
+      for (int c=0; c<ncols; c++)
+      {
+         int k = r*ncols + c;
+         subdomains[r][c] = aux[k];
+      }
+   }
+}
+
+
+int DST::GetSweepToTransfer(const int s, Array<int> directions) const
+{
+   int l1=-1;
+   int nsweeps = swp->nsweeps;
+   Array<int> sweep0;   
+   swp->GetSweep(s,sweep0);
+
+
+   switch (dim)
+   {
+      case 2:
+      for (int l=s; l<nsweeps; l++)
+      {
+         // Rule 1: the transfer source direction has to be similar with 
+         // the sweep direction
+         Array<int> sweep1;   
+         swp->GetSweep(l,sweep1);
+         int ddot = 0;
+         for (int d=0; d<dim; d++) ddot+= sweep1[d] * directions[d];
+         if (ddot <= 0) continue;
+
+         // Rule 2: The horizontal or vertical transfer source cannot be used
+         // Case of horizontal or vertical transfer source 
+         // (it can't be both 0 cause it's skipped)
+         if (directions[0]==0 || directions[1] == 0) 
+         {
+            if (sweep0[0] == -sweep1[0] && sweep0[1] == -sweep1[1]) continue;
+         }
+         l1 = l;
+         break;
+      }
+      break;
+      default:
+      for (int l=s; l<nsweeps; l++)
+      {
+         // Rule 1: (similar directions) the transfer source direction has to be similar with 
+         // the sweep direction
+         Array<int> sweep1;   
+         swp->GetSweep(l,sweep1);
+         int ddot = 0;
+         bool similar = true;
+         for (int d=0; d<dim; d++) 
+         {
+            if (sweep1[d] * directions[d] < 0) similar = false;
+            ddot+= sweep1[d] * directions[d];
+         }
+         if (!similar || ddot<=0) continue; // not similar
+
+         // Rule 2: (oposite directions) the transfer source direction has to be similar with 
+         // the sweep direction
+         // 
+         // check any of the projections onto the planes
+         // (xy, xz, yz)
+
+         if ( (directions[0]==0 && directions[1] != 0) || 
+              (directions[0]!=0 && directions[1] == 0) ||
+              (directions[0]==0 && directions[2] != 0) || 
+              (directions[0]!=0 && directions[2] == 0) ||
+              (directions[2]==0 && directions[1] != 0) || 
+              (directions[2]!=0 && directions[1] == 0) ) 
+         {
+            if (sweep0[0] == -sweep1[0] && 
+                sweep0[1] == -sweep1[1] && 
+                sweep0[2] == -sweep1[2]) continue;
+         }
+
+         l1 = l;
+         break;
+      }
+      break;
+   }
+
+   return l1;   
+}
+
+
+DST::~DST()
+{
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      for (int i=0;i<swp->nsweeps; i++)
+      {
+         delete f_transf[ip][i];
+      }
+      delete f_orig[ip];
+      delete PmlMatInv[ip];
+      delete Optr[ip];
+      delete sqf[ip];
+      // delete PmlMat[ip];
+   }
+   delete dmap;
+   delete part; 
+}
+
+void DST::PlotSolution(Vector & sol, socketstream & sol_sock, int ip, 
+                  bool localdomain) const
+{
+   FiniteElementSpace * fes;
+   if (!localdomain)
+   {
+      fes = bf->FESpace();
+   }
+   else
+   {
+      fes = dmap->fespaces[ip];
+   }
+   Mesh * mesh = fes->GetMesh();
+   GridFunction gf(fes);
+   double * data = sol.GetData();
+   gf.SetData(data);
+   
+   string keys;
+   // if (ip == 0) 
+   keys = "keys mrRljc\n";
+   sol_sock << "solution\n" << *mesh << gf << keys << "valuerange -0.05 0.05 \n"  << flush;
+   // sol_sock << "solution\n" << *mesh << gf << keys << flush;
+}
+
+
+   void DST::MarkOverlapElements()
+   {
+      // cout<< "Compute Overlap Elements (in each possible direction) " << endl;
+      
+      // Lists of elements
+      // x,y,z = +/- 1 ovlp 
+      NovlpElems.resize(nrpatch);
+
+      for (int ip = 0; ip<nrpatch; ip++)
+      {
+         int i,j,k;
+         Getijk(ip,i,j,k);
+         int ijk[dim]; ijk[0] = i; ijk[1]=j;
+         if (dim==3) ijk[2] = k;
+         int nxyz[dim]; nxyz[0] = nx; nxyz[1]=ny; nxyz[2]=nz;
+
+         FiniteElementSpace * fes = dmap->fespaces[ip];
+         Mesh * mesh = fes->GetMesh();
+         NovlpElems[ip].resize(2*dim);
+
+         Vector pmin, pmax;
+         mesh->GetBoundingBox(pmin,pmax);
+         double h = part->MeshSize;
+         // Loop through elements
+         for (int iel=0; iel<mesh->GetNE(); iel++)
+         {
+            // Get element center
+            Vector center(dim);
+            int geom = mesh->GetElementBaseGeometry(iel);
+            ElementTransformation * tr = mesh->GetElementTransformation(iel);
+            tr->Transform(Geometries.GetCenter(geom),center);
+
+            // Assign elements to the appropriate lists
+            for (int d=0;d<dim; d++)
+            {
+               if (ijk[d]>0)
+               {
+                  if (center[d] >= pmin[d]+h*ovlpnrlayers) 
+                  {
+                     NovlpElems[ip][d].Append(iel);
+                  }
+               }
+               else
+               {
+                  NovlpElems[ip][d].Append(iel);
+               }
+               
+               if (ijk[d]<nxyz[d]-1)
+               {
+                  if (center[d] <= pmax[d]-h*ovlpnrlayers) 
+                  {
+                     NovlpElems[ip][dim+d].Append(iel);
+                  }
+               }
+               else
+               {
+                  NovlpElems[ip][dim+d].Append(iel);
+               }
+            }
+         }
+      }   
+   }
+
+
+
+void DST::MarkOverlapDofs()
+{
+   // cout<< "Compute Overlap dofs (in each possible direction) " << endl;
+   NovlpDofs.resize(nrpatch);
+   for (int ip = 0; ip<nrpatch; ip++)
+   {
+      FiniteElementSpace * fes = dmap->fespaces[ip];
+      // Loop through the marked elements
+      NovlpDofs[ip].resize(2*dim);
+      
+      int n = fes->GetTrueVSize();
+      Array<int> marker(n);
+      for (int d=0;d<2*dim; d++)
+      {
+         marker = 0;
+         int m = 0;
+         int melems = NovlpElems[ip][d].Size(); 
+         for (int iel=0; iel<melems; iel++)
+         {
+            Array<int> ElemDofs;
+            int el =  NovlpElems[ip][d][iel];
+            fes->GetElementDofs(el,ElemDofs);
+            int ndof = ElemDofs.Size();
+            for (int i = 0; i<ndof; ++i)
+            {
+               int eldof = ElemDofs[i];
+               int tdof = (eldof >= 0) ? eldof : abs(eldof) - 1; 
+               if (marker[tdof] == 1) continue;
+               marker[tdof] = 1;
+               m++;
+            }
+         }
+         int k = n-m;
+         NovlpDofs[ip][d].SetSize(2*k);
+         int l = 0;
+         for (int i = 0; i<n; i++)
+         {
+            if (marker[i]==0) 
+            {
+               NovlpDofs[ip][d][l] = i;  // real dofs
+               NovlpDofs[ip][d][l+k] = i+n;  // imag dofs
+               l++;
+            }
+         }
+      }
+   }
+}

examples/maxwell-solver/DST/DST.hpp

@@ -0,0 +1,64 @@
+#pragma once
+#include "../common/Utilities.hpp"
+#include "../common/PML.hpp"
+using namespace std;
+using namespace mfem;
+
+
+
+class DST : public Solver//
+{
+private:
+   // Constructor inputs
+   SesquilinearForm *bf=nullptr;
+   Array2D<double> Pmllength;
+   double omega = 0.5;
+   Coefficient * ws;
+   int nrlayers;
+   // 
+   int nrpatch;
+   int dim;
+   int nx, ny, nz;
+   int ovlpnrlayers;
+   MeshPartition * part=nullptr;
+   DofMap * dmap = nullptr;
+   // Auxiliary global vector for transfers
+   std::vector<std::vector<Array<int>>> NovlpElems;
+   std::vector<std::vector<Array<int>>> NovlpDofs;
+   NeighborDofMaps * NeighborMap = nullptr;
+   Array< SesquilinearForm * > sqf;
+   Array< OperatorPtr * > Optr;
+   Array<ComplexSparseMatrix *> PmlMat;
+   Array<ComplexUMFPackSolver *> PmlMatInv;
+   Sweep * swp=nullptr;
+   mutable Array<Vector *> f_orig;
+   mutable std::vector<Array<Vector * >> f_transf;
+   mutable Vector zaux;
+
+
+
+   void MarkOverlapElements();
+   void MarkOverlapDofs();
+   void ComputeOverlapDofMaps();
+   void Getijk(int ip, int & i, int & j, int & k ) const;
+   int GetPatchId(const Array<int> & ijk) const;
+  
+   void SetHelmholtzPmlSystemMatrix(int ip);
+   void SetMaxwellPmlSystemMatrix(int ip);
+
+   void GetChiRes(Vector & res, int ip, Array2D<int> direct) const;                    
+   void GetStepSubdomains(const int sweep, const int step, Array2D<int> & subdomains) const;
+   void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+   int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+   int GetSweepToTransfer(const int s, Array<int> directions) const;
+   void PlotSolution(Vector & sol, socketstream & sol_sock, int ip,bool localdomain) const;
+  
+public:
+   DST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+       double omega_, Coefficient * ws_, int nrlayers_, int nx_=2, int ny_=2, int nz_=2);
+   virtual void SetOperator(const Operator &op) {}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~DST();
+};
+
+

examples/maxwell-solver/DST2D/AdditiveST2D.cpp

@@ -0,0 +1,545 @@
+//Additive Source Transfer Preconditioner
+
+#include "AdditiveST2D.hpp"
+
+
+AdditiveST2D::AdditiveST2D(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+         double omega_, Coefficient * ws_,  int nrlayers_)
+   : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), 
+     bf(bf_), Pmllength(Pmllength_), omega(omega_), ws(ws_), nrlayers(nrlayers_)
+{
+   Mesh * mesh = bf->FESpace()->GetMesh();
+   dim = mesh->Dimension();
+
+   int partition_kind;
+
+   // 1. Ovelapping partition with overlap = 2h 
+   partition_kind = 2; // Non Overlapping partition 
+   int nx=2;
+   int ny=2; 
+   int nz=1;
+   ovlpnrlayers = nrlayers+2;
+   povlp = new MeshPartition(mesh, partition_kind,nx,ny,nz, ovlpnrlayers);
+
+   partition_kind = 1;
+   novlp = new MeshPartition(mesh, partition_kind,nx,ny,nz);
+
+   nxyz[0] = povlp->nxyz[0];
+   nxyz[1] = povlp->nxyz[1];
+   nxyz[2] = povlp->nxyz[2];
+   nrpatch = povlp->nrpatch;
+   subdomains = povlp->subdomains;
+
+
+   ovlp_prob  = new DofMap(bf,povlp); 
+   nvlp_prob  = new DofMap(bf,novlp); 
+   PmlMat.SetSize(nrpatch);
+   PmlMatInv.SetSize(nrpatch);
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      PmlMat[ip] = GetPmlSystemMatrix(ip);
+      PmlMatInv[ip] = new KLUSolver;
+      PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
+   }
+
+   int nsteps = nx + ny - 1;
+
+   f_orig.SetSize(nrpatch);
+   usol.SetSize(nrpatch);
+   f_s.SetSize(nrpatch);
+   f_diag.SetSize(nrpatch);
+   // Construct a simple map used for directions of transfer
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      int n = 2*ovlp_prob->fespaces[ip]->GetTrueVSize(); // (x 2 for complex ) 
+      f_orig[ip] = new Vector(n); *f_orig[ip] = 0.0;
+      usol[ip] = new Vector(n); *usol[ip] = 0.0;
+      f_s[ip].SetSize(nsteps);
+      f_diag[ip].SetSize(nsteps);
+      for (int i=0;i<nsteps; i++)
+      {
+         f_s[ip][i] = new Vector(n); 
+         f_diag[ip][i] = new Vector(n);
+      }
+   }
+}
+
+void AdditiveST2D::Mult(const Vector &r, Vector &z) const
+{
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      *f_orig[ip] = 0.0;
+      *usol[ip] = 0.0;
+      for (int i=0;i< f_s[ip].Size(); i++)
+      {
+         *f_s[ip][i] = 0.0;
+         *f_diag[ip][i] = 0.0;
+      }
+   }
+   socketstream res_sock(vishost, visport);
+   Vector res(r);
+   PlotSolution(res,res_sock,0,false); 
+   // for (int ip=0; ip<nrpatch; ip++)
+   for (int ip=nrpatch-1; ip>=0; ip--)
+   {
+      Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+      r.GetSubVector(*Dof2GlobalDof,*f_orig[ip]);
+
+      // make sure that f_ij is compactly supported in \Omega_ij (non overlapping)
+      int i,j,k;
+      Getijk(ip,i,j,k);
+      Array<int> directions(2); directions = 0;
+      int nx = nxyz[0];
+      int ny = nxyz[1];
+      if (i+1<nx) directions[0] = 1;
+      if (j+1<ny) directions[1] = 1;
+      Vector faux(f_orig[ip]->Size());
+      GetChiRes(*f_orig[ip],faux,ip,directions,ovlpnrlayers);
+      directions = 0.0;
+      if (i>0) directions[0] = -1;
+      if (j>0) directions[1] = -1;
+      *f_orig[ip] = 0.0;
+      GetChiRes(faux,*f_orig[ip],ip,directions,ovlpnrlayers);
+      // Array<int> * nDof2GlobalDof = &nvlp_prob->Dof2GlobalDof[ip];
+      // Vector faux(nDof2GlobalDof->Size());
+      // r.GetSubVector(*nDof2GlobalDof,faux);
+      // res = 0.0;
+      // res.SetSubVector(*nDof2GlobalDof,faux);
+      // res.GetSubVector(*Dof2GlobalDof,*f_orig[ip]);
+   }
+
+   z = 0.0; 
+   Vector znew(z);
+
+   // --------------------------------------------
+   //       Sweep in the direction (1,1)
+   // --------------------------------------------
+   int nx = nxyz[0];
+   int ny = nxyz[1];
+   int nsteps = (nx + ny - 1);
+
+   for (int s = 0; s<nsteps; s++)
+   {
+      for (int ip = 0; ip<nrpatch; ip++)
+      {
+         Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+         int ndofs = Dof2GlobalDof->Size();
+
+         Vector sol_local(ndofs); sol_local = 0.0;
+         Vector res_local(ndofs); res_local = 0.0;
+         if (s==0) 
+         {
+            res_local = *f_orig[ip];
+         }
+         else if (s == 1)
+         {
+            res_local = *f_s[ip][s-1];
+         }
+         else
+         {
+            res_local = *f_s[ip][s-1];
+            res_local += *f_diag[ip][s-2];
+         }
+         // cout << "reslocal norm = " << res_local.Norml2() << endl;
+         if (res_local.Norml2() < 1e-12) continue;
+         PmlMatInv[ip]->Mult(res_local, sol_local);
+         AdditiveTransferSources(s, ip, sol_local);
+         *usol[ip] += sol_local;
+
+         // Array<int>directions(2); directions = 0; 
+         // int i,j,k;
+         // Getijk(ip,i,j,k);
+         // if (i+1<nx) directions[0] = 1;
+         // if (j+1<ny) directions[1] = 1;
+         // Vector cfsol_local;
+         // GetCutOffSolution(sol_local,cfsol_local,ip,directions,ovlpnrlayers,true);
+         // sol_local = cfsol_local;
+         // directions = 0.0;
+         // if (i>0) directions[0] = -1;
+         // if (j>0) directions[1] = -1;
+         // GetCutOffSolution(sol_local,cfsol_local,ip,directions,ovlpnrlayers,true);
+         // znew = 0.0;
+         // znew.SetSubVector(*Dof2GlobalDof, cfsol_local);
+         // z+=znew;
+      }
+      // socketstream sol1_sock(vishost, visport);
+      // PlotSolution(z,sol1_sock,0,false); cin.get();
+   }
+   for (int ip = 0; ip<nrpatch; ip++)
+   {
+      Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+
+      Array<int>directions(2); directions = 0; 
+      int i,j,k;
+      Getijk(ip,i,j,k);
+      if (i+1<nx) directions[0] = 1;
+      if (j+1<ny) directions[1] = 1;
+      Vector cfsol_local;
+      GetCutOffSolution(*usol[ip],cfsol_local,ip,directions,ovlpnrlayers,true);
+      *usol[ip] = cfsol_local;
+      directions = 0.0;
+      if (i>0) directions[0] = -1;
+      if (j>0) directions[1] = -1;
+      GetCutOffSolution(*usol[ip],cfsol_local,ip,directions,ovlpnrlayers,true);
+      znew = 0.0;
+      znew.SetSubVector(*Dof2GlobalDof, cfsol_local);
+      z+=znew;
+   }
+}
+
+
+void AdditiveST2D::GetCutOffSolution(const Vector & sol, Vector & cfsol, 
+                  int ip, Array<int> directions, int nlayers, bool local) const
+{
+
+   // int d = directions.Size();
+   // int directx = directions[0]; // 1,0,-1
+   // int directy = directions[1]; // 1,0,-1
+   // int directz;
+   // if (d ==3) directz = directions[2];
+
+   Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+   
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+   double h = GetUniformMeshElementSize(povlp->patch_mesh[ip]);
+
+   int i, j, k;
+   Getijk(ip,i,j,k);
+   // int nx = nxyz[0];
+   // int ny = nxyz[1];
+   if (directions[0]==1) pmax[0] -= h*nrlayers; 
+   if (directions[1]==1) pmax[1] -= h*nrlayers; 
+
+   if (directions[0]==-1) pmin[0] += h*nrlayers; 
+   if (directions[1]==-1) pmin[1] += h*nrlayers; 
+
+   Array2D<double> pmlh(dim,2); pmlh = 0.0;
+   
+   if (directions[0]==1)
+   {
+      pmlh[0][1] = h*(nlayers-nrlayers-1);
+   }
+   if (directions[0]==-1)
+   {
+      pmlh[0][0] = h*(nlayers-nrlayers-1);
+   }
+   if (directions[1]==1)
+   {
+      pmlh[1][1] = h*(nlayers-nrlayers-1);
+   }
+   if (directions[1]==-1)
+   {
+      pmlh[1][0] = h*(nlayers-nrlayers-1);
+   }
+
+   CutOffFnCoefficient cf(CutOffFncn, pmin, pmax, pmlh);
+   double * data = sol.GetData();
+   FiniteElementSpace * fes;
+   if (!local)
+   {
+      fes = bf->FESpace();
+   }
+   else
+   {
+      fes = ovlp_prob->fespaces[ip];
+   }
+   int n = fes->GetTrueVSize();
+   GridFunction solgf_re(fes, data);
+   GridFunction solgf_im(fes, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fes);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   cfsol.SetSize(sol.Size());
+   cfsol = gf;
+}
+
+
+AdditiveST2D::~AdditiveST2D()
+{
+}
+
+
+void AdditiveST2D::Getijk(int ip, int & i, int & j, int & k) const
+{
+   k = ip/(nxyz[0]*nxyz[1]);
+   j = (ip-k*nxyz[0]*nxyz[1])/nxyz[0];
+   i = (ip-k*nxyz[0]*nxyz[1])%nxyz[0];
+}
+
+int AdditiveST2D::GetPatchId(const Array<int> & ijk) const
+{
+   int d=ijk.Size();
+   int z = (d==2)? 0 : ijk[2];
+   return subdomains(ijk[0],ijk[1],z);
+}
+
+
+
+void AdditiveST2D::AdditiveTransferSources(int s, int ip0, Vector & sol0) const
+{
+ // Find all neighbors of patch ip0
+   int nx = nxyz[0];
+   int ny = nxyz[1];
+   int i0, j0, k0;
+   Getijk(ip0, i0,j0,k0);
+   for (int i=-1; i<2; i++)
+   {
+      int i1 = i0 + i;
+      if (i1 <0 || i1>=nx) continue;
+      for (int j=-1; j<2; j++)
+      {
+         if (i==0 && j==0) continue;
+
+         int j1 = j0 + j;
+         if (j1 <0 || j1>=ny) continue;
+         Array<int> ij1(2); ij1[0] = i1; ij1[1]=j1;
+         int ip1 = GetPatchId(ij1);
+
+         Array<int> directions(2);
+         directions[0] = i;
+         directions[1] = j;
+         Vector cfsol0;
+         GetCutOffSolution(sol0,cfsol0,ip0,directions,ovlpnrlayers,true);
+
+         Vector raux;
+         SourceTransfer(cfsol0,directions,ip0,raux);
+         if (abs(i)+abs(j) == 2)
+         {
+            *f_diag[ip1][s]+=raux;
+         }
+         else
+         {
+            *f_s[ip1][s]+=raux;
+         }
+      }
+   }
+}
+
+SparseMatrix * AdditiveST2D::GetPmlSystemMatrix(int ip)
+{
+   double h = GetUniformMeshElementSize(povlp->patch_mesh[ip]);
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+
+   int i,j,k;
+   int nx = nxyz[0];
+   int ny = nxyz[1];
+   Getijk(ip,i,j,k);
+   if (i == 0 ) length[0][0] = Pmllength[0][0];
+   if (j == 0 ) length[1][0] = Pmllength[1][0];
+   if (i == nx-1 ) length[0][1] = Pmllength[0][1];
+   if (j == ny-1 ) length[1][1] = Pmllength[1][1];
+
+   CartesianPML pml(povlp->patch_mesh[ip], length);
+   pml.SetOmega(omega);
+
+   Array <int> ess_tdof_list;
+   if (povlp->patch_mesh[ip]->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(povlp->patch_mesh[ip]->bdr_attributes.Max());
+      ess_bdr = 1;
+      ovlp_prob->fespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+   ProductCoefficient c2_re(c2_re0, *ws);
+   ProductCoefficient c2_im(c2_im0, *ws);
+   SesquilinearForm a(ovlp_prob->fespaces[ip],ComplexOperator::HERMITIAN);
+
+   a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   a.AddDomainIntegrator(new MassIntegrator(c2_re),
+                         new MassIntegrator(c2_im));
+   a.Assemble();
+
+   OperatorPtr Alocal;
+   a.FormSystemMatrix(ess_tdof_list,Alocal);
+   ComplexSparseMatrix * AZ_ext = Alocal.As<ComplexSparseMatrix>();
+   SparseMatrix * Mat = AZ_ext->GetSystemMatrix();
+   Mat->Threshold(1e-13);
+   return Mat;
+}
+
+void AdditiveST2D::PlotSolution(Vector & sol, socketstream & sol_sock, int ip, 
+                  bool localdomain) const
+{
+   FiniteElementSpace * fes;
+   if (!localdomain)
+   {
+      fes = bf->FESpace();
+   }
+   else
+   {
+      fes = ovlp_prob->fespaces[ip];
+   }
+   Mesh * mesh = fes->GetMesh();
+   GridFunction gf(fes);
+   double * data = sol.GetData();
+   gf.SetData(data);
+   
+   string keys;
+   keys = "keys mrRljc\n";
+   sol_sock << "solution\n" << *mesh << gf << keys << flush;
+}
+
+void AdditiveST2D::PlotMesh(socketstream & mesh_sock, int ip) const
+{
+   FiniteElementSpace * fes = ovlp_prob->fespaces[ip];
+   Mesh * mesh = fes->GetMesh();
+   mesh_sock << "mesh\n" << *mesh << flush;
+}
+
+void AdditiveST2D::SaveSolution(Vector & sol, int ip, bool localdomain) const
+{
+   FiniteElementSpace * fes;
+   if (!localdomain)
+   {
+      fes = bf->FESpace();
+   }
+   else
+   {
+      // fes = ovlp_prob->fespaces[ip];
+      fes = nvlp_prob->fespaces[ip];
+   }
+   Mesh * mesh = fes->GetMesh();
+   int n = fes->GetTrueVSize();
+   GridFunction gf_re(fes);
+   GridFunction gf_im(fes);
+   double * data = sol.GetData();
+   gf_re.SetData(data);
+   gf_im.SetData(&data[n]);
+   cout << "saving mesh no " << ip << endl;
+   string mfilename = "output/mesh_nvlp.";
+   ostringstream mesh_name;
+   mesh_name << mfilename << setfill('0') << setw(6) << ip;
+   ofstream mesh_ofs(mesh_name.str().c_str());
+   mesh_ofs.precision(8);
+   mesh->Print(mesh_ofs);
+
+   string sfilename_re = "output/sol_nvlp.";
+
+   ostringstream solre_name;
+   solre_name << sfilename_re << setfill('0') << setw(6) << ip;
+   ofstream solre_ofs(solre_name.str().c_str());
+   gf_re.Save(solre_ofs);
+
+}
+
+int AdditiveST2D::SourceTransfer(const Vector & Psi0, Array<int> direction, int ip0, Vector & Psi1) const
+{
+   int i0,j0,k0;
+   Getijk(ip0,i0,j0,k0);
+
+   int i1 = i0+direction[0];   
+   int j1 = j0+direction[1];   
+   Array<int> ij(2); ij[0]=i1; ij[1]=j1;
+   int ip1 = GetPatchId(ij);
+
+   MFEM_VERIFY(i1 < nxyz[0] && i1>=0, "SourceTransfer: i1 out of bounds");
+   MFEM_VERIFY(j1 < nxyz[1] && j1>=0, "SourceTransfer: j1 out of bounds");
+
+   Array<int> * Dof2GlobalDof0 = &ovlp_prob->Dof2GlobalDof[ip0];
+   Array<int> * Dof2GlobalDof1 = &ovlp_prob->Dof2GlobalDof[ip1];
+   Psi1.SetSize(Dof2GlobalDof1->Size()); Psi1=0.0;
+   Vector r(2*bf->FESpace()->GetTrueVSize());
+   r = 0.0;
+   r.SetSubVector(*Dof2GlobalDof0,Psi0);
+   Vector zloc(Psi1.Size()); zloc = 0.0;
+   r.GetSubVector(*Dof2GlobalDof1,zloc);
+
+   Vector Psi(Dof2GlobalDof1->Size()); Psi=0.0;
+
+   PmlMat[ip1]->Mult(zloc,Psi);
+   Psi *=-1.0;
+
+   Array<int> direct(2); direct = 0;
+   direct[0] = -direction[0];
+   direct[1] = -direction[1];
+   GetChiRes(Psi, Psi1,ip1,direct, ovlpnrlayers);
+
+   return ip1;
+}
+
+
+
+
+void AdditiveST2D::GetChiRes(const Vector & res, Vector & cfres, 
+                    int ip, Array<int> directions, int nlayers) const
+{
+   // int l,k;
+   // int d = directions.Size();
+   // int directx = directions[0]; // 1,0,-1
+   // int directy = directions[1]; // 1,0,-1
+   // int directz;
+   // if (d ==3) directz = directions[2];
+
+   Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+   double h = GetUniformMeshElementSize(mesh);
+   
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+   Array2D<double> pmlh(dim,2); pmlh = 0.0;
+   int i,j,k;
+   Getijk(ip,i,j,k);
+   if (directions[0]==-1)
+   {
+      pmlh[0][0] = h;
+      pmin[0] += h*(nlayers-1);
+   }
+   if (directions[0]==1)
+   {
+      pmlh[0][1] = h;
+      pmax[0] -= h*(nlayers-1);
+
+   }
+   if (directions[1]==-1)
+   {
+      pmlh[1][0] = h;
+      pmin[1] += h*(nlayers-1);
+   }
+   if (directions[1]==1)
+   {
+      pmlh[1][1] = h;
+      pmax[1] -= h*(nlayers-1);
+   }
+   CutOffFnCoefficient cf(ChiFncn, pmin, pmax, pmlh);
+
+   double * data = res.GetData();
+
+   FiniteElementSpace * fespace;
+   fespace = ovlp_prob->fespaces[ip];
+   
+   int n = fespace->GetTrueVSize();
+
+   GridFunction solgf_re(fespace, data);
+   GridFunction solgf_im(fespace, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fespace);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   cfres.SetSize(res.Size());
+   cfres = gf;
+}

examples/maxwell-solver/DST2D/AdditiveST2D.hpp

@@ -0,0 +1,56 @@
+#pragma once
+#include "../common/Utilities.hpp"
+#include "../common/PML.hpp"
+using namespace std;
+using namespace mfem;
+
+class AdditiveST2D : public Solver//
+{
+private:
+   int nrpatch;
+   int dim;
+   MeshPartition * povlp=nullptr;
+   MeshPartition * novlp=nullptr;
+   int ovlpnrlayers;
+   int nxyz[3];
+   const Operator * A=nullptr;
+   DofMap * ovlp_prob = nullptr;
+   DofMap * nvlp_prob = nullptr;
+   Array<SparseMatrix *> PmlMat;
+   Array<KLUSolver *> PmlMatInv;
+   Array3D<int> subdomains;
+   mutable Array<Vector *> f_orig;
+   mutable Array<Vector *> usol;
+   mutable Array<Array<Vector * >> f_s;
+   mutable Array<Array<Vector * >> f_diag;
+
+   SesquilinearForm *bf=nullptr;
+   Array2D<double> Pmllength;
+   double omega = 0.5;
+   Coefficient * ws;
+   int nrlayers;
+
+
+   SparseMatrix * GetPmlSystemMatrix(int ip);
+  
+   void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+                          int ip, Array<int> directions, int nlayers, bool local=false) const;
+   void GetChiRes(const Vector & res, Vector & cfres,
+                  int ip, Array<int> directions, int nlayers) const;  
+   
+   void AdditiveTransferSources(int step, int ip, Vector & sol_ext) const;
+   int GetPatchId(const Array<int> & ijk) const;
+   void Getijk(int ip, int & i, int & j, int & k ) const;
+   int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+   void PlotSolution(Vector & sol, socketstream & sol_sock, int ip,bool localdomain) const;
+   void SaveSolution(Vector & sol, int ip,bool localdomain) const;
+   void PlotMesh(socketstream & mesh_sock, int ip) const;
+public:
+   AdditiveST2D(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+       double omega_, Coefficient * ws_, int nrlayers_);
+   virtual void SetOperator(const Operator &op) {A = &op;}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~AdditiveST2D();
+};
+
+

examples/maxwell-solver/DST2D/DST2D.cpp

@@ -0,0 +1,699 @@
+//Diagonal Source Transfer Preconditioner
+
+#include "DST2D.hpp"
+
+DST2D::DST2D(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+         double omega_, Coefficient * ws_,  int nrlayers_)
+   : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), 
+     bf(bf_), Pmllength(Pmllength_), omega(omega_), ws(ws_), nrlayers(nrlayers_)
+{
+   Mesh * mesh = bf->FESpace()->GetMesh();
+   dim = mesh->Dimension();
+
+   int partition_kind;
+
+   partition_kind = 2; 
+   int nx=2;
+   int ny=2; 
+   int nz=1;
+   ovlpnrlayers = nrlayers+1;
+   povlp = new MeshPartition(mesh, partition_kind,nx,ny,nz, ovlpnrlayers);
+
+   partition_kind = 1;
+   novlp = new MeshPartition(mesh, partition_kind,nx,ny,nz);
+
+   nxyz[0] = povlp->nxyz[0];
+   nxyz[1] = povlp->nxyz[1];
+   nxyz[2] = povlp->nxyz[2];
+   nrpatch = povlp->nrpatch;
+   subdomains = povlp->subdomains;
+
+   ovlp_prob  = new DofMap(bf->FESpace(),povlp); 
+   nvlp_prob  = new DofMap(bf->FESpace(),novlp); 
+   PmlMat.SetSize(nrpatch);
+   PmlMatInv.SetSize(nrpatch);
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      PmlMat[ip] = GetPmlSystemMatrix(ip);
+      PmlMatInv[ip] = new KLUSolver;
+      PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
+   }
+   nsweeps = pow(2,dim);
+   sweeps.SetSize(nsweeps,dim);
+   // 2D
+   sweeps(0,0) =  1; sweeps(0,1) = 1;
+   sweeps(1,0) = -1; sweeps(1,1) = 1;
+   sweeps(2,0) =  1; sweeps(2,1) =-1;
+   sweeps(3,0) = -1; sweeps(3,1) =-1;
+
+   // Set up src arrays size
+   f_orig.SetSize(nrpatch);
+   f_transf.SetSize(nrpatch);
+   // Construct a simple map used for directions of transfer
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      int n = 2*ovlp_prob->fespaces[ip]->GetTrueVSize(); // (x 2 for complex ) 
+      f_orig[ip] = new Vector(n); *f_orig[ip] = 0.0;
+      f_transf[ip].SetSize(nsweeps);
+      for (int i=0;i<nsweeps; i++)
+      {
+         f_transf[ip][i] = new Vector(n);
+      }
+   }
+}
+
+void DST2D::Mult(const Vector &r, Vector &z) const
+{
+   // char vishost[] = "localhost";
+   // int  visport   = 19916;
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      *f_orig[ip] = 0.0;
+      for (int i=0;i<nsweeps; i++)
+      {
+         *f_transf[ip][i] = 0.0;
+      }
+   }
+
+   // socketstream res_sock(vishost, visport);
+   // Vector res(r);
+   // PlotSolution(res,res_sock,0,false); 
+
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+      
+      r.GetSubVector(*Dof2GlobalDof,*f_orig[ip]);
+
+      // make sure that f_ij is compactly supported in \Omega_ij (non overlapping)
+      int i,j,k;
+      Getijk(ip,i,j,k);
+      int nx = nxyz[0];
+      int ny = nxyz[1];
+      // Array<int> directions(2); directions = 0;
+      // if (i+1<nx) directions[0] = 1;
+      // if (j+1<ny) directions[1] = 1;
+      // Vector faux(*f_orig[ip]);
+      // GetChiRes(*f_orig[ip],faux,ip,directions,ovlpnrlayers);
+      // directions = 0.0;
+      // if (i>0) directions[0] = -1;
+      // if (j>0) directions[1] = -1;
+      // *f_orig[ip] = 0.0;
+      // GetChiRes(faux,*f_orig[ip],ip,directions,ovlpnrlayers);
+
+      Array2D<int> direct(dim,2); direct = 0;
+      if (i>0) direct[0][0] = 1;
+      if (i+1<nx) direct[0][1] = 1;
+      if (j>0) direct[1][0] = 1;
+      if (j+1<ny) direct[1][1] = 1;
+
+      Vector faux(*f_orig[ip]);
+      *f_orig[ip] = 0.0;
+      GetChiRes(faux,*f_orig[ip],ip,direct,ovlpnrlayers);
+
+   }
+
+   z = 0.0; 
+   Vector znew(z);
+
+   // --------------------------------------------
+   //       Sweep in the direction (1,1)
+   // --------------------------------------------
+   int nx = nxyz[0];
+   int ny = nxyz[1];
+
+   int nsteps = nx + ny - 1;
+
+   for (int l=0; l<nsweeps; l++)
+   {
+      for (int s = 0; s<nsteps; s++)
+      {
+         for (int i=nx-1;i>=0; i--)
+         {
+            int j;
+            switch (l)
+            {
+               case 0:  j = s-i;         break;
+               case 1:  j = s-nx+i+1;    break;
+               case 2:  j = nx+i-s-1;    break;
+               default: j = nx+ny-i-s-2; break;
+            }
+            if (j<0 || j>=ny) continue;
+
+            Array<int> ij(2); ij[0] = i; ij[1]=j;
+            int ip = GetPatchId(ij);
+
+            // Solve the PML problem in patch ip with all sources
+            // Original and all transfered 
+            Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+            int ndofs = Dof2GlobalDof->Size();
+
+            Vector sol_local(ndofs); sol_local = 0.0;
+            Vector res_local(ndofs); res_local = 0.0;
+            if (l==0) res_local += *f_orig[ip];
+            res_local += *f_transf[ip][l];
+            if (res_local.Norml2() < 1e-12) continue;
+            PmlMatInv[ip]->Mult(res_local, sol_local);
+            TransferSources(l,ip, sol_local);
+            Array2D<int> direct(dim,2); direct = 0;
+            if (i>0) direct[0][0] = 1;
+            if (i+1<nx) direct[0][1] = 1;
+            if (j>0) direct[1][0] = 1;
+            if (j+1<ny) direct[1][1] = 1;
+            Vector cfsol_local;
+            GetCutOffSolution(sol_local,cfsol_local,ip,direct,ovlpnrlayers,true);
+            
+            znew = 0.0;
+            znew.SetSubVector(*Dof2GlobalDof, cfsol_local);
+            z+=znew;
+         }
+      }
+   }
+}
+
+
+void DST2D::GetCutOffSolution(const Vector & sol, Vector & cfsol, 
+                  int ip, Array<int> directions, int nlayers, bool local) const
+{
+
+   // int d = directions.Size();
+   // int directx = directions[0]; // 1,0,-1
+   // int directy = directions[1]; // 1,0,-1
+   // int directz;
+   // if (d ==3) directz = directions[2];
+
+   Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+   
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+   double h = GetUniformMeshElementSize(povlp->patch_mesh[ip]);
+
+   int i, j, k;
+   Getijk(ip,i,j,k);
+   if (directions[0]==1) pmax[0] -= h*nrlayers; 
+   if (directions[1]==1) pmax[1] -= h*nrlayers; 
+
+   if (directions[0]==-1) pmin[0] += h*nrlayers; 
+   if (directions[1]==-1) pmin[1] += h*nrlayers; 
+
+   Array2D<double> pmlh(dim,2); pmlh = 0.0;
+   
+   if (directions[0]==1)
+   {
+      pmlh[0][1] = h*(nlayers-nrlayers-1);
+   }
+   if (directions[0]==-1)
+   {
+      pmlh[0][0] = h*(nlayers-nrlayers-1);
+   }
+   if (directions[1]==1)
+   {
+      pmlh[1][1] = h*(nlayers-nrlayers-1);
+   }
+   if (directions[1]==-1)
+   {
+      pmlh[1][0] = h*(nlayers-nrlayers-1);
+   }
+
+   CutOffFnCoefficient cf(CutOffFncn, pmin, pmax, pmlh);
+   double * data = sol.GetData();
+   FiniteElementSpace * fes;
+   if (!local)
+   {
+      fes = bf->FESpace();
+   }
+   else
+   {
+      fes = ovlp_prob->fespaces[ip];
+   }
+   int n = fes->GetTrueVSize();
+   GridFunction solgf_re(fes, data);
+   GridFunction solgf_im(fes, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fes);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   cfsol.SetSize(sol.Size());
+   cfsol = gf;
+}
+
+void DST2D::GetCutOffSolution(const Vector & sol, Vector & cfsol, 
+                  int ip, Array2D<int> direct, int nlayers, bool local) const
+{
+
+
+   Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+   
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+   double h = GetUniformMeshElementSize(povlp->patch_mesh[ip]);
+
+
+   int i, j, k;
+   Getijk(ip,i,j,k);
+
+   Array2D<double> pmlh(dim,2); pmlh = 0.0;
+   for (int i=0; i<dim; i++)
+   {
+      if (direct[i][0]==1) pmin[i] += h*nrlayers; 
+      if (direct[i][1]==1) pmax[i] -= h*nrlayers; 
+      for (int j=0; j<2; j++)
+      {
+         if (direct[i][j]==1)
+         {
+            pmlh[i][j] = h*(nlayers-nrlayers-1);
+         }
+      }  
+   }
+
+   CutOffFnCoefficient cf(CutOffFncn, pmin, pmax, pmlh);
+   double * data = sol.GetData();
+   FiniteElementSpace * fes;
+   if (!local)
+   {
+      fes = bf->FESpace();
+   }
+   else
+   {
+      fes = ovlp_prob->fespaces[ip];
+   }
+   int n = fes->GetTrueVSize();
+   GridFunction solgf_re(fes, data);
+   GridFunction solgf_im(fes, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fes);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   cfsol.SetSize(sol.Size());
+   cfsol = gf;
+}
+
+DST2D::~DST2D()
+{
+}
+
+
+void DST2D::Getijk(int ip, int & i, int & j, int & k) const
+{
+   k = ip/(nxyz[0]*nxyz[1]);
+   j = (ip-k*nxyz[0]*nxyz[1])/nxyz[0];
+   i = (ip-k*nxyz[0]*nxyz[1])%nxyz[0];
+}
+
+int DST2D::GetPatchId(const Array<int> & ijk) const
+{
+   int d=ijk.Size();
+   int z = (d==2)? 0 : ijk[2];
+   return subdomains(ijk[0],ijk[1],z);
+}
+
+
+void DST2D::TransferSources(int sweep, int ip0, Vector & sol0) const
+{
+ // Find all neighbors of patch ip0
+   int nx = nxyz[0];
+   int ny = nxyz[1];
+   int i0, j0, k0;
+   Getijk(ip0, i0,j0,k0);
+   int is = sweeps(sweep,0);
+   int js = sweeps(sweep,1);
+   for (int i=-1; i<2; i++)
+   {
+      int i1 = i0 + i;
+      if (i1 <0 || i1>=nx) continue;
+      for (int j=-1; j<2; j++)
+      {
+         if (i==0 && j==0) continue;
+
+         int j1 = j0 + j;
+         if (j1 <0 || j1>=ny) continue;
+         Array<int> ij1(2); ij1[0] = i1; ij1[1]=j1;
+         int ip1 = GetPatchId(ij1);
+
+         for (int l=sweep; l<nsweeps; l++)
+         {
+            // Conditions on sweeps
+            // Rule 1: the transfer source direction has to be similar with 
+            // the sweep direction
+            int il = sweeps(l,0); 
+            int jl = sweeps(l,1);
+            int ddot = il*i + jl * j;
+            if (ddot <= 0) continue;
+
+            // Rule 2: The horizontal or vertical transfer source cannot be used
+            // in a later sweep that with opposite directions
+
+            if (i==0 || j == 0) // Case of horizontal or vertical transfer source
+            {
+               // skip if the two sweeps have opposite direction
+               if (is == -il && js == -jl) continue;
+            }
+
+            Array2D<int> direct(dim,2); direct = 0;
+            if (i==-1) direct[0][0] = 1;
+            if (i==1) direct[0][1] = 1;
+            if (j==-1) direct[1][0] = 1;
+            if (j==1) direct[1][1] = 1;
+            Vector cfsol0;
+            GetCutOffSolution(sol0,cfsol0,ip0,direct,ovlpnrlayers,true);
+
+
+            Array<int> directions(2);
+            directions[0] = i;
+            directions[1] = j;
+            Vector raux;
+            SourceTransfer(cfsol0,directions,ip0,raux);
+            // SourceTransfer1(cfsol0,directions,ip0,raux);
+            *f_transf[ip1][l]+=raux;
+            break;
+         }
+      }  
+   }
+}
+
+SparseMatrix * DST2D::GetPmlSystemMatrix(int ip)
+{
+   double h = GetUniformMeshElementSize(povlp->patch_mesh[ip]);
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+
+   int i,j,k;
+   int nx = nxyz[0];
+   int ny = nxyz[1];
+   Getijk(ip,i,j,k);
+   if (i == 0 ) length[0][0] = Pmllength[0][0];
+   if (j == 0 ) length[1][0] = Pmllength[1][0];
+   if (i == nx-1 ) length[0][1] = Pmllength[0][1];
+   if (j == ny-1 ) length[1][1] = Pmllength[1][1];
+
+   CartesianPML pml(povlp->patch_mesh[ip], length);
+   pml.SetOmega(omega);
+
+   Array <int> ess_tdof_list;
+   if (povlp->patch_mesh[ip]->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(povlp->patch_mesh[ip]->bdr_attributes.Max());
+      ess_bdr = 1;
+      ovlp_prob->fespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+   ProductCoefficient c2_re(c2_re0, *ws);
+   ProductCoefficient c2_im(c2_im0, *ws);
+   SesquilinearForm a(ovlp_prob->fespaces[ip],ComplexOperator::HERMITIAN);
+
+   a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   a.AddDomainIntegrator(new MassIntegrator(c2_re),
+                         new MassIntegrator(c2_im));
+   a.Assemble();
+
+   OperatorPtr Alocal;
+   a.FormSystemMatrix(ess_tdof_list,Alocal);
+   ComplexSparseMatrix * AZ_ext = Alocal.As<ComplexSparseMatrix>();
+   SparseMatrix * Mat = AZ_ext->GetSystemMatrix();
+   Mat->Threshold(1e-13);
+   return Mat;
+}
+
+void DST2D::PlotSolution(Vector & sol, socketstream & sol_sock, int ip, 
+                  bool localdomain) const
+{
+   FiniteElementSpace * fes;
+   if (!localdomain)
+   {
+      fes = bf->FESpace();
+   }
+   else
+   {
+      fes = ovlp_prob->fespaces[ip];
+   }
+   Mesh * mesh = fes->GetMesh();
+   GridFunction gf(fes);
+   double * data = sol.GetData();
+   gf.SetData(data);
+   
+   string keys;
+   // if (ip == 0) 
+   keys = "keys mrRljc\n";
+   // sol_sock << "solution\n" << *mesh << gf << keys << "valuerange -0.1 0.1 \n"  << flush;
+   sol_sock << "solution\n" << *mesh << gf << keys << flush;
+}
+
+void DST2D::PlotMesh(socketstream & mesh_sock, int ip) const
+{
+   FiniteElementSpace * fes = ovlp_prob->fespaces[ip];
+   Mesh * mesh = fes->GetMesh();
+   mesh_sock << "mesh\n" << *mesh << flush;
+}
+
+void DST2D::SaveSolution(Vector & sol, int ip, bool localdomain) const
+{
+   FiniteElementSpace * fes;
+   if (!localdomain)
+   {
+      fes = bf->FESpace();
+   }
+   else
+   {
+      // fes = ovlp_prob->fespaces[ip];
+      fes = nvlp_prob->fespaces[ip];
+   }
+   Mesh * mesh = fes->GetMesh();
+   int n = fes->GetTrueVSize();
+   GridFunction gf_re(fes);
+   GridFunction gf_im(fes);
+   double * data = sol.GetData();
+   gf_re.SetData(data);
+   gf_im.SetData(&data[n]);
+   cout << "saving mesh no " << ip << endl;
+   // string mfilename = "output/globalmesh.";
+   string mfilename = "output/mesh_nvlp.";
+   ostringstream mesh_name;
+   mesh_name << mfilename << setfill('0') << setw(6) << ip;
+   ofstream mesh_ofs(mesh_name.str().c_str());
+   mesh_ofs.precision(8);
+   mesh->Print(mesh_ofs);
+
+   // string sfilename_re = "output/sol_re.";
+   // string sfilename_im = "output/sol_im.";
+   string sfilename_re = "output/sol_nvlp.";
+   // string sfilename_im = "output/sol_im.";
+
+   ostringstream solre_name;
+   solre_name << sfilename_re << setfill('0') << setw(6) << ip;
+   ofstream solre_ofs(solre_name.str().c_str());
+   gf_re.Save(solre_ofs);
+
+   // ostringstream solim_name;
+   // solim_name << sfilename_im << setfill('0') << setw(6) << ip;
+   // ofstream solim_ofs(solim_name.str().c_str());
+   // gf_im.Save(solim_ofs);
+}
+
+void DST2D::SourceTransfer(const Vector & Psi0, Array<int> direction, int ip0, Vector & Psi1) const
+{
+   int i0,j0,k0;
+   Getijk(ip0,i0,j0,k0);
+
+   int i1 = i0+direction[0];   
+   int j1 = j0+direction[1];   
+   Array<int> ij(2); ij[0]=i1; ij[1]=j1;
+   int ip1 = GetPatchId(ij);
+
+   MFEM_VERIFY(i1 < nxyz[0] && i1>=0, "SourceTransfer: i1 out of bounds");
+   MFEM_VERIFY(j1 < nxyz[1] && j1>=0, "SourceTransfer: j1 out of bounds");
+
+   Array<int> * Dof2GlobalDof0 = &ovlp_prob->Dof2GlobalDof[ip0];
+   Array<int> * Dof2GlobalDof1 = &ovlp_prob->Dof2GlobalDof[ip1];
+   Psi1.SetSize(Dof2GlobalDof1->Size()); Psi1=0.0;
+   Vector r(2*bf->FESpace()->GetTrueVSize());
+   r = 0.0;
+   r.SetSubVector(*Dof2GlobalDof0,Psi0);
+   Vector zloc(Psi1.Size()); zloc = 0.0;
+   r.GetSubVector(*Dof2GlobalDof1,zloc);
+
+   Vector Psi(Dof2GlobalDof1->Size()); Psi=0.0;
+
+   PmlMat[ip1]->Mult(zloc,Psi);
+   Psi *=-1.0;
+
+   Array2D<int> direct(dim,2); direct = 0;
+   if (direction[0]==1) direct[0][0] = 1;
+   if (direction[0]==-1) direct[0][1] = 1;
+   if (direction[1]==1) direct[1][0] = 1;
+   if (direction[1]==-1) direct[1][1] = 1;
+
+   GetChiRes(Psi, Psi1,ip1,direct, ovlpnrlayers);
+
+
+}
+
+void DST2D::GetChiRes(const Vector & res, Vector & cfres, 
+                    int ip, Array<int> directions, int nlayers) const
+{
+
+   Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+   double h = GetUniformMeshElementSize(mesh);
+   
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+   Array2D<double> pmlh(dim,2); pmlh = 0.0;
+   int i,j,k;
+   Getijk(ip,i,j,k);
+   if (directions[0]==-1)
+   {
+      pmlh[0][0] = h;
+      pmin[0] += h*(nlayers-1);
+   }
+   if (directions[0]==1)
+   {
+      pmlh[0][1] = h;
+      pmax[0] -= h*(nlayers-1);
+
+   }
+   if (directions[1]==-1)
+   {
+      pmlh[1][0] = h;
+      pmin[1] += h*(nlayers-1);
+   }
+   if (directions[1]==1)
+   {
+      pmlh[1][1] = h;
+      pmax[1] -= h*(nlayers-1);
+   }
+   CutOffFnCoefficient cf(ChiFncn, pmin, pmax, pmlh);
+
+   double * data = res.GetData();
+
+   FiniteElementSpace * fespace;
+   fespace = ovlp_prob->fespaces[ip];
+   
+   int n = fespace->GetTrueVSize();
+
+   GridFunction solgf_re(fespace, data);
+   GridFunction solgf_im(fespace, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fespace);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   cfres.SetSize(res.Size());
+   cfres = gf;
+}
+
+
+void DST2D::GetChiRes(const Vector & res, Vector & cfres, 
+                    int ip, Array2D<int> direct, int nlayers) const
+{
+
+   Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+   double h = GetUniformMeshElementSize(mesh);
+   
+   Array2D<double> pmlh(dim,2); pmlh = 0.0;
+
+   for (int i=0; i<dim; i++)
+   {
+      if (direct[i][0]==1) pmin[i] += h*(nlayers-1); 
+      if (direct[i][1]==1) pmax[i] -= h*(nlayers-1); 
+      for (int j=0; j<2; j++)
+      {
+         if (direct[i][j]==1)
+         {
+            pmlh[i][j] = h;
+         }
+      }  
+   }
+
+   CutOffFnCoefficient cf(ChiFncn, pmin, pmax, pmlh);
+
+   double * data = res.GetData();
+
+   FiniteElementSpace * fespace;
+   fespace = ovlp_prob->fespaces[ip];
+   
+   int n = fespace->GetTrueVSize();
+
+   GridFunction solgf_re(fespace, data);
+   GridFunction solgf_im(fespace, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fespace);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   cfres.SetSize(res.Size());
+   cfres = gf;
+}
+
+
+void DST2D::SourceTransfer1(const Vector & Psi0, Array<int> direction, int ip0, Vector & Psi1) const
+{
+   int i0,j0,k0;
+   Getijk(ip0,i0,j0,k0);
+
+   int i1 = i0+direction[0];   
+   int j1 = j0+direction[1];   
+   Array<int> ij(2); ij[0]=i1; ij[1]=j1;
+   int ip1 = GetPatchId(ij);
+
+   MFEM_VERIFY(i1 < nxyz[0] && i1>=0, "SourceTransfer: i1 out of bounds");
+   MFEM_VERIFY(j1 < nxyz[1] && j1>=0, "SourceTransfer: j1 out of bounds");
+
+   Vector Psi(Psi0.Size());
+   PmlMat[ip0]->Mult(Psi0,Psi);
+   Psi *=-1.0;
+
+   Array<int> * Dof2GlobalDof0 = &ovlp_prob->Dof2GlobalDof[ip0];
+   Array<int> * Dof2GlobalDof1 = &ovlp_prob->Dof2GlobalDof[ip1];
+   Vector r(2*bf->FESpace()->GetTrueVSize());
+   r = 0.0;
+   r.SetSubVector(*Dof2GlobalDof0,Psi);
+   Psi1.SetSize(Dof2GlobalDof1->Size()); Psi1=0.0;
+   r.GetSubVector(*Dof2GlobalDof1,Psi1);
+
+   // Array<int> direct(2); direct = 0;
+   // direct[0] = -direction[0];
+   // direct[1] = -direction[1];
+   Psi = Psi1;
+
+   Array2D<int> direct(dim,2); direct = 0;
+   if (direction[0]==1) direct[0][0] = 1;
+   if (direction[0]==-1) direct[0][1] = 1;
+   if (direction[1]==1) direct[1][0] = 1;
+   if (direction[1]==-1) direct[1][1] = 1;
+
+   GetChiRes(Psi, Psi1,ip1,direct, ovlpnrlayers);
+
+}
+

examples/maxwell-solver/DST2D/DST2D.hpp

@@ -0,0 +1,65 @@
+#pragma once
+#include "../common/Utilities.hpp"
+#include "../common/PML.hpp"
+using namespace std;
+using namespace mfem;
+
+class DST2D : public Solver//
+{
+private:
+   int nrpatch;
+   int dim;
+   MeshPartition * povlp=nullptr;
+   MeshPartition * novlp=nullptr;
+   int ovlpnrlayers;
+   int nxyz[3];
+   const Operator * A=nullptr;
+   DofMap * ovlp_prob = nullptr;
+   DofMap * nvlp_prob = nullptr;
+   Array<SparseMatrix *> PmlMat;
+   Array<KLUSolver *> PmlMatInv;
+   Array3D<int> subdomains;
+   mutable Array<Vector *> f_orig;
+   int nsweeps;
+   Array2D<int> sweeps;
+   mutable Array<Array<Vector * >> f_transf;
+
+   SesquilinearForm *bf=nullptr;
+   Array2D<double> Pmllength;
+   double omega = 0.5;
+   Coefficient * ws;
+   int nrlayers;
+
+
+   SparseMatrix * GetPmlSystemMatrix(int ip);
+  
+   void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+                          int ip, Array<int> directions, int nlayers, bool local=false) const;
+   void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+                          int ip, Array2D<int> directions, int nlayers, bool local=false) const;                          
+   void GetChiRes(const Vector & res, Vector & cfres,
+                  int ip, Array<int> directions, int nlayers) const;  
+   void GetChiRes(const Vector & res, Vector & cfres,
+                  int ip, Array2D<int> directions, int nlayers) const;  
+   void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+   int GetPatchId(const Array<int> & ijk) const;
+   void Getijk(int ip, int & i, int & j, int & k ) const;
+   void SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+   void SourceTransfer1(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+   void PlotSolution(Vector & sol, socketstream & sol_sock, int ip,bool localdomain) const;
+   void SaveSolution(Vector & sol, int ip,bool localdomain) const;
+   void PlotMesh(socketstream & mesh_sock, int ip) const;
+  
+   // void SourceTransfer1(const Vector & Psi0, Array<int> direction, int ip0, Vector & Psi1) const;
+   // void SetSubMeshesAttributes();
+   // void GetRestrCoeffAttr(const Array<int> & directions, Array<int> & attr) const;
+   // double GetSolOvlpNorm(const Vector & sol, const Array<int> & directions, int ip) const;
+public:
+   DST2D(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+       double omega_, Coefficient * ws_, int nrlayers_);
+   virtual void SetOperator(const Operator &op) {A = &op;}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~DST2D();
+};
+
+

examples/maxwell-solver/DST2D/MeshPartition2D.cpp

@@ -0,0 +1,707 @@
+// #include "MeshPartition2D.hpp"
+
+// double GetUniformMeshElementSize(Mesh * mesh)
+// {
+//    int dim = mesh->Dimension();
+//    int nrelem = mesh->GetNE();
+
+//    DenseMatrix J(dim);
+//    double hmin, hmax;
+//    hmin = infinity();
+//    hmax = -infinity();
+//    Vector attr(nrelem);
+//    for (int iel=0; iel<nrelem; ++iel)
+//    {
+//       int geom = mesh->GetElementBaseGeometry(iel);
+//       ElementTransformation *T = mesh->GetElementTransformation(iel);
+//       T->SetIntPoint(&Geometries.GetCenter(geom));
+//       Geometries.JacToPerfJac(geom, T->Jacobian(), J);
+//       attr(iel) = J.Det();
+//       attr(iel) = pow(abs(attr(iel)), 1.0/double(dim));
+//       hmin = min(hmin, attr(iel));
+//       hmax = max(hmax, attr(iel));
+//    }
+//    MFEM_VERIFY(abs(hmin-hmax) < 1e-12, "Case not supported yet")
+
+//    return hmax;
+// }
+
+// Mesh * ExtendMesh(Mesh * mesh, const Array<int> & directions)
+// {
+//     // extrute on one dimension
+//    // flag = 1 +x, -1 -x, 2 +y, -2 +y , 3 +z, -3, -z
+
+//    // copy the original mesh;
+//    Mesh * mesh_orig = new Mesh(*mesh);
+//    if (!directions.Size()) return mesh_orig;
+   
+//    int dim = mesh_orig->Dimension();
+
+//    Mesh * mesh_ext=nullptr;
+
+//    for (int j=0; j<directions.Size(); j++)
+//    {
+//       int d = directions[j];
+//       MFEM_VERIFY(abs(d)<= dim, "Cannot Extend in dimension " << d << ". Dim = " << dim << endl);
+
+//       Vector pmin;
+//       Vector pmax;
+//       mesh_orig->GetBoundingBox(pmin,pmax);
+//       double h = GetUniformMeshElementSize(mesh_orig);
+//       double val;
+//       // find the vertices on the specific boundary
+//       switch (d)
+//       {
+//       case 1:
+//          val = pmax[0];
+//          break;
+//       case -1:
+//          val = pmin[0];
+//          h = -h;
+//          break;    
+//       case 2:
+//          val = pmax[1];
+//          break;
+//       case -2:
+//          val = pmin[1];
+//          h = -h;
+//          break;   
+//       case 3:
+//          val = pmax[2];
+//          break;
+//       case -3:
+//          val = pmin[2];
+//          h = -h;
+//          break;      
+//       }
+//       int k = 0;
+//       for (int i = 0; i<mesh_orig->GetNV(); ++i)
+//       {
+//          double * coords = mesh_orig->GetVertex(i);
+//          switch (abs(d))
+//          {
+//          case 1:
+//             if (coords[0] == val) k++;
+//             break;
+//          case 2:
+//             if (coords[1] == val) k++;
+//             break;   
+//          case 3:
+//             if (coords[2] == val) k++;
+//             break;      
+//          }
+//       }
+//       int nrvertices = mesh_orig->GetNV() + k;
+//       int nrelements = mesh_orig->GetNE() + pow(pow(k,1.0/(dim-1))-1.0,dim-1);
+
+//       mesh_ext = new Mesh(dim, nrvertices, nrelements);
+
+//       // Add existing vertices
+//       Array<int> vmap(mesh_orig->GetNV()); vmap = 0;
+//       k = mesh_orig->GetNV();
+//       for (int i=0; i<mesh_orig->GetNV(); ++i)
+//       {
+//          double * vert = mesh_orig->GetVertex(i);
+//          mesh_ext->AddVertex(vert);
+//          switch (abs(d))
+//          {
+//          case 1:
+//             if (vert[0] == val) 
+//             {
+//                vmap[i] = k;
+//                k++;
+//             }
+//             break;
+//          case 2:
+//             if (vert[1] == val)
+//             {
+//                vmap[i] = k;
+//                k++;
+//             }
+//             break;   
+//          case 3:
+//             if (vert[2] == val)
+//             {
+//                vmap[i] = k;
+//                k++;
+//             }
+//             break;      
+//          }
+//       } 
+//       // Add existing elements
+//       for (int i=0; i<mesh_orig->GetNE(); ++i)
+//       {
+//          Array<int>ind;
+//          mesh_orig->GetElementVertices(i,ind);
+//          if (dim == 2)
+//          {
+//             mesh_ext->AddQuad(ind);
+//          }
+//          else if (dim == 3)
+//          {
+//             mesh_ext->AddHex(ind);
+//          }
+//       } 
+//       // Add new vertices
+//       k = mesh_orig->GetNV();
+//       for (int i=0; i<mesh_orig->GetNV(); ++i)
+//       {
+//          double * vert = mesh_orig->GetVertex(i);
+//          switch (abs(d))
+//          {
+//          case 1:
+//             if (vert[0] == val) 
+//             {
+//                double coords[dim];
+//                coords[0] = vert[0] + h;
+//                coords[1] = vert[1];
+//                if (dim == 3) coords[2] = vert[2];
+//                mesh_ext->AddVertex(coords);
+//             }
+//             break;
+//          case 2:
+//             if (vert[1] == val)
+//             {
+//                double coords[dim];
+//                coords[0] = vert[0];
+//                coords[1] = vert[1] + h;
+//                if (dim == 3) coords[2] = vert[2];
+//                mesh_ext->AddVertex(coords);
+//             }
+//             break;   
+//          case 3:
+//             if (vert[2] == val)
+//             {
+//                double coords[dim];
+//                coords[0] = vert[0];
+//                coords[1] = vert[1];
+//                coords[2] = vert[2] + h;
+//                mesh_ext->AddVertex(coords);
+//             }
+//             break;      
+//          }
+//       }    
+//       // loop through boundary elements and extend in the given direction
+//       for (int i=0; i<mesh_orig->GetNBE(); ++i) 
+//       {
+//          Array<int> vertices;
+//          mesh_orig->GetBdrElementVertices(i,vertices);
+//          if (dim == 2)
+//          {
+//             int ind[4];
+//             if (vmap[vertices[0]] && vmap[vertices[1]])
+//             {
+//                ind[0] = vmap[vertices[0]];
+//                ind[1] = vmap[vertices[1]];
+//                ind[2] = vertices[1];
+//                ind[3] = vertices[0];
+//                mesh_ext->AddQuad(ind);
+//             }
+//          }
+//          else if (dim == 3)
+//          {
+//             int ind[8];
+//             if (vmap[vertices[0]] && vmap[vertices[1]] && vmap[vertices[2]] && vmap[vertices[3]])
+//             {
+//                ind[0] = vmap[vertices[0]];
+//                ind[1] = vmap[vertices[1]];
+//                ind[2] = vmap[vertices[2]];
+//                ind[3] = vmap[vertices[3]];
+//                ind[4] = vertices[0];
+//                ind[5] = vertices[1];
+//                ind[6] = vertices[2];
+//                ind[7] = vertices[3];
+//                mesh_ext->AddHex(ind);
+//             }
+//          }   
+//       }
+//       mesh_ext->FinalizeTopology();
+
+//       if (j<directions.Size()-1)
+//       {
+//          delete mesh_orig;
+//          mesh_orig = mesh_ext;
+//       }
+//    }
+//    delete mesh_orig;
+//    return mesh_ext;
+// }
+
+// // constructor
+   
+// OverlappingCartesianMeshPartition::OverlappingCartesianMeshPartition(Mesh *mesh_,int & nx,int & ny,int & nz) : mesh(mesh_)
+// {  // default overlap size is 2 elements 
+//    int dim = mesh->Dimension();
+//    int n = pow(mesh->GetNE(), 1.0/(double)dim);
+//    if (nx > n) 
+//    {
+//       nx = n;
+//       MFEM_WARNING("Changed partition in the x direction to nx = " << n << endl);
+//    }
+//    if (ny > n) 
+//    {
+//       ny = n;
+//       MFEM_WARNING("Changed partition in the y direction to ny = " << n << endl);
+//    }
+//    if (nz > n) 
+//    {
+//       nz = n;
+//       MFEM_WARNING("Changed partition in the z direction to nz = " << n << endl);
+//    } 
+//    if (dim == 2) nz = 1;
+//    subdomains.SetSize(nx,ny,nz);
+//    nxyz[0] = nx; nxyz[1]=ny; nxyz[2] = nz;
+//    nrpatch = nx*ny*nz;
+//    Vector pmin, pmax;
+//    mesh->GetBoundingBox(pmin, pmax);
+//    double h = GetUniformMeshElementSize(mesh);
+
+//    element_map.resize(nrpatch);
+
+//    double ppt[dim];
+//    Vector pt(ppt, dim);
+//    int nrelem = mesh->GetNE();
+
+//    for (int el = 0; el < nrelem; el++)
+//    {
+//       mesh->GetElementTransformation(el)->Transform(
+//          Geometries.GetCenter(mesh->GetElementBaseGeometry(el)), pt);
+//       // Given the center coordinates determine the patches that this element contributes to
+//       Array<int> idx0(dim);
+//       Array<int> idx1(dim);
+//       Array<int> idx2(dim);
+//       vector<Array<int>> idx(3);
+//       if (dim == 2) idx[2].Append(0);
+
+//       for (int i = 0; i<dim; i++)
+//       {
+//          idx0[i]  = (int)floor(nxyz[i]*((pt(i) - pmin[i])/(pmax[i] - pmin[i])));
+//          idx1[i] = (int)floor(nxyz[i]*((pt(i)+h - pmin[i])/(pmax[i] - pmin[i])));
+//          idx2[i] = (int)floor(nxyz[i]*((pt(i)-h - pmin[i])/(pmax[i] - pmin[i])));
+
+//          if (idx0[i] < 0) idx0[i] = 0;
+//          if (idx0[i] >= nxyz[i]) idx0[i] = nxyz[i]-1;
+         
+//          if (idx1[i] < 0) idx1[i] = 0;
+//          if (idx1[i] >= nxyz[i]) idx1[i] = nxyz[i]-1;
+
+//          if (idx2[i] < 0) idx2[i] = 0;
+//          if (idx2[i] >= nxyz[i]) idx2[i] = nxyz[i]-1;
+//          // convenient to put in one list
+//          idx[i].Append(idx0[i]);
+//          if (idx1[i] != idx0[i]) idx[i].Append(idx1[i]);
+//          if (idx2[i] != idx0[i] && idx2[i] != idx1[i]) idx[i].Append(idx2[i]);
+//       }
+//       // Now loop through all the combinations according to the idx above
+//       // in case of dim = 2 then kk = 0
+//       for (int k=0; k<idx[2].Size(); k++)
+//       {
+//          int kk = idx[2][k];
+//          for (int j=0; j<idx[1].Size(); j++)
+//          {
+//             int jj = idx[1][j];
+//             for (int i=0; i<idx[0].Size(); i++)
+//             {
+//                int ii = idx[0][i];
+//                int ip = kk*nxyz[0]*nxyz[1] + jj*nxyz[0]+ii;
+//                element_map[ip].Append(el);
+//             }
+//          }
+//       }
+//    }
+//    for (int k = 0; k<nz; k++)
+//    {
+//       for (int j = 0; j<ny; j++)
+//       {
+//          for (int i = 0; i<nx; i++)
+//          {
+//             subdomains(i,j,k) = k*ny*nx + j*nx + i;
+//          }
+//       }   
+//    }
+// }
+
+
+// OverlappingCartesianMeshPartition::OverlappingCartesianMeshPartition(Mesh *mesh_,int & nx,int & ny,int & nz, int ovlp_nlayers) : mesh(mesh_)
+// {  // default overlap size is 2 elements 
+//    int dim = mesh->Dimension();
+//    int n = pow(mesh->GetNE(), 1.0/(double)dim);
+//    if (nx > n) 
+//    {
+//       nx = n;
+//       MFEM_WARNING("Changed partition in the x direction to nx = " << n << endl);
+//    }
+//    if (ny > n) 
+//    {
+//       ny = n;
+//       MFEM_WARNING("Changed partition in the y direction to ny = " << n << endl);
+//    }
+//    if (nz > n) 
+//    {
+//       nz = n;
+//       MFEM_WARNING("Changed partition in the z direction to nz = " << n << endl);
+//    } 
+//    if (dim == 2) nz = 1;
+//    subdomains.SetSize(nx,ny,nz);
+//    nxyz[0] = nx; nxyz[1]=ny; nxyz[2] = nz;
+//    nrpatch = nx*ny*nz;
+//    Vector pmin, pmax;
+//    mesh->GetBoundingBox(pmin, pmax);
+//    double h = GetUniformMeshElementSize(mesh);
+//    cout << "h = " << h << endl;
+
+//    // Check that ovlp_size does not exit subdomain size
+//    MFEM_VERIFY((pmax[0]-pmin[0])/nx >= h*ovlp_nlayers, 
+//                 "Check ovlp size in partition"); 
+//                 cout << "pmax[0]-pmin[0])/nx = " << (pmax[0]-pmin[0])/nx << endl;
+//                 cout << "ovlp_nlayers = " << ovlp_nlayers << endl;
+//                 cout << "h*ovlp_nlayers = " << h*ovlp_nlayers << endl;
+//    MFEM_VERIFY((pmax[1]-pmin[1])/ny >= h*ovlp_nlayers, 
+//                "Check ovlp size in partition"); 
+//    if (dim == 3)
+//    {
+//       MFEM_VERIFY((pmax[2]-pmin[2])/nz >= h*ovlp_nlayers, 
+//                "Check ovlp size in partition"); 
+//    }
+//    element_map.resize(nrpatch);
+
+//    double ppt[dim];
+//    Vector pt(ppt, dim);
+//    int nrelem = mesh->GetNE();
+
+//    for (int el = 0; el < nrelem; el++)
+//    {
+//       mesh->GetElementTransformation(el)->Transform(
+//          Geometries.GetCenter(mesh->GetElementBaseGeometry(el)), pt);
+//       // Given the center coordinates determine the patches that this element contributes to
+//       Array<int> idx0(dim);
+//       Array<int> idx1(dim);
+//       Array<int> idx2(dim);
+//       vector<Array<int>> idx(3);
+//       if (dim == 2) idx[2].Append(0);
+
+//       for (int i = 0; i<dim; i++)
+//       {
+//          idx0[i]  = (int)floor(nxyz[i]*((pt(i) - pmin[i])/(pmax[i] - pmin[i])));
+//          idx1[i] = (int)floor(nxyz[i]*((pt(i)+ovlp_nlayers*h - pmin[i])/(pmax[i] - pmin[i])));
+//          idx2[i] = (int)floor(nxyz[i]*((pt(i)-ovlp_nlayers*h - pmin[i])/(pmax[i] - pmin[i])));
+
+//          if (idx0[i] < 0) idx0[i] = 0;
+//          if (idx0[i] >= nxyz[i]) idx0[i] = nxyz[i]-1;
+         
+//          if (idx1[i] < 0) idx1[i] = 0;
+//          if (idx1[i] >= nxyz[i]) idx1[i] = nxyz[i]-1;
+
+//          if (idx2[i] < 0) idx2[i] = 0;
+//          if (idx2[i] >= nxyz[i]) idx2[i] = nxyz[i]-1;
+//          // convenient to put in one list
+//          idx[i].Append(idx0[i]);
+//          if (idx1[i] != idx0[i]) idx[i].Append(idx1[i]);
+//          if (idx2[i] != idx0[i] && idx2[i] != idx1[i]) idx[i].Append(idx2[i]);
+//       }
+//       // Now loop through all the combinations according to the idx above
+//       // in case of dim = 2 then kk = 0
+//       for (int k=0; k<idx[2].Size(); k++)
+//       {
+//          int kk = idx[2][k];
+//          for (int j=0; j<idx[1].Size(); j++)
+//          {
+//             int jj = idx[1][j];
+//             for (int i=0; i<idx[0].Size(); i++)
+//             {
+//                int ii = idx[0][i];
+//                int ip = kk*nxyz[0]*nxyz[1] + jj*nxyz[0]+ii;
+//                element_map[ip].Append(el);
+//             }
+//          }
+//       }
+//    }
+//    for (int k = 0; k<nz; k++)
+//    {
+//       for (int j = 0; j<ny; j++)
+//       {
+//          for (int i = 0; i<nx; i++)
+//          {
+//             subdomains(i,j,k) = k*ny*nx + j*nx + i;
+//          }
+//       }   
+//    }
+// }
+
+// // constructor
+// CartesianMeshPartition::CartesianMeshPartition(Mesh *mesh_,int & nx, int & ny, int & nz) : mesh(mesh_)
+// {
+//    int dim = mesh->Dimension();
+//    nxyz[0] = nx;
+//    nxyz[1] = ny;
+//    nxyz[2] = nz;
+//    nrpatch = nx*ny*nz;
+//    subdomains.SetSize(nx,ny,nz);
+//    Vector pmin, pmax;
+//    mesh->GetBoundingBox(pmin, pmax);
+
+//    int nrelem = mesh->GetNE();
+//    int partitioning[nrelem];
+
+//    // determine the partitioning using the centers of the elements
+//    double ppt[dim];
+//    Vector pt(ppt, dim);
+//    for (int el = 0; el < nrelem; el++)
+//    {
+//       mesh->GetElementTransformation(el)->Transform(
+//          Geometries.GetCenter(mesh->GetElementBaseGeometry(el)), pt);
+//       int part = 0;
+//       for (int i = dim-1; i >= 0; i--)
+//       {
+//          int idx = (int)floor(nxyz[i]*((pt(i) - pmin[i])/(pmax[i] - pmin[i])));
+//          if (idx < 0)
+//          {
+//             idx = 0;
+//          }
+//          if (idx >= nxyz[i])
+//          {
+//             idx = nxyz[i]-1;
+//          }
+//          part = part * nxyz[i] + idx;
+//       }
+//       partitioning[el] = part;
+//    }
+
+//    element_map.resize(nrpatch);
+//    for (int iel = 0; iel < nrelem; iel++)
+//    {
+//       int ip = partitioning[iel];
+//       element_map[ip].Append(iel);
+//    }
+//    for (int k = 0; k<nz; k++)
+//    {
+//       for (int j = 0; j<ny; j++)
+//       {
+//          for (int i = 0; i<nx; i++)
+//          {
+//             subdomains(i,j,k) = k*ny*nx + j*nx + i;
+//          }
+//       }   
+//    }
+// }
+
+// STPOverlappingCartesianMeshPartition::STPOverlappingCartesianMeshPartition(Mesh *mesh_) : mesh(mesh_)
+// {
+//    int dim = mesh->Dimension();
+//    nx = 5;
+//    ny = 1;
+//    nz = 1;
+//    int nxyz[3] = {nx,ny,nz};
+//    // nrpatch = nx*ny*nz;
+
+//    Vector pmin, pmax;
+//    mesh->GetBoundingBox(pmin, pmax);
+
+//    int nrelem = mesh->GetNE();
+//    int partitioning[nrelem];
+
+//    // determine the partitioning using the centers of the elements
+//    double ppt[dim];
+//    Vector pt(ppt, dim);
+//    for (int el = 0; el < nrelem; el++)
+//    {
+//       mesh->GetElementTransformation(el)->Transform(
+//          Geometries.GetCenter(mesh->GetElementBaseGeometry(el)), pt);
+//       int part = 0;
+//       for (int i = dim-1; i >= 0; i--)
+//       {
+//          int idx = (int)floor(nxyz[i]*((pt(i) - pmin[i])/(pmax[i] - pmin[i])));
+//          if (idx < 0)
+//          {
+//             idx = 0;
+//          }
+//          if (idx >= nxyz[i])
+//          {
+//             idx = nxyz[i]-1;
+//          }
+//          part = part * nxyz[i] + idx;
+//       }
+//       partitioning[el] = part;
+//    }
+
+//    std::vector<Array<int>> elem_map;
+//    int npatch = nx*ny*nz;
+//    elem_map.resize(npatch);
+//    for (int iel = 0; iel < nrelem; iel++)
+//    {
+//       int ip = partitioning[iel];
+//       elem_map[ip].Append(iel);
+//    }
+//    // Append the next subdomain to the previous
+//    nrpatch = nx*ny*nz-1;
+//    element_map.resize(nrpatch);
+//    for (int ip = 0; ip<nrpatch; ip++)
+//    {
+//       element_map[ip].Append(elem_map[ip]);
+//       element_map[ip].Append(elem_map[ip+1]);
+//    }
+// }
+
+// MeshPartition::MeshPartition(Mesh* mesh_, int part,int nx, int ny, int nz, int nrlayers): mesh(mesh_)
+// {
+//    partition_kind = part;
+//    if (part == 1)
+//    {
+//       cout << "Non Overlapping Cartesian Partition " << endl;
+//       CartesianMeshPartition partition(mesh,nx, ny, nz);
+//       element_map = partition.element_map;
+//       // subdomains = partition.subdomains;
+//    }
+//    // else if (part == 3 || part == 4)
+//    else if (part == 2)
+//    {
+//       cout << "Overlapping Cartesian Partition " << endl;
+//       OverlappingCartesianMeshPartition partition(mesh,nx, ny, nz,nrlayers);
+//       element_map = partition.element_map;
+//       subdomains = partition.subdomains;
+//       nxyz[0] = partition.nxyz[0];
+//       nxyz[1] = partition.nxyz[1];
+//       nxyz[2] = partition.nxyz[2];
+//    }
+//    else if (part == 3 || part == 4)
+//    // else if (part == 2)
+//    {
+//       cout << "STP Overlapping Cartesian Partition " << endl;
+//       STPOverlappingCartesianMeshPartition partition(mesh);
+//       element_map = partition.element_map;
+//    }
+//    else
+//    {
+//       MFEM_ABORT("Overlapping Vertex based partition not supprorted")
+//    }
+
+//    nrpatch = element_map.size();
+
+//    int dim = mesh->Dimension();
+
+//    patch_mesh.SetSize(nrpatch);
+//    for (int ip = 0; ip<nrpatch; ++ip)
+//    {
+//       int patch_nrelems = element_map[ip].Size();
+//       element_map[ip].SetSize(patch_nrelems);
+//       Array<int> patch_vertices;
+//       for (int iel=0; iel<patch_nrelems; ++iel)
+//       {
+//          // get the vertices list for the element
+//          Array<int> elem_vertices;
+//          int iel_idx = element_map[ip][iel];
+//          mesh->GetElementVertices(iel_idx,elem_vertices);
+//          patch_vertices.Append(elem_vertices);
+//       }
+//       patch_vertices.Sort();
+//       patch_vertices.Unique();
+//       int patch_nrvertices = patch_vertices.Size();
+
+//       // create the mesh
+//       patch_mesh[ip] = new Mesh(dim,patch_nrvertices,patch_nrelems);
+//       // Add the vertices
+//       for (int iv = 0; iv<patch_nrvertices; ++iv)
+//       {
+//          int vert_idx = patch_vertices[iv];
+//          patch_mesh[ip]->AddVertex(mesh->GetVertex(vert_idx));
+//       }
+
+//       // Add the elements (for now search through all the vertices in the patch is needed)
+//       for (int iel=0; iel<patch_nrelems; ++iel)
+//       {
+//          // get the vertices list for the element
+//          Array<int> elem_vertices;
+//          int iel_idx = element_map[ip][iel];
+//          mesh->GetElementVertices(iel_idx,elem_vertices);
+//          int nrvert = elem_vertices.Size();
+//          int ind[nrvert];
+//          for (int iv = 0; iv<nrvert; ++iv)
+//          {
+//             ind[iv] = patch_vertices.FindSorted(elem_vertices[iv]);
+//          }
+//          mfem::Element::Type elem_type = mesh->GetElementType(element_map[ip][iel]);
+
+//          AddElementToMesh(patch_mesh[ip],elem_type,ind);
+
+//       }
+//       patch_mesh[ip]->FinalizeTopology();
+//    }
+// }
+
+// void MeshPartition::AddElementToMesh(Mesh * mesh,mfem::Element::Type elem_type,
+//                                      int * ind)
+// {
+//    switch (elem_type)
+//    {
+//       case Element::QUADRILATERAL:
+//          mesh->AddQuad(ind);
+//          break;
+//       case Element::TRIANGLE :
+//          mesh->AddTri(ind);
+//          break;
+//       case Element::HEXAHEDRON :
+//          mesh->AddHex(ind);
+//          break;
+//       case Element::TETRAHEDRON :
+//          mesh->AddTet(ind);
+//          break;
+//       default:
+//          MFEM_ABORT("Unknown element type");
+//          break;
+//    }
+// }
+
+// void MeshPartition::PrintElementMap()
+// {
+//    mfem::out << "Element map" << endl;
+//    for (int ip = 0; ip<nrpatch; ++ip)
+//    {
+//       mfem::out << "Patch No: " << ip;
+//       mfem::out << ", element map: " ;
+//       element_map[ip].Print(cout,element_map[ip].Size());
+//    }
+// }
+
+// void SaveMeshPartition(Array<Mesh *> meshes, string mfilename, string sfilename)
+// {
+//    int nrmeshes = meshes.Size();
+//    for (int ip = 0; ip<nrmeshes; ++ip)
+//    {
+//       cout << "saving mesh no " << ip << endl;
+//       ostringstream mesh_name;
+//       mesh_name << mfilename << setfill('0') << setw(6) << ip;
+//       ofstream mesh_ofs(mesh_name.str().c_str());
+//       mesh_ofs.precision(8);
+//       meshes[ip]->Print(mesh_ofs);
+//       L2_FECollection L2fec(1,meshes[ip]->Dimension());
+//       FiniteElementSpace L2fes(meshes[ip], &L2fec);
+//       GridFunction x(&L2fes); 
+      
+//       ConstantCoefficient alpha((double)ip);
+//       x.ProjectCoefficient(alpha);
+//       ostringstream sol_name;
+//       sol_name << sfilename << setfill('0') << setw(6) << ip;
+//       ofstream sol_ofs(sol_name.str().c_str());
+//       x.Save(sol_ofs);
+//    }
+// }
+
+// void SaveMesh(Mesh * mesh, string mfilename)
+// {
+//    cout << "saving global mesh " << endl;
+//    ostringstream mesh_name;
+//    mesh_name << mfilename;
+//    ofstream mesh_ofs(mesh_name.str().c_str());
+//    mesh_ofs.precision(8);
+//    mesh->Print(mesh_ofs);
+// }
+
+// MeshPartition::~MeshPartition()
+// {
+//    for (int ip = 0; ip<nrpatch; ++ip)
+//    {
+//       delete patch_mesh[ip];
+//       patch_mesh[ip] = nullptr;
+//    }
+//    patch_mesh.DeleteAll();
+// }

examples/maxwell-solver/DST2D/MeshPartition2D.hpp

@@ -0,0 +1,74 @@
+// #pragma once
+// #include "mfem.hpp"
+// #include <fstream>
+// #include <iostream>
+// using namespace std;
+// using namespace mfem;
+
+// double GetUniformMeshElementSize(Mesh * mesh);
+// Mesh * ExtendMesh(Mesh * mesh, const Array<int> & directions);
+
+// class CartesianMeshPartition 
+// {
+// private:
+//    Mesh *mesh=nullptr;
+// public:
+//    int nrpatch;
+//    int nxyz[3];
+//    std::vector<Array<int>> element_map;
+//    Array3D<int>subdomains;
+//    // constructor
+//    CartesianMeshPartition(Mesh * mesh_,int & nx, int & ny, int & nz);
+//    ~CartesianMeshPartition() {};
+// };
+
+// class OverlappingCartesianMeshPartition 
+// {
+// private:
+//    Mesh *mesh=nullptr;
+// public:
+//    int nrpatch;
+//    int nxyz[3];
+//    std::vector<Array<int>> element_map;
+//    Array3D<int> subdomains;
+//    // constructor
+//    OverlappingCartesianMeshPartition(Mesh * mesh_,int & nx, int & ny, int & nz);
+//    OverlappingCartesianMeshPartition(Mesh * mesh_,int & nx, int & ny, int & nz, int ovlp_nlayers);
+//    ~OverlappingCartesianMeshPartition() {};
+// };
+
+// class STPOverlappingCartesianMeshPartition // Special layered partition for STP
+// {
+// private:
+//    Mesh *mesh=nullptr;
+// public:
+//    int nrpatch;
+//    int nx, ny, nz;
+//    std::vector<Array<int>> element_map;
+//    // constructor
+//    STPOverlappingCartesianMeshPartition(Mesh * mesh_);
+//    ~STPOverlappingCartesianMeshPartition() {};
+// };
+
+// class MeshPartition
+// {
+// private:
+//    Mesh *mesh=nullptr;
+//    void AddElementToMesh(Mesh * mesh,mfem::Element::Type elem_type,int * ind);
+//    void GetNumVertices(int type, mfem::Element::Type & elem_type, int & nrvert);
+//    void PrintElementMap();
+// public:
+//    int nrpatch;
+//    std::vector<Array<int>> element_map;
+//    Array3D<int> subdomains;
+//    Array<Mesh *> patch_mesh;
+//    int partition_kind;
+//    int nxyz[3];
+//    // constructor
+//    MeshPartition(Mesh * mesh_, int part, int mx=1, int my=1, int mz=1, int ovl_nlayers=0);
+//    ~MeshPartition();
+// };
+
+// void SaveMeshPartition(Array<Mesh * > meshes, 
+//                        string mfilename="output/mesh.",
+//                        string sfilename="output/sol.");

examples/maxwell-solver/DST2D/PML2D.cpp

@@ -0,0 +1,170 @@
+// #include "PML2D.hpp"
+
+
+// CartesianPML::CartesianPML(Mesh *mesh_, Array2D<double> length_)
+//    : mesh(mesh_), length(length_)
+// {
+//    dim = mesh->Dimension();
+//    SetBoundaries();
+// }
+
+// void CartesianPML::SetBoundaries()
+// {
+//    comp_dom_bdr.SetSize(dim, 2);
+//    dom_bdr.SetSize(dim, 2);
+//    // initialize with any vertex
+//    for (int i = 0; i < dim; i++)
+//    {
+//       dom_bdr(i, 0) = mesh->GetVertex(0)[i];
+//       dom_bdr(i, 1) = mesh->GetVertex(0)[i];
+//    }
+
+//    for (int i = 0; i < mesh->GetNBE(); i++)
+//    {
+//       Array<int> bdr_vertices;
+//       mesh->GetBdrElementVertices(i, bdr_vertices);
+//       for (int j = 0; j < bdr_vertices.Size(); j++)
+//       {
+//          for (int k = 0; k < dim; k++)
+//          {
+//             dom_bdr(k, 0) = min(dom_bdr(k, 0), mesh->GetVertex(bdr_vertices[j])[k]);
+//             dom_bdr(k, 1) = max(dom_bdr(k, 1), mesh->GetVertex(bdr_vertices[j])[k]);
+//          }
+//       }
+//    }
+
+//    for (int i = 0; i < dim; i++)
+//    {
+//       comp_dom_bdr(i, 0) = dom_bdr(i, 0) + length(i, 0);
+//       comp_dom_bdr(i, 1) = dom_bdr(i, 1) - length(i, 1);
+//    }
+// }
+
+// void CartesianPML::SetAttributes(Mesh *mesh_)
+// {
+//    int nrelem = mesh_->GetNE();
+//    elems.SetSize(nrelem);
+
+//    for (int i = 0; i < nrelem; ++i)
+//    {
+//       elems[i] = 1;
+//       bool in_pml = false;
+//       Element *el = mesh_->GetElement(i);
+//       Array<int> vertices;
+//       // Initialize Attribute
+//       el->SetAttribute(1);
+//       el->GetVertices(vertices);
+//       int nrvert = vertices.Size();
+//       // Check if any vertex is in the pml
+//       for (int iv = 0; iv < nrvert; ++iv)
+//       {
+//          int vert_idx = vertices[iv];
+//          double *coords = mesh_->GetVertex(vert_idx);
+//          for (int comp = 0; comp < dim; ++comp)
+//          {
+//             if (coords[comp] > comp_dom_bdr(comp, 1) ||
+//                 coords[comp] < comp_dom_bdr(comp, 0))
+//             {
+//                in_pml = true;
+//                break;
+//             }
+//          }
+//       }
+//       if (in_pml)
+//       {
+//          elems[i] = 0;
+//          el->SetAttribute(2);
+//       }
+//    }
+//    mesh_->SetAttributes();
+// }
+
+// void CartesianPML::StretchFunction(const Vector &x,
+//                                    vector<complex<double>> &dxs, double omega)
+// {
+//    complex<double> zi = complex<double>(0., 1.);
+
+//    double n = 2.0;
+//    double c = 5.0;
+//    double coeff;
+//    // Stretch in each direction independently
+//    for (int i = 0; i < dim; ++i)
+//    {
+//       dxs[i] = 1.0;
+//       if (x(i) >= comp_dom_bdr(i, 1))
+//       {
+//          coeff = n * c / omega / pow(length(i, 1), n);
+//          dxs[i] = 1.0 + zi * coeff * abs(pow(x(i) - comp_dom_bdr(i, 1), n - 1.0));
+//       }
+//       if (x(i) <= comp_dom_bdr(i, 0))
+//       {
+//          coeff = n * c / omega / pow(length(i, 0), n);
+//          dxs[i] = 1.0 + zi * coeff * abs(pow(x(i) - comp_dom_bdr(i, 0), n - 1.0));
+//       }
+//    }
+// }
+
+
+// double pml_detJ_Re(const Vector & x, CartesianPML * pml)
+// {
+//    int dim = pml->dim;
+//    double omega = pml->omega;
+//    std::vector<std::complex<double>> dxs(dim);
+//    complex<double> det(1.0,0.0);
+//    pml->StretchFunction(x, dxs, omega);
+//    for (int i=0; i<dim; ++i) det *= dxs[i];
+//    return det.real();
+// }
+
+// double pml_detJ_Im(const Vector & x, CartesianPML * pml)
+// {
+//    int dim = pml->dim;
+//    double omega = pml->omega;
+//    std::vector<std::complex<double>> dxs(dim);
+//    complex<double> det(1.0,0.0);
+//    pml->StretchFunction(x, dxs, omega);
+//    for (int i=0; i<dim; ++i) det *= dxs[i];
+//    return det.imag();
+// }
+
+// void pml_detJ_JT_J_inv_Re(const Vector & x, CartesianPML * pml , DenseMatrix & M)
+// {
+//    int dim = pml->dim;
+//    double omega = pml->omega;
+//    std::vector<std::complex<double>> dxs(dim);
+//    complex<double> det(1.0,0.0);
+//    pml->StretchFunction(x, dxs, omega);
+
+//    for (int i = 0; i<dim; ++i)
+//    {
+//       det *= dxs[i];
+//    }
+
+//    M=0.0;
+//    for (int i = 0; i<dim; ++i)
+//    {
+//       M(i,i) = (det / pow(dxs[i],2)).real();
+//    }
+// }
+
+// void pml_detJ_JT_J_inv_Im(const Vector & x, CartesianPML * pml , DenseMatrix & M)
+// {
+//    int dim = pml->dim;
+//    double omega = pml->omega;
+
+//    std::vector<std::complex<double>> dxs(dim);
+//    complex<double> det = 1.0;
+//    pml->StretchFunction(x, dxs, omega);
+
+//    for (int i = 0; i<dim; ++i)
+//    {
+//       det *= dxs[i];
+//    }
+
+//    M=0.0;
+//    for (int i = 0; i<dim; ++i)
+//    {
+//       M(i,i) = (det / pow(dxs[i],2)).imag();
+//    }
+// }
+

examples/maxwell-solver/DST2D/PML2D.hpp

@@ -0,0 +1,101 @@
+// #pragma once
+// #include "mfem.hpp"
+// #include <fstream>
+// #include <iostream>
+
+// using namespace std;
+// using namespace mfem;
+
+// // Class for setting up a simple Cartesian PML region
+// class CartesianPML
+// {
+// private:
+//    Mesh *mesh;
+
+//    // Length of the PML Region in each direction
+//    Array2D<double> length;
+
+//    // Computational Domain Boundary
+//    Array2D<double> comp_dom_bdr;
+
+//    // Domain Boundary
+//    Array2D<double> dom_bdr;
+
+//    // Integer Array identifying elements in the pml
+//    // 0: in the pml, 1: not in the pml
+//    Array<int> elems;
+
+//    // Compute Domain and Computational Domain Boundaries
+//    void SetBoundaries();
+
+// public:
+//    // Constructor
+//    CartesianPML(Mesh *mesh_,Array2D<double> length_);
+
+//    int dim;
+//    double omega;
+//    // Return Computational Domain Boundary
+//    Array2D<double> GetCompDomainBdr() {return comp_dom_bdr;}
+
+//    // Return Domain Boundary
+//    Array2D<double> GetDomainBdr() {return dom_bdr;}
+
+//    // Return Marker list for elements
+//    Array<int> * GetMarkedPMLElements() {return &elems;}
+
+//    // Mark element in the PML region
+//    void SetAttributes(Mesh *mesh_);
+
+//    void SetOmega(double omega_) {omega = omega_;}
+
+//    // PML complex stretching function
+//    void StretchFunction(const Vector &x, vector<complex<double>> &dxs, double omega);
+// };
+
+
+// class PmlCoefficient : public Coefficient
+// {
+// private:
+//    CartesianPML * pml = nullptr;
+//    double (*Function)(const Vector &, CartesianPML * );
+// public:
+//    PmlCoefficient(double (*F)(const Vector &, CartesianPML *), CartesianPML * pml_)
+//       : pml(pml_), Function(F)
+//    {}
+//    virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+//    {
+//       double x[3];
+//       Vector transip(x, 3);
+//       T.Transform(ip, transip);
+//       return ((*Function)(transip, pml));
+//    }
+// };
+
+
+// // This includes scalar coefficients
+// class PmlMatrixCoefficient : public MatrixCoefficient
+// {
+// private:
+//    CartesianPML * pml = nullptr;
+//    void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
+// public:
+//    PmlMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+//                                           DenseMatrix &),
+//                         CartesianPML * pml_)
+//       : MatrixCoefficient(dim), pml(pml_), Function(F)
+//    {}
+//    virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+//                      const IntegrationPoint &ip)
+//    {
+//       double x[3];
+//       Vector transip(x, 3);
+//       T.Transform(ip, transip);
+//       K.SetSize(height, width);
+//       (*Function)(transip, pml, K);
+//    }
+// };
+
+// double pml_detJ_Re(const Vector & x, CartesianPML * pml);
+// double pml_detJ_Im(const Vector & x, CartesianPML * pml);
+// void pml_detJ_JT_J_inv_Re(const Vector & x, CartesianPML * pml , DenseMatrix & M);
+// void pml_detJ_JT_J_inv_Im(const Vector & x, CartesianPML * pml , DenseMatrix & M);

examples/maxwell-solver/DST2D/Utilities2D.cpp

@@ -0,0 +1,382 @@
+// #include "Utilities2D.hpp"
+
+// double CutOffFncn(const Vector &x, const Vector & pmin, const Vector & pmax, const Array2D<double> & h_)
+// {
+//    int dim = pmin.Size();
+//    Vector h0(dim);
+//    Vector h1(dim);
+//    for (int i=0; i<dim; i++)
+//    {
+//       h0(i) = h_[i][0];
+//       h1(i) = h_[i][1];
+//    }
+//    Vector x0(dim);
+//    Vector x1(dim);
+//    x0 = pmin; x0+=h0;
+//    x1 = pmax; x1-=h1;
+
+//    double f = 1.0;
+//    for (int i = 0; i<dim; i++)
+//    {
+//       double val = 1.0;
+//       if( x(i) >= pmax(i) || x(i) <= pmin(i))
+//       {
+//          val = 0.0;
+//       }  
+//       else if (x(i) < pmax(i) && x(i) >= x1(i))
+//       {
+//          if(h1(i) != 0.0)
+//             // val = (x(i)-pmax(i))/(x1(i)-pmax(i)); 
+//             val = pow((x(i)-pmax(i))/(x1(i)-pmax(i)),1.0); 
+//       }
+//       else if (x(i) > pmin(i) && x(i) <= x0(i))
+//       {
+//          if (h0(i) != 0.0)
+//             // val = (x(i)-pmin(i))/(x0(i)-pmin(i)); 
+//             val = pow((x(i)-pmin(i))/(x0(i)-pmin(i)),1.0); 
+//       }
+
+//       if (h0(i) == 0 && x(i) <= x1(i))
+//       {
+//          val = 1.0;
+//       }
+//       if (h1(i) == 0 && x(i) >= x0(i))
+//       {
+//          val = 1.0;
+//       }
+//       f *= val;
+//    }
+//    return f;
+// }
+
+// double ChiFncn(const Vector &x, const Vector & pmin, const Vector & pmax, const Array2D<double> & h_)
+// {
+//    int dim = pmin.Size();
+//    Vector h0(dim);
+//    Vector h1(dim);
+//    for (int i=0; i<dim; i++)
+//    {
+//       h0(i) = h_[i][0];
+//       h1(i) = h_[i][1];
+//    }
+//    Vector x0(dim);
+//    Vector x1(dim);
+//    x0 = pmin; x0+=h0;
+//    x1 = pmax; x1-=h1;
+
+//    double f = 1.0;
+//    for (int i = 0; i<dim; i++)
+//    {
+//       double val = 1.0;
+//       if( x(i) >= pmax(i) || x(i) <= pmin(i))
+//       {
+//          val = 0.0;
+//       }  
+//       else if (x(i) < pmax(i) && x(i) >= x1(i))
+//       {
+//          if(h1(i) != 0.0)
+//             // val = (x(i)-pmax(i))/(x1(i)-pmax(i)); 
+//             // This function has to be changed to smth more reasonable
+//             val = pow((x(i)-pmax(i))/(x1(i)-pmax(i)),100.0); 
+//       }
+//       else if (x(i) > pmin(i) && x(i) <= x0(i))
+//       {
+//          if (h0(i) != 0.0)
+//             // val = (x(i)-pmin(i))/(x0(i)-pmin(i)); 
+//             val = pow((x(i)-pmin(i))/(x0(i)-pmin(i)),100.0); 
+//       }
+
+//       if (h0(i) == 0 && x(i) <= x1(i))
+//       {
+//          val = 1.0;
+//       }
+//       if (h1(i) == 0 && x(i) >= x0(i))
+//       {
+//          val = 1.0;
+//       }
+//       f *= val;
+//    }
+//    return f;
+// }
+
+
+// DofMap::DofMap(SesquilinearForm * bf_ , MeshPartition * partition_) 
+//                : bf(bf_), partition(partition_)
+// {
+//    // int partition_kind = partition->partition_kind;
+//    // MFEM_VERIFY(partition_kind == 1, "Check Partition kind");
+//    fespace = bf->FESpace();
+//    // Mesh * mesh = fespace->GetMesh();
+//    const FiniteElementCollection * fec = fespace->FEColl();
+//    nrpatch = partition->nrpatch;
+
+//    fespaces.SetSize(nrpatch);
+
+//    Dof2GlobalDof.resize(nrpatch);
+
+//    for (int ip=0; ip<nrpatch; ++ip)
+//    {
+//       // create finite element spaces for each patch 
+//       fespaces[ip] = new FiniteElementSpace(partition->patch_mesh[ip],fec);
+
+//       // construct the patch tdof to global tdof map
+//       int nrdof = fespaces[ip]->GetTrueVSize();
+//       Dof2GlobalDof[ip].SetSize(2*nrdof);
+
+//       // loop through the elements in the patch
+//       for (int iel = 0; iel<partition->element_map[ip].Size(); ++iel)
+//       {
+//          // index in the global mesh
+//          int iel_idx = partition->element_map[ip][iel];
+//          // get the dofs of this element
+//          Array<int> ElemDofs;
+//          Array<int> GlobalElemDofs;
+//          fespaces[ip]->GetElementDofs(iel,ElemDofs);
+//          fespace->GetElementDofs(iel_idx,GlobalElemDofs);
+//          // the sizes have to match
+//          MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+//                      "Size inconsistency");
+//          // loop through the dofs and take into account the signs;
+//          int ndof = ElemDofs.Size();
+//          for (int i = 0; i<ndof; ++i)
+//          {
+//             int pdof_ = ElemDofs[i];
+//             int gdof_ = GlobalElemDofs[i];
+//             int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+//             int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+//             Dof2GlobalDof[ip][pdof] = gdof;
+//             Dof2GlobalDof[ip][pdof+nrdof] = gdof+fespace->GetTrueVSize();
+//          }
+//       }
+//    }
+// }
+
+// DofMap::DofMap(SesquilinearForm * bf_ , MeshPartition * partition_, int nrlayers) 
+//                : bf(bf_), partition(partition_)
+// {
+
+//    nx = partition->nxyz[0];
+//    ny = partition->nxyz[1];
+//    nz = partition->nxyz[2];
+
+//    int partition_kind = partition->partition_kind;
+//    fespace = bf->FESpace();
+//    // Mesh * mesh = fespace->GetMesh();
+//    const FiniteElementCollection * fec = fespace->FEColl();
+//    nrpatch = partition->nrpatch;
+
+//    fespaces.SetSize(nrpatch);
+//    PmlMeshes.SetSize(nrpatch);
+//    // Extend patch meshes to include pml
+
+//    for  (int ip = 0; ip<nrpatch; ip++)
+//    {
+//       int k = ip/(nx*ny);
+//       int j = (ip-k*nx*ny)/nx;
+//       int i = (ip-k*nx*ny)%nx;
+      
+//       Array<int> directions;
+//       if (i > 0)
+//       {
+//          for (int i=0; i<nrlayers; i++)
+//          {
+//             directions.Append(-1);
+//          }
+//       }
+//       if (j > 0)
+//       {
+//          for (int i=0; i<nrlayers; i++)
+//          {
+//             directions.Append(-2);
+//          }
+//       }
+//       if (k > 0)
+//       {
+//          for (int i=0; i<nrlayers; i++)
+//          {
+//             directions.Append(-3);
+//          }
+//       }
+//       if (i < nx-1)
+//       {
+//          for (int i=0; i<nrlayers; i++)
+//          {
+//             if (partition_kind == 3 || partition_kind == 2) directions.Append(1);
+//          }
+//       }
+//       if (j < ny-1)
+//       {
+//          for (int i=0; i<nrlayers; i++)
+//          {
+//             if (partition_kind == 3 || partition_kind == 2) directions.Append(2);
+//          }
+//       }
+//       if (k < nz-1)
+//       {
+//          for (int i=0; i<nrlayers; i++)
+//          {
+//             if (partition_kind == 3 || partition_kind == 2) directions.Append(1);
+//          }
+//       }
+//       PmlMeshes[ip] = ExtendMesh(partition->patch_mesh[ip],directions);
+//    }
+
+//    // Save PML_meshes
+//    string meshpath;
+//    string solpath;
+//    if (partition_kind == 3 || partition_kind == 2) 
+//    {
+//       meshpath = "output/mesh_ovlp_pml.";
+//       solpath = "output/sol_ovlp_pml.";
+//    }
+//    else if (partition_kind == 4)
+//    {
+//       meshpath = "output/mesh_novlp_pml.";
+//       solpath = "output/sol_novlp_pml.";
+//    }
+//    else
+//    {
+//       MFEM_ABORT("This partition kind not supported yet");
+//    }
+   
+//    // SaveMeshPartition(PmlMeshes, meshpath, solpath);
+
+//    PmlFespaces.SetSize(nrpatch);
+//    Dof2GlobalDof.resize(nrpatch);
+//    Dof2PmlDof.resize(nrpatch);
+
+//    for (int ip=0; ip<nrpatch; ++ip)
+//    {
+//       // create finite element spaces for each patch 
+//       fespaces[ip] = new FiniteElementSpace(partition->patch_mesh[ip],fec);
+//       PmlFespaces[ip] = new FiniteElementSpace(PmlMeshes[ip],fec);
+
+//       // construct the patch tdof to global tdof map
+//       int nrdof = fespaces[ip]->GetTrueVSize();
+//       Dof2GlobalDof[ip].SetSize(2*nrdof);
+//       Dof2PmlDof[ip].SetSize(2*nrdof);
+
+//       // build dof maps between patch and extended patch
+//       // loop through the patch elements and constract the dof map
+//       // The same elements in the extended mesh have the same ordering (but not the dofs)
+
+//       // loop through the elements in the patch
+//       for (int iel = 0; iel<partition->element_map[ip].Size(); ++iel)
+//       {
+//          // index in the global mesh
+//          int iel_idx = partition->element_map[ip][iel];
+//          // get the dofs of this element
+//          Array<int> ElemDofs;
+//          Array<int> PmlElemDofs;
+//          Array<int> GlobalElemDofs;
+//          fespaces[ip]->GetElementDofs(iel,ElemDofs);
+//          PmlFespaces[ip]->GetElementDofs(iel,PmlElemDofs);
+//          fespace->GetElementDofs(iel_idx,GlobalElemDofs);
+//          // the sizes have to match
+//          MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+//                      "Size inconsistency");
+//          MFEM_VERIFY(ElemDofs.Size() == PmlElemDofs.Size(),
+//                      "Size inconsistency");            
+//          // loop through the dofs and take into account the signs;
+//          int ndof = ElemDofs.Size();
+//          for (int i = 0; i<ndof; ++i)
+//          {
+//             int pdof_ = ElemDofs[i];
+//             int gdof_ = GlobalElemDofs[i];
+//             int pmldof_ = PmlElemDofs[i];
+//             int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+//             int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+//             int pmldof = (pmldof_ >= 0) ? pmldof_ : abs(pmldof_) - 1;
+
+//             Dof2GlobalDof[ip][pdof] = gdof;
+//             Dof2GlobalDof[ip][pdof+nrdof] = gdof+fespace->GetTrueVSize();
+//             Dof2PmlDof[ip][pdof] = pmldof;
+//             Dof2PmlDof[ip][pdof+nrdof] = pmldof+PmlFespaces[ip]->GetTrueVSize();
+//          }
+//       }
+//    }
+// }
+
+
+// LocalDofMap::LocalDofMap(const FiniteElementCollection * fec_, MeshPartition * part1_, 
+//                MeshPartition * part2_):fec(fec_), part1(part1_), part2(part2_)
+// {
+//    // Each overlapping patch has 2 non-overlapping subdomains
+//    // Thre are n non-overlapping and and n-1 overlapping subdomains
+//    int nrpatch = part2->nrpatch;
+//    MFEM_VERIFY(part1->nrpatch-1 == part2->nrpatch, "Check number of subdomains");
+
+//    cout << "Constructing local dof maps" << endl; 
+//    map1.resize(nrpatch);
+//    map2.resize(nrpatch);
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       // Get the 3 meshes involved
+//       Mesh * mesh = part2->patch_mesh[ip];
+//       Mesh * mesh1 = part1->patch_mesh[ip];
+//       Mesh * mesh2 = part1->patch_mesh[ip+1];
+
+//       // Define the fespaces
+//       FiniteElementSpace fespace(mesh, fec);
+//       FiniteElementSpace fespace1(mesh1, fec);
+//       FiniteElementSpace fespace2(mesh2, fec);
+
+//       int ndof1 = fespace1.GetTrueVSize();
+//       int ndof2 = fespace2.GetTrueVSize();
+
+//       map1[ip].SetSize(2*ndof1); // times 2 because it's complex
+//       map2[ip].SetSize(2*ndof2); // times 2 because it's complex
+
+//       // loop through the elements in the patches
+//       // map 1 is constructed by the first half of elements
+//       // map 2 is constructed by the second half of elements
+
+//       for (int iel = 0; iel<part1->element_map[ip].Size(); ++iel)
+//       {
+//          // index in the overlapping mesh
+//          int iel_idx = iel;
+//          Array<int> ElemDofs;
+//          Array<int> GlobalElemDofs;
+//          fespace1.GetElementDofs(iel,ElemDofs);
+//          fespace.GetElementDofs(iel_idx,GlobalElemDofs);
+//          // the sizes have to match
+//          MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+//                      "Size inconsistency");
+//          // loop through the dofs and take into account the signs;
+//          int ndof = ElemDofs.Size();
+//          for (int i = 0; i<ndof; ++i)
+//          {
+//             int pdof_ = ElemDofs[i];
+//             int gdof_ = GlobalElemDofs[i];
+//             int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+//             int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+//             map1[ip][pdof] = gdof;
+//             map1[ip][pdof+ndof1] = gdof+fespace.GetTrueVSize();
+//          }
+//       }
+//       for (int iel = 0; iel<part1->element_map[ip+1].Size(); ++iel)
+//       {
+//          // index in the overlapping mesh
+//          int k = part1->element_map[ip].Size();
+//          int iel_idx = iel+k;
+//          Array<int> ElemDofs;
+//          Array<int> GlobalElemDofs;
+//          fespace2.GetElementDofs(iel,ElemDofs);
+//          fespace.GetElementDofs(iel_idx,GlobalElemDofs);
+//          // the sizes have to match
+//          MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+//                      "Size inconsistency");
+//          // loop through the dofs and take into account the signs;
+//          int ndof = ElemDofs.Size();
+//          for (int i = 0; i<ndof; ++i)
+//          {
+//             int pdof_ = ElemDofs[i];
+//             int gdof_ = GlobalElemDofs[i];
+//             int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+//             int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+//             map2[ip][pdof] = gdof;
+//             map2[ip][pdof+ndof2] = gdof+fespace.GetTrueVSize();
+//          }
+//       }
+//    }
+// }

examples/maxwell-solver/DST2D/Utilities2D.hpp

@@ -0,0 +1,103 @@
+// #pragma once
+// #include "MeshPartition2D.hpp"
+
+// struct hash_pair {
+//    template <class T1, class T2>
+//    size_t operator()(const pair<T1, T2>& p) const{
+//       auto hash1 = hash<T1>{}(p.first);
+//       auto hash2 = hash<T2>{}(p.second);
+//       return hash1 ^ hash2;
+//    }
+// };
+
+// struct UniqueIndexGenerator
+// {
+//    int counter = 0;
+//    std::unordered_map<pair<int,int>,int, hash_pair> idx;
+//    int Get(int i, int j)
+//    {
+//       pair<int,int> p1(i,j);
+//       std::unordered_map<pair<int,int>,int, hash_pair>::iterator f = idx.find(p1);
+//       if (f == idx.end())
+//       {
+//          idx[p1] = counter;
+//          return counter++;
+//       }
+//       else
+//       {
+//          return (*f).second;
+//       }
+//    }
+//    void Reset()
+//    {
+//       counter = 0;
+//       idx.clear();
+//    }
+// };
+
+
+
+// // Function coefficient that takes the boundingbox of the mesh as an input
+// class CutOffFnCoefficient : public Coefficient
+// {
+// private:
+//    double (*Function)(const Vector &, const Vector &, const Vector &, const Array2D<double> &);
+//    Vector pmin, pmax;
+//    Array2D<double> h; // specify the with of the cutoff function (h in each direction)
+   
+
+// public:
+//    CutOffFnCoefficient(double (*F)(const Vector &, const Vector &, const Vector &, const Array2D<double> &), 
+//                              const Vector & pmin_, const Vector & pmax_, Array2D<double> & h_)
+//       : Function(F), pmin(pmin_), pmax(pmax_), h(h_)
+//    {}
+//    virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+//    {
+//       double x[3];
+//       Vector transip(x, 3);
+//       T.Transform(ip, transip);
+//       return ((*Function)(transip, pmin, pmax, h));
+//    }
+// };
+
+// double CutOffFncn(const Vector &x, const Vector & pmin, 
+// const Vector & pmax, const Array2D<double> & h_);
+// double ChiFncn(const Vector &x, const Vector & pmin, 
+// const Vector & pmax, const Array2D<double> & h_);
+
+// class DofMap // Constructs dof maps for a given partition
+// {
+//    FiniteElementSpace *fespace=nullptr;
+//    SesquilinearForm * bf=nullptr;
+//    MeshPartition * partition=nullptr;
+// public:
+//    int nrpatch, nx, ny, nz;
+//    vector<Array<int>> Dof2GlobalDof;
+//    vector<Array<int>> Dof2PmlDof;
+//    Array<Mesh *> PmlMeshes;
+//    Array<FiniteElementSpace *> fespaces;
+//    Array<FiniteElementSpace *> PmlFespaces;
+//    // constructor
+//    // Non PML contructor dof map
+//    DofMap(SesquilinearForm * bf_, MeshPartition * partition_);
+//    // PML
+//    DofMap(SesquilinearForm * bf_ , MeshPartition * partition_, int nrlayers);
+//    ~DofMap();
+// };
+
+
+// class LocalDofMap // Constructs dof mapbetween two partitions
+// {
+//    const FiniteElementCollection *fec=nullptr;
+//    MeshPartition * part1=nullptr;
+//    MeshPartition * part2=nullptr;
+// public:
+//    int nrpatch, nx, ny, nz;
+//    vector<Array<int>> map1;
+//    vector<Array<int>> map2;
+//    // constructor
+//    LocalDofMap(const FiniteElementCollection * fec_, MeshPartition * part1_, 
+//                MeshPartition * part2_);
+//    ~LocalDofMap();
+// };
+

examples/maxwell-solver/DST2D/back/DSTnew.cpp

@@ -0,0 +1,668 @@
+// //Diagonal Source Transfer Preconditioner
+
+// #include "DST.hpp"
+
+
+// DST::DST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+//          double omega_, Coefficient * ws_,  int nrlayers_)
+//    : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), 
+//      bf(bf_), Pmllength(Pmllength_), omega(omega_), ws(ws_), nrlayers(nrlayers_)
+// {
+//    Mesh * mesh = bf->FESpace()->GetMesh();
+//    dim = mesh->Dimension();
+
+//    // ----------------- Step 1 --------------------
+//    // Introduce 2 layered partitios of the domain 
+//    // 
+//    int partition_kind;
+
+//    // 1. Ovelapping partition with overlap = 2h 
+//    partition_kind = 2; // Non Overlapping partition 
+//    int nx=4;
+//    int ny=1; 
+//    int nz=1;
+
+//    povlp = new MeshPartition(mesh, partition_kind,nx,ny,nz, nrlayers);
+//    nxyz[0] = povlp->nxyz[0];
+//    nxyz[1] = povlp->nxyz[1];
+//    nxyz[2] = povlp->nxyz[2];
+//    nrpatch = povlp->nrpatch;
+//    subdomains = povlp->subdomains;
+
+//    //
+//    // ----------------- Step 1a -------------------
+//    // Save the partition for visualization
+//    // SaveMeshPartition(povlp->patch_mesh, "output/mesh_ovlp.", "output/sol_ovlp.");
+
+//    ovlp_prob  = new DofMap(bf,povlp); 
+//    PmlMat.SetSize(nrpatch);
+//    PmlMatInv.SetSize(nrpatch);
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       PmlMat[ip] = GetPmlSystemMatrix(ip);
+//       PmlMatInv[ip] = new KLUSolver;
+//       PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
+//    }
+//    nsweeps = pow(2,dim);
+//    sweeps.SetSize(nsweeps,dim);
+//    // 2D
+//    sweeps(0,0) =  1; sweeps(0,1) = 1;
+//    sweeps(1,0) = -1; sweeps(1,1) = 1;
+//    sweeps(2,0) =  1; sweeps(2,1) =-1;
+//    sweeps(3,0) = -1; sweeps(3,1) =-1;
+
+//    // Set up src arrays size
+//    f_orig.SetSize(nrpatch);
+//    f_transf.SetSize(nrpatch);
+//    // Construct a simple map used for directions of transfer
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       int n = 2*ovlp_prob->fespaces[ip]->GetTrueVSize(); // (x 2 for complex ) 
+//       f_orig[ip] = new Vector(n); *f_orig[ip] = 0.0;
+//       f_transf[ip].SetSize(nsweeps);
+//       for (int i=0;i<nsweeps; i++)
+//       {
+//          f_transf[ip][i] = new Vector(n);
+//       }
+//    }
+// }
+
+
+
+// void DST::Mult(const Vector &r, Vector &z) const
+// {
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       *f_orig[ip] = 0.0;
+//       for (int i=0;i<nsweeps; i++)
+//       {
+//          *f_transf[ip][i] = 0.0;
+//       }
+//    }
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+//       r.GetSubVector(*Dof2GlobalDof,*f_orig[ip]);
+//    }
+
+//    char vishost[] = "localhost";
+//    int  visport   = 19916;
+//    z = 0.0; 
+//    Vector znew(z);
+//    Vector z1(z);
+//    Vector z2(z);
+
+//    // --------------------------------------------
+//    //       Sweep in the direction (1,1)
+//    // --------------------------------------------
+//    int nx = nxyz[0];
+//    int ny = nxyz[1];
+
+//    int nsteps = nx + ny - 1;
+
+//    for (int l=0; l<1; l++)
+//    {
+//       for (int s = 0; s<nsteps; s++)
+//       {
+//          // the patches involved are the ones such that
+//          // i+j = s
+//          // cout << "Step no: " << s << endl;
+//          for (int i=0;i<nx; i++)
+//          {
+//             int j;
+//             switch (l)
+//             {
+//                case 0:  j = s-i;         break;
+//                case 1:  j = s-nx+i+1;    break;
+//                case 2:  j = nx+i-s-1;    break;
+//                default: j = nx+ny-i-s-2; break;
+//             }
+//             if (j<0 || j>=ny) continue;
+//             // cout << "Patch no: (" << i <<"," << j << ")" << endl; 
+
+//             // find patch id
+//             Array<int> ij(2); ij[0] = i; ij[1]=j;
+//             int ip = GetPatchId(ij);
+//             // cout << "ip = " << ip << endl;
+
+//             // Solve the PML problem in patch ip with all sources
+//             // Original and all transfered (maybe some of them)
+//             Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+//             int ndofs = Dof2GlobalDof->Size();
+
+//             Vector sol_local(ndofs); sol_local = 0.0;
+//             Vector res_local(ndofs); res_local = 0.0;
+//             if (l==0) res_local += *f_orig[ip];
+//             // res_local += *f_orig[ip];
+//             res_local += *f_transf[ip][l];
+//             // Extend by zero to the PML mesh
+//             // if (res_local.Norml2() < 1e-11) continue;
+//             PmlMatInv[ip]->Mult(res_local, sol_local);
+
+//             TransferSources(l,ip, sol_local);
+
+//             // cut off the ip solution to all possible directions
+//             Array<int>directions(2); directions = 0; 
+            
+//             if (i+1<nx) directions[0] = 1;
+//             if (j+1<ny) directions[1] = 1;
+//             Vector cfsol_local;
+//             GetCutOffSolution(sol_local,cfsol_local,ip,directions,nrlayers,true);
+//             sol_local = cfsol_local;
+//             directions = 0.0;
+//             if (i>0) directions[0] = -1;
+//             if (j>0) directions[1] = -1;
+//             GetCutOffSolution(sol_local,cfsol_local,ip,directions,nrlayers,true);
+//             znew = 0.0;
+//             // znew.SetSubVector(*Dof2GlobalDof, cfsol_local);
+//             znew.SetSubVector(*Dof2GlobalDof, sol_local);
+//             z+=znew;
+//          }
+//          socketstream zsock(vishost, visport);
+//          PlotSolution(z,zsock,0); cin.get();
+//       }
+//    }
+// }
+
+
+// void DST::GetCutOffSolution(const Vector & sol, Vector & cfsol, 
+//                   int ip, Array<int> directions, int nlayers, bool local) const
+// {
+
+//    int d = directions.Size();
+//    int directx = directions[0]; // 1,0,-1
+//    int directy = directions[1]; // 1,0,-1
+//    int directz;
+//    if (d ==3) directz = directions[2];
+
+//    Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+   
+//    Vector pmin, pmax;
+//    mesh->GetBoundingBox(pmin, pmax);
+//    double h = GetUniformMeshElementSize(povlp->patch_mesh[ip]);
+//    Array2D<double> pmlh(dim,2); pmlh = 0.0;
+   
+//    if (directions[0]==1)
+//    {
+//       pmlh[0][1] = h*nlayers;
+//    }
+//    if (directions[0]==-1)
+//    {
+//       pmlh[0][0] = h*nlayers;
+//    }
+//    if (directions[1]==1)
+//    {
+//       pmlh[1][1] = h*nlayers;
+//    }
+//    if (directions[1]==-1)
+//    {
+//       pmlh[1][0] = h*nlayers;
+//    }
+
+//    CutOffFnCoefficient cf(CutOffFncn, pmin, pmax, pmlh);
+
+//    double * data = sol.GetData();
+
+//    FiniteElementSpace * fes;
+//    if (!local)
+//    {
+//       fes = bf->FESpace();
+//    }
+//    else
+//    {
+//       fes = ovlp_prob->fespaces[ip];
+//    }
+   
+//    int n = fes->GetTrueVSize();
+
+//    GridFunction solgf_re(fes, data);
+//    GridFunction solgf_im(fes, &data[n]);
+
+
+//    GridFunctionCoefficient coeff1_re(&solgf_re);
+//    GridFunctionCoefficient coeff1_im(&solgf_im);
+
+//    ProductCoefficient prod_re(coeff1_re, cf);
+//    ProductCoefficient prod_im(coeff1_im, cf);
+
+//    ComplexGridFunction gf(fes);
+//    gf.ProjectCoefficient(prod_re,prod_im);
+
+//    cfsol.SetSize(sol.Size());
+//    cfsol = gf;
+// }
+
+// void DST::GetChiRes(const Vector & res, Vector & cfres, 
+//                        int ip, Array<int> directions, int nlayers) const
+// {
+//    // int l,k;
+//    int d = directions.Size();
+//    int directx = directions[0]; // 1,0,-1
+//    int directy = directions[1]; // 1,0,-1
+//    int directz;
+//    if (d ==3) directz = directions[2];
+
+//    Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+//    double h = GetUniformMeshElementSize(mesh);
+   
+//    Vector pmin, pmax;
+//    mesh->GetBoundingBox(pmin, pmax);
+
+//    Array2D<double> pmlh(dim,2); pmlh = 0.0;
+   
+//    if (directions[0]==1)
+//    {
+//       pmlh[0][1] = h*nlayers;
+//    }
+//    if (directions[0]==-1)
+//    {
+//       pmlh[0][0] = h*nlayers;
+//    }
+//    if (directions[1]==1)
+//    {
+//       pmlh[1][1] = h*nlayers;
+//    }
+//    if (directions[1]==-1)
+//    {
+//       pmlh[1][0] = h*nlayers;
+//    }
+
+//    CutOffFnCoefficient cf(ChiFncn, pmin, pmax, pmlh);
+
+//    double * data = res.GetData();
+
+//    FiniteElementSpace * fespace;
+//    fespace = ovlp_prob->fespaces[ip];
+   
+//    int n = fespace->GetTrueVSize();
+
+//    GridFunction solgf_re(fespace, data);
+//    GridFunction solgf_im(fespace, &data[n]);
+
+//    GridFunctionCoefficient coeff1_re(&solgf_re);
+//    GridFunctionCoefficient coeff1_im(&solgf_im);
+
+//    ProductCoefficient prod_re(coeff1_re, cf);
+//    ProductCoefficient prod_im(coeff1_im, cf);
+
+//    ComplexGridFunction gf(fespace);
+//    gf.ProjectCoefficient(prod_re,prod_im);
+
+//    cfres.SetSize(res.Size());
+//    cfres = gf;
+// }
+
+
+// DST::~DST()
+// {
+// }
+
+
+// void DST::Getijk(int ip, int & i, int & j, int & k) const
+// {
+//    k = ip/(nxyz[0]*nxyz[1]);
+//    j = (ip-k*nxyz[0]*nxyz[1])/nxyz[0];
+//    i = (ip-k*nxyz[0]*nxyz[1])%nxyz[0];
+// }
+
+// int DST::GetPatchId(const Array<int> & ijk) const
+// {
+//    int d=ijk.Size();
+//    int z = (dim==2)? 0 : ijk[2];
+//    return subdomains(ijk[0],ijk[1],z);
+// }
+
+
+// void DST::TransferSources(int sweep, int ip0, Vector & sol0) const
+// {
+//  // Find all neighbors of patch ip0
+//    int nx = nxyz[0];
+//    int ny = nxyz[1];
+//    int i0, j0, k0;
+//    Getijk(ip0, i0,j0,k0);
+//    // cout << "Transfer to : " << endl;
+//    // loop through possible directions
+//    for (int i=-1; i<2; i++)
+//    {
+//       int i1 = i0 + i;
+//       if (i1 <0 || i1>=nx) continue;
+//       for (int j=-1; j<2; j++)
+//       {
+//          if (i==0 && j==0) continue;
+//          int j1 = j0 + j;
+//          if (j1 <0 || j1>=ny) continue;
+//          // cout << "(" << i1 << "," << j1 <<"), ";
+//          // Find ip 1
+//          Array<int> ij1(2); ij1[0] = i1; ij1[1]=j1;
+//          int ip1 = GetPatchId(ij1);
+//          // cout << "ip1 = " << ip1;
+//          // cout << " in the direction of (" << i <<", " <<j <<")" << endl;
+//          Array<int> directions(2);
+//          directions[0] = i;
+//          directions[1] = j;
+//          Vector cfsol0;
+//          GetCutOffSolution(sol0,cfsol0,ip0,directions,nrlayers,true);
+         
+//          // Transfer solution to ip1;
+//          Array<int> * Dof2GlobalDof0 = &ovlp_prob->Dof2GlobalDof[ip0];
+//          Array<int> * Dof2GlobalDof1 = &ovlp_prob->Dof2GlobalDof[ip1];
+
+//          Vector znew(2*bf->FESpace()->GetTrueVSize());
+//          znew = 0.0;
+//          znew.SetSubVector(*Dof2GlobalDof0,sol0);
+
+//          Vector sol1(Dof2GlobalDof1->Size()); sol1 = 0.0;
+//          Vector res1(Dof2GlobalDof1->Size()); res1 = 0.0;
+//          znew.GetSubVector(*Dof2GlobalDof1,sol1);
+//          PmlMat[ip1]->Mult(sol1,res1);
+//          res1 *=-1.0;
+
+//          // remove the source in the pml restrict to the non overlapping subdomain
+
+//          Array<int> direct(2); direct = 0;
+//          if (i1>0) direct[0] = -1; 
+//          if (j1>0) direct[1] = -1;
+//          Vector cfraux(res1.Size()); cfraux = 0.0; 
+//          GetChiRes(res1, cfraux,ip1,direct, nrlayers);
+//          direct = 0;
+//          if (i1+1<nx) direct[0] = 1; 
+//          if (j1+1<ny) direct[1] = 1; 
+//          GetChiRes(cfraux, res1,ip1,direct, nrlayers);
+
+
+//          // Find the minumum sweep number that to transfer the source that 
+//          // satisfies the two rules
+//          for (int l=sweep; l<nsweeps; l++)
+//          {
+//             // Conditions on sweeps
+//             // Rule 1: the transfer source direction has to be similar with 
+//             // the sweep direction
+//             int is = sweeps(l,0); 
+//             int js = sweeps(l,1);
+//             int ddot = is*i + js * j;
+//             // cout << "(i,j) = (" << i <<"," <<j <<")" << endl;
+//             // cout << "(is,js) = (" << is <<"," <<js <<")" << endl;
+//             // cout << "ip0 , ip1 = " << ip0 << ", " << ip1 << endl;
+//             if (ddot <= 0) continue;
+
+//             // Rule 2: The horizontal or vertical transfer source cannot be used
+//             // in a later sweep that with opposite directions
+
+//             if (i==0 || j == 0) // Case of horizontal or vertical transfer source
+//             {
+//                int il = sweeps(l,0);
+//                int jl = sweeps(l,1);
+//                // skip if the two sweeps have opposite direction
+//                if (is == -il && js == -jl) continue;
+//             }
+//             // cout << "Passing ip0 = " << ip0 << " to ip1 = " << ip1 
+//                //   << " to sweep no l = " << l << endl;  
+//             MFEM_VERIFY(f_transf[ip1][l]->Size()==res1.Size(), 
+//                         "Transfer Sources: inconsistent size");
+//             *f_transf[ip1][l]+=res1;
+//             break;
+//          }
+//       }  
+//    }
+// }
+
+
+
+
+// SparseMatrix * DST::GetPmlSystemMatrix(int ip)
+// {
+//    double h = GetUniformMeshElementSize(povlp->patch_mesh[ip]);
+//    Array2D<double> length(dim,2);
+//    length = h*(nrlayers);
+
+//    CartesianPML pml(povlp->patch_mesh[ip], length);
+//    pml.SetOmega(omega);
+
+//    Array <int> ess_tdof_list;
+//    if (povlp->patch_mesh[ip]->bdr_attributes.Size())
+//    {
+//       Array<int> ess_bdr(povlp->patch_mesh[ip]->bdr_attributes.Max());
+//       ess_bdr = 1;
+//       ovlp_prob->fespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+//    }
+
+//    ConstantCoefficient one(1.0);
+//    ConstantCoefficient sigma(-pow(omega, 2));
+//    PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+//    PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+//    PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+//    PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+//    ProductCoefficient c2_re0(sigma, detJ_re);
+//    ProductCoefficient c2_im0(sigma, detJ_im);
+//    ProductCoefficient c2_re(c2_re0, *ws);
+//    ProductCoefficient c2_im(c2_im0, *ws);
+//    SesquilinearForm a(ovlp_prob->fespaces[ip],ComplexOperator::HERMITIAN);
+
+//    a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+//                          new DiffusionIntegrator(c1_im));
+//    a.AddDomainIntegrator(new MassIntegrator(c2_re),
+//                          new MassIntegrator(c2_im));
+//    a.Assemble();
+
+//    OperatorPtr Alocal;
+//    a.FormSystemMatrix(ess_tdof_list,Alocal);
+//    ComplexSparseMatrix * AZ_ext = Alocal.As<ComplexSparseMatrix>();
+//    SparseMatrix * Mat = AZ_ext->GetSystemMatrix();
+//    Mat->Threshold(0.0);
+//    return Mat;
+// }
+
+// void DST::PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const
+// {
+//    FiniteElementSpace * fespace = bf->FESpace();
+//    Mesh * mesh = fespace->GetMesh();
+//    GridFunction gf(fespace);
+//    double * data = sol.GetData();
+//    gf.SetData(data);
+   
+//    string keys;
+//    if (ip == 0) keys = "keys mrRljc\n";
+//    // sol_sock << "solution\n" << *mesh << gf << keys << "valuerange -0.1 0.1 \n"  << flush;
+//    sol_sock << "solution\n" << *mesh << gf << keys << flush;
+// }
+
+
+// // void DST::SetSubMeshesAttributes()
+// // {
+// //    // For each subdomain there are 2 associated meshes. 
+// //    // The one from non-overlapping partitioning and the one from 
+// //    // an overlapping one. We need to mark the elements according to the 
+// //    // following diagram
+// //    //        _______________________________________
+// //    //       |       |                       |       |
+// //    //       |   3   |          4            |   5   |
+// //    //       |_______|_______________________|_______|
+// //    //       |       |                       |       |
+// //    //       |       |                       |       |
+// //    //       |       |                       |       |
+// //    //       |   2   |          9            |   6   |
+// //    //       |       |                       |       |
+// //    //       |       |                       |       |
+// //    //       |       |                       |       |
+// //    //       |_______|_______________________|_______|
+// //    //       |       |                       |       |
+// //    //       |   1   |          8            |   7   |
+// //    //       |_______|_______________________|_______|
+
+// //    for (int ip=0; ip<nrpatch; ip++)
+// //    {
+// //       Mesh * mesh1 = nvlp_prob->fespaces[ip]->GetMesh();
+// //       Vector pmin, pmax;
+// //       mesh1->GetBoundingBox(pmin,pmax);
+// //       Mesh * mesh = ovlp_prob->fespaces[ip]->GetMesh();
+// //       int dim=mesh->Dimension();
+// //       for (int iel=0; iel<mesh->GetNE(); iel++)
+// //       {
+// //          Vector center(dim);
+// //          int geom = mesh->GetElementBaseGeometry(iel);
+// //          ElementTransformation * tr = mesh->GetElementTransformation(iel);
+// //          tr->Transform(Geometries.GetCenter(geom), center);
+// //          int attr = 9;
+// //          if (center[0] < pmin[0])
+// //          {
+// //             if (center[1] < pmin[1])
+// //             {
+// //                attr = 1;
+// //             }
+// //             else if (center[1] > pmax[1])
+// //             {
+// //                attr = 3;
+// //             }
+// //             else
+// //             {
+// //                attr = 2;
+// //             }
+// //          }
+// //          else if (center[0] < pmax[0])
+// //          {
+// //             if (center[1] < pmin[1])
+// //             {
+// //                attr = 8;
+// //             }
+// //             else if (center[1] > pmax[1])
+// //             {
+// //                attr = 4;
+// //             }
+// //          }
+// //          else
+// //          {
+// //             if (center[1] < pmin[1])
+// //             {
+// //                attr = 7;
+// //             }
+// //             else if (center[1] > pmax[1])
+// //             {
+// //                attr = 5;
+// //             }
+// //             else
+// //             {
+// //                attr = 6;
+// //             }
+            
+// //          }
+// //          mesh->SetAttribute(iel,attr);
+// //       }
+// //       mesh->SetAttributes();
+// //    }
+// // }
+
+// // void DST::GetRestrCoeffAttr(const Array<int> & directions, Array<int> & attr) const
+// // {
+// //    attr.SetSize(9); attr = 1;
+// //    // Set the attributes of the restricted coeff
+// //    if (directions[0] == 1)
+// //    {
+// //       if (directions[1] == 0)
+// //       {
+// //          attr[0] = 0;
+// //          attr[1] = 0;
+// //          attr[2] = 0;
+// //       }
+// //       else if (directions[1] == 1)
+// //       {
+// //          attr[0] = 0;
+// //          attr[1] = 0;
+// //          attr[2] = 0;
+// //          attr[7] = 0;
+// //          attr[6] = 0;
+// //       }
+// //       else if (directions[1] == -1)
+// //       {
+// //          attr[0] = 0;
+// //          attr[1] = 0;
+// //          attr[2] = 0;
+// //          attr[3] = 0;
+// //          attr[4] = 0;
+// //       }
+// //    }
+// //    else if (directions[0] == 0)
+// //    {
+// //       if (directions[1] == 1)
+// //       {
+// //          attr[0] = 0;
+// //          attr[7] = 0;
+// //          attr[6] = 0;
+// //       }
+// //       else if (directions[1] == -1)
+// //       {
+// //          attr[2] = 0;
+// //          attr[3] = 0;
+// //          attr[4] = 0;
+// //       }
+// //    }
+// //    if (directions[0] == -1)
+// //    {
+// //       if (directions[1] == 0)
+// //       {
+// //          attr[4] = 0;
+// //          attr[5] = 0;
+// //          attr[6] = 0;
+// //       }
+// //       else if (directions[1] == 1)
+// //       {
+// //          attr[0] = 0;
+// //          attr[7] = 0;
+// //          attr[6] = 0;
+// //          attr[5] = 0;
+// //          attr[4] = 0;
+// //       }
+// //       else if (directions[1] == -1)
+// //       {
+// //          attr[2] = 0;
+// //          attr[3] = 0;
+// //          attr[4] = 0;
+// //          attr[5] = 0;
+// //          attr[6] = 0;
+// //       }
+// //    }
+// // }
+
+// // double DST::GetSolOvlpNorm(const Vector & sol, 
+// //                      const Array<int> & directions, int ip) const
+// // {
+// //    FiniteElementSpace * fes = ovlp_prob->fespaces[ip];
+// //    Mesh * mesh = fes->GetMesh();
+   
+// //    int n = fes->GetTrueVSize();
+// //    GridFunction gf_re(fes);
+// //    GridFunction gf_im(fes);
+// //    double * data = sol.GetData();
+// //    gf_re.SetData(data);
+// //    gf_im.SetData(&data[0]);
+
+// //    Array<int> elems(mesh->GetNE()); elems = 0;
+// //    // Find the elements in the ovlp
+// //    Array<int> attr;
+// //    Array<int> direct(2); 
+// //    direct[0] = -directions[0];
+// //    direct[1] = -directions[1];
+// //    GetRestrCoeffAttr(direct,attr);
+// //    for (int iel = 0; iel<mesh->GetNE(); iel++)
+// //    {
+      
+// //       int i = mesh->GetAttribute(iel);
+// //       if (attr[i-1] == 0)
+// //       {
+// //          // elems.Append(iel);
+// //          elems[iel] = 1;
+// //       }
+// //    }
+
+// //    ConstantCoefficient zero(0.0);
+// //    GridFunction error(fes);
+
+// //    gf_re.ComputeElementL2Errors(zero, error);
+
+// //    double norm = 0.0;
+// //    for (int iel = 0; iel<mesh->GetNE(); iel++)
+// //    {
+// //       if (elems[iel] == 1) norm+=error[iel];
+// //    }
+
+// //    return norm;
+// // }

examples/maxwell-solver/DST2D/back/DSTnew.hpp

@@ -0,0 +1,52 @@
+// #pragma once
+// #include "Utilities.hpp"
+// #include "PML.hpp"
+// using namespace std;
+// using namespace mfem;
+
+// class DST : public Solver//
+// {
+// private:
+//    int nrpatch;
+//    int dim;
+//    SesquilinearForm *bf=nullptr;
+//    MeshPartition * povlp=nullptr;
+//    double omega = 0.5;
+//    Coefficient * ws;
+//    int nrlayers;
+//    int nxyz[3];
+//    const Operator * A=nullptr;
+//    DofMap * ovlp_prob = nullptr;
+//    Array<SparseMatrix *> PmlMat;
+//    Array<KLUSolver *> PmlMatInv;
+//    Array2D<double> Pmllength;
+//    Array3D<int> subdomains;
+//    mutable Array<Vector *> f_orig;
+//    int ntransf_directions;
+//    int nsweeps;
+//    Array2D<int> sweeps;
+//    Array<int> dirx;
+//    Array<int> diry;
+//    Array<int> dirz;
+//    mutable Array<Array<Vector * >> f_transf;
+//    Array<Array<Vector * >> usol;
+
+//    SparseMatrix * GetPmlSystemMatrix(int ip);
+//    void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+//    void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+//                           int ip, Array<int> directions, int nlayers, bool local=false) const;
+//    void GetChiRes(const Vector & res, Vector & cfres,
+//                int ip, Array<int> directions, int nlayers) const;                             
+//    void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+//    int GetPatchId(const Array<int> & ijk) const;
+//    void Getijk(int ip, int & i, int & j, int & k ) const;
+//    int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+// public:
+//    DST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+//        double omega_, Coefficient * ws_, int nrlayers_);
+//    virtual void SetOperator(const Operator &op) {A = &op;}
+//    virtual void Mult(const Vector &r, Vector &z) const;
+//    virtual ~DST();
+// };
+
+

examples/maxwell-solver/DST2D/back/DiagST.hpp

@@ -0,0 +1,122 @@
+// // #pragma once
+// // #include "Utilities.hpp"
+// // #include "PML.hpp"
+// // using namespace std;
+// // using namespace mfem;
+
+// // class DiagST : public Solver//
+// // {
+// // private:
+// //    int nrpatch;
+// //    int dim;
+// //    SesquilinearForm *bf=nullptr;
+// //    MeshPartition * povlp=nullptr;
+// //    double omega = 0.5;
+// //    Coefficient * ws;
+// //    int nrlayers;
+// //    int ovlpnrlayers;
+// //    int nxyz[3];
+// //    const Operator * A=nullptr;
+// //    Vector B;
+// //    DofMap * ovlp_prob = nullptr;
+// //    Array<SparseMatrix *> PmlMat;
+// //    Array<KLUSolver *> PmlMatInv;
+// //    Array2D<double> Pmllength;
+// //    Array3D<int> subdomains;
+// //    mutable Array<Vector *> f_orig;
+// //    int ntransf_directions;
+// //    int nsweeps;
+// //    Array2D<int> sweeps;
+// //    Array<int> dirx;
+// //    Array<int> diry;
+// //    Array<int> dirz;
+// //    mutable Array<Array<Vector * >> f_transf;
+// //    Array<Array<Vector * >> usol;
+
+// //    SparseMatrix * GetPmlSystemMatrix(int ip);
+// //    void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+// //    // void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+// //                         //   int ip, Array<int> directions, bool local=false) const;
+// //    void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+// //                         int ip, Array<int> directions, int ovlpnlayers, bool local=false) const;                       
+// //    void GetChiRes(const Vector & res, Vector & cfres,
+// //                   int ip, Array<int> directions, int nlayers) const;                       
+// //    void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+// //    int  GetDirectionId(const Array<int> & ijk) const;
+// //    void GetDirectionijk(int id, Array<int> & ijk) const;
+// //    void ConstructDirectionsMap();
+// //    int GetPatchId(const Array<int> & ijk) const;
+// //    void Getijk(int ip, int & i, int & j, int & k ) const;
+// //    int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+// // public:
+// //    DiagST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+// //        double omega_, Coefficient * ws_, int nrlayers_);
+// //    void SetLoadVector(Vector load) { B = load;}
+// //    virtual void SetOperator(const Operator &op) {A = &op;}
+// //    virtual void Mult(const Vector &r, Vector &z) const;
+// //    virtual ~DiagST();
+// // };
+
+// #pragma once
+// #include "Utilities.hpp"
+// #include "PML.hpp"
+// using namespace std;
+// using namespace mfem;
+
+// class DiagST : public Solver//
+// {
+// private:
+//    int nrpatch;
+//    int dim;
+//    SesquilinearForm *bf=nullptr;
+//    MeshPartition * povlp=nullptr;
+//    MeshPartition * novlp=nullptr;
+//    double omega = 0.5;
+//    Coefficient * ws;
+//    int nrlayers;
+//    int ovlpnrlayers;
+//    int nxyz[3];
+//    const Operator * A=nullptr;
+//    Vector B;
+//    DofMap * ovlp_prob = nullptr;
+//    DofMap * nvlp_prob = nullptr;
+//    Array<SparseMatrix *> PmlMat;
+//    Array<KLUSolver *> PmlMatInv;
+//    Array2D<double> Pmllength;
+//    Array3D<int> subdomains;
+//    mutable Array<Vector *> f_orig;
+//    int ntransf_directions;
+//    int nsweeps;
+//    Array2D<int> sweeps;
+//    Array<int> dirx;
+//    Array<int> diry;
+//    Array<int> dirz;
+//    mutable Array<Array<Vector * >> f_transf;
+//    Array<Vector * > usol;
+
+//    SparseMatrix * GetPmlSystemMatrix(int ip);
+//    void PlotSolution(Vector & sol, socketstream & sol_sock, int ip, bool localdomain = false, bool pmldomain = false) const;
+//    // void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+//                         //   int ip, Array<int> directions, bool local=false) const;
+//    void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+//                         int ip, Array<int> directions, int ovlpnlayers, bool local=false) const;                       
+//    void GetChiRes(const Vector & res, Vector & cfres,
+//                   int ip, Array<int> directions, int nlayers) const;                       
+//    void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+//    int  GetDirectionId(const Array<int> & ijk) const;
+//    void GetDirectionijk(int id, Array<int> & ijk) const;
+//    void ConstructDirectionsMap();
+//    int GetPatchId(const Array<int> & ijk) const;
+//    void Getijk(int ip, int & i, int & j, int & k ) const;
+//    int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+// public:
+//    DiagST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+//        double omega_, Coefficient * ws_, int nrlayers_);
+//    void SetLoadVector(Vector load) { B = load;}
+//    virtual void SetOperator(const Operator &op) {A = &op;}
+//    virtual void Mult(const Vector &r, Vector &z) const;
+//    virtual ~DiagST();
+// };
+
+
+

examples/maxwell-solver/DST2D/back/DiagST1.hpp

@@ -0,0 +1,120 @@
+// // #pragma once
+// // #include "Utilities.hpp"
+// // #include "PML.hpp"
+// // using namespace std;
+// // using namespace mfem;
+
+// // class DiagST : public Solver//
+// // {
+// // private:
+// //    int nrpatch;
+// //    int dim;
+// //    SesquilinearForm *bf=nullptr;
+// //    MeshPartition * povlp=nullptr;
+// //    double omega = 0.5;
+// //    Coefficient * ws;
+// //    int nrlayers;
+// //    int ovlpnrlayers;
+// //    int nxyz[3];
+// //    const Operator * A=nullptr;
+// //    Vector B;
+// //    DofMap * ovlp_prob = nullptr;
+// //    Array<SparseMatrix *> PmlMat;
+// //    Array<KLUSolver *> PmlMatInv;
+// //    Array2D<double> Pmllength;
+// //    Array3D<int> subdomains;
+// //    mutable Array<Vector *> f_orig;
+// //    int ntransf_directions;
+// //    int nsweeps;
+// //    Array2D<int> sweeps;
+// //    Array<int> dirx;
+// //    Array<int> diry;
+// //    Array<int> dirz;
+// //    mutable Array<Array<Vector * >> f_transf;
+// //    Array<Array<Vector * >> usol;
+
+// //    SparseMatrix * GetPmlSystemMatrix(int ip);
+// //    void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+// //    // void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+// //                         //   int ip, Array<int> directions, bool local=false) const;
+// //    void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+// //                         int ip, Array<int> directions, int ovlpnlayers, bool local=false) const;                       
+// //    void GetChiRes(const Vector & res, Vector & cfres,
+// //                   int ip, Array<int> directions, int nlayers) const;                       
+// //    void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+// //    int  GetDirectionId(const Array<int> & ijk) const;
+// //    void GetDirectionijk(int id, Array<int> & ijk) const;
+// //    void ConstructDirectionsMap();
+// //    int GetPatchId(const Array<int> & ijk) const;
+// //    void Getijk(int ip, int & i, int & j, int & k ) const;
+// //    int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+// // public:
+// //    DiagST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+// //        double omega_, Coefficient * ws_, int nrlayers_);
+// //    void SetLoadVector(Vector load) { B = load;}
+// //    virtual void SetOperator(const Operator &op) {A = &op;}
+// //    virtual void Mult(const Vector &r, Vector &z) const;
+// //    virtual ~DiagST();
+// // };
+
+// #pragma once
+// #include "Utilities.hpp"
+// #include "PML.hpp"
+// using namespace std;
+// using namespace mfem;
+
+// class DiagST : public Solver//
+// {
+// private:
+//    int nrpatch;
+//    int dim;
+//    SesquilinearForm *bf=nullptr;
+//    MeshPartition * povlp=nullptr;
+//    double omega = 0.5;
+//    Coefficient * ws;
+//    int nrlayers;
+//    int ovlpnrlayers;
+//    int nxyz[3];
+//    const Operator * A=nullptr;
+//    Vector B;
+//    DofMap * ovlp_prob = nullptr;
+//    Array<SparseMatrix *> PmlMat;
+//    Array<KLUSolver *> PmlMatInv;
+//    Array2D<double> Pmllength;
+//    Array3D<int> subdomains;
+//    mutable Array<Vector *> f_orig;
+//    int ntransf_directions;
+//    int nsweeps;
+//    Array2D<int> sweeps;
+//    Array<int> dirx;
+//    Array<int> diry;
+//    Array<int> dirz;
+//    mutable Array<Array<Vector * >> f_transf;
+//    Array<Array<Vector * >> usol;
+
+//    SparseMatrix * GetPmlSystemMatrix(int ip);
+//    void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+//    // void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+//                         //   int ip, Array<int> directions, bool local=false) const;
+//    void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+//                         int ip, Array<int> directions, int ovlpnlayers, bool local=false) const;                       
+//    void GetChiRes(const Vector & res, Vector & cfres,
+//                   int ip, Array<int> directions, int nlayers) const;                       
+//    void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+//    int  GetDirectionId(const Array<int> & ijk) const;
+//    void GetDirectionijk(int id, Array<int> & ijk) const;
+//    void ConstructDirectionsMap();
+//    int GetPatchId(const Array<int> & ijk) const;
+//    void Getijk(int ip, int & i, int & j, int & k ) const;
+//    int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+// public:
+//    DiagST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+//        double omega_, Coefficient * ws_, int nrlayers_);
+//    void SetLoadVector(Vector load) { B = load;}
+//    virtual void SetOperator(const Operator &op) {A = &op;}
+//    virtual void Mult(const Vector &r, Vector &z) const;
+//    virtual ~DiagST();
+// };
+
+
+

examples/maxwell-solver/DST2D/back/DiagST2.cpp

@@ -0,0 +1,696 @@
+// //Diagonal Source Transfer Preconditioner
+
+// #include "DiagST.hpp"
+
+// DiagST::DiagST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+//          double omega_, Coefficient * ws_,  int nrlayers_)
+//    : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), 
+//      bf(bf_), Pmllength(Pmllength_), omega(omega_), ws(ws_), nrlayers(nrlayers_)
+// {
+//    Mesh * mesh = bf->FESpace()->GetMesh();
+//    dim = mesh->Dimension();
+
+//    // ----------------- Step 1 --------------------
+//    // Introduce 2 layered partitios of the domain 
+//    // 
+//    int partition_kind;
+
+//    // 1. Ovelapping partition with overlap = 2h 
+//    partition_kind = 2; // Non Overlapping partition 
+//    int nx=3;
+//    int ny=3; 
+//    int nz=1;
+//    povlp = new MeshPartition(mesh, partition_kind,nx,ny,nz);
+//    nxyz[0] = povlp->nxyz[0];
+//    nxyz[1] = povlp->nxyz[1];
+//    nxyz[2] = povlp->nxyz[2];
+//    nrpatch = povlp->nrpatch;
+//    // cout<< "nrpatch = " << nrpatch << endl;
+//    // cout << "nx = " << nx << endl;
+//    // cout << "ny = " << ny << endl;
+//    // cout << "nz = " << nz << endl;
+//    subdomains = povlp->subdomains;
+//    // for (int k = 0; k<nxyz[2]; k++)
+//    // {
+//    //    for (int j = 0; j<nxyz[1]; j++)
+//    //    {
+//    //       for (int i = 0; i<nxyz[0]; i++)
+//    //       {
+//    //          Array<int> ijk(3);
+//    //          ijk[0]=i;
+//    //          ijk[1]=j;
+//    //          ijk[2]=k;
+//    //          // cout << "("<<i<<","<<j<<","<<k<<") = " << povlp->subdomains(i,j,k) << endl;
+//    //          cout << "("<<i<<","<<j<<","<<k<<") = " << GetPatchId(ijk) << endl;
+//    //       }
+//    //    }   
+//    // }
+
+//    // for (int ip = 0; ip<nrpatch; ip++)
+//    // {
+//    //    int i, j, k;
+//    //    Getijk(ip, i,j,k);
+//    //    cout << "ip = " << ip << ": ("<<i<<","<<j<<","<<k<<")"<< endl;
+//    // }
+
+   
+
+//    //
+//    // ----------------- Step 1a -------------------
+//    // Save the partition for visualization
+//    // SaveMeshPartition(povlp->patch_mesh, "output/mesh_ovlp.", "output/sol_ovlp.");
+
+//    // // // ------------------Step 2 --------------------
+//    // // // Construct the dof maps from subdomains to global (for the extended and not)
+//    ovlp_prob  = new DofMap(bf,povlp,nrlayers); 
+
+//    // ------------------Step 3 --------------------
+//    // Assemble the PML Problem matrices and factor them
+//    PmlMat.SetSize(nrpatch);
+//    PmlMatInv.SetSize(nrpatch);
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       PmlMat[ip] = GetPmlSystemMatrix(ip);
+//       PmlMatInv[ip] = new KLUSolver;
+//       PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
+//    }
+
+//    // Set up src arrays size
+//    f_orig.SetSize(nrpatch);
+//    f_transf.SetSize(nrpatch);
+
+
+//    // Construct a simple map used for directions of transfer
+//    ConstructDirectionsMap();
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       int n = 2*ovlp_prob->fespaces[ip]->GetTrueVSize(); // (x 2 for complex ) 
+//       f_orig[ip] = new Vector(n); *f_orig[ip] = 0.0;
+//       f_transf[ip].SetSize(ntransf_directions);
+//       for (int i=0;i<ntransf_directions; i++)
+//       {
+//          f_transf[ip][i] = new Vector(n); *f_transf[ip][i] = 0.0;
+//       }
+//    }
+// }
+
+// SparseMatrix * DiagST::GetPmlSystemMatrix(int ip)
+// {
+//    double h = GetUniformMeshElementSize(ovlp_prob->PmlMeshes[ip]);
+//    Array2D<double> length(dim,2);
+//    length = h*(nrlayers);
+
+//    CartesianPML pml(ovlp_prob->PmlMeshes[ip], length);
+//    pml.SetOmega(omega);
+
+//    Array <int> ess_tdof_list;
+//    if (ovlp_prob->PmlMeshes[ip]->bdr_attributes.Size())
+//    {
+//       Array<int> ess_bdr(ovlp_prob->PmlMeshes[ip]->bdr_attributes.Max());
+//       ess_bdr = 1;
+//       ovlp_prob->PmlFespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+//    }
+
+//    ConstantCoefficient one(1.0);
+//    ConstantCoefficient sigma(-pow(omega, 2));
+
+//    PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+//    PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+//    PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+//    PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+//    ProductCoefficient c2_re0(sigma, detJ_re);
+//    ProductCoefficient c2_im0(sigma, detJ_im);
+
+//    ProductCoefficient c2_re(c2_re0, *ws);
+//    ProductCoefficient c2_im(c2_im0, *ws);
+
+//    SesquilinearForm a(ovlp_prob->PmlFespaces[ip],ComplexOperator::HERMITIAN);
+
+//    a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+//                          new DiffusionIntegrator(c1_im));
+//    a.AddDomainIntegrator(new MassIntegrator(c2_re),
+//                          new MassIntegrator(c2_im));
+//    a.Assemble();
+
+//    OperatorPtr Alocal;
+//    a.FormSystemMatrix(ess_tdof_list,Alocal);
+//    ComplexSparseMatrix * AZ_ext = Alocal.As<ComplexSparseMatrix>();
+//    SparseMatrix * Mat = AZ_ext->GetSystemMatrix();
+//    Mat->Threshold(0.0);
+//    return Mat;
+// }
+
+
+// void DiagST::Mult(const Vector &r, Vector &z) const
+// {
+//    // Step 0
+//    // Restrict original sources to the patches
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       *f_orig[ip] = 0.0;
+//       for (int i=0;i<ntransf_directions; i++)
+//       {
+//          *f_transf[ip][i] = 0.0;
+//       }
+//    }
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+//       r.GetSubVector(*Dof2GlobalDof,*f_orig[ip]);
+//    }
+
+//    char vishost[] = "localhost";
+//    int  visport   = 19916;
+//    z = 0.0; 
+//    Vector rnew(r);
+//    Vector znew(z);
+//    znew = 0.0;
+   
+//    // in 2D there are a total of 4 sweeps
+//    // with nx + ny - 1 serial steps each
+//    // --------------------------------------------
+//    //       Sweep in the direction (1,1)
+//    // --------------------------------------------
+//    int nx = nxyz[0];
+//    int ny = nxyz[1];
+
+//    int nsteps = nx + ny - 1;
+//    // loop through the steps
+//    Array<int> sweep_direction(2); sweep_direction = 1;
+//    for (int s = 0; s<nsteps; s++)
+//    {
+//       // the patches involved are the ones such that
+//       // i+j = s
+//       // cout << "Step no: " << s << endl;
+//       for (int i=0;i<nx; i++)
+//       {
+//          int j = s-i;
+//          if (j<0 || j>=ny) continue;
+//          // cout << "Patch no: (" << i <<"," << j << ")" << endl; 
+
+//          // find patch id
+//          Array<int> ij(2); ij[0] = i; ij[1]=j;
+//          int ip = GetPatchId(ij);
+//          // cout << "ip = " << ip << endl;
+
+//          // Solve the PML problem in patch ip with all sources
+//          // Original and all transfered (maybe some of them)
+//          Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+//          Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip];
+//          int ndofs = Dof2GlobalDof->Size();
+//          Vector sol_local(ndofs);
+//          Vector res_local(ndofs);
+//          res_local = *f_orig[ip];
+
+//          // RULE 3.1 (form Leng & Ju paper)
+//          for (int nd=0; nd<ntransf_directions; nd++)
+//          {
+//             // only the transfer sourcers in the similar direction
+//             // of the sweep should be used
+//             Array<int> ijk(2);
+//             GetDirectionijk(nd,ijk);
+//             ijk[0]*=-1; ijk[1]*=-1;
+//             if (sweep_direction[0]*ijk[0] + sweep_direction[1]*ijk[1] > 0)
+//             {
+
+//              // INSTEAD OF MULTIPLE COPIES FOR EACH DIRECTION 
+//              // USE MULTIPLE COPIES FOR EACH SWEEP FOR EACH SUBDOMAIN
+//              // i.e, each subdomain will have 4 different transfer sources 
+//              // which you accumulate as you go.     
+
+
+//                res_local += *f_transf[ip][nd];
+//             }
+//          }
+//          // Extend by zero to the PML mesh
+//          int nrdof_ext = PmlMat[ip]->Height();
+      
+//          Vector res_ext(nrdof_ext); res_ext = 0.0;
+//          Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+
+//          res_ext.SetSubVector(*Dof2PmlDof,res_local);
+//          PmlMatInv[ip]->Mult(res_ext, sol_ext);
+
+//          // Multiply with the cutoff functions, find the new sources and 
+//          // and propagate to all neighboring subdomains 
+//          // (possible 8 in 2D, 26 in 3D)
+//          TransferSources(ip, sol_ext);
+//          Vector cfsol_ext(sol_ext.Size());
+//          // cut off the ip solution to all possible directions
+//          Array<int>directions(2); directions = 0; 
+//          if (i+1<nx) directions[0] = 1;
+//          if (j+1<ny) directions[1] = 1;
+//          GetCutOffSolution(sol_ext,cfsol_ext,ip,directions,true);
+//          // directions = 0;
+//          // if (i-1>=0) directions[0] = -1;
+//          // if (j-1>=0) directions[1] = -1;
+//          // sol_ext = cfsol_ext;
+//          // GetCutOffSolution(sol_ext,cfsol_ext,ip,directions,true);
+//          cfsol_ext.GetSubVector(*Dof2PmlDof, sol_local);
+//          znew = 0.0;
+//          znew.SetSubVector(*Dof2GlobalDof, sol_local);
+//          z+=znew;
+//       }
+//       // socketstream zsock(vishost, visport);
+//       // PlotSolution(z,zsock,0);
+//       // cin.get();
+//    }
+   
+
+// }
+
+// void DiagST::PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const
+// {
+//    FiniteElementSpace * fespace = bf->FESpace();
+//    Mesh * mesh = fespace->GetMesh();
+//    GridFunction gf(fespace);
+//    double * data = sol.GetData();
+//    // gf.SetData(&data[fespace->GetTrueVSize()]);
+//    gf.SetData(data);
+   
+//    string keys;
+//    if (ip == 0) keys = "keys mrRljc\n";
+//    sol_sock << "solution\n" << *mesh << gf << keys << "valuerange -0.1 0.1 \n"  << flush;
+// }
+
+// void DiagST::GetCutOffSolution(const Vector & sol, Vector & cfsol, 
+//                                int ip0, Array<int> directions, bool local) const
+// {
+//    // int l,k;
+//    int d = directions.Size();
+//    int directx = directions[0]; // 1,0,-1
+//    int directy = directions[1]; // 1,0,-1
+//    int directz;
+//    if (d ==3) directz = directions[2];
+
+//    // cout << "ip0 = " << ip0 << endl; 
+
+//    int i0, j0, k0;
+//    Getijk(ip0,i0, j0, k0);
+//    // cout << "(i0,j0) = " << "(" <<i0 <<","<<j0<<")" << endl;
+
+//    // 2D for now...
+//    // Find the id of the neighboring patch
+//    int i1 = i0 + directx;
+//    int j1 = j0 + directy;
+//    MFEM_VERIFY(i1 < nxyz[0] && i1>=0, "GetCutOffSolution: i1 out of bounds");
+//    MFEM_VERIFY(j1 < nxyz[1] && j1>=0, "GetCutOffSolution: j1 out of bounds");
+   
+//    Array<int> ijk(d);
+//    ijk[0] = i1;
+//    ijk[1] = j1;
+//    int ip1 = GetPatchId(ijk);
+
+//    // cout << "ip1 = " << ip1 << endl;
+//    // cout << "(i1,j1) = " << "(" << i1 <<","<<j1<<")" << endl;
+
+//    Mesh * mesh0 = ovlp_prob->fespaces[ip0]->GetMesh();
+//    Mesh * mesh1 = ovlp_prob->fespaces[ip1]->GetMesh();
+   
+//    Vector pmin0, pmax0;
+//    Vector pmin1, pmax1;
+//    mesh0->GetBoundingBox(pmin0, pmax0);
+//    mesh1->GetBoundingBox(pmin1, pmax1);
+
+//    Array2D<double> h(dim,2); h = 0.0;
+   
+//    if (directions[0]==1)
+//    {
+//       h[0][1] = pmax0[0] - pmin1[0];
+//    }
+//    if (directions[0]==-1)
+//    {
+//       h[0][0] = pmax1[0] - pmin0[0];
+//    }
+//    if (directions[1]==1)
+//    {
+//       h[1][1] = pmax0[1] - pmin1[1];
+//    }
+//    if (directions[1]==-1)
+//    {
+//       h[1][0] = pmax1[1] - pmin0[1];
+//    }
+
+//    CutOffFnCoefficient cf(CutOffFncn, pmin0, pmax0, h);
+
+//    double * data = sol.GetData();
+
+//    FiniteElementSpace * fespace;
+//    if (!local)
+//    {
+//       fespace = bf->FESpace();
+//    }
+//    else
+//    {
+//       fespace = ovlp_prob->PmlFespaces[ip0];
+//    }
+   
+//    int n = fespace->GetTrueVSize();
+//    // GridFunction cutF(fespace);
+//    // cutF.ProjectCoefficient(cf);
+//    // char vishost[] = "localhost";
+//    // int  visport   = 19916;
+
+//    // socketstream sub_sock1(vishost, visport);
+//    // sub_sock1 << "solution\n" << *fespace->GetMesh() << cutF << flush;
+//    // cin.get();
+
+
+//    GridFunction solgf_re(fespace, data);
+//    GridFunction solgf_im(fespace, &data[n]);
+
+//    // socketstream sub_sock(vishost, visport);
+//    // sub_sock << "solution\n" << *fespace->GetMesh() << solgf_re << flush;
+//    // cin.get();
+
+
+//    GridFunctionCoefficient coeff1_re(&solgf_re);
+//    GridFunctionCoefficient coeff1_im(&solgf_im);
+
+//    ProductCoefficient prod_re(coeff1_re, cf);
+//    ProductCoefficient prod_im(coeff1_im, cf);
+
+//    ComplexGridFunction gf(fespace);
+//    gf.ProjectCoefficient(prod_re,prod_im);
+
+//    cfsol.SetSize(sol.Size());
+//    cfsol = gf;
+//    // socketstream sub_sock2(vishost, visport);
+//    // sub_sock2 << "solution\n" << *fespace->GetMesh() << gf.real() << flush;
+//    // cin.get();
+// }
+
+// DiagST::~DiagST()
+// {
+//    for (int ip = 0; ip<nrpatch; ++ip)
+//    {
+//       delete PmlMatInv[ip];
+//       delete PmlMat[ip];
+//    }
+//    PmlMat.DeleteAll();
+//    PmlMatInv.DeleteAll();
+//    for (int ip=0; ip<nrpatch; ip++)
+//    {
+//       delete f_orig[ip];
+//       for (int i=0;i<ntransf_directions; i++)
+//       {
+//          delete f_transf[ip][i];
+//       }
+//    }
+// }
+
+// void DiagST::Getijk(int ip, int & i, int & j, int & k) const
+// {
+//    k = ip/(nxyz[0]*nxyz[1]);
+//    j = (ip-k*nxyz[0]*nxyz[1])/nxyz[0];
+//    i = (ip-k*nxyz[0]*nxyz[1])%nxyz[0];
+// }
+
+// int DiagST::GetPatchId(const Array<int> & ijk) const
+// {
+//    int d=ijk.Size();
+//    if (d==2)
+//    {
+//       return subdomains(ijk[0],ijk[1],0);
+//    }
+//    else
+//    {
+//       return subdomains(ijk[0],ijk[1],ijk[2]);
+//    }
+// }
+
+// int DiagST::SourceTransfer(const Vector & Psi0, Array<int> direction, int ip0, Vector & Psi1) const
+// {
+//    // For now 2D problems only
+//    // Directions
+//    // direction (1,1)
+//    int i0,j0,k0;
+//    Getijk(ip0,i0,j0,k0);
+
+//    int i1 = i0+direction[0];   
+//    int j1 = j0+direction[1];   
+//    Array<int> ij(2); ij[0]=i1; ij[1]=j1;
+//    int ip1 = GetPatchId(ij);
+
+//    MFEM_VERIFY(i1 < nxyz[0] && i1>=0, "SourceTransfer: i1 out of bounds");
+//    MFEM_VERIFY(j1 < nxyz[1] && j1>=0, "SourceTransfer: j1 out of bounds");
+
+//    Array<int> * Dof2GlobalDof0 = &ovlp_prob->Dof2GlobalDof[ip0];
+//    Array<int> * Dof2GlobalDof1 = &ovlp_prob->Dof2GlobalDof[ip1];
+//    Psi1.SetSize(Dof2GlobalDof1->Size()); Psi1=0.0;
+//    Vector r(2*bf->FESpace()->GetTrueVSize());
+//    r = 0.0;
+//    r.SetSubVector(*Dof2GlobalDof0,Psi0);
+//    r.GetSubVector(*Dof2GlobalDof1,Psi1);
+//    return ip1;
+// }
+
+// void DiagST::ConstructDirectionsMap()
+// {
+//    // total of 8 possible directions of transfer (2D)
+//    // form left            ( 1 ,  0)
+//    // form left-above      ( 1 , -1)
+//    // form left-below      ( 1 ,  1)
+//    // form right           (-1 ,  0)
+//    // form right-below     (-1 ,  1)
+//    // form right-above     (-1 , -1)
+//    // form above           ( 0 , -1)
+//    // form below           ( 0 ,  1)
+//    ntransf_directions = pow(3,dim);
+
+//    dirx.SetSize(ntransf_directions);
+//    diry.SetSize(ntransf_directions);
+//    int n=3;
+//    Array<int> ijk(dim);
+//    if (dim==2)
+//    {
+//       for (int i=-1; i<=1; i++) // directions x
+//       {
+//          for (int j=-1; j<=1; j++) // directions y
+//          {
+//             ijk[0]=i;
+//             ijk[1]=j;
+//             int k=GetDirectionId(ijk);
+//             dirx[k]=i;
+//             diry[k]=j;
+//          }
+//       }
+//    }
+//    else if (dim==3)
+//    {
+//       dirz.SetSize(ntransf_directions);
+//       for (int i=-1; i<=1; i++) // directions x
+//       {
+//          for (int j=-1; j<=1; j++) // directions y
+//          {
+//             for (int k=-1; k<=1; k++) // directions zß
+//             {
+//                ijk[0]=i;
+//                ijk[1]=j;
+//                ijk[2]=k;
+//                int l=GetDirectionId(ijk);
+//                dirx[l]=i;
+//                diry[l]=j;
+//                dirz[l]=k;
+//             }
+//          }
+//       }
+//    }
+
+//    // cout << "dirx = " << endl;
+//    // dirx.Print(cout,ntransf_directions);
+//    // cout << "diry = " << endl;
+//    // diry.Print(cout,ntransf_directions);
+
+//    if (dim==2)
+//    {
+//       for (int id=0; id<9; id++)
+//       {
+//          GetDirectionijk(id,ijk);
+//          // cout << "for id = " << id << ": (" <<ijk[0] << ", " << ijk[1] << ")" << endl;
+//       }
+//    }
+//    else
+//    {
+//       cout << "dirz = " << endl;
+//       dirz.Print(cout,ntransf_directions);
+//       for (int id=0; id<27; id++)
+//       {
+//          GetDirectionijk(id,ijk);
+//          // cout << "for id = " << id << ": (" <<ijk[0] << ", " <<ijk[1] << ", " << ijk[2] << ")" << endl;
+//       }
+//    }
+// }
+
+// int DiagST::GetDirectionId(const Array<int> & ijk) const
+// {
+//    int d = ijk.Size();
+//    int n=3;
+//    if (d==2)
+//    {
+//       return (ijk[0]+1)*n+(ijk[1]+1);
+//    }
+//    else
+//    {
+//       return (ijk[0]+1)*n*n+(ijk[1]+1)*n+ijk[2]+1;
+//    }
+// }
+
+// void DiagST::GetDirectionijk(int id, Array<int> & ijk) const
+// {
+//    int d = ijk.Size();
+//    int n=3;
+//    if (d==2)
+//    {
+//       ijk[0]=id/n - 1;
+//       ijk[1]=id%n - 1;
+//    }
+//    else
+//    {
+//       ijk[0]=id/(n*n)-1;
+//       ijk[1]=(id-(ijk[0]+1)*n*n)/n - 1;
+//       ijk[2]=(id-(ijk[0]+1)*n*n)%n - 1;
+//    }
+//    // cout << "ijk = " ; ijk.Print();
+// }
+
+
+
+
+// void DiagST::TransferSources(int ip0, Vector & sol_ext) const
+// {
+//    // Find all neighbors of patch ip
+//    int nx = nxyz[0];
+//    int ny = nxyz[1];
+//    int i0, j0, k0;
+//    Getijk(ip0, i0,j0,k0);
+//    // cout << "Transfer to : " << endl;
+//    // loop through possible directions
+//    for (int i=-1; i<2; i++)
+//    {
+//       int i1 = i0 + i;
+//       if (i1 <0 || i1>=nx) continue;
+//       for (int j=-1; j<2; j++)
+//       {
+//          int j1 = j0 + j;
+//          if (j1 <0 || j1>=ny) continue;
+//          // cout << "(" << i1 << "," << j1 <<"), ";
+//          // Find ip 1
+//          Array<int> ij1(2); ij1[0] = i1; ij1[1]=j1;
+//          int ip1 = GetPatchId(ij1);
+//          // cout << "ip1 = " << ip1;
+//          // cout << " in the direction of (" << i <<", " <<j <<")" << endl;
+//          Array<int> directions(2);
+//          directions[0] = i;
+//          directions[1] = j;
+//          Vector cfsol_ext;
+//          Vector res_ext(sol_ext.Size());
+//          GetCutOffSolution(sol_ext,cfsol_ext,ip0,directions,true);
+//          // sol_ext = cfsol_ext;
+//          // Calculate source to be transfered
+//          PmlMat[ip0]->Mult(cfsol_ext, res_ext); res_ext*= -1.0;
+//          Array<int> *Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip0];
+//          Vector res_local(Dof2PmlDof->Size()); res_local = 0.0;
+//          res_ext.GetSubVector(*Dof2PmlDof,res_local);
+//          // Find the direction id to store the transfered source
+//          Array<int> dij(2); dij[0] = -i; dij[1] = -j; 
+//          int did = GetDirectionId(dij);
+//          int jp1 = SourceTransfer(res_local,directions,ip0,*f_transf[ip1][did]);
+//          MFEM_VERIFY(ip1 == jp1, "Check SourceTransfer patch id");
+//       }  
+//    }
+// }
+
+
+
+
+
+
+// // void DiagST::GetCutOffSolution(Vector & sol, int ip, int direction, bool local) const
+// // {
+// //    int l,k;
+// //    k=(direction == 1)? ip: ip-1;
+// //    l=(direction == 1)? ip+1: ip;
+
+// //    Mesh * mesh1 = ovlp_prob->fespaces[k]->GetMesh();
+// //    Mesh * mesh2 = ovlp_prob->fespaces[l]->GetMesh();
+   
+// //    Vector pmin1, pmax1;
+// //    Vector pmin2, pmax2;
+// //    mesh1->GetBoundingBox(pmin1, pmax1);
+// //    mesh2->GetBoundingBox(pmin2, pmax2);
+
+// //    Array2D<double> h(dim,2); h = 0.0;
+   
+// //    Vector pmin, pmax;
+// //    if (direction == 1)
+// //    {
+// //       h[0][1] = pmax1[0] - pmin2[0];
+// //       CutOffFnCoefficient cf(CutOffFncn, pmin1, pmax1, h);
+// //       pmin = pmin1;
+// //       pmax = pmax1;
+// //    }
+// //    else if (direction == -1)
+// //    {
+// //       h[0][0] = pmax1[0] - pmin2[0];
+// //       pmin = pmin2;
+// //       pmax = pmax2;
+// //    }
+// //    CutOffFnCoefficient cf(CutOffFncn, pmin, pmax, h);
+
+// //    double * data = sol.GetData();
+
+// //    FiniteElementSpace * fespace;
+// //    if (!local)
+// //    {
+// //       fespace = bf->FESpace();
+// //    }
+// //    else
+// //    {
+// //       if (direction == 1)
+// //       {
+// //          fespace = ovlp_prob->PmlFespaces[k];
+// //       }
+// //       else
+// //       {
+// //          fespace = ovlp_prob->PmlFespaces[l];
+// //       }
+// //    }
+   
+// //    int n = fespace->GetTrueVSize();
+// //    GridFunction cutF(fespace);
+// //    cutF.ProjectCoefficient(cf);
+// //    // char vishost[] = "localhost";
+// //    // int  visport   = 19916;
+
+   
+
+// //    // socketstream sub_sock1(vishost, visport);
+// //    // sub_sock1 << "solution\n" << *fespace->GetMesh() << cutF << flush;
+// //    // cin.get();
+
+
+// //    GridFunction solgf_re(fespace, data);
+
+// //    // socketstream sub_sock(vishost, visport);
+// //    // sub_sock << "solution\n" << *fespace->GetMesh() << solgf_re << flush;
+// //    // cin.get();
+
+// //    GridFunction solgf_im(fespace, &data[n]);
+
+// //    GridFunctionCoefficient coeff1_re(&solgf_re);
+// //    GridFunctionCoefficient coeff1_im(&solgf_im);
+
+// //    ProductCoefficient prod_re(coeff1_re, cf);
+// //    ProductCoefficient prod_im(coeff1_im, cf);
+
+// //    ComplexGridFunction gf(fespace);
+// //    gf.ProjectCoefficient(prod_re,prod_im);
+
+// //    sol = gf;
+// //    // socketstream sub_sock2(vishost, visport);
+// //    // sub_sock2 << "solution\n" << *fespace->GetMesh() << gf.real() << flush;
+// //    // cin.get();
+// // }

examples/maxwell-solver/DST2D/back/DiagST2.hpp

@@ -0,0 +1,53 @@
+// #pragma once
+// #include "Utilities.hpp"
+// #include "PML.hpp"
+// using namespace std;
+// using namespace mfem;
+
+// class DiagST : public Solver//
+// {
+// private:
+//    int nrpatch;
+//    int dim;
+//    SesquilinearForm *bf=nullptr;
+//    MeshPartition * povlp=nullptr;
+//    double omega = 0.5;
+//    Coefficient * ws;
+//    int nrlayers;
+//    int nxyz[3];
+//    const Operator * A=nullptr;
+//    Vector B;
+//    DofMap * ovlp_prob = nullptr;
+//    Array<SparseMatrix *> PmlMat;
+//    Array<KLUSolver *> PmlMatInv;
+//    Array2D<double> Pmllength;
+//    Array3D<int> subdomains;
+//    mutable Array<Vector *> f_orig;
+//    int ntransf_directions;
+//    UniqueIndexGenerator gen;
+//    Array<int> dirx;
+//    Array<int> diry;
+//    Array<int> dirz;
+//    mutable Array<Array<Vector * >> f_transf;
+
+//    SparseMatrix * GetPmlSystemMatrix(int ip);
+//    void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+//    void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+//                           int ip, Array<int> directions, bool local=false) const;
+//    void TransferSources(int ip, Vector & sol_ext) const;
+//    int  GetDirectionId(const Array<int> & ijk) const;
+//    void GetDirectionijk(int id, Array<int> & ijk) const;
+//    void ConstructDirectionsMap();
+//    int GetPatchId(const Array<int> & ijk) const;
+//    void Getijk(int ip, int & i, int & j, int & k ) const;
+//    int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+// public:
+//    DiagST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+//        double omega_, Coefficient * ws_, int nrlayers_);
+//    void SetLoadVector(Vector load) { B = load;}
+//    virtual void SetOperator(const Operator &op) {A = &op;}
+//    virtual void Mult(const Vector &r, Vector &z) const;
+//    virtual ~DiagST();
+// };
+
+

examples/maxwell-solver/DST2D/back/DiagSTnew.hpp

@@ -0,0 +1,120 @@
+// #pragma once
+// #include "Utilities.hpp"
+// #include "PML.hpp"
+// using namespace std;
+// using namespace mfem;
+
+// class DiagST : public Solver//
+// {
+// private:
+//    int nrpatch;
+//    int dim;
+//    SesquilinearForm *bf=nullptr;
+//    MeshPartition * povlp=nullptr;
+//    double omega = 0.5;
+//    Coefficient * ws;
+//    int nrlayers;
+//    int ovlpnrlayers;
+//    int nxyz[3];
+//    const Operator * A=nullptr;
+//    Vector B;
+//    DofMap * ovlp_prob = nullptr;
+//    Array<SparseMatrix *> PmlMat;
+//    Array<KLUSolver *> PmlMatInv;
+//    Array2D<double> Pmllength;
+//    Array3D<int> subdomains;
+//    mutable Array<Vector *> f_orig;
+//    int ntransf_directions;
+//    int nsweeps;
+//    Array2D<int> sweeps;
+//    Array<int> dirx;
+//    Array<int> diry;
+//    Array<int> dirz;
+//    mutable Array<Array<Vector * >> f_transf;
+//    Array<Array<Vector * >> usol;
+
+//    SparseMatrix * GetPmlSystemMatrix(int ip);
+//    void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+//    // void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+//                         //   int ip, Array<int> directions, bool local=false) const;
+//    void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+//                         int ip, Array<int> directions, int ovlpnlayers, bool local=false) const;                       
+//    void GetChiRes(const Vector & res, Vector & cfres,
+//                   int ip, Array<int> directions, int nlayers) const;                       
+//    void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+//    int  GetDirectionId(const Array<int> & ijk) const;
+//    void GetDirectionijk(int id, Array<int> & ijk) const;
+//    void ConstructDirectionsMap();
+//    int GetPatchId(const Array<int> & ijk) const;
+//    void Getijk(int ip, int & i, int & j, int & k ) const;
+//    int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+// public:
+//    DiagST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+//        double omega_, Coefficient * ws_, int nrlayers_);
+//    void SetLoadVector(Vector load) { B = load;}
+//    virtual void SetOperator(const Operator &op) {A = &op;}
+//    virtual void Mult(const Vector &r, Vector &z) const;
+//    virtual ~DiagST();
+// };
+
+#pragma once
+#include "Utilities.hpp"
+#include "PML.hpp"
+using namespace std;
+using namespace mfem;
+
+class DiagST : public Solver//
+{
+private:
+   int nrpatch;
+   int dim;
+   SesquilinearForm *bf=nullptr;
+   MeshPartition * povlp=nullptr;
+   double omega = 0.5;
+   Coefficient * ws;
+   int nrlayers;
+   int ovlpnrlayers;
+   int nxyz[3];
+   const Operator * A=nullptr;
+   Vector B;
+   DofMap * ovlp_prob = nullptr;
+   Array<SparseMatrix *> PmlMat;
+   Array<KLUSolver *> PmlMatInv;
+   Array2D<double> Pmllength;
+   Array3D<int> subdomains;
+   mutable Array<Vector *> f_orig;
+   int ntransf_directions;
+   int nsweeps;
+   Array2D<int> sweeps;
+   Array<int> dirx;
+   Array<int> diry;
+   Array<int> dirz;
+   mutable Array<Array<Vector * >> f_transf;
+   Array<Array<Vector * >> usol;
+
+   SparseMatrix * GetPmlSystemMatrix(int ip);
+   void PlotSolution(Vector & sol, socketstream & sol_sock, int ip, bool localdomain = false, bool pmldomain = false) const;
+   // void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+                        //   int ip, Array<int> directions, bool local=false) const;
+   void GetCutOffSolution(const Vector & sol, Vector & cfsol,
+                        int ip, Array<int> directions, int ovlpnlayers, bool local=false) const;                       
+   void GetChiRes(const Vector & res, Vector & cfres,
+                  int ip, Array<int> directions, int nlayers) const;                       
+   void TransferSources(int sweep, int ip, Vector & sol_ext) const;
+   int  GetDirectionId(const Array<int> & ijk) const;
+   void GetDirectionijk(int id, Array<int> & ijk) const;
+   void ConstructDirectionsMap();
+   int GetPatchId(const Array<int> & ijk) const;
+   void Getijk(int ip, int & i, int & j, int & k ) const;
+   int SourceTransfer(const Vector & Psi0, Array<int> direction, int ip, Vector & Psi1) const;
+public:
+   DiagST(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+       double omega_, Coefficient * ws_, int nrlayers_);
+   void SetLoadVector(Vector load) { B = load;}
+   virtual void SetOperator(const Operator &op) {A = &op;}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~DiagST();
+};
+
+
+

examples/maxwell-solver/DST2D/back/ex1_testL2f.cpp

@@ -0,0 +1,224 @@
+// //                                MFEM Example 1
+// //
+// // Compile with: make ex1
+// //
+
+// #include "mfem.hpp"
+// #include <fstream>
+// #include <iostream>
+
+// using namespace std;
+// using namespace mfem;
+
+// void SetElemAttr(Mesh * mesh);
+// double SolExact(const Vector & x);
+// double ChiExact(const Vector & x);
+// double BumpFncn(const Vector & x);
+
+// int main(int argc, char *argv[])
+// {
+//    // 1. Parse command-line options.
+//    const char *mesh_file = "../data/star.mesh";
+//    int order = 1;
+//    bool visualization = true;
+
+//    OptionsParser args(argc, argv);
+//    args.AddOption(&mesh_file, "-m", "--mesh",
+//                   "Mesh file to use.");
+//    args.AddOption(&order, "-o", "--order",
+//                   "Finite element order (polynomial degree) or -1 for"
+//                   " isoparametric space.");
+//    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+//                   "--no-visualization",
+//                   "Enable or disable GLVis visualization.");
+//    args.Parse();
+//    if (!args.Good())
+//    {
+//       args.PrintUsage(cout);
+//       return 1;
+//    }
+//    args.PrintOptions(cout);
+
+//    Mesh *mesh = new Mesh(mesh_file, 1, 1);
+//    int dim = mesh->Dimension();
+
+//    int ref_levels = 4;
+//    for (int l = 0; l < ref_levels; l++)
+//    {
+//       mesh->UniformRefinement();
+//    }
+
+//    // SetElemAttr(mesh);
+
+//    // Array<int> attr;
+//    // if (mesh->attributes.Size())
+//    // {
+//    //    attr.SetSize(mesh->attributes.Max());
+//    //    attr = 0;   attr[1] = 1;
+//    // }
+
+
+//    FiniteElementCollection *fec = new H1_FECollection(order, dim);
+//    FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+//    cout << "Number of finite element unknowns: "
+//         << fespace->GetTrueVSize() << endl;
+
+//    Array<int> ess_tdof_list;
+
+//    // mesh->bdr_attributes.Print();
+//    // if (mesh->bdr_attributes.Size())
+//    // {
+//    //    Array<int> ess_bdr(mesh->bdr_attributes.Max());
+//    //    ess_bdr = 0;
+//    //    // ess_bdr[3] = 1;
+//    //    // ess_bdr[1] = 1;
+//    //    fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+//    // }
+
+
+//    // LinearForm *b = new LinearForm(fespace);
+//    // ConstantCoefficient one(1.0);
+//    // RestrictedCoefficient restr(one,attr);
+
+
+//    // b->AddDomainIntegrator(new DomainLFIntegrator(restr));
+//    // b->AddDomainIntegrator(new DomainLFIntegrator(one));
+//    // b->Assemble();
+
+//    // GridFunction x(fespace);
+//    // FunctionCoefficient chi(ChiExact);
+//    // x.ProjectCoefficient(chi);
+//    // x = 0.0;
+
+//    // BilinearForm *a = new BilinearForm(fespace);
+//    // // a->AddDomainIntegrator(new DiffusionIntegrator(one));
+//    // ConstantCoefficient epsilon(0.000001);
+//    // a->AddDomainIntegrator(new DiffusionIntegrator(epsilon));
+//    // a->AddDomainIntegrator(new MassIntegrator(one));
+//    // a->Assemble();
+
+//    // OperatorPtr A;
+//    // Vector B, X;
+//    // a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+//    // cout << "Size of linear system: " << A->Height() << endl;
+
+
+//    // UMFPackSolver umf_solver;
+//    // umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+//    // umf_solver.SetOperator(*A);
+//    // umf_solver.Mult(B, X);
+
+
+//    // a->RecoverFEMSolution(X, *b, x);
+
+
+//    GridFunction bump(fespace);
+//    FunctionCoefficient c1(BumpFncn);
+//    bump.ProjectCoefficient(c1);
+
+
+//    // GridFunction uex(fespace);
+//    // FunctionCoefficient u_ex(SolExact);
+//    // uex.ProjectCoefficient(u_ex);
+
+//    // int order_quad = max(2, 2 * order + 1);
+//    // const IntegrationRule *irs[Geometry::NumGeom];
+//    // for (int i = 0; i < Geometry::NumGeom; ++i)
+//    // {
+//    //    irs[i] = &(IntRules.Get(i, order_quad));
+//    // }
+//    // double l2error = x.ComputeL2Error(u_ex, irs);
+//    // cout << "l2error = "<< l2error << endl;
+//    // 14. Send the solution by socket to a GLVis server.
+//    if (visualization)
+//    {
+//       char vishost[] = "localhost";
+//       int  visport   = 19916;
+//       socketstream sol_sock(vishost, visport);
+//       sol_sock.precision(8);
+//       // sol_sock << "solution\n" << *mesh << x << flush;
+//       sol_sock << "solution\n" << *mesh << bump << flush;
+
+//       // socketstream ex_sock(vishost, visport);
+//       // ex_sock.precision(8);
+//       // ex_sock << "solution\n" << *mesh << uex << flush;
+
+//       // GridFunction err(uex);
+//       // err-= x;
+//       // socketstream diff_sock(vishost, visport);
+//       // diff_sock.precision(8);
+//       // diff_sock << "solution\n" << *mesh << err << flush;
+
+//    }
+
+//    // 15. Free the used memory.
+//    // delete a;
+//    // delete b;
+//    delete fespace;
+//    delete fec;
+//    delete mesh;
+
+//    return 0;
+// }
+
+
+// void SetElemAttr(Mesh * mesh)
+// {
+//    int dim=mesh->Dimension();
+//    double h = 1.0/sqrt(mesh->GetNE());
+//    for (int iel=0; iel<mesh->GetNE(); iel++)
+//    {
+//       Vector center(dim);
+//       int geom = mesh->GetElementBaseGeometry(iel);
+//       ElementTransformation * tr = mesh->GetElementTransformation(iel);
+//       tr->Transform(Geometries.GetCenter(geom), center);
+//       int attr = (center[0] < 15*h) ? 1 : 2;
+//       mesh->SetAttribute(iel,attr);
+//    }
+//    mesh->SetAttributes();
+// }
+
+// double SolExact(const Vector & x)
+// {
+//    double u;
+//    if (x(0) < 0.5) 
+//    {
+//       u = x(0)/8.0;
+//    }
+//    else
+//    {
+//       u = - x(0)*x(0)/2.0 + 5.0 * x(0) / 8.0 - 1.0/8.0;
+//    }
+
+//    return u;
+// }
+
+// double ChiExact(const Vector & x)
+// {
+//    double u;
+//    if (x(0) == 0.0) 
+//    {
+//       u = 0.0;
+//    }
+//    else
+//    {
+//       u = 0.0;
+//    }
+
+//    return u;
+// }
+
+// double BumpFncn(const Vector & x)
+// {
+//    double u;
+//    if (x(0) == 0.0) 
+//    {
+//       u = 0.0;
+//    }
+//    else
+//    {
+//       u = exp(- 0.01/(1.0-pow(x(0)-1.0,2)));
+//    }
+
+//    return u;
+// }

examples/maxwell-solver/DST2D/back/helmholtz1.cpp

@@ -0,0 +1,468 @@
+// //
+// // Compile with: make helmholtz
+// //
+// // Sample runs:  helmholtz -m ../data/one-hex.mesh
+// //               helmholtz -m ../data/fichera.mesh
+// //               helmholtz -m ../data/fichera-mixed.mesh
+// //
+// // Description:  This example code demonstrates the use of MFEM to define a
+// //               simple finite element discretization of the Helmholtz problem
+// //               -Delta p - omega^2 p = 1 with impedance boundary condition.
+// //
+// #include "mfem.hpp"
+// #include <fstream>
+// #include <iostream>
+// // #include "DiagST.hpp"
+// #include "DST.hpp"
+
+// using namespace std;
+// using namespace mfem;
+
+// // Exact solution and r.h.s., see below for implementation.
+// double f_exact_Re(const Vector &x);
+// double f_exact_Im(const Vector &x);
+
+// double wavespeed(const Vector &x);
+
+
+// int dim;
+// double omega;
+// int sol = 1;
+// bool pml = false;
+// double length = 1.0;
+// double pml_length = 0.25;
+// bool scatter = false;
+// Array2D<double>comp_bdr;
+
+// #ifndef MFEM_USE_SUPERLU
+// #error This example requires that MFEM is built with MFEM_USE_PETSC=YES
+// #endif
+
+// int main(int argc, char *argv[])
+// {
+
+//    // 2. Parse command-line options.
+//    // geometry file
+//    const char *mesh_file = "../../data/one-hex.mesh";
+//    // finite element order of approximation
+//    int order = 1;
+//    // static condensation flag
+//    bool static_cond = false;
+//    bool visualization = 1;
+//    // number of wavelengths
+//    double k = 0.5;
+//    // number of mg levels
+//    int ref = 1;
+//    // dimension
+//    int nd = 2;
+
+//    // optional command line inputs
+//    OptionsParser args(argc, argv);
+//    args.AddOption(&mesh_file, "-m", "--mesh",
+//                   "Mesh file to use.");
+//    args.AddOption(&order, "-o", "--order",
+//                   "Finite element order (polynomial degree) or -1 for"
+//                   " isoparametric space.");
+//    args.AddOption(&nd, "-nd", "--dim","Problem space dimension");
+//    args.AddOption(&sol, "-sol", "--exact",
+//                   "Exact solution flag - 0:polynomial, 1: plane wave, -1: unknown exact");
+//    args.AddOption(&k, "-k", "--wavelengths",
+//                   "Number of wavelengths.");
+//    args.AddOption(&pml, "-pml", "--pml", "-no-pml",
+//                   "--no-pml", "Enable PML.");
+//    args.AddOption(&pml_length, "-pml_length", "--pml_length",
+//                   "Length of the PML region in each direction");
+//    args.AddOption(&length, "-length", "--length",
+//                   "length of the domainin in each direction.");
+//    args.AddOption(&ref, "-ref", "--ref",
+//                   "Number of Refinements.");
+//    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+//                   "--no-static-condensation", "Enable static condensation.");
+//    args.AddOption(&scatter, "-scat", "--scattering-prob", "-no-scat",
+//                   "--no-scattering", "Solve a scattering problem");
+//    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+//                   "--no-visualization",
+//                   "Enable or disable GLVis visualization.");
+//    args.Parse();
+//    // check if the inputs are correct
+//    if (!args.Good())
+//    {
+// 		args.PrintUsage(cout);
+// 		return 1;
+//    }
+// 	args.PrintOptions(cout);
+//    // Angular frequency
+//    omega = 2.0 * M_PI * k;
+
+//    // 3. Read the mesh from the given mesh file.
+//    Mesh *mesh;
+
+//    if (nd == 2)
+//    {
+//       // mesh = new Mesh(mesh_file,1,1);
+//       mesh = new Mesh(4, 4, Element::QUADRILATERAL, true, length, length, false);
+//    }
+//    else
+//    {
+//       mesh = new Mesh(1, 1, 1, Element::HEXAHEDRON, true, length, length, length,false);
+//    }
+
+//    // 3. Executing uniform h-refinement
+//    for (int i = 0; i < ref; i++ )
+//    {
+//       mesh->UniformRefinement();
+//    }
+//    dim = mesh->Dimension();
+
+//    double hl = GetUniformMeshElementSize(mesh);
+//    Vector pmin, pmax;
+//    mesh->GetBoundingBox(pmin,pmax);
+//    double domain_length = pmax[0] - pmin[0];
+//    double pml_thickness = 0.125/domain_length;
+//    // int nrlayers = pml_thickness/hl;
+//    int nrlayers = 2;
+//    Array<int> directions;
+   
+//    for (int i = 0; i<nrlayers; i++)
+//    {
+//       for (int comp=0; comp<dim; ++comp)
+//       {
+//          // directions.Append(comp+1);
+//          // directions.Append(-comp-1);
+//       }
+//    }
+//    // Find uniform h size of the original mesh
+//    cout << "pml layers = " << nrlayers << endl;
+//    cout << "pml length = " << hl*nrlayers << endl;
+//    Mesh *mesh_ext = ExtendMesh(mesh,directions);
+
+
+//    Array2D<double> lengths(dim,2);
+//    lengths = hl*nrlayers;
+//    // lengths[0][1] = 0.0;
+//    // lengths[1][1] = 0.0;
+//    // lengths[1][0] = 0.0;
+//    // lengths[0][0] = 0.0;
+//    CartesianPML pml(mesh_ext,lengths);
+//    pml.SetOmega(omega);
+//    comp_bdr.SetSize(dim,2);
+//    comp_bdr = pml.GetCompDomainBdr(); 
+
+//    // 6. Define a finite element space on the mesh.
+//    FiniteElementCollection *fec = new H1_FECollection(order, dim);
+//    FiniteElementSpace *fespace = new FiniteElementSpace(mesh_ext, fec);
+
+//    // 6. Set up the linear form (Real and Imaginary part)
+//    FunctionCoefficient f_Re(f_exact_Re);
+//    FunctionCoefficient f_Im(f_exact_Im);
+
+//    // ParLinearForm *b_Re(new ParLinearForm);
+//    ComplexLinearForm b(fespace, ComplexOperator::HERMITIAN);
+//    b.AddDomainIntegrator(new DomainLFIntegrator(f_Re),
+//                          new DomainLFIntegrator(f_Im));
+//    b.real().Vector::operator=(0.0);
+//    b.imag().Vector::operator=(0.0);
+//    b.Assemble();
+
+//    // 7. Set up the bilinear form (Real and Imaginary part)
+//    ConstantCoefficient one(1.0);
+//    ConstantCoefficient sigma(-pow(omega, 2));
+
+//    FunctionCoefficient ws(wavespeed);
+
+//    PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+//    PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+//    PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+//    PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+//    ProductCoefficient c2_re0(sigma, detJ_re);
+//    ProductCoefficient c2_im0(sigma, detJ_im);
+
+//    ProductCoefficient c2_re(c2_re0, ws);
+//    ProductCoefficient c2_im(c2_im0, ws);
+
+
+//    SesquilinearForm a(fespace,ComplexOperator::HERMITIAN);
+
+//    a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+//                          new DiffusionIntegrator(c1_im));
+//    a.AddDomainIntegrator(new MassIntegrator(c2_re),new MassIntegrator(c2_im));
+
+//    a.Assemble();
+//    a.Finalize();
+
+//    Array<int> ess_tdof_list;
+//    Array<int> ess_bdr(mesh_ext->bdr_attributes.Max());
+//    ess_bdr = 1;
+//    fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+//    // Solution grid function
+//    ComplexGridFunction p_gf(fespace);
+//    OperatorHandle Ah;
+//    Vector X, B;
+
+//    a.FormLinearSystem(ess_tdof_list, p_gf, b, Ah, X, B);
+
+//    ComplexSparseMatrix * AZ = Ah.As<ComplexSparseMatrix>();
+//    SparseMatrix * A = AZ->GetSystemMatrix();
+
+
+//    cout << "Size of fine grid system: "
+//          << A->Height() << " x " << A->Width() << endl;
+
+
+//    DST S(&a,lengths, omega, &ws, nrlayers);
+//    // DiagST S(&a,lengths, omega, &ws, nrlayers);
+// 	S.SetOperator(*A);
+//    // S.SetLoadVector(B);
+   
+
+
+
+//    X = 0.0;
+// 	GMRESSolver gmres;
+// 	gmres.SetPreconditioner(S);
+// 	gmres.SetOperator(*A);
+// 	gmres.SetRelTol(1e-8);
+// 	gmres.SetMaxIter(50);
+// 	gmres.SetPrintLevel(1);
+// 	gmres.Mult(B, X);
+
+
+
+
+//    int n= 20;
+//    X = 0.0;
+//    Vector z(X.Size()); z = 0.0;
+//    Vector r(B);
+//    Vector ztemp(r.Size());
+//    Vector Ax(X.Size());
+//    double tol = 1e-8;
+//    cout << endl;
+   
+//    for (int i = 0; i<n; i++)
+//    {
+//       A->Mult(X,Ax); Ax *=-1.0;
+//       r = b; r+=Ax;
+//       cout << "   ST Solver   Iteration :   " << i <<"  || r || = " <<  r.Norml2() << endl;
+//       if (r.Norml2() < tol) 
+//       {
+//          cout << "Convergence in " << i+1 << " iterations" << endl;
+//          break;
+//       }
+//       S.Mult(r,z); 
+//       X += z;
+
+//       // X1-=z;
+
+//          // p_gf = 0.0;
+//          // a.RecoverFEMSolution(X,B,p_gf);
+//          // char vishost[] = "localhost";
+//          // int  visport   = 19916;
+//          // string keys;
+//          // if (dim ==2 )
+//          // {
+//          //    keys = "keys mrRljc\n";
+//          // }
+//          // else
+//          // {
+//          //    keys = "keys mc\n";
+//          // }
+//          // socketstream sol1_sock_re(vishost, visport);
+//          // sol1_sock_re.precision(8);
+//          // sol1_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+//          //             "window_title 'Numerical Pressure (real part)' "
+//          //             << keys << flush;
+//          // cin.get();
+//    }
+
+//    KLUSolver klu(*A);
+//    Vector X1(X.Size());
+//    klu.Mult(B,X1);
+//    X1-= X;
+
+
+//    if (visualization)
+//    {
+//       char vishost[] = "localhost";
+//       int  visport   = 19916;
+//       string keys;
+//       if (dim ==2 )
+//       {
+//          keys = "keys mrRljc\n";
+//       }
+//       else
+//       {
+//          keys = "keys mc\n";
+//       }
+//       socketstream st_sock_re(vishost, visport);
+//       st_sock_re.precision(8);
+
+//       a.RecoverFEMSolution(X,B,p_gf);
+//       st_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+//                   "window_title 'Numerical Pressure (real part from KLU)' "
+//                   << keys << flush;
+//       a.RecoverFEMSolution(X1,B,p_gf);
+//       socketstream sol_sock_re(vishost, visport);
+//       sol_sock_re.precision(8);
+//       sol_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+//                   "window_title 'Numerical Pressure (real part from KLU)' "
+//                   << keys << flush;
+//                   // << keys << "valuerange -0.1 0.1 \n" << flush;
+//       // socketstream diff_sock_re(vishost, visport);
+//       // diff_sock_re.precision(8);
+//       // diff_sock_re << "solution\n" << *mesh_ext << p_gf1.real() <<
+//       //             "window_title 'Numerical Pressure (real part from KLU)' "
+//       //             << keys << flush;
+
+                                  
+//    }
+//    delete fespace;
+//    delete fec;
+// 	delete mesh_ext;
+// 	delete mesh;
+//    return 0;
+// }
+
+
+// //calculate RHS from exact solution f = - \Delta u
+// double f_exact_Re(const Vector &x)
+// {
+//    double f_re = 0.0;
+//    double x0 = length/2.0;
+//    double x1 = length/2.0;
+//    double x2 = length/2.0;
+//    // x0 = 0.59;
+//    // x0 = 0.19;
+//    x0 = 0.5;
+//    // x1 = 0.768;
+//    // x1 = 0.168;
+//    x1 = 0.5;
+//    double alpha,beta;
+//    // double n = 5.0*omega/M_PI;
+//    double n = 4.0*omega/M_PI;
+//    // double n = 1.0;
+//    // double coeff = pow(n,2)/M_PI;
+//    beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+//    if (dim == 3) { beta += pow(x2-x(2),2); }
+//    // alpha = -pow(n,2) * beta;
+//    // double coeff = pow(n,2)/M_PI;
+//    double coeff = 16.0*omega*omega/M_PI/M_PI/M_PI;
+//    alpha = -pow(n,2) * beta;
+//    f_re = coeff*exp(alpha);
+
+//    x0 = 0.85;
+//    x1 = 0.15;
+//    beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+//    if (dim == 3) { beta += pow(x2-x(2),2); }
+//    alpha = -pow(n,2) * beta;
+//    // f_re += coeff*exp(alpha);
+
+//    // x0 = 0.5;
+//    // x1 = 0.8;
+//    // beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+//    // if (dim == 3) { beta += pow(x2-x(2),2); }
+//    // alpha = -pow(n,2) * beta;
+//    // f_re += coeff*exp(alpha);
+
+//    bool in_pml = false;
+//    for (int i = 0; i<dim; i++)
+//    {
+//       if (x(i)<=comp_bdr(i,0) || x(i)>=comp_bdr(i,1))
+//       {
+//          in_pml = true;
+//          break;
+//       }
+//    }
+//    if (in_pml) f_re = 0.0;
+
+//    return f_re;
+
+// }
+// double f_exact_Im(const Vector &x)
+// {
+//    double f_im;
+//    f_im = 0.0;
+//    return f_im;
+// }
+
+// double wavespeed(const Vector &x)
+// {
+//    double ws;
+//    // if (x(0) <= 0.25)
+//    // {
+//    //    ws = 1.0;
+//    // }
+//    // else if(x(0)<=0.5)
+//    // {
+//    //    ws = 1.0;
+//    // }
+//    // else if(x(0)<=0.75)
+//    // {
+//    //    ws = 0.75;
+//    //    // ws = 0.5;
+//    // }
+//    // else 
+//    // {
+//    //    ws = 0.75;
+//    //    // ws = 1.0;
+//    // }
+//    // if (x(1) <= 1.0/3.0)
+//    // {
+//    //    ws = 2.0;
+//    // }
+//    // else if(x(1)<=2.0/3.0)
+//    // {
+//    //    ws = 1.0;
+//    // }
+//    // else 
+//    // {
+//    //    // ws = 0.75;
+//    //    ws = 0.25;
+//    // }
+
+//    // if (x(0) <= 0.33)
+//    // {
+//    //    ws = 1.0;
+//    // }
+//    // else if(x(0)<=0.66)
+//    // {
+//    //    ws = -0.65 + 5.0*x(0);
+//    // }
+//    // else 
+//    // {
+//    //    ws = 2.65;
+//    //    // ws = 0.5;
+//    // }
+
+
+//    ws = 1.0;
+//    return ws;
+// }
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

examples/maxwell-solver/DST2D/helmholtz.cpp

@@ -0,0 +1,500 @@
+//
+// Compile with: make helmholtz
+//
+// Sample runs:  helmholtz -m ../data/one-hex.mesh
+//               helmholtz -m ../data/fichera.mesh
+//               helmholtz -m ../data/fichera-mixed.mesh
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Helmholtz problem
+//               -Delta p - omega^2 p = 1 with impedance boundary condition.
+//
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "DST2D.hpp"
+#include "AdditiveST2D.hpp"
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution and r.h.s., see below for implementation.
+double f_exact_Re(const Vector &x);
+double f_exact_Im(const Vector &x);
+
+double wavespeed(const Vector &x);
+
+
+int dim;
+double omega;
+int sol = 1;
+bool pml = false;
+double length = 1.0;
+double pml_length = 0.25;
+bool scatter = false;
+Array2D<double>comp_bdr;
+
+#ifndef MFEM_USE_SUPERLU
+#error This example requires that MFEM is built with MFEM_USE_PETSC=YES
+#endif
+
+int main(int argc, char *argv[])
+{
+
+   // 2. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../../data/one-hex.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // static condensation flag
+   bool static_cond = false;
+   bool visualization = 1;
+   // number of wavelengths
+   double k = 0.5;
+   // number of mg levels
+   int ref = 1;
+   // dimension
+   int nd = 2;
+
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nd, "-nd", "--dim","Problem space dimension");
+   args.AddOption(&sol, "-sol", "--exact",
+                  "Exact solution flag - 0:polynomial, 1: plane wave, -1: unknown exact");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&pml, "-pml", "--pml", "-no-pml",
+                  "--no-pml", "Enable PML.");
+   args.AddOption(&pml_length, "-pml_length", "--pml_length",
+                  "Length of the PML region in each direction");
+   args.AddOption(&length, "-length", "--length",
+                  "length of the domainin in each direction.");
+   args.AddOption(&ref, "-ref", "--ref",
+                  "Number of Refinements.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&scatter, "-scat", "--scattering-prob", "-no-scat",
+                  "--no-scattering", "Solve a scattering problem");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+		args.PrintUsage(cout);
+		return 1;
+   }
+	args.PrintOptions(cout);
+   // Angular frequency
+   omega = 2.0 * M_PI * k;
+
+   // 3. Read the mesh from the given mesh file.
+   Mesh *mesh;
+
+   if (nd == 2)
+   {
+      // mesh = new Mesh(mesh_file,1,1);
+      mesh = new Mesh(4, 4, Element::QUADRILATERAL, true, length, length, false);
+   }
+   else
+   {
+      mesh = new Mesh(1, 1, 1, Element::HEXAHEDRON, true, length, length, length,false);
+   }
+
+   // 3. Executing uniform h-refinement
+   for (int i = 0; i < ref; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+   dim = mesh->Dimension();
+
+   double hl = GetUniformMeshElementSize(mesh);
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin,pmax);
+   // double domain_length = pmax[0] - pmin[0];
+   // double pml_thickness = 0.125/domain_length;
+   // int nrlayers = pml_thickness/hl;
+   int nrlayers = 4;
+   Array<int> directions;
+   
+   for (int i = 0; i<nrlayers; i++)
+   {
+      for (int comp=0; comp<dim; ++comp)
+      {
+         directions.Append(comp+1);
+         directions.Append(-comp-1);
+      }
+   }
+   // Find uniform h size of the original mesh
+   cout << "pml layers = " << nrlayers << endl;
+   cout << "pml length = " << hl*nrlayers << endl;
+   Mesh *mesh_ext = ExtendMesh(mesh,directions);
+
+
+   Array2D<double> lengths(dim,2);
+   lengths = hl*nrlayers;
+   // lengths[0][1] = 0.0;
+   // lengths[1][1] = 0.0;
+   // lengths[1][0] = 0.0;
+   // lengths[0][0] = 0.0;
+   CartesianPML pml(mesh_ext,lengths);
+   pml.SetOmega(omega);
+   comp_bdr.SetSize(dim,2);
+   comp_bdr = pml.GetCompDomainBdr(); 
+
+   // 6. Define a finite element space on the mesh.
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh_ext, fec);
+
+   // 6. Set up the linear form (Real and Imaginary part)
+   FunctionCoefficient f_Re(f_exact_Re);
+   FunctionCoefficient f_Im(f_exact_Im);
+
+   // ParLinearForm *b_Re(new ParLinearForm);
+   ComplexLinearForm b(fespace, ComplexOperator::HERMITIAN);
+   b.AddDomainIntegrator(new DomainLFIntegrator(f_Re),
+                         new DomainLFIntegrator(f_Im));
+   b.real().Vector::operator=(0.0);
+   b.imag().Vector::operator=(0.0);
+   b.Assemble();
+
+   // 7. Set up the bilinear form (Real and Imaginary part)
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+   FunctionCoefficient ws(wavespeed);
+
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+
+   ProductCoefficient c2_re(c2_re0, ws);
+   ProductCoefficient c2_im(c2_im0, ws);
+
+
+   SesquilinearForm a(fespace,ComplexOperator::HERMITIAN);
+
+   a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   a.AddDomainIntegrator(new MassIntegrator(c2_re),new MassIntegrator(c2_im));
+
+   a.Assemble();
+   a.Finalize();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(mesh_ext->bdr_attributes.Max());
+   ess_bdr = 1;
+   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   // Solution grid function
+   ComplexGridFunction p_gf(fespace);
+   OperatorHandle Ah;
+   Vector X, B;
+
+   a.FormLinearSystem(ess_tdof_list, p_gf, b, Ah, X, B);
+
+   ComplexSparseMatrix * AZ = Ah.As<ComplexSparseMatrix>();
+   SparseMatrix * A = AZ->GetSystemMatrix();
+
+
+   cout << "Size of fine grid system: "
+         << A->Height() << " x " << A->Width() << endl;
+
+
+   DST2D S1(&a,lengths, omega, &ws, nrlayers);
+   // AdditiveST S2(&a,lengths, omega, &ws, nrlayers);
+   
+
+   StopWatch chrono;
+
+   // chrono.Clear();
+   // chrono.Start();
+   X = 0.0;
+	GMRESSolver gmres;
+	// gmres.iterative_mode = true;
+   gmres.SetPreconditioner(S1);
+	gmres.SetOperator(*A);
+	gmres.SetRelTol(1e-10);
+	gmres.SetMaxIter(50);
+	gmres.SetPrintLevel(1);
+	gmres.Mult(B, X);
+
+   // X = 0.0;
+	// gmres.SetPreconditioner(S2);
+	// gmres.Mult(B, X);
+
+   // chrono.Stop();
+   // cout << "GMRES time: " << chrono.RealTime() << endl; 
+
+   X = 0.0;
+   SLISolver sli;
+   sli.iterative_mode = true;
+   sli.SetPreconditioner(S1);
+   sli.SetOperator(*A);
+   sli.SetRelTol(1e-10);
+   sli.SetMaxIter(50);
+   sli.SetPrintLevel(1);
+   sli.Mult(B,X);
+
+   // int n= 200;
+   // X = 0.0;
+   // Vector z(X.Size()); z = 0.0;
+   // Vector r(B);
+   // Vector ztemp(r.Size());
+   // Vector Ax(X.Size());
+   // double tol = 1e-10;
+   // cout << endl;
+   // chrono.Clear();
+   // chrono.Start();
+   // for (int i = 0; i<n; i++)
+   // {
+   //    A->Mult(X,Ax); Ax *=-1.0;
+   //    r = b; r+=Ax;
+   //    cout << "   ST Solver   Iteration :   " << i <<"  || r || = " <<  r.Norml2() << endl;
+   //    if (r.Norml2() < tol) 
+   //    {
+   //       cout << "Convergence in " << i << " iterations" << endl;
+   //       break;
+   //    }
+   //    S1.Mult(r,z); 
+   //    X += z;
+
+   //    // X1-=z;
+   //    // p_gf = 0.0;
+   //    // a.RecoverFEMSolution(X,B,p_gf);
+   //    //    char vishost[] = "localhost";
+   //    //    int  visport   = 19916;
+   //    //    string keys;
+   //    //    if (dim ==2 )
+   //    //    {
+   //    //       keys = "keys mrRljc\n";
+   //    //    }
+   //    //    else
+   //    //    {
+   //    //       keys = "keys mc\n";
+   //    //    }
+   //    //    socketstream sol1_sock_re(vishost, visport);
+   //    //    sol1_sock_re.precision(8);
+   //    //    sol1_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+   //    //                "window_title 'Numerical Pressure (real part)' "
+   //    //                << keys << flush;
+   //    //    cin.get();
+   // }
+
+   // chrono.Stop();
+   // cout << "Solver time: " << chrono.RealTime() << endl; 
+
+   a.RecoverFEMSolution(X,B,p_gf);
+
+   KLUSolver klu(*A);
+   Vector X1(X.Size());
+   klu.Mult(B,X1);
+   X1-= X;
+
+   ComplexGridFunction error_gf(fespace);
+
+   a.RecoverFEMSolution(X1,B,error_gf);
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      string keys;
+      if (dim ==2 )
+      {
+         keys = "keys mrRljc\n";
+      }
+      else
+      {
+         keys = "keys mc\n";
+      }
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re.precision(8);
+      sol_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+                  "window_title 'Numerical Pressure (real part from DST)' "
+                  // << keys << flush;
+                  << keys << "valuerange -0.08 0.08 \n" << flush;
+      socketstream err_sock_re(vishost, visport);
+      err_sock_re.precision(8);
+      err_sock_re << "solution\n" << *mesh_ext << error_gf.real() <<
+                  "window_title 'Numerical Pressure (real part from KLU)' "
+                  << keys << flush;
+   }
+   delete fespace;
+   delete fec;
+	delete mesh_ext;
+	delete mesh;
+   return 0;
+}
+
+
+//calculate RHS from exact solution f = - \Delta u
+double f_exact_Re(const Vector &x)
+{
+   double f_re = 0.0;
+   double x0 = length/2.0;
+   double x1 = length/2.0;
+   double x2 = length/2.0;
+   // x0 = 0.59;
+   // x0 = 0.19;
+   x0 = 0.5;
+   // x1 = 0.768;
+   // x1 = 0.168;
+   x1 = 0.5;
+   double alpha,beta;
+   // double n = 5.0*omega/M_PI;
+   double n = 4.0*omega/M_PI;
+   // double n = 1.0;
+   // double coeff = pow(n,2)/M_PI;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   // alpha = -pow(n,2) * beta;
+   // double coeff = pow(n,2)/M_PI;
+   double coeff = 16.0*omega*omega/M_PI/M_PI/M_PI;
+   alpha = -pow(n,2) * beta;
+   f_re = coeff*exp(alpha);
+
+   x0 = 0.85;
+   x1 = 0.15;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   alpha = -pow(n,2) * beta;
+   // f_re += coeff*exp(alpha);
+
+   x0 = 0.8;
+   x1 = 0.4;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   alpha = -pow(n,2) * beta;
+   // f_re += coeff*exp(alpha);
+
+   bool in_pml = false;
+   for (int i = 0; i<dim; i++)
+   {
+      if (x(i)<=comp_bdr(i,0) || x(i)>=comp_bdr(i,1))
+      {
+         in_pml = true;
+         break;
+      }
+   }
+   if (in_pml) f_re = 0.0;
+
+   return f_re;
+
+}
+double f_exact_Im(const Vector &x)
+{
+   double f_im;
+   f_im = 0.0;
+   return f_im;
+}
+
+double wavespeed(const Vector &x)
+{
+   double ws;
+   // if (x(0) <= 0.25)
+   // {
+   //    ws = 1.0;
+   // }
+   // else if(x(0)<=0.5)
+   // {
+   //    ws = 1.0;
+   // }
+   // else if(x(0)<=0.75)
+   // {
+   //    ws = 0.75;
+   //    // ws = 0.5;
+   // }
+   // else 
+   // {
+   //    ws = 0.75;
+   //    // ws = 1.0;
+   // }
+   // if (x(1) <= 1.0/3.0)
+   // {
+   //    ws = 2.0;
+   // }
+   // else if(x(1)<=2.0/3.0)
+   // {
+   //    ws = 1.0;
+   // }
+   // else 
+   // {
+   //    // ws = 0.75;
+   //    ws = 0.25;
+   // }
+
+   // if (x(0) <= 0.33)
+   // {
+   //    ws = 1.0;
+   // }
+   // else if(x(0)<=0.66)
+   // {
+   //    ws = -0.65 + 5.0*x(0);
+   // }
+   // else 
+   // {
+   //    ws = 2.65;
+   //    // ws = 0.5;
+   // }
+
+   // if (x(0) <= x(1) && x(1) >= 1.0-x(0))
+   // {
+   //    ws = 1.0;
+   // }
+   // else if (x(0) > x(1) && x(1) >= 1.0-x(0))
+   // {
+   //    ws = 3.0;
+   // }
+   // else if (x(0) <= x(1) && x(1) < 1.0-x(0))
+   // {
+   //    ws = 2.0;
+   // }
+   // else
+   // {
+   //    ws = 4.0;
+   // }
+   
+
+
+   ws = 1.0;
+   return ws;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

examples/maxwell-solver/DST2D/makefile

@@ -0,0 +1,61 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/maxwell-solver/DST2D,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = helmholtz
+PAR_EXAMPLES = 
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean
+.PRECIOUS: %.o
+
+COMMON_O= ../common/PML.o ../common/MeshPartition.o ../common/Utilities.o DST2D.o AdditiveST2D.o
+
+# Remove built-in rules
+%: %.cpp
+%.o: %.cpp
+
+all: $(EXAMPLES)
+
+# Rules for building the EXAMPLES
+
+%: $(SRC)%.cpp $(COMMON_O) $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(COMMON_O) $(MFEM_LIBS)
+
+# Rules for compiling miniapp dependencies
+$(COMMON_O) $($(EXAMPLES)): \
+%.o: $(SRC)%.cpp $(SRC)%.hpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $(<) -o $(@)
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+	rm output/*
+
+

examples/maxwell-solver/FOSLS.cpp

@@ -0,0 +1,475 @@
+#include "FOSLS.hpp"
+
+
+ComplexMaxwellFOSLS::ComplexMaxwellFOSLS(ParFiniteElementSpace * fes_) : fes(fes_)
+{ };
+
+void ComplexMaxwellFOSLS::SetLoadData(Array<VectorFunctionCoefficient *> & loads_)
+{
+   loads = loads_;
+}
+void ComplexMaxwellFOSLS::SetEssentialData(Array<VectorFunctionCoefficient *> &  ess_data_)
+{
+   ess_data = ess_data_;
+}
+
+void ComplexMaxwellFOSLS::GetFOSLSLinearSystem(Array2D<HypreParMatrix *> & A_, 
+                                               BlockVector & X_,
+                                               BlockVector & Rhs_)
+{
+   if (A.NumCols() == 0)
+   {
+      FormSystem(true);
+   }
+   A_ = A;
+   X_ = X;
+   Rhs_ = Rhs;
+}
+
+void ComplexMaxwellFOSLS::GetFOSLSMatrix(Array2D<HypreParMatrix *> & A_)
+{
+   if (A.NumCols() == 0)
+   {
+      FormSystem(false);
+   }
+   A_ = A;
+}
+
+void ComplexMaxwellFOSLS::FormSystem(bool system)
+{
+   // HYPRE_Int size = fes->GlobalTrueVSize();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   pmesh = fes->GetParMesh();
+   if (pmesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   VectorFunctionCoefficient * E_ex_re = ess_data[0];
+   // VectorFunctionCoefficient * H_ex_re = ess_data[1];
+   VectorFunctionCoefficient * E_ex_im = ess_data[2];
+   // VectorFunctionCoefficient * H_ex_im = ess_data[3];
+
+   // VectorFunctionCoefficient * f_ex_re = loads[0];
+   VectorFunctionCoefficient * g_ex_re = loads[1];
+   // VectorFunctionCoefficient * f_ex_im = loads[2];
+   VectorFunctionCoefficient * g_ex_im = loads[3];
+
+   int n = fes->GetVSize();
+   int N = fes->GetTrueVSize();
+   block_offsets.SetSize(5);
+   block_offsets[0] = 0;
+   block_offsets[1] = n;
+   block_offsets[2] = n;
+   block_offsets[3] = n;
+   block_offsets[4] = n;
+   block_offsets.PartialSum();
+
+   block_trueOffsets.SetSize(5);
+   block_trueOffsets[0] = 0;
+   block_trueOffsets[1] = N;
+   block_trueOffsets[2] = N;
+   block_trueOffsets[3] = N;
+   block_trueOffsets[4] = N;
+   block_trueOffsets.PartialSum();
+
+   ParGridFunction E_gf_re, E_gf_im, H_gf_re, H_gf_im;
+
+   if(system)
+   {
+      x.Update(block_offsets);
+      rhs.Update(block_offsets);
+      X.Update(block_trueOffsets);
+      Rhs.Update(block_trueOffsets);
+      x = 0.0;  rhs = 0.0; X = 0.0;  Rhs = 0.0;
+
+      E_gf_re.MakeRef(fes,x.GetBlock(0)); E_gf_re = 0.0;
+      H_gf_re.MakeRef(fes,x.GetBlock(1)); H_gf_re = 0.0;
+      E_gf_im.MakeRef(fes,x.GetBlock(2)); E_gf_im = 0.0;
+      H_gf_im.MakeRef(fes,x.GetBlock(3)); H_gf_im = 0.0;
+
+      E_gf_re.ProjectCoefficient(*E_ex_re);
+      E_gf_im.ProjectCoefficient(*E_ex_im);
+   }
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient negomeg(-omega);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega * omega);
+   ScalarVectorProductCoefficient wJi(omeg,*g_ex_im);
+   ScalarVectorProductCoefficient negJr(negone,*g_ex_re);
+   ScalarVectorProductCoefficient negwJr(negomeg,*g_ex_re);
+   ScalarVectorProductCoefficient negJi(negone,*g_ex_im);
+
+   ParLinearForm b0, b1, b2, b3;
+
+   if(system)
+   {
+      b0.Update(fes,rhs.GetBlock(0),0);
+      b1.Update(fes,rhs.GetBlock(1),0);
+      b2.Update(fes,rhs.GetBlock(2),0);
+      b3.Update(fes,rhs.GetBlock(3),0);
+      b0.AddDomainIntegrator(new VectorFEDomainLFIntegrator(wJi));
+      b1.AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(negJr));
+      b2.AddDomainIntegrator(new VectorFEDomainLFIntegrator(negwJr));
+      b3.AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(negJi));
+      b0.Assemble();
+      b1.Assemble();
+      b2.Assemble();
+      b3.Assemble();
+   }
+   A.SetSize(4,4); 
+   for (int i = 0; i<4; i++)
+   {
+      for (int j = 0; j<4; j++)
+      {
+         A[i][j] = nullptr;
+      }
+   }
+
+   ParBilinearForm a00(fes);
+   a00.AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a00.Assemble();
+   if (system)
+   {
+      a00.EliminateEssentialBC(ess_bdr,x.GetBlock(0),rhs.GetBlock(0),mfem::Operator::DIAG_ONE);
+   }
+   else
+   {
+      a00.EliminateEssentialBC(ess_bdr);
+   }
+   a00.Finalize();
+   A[0][0] = a00.ParallelAssemble();
+
+   ParMixedBilinearForm a03(fes,fes);
+   a03.AddDomainIntegrator(new MixedVectorCurlIntegrator(negomeg));
+   a03.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(negomeg));
+   a03.Assemble();
+   a03.EliminateTestDofs(ess_bdr);
+   a03.Finalize();
+   A[0][3] = a03.ParallelAssemble();
+
+   ParBilinearForm a11(fes);
+   a11.AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a11.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a11.Assemble();
+   a11.Finalize();
+   A[1][1] = a11.ParallelAssemble();
+
+   ParMixedBilinearForm a21(fes,fes);
+   a21.AddDomainIntegrator(new MixedVectorCurlIntegrator(omeg));
+   a21.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(omeg));
+   a21.Assemble();
+   a21.EliminateTestDofs(ess_bdr);
+   a21.Finalize();
+   A[2][1] = a21.ParallelAssemble();
+
+   if (system)
+   {
+      ParMixedBilinearForm a12(fes,fes);
+      a12.AddDomainIntegrator(new MixedVectorCurlIntegrator(omeg));
+      a12.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(omeg));
+      a12.Assemble();
+      a12.EliminateTrialDofs(ess_bdr,x.GetBlock(2),rhs.GetBlock(1));
+      a12.Finalize();
+      A[1][2] = a12.ParallelAssemble();
+   }
+   else
+   {
+      A[1][2] = A[2][1]->Transpose();
+   }
+
+
+   ParBilinearForm a22(fes);
+   a22.AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a22.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a22.Assemble();
+   if (system)
+   {
+      a22.EliminateEssentialBC(ess_bdr,x.GetBlock(2),rhs.GetBlock(2),mfem::Operator::DIAG_ONE);
+   }
+   else
+   {
+      a22.EliminateEssentialBC(ess_bdr);
+   }
+   a22.Finalize();
+   A[2][2] = a22.ParallelAssemble();
+
+   if (system)
+   {
+      ParMixedBilinearForm a30(fes,fes);
+      a30.AddDomainIntegrator(new MixedVectorCurlIntegrator(negomeg));
+      a30.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(negomeg));
+      a30.Assemble();
+      a30.EliminateTrialDofs(ess_bdr,x.GetBlock(0),rhs.GetBlock(3));
+      a30.Finalize();
+      A[3][0] = a30.ParallelAssemble();
+   }
+   else
+   {
+      A[3][0] = A[0][3]->Transpose();
+   }
+
+   ParBilinearForm a33(fes);
+   a33.AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a33.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a33.Assemble();
+   a33.Finalize();
+   A[3][3] = a33.ParallelAssemble();
+
+   if (system)
+   {
+      for (int i = 0; i<4; i++)
+      {
+         fes->GetRestrictionMatrix()->Mult(x.GetBlock(i), X.GetBlock(i));
+         fes->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(i),Rhs.GetBlock(i));
+      }
+   }
+};
+
+HelmholtzFOSLS::HelmholtzFOSLS(Array<ParFiniteElementSpace * > & fes_, 
+bool definite_, bool complex_) : fes(fes_), definite(definite_), complex(complex_)
+{ 
+   n = complex ? 2 : 1;
+   Init();
+};
+
+void HelmholtzFOSLS::Init()
+{
+   f.SetSize(n);
+   Q.SetSize(n);
+   p_ex_coeff.SetSize(n);
+   u_ex_coeff.SetSize(n);
+   for (int i = 0; i<n; i++)
+   {
+      f[i] = nullptr;
+      Q[i] = nullptr;
+      p_ex_coeff[i] = nullptr;
+      u_ex_coeff[i] = nullptr;
+   }
+}
+
+void HelmholtzFOSLS::SetLoadData(Array<FunctionCoefficient * > & f_)
+{
+   f = f_;
+}
+void HelmholtzFOSLS::SetLoadData(Array<VectorFunctionCoefficient * > & Q_)
+{
+   Q = Q_;
+}
+
+void HelmholtzFOSLS::SetEssentialData(Array<FunctionCoefficient * > & p_ex_coeff_)
+{
+   p_ex_coeff = p_ex_coeff_;
+}
+void HelmholtzFOSLS::SetEssentialData(Array<VectorFunctionCoefficient * > & u_ex_coeff_)
+{
+   u_ex_coeff = u_ex_coeff_;
+}
+
+void HelmholtzFOSLS::GetFOSLSLinearSystem(Array2D<HypreParMatrix *> & A_, 
+                                          BlockVector & X_,
+                                          BlockVector & Rhs_)
+{
+   if (A.NumCols() == 0)
+   {
+      FormSystem(true);
+   }
+   A_ = A;
+   X_ = X;
+   Rhs_ = Rhs;
+}
+
+void HelmholtzFOSLS::GetFOSLSMatrix(Array2D<HypreParMatrix *> & A_)
+{
+   if (A.NumCols() == 0)
+   {
+      FormSystem(false);
+   }
+   A_ = A;
+}
+
+void HelmholtzFOSLS::FormSystem(bool system)
+{
+   // HYPRE_Int size = fes[0]->GlobalTrueVSize() + fes[1]->GlobalTrueVSize();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   pmesh = fes[0]->GetParMesh();
+   if (pmesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+   int blksize = complex ? 5 : 3;
+   block_offsets.SetSize(blksize);
+   block_offsets[0] = 0;
+   block_offsets[1] = fes[0]->GetVSize();
+   block_offsets[2] = fes[1]->GetVSize();
+   if (complex)
+   {
+      block_offsets[3] = fes[0]->GetVSize();
+      block_offsets[4] = fes[1]->GetVSize();
+   }
+   block_offsets.PartialSum();
+
+   block_trueOffsets.SetSize(blksize);
+   block_trueOffsets[0] = 0;
+   block_trueOffsets[1] = fes[0]->GetTrueVSize();
+   block_trueOffsets[2] = fes[1]->GetTrueVSize();
+   if (complex)
+   {
+      block_trueOffsets[3] = fes[0]->GetTrueVSize();
+      block_trueOffsets[4] = fes[1]->GetTrueVSize();
+   }
+   block_trueOffsets.PartialSum();
+
+   ParGridFunction p_gf_re, u_gf_re, p_gf_im, u_gf_im;
+
+   if(system)
+   {
+      x.Update(block_offsets);
+      rhs.Update(block_offsets);
+      X.Update(block_trueOffsets);
+      Rhs.Update(block_trueOffsets);
+      x = 0.0;  rhs = 0.0; X = 0.0;  Rhs = 0.0;
+
+      p_gf_re.MakeRef(fes[0],x.GetBlock(0)); p_gf_re = 0.0;
+      u_gf_re.MakeRef(fes[1],x.GetBlock(1)); u_gf_re = 0.0;
+
+      if (complex)
+      {
+         p_gf_im.MakeRef(fes[0],x.GetBlock(2)); p_gf_im = 0.0;
+         u_gf_im.MakeRef(fes[1],x.GetBlock(3)); u_gf_im = 0.0;
+      }
+
+      if (p_ex_coeff[0])
+      {
+         p_gf_re.ProjectCoefficient(*p_ex_coeff[0]);
+      }
+      if (u_ex_coeff[0])
+      {
+         u_gf_re.ProjectCoefficient(*u_ex_coeff[0]);
+      }
+      if (complex)
+      {
+         if (p_ex_coeff[1])
+         {
+            p_gf_im.ProjectCoefficient(*p_ex_coeff[1]);
+         }
+         if (u_ex_coeff[1])
+         {
+            u_gf_im.ProjectCoefficient(*u_ex_coeff[1]);
+         }
+      }
+   }
+
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negomeg(-omega);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega * omega);
+   ProductCoefficient omega_f(omeg,*f[0]);
+   ProductCoefficient neg_f(negone,*f[0]);
+
+   ParLinearForm b0, b1;
+
+   if(system)
+   {
+      b0.Update(fes[0],rhs.GetBlock(0),0);
+      b1.Update(fes[1],rhs.GetBlock(1),0);
+      b0.AddDomainIntegrator(new DomainLFIntegrator(omega_f));
+      if (definite)
+      {
+         b1.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(neg_f));
+      }
+      else
+      {
+         b1.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(*f[0]));
+      }
+      b0.Assemble();
+      b1.Assemble();
+   }
+   A.SetSize(2,2); 
+   for (int i = 0; i<2; i++)
+   {
+      for (int j = 0; j<2; j++)
+      {
+         A[i][j] = nullptr;
+      }
+   }
+
+   ParBilinearForm a00(fes[0]);
+   a00.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a00.AddDomainIntegrator(new MassIntegrator(omeg2));
+   a00.Assemble();
+   if (system)
+   {
+      a00.EliminateEssentialBC(ess_bdr,x.GetBlock(0),rhs.GetBlock(0),mfem::Operator::DIAG_ONE);
+   }
+   else
+   {
+      a00.EliminateEssentialBC(ess_bdr,mfem::Operator::DIAG_ONE);
+   }
+   a00.Finalize();
+   A[0][0] = a00.ParallelAssemble();
+
+   ParMixedBilinearForm a01(fes[1],fes[0]);
+   if (definite)
+   {
+      a01.AddDomainIntegrator(new MixedScalarDivergenceIntegrator(negomeg));
+   }
+   else
+   {
+      a01.AddDomainIntegrator(new MixedScalarDivergenceIntegrator(omeg));
+   }
+   a01.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg));
+   a01.Assemble();
+   a01.EliminateTestDofs(ess_bdr);
+   a01.Finalize();
+   A[0][1] = a01.ParallelAssemble();
+
+   if (system)
+   {
+      ParMixedBilinearForm a10(fes[0],fes[1]);
+      if (definite)
+      {
+         a10.AddDomainIntegrator(new MixedScalarWeakGradientIntegrator(omeg));
+      }
+      else
+      {
+         a10.AddDomainIntegrator(new MixedScalarWeakGradientIntegrator(negomeg));
+      }
+      a10.AddDomainIntegrator(new MixedVectorGradientIntegrator(negomeg));
+      a10.Assemble();
+      a10.EliminateTrialDofs(ess_bdr,x.GetBlock(0),rhs.GetBlock(1));
+      a10.Finalize();
+      A[1][0] = a10.ParallelAssemble();
+   }
+   else
+   {
+      A[1][0] = A[0][1]->Transpose();
+   }
+
+   ParBilinearForm a11(fes[1]);
+   a11.AddDomainIntegrator(new DivDivIntegrator(one));
+   a11.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a11.Assemble();
+   a11.Finalize();
+   A[1][1] = a11.ParallelAssemble();
+
+
+   if (system)
+   {
+      fes[0]->GetRestrictionMatrix()->Mult(x.GetBlock(0), X.GetBlock(0));
+      fes[1]->GetRestrictionMatrix()->Mult(x.GetBlock(1), X.GetBlock(1));
+      fes[0]->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(0),Rhs.GetBlock(0));
+      fes[1]->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(1),Rhs.GetBlock(1));
+   }
+}

examples/maxwell-solver/FOSLS.hpp

@@ -0,0 +1,77 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+using namespace std;
+using namespace mfem;
+
+
+class ComplexMaxwellFOSLS 
+{
+private:
+   ParFiniteElementSpace * fes = nullptr;
+   ParMesh * pmesh = nullptr;
+   double omega = 1.0;
+   Array<VectorFunctionCoefficient *> loads;
+   Array<VectorFunctionCoefficient *> ess_data;
+   Array2D<HypreParMatrix *> A;
+   BlockVector x,rhs;
+   BlockVector X,Rhs;
+   Array<int> block_offsets;
+   Array<int> block_trueOffsets;
+
+   void FormSystem(bool system = true);
+public:
+   ComplexMaxwellFOSLS(ParFiniteElementSpace * fes_);
+   void SetOmega(double omega_) { omega = omega_; }
+   void SetLoadData(Array<VectorFunctionCoefficient *> & loads_);
+   void SetEssentialData(Array<VectorFunctionCoefficient *> & ess_data_);
+   void GetFOSLSLinearSystem(Array2D<HypreParMatrix *> & A_, 
+                             BlockVector & X_,
+                             BlockVector & Rhs_);
+   void GetFOSLSMatrix(Array2D<HypreParMatrix *> & A_);
+};
+
+
+   // -------------------------------------------------------------------
+   // |   |            p             |           u           |   RHS    | 
+   // -------------------------------------------------------------------
+   // | q | (gradp,gradq) + w^2(p,q) | w(divu,q)-w(u, gradq) |  w(f,q)  |
+   // |   |                          |                       |          |
+   // | v | w(p,divv) - w(gradp,v)   | (divu,divv) + w^2(u,v)| (f,divv) |
+
+class HelmholtzFOSLS 
+{
+private:
+   Array<ParFiniteElementSpace * > fes;
+   bool definite;
+   bool complex;
+   ParMesh * pmesh = nullptr;
+   double omega = 1.0;
+   int n; // 
+   Array<FunctionCoefficient * > f;
+   Array<VectorFunctionCoefficient * > Q;
+   Array<FunctionCoefficient * > p_ex_coeff;
+   Array<VectorFunctionCoefficient * > u_ex_coeff;
+   Array2D<HypreParMatrix *> A;
+   BlockVector x,rhs;
+   BlockVector X,Rhs;
+   Array<int> block_offsets;
+   Array<int> block_trueOffsets;
+
+   void FormSystem(bool system = true);
+   void Init();
+public:
+   HelmholtzFOSLS(Array<ParFiniteElementSpace * > & fes_, 
+                  bool definite_ = false,
+                  bool complex_ = false);
+   void SetOmega(double omega_) { omega = omega_; }
+   void SetLoadData(Array<FunctionCoefficient * > & f_);
+   void SetLoadData(Array<VectorFunctionCoefficient * > & Q_);
+   void SetEssentialData(Array<FunctionCoefficient * > & p_ex_coeff_);
+   void SetEssentialData(Array<VectorFunctionCoefficient * > & u_ex_coeff_);
+   void GetFOSLSLinearSystem(Array2D<HypreParMatrix *> & A_, 
+                             BlockVector & X_,
+                             BlockVector & Rhs_);
+   void GetFOSLSMatrix(Array2D<HypreParMatrix *> & A_);
+};

examples/maxwell-solver/FOSLS2D_maxwell.cpp

@@ -0,0 +1,310 @@
+// Example run: ./FOSLS2D_maxwell -ref 4  -o 3 -sol 1 -k 3.0
+
+//  ∇ × E - ω H = 0
+// -ω E + ∇ × H = J
+
+// --------------------------------------------------------------------------
+// |   |            E             |             H              |     RHS    | 
+// --------------------------------------------------------------------------
+// | F | (∇ × E,∇ × F)+ ω^2 (E,F) | - ω (∇ × H,F) - ω (H,curF) | - ω (J,F)  |
+// |   |                          |                            |            |
+// | G |-ω (E,∇ × G)-ω (∇ × E,G)  | (∇ × H,∇ × G)+ ω^2(H,G)    | (J,∇ × G)  |
+
+// for E in H1 (scalar) we have ∇ × E = [0 1;-1 0] ∇ E
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// Define exact solution
+double E_exact(const Vector &x);
+void H_exact(const Vector &x, Vector &H);
+double frhs(const Vector &x);
+void fvrhs(const Vector &x, Vector &f);
+void get_maxwell_solution(const Vector &x, double & E, Vector & curlE, double & curl2E);
+
+int dim;
+double omega;
+int isol = 0;
+
+int main(int argc, char *argv[])
+{
+   StopWatch chrono;
+
+   // 1. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../data/star.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // visualization flag
+   bool visualization = 1;
+   int ref = 1;
+   // number of wavelengths
+   double k = 0.6;
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&ref, "-ref", "--init-refinements",
+                  "Number of initial mesh refinements");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&isol, "-sol", "--solution",
+                  "Exact Solution: 0) Polynomial, 1) Sinusoidal.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   omega = 2.0 * M_PI * k;
+
+   // Mesh mesh(1, 1, Element::QUADRILATERAL, true, 1.0, 1.0, false);
+   Mesh mesh(mesh_file, 1, 1);
+
+   dim = mesh.Dimension();
+   if (dim == 3) {MFEM_ABORT("This is 2D Maxwell")};
+
+   for (int i = 0; i < ref; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   H1_FECollection H1fec(order,dim);
+   FiniteElementSpace H1fes(&mesh, &H1fec);
+
+   ND_FECollection NDfec(order, dim);
+   FiniteElementSpace NDfes(&mesh, &NDfec);
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      // Essential BC on E. Nothing on H
+      H1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+   Array<int> block_offsets(3);
+   block_offsets[0] = 0;
+   block_offsets[1] = H1fes.GetVSize();
+   block_offsets[2] = NDfes.GetVSize();
+   block_offsets.PartialSum();
+
+   BlockVector x(block_offsets), b(block_offsets);
+   x = 0.0;
+   b = 0.0;
+
+   FunctionCoefficient Eex(E_exact);
+   VectorFunctionCoefficient Hex(dim, H_exact);
+   GridFunction E_gf;
+   GridFunction H_gf;
+   E_gf.MakeRef(&H1fes, x.GetBlock(0));
+   E_gf.ProjectBdrCoefficient(Eex,ess_bdr);
+   H_gf.MakeRef(&NDfes, x.GetBlock(1));
+
+   FunctionCoefficient f(frhs);
+   ProductCoefficient f_E(-omega, f);
+   VectorFunctionCoefficient f_H(1,fvrhs);
+   LinearForm b_E;
+   b_E.Update(&H1fes, b.GetBlock(0), 0);
+   b_E.AddDomainIntegrator(new DomainLFIntegrator(f_E));
+   b_E.Assemble();
+
+   LinearForm b_H;
+   b_H.Update(&NDfes, b.GetBlock(1), 0);
+   b_H.AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(f_H));
+   b_H.Assemble();
+
+   // 7. Bilinear form a(.,.) on the finite element space
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient omeg2(pow(omega, 2));
+   ConstantCoefficient negomega(-(omega));
+   DenseMatrix mat(2);
+   mat(0,0) = 0.; mat(0,1) = 1.;
+   mat(1,0) = -1.; mat(1,1) = 0.;
+   MatrixConstantCoefficient rot(mat);
+
+   BilinearForm a_EE(&H1fes);
+   a_EE.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a_EE.AddDomainIntegrator(new MassIntegrator(omeg2));
+   a_EE.Assemble();
+   a_EE.EliminateEssentialBC(ess_bdr, x.GetBlock(0), b.GetBlock(0));
+   a_EE.Finalize();
+   SparseMatrix &A_EE = a_EE.SpMat();
+
+   ScalarMatrixProductCoefficient c1(-omega, rot);
+   MixedBilinearForm a_EH(&H1fes,&NDfes);
+   // - omega (rot grad E, G) - (omega E, curl G)
+   a_EH.AddDomainIntegrator(new MixedVectorGradientIntegrator(c1));
+   a_EH.AddDomainIntegrator(new MixedScalarWeakCurlIntegrator(negomega));
+   a_EH.Assemble();
+   a_EH.EliminateTrialDofs(ess_bdr, x.GetBlock(0), b.GetBlock(1));
+   a_EH.Finalize();
+   SparseMatrix &A_EH = a_EH.SpMat();
+   SparseMatrix * A_HE = Transpose(A_EH);
+
+   BilinearForm a_HH(&NDfes);
+   a_HH.AddDomainIntegrator(new CurlCurlIntegrator(one)); // one is the coeff
+   a_HH.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2)); // one is the coeff
+   a_HH.Assemble();
+   a_HH.Finalize();
+   SparseMatrix &A_HH = a_HH.SpMat();
+
+   BlockMatrix LS_Maxwellop(block_offsets);
+   LS_Maxwellop.SetBlock(0, 0, &A_EE);
+   LS_Maxwellop.SetBlock(0, 1, A_HE);
+   LS_Maxwellop.SetBlock(1, 0, &A_EH);
+   LS_Maxwellop.SetBlock(1, 1, &A_HH);
+
+   UMFPackSolver invE;
+   invE.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   invE.SetOperator(LS_Maxwellop.GetBlock(0,0));
+
+   UMFPackSolver invH;
+   invH.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   invH.SetOperator(LS_Maxwellop.GetBlock(1,1));
+
+   BlockDiagonalPreconditioner prec(block_offsets);
+   prec.SetDiagonalBlock(0, &invE);
+   prec.SetDiagonalBlock(1, &invH);
+
+   int maxit(5000);
+   double rtol(1.e-16);
+   double atol(0.0);
+
+   CGSolver pcg;
+   pcg.SetAbsTol(atol);
+   pcg.SetRelTol(rtol);
+   pcg.SetMaxIter(maxit);
+   pcg.SetOperator(LS_Maxwellop);
+   pcg.SetPreconditioner(prec);
+   pcg.SetPrintLevel(3);
+   pcg.Mult(b, x);
+
+   int order_quad = max(2, 2 * order + 1);
+   const IntegrationRule *irs[Geometry::NumGeom];
+   for (int i = 0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+
+   double Error_E = E_gf.ComputeL2Error(Eex, irs);
+   double Error_H = H_gf.ComputeL2Error(Hex, irs);
+   
+   cout << "|| E_h - E || = " << Error_E << "\n";
+   cout << "|| H_h - H || = " << Error_H << "\n";
+   cout << "Total error = " << sqrt(Error_H*Error_H+Error_E*Error_E) << "\n";
+
+   GridFunction E_exgf(&H1fes);
+   E_exgf.ProjectCoefficient(Eex);
+
+   GridFunction H_exgf(&NDfes);
+   H_exgf.ProjectCoefficient(Hex);
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int visport = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      socketstream ex_sock(vishost, visport);
+      ex_sock.precision(8);
+      socketstream sol_sockH(vishost, visport);
+      sol_sockH.precision(8);
+      socketstream ex_sockH(vishost, visport);
+      ex_sockH.precision(8);
+      sol_sock << "solution\n"
+               << mesh << E_gf << "window_title 'Numerical E'" << "keys rRljc\n"
+               << flush;
+      ex_sock << "solution\n"
+               << mesh << E_exgf << "window_title 'Exact E'" << "keys rRljc\n"
+               << flush;
+      sol_sockH << "solution\n"
+               << mesh << H_gf << "window_title 'Numerical H'" << "keys rRljc\n"
+               << flush;
+      ex_sockH << "solution\n"
+               << mesh << H_exgf << "window_title 'Exact H'" << "keys rRljc\n"
+               << flush;        
+   }
+   delete A_HE;
+   return 0;
+}
+
+double E_exact(const Vector &x)
+{
+   double E, curl2E;
+   Vector curlE(2);
+   get_maxwell_solution(x, E, curlE, curl2E);
+   return E;  //Scalar
+}
+
+//define exact solution
+void H_exact(const Vector &x, Vector &H)
+{
+   double E, curl2E;
+   Vector curlE(2);
+   get_maxwell_solution(x, E, curlE, curl2E);
+   H[0] = curlE[0]/omega;
+   H[1] = curlE[1]/omega;
+}
+
+double frhs(const Vector &x)
+{
+   double E, curl2E;
+   Vector curlE(2);
+   get_maxwell_solution(x, E, curlE, curl2E);
+
+   // - omega E + curl H = f
+   // - omega E + curl (curl E) / omega = f
+   double f = - omega * E + curl2E / omega;
+   return f;
+}
+
+void fvrhs(const Vector &x, Vector &f)
+{
+   double E, curl2E;
+   Vector curlE(2);
+   get_maxwell_solution(x, E, curlE, curl2E);
+   f[0] = - omega * E + curl2E / omega;
+}
+
+
+void get_maxwell_solution(const Vector &X, double & E, Vector & curlE, double & curl2E)
+{
+   double x = X[0];
+   double y = X[1];
+   double Ex, Ey, Exx, Eyy;
+   if (isol == 0) // polynomial
+   {
+      E = x * (1.0 - x) * y * (1.0 - y);
+      Ex = (1.0 - 2.0 * x) * y * (1.0 - y);
+      Ey = x * (1.0 - x) * (1.0 - 2.0 * y);
+
+      Exx = -2.0 * y * (1.0 - y);
+      Eyy = -2.0 * x * (1.0 - x);
+   }
+   else 
+   {
+      double s = omega * (y+x);
+      E = cos(s);
+      Ex = -omega * sin(s);  
+      Ey = Ex;
+      Exx = - omega * omega * E;  
+      Eyy = Exx;  
+   }
+   curlE[0] = Ey;
+   curlE[1] = -Ex;
+   curl2E = -Exx - Eyy;
+}

examples/maxwell-solver/LS-LOR-results.txt

@@ -0,0 +1,42 @@
+./LS-helmholtzp_lor  -o 4  -m ../../data/inline-quad.mesh
+
+
+
+
+omega/2pi |   dof_H1  | dof_RT  |    H1_err    |    Hdiv_err  |   Exact LOR   |  Inexact LOR  |   AMG/AMS ho  |
+--------------------------------------------------------------------------------------------------------------   
+5         |   1089    | 2112    | 7.5044e-02   |  6.8128e-02  | 47 (0.551404) | 180 (0.16153) | 155 (0.914495)
+10        |   4225    | 8320    | 7.2461e-02   |  6.6677e-02  | 64 (2.23832)  | 197 (0.931944)| 164 (0.918901)
+20        |   16641   | 33024   | 7.1488e-02   |  6.6131e-02  | 79 (9.42402)  | 292 (4.14552) | 246 (11.3769)
+40        |   66049   | 131584  | 6.9699e-02   |  6.5160e-02  | 87 (39.8902)  | 380 (25.3352) | 251 (48.977)
+80        |  263169   | 525312  | 6.7949e-02   |  6.4224e-02  | 98 (179.461)  | 401 (114.649) | 316 (271.633)
+160       |  1050625  | 2099200 | 6.7005e-02   |  6.3724e-02  | 96 (1791.43)  | 377 (569.546) | 299 (1277.63) 
+
+
+omega/2pi |   dof_H1  |   H1_err |   Exact LOR  |
+--------------------------------------------------------------------------------------------------------------   
+5         |    1089   | 4.2337e-02 |  16 (0.176056)
+10        |    4225   | 4.2329e-02 |  30 (1.02578)
+20        |   16641   | 4.2328e-02 |  209 (16.7772)
+40        |   66049   | 4.2327e-02 | >2000 (600.249)
+80        |  263169   |     *      | >2000 (2818.08) 
+160       | 1050625   |     *      |   
+
+
+srun -n 4 ./LS_maxwellp -rnum 2.0 -o 3 -sr 2 -pr 0 -no-vis -m ../data/inline-hex.mesh -solution 1
+omega/2pi |   dof (x4) |  L2 err rel | ranks |  AMG/AMS ho  |
+--------------------------------------------------------------------------------------------------------------   
+2         |   45000    |  0.0129377  |    4  |  143 (31.2249)|
+4         |   345744   |  0.0345918  |   32  |  262 (108.072)| 
+8         |  2709792   |  0.0547445  |  256  |  354 (244.849)| 
+16        |
+
+// Complex indefinite Helmholtz
+./LS_complex_helmholtzp -o 5 -solution 1 -sr 0 -rnum 4.0
+omega/2pi |   dof_H1  | dof_RT  |    H1_err    |    Hdiv_err  |   Exact LOR   |  Inexact LOR  |   AMG/AMS ho  |
+--------------------------------------------------------------------------------------------------------------   
+4         |   1681    | 3280    | 1.0107e-03   |  8.8146e-04  |  71 (2.2144)  | 426 (1.35841) | 329 (3.65064) 
+8         |   6561    | 12960   | 1.4215e-03   |  1.2339e-03  | 128 (13.258)  | 625 (8.7082)  | 481 (24.1927)
+16        |   25921   | 51520   | 2.3111e-02   |  9.9057e-03  | 239 (88.5967) |1045 (57.6934) | 724 (149.518)
+32        |   103041  | 205440  | 1.4054e-02   |  1.4147e-02  | 286 (408.915) |1153 (279.299) | 860 (748.056)
+64        |   410881  | 820480  | 7.9836e-01   |  2.8242e-02  | 574 (3819.05) |2798 (2860.15) | 1974 (6881.4)

examples/maxwell-solver/LS-helmholtzp.cpp

@@ -0,0 +1,378 @@
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+void gradp_exact(const Vector &x, Vector &gradu);
+double divu_exact(const Vector &x);
+double d2_exact(const Vector &x);
+
+int dim;
+double omega;
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = 1;
+   int sr = 1;
+   int pr = 1;
+   double rnum=1.0;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&sr, "-sr", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pr", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");                  
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+
+
+   omega = 2.0 * M_PI * rnum;
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution.
+   for (int i = 0; i < sr; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   // 6. Define a parallel finite element space on the parallel mesh.
+   FiniteElementCollection *H1fec = new H1_FECollection(order,dim); 
+   ParFiniteElementSpace *H1fespace = new ParFiniteElementSpace(pmesh, H1fec);
+
+   FiniteElementCollection *RTfec = new RT_FECollection(order,dim); 
+   ParFiniteElementSpace *RTfespace = new ParFiniteElementSpace(pmesh, RTfec);
+
+
+   // -------------------------------------------------------------------
+   // |   |            p             |           u           |   RHS    | 
+   // -------------------------------------------------------------------
+   // | q | (gradp,gradq) + w^2(p,q) | w(divu,q)-w(u, gradq) |  w(f,q)  |
+   // |   |                          |                       |          |
+   // | v | w(p,divv) - w(gradp,v)   | (divu,divv) + w^2(u,v)| (f,divv) |
+
+
+   // omega(f,q) 
+   ParLinearForm b_q(H1fespace);
+   ConstantCoefficient omeg(omega);
+   FunctionCoefficient f_rhs(rhs_func);
+   ProductCoefficient omega_f(omeg,f_rhs);
+   b_q.AddDomainIntegrator(new DomainLFIntegrator(omega_f));
+   // (f, div v)
+   ParLinearForm b_v(RTfespace);
+#ifdef DEFINITE
+   ConstantCoefficient negone(-1.0);
+   ProductCoefficient neg_f(negone,f_rhs);
+   b_v.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(neg_f));
+#else    
+   b_v.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(f_rhs));
+#endif
+
+   ParBilinearForm a_qp(H1fespace);
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negomeg(-omega);
+   ConstantCoefficient omeg2(omega*omega);
+   // (grad p, grad q) + \omega^2 (p,q)
+   a_qp.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a_qp.AddDomainIntegrator(new MassIntegrator(omeg2));
+
+   ParMixedBilinearForm a_qu(RTfespace, H1fespace);
+#ifdef DEFINITE
+   // -w(divu,q)
+   a_qu.AddDomainIntegrator(new MixedScalarDivergenceIntegrator(negomeg));
+#else   
+   // w(divu,q)
+   a_qu.AddDomainIntegrator(new MixedScalarDivergenceIntegrator(omeg));
+#endif
+   // -w(u, gradq)
+   a_qu.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg));
+   // w(p,divv) - w(gradp,v)
+   ParMixedBilinearForm a_vp(H1fespace, RTfespace);
+#ifdef DEFINITE
+   // -w(p,divv)
+   a_vp.AddDomainIntegrator(new MixedScalarWeakGradientIntegrator(omeg));
+#else
+   // w(p,divv)
+   a_vp.AddDomainIntegrator(new MixedScalarWeakGradientIntegrator(negomeg));
+#endif
+   // - w(gradp,v)
+   a_vp.AddDomainIntegrator(new MixedVectorGradientIntegrator(negomeg));
+
+   ParBilinearForm a_vu(RTfespace);
+   a_vu.AddDomainIntegrator(new DivDivIntegrator(one));
+   a_vu.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+
+
+   ConvergenceStudy ratesH1;
+   ConvergenceStudy ratesRT;
+   FunctionCoefficient p_ex(p_exact);
+   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
+   VectorFunctionCoefficient u_ex(dim,u_exact);
+   FunctionCoefficient divu_ex(divu_exact);
+   ParGridFunction p_gf, u_gf;
+
+   for (int l = 0; l <= pr; l++)
+   {
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh->bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+         ess_bdr = 1;
+         H1fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      Array<int> block_offsets(3);
+      block_offsets[0] = 0;
+      block_offsets[1] = H1fespace->GetVSize();
+      block_offsets[2] = RTfespace->GetVSize();
+      block_offsets.PartialSum();
+
+      Array<int> block_trueOffsets(3);
+      block_trueOffsets[0] = 0;
+      block_trueOffsets[1] = H1fespace->TrueVSize();
+      block_trueOffsets[2] = RTfespace->TrueVSize();
+      block_trueOffsets.PartialSum();
+
+      BlockVector x(block_offsets), rhs(block_offsets);
+      BlockVector trueX(block_trueOffsets), trueRhs(block_trueOffsets);
+      x = 0.0;  rhs = 0.0;
+      trueX = 0.0;  trueRhs = 0.0;
+
+      p_gf.MakeRef(H1fespace,x.GetBlock(0));
+      p_gf.ProjectBdrCoefficient(p_ex,ess_bdr);
+
+      u_gf.MakeRef(RTfespace,x.GetBlock(1));
+      u_gf = 0.0;
+
+      b_q.Update(H1fespace,rhs.GetBlock(0),0);
+      b_q.Assemble();
+
+      b_v.Update(RTfespace,rhs.GetBlock(1),0);
+      b_v.Assemble();
+
+
+      a_qp.Assemble();
+      a_qp.EliminateEssentialBC(ess_bdr,x.GetBlock(0),rhs.GetBlock(0));
+      a_qp.Finalize();
+      HypreParMatrix * A_qp = a_qp.ParallelAssemble();
+
+      a_qu.Assemble();
+      a_qu.EliminateTestDofs(ess_bdr);
+      a_qu.Finalize();
+      HypreParMatrix * A_qu = a_qu.ParallelAssemble();
+
+
+      a_vp.Assemble();
+      a_vp.EliminateTrialDofs(ess_bdr,x.GetBlock(0),rhs.GetBlock(1));
+      a_vp.Finalize();
+      HypreParMatrix * A_vp = a_vp.ParallelAssemble();
+
+      a_vu.Assemble();
+      a_vu.Finalize();
+      HypreParMatrix * A_vu = a_vu.ParallelAssemble();
+
+
+
+      H1fespace->GetRestrictionMatrix()->Mult(x.GetBlock(0), trueX.GetBlock(0));
+      H1fespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(0),trueRhs.GetBlock(0));
+
+      RTfespace->GetRestrictionMatrix()->Mult(x.GetBlock(1), trueX.GetBlock(1));
+      RTfespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(1),trueRhs.GetBlock(1));
+
+
+      Array2D<HypreParMatrix *> Ah(2,2);
+      Ah[0][0] = A_qp; 
+      Ah[0][1] = A_qu;
+      Ah[1][0] = A_vp;
+      Ah[1][1] = A_vu;
+      HypreParMatrix * A = HypreParMatrixFromBlocks(Ah);
+
+      HypreBoomerAMG amg_p(*A_qp);
+      amg_p.SetPrintLevel(0);
+
+      Solver *prec = nullptr;
+      if (dim == 2) 
+      {
+         prec = new HypreAMS(*A_vu,RTfespace);
+         dynamic_cast<HypreAMS *>(prec)->SetPrintLevel(0);
+      }
+      else
+      {
+         prec = new HypreADS(*A_vu,RTfespace);
+         dynamic_cast<HypreADS *>(prec)->SetPrintLevel(0);
+      }
+
+      BlockDiagonalPreconditioner M(block_trueOffsets);
+      // BlockDiagonalMultiplicativePreconditioner M(block_trueOffsets);
+      // M.SetOperator(*A);
+      M.SetDiagonalBlock(0,&amg_p);
+      ScaledOperator S(prec,1.0);
+      M.SetDiagonalBlock(1,&S);
+
+      StopWatch chrono;
+      chrono.Clear();
+      chrono.Start();
+      // GMRESSolver cg(MPI_COMM_WORLD);
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-6);
+      // cg.SetAbsTol(1e-6);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(1);
+      cg.SetPreconditioner(M);
+      cg.SetOperator(*A);
+      cg.Mult(trueRhs, trueX);
+      delete prec;
+      chrono.Stop();
+      cout << "PCG time " << chrono.RealTime() << endl;
+
+      chrono.Clear();
+      chrono.Start();
+      MUMPSSolver mumps;
+      mumps.SetPrintLevel(0);
+      mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
+      mumps.SetOperator(*A);
+      Vector trueY(trueX.Size());
+      mumps.Mult(trueRhs,trueY);
+      chrono.Stop();
+      cout << "MUMPS time " << chrono.RealTime() << endl;
+
+
+
+
+      delete A;
+      delete A_vu;
+      delete A_qp;
+      delete A_vp;
+      delete A_qu;
+
+      p_gf = 0.0;
+      u_gf = 0.0;
+      p_gf.Distribute(&(trueX.GetBlock(0)));
+      u_gf.Distribute(&(trueX.GetBlock(1)));
+
+      ratesH1.AddH1GridFunction(&p_gf,&p_ex,&gradp_ex);
+      ratesRT.AddHdivGridFunction(&u_gf,&u_ex,&divu_ex);
+
+      if (l==pr) break;
+
+      pmesh->UniformRefinement();
+      H1fespace->Update();
+      RTfespace->Update();
+      a_qp.Update();
+      a_qu.Update();
+      a_vp.Update();
+      a_vu.Update();
+      b_q.Update();
+      b_v.Update();
+      p_gf.Update();
+      u_gf.Update();
+   }
+   ratesH1.Print(true);
+   ratesRT.Print(true);
+
+   // 10. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << p_gf <<
+               "window_title 'Numerical Pressure (real part)' "
+               << flush;
+   }
+
+   // // 11. Free the used memory.
+   delete H1fespace;
+   delete RTfespace;
+   delete H1fec;
+   delete RTfec;
+   delete pmesh;
+   MPI_Finalize();
+   return 0;
+}
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return divu + omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   return sin(omega*x.Sum());
+}
+
+void gradp_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   grad = omega * cos(omega * x.Sum());
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   gradp_exact(x,u);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   return d2_exact(x)/omega;
+}
+
+double d2_exact(const Vector &x)
+{
+   return -dim * omega * omega * sin(omega*x.Sum());
+}

examples/maxwell-solver/LS-helmholtzp_lor.cpp

@@ -0,0 +1,310 @@
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "FOSLS.hpp"
+#include "lor.hpp"
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+
+
+
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+void gradp_exact(const Vector &x, Vector &gradu);
+double divu_exact(const Vector &x);
+double d2_exact(const Vector &x);
+
+#ifdef DEFINITE   
+   bool definite = true;
+#else
+   bool definite = false;
+#endif
+int dim;
+double omega;
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = 1;
+   int sr = 1;
+   int pr = 1;
+   double rnum=1.0;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&sr, "-sr", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pr", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");                  
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   omega = 2.0 * M_PI * rnum;
+
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution.
+   for (int i = 0; i < sr; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   int btype = BasisType::GaussLobatto;
+   ParMesh pmesh_lor(pmesh, order, btype);
+
+   unique_ptr<FiniteElementCollection> H1fec_ho, H1fec_lor;
+   unique_ptr<FiniteElementCollection> RTfec_ho, RTfec_lor;
+
+   H1fec_ho.reset(new H1_FECollection(order, dim));
+   H1fec_lor.reset(new H1_FECollection(1, dim));
+   RTfec_ho.reset(new RT_FECollection(order-1, dim, BasisType::GaussLobatto, BasisType::Integrated));
+   RTfec_lor.reset(new RT_FECollection(0, dim, BasisType::GaussLobatto, BasisType::Integrated));
+     
+   ParFiniteElementSpace H1fes_ho(pmesh, H1fec_ho.get());
+   ParFiniteElementSpace H1fes_lor(&pmesh_lor, H1fec_lor.get());
+   ParFiniteElementSpace RTfes_ho(pmesh, RTfec_ho.get());
+   ParFiniteElementSpace RTfes_lor(&pmesh_lor, RTfec_lor.get());
+
+   Array<int> block_trueOffsets(3);
+   block_trueOffsets[0] = 0;
+   block_trueOffsets[1] = H1fes_ho.TrueVSize();
+   block_trueOffsets[2] = RTfes_ho.TrueVSize();
+   block_trueOffsets.PartialSum();
+
+   BlockVector trueX(block_trueOffsets), trueRhs(block_trueOffsets);
+   trueX = 0.0;  trueRhs = 0.0;
+
+   FunctionCoefficient p_ex(p_exact);
+   FunctionCoefficient f_rhs(rhs_func);
+   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
+   VectorFunctionCoefficient u_ex(dim,u_exact);
+   FunctionCoefficient divu_ex(divu_exact);
+
+   Vector trueY(trueX);
+   Vector trueZ(trueX);
+
+   Array<ParFiniteElementSpace *> fes_ho(2);
+   fes_ho[0] = &H1fes_ho;
+   fes_ho[1] = &RTfes_ho;
+
+   HelmholtzFOSLS ho_system(fes_ho,definite);
+
+   ho_system.SetOmega(omega);
+   Array<FunctionCoefficient * > F_rhs(1);
+   F_rhs[0] = &f_rhs;
+   ho_system.SetLoadData(F_rhs);
+
+   Array<FunctionCoefficient * > P_ex(1);
+   P_ex[0] = &p_ex;
+   ho_system.SetEssentialData(P_ex);
+
+   Array<ParFiniteElementSpace *> fes_lor(2);
+   fes_lor[0] = &H1fes_lor;
+   fes_lor[1] = &RTfes_lor;
+   HelmholtzFOSLS lor_system(fes_lor,definite);
+   lor_system.SetOmega(omega);
+
+   Array2D<HypreParMatrix *> Ah_ho(2,2);
+   ho_system.GetFOSLSLinearSystem(Ah_ho,trueX,trueRhs);
+
+   Array2D<HypreParMatrix *> Ah_lor(2,2);
+   lor_system.GetFOSLSMatrix(Ah_lor);
+   HypreParMatrix * A_ho = HypreParMatrixFromBlocks(Ah_ho);
+   HypreParMatrix * A_lor = HypreParMatrixFromBlocks(Ah_lor);
+
+   HypreBoomerAMG * amg_p = new HypreBoomerAMG(*Ah_ho[0][0]);
+   amg_p->SetPrintLevel(0);
+   HypreBoomerAMG * amg_lor_p = new HypreBoomerAMG(*Ah_lor[0][0]);
+   amg_lor_p->SetPrintLevel(0);
+
+   Solver *prec = nullptr;
+   Solver *prec_lor = nullptr;
+   if (dim == 2) 
+   {
+      prec = new HypreAMS(*Ah_ho[1][1],&RTfes_ho);
+      dynamic_cast<HypreAMS *>(prec)->SetPrintLevel(0);
+      prec_lor = new HypreAMS(*Ah_lor[1][1],&RTfes_lor);
+      dynamic_cast<HypreAMS *>(prec_lor)->SetPrintLevel(0);
+   }
+   else
+   {
+      prec = new HypreADS(*Ah_ho[1][1],&RTfes_ho);
+      dynamic_cast<HypreADS *>(prec)->SetPrintLevel(0);
+      prec_lor = new HypreADS(*Ah_lor[1][1],&RTfes_lor);
+      dynamic_cast<HypreADS *>(prec_lor)->SetPrintLevel(0);
+   }
+
+   BlockDiagonalPreconditioner M(block_trueOffsets);
+   BlockDiagonalPreconditioner M_lor2(block_trueOffsets);
+
+   FiniteElement::MapType t = FiniteElement::H_DIV;
+   Array<int> perm = ComputeVectorFE_LORPermutation(RTfes_ho, RTfes_lor, t);
+   
+   RealLORSolver M_lor(*A_lor, perm);
+   M.SetDiagonalBlock(0,amg_p);
+   ScaledOperator S(prec,1.0);
+   M.SetDiagonalBlock(1,&S);
+
+   M_lor2.SetDiagonalBlock(0,amg_lor_p);
+   ScaledOperator S_lor(prec_lor,1.0);
+   M_lor2.SetDiagonalBlock(1,&S_lor);
+
+   RealLORSolver M_lor_inexact(*A_lor, perm, false, &M_lor2);
+
+   StopWatch chrono;
+   chrono.Clear();
+   chrono.Start();
+   // GMRESSolver cg(MPI_COMM_WORLD);
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-6);
+   // cg.SetAbsTol(1e-6);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetOperator(*A_ho);
+   // cg.SetPreconditioner(M);
+   cg.SetPreconditioner(M_lor);
+   cg.Mult(trueRhs, trueX);
+   chrono.Stop();
+   cout << "LOR exact - PCG time " << chrono.RealTime() << endl;
+
+   chrono.Clear();
+   chrono.Start();
+   cg.SetPreconditioner(M_lor_inexact);
+   cg.Mult(trueRhs, trueY);
+
+   chrono.Stop();
+   cout << "LOR inexact PCG time " << chrono.RealTime() << endl;
+
+   chrono.Clear();
+   chrono.Start();
+   cg.SetPreconditioner(M);
+   cg.Mult(trueRhs, trueZ);
+   
+
+   chrono.Stop();
+   cout << "AMG/AMS PCG time " << chrono.RealTime() << endl;
+
+   for (int i = 0; i<2; i++)
+   {
+      for (int j = 0; j<2; j++)
+      {
+         delete Ah_ho[i][j];
+         delete Ah_lor[i][j];
+      }
+   }
+
+   ParGridFunction p_gf(&H1fes_ho);
+   ParGridFunction u_gf(&RTfes_ho);
+   ParGridFunction p_zero(&H1fes_ho);
+   ParGridFunction u_zero(&RTfes_ho);
+   p_gf = 0.0; p_zero = 0.0;
+   u_gf = 0.0; u_zero = 0.0;
+   p_gf.Distribute(&(trueX.GetBlock(0)));
+   u_gf.Distribute(&(trueX.GetBlock(1)));
+
+   double H1_error = p_gf.ComputeH1Error(&p_ex,&gradp_ex);
+   double H1_norm  = p_zero.ComputeH1Error(&p_ex,&gradp_ex);
+   double Hdiv_error = u_gf.ComputeHDivError(&u_ex,&divu_ex);
+   double Hdiv_norm = u_zero.ComputeHDivError(&u_ex,&divu_ex);
+
+
+   if (myid == 0)
+   {
+      cout << "H1 rel error     = " << H1_error/H1_norm << endl;
+      cout << "H(div) rel error = " << Hdiv_error/Hdiv_norm << endl;
+   }
+
+   // 10. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << p_gf <<
+               "window_title 'Numerical Pressure (real part)' "
+               << flush;
+   }
+
+   // // 11. Free the used memory.
+   delete amg_lor_p;
+   delete amg_p;
+   delete prec;
+   delete prec_lor;
+   delete pmesh;
+   MPI_Finalize();
+   return 0;
+}
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return divu + omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   return sin(omega*x.Sum());
+}
+
+void gradp_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   grad = omega * cos(omega * x.Sum());
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   gradp_exact(x,u);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   return d2_exact(x)/omega;
+}
+
+double d2_exact(const Vector &x)
+{
+   return -dim * omega * omega * sin(omega*x.Sum());
+}

examples/maxwell-solver/LS_complex_helmholtzp.cpp

@@ -0,0 +1,734 @@
+
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "FOSLS.hpp"
+#include "lor.hpp"
+
+using namespace std;
+using namespace mfem;
+
+int dim;
+double omega;
+int exact = 0;
+
+void helmholtz_solution(const Vector &x, complex<double> & sol, 
+                        std::vector<complex<double>> & grad,
+                        complex<double> & grad2);
+
+double p_exact_re(const Vector &x);
+void u_exact_re(const Vector &x, Vector &u);
+double p_exact_im(const Vector &x);
+void u_exact_im(const Vector &x, Vector &u);
+void gradp_exact_re(const Vector &x, Vector &gradu);
+double divu_exact_re(const Vector &x);
+void gradp_exact_im(const Vector &x, Vector &gradu);
+double divu_exact_im(const Vector &x);
+
+
+void f_exact_re(const Vector &x, Vector &f);
+double g_exact_re(const Vector &x);
+void f_exact_im(const Vector &x, Vector &f);
+double g_exact_im(const Vector &x);
+void plotfield(socketstream &,ParMesh * pmesh,const ParGridFunction & , string &);
+
+// ----------------------------------------------------------------------
+// |   |            p             |             u           |    RHS    | 
+// ----------------------------------------------------------------------
+// | q | (grad p,grad q)+w^2(p,q) |-iw(div u,q)+iw(u,grad q)| -iw(f,q)  |
+// |   |                          |                         |           |
+// | v | iw(p,div v)-iw(grad p,v) | (div u,div v)+w^2(u,v)  | (f,div v) |
+// ----------------------------------------------------------------------
+int main(int argc, char *argv[])
+{
+    // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = 1;
+   int sr = 1;
+   int pr = 1;
+   double rnum=1.0;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&sr, "-sr", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pr", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");                  
+   args.AddOption(&exact, "-solution", "--exact_solution",
+                  "Exact solution : 0-polynomial, 1-plane wave");                  
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   omega = 2.0 * M_PI * rnum;
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+
+   dim = mesh->Dimension();
+
+   for (int i = 0; i < sr; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   for (int i = 0; i < pr; i++ )
+   {
+      pmesh->UniformRefinement();
+   }
+
+   int btype = BasisType::GaussLobatto;
+   ParMesh pmesh_lor(pmesh, order, btype);
+
+   unique_ptr<FiniteElementCollection> H1fec_ho, H1fec_lor;
+   unique_ptr<FiniteElementCollection> RTfec_ho, RTfec_lor;
+
+   H1fec_ho.reset(new H1_FECollection(order, dim));
+   H1fec_lor.reset(new H1_FECollection(1, dim));
+   RTfec_ho.reset(new RT_FECollection(order-1, dim, BasisType::GaussLobatto, BasisType::Integrated));
+   RTfec_lor.reset(new RT_FECollection(0, dim, BasisType::GaussLobatto, BasisType::Integrated));
+   
+   ParFiniteElementSpace H1fes_ho(pmesh, H1fec_ho.get());
+   ParFiniteElementSpace H1fes_lor(&pmesh_lor, H1fec_lor.get());
+   ParFiniteElementSpace RTfes_ho(pmesh, RTfec_ho.get());
+   ParFiniteElementSpace RTfes_lor(&pmesh_lor, RTfec_lor.get());
+
+   HYPRE_Int H1size = H1fes_ho.GlobalTrueVSize();
+   HYPRE_Int RTsize = RTfes_ho.GlobalTrueVSize();
+
+   if (myid == 0)
+   {
+      cout << "Number of H1 True Dofs = " << H1size << endl;
+      cout << "Number of RT True Dofs = " << RTsize << endl;
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (pmesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      H1fes_ho.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   FunctionCoefficient p_ex_re(p_exact_re);
+   VectorFunctionCoefficient u_ex_re(dim,u_exact_re);
+   FunctionCoefficient p_ex_im(p_exact_im);
+   VectorFunctionCoefficient u_ex_im(dim,u_exact_im);
+
+   VectorFunctionCoefficient f_ex_re(dim,f_exact_re);
+   FunctionCoefficient g_ex_re(g_exact_re);
+   VectorFunctionCoefficient f_ex_im(dim,f_exact_im);
+   FunctionCoefficient g_ex_im(g_exact_im);
+
+   int n0 = H1fes_ho.GetVSize();
+   int N0 = H1fes_ho.GetTrueVSize();
+   int n1 = RTfes_ho.GetVSize();
+   int N1 = RTfes_ho.GetTrueVSize();
+   Array<int> block_offsets(5);
+   block_offsets[0] = 0;
+   block_offsets[1] = n0;
+   block_offsets[2] = n1;
+   block_offsets[3] = n0;
+   block_offsets[4] = n1;
+   block_offsets.PartialSum();
+
+   Array<int> block_trueOffsets(5);
+   block_trueOffsets[0] = 0;
+   block_trueOffsets[1] = N0;
+   block_trueOffsets[2] = N1;
+   block_trueOffsets[3] = N0;
+   block_trueOffsets[4] = N1;
+   block_trueOffsets.PartialSum();
+
+   BlockVector x(block_offsets), rhs(block_offsets);
+   BlockVector X(block_trueOffsets), Rhs(block_trueOffsets);
+   x = 0.0;  rhs = 0.0; X = 0.0;  Rhs = 0.0;
+
+   ParGridFunction p_gf_re, p_gf_im, u_gf_re, u_gf_im;
+
+   p_gf_re.MakeRef(&H1fes_ho,x.GetBlock(0)); p_gf_re = 0.0;
+   u_gf_re.MakeRef(&RTfes_ho,x.GetBlock(1)); u_gf_re = 0.0;
+   p_gf_im.MakeRef(&H1fes_ho,x.GetBlock(2)); p_gf_im = 0.0;
+   u_gf_im.MakeRef(&RTfes_ho,x.GetBlock(3)); u_gf_im = 0.0;
+
+   // E_gf_re.ProjectBdrCoefficientTangent(E_ex_re,ess_bdr);
+   // E_gf_im.ProjectBdrCoefficientTangent(E_ex_im,ess_bdr);
+   p_gf_re.ProjectCoefficient(p_ex_re);
+   p_gf_im.ProjectCoefficient(p_ex_im);
+
+// ----------------------------------------------------------------------
+// |   |            p             |             u           |    RHS    | 
+// ----------------------------------------------------------------------
+// | q | (grad p,grad q)+w^2(p,q) |-iw(div u,q)+iw(u,grad q)| -iw(g,q)  |
+// |   |                          |                         |           |
+// | v | iw(p,div v)-iw(grad p,v) | (div u,div v)+w^2(u,v)  | (g,div v) |
+// ----------------------------------------------------------------------
+
+   // for convinience we convert the above 2 x 2 blocks to 4 x 4 in order
+   // to accomodate complex valued operators
+
+   // A = (grad p,grad q)+w^2(p,q)
+   // B = (div u,div v)+w^2(u,v)
+   // C = -w(div u,q) + w(u,grad q)
+   // D = w(p,div v)-w(grad p,v)
+   // b0 = w(g_im,q)
+   // b1 = (g_re,div v)
+   // b2 = -w(g_re,q)
+   // b3 = (g_im,div v)
+
+   // | A   0   0  -C |  | p_re |     | b0 |
+   // | 0   B  -D   0 |  | u_re |  =  | b1 |
+   // | 0   C   A   0 |  | p_im |     | b2 |
+   // | D   0   0   B |  | u_im |     | b3 |
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient negomeg(-omega);
+   ConstantCoefficient omeg2(omega * omega);
+
+   ProductCoefficient wgi(omeg,g_ex_im);
+   ProductCoefficient negwgr(negomeg,g_ex_re);
+
+   ParLinearForm b0, b1, b2, b3;
+   b0.Update(&H1fes_ho,rhs.GetBlock(0),0);
+   b1.Update(&RTfes_ho,rhs.GetBlock(1),0);
+   b2.Update(&H1fes_ho,rhs.GetBlock(2),0);
+   b3.Update(&RTfes_ho,rhs.GetBlock(3),0);
+
+
+   b0.AddDomainIntegrator(new DomainLFIntegrator(wgi));
+   b1.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(g_ex_re));
+   b2.AddDomainIntegrator(new DomainLFIntegrator(negwgr));
+   b3.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(g_ex_im));
+   
+   b0.Assemble();
+   b1.Assemble();
+   b2.Assemble();
+   b3.Assemble();
+
+   Array2D<HypreParMatrix *> Ah(4,4); 
+   for (int i = 0; i<4; i++)
+   {
+      for (int j = 0; j<4; j++)
+      {
+         Ah[i][j] = nullptr;
+      }
+   }
+
+   // A00 = (grad p,grad q)+w^2(p,q)
+   ParBilinearForm a00(&H1fes_ho);
+   a00.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a00.AddDomainIntegrator(new MassIntegrator(omeg2));
+   a00.Assemble();
+   a00.EliminateEssentialBC(ess_bdr,x.GetBlock(0),rhs.GetBlock(0),mfem::Operator::DIAG_ONE);
+   a00.Finalize();
+   Ah[0][0] = a00.ParallelAssemble();
+
+   // -C = w(div u,q) - w(u,grad q)
+   ParMixedBilinearForm a03(&RTfes_ho,&H1fes_ho);
+   // w(divu,q)
+   a03.AddDomainIntegrator(new MixedScalarDivergenceIntegrator(omeg));
+   // -w(u, gradq)
+   a03.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg));
+   a03.Assemble();
+   a03.EliminateTestDofs(ess_bdr);
+   a03.Finalize();
+   Ah[0][3] = a03.ParallelAssemble();
+
+   // A11 = (div u,div v)+w^2(u,v)
+   ParBilinearForm a11(&RTfes_ho);
+   a11.AddDomainIntegrator(new DivDivIntegrator(one));
+   a11.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a11.Assemble();
+   a11.Finalize();
+   Ah[1][1] = a11.ParallelAssemble();
+
+   // A12 = -w(p,div v)+w(grad p,v)
+   ParMixedBilinearForm a12(&H1fes_ho,&RTfes_ho);
+   // -w(p,divv)
+   a12.AddDomainIntegrator(new MixedScalarWeakGradientIntegrator(omeg));
+   // w(grad p,v)
+   a12.AddDomainIntegrator(new MixedVectorGradientIntegrator(omeg));
+   a12.Assemble();
+   a12.EliminateTrialDofs(ess_bdr,x.GetBlock(2),rhs.GetBlock(1));
+   a12.Finalize();
+   Ah[1][2] = a12.ParallelAssemble();
+
+   // A21 = -w(div u,q) + w(u,grad q)
+   ParMixedBilinearForm a21(&RTfes_ho,&H1fes_ho);
+   // -w(div u,q) 
+   a21.AddDomainIntegrator(new MixedScalarDivergenceIntegrator(negomeg));
+   // w(u,grad q)
+   a21.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(negomeg));
+   a21.Assemble();
+   a21.EliminateTestDofs(ess_bdr);
+   a21.Finalize();
+   Ah[2][1] = a21.ParallelAssemble();
+
+   // A22 = (grad p,grad q)+w^2(p,q)
+   ParBilinearForm a22(&H1fes_ho);
+   a22.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a22.AddDomainIntegrator(new MassIntegrator(omeg2));
+   a22.Assemble();
+   a22.EliminateEssentialBC(ess_bdr,x.GetBlock(2),rhs.GetBlock(2),mfem::Operator::DIAG_ONE);
+   a22.Finalize();
+   Ah[2][2] = a22.ParallelAssemble();
+
+
+   // A30 = w(p,div v)-w(grad p,v)
+   ParMixedBilinearForm a30(&H1fes_ho,&RTfes_ho);
+   // w(p,div v)
+   a30.AddDomainIntegrator(new MixedScalarWeakGradientIntegrator(negomeg));
+   // -w(grad p,v)
+   a30.AddDomainIntegrator(new MixedVectorGradientIntegrator(negomeg));
+   a30.Assemble();
+   a30.EliminateTrialDofs(ess_bdr,x.GetBlock(0),rhs.GetBlock(3));
+   a30.Finalize();
+   Ah[3][0] = a30.ParallelAssemble();
+
+   ParBilinearForm a33(&RTfes_ho);
+   a33.AddDomainIntegrator(new DivDivIntegrator(one));
+   a33.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a33.Assemble();
+   a33.Finalize();
+   Ah[3][3] = a33.ParallelAssemble();
+
+   for (int i = 0; i<2; i++)
+   {
+      H1fes_ho.GetRestrictionMatrix()->Mult(x.GetBlock(2*i), X.GetBlock(2*i));
+      H1fes_ho.GetProlongationMatrix()->MultTranspose(rhs.GetBlock(2*i),Rhs.GetBlock(2*i));
+      RTfes_ho.GetRestrictionMatrix()->Mult(x.GetBlock(2*i+1), X.GetBlock(2*i+1));
+      RTfes_ho.GetProlongationMatrix()->MultTranspose(rhs.GetBlock(2*i+1),Rhs.GetBlock(2*i+1));
+   }
+
+   HypreParMatrix * A = HypreParMatrixFromBlocks(Ah);
+
+
+// -----------------------------------------------------
+//        L O R    P R E C O N D I T I O N E R 
+// -----------------------------------------------------
+   Array2D<HypreParMatrix *> Ah_lor(4,4); 
+   for (int i = 0; i<4; i++)
+   {
+      for (int j = 0; j<4; j++)
+      {
+         Ah_lor[i][j] = nullptr;
+      }
+   }
+
+   ParBilinearForm a00_lor(&H1fes_lor);
+   a00_lor.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a00_lor.AddDomainIntegrator(new MassIntegrator(omeg2));
+   a00_lor.Assemble();
+   a00_lor.EliminateEssentialBC(ess_bdr,mfem::Operator::DIAG_ONE);
+   a00_lor.Finalize();
+   Ah_lor[0][0] = a00_lor.ParallelAssemble();
+
+   ParMixedBilinearForm a03_lor(&RTfes_lor,&H1fes_lor);
+   a03_lor.AddDomainIntegrator(new MixedScalarDivergenceIntegrator(omeg));
+   a03_lor.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg));
+   a03_lor.Assemble();
+   a03_lor.EliminateTestDofs(ess_bdr);
+   a03_lor.Finalize();
+   Ah_lor[0][3] = a03_lor.ParallelAssemble();
+   Ah_lor[3][0] = Ah_lor[0][3]->Transpose();
+
+   ParBilinearForm a11_lor(&RTfes_lor);
+   a11_lor.AddDomainIntegrator(new DivDivIntegrator(one));
+   a11_lor.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a11_lor.Assemble();
+   a11_lor.Finalize();
+   Ah_lor[1][1] = a11_lor.ParallelAssemble();
+
+
+   ParMixedBilinearForm a21_lor(&RTfes_lor,&H1fes_lor);
+   a21_lor.AddDomainIntegrator(new MixedScalarDivergenceIntegrator(negomeg));
+   a21_lor.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(negomeg));
+   a21_lor.Assemble();
+   a21_lor.EliminateTestDofs(ess_bdr);
+   a21_lor.Finalize();
+   Ah_lor[2][1] = a21_lor.ParallelAssemble();
+   Ah_lor[1][2] = Ah_lor[2][1]->Transpose();
+
+   ParBilinearForm a22_lor(&H1fes_lor);
+   a22_lor.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a22_lor.AddDomainIntegrator(new MassIntegrator(omeg2));
+   a22_lor.Assemble();
+   a22_lor.EliminateEssentialBC(ess_bdr,mfem::Operator::DIAG_ONE);
+   a22_lor.Finalize();
+   Ah_lor[2][2] = a22_lor.ParallelAssemble();
+
+   ParBilinearForm a33_lor(&RTfes_lor);
+   a33_lor.AddDomainIntegrator(new DivDivIntegrator(one));
+   a33_lor.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a33_lor.Assemble();
+   a33_lor.Finalize();
+   Ah_lor[3][3] = a33_lor.ParallelAssemble();
+
+   HypreParMatrix * A_lor = HypreParMatrixFromBlocks(Ah_lor);
+
+
+// -----------------------------------------------------
+// -----------------------------------------------------
+
+   FiniteElement::MapType t = FiniteElement::H_DIV;
+   Array<int> perm = ComputeVectorFE_LORPermutation(RTfes_ho, RTfes_lor, t);
+
+   HypreBoomerAMG * amg_p0 = new HypreBoomerAMG(*Ah[0][0]);
+   amg_p0->SetPrintLevel(0);
+   HypreBoomerAMG * amg_lor_p0 = new HypreBoomerAMG(*Ah_lor[0][0]);
+   amg_lor_p0->SetPrintLevel(0);
+   HypreBoomerAMG * amg_p2 = new HypreBoomerAMG(*Ah[2][2]);
+   amg_p2->SetPrintLevel(0);
+   HypreBoomerAMG * amg_lor_p2 = new HypreBoomerAMG(*Ah_lor[2][2]);
+   amg_lor_p2->SetPrintLevel(0);
+
+   Solver *prec1 = nullptr;
+   Solver *prec3 = nullptr;
+   Solver *prec1_lor = nullptr;
+   Solver *prec3_lor = nullptr;
+   if (dim == 2) 
+   {
+      prec1 = new HypreAMS(*Ah[1][1],&RTfes_ho);
+      dynamic_cast<HypreAMS *>(prec1)->SetPrintLevel(0);
+      prec3 = new HypreAMS(*Ah[3][3],&RTfes_ho);
+      dynamic_cast<HypreAMS *>(prec3)->SetPrintLevel(0);
+      prec1_lor = new HypreAMS(*Ah_lor[1][1],&RTfes_lor);
+      dynamic_cast<HypreAMS *>(prec1_lor)->SetPrintLevel(0);
+      prec3_lor = new HypreAMS(*Ah_lor[3][3],&RTfes_lor);
+      dynamic_cast<HypreAMS *>(prec3_lor)->SetPrintLevel(0);
+   }
+   else
+   {
+      prec1 = new HypreADS(*Ah[1][1],&RTfes_ho);
+      dynamic_cast<HypreADS *>(prec1)->SetPrintLevel(0);
+      prec3 = new HypreADS(*Ah[3][3],&RTfes_ho);
+      dynamic_cast<HypreADS *>(prec3)->SetPrintLevel(0);
+      prec1_lor = new HypreADS(*Ah_lor[1][1],&RTfes_lor);
+      dynamic_cast<HypreADS *>(prec1_lor)->SetPrintLevel(0);
+      prec3_lor = new HypreADS(*Ah_lor[3][3],&RTfes_lor);
+      dynamic_cast<HypreADS *>(prec3_lor)->SetPrintLevel(0);
+   }
+   // 1st preconditioner: Exact LOR with direct solver
+   ComplexLORSolver M_lor_exact(*A_lor, perm);
+
+   // 2nd preconditioner: AMG/AMS on the high order system
+   BlockDiagonalPreconditioner M(block_trueOffsets);
+   M.SetDiagonalBlock(0,amg_p0);
+   M.SetDiagonalBlock(1,prec1);
+   M.SetDiagonalBlock(2,amg_p2);
+   M.SetDiagonalBlock(3,prec3);
+
+   // 3rd preconditioner: AMG/AMS on the LOR system
+   BlockDiagonalPreconditioner M_lor2(block_trueOffsets);
+   M_lor2.SetDiagonalBlock(0,amg_lor_p0);
+   M_lor2.SetDiagonalBlock(1,prec1_lor);
+   M_lor2.SetDiagonalBlock(2,amg_lor_p2);
+   M_lor2.SetDiagonalBlock(3,prec3_lor);
+
+   ComplexLORSolver M_lor(*A_lor, perm,false,&M_lor2);
+
+   Vector Y(X), Z(X);
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-6);
+   cg.SetMaxIter(5000);
+   cg.SetPrintLevel(3);
+   cg.SetOperator(*A);
+   StopWatch chrono;
+   chrono.Clear();
+   chrono.Start();
+   cg.SetPreconditioner(M_lor_exact);
+   cg.Mult(Rhs, X);
+   chrono.Stop();
+   cout << "PCG Exact LOR time = " << chrono.RealTime() << endl;
+
+   chrono.Clear();
+   chrono.Start();
+   cg.SetPreconditioner(M_lor);
+   cg.Mult(Rhs, Y);
+   chrono.Stop();
+   cout << "PCG AMG/AMS LOR time = " << chrono.RealTime() << endl;
+
+   chrono.Clear();
+   chrono.Start();
+   cg.SetPreconditioner(M);
+   cg.Mult(Rhs, Z);
+   chrono.Stop();
+   cout << "PCG AMG/AMS HO time = " << chrono.RealTime() << endl;
+   {
+      MUMPSSolver mumps;
+      mumps.SetPrintLevel(0);
+      mumps.SetOperator(*A);
+      mumps.Mult(Rhs,X);
+   }
+
+
+   p_gf_re = 0.0;
+   p_gf_im = 0.0;
+   u_gf_re = 0.0;
+   u_gf_im = 0.0;
+
+   p_gf_re.Distribute(&(X.GetBlock(0)));
+   u_gf_re.Distribute(&(X.GetBlock(1)));
+   p_gf_im.Distribute(&(X.GetBlock(2)));
+   u_gf_im.Distribute(&(X.GetBlock(3)));
+
+   ConvergenceStudy ratesH1;
+   ConvergenceStudy ratesRT;
+
+   VectorFunctionCoefficient gradp_ex(dim,gradp_exact_re);
+   FunctionCoefficient divu_ex(divu_exact_re);
+
+   ratesH1.AddH1GridFunction(&p_gf_re,&p_ex_re,&gradp_ex);
+   ratesRT.AddHdivGridFunction(&u_gf_re,&u_ex_re,&divu_ex);
+
+   ratesH1.Print(true);
+   ratesRT.Print(true);
+
+   // 10. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << p_gf_re <<
+               "window_title 'Numerical Pressure (real part)' "
+               << flush;
+      socketstream sol_sockex(vishost, visport);
+      ParGridFunction p_ex(&H1fes_ho);
+      p_ex.ProjectCoefficient(p_ex_re);
+      sol_sockex << "parallel " << num_procs << " " << myid << "\n";
+      sol_sockex.precision(8);
+      sol_sockex << "solution\n" << *pmesh << p_ex <<
+               "window_title 'Exact Pressure (real part)' "
+               << flush;         
+   }
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+
+
+void helmholtz_solution(const Vector &X, complex<double> &sol, 
+                                         std::vector<complex<double>> &grad,
+                                         complex<double> &grad2)
+{
+   double x = X(0), y = X(1);
+   double z;
+   if (dim == 3 ) z = X(2);
+
+   complex<double> zi(0,1);
+   if (exact == 0)
+   {
+      if (dim == 2)
+      {
+         sol = x*(1.0-x) * y*(1.0-y);
+
+         grad[0] = (1.0 - 2*x) * y*(1.0 - y);
+         grad[1] = (1.0 - 2*y) * x*(1.0 - x);
+         grad2 = -2 * y*(1.0 - y) - 2 * x*(1.0 - x);
+      }
+      else
+      {
+         sol = x*(1.0-x) * y*(1.0-y) * z*(1.0-z);
+         grad[0] = (1.0 - 2*x) * y*(1.0 - y) * z*(1.0-z);
+         grad[1] = (1.0 - 2*y) * x*(1.0 - x) * z*(1.0-z);
+         grad[2] = (1.0 - 2*z) * x*(1.0 - x) * y*(1.0-y);
+         grad2 = -2 * y*(1.0 - y) * z*(1.0-z)  
+                 -2 * x*(1.0 - x) * z*(1.0-z) 
+                 -2 * x*(1.0 - x) * y*(1.0-y);
+      }
+   }
+   else
+   {
+      complex<double> alpha;
+      if (dim == 2)
+      {
+         alpha = zi * omega / sqrt(2);
+         sol = exp(alpha*(x+y));
+         grad[0] = alpha * sol;
+         grad[1] = alpha * sol;
+         grad2 = 2.0*alpha*alpha*sol;
+      }
+      else
+      {
+         alpha = zi * omega / sqrt(3);
+         sol = exp(alpha*(x+y+z));
+         grad[0] = alpha * sol;
+         grad[1] = alpha * sol;
+         grad[2] = alpha * sol;
+         grad2 = 3.0*alpha*alpha*sol;
+      }
+   }
+}
+
+double p_exact_re(const Vector &x)
+{
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+   return sol.real();
+}
+
+double p_exact_im(const Vector &x)
+{
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+   return sol.imag();
+}
+
+void gradp_exact_re(const Vector &x, Vector &gradp)
+{
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+   for (int i=0; i<dim; i++)
+   {
+      gradp[i] = grad[i].real();
+   }
+}
+void gradp_exact_im(const Vector &x, Vector &gradp)
+{
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+   for (int i=0; i<dim; i++)
+   {
+      gradp[i] = grad[i].real();
+   }
+}
+
+
+void u_exact_re(const Vector &x, Vector &u)
+{
+   complex<double> zi(0,1);
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+   // u = i grad p / w 
+   for (int i=0; i<dim; i++)
+   {
+      u[i] = (zi * grad[i]/omega).real();
+   }
+}
+
+void u_exact_im(const Vector &x, Vector &u)
+{
+   complex<double> zi(0,1);
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+   // u = i grad p / w 
+   for (int i=0; i<dim; i++)
+   {
+      u[i] = (zi * grad[i]/omega).imag();
+   }
+}
+
+double divu_exact_re(const Vector &x)
+{
+   complex<double> zi(0,1);
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+   return (zi/omega * grad2).real();
+
+}
+double divu_exact_im(const Vector &x)
+{
+   complex<double> zi(0,1);
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+   return (zi/omega * grad2).imag();
+}
+
+void f_exact_re(const Vector &x, Vector &f)
+{
+   f = 0.0;
+}
+
+void f_exact_im(const Vector &x, Vector &f)
+{
+   f = 0.0;
+}
+
+double g_exact_re(const Vector &x)
+{
+   // f = i omega p + div u  
+   // f = i / omega *( omega * omega p + grad2) 
+   complex<double> zi(0,1);
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+
+   return (zi / omega *(omega * omega * sol + grad2)).real();
+}
+
+double g_exact_im(const Vector &x)
+{
+   // f = i omega p + div u  
+   // f = i / omega *( omega * omega p + grad2) 
+   complex<double> zi(0,1);
+   complex<double>sol;
+   std::vector<complex<double>>grad(dim);
+   complex<double>grad2;
+   helmholtz_solution(x,sol,grad,grad2);
+
+   return (zi / omega *(omega * omega * sol + grad2)).imag();
+}
+
+void plotfield(socketstream & socket, ParMesh * pmesh, const ParGridFunction & pgf, string & title )
+{
+   int num_procs, myid;
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+   ostringstream oss;
+   oss << title;
+   socket << "parallel " << num_procs << " " << myid << "\n";
+   socket.precision(8);
+   socket << "solution\n" << *pmesh << pgf 
+          << "window_title '" << oss.str() << "'" << flush;
+}

examples/maxwell-solver/LS_maxwell_lor.cpp

@@ -0,0 +1,419 @@
+
+
+//                                MFEM Example multigrid-grid Cycle
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "FOSLS.hpp"
+
+using namespace std;
+using namespace mfem;
+
+void maxwell_solution(const Vector &x, std::vector<complex<double>> & sol, 
+                                       std::vector<complex<double>> & curl,
+                                       std::vector<complex<double>> & curl2);
+
+void E_exact_re(const Vector &x, Vector &E);
+void H_exact_re(const Vector &x, Vector &H);
+void E_exact_im(const Vector &x, Vector &E);
+void H_exact_im(const Vector &x, Vector &H);
+
+void f_exact_re(const Vector &x, Vector &f);
+void g_exact_re(const Vector &x, Vector &g);
+void f_exact_im(const Vector &x, Vector &f);
+void g_exact_im(const Vector &x, Vector &g);
+void plotfield(socketstream &,ParMesh * pmesh,const ParGridFunction & , string &);
+
+int dim;
+double omega;
+int exact = 0;
+
+   // ----------------------------------------------------------------------
+   // |   |            E             |             H          |     RHS    | 
+   // ----------------------------------------------------------------------
+   // | F | (curlE,curlF)+w^2(E,F)   | iw(curlH,F)+iw(H,curF) | -iw(J,F)   |
+   // |   |                          |                        |            |
+   // | G |-iw(E,curlG)-iw(curlE,G)  | (curlH,curlG)+w^2(H,G) | -(J,curlG) |
+
+int main(int argc, char *argv[])
+{
+    // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/inline-hex.mesh";
+   int order = 1;
+   bool visualization = 1;
+   int sr = 1;
+   int pr = 1;
+   double rnum=1.0;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&sr, "-sr", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pr", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");                  
+   args.AddOption(&exact, "-solution", "--exact_solution",
+                  "Exact solution : 0-polynomial, 1-plane wave");                  
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   omega = 2.0 * M_PI * rnum;
+   // omega = rnum;
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+
+   dim = mesh->Dimension();
+
+   MFEM_VERIFY(dim == 3, "only 3D problems supported by this formulation");
+
+   for (int i = 0; i < sr; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   for (int i = 0; i < pr; i++ )
+   {
+      pmesh->UniformRefinement();
+   }
+
+   FiniteElementCollection *fec = new ND_FECollection(order,dim); 
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+
+   if (myid == 0)
+   {
+      cout << "Number of True Dofs = " << size << endl;
+   }
+
+   VectorFunctionCoefficient E_ex_re(dim,E_exact_re);
+   VectorFunctionCoefficient H_ex_re(dim,H_exact_re);
+   VectorFunctionCoefficient E_ex_im(dim,E_exact_im);
+   VectorFunctionCoefficient H_ex_im(dim,H_exact_im);
+   VectorFunctionCoefficient f_ex_re(dim,f_exact_re);
+   VectorFunctionCoefficient g_ex_re(dim,g_exact_re);
+   VectorFunctionCoefficient f_ex_im(dim,f_exact_im);
+   VectorFunctionCoefficient g_ex_im(dim,g_exact_im);
+
+   int n = fespace->GetVSize();
+   int N = fespace->GetTrueVSize();
+   Array<int> block_offsets(5);
+   block_offsets = n; 
+   block_offsets[0] = 0;
+   block_offsets.PartialSum();
+
+   Array<int> block_trueOffsets(5);
+   block_trueOffsets = N;
+   block_trueOffsets[0] = 0;
+   block_trueOffsets.PartialSum();
+
+   BlockVector X(block_trueOffsets), Rhs(block_trueOffsets);
+   X = 0.0;  Rhs = 0.0;
+
+   ComplexMaxwellFOSLS fosls(fespace);
+   fosls.SetOmega(omega);
+   Array<VectorFunctionCoefficient * > ess_data(4);
+   ess_data[0] = &E_ex_re;
+   ess_data[1] = &H_ex_re;
+   ess_data[2] = &E_ex_im;
+   ess_data[3] = &H_ex_im;
+   fosls.SetEssentialData(ess_data);
+   Array<VectorFunctionCoefficient * > loads(4);
+   loads[0] = &f_ex_re;
+   loads[1] = &g_ex_re;
+   loads[2] = &f_ex_im;
+   loads[3] = &g_ex_im;
+   fosls.SetLoadData(loads);
+
+   Array2D<HypreParMatrix *> Ah;
+   fosls.GetFOSLSLinearSystem(Ah,X,Rhs);
+
+   HypreParMatrix * A = HypreParMatrixFromBlocks(Ah);
+   
+   HypreAMS ams0(*Ah[0][0],fespace);
+   HypreAMS ams1(*Ah[1][1],fespace);
+
+   BlockDiagonalPreconditioner prec(block_trueOffsets);
+   prec.SetDiagonalBlock(0,&ams0);
+   prec.SetDiagonalBlock(1,&ams1);
+   prec.SetDiagonalBlock(2,&ams0);
+   prec.SetDiagonalBlock(3,&ams1);
+
+   StopWatch chrono;
+   chrono.Clear();
+   chrono.Start();
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-6);
+   // cg.SetAbsTol(1e-6);
+   cg.SetMaxIter(400);
+   cg.SetPrintLevel(1);
+   cg.SetOperator(*A);
+   cg.SetPreconditioner(prec);
+   cg.Mult(Rhs, X);
+   chrono.Stop();
+   double t1 = chrono.RealTime();
+   if (myid == 0)
+   {
+      cout << "PCG time = " << t1 << endl;
+   }
+   
+   // {
+   //    MUMPSSolver mumps;
+   //    mumps.SetPrintLevel(0);
+   //    mumps.SetOperator(*A);
+   //    mumps.Mult(Rhs,X);
+   // }
+   ParGridFunction E_gf_re(fespace);
+   ParGridFunction H_gf_re(fespace);
+   ParGridFunction E_gf_im(fespace);
+   ParGridFunction H_gf_im(fespace);
+   E_gf_re = 0.0;
+   E_gf_im = 0.0;
+   H_gf_re = 0.0;
+   H_gf_im = 0.0;
+
+   E_gf_re.Distribute(&(X.GetBlock(0)));
+   H_gf_re.Distribute(&(X.GetBlock(1)));
+   E_gf_im.Distribute(&(X.GetBlock(2)));
+   H_gf_im.Distribute(&(X.GetBlock(3)));
+
+   double E_re_L2_Error = E_gf_re.ComputeL2Error(E_ex_re);
+   double E_im_L2_Error = E_gf_im.ComputeL2Error(E_ex_im);
+   double H_re_L2_Error = H_gf_re.ComputeL2Error(H_ex_re);
+   double H_im_L2_Error = H_gf_im.ComputeL2Error(H_ex_im);
+
+   ParGridFunction zero(fespace);
+   zero = 0.0;
+   double E_re_L2_norm = zero.ComputeL2Error(E_ex_re);
+   double E_im_L2_norm = zero.ComputeL2Error(E_ex_im);
+   double H_re_L2_norm = zero.ComputeL2Error(H_ex_re);
+   double H_im_L2_norm = zero.ComputeL2Error(H_ex_im);
+   if (myid == 0)
+   {
+      cout << "E_re L2 Error = " << E_re_L2_Error/E_re_L2_norm << endl;
+      cout << "E_im L2 Error = " << E_im_L2_Error/E_im_L2_norm << endl;
+      cout << "H_re L2 Error = " << H_re_L2_Error/H_re_L2_norm << endl;
+      cout << "H_im L2 Error = " << H_im_L2_Error/H_im_L2_norm << endl;
+   }   
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock0(vishost, visport);
+      socketstream sol_sock1(vishost, visport);
+      socketstream sol_sock2(vishost, visport);
+      socketstream sol_sock3(vishost, visport);
+      string str0 = "E_re";
+      plotfield(sol_sock0,pmesh, E_gf_re,str0);
+      string str1 = "E_im";
+      plotfield(sol_sock1,pmesh,E_gf_im,str1);
+      string str2 = "H_re";
+      plotfield(sol_sock2,pmesh,H_gf_re,str2);
+      string str3 = "H_im";
+      plotfield(sol_sock3,pmesh,H_gf_im,str3);
+
+      ParGridFunction E_exact_re(fespace);
+      ParGridFunction E_exact_im(fespace);
+      ParGridFunction H_exact_re(fespace);
+      ParGridFunction H_exact_im(fespace);
+      E_exact_re.ProjectCoefficient(E_ex_re);
+      E_exact_im.ProjectCoefficient(E_ex_im);
+      H_exact_re.ProjectCoefficient(H_ex_re);
+      H_exact_im.ProjectCoefficient(H_ex_im);
+
+      socketstream sol_sock_ex0(vishost, visport);
+      socketstream sol_sock_ex1(vishost, visport);
+      socketstream sol_sock_ex2(vishost, visport);
+      socketstream sol_sock_ex3(vishost, visport);
+      str0 = "E_exact_re";
+      plotfield(sol_sock_ex0,pmesh,E_exact_re,str0);
+      str1 = "E_exact_im";
+      plotfield(sol_sock_ex1,pmesh,E_exact_im,str1);
+      str2 = "H_exact_re";
+      plotfield(sol_sock_ex2,pmesh,H_exact_re,str2);
+      str3 = "H_exact_im";
+      plotfield(sol_sock_ex3,pmesh,H_exact_im,str3);
+   }
+
+   MPI_Finalize();
+   return 0;
+}
+
+
+void maxwell_solution(const Vector &X, std::vector<complex<double>> &sol, 
+                                       std::vector<complex<double>> &curl,
+                                       std::vector<complex<double>> &curl2)
+{
+   double x = X(0), y = X(1), z = X(2);
+
+   complex<double> zi(0,1);
+   if (exact == 0)
+   {
+      sol[0] = y*(1.0-y)*z*(1.0-z) + zi * 2.0;
+      sol[1] = y*x*(1.0-x)*z*(1.0-z)+ zi * 2.0;
+      sol[2] = x*(1.0-x)*y*(1.0-y) + zi * 2.0;
+
+      curl[0] = (1.0-x)*x*(y*(2.0*z-3.0)+1.0);
+      curl[1] = 2.0*(1.0-y)*y*(x-z);
+      curl[2] = (z-1.0)*z*(y*(2*x-3)+1.0);
+
+      curl2[0] = (2.0*x-3.0)*(z-1.0)*z-2.0*y*y+2*y;
+      curl2[1] = -2.0*y*(x*x-x+(z-1.0)*z);
+      curl2[2] = 2*(x*(1.5-z)+x*x*(z-1.5)-y*y+y);
+   }
+   else
+   {
+      complex<double> alpha = zi * omega / sqrt(3);
+      sol[0] = exp(alpha*(x+y+z));
+      sol[1] = 0.0;
+      sol[2] = 0.0;
+
+      curl[0] = 0.0;
+      curl[1] = alpha * sol[0];
+      curl[2] = -alpha * sol[0];
+
+      curl2[0] = -2.0 * alpha * alpha * sol[0];
+      curl2[1] = alpha * alpha * sol[0];
+      curl2[2] = curl2[1];
+   }
+
+
+   // sol[0] = 1.0 + 2.0*zi;
+   // sol[1] = 1.0 + 2.0*zi;
+   // sol[2] = 1.0 + 2.0*zi;
+   // curl[0] = 0.0;
+   // curl[1] =0.0;
+   // curl[2] =0.0;
+   // curl2[0] =0.0;
+   // curl2[1] =0.0;
+   // curl2[2] =0.0;  
+
+}
+
+void E_exact_re(const Vector &x, Vector &E)
+{
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      E(i) = sol[i].real();
+   }
+}
+void H_exact_re(const Vector &x, Vector &H)
+{
+   complex<double> zi(0,1);
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   // H = i curlE / w 
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      H[i] = (zi * curl[i]/omega).real();
+   }
+}
+void E_exact_im(const Vector &x, Vector &E)
+{
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      E(i) = sol[i].imag();
+   }
+}
+void H_exact_im(const Vector &x, Vector &H)
+{
+   complex<double> zi(0,1);
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   // H = i curlE / w 
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      H[i] = (zi * curl[i]/omega).imag();
+   }
+}
+
+void f_exact_re(const Vector &x, Vector &f)
+{
+   f = 0.0;
+}
+void g_exact_re(const Vector &x, Vector &g)
+{
+   // J = i omega E - curl H 
+   // J = - i / omega (curl curl E - omega * omega E)
+   complex<double> zi(0,1);
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      g(i) = (-zi / omega *(curl2[i] - omega * omega * sol[i])).real();
+   }
+}
+void f_exact_im(const Vector &x, Vector &f)
+{
+   f = 0.0;
+}
+void g_exact_im(const Vector &x, Vector &g)
+{
+   // J = i omega E - curl H 
+   // J = - i / omega (curl curl E - omega * omega E)
+   complex<double> zi(0,1);
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      g(i) = (-zi / omega *(curl2[i] - omega * omega * sol[i])).imag();
+   }
+}
+
+
+void plotfield(socketstream & socket, ParMesh * pmesh, const ParGridFunction & pgf, string & title )
+{
+   int num_procs, myid;
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+   ostringstream oss;
+   oss << title;
+   socket << "parallel " << num_procs << " " << myid << "\n";
+   socket.precision(8);
+   socket << "solution\n" << *pmesh << pgf 
+          << "window_title '" << oss.str() << "'" << flush;
+}

examples/maxwell-solver/LS_maxwellp.cpp

@@ -0,0 +1,568 @@
+
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "FOSLS.hpp"
+
+using namespace std;
+using namespace mfem;
+
+void maxwell_solution(const Vector &x, std::vector<complex<double>> & sol, 
+                                       std::vector<complex<double>> & curl,
+                                       std::vector<complex<double>> & curl2);
+
+void E_exact_re(const Vector &x, Vector &E);
+void H_exact_re(const Vector &x, Vector &H);
+void E_exact_im(const Vector &x, Vector &E);
+void H_exact_im(const Vector &x, Vector &H);
+
+void f_exact_re(const Vector &x, Vector &f);
+void g_exact_re(const Vector &x, Vector &g);
+void f_exact_im(const Vector &x, Vector &f);
+void g_exact_im(const Vector &x, Vector &g);
+void plotfield(socketstream &,ParMesh * pmesh,const ParGridFunction & , string &);
+
+int dim;
+double omega;
+int exact = 0;
+
+   // ----------------------------------------------------------------------
+   // |   |            E             |             H          |     RHS    | 
+   // ----------------------------------------------------------------------
+   // | F | (curlE,curlF)+w^2(E,F)   | iw(curlH,F)+iw(H,curF) | -iw(J,F)   |
+   // |   |                          |                        |            |
+   // | G |-iw(E,curlG)-iw(curlE,G)  | (curlH,curlG)+w^2(H,G) | -(J,curlG) |
+
+int main(int argc, char *argv[])
+{
+    // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/inline-hex.mesh";
+   int order = 1;
+   bool visualization = 1;
+   int sr = 1;
+   int pr = 1;
+   double rnum=1.0;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&sr, "-sr", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pr", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");                  
+   args.AddOption(&exact, "-solution", "--exact_solution",
+                  "Exact solution : 0-polynomial, 1-plane wave");                  
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   omega = 2.0 * M_PI * rnum;
+   // omega = rnum;
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+
+   dim = mesh->Dimension();
+
+   MFEM_VERIFY(dim == 3, "only 3D problems supported by this formulation");
+
+   for (int i = 0; i < sr; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   for (int i = 0; i < pr; i++ )
+   {
+      pmesh->UniformRefinement();
+   }
+
+   FiniteElementCollection *fec = new ND_FECollection(order,dim); 
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+
+   if (myid == 0)
+   {
+      cout << "Number of True Dofs = " << size << endl;
+   }
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (pmesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+
+   VectorFunctionCoefficient E_ex_re(dim,E_exact_re);
+   VectorFunctionCoefficient H_ex_re(dim,H_exact_re);
+   VectorFunctionCoefficient E_ex_im(dim,E_exact_im);
+   VectorFunctionCoefficient H_ex_im(dim,H_exact_im);
+
+   VectorFunctionCoefficient f_ex_re(dim,f_exact_re);
+   VectorFunctionCoefficient g_ex_re(dim,g_exact_re);
+   VectorFunctionCoefficient f_ex_im(dim,f_exact_im);
+   VectorFunctionCoefficient g_ex_im(dim,g_exact_im);
+
+   int n = fespace->GetVSize();
+   int N = fespace->GetTrueVSize();
+   Array<int> block_offsets(5);
+   block_offsets[0] = 0;
+   block_offsets[1] = n;
+   block_offsets[2] = n;
+   block_offsets[3] = n;
+   block_offsets[4] = n;
+   block_offsets.PartialSum();
+
+   Array<int> block_trueOffsets(5);
+   block_trueOffsets[0] = 0;
+   block_trueOffsets[1] = N;
+   block_trueOffsets[2] = N;
+   block_trueOffsets[3] = N;
+   block_trueOffsets[4] = N;
+   block_trueOffsets.PartialSum();
+
+   BlockVector x(block_offsets), rhs(block_offsets);
+   BlockVector X(block_trueOffsets), Rhs(block_trueOffsets);
+   x = 0.0;  rhs = 0.0; X = 0.0;  Rhs = 0.0;
+
+   ParGridFunction E_gf_re, E_gf_im, H_gf_re, H_gf_im;
+
+   E_gf_re.MakeRef(fespace,x.GetBlock(0)); E_gf_re = 0.0;
+   H_gf_re.MakeRef(fespace,x.GetBlock(1)); H_gf_re = 0.0;
+   E_gf_im.MakeRef(fespace,x.GetBlock(2)); E_gf_im = 0.0;
+   H_gf_im.MakeRef(fespace,x.GetBlock(3)); H_gf_im = 0.0;
+
+   // E_gf_re.ProjectBdrCoefficientTangent(E_ex_re,ess_bdr);
+   // E_gf_im.ProjectBdrCoefficientTangent(E_ex_im,ess_bdr);
+   E_gf_re.ProjectCoefficient(E_ex_re);
+   E_gf_im.ProjectCoefficient(E_ex_im);
+
+   // ----------------------------------------------------------------------
+   // |   |            E             |             H          |     RHS    | 
+   // ----------------------------------------------------------------------
+   // | F | (curlE,curlF)+w^2(E,F)   | iw(curlH,F)+iw(H,curF) | -iw(J,F)   |
+   // |   |                          |                        |            |
+   // | G |-iw(E,curlG)-iw(curlE,G)  | (curlH,curlG)+w^2(H,G) | -(J,curlG) |
+   // ----------------------------------------------------------------------
+
+   // for convinience we convert the above 2 x 2 blocks to 4 x 4 in order
+   // to accomodate complex valued operators
+
+   // A = (curlE,curlF)+w^2(E,F)
+   // B = w(curlH,F)+w(H,curF)
+   // b0 = w(J_im,F)
+   // b1 = -(J_re,curlG)
+   // b2 = -w(J_re,G)
+   // b3 = -(J_im,G)
+
+   // | A   0   0  -B |  | E_re |     | b0 |
+   // | 0   A   B   0 |  | H_re |  =  | b1 |
+   // | 0   B   A   0 |  | E_Im |     | b2 |
+   // |-B   0   0   A |  | H_im |     | b3 |
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient negomeg(-omega);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega * omega);
+
+
+   ScalarVectorProductCoefficient wJi(omeg,g_ex_im);
+   ScalarVectorProductCoefficient negJr(negone,g_ex_re);
+   ScalarVectorProductCoefficient negwJr(negomeg,g_ex_re);
+   ScalarVectorProductCoefficient negJi(negone,g_ex_im);
+
+
+   ParLinearForm b0(fespace);
+   ParLinearForm b1(fespace);
+   ParLinearForm b2(fespace);
+   ParLinearForm b3(fespace);
+   b0.Update(fespace,rhs.GetBlock(0),0);
+   b1.Update(fespace,rhs.GetBlock(1),0);
+   b2.Update(fespace,rhs.GetBlock(2),0);
+   b3.Update(fespace,rhs.GetBlock(3),0);
+   
+   b0.AddDomainIntegrator(new VectorFEDomainLFIntegrator(wJi));
+   b1.AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(negJr));
+   b2.AddDomainIntegrator(new VectorFEDomainLFIntegrator(negwJr));
+   b3.AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(negJi));
+   
+   b0.Assemble();
+   b1.Assemble();
+   b2.Assemble();
+   b3.Assemble();
+
+   Array2D<HypreParMatrix *> Ah(4,4); 
+   for (int i = 0; i<4; i++)
+   {
+      for (int j = 0; j<4; j++)
+      {
+         Ah[i][j] = nullptr;
+      }
+   }
+
+
+   ParBilinearForm a00(fespace);
+   a00.AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a00.Assemble();
+   a00.EliminateEssentialBC(ess_bdr,x.GetBlock(0),rhs.GetBlock(0),mfem::Operator::DIAG_ONE);
+   a00.Finalize();
+   Ah[0][0] = a00.ParallelAssemble();
+
+   ParMixedBilinearForm a03(fespace,fespace);
+   a03.AddDomainIntegrator(new MixedVectorCurlIntegrator(negomeg));
+   a03.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(negomeg));
+   a03.Assemble();
+   a03.EliminateTestDofs(ess_bdr);
+   a03.Finalize();
+   Ah[0][3] = a03.ParallelAssemble();
+
+   ParBilinearForm a11(fespace);
+   a11.AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a11.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a11.Assemble();
+   a11.Finalize();
+   Ah[1][1] = a11.ParallelAssemble();
+
+   ParMixedBilinearForm a12(fespace,fespace);
+   a12.AddDomainIntegrator(new MixedVectorCurlIntegrator(omeg));
+   a12.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(omeg));
+   a12.Assemble();
+   a12.EliminateTrialDofs(ess_bdr,x.GetBlock(2),rhs.GetBlock(1));
+   a12.Finalize();
+   Ah[1][2] = a12.ParallelAssemble();
+
+   ParMixedBilinearForm a21(fespace,fespace);
+   a21.AddDomainIntegrator(new MixedVectorCurlIntegrator(omeg));
+   a21.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(omeg));
+   a21.Assemble();
+   a21.EliminateTestDofs(ess_bdr);
+   a21.Finalize();
+   Ah[2][1] = a21.ParallelAssemble();
+   // Ah[2][1] = Ah[1][2]->Transpose();
+   // (*Ah[2][1]) *=-1.0;
+
+   ParBilinearForm a22(fespace);
+   a22.AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a22.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a22.Assemble();
+   a22.EliminateEssentialBC(ess_bdr,x.GetBlock(2),rhs.GetBlock(2),mfem::Operator::DIAG_ONE);
+   a22.Finalize();
+   Ah[2][2] = a22.ParallelAssemble();
+
+   ParMixedBilinearForm a30(fespace,fespace);
+   a30.AddDomainIntegrator(new MixedVectorCurlIntegrator(negomeg));
+   a30.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(negomeg));
+   a30.Assemble();
+   a30.EliminateTrialDofs(ess_bdr,x.GetBlock(0),rhs.GetBlock(3));
+   a30.Finalize();
+   Ah[3][0] = a30.ParallelAssemble();
+   // Ah[3][0] = Ah[0][3]->Transpose();
+   // (*Ah[3][0])*=-1.0;
+
+   ParBilinearForm a33(fespace);
+   a33.AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a33.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2));
+   a33.Assemble();
+   a33.Finalize();
+   Ah[3][3] = a33.ParallelAssemble();
+   // Ah[3][3] = Ah[2][2];
+
+   // HypreParMatrix * diff = new HypreParMatrix(*Ah[0][3]);
+   // *diff += *Ah[3][0];
+
+
+   for (int i = 0; i<4; i++)
+   {
+      fespace->GetRestrictionMatrix()->Mult(x.GetBlock(i), X.GetBlock(i));
+      fespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(i),Rhs.GetBlock(i));
+   }
+
+   HypreParMatrix * A = HypreParMatrixFromBlocks(Ah);
+   
+   HypreAMS ams0(*Ah[0][0],fespace);
+   HypreAMS ams1(*Ah[1][1],fespace);
+
+   BlockDiagonalPreconditioner prec(block_trueOffsets);
+   prec.SetDiagonalBlock(0,&ams0);
+   prec.SetDiagonalBlock(1,&ams1);
+   prec.SetDiagonalBlock(2,&ams0);
+   prec.SetDiagonalBlock(3,&ams1);
+
+   StopWatch chrono;
+   chrono.Clear();
+   chrono.Start();
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-6);
+   // cg.SetAbsTol(1e-6);
+   cg.SetMaxIter(400);
+   cg.SetPrintLevel(1);
+   cg.SetOperator(*A);
+   cg.SetPreconditioner(prec);
+   cg.Mult(Rhs, X);
+   chrono.Stop();
+   double t1 = chrono.RealTime();
+   if (myid == 0)
+   {
+      cout << "PCG time = " << t1 << endl;
+   }
+   
+   // {
+   //    MUMPSSolver mumps;
+   //    mumps.SetPrintLevel(0);
+   //    mumps.SetOperator(*A);
+   //    mumps.Mult(Rhs,X);
+   // }
+   E_gf_re = 0.0;
+   E_gf_im = 0.0;
+   H_gf_re = 0.0;
+   H_gf_im = 0.0;
+
+   E_gf_re.Distribute(&(X.GetBlock(0)));
+   H_gf_re.Distribute(&(X.GetBlock(1)));
+   E_gf_im.Distribute(&(X.GetBlock(2)));
+   H_gf_im.Distribute(&(X.GetBlock(3)));
+
+   double E_re_L2_Error = E_gf_re.ComputeL2Error(E_ex_re);
+   double E_im_L2_Error = E_gf_im.ComputeL2Error(E_ex_im);
+   double H_re_L2_Error = H_gf_re.ComputeL2Error(H_ex_re);
+   double H_im_L2_Error = H_gf_im.ComputeL2Error(H_ex_im);
+
+   ParGridFunction zero(fespace);
+   zero = 0.0;
+   double E_re_L2_norm = zero.ComputeL2Error(E_ex_re);
+   double E_im_L2_norm = zero.ComputeL2Error(E_ex_im);
+   double H_re_L2_norm = zero.ComputeL2Error(H_ex_re);
+   double H_im_L2_norm = zero.ComputeL2Error(H_ex_im);
+   if (myid == 0)
+   {
+      cout << "E_re L2 Error = " << E_re_L2_Error/E_re_L2_norm << endl;
+      cout << "E_im L2 Error = " << E_im_L2_Error/E_im_L2_norm << endl;
+      cout << "H_re L2 Error = " << H_re_L2_Error/H_re_L2_norm << endl;
+      cout << "H_im L2 Error = " << H_im_L2_Error/H_im_L2_norm << endl;
+   }   
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock0(vishost, visport);
+      socketstream sol_sock1(vishost, visport);
+      socketstream sol_sock2(vishost, visport);
+      socketstream sol_sock3(vishost, visport);
+      string str0 = "E_re";
+      plotfield(sol_sock0,pmesh, E_gf_re,str0);
+      string str1 = "E_im";
+      plotfield(sol_sock1,pmesh,E_gf_im,str1);
+      string str2 = "H_re";
+      plotfield(sol_sock2,pmesh,H_gf_re,str2);
+      string str3 = "H_im";
+      plotfield(sol_sock3,pmesh,H_gf_im,str3);
+
+      ParGridFunction E_exact_re(fespace);
+      ParGridFunction E_exact_im(fespace);
+      ParGridFunction H_exact_re(fespace);
+      ParGridFunction H_exact_im(fespace);
+      E_exact_re.ProjectCoefficient(E_ex_re);
+      E_exact_im.ProjectCoefficient(E_ex_im);
+      H_exact_re.ProjectCoefficient(H_ex_re);
+      H_exact_im.ProjectCoefficient(H_ex_im);
+
+      socketstream sol_sock_ex0(vishost, visport);
+      socketstream sol_sock_ex1(vishost, visport);
+      socketstream sol_sock_ex2(vishost, visport);
+      socketstream sol_sock_ex3(vishost, visport);
+      str0 = "E_exact_re";
+      plotfield(sol_sock_ex0,pmesh,E_exact_re,str0);
+      str1 = "E_exact_im";
+      plotfield(sol_sock_ex1,pmesh,E_exact_im,str1);
+      str2 = "H_exact_re";
+      plotfield(sol_sock_ex2,pmesh,H_exact_re,str2);
+      str3 = "H_exact_im";
+      plotfield(sol_sock_ex3,pmesh,H_exact_im,str3);
+   }
+
+   MPI_Finalize();
+   return 0;
+}
+
+
+void maxwell_solution(const Vector &X, std::vector<complex<double>> &sol, 
+                                       std::vector<complex<double>> &curl,
+                                       std::vector<complex<double>> &curl2)
+{
+   double x = X(0), y = X(1), z = X(2);
+
+   complex<double> zi(0,1);
+   if (exact == 0)
+   {
+      sol[0] = y*(1.0-y)*z*(1.0-z) + zi * 2.0;
+      sol[1] = y*x*(1.0-x)*z*(1.0-z)+ zi * 2.0;
+      sol[2] = x*(1.0-x)*y*(1.0-y) + zi * 2.0;
+
+      curl[0] = (1.0-x)*x*(y*(2.0*z-3.0)+1.0);
+      curl[1] = 2.0*(1.0-y)*y*(x-z);
+      curl[2] = (z-1.0)*z*(y*(2*x-3)+1.0);
+
+      curl2[0] = (2.0*x-3.0)*(z-1.0)*z-2.0*y*y+2*y;
+      curl2[1] = -2.0*y*(x*x-x+(z-1.0)*z);
+      curl2[2] = 2*(x*(1.5-z)+x*x*(z-1.5)-y*y+y);
+   }
+   else
+   {
+      complex<double> alpha = zi * omega / sqrt(3);
+      sol[0] = exp(alpha*(x+y+z));
+      sol[1] = 0.0;
+      sol[2] = 0.0;
+
+      curl[0] = 0.0;
+      curl[1] = alpha * sol[0];
+      curl[2] = -alpha * sol[0];
+
+      curl2[0] = -2.0 * alpha * alpha * sol[0];
+      curl2[1] = alpha * alpha * sol[0];
+      curl2[2] = curl2[1];
+   }
+
+
+   // sol[0] = 1.0 + 2.0*zi;
+   // sol[1] = 1.0 + 2.0*zi;
+   // sol[2] = 1.0 + 2.0*zi;
+   // curl[0] = 0.0;
+   // curl[1] =0.0;
+   // curl[2] =0.0;
+   // curl2[0] =0.0;
+   // curl2[1] =0.0;
+   // curl2[2] =0.0;  
+
+}
+
+void E_exact_re(const Vector &x, Vector &E)
+{
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      E(i) = sol[i].real();
+   }
+}
+void H_exact_re(const Vector &x, Vector &H)
+{
+   complex<double> zi(0,1);
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   // H = i curlE / w 
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      H[i] = (zi * curl[i]/omega).real();
+   }
+}
+void E_exact_im(const Vector &x, Vector &E)
+{
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      E(i) = sol[i].imag();
+   }
+}
+void H_exact_im(const Vector &x, Vector &H)
+{
+   complex<double> zi(0,1);
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   // H = i curlE / w 
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      H[i] = (zi * curl[i]/omega).imag();
+   }
+}
+
+void f_exact_re(const Vector &x, Vector &f)
+{
+   f = 0.0;
+}
+void g_exact_re(const Vector &x, Vector &g)
+{
+   // J = i omega E - curl H 
+   // J = - i / omega (curl curl E - omega * omega E)
+   complex<double> zi(0,1);
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      g(i) = (-zi / omega *(curl2[i] - omega * omega * sol[i])).real();
+   }
+}
+void f_exact_im(const Vector &x, Vector &f)
+{
+   f = 0.0;
+}
+void g_exact_im(const Vector &x, Vector &g)
+{
+   // J = i omega E - curl H 
+   // J = - i / omega (curl curl E - omega * omega E)
+   complex<double> zi(0,1);
+   std::vector<complex<double>>sol(3);
+   std::vector<complex<double>>curl(3);
+   std::vector<complex<double>>curl2(3);
+   maxwell_solution(x,sol,curl,curl2);
+   for (int i=0; i<dim; i++)
+   {
+      g(i) = (-zi / omega *(curl2[i] - omega * omega * sol[i])).imag();
+   }
+}
+
+
+void plotfield(socketstream & socket, ParMesh * pmesh, const ParGridFunction & pgf, string & title )
+{
+   int num_procs, myid;
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+   ostringstream oss;
+   oss << title;
+   socket << "parallel " << num_procs << " " << myid << "\n";
+   socket.precision(8);
+   socket << "solution\n" << *pmesh << pgf 
+          << "window_title '" << oss.str() << "'" << flush;
+}

examples/maxwell-solver/ParDST/DofMapsDST.cpp

@@ -0,0 +1,960 @@
+#include "DofMapsDST.hpp"
+
+double testcoeff(const Vector & x)
+{
+   return sin(3*M_PI*(x.Sum()));
+}
+
+int get_rank(int tdof, std::vector<int> & tdof_offsets)
+{
+   int size = tdof_offsets.size();
+   if (size == 1) { return 0; }
+   std::vector<int>::iterator up;
+   up=std::upper_bound(tdof_offsets.begin(), tdof_offsets.end(),tdof); // 
+   return std::distance(tdof_offsets.begin(),up)-1;
+}
+
+void ComputeTdofOffsets(const MPI_Comm & comm, const ParFiniteElementSpace * pfes, 
+                        std::vector<int> & tdof_offsets)
+{
+   int num_procs;
+   MPI_Comm_size(comm, &num_procs);
+   tdof_offsets.resize(num_procs);
+   int mytoffset = pfes->GetMyTDofOffset();
+   MPI_Allgather(&mytoffset,1,MPI_INT,&tdof_offsets[0],1,MPI_INT,comm);
+}
+
+void GetSubdomainijk(int ip, const Array<int> nxyz, Array<int> & ijk)
+{
+   ijk.SetSize(3);
+   ijk[2] = ip/(nxyz[0]*nxyz[1]);
+   ijk[1] = (ip-ijk[2]*nxyz[0]*nxyz[1])/nxyz[0];
+   ijk[0] = (ip-ijk[2]*nxyz[0]*nxyz[1])%nxyz[0];
+}
+void GetDirectionijk(int id, Array<int> & ijk)
+{
+   ijk.SetSize(3);
+   int n = 3;
+   ijk[2] = id/(n*n) - 1;
+   ijk[1] = (id-(ijk[2]+1)*n*n)/n - 1;
+   ijk[0] = (id-(ijk[2]+1)*n*n)%n - 1;
+}
+
+int GetSubdomainId(const Array<int> nxyz, Array<int> & ijk)
+{
+   int dim=ijk.Size();
+   int k = (dim==2)? 0 : ijk[2];
+   return k*nxyz[1]*nxyz[0] + ijk[1]*nxyz[0] + ijk[0];
+}
+
+int GetDirectionId(const Array<int> & ijk)
+{
+   int n = 3;
+   int dim = ijk.Size();
+   int k = (dim == 2) ? -1 : ijk[2];
+   return (k+1)*n*n + (ijk[1]+1)*n + ijk[0]+1; 
+}
+
+void DofMaps::Init()
+{
+   comm = pfes->GetComm();
+   MPI_Comm_size(comm, &num_procs);
+   MPI_Comm_rank(comm, &myid);
+
+   dim = pfes->GetParMesh()->Dimension();
+   ComputeTdofOffsets(comm, pfes, tdof_offsets);
+   myelemoffset = part->myelem_offset;
+   mytoffset = pfes->GetMyTDofOffset();
+   subdomain_rank = part->subdomain_rank;
+   nrsubdomains = part->nrsubdomains;
+   nxyz.SetSize(3);
+   for (int i = 0; i<3; i++) { nxyz[i] = part->nxyz[i]; }
+
+   //compute sign factors for tdofs
+   int lsize = pfes->GetVSize();
+   int tsize = pfes->GetTrueVSize();
+   tdof_sign.SetSize(tsize); 
+   for (int i = 0; i<lsize; i++)
+   {
+      int j = pfes->GetGlobalTDofNumber(i);
+      if (j<mytoffset || j>=mytoffset+tsize) continue;
+      tdof_sign[j-mytoffset] = pfes->GetDofSign(i);
+   }
+}
+
+DofMaps::DofMaps(ParFiniteElementSpace *pfes_, ParMeshPartition * part_, bool CompFlag_)
+: pfes(pfes_), part(part_), CompFlag(CompFlag_)
+{
+   Init();
+   Setup();
+}
+
+void DofMaps::Setup()
+{
+   // Setup the local FiniteElementSpaces
+   const FiniteElementCollection * fec = pfes->FEColl();
+   fes.SetSize(nrsubdomains);
+   for (int i = 0; i<nrsubdomains; i++)
+   {
+      fes[i] = nullptr; // initialize with null on all procs
+      if (myid == subdomain_rank[i])
+      {
+         fes[i] = new FiniteElementSpace(part->subdomain_mesh[i],fec);
+      }
+   }
+   // cout << "Computing Overlap Tdofs" << endl;
+   SubdomainToSubdomainMapsSetup();
+   // TestSubdomainToSubdomainMaps();
+
+   SubdomainToGlobalMapsSetup();
+   // TestSubdomainToGlobalMaps();
+}
+
+void DofMaps::SubdomainToSubdomainMapsSetup()
+{
+   ComputeOvlpElems();
+   ComputeOvlpTdofs();
+}
+
+void DofMaps::AddElementToOvlpLists(int l, int iel, 
+                            const Array<bool> & neg, const Array<bool> & pos)
+{
+   int kbeg = (dim == 2) ? 0 : -1;
+   int kend = (dim == 2) ? 0 :  1;
+   Array<int> dijk(3);
+   for (int k = kbeg; k<=kend; k++)
+   {
+      if (dim == 3)
+      {
+         if (k == -1 && !neg[2]) continue;   
+         if (k ==  1 && !pos[2]) continue;   
+      }   
+
+      for (int j = -1; j<=1; j++)
+      {
+         if (j== -1 && !neg[1]) continue;   
+         if (j==  1 && !pos[1]) continue;   
+         for (int i = -1; i<=1; i++)
+         {
+            // cases to skip 
+            if (i==-1 && !neg[0]) continue;   
+            if (i== 1 && !pos[0]) continue;   
+
+            if (i==0 && j==0 && k == 0) continue;
+            dijk[0] = i; dijk[1] = j; dijk[2] = (dim==2)?-1 : k;  
+            int DirId = GetDirectionId(dijk);
+            OvlpElems[l][DirId].Append(iel);
+         }
+      }
+   }
+}
+
+void DofMaps::ComputeOvlpElems()
+{
+   // first compute the element in the overlaps
+   OvlpElems.resize(nrsubdomains);
+   int nlayers = 2*part->OvlpNlayers; 
+   // loop through subdomains
+   for (int l = 0; l<nrsubdomains; l++)
+   {
+      if (myid == subdomain_rank[l])
+      {
+         Array<int> ijk;
+         GetSubdomainijk(l,nxyz,ijk);
+         Mesh * mesh = part->subdomain_mesh[l];
+         OvlpElems[l].resize(pow(3,dim)); 
+         Vector pmin, pmax;
+         mesh->GetBoundingBox(pmin,pmax);
+         double h = part->MeshSize;
+         // loop through the elements in the mesh and assign them to the 
+         // appropriate lists of overlaps
+         for (int iel=0; iel< mesh->GetNE(); iel++)
+         {
+            // Get element center
+            Vector center(dim);
+            int geom = mesh->GetElementBaseGeometry(iel);
+            ElementTransformation * tr = mesh->GetElementTransformation(iel);
+            tr->Transform(Geometries.GetCenter(geom),center);
+
+            Array<bool> pos(dim); pos = false;
+            Array<bool> neg(dim); neg = false;
+            // loop through dimensions   
+            for (int d=0;d<dim; d++)
+            {
+               if (ijk[d]>0 && center[d] < pmin[d]+h*nlayers)
+               {
+                  neg[d] = true;
+               }
+
+               if (ijk[d]<nxyz[d]-1 && center[d] > pmax[d]-h*nlayers) 
+               {
+                  pos[d] = true;
+               }
+            }
+            // Add the element to the appropriate lists
+            AddElementToOvlpLists(l,iel,neg,pos);        
+         }   
+      }
+   }
+}
+
+void DofMaps::ComputeOvlpTdofs()
+{
+   OvlpTDofs.resize(nrsubdomains);
+   int nrneighbors = pow(3,dim); // including its self
+
+   // loop through subdomains
+   for (int l = 0; l<nrsubdomains; l++)
+   {
+      if (myid != subdomain_rank[l]) continue;
+      int ntdofs = fes[l]->GetTrueVSize();
+      Array<int> tdof_marker(ntdofs); 
+      OvlpTDofs[l].resize(nrneighbors);
+      // loop through neighboring directions/neighbors
+      for (int d=0; d<nrneighbors; d++)
+      {
+         tdof_marker = 0;
+         Array<int> tdoflist;
+         // Get the direction
+         Array<int> dijk;
+         GetDirectionijk(l,dijk);
+         int nel = OvlpElems[l][d].Size();
+         Array<int>Elems = OvlpElems[l][d];
+         for (int iel = 0; iel<nel; ++iel)
+         {
+            int jel = Elems[iel];   
+            Array<int> ElemDofs;
+
+            fes[l]->GetElementDofs(jel,ElemDofs);
+            int ndof = ElemDofs.Size();
+            for (int i = 0; i<ndof; ++i)
+            {
+               int dof_ = ElemDofs[i];
+               int dof = (dof_ >= 0) ? dof_ : abs(dof_) - 1;
+               if (!tdof_marker[dof])
+               {
+                  tdoflist.Append(dof); // dofs of ip0 in ovlp
+                  tdof_marker[dof] = 1;
+               }
+            }
+         }
+         OvlpTDofs[l][d] = tdoflist; 
+         if (CompFlag)
+         {
+            for (int i=0; i<tdoflist.Size(); i++)
+            {
+               tdoflist[i] += fes[l]->GetTrueVSize();
+            }
+            OvlpTDofs[l][d].Append(tdoflist);
+         }
+      }
+   }
+}
+
+void DofMaps::PrintOvlpTdofs()
+{
+   int nrneighbors = pow(3,dim); // including its self
+   if (myid == 0)
+   {
+      for (int i = 0; i<nrsubdomains; i++)
+      {
+         if (myid != subdomain_rank[i]) continue;
+         Array<int> ijk;
+         GetSubdomainijk(i,nxyz,ijk);
+         cout << "subdomain = " ; ijk.Print();
+         cout << "myid = " << myid << endl;
+         cout << "ip   = " << i << endl;
+         for (int d = 0; d<nrneighbors; d++)
+         {
+            Array<int> dijk;
+            GetDirectionijk(d,dijk);
+            cout << "direction = " ; dijk.Print();
+
+            if (OvlpTDofs[i][d].Size())
+            {
+               cout << "OvlpTdofs = " ; 
+               OvlpTDofs[i][d].Print(cout,OvlpTDofs[i][d].Size() );
+            }
+         }
+      }
+   }
+}
+
+void DofMaps::TransferToNeighbors(const Array<int> & SubdomainIds, const Array<Vector *> & x,
+std::vector<std::vector<Vector * >> & OvlpSol)
+{
+   // 2D for now....
+   MFEM_VERIFY(SubdomainIds.Size() == x.Size(), "TransferToNeighbors: Size inconsistency");
+   int nrsendIds = SubdomainIds.Size();
+   int nrneighbors = pow(3,dim);
+   MPI_Request *recv_requests = new MPI_Request[nrsendIds*nrneighbors];
+   MPI_Request *send_requests = new MPI_Request[nrsendIds*nrneighbors];
+   MPI_Status  *recv_statuses = new MPI_Status[nrsendIds*nrneighbors];
+   MPI_Status  *send_statuses = new MPI_Status[nrsendIds*nrneighbors];
+   Array<Vector * > send_buffer(nrsendIds*nrneighbors);
+   Array<Vector * > recv_buffer(nrsendIds*nrneighbors);
+   int send_counter = 0;
+   int recv_counter = 0;
+   for (int is = 0; is<nrsendIds; is++)
+   {
+      int i0 = SubdomainIds[is];
+      Array<int> ijk;
+      GetSubdomainijk(i0,nxyz,ijk);
+      for (int d=0;d<nrneighbors; d++)
+      {
+         Array<int>directions;
+         GetDirectionijk(d,directions);
+
+         if (dim == 2 && directions[0] == 0 && directions[1] == 0) continue;
+         if (dim == 3 && directions[0] == 0 
+                      && directions[1] == 0 
+                      && directions[2] == 0) continue;
+         int i = ijk[0] + directions[0];
+         if (i<0 || i>=nxyz[0]) continue;
+         int j = ijk[1] + directions[1];
+         if (j<0 || j>=nxyz[1]) continue;
+         int k = (dim ==3 ) ? ijk[2] + directions[2] : 0;
+         if (k<0 || k>=nxyz[2]) continue;
+         Array<int>ijk1(3);
+         ijk1[0] = i;
+         ijk1[1] = j;
+         ijk1[2] = k;
+         int i1 = GetSubdomainId(nxyz,ijk1);
+         if (myid == subdomain_rank[i0])
+         {
+            Array<int> tdofs0 = OvlpTDofs[i0][d]; // map of dofs in the overlap
+            send_buffer[send_counter] = new Vector(tdofs0.Size());
+            x[is]->GetSubVector(tdofs0,*send_buffer[send_counter]);
+            // Destination rank
+            int dest = subdomain_rank[i1];
+            int tag = i0 * nrneighbors + d;
+
+            int count = tdofs0.Size();
+            MPI_Isend(send_buffer[send_counter]->GetData(),count,MPI_DOUBLE,dest,
+                      tag,comm,&send_requests[send_counter]);
+            send_counter++;
+
+         }
+         if (myid == subdomain_rank[i1])
+         {
+            Array<int> direction1(3); direction1 = -1;
+            for (int dd=0;dd<dim;dd++) 
+            { 
+               direction1[dd] = -directions[dd];
+            }
+            int d1 = GetDirectionId(direction1);
+
+            int count = OvlpTDofs[i1][d1].Size();
+            recv_buffer[recv_counter] = new Vector(count); 
+            int src = subdomain_rank[i0];
+            int tag = i0 * nrneighbors + d;
+            MPI_Irecv(recv_buffer[recv_counter]->GetData(), count,MPI_DOUBLE,src,
+                      tag,comm, &recv_requests[recv_counter]);
+            recv_counter++;
+         }
+      }   
+   }
+   MPI_Waitall(send_counter, send_requests, send_statuses);
+   MPI_Waitall(recv_counter, recv_requests, recv_statuses);
+
+   delete [] send_statuses;
+   delete [] send_requests;
+   delete [] recv_statuses;
+   delete [] recv_requests;
+
+   for (int i = 0; i<send_counter; i++)
+   {
+      delete send_buffer[i];
+   }
+   send_buffer.DeleteAll();
+
+
+   // Extract the transfered solutions
+   recv_counter = 0;
+   for (int is = 0; is<nrsendIds; is++)
+   {
+      int i0 = SubdomainIds[is];
+      Array<int> ijk;
+      GetSubdomainijk(i0,nxyz,ijk);
+      for (int d=0;d<nrneighbors; d++)
+      {
+         Array<int>directions;
+         GetDirectionijk(d,directions);
+         if (dim == 2 && directions[0] == 0 && directions[1] == 0) continue;
+         if (dim == 3 && directions[0] == 0 
+                      && directions[1] == 0 
+                      && directions[2] == 0) continue;
+         int i = ijk[0] + directions[0];
+         if (i<0 || i>=nxyz[0]) continue;
+         int j = ijk[1] + directions[1];
+         if (j<0 || j>=nxyz[1]) continue;
+         int k = (dim ==3 ) ? ijk[2] + directions[2] : 0;
+         if (k<0 || k>=nxyz[2]) continue;
+
+         Array<int>ijk1(3);
+         ijk1[0] = i;
+         ijk1[1] = j;
+         ijk1[2] = k;
+         int i1 = GetSubdomainId(nxyz,ijk1);
+         if (myid == subdomain_rank[i1])
+         {
+            Array<int> direction1(3); direction1 = -1;
+            for (int d=0;d<dim;d++) 
+            { 
+               direction1[d] = -directions[d];
+            }
+            int d1 = GetDirectionId(direction1);
+            Array<int> tdofs1 = OvlpTDofs[i1][d1];
+            if (!OvlpSol[i1][d1])
+            {
+               OvlpSol[i1][d1] = new Vector(2*fes[i1]->GetTrueVSize());
+            }
+            *OvlpSol[i1][d1] = 0.0;
+            OvlpSol[i1][d1]->SetSubVector(tdofs1,*recv_buffer[recv_counter]);
+            recv_counter++;
+         }
+      }   
+   }
+   for (int i = 0; i<recv_counter; i++)
+   {
+      delete recv_buffer[i];
+   }
+   recv_buffer.DeleteAll();
+}
+
+void DofMaps::TestSubdomainToSubdomainMaps()
+{
+      // testing inter-subdomain communication
+   FunctionCoefficient c1(testcoeff);
+   int nrsub = nrsubdomains;
+   Array<int> subdomain_ids(nrsub);
+   Array<Vector*> x(nrsub);
+   for (int i = 0; i<nrsub; i++)
+   {
+      x[i] = nullptr;
+      subdomain_ids[i] = i;
+      if (fes[i])
+      {
+         ComplexGridFunction gf(fes[i]);
+         gf = 0.0;
+         gf.ProjectCoefficient(c1,c1);
+         x[i] = new Vector(2*fes[i]->GetTrueVSize());
+         *x[i] = gf;
+      }
+   }
+
+   std::vector<std::vector<Vector * >> OvlpSol;
+
+   OvlpSol.resize(nrsubdomains);
+   int nrneighbors = pow(3,dim);
+   for (int ip = 0; ip<nrsubdomains; ip++)
+   {
+      if (myid == subdomain_rank[ip])
+      {
+         OvlpSol[ip].resize(nrneighbors);
+      }
+   }
+
+
+   TransferToNeighbors(subdomain_ids,x,OvlpSol);
+
+   string keys = "keys amrRljc\n";
+   for (int i0 = 0 ; i0< nrsubdomains; i0++)
+   {
+      if (fes[i0])
+      {
+         ComplexGridFunction gf0(fes[i0]);
+         for (int d = 0; d<nrneighbors; d++)
+         {
+            if(OvlpSol[i0][d])
+            {
+               Array<int>dijk;
+               GetDirectionijk(d,dijk);
+               Array<int>ijk;
+               GetSubdomainijk(i0,nxyz,ijk);
+               ostringstream oss;
+               oss << "myid: " << myid 
+                     << ", subdomain: (" << ijk[0] << "," << ijk[1] <<")"
+                     << ", direction: (" << dijk[0] << "," << dijk[1] <<")";
+
+               gf0 = 0.0;
+               gf0.real().SetVector(*OvlpSol[i0][d],0);
+               gf0.imag().SetVector(*OvlpSol[i0][d],fes[i0]->GetTrueVSize());
+               char vishost[] = "localhost";
+               int  visport   = 19916;
+               socketstream sol_sock(vishost, visport);
+               sol_sock.precision(8);
+               sol_sock << "solution\n" << *(part->subdomain_mesh[i0]) << gf0.real() 
+               << keys
+               << "window_title '" << oss.str() << "'" << flush;
+            }
+         }
+      }
+   }
+   for (int i = 0; i<nrsub; i++)
+   {
+      delete x[i];
+   }
+}
+
+void DofMaps::SubdomainToGlobalMapsSetup()
+{
+   // workspace for MPI_AlltoAll
+   send_count.SetSize(num_procs);  send_count = 0;
+   send_displ.SetSize(num_procs);  send_displ = 0; 
+   recv_count.SetSize(num_procs);  recv_count = 0;
+   recv_displ.SetSize(num_procs);  recv_displ = 0;
+
+   // 1. Communicate to the subdomain rank the list of tdofs 
+   // a. Compute send count
+   for (int ip = 0; ip<nrsubdomains; ++ip)
+   {
+      // avoid any communication if on subdomain rank
+      int nel = part->local_element_map[ip].Size();
+
+      for (int iel = 0; iel<nel; iel++)
+      {
+         int elem_idx = part->local_element_map[ip][iel] - myelemoffset;
+         // int ndofs = local_tdofs[ip].Size();
+         int ndofs = pfes->GetFE(elem_idx)->GetDof();
+
+         send_count[subdomain_rank[ip]] += 2 + ndofs; 
+      }
+   }
+   // b. Compute receive count
+   MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,comm);
+   for (int k=0; k<num_procs-1; k++)
+   {
+      send_displ[k+1] = send_displ[k] + send_count[k];
+      recv_displ[k+1] = recv_displ[k] + recv_count[k];
+   }
+   sbuff_size = send_count.Sum();
+   rbuff_size = recv_count.Sum();
+   // c. Allocate and fill the send buffer
+   Array<int> sendbuf(sbuff_size);  sendbuf = 0;
+   Array<int> soffs(num_procs); soffs = 0;
+   for (int ip = 0; ip<nrsubdomains; ++ip)
+   {
+      int nel = part->local_element_map[ip].Size();
+      for (int iel = 0; iel<nel; iel++)
+      {
+         int elem_idx = part->local_element_map[ip][iel] - myelemoffset;
+         Array<int>ElemDofs;
+         pfes->GetElementDofs(elem_idx,ElemDofs);
+         int ndofs = ElemDofs.Size();
+
+         int j = send_displ[subdomain_rank[ip]] + soffs[subdomain_rank[ip]];
+         sendbuf[j] = ip;
+         sendbuf[j+1] = ndofs;
+
+         for (int k = 0; k < ndofs ; ++k)
+         {
+            int edof_ = ElemDofs[k];
+            int edof = (edof_ >= 0) ? edof_ : abs(edof_) - 1;
+            sendbuf[j+2+k] = pfes->GetGlobalTDofNumber(edof);
+         }
+         soffs[subdomain_rank[ip]] +=  2 + ndofs;
+      }
+   }
+
+   // d. Communication
+   Array<int> recvbuf(rbuff_size);
+   MPI_Alltoallv(sendbuf, send_count, send_displ, MPI_INT, recvbuf,
+                 recv_count, recv_displ, MPI_INT, comm);
+
+   // 3. Extract from recv_buffer
+   std::vector<Array<int>> global_tdofs(nrsubdomains);
+   int k=0;
+   while (k<rbuff_size)
+   {
+      int ip = recvbuf[k++]; 
+      int ndofs = recvbuf[k++];
+      for (int i = 0; i < ndofs; ++i)
+      {
+         global_tdofs[ip].Append(recvbuf[i+k]); 
+      }
+      k += ndofs;
+   }
+
+   SubdomainGTrueDofs.resize(nrsubdomains);
+   // 4. Construct SubdomainTdof to Global mesh tdof maps
+   for (int ip=0; ip<nrsubdomains; ++ip)
+   {
+      if (myid != subdomain_rank[ip]) continue;
+      int nrdof = fes[ip]->GetTrueVSize();
+
+      SubdomainGTrueDofs[ip].SetSize(nrdof);
+      int nel = part->element_map[ip].Size();
+      int k = 0;
+      for (int iel = 0; iel<nel; ++iel)
+      {
+         Array<int> elem_dofs;
+         fes[ip]->GetElementDofs(iel,elem_dofs);
+         int ndof = elem_dofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int edof_ = elem_dofs[i];
+            int edof = (edof_ >= 0) ? edof_ : abs(edof_) - 1;
+            // rearranging dofs from serial fespace to pfes ordering
+            SubdomainGTrueDofs[ip][edof] = global_tdofs[ip][k++];
+         }
+      }
+   }
+
+   // 5. Communicate SubdomainGTrueDofs to participating ranks 
+   send_count = 0;  send_displ = 0;
+   recv_count = 0;  recv_displ = 0;
+
+   for (int ip = 0; ip < nrsubdomains; ++ip)
+   {
+      if (myid != subdomain_rank[ip]) continue;
+      int ndofs = SubdomainGTrueDofs[ip].Size();
+      for (int i = 0; i<ndofs; ++i)
+      {
+         int tdof = SubdomainGTrueDofs[ip][i];
+         int rank = get_rank(tdof,tdof_offsets);
+         if (rank == subdomain_rank[ip]) continue; // <--------------
+         send_count[rank] += 2; // 1 for the dof and 1 for the ip that goes to 
+      }
+   }
+
+   // communicate so that recv_count is constructed
+   MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,comm);
+   //
+   for (int k=0; k<num_procs-1; k++)
+   {
+      send_displ[k+1] = send_displ[k] + send_count[k];
+      recv_displ[k+1] = recv_displ[k] + recv_count[k];
+   }
+   sbuff_size = send_count.Sum();
+   rbuff_size = recv_count.Sum();
+
+   sendbuf.SetSize(sbuff_size);  
+   sendbuf = 0; soffs = 0;
+
+   for (int ip = 0; ip < nrsubdomains; ip++)
+   {
+      if (myid != subdomain_rank[ip]) continue;
+      int ndofs = SubdomainGTrueDofs[ip].Size();
+      // loop through dofs
+      for (int i = 0; i<ndofs; ++i)
+      {
+         int tdof = SubdomainGTrueDofs[ip][i];
+         int irank = get_rank(tdof,tdof_offsets);
+         if (irank == subdomain_rank[ip]) continue; // <--------------
+         int j = send_displ[irank] + soffs[irank];
+         sendbuf[j] = ip;
+         sendbuf[j+1] = SubdomainGTrueDofs[ip][i];
+         soffs[irank] +=  2 ;
+      }
+   }
+
+   recvbuf.SetSize(rbuff_size);
+   MPI_Alltoallv(sendbuf, send_count, send_displ, MPI_INT, recvbuf,
+                 recv_count, recv_displ, MPI_INT, comm);
+
+   // List of tdofs owned by the processor for subdomains not owned
+   SubdomainLTrueDofs.resize(nrsubdomains); 
+   for (int k=0; k<rbuff_size/2; k++)
+   {
+      int ip = recvbuf[2*k];
+      int tdof = recvbuf[2*k+1];
+      SubdomainLTrueDofs[ip].Append(tdof);
+   }    
+}
+
+// Restriction of global residual to subdomain residuals
+void DofMaps::GlobalToSubdomains(const Vector & y, Array<Vector*> & x)
+{
+   send_count = 0; send_displ = 0;
+   recv_count = 0; recv_displ = 0;
+
+   // Compute send_counts
+   int m = (CompFlag) ? 2 : 1 ;
+   for (int ip = 0; ip < nrsubdomains; ip++)
+   {
+      if (myid == subdomain_rank[ip]) continue;  // <---------------
+      int ndofs = SubdomainLTrueDofs[ip].Size();
+      send_count[subdomain_rank[ip]] += m * ndofs;
+   }
+
+    // communicate so that recv_count is constructed
+   MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,comm);
+
+   for (int k=0; k<num_procs-1; k++)
+   {
+      send_displ[k+1] = send_displ[k] + send_count[k];
+      recv_displ[k+1] = recv_displ[k] + recv_count[k];
+   }
+   sbuff_size = send_count.Sum();
+   rbuff_size = recv_count.Sum();
+
+   Array<double> sendbuf(sbuff_size); sendbuf = 0;
+   Array<int> soffs(num_procs); soffs = 0;
+
+   for (int ip = 0; ip < nrsubdomains; ip++)
+   {
+      if (myid == subdomain_rank[ip]) continue;  // <---------------
+      int ndofs = SubdomainLTrueDofs[ip].Size();
+      for (int i = 0; i<ndofs; i++)
+      {
+         int tdof = SubdomainLTrueDofs[ip][i];
+         int j = send_displ[subdomain_rank[ip]] + soffs[subdomain_rank[ip]];
+         soffs[subdomain_rank[ip]] +=m;
+         int k = tdof - mytoffset;
+         // sendbuf[j] = y[k];
+         sendbuf[j] = tdof_sign[k]*y[k];
+         if (CompFlag)
+         {  // if complex valued 
+            int tsize = pfes->GetTrueVSize();
+            // sendbuf[j+1] = y[k+tsize];
+            sendbuf[j+1] = tdof_sign[k]*y[k+tsize];
+         }
+      }
+   }
+
+   // communication
+   Array<double> recvbuf(rbuff_size);
+   MPI_Alltoallv(sendbuf, send_count, send_displ, MPI_DOUBLE, recvbuf,
+                 recv_count, recv_displ, MPI_DOUBLE, comm);
+   Array<int> roffs(num_procs);
+   roffs = 0;
+   // Now each process will construct the res vector
+   x.SetSize(nrsubdomains);
+   for (int ip = 0; ip < nrsubdomains; ip++)
+   {
+      if (myid != subdomain_rank[ip]) continue;
+      int ndof = SubdomainGTrueDofs[ip].Size();
+      if (!x[ip]) x[ip] = new Vector(m*ndof); 
+      *x[ip] = 0.0;
+      // extract the data from receiv buffer
+      for (int i=0; i<ndof; i++)
+      {
+         // pick up the tdof and find its rank
+         int tdof = SubdomainGTrueDofs[ip][i];
+         int tdof_rank = get_rank(tdof,tdof_offsets);
+         if (tdof_rank != subdomain_rank[ip])  // <---------------
+         {
+            int k = recv_displ[tdof_rank] + roffs[tdof_rank];
+            roffs[tdof_rank] += m;
+            (*x[ip])[i] = recvbuf[k];
+            if (CompFlag)
+            {  
+               (*x[ip])[i+ndof] = recvbuf[k+1];
+            }
+         }
+         else
+         {
+            int k = tdof - mytoffset;
+            // (*x[ip])[i] = y[k];
+            (*x[ip])[i] = tdof_sign[k]*y[k];
+            if (CompFlag)
+            {  
+               int gtsize = pfes->GetTrueVSize();
+               (*x[ip])[i+ndof] = tdof_sign[k]*y[k+gtsize];
+            }
+         }
+      }
+   }
+}
+
+// Prolongation of subdomain solutions to the global solution
+void DofMaps::SubdomainsToGlobal(const Array<Vector*> & x, Vector & y)
+{
+   send_count = 0; send_displ = 0;
+   recv_count = 0; recv_displ = 0;
+
+   // Compute send_counts
+   int m = (CompFlag) ? 2 : 1 ;
+   for (int ip = 0; ip < nrsubdomains; ip++)
+   {
+      if (myid != subdomain_rank[ip]) continue;
+      int ndofs = SubdomainGTrueDofs[ip].Size();
+      for (int i=0; i<ndofs; i++)
+      {
+         // pick up the tdof and find its rank
+         int tdof = SubdomainGTrueDofs[ip][i];
+         int tdof_rank = get_rank(tdof,tdof_offsets);
+         if (tdof_rank == subdomain_rank[ip]) continue;
+         send_count[tdof_rank] +=m;
+      }
+   }
+
+   MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,comm);
+
+   for (int k=0; k<num_procs-1; k++)
+   {
+      send_displ[k+1] = send_displ[k] + send_count[k];
+      recv_displ[k+1] = recv_displ[k] + recv_count[k];
+   }
+   sbuff_size = send_count.Sum();
+   rbuff_size = recv_count.Sum();
+
+   Array<double> sendbuf(sbuff_size); sendbuf = 0;
+   Array<int> soffs(num_procs); soffs = 0;
+
+   for (int ip = 0; ip < nrsubdomains; ip++)
+   {
+      if (myid != subdomain_rank[ip]) continue;
+      int ndofs = SubdomainGTrueDofs[ip].Size();
+      // loop through dofs
+      for (int i=0; i<ndofs; i++)
+      {
+         //  pick up the dof and find its tdof_rank
+         int tdof = SubdomainGTrueDofs[ip][i];
+         int tdof_rank = get_rank(tdof,tdof_offsets);
+         // offset
+         if (tdof_rank == subdomain_rank[ip]) continue;
+         int k = send_displ[tdof_rank] + soffs[tdof_rank];
+         soffs[tdof_rank] +=m;
+         sendbuf[k] = (*x[ip])[i];
+         if (CompFlag)
+         {
+            sendbuf[k+1] = (*x[ip])[i+ndofs];
+         }
+      }
+   }
+
+   Array<double> recvbuf(rbuff_size);
+   Array<int> roffs(num_procs); roffs = 0;
+   MPI_Alltoallv(sendbuf, send_count, send_displ, MPI_DOUBLE, recvbuf,
+                 recv_count, recv_displ, MPI_DOUBLE, comm);
+
+   for (int ip = 0; ip < nrsubdomains; ip++)
+   {
+      if (myid == subdomain_rank[ip]) 
+      {
+         int ndofs = SubdomainGTrueDofs[ip].Size();
+         for (int i = 0; i<ndofs; i++)
+         {
+            int tdof = SubdomainGTrueDofs[ip][i];
+            int k = tdof - mytoffset;
+            if (k<0 || k>=pfes->GetTrueVSize()) continue;
+            y[k] += tdof_sign[k] * (*x[ip])[i];
+            if (CompFlag)
+            {  
+               int gtsize = pfes->GetTrueVSize();
+               y[k+gtsize] += tdof_sign[k]*(*x[ip])[i+ndofs];
+            }
+         }
+      }
+      else
+      {
+         int ndofs = SubdomainLTrueDofs[ip].Size();
+         for (int i = 0; i<ndofs; i++)
+         {
+            int tdof = SubdomainLTrueDofs[ip][i];
+            int k = tdof - mytoffset;
+            int j = recv_displ[subdomain_rank[ip]] + roffs[subdomain_rank[ip]];
+            roffs[subdomain_rank[ip]] +=m;
+            y[k] += tdof_sign[k] * recvbuf[j];
+            if (CompFlag)
+            {
+               int tsize = pfes->GetTrueVSize();
+               y[k+tsize] += tdof_sign[k]*recvbuf[j+1];
+            }
+         }
+      }
+   }              
+}
+
+void DofMaps::TestSubdomainToGlobalMaps()
+{
+   cout << "Testing Subdomain To Global Maps" << endl;
+   FunctionCoefficient c1(testcoeff);
+   Array<Vector*> x(nrsubdomains);
+   Vector y(pfes->GetTrueVSize()); y = 0.0;
+   for (int i = 0 ; i<nrsubdomains; i++)
+   {
+      if (myid != subdomain_rank[i]) continue;
+      x[i] = new Vector(fes[i]->GetTrueVSize());
+      GridFunction gf(fes[i]);
+      gf = 0.0;
+
+      if (i==3) gf.ProjectCoefficient(c1);
+      *x[i] = gf;
+   }
+
+   SubdomainsToGlobal(x,y);
+
+   // cout << "1: myid = " <<  myid << ", y = "; y.Print();
+
+   string keys = (dim==2) ? "keys amrRljc\n": "keys m\n";
+   ParGridFunction pgf(pfes);
+
+   const Operator &P = *pfes->GetProlongationMatrix();
+   P.Mult(y, pgf);
+   
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock(vishost, visport);
+   sol_sock.precision(8);
+   sol_sock << "parallel " << num_procs << " " << myid << "\n"
+               << "solution\n" << *pfes->GetParMesh() << pgf 
+               << keys << flush;
+
+   ParGridFunction pgf1(pfes);
+   pgf1.ProjectCoefficient(c1);
+   Vector y1(pfes->GetTrueVSize());
+   const SparseMatrix * R = pfes->GetRestrictionMatrix();
+
+   R->Mult(pgf1,y1);
+   // P.MultTranspose(pgf1,y1);
+   Array<Vector*> x1;
+   GlobalToSubdomains(y1,x1);
+
+
+   // for (int i = 0 ; i<nrsubdomains; i++)
+   // {
+   //    if (myid != subdomain_rank[i]) continue;
+   //    ostringstream mesh_name;
+   //    mesh_name << "output/mesh." << setfill('0') << setw(6) << i;
+   //    ofstream mesh_ofs(mesh_name.str().c_str());
+   //    mesh_ofs.precision(8);
+   //    fes[i]->GetMesh()->Print(mesh_ofs);
+   //    GridFunction gf(fes[i]);
+   //    gf = x1[i];
+   //    ostringstream gf_name;
+   //    gf_name << "output/gf." << setfill('0') << setw(6) << i;
+   //    ofstream gf_ofs(gf_name.str().c_str());
+   //    gf_ofs.precision(8);
+   //    gf.Save(gf_ofs);
+   // }
+
+
+
+   int nrsub = nrsubdomains;
+   for (int i = 0 ; i<nrsub; i++)
+   {
+      if (myid == subdomain_rank[i])
+      {
+         socketstream sol_sock1(vishost, visport);
+         sol_sock1.precision(8);
+         sol_sock1 << "parallel " << nrsub << " " << i << "\n";
+         GridFunction gf(fes[i]);
+         GridFunction gf1(fes[i]);
+         gf1.ProjectCoefficient(c1);
+         gf = *x1[i];
+         gf1-=gf;
+         cout << "ip, Diff norm = " <<i<<", " << gf1.Norml2() << endl;
+         sol_sock1 << "solution\n" << *fes[i]->GetMesh() << gf
+                   << keys << flush;
+      }
+      MPI_Barrier(MPI_COMM_WORLD);
+   }
+
+   socketstream gf_sock(vishost, visport);
+   gf_sock.precision(8);
+   gf_sock << "parallel " << num_procs << " " << myid << "\n"
+           << "solution\n" << *pfes->GetParMesh() << pgf1 
+           << keys << flush;
+}
+
+
+DofMaps::~DofMaps()
+{
+   for (int i = 0; i<nrsubdomains; i++)
+   {
+      delete fes[i];
+   }
+}

examples/maxwell-solver/ParDST/DofMapsDST.hpp

@@ -0,0 +1,110 @@
+#pragma once
+#include "../common/Utilities.hpp"
+#include "../common/PML.hpp"
+#include "../DST/DST.hpp"
+using namespace std;
+using namespace mfem;
+
+double testcoeff(const Vector & x);
+int get_rank(int tdof, std::vector<int> & tdof_offsets);
+
+
+void ComputeTdofOffsets(const MPI_Comm & comm, const ParFiniteElementSpace * pfes, 
+                        std::vector<int> & tdof_offsets);
+
+void GetSubdomainijk(int ip, const Array<int> nxyz, Array<int> & ijk);
+void GetDirectionijk(int id, Array<int> & ijk);
+int GetSubdomainId(const Array<int> nxyz, Array<int> & ijk);
+int GetDirectionId(const Array<int> & ijk);
+
+
+// class handling two types of dof maps
+// 1. Subdomain truedofs ---> Global truedofs
+// 2. Subdomain truedofs ---> Neighbor truedofs 
+class DofMaps
+{
+private:
+   // The FE space of the problem (H1/Hcurl)
+   ParFiniteElementSpace *pfes = nullptr;
+
+   // The given partition of the parmesh
+   ParMeshPartition *part = nullptr;
+   // partition in x-y-z
+   Array<int> nxyz;
+
+   // MPI parameters
+   MPI_Comm comm = MPI_COMM_WORLD;
+   int num_procs, myid;
+
+   // true dof offset and element offset of the processor
+   vector<int> tdof_offsets;
+   int mytoffset;
+   int myelemoffset;
+   
+   int dim;
+   // Total number of subdomains
+   int nrsubdomains;
+   
+   // Array specifying the subdomain rank
+   Array<int> subdomain_rank;
+
+   // Complex flag
+   bool CompFlag;
+
+   // sign factors
+   Array<int> tdof_sign;
+   // Initializing mpi and helper parameters
+   void Init();
+
+   // 1. Setting up the subdomains FE spaces
+   // 2. Setting up the subdomains-to-subdomains maps
+   // 3. Setting up the subdomain-to-global maps
+   void Setup();
+
+   // -----------------------------------------------
+   //           Subdomain to Subdomain maps
+   // -----------------------------------------------
+   std::vector<std::vector<Array<int>>> OvlpElems;
+   void AddElementToOvlpLists(int l, int iel, 
+                            const Array<bool> & neg, 
+                            const Array<bool> & pos);
+   std::vector<std::vector<Array<int>>> OvlpTDofs;
+   void SubdomainToSubdomainMapsSetup();
+   void ComputeOvlpElems();
+   void ComputeOvlpTdofs();
+   void PrintOvlpTdofs();
+
+   // -----------------------------------------------
+   //           Subdomain to Global maps
+   // -----------------------------------------------
+   std::vector<Array<int>> SubdomainGTrueDofs; // Subdomain Tdofs to Global Tdofs
+   std::vector<Array<int>> SubdomainLTrueDofs; // Subdomain Tdofs to Local (on rank) Tdofs
+
+   Array<int> send_count, send_displ;  
+   Array<int> recv_count, recv_displ;  
+   int sbuff_size = 0;
+   int rbuff_size = 0;
+   void SubdomainToGlobalMapsSetup();
+
+   // Testing
+   void TestSubdomainToGlobalMaps();
+   void TestSubdomainToSubdomainMaps();
+
+public:
+   // constructor
+
+   // FiniteElementSpaces of the subdomains
+   Array<FiniteElementSpace *> fes;
+
+   DofMaps(ParFiniteElementSpace *fespace_, ParMeshPartition * part_, bool CompFlag_ = false);
+   ~DofMaps();
+   // Transfering from subdomains SubdomainIds to all their neighbors
+   void TransferToNeighbors(const Array<int> & SubdomainIds, const Array<Vector *> & x, 
+   std::vector<std::vector<Vector * >> & OvlpSol);
+
+   // Prolongation of subdomain solutions to the global solution
+   void SubdomainsToGlobal(const Array<Vector*> & x, Vector & y);
+   // Restriction of global residual to subdomain residuals
+   // bool comp: true for complex valued problems
+   void GlobalToSubdomains(const Vector & y, Array<Vector*> & x);
+};

examples/maxwell-solver/ParDST/ParDST.cpp

@@ -0,0 +1,941 @@
+//Parallel Diagonal Source Transfer Preconditioner
+
+#include "ParDST.hpp"
+
+ParDST::ParDST(ParSesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+         double omega_, Coefficient * Q_,  
+         int nrlayers_ , int nx_, int ny_, int nz_, 
+         BCType bc_type_, Coefficient * LossCoeff_)
+   : Solver(2*bf_->ParFESpace()->GetTrueVSize(), 2*bf_->ParFESpace()->GetTrueVSize()), 
+     bf(bf_), Pmllength(Pmllength_), omega(omega_), 
+     Q(Q_), nrlayers(nrlayers_), bc_type(bc_type_), LossCoeff(LossCoeff_)
+{
+   nx = nx_; ny = ny_; nz = nz_;
+   Init();
+}
+ParDST::ParDST(ParSesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+         double omega_, MatrixCoefficient * MQ_,  
+         int nrlayers_ , int nx_, int ny_, int nz_, 
+         BCType bc_type_, Coefficient * LossCoeff_)
+   : Solver(2*bf_->ParFESpace()->GetTrueVSize(), 2*bf_->ParFESpace()->GetTrueVSize()), 
+     bf(bf_), Pmllength(Pmllength_), omega(omega_), 
+     MQ(MQ_), nrlayers(nrlayers_), bc_type(bc_type_), LossCoeff(LossCoeff_)
+{
+   nx = nx_; ny = ny_; nz = nz_;
+   Init();
+}
+
+void ParDST::Init()
+{
+   pfes = bf->ParFESpace();
+   fec = pfes->FEColl();
+ 
+   comm = pfes->GetComm();
+   MPI_Comm_size(comm, &num_procs);
+   MPI_Comm_rank(comm, &myid);
+
+   //1. Indentify problem ... Helmholtz or Maxwell
+   prob_kind = fec->GetContType();
+   if (myid == 0)
+   {
+      cout << " 1. Indentify problem to be solved ... " << endl;
+      if (prob_kind == 0) cout << "    Helmholtz" << endl;
+      if (prob_kind == 1) cout << "    Maxwell" << endl; 
+   }
+
+   //2. Create the parallel mesh partition
+   pmesh = pfes->GetParMesh();
+   dim = pmesh->Dimension();
+   if (myid == 0)
+   {
+      cout << "\n 2. Generating ParMesh partitioning ... " << endl;
+   }
+   ovlpnrlayers = nrlayers+1;
+   part = new ParMeshPartition(pmesh,nx,ny,nz,ovlpnrlayers);
+   nxyz.SetSize(3);
+   nxyz[0] = nx = part->nxyz[0]; 
+   nxyz[1] = ny = part->nxyz[1];  
+   nxyz[2] = nz = part->nxyz[2];
+
+   nrsubdomains = part->nrsubdomains;
+   SubdomainRank = part->subdomain_rank;
+
+   for (int ip = 0; ip<nrsubdomains; ip++)
+   {
+      if (myid == SubdomainRank[ip])
+      {
+         RankSubdomains.Append(ip);
+      }
+   }
+
+   cout << " myid: " << myid 
+        << ", nrsubdomains: " << RankSubdomains.Size() << endl;
+
+   MPI_Barrier(comm);
+
+   if (myid == 0)
+   {
+      cout << "    Done ! " << endl;
+   }
+   //3. Setup info for sweeps
+   if (myid == 0)
+   {
+      cout << "\n 3. Computing sweeps info ..." << endl; 
+   }
+   sweeps = new Sweep(dim);
+   if (myid == 0)
+   {
+      cout << "    Done ! " << endl;
+   }
+   //4. Create LocalToGlobal maps 
+   //   (local GridFunctions/Vector to Global ParGridFunction/Vector) 
+   if (myid == 0)
+   {
+      cout << "\n 4. Computing true dofs maps ..." << endl; 
+   }
+
+   // if (myid == SubdomainRank[0])
+   // {
+   //    cout << "myid = " << myid << endl;
+   //    char vishost[] = "localhost";
+   //    int visport = 19916;
+   //    socketstream mesh_sock1(vishost, visport);
+   //    mesh_sock1.precision(8);
+   //    mesh_sock1 << "mesh\n"
+   //               << *part->subdomain_mesh[0] << "window_title 'Subdomain'" << flush;
+   //    part->subdomain_mesh[0]->Print();
+   
+   // }
+   bool comp = true;
+
+   dmaps = new DofMaps(pfes,part, comp);
+   
+   if (myid == 0)
+   {
+      cout << "    Done ! " << endl;
+   }
+   // 4. Setting up the local problems 
+   if (myid == 0)
+   {
+      cout << "\n 5. Setting up the subdomain problems ..." << endl; 
+   }
+
+   SetupSubdomainProblems();
+   
+   if (myid == 0)
+   {
+      cout << "    Done ! " << endl;
+   }
+   if (myid == 0)
+   {
+      cout << "\n 6. Mark subdomain overlap truedofs ..." << endl; 
+   }
+   MarkSubdomainOverlapDofs(comp);
+   if (myid == 0)
+   {
+      cout << "    Done ! " << endl;
+   }
+}
+
+void ParDST::Mult(const Vector &r, Vector &z) const
+{
+   // Initialize transfered residuals to 0.0;
+   for (int ip=0; ip<nrsubdomains; ip++)
+   {
+      if (myid != SubdomainRank[ip]) continue;
+      for (int i=0;i<sweeps->nsweeps; i++)
+      {
+         *f_transf[ip][i] = 0.0;
+      }
+   }
+
+   // restrict given residual to subdomains
+   dmaps->GlobalToSubdomains(r,f_orig);
+
+   for (int ip=0; ip<nrsubdomains; ip++)
+   {
+      if (myid != SubdomainRank[ip]) continue;
+      Array<int> ijk(3);
+      GetSubdomainijk(ip,nxyz,ijk);
+      Array2D<int> direct(dim,2); direct = 0;
+      for (int d=0;d<dim; d++)
+      {
+         if (ijk[d] > 0) direct[d][0] = 1; 
+         if (ijk[d] < part->nxyz[d]-1) direct[d][1] = 1; 
+      }
+      GetChiRes(*f_orig[ip],ip,direct);
+   }
+
+   z = 0.0; 
+   int nsweeps = sweeps->nsweeps;
+   // 1. Loop through sweeps
+   if (dim == 3 && nz == 1) { nsweeps = 4; } // x-y partition only;
+   for (int l=0; l<nsweeps; l++)
+   {  
+      // cout << "sweep = " << l << endl;
+      int nsteps = GetSweepNumSteps(l);
+      // 2. loop through diagonals/steps of each sweep   
+      for (int s = 0; s<nsteps; s++)
+      {
+         // cout << "step = " << s << endl;
+         Array2D<int> subdomains;
+         GetStepSubdomains(l,s,subdomains);
+         // cout << "subdomains = " << endl;
+         // subdomains.Print(cout, subdomains.NumCols());
+         // cin.get();
+
+         int nsubdomains = subdomains.NumRows();
+
+         // 3. Loop through the subdomains on the diagonal
+         Array<int> subdomain_ids;
+         for (int sb=0; sb < nsubdomains; sb++)
+         {
+            Array<int> ijk(dim); ijk = 0;
+            for (int d=0; d<dim; d++) ijk[d] = subdomains[sb][d]; 
+            int ip = GetSubdomainId(nxyz,ijk);
+            subdomain_ids.Append(ip);
+            if (myid != SubdomainRank[ip]) continue;
+
+            int n = dmaps->fes[ip]->GetTrueVSize();
+            Vector res_local(2*n); res_local = 0.0;
+
+            if (l==0)  { res_local += *f_orig[ip]; }
+            res_local += *f_transf[ip][l];
+            if (res_local.Norml2() < 1e-12) 
+            {
+               *subdomain_sol[ip] = 0.0;
+               continue;
+            }
+            
+            // char vishost[] = "localhost";
+            // int visport = 19916;
+
+            // socketstream res_sock(vishost, visport);
+            // PlotLocal(res_local,res_sock,ip);
+
+            PmlMatInv[ip]->Mult(res_local, *subdomain_sol[ip]);
+            // GetSubdomainijk(ip,nxyz,ijk);
+            // Array2D<int> direct(dim,2); direct = 0;
+            // for (int d=0;d<dim; d++)
+            // {
+            //    if (ijk[d] > 0) direct[d][0] = 1; 
+            //    if (ijk[d] < part->nxyz[d]-1) direct[d][1] = 1; 
+            // }
+            // cout << "direct = " ; direct.Print();
+            // GetChiRes(*subdomain_sol[ip],ip,direct);
+
+            // socketstream sol_sock1(vishost, visport);
+            // PlotLocal(*subdomain_sol[ip],sol_sock1,ip);
+            // cout << "ip = " << ip << endl;
+            // cin.get();
+         }
+         // 4. Transfer solutions to neighbors so that the subdomain
+         // residuals are updated
+         TransferSources(l,subdomain_ids);
+      }
+      // 5. Update the global solution 
+      dmaps->SubdomainsToGlobal(subdomain_sol,z);
+      // char vishost[] = "localhost";
+      // int visport = 19916;
+      // socketstream sol_sock1(vishost, visport);
+      // PlotGlobal(z,sol_sock1);
+      // cin.get();
+   }
+   
+}
+
+void ParDST::SetupSubdomainProblems()
+{
+   sqf.SetSize(nrsubdomains);
+   Optr.SetSize(nrsubdomains);
+   PmlMat.SetSize(nrsubdomains);
+   PmlMatInv.SetSize(nrsubdomains);
+   f_orig.SetSize(nrsubdomains);
+   f_transf.resize(nrsubdomains);
+   subdomain_sol.SetSize(nrsubdomains);
+   for (int ip=0; ip<nrsubdomains; ip++)
+   {
+      sqf[ip] = nullptr;
+      f_orig[ip] = nullptr;
+      subdomain_sol[ip] = nullptr;
+      PmlMat[ip] = nullptr;
+      PmlMatInv[ip] = nullptr;
+      Optr[ip] = nullptr;
+
+      if (myid != SubdomainRank[ip]) continue;
+      subdomain_sol[ip] = new Vector(2*dmaps->fes[ip]->GetTrueVSize());
+      if (prob_kind == 0)
+      {
+         SetHelmholtzPmlSystemMatrix(ip);
+      }
+      else if (prob_kind == 1)
+      {
+         SetMaxwellPmlSystemMatrix(ip);
+      }
+      PmlMat[ip] = Optr[ip]->As<ComplexSparseMatrix>();
+
+      PmlMatInv[ip] = new ComplexUMFPackSolver;
+      PmlMatInv[ip]->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
+
+      // HYPRE_Int rowstarts[2]; rowstarts[0] = 0;
+      // rowstarts[1] = dmaps->fes[ip]->GetTrueVSize();
+      // HypreParMatrix * HypreMat_r =
+      //    new HypreParMatrix(MPI_COMM_SELF,rowstarts[1],rowstarts,
+      //                       &(PmlMat[ip]->real()));
+      // HypreParMatrix * HypreMat_i =
+      //    new HypreParMatrix(MPI_COMM_SELF,rowstarts[1],rowstarts,
+      //                       &(PmlMat[ip]->imag()));
+      // ComplexHypreParMatrix * HypreMat =
+      //    new ComplexHypreParMatrix(HypreMat_r,HypreMat_i,true,true);
+      // PmlMatInv[ip] = new ComplexMUMPSSolver;
+      // PmlMatInv[ip]->SetOperator(*HypreMat);
+      // delete HypreMat;
+      int ndofs = dmaps->fes[ip]->GetTrueVSize();
+      f_transf[ip].SetSize(sweeps->nsweeps);
+      for (int i=0;i<sweeps->nsweeps; i++)
+      {
+         f_transf[ip][i] = new Vector(2*ndofs);
+      }
+   }
+}
+
+
+
+void ParDST::SetHelmholtzPmlSystemMatrix(int ip)
+{
+   MFEM_VERIFY(part->subdomain_mesh[ip], "Null mesh pointer");
+   Mesh * mesh = part->subdomain_mesh[ip];
+   double h = part->MeshSize;
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+
+   Array<int> ijk;
+   GetSubdomainijk(ip,nxyz,ijk);
+   int i = ijk[0];
+   int j = ijk[1];
+   int k = ijk[2];
+
+   if (i == 0 )    length[0][0] = Pmllength[0][0];
+   if (i == nx-1 ) length[0][1] = Pmllength[0][1];
+   if (dim  > 1)
+   {
+      if (j == 0 )    length[1][0] = Pmllength[1][0];
+      if (j == ny-1 ) length[1][1] = Pmllength[1][1];
+   }
+   if (dim  == 3)
+   {
+      if (k == 0 )    length[2][0] = Pmllength[2][0];
+      if (k == nz-1 ) length[2][1] = Pmllength[2][1];
+   }
+   
+   CartesianPML pml(mesh, length);
+   pml.SetOmega(omega);
+
+   Array <int> ess_tdof_list;
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = (bc_type == BCType::DIRICHLET) ? 1 : 0;
+      dmaps->fes[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+   ProductCoefficient c2_re(c2_re0, *Q);
+   ProductCoefficient c2_im(c2_im0, *Q);
+   sqf[ip] = new SesquilinearForm (dmaps->fes[ip],bf->GetConvention());
+
+   sqf[ip]->AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   sqf[ip]->AddDomainIntegrator(new MassIntegrator(c2_re),
+                         new MassIntegrator(c2_im));
+   sqf[ip]->Assemble();
+
+   Optr[ip] = new OperatorPtr;
+   sqf[ip]->FormSystemMatrix(ess_tdof_list,*Optr[ip]);
+}
+
+void ParDST::SetMaxwellPmlSystemMatrix(int ip)
+{
+   MFEM_VERIFY(part->subdomain_mesh[ip], "Null mesh pointer");
+   Mesh * mesh = part->subdomain_mesh[ip];
+   double h = part->MeshSize;
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+
+   Array<int> ijk;
+   GetSubdomainijk(ip,nxyz,ijk);
+   int i = ijk[0];
+   int j = ijk[1];
+   int k = ijk[2];
+
+   if (i == 0 )    length[0][0] = Pmllength[0][0];
+   if (i == nx-1 ) length[0][1] = Pmllength[0][1];
+   if (dim  > 1)
+   {
+      if (j == 0 )    length[1][0] = Pmllength[1][0];
+      if (j == ny-1 ) length[1][1] = Pmllength[1][1];
+   }
+   if (dim  == 3)
+   {
+      if (k == 0 )    length[2][0] = Pmllength[2][0];
+      if (k == nz-1 ) length[2][1] = Pmllength[2][1];
+   }
+   
+   CartesianPML pml(mesh, length);
+   pml.SetOmega(omega);
+   pml.SetAttributes(mesh);
+
+   Array <int> ess_tdof_list;
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = (bc_type == BCType::DIRICHLET) ? 1 : 0;
+      dmaps->fes[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   Array<int> attr;
+   Array<int> attrPML;
+   if (mesh->attributes.Size())
+   {
+      attr.SetSize(mesh->attributes.Max());
+      attrPML.SetSize(mesh->attributes.Max());
+      attr = 0;   attr[0] = 1;
+      attrPML = 0;
+      if (mesh->attributes.Max() > 1)
+      {
+         attrPML[1] = 1;
+      }
+   }
+
+   // Integrators inside the computational domain (excluding the PML region)
+   double mu = 1.0;
+   double epsilon = 1.0;
+   ConstantCoefficient muinv(1.0/mu);
+   ConstantCoefficient omeg(-pow(omega, 2)* epsilon);
+   RestrictedCoefficient * restr_loss = nullptr;
+
+   RestrictedCoefficient restr_muinv(muinv,attr);
+   RestrictedCoefficient restr_omeg(omeg,attr);
+
+   sqf[ip] = new SesquilinearForm(dmaps->fes[ip],bf->GetConvention());
+   sqf[ip]->SetDiagonalPolicy(mfem::Matrix::DIAG_ONE);
+
+   sqf[ip]->AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv),NULL);
+   sqf[ip]->AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);
+
+   if (LossCoeff) 
+   {
+      restr_loss = new RestrictedCoefficient(*LossCoeff,attr);
+      // sqf[ip]->AddDomainIntegrator(NULL, new VectorFEMassIntegrator(*restr_loss));      
+      sqf[ip]->AddDomainIntegrator(NULL, new VectorFEMassIntegrator(*LossCoeff));      
+   }
+
+
+   int cdim = (dim == 2) ? 1 : dim;
+
+   PmlMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, &pml);
+   PmlMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, &pml);
+   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+
+
+   PmlMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,&pml);
+   ScalarMatrixProductCoefficient c2_Re0(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im0(omeg,pml_c2_Im);
+
+   MatrixCoefficient * c2_Re=nullptr;
+   MatrixCoefficient * c2_Im=nullptr;
+
+   if (Q)
+   {
+      c2_Re = new ScalarMatrixProductCoefficient(*Q,c2_Re0);
+      c2_Im = new ScalarMatrixProductCoefficient(*Q,c2_Im0);
+   }
+   else if (MQ)
+   {
+      c2_Re = new MatrixMatrixProductCoefficient(c2_Re0,*MQ);
+      c2_Im = new MatrixMatrixProductCoefficient(c2_Im0,*MQ);
+   }
+   
+   MatrixRestrictedCoefficient restr_c2_Re(*c2_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c2_Im(*c2_Im,attrPML);
+
+
+   sqf[ip]->AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
+                                new CurlCurlIntegrator(restr_c1_Im));
+   sqf[ip]->AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
+                                new VectorFEMassIntegrator(restr_c2_Im));
+   sqf[ip]->Assemble();
+   Optr[ip] = new OperatorPtr;
+   sqf[ip]->FormSystemMatrix(ess_tdof_list,*Optr[ip]);
+   delete c2_Re;
+   delete c2_Im;
+   if (LossCoeff)  delete restr_loss;
+}
+
+
+void ParDST::MarkSubdomainOverlapDofs(const bool comp)
+{
+   // First mark the elements
+         // cout<< "Compute Overlap Elements (in each possible direction) " << endl;
+      // Lists of elements
+      // x,y,z = +/- 1 ovlp 
+   NovlpElems.resize(nrsubdomains);
+
+   for (int ip = 0; ip<nrsubdomains; ip++)
+   {
+      if (myid != SubdomainRank[ip]) continue;
+      Array<int> ijk;
+      GetSubdomainijk(ip,nxyz,ijk);
+
+      Mesh * mesh = dmaps->fes[ip]->GetMesh();
+      NovlpElems[ip].resize(2*dim);
+
+      Vector pmin, pmax;
+      mesh->GetBoundingBox(pmin,pmax);
+      double h = part->MeshSize;
+      // Loop through elements
+      for (int iel=0; iel<mesh->GetNE(); iel++)
+      {
+         // Get element center
+         Vector center(dim);
+         int geom = mesh->GetElementBaseGeometry(iel);
+         ElementTransformation * tr = mesh->GetElementTransformation(iel);
+         tr->Transform(Geometries.GetCenter(geom),center);
+
+         // Assign elements to the appropriate lists
+         for (int d=0;d<dim; d++)
+         {
+            if (ijk[d]>0)
+            {
+               if (center[d] >= pmin[d]+h*ovlpnrlayers) 
+               {
+                  NovlpElems[ip][d].Append(iel);
+               }
+            }
+            else
+            {
+               NovlpElems[ip][d].Append(iel);
+            }
+               
+            if (ijk[d]<nxyz[d]-1)
+            {
+               if (center[d] <= pmax[d]-h*ovlpnrlayers) 
+               {
+                  NovlpElems[ip][dim+d].Append(iel);
+               }
+            }
+            else
+            {
+               NovlpElems[ip][dim+d].Append(iel);
+            }
+         }
+      }
+   }
+
+      // mark dofs
+   NovlpDofs.resize(nrsubdomains);
+   int mm = (comp) ? 2 : 1; // complex or real valued
+   for (int ip = 0; ip<nrsubdomains; ip++)
+   {
+      if (myid != SubdomainRank[ip]) continue;
+      FiniteElementSpace * fes = dmaps->fes[ip];
+      // Loop through the marked elements
+      NovlpDofs[ip].resize(2*dim);
+      int n = fes->GetTrueVSize();
+      Array<int> marker(n);
+      for (int d=0;d<2*dim; d++)
+      {
+         marker = 0;
+         int m = 0;
+         int melems = NovlpElems[ip][d].Size(); 
+         for (int iel=0; iel<melems; iel++)
+         {
+            Array<int> ElemDofs;
+            int el =  NovlpElems[ip][d][iel];
+            fes->GetElementDofs(el,ElemDofs);
+            int ndof = ElemDofs.Size();
+            for (int i = 0; i<ndof; ++i)
+            {
+               int eldof = ElemDofs[i];
+               int tdof = (eldof >= 0) ? eldof : abs(eldof) - 1; 
+               if (marker[tdof] == 1) continue;
+               marker[tdof] = 1;
+               m++;
+            }
+         }
+         int k = mm*(n-m);
+         NovlpDofs[ip][d].SetSize(k);
+         int l = 0;
+         for (int i = 0; i<n; i++)
+         {
+            if (marker[i]==0) 
+            {
+               NovlpDofs[ip][d][l] = i;  // real dofs
+               if (comp) 
+               {
+                  NovlpDofs[ip][d][l+k/2] = i+fes->GetTrueVSize();
+               }
+               l++;
+            }
+         }
+      }
+   }
+}
+
+void ParDST::GetChiRes(Vector & res, int ip, Array2D<int> direct) const
+{
+   for (int d=0; d<dim; d++)
+   {
+      // negative direction
+      if (direct[d][0]==1) res.SetSubVector(NovlpDofs[ip][d],0.0);
+      // possitive direction
+      if (direct[d][1]==1) res.SetSubVector(NovlpDofs[ip][d+dim],0.0);
+   }
+}
+
+
+
+void ParDST::PlotLocal(Vector & sol, socketstream & sol_sock, int ip) const
+{
+   FiniteElementSpace * fes = dmaps->fes[ip];
+   Mesh * mesh = fes->GetMesh();
+   GridFunction gf(fes);
+   double * data = sol.GetData();
+   gf.SetData(data);
+   
+   string keys;
+   keys = "keys mrRljc\n";
+   sol_sock << "solution\n" << *mesh << gf << keys << flush;
+}
+
+void ParDST::PlotGlobal(Vector & sol, socketstream & sol_sock) const
+{
+   ParMesh * pmesh = pfes->GetParMesh();
+   ParGridFunction pgf(pfes);
+   double * data = sol.GetData();
+   pgf.SetData(data);
+   string keys;
+   keys = "keys mrRljc\n";
+   sol_sock << "solution\n" << *pmesh << pgf << keys << flush;
+}
+
+
+double ParDST::GetSweepNumSteps(const int sweep) const
+{
+   int nsteps;
+   switch(dim)
+   {
+      case 1: nsteps = nx; break;
+      case 2: nsteps = nx+ny-1; break;
+      default: nsteps = nx+ny+nz-2; break;
+   }
+   return nsteps;
+}
+
+void ParDST::GetStepSubdomains(const int sweep, const int step, Array2D<int> & subdomains) const
+{
+   Array<int> aux;
+
+   switch(dim)
+   {
+      case 2: 
+      for (int i=nx-1;i>=0; i--)
+      {  
+         int j;
+         switch (sweep)
+         {
+            case 0:  j = step-i;         break;
+            case 1:  j = step-nx+i+1;    break;
+            case 2:  j = nx+i-step-1;    break;
+            default: j = nx+ny-i-step-2; break;
+         }
+         if (j<0 || j>=ny) continue;
+         aux.Append(i); aux.Append(j);
+      }
+      break; 
+      default:
+      for (int i=nx-1;i>=0; i--)
+      {  
+         for (int j=ny-1;j>=0; j--)
+         {
+            int k;
+            switch (sweep)
+            {
+               case 0:  k = step-i-j;            break;
+               case 1:  k = step-nx+i+1-j;       break;
+               case 2:  k = step-ny+j+1-i;       break;
+               case 3:  k = step-nx-ny+i+j+2;    break;
+               case 4:  k = i+j+nz-1-step;       break;
+               case 5:  k = nx+nz-i+j-step-2;    break;
+               case 6:  k = ny+nz+i-j-step-2;    break;
+               default: k = nx+ny+nz-i-j-step-3; break;
+            }
+            if (k<0 || k>=nz) continue;   
+            aux.Append(i); aux.Append(j); aux.Append(k); 
+         }
+      }
+      break;
+   }
+
+   int nrows = aux.Size()/dim;
+   int ncols = dim;
+
+   subdomains.SetSize(nrows,ncols);
+   for (int r=0;r<nrows; r++)
+   {
+      for (int c=0; c<ncols; c++)
+      {
+         int k = r*ncols + c;
+         subdomains[r][c] = aux[k];
+      }
+   }
+}
+
+
+void ParDST::TransferSources(int sweep, const Array<int> & subdomain_ids) const
+{
+   OvlpSol.resize(nrsubdomains);
+   int nrneighbors = pow(3,dim);
+   for (int ip = 0; ip<nrsubdomains; ip++)
+   {
+      if (myid == SubdomainRank[ip])
+      {
+         OvlpSol[ip].resize(nrneighbors);
+      }
+   }
+   int m = subdomain_ids.Size();
+   Array<Vector *> x(m);
+   for (int i = 0; i<m; i++)
+   {
+      x[i] = nullptr;
+      int ip = subdomain_ids[i];
+      if (myid != SubdomainRank[ip]) continue;
+      x[i] = new Vector(subdomain_sol[ip]->GetData(),subdomain_sol[ip]->Size());
+   }
+   dmaps->TransferToNeighbors(subdomain_ids,x,OvlpSol);
+   for (int i = 0; i<m; i++)
+   {
+      delete x[i]; x[i] = nullptr;
+   }
+   // Update residuals
+//   Find all neighbors of patch ip0
+   for (int is = 0; is<m; is++)
+   {
+      int ip0 = subdomain_ids[is];
+      Array<int> ijk;
+      Array<int> ijk1(3);
+      GetSubdomainijk(ip0,nxyz,ijk);
+      // cout << "Subdomain to transfer its sources: " << "(" <<ijk[0] << "," << ijk[1] << ")" <<endl;
+      Array<int> directions(3);   
+      for (int i=-1; i<2; i++)
+      {
+         int i1 = ijk[0] + i;
+         if (i1 <0 || i1>=nx) continue;
+         directions[0] = i;
+         ijk1[0] = i1;
+         for (int j=-1; j<2; j++)
+         {
+            int j1 = ijk[1] + j;
+            if (j1 <0 || j1>=ny) continue;
+            directions[1] = j;
+            ijk1[1] = j1;
+            int kbeg = (dim == 2) ? 0 : -1;
+            int kend = (dim == 2) ? 1 :  2;
+            for (int k=kbeg; k<kend; k++)
+            {
+               int k1 = ijk[2] + k;
+               if (k1 <0 || k1>=nz) continue;
+               directions[2] = (dim == 3) ? k : -1 ;
+               if (i==0 && j==0 && k==0) continue;
+
+               int l = GetSweepToTransfer(sweep,directions);
+               // cout << "in the direction " ; directions.Print(); 
+               // cout << "sweep of transfer = " << l << endl;
+               if (l == -1) continue;
+               ijk1[2] = k1;
+               int ip1 = GetSubdomainId(nxyz,ijk1);
+
+               if (myid != SubdomainRank[ip1]) continue;
+               Array<int>directions1(3); directions1 = -1;
+               for (int i = 0; i<dim; i++) directions1[i] = -directions[i];
+               int dir = GetDirectionId(directions1);
+               int n = dmaps->fes[ip1]->GetTrueVSize();
+               Vector res(2*n);
+               PmlMat[ip1]->Mult(*OvlpSol[ip1][dir],res);
+
+               Array2D<int> direct(dim,2); direct = 0;
+               for (int d = 0; d<dim; d++)
+               {
+                  if (directions[d]==1)  direct[d][0] = 1;
+                  if (directions[d]==-1) direct[d][1] = 1;
+               }   
+               GetChiRes(res,ip1,direct);
+               *f_transf[ip1][l] -= res;
+            }
+         }  
+      }
+      // cin.get();
+   }
+
+   for (int ip = 0; ip<nrsubdomains; ip++)
+   {
+      if (myid == SubdomainRank[ip])
+      {
+         for (int i = 0; i<nrneighbors; i++)
+         {
+            if (OvlpSol[ip][i])
+            {
+               delete OvlpSol[ip][i];
+            }
+         }
+         OvlpSol[ip].clear();
+      }
+   }
+}
+
+int ParDST::GetSweepToTransfer(const int s, Array<int> directions) const
+{
+   int l1=-1;
+   int nsweeps = sweeps->nsweeps;
+   Array<int> sweep0;   
+   sweeps->GetSweep(s,sweep0);
+   switch (dim)
+   {
+      case 2:
+      for (int l=s; l<nsweeps; l++)
+      {
+         // Rule 1: the transfer source direction has to be similar with 
+         // the sweep direction
+         Array<int> sweep1;   
+         sweeps->GetSweep(l,sweep1);
+         int ddot = 0;
+         for (int d=0; d<dim; d++) ddot+= sweep1[d] * directions[d];
+         if (ddot <= 0) continue;
+
+         // Rule 2: The horizontal or vertical transfer source cannot be used
+         // Case of horizontal or vertical transfer source 
+         // (it can't be both 0 cause it's skipped)
+         if (directions[0]==0 || directions[1] == 0) 
+         {
+            if (sweep0[0] == -sweep1[0] && sweep0[1] == -sweep1[1]) continue;
+         }
+         l1 = l;
+         break;
+      }
+      break;
+      default:
+      for (int l=s; l<nsweeps; l++)
+      {
+         // Rule 1: (similar directions) the transfer source direction has to be similar with 
+         // the sweep direction
+         Array<int> sweep1;   
+         sweeps->GetSweep(l,sweep1);
+         int ddot = 0;
+         bool similar = true;
+         for (int d=0; d<dim; d++) 
+         {
+            if (sweep1[d] * directions[d] < 0) similar = false;
+            ddot+= sweep1[d] * directions[d];
+         }
+         if (!similar || ddot<=0) continue; // not similar
+
+         // Rule 2: (oposite directions) the transfer source direction has to be similar with 
+         // the sweep direction
+         // 
+         // check any of the projections onto the planes
+         // (xy, xz, yz)
+
+         if ( (directions[0]==0 && directions[1] != 0) || 
+              (directions[0]!=0 && directions[1] == 0) ||
+              (directions[0]==0 && directions[2] != 0) || 
+              (directions[0]!=0 && directions[2] == 0) ||
+              (directions[2]==0 && directions[1] != 0) || 
+              (directions[2]!=0 && directions[1] == 0) ) 
+         {
+            if (sweep0[0] == -sweep1[0] && 
+                sweep0[1] == -sweep1[1] && 
+                sweep0[2] == -sweep1[2]) continue;
+         }
+         l1 = l;
+         break;
+      }
+      break;
+   }
+
+   return l1;   
+}
+
+void ParDST::CorrectOrientation(int ip,Vector &x) const
+{
+   FiniteElementSpace * fespace = dmaps->fes[ip];
+   Mesh * mesh = fespace->GetMesh();
+   int nrelems = mesh->GetNE();
+   // GridFunction test; 
+   // test.SetFromTrueDofs(x)
+   Array<int> signs(fespace->GetTrueVSize()); signs = 0;
+   for (int iel=0; iel<nrelems; iel++)
+   {
+      Array<int> ElemDofs;
+      fespace->GetElementDofs(iel,ElemDofs);
+      int ndofs = ElemDofs.Size();
+      ElemDofs.Print();
+      for (int i = 0; i< ndofs; i++)
+      {
+         int pdof_ = ElemDofs[i];
+         if (pdof_ < 0) 
+         {
+            signs[abs(pdof_)-1] += 1.0 ; 
+         }
+         else
+         {
+            signs[pdof_] -= 1.0 ; 
+         }
+      }
+   }
+
+   cout << "signs = " ; signs.Print();
+   for (int i = 0; i<fespace->GetTrueVSize(); i++)
+   {
+      if (signs[i]<0) 
+      {
+         x(i) *= -1.0;
+         x(i+fespace->GetTrueVSize()) *= -1.0;
+      }
+   }
+}
+
+
+ParDST::~ParDST()
+{
+
+   for (int ip=0; ip<nrsubdomains; ip++)
+   {
+      delete Optr[ip];
+      delete subdomain_sol[ip];
+      delete PmlMatInv[ip];
+      delete sqf[ip];
+      if (myid != SubdomainRank[ip]) continue;
+      for (int i=0;i<sweeps->nsweeps; i++)
+      {
+         delete f_transf[ip][i];
+      }
+      delete f_orig[ip];
+   }
+   f_orig.DeleteAll();
+   delete dmaps;
+   delete sweeps;
+   delete part;
+
+}

examples/maxwell-solver/ParDST/ParDST.hpp

@@ -0,0 +1,82 @@
+#pragma once
+#include "../common/Utilities.hpp"
+#include "../common/PML.hpp"
+#include "../DST/DST.hpp"
+#include "DofMapsDST.hpp"
+using namespace std;
+using namespace mfem;
+
+
+class ParDST : public Solver//
+{
+public:
+   enum BCType
+   {
+       NEUMANN,
+       DIRICHLET
+   };
+   ParDST(ParSesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+       double omega_, Coefficient * Q_, int nrlayers_, 
+       int nx_=2, int ny_=2, int nz_=2, 
+       BCType bc_type_ = BCType::DIRICHLET, Coefficient * LossCoeff_ = nullptr);
+   ParDST(ParSesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+       double omega_, MatrixCoefficient * MQ_, int nrlayers_, int nx_=2, int ny_=2, int nz_=2, 
+       BCType bc_type_ = BCType::DIRICHLET, Coefficient * LossCoeff_ = nullptr);
+   virtual void SetOperator(const Operator &op) {}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~ParDST();
+private:
+   MPI_Comm comm = MPI_COMM_WORLD;
+   int num_procs, myid;
+   // Constructor inputs
+   int prob_kind;
+   ParSesquilinearForm *bf=nullptr;
+   ParFiniteElementSpace * pfes = nullptr;
+   ParMesh * pmesh = nullptr;
+   ParMeshPartition * part = nullptr;
+   Array<int> SubdomainRank;
+   Array<int> RankSubdomains;
+   const FiniteElementCollection * fec = nullptr;
+   Array2D<double> Pmllength;
+   int dim = 2;
+   double omega = 0.5;
+   Coefficient * Q=nullptr;
+   MatrixCoefficient * MQ=nullptr;
+   int nrlayers;
+   BCType bc_type = BCType::DIRICHLET;
+   Coefficient * LossCoeff=nullptr;
+   
+   int ovlpnrlayers;
+   int nrsubdomains = 0;
+   int nx,ny,nz;
+   Array<int> nxyz;
+   Sweep * sweeps = nullptr;
+   DofMaps * dmaps = nullptr;
+   Array< SesquilinearForm * > sqf;
+   Array< OperatorPtr * > Optr;
+   Array<ComplexSparseMatrix *> PmlMat;
+   Array<ComplexUMFPackSolver *> PmlMatInv;
+//    Array<ComplexMUMPSSolver *> PmlMatInv;
+   mutable Array<Vector *> f_orig;
+   mutable std::vector<Array<Vector * >> f_transf;
+   mutable Array<Vector * > subdomain_sol;
+   mutable std::vector<std::vector<Vector * >> OvlpSol;
+   void SetupSubdomainProblems();
+   std::vector<std::vector<Array<int>>> NovlpElems;
+   std::vector<std::vector<Array<int>>> NovlpDofs;
+   void MarkSubdomainOverlapDofs(const bool comp = false);
+   void SetHelmholtzPmlSystemMatrix(int ip);   
+   void SetMaxwellPmlSystemMatrix(int ip);
+   void GetChiRes(Vector & res, int ip, Array2D<int> direct) const;
+   void PlotLocal(Vector & sol, socketstream & sol_sock, int ip) const;
+   void PlotGlobal(Vector & sol, socketstream & sol_sock) const;
+   double GetSweepNumSteps(const int sweep) const;
+   void GetStepSubdomains(const int sweep, const int step, Array2D<int> & subdomains) const;
+   void TransferSources(int sweep, const Array<int> & subdomain_ids) const;
+   int GetSweepToTransfer(const int s, Array<int> directions) const;
+   void CorrectOrientation(int ip, Vector & x) const;
+   void Init();
+
+};
+
+

examples/maxwell-solver/ST/PST.cpp

@@ -0,0 +1,795 @@
+// Pure Source Transfer Preconditioner
+#include "PST.hpp"
+
+PSTP::PSTP(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+         double omega_, Coefficient * ws_,  int nrlayers_)
+   : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), 
+     bf(bf_), Pmllength(Pmllength_), omega(omega_), ws(ws_), nrlayers(nrlayers_)
+{
+   Mesh * mesh = bf->FESpace()->GetMesh();
+   dim = mesh->Dimension();
+
+   // ----------------- Step 1 --------------------
+   // Introduce 2 layered partitios of the domain 
+   // 
+   int partition_kind;
+   // 1. Non ovelapping 
+   partition_kind = 1; // Non Ovelapping partition 
+   pnovlp = new MeshPartition(mesh, partition_kind);
+
+   // 2. Overlapping to the right
+   partition_kind = 3; // Overlapping partition for the full space
+   povlp = new MeshPartition(mesh, partition_kind);
+
+   nrpatch = povlp->nrpatch;
+   MFEM_VERIFY(povlp->nrpatch+1 == pnovlp->nrpatch,"Check nrpatch");
+
+
+   lmap = new LocalDofMap(bf->FESpace()->FEColl(),pnovlp,povlp);
+
+   // Given the two partitions create a dof map between the non-ovelapping 
+   // subdomain dofs and the overlapping ones
+
+
+   //
+   // ----------------- Step 1a -------------------
+   // Save the partition for visualization
+   // SaveMeshPartition(povlp->patch_mesh, "output/mesh_ovlp.", "output/sol_ovlp.");
+   // SaveMeshPartition(pnovlp->patch_mesh, "output/mesh_novlp.", "output/sol_novlp.");
+
+   // ------------------Step 2 --------------------
+   // Construct the dof maps from subdomains to global (for the extended and not)
+   // The non ovelapping is extended on the left by pml (halfspace problem)
+   // The overlapping is extended left and right by pml (unbounded domain problem)
+   novlp_prob = new DofMap(bf,pnovlp);
+   ovlp_prob  = new DofMap(bf,povlp,nrlayers); 
+
+   // Given 
+
+   // ------------------Step 3 --------------------
+   // Assemble the PML Problem matrices and factor them
+   PmlMat.SetSize(nrpatch);
+   PmlMatInv.SetSize(nrpatch);
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      PmlMat[ip] = GetPmlSystemMatrix(ip);
+      PmlMatInv[ip] = new KLUSolver;
+      PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
+   }
+}
+
+SparseMatrix * PSTP::GetPmlSystemMatrix(int ip)
+{
+   double h = GetUniformMeshElementSize(ovlp_prob->PmlMeshes[ip]);
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+   if (ip == nrpatch-1 || ip == 0) 
+   {
+      length[0][0] = Pmllength[0][0];
+      length[0][1] = Pmllength[0][1];
+   }
+   length[1][0] = Pmllength[1][0];
+   length[1][1] = Pmllength[1][1];
+
+   CartesianPML pml(ovlp_prob->PmlMeshes[ip], length);
+   pml.SetOmega(omega);
+
+   Array <int> ess_tdof_list;
+   if (ovlp_prob->PmlMeshes[ip]->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(ovlp_prob->PmlMeshes[ip]->bdr_attributes.Max());
+      ess_bdr = 1;
+      ovlp_prob->PmlFespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+
+   ProductCoefficient c2_re(c2_re0, *ws);
+   ProductCoefficient c2_im(c2_im0, *ws);
+
+   SesquilinearForm a(ovlp_prob->PmlFespaces[ip],ComplexOperator::HERMITIAN);
+
+   a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   a.AddDomainIntegrator(new MassIntegrator(c2_re),
+                         new MassIntegrator(c2_im));
+   a.Assemble();
+
+   OperatorPtr Alocal;
+   a.FormSystemMatrix(ess_tdof_list,Alocal);
+   ComplexSparseMatrix * AZ_ext = Alocal.As<ComplexSparseMatrix>();
+   SparseMatrix * Mat = AZ_ext->GetSystemMatrix();
+   Mat->Threshold(0.0);
+   return Mat;
+}
+
+void PSTP::Mult(const Vector &r, Vector &z) const
+{
+   z = 0.0; 
+   res.SetSize(nrpatch);
+   Vector rnew(r);
+   Vector znew(z);
+   Vector z1(z);
+   Vector z2(z);
+   Vector raux(znew.Size());
+   Vector res_local, sol_local;
+   znew = 0.0;
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   Array<Vector> fsol(nrpatch+1);
+   Array<Vector> bsol(nrpatch+1);
+
+   // source transfer algorithm
+   for (int ip = 0; ip < nrpatch; ip++)
+   {
+      // cout << "ip = " << ip << endl;
+      Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+      Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip];
+      int ndofs = Dof2GlobalDof->Size();
+      res_local.SetSize(ndofs);
+      sol_local.SetSize(ndofs);
+
+      rnew.GetSubVector(*Dof2GlobalDof, res_local);
+
+      int nrdof_ext = PmlMat[ip]->Height();
+         
+      Vector res_ext(nrdof_ext); res_ext = 0.0;
+      Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+
+      res_ext.SetSubVector(*Dof2PmlDof,res_local.GetData());
+      PmlMatInv[ip]->Mult(res_ext, sol_ext);
+
+      sol_ext.GetSubVector(*Dof2PmlDof,sol_local);
+
+      znew = 0.0;
+      znew.SetSubVector(*Dof2GlobalDof,sol_local);
+
+      // cout << "ip+1 = " << ip+1 << endl;
+      Array<int> * nDof2GlobalDof = &novlp_prob->Dof2GlobalDof[ip+1];
+      fsol[ip+1].SetSize(nDof2GlobalDof->Size());
+      znew.GetSubVector(*nDof2GlobalDof,fsol[ip+1]);
+
+      socketstream subsol_sock(vishost, visport);
+      // PlotSolution(znew, subsol_sock,ip); cin.get();
+
+      // z.AddElementVector(*Dof2GlobalDof,sol_local);
+      int direction = 1;
+      if (ip <nrpatch-1) GetCutOffSolution(znew, ip, direction);
+      
+
+      if (ip != 0) z1+=znew;
+      // PlotSolution(z, subsol_sock,1); cin.get();
+
+      A->Mult(znew, raux);
+      rnew -= raux;
+      // PlotSolution(rnew, subsol_sock,ip); cin.get();
+
+   }
+
+   // socketstream subsol1_sock(vishost, visport);
+   // PlotSolution(z1, subsol1_sock,0); 
+
+   rnew = r;
+   for (int ip = nrpatch-1; ip >=0; ip--)
+   {
+      Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+      Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip];
+      int ndofs = Dof2GlobalDof->Size();
+      res_local.SetSize(ndofs);
+      sol_local.SetSize(ndofs);
+
+      rnew.GetSubVector(*Dof2GlobalDof, res_local);
+
+      //-----------------------------------------------
+      // Extend by zero to the extended mesh
+      int nrdof_ext = PmlMat[ip]->Height();
+         
+      Vector res_ext(nrdof_ext); res_ext = 0.0;
+      Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+
+      res_ext.SetSubVector(*Dof2PmlDof,res_local.GetData());
+      PmlMatInv[ip]->Mult(res_ext, sol_ext);
+
+      sol_ext.GetSubVector(*Dof2PmlDof,sol_local);
+
+      znew = 0.0;
+      znew.SetSubVector(*Dof2GlobalDof,sol_local);
+
+
+      Array<int> * nDof2GlobalDof = &novlp_prob->Dof2GlobalDof[ip];
+      bsol[ip].SetSize(nDof2GlobalDof->Size());
+      znew.GetSubVector(*nDof2GlobalDof,bsol[ip]);
+      // cout << "ip = " << ip << endl;
+
+      // PlotSolution(znew, subsol_sock,ip); cin.get();
+
+      // z.AddElementVector(*Dof2GlobalDof,sol_local);
+
+
+      int direction = -1;
+      if (ip>0) GetCutOffSolution(znew, ip-1, direction);
+
+      if (ip != nrpatch-1) z2+=znew;
+      // PlotSolution(z, subsol_sock,1); cin.get();
+
+      A->Mult(znew, raux);
+      rnew -= raux;
+      // PlotSolution(rnew, subsol_sock,ip); cin.get();
+
+   }
+   // socketstream subsol2_sock(vishost, visport);
+   // PlotSolution(z2, subsol2_sock,0); cin.get();
+
+
+   // construct solution z by z1 and z2
+   // vizualize solutions 
+   // Forward solutions
+   // socketstream subsol3_sock(vishost, visport);
+   for (int ip = 0; ip<nrpatch; ip++)
+   {
+      // cout << "ip = " << ip << endl;
+      znew = 0.0;
+      Array<int> * nDof2GlobalDof = &novlp_prob->Dof2GlobalDof[ip+1];
+      znew.SetSubVector(*nDof2GlobalDof,fsol[ip+1]);
+      // PlotSolution(znew, subsol3_sock,0); cin.get();
+   }
+
+      // Backward solutions
+   // socketstream subsol4_sock(vishost, visport);
+   for (int ip = 0; ip<nrpatch; ip++)
+   {
+      znew = 0.0;
+      Array<int> * nDof2GlobalDof = &novlp_prob->Dof2GlobalDof[ip];
+      znew.SetSubVector(*nDof2GlobalDof,bsol[ip]);
+      // PlotSolution(znew, subsol4_sock,0); cin.get();
+   }
+
+
+   Array<Vector> gsol(nrpatch+1);
+   // socketstream subsol5_sock(vishost, visport);
+
+   for (int ip = 0; ip<=nrpatch; ip++)
+   {
+      if (ip == 0) 
+      {
+         gsol[ip].SetSize(bsol[ip].Size());
+         gsol[ip] = bsol[ip];
+      }
+      else if (ip == nrpatch)
+      {
+         gsol[ip].SetSize(fsol[ip].Size());
+         gsol[ip] = fsol[ip];
+      }
+      else
+      {
+         gsol[ip].SetSize(fsol[ip].Size());
+         gsol[ip] = 0.0;
+         gsol[ip] += bsol[ip];
+         gsol[ip] += fsol[ip];
+      }
+      
+      znew = 0.0;
+      Array<int> * nDof2GlobalDof = &novlp_prob->Dof2GlobalDof[ip];
+      znew.SetSubVector(*nDof2GlobalDof,gsol[ip]);
+      // PlotSolution(znew, subsol5_sock,0); cin.get();
+      z.SetSubVector(*nDof2GlobalDof,gsol[ip]);
+   }
+
+
+   // required for visualization
+   // char vishost[] = "localhost";
+   // int  visport   = 19916;
+   // socketstream subsol_sock(vishost, visport);
+   // socketstream subsol1_sock(vishost, visport);
+   // socketstream subsol2_sock(vishost, visport);
+   // socketstream subsol3_sock(vishost, visport);
+
+   // // Initialize correction
+   // z = 0.0; 
+   // Vector fpml;
+   // Vector zpml;
+   // Vector z1(z);
+   // Vector res(z);
+
+   // // Construct the sources in each non-overlapping subdomain by restricting 
+   // // the global source
+   // Array<Vector> fn(nrpatch+1);
+   // Array<Vector> ftransf(nrpatch+1);
+   // for (int ip=0; ip<=nrpatch; ip++)
+   // {
+   //    Array<int> *Dof2GDof = &novlp_prob->Dof2GlobalDof[ip];
+   //    fn[ip].SetSize(Dof2GDof->Size());
+   //    ftransf[ip].SetSize(Dof2GDof->Size());
+   //    r.GetSubVector(*Dof2GDof,fn[ip]);
+   // }
+
+   // // source transfer algorithm 1 (forward sweep)
+   // Vector f;
+   // for (int ip = 0; ip < nrpatch; ip++)
+   // {
+   //    // construct the source in the overlapping PML problem
+   //    if (ip == 0) ftransf[ip] = fn[ip];
+
+   //    int ndof = ovlp_prob->Dof2GlobalDof[ip].Size();
+   //    f.SetSize(ndof); f = 0.0;
+   //    f.SetSubVector(lmap->map1[ip],ftransf[ip]);
+   //    f.SetSubVector(lmap->map2[ip],fn[ip+1]);
+
+   //    // Extend to the pml problem and solve for the local pml solution
+   //    Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip];
+   //    int ndof_pml = PmlMat[ip]->Height();
+   //    fpml.SetSize(ndof_pml); fpml=0.0;
+   //    zpml.SetSize(ndof_pml); zpml=0.0;
+   //    fpml.SetSubVector(*Dof2PmlDof,f);
+   //    // Solve the pml problem
+   //    PmlMatInv[ip]->Mult(fpml, zpml);
+   //    // PlotLocalSolution(zpml,subsol_sock,ip); cin.get();
+
+   //    //--------------------------------------------------
+   //    // Save the solution to the global solution
+   //     // restrict to non-pml problem
+   //    Vector sol(ndof); 
+   //    zpml.GetSubVector(*Dof2PmlDof, sol);
+   //    // restrict to the non-ovlp subdomain
+   //    // z1.AddElementVector(ovlp_prob->Dof2GlobalDof[ip],sol);
+
+   //    int m = lmap->map2[ip].Size();
+   //    Vector soll(m);
+   //    sol.GetSubVector(lmap->map2[ip],soll);
+   //    // prolong to the global solution
+   //    z.SetSubVector(novlp_prob->Dof2GlobalDof[ip+1],soll);
+
+   //    // PlotSolution(z,subsol1_sock,0); 
+   //    // PlotSolution(z1,subsol2_sock,0); 
+
+   //    //--------------------------------------------------
+
+   //    if (ip == nrpatch-1) continue;
+
+   //    int direction = 1;
+   //    GetCutOffSol(zpml, ip, direction);
+   //    // PlotLocalSolution(zpml,subsol3_sock,ip); cin.get();
+
+   //    // Calculate source to be trasfered to the pml mesh
+   //    Vector respml(zpml.Size());
+   //    PmlMat[ip]->Mult(zpml,respml);
+
+   //    // PlotLocalSolution(respml,subsol_sock,ip); cin.get();
+   //    // restrict to non-pml problem
+   //    Vector res(ndof); 
+   //    respml.GetSubVector(*Dof2PmlDof, res);
+   //    // source to be transfered
+   //    res.GetSubVector(lmap->map2[ip],ftransf[ip+1]);
+
+   //    // restrict to nonpml problem
+   //    // restrict to non-pml problem
+   //    // Vector sol1(ndof); 
+   //    // zpml.GetSubVector(*Dof2PmlDof, sol1);
+   //    // // prolong to global sol
+   //    // z1 = 0.0;
+   //    // Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+   //    // z1.SetSubVector(*Dof2GlobalDof, sol1);
+   //    // // calculate new source
+   //    // A->Mult(z1,res);
+
+   //    // //restrict to subdomain ip+1
+   //    // Array<int> * nDof2GlobalDof = &novlp_prob->Dof2GlobalDof[ip+1];
+   //    // res.GetSubVector(*nDof2GlobalDof,ftransf[ip+1]);
+   // }
+
+   // // source transfer algorithm 2 (backward sweep)
+   // for (int ip = nrpatch-1; ip >= 0; ip--)
+   // {
+   //    // construct the source in the overlapping PML problem
+   //    if (ip == nrpatch-1) ftransf[ip+1] = fn[ip+1];
+
+   //    int ndof = ovlp_prob->Dof2GlobalDof[ip].Size();
+   //    f.SetSize(ndof); f = 0.0;
+   //    f.SetSubVector(lmap->map1[ip],fn[ip]);
+   //    f.SetSubVector(lmap->map2[ip],ftransf[ip+1]);
+
+   //    // Extend to the pml problem and solve for the local pml solution
+   //    Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip];
+   //    int ndof_pml = PmlMat[ip]->Height();
+   //    fpml.SetSize(ndof_pml); fpml=0.0;
+   //    zpml.SetSize(ndof_pml); zpml=0.0;
+   //    fpml.SetSubVector(*Dof2PmlDof,f);
+   //    // Solve the pml problem
+   //    PmlMatInv[ip]->Mult(fpml, zpml);
+   //    PlotLocalSolution(zpml,subsol_sock,ip); cin.get();
+
+   //    //--------------------------------------------------
+   //    // Save the solution to the global solution
+   //    // restrict to non-pml problem
+   //    Vector sol(ndof); 
+   //    zpml.GetSubVector(*Dof2PmlDof, sol);
+   //    // restrict to the non-ovlp subdomain
+   //    int m = lmap->map1[ip].Size();
+   //    Vector soll(m);
+   //    sol.GetSubVector(lmap->map1[ip],soll);
+   //    // prolong to the global solution
+   //    z.AddElementVector(novlp_prob->Dof2GlobalDof[ip],soll);
+
+   //    // PlotSolution(z,subsol_sock,0); cin.get();
+
+   //    //--------------------------------------------------
+
+   //    if (ip == 0) continue;
+
+   //    int direction = -1;
+   //    GetCutOffSol(zpml, ip-1, direction);
+   //    PlotLocalSolution(zpml,subsol_sock,ip); cin.get();
+
+   //    // Calculate source to be trasfered to the pml mesh
+   //    Vector respml(zpml.Size());
+   //    PmlMat[ip]->Mult(zpml,respml);
+
+   //    // PlotLocalSolution(respml,subsol_sock,ip); cin.get();
+   //    // restrict to non-pml problem
+   //    Vector res(ndof); 
+   //    respml.GetSubVector(*Dof2PmlDof, res);
+   //    // source to be transfered
+   //    res.GetSubVector(lmap->map2[ip],ftransf[ip]);
+   // }
+
+
+   // PlotSolution(z,subsol_sock,0); cin.get();
+
+
+   // res.SetSize(nrpatch);
+   // Vector rnew(r);
+   // Vector rnew2(r);
+   // Vector znew(z);
+   // Vector znew1(z);
+   // Vector znew2(z);
+   // Vector raux(znew.Size());
+   // Vector res_local, sol_local;
+   // znew = 0.0;
+   // znew1= 0.0;
+   // znew2= 0.0;
+
+   // char vishost[] = "localhost";
+   // int  visport   = 19916;
+   // socketstream subsol_sock(vishost, visport);
+
+   // std::vector<Vector*> zloc;
+   // zloc.resize(nrpatch+1);
+
+   // // allocate memory and initialize
+   // for (int ip = 0; ip <= nrpatch; ip++)
+   // {
+   //    int n = novlp_prob->Dof2GlobalDof[ip].Size();
+   //    zloc[ip] = new Vector(n); *zloc[ip]=0.0;
+   // }
+
+
+   // // source transfer algorithm 1 (forward sweep)
+   // for (int ip = 0; ip < nrpatch; ip++)
+   // {
+   //    Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+   //    Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip];
+   //    int ndofs = Dof2GlobalDof->Size();
+   //    res_local.SetSize(ndofs);
+   //    sol_local.SetSize(ndofs);
+
+   //    rnew.GetSubVector(*Dof2GlobalDof, res_local);
+
+   //    //-----------------------------------------------
+   //    // Extend by zero to the extended mesh
+   //    int nrdof_ext = PmlMat[ip]->Height();
+   //    Vector res_ext(nrdof_ext); res_ext = 0.0;
+   //    Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+   //    res_ext.SetSubVector(*Dof2PmlDof,res_local.GetData());
+   //    PmlMatInv[ip]->Mult(res_ext, sol_ext);
+   //    sol_ext.GetSubVector(*Dof2PmlDof,sol_local);
+   //    znew = 0.0;
+   //    znew.SetSubVector(*Dof2GlobalDof,sol_local);
+
+   //    Array<int> * Dof2GDof = &novlp_prob->Dof2GlobalDof[ip+1];
+   //    int n = Dof2GDof->Size();
+   //    Vector nsol(n);
+   //    znew.GetSubVector(*Dof2GDof, nsol);
+   //    *zloc[ip+1] += nsol;
+      
+   //    int direction = 1;
+      // if (ip < nrpatch-1) GetCutOffSolution(znew, ip, direction);
+
+   //    A->Mult(znew, raux);
+   //    rnew -= raux;
+   //    znew1 += znew;
+
+
+   // }
+
+   // PlotSolution(znew1, subsol_sock,0); cin.get();
+
+   // // source transfer algorithm 2 (backward sweep)
+   // for (int ip = nrpatch-1; ip >=0; ip--)
+   // {
+   //    Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+   //    Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip];
+   //    int ndofs = Dof2GlobalDof->Size();
+   //    res_local.SetSize(ndofs);
+   //    sol_local.SetSize(ndofs);
+   //    rnew2.GetSubVector(*Dof2GlobalDof, res_local);
+
+   //    //-----------------------------------------------
+   //    // Extend by zero to the extended mesh
+   //    int nrdof_ext = PmlMat[ip]->Height();
+   //    Vector res_ext(nrdof_ext); res_ext = 0.0;
+   //    Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+   //    res_ext.SetSubVector(*Dof2PmlDof,res_local.GetData());
+   //    PmlMatInv[ip]->Mult(res_ext, sol_ext);
+   //    sol_ext.GetSubVector(*Dof2PmlDof,sol_local);
+   //    znew = 0.0;
+   //    znew.SetSubVector(*Dof2GlobalDof,sol_local);
+
+
+
+   //    Array<int> * Dof2GDof = &novlp_prob->Dof2GlobalDof[ip];
+   //    int n = Dof2GDof->Size();
+   //    Vector nsol(n);
+   //    znew.GetSubVector(*Dof2GDof, nsol);
+   //    *zloc[ip] += nsol;
+      
+   //    int direction = -1;
+   //    if (ip > 0) GetCutOffSolution(znew, ip-1, direction);
+
+   //    A->Mult(znew, raux);
+   //    rnew2 -= raux;
+   //    znew2 += znew;
+   // }
+
+   // // PlotSolution(znew2, subsol_sock,0); cin.get();
+
+   // // propagate to global dofs 
+   // z = 0.0;
+   // for (int ip = 0; ip <= nrpatch; ip++)
+   // {
+   //    Array<int> Dof2GDof = novlp_prob->Dof2GlobalDof[ip];
+   //    z.AddElementVector(Dof2GDof,*zloc[ip]);
+   // }
+
+   // PlotSolution(z, subsol_sock,0); cin.get();
+
+
+
+}
+
+void PSTP::PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const
+{
+   FiniteElementSpace * fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   GridFunction gf(fespace);
+   double * data = sol.GetData();
+   gf.SetData(data);
+   
+   string keys = "keys z\n";
+   if (ip ==0) keys = "keys rRljc\n";
+   sol_sock << "solution\n" << *mesh << gf << flush;
+   
+}
+
+
+void PSTP::PlotLocalSolution(Vector & sol, socketstream & sol_sock, int ip) const
+{
+   FiniteElementSpace * fespace = ovlp_prob->PmlFespaces[ip];
+   Mesh * mesh = fespace->GetMesh();
+   GridFunction gf(fespace);
+   double * data = sol.GetData();
+   gf.SetData(data);
+   
+   string keys = "keys z\n";
+   if (ip ==0) keys = "keys rRljc\n";
+   sol_sock << "solution\n" << *mesh << gf << flush;
+   
+}
+
+void PSTP::GetCutOffSolution(Vector & sol, int ip, int direction) const
+{
+
+   int l,k;
+
+   l=(direction == 1)? ip+1: ip;
+   k=(direction == 1)? ip: ip+1;
+
+   Mesh * mesh1 = ovlp_prob->fespaces[l]->GetMesh();
+   Mesh * mesh2 = ovlp_prob->fespaces[k]->GetMesh();
+   
+   Vector pmin1, pmax1;
+   Vector pmin2, pmax2;
+   mesh1->GetBoundingBox(pmin1, pmax1);
+   mesh2->GetBoundingBox(pmin2, pmax2);
+
+   Array2D<double> h(dim,2);
+   
+   h[0][0] = pmin2[0] - pmin1[0];
+   h[0][1] = pmax2[0] - pmin1[0];
+   h[1][0] = pmin2[1] - pmin1[1];
+   h[1][1] = pmax2[1] - pmax1[1];
+
+   if (direction == 1)
+   {
+      h[0][0] = 0.0;
+   }
+   else if (direction == -1)
+   {
+      h[0][1] = 0.0;
+   }
+   CutOffFnCoefficient cf(CutOffFncn, pmin2, pmax2, h);
+   double * data = sol.GetData();
+
+   FiniteElementSpace * fespace = bf->FESpace();
+   int n = fespace->GetTrueVSize();
+
+   GridFunction solgf_re(fespace, data);
+   GridFunction solgf_im(fespace, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fespace);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   sol = gf;
+}
+
+PSTP::~PSTP()
+{
+   for (int ip = 0; ip<nrpatch; ++ip)
+   {
+      delete PmlMatInv[ip];
+      delete PmlMat[ip];
+   }
+   PmlMat.DeleteAll();
+   PmlMatInv.DeleteAll();
+}
+
+void PSTP::GetCutOffSol(Vector & sol, int ip, int direction) const
+{
+   int l,k;
+   l=(direction == 1)? ip+1: ip;
+   k=(direction == 1)? ip: ip+1;
+
+   Mesh * mesh1 = ovlp_prob->fespaces[l]->GetMesh();
+   Mesh * mesh2 = ovlp_prob->fespaces[k]->GetMesh();
+   
+   Vector pmin1, pmax1;
+   Vector pmin2, pmax2;
+   mesh1->GetBoundingBox(pmin1, pmax1);
+   mesh2->GetBoundingBox(pmin2, pmax2);
+
+   Array2D<double> h(dim,2);
+   
+   h[0][0] = pmin2[0] - pmin1[0];
+   h[0][1] = pmax2[0] - pmin1[0];
+   h[1][0] = pmin2[1] - pmin1[1];
+   h[1][1] = pmax2[1] - pmax1[1];
+
+   if (direction == 1)
+   {
+      h[0][0] = 0.0;
+   }
+   else if (direction == -1)
+   {
+      h[0][1] = 0.0;
+   }
+   CutOffFnCoefficient cf(CutOffFncn, pmin2, pmax2, h);
+   double * data = sol.GetData();
+
+   int m = (direction == 1) ? ip : ip+1;
+   FiniteElementSpace * fespace = ovlp_prob->PmlFespaces[m];
+   int n = fespace->GetTrueVSize();
+
+   GridFunction solgf_re(fespace, data);
+   GridFunction solgf_im(fespace, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fespace);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   sol = gf;
+}
+
+
+
+LocalDofMap::LocalDofMap(const FiniteElementCollection * fec_, MeshPartition * part1_, 
+               MeshPartition * part2_):fec(fec_), part1(part1_), part2(part2_)
+{
+   // Each overlapping patch has 2 non-overlapping subdomains
+   // Thre are n non-overlapping and and n-1 overlapping subdomains
+   int nrpatch = part2->nrpatch;
+   MFEM_VERIFY(part1->nrpatch-1 == part2->nrpatch, "Check number of subdomains");
+
+   cout << "Constructing local dof maps" << endl; 
+   map1.resize(nrpatch);
+   map2.resize(nrpatch);
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      // Get the 3 meshes involved
+      Mesh * mesh = part2->patch_mesh[ip];
+      Mesh * mesh1 = part1->patch_mesh[ip];
+      Mesh * mesh2 = part1->patch_mesh[ip+1];
+
+      // Define the fespaces
+      FiniteElementSpace fespace(mesh, fec);
+      FiniteElementSpace fespace1(mesh1, fec);
+      FiniteElementSpace fespace2(mesh2, fec);
+
+      int ndof1 = fespace1.GetTrueVSize();
+      int ndof2 = fespace2.GetTrueVSize();
+
+      map1[ip].SetSize(2*ndof1); // times 2 because it's complex
+      map2[ip].SetSize(2*ndof2); // times 2 because it's complex
+
+      // loop through the elements in the patches
+      // map 1 is constructed by the first half of elements
+      // map 2 is constructed by the second half of elements
+
+      for (int iel = 0; iel<part1->element_map[ip].Size(); ++iel)
+      {
+         // index in the overlapping mesh
+         int iel_idx = iel;
+         Array<int> ElemDofs;
+         Array<int> GlobalElemDofs;
+         fespace1.GetElementDofs(iel,ElemDofs);
+         fespace.GetElementDofs(iel_idx,GlobalElemDofs);
+         // the sizes have to match
+         MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+                     "Size inconsistency");
+         // loop through the dofs and take into account the signs;
+         int ndof = ElemDofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = ElemDofs[i];
+            int gdof_ = GlobalElemDofs[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            map1[ip][pdof] = gdof;
+            map1[ip][pdof+ndof1] = gdof+fespace.GetTrueVSize();
+         }
+      }
+      for (int iel = 0; iel<part1->element_map[ip+1].Size(); ++iel)
+      {
+         // index in the overlapping mesh
+         int k = part1->element_map[ip].Size();
+         int iel_idx = iel+k;
+         Array<int> ElemDofs;
+         Array<int> GlobalElemDofs;
+         fespace2.GetElementDofs(iel,ElemDofs);
+         fespace.GetElementDofs(iel_idx,GlobalElemDofs);
+         // the sizes have to match
+         MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+                     "Size inconsistency");
+         // loop through the dofs and take into account the signs;
+         int ndof = ElemDofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = ElemDofs[i];
+            int gdof_ = GlobalElemDofs[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            map2[ip][pdof] = gdof;
+            map2[ip][pdof+ndof2] = gdof+fespace.GetTrueVSize();
+         }
+      }
+   }
+}

examples/maxwell-solver/ST/PST.hpp

@@ -0,0 +1,62 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "ST.hpp"
+using namespace std;
+using namespace mfem;
+
+
+class LocalDofMap // Constructs dof mapbetween two partitions
+{
+   const FiniteElementCollection *fec=nullptr;
+   MeshPartition * part1=nullptr;
+   MeshPartition * part2=nullptr;
+public:
+   int nrpatch, nx, ny, nz;
+   vector<Array<int>> map1;
+   vector<Array<int>> map2;
+   // constructor
+   LocalDofMap(const FiniteElementCollection * fec_, MeshPartition * part1_, 
+               MeshPartition * part2_);
+   ~LocalDofMap();
+};
+
+
+class PSTP : public Solver//
+{
+private:
+   int nrpatch;
+   int dim;
+   SesquilinearForm *bf=nullptr;
+   MeshPartition * povlp;
+   MeshPartition * pnovlp;
+   double omega = 0.5;
+   Coefficient * ws;
+   int nrlayers;
+   const Operator * A;
+   Vector B;
+   DofMap * ovlp_prob = nullptr;
+   DofMap * novlp_prob = nullptr;
+   LocalDofMap * lmap=nullptr;
+   Array<SparseMatrix *> PmlMat;
+   Array<KLUSolver *> PmlMatInv;
+   Array2D<double> Pmllength;
+   mutable Array<Vector * > res;
+
+   SparseMatrix * GetPmlSystemMatrix(int ip);
+   void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+   void PlotLocalSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+   void GetCutOffSolution(Vector & sol, int ip, int direction) const;
+   void GetCutOffSol(Vector & sol, int ip, int direction) const;
+
+public:
+   PSTP(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+       double omega_, Coefficient * ws_, int nrlayers_);
+   void SetLoadVector(Vector load) { B = load;}
+   virtual void SetOperator(const Operator &op) {A = &op;}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~PSTP();
+};
+
+

examples/maxwell-solver/ST/ST.cpp

@@ -0,0 +1,617 @@
+//Source Transfer Preconditioner
+
+#include "ST.hpp"
+
+DofMap::DofMap(SesquilinearForm * bf_ , MeshPartition * partition_) 
+               : bf(bf_), partition(partition_)
+{
+   int partition_kind = partition->partition_kind;
+   MFEM_VERIFY(partition_kind == 1, "Check Partition kind");
+   fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   const FiniteElementCollection * fec = fespace->FEColl();
+   nrpatch = partition->nrpatch;
+
+   fespaces.SetSize(nrpatch);
+
+   Dof2GlobalDof.resize(nrpatch);
+
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // create finite element spaces for each patch 
+      fespaces[ip] = new FiniteElementSpace(partition->patch_mesh[ip],fec);
+
+      // construct the patch tdof to global tdof map
+      int nrdof = fespaces[ip]->GetTrueVSize();
+      Dof2GlobalDof[ip].SetSize(2*nrdof);
+
+      // loop through the elements in the patch
+      for (int iel = 0; iel<partition->element_map[ip].Size(); ++iel)
+      {
+         // index in the global mesh
+         int iel_idx = partition->element_map[ip][iel];
+         // get the dofs of this element
+         Array<int> ElemDofs;
+         Array<int> GlobalElemDofs;
+         fespaces[ip]->GetElementDofs(iel,ElemDofs);
+         fespace->GetElementDofs(iel_idx,GlobalElemDofs);
+         // the sizes have to match
+         MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+                     "Size inconsistency");
+         // loop through the dofs and take into account the signs;
+         int ndof = ElemDofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = ElemDofs[i];
+            int gdof_ = GlobalElemDofs[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            Dof2GlobalDof[ip][pdof] = gdof;
+            Dof2GlobalDof[ip][pdof+nrdof] = gdof+fespace->GetTrueVSize();
+         }
+      }
+   }
+}
+
+
+DofMap::DofMap(SesquilinearForm * bf_ , MeshPartition * partition_, int nrlayers) 
+               : bf(bf_), partition(partition_)
+{
+   int partition_kind = partition->partition_kind;
+   fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   const FiniteElementCollection * fec = fespace->FEColl();
+   nrpatch = partition->nrpatch;
+
+   fespaces.SetSize(nrpatch);
+   PmlMeshes.SetSize(nrpatch);
+   // Extend patch meshes to include pml
+   for  (int ip = 0; ip<nrpatch; ip++)
+   {
+      Array<int> directions;
+      if (ip > 0)
+      {
+         for (int i=0; i<nrlayers; i++)
+         {
+            directions.Append(-1);
+         }
+      }
+      if (ip < nrpatch-1)
+      {
+         for (int i=0; i<nrlayers; i++)
+         {
+            if (partition_kind == 3) directions.Append(1);
+         }
+      }
+      PmlMeshes[ip] = ExtendMesh(partition->patch_mesh[ip],directions);
+   }
+
+   // Save PML_meshes
+   string meshpath;
+   string solpath;
+   if (partition_kind == 3) 
+   {
+      meshpath = "output/mesh_ovlp_pml.";
+      solpath = "output/sol_ovlp_pml.";
+   }
+   else if (partition_kind == 4)
+   {
+      meshpath = "output/mesh_novlp_pml.";
+      solpath = "output/sol_novlp_pml.";
+   }
+   else
+   {
+      MFEM_ABORT("This partition kind not supported yet");
+   }
+   
+   // SaveMeshPartition(PmlMeshes, meshpath, solpath);
+
+   PmlFespaces.SetSize(nrpatch);
+   Dof2GlobalDof.resize(nrpatch);
+   Dof2PmlDof.resize(nrpatch);
+
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // create finite element spaces for each patch 
+      fespaces[ip] = new FiniteElementSpace(partition->patch_mesh[ip],fec);
+      PmlFespaces[ip] = new FiniteElementSpace(PmlMeshes[ip],fec);
+
+      // construct the patch tdof to global tdof map
+      int nrdof = fespaces[ip]->GetTrueVSize();
+      Dof2GlobalDof[ip].SetSize(2*nrdof);
+      Dof2PmlDof[ip].SetSize(2*nrdof);
+
+      // build dof maps between patch and extended patch
+      // loop through the patch elements and constract the dof map
+      // The same elements in the extended mesh have the same ordering (but not the dofs)
+
+      // loop through the elements in the patch
+      for (int iel = 0; iel<partition->element_map[ip].Size(); ++iel)
+      {
+         // index in the global mesh
+         int iel_idx = partition->element_map[ip][iel];
+         // get the dofs of this element
+         Array<int> ElemDofs;
+         Array<int> PmlElemDofs;
+         Array<int> GlobalElemDofs;
+         fespaces[ip]->GetElementDofs(iel,ElemDofs);
+         PmlFespaces[ip]->GetElementDofs(iel,PmlElemDofs);
+         fespace->GetElementDofs(iel_idx,GlobalElemDofs);
+         // the sizes have to match
+         MFEM_VERIFY(ElemDofs.Size() == GlobalElemDofs.Size(),
+                     "Size inconsistency");
+         MFEM_VERIFY(ElemDofs.Size() == PmlElemDofs.Size(),
+                     "Size inconsistency");            
+         // loop through the dofs and take into account the signs;
+         int ndof = ElemDofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = ElemDofs[i];
+            int gdof_ = GlobalElemDofs[i];
+            int pmldof_ = PmlElemDofs[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            int pmldof = (pmldof_ >= 0) ? pmldof_ : abs(pmldof_) - 1;
+
+            Dof2GlobalDof[ip][pdof] = gdof;
+            Dof2GlobalDof[ip][pdof+nrdof] = gdof+fespace->GetTrueVSize();
+            Dof2PmlDof[ip][pdof] = pmldof;
+            Dof2PmlDof[ip][pdof+nrdof] = pmldof+PmlFespaces[ip]->GetTrueVSize();
+         }
+      }
+   }
+}
+
+
+STP::STP(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+         double omega_, Coefficient * ws_,  int nrlayers_)
+   : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), 
+     bf(bf_), Pmllength(Pmllength_), omega(omega_), ws(ws_), nrlayers(nrlayers_)
+{
+   Mesh * mesh = bf->FESpace()->GetMesh();
+   dim = mesh->Dimension();
+
+   // ----------------- Step 1 --------------------
+   // Introduce 2 layered partitios of the domain 
+   // 
+   int partition_kind;
+   // 1. Non ovelapping 
+   partition_kind = 4; // Ovelapping partition for the halfspace problem
+   pnovlp = new MeshPartition(mesh, partition_kind);
+
+   // 2. Overlapping to the right
+   partition_kind = 3; // Ovelapping partition for the full space
+   povlp = new MeshPartition(mesh, partition_kind);
+   nrpatch = pnovlp->nrpatch;
+   //
+   // ----------------- Step 1a -------------------
+   // Save the partition for visualization
+   // SaveMeshPartition(povlp->patch_mesh, "output/mesh_ovlp.", "output/sol_ovlp.");
+   // SaveMeshPartition(pnovlp->patch_mesh, "output/mesh_novlp.", "output/sol_novlp.");
+
+   // ------------------Step 2 --------------------
+   // Construct the dof maps from subdomains to global (for the extended and not)
+   // The non ovelapping is extended on the left by pml (halfspace problem)
+   // The overlapping is extended left and right by pml (unbounded domain problem)
+   novlp_prob = new DofMap(bf,pnovlp,nrlayers);
+   ovlp_prob  = new DofMap(bf,povlp,nrlayers); 
+
+   // ------------------Step 3 --------------------
+   // Assemble the PML Problem matrices and factor them
+   PmlMat.SetSize(nrpatch);
+   PmlMatInv.SetSize(nrpatch);
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      PmlMat[ip] = GetPmlSystemMatrix(ip);
+      PmlMatInv[ip] = new KLUSolver;
+      PmlMatInv[ip]->SetOperator(*PmlMat[ip]);
+   }
+
+   HalfSpaceMat.SetSize(nrpatch);
+   HalfSpaceMatInv.SetSize(nrpatch);
+   HalfSpaceForms.SetSize(nrpatch);
+   for (int ip=0; ip<nrpatch; ip++)
+   {
+      HalfSpaceMat[ip] = GetHalfSpaceSystemMatrix(ip);
+      HalfSpaceMatInv[ip] = new KLUSolver;
+      HalfSpaceMatInv[ip]->SetOperator(*HalfSpaceMat[ip]);
+   }
+}
+
+SparseMatrix * STP::GetPmlSystemMatrix(int ip)
+{
+   double h = GetUniformMeshElementSize(ovlp_prob->PmlMeshes[ip]);
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+   if (ip == nrpatch-1 || ip == 0) 
+   {
+      length[0][0] = Pmllength[0][0];
+      length[0][1] = Pmllength[0][1];
+   }
+   length[1][0] = Pmllength[1][0];
+   length[1][1] = Pmllength[1][1];
+
+   CartesianPML pml(ovlp_prob->PmlMeshes[ip], length);
+   pml.SetOmega(omega);
+
+   Array <int> ess_tdof_list;
+   if (ovlp_prob->PmlMeshes[ip]->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(ovlp_prob->PmlMeshes[ip]->bdr_attributes.Max());
+      ess_bdr = 1;
+      ovlp_prob->PmlFespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+
+   ProductCoefficient c2_re(c2_re0, *ws);
+   ProductCoefficient c2_im(c2_im0, *ws);
+
+   SesquilinearForm a(ovlp_prob->PmlFespaces[ip],ComplexOperator::HERMITIAN);
+
+   a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   a.AddDomainIntegrator(new MassIntegrator(c2_re),
+                         new MassIntegrator(c2_im));
+   a.Assemble();
+
+   OperatorPtr Alocal;
+   a.FormSystemMatrix(ess_tdof_list,Alocal);
+   ComplexSparseMatrix * AZ_ext = Alocal.As<ComplexSparseMatrix>();
+   SparseMatrix * Mat = AZ_ext->GetSystemMatrix();
+   Mat->Threshold(0.0);
+   return Mat;
+}
+
+SparseMatrix * STP::GetHalfSpaceSystemMatrix(int ip)
+{
+   double h = GetUniformMeshElementSize(novlp_prob->PmlMeshes[ip]);
+   Array2D<double> length(dim,2);
+   length = h*(nrlayers);
+   if (ip == nrpatch-1 || ip == 0) 
+   {
+      length[0][0] = Pmllength[0][0];
+   }
+   length[1][0] = Pmllength[1][0];
+   length[1][1] = Pmllength[1][1];
+   length[0][1] = 0.0;
+
+   CartesianPML pml(novlp_prob->PmlMeshes[ip], length);
+   pml.SetOmega(omega);
+
+   Array <int> ess_tdof_list;
+   if (novlp_prob->PmlMeshes[ip]->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(ovlp_prob->PmlMeshes[ip]->bdr_attributes.Max());
+      ess_bdr = 1;
+      novlp_prob->PmlFespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+
+   ProductCoefficient c2_re(c2_re0, *ws);
+   ProductCoefficient c2_im(c2_im0, *ws);
+
+   HalfSpaceForms[ip] = new SesquilinearForm(novlp_prob->PmlFespaces[ip],
+                        ComplexOperator::HERMITIAN);
+   HalfSpaceForms[ip]->AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   HalfSpaceForms[ip]->AddDomainIntegrator(new MassIntegrator(c2_re),
+                         new MassIntegrator(c2_im));
+   HalfSpaceForms[ip]->Assemble();
+
+   OperatorPtr Alocal;
+   HalfSpaceForms[ip]->FormSystemMatrix(ess_tdof_list, Alocal);
+
+   ComplexSparseMatrix * AZ_ext = Alocal.As<ComplexSparseMatrix>();
+   SparseMatrix * Mat = AZ_ext->GetSystemMatrix();
+   Mat->Threshold(0.0);
+   return Mat;
+}
+
+void STP::SolveHalfSpaceLinearSystem(int ip, Vector &x, Vector & load) const
+{
+   Array <int> ess_tdof_list;
+   if (novlp_prob->PmlMeshes[ip]->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(ovlp_prob->PmlMeshes[ip]->bdr_attributes.Max());
+      ess_bdr = 1;
+      novlp_prob->PmlFespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   OperatorHandle Ah;
+   Vector X,Modload;
+
+   HalfSpaceForms[ip]->FormLinearSystem(ess_tdof_list,x,load,
+   Ah,X,Modload);
+   HalfSpaceMatInv[ip]->Mult(Modload,X);
+   HalfSpaceForms[ip]->RecoverFEMSolution(X,Modload,x);
+}
+
+
+void STP::Mult(const Vector &r, Vector &z) const
+{
+   z = 0.0; 
+   res.SetSize(nrpatch);
+   Vector rnew(r);
+   Vector znew(z);
+   Vector z1(z);
+   Vector raux(znew.Size());
+   Vector res_local, sol_local;
+   znew = 0.0;
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   // socketstream subsol_sock1(vishost, visport);
+   // socketstream subsol_sock(vishost, visport);
+
+   // source transfer algorithm
+   for (int ip = 0; ip < nrpatch; ip++)
+   {
+      Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+      Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[ip];
+      int ndofs = Dof2GlobalDof->Size();
+      res_local.SetSize(ndofs);
+      sol_local.SetSize(ndofs);
+
+      rnew.GetSubVector(*Dof2GlobalDof, res_local);
+
+      // store residuals for the non overlapping partition
+      Array<int> * nDof2GlobalDof;
+      if (ip == nrpatch-1 )
+      {
+         nDof2GlobalDof = &ovlp_prob->Dof2GlobalDof[ip];
+      }
+      else
+      {
+         nDof2GlobalDof = &novlp_prob->Dof2GlobalDof[ip];
+      }
+      int mdofs = nDof2GlobalDof->Size();
+      res[ip] = new Vector(mdofs);
+      rnew.GetSubVector(*nDof2GlobalDof, *res[ip]);
+      if (ip == nrpatch-1) continue;
+
+      //-----------------------------------------------
+      // Extend by zero to the extended mesh
+      int nrdof_ext = PmlMat[ip]->Height();
+         
+      Vector res_ext(nrdof_ext); res_ext = 0.0;
+      Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+
+      res_ext.SetSubVector(*Dof2PmlDof,res_local.GetData());
+      PmlMatInv[ip]->Mult(res_ext, sol_ext);
+
+      sol_ext.GetSubVector(*Dof2PmlDof,sol_local);
+
+      znew = 0.0;
+      znew.SetSubVector(*Dof2GlobalDof,sol_local);
+
+      // PlotSolution(znew, subsol_sock,ip); cin.get();
+
+      // z.AddElementVector(*Dof2GlobalDof,sol_local);
+      int direction = 1;
+      GetCutOffSolution(znew, ip, direction);
+
+      z1+=znew;
+      // PlotSolution(z, subsol_sock,1); cin.get();
+
+      A->Mult(znew, raux);
+      rnew -= raux;
+      // PlotSolution(rnew, subsol_sock,ip); cin.get();
+
+   }
+
+   // solution stage
+   // First solve the nrpatch-1 problem (last subdomain)
+   // extend residual to all around pml
+   int nrdof_ext = PmlMat[nrpatch-1]->Height();
+   Vector res_ext(nrdof_ext); res_ext = 0.0;
+   Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+   Array<int> * Dof2GlobalDof = &ovlp_prob->Dof2GlobalDof[nrpatch-1];
+   Array<int> * Dof2PmlDof = &ovlp_prob->Dof2PmlDof[nrpatch-1];
+   res_ext.SetSubVector(*Dof2PmlDof,*res[nrpatch-1]);
+   PmlMatInv[nrpatch-1]->Mult(res_ext, sol_ext);
+   int ndofs = Dof2GlobalDof->Size();
+   sol_local.SetSize(ndofs);
+   sol_ext.GetSubVector(*Dof2PmlDof,sol_local);
+   znew = 0.0;
+   znew.SetSubVector(*Dof2GlobalDof,sol_local);
+   z.SetSubVector(*Dof2GlobalDof,sol_local);
+   z1+=znew;
+
+
+   // z = z1;
+
+   // PlotSolution(z1, subsol_sock1,0); cin.get();
+
+   // backward sweep for half space problems
+
+   Vector z_loc(z.Size());
+   for (int ip = nrpatch-2; ip >= 0; ip--)
+   {
+      // Get solution from previous layer
+      Array<int> * Dof2GlobalDof = &novlp_prob->Dof2GlobalDof[ip];
+      Array<int> * Dof2PmlDof = &novlp_prob->Dof2PmlDof[ip];
+      int ndof = Dof2GlobalDof->Size();
+      Vector sol_loc(ndof);      
+      znew.GetSubVector(* Dof2GlobalDof, sol_loc);
+
+      // extend by zero to the halfspace pml problem
+      FiniteElementSpace * subfespace = novlp_prob->PmlFespaces[ip];
+      int mdof = 2*subfespace->GetTrueVSize();
+      Vector sol_pml(mdof); sol_pml = 0.0;
+      sol_pml.SetSubVector(* Dof2PmlDof, sol_loc);
+      Mesh * submesh = subfespace->GetMesh();
+
+      // Set to zero the non boundary dofs
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr(submesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      subfespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      int n = ess_tdof_list.Size();
+      for (int i=0; i<n; i++)
+      {
+         ess_tdof_list.Append(ess_tdof_list[i]+mdof/2);
+      }
+      sol_pml.SetSubVectorComplement(ess_tdof_list,0.0);
+      
+      // Set up the halfspace problem
+      // extend the residual by zero to pml region
+      Vector pmlres(sol_pml.Size()); pmlres = 0.0;
+      pmlres.SetSubVector(* Dof2PmlDof,*res[ip]);
+      SolveHalfSpaceLinearSystem(ip, sol_pml, pmlres);
+
+      sol_loc = 0.0;
+      sol_pml.GetSubVector(* Dof2PmlDof, sol_loc);
+
+      z_loc = 0.0;
+      z_loc.SetSubVector(* Dof2GlobalDof, sol_loc);
+      znew = z_loc;
+      z.SetSubVector(* Dof2GlobalDof, sol_loc);
+   }
+      // PlotSolution(z, subsol_sock,1); cin.get();
+
+}
+
+void STP::PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const
+{
+   FiniteElementSpace * fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   GridFunction gf(fespace);
+   double * data = sol.GetData();
+   gf.SetData(data);
+   
+   string keys;
+   if (ip == 0) keys = "keys mrRljc\n";
+   sol_sock << "solution\n" << *mesh << gf << keys << flush;
+}
+
+void STP::GetCutOffSolution(Vector & sol, int ip, int direction) const
+{
+   int l,k;
+   l=(direction == 1)? ip+1: ip;
+   k=(direction == 1)? ip: ip+1;
+
+   Mesh * mesh1 = ovlp_prob->fespaces[l]->GetMesh();
+   Mesh * mesh2 = ovlp_prob->fespaces[k]->GetMesh();
+   
+   Vector pmin1, pmax1;
+   Vector pmin2, pmax2;
+   mesh1->GetBoundingBox(pmin1, pmax1);
+   mesh2->GetBoundingBox(pmin2, pmax2);
+
+   Array2D<double> h(dim,2);
+   
+   h[0][0] = pmin2[0] - pmin1[0];
+   h[0][1] = pmax2[0] - pmin1[0];
+   h[1][0] = pmin2[1] - pmin1[1];
+   h[1][1] = pmax2[1] - pmax1[1];
+
+   if (direction == 1)
+   {
+      h[0][0] = 0.0;
+   }
+   else if (direction == -1)
+   {
+      h[0][1] = 0.0;
+   }
+   CutOffFnCoefficient cf(CutOffFncn, pmin2, pmax2, h);
+   double * data = sol.GetData();
+
+   FiniteElementSpace * fespace = bf->FESpace();
+   int n = fespace->GetTrueVSize();
+
+   GridFunction solgf_re(fespace, data);
+   GridFunction solgf_im(fespace, &data[n]);
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(fespace);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   sol = gf;
+}
+
+STP::~STP()
+{
+   for (int ip = 0; ip<nrpatch; ++ip)
+   {
+      delete HalfSpaceForms[ip];
+      delete HalfSpaceMat[ip];
+      delete HalfSpaceMatInv[ip];
+      delete PmlMatInv[ip];
+      delete PmlMat[ip];
+   }
+   HalfSpaceForms.DeleteAll();
+   HalfSpaceMat.DeleteAll();
+   HalfSpaceMatInv.DeleteAll();
+   PmlMat.DeleteAll();
+   PmlMatInv.DeleteAll();
+}
+
+
+double CutOffFncn(const Vector &x, const Vector & pmin, const Vector & pmax, const Array2D<double> & h_)
+{
+   int dim = pmin.Size();
+   Vector h0(dim);
+   Vector h1(dim);
+   for (int i=0; i<dim; i++)
+   {
+      h0(i) = h_[i][0];
+      h1(i) = h_[i][1];
+   }
+   Vector x0(dim);
+   x0 = pmax; x0-=h1;
+   Vector x1(dim);
+   x1 = pmin; x1+=h0;
+
+   double f = 1.0;
+
+   for (int i = 0; i<dim; i++)
+   {
+      double val = 1.0;
+      if( x(i) > pmax(i) || x(i) < pmin(i))
+      {
+         val = 0.0;
+      }  
+      else if (x(i) <= pmax(i) && x(i) >= x0(i))
+      {
+         if(x0(i)-pmax(i) != 0.0)
+            val = (x(i)-pmax(i))/(x0(i)-pmax(i)); 
+      }
+      else if (x(i) >= pmin(i) && x(i) <= x1(i))
+      {
+         if (x1(i)-pmin(i) != 0.0)
+            val = (x(i)-pmin(i))/(x1(i)-pmin(i)); 
+      }
+      else
+      {
+         val = 1.0;
+      }
+      f *= val;
+   }
+   return f;
+}

examples/maxwell-solver/ST/ST.hpp

@@ -0,0 +1,96 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "complex_additive_schwarz.hpp"
+using namespace std;
+using namespace mfem;
+
+
+
+// Function coefficient that takes the boundingbox of the mesh as an input
+class CutOffFnCoefficient : public Coefficient
+{
+private:
+   double (*Function)(const Vector &, const Vector &, const Vector &, const Array2D<double> &);
+   Vector pmin, pmax;
+   Array2D<double> h; // specify the with of the cutoff function (h in each direction)
+   
+
+public:
+   CutOffFnCoefficient(double (*F)(const Vector &, const Vector &, const Vector &, const Array2D<double> &), 
+                             const Vector & pmin_, const Vector & pmax_, Array2D<double> & h_)
+      : Function(F), pmin(pmin_), pmax(pmax_), h(h_)
+   {}
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+   {
+      double x[3];
+      Vector transip(x, 3);
+      T.Transform(ip, transip);
+      return ((*Function)(transip, pmin, pmax, h));
+   }
+};
+
+double CutOffFncn(const Vector &x, const Vector &pmax, const Vector &pmin, const Array2D<double> & h_);
+
+
+class DofMap // Constructs dof maps for a given partition
+{
+   FiniteElementSpace *fespace=nullptr;
+   SesquilinearForm * bf=nullptr;
+   MeshPartition * partition=nullptr;
+public:
+   int nrpatch, nx, ny, nz;
+   vector<Array<int>> Dof2GlobalDof;
+   vector<Array<int>> Dof2PmlDof;
+   Array<Mesh *> PmlMeshes;
+   Array<FiniteElementSpace *> fespaces;
+   Array<FiniteElementSpace *> PmlFespaces;
+   // constructor
+   // Non PML contructor dof map
+   DofMap(SesquilinearForm * bf_, MeshPartition * partition_);
+   // PML
+   DofMap(SesquilinearForm * bf_ , MeshPartition * partition_, int nrlayers);
+   ~DofMap();
+};
+
+
+class STP : public Solver//
+{
+private:
+   int nrpatch;
+   int dim;
+   SesquilinearForm *bf=nullptr;
+   MeshPartition * povlp;
+   MeshPartition * pnovlp;
+   double omega = 0.5;
+   Coefficient * ws;
+   int nrlayers;
+   const Operator * A=nullptr;
+   Vector B;
+   DofMap * ovlp_prob = nullptr;
+   DofMap * novlp_prob = nullptr;
+   Array<SesquilinearForm *> HalfSpaceForms;
+   Array<SparseMatrix *> PmlMat;
+   Array<SparseMatrix *> HalfSpaceMat;
+   Array<KLUSolver *> PmlMatInv;
+   Array<KLUSolver *> HalfSpaceMatInv;
+   Array2D<double> Pmllength;
+   mutable Array<Vector * > res;
+
+   SparseMatrix * GetPmlSystemMatrix(int ip);
+   SparseMatrix * GetHalfSpaceSystemMatrix(int ip);
+   void SolveHalfSpaceLinearSystem(int ip, Vector & x, Vector & load) const;
+   void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+   void GetCutOffSolution(Vector & sol, int ip, int direction) const;
+
+public:
+   STP(SesquilinearForm * bf_, Array2D<double> & Pmllength_, 
+       double omega_, Coefficient * ws_, int nrlayers_);
+   void SetLoadVector(Vector load) { B = load;}
+   virtual void SetOperator(const Operator &op) {A = &op;}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~STP();
+};
+
+

examples/maxwell-solver/ST/SourceTransfer.cpp

@@ -0,0 +1,473 @@
+#include "SourceTransfer.hpp"
+
+STPmlPatchAssembly::STPmlPatchAssembly(SesquilinearForm * bf_, Array<int> & ess_tdofs, 
+                                   double omega_, int nrlayers_, int part) 
+                                   : bf(bf_), omega(omega_), nrlayers(nrlayers_)
+{
+   fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   int dim = mesh->Dimension();
+   const FiniteElementCollection *fec = fespace->FEColl();
+
+   p = new MeshPartition(mesh, part);
+   nx = p->nx;
+   ny = p->ny;
+   nz = p->nz;
+   // SaveMeshPartition(p->patch_mesh);
+   nrpatch = p->nrpatch;
+   patch_fespaces.SetSize(nrpatch);
+   patch_meshes_ext.SetSize(nrpatch);
+   patch_fespaces_ext.SetSize(nrpatch);
+   dof2extdof_map.resize(nrpatch);
+   patch_dof_map.resize(nrpatch);
+   patch_mat.SetSize(nrpatch);
+   patch_mat_ext.SetSize(nrpatch);
+   patch_mat_inv.SetSize(nrpatch);
+   patch_mat_inv_ext.SetSize(nrpatch);
+   ess_tdof_list.resize(nrpatch);
+   ess_tdof_list_ext.resize(nrpatch);
+
+   // construct extended meshes for the pml
+
+   int ip = -1;
+   for (int kz = 0; kz<nz; kz++)
+   {
+      for (int ky = 0; ky<ny; ky++)
+      {
+         for (int kx = 0; kx<nx; kx++)
+         {
+            ip++;
+            Array<int> ext_directions;
+            for (int j=0; j<nrlayers; ++j)// one more layer of extension (epsilon layer)
+            {
+               for (int comp=0; comp<dim; ++comp)
+               {
+                  if (comp == 0 && kx != 0)
+                  {
+                     ext_directions.Append(-comp-1);
+                  }
+                  if (comp == 0 && kx != nx-1)
+                  {
+                     ext_directions.Append(comp+1);
+                  }
+                  if (comp == 1 && ky != 0)
+                  {
+                     ext_directions.Append(-comp-1);
+                  }
+                  if (comp == 1 && ky != ny-1)
+                  {
+                     ext_directions.Append(comp+1);
+                  }
+                  if (comp == 2 && kz != 0)
+                  {
+                     // ext_directions.Append(-comp-1);
+                  }
+                  if (comp == 2 && kz != nz-1)
+                  {
+                     // ext_directions.Append(comp+1);
+                  }
+               }
+            }
+            if (ip < nrpatch-1)
+            {
+               // ext_directions.Append(1);
+               // ext_directions.Append(1);
+               // ext_directions.Append(1);
+               // ext_directions.Append(1);
+            }
+            patch_meshes_ext[ip] = ExtendMesh(p->patch_mesh[ip],ext_directions);
+         }
+      }
+   }
+   // SaveMeshPartition(patch_meshes_ext, "output/ext_mesh.", "output/ext_sol.");
+   // // cout << p->patch_mesh[0]->GetNE() << endl;
+
+
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // create finite element spaces for each patch // This might be avoided
+      patch_fespaces[ip] = new FiniteElementSpace(p->patch_mesh[ip],fec);
+      // create finite element spaces on the extented (PML) meshes
+      patch_fespaces_ext[ip] = new FiniteElementSpace(patch_meshes_ext[ip],fec);
+
+      // construct the patch tdof to global tdof map
+      int nrdof = patch_fespaces[ip]->GetTrueVSize();
+      patch_dof_map[ip].SetSize(2*nrdof);
+      dof2extdof_map[ip].SetSize(2*nrdof);
+
+      // build dof maps between patch and extended patch
+      // loop through the patch elements and constract the dof map
+      // The same elements in the extended mesh have the same ordering (but not the dofs)
+
+      // loop through the elements in the patch
+      for (int iel = 0; iel<p->element_map[ip].Size(); ++iel)
+      {
+         // index in the global mesh
+         int iel_idx = p->element_map[ip][iel];
+         // get the dofs of this element
+         Array<int> patch_elem_dofs;
+         Array<int> patch_elem_dofs_ext;
+         Array<int> global_elem_dofs;
+         patch_fespaces[ip]->GetElementDofs(iel,patch_elem_dofs);
+         patch_fespaces_ext[ip]->GetElementDofs(iel,patch_elem_dofs_ext);
+         fespace->GetElementDofs(iel_idx,global_elem_dofs);
+         // the sizes have to match
+         MFEM_VERIFY(patch_elem_dofs.Size() == global_elem_dofs.Size(),
+                     "Size inconsistency");
+         MFEM_VERIFY(patch_elem_dofs.Size() == patch_elem_dofs_ext.Size(),
+                     "Size inconsistency");            
+         // loop through the dofs and take into account the signs;
+         int ndof = patch_elem_dofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = patch_elem_dofs[i];
+            int gdof_ = global_elem_dofs[i];
+            int extdof_ = patch_elem_dofs_ext[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            int extdof = (extdof_ >= 0) ? extdof_ : abs(extdof_) - 1;
+            patch_dof_map[ip][pdof] = gdof;
+            patch_dof_map[ip][pdof+nrdof] = gdof+fespace->GetTrueVSize();
+            dof2extdof_map[ip][pdof] = extdof;
+            dof2extdof_map[ip][pdof+nrdof] = extdof+patch_fespaces_ext[ip]->GetTrueVSize();
+         }
+      }
+      // // Define the patch bilinear form and apply boundary conditions (only the LHS)
+      // Array <int> ess_temp_list;
+      // if (p->patch_mesh[ip]->bdr_attributes.Size())
+      // {
+      //    Array<int> ess_bdr(p->patch_mesh[ip]->bdr_attributes.Max());
+      //    ess_bdr = 0;
+      //    patch_fespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_temp_list);
+      // }
+
+      Array <int> ess_list_ext;
+      if (patch_meshes_ext[ip]->bdr_attributes.Size())
+      {
+         Array<int> ess_bdr(patch_meshes_ext[ip]->bdr_attributes.Max());
+         ess_bdr = 1;
+         patch_fespaces_ext[ip]->GetEssentialTrueDofs(ess_bdr, ess_list_ext);
+      }
+      ess_tdof_list_ext[ip] = ess_list_ext;
+
+      // // Adjust the essential tdof list for each patch
+      // for (int i=0; i<ess_temp_list.Size(); i++)
+      // {
+      //    int ldof = ess_temp_list[i];
+      //    int tdof = patch_dof_map[ip][ldof];
+      //    // check the kind of this tdof
+      //    if (!global_tdofs[tdof]) ess_tdof_list[ip].Append(ldof);
+      // }
+
+      // SesquilinearForm a(patch_fespaces[ip], &bf->real(), &bf->imag());
+      
+      //-----------------PML FORMULATION----------------------------
+      Array2D<double> length(dim,2);
+      double h = GetUniformMeshElementSize(patch_meshes_ext[ip]);
+      length = h*(nrlayers);
+
+      if (ip < nrpatch-1)
+      {
+         // length(0,1) = h*(nrlayers+4);
+      }
+      // if (ip != 0) 
+      // {
+      //    length(0,0) = 0.0;
+      // }
+      // // length = h * nrlayers;
+      // // if (ip != 0)
+      // // {
+      // //    length(0,0) = 0.0;
+      // //    length(1,0) = 0.0;
+      // // }
+      // // length(0,1) = h * nrlayers;
+      // // length(1,1) = h * nrlayers;
+      // // if (ip == 1 || ip == 2 || ip == 3) length(1,0) = h * nrlayers;
+      // // if (ip == 4 || ip == 8 || ip == 12) length(0,0) = h * nrlayers;
+
+
+      CartesianPML pml(patch_meshes_ext[ip], length);
+      pml.SetOmega(omega);
+
+      ConstantCoefficient one(1.0);
+      ConstantCoefficient sigma(-pow(omega, 2));
+
+      PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+      PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+      PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+      PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+      ProductCoefficient c2_re(sigma, detJ_re);
+      ProductCoefficient c2_im(sigma, detJ_im);
+
+      SesquilinearForm a_ext(patch_fespaces_ext[ip],ComplexOperator::HERMITIAN);
+
+      a_ext.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                                new DiffusionIntegrator(c1_im));
+      a_ext.AddDomainIntegrator(new MassIntegrator(c2_re),
+                                new MassIntegrator(c2_im));
+
+      //------------------------------------------------------------
+
+      // a.Assemble();
+      a_ext.Assemble();
+      // OperatorPtr Alocal;
+      // a.FormSystemMatrix(ess_tdof_list[ip],Alocal);
+      // ComplexSparseMatrix * AZ = Alocal.As<ComplexSparseMatrix>();
+      // patch_mat[ip] = AZ->GetSystemMatrix();
+      // patch_mat[ip]->Threshold(0.0);
+      // // Save the inverse
+      // patch_mat_inv[ip] = new KLUSolver;
+      // patch_mat_inv[ip]->SetOperator(*patch_mat[ip]);
+
+
+      OperatorPtr Alocal_ext;
+      a_ext.FormSystemMatrix(ess_list_ext,Alocal_ext);
+      ComplexSparseMatrix * AZ_ext = Alocal_ext.As<ComplexSparseMatrix>();
+      patch_mat_ext[ip] = AZ_ext->GetSystemMatrix();
+      patch_mat_ext[ip]->Threshold(0.0);
+      patch_mat_inv_ext[ip] = new KLUSolver;
+      patch_mat_inv_ext[ip]->SetOperator(*patch_mat_ext[ip]);
+
+
+      // delete patch_fespaces[ip];
+      // delete patch_fespaces_ext[ip];
+   }
+   // delete p;
+}
+
+STPmlPatchAssembly::~STPmlPatchAssembly()
+{
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // delete patch_fespaces[ip]; patch_fespaces[ip]=nullptr;
+      delete patch_fespaces[ip];
+      delete patch_fespaces_ext[ip];
+      delete patch_meshes_ext[ip];
+      patch_meshes_ext[ip]=nullptr;
+      // delete patch_mat_inv[ip];
+      delete patch_mat_inv_ext[ip];
+      // patch_mat_inv[ip]=nullptr;
+      patch_mat_inv_ext[ip]=nullptr;
+      // delete patch_mat[ip];
+      delete patch_mat_ext[ip];
+      // patch_mat[ip]=nullptr;
+      patch_mat_ext[ip]=nullptr;
+   }
+   // patch_fespaces.DeleteAll();
+   patch_meshes_ext.DeleteAll();
+   patch_mat_ext.DeleteAll();
+   // patch_mat.DeleteAll();
+   // patch_mat_inv.DeleteAll();
+   // patch_mat_inv.DeleteAll();
+   // delete p;
+}
+
+
+
+
+void SourceTransferPrecond::GetCutOffSolution(Vector & sol, int ip) const
+{
+
+   Mesh * mesh = p->patch_fespaces[ip]->GetMesh();
+   int n = p->patch_fespaces[ip]->GetTrueVSize();
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+   int dim = mesh->Dimension();
+   double hl = GetUniformMeshElementSize(mesh);
+   Array2D<double> h(dim,2);
+   h[0][0] = 0.0;
+   h[0][1] = hl;
+   h[1][0] = 0.0;
+   h[1][1] = 0.0;
+   CutOffFunctionCoefficient cf(CutOffFn, pmin, pmax, h);
+
+   double * data = sol.GetData();
+
+   GridFunction solgf_re(p->patch_fespaces[ip], data);
+   GridFunction solgf_im(p->patch_fespaces[ip], &data[n]);
+
+
+   GridFunctionCoefficient coeff1_re(&solgf_re);
+   GridFunctionCoefficient coeff1_im(&solgf_im);
+
+   ProductCoefficient prod_re(coeff1_re, cf);
+   ProductCoefficient prod_im(coeff1_im, cf);
+
+   ComplexGridFunction gf(p->patch_fespaces[ip]);
+   gf.ProjectCoefficient(prod_re,prod_im);
+
+   sol = gf;
+}
+
+
+
+
+
+
+SourceTransferPrecond::SourceTransferPrecond(SesquilinearForm * bf_, Array<int> & ess_tdofs, double omega_, int nrlayers_, int i)
+   : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), bf(bf_), omega(omega_), nrlayers(nrlayers_),
+     part(i)
+{
+   p = new STPmlPatchAssembly(bf_, ess_tdofs, omega, nrlayers, part);
+   nrpatch = p->nrpatch;
+}
+
+void SourceTransferPrecond::Mult(const Vector &r, Vector &z) const
+{
+   z = 0.0;
+   Vector rnew(r);
+   Vector znew(z);
+   Vector raux(znew.Size());
+   Vector res_local, sol_local;
+   Array<int> visit(znew.Size());
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+
+   // zero out sources from other subdomains
+   // save the first subdomain
+   // rnew = 0.0;
+   // Array<int> * dof_map0 = &p->patch_dof_map[0];
+   // int ndofs = dof_map0->Size();
+   // res_local.SetSize(ndofs);
+   // r.GetSubVector(*dof_map0, res_local);
+   // rnew.SetSubVector(*dof_map0,res_local.GetData());
+
+
+   // socketstream sol_sock(vishost, visport);
+   // sol_sock.precision(8);
+   // socketstream res_sock(vishost, visport);
+   // res_sock.precision(8);
+   // cout << "nrpatch = " << nrpatch << endl; 
+   for (int iter = 0; iter < maxit; iter++)
+   {
+      znew = 0.0;
+      visit = 0;
+      for (int ip = 0; ip < nrpatch; ip++)
+      {
+         // cout << "ip = " << ip << endl;
+         Array<int> * dof_map = &p->patch_dof_map[ip];
+         int ndofs = dof_map->Size();
+         res_local.SetSize(ndofs);
+         sol_local.SetSize(ndofs);
+
+         rnew.GetSubVector(*dof_map, res_local);
+
+         //-----------------------------------------------
+         // Extend by zero to the extended mesh
+         int nrdof_ext = p->patch_mat_ext[ip]->Height();
+         
+         Vector res_ext(nrdof_ext); res_ext = 0.0;
+         Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+
+         res_ext.SetSubVector(p->dof2extdof_map[ip],res_local.GetData());
+
+         p->patch_mat_inv_ext[ip]->Mult(res_ext, sol_ext);
+
+
+
+         sol_ext.GetSubVector(p->dof2extdof_map[ip],sol_local);
+
+         // Smooth the solution before transfer
+         // if (ip < nrpatch-1) GetCutOffSolution(sol_local, ip);
+
+         if (type == 1) znew = 0.0;
+         znew.AddElementVector(*dof_map,sol_local);
+         // zero out the contributions to the dofs which are already updated
+         // for (int i = 0; i<ndofs; i++)
+         // {
+         //    int j = (*dof_map)[i];
+         //    if (visit[j])
+         //    {
+         //       znew(j) = 0.0;
+         //    }
+         //    else
+         //    {
+         //       visit[j] = 1;
+         //    }
+         // }
+         if (type == 1)
+         {
+            z.Add(theta, znew);
+            A->Mult(znew, raux);
+            rnew -= raux;
+         }
+         // PlotSolution(z, sol_sock, ip); cin.get();
+         // PlotSolution(rnew, res_sock, ip); cin.get();
+      }
+      if (type == 0)
+      {
+         z.Add(theta, znew);
+         A->Mult(znew, raux);
+         rnew -= raux;
+      }
+      // Update residual
+      if (iter + 1 < maxit)
+      {
+         A->Mult(znew, raux);
+         rnew -= raux;
+      }
+   }
+   // PlotSolution(rnew, sol_sock, 0); cin.get();
+}
+
+
+void SourceTransferPrecond::PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const
+{
+   FiniteElementSpace * fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   ComplexGridFunction gf(fespace);
+   bf->RecoverFEMSolution(sol,B,gf);
+   
+   string keys;
+   if (ip == 0) keys = "keys mrRljc\n";
+   sol_sock << "solution\n" << *mesh << gf.imag() << keys << flush;
+}
+
+SourceTransferPrecond::~SourceTransferPrecond(){ }
+
+
+
+double CutOffFn(const Vector &x, const Vector & pmin, const Vector & pmax, const Array2D<double> & h_)
+{
+   int dim = pmin.Size();
+   Vector h0(dim);
+   Vector h1(dim);
+   for (int i=0; i<dim; i++)
+   {
+      h0(i) = h_[i][0];
+      h1(i) = h_[i][1];
+   }
+   Vector x0(dim);
+   x0 = pmax; x0-=h1;
+   Vector x1(dim);
+   x1 = pmin; x1+=h0;
+
+   double f = 1.0;
+
+   for (int i = 0; i<dim; i++)
+   {
+      double val = 1.0;
+      if( x(i) > pmax(i) || x(i) < pmin(i))
+      {
+         val = 0.0;
+      }  
+      else if (x(i) <= pmax(i) && x(i) >= x0(i))
+      {
+         if(x0(i)-pmax(i) != 0.0)
+            val = (x(i)-pmax(i))/(x0(i)-pmax(i)); 
+      }
+      else if (x(i) >= pmin(i) && x(i) <= x1(i))
+      {
+         if (x1(i)-pmin(i) != 0.0)
+            val = (x(i)-pmin(i))/(x1(i)-pmin(i)); 
+      }
+      else
+      {
+         val = 1.0;
+      }
+      f *= val;
+   }
+   return f;
+}

examples/maxwell-solver/ST/SourceTransfer.hpp

@@ -0,0 +1,92 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "complex_additive_schwarz.hpp"
+using namespace std;
+using namespace mfem;
+
+class STPmlPatchAssembly
+{
+   FiniteElementSpace *fespace=nullptr;
+   SesquilinearForm *bf=nullptr;
+   double omega = 0.5;
+   int nrlayers = 4;
+public:
+   int nrpatch, nx, ny, nz;
+   MeshPartition * p;
+   Array<FiniteElementSpace *> patch_fespaces;
+   Array<FiniteElementSpace *> patch_fespaces_ext;
+   Array<Mesh *> patch_meshes_ext;
+   std::vector<Array<int>> patch_dof_map;
+   std::vector<Array<int>> complex_patch_dof_map;
+   std::vector<Array<int>> dof2extdof_map;
+   Array<SparseMatrix *> patch_mat;
+   Array<SparseMatrix *> patch_mat_ext;
+   Array<KLUSolver * > patch_mat_inv_ext;
+   Array<KLUSolver * > patch_mat_inv;
+   std::vector<Array<int>> ess_tdof_list;
+   std::vector<Array<int>> ess_tdof_list_ext;
+
+   // constructor
+   STPmlPatchAssembly(SesquilinearForm * bf_, Array<int> & ess_tdofs, double omega_, int nrlayers_, int part);
+
+   ~STPmlPatchAssembly();
+};
+
+
+class SourceTransferPrecond : public Solver//
+{
+private:
+   int nrpatch;
+   int maxit = 1;
+   SesquilinearForm *bf=nullptr;
+   int type = 0;
+   double theta = 0.5;
+   double omega = 0.5;
+   int nrlayers;
+   int part;
+   STPmlPatchAssembly * p;
+   const Operator * A;
+   Vector B;
+   void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+   void GetCutOffSolution(Vector & sol, int ip) const;
+
+   
+public:
+   SourceTransferPrecond(SesquilinearForm * bf_, Array<int> & ess_tdofs, double omega_, int nrlayers_, int i = 0);
+   void SetNumSmoothSteps(const int iter) { maxit = iter;}
+   void SetLoadVector(Vector load) { B = load;}
+   void SetSmoothType(int itype) { type = itype;}
+   void SetDumpingParam(const double & dump_param) {theta = dump_param;}
+   void SetOmega(const double & omega_) {omega = omega_;}
+   virtual void SetOperator(const Operator &op) {A = &op;}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~SourceTransferPrecond();
+};
+
+
+// Function coefficient that takes the boundingbox of the mesh as an input
+class CutOffFunctionCoefficient : public Coefficient
+{
+private:
+   double (*Function)(const Vector &, const Vector &, const Vector &, const Array2D<double> &);
+   Vector pmin, pmax;
+   Array2D<double> h; // specify the with of the cutoff function (h in each direction)
+   
+
+public:
+   CutOffFunctionCoefficient(double (*F)(const Vector &, const Vector &, const Vector &, const Array2D<double> &), 
+                             const Vector & pmin_, const Vector & pmax_, Array2D<double> & h_)
+      : Function(F), pmin(pmin_), pmax(pmax_), h(h_)
+   {}
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+   {
+      double x[3];
+      Vector transip(x, 3);
+      T.Transform(ip, transip);
+      return ((*Function)(transip, pmin, pmax, h));
+   }
+};
+
+double CutOffFn(const Vector &x, const Vector &pmax, const Vector &pmin, const Array2D<double> & h_);

examples/maxwell-solver/ST/additive_schwarz.cpp

@@ -0,0 +1,823 @@
+#include "additive_schwarz.hpp"
+
+
+// constructor
+OverlappingCartesianMeshPartition::OverlappingCartesianMeshPartition(Mesh *mesh_) : mesh(mesh_)
+{  // default overlap size is 2 elements 
+   int dim = mesh->Dimension();
+   int n = pow(mesh->GetNE(), 1.0/(double)dim);
+   nx = 16;
+   ny = 1;
+   nz = 1;
+   if (nx > n) 
+   {
+      nx = n;
+      MFEM_WARNING("Changed partition in the x direction to nx = " << n << endl);
+   }
+   if (ny > n) 
+   {
+      ny = n;
+      MFEM_WARNING("Changed partition in the y direction to ny = " << n << endl);
+   }
+   if (nz > n) 
+   {
+      nz = n;
+      MFEM_WARNING("Changed partition in the z direction to nz = " << n << endl);
+   } 
+   if (dim == 2) nz = 1;
+   int nxyz[3] = {nx,ny,nz};
+   nrpatch = nx*ny*nz;
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+   double h = GetUniformMeshElementSize(mesh);
+
+   element_map.resize(nrpatch);
+
+   double ppt[dim];
+   Vector pt(ppt, dim);
+   int nrelem = mesh->GetNE();
+
+   for (int el = 0; el < nrelem; el++)
+   {
+      mesh->GetElementTransformation(el)->Transform(
+         Geometries.GetCenter(mesh->GetElementBaseGeometry(el)), pt);
+      // Given the center coordinates determine the patches that this element contributes to
+      Array<int> idx0(dim);
+      Array<int> idx1(dim);
+      Array<int> idx2(dim);
+      vector<Array<int>> idx(3);
+      if (dim == 2) idx[2].Append(0);
+
+      for (int i = 0; i<dim; i++)
+      {
+         idx0[i]  = (int)floor(nxyz[i]*((pt(i) - pmin[i])/(pmax[i] - pmin[i])));
+         idx1[i] = (int)floor(nxyz[i]*((pt(i)-2*h - pmin[i])/(pmax[i] - pmin[i])));
+         idx2[i] = (int)floor(nxyz[i]*((pt(i)-h - pmin[i])/(pmax[i] - pmin[i])));
+
+         if (idx0[i] < 0) idx0[i] = 0;
+         if (idx0[i] >= nxyz[i]) idx0[i] = nxyz[i]-1;
+         
+         if (idx1[i] < 0) idx1[i] = 0;
+         if (idx1[i] >= nxyz[i]) idx1[i] = nxyz[i]-1;
+
+         if (idx2[i] < 0) idx2[i] = 0;
+         if (idx2[i] >= nxyz[i]) idx2[i] = nxyz[i]-1;
+         // convenient to put in one list
+         idx[i].Append(idx0[i]);
+         if (idx1[i] != idx0[i]) idx[i].Append(idx1[i]);
+         if (idx2[i] != idx0[i] && idx2[i] != idx1[i]) idx[i].Append(idx2[i]);
+      }
+      // Now loop through all the combinations according to the idx above
+      // in case of dim = 2 then kk = 0
+      for (int k=0; k<idx[2].Size(); k++)
+      {
+         int kk = idx[2][k];
+         for (int j=0; j<idx[1].Size(); j++)
+         {
+            int jj = idx[1][j];
+            for (int i=0; i<idx[0].Size(); i++)
+            {
+               int ii = idx[0][i];
+               int ip = kk*nxyz[0]*nxyz[1] + jj*nxyz[0]+ii;
+               element_map[ip].Append(el);
+            }
+         }
+      }
+   }
+}
+
+
+// constructor
+CartesianMeshPartition::CartesianMeshPartition(Mesh *mesh_) : mesh(mesh_)
+{
+   int dim = mesh->Dimension();
+   nx = 5;
+   ny = 1;
+   nz = 1;
+   int nxyz[3] = {nx,ny,nz};
+   nrpatch = nx*ny*nz;
+
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+
+   int nrelem = mesh->GetNE();
+   int partitioning[nrelem];
+
+   // determine the partitioning using the centers of the elements
+   double ppt[dim];
+   Vector pt(ppt, dim);
+   for (int el = 0; el < nrelem; el++)
+   {
+      mesh->GetElementTransformation(el)->Transform(
+         Geometries.GetCenter(mesh->GetElementBaseGeometry(el)), pt);
+      int part = 0;
+      for (int i = dim-1; i >= 0; i--)
+      {
+         int idx = (int)floor(nxyz[i]*((pt(i) - pmin[i])/(pmax[i] - pmin[i])));
+         if (idx < 0)
+         {
+            idx = 0;
+         }
+         if (idx >= nxyz[i])
+         {
+            idx = nxyz[i]-1;
+         }
+         part = part * nxyz[i] + idx;
+      }
+      partitioning[el] = part;
+   }
+
+   element_map.resize(nrpatch);
+   for (int iel = 0; iel < nrelem; iel++)
+   {
+      int ip = partitioning[iel];
+      element_map[ip].Append(iel);
+   }
+}
+
+
+STPOverlappingCartesianMeshPartition::STPOverlappingCartesianMeshPartition(Mesh *mesh_) : mesh(mesh_)
+{
+   int dim = mesh->Dimension();
+   nx = 9;
+   ny = 1;
+   nz = 1;
+   int nxyz[3] = {nx,ny,nz};
+   // nrpatch = nx*ny*nz;
+
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin, pmax);
+
+   int nrelem = mesh->GetNE();
+   int partitioning[nrelem];
+
+   // determine the partitioning using the centers of the elements
+   double ppt[dim];
+   Vector pt(ppt, dim);
+   for (int el = 0; el < nrelem; el++)
+   {
+      mesh->GetElementTransformation(el)->Transform(
+         Geometries.GetCenter(mesh->GetElementBaseGeometry(el)), pt);
+      int part = 0;
+      for (int i = dim-1; i >= 0; i--)
+      {
+         int idx = (int)floor(nxyz[i]*((pt(i) - pmin[i])/(pmax[i] - pmin[i])));
+         if (idx < 0)
+         {
+            idx = 0;
+         }
+         if (idx >= nxyz[i])
+         {
+            idx = nxyz[i]-1;
+         }
+         part = part * nxyz[i] + idx;
+      }
+      partitioning[el] = part;
+   }
+   
+   // element_map.resize(nrpatch);
+   // for (int iel = 0; iel < nrelem; iel++)
+   // {
+   //    int ip = partitioning[iel];
+   //    element_map[ip].Append(iel);
+   // }
+   // // Append the next subdomain to the previous
+   // for (int ip = 0; ip<nrpatch-1; ip++)
+   // {
+   //    element_map[ip].Append(element_map[ip+1]);
+   // }
+
+
+   std::vector<Array<int>> elem_map;
+   int npatch = nx*ny*nz;
+   elem_map.resize(npatch);
+   for (int iel = 0; iel < nrelem; iel++)
+   {
+      int ip = partitioning[iel];
+      elem_map[ip].Append(iel);
+   }
+   // Append the next subdomain to the previous
+   nrpatch = nx*ny*nz-1;
+   element_map.resize(nrpatch);
+   for (int ip = 0; ip<nrpatch; ip++)
+   {
+      element_map[ip].Append(elem_map[ip]);
+      element_map[ip].Append(elem_map[ip+1]);
+   }
+
+
+
+}
+
+// constructor
+VertexMeshPartition::VertexMeshPartition(Mesh *mesh_) : mesh(mesh_)
+{
+   nrpatch = mesh->GetNV();
+   element_map.resize(nrpatch);
+   //every element will contribute to the the patches of its vertices
+   // loop through the elements
+   int nrelems = mesh->GetNE();
+   for (int iel=0; iel<nrelems; ++iel)
+   {
+      // get element vertex index
+      Array<int> vertices;
+      mesh->GetElementVertices(iel,vertices);
+      int nrvert = vertices.Size();
+      // fill in the element contribution lists
+      for (int iv = 0; iv< nrvert; ++iv)
+      {
+         int ip = vertices[iv];
+         element_map[ip].Append(iel);
+      }
+   }
+}
+
+
+MeshPartition::MeshPartition(Mesh* mesh_, int part): mesh(mesh_)
+{
+   partition_kind = part;
+   if (part == 1)
+   {
+      cout << "Non Overlapping Cartesian Partition " << endl;
+      CartesianMeshPartition partition(mesh);
+      element_map = partition.element_map;
+      nx = partition.nx;
+      ny = partition.ny;
+      nz = partition.nz;
+   }
+   // else if (part == 3 || part == 4)
+   else if (part == 2)
+   {
+      cout << "Overlapping Cartesian Partition " << endl;
+      OverlappingCartesianMeshPartition partition(mesh);
+      element_map = partition.element_map;
+      nx = partition.nx;
+      ny = partition.ny;
+      nz = partition.nz;
+   }
+   else if (part == 3 || part == 4)
+   // else if (part == 2)
+   {
+      cout << "STP Overlapping Cartesian Partition " << endl;
+      STPOverlappingCartesianMeshPartition partition(mesh);
+      element_map = partition.element_map;
+      nx = partition.nx;
+      ny = partition.ny;
+      nz = partition.nz;
+   }
+   else
+   {
+      cout << "Overlapping Vertex based partition " << endl;
+      VertexMeshPartition partition(mesh);
+      element_map = partition.element_map;
+      partition_kind = 0;
+   }
+
+   nrpatch = element_map.size();
+   int dim = mesh->Dimension();
+
+   patch_mesh.SetSize(nrpatch);
+   for (int ip = 0; ip<nrpatch; ++ip)
+   {
+      int patch_nrelems = element_map[ip].Size();
+      element_map[ip].SetSize(patch_nrelems);
+      // need to ensure that a vertex is not added more than once
+      // and that the ordering of vertices is known for when the element is added
+      // create a list of for this patch including possible repetitions
+      // loop through elements in the patch
+      Array<int> patch_vertices;
+      for (int iel=0; iel<patch_nrelems; ++iel)
+      {
+         // get the vertices list for the element
+         Array<int> elem_vertices;
+         int iel_idx = element_map[ip][iel];
+         mesh->GetElementVertices(iel_idx,elem_vertices);
+         patch_vertices.Append(elem_vertices);
+      }
+      patch_vertices.Sort();
+      patch_vertices.Unique();
+      int patch_nrvertices = patch_vertices.Size();
+
+      // create the mesh
+      patch_mesh[ip] = new Mesh(dim,patch_nrvertices,patch_nrelems);
+      // Add the vertices
+      for (int iv = 0; iv<patch_nrvertices; ++iv)
+      {
+         int vert_idx = patch_vertices[iv];
+         patch_mesh[ip]->AddVertex(mesh->GetVertex(vert_idx));
+      }
+
+      // Add the elements (for now search through all the vertices in the patch is needed)
+      for (int iel=0; iel<patch_nrelems; ++iel)
+      {
+         // get the vertices list for the element
+         Array<int> elem_vertices;
+         int iel_idx = element_map[ip][iel];
+         mesh->GetElementVertices(iel_idx,elem_vertices);
+         int nrvert = elem_vertices.Size();
+         int ind[nrvert];
+         for (int iv = 0; iv<nrvert; ++iv)
+         {
+            ind[iv] = patch_vertices.FindSorted(elem_vertices[iv]);
+         }
+         mfem::Element::Type elem_type = mesh->GetElementType(element_map[ip][iel]);
+
+         AddElementToMesh(patch_mesh[ip],elem_type,ind);
+
+      }
+      patch_mesh[ip]->FinalizeTopology();
+   }
+}
+
+
+void MeshPartition::AddElementToMesh(Mesh * mesh,mfem::Element::Type elem_type,
+                                     int * ind)
+{
+   switch (elem_type)
+   {
+      case Element::QUADRILATERAL:
+         mesh->AddQuad(ind);
+         break;
+      case Element::TRIANGLE :
+         mesh->AddTri(ind);
+         break;
+      case Element::HEXAHEDRON :
+         mesh->AddHex(ind);
+         break;
+      case Element::TETRAHEDRON :
+         mesh->AddTet(ind);
+         break;
+      default:
+         MFEM_ABORT("Unknown element type");
+         break;
+   }
+}
+
+void MeshPartition::PrintElementMap()
+{
+   mfem::out << "Element map" << endl;
+   for (int ip = 0; ip<nrpatch; ++ip)
+   {
+      mfem::out << "Patch No: " << ip;
+      mfem::out << ", element map: " ;
+      element_map[ip].Print(cout,element_map[ip].Size());
+   }
+}
+
+void SaveMeshPartition(Array<Mesh *> meshes, string mfilename, string sfilename)
+{
+   int nrmeshes = meshes.Size();
+   for (int ip = 0; ip<nrmeshes; ++ip)
+   {
+      cout << "saving mesh no " << ip << endl;
+      ostringstream mesh_name;
+      mesh_name << mfilename << setfill('0') << setw(6) << ip;
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      meshes[ip]->Print(mesh_ofs);
+      L2_FECollection L2fec(1,meshes[ip]->Dimension());
+      FiniteElementSpace L2fes(meshes[ip], &L2fec);
+      GridFunction x(&L2fes); 
+      
+      ConstantCoefficient alpha((double)ip);
+      x.ProjectCoefficient(alpha);
+      ostringstream sol_name;
+      sol_name << sfilename << setfill('0') << setw(6) << ip;
+      ofstream sol_ofs(sol_name.str().c_str());
+      x.Save(sol_ofs);
+   }
+}
+MeshPartition::~MeshPartition()
+{
+   for (int ip = 0; ip<nrpatch; ++ip)
+   {
+      delete patch_mesh[ip];
+      patch_mesh[ip] = nullptr;
+   }
+   patch_mesh.DeleteAll();
+}
+
+// constructor
+PatchAssembly::PatchAssembly(BilinearForm *bf_, Array<int> & ess_tdofs, int part) : bf(bf_)
+{
+   fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   const FiniteElementCollection *fec = fespace->FEColl();
+
+   // list of dofs to distiguish between interior/boundary and essential
+   Array<int> global_tdofs(fespace->GetTrueVSize());
+   Array<int> bdr_tdofs(fespace->GetTrueVSize());
+   global_tdofs = 0;
+   // Mark boundary dofs and ess_dofs
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, bdr_tdofs);
+   }
+
+   // mark boundary dofs
+   for (int i = 0; i<bdr_tdofs.Size(); i++) global_tdofs[bdr_tdofs[i]] = 1;
+   // overwrite flag for essential dofs
+   for (int i = 0; i<ess_tdofs.Size(); i++) global_tdofs[ess_tdofs[i]] = 0;
+
+   MeshPartition * p = new MeshPartition(mesh, part);
+   // SaveMeshPartition(p->patch_mesh);
+   nrpatch = p->nrpatch;
+   patch_fespaces.SetSize(nrpatch);
+   patch_dof_map.resize(nrpatch);
+   patch_mat.SetSize(nrpatch);
+   patch_mat_inv.SetSize(nrpatch);
+   ess_tdof_list.resize(nrpatch);
+   ess_int_tdofs.resize(nrpatch);
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // create finite element spaces for each patch
+      patch_fespaces[ip] = new FiniteElementSpace(p->patch_mesh[ip],fec);
+      // construct the patch tdof to global tdof map
+      int nrdof = patch_fespaces[ip]->GetTrueVSize();
+      patch_dof_map[ip].SetSize(nrdof);
+      // loop through the elements in the patch
+      for (int iel = 0; iel<p->element_map[ip].Size(); ++iel)
+      {
+         // index in the global mesh
+         int iel_idx = p->element_map[ip][iel];
+         // get the dofs of this element
+         Array<int> patch_elem_dofs;
+         Array<int> global_elem_dofs;
+         patch_fespaces[ip]->GetElementDofs(iel,patch_elem_dofs);
+         fespace->GetElementDofs(iel_idx,global_elem_dofs);
+         // the sizes have to match
+         MFEM_VERIFY(patch_elem_dofs.Size() == global_elem_dofs.Size(),
+                     "Size inconsistency");
+         // loop through the dofs and take into account the signs;
+         int ndof = patch_elem_dofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = patch_elem_dofs[i];
+            int gdof_ = global_elem_dofs[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            patch_dof_map[ip][pdof] = gdof;
+         }
+      }
+      // Define the patch bilinear form and apply boundary conditions (only the LHS)
+      Array <int> ess_temp_list;
+      if (p->patch_mesh[ip]->bdr_attributes.Size())
+      {
+         Array<int> ess_bdr(p->patch_mesh[ip]->bdr_attributes.Max());
+         ess_bdr = 1;
+         patch_fespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_temp_list);
+      }
+
+      // Adjust the essential tdof list for each patch
+      for (int i=0; i<ess_temp_list.Size(); i++)
+      {
+         int ldof = ess_temp_list[i];
+         int tdof = patch_dof_map[ip][ldof];
+         // check the kind of this tdof
+         if (!global_tdofs[tdof]) ess_tdof_list[ip].Append(ldof);
+      }
+
+      BilinearForm a(patch_fespaces[ip], bf);
+      a.Assemble();
+      OperatorPtr Alocal;
+      a.FormSystemMatrix(ess_tdof_list[ip],Alocal);
+      delete patch_fespaces[ip];
+      patch_mat[ip] = new SparseMatrix((SparseMatrix&)(*Alocal));
+      patch_mat[ip]->Threshold(0.0);
+      // Save the inverse
+      patch_mat_inv[ip] = new KLUSolver;
+      patch_mat_inv[ip]->SetOperator(*patch_mat[ip]);
+   }
+   delete p;
+}
+
+void PatchAssembly::print_patch_dof_map()
+{
+   mfem::out << "Patch dof map" << endl;
+   for (int ip = 0; ip<nrpatch; ++ip)
+   {
+      mfem::out << "Patch No: " << ip;
+      mfem::out << ", dof map: " ;
+      patch_dof_map[ip].Print(cout,patch_dof_map[ip].Size());
+   }
+}
+
+PatchAssembly::~PatchAssembly()
+{
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // delete patch_fespaces[ip]; patch_fespaces[ip]=nullptr;
+      delete patch_mat_inv[ip];
+      patch_mat_inv[ip]=nullptr;
+      delete patch_mat[ip];
+      patch_mat[ip]=nullptr;
+   }
+   patch_fespaces.DeleteAll();
+   patch_mat.DeleteAll();
+   patch_mat_inv.DeleteAll();
+}
+
+AddSchwarz::AddSchwarz(BilinearForm * bf_, Array<int> & global_ess_tdof_list, int i)
+   : Solver(bf_->FESpace()->GetTrueVSize(), bf_->FESpace()->GetTrueVSize()),
+     part(i)
+{
+   p = new PatchAssembly(bf_, global_ess_tdof_list,  part);
+   nrpatch = p->nrpatch;
+}
+
+void AddSchwarz::Mult(const Vector &r, Vector &z) const
+{
+   z = 0.0;
+   Vector rnew(r);
+   Vector znew(z);
+   Vector raux(znew.Size());
+   Vector res_local, sol_local;
+   for (int iter = 0; iter < maxit; iter++)
+   {
+      znew = 0.0;
+      for (int ip = 0; ip < nrpatch; ip++)
+      {
+         Array<int> * dof_map = &p->patch_dof_map[ip];
+         int ndofs = dof_map->Size();
+         res_local.SetSize(ndofs);
+         sol_local.SetSize(ndofs);
+
+         rnew.GetSubVector(*dof_map, res_local);
+         Array<int> ess_bdr_indices = p->ess_tdof_list[ip];
+         // for the overlapping case
+         // zero out the entries corresponding to the ess_bdr
+         p->patch_mat_inv[ip]->Mult(res_local, sol_local);
+         if (!part) { sol_local.SetSubVector(ess_bdr_indices,0.0); }
+         znew.AddElementVector(*dof_map,sol_local);
+      }
+      // Relaxation parameter
+      znew *= theta;
+      z += znew;
+      // Update residual
+      if (iter + 1 < maxit)
+      {
+         A->Mult(znew, raux);
+         rnew -= raux;
+      }
+   }
+}
+
+AddSchwarz::~AddSchwarz()
+{
+   delete p;
+}
+
+
+
+double GetUniformMeshElementSize(Mesh * mesh)
+{
+   int dim = mesh->Dimension();
+   int nrelem = mesh->GetNE();
+
+   DenseMatrix J(dim);
+   double hmin, hmax;
+   hmin = infinity();
+   hmax = -infinity();
+   Vector attr(nrelem);
+
+   for (int iel=0; iel<nrelem; ++iel)
+   {
+      int geom = mesh->GetElementBaseGeometry(iel);
+      ElementTransformation *T = mesh->GetElementTransformation(iel);
+      T->SetIntPoint(&Geometries.GetCenter(geom));
+      Geometries.JacToPerfJac(geom, T->Jacobian(), J);
+      attr(iel) = J.Det();
+      attr(iel) = pow(abs(attr(iel)), 1.0/double(dim));
+      hmin = min(hmin, attr(iel));
+      hmax = max(hmax, attr(iel));
+   }
+
+   MFEM_VERIFY(abs(hmin-hmax) < 1e-12, "Case not supported yet")
+
+   return hmax;
+}
+
+
+
+Mesh * ExtendMesh(Mesh * mesh, const Array<int> & directions)
+{
+    // extrute on one dimension
+   // flag = 1 +x, -1 -x, 2 +y, -2 +y , 3 +z, -3, -z
+
+   // copy the original mesh;
+   Mesh * mesh_orig = new Mesh(*mesh);
+   if (!directions.Size()) return mesh_orig;
+   
+   int dim = mesh_orig->Dimension();
+
+   Mesh * mesh_ext=nullptr;
+
+   for (int j=0; j<directions.Size(); j++)
+   {
+      int d = directions[j];
+      MFEM_VERIFY(abs(d)<= dim, "Cannot Extend in dimension " << d << ". Dim = " << dim << endl);
+
+      Vector pmin;
+      Vector pmax;
+      mesh_orig->GetBoundingBox(pmin,pmax);
+
+      // DenseMatrix J(dim);
+      // double hmin, hmax;
+      // hmin = infinity();
+      // hmax = -infinity();
+      // Vector attr(nrelem);
+      // // element size
+
+      // for (int iel=0; iel<nrelem; ++iel)
+      // {
+      //    int geom = mesh_orig->GetElementBaseGeometry(iel);
+      //    ElementTransformation *T = mesh_orig->GetElementTransformation(iel);
+      //    T->SetIntPoint(&Geometries.GetCenter(geom));
+      //    Geometries.JacToPerfJac(geom, T->Jacobian(), J);
+      //    attr(iel) = J.Det();
+      //    attr(iel) = pow(abs(attr(iel)), 1.0/double(dim));
+      //    hmin = min(hmin, attr(iel));
+      //    hmax = max(hmax, attr(iel));
+      // }
+      // MFEM_VERIFY(hmin==hmax, "Case not supported yet")
+      double h = GetUniformMeshElementSize(mesh_orig);
+      double val;
+      // find the vertices on the specific boundary
+      switch (d)
+      {
+      case 1:
+         val = pmax[0];
+         break;
+      case -1:
+         val = pmin[0];
+         h = -h;
+         break;    
+      case 2:
+         val = pmax[1];
+         break;
+      case -2:
+         val = pmin[1];
+         h = -h;
+         break;   
+      case 3:
+         val = pmax[2];
+         break;
+      case -3:
+         val = pmin[2];
+         h = -h;
+         break;      
+      }
+      int k = 0;
+      for (int i = 0; i<mesh_orig->GetNV(); ++i)
+      {
+         double * coords = mesh_orig->GetVertex(i);
+         switch (abs(d))
+         {
+         case 1:
+            if (coords[0] == val) k++;
+            break;
+         case 2:
+            if (coords[1] == val) k++;
+            break;   
+         case 3:
+            if (coords[2] == val) k++;
+            break;      
+         }
+      }
+      int nrvertices = mesh_orig->GetNV() + k;
+      int nrelements = mesh_orig->GetNE() + pow(pow(k,1.0/(dim-1))-1.0,dim-1);
+
+      mesh_ext = new Mesh(dim, nrvertices, nrelements);
+
+      // Add existing vertices
+      Array<int> vmap(mesh_orig->GetNV()); vmap = 0;
+      k = mesh_orig->GetNV();
+      for (int i=0; i<mesh_orig->GetNV(); ++i)
+      {
+         double * vert = mesh_orig->GetVertex(i);
+         mesh_ext->AddVertex(vert);
+         switch (abs(d))
+         {
+         case 1:
+            if (vert[0] == val) 
+            {
+               vmap[i] = k;
+               k++;
+            }
+            break;
+         case 2:
+            if (vert[1] == val)
+            {
+               vmap[i] = k;
+               k++;
+            }
+            break;   
+         case 3:
+            if (vert[2] == val)
+            {
+               vmap[i] = k;
+               k++;
+            }
+            break;      
+         }
+      } 
+      // Add existing elements
+      for (int i=0; i<mesh_orig->GetNE(); ++i)
+      {
+         Array<int>ind;
+         mesh_orig->GetElementVertices(i,ind);
+         if (dim == 2)
+         {
+            mesh_ext->AddQuad(ind);
+         }
+         else if (dim == 3)
+         {
+            mesh_ext->AddHex(ind);
+         }
+      } 
+      // Add new vertices
+      k = mesh_orig->GetNV();
+      for (int i=0; i<mesh_orig->GetNV(); ++i)
+      {
+         double * vert = mesh_orig->GetVertex(i);
+         switch (abs(d))
+         {
+         case 1:
+            if (vert[0] == val) 
+            {
+               double coords[dim];
+               coords[0] = vert[0] + h;
+               coords[1] = vert[1];
+               if (dim == 3) coords[2] = vert[2];
+               mesh_ext->AddVertex(coords);
+            }
+            break;
+         case 2:
+            if (vert[1] == val)
+            {
+               double coords[dim];
+               coords[0] = vert[0];
+               coords[1] = vert[1] + h;
+               if (dim == 3) coords[2] = vert[2];
+               mesh_ext->AddVertex(coords);
+            }
+            break;   
+         case 3:
+            if (vert[2] == val)
+            {
+               double coords[dim];
+               coords[0] = vert[0];
+               coords[1] = vert[1];
+               coords[2] = vert[2] + h;
+               mesh_ext->AddVertex(coords);
+            }
+            break;      
+         }
+      }    
+      // loop through boundary elements and extend in the given direction
+      for (int i=0; i<mesh_orig->GetNBE(); ++i) 
+      {
+         Array<int> vertices;
+         mesh_orig->GetBdrElementVertices(i,vertices);
+         if (dim == 2)
+         {
+            int ind[4];
+            if (vmap[vertices[0]] && vmap[vertices[1]])
+            {
+               ind[0] = vmap[vertices[0]];
+               ind[1] = vmap[vertices[1]];
+               ind[2] = vertices[1];
+               ind[3] = vertices[0];
+               mesh_ext->AddQuad(ind);
+            }
+         }
+         else if (dim == 3)
+         {
+            int ind[8];
+            if (vmap[vertices[0]] && vmap[vertices[1]] && vmap[vertices[2]] && vmap[vertices[3]])
+            {
+               ind[0] = vmap[vertices[0]];
+               ind[1] = vmap[vertices[1]];
+               ind[2] = vmap[vertices[2]];
+               ind[3] = vmap[vertices[3]];
+               ind[4] = vertices[0];
+               ind[5] = vertices[1];
+               ind[6] = vertices[2];
+               ind[7] = vertices[3];
+               mesh_ext->AddHex(ind);
+            }
+         }   
+      }
+      mesh_ext->FinalizeTopology();
+
+      if (j<directions.Size()-1)
+      {
+         delete mesh_orig;
+         mesh_orig = mesh_ext;
+      }
+   }
+   delete mesh_orig;
+   return mesh_ext;
+}

examples/maxwell-solver/ST/additive_schwarz.hpp

@@ -0,0 +1,133 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+using namespace std;
+using namespace mfem;
+
+
+class OverlappingCartesianMeshPartition 
+{
+private:
+   Mesh *mesh=nullptr;
+public:
+   int nrpatch;
+   int nx, ny, nz;
+   std::vector<Array<int>> element_map;
+   // constructor
+   OverlappingCartesianMeshPartition(Mesh * mesh_);
+   ~OverlappingCartesianMeshPartition() {};
+};
+
+class STPOverlappingCartesianMeshPartition // Special layered partition for STP
+{
+private:
+   Mesh *mesh=nullptr;
+public:
+   int nrpatch;
+   int nx, ny, nz;
+   std::vector<Array<int>> element_map;
+   // constructor
+   STPOverlappingCartesianMeshPartition(Mesh * mesh_);
+   ~STPOverlappingCartesianMeshPartition() {};
+};
+
+
+class CartesianMeshPartition // for now every vertex defines a patch
+{
+private:
+   Mesh *mesh=nullptr;
+public:
+   int nrpatch;
+   int nx, ny, nz;
+   std::vector<Array<int>> element_map;
+   // constructor
+   CartesianMeshPartition(Mesh * mesh_);
+   ~CartesianMeshPartition() {};
+};
+
+class VertexMeshPartition // for now every vertex defines a patch
+{
+private:
+   Mesh *mesh=nullptr;
+public:
+   int nrpatch;
+   // map local (patch) element to global (original mesh) element
+   std::vector<Array<int>> element_map; 
+   // constructor
+   VertexMeshPartition(Mesh * mesh_);
+   ~VertexMeshPartition() {};
+};
+
+class MeshPartition
+{
+private:
+   Mesh *mesh=nullptr;
+   void AddElementToMesh(Mesh * mesh,mfem::Element::Type elem_type,int * ind);
+   void GetNumVertices(int type, mfem::Element::Type & elem_type, int & nrvert);
+   void PrintElementMap();
+public:
+   int nrpatch;
+   int nx, ny, nz;
+   std::vector<Array<int>> element_map;
+   Array<Mesh *> patch_mesh;
+   int partition_kind;
+   // constructor
+   MeshPartition(Mesh * mesh_, int part);
+   ~MeshPartition();
+};
+
+void SaveMeshPartition(Array<Mesh * > meshes, 
+                       string mfilename="output/mesh.",
+                       string sfilename="output/sol.");
+
+
+class PatchAssembly // for now every vertex defines a patch
+{
+   FiniteElementSpace *fespace=nullptr;
+   BilinearForm *bf=nullptr;
+   void print_patch_dof_map();
+public:
+   int nrpatch;
+   Array<FiniteElementSpace *> patch_fespaces;
+   std::vector<Array<int>> patch_dof_map;
+   Array<SparseMatrix *> patch_mat;
+   Array<KLUSolver * > patch_mat_inv;
+   std::vector<Array<int>> ess_tdof_list;
+   std::vector<Array<int>> ess_int_tdofs;
+
+   // constructor
+   PatchAssembly(BilinearForm * bf_, Array<int> & ess_tdofs, int part);
+   ~PatchAssembly();
+};
+
+class AddSchwarz : public Solver//
+{
+private:
+   int nrpatch;
+   int maxit = 1;
+   int part;
+   double theta = 0.5;
+   PatchAssembly * p;
+   const Operator * A;
+public:
+   AddSchwarz(BilinearForm * bf_, Array<int> & ess_tdofs, int i = 0);
+   void SetNumSmoothSteps(const int iter)
+   {
+      maxit = iter;
+   }
+   void SetDumpingParam(const double dump_param)
+   {
+      theta = dump_param;
+   }
+   virtual void SetOperator(const Operator &op)
+   {
+      A = &op;
+   }
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~AddSchwarz();
+};
+
+Mesh * ExtendMesh(Mesh * mesh, const Array<int> & directions);
+
+double GetUniformMeshElementSize(Mesh * mesh);

examples/maxwell-solver/ST/additive_schwarzp.hpp

@@ -0,0 +1,182 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include <unordered_map>
+
+using namespace std;
+using namespace mfem;
+
+
+struct UniqueIndexGenerator
+{
+   int counter = 0;
+   std::unordered_map<int,int> idx;
+   int Get(int i)
+   {
+      std::unordered_map<int,int>::iterator f = idx.find(i);
+      if (f == idx.end())
+      {
+         idx[i] = counter;
+         return counter++;
+      }
+      else
+      {
+         return (*f).second;
+      }
+   }
+   void Reset()
+   {
+      counter = 0;
+      idx.clear();
+   }
+};
+
+
+class CartesianParMeshPartition // for now every vertex defines a patch
+{
+private:
+   ParMesh *pmesh=nullptr;
+public:
+   int nrpatch;
+   Array<int> patch_rank;
+   std::vector<Array<int>> local_element_map;
+   // constructor
+   CartesianParMeshPartition(ParMesh * pmesh_);
+   ~CartesianParMeshPartition() {};
+};
+
+class VertexParMeshPartition
+{
+private:
+   ParMesh *pmesh=nullptr;
+public:
+   int nrpatch;
+   Array<int> patch_rank;
+   std::vector<Array<int>> local_element_map;
+   // constructor
+   VertexParMeshPartition(ParMesh * pmesh_);
+   ~VertexParMeshPartition() {};
+};
+
+class ParMeshPartition
+{
+private:
+   MPI_Comm comm;
+   ParMesh *pmesh=nullptr;
+   void AddElementToMesh(Mesh * mesh,mfem::Element::Type elem_type,int * ind);
+   void GetNumVertices(int type, mfem::Element::Type & elem_type, int & nrvert);
+   void SaveMeshPartition();
+public:
+   int nrpatch;
+   int myelem_offset = 0;
+   Array<int> patch_rank;
+   std::vector<Array<int>> element_map;
+   std::vector<Array<int>> local_element_map;
+   Array<Mesh *> patch_mesh;
+   // constructor
+   ParMeshPartition(ParMesh * pmesh_, int part);
+   ~ParMeshPartition();
+};
+
+class ParPatchDofInfo
+{
+public:
+   MPI_Comm comm = MPI_COMM_WORLD;
+   int nrpatch;
+   Array<int> patch_rank;
+   vector<Array<int>> PatchGlobalTrueDofs; // list of all the true dofs in a patch
+   vector<Array<int>> PatchTrueDofs; // list of only
+   Array<FiniteElementSpace *> patch_fespaces;
+   std::vector<Array<int>> patch_dof_map;
+   ParMeshPartition * p;
+   // constructor
+   ParPatchDofInfo(ParFiniteElementSpace *fespace, int part);
+   // void Print();
+   ~ParPatchDofInfo();
+};
+
+
+
+class ParPatchAssembly // for now every vertex defines a patch
+{
+private:
+   std::vector<int> tdof_offsets;
+   ParBilinearForm *bf=nullptr;
+   void compute_trueoffsets();
+   void AssemblePatchMatrices(ParPatchDofInfo * p);
+   void print_patch_dof_map() {};
+public:
+   MPI_Comm comm;
+   int nrpatch;
+   ParFiniteElementSpace *fespace=nullptr;
+   Array<int> patch_rank;
+   std::vector<Array<int>> patch_true_dofs;
+   std::vector<Array<int>> patch_local_dofs;
+
+   Array<SparseMatrix *> patch_mat;
+   Array<BilinearForm * > patch_bilinear_forms;
+   Array<KLUSolver * > patch_mat_inv;
+   std::vector<Array<int>> ess_tdof_list;
+
+   // constructor
+   ParPatchAssembly(ParBilinearForm * bf_, int part);
+   int get_rank(int tdof);
+   ~ParPatchAssembly();
+};
+
+
+class ParPatchRestriction
+{
+private:
+   MPI_Comm comm;
+   int num_procs, myid;
+   Array<int> patch_rank;
+   ParPatchAssembly * P;
+   int nrpatch;
+   Array<int> send_count;
+   Array<int> send_displ;
+   Array<int> recv_count;
+   Array<int> recv_displ;
+   int sbuff_size, rbuff_size;
+public:
+   ParPatchRestriction(ParPatchAssembly * P_);
+   // void Mult(const Vector & r , Array<BlockVector *> & res);
+   void Mult(const Vector & r , std::vector<Vector  > & res);
+   // void MultTranspose(const Array<BlockVector*> & sol, Vector & z);
+   void MultTranspose(const std::vector<Vector  > & sol, Vector & z);
+   virtual ~ParPatchRestriction() {}
+};
+
+
+class ParAddSchwarz : public Solver//
+{
+private:
+   MPI_Comm comm;
+   int nrpatch;
+   int part;
+   int maxit = 1;
+   double theta = 0.5;
+   ParPatchAssembly * p;
+   const Operator * A;
+   ParPatchRestriction * R;
+public:
+   ParAddSchwarz(ParBilinearForm * bf_, int i = 0);
+
+   void SetNumSmoothSteps(const int iter)
+   {
+      maxit = iter;
+   }
+   void SetDumpingParam(const double dump_param)
+   {
+      theta = dump_param;
+   }
+   virtual void SetOperator(const Operator &op)
+   {
+      A = &op;
+   }
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~ParAddSchwarz();
+};
+
+

examples/maxwell-solver/ST/complex_additive_schwarz.cpp

@@ -0,0 +1,348 @@
+#include "complex_additive_schwarz.hpp"
+
+ComplexPatchAssembly::ComplexPatchAssembly(SesquilinearForm * bf_, Array<int> & ess_tdofs, int part) : bf(bf_)
+{
+   fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   int dim = mesh->Dimension();
+   const FiniteElementCollection *fec = fespace->FEColl();
+
+   // list of dofs to distiguish between interior/boundary and essential
+   Array<int> global_tdofs(fespace->GetTrueVSize());
+   Array<int> bdr_tdofs(fespace->GetTrueVSize());
+   global_tdofs = 0;
+   // Mark boundary dofs and ess_dofs
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, bdr_tdofs);
+   }
+
+   // mark boundary dofs
+   for (int i = 0; i<bdr_tdofs.Size(); i++) global_tdofs[bdr_tdofs[i]] = 1;
+   // overwrite flag for essential dofs
+   for (int i = 0; i<ess_tdofs.Size(); i++) global_tdofs[ess_tdofs[i]] = 0;
+
+
+   MeshPartition * p = new MeshPartition(mesh, part);
+   nx = p->nx;
+   ny = p->ny;
+   nz = p->nz;
+   // SaveMeshPartition(p->patch_mesh);
+   nrpatch = p->nrpatch;
+   patch_fespaces.SetSize(nrpatch);
+   patch_meshes_ext.SetSize(nrpatch);
+   patch_fespaces_ext.SetSize(nrpatch);
+   dof2extdof_map.resize(nrpatch);
+   patch_dof_map.resize(nrpatch);
+   patch_mat.SetSize(nrpatch);
+   patch_mat_ext.SetSize(nrpatch);
+   patch_mat_inv.SetSize(nrpatch);
+   patch_mat_inv_ext.SetSize(nrpatch);
+   ess_tdof_list.resize(nrpatch);
+
+   // construct extended meshes
+   int ip = -1;
+   int nrlayers = 0;
+   if (!part)
+   {
+      for (int ip = 0; ip<nrpatch; ip++)
+      {
+         patch_meshes_ext[ip] = new Mesh(*p->patch_mesh[ip]);
+      }
+   }
+   else
+   {   
+      for (int kz = 0; kz<nz; kz++)
+      {
+         for (int ky = 0; ky<ny; ky++)
+         {
+            for (int kx = 0; kx<nx; kx++)
+            {
+               ip++;
+               Array<int> ext_directions;
+               for (int j=0; j<nrlayers; ++j)
+               {
+                  for (int comp=0; comp<dim; ++comp)
+                  {
+                     if (comp == 0 && kx != 0)
+                     {
+                        ext_directions.Append(-comp-1);
+                     }
+                     if (comp == 0 && kx != nx-1)
+                     {
+                        ext_directions.Append(comp+1);
+                     }
+                     if (comp == 1 && ky != 0)
+                     {
+                        ext_directions.Append(-comp-1);
+                     }
+                     if (comp == 1 && ky != ny-1)
+                     {
+                        ext_directions.Append(comp+1);
+                     }
+                     if (comp == 2 && kz != 0)
+                     {
+                        ext_directions.Append(-comp-1);
+                     }
+                     if (comp == 2 && kz != ny-1)
+                     {
+                        ext_directions.Append(comp+1);
+                     }
+                  }
+               }
+               patch_meshes_ext[ip] = ExtendMesh(p->patch_mesh[ip],ext_directions);
+            }
+         }
+      }
+   }
+
+   
+   // SaveMeshPartition(patch_meshes_ext, "output/ext_mesh.", "output/ext_sol.");
+
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // create finite element spaces for each patch // This might be avoided
+      patch_fespaces[ip] = new FiniteElementSpace(p->patch_mesh[ip],fec);
+      // create finite element spaces on the extented (PML) meshes
+      patch_fespaces_ext[ip] = new FiniteElementSpace(patch_meshes_ext[ip],fec);
+
+      // construct the patch tdof to global tdof map
+      int nrdof = patch_fespaces[ip]->GetTrueVSize();
+      patch_dof_map[ip].SetSize(2*nrdof);
+      dof2extdof_map[ip].SetSize(2*nrdof);
+
+      // build dof maps between patch and extended patch
+      //loop through the patch elements and constract the dof map
+      // The same elements in the extended mesh have the same ordering (but not the dofs)
+
+      // loop through the elements in the patch
+      for (int iel = 0; iel<p->element_map[ip].Size(); ++iel)
+      {
+         // index in the global mesh
+         int iel_idx = p->element_map[ip][iel];
+         // get the dofs of this element
+         Array<int> patch_elem_dofs;
+         Array<int> patch_elem_dofs_ext;
+         Array<int> global_elem_dofs;
+         patch_fespaces[ip]->GetElementDofs(iel,patch_elem_dofs);
+         patch_fespaces_ext[ip]->GetElementDofs(iel,patch_elem_dofs_ext);
+         fespace->GetElementDofs(iel_idx,global_elem_dofs);
+         // the sizes have to match
+         MFEM_VERIFY(patch_elem_dofs.Size() == global_elem_dofs.Size(),
+                     "Size inconsistency");
+         MFEM_VERIFY(patch_elem_dofs.Size() == patch_elem_dofs_ext.Size(),
+                     "Size inconsistency");            
+         // loop through the dofs and take into account the signs;
+         int ndof = patch_elem_dofs.Size();
+         for (int i = 0; i<ndof; ++i)
+         {
+            int pdof_ = patch_elem_dofs[i];
+            int gdof_ = global_elem_dofs[i];
+            int extdof_ = patch_elem_dofs_ext[i];
+            int pdof = (pdof_ >= 0) ? pdof_ : abs(pdof_) - 1;
+            int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
+            int extdof = (extdof_ >= 0) ? extdof_ : abs(extdof_) - 1;
+            patch_dof_map[ip][pdof] = gdof;
+            patch_dof_map[ip][pdof+nrdof] = gdof+fespace->GetTrueVSize();
+            dof2extdof_map[ip][pdof] = extdof;
+            dof2extdof_map[ip][pdof+nrdof] = extdof+patch_fespaces_ext[ip]->GetTrueVSize();
+         }
+      }
+      // Define the patch bilinear form and apply boundary conditions (only the LHS)
+      Array <int> ess_temp_list;
+      if (p->patch_mesh[ip]->bdr_attributes.Size())
+      {
+         Array<int> ess_bdr(p->patch_mesh[ip]->bdr_attributes.Max());
+         ess_bdr = 0;
+         patch_fespaces[ip]->GetEssentialTrueDofs(ess_bdr, ess_temp_list);
+      }
+
+      Array <int> ess_list_ext;
+      if (patch_meshes_ext[ip]->bdr_attributes.Size())
+      {
+         Array<int> ess_bdr(patch_meshes_ext[ip]->bdr_attributes.Max());
+         ess_bdr = 0;
+         patch_fespaces_ext[ip]->GetEssentialTrueDofs(ess_bdr, ess_list_ext);
+      }
+
+      // Adjust the essential tdof list for each patch
+      for (int i=0; i<ess_temp_list.Size(); i++)
+      {
+         int ldof = ess_temp_list[i];
+         int tdof = patch_dof_map[ip][ldof];
+         // check the kind of this tdof
+         if (!global_tdofs[tdof]) ess_tdof_list[ip].Append(ldof);
+      }
+
+      SesquilinearForm a(patch_fespaces[ip], &bf->real(), &bf->imag());
+      SesquilinearForm a_ext(patch_fespaces_ext[ip], &bf->real(), &bf->imag());
+      
+      a.Assemble();
+      a_ext.Assemble();
+      OperatorPtr Alocal;
+      a.FormSystemMatrix(ess_tdof_list[ip],Alocal);
+      ComplexSparseMatrix * AZ = Alocal.As<ComplexSparseMatrix>();
+      patch_mat[ip] = AZ->GetSystemMatrix();
+      patch_mat[ip]->Threshold(0.0);
+      // Save the inverse
+      patch_mat_inv[ip] = new KLUSolver;
+      patch_mat_inv[ip]->SetOperator(*patch_mat[ip]);
+
+
+      OperatorPtr Alocal_ext;
+      a_ext.FormSystemMatrix(ess_list_ext,Alocal_ext);
+      ComplexSparseMatrix * AZ_ext = Alocal_ext.As<ComplexSparseMatrix>();
+      patch_mat_ext[ip] = AZ_ext->GetSystemMatrix();
+      patch_mat_ext[ip]->Threshold(0.0);
+      patch_mat_inv_ext[ip] = new KLUSolver;
+      patch_mat_inv_ext[ip]->SetOperator(*patch_mat_ext[ip]);
+
+
+      delete patch_fespaces[ip];
+      delete patch_fespaces_ext[ip];
+   }
+   delete p;
+}
+
+ComplexPatchAssembly::~ComplexPatchAssembly()
+{
+   for (int ip=0; ip<nrpatch; ++ip)
+   {
+      // delete patch_fespaces[ip]; patch_fespaces[ip]=nullptr;
+      delete patch_meshes_ext[ip];
+      patch_meshes_ext[ip]=nullptr;
+      delete patch_mat_inv[ip];
+      patch_mat_inv[ip]=nullptr;
+      delete patch_mat[ip];
+      patch_mat[ip]=nullptr;
+   }
+   patch_fespaces.DeleteAll();
+   patch_meshes_ext.DeleteAll();
+   patch_mat.DeleteAll();
+   patch_mat_inv.DeleteAll();
+}
+
+
+
+ComplexAddSchwarz::ComplexAddSchwarz(SesquilinearForm * bf_, Array<int> & ess_tdofs, int i)
+   : Solver(2*bf_->FESpace()->GetTrueVSize(), 2*bf_->FESpace()->GetTrueVSize()), bf(bf_),
+     part(i)
+{
+   p = new ComplexPatchAssembly(bf_, ess_tdofs, part);
+   nrpatch = p->nrpatch;
+}
+
+void  ComplexAddSchwarz::Mult(const Vector &r, Vector &z) const
+{
+   z = 0.0;
+   Vector rnew(r);
+   Vector znew(z);
+   Vector raux(znew.Size());
+   Vector res_local, sol_local;
+   Array<int> visit(znew.Size());
+   // char vishost[] = "localhost";
+   // int  visport   = 19916;
+
+   // socketstream sol_sock(vishost, visport);
+   // sol_sock.precision(8);
+   for (int iter = 0; iter < maxit; iter++)
+   {
+      znew = 0.0;
+      visit = 0;
+      for (int ip = 0; ip < nrpatch; ip++)
+      {
+         Array<int> * dof_map = &p->patch_dof_map[ip];
+         int ndofs = dof_map->Size();
+         res_local.SetSize(ndofs);
+         sol_local.SetSize(ndofs);
+
+         rnew.GetSubVector(*dof_map, res_local);
+
+         //-----------------------------------------------
+         // Extend by zero to the extended mesh
+         int nrdof_ext = p->patch_mat_ext[ip]->Height();
+         
+         Vector res_ext(nrdof_ext); res_ext = 0.0;
+         Vector sol_ext(nrdof_ext); sol_ext = 0.0;
+
+         res_ext.SetSubVector(p->dof2extdof_map[ip],res_local.GetData());
+
+         p->patch_mat_inv_ext[ip]->Mult(res_ext, sol_ext);
+
+         sol_ext.GetSubVector(p->dof2extdof_map[ip],sol_local);
+
+
+         //-----------------------------------------------
+
+         // p->patch_mat_inv[ip]->Mult(res_local, sol_local);
+
+         // for the overlapping case
+         // zero out the entries corresponding to the ess_bdr
+         Array<int> ess_bdr_indices_re = p->ess_tdof_list[ip]; // real part
+         Array<int> ess_bdr_indices(2*ess_bdr_indices_re.Size()); //imag part
+
+         for (int i = 0; i< ess_bdr_indices_re.Size(); i++)
+         {
+            ess_bdr_indices[i] = ess_bdr_indices_re[i];
+            ess_bdr_indices[i+ess_bdr_indices_re.Size()] = ess_bdr_indices_re[i]+ndofs/2;
+         }
+         if (!part) 
+         { 
+            sol_local.SetSubVector(ess_bdr_indices,0.0); 
+         }
+         if (type == 1) znew = 0.0;
+
+         znew.AddElementVector(*dof_map,sol_local);
+         // zero out the contributions to the dofs which are already updated
+         if (type == 1)
+         {
+            for (int i = 0; i<ndofs; i++)
+            {
+               int j = (*dof_map)[i];
+               if (visit[j])
+               {
+                  znew(j) = 0.0;
+               }
+               else
+               {
+                  visit[j] = 1;
+               }
+            }
+            z.Add(theta, znew);
+            A->Mult(znew, raux);
+            rnew -= raux;
+         }
+         // PlotSolution(z, sol_sock, ip); cin.get();
+      }
+      if (type == 0)
+      {
+         z.Add(theta, znew);
+         A->Mult(znew, raux);
+         rnew -= raux;
+      }
+      // Update residual
+      if (iter + 1 < maxit)
+      {
+         A->Mult(znew, raux);
+         rnew -= raux;
+      }
+   }
+   // PlotSolution(z, sol_sock, 0); cin.get();
+}
+
+
+void ComplexAddSchwarz::PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const
+{
+   FiniteElementSpace * fespace = bf->FESpace();
+   Mesh * mesh = fespace->GetMesh();
+   ComplexGridFunction gf(fespace);
+   bf->RecoverFEMSolution(sol,B,gf);
+   
+   string keys;
+   if (ip == 0) keys = "keys mrRljc\n";
+   sol_sock << "solution\n" << *mesh << gf.real() << keys << flush;
+}
+
+ComplexAddSchwarz::~ComplexAddSchwarz(){ delete p;}

examples/maxwell-solver/ST/complex_additive_schwarz.hpp

@@ -0,0 +1,58 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "additive_schwarz.hpp"
+#include "pml.hpp"
+using namespace std;
+using namespace mfem;
+
+class ComplexPatchAssembly
+{
+   FiniteElementSpace *fespace=nullptr;
+   SesquilinearForm *bf=nullptr;
+public:
+   int nrpatch, nx, ny, nz;
+   Array<FiniteElementSpace *> patch_fespaces;
+   Array<FiniteElementSpace *> patch_fespaces_ext;
+   Array<Mesh *> patch_meshes_ext;
+   std::vector<Array<int>> patch_dof_map;
+   std::vector<Array<int>> complex_patch_dof_map;
+   std::vector<Array<int>> dof2extdof_map;
+   Array<SparseMatrix *> patch_mat;
+   Array<SparseMatrix *> patch_mat_ext;
+   Array<KLUSolver * > patch_mat_inv_ext;
+   Array<KLUSolver * > patch_mat_inv;
+   std::vector<Array<int>> ess_tdof_list;
+
+   // constructor
+   ComplexPatchAssembly(SesquilinearForm * bf_, Array<int> & ess_tdofs, int part);
+   ~ComplexPatchAssembly();
+};
+
+
+class ComplexAddSchwarz : public Solver//
+{
+private:
+   int nrpatch;
+   int maxit = 1;
+   SesquilinearForm *bf=nullptr;
+   int part;
+   int type = 0;
+   double theta = 0.5;
+   ComplexPatchAssembly * p;
+   const Operator * A;
+   Vector B;
+   void PlotSolution(Vector & sol, socketstream & sol_sock, int ip) const;
+
+   
+public:
+   ComplexAddSchwarz(SesquilinearForm * bf_, Array<int> & ess_tdofs, int i = 0);
+   void SetNumSmoothSteps(const int iter) { maxit = iter;}
+   void SetLoadVector(Vector load) { B = load;}
+   void SetSmoothType(int itype) { type = itype;}
+   void SetDumpingParam(const double dump_param) {theta = dump_param;}
+   virtual void SetOperator(const Operator &op) {A = &op;}
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~ComplexAddSchwarz();
+};

examples/maxwell-solver/ST/complex_additive_schwarzp.cpp

@@ -0,0 +1,88 @@
+#include "complex_additive_schwarzp.hpp"
+
+// constructor
+ComplexParPatchAssembly::ComplexParPatchAssembly(ParSesquilinearForm * bf_) :
+   bf(bf_)
+{
+
+
+
+}
+
+
+void ComplexParPatchAssembly::AssemblePatchMatrices(ParPatchDofInfo * p)
+{
+   // patch_mat.SetSize(nrpatch);
+   // patch_bilinear_forms.SetSize(nrpatch);
+   // patch_mat_inv.SetSize(nrpatch);
+   // ess_tdof_list.resize(nrpatch);
+   // for (int ip=0; ip<nrpatch; ++ip)
+   // {
+   //    patch_bilinear_forms[ip] = nullptr;
+   //    patch_mat_inv[ip] = nullptr;
+   //    patch_mat[ip] = nullptr;
+   //    if (p->p->patch_mesh[ip])
+   //    {
+   //       // Define the patch bilinear form and apply boundary conditions (only the LHS)
+   //       FiniteElementSpace * patch_fespace = p->patch_fespaces[ip];
+   //       Mesh * patch_mesh = p->p->patch_mesh[ip];
+   //       if (patch_mesh->bdr_attributes.Size())
+   //       {
+   //          Array<int> ess_bdr(patch_mesh->bdr_attributes.Max());
+   //          ess_bdr = 1;
+   //          patch_fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list[ip]);
+   //       }
+   //       patch_bilinear_forms[ip] = new SesquilinearForm(patch_fespace, bf);
+   //       patch_bilinear_forms[ip]->Assemble();
+   //       OperatorPtr Alocal;
+   //       // need to add the method FormSystemMatrix to complex_fem
+   //       // patch_bilinear_forms[ip]->FormSystemMatrix(ess_tdof_list[ip],Alocal);
+   //       patch_mat[ip] = &(SparseMatrix&)(*Alocal);
+   //       patch_mat[ip]->Threshold(0.0);
+   //       // Save the inverse
+   //       patch_mat_inv[ip] = new KLUSolver;
+   //       patch_mat_inv[ip]->SetOperator(*patch_mat[ip]);
+   //    }
+   // }
+}
+
+
+ComplexParPatchAssembly::~ComplexParPatchAssembly() {};
+
+
+ComplexParPatchRestriction::ComplexParPatchRestriction(ComplexParPatchAssembly *
+                                                       P_)
+{}
+
+void ComplexParPatchRestriction::Mult(const Vector & r ,
+                                      std::vector<Vector > & res)
+{}
+
+
+void ComplexParPatchRestriction::MultTranspose(const std::vector<Vector > & sol,
+                                               Vector & z)
+{}
+
+
+ComplexParAddSchwarz::ComplexParAddSchwarz(ParSesquilinearForm * pbf_)
+   : Solver(2*pbf_->ParFESpace()->GetTrueVSize(),
+            2*pbf_->ParFESpace()->GetTrueVSize())
+{
+   // cout << "Testing ComplexParAddSchwarz" << endl;
+   // comm = pbf_->ParFESpace()->GetComm();
+   // p = new ComplexParPatchAssembly(pbf_);
+   // nrpatch = p->nrpatch;
+   // R = new ComplexParPatchRestriction(p);
+}
+
+void ComplexParAddSchwarz::Mult(const Vector &r, Vector &z) const
+{}
+
+ComplexParAddSchwarz::~ComplexParAddSchwarz()
+{
+   // delete p;
+   // delete R;
+}
+
+
+

examples/maxwell-solver/ST/complex_additive_schwarzp.hpp

@@ -0,0 +1,87 @@
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "additive_schwarzp.hpp"
+using namespace std;
+using namespace mfem;
+
+
+
+class ComplexParPatchAssembly
+{
+   // std::vector<int> tdof_offsets;
+   ParSesquilinearForm * bf=nullptr;
+   void compute_trueoffsets();
+   void AssemblePatchMatrices(ParPatchDofInfo * p);
+public:
+   // MPI_Comm comm;
+   // int nrpatch;
+   // ParFiniteElementSpace *fespace=nullptr;
+   // Array<int> patch_rank;
+   // std::vector<Array<int>> patch_true_dofs;
+   // std::vector<Array<int>> patch_local_dofs;
+   // Array<SparseMatrix *> patch_mat;
+   // Array<SesquilinearForm * > patch_bilinear_forms;
+   // Array<KLUSolver * > patch_mat_inv;
+   // std::vector<Array<int>> ess_tdof_list;
+
+   // constructor
+   ComplexParPatchAssembly(ParSesquilinearForm * bf_);
+   int get_rank(int tdof);
+   ~ComplexParPatchAssembly();
+};
+
+
+class ComplexParPatchRestriction
+{
+private:
+   // MPI_Comm comm;
+   // int num_procs, myid;
+   // Array<int> patch_rank;
+   // ParPatchAssembly * P;
+   // int nrpatch;
+   // Array<int> send_count;
+   // Array<int> send_displ;
+   // Array<int> recv_count;
+   // Array<int> recv_displ;
+   // int sbuff_size, rbuff_size;
+public:
+   ComplexParPatchRestriction(ComplexParPatchAssembly * P_);
+   void Mult(const Vector & r , std::vector<Vector > & res);
+   void MultTranspose(const std::vector<Vector > & sol, Vector & z);
+   virtual ~ComplexParPatchRestriction() {}
+};
+
+
+class ComplexParAddSchwarz : public Solver//
+{
+private:
+   // MPI_Comm comm;
+   // int nrpatch;
+   // int maxit = 1;
+   // double theta = 0.5;
+   // FiniteElementSpace *fespace=nullptr;
+   // ComplexParPatchAssembly * p;
+   // const Operator * A;
+   // ParSesquilinearForm * pbf;
+   // ComplexParPatchRestriction * R;
+public:
+   ComplexParAddSchwarz(ParSesquilinearForm * pbf_);
+   void SetNumSmoothSteps(const int iter)
+   {
+      // maxit = iter;
+   }
+   void SetDumpingParam(const double dump_param)
+   {
+      // theta = dump_param;
+   }
+   virtual void SetOperator(const Operator &op)
+   {
+      // A = &op;
+   }
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~ComplexParAddSchwarz();
+};
+
+

examples/maxwell-solver/ST/example1.cpp

@@ -0,0 +1,150 @@
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "additive_schwarz.hpp"
+#include "schwarz.hpp"
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/star.mesh";
+   // const char *mesh_file = "../../../data/beam-quad.mesh";
+   int order = 1;
+   int ref_levels = 1;
+   bool visualization = true;
+   StopWatch chrono;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&ref_levels, "-ref", "--ref_levels",
+                  "Number of uniform h-refinements");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+
+   Mesh *mesh;
+   // mesh = new Mesh(mesh_file, 1, 1);
+   mesh = new Mesh(1, 1, Element::QUADRILATERAL, true, 1, 1, false);
+   int dim = mesh->Dimension();
+
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   // FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   FiniteElementSpace * fespace = new FiniteElementSpace(mesh, fec);
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (mesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
+   //    the basis functions in the finite element fespace.
+   LinearForm *b = new LinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   // 8. Define the solution vector x as a finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   GridFunction x(fespace);
+   x = 1.0;
+
+   // 9. Set up the bilinear form a(.,.) on the finite element space
+   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //    domain integrator.
+   BilinearForm *a = new BilinearForm(fespace);
+   a->SetDiagonalPolicy(mfem::Matrix::DIAG_ONE);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 10. Assemble the bilinear form and the corresponding linear system,
+   //     applying any necessary transformations such as: eliminating boundary
+   //     conditions, applying conforming constraints for non-conforming AMR,
+   //     static condensation, etc.
+   a->Assemble();
+
+   OperatorPtr A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   cout << "Size of linear system: " << A->Height() << endl;
+
+   AddSchwarz * prec = new AddSchwarz(a,ess_tdof_list, 0);
+   prec->SetOperator((SparseMatrix&)(*A));
+   prec->SetNumSmoothSteps(1);
+   prec->SetDumpingParam(0.5);
+
+   SchwarzSmoother * prec2 = new SchwarzSmoother(mesh,0,fespace,&(SparseMatrix&)(*A),ess_bdr);
+   prec2->SetNumSmoothSteps(1);
+   prec2->SetDumpingParam(0.5);
+
+
+   int maxit = 2000;
+   double rtol = 1e-8;
+   double atol = 1e-8;
+   Vector X0(X);
+   CGSolver pcg;
+   pcg.iterative_mode = false;
+   pcg.SetPrintLevel(1);
+   pcg.SetMaxIter(maxit);
+   pcg.SetRelTol(rtol);
+   pcg.SetAbsTol(atol);
+   pcg.SetPreconditioner(*prec);
+   pcg.SetOperator((SparseMatrix&)(*A));
+   pcg.Mult(B, X0);
+
+   X0 = X;
+   pcg.SetPreconditioner(*prec2);
+   pcg.Mult(B, X0);
+
+   // 12. Recover the solution as a finite element grid function.
+   a->RecoverFEMSolution(X0, *b, x);
+
+   // 14. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream mesh_sock(vishost, visport);
+      mesh_sock.precision(8);
+      mesh_sock << "mesh\n" << *mesh << flush;
+
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *mesh << x << "keys rRjmc" << flush;
+   }
+
+   // 15. Free the used memory.
+   delete prec;
+   delete a;
+   delete b;
+   delete fespace;
+   delete fec;
+   delete mesh;
+
+   return 0;
+}

examples/maxwell-solver/ST/example1p.cpp

@@ -0,0 +1,177 @@
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "additive_schwarzp.hpp"
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   StopWatch chrono;
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../../data/star.mesh";
+   int order = 1;
+   int ref_levels = 1;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&ref_levels, "-ref", "--ref_levels",
+                  "Number of uniform h-refinements");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+
+   // Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   Mesh * mesh = new Mesh(1, 1, Element::QUADRILATERAL, true, 1, 1, false);
+   // Mesh * mesh = new Mesh(1, 1,1, Element::HEXAHEDRON, true, 1, 1, 1, false);
+   int dim = mesh->Dimension();
+
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   // FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ParLinearForm *b = new ParLinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   ParGridFunction x(fespace);
+   x = 0.0;
+
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   chrono.Clear();
+   chrono.Start();
+   a->Assemble();
+
+   HypreParMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+   chrono.Stop();
+   if (myid == 0) { cout << "Form Linear System time: " << chrono.RealTime() << endl; }
+
+   // Array<int>elem_vertices;
+   // for (int iel = 0; iel<pmesh->GetNE(); iel++)
+   // {
+   //    pmesh->GetElementVertices(iel,elem_vertices);
+   //    cout << "myid, iel: " << myid <<", " << iel << ", " ; elem_vertices.Print(cout,10);
+   // }
+
+   Array<double> times(4);
+
+   chrono.Clear();
+   chrono.Start();
+   ParAddSchwarz *prec = new ParAddSchwarz(a,0);
+   prec->SetOperator(A);
+   prec->SetNumSmoothSteps(1);
+   prec->SetDumpingParam(0.5);
+
+   chrono.Stop();
+   times[0] = chrono.RealTime();
+
+   int maxit = 200;
+   double rtol = 1e-8;
+   double atol = 1e-8;
+   X = 0.0;
+   CGSolver pcg(MPI_COMM_WORLD);
+   pcg.SetPrintLevel(1);
+   pcg.SetMaxIter(maxit);
+   pcg.SetRelTol(rtol);
+   pcg.SetAbsTol(atol);
+   pcg.SetPreconditioner(*prec);
+   pcg.SetOperator(A);
+
+   chrono.Clear();
+   chrono.Start();
+   pcg.Mult(B, X);
+   chrono.Stop();
+   times[1] = chrono.RealTime();
+   delete prec;
+
+   if (myid == 0)
+   {
+      cout << "prec construction time: " << times[0] << endl;
+      cout << "PCG solution time:      " << times[1] << endl;
+   }
+
+   a->RecoverFEMSolution(X, *b, x);
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      // socketstream mesh_sock(vishost, visport);
+      // mesh_sock << "parallel " << num_procs << " " << myid << "\n";
+      // mesh_sock.precision(8);
+      // mesh_sock << "mesh\n" << *pmesh  << "keys n/n" << flush;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh  << x <<"keys " << flush;
+   }
+
+   // // 17. Free the used memory.
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0)
+   {
+      delete fec;
+   }
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}

examples/maxwell-solver/ST/helmholtz.cpp

@@ -0,0 +1,306 @@
+//
+// Compile with: make helmholtz
+//
+// Sample runs:  helmholtz -m ../data/one-hex.mesh
+//               helmholtz -m ../data/fichera.mesh
+//               helmholtz -m ../data/fichera-mixed.mesh
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Helmholtz problem
+//               -Delta p - omega^2 p = 1 with impedance boundary condition.
+//
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "complex_additive_schwarz.hpp"
+#include "schwarz.hpp"
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution and r.h.s., see below for implementation.
+double f_exact_Re(const Vector &x);
+double f_exact_Im(const Vector &x);
+
+int dim;
+double omega;
+int sol = 1;
+bool pml = false;
+double length = 1.0;
+double pml_length = 0.25;
+bool scatter = false;
+
+#ifndef MFEM_USE_SUPERLU
+#error This example requires that MFEM is built with MFEM_USE_PETSC=YES
+#endif
+
+int main(int argc, char *argv[])
+{
+
+   // 2. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../../data/one-hex.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // static condensation flag
+   bool static_cond = false;
+   bool visualization = 1;
+   // number of wavelengths
+   double k = 0.5;
+   // number of mg levels
+   int ref = 1;
+   // dimension
+   int nd = 2;
+
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nd, "-nd", "--dim","Problem space dimension");
+   args.AddOption(&sol, "-sol", "--exact",
+                  "Exact solution flag - 0:polynomial, 1: plane wave, -1: unknown exact");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&pml, "-pml", "--pml", "-no-pml",
+                  "--no-pml", "Enable PML.");
+   args.AddOption(&pml_length, "-pml_length", "--pml_length",
+                  "Length of the PML region in each direction");
+   args.AddOption(&length, "-length", "--length",
+                  "length of the domainin in each direction.");
+   args.AddOption(&ref, "-ref", "--ref",
+                  "Number of Refinements.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&scatter, "-scat", "--scattering-prob", "-no-scat",
+                  "--no-scattering", "Solve a scattering problem");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+		args.PrintUsage(cout);
+		return 1;
+   }
+	args.PrintOptions(cout);
+   // Angular frequency
+   omega = 2.0 * M_PI * k;
+
+   // 3. Read the mesh from the given mesh file.
+   Mesh *mesh;
+
+   if (nd == 2)
+   {
+      // mesh = new Mesh(mesh_file,1,1);
+      mesh = new Mesh(1, 1, Element::QUADRILATERAL, true, length, length, false);
+   }
+   else
+   {
+      mesh = new Mesh(1, 1, 1, Element::HEXAHEDRON, true, length, length, length,false);
+   }
+
+   // 3. Executing uniform h-refinement
+   for (int i = 0; i < ref; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+   dim = mesh->Dimension();
+
+
+   // 6. Define a finite element space on the mesh.
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+
+   // 6. Set up the linear form (Real and Imaginary part)
+   FunctionCoefficient f_Re(f_exact_Re);
+   FunctionCoefficient f_Im(f_exact_Im);
+
+   // ParLinearForm *b_Re(new ParLinearForm);
+   ComplexLinearForm b(fespace, ComplexOperator::HERMITIAN);
+   b.AddDomainIntegrator(new DomainLFIntegrator(f_Re),
+                         new DomainLFIntegrator(f_Im));
+   b.real().Vector::operator=(0.0);
+   b.imag().Vector::operator=(0.0);
+   b.Assemble();
+
+   // 7. Set up the bilinear form (Real and Imaginary part)
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+   SesquilinearForm a(fespace,ComplexOperator::HERMITIAN);
+   ConstantCoefficient impedance(omega);
+
+
+   Array<int> bdr_attr(mesh->bdr_attributes.Max());
+   bdr_attr = 1;
+   RestrictedCoefficient imp_rest(impedance,bdr_attr);
+   a.AddDomainIntegrator(new DiffusionIntegrator(one),NULL);
+   a.AddDomainIntegrator(new MassIntegrator(sigma),NULL);
+   a.AddBoundaryIntegrator(NULL,new BoundaryMassIntegrator(imp_rest));
+   a.Assemble();
+   a.Finalize();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(mesh->bdr_attributes.Max());
+   ess_bdr = 0;
+   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   // Solution grid function
+   ComplexGridFunction p_gf(fespace);
+
+   OperatorHandle Ah;
+   Vector X, B;
+
+   a.FormLinearSystem(ess_tdof_list, p_gf, b, Ah, X, B);
+
+   ComplexSparseMatrix * AZ = Ah.As<ComplexSparseMatrix>();
+   SparseMatrix * A = AZ->GetSystemMatrix();
+
+
+      cout << "Size of fine grid system: "
+           << A->Height() << " x " << A->Width() << endl;
+
+
+
+
+   ComplexAddSchwarz S(&a,ess_tdof_list, 1);
+	S.SetOperator(*A);
+   S.SetSmoothType(0);
+   S.SetLoadVector(B);
+   // S.SetNumSmoothSteps(7);
+   S.SetDumpingParam(1.0);
+
+	BlkSchwarzSmoother * BlkS = new BlkSchwarzSmoother(mesh,0,fespace,A);
+
+	X = 0.0;
+	GMRESSolver gmres;
+	gmres.SetPreconditioner(*BlkS);
+	gmres.SetOperator(*A);
+	gmres.SetRelTol(1e-4);
+	gmres.SetMaxIter(500);
+	gmres.SetPrintLevel(1);
+	gmres.Mult(B, X);
+
+
+	X = 0.0;
+	gmres.SetPreconditioner(S);
+	gmres.Mult(B, X);
+
+   KLUSolver klu(*A);
+   klu.Mult(B,X);
+
+
+   a.RecoverFEMSolution(X,B,p_gf);
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      string keys;
+      if (dim ==2 )
+      {
+         keys = "keys mrRljc\n";
+      }
+      else
+      {
+         keys = "keys mc\n";
+      }
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re.precision(8);
+      sol_sock_re << "solution\n" << *mesh << p_gf.real() <<
+                  "window_title 'Numerical Pressure (real part): (KLU solver)' "
+                  << keys << flush;
+   }
+
+   delete fespace;
+   delete fec;
+	delete mesh;
+   return 0;
+}
+
+
+//calculate RHS from exact solution f = - \Delta u
+double f_exact_Re(const Vector &x)
+{
+   double f_re = 0.0;
+   double x0 = length/2.0;
+   double x1 = length/2.0;
+   double x2 = length/2.0;
+   x0 = 0.1;
+   x1 = 0.1;
+   double alpha,beta;
+   double n = 5.0 * omega/M_PI;
+   double coeff = pow(n,2)/M_PI;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   alpha = -pow(n,2) * beta;
+   f_re = coeff*exp(alpha);
+
+   // x0 = 0.9;
+   // x1 = 0.9;
+   // n = 5.0 * omega/M_PI;
+   // coeff = pow(n,2)/M_PI;
+   // beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   // if (dim == 3) { beta += pow(x2-x(2),2); }
+   // alpha = -pow(n,2) * beta;
+   // f_re += coeff*exp(alpha);
+   
+   // x0 = 0.9;
+   // x1 = 0.1;
+   // n = 5.0 * omega/M_PI;
+   // coeff = pow(n,2)/M_PI;
+   // beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   // if (dim == 3) { beta += pow(x2-x(2),2); }
+   // alpha = -pow(n,2) * beta;
+   // f_re += coeff*exp(alpha);
+
+   // x0 = 0.1;
+   // x1 = 0.9;
+   // n = 5.0 * omega/M_PI;
+   // coeff = pow(n,2)/M_PI;
+   // beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   // if (dim == 3) { beta += pow(x2-x(2),2); }
+   // alpha = -pow(n,2) * beta;
+   // f_re += coeff*exp(alpha);
+
+   return f_re;
+}
+double f_exact_Im(const Vector &x)
+{
+   double f_im;
+   f_im = 0.0;
+   return f_im;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

examples/maxwell-solver/ST/helmholtz_pml.cpp

@@ -0,0 +1,368 @@
+//
+// Compile with: make helmholtz
+//
+// Sample runs:  helmholtz -m ../data/one-hex.mesh
+//               helmholtz -m ../data/fichera.mesh
+//               helmholtz -m ../data/fichera-mixed.mesh
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Helmholtz problem
+//               -Delta p - omega^2 p = 1 with impedance boundary condition.
+//
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "pml.hpp"
+#include "LSweepsPrecond.hpp"
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution and r.h.s., see below for implementation.
+double f_exact_Re(const Vector &x);
+double f_exact_Im(const Vector &x);
+
+
+
+int dim;
+double omega;
+int sol = 1;
+bool pml = false;
+double length = 1.0;
+double pml_length = 0.25;
+bool scatter = false;
+Array2D<double>comp_bdr;
+
+
+#ifndef MFEM_USE_SUPERLU
+#error This example requires that MFEM is built with MFEM_USE_PETSC=YES
+#endif
+
+int main(int argc, char *argv[])
+{
+
+   // 2. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../../data/one-hex.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // static condensation flag
+   bool static_cond = false;
+   bool visualization = 1;
+   // number of wavelengths
+   double k = 0.5;
+   // number of mg levels
+   int ref = 1;
+   // dimension
+   int nd = 2;
+
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nd, "-nd", "--dim","Problem space dimension");
+   args.AddOption(&sol, "-sol", "--exact",
+                  "Exact solution flag - 0:polynomial, 1: plane wave, -1: unknown exact");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&pml, "-pml", "--pml", "-no-pml",
+                  "--no-pml", "Enable PML.");
+   args.AddOption(&pml_length, "-pml_length", "--pml_length",
+                  "Length of the PML region in each direction");
+   args.AddOption(&length, "-length", "--length",
+                  "length of the domainin in each direction.");
+   args.AddOption(&ref, "-ref", "--ref",
+                  "Number of Refinements.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&scatter, "-scat", "--scattering-prob", "-no-scat",
+                  "--no-scattering", "Solve a scattering problem");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+		args.PrintUsage(cout);
+		return 1;
+   }
+	args.PrintOptions(cout);
+   // Angular frequency
+   omega = 2.0 * M_PI * k;
+
+   // 3. Read the mesh from the given mesh file.
+   Mesh *mesh;
+
+   if (nd == 2)
+   {
+      // mesh = new Mesh(mesh_file,1,1);
+      mesh = new Mesh(1, 1, Element::QUADRILATERAL, true, length, length, false);
+   }
+   else
+   {
+      mesh = new Mesh(1, 1, 1, Element::HEXAHEDRON, true, length, length, length,false);
+   }
+
+   // 3. Executing uniform h-refinement
+   for (int i = 0; i < ref; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+   dim = mesh->Dimension();
+
+   Array<int> directions;
+   int nrlayers = 4;
+   
+   for (int i = 0; i<nrlayers; i++)
+   {
+      for (int comp=0; comp<dim; ++comp)
+      {
+         directions.Append(comp+1);
+         directions.Append(-comp-1);
+      }
+   }
+   // Find uniform h size of the original mesh
+   double h = GetUniformMeshElementSize(mesh);
+   cout << "pml length = " << h*nrlayers << endl;
+   Mesh *mesh_ext = ExtendMesh(mesh,directions);
+
+   Array2D<double> lengths(dim,2);
+   lengths = h*nrlayers;
+   CartesianPML pml(mesh_ext,lengths);
+   pml.SetOmega(omega);
+   comp_bdr.SetSize(dim,2);
+   comp_bdr = pml.GetCompDomainBdr(); 
+
+   // 6. Define a finite element space on the mesh.
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh_ext, fec);
+
+   // 6. Set up the linear form (Real and Imaginary part)
+   FunctionCoefficient f_Re(f_exact_Re);
+   FunctionCoefficient f_Im(f_exact_Im);
+
+   // ParLinearForm *b_Re(new ParLinearForm);
+   ComplexLinearForm b(fespace, ComplexOperator::HERMITIAN);
+   b.AddDomainIntegrator(new DomainLFIntegrator(f_Re),
+                         new DomainLFIntegrator(f_Im));
+   b.real().Vector::operator=(0.0);
+   b.imag().Vector::operator=(0.0);
+   b.Assemble();
+
+   // 7. Set up the bilinear form (Real and Imaginary part)
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+
+
+
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+   ProductCoefficient c2_re(sigma, detJ_re);
+   ProductCoefficient c2_im(sigma, detJ_im);
+
+   SesquilinearForm a(fespace,ComplexOperator::HERMITIAN);
+
+   a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   a.AddDomainIntegrator(new MassIntegrator(c2_re),new MassIntegrator(c2_im));
+
+   a.Assemble();
+   a.Finalize();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(mesh_ext->bdr_attributes.Max());
+   ess_bdr = 1;
+   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   // Solution grid function
+   ComplexGridFunction p_gf(fespace);
+
+   OperatorHandle Ah;
+   Vector X, B;
+
+   a.FormLinearSystem(ess_tdof_list, p_gf, b, Ah, X, B);
+
+   ComplexSparseMatrix * AZ = Ah.As<ComplexSparseMatrix>();
+   SparseMatrix * A = AZ->GetSystemMatrix();
+
+
+   cout << "Size of fine grid system: "
+         << A->Height() << " x " << A->Width() << endl;
+
+
+   LSweepsPrecond S(&a,ess_tdof_list, omega,nrlayers, 1);
+	S.SetOperator(*A);
+   S.SetSmoothType(1);
+   S.SetLoadVector(B);
+   S.SetDumpingParam(1.0);
+   
+   // X = 0.0;
+	// GMRESSolver gmres;
+	// gmres.SetPreconditioner(S);
+	// gmres.SetOperator(*A);
+	// gmres.SetRelTol(1e-8);
+	// gmres.SetMaxIter(500);
+	// gmres.SetPrintLevel(1);
+	// gmres.Mult(B, X);
+
+
+
+
+
+   X = 0.0;
+   Vector z(X.Size()); z = 0.0;
+   Vector r(B);
+   // r = B;
+   Vector ztemp(r.Size());
+
+   int n= 1;
+
+   Vector Ax(X.Size());
+   for (int i = 0; i<n; i++)
+   {
+      A->Mult(X,Ax); Ax *=-1.0;
+      r = b; r+=Ax;
+      // A->AddMult(X,r,-1.0); //r = r-Ax
+      cout << "residual norm =" << r.Norml2() << endl;
+      // S.Mult(r,z); 
+      S.Mult(r,z); 
+      cout << "correction norm =" << z.Norml2() << endl;
+
+      X += z;
+
+      cout << "solution norm =" << X.Norml2() << endl;
+
+      p_gf = 0.0;
+      a.RecoverFEMSolution(X,B,p_gf);
+
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         string keys;
+         if (dim ==2 )
+         {
+            keys = "keys mrRljc\n";
+         }
+         else
+         {
+            keys = "keys mc\n";
+         }
+         socketstream sol_sock_re(vishost, visport);
+         sol_sock_re.precision(8);
+         sol_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+                     "window_title 'Numerical Pressure (real part)' "
+                     << keys << flush;
+      cout << "Iteration " << i << endl;
+      cin.get();
+   }
+
+
+
+   KLUSolver klu(*A);
+   klu.Mult(B,X);
+   ComplexGridFunction p_gf1(fespace);
+
+   a.RecoverFEMSolution(X,B,p_gf1);
+
+   p_gf1 -= p_gf;
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      string keys;
+      if (dim ==2 )
+      {
+         keys = "keys mrRljc\n";
+      }
+      else
+      {
+         keys = "keys mc\n";
+      }
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re.precision(8);
+      sol_sock_re << "solution\n" << *mesh_ext << p_gf1.real() <<
+                  "window_title 'Numerical Pressure (real part from KLU)' "
+                  << keys << flush;
+   }
+
+   delete fespace;
+   delete fec;
+	delete mesh_ext;
+	delete mesh;
+   return 0;
+}
+
+
+//calculate RHS from exact solution f = - \Delta u
+double f_exact_Re(const Vector &x)
+{
+   double f_re = 0.0;
+   double x0 = length/2.0;
+   double x1 = length/2.0;
+   double x2 = length/2.0;
+   x0 = 0.0;
+   x1 = 0.0;
+   double alpha,beta;
+   double n = 5.0 * omega/M_PI;
+   double coeff = pow(n,2)/M_PI;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   alpha = -pow(n,2) * beta;
+   f_re = coeff*exp(alpha);
+
+   bool in_pml = false;
+   for (int i = 0; i<dim; i++)
+   {
+      if (x(i)<=comp_bdr(i,0) || x(i)>=comp_bdr(i,1))
+      {
+         in_pml = true;
+         break;
+      }
+   }
+   if (in_pml) f_re = 0.0;
+
+   return f_re;
+
+}
+double f_exact_Im(const Vector &x)
+{
+   double f_im;
+   f_im = 0.0;
+   return f_im;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

examples/maxwell-solver/ST/helmholtz_pml_ST.cpp

@@ -0,0 +1,448 @@
+//
+// Compile with: make helmholtz
+//
+// Sample runs:  helmholtz -m ../data/one-hex.mesh
+//               helmholtz -m ../data/fichera.mesh
+//               helmholtz -m ../data/fichera-mixed.mesh
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Helmholtz problem
+//               -Delta p - omega^2 p = 1 with impedance boundary condition.
+//
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "pml.hpp"
+// #include "PST.hpp"
+#include "ST.hpp"
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution and r.h.s., see below for implementation.
+double f_exact_Re(const Vector &x);
+double f_exact_Im(const Vector &x);
+
+double wavespeed(const Vector &x);
+
+
+int dim;
+double omega;
+int sol = 1;
+bool pml = false;
+double length = 1.0;
+double pml_length = 0.25;
+bool scatter = false;
+Array2D<double>comp_bdr;
+
+#ifndef MFEM_USE_SUPERLU
+#error This example requires that MFEM is built with MFEM_USE_PETSC=YES
+#endif
+
+int main(int argc, char *argv[])
+{
+
+   // 2. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../../data/one-hex.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // static condensation flag
+   bool static_cond = false;
+   bool visualization = 1;
+   // number of wavelengths
+   double k = 0.5;
+   // number of mg levels
+   int ref = 1;
+   // dimension
+   int nd = 2;
+
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nd, "-nd", "--dim","Problem space dimension");
+   args.AddOption(&sol, "-sol", "--exact",
+                  "Exact solution flag - 0:polynomial, 1: plane wave, -1: unknown exact");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&pml, "-pml", "--pml", "-no-pml",
+                  "--no-pml", "Enable PML.");
+   args.AddOption(&pml_length, "-pml_length", "--pml_length",
+                  "Length of the PML region in each direction");
+   args.AddOption(&length, "-length", "--length",
+                  "length of the domainin in each direction.");
+   args.AddOption(&ref, "-ref", "--ref",
+                  "Number of Refinements.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&scatter, "-scat", "--scattering-prob", "-no-scat",
+                  "--no-scattering", "Solve a scattering problem");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+		args.PrintUsage(cout);
+		return 1;
+   }
+	args.PrintOptions(cout);
+   // Angular frequency
+   omega = 2.0 * M_PI * k;
+
+   // 3. Read the mesh from the given mesh file.
+   Mesh *mesh;
+
+   if (nd == 2)
+   {
+      // mesh = new Mesh(mesh_file,1,1);
+      mesh = new Mesh(1, 1, Element::QUADRILATERAL, true, length, length, false);
+   }
+   else
+   {
+      mesh = new Mesh(1, 1, 1, Element::HEXAHEDRON, true, length, length, length,false);
+   }
+
+   // 3. Executing uniform h-refinement
+   for (int i = 0; i < ref; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+   dim = mesh->Dimension();
+
+   double hl = GetUniformMeshElementSize(mesh);
+   Vector pmin, pmax;
+   mesh->GetBoundingBox(pmin,pmax);
+   double domain_length = pmax[0] - pmin[0];
+   double pml_thickness = 0.25/domain_length;
+   int nrlayers = pml_thickness/hl;
+   // int nrlayers = 4;
+   Array<int> directions;
+   
+   for (int i = 0; i<nrlayers; i++)
+   {
+      for (int comp=0; comp<dim; ++comp)
+      {
+         directions.Append(comp+1);
+         directions.Append(-comp-1);
+      }
+   }
+   // Find uniform h size of the original mesh
+   cout << "pml layers = " << nrlayers << endl;
+   cout << "pml length = " << hl*nrlayers << endl;
+   Mesh *mesh_ext = ExtendMesh(mesh,directions);
+
+
+   Array2D<double> lengths(dim,2);
+   lengths = hl*nrlayers;
+   // lengths[0][1] = 0.0;
+   // lengths[1][1] = 0.0;
+   // lengths[1][0] = 0.0;
+   // lengths[0][0] = 0.0;
+   CartesianPML pml(mesh_ext,lengths);
+   pml.SetOmega(omega);
+   comp_bdr.SetSize(dim,2);
+   comp_bdr = pml.GetCompDomainBdr(); 
+
+   // 6. Define a finite element space on the mesh.
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh_ext, fec);
+
+   // 6. Set up the linear form (Real and Imaginary part)
+   FunctionCoefficient f_Re(f_exact_Re);
+   FunctionCoefficient f_Im(f_exact_Im);
+
+   // ParLinearForm *b_Re(new ParLinearForm);
+   ComplexLinearForm b(fespace, ComplexOperator::HERMITIAN);
+   b.AddDomainIntegrator(new DomainLFIntegrator(f_Re),
+                         new DomainLFIntegrator(f_Im));
+   b.real().Vector::operator=(0.0);
+   b.imag().Vector::operator=(0.0);
+   b.Assemble();
+
+   // 7. Set up the bilinear form (Real and Imaginary part)
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+   FunctionCoefficient ws(wavespeed);
+
+   PmlMatrixCoefficient c1_re(dim,pml_detJ_JT_J_inv_Re,&pml);
+   PmlMatrixCoefficient c1_im(dim,pml_detJ_JT_J_inv_Im,&pml);
+
+   PmlCoefficient detJ_re(pml_detJ_Re,&pml);
+   PmlCoefficient detJ_im(pml_detJ_Im,&pml);
+
+   ProductCoefficient c2_re0(sigma, detJ_re);
+   ProductCoefficient c2_im0(sigma, detJ_im);
+
+   ProductCoefficient c2_re(c2_re0, ws);
+   ProductCoefficient c2_im(c2_im0, ws);
+
+
+   SesquilinearForm a(fespace,ComplexOperator::HERMITIAN);
+
+   a.AddDomainIntegrator(new DiffusionIntegrator(c1_re),
+                         new DiffusionIntegrator(c1_im));
+   a.AddDomainIntegrator(new MassIntegrator(c2_re),new MassIntegrator(c2_im));
+
+   a.Assemble();
+   a.Finalize();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(mesh_ext->bdr_attributes.Max());
+   ess_bdr = 1;
+   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   // Solution grid function
+   ComplexGridFunction p_gf(fespace);
+
+   OperatorHandle Ah;
+   Vector X, B;
+
+   a.FormLinearSystem(ess_tdof_list, p_gf, b, Ah, X, B);
+
+   ComplexSparseMatrix * AZ = Ah.As<ComplexSparseMatrix>();
+   SparseMatrix * A = AZ->GetSystemMatrix();
+
+
+   cout << "Size of fine grid system: "
+         << A->Height() << " x " << A->Width() << endl;
+
+
+   // PSTP S(&a,lengths, omega, &ws, nrlayers);
+   STP S(&a,lengths, omega, &ws, nrlayers);
+	S.SetOperator(*A);
+   // S.SetLoadVector(B);
+   
+
+
+
+   X = 0.0;
+	GMRESSolver gmres;
+	gmres.SetPreconditioner(S);
+	gmres.SetOperator(*A);
+	gmres.SetRelTol(1e-8);
+	gmres.SetMaxIter(50);
+	gmres.SetPrintLevel(1);
+	gmres.Mult(B, X);
+
+
+
+
+   int n= 50;
+   X = 0.0;
+   Vector z(X.Size()); z = 0.0;
+   Vector r(B);
+   Vector ztemp(r.Size());
+   Vector Ax(X.Size());
+   double tol = 1e-8;
+   cout << endl;
+   
+   for (int i = 0; i<n; i++)
+   {
+      A->Mult(X,Ax); Ax *=-1.0;
+      r = b; r+=Ax;
+      cout << "   ST Solver   Iteration :   " << i <<"  || r || = " <<  r.Norml2() << endl;
+      if (r.Norml2() < tol) 
+      {
+         // cout << "Convergence in " << i+1 << " iterations" << endl;
+         break;
+      }
+      S.Mult(r,z); 
+      X += z;
+      // p_gf = 0.0;
+      // a.RecoverFEMSolution(X,B,p_gf);
+      //    char vishost[] = "localhost";
+      //    int  visport   = 19916;
+      //    string keys;
+      //    if (dim ==2 )
+      //    {
+      //       keys = "keys mrRljc\n";
+      //    }
+      //    else
+      //    {
+      //       keys = "keys mc\n";
+      //    }
+      //    socketstream sol1_sock_re(vishost, visport);
+      //    sol1_sock_re.precision(8);
+      //    sol1_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+      //                "window_title 'Numerical Pressure (real part)' "
+      //                << keys << flush;
+   }
+
+   // KLUSolver klu(*A);
+   // klu.Mult(B,X);
+   a.RecoverFEMSolution(X,B,p_gf);
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      string keys;
+      if (dim ==2 )
+      {
+         keys = "keys mrRljc\n";
+      }
+      else
+      {
+         keys = "keys mc\n";
+      }
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re.precision(8);
+      sol_sock_re << "solution\n" << *mesh_ext << p_gf.real() <<
+                  "window_title 'Numerical Pressure (real part from KLU)' "
+                  << keys << flush;
+      // socketstream diff_sock_re(vishost, visport);
+      // diff_sock_re.precision(8);
+      // diff_sock_re << "solution\n" << *mesh_ext << p_gf1.real() <<
+      //             "window_title 'Numerical Pressure (real part from KLU)' "
+      //             << keys << flush;
+
+                                  
+   }
+   delete fespace;
+   delete fec;
+	delete mesh_ext;
+	delete mesh;
+   return 0;
+}
+
+
+//calculate RHS from exact solution f = - \Delta u
+double f_exact_Re(const Vector &x)
+{
+   double f_re = 0.0;
+   double x0 = length/2.0;
+   double x1 = length/2.0;
+   double x2 = length/2.0;
+   x0 = 0.1;
+   x1 = 0.5;
+   double alpha,beta;
+   double n = 5.0*omega/M_PI;
+   // double n = 1.0;
+   double coeff = pow(n,2)/M_PI;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   alpha = -pow(n,2) * beta;
+   f_re = coeff*exp(alpha);
+
+   // x0 = 0.9;
+   // x1 = 0.5;
+   // beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   // if (dim == 3) { beta += pow(x2-x(2),2); }
+   // alpha = -pow(n,2) * beta;
+   // f_re += coeff*exp(alpha);
+
+   // x0 = 0.5;
+   // x1 = 0.8;
+   // beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   // if (dim == 3) { beta += pow(x2-x(2),2); }
+   // alpha = -pow(n,2) * beta;
+   // f_re += coeff*exp(alpha);
+
+   bool in_pml = false;
+   for (int i = 0; i<dim; i++)
+   {
+      if (x(i)<=comp_bdr(i,0) || x(i)>=comp_bdr(i,1))
+      {
+         in_pml = true;
+         break;
+      }
+   }
+   if (in_pml) f_re = 0.0;
+
+   return f_re;
+
+}
+double f_exact_Im(const Vector &x)
+{
+   double f_im;
+   f_im = 0.0;
+   return f_im;
+}
+
+double wavespeed(const Vector &x)
+{
+   double ws;
+   // if (x(0) <= 0.25)
+   // {
+   //    ws = 1.0;
+   // }
+   // else if(x(0)<=0.5)
+   // {
+   //    ws = 1.0;
+   // }
+   // else if(x(0)<=0.75)
+   // {
+   //    ws = 0.75;
+   //    // ws = 0.5;
+   // }
+   // else 
+   // {
+   //    ws = 0.75;
+   //    // ws = 1.0;
+   // }
+   // if (x(1) <= 1.0/3.0)
+   // {
+   //    ws = 2.0;
+   // }
+   // else if(x(1)<=2.0/3.0)
+   // {
+   //    ws = 1.0;
+   // }
+   // else 
+   // {
+   //    // ws = 0.75;
+   //    ws = 0.25;
+   // }
+
+   // if (x(0) <= 0.33)
+   // {
+   //    ws = 1.0;
+   // }
+   // else if(x(0)<=0.66)
+   // {
+   //    ws = -0.65 + 5.0*x(0);
+   // }
+   // else 
+   // {
+   //    ws = 2.65;
+   //    // ws = 0.5;
+   // }
+
+
+   ws = 1.0;
+   return ws;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

examples/maxwell-solver/ST/helmholtzp.cpp

@@ -0,0 +1,317 @@
+//
+// Compile with: make helmholtz
+//
+// Sample runs:  helmholtz -m ../data/one-hex.mesh
+//               helmholtz -m ../data/fichera.mesh
+//               helmholtz -m ../data/fichera-mixed.mesh
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Helmholtz problem
+//               -Delta p - omega^2 p = 1 with impedance boundary condition.
+//
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "complex_additive_schwarzp.hpp"
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution and r.h.s., see below for implementation.
+double f_exact_Re(const Vector &x);
+double f_exact_Im(const Vector &x);
+
+int dim;
+double omega;
+int sol = 1;
+bool pml = false;
+double length = 1.0;
+double pml_length = 0.25;
+bool scatter = false;
+
+#ifndef MFEM_USE_SUPERLU
+#error This example requires that MFEM is built with MFEM_USE_PETSC=YES
+#endif
+
+int main(int argc, char *argv[])
+{
+
+   // 1. Initialise MPI
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);                    // Initialise MPI
+   MPI_Comm_size(MPI_COMM_WORLD,
+                 &num_procs); //total number of processors available
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);      // Determine process identifier
+
+   //-----------------------------------------------------------------------------
+
+   // 2. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../../data/one-hex.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // static condensation flag
+   bool static_cond = false;
+   bool visualization = 1;
+   // number of wavelengths
+   double k = 0.5;
+   // number of mg levels
+   int ref = 1;
+   // number of initial ref
+   int initref = 1;
+   // dimension
+   int nd = 2;
+
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nd, "-nd", "--dim","Problem space dimension");
+   args.AddOption(&sol, "-sol", "--exact",
+                  "Exact solution flag - 0:polynomial, 1: plane wave, -1: unknown exact");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&pml, "-pml", "--pml", "-no-pml",
+                  "--no-pml", "Enable PML.");
+   args.AddOption(&pml_length, "-pml_length", "--pml_length",
+                  "Length of the PML region in each direction");
+   args.AddOption(&length, "-length", "--length",
+                  "length of the domainin in each direction.");
+   args.AddOption(&ref, "-ref", "--ref",
+                  "Number of Refinements.");
+   args.AddOption(&initref, "-initref", "--initref",
+                  "Number of initial refinements.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&scatter, "-scat", "--scattering-prob", "-no-scat",
+                  "--no-scattering", "Solve a scattering problem");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+   // Angular frequency
+   omega = 2.0 * M_PI * k;
+
+   // 3. Read the mesh from the given mesh file.
+   Mesh *mesh;
+
+   if (nd == 2)
+   {
+      mesh = new Mesh(1, 1, Element::QUADRILATERAL, true, length, length, false);
+   }
+   else
+   {
+      mesh = new Mesh(1, 1, 1, Element::HEXAHEDRON, true, length, length, length,
+                      false);
+   }
+
+   // 3. Executing uniform h-refinement
+   for (int i = 0; i < initref; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+   dim = mesh->Dimension();
+
+   // 5. Define a parallel mesh and delete the serial mesh.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   // ----------------------------------------------------------------------------
+
+   for (int i = 0; i < ref; i++)
+   {
+      pmesh->UniformRefinement();
+   }
+
+   // 6. Define a finite element space on the mesh.
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+
+   // 6. Set up the linear form (Real and Imaginary part)
+   FunctionCoefficient f_Re(f_exact_Re);
+   FunctionCoefficient f_Im(f_exact_Im);
+
+   // ParLinearForm *b_Re(new ParLinearForm);
+   ParComplexLinearForm b(fespace, ComplexOperator::HERMITIAN);
+   b.AddDomainIntegrator(new DomainLFIntegrator(f_Re),
+                         new DomainLFIntegrator(f_Im));
+   b.real().Vector::operator=(0.0);
+   b.imag().Vector::operator=(0.0);
+   b.Assemble();
+
+   // 7. Set up the bilinear form (Real and Imaginary part)
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(-pow(omega, 2));
+
+   ParSesquilinearForm * a = new ParSesquilinearForm(fespace,
+                                                     ComplexOperator::HERMITIAN);
+   ConstantCoefficient impedance(omega);
+
+
+   Array<int> bdr_attr(pmesh->bdr_attributes.Max());
+   bdr_attr = 1;
+   RestrictedCoefficient imp_rest(impedance,bdr_attr);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one),NULL);
+   a->AddDomainIntegrator(new MassIntegrator(sigma),NULL);
+   a->AddBoundaryIntegrator(NULL,new BoundaryMassIntegrator(imp_rest));
+   a->Assemble();
+   a->Finalize();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+   ess_bdr = 0;
+   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   // Solution grid function
+   ParComplexGridFunction p_gf(fespace);
+   ParComplexGridFunction p_gf_ex(fespace);
+
+   OperatorHandle Ah;
+   Vector X, B;
+
+   a->FormLinearSystem(ess_tdof_list, p_gf, b, Ah, X, B);
+
+   ComplexHypreParMatrix * AZ = Ah.As<ComplexHypreParMatrix>();
+   HypreParMatrix * A = AZ->GetSystemMatrix();
+
+
+   if (myid == 0)
+   {
+      cout << "Size of fine grid system: "
+           << A->GetGlobalNumRows() << " x " << A->GetGlobalNumCols() << endl;
+   }
+
+   SuperLURowLocMatrix * Arow = new SuperLURowLocMatrix(*A);
+   SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
+   superlu->SetPrintStatistics(false);
+   superlu->SetSymmetricPattern(true);
+   superlu->SetColumnPermutation(superlu::PARMETIS);
+   superlu->SetOperator(*Arow);
+   superlu->Mult(B,X);
+
+   a->RecoverFEMSolution(X,B,p_gf);
+
+
+
+   ComplexParAddSchwarz * test = new ComplexParAddSchwarz(a);
+   delete test;
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      string keys;
+      if (dim ==2 )
+      {
+         keys = "keys mrRljc\n";
+      }
+      else
+      {
+         keys = "keys mc\n";
+      }
+      socketstream sol_sock_re(vishost, visport);
+      sol_sock_re << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_re.precision(8);
+      sol_sock_re << "solution\n" << *pmesh << p_gf.real() <<
+                  "window_title 'Numerical Pressure (real part)' "
+                  << keys << flush;
+   }
+
+   delete a;
+   delete fespace;
+   delete fec;
+   delete pmesh;
+   MPI_Finalize();
+   return 0;
+}
+
+
+//calculate RHS from exact solution f = - \Delta u
+double f_exact_Re(const Vector &x)
+{
+   double f_re = 0.0;
+   double x0 = length/2.0;
+   double x1 = length/2.0;
+   double x2 = length/2.0;
+   double alpha,beta;
+   double n = 5.0 * omega/M_PI;
+   double coeff = pow(n,2)/M_PI;
+   beta = pow(x0-x(0),2) + pow(x1-x(1),2);
+   if (dim == 3) { beta += pow(x2-x(2),2); }
+   alpha = -pow(n,2) * beta;
+   f_re = coeff*exp(alpha);
+   return f_re;
+}
+double f_exact_Im(const Vector &x)
+{
+   double f_im;
+   f_im = 0.0;
+   return f_im;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+// int ndofs = nodes->FESpace()->GetNDofs();
+//    Vector xcoords(ndofs), ycoords(ndofs), zcoords(ndofs);
+
+//    for (int comp = 0; comp < nodes->FESpace()->GetVDim(); comp++)
+//    {
+//       for (int i = 0; i < ndofs; i++)
+//       {
+//          if (comp == 0)
+//          {
+//             xcoords(i) = *nodes[nodes->FESpace()->DofToVDof(i, comp)];
+//          }
+//          else if (comp == 1)
+//          {
+//             ycoords(i) = *nodes[nodes->FESpace()->DofToVDof(i, comp)];
+//          }
+//          else if (comp == 2)
+//          {
+//             zcoords(i) = *nodes[nodes->FESpace()->DofToVDof(i, comp)];
+//          }
+//       }
+//    }