Adding cusparse ILU and incomplete Cholesky solvers on GPU.

.gitignore

@@ -175,7 +175,6 @@ miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
 miniapps/meshing/minimal-surface
 miniapps/meshing/pminimal-surface
-miniapps/meshing/polar-nc
 
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
@@ -188,7 +187,6 @@ miniapps/meshing/extruder.mesh
 miniapps/meshing/trimmer.mesh
 miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
-miniapps/meshing/polar-nc.mesh
 
 miniapps/performance/ex1
 miniapps/performance/ex1p
@@ -235,7 +233,6 @@ miniapps/nurbs/mode_*
 miniapps/nurbs/Example1*
 
 miniapps/gslib/field-diff
-miniapps/gslib/field-interp
 miniapps/gslib/findpts
 miniapps/gslib/pfindpts
 
@@ -262,10 +259,5 @@ tests/scripts/*.err
 tests/scripts/*.out
 tests/scripts/*.msg
 
-# Other tests
-tests/convergence/rates
-tests/convergence/prates
-tests/par-mesh-format/ex1p
-
 # VPATH builds
 build-*/*

.gitlab-ci.yml

@@ -71,8 +71,6 @@ stages:
   - build
   - test
   - deallocate
-  - lassen_build
-  - lassen_test
   - baseline_check
   - baseline_publish
 
@@ -81,11 +79,7 @@ stages:
 # TODO: updating tests and tpls is not necessary anymore since pipelines are
 # now using unique directories so repo are never shared with another pipeline.
 # This is not memory efficient (we keep a lot of data), hence this reminder.
-# Setup
-setup:
-  tags:
-    - shell
-    - quartz
+.setup:
   stage: setup
   variables:
     GIT_STRATEGY: none
@@ -106,15 +100,6 @@ setup:
   before_script:
     - module load gcc/6.1.0
 
-# On lassen
-.with_gcc_8_3_1:
-  variables:
-    TOOLCHAIN: gcc_8_3_1
-    CXX: g++
-    CC: gcc
-  before_script:
-    - module load gcc/8.3.1
-
 .with_gcc_4_9_3:
   variables:
     TOOLCHAIN: gcc_4_9_3
@@ -305,4 +290,3 @@ setup:
 # The list on jobs is defined in machine-specific files.
 include:
   - local: .gitlab/quartz.yml
-  - local: .gitlab/lassen.yml

.gitlab/lassen.yml

@@ -1,57 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Lassen machine at LLNL
-
-.on_lassen:
-  tags:
-    - shell
-    - lassen
-  variables:
-    PLAT: lassen
-
-# Build MFEM
-build_mfem_ser_lassen:
-  extends: [.with_gcc_8_3_1, .on_lassen]
-  needs: [setup]
-  stage: lassen_build
-  script:
-    - mkdir -p ${BUILD_PATH}
-    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
-    - lalloc 1 -W 5 -q pdebug make -j cuda CUDA_ARCH=sm_70
-
-build_mfem_debug_ser_lassen:
-  extends: [.with_gcc_8_3_1, .on_lassen]
-  needs: [setup]
-  stage: lassen_build
-  script:
-    - mkdir -p ${BUILD_PATH}
-    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
-    - lalloc 1 -W 5 -q pdebug make -j cuda MFEM_DEBUG="YES" CUDA_ARCH=sm_70
-
-# Sanity check
-sanitycheck_mfem_ser_lassen:
-  extends: [.with_gcc_8_3_1, .on_lassen]
-  stage: lassen_test
-  needs: [build_mfem_ser_lassen]
-  script:
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
-    - lalloc 1 -W 15 -q pdebug make -j test
-
-sanitycheck_mfem_debug_ser_lassen:
-  extends: [.with_gcc_8_3_1, .on_lassen]
-  stage: lassen_test
-  needs: [build_mfem_debug_ser_lassen]
-  script:
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
-    - lalloc 1 -W 30 -q pdebug make -j test

.gitlab/quartz.yml

@@ -22,6 +22,10 @@
     MAKE_PAR: 6
     BASELINE_PAR: 18
 
+# Setup
+setup_quartz:
+  extends: [.setup, .on_quartz]
+
 # Allocate
 allocate_quartz:
   variables:

CHANGELOG

@@ -38,11 +38,6 @@ Meshing improvements
 - Added complete action of the TMOP Integrator to account for the spatial
   derivatives of discrete and analytic targets.
 
-- Added support for initialization of (serial) non-conforming meshes. Hanging
-  nodes can be marked with Mesh::AddVertexParents when building the mesh with
-  the "init" constructor. The usage is demonstrated in a new meshing miniapp
-  (polar-nc) which generates meshes that are non-conforming from the start.
-
 Performance improvements
 ------------------------
 - Added support for explicit vectorization in the high-performance templated
@@ -67,14 +62,8 @@ Improved GPU capabilities
 
 - Added CUDA support for sparse matrix-vector multiplication with cuSPARSE.
 
-- Added AmgX solver class for simple integration with NVIDIA's multigrid library.
-
 - Added support for BlockOperator on GPU. See the updated Example 5.
 
-- Added partial assembly and GPU support for complex operators, including the
-  classes ComplexOperator, [Par]ComplexGridFunction, [Par]ComplexLinearForm, and
-  [Par]SesquilinearForm. See the updated Example 22.
-
 Discretization improvements
 ---------------------------
 - Added support for matrix-free interpolation and restriction operators between
@@ -103,14 +92,6 @@ Discretization improvements
 
 - Added support face integrals on the boundaries of NURBS meshes.
 
-- Added support for interpolation of functions in L2, H(div) and H(curl)
-  spaces using GSLIB-FindPoints.
-
-- Added support for computing asymptotic error estimates and convergence rates
-  for the whole de Rham sequence based on the new class ConvergenceStudy and new
-  member methods in GridFunction and ParGridFunction. See the rates.cpp file in
-  the tests/convergence directory for sample usage.
-
 Linear and nonlinear solvers
 ----------------------------
 - Added power method to iteratively estimate the largest eigenvalue and the
@@ -136,9 +117,6 @@ Linear and nonlinear solvers
 
 - Added support for the SLEPc eigensolver package.
 
-- Added partially assembled convergent diagonal preconditioner for adaptively
-  refined meshes (i.e. non-conforming finite element spaces), see Example 6/6p.
-
 New and updated examples and miniapps
 -------------------------------------
 - Added a new example, Example 25/25p, to demonstrate the use of a Perfectly
@@ -175,9 +153,6 @@ New and updated examples and miniapps
 - Added a new meshing miniapp, Minimal Surface, which solves Plateau's problem:
   the Dirichlet problem for the minimal surface equation.
 
-- Added a new meshing miniapp, Polar NC, which demonstrates the construction of
-  polar non-conforming meshes.
-
 - Added partial assembly support to Example 4/4p and Example 5/5p, with diagonal
   preconditioning.
 
@@ -192,47 +167,30 @@ New and updated examples and miniapps
   mesh based on element attributes. Any newly exposed boundary elements are
   assigned attribute numbers related to the trimmed element attributes.
 
-- Added a new miniapp (field-interp) that demonstrates transfer of grid function
-  between different meshes using GSLIB-FindPoints.
-
-- Added diagonal preconditioner in Example 6/6p for partial assembly with AMR.
-
 - Added device support in Example 5/5p.
 
-- Added partial assembly and device support to Example 22/22p, with diagonal
-  preconditioning.
-
 - Added the option to plot a function in Mesh Explorer.
 
 Improved testing
 ----------------
-- Upgraded the Catch unit test framework from version 1.6.1 to version 2.13.0.
-
 - Added a GitLab pipeline that automates PR testing on supercomputing systems
   and Linux clusters at Lawrence Livermore National Lab (LLNL). This can be
   triggered only by LLNL developers, see .gitlab-ci.yml, the .gitlab directory
   and the updated CONTRIBUTING.md file.
 
-- Added testing of the parallel mesh format in tests/par-mesh-format.
-
 Miscellaneous
 -------------
 - Added support for ADIOS2 for parallel I/O with ParaView visualization. The
   classes adios2stream and ADIOS2DataCollection are introduced in mfem as the
   interfaces to generate ADIOS2 Binary Pack (BP4) directory datasets for the
-  entire spatial and temporal node data. Cell centered data is accessible by
-  ADIOS2 data readers (e.g. Python), but currently not yet implement as of
-  ParaView v5.8.1. In addition, ADIOS2 allows for setting a user-defined number
-  of data substreams/subfiles at scale. See examples 5, 9, 12, 16.
+  entire spatial and temporal data. In addition, ADIOS2 allows for setting a
+  user-defined number of data substreams/subfiles. See examples 5, 9, 12, 16.
 
 - The integration order used in the ComputeLpError and ComputeElementLpError
   methods of class GridFunction has been increased.
 
 - Various other simplifications, extensions, and bugfixes in the code.
 
-- Renamed "Backend::DEBUG" to "Backend::DEBUG_DEVICE" to avoid conflicts,
-  as DEBUG is sometimes used as a macro.
-
 
 Version 4.1, released on March 10, 2020
 =======================================

CMakeLists.txt

@@ -295,11 +295,6 @@ if (MFEM_USE_CEED)
   find_package(libCEED REQUIRED)
 endif()
 
-if (MFEM_USE_AMGX)
-  include_directories(${AMGX_DIR}/include)
-  link_directories(${AMGX_DIR}/lib64 -L${AMGX_DIR}/lib -Xlinker=-rpath=${CUDA_HOME}/lib64)
-endif()
-
 if (MFEM_USE_CONDUIT)
     find_package(Conduit REQUIRED conduit relay blueprint )
 endif()
@@ -377,7 +372,7 @@ endif()
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
     SLEPC MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
     MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE AMGX)
+    CUSPARSE)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")

INSTALL

@@ -350,12 +350,6 @@ MFEM_USE_STRUMPACK = YES/NO
    classes. When enabled, this option uses the STRUMPACK_* library options, see
    below.
 
-MFEM_USE_AMGX = YES/NO
-   Enable MFEM functionality based on the AMGX multigrid library from 
-   NVIDIA. When enabled, this options enables the user to use SparseMatrices
-   and HypreParMatrices to solve linear systems using routines from the AMGX
-   library.
-
 MFEM_USE_GINKGO = YES/NO
    Enable MFEM functionality based on the Ginkgo library, which provides
    iterative linear solvers and preconditioners with OpenMP, CUDA backends, see
@@ -656,11 +650,6 @@ The specific libraries and their options are:
   Options: CUDA_CXX, CUDA_ARCH, CUDA_OPT, CUDA_LIB.
   Versions: CUDA >= 9.1, older versions may work too.
 
-- AMGX (optional), used when MFEM_USE_AMGX = YES.
-  URL: https://github.com/NVIDIA/AMGX
-  Options: AMGX_OPT, AMGX_LIB.
-  Versions: AMGX >= 2.1, older versions may work too.
-
 - HIP (optional), used when MFEM_USE_HIP = YES.
   URL: https://rocm.github.io/ROCmInstall.html
   Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB.
@@ -674,7 +663,7 @@ The specific libraries and their options are:
   URL: https://github.com/CEED/libCEED
        https://ceed.exascaleproject.org/libceed
   Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED > 0.6, git-hash bdfed75.
+  Versions: libCEED > 0.6, git-hash fe5822c.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
   Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
@@ -817,7 +806,6 @@ MFEM_USE_MESQUITE
 MFEM_USE_SUITESPARSE
 MFEM_USE_SUPERLU
 MFEM_USE_STRUMPACK
-MFEM_USE_AMGX
 MFEM_USE_GINKGO
 MFEM_USE_GNUTLS
 MFEM_USE_NETCDF

config/cmake/config.hpp.in

@@ -126,9 +126,6 @@
 // Requires a CUDA compiler (nvcc).
 #cmakedefine MFEM_USE_CUDA
 
-// Enable MFEM functionality based on the AMGX library
-#cmakedefine MFEM_USE_AMGX
-
 // Enable MFEM functionality based on the RAJA library
 #cmakedefine MFEM_USE_RAJA
 

config/cmake/modules/FindADIOS2.cmake

@@ -38,19 +38,7 @@ if(NOT ADIOS2_FOUND)
   endif()
 
   find_path(ADIOS2_INCLUDE_DIR adios2.h ${ADIOS2_INCLUDE_OPTS})
-  
-  # adios2 version 2.5.0 
-  find_library(ADIOS2_LIBRARY NAMES adios2 ${ADIOS2_LIBRARY_OPTS})
-  
-  # adios2 version 2.6.0 and onwards
-  if(NOT ADIOS2_LIBRARY)
-  	find_library(ADIOS2_CXX11_MPI_LIBRARY NAMES adios2_cxx11_mpi ${ADIOS2_LIBRARY_OPTS})
-  	find_library(ADIOS2_CXX11_LIBRARY NAMES adios2_cxx11 ${ADIOS2_LIBRARY_OPTS})
-  	set(ADIOS2_LIBRARY ${ADIOS2_CXX11_MPI_LIBRARY} ${ADIOS2_CXX11_LIBRARY})
-  	if(MFEM_USE_MPI)
-  	  add_definitions(-DADIOS2_USE_MPI)
-  	endif()
-  endif() 
+  find_library(ADIOS2_LIBRARY NAMES adios2 ${ADIOS2_LIBRARY_OPTS}) 
 
   include(FindPackageHandleStandardArgs)
   find_package_handle_standard_args(ADIOS2

config/config.hpp.in

@@ -137,9 +137,6 @@
 // Requires a CUDA compiler (nvcc).
 // #define MFEM_USE_CUDA
 
-// Enable MFEM functionality based on the AMGX library.
-// #define MFEM_USE_AMGX
-
 // Build the AMD GPU/HIP-enabled version of the MFEM library.
 // Requires a HIP compiler (hipcc).
 // #define MFEM_USE_HIP

config/config.mk.in

@@ -45,7 +45,6 @@ MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_HIOP          = @MFEM_USE_HIOP@
 MFEM_USE_GSLIB         = @MFEM_USE_GSLIB@
 MFEM_USE_CUDA          = @MFEM_USE_CUDA@
-MFEM_USE_AMGX          = @MFEM_USE_AMGX@
 MFEM_USE_HIP           = @MFEM_USE_HIP@
 MFEM_USE_RAJA          = @MFEM_USE_RAJA@
 MFEM_USE_OCCA          = @MFEM_USE_OCCA@

config/defaults.cmake

@@ -83,11 +83,6 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
-if (MFEM_USE_CUDA)
-   # This is only necessary when hypre is built with cuda:
-   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
-       "Libraries that HYPRE depends on.")
-endif()
 
 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")
 
@@ -187,7 +182,6 @@ set(HIOP_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
-set(AMGX_DIR "${MFEM_DIR}/../amgx" CACHE PATH "Path to AMGX")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
 

config/defaults.mk

@@ -133,7 +133,6 @@ MFEM_USE_PUMI          = NO
 MFEM_USE_HIOP          = NO
 MFEM_USE_GSLIB         = NO
 MFEM_USE_CUDA          = NO
-MFEM_USE_AMGX          = NO
 MFEM_USE_HIP           = NO
 MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
@@ -155,10 +154,6 @@ LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
 HYPRE_DIR = @MFEM_DIR@/../hypre/src/hypre
 HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
-ifeq (YES,$(MFEM_USE_CUDA))
-   # This is only necessary when hypre is built with cuda:
-   HYPRE_LIB += -lcusparse -lcurand
-endif
 
 # METIS library configuration
 ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK),NONO)
@@ -377,11 +372,6 @@ UMPIRE_DIR = @MFEM_DIR@/../umpire
 UMPIRE_OPT = -I$(UMPIRE_DIR)/include
 UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire
 
-# AMGX library configuration
-AMGX_DIR = @MFEM_DIR@/../amgx
-AMGX_OPT = -I$(AMGX_DIR)/include
-AMGX_LIB = -L$(CUDA_HOME)/lib64 -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx -Xlinker=-rpath=$(CUDA_HOME)/lib64
-
 # If YES, enable some informational messages
 VERBOSE = NO
 

config/sample-runs.sh

@@ -78,14 +78,6 @@ groups_parallel=(
    "miniapps/electromagnetics"
    "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
-'"convergence"
-   "Convergence tests:"
-   "tests/convergence"
-   "diffusion.cpp"'
-'"par-mesh-format"
-   "Parallel mesh tests:"
-   "tests/par-mesh-format"
-   "ex1p.cpp"'
 )
 # All groups serial + parallel runs mixed in the same group:
 groups_all=(
@@ -115,14 +107,6 @@ groups_all=(
    "miniapps/electromagnetics"
    "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
-'"convergence"
-   "Convergence tests:"
-   "tests/convergence"
-   "diffusion.cpp"'
-'"par-mesh-format"
-   "Parallel mesh tests:"
-   "tests/par-mesh-format"
-   "ex1p.cpp"'
 )
 make_all="all"
 base_timeformat=$'real: %3Rs  user: %3Us  sys: %3Ss  %%cpu: %P'
@@ -396,15 +380,10 @@ function timed_run()
 # This function is used to execute the sample runs
 function go()
 {
-   # Strip leading and trailing spaces from $1 and store the result in cmd_line
-   shopt -s extglob
-   local cmd_line="${1##+( )}"
-   cmd_line="${cmd_line%%+( )}"
-   shopt -u extglob
-   eval local cmd=(${cmd_line})
+   local cmd=("$@")
    local res=""
    echo $sep
-   echo "<${group}>" "${cmd_line}"
+   echo "<${group}>" "${cmd[@]}"
    echo $sep
    if [ "${timing}" == "yes" ]; then
       timed_run "${cmd[@]}"
@@ -416,15 +395,15 @@ function go()
    else
       res="${red}FAILED${none}"
    fi
-   printf "[${res}] <${group}> ${cmd_line}\n"
+   printf "[${res}] <${group}> ${cmd[*]}\n"
    if [ "${timing}" == "yes" ]; then
       printf "Run time: %s\n" "${timer}"
       timer=(${timer})
       timer="${timer[1]}"
-      printf -v line "[$res](%8s) ${cmd_line}" "$timer"
+      printf -v line "[$res](%8s) ${cmd[*]}" "$timer"
       summary=("${summary[@]}" "$line")
    else
-      summary=("${summary[@]}" "[${res}] ${cmd_line}")
+      summary=("${summary[@]}" "[${res}] ${cmd[*]}")
    fi
    echo $sep
 }
@@ -459,7 +438,7 @@ function go_group()
       fi
       for run in "${runs[@]}"; do
          if [ "${run}" == "" ]; then continue; fi
-         eval go \"\${run_prefix} \${run} \${run_suffix}\" $output
+         eval go \${run_prefix} \${run} \${run_suffix} $output
       done
    done
    ${make} clean-exec
@@ -525,7 +504,7 @@ function echo_run()
 {
    echo "   $@"
    { echo "   $@"; echo "$sep";
-     eval "$@"
+     "$@"
      echo "$sep"; } >> "$echo_log" 2>&1
 }
 
@@ -545,28 +524,6 @@ function build_all()
    echo_run ${make} config ${mfem_config} || exit 1
    echo_run ${make} ${make_j} || exit 1
    echo_run ${make} ${make_all} ${make_j} || exit 1
-   # Build groups in directories other than the directories built by 'make all':
-   for group_params in "${groups[@]}"; do
-      eval params=(${group_params})
-      group_dir="${params[2]}"
-      case "$group_dir" in
-         (examples*|miniapps*)
-            # Built by 'make all'
-            ;;
-         (*)
-            if [ "${mfem_dir}" != "${mfem_build_dir}" ]; then
-               echo_run mkdir -p "${group_dir}" || exit 1
-               echo_run cd "${group_dir}" || exit 1
-               echo_run cp -af "${mfem_dir}/${group_dir}/makefile" . || exit 1
-            else
-               echo_run cd "${group_dir}" || exit 1
-            fi
-            echo_run ${make} clean || exit 1
-            echo_run ${make} MFEM_DIR="${mfem_dir}" ${make_j} || exit 1
-            echo_run cd "${mfem_build_dir}" || exit 1
-            ;;
-      esac
-   done
 }
 
 # Function that runs all sample runs, given by the array variable "groups".

doc/CodeDocumentation.dox

@@ -149,7 +149,6 @@ namespace mfem {
  * - <a class="el" href="toroid_8cpp_source.html">Toroid</a>: generate simple toroidal meshes
  * - <a class="el" href="twist_8cpp_source.html">Twist</a>: generate simple periodic meshes
  * - <a class="el" href="minimal-surface_8cpp_source.html">Minimal Surface</a>: compute minimal surfaces, <a class="el" href="minimal-surface_8cpp_source.html">serial</a> and <a class="el" href="pminimal-surface_8cpp_source.html">parallel</a> versions
- * - <a class="el" href="polar-nc_8cpp_source.html">Polar NC</a>: generate polar non-conforming meshes
  * - <a class="el" href="shaper_8cpp_source.html">Shaper</a>: resolve material interfaces by mesh refinement
  * - <a class="el" href="extruder_8cpp_source.html">Extruder</a>: extrude a low-dimensional mesh into a higher dimension
  * - <a class="el" href="mesh-explorer_8cpp_source.html">Mesh Explorer</a>: visualize and manipulate meshes
@@ -162,7 +161,6 @@ namespace mfem {
  * - <a class="el" href="lor-transfer_8cpp_source.html">LOR Transfer</a>: map functions between high-order and low-order refined spaces
  * - <a class="el" href="findpts_8cpp_source.html">Find Points</a>: evaluate grid function in physical space, <a class="el" href="findpts_8cpp_source.html">serial</a> and <a class="el" href="pfindpts_8cpp_source.html">parallel</a> versions
  * - <a class="el" href="field-diff_8cpp_source.html">Field Diff</a>: compare grid functions on different meshes
- * - <a class="el" href="field-interp_8cpp_source.html">Field Interp</a>: transfer a grid functions betwen meshes
  * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
  * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
  *

examples/amgx/amg_pcg.json

@@ -1,38 +0,0 @@
-{
-    "config_version": 2,
-    "solver": {
-        "preconditioner": {
-            "print_grid_stats": 1,
-            "print_vis_data": 0,
-            "solver": "AMG",
-            "smoother": {
-                "scope": "jacobi",
-                "solver": "BLOCK_JACOBI",
-                "relaxation_factor": 0.7,
-                "monitor_residual": 0,
-                "print_solve_stats": 0
-            },
-            "print_solve_stats": 0,
-            "presweeps": 1,
-            "interpolator": "D2",
-            "max_row_sum" : 0.9,
-            "strength_threshold" : 0.25,
-            "max_iters": 1,
-            "monitor_residual": 0,
-            "store_res_history": 0,
-            "scope": "amg",
-            "max_levels": 100,
-            "cycle": "V",
-            "postsweeps": 1
-        },
-        "solver": "PCG",
-        "print_solve_stats": 1,
-        "obtain_timings": 1,
-        "max_iters": 100,
-        "monitor_residual": 1,
-        "convergence": "RELATIVE_MAX",
-        "scope": "main",
-        "tolerance": 1e-12,
-        "norm": "L2"
-    }
-}

examples/amgx/ex1.cpp

@@ -1,271 +0,0 @@
-//                                MFEM Example 1
-//
-// Compile with: make ex1
-//
-// Sample runs:  ex1 -m ../../data/square-disc.mesh
-//               ex1 -m ../../data/star.mesh
-//               ex1 -m ../../data/star-mixed.mesh
-//               ex1 -m ../../data/escher.mesh
-//               ex1 -m ../../data/fichera.mesh
-//               ex1 -m ../../data/fichera-mixed.mesh
-//               ex1 -m ../../data/toroid-wedge.mesh
-//               ex1 -m ../../data/periodic-annulus-sector.msh
-//               ex1 -m ../../data/periodic-torus-sector.msh
-//               ex1 -m ../../data/square-disc-p2.vtk -o 2
-//               ex1 -m ../../data/square-disc-p3.mesh -o 3
-//               ex1 -m ../../data/square-disc-nurbs.mesh -o -1
-//               ex1 -m ../../data/star-mixed-p2.mesh -o 2
-//               ex1 -m ../../data/disc-nurbs.mesh -o -1
-//               ex1 -m ../../data/pipe-nurbs.mesh -o -1
-//               ex1 -m ../../data/fichera-mixed-p2.mesh -o 2
-//               ex1 -m ../../data/star-surf.mesh
-//               ex1 -m ../../data/square-disc-surf.mesh
-//               ex1 -m ../../data/inline-segment.mesh
-//               ex1 -m ../../data/amr-quad.mesh
-//               ex1 -m ../../data/amr-hex.mesh
-//               ex1 -m ../../data/fichera-amr.mesh
-//               ex1 -m ../../data/mobius-strip.mesh
-//               ex1 -m ../../data/mobius-strip.mesh -o -1 -sc
-//
-// Device sample runs:
-//               ex1 -pa -d cuda
-//               ex1 -pa -d raja-cuda
-//               ex1 -pa -d occa-cuda
-//               ex1 -pa -d raja-omp
-//               ex1 -pa -d occa-omp
-//               ex1 -pa -d ceed-cpu
-//             * ex1 -pa -d ceed-cuda
-//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
-//               ex1 -m ../../data/beam-hex.mesh -pa -d cuda
-//               ex1 -m ../../data/beam-tet.mesh -pa -d ceed-cpu
-//               ex1 -m ../../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
-//
-// AmgX sample runs:
-//               ./ex1 --amgx-file multi_gs.json --amgx-solver
-//               ./ex1 --amgx-file precon.json --amgx-preconditioner
-//               ./ex1 --amgx-file multi_gs.json --amgx-solver -d cuda
-//               ./ex1 --amgx-file precon.json --amgx-preconditioner -d cuda
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   bool amgx_solver = true;
-   const char* amgx_json_file = ""; // jason file for amgx
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&amgx_json_file, "--amgx-file", "--amgx-file",
-                  "AMGX solver config file (overrides --amgx-solver, --amgx-verbose)");
-   args.AddOption(&amgx_solver, "--amgx-solver", "--amgx-solver",
-                  "--amgx-preconditioner",
-                  "--amgx-preconditioner",
-                  "Configure AMGX as solver or preconditioner.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   MFEM_VERIFY(!pa && strcmp(amgx_json_file,"") != 0,
-               "An AmgX json file is needed for this example \n");
-
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 50,000
-   //    elements.
-   {
-      int ref_levels =
-         (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order. If order < 1, we
-   //    instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   bool delete_fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
-   }
-   else if (mesh.GetNodes())
-   {
-      fec = mesh.GetNodes()->OwnFEC();
-      delete_fec = false;
-      cout << "Using isoparametric FEs: " << fec->Name() << endl;
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
-   }
-   FiniteElementSpace fespace(&mesh, fec);
-   cout << "Number of finite element unknowns: "
-        << fespace.GetTrueVSize() << endl;
-
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   LinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 8. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(&fespace);
-   x = 0.0;
-
-   // 9. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   BilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 10. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: eliminating boundary
-   //     conditions, applying conforming constraints for non-conforming AMR,
-   //     static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   cout << "Size of linear system: " << A->Height() << endl;
-
-   // 11. Solve the linear system A X = B.
-   if (!pa)
-   {
-      AmgXSolver amgx;
-
-      amgx.ReadParameters(amgx_json_file, AmgXSolver::EXTERNAL);
-      amgx.InitSerial();
-      amgx.SetOperator(*A.As<SparseMatrix>());
-
-      if (amgx_solver)
-      {
-         amgx.Mult(B,X);
-      }
-      else
-      {
-         PCG(*A.As<SparseMatrix>(), amgx, B, X, 3, 40, 1e-12, 0.0);
-      }
-
-   }
-   else // Jacobi preconditioning in partial assembly mode
-   {
-      if (UsesTensorBasis(fespace))
-      {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
-         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-      }
-      else
-      {
-         CG(*A, B, X, 1, 400, 1e-12, 0.0);
-      }
-   }
-
-   // 12. Recover the solution as a finite element grid function.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 13. Save the refined mesh and the solution. This output can be viewed later
-   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh.Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-   // 14. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << mesh << x << flush;
-   }
-
-   // 15. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
-
-   return 0;
-}

examples/amgx/ex1p.cpp

@@ -1,322 +0,0 @@
-//                       MFEM Example 1 - Parallel Version
-//
-// Compile with: make ex1p
-//
-// Sample runs:  mpirun -np 4 ex1p -m ../../data/square-disc.mesh
-//               mpirun -np 4 ex1p -m ../../data/star.mesh
-//               mpirun -np 4 ex1p -m ../../data/star-mixed.mesh
-//               mpirun -np 4 ex1p -m ../../data/escher.mesh
-//               mpirun -np 4 ex1p -m ../../data/fichera.mesh
-//               mpirun -np 4 ex1p -m ../../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -m ../../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../../data/periodic-annulus-sector.msh
-//               mpirun -np 4 ex1p -m ../../data/periodic-torus-sector.msh
-//               mpirun -np 4 ex1p -m ../../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex1p -m ../../data/square-disc-p3.mesh -o 3
-//               mpirun -np 4 ex1p -m ../../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/star-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../../data/disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex1p -m ../../data/fichera-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../../data/star-surf.mesh
-//               mpirun -np 4 ex1p -m ../../data/square-disc-surf.mesh
-//               mpirun -np 4 ex1p -m ../../data/inline-segment.mesh
-//               mpirun -np 4 ex1p -m ../../data/amr-quad.mesh
-//               mpirun -np 4 ex1p -m ../../data/amr-hex.mesh
-//               mpirun -np 4 ex1p -m ../../data/mobius-strip.mesh
-//               mpirun -np 4 ex1p -m ../../data/mobius-strip.mesh -o -1 -sc
-//
-// Device sample runs:
-//               mpirun -np 4 ex1p -pa -d cuda
-//               mpirun -np 4 ex1p -pa -d occa-cuda
-//               mpirun -np 4 ex1p -pa -d raja-omp
-//               mpirun -np 4 ex1p -pa -d ceed-cpu
-//             * mpirun -np 4 ex1p -pa -d ceed-cuda
-//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
-//               mpirun -np 4 ex1p -m ../../data/beam-tet.mesh -pa -d ceed-cpu
-//
-// AmgX sample runs:
-//
-//               mpirun -n 40 ./ex1p --amgx-file amg_pcg.json
-//               lrun -n 4 ./ex1p --amgx-file amg_pcg.json --amgx-mpi-gpu-exclusive
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   bool amgx_mpi_teams = true;
-   const char* amgx_json_file = ""; // jason file for amgx
-   int ndevices = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&amgx_json_file, "--amgx-file", "--amgx-file",
-                  "AMGX solver config file (overrides --amgx-solver, --amgx-verbose)");
-   args.AddOption(&amgx_mpi_teams, "--amgx-mpi-teams", "--amgx-mpi-teams",
-                  "--amgx-mpi-gpu-exclusive", "--amgx-mpi-gpu-exclusive",
-                  "Create MPI teams when using AMGX.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&ndevices, "-nd","--nd","Number of GPU devices.");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-
-      MFEM_VERIFY(!pa == true && strcmp(amgx_json_file,"") != 0,
-                  "An AmgX json file is needed for this example \n");
-   }
-
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 10,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh.UniformRefinement();
-      }
-   }
-
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   bool delete_fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
-   }
-   else if (pmesh.GetNodes())
-   {
-      fec = pmesh.GetNodes()->OwnFEC();
-      delete_fec = false;
-      if (myid == 0)
-      {
-         cout << "Using isoparametric FEs: " << fec->Name() << endl;
-      }
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
-   }
-   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_Int size = fespace.GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 8. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 9. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 10. Define the solution vector x as a parallel finite element grid function
-   //     corresponding to fespace. Initialize x with initial guess of zero,
-   //     which satisfies the boundary conditions.
-   ParGridFunction x(&fespace);
-   x = 0.0;
-
-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
-   ParBilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 12. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   // 13. Solve the linear system A X = B.
-   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-   //     * With partial assembly, use Jacobi smoothing, for now.
-   Solver *prec = NULL;
-   if (!pa)
-   {
-
-      AmgXSolver amgx;
-
-      amgx.ReadParameters(amgx_json_file, AmgXSolver::EXTERNAL);
-
-      if (amgx_mpi_teams)
-      {
-         //Forms MPI teams to load balance between mpi ranks and gpus
-         amgx.InitMPITeams(MPI_COMM_WORLD, ndevices);
-      }
-      else
-      {
-         //Assumes MPI == number of devices
-         amgx.InitExclusiveGPU(MPI_COMM_WORLD);
-      }
-
-      amgx.SetOperator(*A.As<HypreParMatrix>());
-
-      amgx.Mult(B, X);
-
-   }
-   else
-   {
-
-      if (UsesTensorBasis(fespace))
-      {
-         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
-      }
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(1);
-      if (prec) { cg.SetPreconditioner(*prec); }
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
-
-   }
-
-   // 14. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
-   MPI_Finalize();
-
-   return 0;
-}

examples/amgx/makefile

@@ -1,72 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../..
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/amgx/,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-# Use the MFEM install directory
-# MFEM_INSTALL_DIR = ../../mfem
-# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
--include $(CONFIG_MK)
-
-SEQ_EXAMPLES = ex1
-PAR_EXAMPLES = ex1p
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES)
-
-ifeq ($(MFEM_USE_AMGX),NO)
-$(EXAMPLES):
-	$(error MFEM is not configured with AMGX)
-endif
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-SERIAL_NAME := Serial AMGX example
-PARALLEL_NAME := Parallel AMGX example
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
-%-test-seq: %
-	@$(call mfem-test,$<,, $(SERIAL_NAME))
-
-# Testing: "test" target and mfem-test* variables are defined in config/test.mk
-
-# Generate an error message if the MFEM library is not built and exit
-$(MFEM_LIB_FILE):
-	$(error The MFEM library is not build)
-
-clean: clean-build
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints

examples/amgx/multi_gs.json

@@ -1,24 +0,0 @@
-{
-    "config_version": 2,
-    "solver": {
-        "max_uncolored_percentage": 0.15,
-        "algorithm": "AGGREGATION",
-        "solver": "AMG",
-        "smoother": "MULTICOLOR_GS",
-        "presweeps": 1,
-        "symmetric_GS" : 1,
-        "selector": "SIZE_2",
-        "coarsest_sweeps": 10,
-        "max_iters": 10000,
-        "postsweeps": 1,
-        "scope": "main",
-        "max_levels": 1000,
-        "matrix_coloring_scheme" : "MIN_MAX",
-        "tolerance": 0.0000001,
-        "print_solve_stats": 1,
-        "obtain_timings": 1,
-        "monitor_residual": 1,
-        "norm": "L2",
-        "cycle": "V"
-    }
-}

examples/amgx/precon.json

@@ -1,21 +0,0 @@
-{
-    "config_version": 2,
-    "solver": {
-        "max_uncolored_percentage": 0.15,
-        "algorithm": "AGGREGATION",
-        "solver": "AMG",
-        "smoother": "MULTICOLOR_GS",
-        "presweeps": 1,
-        "symmetric_GS" : 1,
-        "selector": "SIZE_2",
-        "coarsest_sweeps": 10,
-        "max_iters": 2,
-        "postsweeps": 1,
-        "scope": "main",
-        "max_levels": 1000,
-        "matrix_coloring_scheme" : "MIN_MAX",
-        "tolerance": 0.0,
-        "norm": "L2",
-        "cycle": "V"
-    }
-}

examples/ex1.cpp

@@ -71,7 +71,6 @@ int main(int argc, char *argv[])
    bool pa = false;
    const char *device_config = "cpu";
    bool visualization = true;
-   bool amgx = false;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -83,8 +82,6 @@ int main(int argc, char *argv[])
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&amgx, "-amgx", "--amgx-precon", "-no-amgx",
-                  "--no-amgx-precon", "Use AmgX V-cycle as preconditioner for CG.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -178,7 +175,8 @@ int main(int argc, char *argv[])
    //    domain integrator.
    BilinearForm a(&fespace);
    if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   //a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a.AddDomainIntegrator(new MassIntegrator(one));
 
    // 10. Assemble the bilinear form and the corresponding linear system,
    //     applying any necessary transformations such as: eliminating boundary
@@ -187,16 +185,50 @@ int main(int argc, char *argv[])
    if (static_cond) { a.EnableStaticCondensation(); }
    a.Assemble();
 
-   OperatorPtr A;
+   OperatorPtr A, As;
    Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   Array<int> empty_list;
+   a.FormSystemMatrix(empty_list, As);
+   //a.FormLinearSystem(empty_list, x, b, A, X, B);
+   //a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
 
-   cout << "Size of linear system: " << A->Height() << endl;
+   //cout << "Size of linear system: " << A->Height() << endl;
 
    // 11. Solve the linear system A X = B.
-   if (pa)
+   if (!pa)
+   {
+#ifndef MFEM_USE_SUITESPARSE
+      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
+      //GSSmoother M((SparseMatrix&)(*A));
+
+      //SparseMatrix &Asp = *As.As<SparseMatrix>();
+      SparseMatrix &Asp = a.SpMat();
+
+      Asp.Finalize();
+      Asp.SortColumnIndices();
+
+      Vector tmpx(B.Size());
+      Vector tmpy(B.Size());
+      tmpx = 1.0;
+      tmpy = 0.0;
+
+      //As.As<SparseMatrix>()->Mult(tmpx, tmpy);
+      Asp.Mult(tmpx, tmpy);
+
+      //IncompleteCholesky M(*As.As<SparseMatrix>());
+      IncompleteCholesky M(Asp);
+      //ILUcusparse M(*A.As<SparseMatrix>());
+      PCG(*As, M, B, X, 1, 200, 1e-12, 0.0);
+#else
+      // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+      UMFPackSolver umf_solver;
+      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver.SetOperator(*A);
+      umf_solver.Mult(B, X);
+#endif
+   }
+   else // Jacobi preconditioning in partial assembly mode
    {
-      // Jacobi preconditioning in partial assembly mode
       if (UsesTensorBasis(fespace))
       {
          OperatorJacobiSmoother M(a, ess_tdof_list);
@@ -207,31 +239,6 @@ int main(int argc, char *argv[])
          CG(*A, B, X, 1, 400, 1e-12, 0.0);
       }
    }
-   else if (amgx)
-   {
-#if defined(MFEM_USE_AMGX)
-      bool amgx_verbose = false;
-      AmgXSolver amgx(AmgXSolver::PRECONDITIONER, amgx_verbose);
-      amgx.SetOperator(*A.As<SparseMatrix>());
-      PCG(*A, amgx, B, X, 1, 200, 1e-12, 0.0);
-#else
-      mfem_error("MFEM not configured with AMGX \n");
-#endif
-   }
-   else
-   {
-#ifndef MFEM_USE_SUITESPARSE
-      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-      GSSmoother M((SparseMatrix&)(*A));
-      PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
-#else
-      // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-      UMFPackSolver umf_solver;
-      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver.SetOperator(*A);
-      umf_solver.Mult(B, X);
-#endif
-   }
 
    // 12. Recover the solution as a finite element grid function.
    a.RecoverFEMSolution(X, b, x);

examples/ex1p.cpp

@@ -73,7 +73,6 @@ int main(int argc, char *argv[])
    bool pa = false;
    const char *device_config = "cpu";
    bool visualization = true;
-   bool amgx = false;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -85,8 +84,6 @@ int main(int argc, char *argv[])
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&amgx, "-amgx", "--amgx-precon", "-no-amgx",
-                  "--no-amgx-precon", "Use AmgX V-cycle as preconditioner for CG.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -125,7 +122,7 @@ int main(int argc, char *argv[])
    {
       int ref_levels =
          (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
+      for (int l = 0; l < ref_levels-1; l++)
       {
          mesh.UniformRefinement();
       }
@@ -137,7 +134,7 @@ int main(int argc, char *argv[])
    ParMesh pmesh(MPI_COMM_WORLD, mesh);
    mesh.Clear();
    {
-      int par_ref_levels = 2;
+      int par_ref_levels = 1;
       for (int l = 0; l < par_ref_levels; l++)
       {
          pmesh.UniformRefinement();
@@ -219,9 +216,15 @@ int main(int argc, char *argv[])
    Vector B, X;
    a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
 
+   SparseMatrix Asp;
+   A.As<HypreParMatrix>()->GetDiag(Asp);
+   Vector diag;
+
+   StopWatch sw;
+   sw.Start();
+
    // 13. Solve the linear system A X = B.
    //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-   //     * If AmgX is available solve using amg preconditioner.
    //     * With partial assembly, use Jacobi smoothing, for now.
    Solver *prec = NULL;
    if (pa)
@@ -231,19 +234,16 @@ int main(int argc, char *argv[])
          prec = new OperatorJacobiSmoother(a, ess_tdof_list);
       }
    }
-   else if (amgx)
-   {
-#if defined(MFEM_USE_AMGX)
-      bool amgx_verbose = false;
-      prec = new AmgXSolver(MPI_COMM_WORLD, AmgXSolver::PRECONDITIONER,
-                            amgx_verbose);
-#else
-      mfem_error("MFEM not configured with AMGX \n");
-#endif
-   }
    else
    {
-      prec = new HypreBoomerAMG;
+      //prec = new HypreBoomerAMG;
+      Asp.Finalize();
+      Asp.SortColumnIndices();
+
+      Asp.GetDiag(diag);
+      prec = new OperatorJacobiSmoother(diag, ess_tdof_list);
+      //prec = new IncompleteCholesky(Asp);
+      //prec = new ILUcusparse(Asp);
    }
    CGSolver cg(MPI_COMM_WORLD);
    cg.SetRelTol(1e-12);
@@ -254,6 +254,9 @@ int main(int argc, char *argv[])
    cg.Mult(B, X);
    delete prec;
 
+   sw.Stop();
+   cout << "Step 13 solve time " << sw.RealTime() << endl;
+
    // 14. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.
    a.RecoverFEMSolution(X, b, x);

examples/ex1p_helmholtz.cpp

@@ -1,503 +0,0 @@
-//                       MFEM Example 1 - Parallel Version
-//
-// Compile with: make ex1p
-//
-// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
-//               mpirun -np 4 ex1p -m ../data/star.mesh
-//               mpirun -np 4 ex1p -m ../data/star-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/escher.mesh
-//               mpirun -np 4 ex1p -m ../data/fichera.mesh
-//               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
-//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/star-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/fichera-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-double u_exact(const Vector &x);
-double f_exact(const Vector &x);
-
-// #define FORM_DEFINITE
-#define USE_GMRES
-
-#define USE_CSL
-
-#define K2 250.0
-
-int dim;
-double kappa;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../data/beam-tet.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   dim = mesh->Dimension();
-
-   kappa = 2.0 * M_PI;
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 10,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(100000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh->UniformRefinement();
-      }
-   }
-
-   {
-      double minsize = pmesh->GetElementSize(0);
-      double maxsize = minsize;
-      for (int i=1; i<pmesh->GetNE(); ++i)
-      {
-         const double size_i = pmesh->GetElementSize(i);
-         minsize = std::min(minsize, size_i);
-         maxsize = std::max(maxsize, size_i);
-      }
-
-      cout << myid << ": Element size range: (" << minsize << ", " << maxsize << ")"
-           << endl;
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (pmesh->GetNodes())
-   {
-      fec = pmesh->GetNodes()->OwnFEC();
-      if (myid == 0)
-      {
-         cout << "Using isoparametric FEs: " << fec->Name() << endl;
-      }
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh->bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm *b = new ParLinearForm(fespace);
-
-   //ConstantCoefficient bcoef(1.0);
-   FunctionCoefficient bcoef(f_exact);
-
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient zero(0.0);
-   ConstantCoefficient neg(-K2);
-   ConstantCoefficient pos(K2);
-   b->AddDomainIntegrator(new DomainLFIntegrator(bcoef));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   ParGridFunction x(fespace);
-   x = 0.0;
-
-   // 10. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   a->AddDomainIntegrator(new MassIntegrator(neg));
-
-#ifdef FORM_DEFINITE
-   ParBilinearForm *adef = new ParBilinearForm(fespace);
-   adef->AddDomainIntegrator(new DiffusionIntegrator(one));
-   adef->AddDomainIntegrator(new MassIntegrator(pos));
-
-   if (static_cond) { adef->EnableStaticCondensation(); }
-   adef->Assemble();
-
-   ParGridFunction xdef(fespace);
-   xdef = 0.0;
-
-   ParLinearForm *bdef = new ParLinearForm(fespace);
-   bdef->AddDomainIntegrator(new DomainLFIntegrator(bcoef));
-   bdef->Assemble();
-
-   HypreParMatrix Adef;
-   Vector Bdef, Xdef;
-   adef->FormLinearSystem(ess_tdof_list, xdef, *bdef, Adef, Xdef, Bdef);
-#endif
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   HypreParMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   if (myid == 0)
-   {
-      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
-   }
-
-   //A.Print("helmholtz");
-
-   // 12. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
-   //     preconditioner from hypre.
-#ifdef FORM_DEFINITE
-   HypreSolver *amg = new HypreBoomerAMG(Adef);
-#else
-   HypreSolver *amg = new HypreBoomerAMG(A);
-#endif
-
-   const bool fullDirect = true;
-
-   if (fullDirect)
-   {
-#ifdef USE_CSL
-      Vector Bdef, Xdef;
-
-      ParBilinearForm *Mform =  new ParBilinearForm(fespace);
-      Mform->AddDomainIntegrator(new MassIntegrator(pos));
-      Mform->Assemble();
-
-      HypreParMatrix Mmat, Smat, Mcopy;
-      Mform->FormLinearSystem(ess_tdof_list, x, *b, Mmat, Xdef, Bdef);
-      Mform->FormLinearSystem(ess_tdof_list, x, *b, Mcopy, Xdef,
-                              Bdef);  // There must be a better way than creating two identical matrices.
-
-      ParBilinearForm *Sform =  new ParBilinearForm(fespace);
-      Sform->AddDomainIntegrator(new DiffusionIntegrator(one));
-      Sform->Assemble();
-
-      Sform->FormLinearSystem(ess_tdof_list, x, *b, Smat, Xdef, Bdef);
-
-      const double beta1 = 1.0;
-      const double beta2 = 1.0;
-
-      Mmat *= -beta1;
-
-      HypreParMatrix * cslRe = ParAdd(&Smat, &Mmat);
-
-      Mcopy *= beta2;
-
-      ComplexHypreParMatrix chpm(cslRe, &Mcopy, false, false);
-
-      HypreParMatrix *cSysMat = chpm.GetSystemMatrix();
-
-      Array<int> block_trueOffsets(3); // number of variables + 1
-      block_trueOffsets[0] = 0;
-      block_trueOffsets[1] = fespace->TrueVSize();
-      block_trueOffsets[2] = fespace->TrueVSize();
-      block_trueOffsets.PartialSum();
-
-      // Note that B is of true size.
-      BlockVector trueY(block_trueOffsets), trueX(block_trueOffsets),
-                  trueRhs(block_trueOffsets);
-
-      trueRhs.GetBlock(0) = B;
-      trueRhs.GetBlock(1) = 0.0;
-
-      Operator * Arow = new STRUMPACKRowLocMatrix(*cSysMat);
-
-      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-      strumpack->SetPrintFactorStatistics(true);
-      strumpack->SetPrintSolveStatistics(false);
-      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
-      // strumpack->SetSymmetricPattern(true);
-      strumpack->SetOperator(*Arow);
-      strumpack->SetFromCommandLine();
-      //Solver * precond = strumpack;
-
-      // strumpack->Mult(B, X);
-
-      BlockOperator blockDiagA(block_trueOffsets);
-      for (int i=0; i<2; ++i)
-      {
-         blockDiagA.SetDiagonalBlock(i, &A);
-      }
-
-      ProductOperator prod(&blockDiagA, strumpack, false, false);
-      //GMRESSolver *gmres = new GMRESSolver(fespace->GetComm());
-      BiCGSTABSolver *gmres = new BiCGSTABSolver(fespace->GetComm());
-
-      gmres->SetOperator(prod);
-      gmres->SetRelTol(1e-8);
-      gmres->SetMaxIter(10000);
-      gmres->SetPrintLevel(1);
-
-      gmres->Mult(trueRhs, trueY);
-      strumpack->Mult(trueY, trueX);
-
-      X = trueX.GetBlock(0);
-      double xim2 = trueX.GetBlock(1).Norml2();
-      xim2 *= xim2;
-      double sumxim2 = 0.0;
-
-      MPI_Allreduce(&xim2, &sumxim2, 1, MPI_DOUBLE, MPI_SUM, fespace->GetComm());
-
-      if (myid == 0)
-      {
-         cout << myid << ": norm of Xim " << trueX.GetBlock(1).Norml2() << ", global " <<
-              sqrt(sumxim2) << endl;
-      }
-
-      delete gmres;
-      delete strumpack;
-      delete Arow;
-#else
-      Operator * Arow = new STRUMPACKRowLocMatrix(A);
-
-      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-      strumpack->SetPrintFactorStatistics(true);
-      strumpack->SetPrintSolveStatistics(false);
-      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
-      // strumpack->SetSymmetricPattern(true);
-      strumpack->SetOperator(*Arow);
-      strumpack->SetFromCommandLine();
-      //Solver * precond = strumpack;
-
-      strumpack->Mult(B, X);
-
-      delete strumpack;
-      delete Arow;
-#endif
-   }
-   else
-   {
-#ifdef USE_GMRES
-      HypreGMRES *gmres = new HypreGMRES(A);
-      gmres->SetTol(1e-12);
-      gmres->SetMaxIter(1000);
-      gmres->SetPrintLevel(10);
-      gmres->SetPreconditioner(*amg);
-      gmres->Mult(B, X);
-      delete gmres;
-#else
-      HyprePCG *pcg = new HyprePCG(A);
-      pcg->SetTol(1e-12);
-      pcg->SetMaxIter(100);
-      pcg->SetPrintLevel(2);
-      pcg->SetPreconditioner(*amg);
-      pcg->Mult(B, X);
-#endif
-   }
-
-   /*
-   HYPRE_ParCSRMatrix* amgP = amg->Get_Restriction();
-   HypreParMatrix P0(amgP[0], false);
-   HypreParMatrix P1(amgP[1], false);
-   HypreParMatrix P2(amgP[2], false);
-   //HypreParMatrix P3(amgP[3], false);
-
-   P0.Print("P0");
-   */
-
-   // 13. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-
-   // Compute and print the L^2 norm of the error.
-   {
-      FunctionCoefficient uex(u_exact);
-
-      double err = x.ComputeL2Error(uex);
-      double xnrm = x.ComputeL2Error(zero);
-      ParGridFunction zerogf(fespace);
-      zerogf = 0.0;
-      double normE = zerogf.ComputeL2Error(uex);
-      if (myid == 0)
-      {
-         cout << "|| E_h - E ||_{L^2} = " << err << endl;
-         cout << "|| E_h ||_{L^2} = " << xnrm << endl;
-         cout << "|| E ||_{L^2} = " << normE << endl;
-      }
-   }
-
-   // 14. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 15. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 16. Free the used memory.
-   //delete pcg;
-   delete amg;
-   delete a;
-   delete b;
-   delete fespace;
-   if (order > 0) { delete fec; }
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}
-
-
-double u_exact(const Vector & x)
-{
-   double xi(x(0));
-   double yi(x(1));
-   double zi(1.0);
-
-   if (x.Size() == 3)
-   {
-      zi = x(2);
-   }
-
-   return sin(kappa*xi)*sin(kappa*yi)*sin(kappa*zi);
-}
-
-double f_exact(const Vector &x)
-{
-   double xi(x(0));
-   double yi(x(1));
-   double zi(1.0);
-
-   if (x.Size() == 3)
-   {
-      zi = x(2);
-   }
-
-   const double s = 1.0;
-
-   return ((3.0*kappa*kappa) - (s*K2)) * sin(kappa*xi)*sin(kappa*yi)*sin(
-             kappa*zi) / s;
-}

examples/ex22.cpp

@@ -6,19 +6,17 @@
 //               ex22 -m ../data/inline-tri.mesh -o 3
 //               ex22 -m ../data/inline-quad.mesh -o 3
 //               ex22 -m ../data/inline-quad.mesh -o 3 -p 1
-//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa
 //               ex22 -m ../data/inline-quad.mesh -o 3 -p 2
 //               ex22 -m ../data/inline-tet.mesh -o 2
 //               ex22 -m ../data/inline-hex.mesh -o 2
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 1
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2
-//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa
 //               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0
 //
-// Device sample runs:
-//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa -d cuda
-//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa -d cuda
-//               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0 -pa -d cuda
+// With partial assembly:
+//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa
+//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa
+//               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0 -pa
 //
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
@@ -84,7 +82,6 @@ int main(int argc, char *argv[])
    bool herm_conv = true;
    bool exact_sol = true;
    bool pa = false;
-   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -117,8 +114,6 @@ int main(int argc, char *argv[])
                   "Enable or disable GLVis visualization.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -148,18 +143,13 @@ int main(int argc, char *argv[])
    ComplexOperator::Convention conv =
       herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
 
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes
    //    with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 4. Refine the mesh to increase resolution. In this example we do
+   // 3. Refine the mesh to increase resolution. In this example we do
    //    'ref_levels' of uniform refinement where the user specifies
    //    the number of levels with the '-r' option.
    for (int l = 0; l < ref_levels; l++)
@@ -167,7 +157,7 @@ int main(int argc, char *argv[])
       mesh->UniformRefinement();
    }
 
-   // 5. Define a finite element space on the mesh. Here we use continuous
+   // 4. Define a finite element space on the mesh. Here we use continuous
    //    Lagrange, Nedelec, or Raviart-Thomas finite elements of the specified
    //    order.
    if (dim == 1 && prob != 0 )
@@ -189,7 +179,7 @@ int main(int argc, char *argv[])
    cout << "Number of finite element unknowns: " << fespace->GetTrueVSize()
         << endl;
 
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
    //    In this example, the boundary conditions are defined based on the type
    //    of mesh and the problem type.
    Array<int> ess_tdof_list;
@@ -201,12 +191,12 @@ int main(int argc, char *argv[])
       fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
 
-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
    //    the FEM linear system.
    ComplexLinearForm b(fespace, conv);
    b.Vector::operator=(0.0);
 
-   // 8. Define the solution vector u as a complex finite element grid function
+   // 7. Define the solution vector u as a complex finite element grid function
    //    corresponding to fespace. Initialize u with initial guess of 1+0i or
    //    the exact solution if it is known.
    ComplexGridFunction u(fespace);
@@ -228,6 +218,7 @@ int main(int argc, char *argv[])
    VectorConstantCoefficient zeroVecCoef(zeroVec);
    VectorConstantCoefficient oneVecCoef(oneVec);
 
+   u = 0.0;
    switch (prob)
    {
       case 0:
@@ -280,7 +271,7 @@ int main(int argc, char *argv[])
                  << "window_title 'Exact: Imaginary Part'" << flush;
    }
 
-   // 9. Set up the sesquilinear form a(.,.) on the finite element space
+   // 8. Set up the sesquilinear form a(.,.) on the finite element space
    //    corresponding to the damped harmonic oscillator operator of the
    //    appropriate type:
    //
@@ -323,7 +314,7 @@ int main(int argc, char *argv[])
       default: break; // This should be unreachable
    }
 
-   // 9a. Set up the bilinear form for the preconditioner corresponding to the
+   // 8a. Set up the bilinear form for the preconditioner corresponding to the
    //     appropriate operator
    //
    //      0) A scalar H1 field
@@ -358,9 +349,9 @@ int main(int argc, char *argv[])
       default: break; // This should be unreachable
    }
 
-   // 10. Assemble the form and the corresponding linear system, applying any
-   //     necessary transformations such as: assembly, eliminating boundary
-   //     conditions, conforming constraints for non-conforming AMR, etc.
+   // 9. Assemble the form and the corresponding linear system, applying any
+   //    necessary transformations such as: assembly, eliminating boundary
+   //    conditions, conforming constraints for non-conforming AMR, etc.
    a->Assemble();
    pcOp->Assemble();
 
@@ -371,7 +362,7 @@ int main(int argc, char *argv[])
 
    cout << "Size of linear system: " << A->Width() << endl << endl;
 
-   // 11. Define and apply a GMRES solver for AU=B with a block diagonal
+   // 10. Define and apply a GMRES solver for AU=B with a block diagonal
    //     preconditioner based on the appropriate sparse smoother.
    {
       Array<int> blockOffsets;
@@ -428,7 +419,7 @@ int main(int argc, char *argv[])
       gmres.Mult(B, U);
    }
 
-   // 12. Recover the solution as a finite element grid function and compute the
+   // 11. Recover the solution as a finite element grid function and compute the
    //     errors if the exact solution is known.
    a->RecoverFEMSolution(U, b, u);
 
@@ -460,7 +451,7 @@ int main(int argc, char *argv[])
       cout << endl;
    }
 
-   // 13. Save the refined mesh and the solution. This output can be viewed
+   // 12. Save the refined mesh and the solution. This output can be viewed
    //     later using GLVis: "glvis -m mesh -g sol".
    {
       ofstream mesh_ofs("refined.mesh");
@@ -475,7 +466,7 @@ int main(int argc, char *argv[])
       u.imag().Save(sol_i_ofs);
    }
 
-   // 14. Send the solution by socket to a GLVis server.
+   // 13. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -534,7 +525,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 15. Free the used memory.
+   // 14. Free the used memory.
    delete a;
    delete u_exact;
    delete pcOp;

examples/ex22p.cpp

@@ -7,18 +7,16 @@
 //               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3
 //               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3 -p 1
 //               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3 -p 2
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa
 //               mpirun -np 4 ex22p -m ../data/inline-tet.mesh -o 2
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 1
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 2
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa
 //               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
 //
-// Device sample runs:
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa -d cuda
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa -d cuda
-//               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0 -pa -d cuda
+// With partial assembly:
+//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa
+//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa
+//               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0 -pa
 //
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
@@ -48,6 +46,7 @@
 //               We recommend viewing examples 1, 3 and 4 before viewing this
 //               example.
 
+
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
@@ -91,7 +90,6 @@ int main(int argc, char *argv[])
    bool herm_conv = true;
    bool exact_sol = true;
    bool pa = false;
-   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -126,8 +124,6 @@ int main(int argc, char *argv[])
                   "Enable or disable GLVis visualization.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -164,24 +160,19 @@ int main(int argc, char *argv[])
    ComplexOperator::Convention conv =
       herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
 
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Read the (serial) mesh from the given mesh file on all processors. We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 5. Refine the serial mesh on all processors to increase the resolution.
+   // 4. Refine the serial mesh on all processors to increase the resolution.
    for (int l = 0; l < ser_ref_levels; l++)
    {
       mesh->UniformRefinement();
    }
 
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -191,7 +182,7 @@ int main(int argc, char *argv[])
       pmesh->UniformRefinement();
    }
 
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
    //    use continuous Lagrange, Nedelec, or Raviart-Thomas finite elements of
    //    the specified order.
    if (dim == 1 && prob != 0 )
@@ -219,7 +210,7 @@ int main(int argc, char *argv[])
       cout << "Number of finite element unknowns: " << size << endl;
    }
 
-   // 8. Determine the list of true (i.e. parallel conforming) essential
+   // 7. Determine the list of true (i.e. parallel conforming) essential
    //    boundary dofs. In this example, the boundary conditions are defined
    //    based on the type of mesh and the problem type.
    Array<int> ess_tdof_list;
@@ -231,14 +222,14 @@ int main(int argc, char *argv[])
       fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
 
-   // 9. Set up the parallel linear form b(.) which corresponds to the
+   // 8. Set up the parallel linear form b(.) which corresponds to the
    //    right-hand side of the FEM linear system.
    ParComplexLinearForm b(fespace, conv);
    b.Vector::operator=(0.0);
 
-   // 10. Define the solution vector u as a parallel complex finite element grid
-   //     function corresponding to fespace. Initialize u with initial guess of
-   //     1+0i or the exact solution if it is known.
+   // 9. Define the solution vector u as a parallel complex finite element grid
+   //    function corresponding to fespace. Initialize u with initial guess of
+   //    1+0i or the exact solution if it is known.
    ParComplexGridFunction u(fespace);
    ParComplexGridFunction * u_exact = NULL;
    if (exact_sol) { u_exact = new ParComplexGridFunction(fespace); }
@@ -258,6 +249,7 @@ int main(int argc, char *argv[])
    VectorConstantCoefficient zeroVecCoef(zeroVec);
    VectorConstantCoefficient oneVecCoef(oneVec);
 
+   u = 0.0;
    switch (prob)
    {
       case 0:
@@ -312,7 +304,7 @@ int main(int argc, char *argv[])
                  << "window_title 'Exact: Imaginary Part'" << flush;
    }
 
-   // 11. Set up the parallel sesquilinear form a(.,.) on the finite element
+   // 10. Set up the parallel sesquilinear form a(.,.) on the finite element
    //     space corresponding to the damped harmonic oscillator operator of the
    //     appropriate type:
    //
@@ -355,7 +347,7 @@ int main(int argc, char *argv[])
       default: break; // This should be unreachable
    }
 
-   // 11a. Set up the parallel bilinear form for the preconditioner
+   // 10a. Set up the parallel bilinear form for the preconditioner
    //      corresponding to the appropriate operator
    //
    //      0) A scalar H1 field
@@ -389,7 +381,7 @@ int main(int argc, char *argv[])
       default: break; // This should be unreachable
    }
 
-   // 12. Assemble the parallel bilinear form and the corresponding linear
+   // 11. Assemble the parallel bilinear form and the corresponding linear
    //     system, applying any necessary transformations such as: parallel
    //     assembly, eliminating boundary conditions, applying conforming
    //     constraints for non-conforming AMR, etc.
@@ -407,7 +399,7 @@ int main(int argc, char *argv[])
            << 2 * fespace->GlobalTrueVSize() << endl << endl;
    }
 
-   // 13. Define and apply a parallel FGMRES solver for AU=B with a block
+   // 12. Define and apply a parallel FGMRES solver for AU=B with a block
    //     diagonal preconditioner based on the appropriate multigrid
    //     preconditioner from hypre.
    {
@@ -468,7 +460,7 @@ int main(int argc, char *argv[])
       fgmres.SetPrintLevel(1);
       fgmres.Mult(B, U);
    }
-   // 14. Recover the parallel grid function corresponding to U. This is the
+   // 13. Recover the parallel grid function corresponding to U. This is the
    //     local finite element solution on each processor.
    a->RecoverFEMSolution(U, b, u);
 
@@ -503,7 +495,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 15. Save the refined mesh and the solution in parallel. This output can be
+   // 14. Save the refined mesh and the solution in parallel. This output can be
    //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
    {
       ostringstream mesh_name, sol_r_name, sol_i_name;
@@ -523,7 +515,7 @@ int main(int argc, char *argv[])
       u.imag().Save(sol_i_ofs);
    }
 
-   // 16. Send the solution by socket to a GLVis server.
+   // 15. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -588,7 +580,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 17. Free the used memory.
+   // 16. Free the used memory.
    delete a;
    delete u_exact;
    delete pcOp;

examples/ex25.cpp

@@ -82,27 +82,24 @@ public:
 };
 
 // Class for returning the PML coefficients of the bilinear form
-class PMLDiagMatrixCoefficient : public VectorCoefficient
+class PMLMatrixCoefficient : public MatrixCoefficient
 {
 private:
    CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , Vector &);
+   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                              Vector &),
-                            CartesianPML * pml_)
-      : VectorCoefficient(dim), pml(pml_), Function(F)
+   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                          DenseMatrix &),
+                        CartesianPML * pml_)
+      : MatrixCoefficient(dim), pml(pml_), Function(F)
    {}
-
-   using VectorCoefficient::Eval;
-
-   virtual void Eval(Vector &K, ElementTransformation &T,
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                      const IntegrationPoint &ip)
    {
       double x[3];
       Vector transip(x, 3);
       T.Transform(ip, transip);
-      K.SetSize(vdim);
+      K.SetSize(height, width);
       (*Function)(transip, pml, K);
    }
 };
@@ -119,13 +116,13 @@ void source(const Vector &x, Vector & f);
 
 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
 
 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -368,19 +365,19 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);
 
    int cdim = (dim == 2) ? 1 : dim;
-   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
-   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
 
-   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
-   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
 
    // Integrators inside the PML region
    a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -422,13 +419,13 @@ int main(int argc, char *argv[])
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));
 
-      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
-      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
+      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
 
-      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
 
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));
@@ -766,7 +763,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
    }
 }
 
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -777,13 +774,14 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
       det *= dxs[i];
    }
 
+   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      D(i) = (det / pow(dxs[i], 2)).real();
+      M(i, i) = (det / pow(dxs[i], 2)).real();
    }
 }
 
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -794,13 +792,14 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
       det *= dxs[i];
    }
 
+   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      D(i) = (det / pow(dxs[i], 2)).imag();
+      M(i, i) = (det / pow(dxs[i], 2)).imag();
    }
 }
 
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -811,13 +810,14 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
       det *= dxs[i];
    }
 
+   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      D(i) = abs(det / pow(dxs[i], 2));
+      M(i, i) = abs(det / pow(dxs[i], 2));
    }
 }
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -831,18 +831,19 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
    // in the 2D case the coefficient is scalar 1/det(J)
    if (dim == 2)
    {
-      D = (1.0 / det).real();
+      M = (1.0 / det).real();
    }
    else
    {
+      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         D(i) = (pow(dxs[i], 2) / det).real();
+         M(i, i) = (pow(dxs[i], 2) / det).real();
       }
    }
 }
 
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -855,18 +856,19 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)
 
    if (dim == 2)
    {
-      D = (1.0 / det).imag();
+      M = (1.0 / det).imag();
    }
    else
    {
+      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         D(i) = (pow(dxs[i], 2) / det).imag();
+         M(i, i) = (pow(dxs[i], 2) / det).imag();
       }
    }
 }
 
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -879,13 +881,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)
 
    if (dim == 2)
    {
-      D = abs(1.0 / det);
+      M = abs(1.0 / det);
    }
    else
    {
+      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         D(i) = abs(pow(dxs[i], 2) / det);
+         M(i, i) = abs(pow(dxs[i], 2) / det);
       }
    }
 }

examples/ex25p.cpp

@@ -82,27 +82,24 @@ public:
 };
 
 // Class for returning the PML coefficients of the bilinear form
-class PMLDiagMatrixCoefficient : public VectorCoefficient
+class PMLMatrixCoefficient : public MatrixCoefficient
 {
 private:
    CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , Vector &);
+   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                              Vector &),
-                            CartesianPML * pml_)
-      : VectorCoefficient(dim), pml(pml_), Function(F)
+   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                          DenseMatrix &),
+                        CartesianPML * pml_)
+      : MatrixCoefficient(dim), pml(pml_), Function(F)
    {}
-
-   using VectorCoefficient::Eval;
-
-   virtual void Eval(Vector &K, ElementTransformation &T,
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                      const IntegrationPoint &ip)
    {
       double x[3];
       Vector transip(x, 3);
       T.Transform(ip, transip);
-      K.SetSize(vdim);
+      K.SetSize(height, width);
       (*Function)(transip, pml, K);
    }
 };
@@ -119,13 +116,13 @@ void source(const Vector &x, Vector & f);
 
 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
 
 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -396,19 +393,19 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);
 
    int cdim = (dim == 2) ? 1 : dim;
-   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
-   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
 
-   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
-   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
 
    // Integrators inside the PML region
    a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -456,13 +453,13 @@ int main(int argc, char *argv[])
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));
 
-      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
-      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
+      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
 
-      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
 
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));
@@ -822,7 +819,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
    }
 }
 
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -833,13 +830,14 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
       det *= dxs[i];
    }
 
+   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      D(i) = (det / pow(dxs[i], 2)).real();
+      M(i, i) = (det / pow(dxs[i], 2)).real();
    }
 }
 
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -850,13 +848,14 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
       det *= dxs[i];
    }
 
+   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      D(i) = (det / pow(dxs[i], 2)).imag();
+      M(i, i) = (det / pow(dxs[i], 2)).imag();
    }
 }
 
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -867,13 +866,14 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
       det *= dxs[i];
    }
 
+   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      D(i) = abs(det / pow(dxs[i], 2));
+      M(i, i) = abs(det / pow(dxs[i], 2));
    }
 }
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -887,18 +887,19 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
    // in the 2D case the coefficient is scalar 1/det(J)
    if (dim == 2)
    {
-      D = (1.0 / det).real();
+      M = (1.0 / det).real();
    }
    else
    {
+      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         D(i) = (pow(dxs[i], 2) / det).real();
+         M(i, i) = (pow(dxs[i], 2) / det).real();
       }
    }
 }
 
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -911,18 +912,19 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)
 
    if (dim == 2)
    {
-      D = (1.0 / det).imag();
+      M = (1.0 / det).imag();
    }
    else
    {
+      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         D(i) = (pow(dxs[i], 2) / det).imag();
+         M(i, i) = (pow(dxs[i], 2) / det).imag();
       }
    }
 }
 
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -935,13 +937,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)
 
    if (dim == 2)
    {
-      D = abs(1.0 / det);
+      M = abs(1.0 / det);
    }
    else
    {
+      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         D(i) = abs(pow(dxs[i], 2) / det);
+         M(i, i) = abs(pow(dxs[i], 2) / det);
       }
    }
 }

examples/ex2p.cpp

@@ -60,7 +60,6 @@ int main(int argc, char *argv[])
    bool static_cond = false;
    bool visualization = 1;
    bool amg_elast = 0;
-   bool reorder_space = false;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -76,8 +75,6 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
-   args.AddOption(&reorder_space, "-nodes", "--by-nodes", "-vdim", "--by-vdim",
-                  "Use byNODES ordering of vector space instead of byVDIM");
    args.Parse();
    if (!args.Good())
    {
@@ -159,14 +156,7 @@ int main(int argc, char *argv[])
    else
    {
       fec = new H1_FECollection(order, dim);
-      if (reorder_space)
-      {
-         fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byNODES);
-      }
-      else
-      {
-         fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
-      }
+      fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
    }
    HYPRE_Int size = fespace->GlobalTrueVSize();
    if (myid == 0)
@@ -259,7 +249,7 @@ int main(int argc, char *argv[])
    }
    else
    {
-      amg->SetSystemsOptions(dim, reorder_space);
+      amg->SetSystemsOptions(dim);
    }
    HyprePCG *pcg = new HyprePCG(A);
    pcg->SetTol(1e-8);

examples/ex3p.cpp

@@ -158,9 +158,9 @@ int main(int argc, char *argv[])
    //    by marking all the boundary attributes from the mesh as essential
    //    (Dirichlet) and converting them to a list of true dofs.
    Array<int> ess_tdof_list;
-   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
    if (pmesh->bdr_attributes.Size())
    {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
       ess_bdr = 1;
       fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
@@ -207,36 +207,18 @@ int main(int argc, char *argv[])
    // 13. Solve the system AX=B using PCG with the AMS preconditioner from hypre
    //     (in the full assembly case) or CG with Jacobi preconditioner (in the
    //     partial assembly case).
-   if (pa) // matrix-free auxiliary space solver with PA
+
+   if (pa) // Jacobi preconditioning in partial assembly mode
    {
-      StopWatch sw;
-      sw.Clear();
-      sw.Start();
-
-      MatrixFreeAMS *ams = new MatrixFreeAMS(*a, *A, *fespace, muinv, sigma, NULL,
-                                             ess_bdr); //, 2, 2);
-
-      //OperatorJacobiSmoother jacobi(*a, ess_tdof_list);
+      OperatorJacobiSmoother Jacobi(*a, ess_tdof_list);
 
       CGSolver cg(MPI_COMM_WORLD);
       cg.SetRelTol(1e-12);
       cg.SetMaxIter(1000);
       cg.SetPrintLevel(1);
       cg.SetOperator(*A);
-      cg.SetPreconditioner(*ams);
-      //cg.SetPreconditioner(jacobi);
-      //cg.iterative_mode = false;
-      sw.Stop();
-      cout << myid << ": MatrixFreeAMS-CG setup time " << sw.RealTime() << endl;
-
-      sw.Clear();
-      sw.Start();
-
+      cg.SetPreconditioner(Jacobi);
       cg.Mult(B, X);
-
-      sw.Stop();
-      cout << myid << ": MatrixFreeAMS-CG solve time " << sw.RealTime() << endl;
-      ams->PrintTimings(myid);
    }
    else
    {

examples/ex3p_airy.cpp

@@ -1,425 +0,0 @@
-//                       MFEM Example 3 - Parallel Version
-//
-// Compile with: make ex3p
-//
-// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
-//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
-//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/escher.mesh
-//               mpirun -np 4 ex3p -m ../data/escher.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/fichera.mesh
-//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
-//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
-//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
-//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
-//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
-//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
-//
-// Description:  This example code solves a simple electromagnetic diffusion
-//               problem corresponding to the second order definite Maxwell
-//               equation curl curl E + E = f with boundary condition
-//               E x n = <given tangential field>. Here, we use a given exact
-//               solution E and compute the corresponding r.h.s. f.
-//               We discretize with Nedelec finite elements in 2D or 3D.
-//
-//               The example demonstrates the use of H(curl) finite element
-//               spaces with the curl-curl and the (vector finite element) mass
-//               bilinear form, as well as the computation of discretization
-//               error when the exact solution is known. Static condensation is
-//               also illustrated.
-//
-//               We recommend viewing examples 1-2 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-#define TEST_AIRY
-
-#ifdef TEST_AIRY
-#include "gsl_sf_airy.h"
-
-#define XSHIFT 0.0 // 0.25
-#endif
-
-// Exact solution, E, and r.h.s., f. See below for implementation.
-void E_exact(const Vector &, Vector &);
-void f_exact(const Vector &, Vector &);
-double freq = 1.0, kappa;
-int dim;
-
-//#define K2_VALUE 11.9
-//#define K2_VALUE 2.1
-//#define K2_VALUE 1500.9
-#define K2_VALUE 10981.41589009910  // 104.792251097584^2 or 5 GHz
-
-void test_Airy_epsilon(const Vector &x, Vector &e)
-{
-   e(0) = 1.0;
-   e(1) = 1.0;
-   e(2) = (4.0 * (x(0) + XSHIFT)) - 1.0;
-
-   e *= -K2_VALUE;
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   //const char *mesh_file = "../data/beam-tet.mesh";
-   const char *mesh_file = "../data/inline-tetHalf.mesh";
-   //const char *mesh_file = "inline-tetSlab.mesh";
-   //const char *mesh_file = "../data/inline-hexHalf.mesh";
-   //const char *mesh_file = "../data/inline-tet.mesh";
-   int order = 2;
-   bool static_cond = false;
-   bool visualization = 1;
-#ifdef MFEM_USE_STRUMPACK
-   bool use_strumpack = false;
-#endif
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
-                  " solution.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-#ifdef MFEM_USE_STRUMPACK
-   args.AddOption(&use_strumpack, "-strumpack", "--strumpack-solver",
-                  "-no-strumpack", "--no-strumpack-solver",
-                  "Use STRUMPACK's double complex linear solver.");
-#endif
-
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-      cout << "Using k2 " << K2_VALUE << endl;
-   }
-   kappa = freq * M_PI;
-
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(100000./mesh->GetNE())/log(2.)/dim);
-      ref_levels = 0;
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh->UniformRefinement();
-      }
-   }
-   pmesh->ReorientTetMesh();
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use the Nedelec finite elements of the specified order.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   long globalNE = pmesh->GetGlobalNE();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-      cout << "Number of mesh elements: " << globalNE << endl;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh->bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
-   //    basis functions in the finite element fespace.
-   VectorFunctionCoefficient f(sdim, f_exact);
-   ParLinearForm *b = new ParLinearForm(fespace);
-   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x by projecting the exact
-   //    solution. Note that only values from the boundary edges will be used
-   //    when eliminating the non-homogeneous boundary condition to modify the
-   //    r.h.s. vector b.
-   ParGridFunction x(fespace);
-   ParGridFunction solproj(fespace);
-   VectorFunctionCoefficient E(sdim, E_exact);
-   x.ProjectCoefficient(E);
-
-   solproj = x;
-
-   // 10. Set up the parallel bilinear form corresponding to the EM diffusion
-   //     operator curl muinv curl + sigma I, by adding the curl-curl and the
-   //     mass domain integrators.
-   Coefficient *muinv = new ConstantCoefficient(1.0);
-   Coefficient *sigma = new ConstantCoefficient(-K2_VALUE);
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
-#ifdef TEST_AIRY
-   VectorFunctionCoefficient epsilon(3, test_Airy_epsilon);
-   a->AddDomainIntegrator(new VectorFEMassIntegrator(epsilon));
-#else
-   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
-#endif
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   HypreParMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   if (myid == 0)
-   {
-      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
-   }
-
-   StopWatch chrono;
-   chrono.Clear();
-   chrono.Start();
-
-#ifdef MFEM_USE_STRUMPACK
-   if (use_strumpack)
-   {
-      Operator * Arow = new STRUMPACKRowLocMatrix(A);
-
-      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-      strumpack->SetPrintFactorStatistics(true);
-      strumpack->SetPrintSolveStatistics(false);
-      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
-      // strumpack->SetSymmetricPattern(true);
-      strumpack->SetOperator(*Arow);
-      strumpack->SetFromCommandLine();
-      //Solver * precond = strumpack;
-
-      strumpack->Mult(B, X);
-      //X = 0.0;
-
-      {
-         // Check residual
-         Vector res(X.Size());
-         Vector ssol(X.Size());
-         ssol = X;
-
-         const double Bnrm = B.Norml2();
-         const double Bnrm2 = Bnrm*Bnrm;
-
-         A.Mult(ssol, res);
-         res -= B;
-
-         const double Rnrm = res.Norml2();
-         const double Rnrm2 = Rnrm*Rnrm;
-
-         double sumBnrm2 = 0.0;
-         double sumRnrm2 = 0.0;
-         MPI_Allreduce(&Bnrm2, &sumBnrm2, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-         MPI_Allreduce(&Rnrm2, &sumRnrm2, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-
-         if (myid == 0)
-         {
-            cout << myid << ": STRUMPACK residual norm " << sqrt(sumRnrm2) << ", B norm " <<
-                 sqrt(sumBnrm2) << endl;
-         }
-      }
-
-      delete strumpack;
-      delete Arow;
-   }
-   else
-#endif
-   {
-      // 12. Define and apply a parallel PCG solver for AX=B with the AMS
-      //     preconditioner from hypre.
-      ParFiniteElementSpace *prec_fespace =
-         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
-      HypreSolver *ams = new HypreAMS(A, prec_fespace);
-      HyprePCG *pcg = new HyprePCG(A);
-      pcg->SetTol(1e-12);
-      pcg->SetMaxIter(500);
-      pcg->SetPrintLevel(2);
-      pcg->SetPreconditioner(*ams);
-      pcg->Mult(B, X);
-
-      delete pcg;
-      delete ams;
-   }
-
-   chrono.Stop();
-   if (myid == 0)
-   {
-      cout << "Solver time " << chrono.RealTime() << endl;
-   }
-
-   // 13. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-
-   // 14. Compute and print the L^2 norm of the error.
-   {
-      double err = x.ComputeL2Error(E);
-      ParGridFunction x0(fespace);
-      x0 = 0.0;
-
-      double solnrm = x0.ComputeL2Error(E);
-
-      if (myid == 0)
-      {
-         cout << "\n|| E_h - E ||_{L^2} = " << err << ", relative error " << err / solnrm
-              << endl;
-      }
-   }
-
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-      //solproj.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   delete a;
-   delete sigma;
-   delete muinv;
-   delete b;
-   delete fespace;
-   delete fec;
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}
-
-
-void E_exact(const Vector &x, Vector &E)
-{
-   if (dim == 3)
-   {
-#ifdef TEST_AIRY
-      const double y = (4.0 * (x(0) + XSHIFT)) - 1.0;
-      const double k = sqrt(K2_VALUE);
-      const double beta = pow(0.25 * k, 2.0/3.0);
-
-      E(0) = 0.0;
-      E(1) = 0.0;
-      E(2) = gsl_sf_airy_Ai(-beta * y, GSL_PREC_DOUBLE);
-#else
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(2));
-      E(2) = sin(kappa * x(0));
-#endif
-   }
-   else
-   {
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(0));
-      if (x.Size() == 3) { E(2) = 0.0; }
-   }
-}
-
-void f_exact(const Vector &x, Vector &f)
-{
-   if (dim == 3)
-   {
-#ifdef TEST_AIRY
-      f = 0.0;
-#else
-      f(0) = (-K2_VALUE + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (-K2_VALUE + kappa * kappa) * sin(kappa * x(2));
-      f(2) = (-K2_VALUE + kappa * kappa) * sin(kappa * x(0));
-#endif
-   }
-   else
-   {
-      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
-      if (x.Size() == 3) { f(2) = 0.0; }
-   }
-}

examples/ex3p_cfosls.cpp

@@ -1,848 +0,0 @@
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-#include "gsl_sf_airy.h"
-
-#include "multigrid.hpp"
-
-using namespace std;
-using namespace mfem;
-
-
-//#define DIRECT_SOLVER
-
-// Define exact solution
-void E_exact(const Vector & x, Vector & E);
-void H_exact(const Vector & x, Vector & H);
-void f_exact_H(const Vector & x, Vector & f_H);
-void get_maxwell_solution(const Vector & x, double E[], double curlE[],
-                          double curl2E[]);
-void epsilon_func(const Vector &x, Vector &M);
-void epsilon2_func(const Vector &x, Vector &M);
-void epsilon_func_mat(const Vector &x, DenseMatrix &M);
-
-int dim;
-double omega;
-int sol = 3;
-
-
-class FOSLSSolver : public Solver
-{
-public:
-   FOSLSSolver(ParFiniteElementSpace *fespace_, std::vector<HypreParMatrix*>& P,
-               const int myid_)
-      : Solver(2 * fespace_->GetTrueVSize()), M_inv(MPI_COMM_WORLD),
-        fespace(fespace_),
-        n(fespace_->GetTrueVSize()), nfull(fespace_->GetVSize()), LSpcg(MPI_COMM_WORLD),
-        myid(myid_)
-   {
-      z.SetSize(n);
-      Minv_x.SetSize(n);
-
-      ParMesh *pmesh = fespace->GetParMesh();
-      int dim = pmesh->Dimension();
-      int sdim = pmesh->SpaceDimension();
-
-      VectorFunctionCoefficient epsilon(dim, epsilon_func);
-      VectorFunctionCoefficient epsilonT(epsilon);  // transpose of epsilon
-      VectorFunctionCoefficient epsilon2(dim,epsilon2_func);
-      ConstantCoefficient pos(omega);
-      ConstantCoefficient sigma(omega*omega);
-      ScalarVectorProductCoefficient coeff(pos,epsilon);
-      ScalarVectorProductCoefficient coeffT(pos,epsilonT);
-      ScalarVectorProductCoefficient coeff2(sigma,epsilon2);
-
-      bM = new ParBilinearForm(fespace);
-      bM->AddDomainIntegrator(new VectorFEMassIntegrator());
-      bM->Assemble();
-      bM->Finalize();
-
-      bM_eps = new ParBilinearForm(fespace);
-      bM_eps->AddDomainIntegrator(new VectorFEMassIntegrator(epsilonT));
-      bM_eps->Assemble();
-      bM_eps->Finalize();
-
-      bM_curl = new ParMixedBilinearForm(fespace,fespace);
-      bM_curl->AddDomainIntegrator(new MixedVectorWeakCurlIntegrator());
-      bM_curl->Assemble();
-      bM_curl->Finalize();
-
-      Array<int> ess_tdof_list;  // empty
-
-      bM->FormSystemMatrix(ess_tdof_list, M);
-      bM_eps->FormSystemMatrix(ess_tdof_list, M_eps);
-      //bM_curl->FormColSystemMatrix(ess_tdof_list, M_curl);
-      {
-         OperatorPtr M_curl_ptr;
-         bM_curl->FormRectangularSystemMatrix(ess_tdof_list, ess_tdof_list, M_curl_ptr);
-         M_curl = M_curl_ptr.As<HypreParMatrix>();
-      }
-
-      M_inv.SetAbsTol(1.0e-12);
-      M_inv.SetRelTol(1.0e-12);
-      M_inv.SetMaxIter(100);
-      M_inv.SetOperator(M);
-      M_inv.SetPrintLevel(0);
-
-      block_trueOffsets.SetSize(5);
-
-      block_trueOffsets[0] = 0;
-      block_trueOffsets[1] = n;
-      block_trueOffsets[2] = n;
-      block_trueOffsets[3] = n;
-      block_trueOffsets[4] = n;
-      block_trueOffsets.PartialSum();
-
-      trueRhs = new BlockVector(block_trueOffsets);
-      trueSol = new BlockVector(block_trueOffsets);
-
-      //    _           _    _  _       _  _
-      //   |             |  |    |     |    |
-      //   |  A00   A01  |  | E  |     |F_E |
-      //   |             |  |    |  =  |    |
-      //   |  A10   A11  |  | H  |     |F_G |
-      //   |_           _|  |_  _|     |_  _|
-      //
-      // A00 = (curl E, curl F) + \omega^2 (E,F)
-      // A01 = - \omega *( (curl E, F) + (E,curl F)
-      // A10 = - \omega *( (curl H, G) + (H,curl G)
-      // A11 = (curl H, curl G) + \omega^2 (H,G)
-
-      ParBilinearForm *a_EE = new ParBilinearForm(fespace);
-      a_EE->AddDomainIntegrator(new CurlCurlIntegrator());
-      a_EE->AddDomainIntegrator(new VectorFEMassIntegrator(coeff2));
-      a_EE->AddBoundaryIntegrator(new VectorFEMassIntegrator());
-      a_EE->Assemble();
-      a_EE->Finalize();
-      HypreParMatrix *A_EE = new HypreParMatrix;
-      a_EE->FormSystemMatrix(ess_tdof_list, *A_EE);
-
-      ParBilinearForm *a_HH = new ParBilinearForm(fespace);
-      a_HH->AddDomainIntegrator(new CurlCurlIntegrator());
-      a_HH->AddDomainIntegrator(new VectorFEMassIntegrator(sigma));
-      a_HH->AddBoundaryIntegrator(new VectorFEMassIntegrator());
-      a_HH->Assemble();
-      a_HH->Finalize();
-      HypreParMatrix *A_HH = new HypreParMatrix;
-      a_HH->FormSystemMatrix(ess_tdof_list, *A_HH);
-
-      ParBilinearForm *a_tang = new ParBilinearForm(fespace);
-      a_tang->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator(1.0));
-      a_tang->Assemble();
-      a_tang->Finalize();
-      OperatorHandle A_tang_ptr;
-      a_tang->FormSystemMatrix(ess_tdof_list, A_tang_ptr);
-      HypreParMatrix *A_tang = A_tang_ptr.As<HypreParMatrix>();
-
-      // (k curl u, eps v) + (k u, curl v)
-      ParMixedBilinearForm *a_mix1 = new ParMixedBilinearForm(fespace,fespace);
-      a_mix1->AddDomainIntegrator(new MixedVectorCurlIntegrator(coeffT));
-      a_mix1->AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(pos));
-      a_mix1->Assemble();
-      a_mix1->Finalize();
-      HypreParMatrix *A_mix1 = NULL; // new HypreParMatrix;
-      //a_mix1->FormColSystemMatrix(ess_tdof_list, *A_mix1);
-      {
-         OperatorPtr A_mix1_ptr;
-         a_mix1->FormRectangularSystemMatrix(ess_tdof_list, ess_tdof_list, A_mix1_ptr);
-         A_mix1 = A_mix1_ptr.As<HypreParMatrix>();
-      }
-
-      // (k curl u, v) + (k eps u, curl v)
-      ParMixedBilinearForm *a_mix2 = new ParMixedBilinearForm(fespace,fespace);
-      a_mix2->AddDomainIntegrator(new MixedVectorCurlIntegrator(pos));
-      a_mix2->AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(coeff));
-      a_mix2->Assemble();
-      a_mix2->Finalize();
-      HypreParMatrix *A_mix2 = NULL; // new HypreParMatrix;
-      //a_mix2->FormColSystemMatrix(ess_tdof_list, *A_mix2);
-      {
-         OperatorPtr A_mix2_ptr;
-         a_mix2->FormRectangularSystemMatrix(ess_tdof_list, ess_tdof_list, A_mix2_ptr);
-         A_mix2 = A_mix2_ptr.As<HypreParMatrix>();
-      }
-
-      BlockOperator *LS_Maxwellop = new BlockOperator(block_trueOffsets);
-      const int numBlocks = 4;
-
-      LS_Maxwellop->SetBlock(0, 0, A_EE);
-      LS_Maxwellop->SetBlock(1, 0, A_mix2, -1.0); // no bc
-      LS_Maxwellop->SetBlock(3, 0, A_tang, -1.0);
-      LS_Maxwellop->SetBlock(0, 1, A_mix1, -1.0);  // no bc
-      LS_Maxwellop->SetBlock(1, 1, A_HH);
-      LS_Maxwellop->SetBlock(2, 1, A_tang, -1.0);  // other rotation
-      LS_Maxwellop->SetBlock(1, 2, A_tang);
-      LS_Maxwellop->SetBlock(2, 2, A_EE);
-      LS_Maxwellop->SetBlock(3, 2, A_mix2, -1.0);  // no bc
-      LS_Maxwellop->SetBlock(0, 3, A_tang);  // other rotation
-      LS_Maxwellop->SetBlock(2, 3, A_mix1, -1.0);  // no bc
-      LS_Maxwellop->SetBlock(3, 3, A_HH);
-
-      // Set up the preconditioner
-      Array2D<HypreParMatrix*> blockA(numBlocks, numBlocks);
-      Array2D<double> blockAcoef(numBlocks, numBlocks);
-
-      for (int i=0; i<numBlocks; ++i)
-      {
-         for (int j=0; j<numBlocks; ++j)
-         {
-            if (LS_Maxwellop->IsZeroBlock(i,j) == 0)
-            {
-               blockA(i,j) = static_cast<HypreParMatrix *>(&LS_Maxwellop->GetBlock(i,j));
-               blockAcoef(i,j) = LS_Maxwellop->GetBlockCoef(i,j);
-            }
-            else
-            {
-               blockA(i,j) = NULL;
-               blockAcoef(i,j) = 1.0;
-            }
-         }
-      }
-
-      LSpcg.SetAbsTol(1.0e-12);
-      LSpcg.SetRelTol(1.0e-8);
-      LSpcg.SetMaxIter(2000);
-      LSpcg.SetOperator(*LS_Maxwellop);
-      LSpcg.SetPrintLevel(1);
-
-      BlockMGSolver * precMG = NULL;
-
-#ifdef DIRECT_SOLVER
-      std::vector<std::vector<int> > blockProcOffsets(numBlocks);
-      std::vector<std::vector<int> > all_block_num_loc_rows(numBlocks);
-      Array2D<SparseMatrix*> Asp;
-      Asp.SetSize(numBlocks,numBlocks);
-
-      {
-         int nprocs, rank;
-         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-         MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-
-         std::vector<int> allnumrows(nprocs);
-         const int blockNumRows = n;
-         MPI_Allgather(&blockNumRows, 1, MPI_INT, allnumrows.data(), 1, MPI_INT,
-                       MPI_COMM_WORLD);
-
-         for (int b=0; b<numBlocks; ++b)
-         {
-            blockProcOffsets[b].resize(nprocs);
-            all_block_num_loc_rows[b].resize(nprocs);
-
-            for (int j=0; j<numBlocks; ++j)
-            {
-               Asp(b,j) = NULL;
-            }
-         }
-
-         blockProcOffsets[0][0] = 0;
-         for (int i=0; i<nprocs-1; ++i)
-         {
-            blockProcOffsets[0][i+1] = blockProcOffsets[0][i] + allnumrows[i];
-         }
-
-         for (int i=0; i<nprocs; ++i)
-         {
-            for (int b=0; b<numBlocks; ++b)
-            {
-               all_block_num_loc_rows[b][i] = allnumrows[i];
-            }
-
-            for (int b=1; b<numBlocks; ++b)
-            {
-               blockProcOffsets[b][i] = blockProcOffsets[0][i];
-            }
-         }
-      }
-
-      LSH = CreateHypreParMatrixFromBlocks(MPI_COMM_WORLD, block_trueOffsets, blockA,
-                                           Asp,
-                                           blockAcoef, blockProcOffsets, all_block_num_loc_rows);
-
-      invLSH = CreateStrumpackSolver(new STRUMPACKRowLocMatrix(*LSH), MPI_COMM_WORLD);
-#else
-      precMG = new BlockMGSolver(LS_Maxwellop->Height(), LS_Maxwellop->Width(),
-                                 blockA, blockAcoef, P);
-      precMG->SetTheta(0.5);
-      LSpcg.SetPreconditioner(*precMG);
-#endif
-   }
-
-   void SetOperator(const Operator &op) { }
-
-   void Mult(const Vector &x, Vector &y) const
-   {
-      // Solve (curl E, curl v) - k^2 (eps E, v) + ik <pi(u), pi(v)> = (x, v), with no BC,
-      // where x is complex, using FOSLS. This is the Galerkin discretization of
-      // curl curl u - k^2 eps u = x, with ik n x u x n - n x curl u = 0 on the boundary.
-
-      MFEM_VERIFY(x.Size() == 2*n, "");
-
-      (*trueRhs) = 0.0;
-
-      for (int i=0; i<n; ++i)
-      {
-         z[i] = x[i];   // Set z = x_Re
-      }
-
-      M_inv.Mult(z, Minv_x);
-      M_eps.Mult(Minv_x, z);
-
-      trueRhs->GetBlock(0) -= z;
-
-      M_curl->Mult(Minv_x, z);
-      z *= 1.0 / omega;
-
-      trueRhs->GetBlock(1) = z;
-
-      for (int i=0; i<n; ++i)
-      {
-         z[i] = x[n + i];   // Set z = x_Im
-      }
-
-      M_inv.Mult(z, Minv_x);
-      M_eps.Mult(Minv_x, z);
-
-      trueRhs->GetBlock(2) -= z;
-
-      M_curl->Mult(Minv_x, z);
-      z *= 1.0 / omega;
-
-      trueRhs->GetBlock(3) += z;
-
-#ifdef DIRECT_SOLVER
-      invLSH->Mult(*trueRhs, *trueSol);
-#else
-      LSpcg.Mult(*trueRhs, *trueSol);
-#endif
-
-      for (int i=0; i<n; ++i)
-      {
-         y[i] = trueSol->GetBlock(0)[i];   // Set y_Re = E_Re
-      }
-
-      for (int i=0; i<n; ++i)
-      {
-         y[n + i] = trueSol->GetBlock(2)[i];   // Set y_Im = E_Im
-      }
-   }
-
-   void ComplexFOSLSTest()
-   {
-      Array<int> block_offsets;
-
-      block_offsets.SetSize(5);
-
-      block_offsets[0] = 0;
-      block_offsets[1] = nfull;
-      block_offsets[2] = nfull;
-      block_offsets[3] = nfull;
-      block_offsets[4] = nfull;
-      block_offsets.PartialSum();
-
-      BlockVector rhs(block_offsets);
-      BlockVector rhsIm(block_offsets);
-
-      rhs = 0.0;
-      rhsIm = 0.0;
-
-      const double ci = 0.0; // 3.3;
-
-      // Exact complex solution: E = Er + i Ei = Epw + ci i Epw, where Epw is E_exact.
-
-      // Set up the linear form with the real part Fr only.
-      const int sdim = 3;
-      VectorFunctionCoefficient Eex(sdim, E_exact);
-      VectorFunctionCoefficient Hex(sdim, H_exact);
-      ConstantCoefficient negOne(-1.0);
-      VectorFunctionCoefficient mEex(sdim, E_exact, &negOne);
-      VectorFunctionCoefficient mHex(sdim, H_exact, &negOne);
-
-      ConstantCoefficient neg(-omega);
-      ConstantCoefficient pos(omega);
-      VectorFunctionCoefficient f_H(3,f_exact_H);  // f / omega
-      ScalarVectorProductCoefficient sf_H(neg,f_H);
-      ScalarVectorProductCoefficient spf_H(pos,f_H);
-      ScalarVectorProductCoefficient mf_H(negOne,f_H);
-
-      VectorFunctionCoefficient epsilon(3, epsilon_func);
-      VectorFunctionCoefficient epsilonT(3, epsilon_func);  // transpose of epsilon
-      MatrixFunctionCoefficient epsilonTmat(3,
-                                            epsilon_func_mat);  // transpose of epsilon
-
-      MatVecCoefficient epsT_spf_H(epsilonTmat, spf_H);
-      MatVecCoefficient epsT_sf_H(epsilonTmat, sf_H);
-
-      ParLinearForm *b_E = new ParLinearForm;
-      b_E->Update(fespace, rhs.GetBlock(0), 0);
-      b_E->AddDomainIntegrator(new VectorFEDomainLFIntegrator(
-                                  epsT_sf_H)); // (k^{-1} Fr, -k eps Qr)
-      b_E->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
-                                    Eex));  // <n x E_Re x n, n x Q_Re x n>
-      b_E->Assemble();
-
-      ParLinearForm *b_H = new ParLinearForm;
-      b_H->Update(fespace, rhs.GetBlock(1), 0);
-      b_H->AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(
-                                  f_H));  // (k^{-1} Fr, curl Rr)
-      b_H->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
-                                    Hex));  // <n x H_Re, n x R_Re>
-      b_H->Assemble();
-
-      ParLinearForm *b_E_Im = new ParLinearForm;
-      b_E_Im->Update(fespace, rhs.GetBlock(2), 0);
-      b_E_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
-                                       mHex));  // <n x H_Re, n x Q_Im x n>
-      b_E_Im->Assemble();
-
-      ParLinearForm *b_H_Im = new ParLinearForm;
-      b_H_Im->Update(fespace, rhs.GetBlock(3), 0);
-      b_H_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
-                                       mEex));  // -<n x E_Re x n, n x R_Im>
-      b_H_Im->Assemble();
-
-      // Add the imaginary part Fi.
-
-      ParLinearForm *b_Ei = new ParLinearForm;
-      b_Ei->Update(fespace, rhsIm.GetBlock(0), 0);
-      b_Ei->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
-                                     Hex));  // -<n x H_Im, n x Q_Re x n>
-      b_Ei->Assemble();
-
-      ParLinearForm *b_Hi = new ParLinearForm;
-      b_Hi->Update(fespace, rhsIm.GetBlock(1), 0);
-      b_Hi->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
-                                     Eex));  // <n x E_Im x n, n x R_Re>
-      b_Hi->Assemble();
-
-      ParLinearForm *b_Ei_Im = new ParLinearForm;
-      b_Ei_Im->Update(fespace, rhsIm.GetBlock(2), 0);
-      b_Ei_Im->AddDomainIntegrator(new VectorFEDomainLFIntegrator(
-                                      epsT_sf_H)); // -(k^{-1} Fi, k eps Qi)
-      b_Ei_Im->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
-                                        Eex));  // <n x E_Im x n, n x Q_Im x n>
-      b_Ei_Im->Assemble();
-
-      ParLinearForm *b_Hi_Im = new ParLinearForm;
-      b_Hi_Im->Update(fespace, rhsIm.GetBlock(3), 0);
-      b_Hi_Im->AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(
-                                      f_H)); // (k^{-1} Fi, curl Ri)
-      b_Hi_Im->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
-                                        Hex));  // <n x H_Im, n x R_Im>
-      b_Hi_Im->Assemble();
-
-      rhsIm *= ci;
-      rhs += rhsIm;
-
-      for (int i=0; i<4; ++i)
-      {
-         fespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(i),
-                                                         trueRhs->GetBlock(i));
-      }
-
-#ifdef DIRECT_SOLVER
-      invLSH->Mult(*trueRhs, *trueSol);
-#else
-      LSpcg.Mult(*trueRhs, *trueSol);
-#endif
-
-      // Check error
-      ParGridFunction E_gf(fespace);
-      int order = 2;
-      int order_quad = std::max(2, 2*order+1);
-      const IntegrationRule *irs[Geometry::NumGeom];
-      for (int i=0; i < Geometry::NumGeom; ++i)
-      {
-         irs[i] = &(IntRules.Get(i, order_quad));
-      }
-
-      ParMesh *pmesh = fespace->GetParMesh();
-
-      // Check error of real part
-
-      E_gf.SetFromTrueDofs(trueSol->GetBlock(0));
-      double Error_E = E_gf.ComputeL2Error(Eex, irs);
-      double norm_E = ComputeGlobalLpNorm(2, Eex, *pmesh, irs);
-
-      cout << myid << ": real error " << Error_E << " relative to " << norm_E << endl;
-
-      {
-         ostringstream mesh_name, sol_name;
-         mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-         sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-         ofstream mesh_ofs(mesh_name.str().c_str());
-         mesh_ofs.precision(8);
-         pmesh->Print(mesh_ofs);
-
-         ofstream sol_ofs(sol_name.str().c_str());
-         sol_ofs.precision(8);
-         E_gf.Save(sol_ofs);
-      }
-
-      // Check error of imaginary part
-      E_gf.SetFromTrueDofs(trueSol->GetBlock(2));
-      const double iml2 = E_gf.Norml2();
-      E_gf *= ci == 0.0 ? 0.0 : (1.0 / ci);
-      Error_E = E_gf.ComputeL2Error(Eex, irs);
-
-      cout << myid << ": imag error " << Error_E << " relative to " << norm_E <<
-           ", l2 norm " << iml2 << endl;
-   }
-
-private:
-
-   BlockVector *trueRhs, *trueSol;
-
-   Array<int> block_trueOffsets;
-
-   ParBilinearForm *bM, *bM_eps;
-   ParMixedBilinearForm *bM_curl;
-
-   HypreParMatrix M, M_eps;
-   HypreParMatrix *M_curl;
-
-   CGSolver M_inv;
-
-   const int n;
-   const int nfull;
-   const int myid;
-
-   mutable Vector z, Minv_x;
-
-   CGSolver LSpcg;
-
-   STRUMPACKSolver *invLSH;
-   HypreParMatrix *LSH;
-
-   ParFiniteElementSpace *fespace;
-};
-
-int main(int argc, char *argv[])
-{
-   StopWatch chrono;
-
-   // 1. Initialize MPI
-   int num_procs, myid;
-   MPI_Init(&argc, &argv); // Initialize MPI
-   MPI_Comm_size(MPI_COMM_WORLD,
-                 &num_procs); //total number of processors available
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid); // Determine process identifier
-   // 1. Parse command-line options.
-   // geometry file
-   const char *mesh_file = "../data/inline-hex.mesh";
-   // finite element order of approximation
-   int order = 1;
-   // static condensation flag
-   bool static_cond = false;
-   // visualization flag
-   bool visualization = 1;
-   // number of wavelengths
-   double k = 1.0;
-   // number of mg levels
-   int ref_levels = 1;
-   // number of initial ref
-   int initref = 1;
-
-   // optional command line inputs
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&k, "-k", "--wavelengths",
-                  "Number of wavelengths.");
-   args.AddOption(&ref_levels, "-ref", "--ref_levels",
-                  "Number of Refinements.");
-   args.AddOption(&initref, "-initref", "--initref",
-                  "Number of initial refinements.");
-   args.AddOption(&sol, "-sol", "--exact",
-                  "Exact solution flag - "
-                  " 1:sinusoidal, 2: point source, 3: plane wave");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   // check if the inputs are correct
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // Angular frequency
-   //omega = 2.0*k*M_PI;
-   omega = k;
-
-   // 2. Read the mesh from the given mesh file.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-
-   if (sol == 4)
-   {
-      mesh->EnsureNodes();
-      GridFunction *nodes = mesh->GetNodes();
-      (*nodes) *= 0.5;
-   }
-
-   dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 3. Executing uniform h-refinement
-   for (int i = 0; i < initref; i++ )
-   {
-      mesh->UniformRefinement();
-   }
-
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-
-   // 4. Define a finite element space on the mesh.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   std::vector<ParFiniteElementSpace * > fespaces(ref_levels+1);
-   std::vector<ParMesh * > ParMeshes(ref_levels+1);
-   std::vector<HypreParMatrix*> P(ref_levels);
-
-   for (int i = 0; i < ref_levels; i++)
-   {
-      ParMeshes[i] =new ParMesh(*pmesh);
-      fespaces[i] = new ParFiniteElementSpace(*fespace, *ParMeshes[i]);
-      pmesh->UniformRefinement();
-      // Update fespace
-      fespace->Update();
-      OperatorHandle Tr(Operator::Hypre_ParCSR);
-      fespace->GetTrueTransferOperator(*fespaces[i], Tr);
-      Tr.SetOperatorOwner(false);
-      Tr.Get(P[i]);
-   }
-   fespaces[ref_levels] = new ParFiniteElementSpace(*fespace);
-
-   FOSLSSolver fosls(fespace, P, myid);
-
-   fosls.ComplexFOSLSTest();
-
-   for (auto p: ParMeshes) { delete p; }
-   for (auto p: fespaces) { delete p; }
-   for (auto p: P) { delete p; }
-   ParMeshes.clear();
-   fespaces.clear();
-   P.clear();
-   delete fec;
-   delete fespace;
-   delete pmesh;
-
-   MPI_Finalize();
-   return 0;
-}
-
-//define exact solution
-void E_exact(const Vector &x, Vector &E)
-{
-   double curlE[3], curl2E[3];
-   get_maxwell_solution(x, E, curlE, curl2E);
-}
-
-void H_exact(const Vector &x, Vector &H)
-{
-   double E[3], curlE[3], curl2E[3];
-   get_maxwell_solution(x, E, curlE, curl2E);
-   for (int i = 0; i<3; i++) { H(i) = curlE[i]/omega; }
-}
-
-
-void f_exact_H(const Vector &x, Vector &f)
-{
-   // curl H - omega E = f
-   // = curl (curl E / omega) - omega E
-   f = 0.0;
-   if (sol !=4)
-   {
-      double E[3], curlE[3], curl2E[3];
-      get_maxwell_solution(x, E, curlE, curl2E);
-      f(0) = curl2E[0] / omega - omega * E[0];
-      f(1) = curl2E[1] / omega - omega * E[1];
-      f(2) = curl2E[2] / omega - omega * E[2];
-   }
-}
-
-void get_maxwell_solution(const Vector &X, double E[], double curlE[],
-                          double curl2E[])
-{
-   double x = X[0];
-   double y = X[1];
-   double z = X[2];
-
-   if (sol ==-1)
-   {
-      E[0] = y * z * (1.0 - y) * (1.0 - z);
-      E[1] = x * y * z * (1.0 - x) * (1.0 - z);
-      E[2] = x * y * (1.0 - x) * (1.0 - y);
-
-      curlE[0] = -(x-1.0) * x * (y*(2.0*z-3.0)+1.0);
-      curlE[1] = -2.0*(y-1.0)*y*(x-z);
-      curlE[2] = (z-1)*z*(1.0+y*(2.0*x-3.0));
-
-      curl2E[0] = 2.0 * y * (1.0 - y) - (2.0 * x - 3.0) * z * (1 - z);
-      curl2E[1] = 2.0 * y * (x * (1.0 - x) + (1.0 - z) * z);
-      curl2E[2] = 2.0 * y * (1.0 - y) + x * (3.0 - 2.0 * z) * (1.0 - x);
-   }
-   else if (sol == 0) // polynomial
-   {
-      // Polynomial vanishing on the boundary
-      E[0] = y * z * (1.0 - y) * (1.0 - z);
-      E[1] = (1.0 - x) * x * y * (1.0 - z) * z;
-      E[2] = (1.0 - x) * x * (1.0 - y) * y;
-      //
-      curlE[0] = -(-1.0 + x) * x * (1.0 + y * (-3.0 + 2.0 * z));
-      curlE[1] = -2.0 * (-1.0 + y) * y * (x - z);
-      curlE[2] = (1.0 + (-3.0 + 2.0 * x) * y) * (-1.0 + z) * z;
-
-      curl2E[0] = -2.0 * (-1.0 + y) * y + (-3.0 + 2.0 * x) * (-1.0 + z) * z;
-      curl2E[1] = -2.0 * y * (-x + x * x + (-1.0 + z) * z);
-      curl2E[2] = -2.0 * (-1.0 + y) * y + (-1.0 + x) * x * (-3.0 + 2.0 * z);
-   }
-   else if (sol == 1) // sinusoidal
-   {
-      E[0] = sin(omega * y);
-      E[1] = sin(omega * z);
-      E[2] = sin(omega * x);
-
-      curlE[0] = -omega * cos(omega * z);
-      curlE[1] = -omega * cos(omega * x);
-      curlE[2] = -omega * cos(omega * y);
-
-      curl2E[0] = omega * omega * E[0];
-      curl2E[1] = omega * omega * E[1];
-      curl2E[2] = omega * omega * E[2];
-   }
-   else if (sol == 2) // point source
-   {
-      // shift to avoid singularity
-      double x0 = x + 0.1;
-      double x1 = y + 0.1;
-      double x2 = z + 0.1;
-      double r = sqrt(x0 * x0 + x1 * x1 + x2 * x2);
-
-      E[0] = cos(omega * r);
-      E[1] = 0.0;
-      E[2] = 0.0;
-
-      double r_x = x0 / r;
-      double r_y = x1 / r;
-      double r_z = x2 / r;
-      double r_xy = -(r_x / r) * r_y;
-      double r_xz = -(r_x / r) * r_z;
-      double r_yx = r_xy;
-      double r_yy = (1.0 / r) * (1.0 - r_y * r_y);
-      double r_zx = r_xz;
-      double r_zz = (1.0 / r) * (1.0 - r_z * r_z);
-
-      curlE[0] = 0.0;
-      curlE[1] = -omega * r_z * sin(omega * r);
-      curlE[2] =  omega * r_y * sin(omega * r);
-
-      curl2E[0] = omega * ((r_yy + r_zz) * sin(omega * r) +
-                           (omega * r_y * r_y + omega * r_z * r_z) * cos(omega * r));
-      curl2E[1] = -omega * (r_yx * sin(omega * r) + omega * r_y * r_x * cos(
-                               omega * r));
-      curl2E[2] = -omega * (r_zx * sin(omega * r) + omega * r_z * r_x * cos(
-                               omega * r));
-   }
-   else if (sol == 3) // plane wave
-   {
-      double coeff = omega / sqrt(3.0);
-      E[0] = cos(coeff * (x + y + z));
-      E[1] = 0.0;
-      E[2] = 0.0;
-
-      curlE[0] = 0.0;
-      curlE[1] = -coeff * sin(coeff * (x + y + z));
-      curlE[2] = coeff * sin(coeff * (x + y + z));
-
-      curl2E[0] = 2.0 * coeff * coeff * E[0];
-      curl2E[1] = -coeff * coeff * E[0];
-      curl2E[2] = -coeff * coeff * E[0];
-   }
-   else if (sol == -1)
-   {
-      E[0] = cos(omega * y);
-      E[1] = 0.0;
-
-      curlE[0] = 0.0;
-      curlE[1] = 0.0;
-      curlE[2] = -omega * sin(omega * y);
-
-      curl2E[0] = omega*omega * cos(omega*y);
-      curl2E[1] = 0.0;
-      curl2E[2] = 0.0;
-   }
-   else if (sol == 4) // Airy function
-   {
-      E[0] = 0;
-      E[1] = 0;
-      // double b = -pow(omega/4.0,2.0/3.0)*(4.0*x(0)-1.0);
-      double b = -pow(omega/4.0,2.0/3.0)*(4.0*x-1.0);
-      //E[2] = boost::math::airy_ai(b);
-      E[2] = gsl_sf_airy_Ai(b, GSL_PREC_DOUBLE);
-
-      curlE[0] = 0.0;
-      curlE[1] = 4.0 * pow(omega/4.0,2.0/3.0) * gsl_sf_airy_Ai_deriv(b,
-                                                                     GSL_PREC_DOUBLE);
-      curlE[2] = 0.0;
-
-      // not used
-      curl2E[0] = 0.0;
-      curl2E[1] = 0.0;
-      curl2E[2] = 0.0;
-   }
-}
-
-void epsilon_func(const Vector &x, Vector &M)
-{
-   M.SetSize(3);
-
-   M = 1.0;
-   if (sol == 4)
-   {
-      M[2] = 4.0*x(0)-1.0;
-   }
-}
-
-void epsilon2_func(const Vector &x, Vector &M)
-{
-   M.SetSize(3);
-
-   M = 1.0;
-   if (sol == 4)
-   {
-      M[2] = (4.0*x(0)-1.0) * (4.0*x(0)-1.0);
-   }
-}
-
-void epsilon_func_mat(const Vector &x, DenseMatrix &M)
-{
-   M.SetSize(3);
-
-   M = 0.0;
-   M(0,0) = 1.0;
-   M(1,1) = 1.0;
-   if (sol != 4)
-   {
-      M(2,2) = 1.0;
-   }
-   else
-   {
-      M(2,2) = 4.0*x(0)-1.0;
-   }
-}

examples/ex3p_fosls.cpp

@@ -1,764 +0,0 @@
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-#include "gsl_sf_airy.h"
-
-#include "multigrid.hpp"
-
-using namespace std;
-using namespace mfem;
-
-#define COMPLEX_IMPEDANCE  // Whether to use impedance BC and solve a complex system
-
-// Define exact solution
-void E_exact(const Vector & x, Vector & E);
-void H_exact(const Vector & x, Vector & H);
-void f_exact_H(const Vector & x, Vector & f_H);
-void get_maxwell_solution(const Vector & x, double E[], double curlE[],
-                          double curl2E[]);
-void epsilon_func(const Vector &x, DenseMatrix &M);
-void epsilon2_func(const Vector &x, DenseMatrix &M);
-
-int dim;
-double omega;
-int sol = 4;
-
-int main(int argc, char *argv[])
-{
-   StopWatch chrono;
-
-   // 1. Initialize MPI
-   int num_procs, myid;
-   MPI_Init(&argc, &argv); // Initialize MPI
-   MPI_Comm_size(MPI_COMM_WORLD,
-                 &num_procs); //total number of processors available
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid); // Determine process identifier
-   // 1. Parse command-line options.
-   // geometry file
-   const char *mesh_file = "../data/inline-hex.mesh";
-   // finite element order of approximation
-   int order = 1;
-   // static condensation flag
-   bool static_cond = false;
-   // visualization flag
-   bool visualization = 1;
-   // number of wavelengths
-   double k = 1.0;
-   // number of mg levels
-   int ref_levels = 1;
-   // number of initial ref
-   int initref = 1;
-
-   // optional command line inputs
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&k, "-k", "--wavelengths",
-                  "Number of wavelengths.");
-   args.AddOption(&ref_levels, "-ref", "--ref_levels",
-                  "Number of Refinements.");
-   args.AddOption(&initref, "-initref", "--initref",
-                  "Number of initial refinements.");
-   args.AddOption(&sol, "-sol", "--exact",
-                  "Exact solution flag - "
-                  " 1:sinusoidal, 2: point source, 3: plane wave");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   // check if the inputs are correct
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // Angular frequency
-   //omega = 2.0*k*M_PI;
-   omega = k;
-
-   // 2. Read the mesh from the given mesh file.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-
-   if (sol == 4)
-   {
-      mesh->EnsureNodes();
-      GridFunction *nodes = mesh->GetNodes();
-      (*nodes) *= 0.5;
-   }
-
-   dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 3. Executing uniform h-refinement
-   for (int i = 0; i < initref; i++ )
-   {
-      mesh->UniformRefinement();
-   }
-
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-
-   // 4. Define a finite element space on the mesh.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   std::vector<ParFiniteElementSpace * > fespaces(ref_levels+1);
-   std::vector<ParMesh * > ParMeshes(ref_levels+1);
-   std::vector<HypreParMatrix*> P(ref_levels);
-
-   for (int i = 0; i < ref_levels; i++)
-   {
-      ParMeshes[i] =new ParMesh(*pmesh);
-      fespaces[i] = new ParFiniteElementSpace(*fespace, *ParMeshes[i]);
-      pmesh->UniformRefinement();
-      // Update fespace
-      fespace->Update();
-      OperatorHandle Tr(Operator::Hypre_ParCSR);
-      fespace->GetTrueTransferOperator(*fespaces[i], Tr);
-      Tr.SetOperatorOwner(false);
-      Tr.Get(P[i]);
-   }
-   fespaces[ref_levels] = new ParFiniteElementSpace(*fespace);
-
-   Array<int> ess_tdof_listE;
-   Array<int> ess_tdof_listH;
-   Array<int> ess_bdrE(pmesh->bdr_attributes.Max());
-   Array<int> ess_bdrH(pmesh->bdr_attributes.Max());
-#ifdef COMPLEX_IMPEDANCE
-   ess_bdrE = 0;
-   ess_bdrH = 0;  // Neumann
-#else
-   ess_bdrE = 1;
-   ess_bdrH = 0;  // Neumann
-#endif
-   fespace->GetEssentialTrueDofs(ess_bdrE, ess_tdof_listE);
-   fespace->GetEssentialTrueDofs(ess_bdrH, ess_tdof_listH);
-
-#ifdef COMPLEX_IMPEDANCE
-   Array<int> bdr_attr(pmesh->bdr_attributes.Max());
-   bdr_attr = 1;
-   ConstantCoefficient impedance(omega);
-   RestrictedCoefficient imp_rest(impedance, bdr_attr);
-   Array<int> block_offsets(5);
-   Array<int> block_trueOffsets(5);
-#else
-   Array<int> block_offsets(3);
-   Array<int> block_trueOffsets(3);
-#endif
-   block_offsets[0] = 0;
-   block_offsets[1] = fespace->GetVSize();
-   block_offsets[2] = fespace->GetVSize();
-#ifdef COMPLEX_IMPEDANCE
-   block_offsets[3] = fespace->GetVSize();
-   block_offsets[4] = fespace->GetVSize();
-#endif
-   block_offsets.PartialSum();
-
-   block_trueOffsets[0] = 0;
-   block_trueOffsets[1] = fespace->TrueVSize();
-   block_trueOffsets[2] = fespace->TrueVSize();
-#ifdef COMPLEX_IMPEDANCE
-   block_trueOffsets[3] = fespace->TrueVSize();
-   block_trueOffsets[4] = fespace->TrueVSize();
-#endif
-   block_trueOffsets.PartialSum();
-
-   //    _           _    _  _       _  _
-   //   |             |  |    |     |    |
-   //   |  A00   A01  |  | E  |     |F_E |
-   //   |             |  |    |  =  |    |
-   //   |  A10   A11  |  | H  |     |F_G |
-   //   |_           _|  |_  _|     |_  _|
-   //
-   // A00 = (curl E, curl F) + \omega^2 (E,F)
-   // A01 = - \omega *( (curl E, F) + (E,curl F)
-   // A10 = - \omega *( (curl H, G) + (H,curl G)
-   // A11 = (curl H, curl G) + \omega^2 (H,G)
-
-   BlockVector x(block_offsets), rhs(block_offsets);
-   BlockVector trueX(block_trueOffsets), trueRhs(block_trueOffsets);
-
-   x = 0.0;
-   rhs = 0.0;
-   trueX = 0.0;
-   trueRhs = 0.0;
-
-   VectorFunctionCoefficient Eex(sdim, E_exact);
-   ConstantCoefficient negOne(-1.0);
-   VectorFunctionCoefficient mEex(sdim, E_exact, &negOne);
-
-   ParGridFunction * E_gf = new ParGridFunction;
-   ParGridFunction * Exact_gf = new ParGridFunction(fespace);
-   E_gf->MakeRef(fespace, x.GetBlock(0));
-   E_gf->ProjectCoefficient(Eex);
-   Exact_gf->ProjectCoefficient(Eex);
-
-   VectorFunctionCoefficient Hex(sdim, H_exact);
-   ParGridFunction * H_gf = new ParGridFunction;
-   H_gf->MakeRef(fespace, x.GetBlock(1));
-   H_gf->ProjectCoefficient(Hex);
-
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient sigma(pow(omega, 2));
-   ConstantCoefficient neg(-abs(omega));
-   ConstantCoefficient pos(abs(omega));
-
-   MatrixFunctionCoefficient epsilon(dim,epsilon_func);
-   TransposeMatrixCoefficient epsilonT(epsilon);  // transpose of epsilon
-   MatrixFunctionCoefficient epsilon2(dim,epsilon2_func);
-   ScalarMatrixProductCoefficient coeff(neg,epsilon);
-   ScalarMatrixProductCoefficient coeff2(sigma,epsilon2);
-
-   // 6. Set up the linear form
-   VectorFunctionCoefficient f_H(sdim,f_exact_H);
-   ScalarVectorProductCoefficient sf_H(neg,f_H);
-
-   MatVecCoefficient epsT_sf_H(epsilonT, sf_H);
-
-   ParLinearForm *b_E = new ParLinearForm;
-   b_E->Update(fespace, rhs.GetBlock(0), 0);
-   b_E->AddDomainIntegrator(new VectorFEDomainLFIntegrator(epsT_sf_H));
-#ifdef COMPLEX_IMPEDANCE
-   b_E->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
-                                 Eex));  // <g_Im, n x Q x n> = <n x E_Re x n, n x Q x n>
-
-   ParLinearForm *b_E_Im = new ParLinearForm;
-   b_E_Im->Update(fespace, rhs.GetBlock(2), 0);
-   b_E_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
-                                    Hex));  // -<g_Re, n x Q x n> = <n x H_Re, n x Q x n>
-   b_E_Im->Assemble();
-#endif
-   b_E->Assemble();
-
-   ParLinearForm *b_H = new ParLinearForm;
-   b_H->Update(fespace, rhs.GetBlock(1), 0);
-   b_H->AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(f_H));
-#ifdef COMPLEX_IMPEDANCE
-   b_H->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
-                                 Hex));  // -<g_Re, n x R> = <n x H_Re, n x R>
-
-   ParLinearForm *b_H_Im = new ParLinearForm;
-   b_H_Im->Update(fespace, rhs.GetBlock(3), 0);
-   b_H_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
-                                    mEex));  // -<g_Im, n x R> = -<n x E_Re x n, n x R>
-   b_H_Im->Assemble();
-#endif
-   b_H->Assemble();
-
-#ifdef COMPLEX_IMPEDANCE
-   fespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(2),
-                                                   trueRhs.GetBlock(2));
-   fespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(3),
-                                                   trueRhs.GetBlock(3));
-#endif
-
-   // 7. Bilinear form a(.,.) on the finite element space
-   ParBilinearForm *a_EE = new ParBilinearForm(fespace);
-   a_EE->AddDomainIntegrator(new CurlCurlIntegrator(one));
-   a_EE->AddDomainIntegrator(new VectorFEMassIntegrator(coeff2));
-
-#ifdef COMPLEX_IMPEDANCE
-   a_EE->AddBoundaryIntegrator(new VectorFEMassIntegrator());
-   //a_EE->AddBoundaryIntegrator(new VectorFEMassIntegrator(imp_rest));
-   //a_EE->AddBoundaryIntegrator(new BoundaryMassIntegrator(imp_rest));
-   //a_EE->AddBdrFaceIntegrator(new BoundaryMassIntegrator(imp_rest));
-#endif
-
-   a_EE->Assemble();
-   a_EE->Finalize();
-   HypreParMatrix *A_EE = new HypreParMatrix;
-   a_EE->FormLinearSystem(ess_tdof_listE, x.GetBlock(0), rhs.GetBlock(0), *A_EE,
-                          trueX.GetBlock(0), trueRhs.GetBlock(0));
-
-   ParBilinearForm *a_HH = new ParBilinearForm(fespace);
-   a_HH->AddDomainIntegrator(new CurlCurlIntegrator(one)); // one is the coeff
-   a_HH->AddDomainIntegrator(new VectorFEMassIntegrator(sigma));
-
-#ifdef COMPLEX_IMPEDANCE
-   a_HH->AddBoundaryIntegrator(new VectorFEMassIntegrator());
-   //a_HH->AddBoundaryIntegrator(new VectorFEMassIntegrator(imp_rest));
-   //a_HH->AddBoundaryIntegrator(new BoundaryMassIntegrator(imp_rest));
-#endif
-
-   a_HH->Assemble();
-   a_HH->Finalize();
-   HypreParMatrix *A_HH = new HypreParMatrix;
-   a_HH->FormLinearSystem(ess_tdof_listH, x.GetBlock(1), rhs.GetBlock(1), *A_HH,
-                          trueX.GetBlock(1), trueRhs.GetBlock(1));
-
-   ParMixedBilinearForm *a_HE = new ParMixedBilinearForm(fespace,fespace);
-   a_HE->AddDomainIntegrator(new MixedVectorCurlIntegrator(neg));
-   a_HE->AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(coeff));
-   a_HE->Assemble();
-   a_HE->Finalize();
-   HypreParMatrix *A_HE = new HypreParMatrix;
-   a_HE->FormColLinearSystem(ess_tdof_listE,x.GetBlock(0),rhs.GetBlock(1),*A_HE,
-                             trueX.GetBlock(0),trueRhs.GetBlock(1));
-
-   HypreParMatrix *A_EH = A_HE->Transpose();
-
-#ifdef COMPLEX_IMPEDANCE
-   ParBilinearForm *a_EH_Im = new ParBilinearForm(fespace);
-   //a_EH_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator(imp_rest));
-   //a_EH_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator(impedance));
-   //a_EH_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator(omega));
-   a_EH_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator());
-   a_EH_Im->Assemble();
-   a_EH_Im->Finalize();
-
-   OperatorHandle A_EH_Im_ptr;
-   a_EH_Im->FormSystemMatrix(ess_tdof_listE,
-                             A_EH_Im_ptr);  // empty ess_tdof_list for impedance
-
-   HypreParMatrix *A_EH_Im = A_EH_Im_ptr.As<HypreParMatrix>();
-
-   HypreParMatrix *A_HE_Im = A_EH_Im->Transpose();
-#endif
-
-   BlockOperator *LS_Maxwellop = new BlockOperator(block_trueOffsets);
-#ifdef COMPLEX_IMPEDANCE
-   const int numBlocks = 4;
-#else
-   const int numBlocks = 2;
-#endif
-
-   LS_Maxwellop->SetBlock(0, 0, A_EE);
-   LS_Maxwellop->SetBlock(0, 1, A_EH);
-   LS_Maxwellop->SetBlock(1, 0, A_HE);
-   LS_Maxwellop->SetBlock(1, 1, A_HH);
-
-#ifdef COMPLEX_IMPEDANCE
-   LS_Maxwellop->SetBlock(0, 3, A_EH_Im);
-   LS_Maxwellop->SetBlock(1, 2, A_HE_Im, -1.0);
-
-   LS_Maxwellop->SetBlock(2, 1, A_EH_Im, -1.0);
-   LS_Maxwellop->SetBlock(3, 0, A_HE_Im);
-
-   LS_Maxwellop->SetBlock(2, 2, A_EE);
-   LS_Maxwellop->SetBlock(2, 3, A_EH);
-   LS_Maxwellop->SetBlock(3, 2, A_HE);
-   LS_Maxwellop->SetBlock(3, 3, A_HH);
-#endif
-
-   if (myid == 0)
-   {
-      cout << "Size of fine grid system: "
-           << 2.0 * A_EE->GetGlobalNumRows() << " x " << 2.0* A_EE->GetGlobalNumCols() <<
-           endl;
-   }
-
-   // Set up the preconditioner
-   Array2D<HypreParMatrix*> blockA(numBlocks, numBlocks);
-   Array2D<double> blockAcoef(numBlocks, numBlocks);
-
-   for (int i=0; i<numBlocks; ++i)
-   {
-      for (int j=0; j<numBlocks; ++j)
-      {
-         if (LS_Maxwellop->IsZeroBlock(i,j) == 0)
-         {
-            blockA(i,j) = static_cast<HypreParMatrix *>(&LS_Maxwellop->GetBlock(i,j));
-            blockAcoef(i,j) = LS_Maxwellop->GetBlockCoef(i,j);
-         }
-         else
-         {
-            blockA(i,j) = NULL;
-            blockAcoef(i,j) = 1.0;
-         }
-      }
-   }
-
-   // // double nnz = A_HH->NNZ();
-   // // double ndof = A_HH->GetGlobalNumRows();
-   // // double est_mem_b = nnz*12.0 + (ndof+1.0)*4;
-   // // double gb = est_mem_b*4.0/pow(1024.0,3);
-
-   // // mfem::out << "Estimated memory taken by the global matrix: " <<  gb << endl;
-
-   int maxit(2000);
-   double rtol(1.e-8);
-   double atol(1.e-12);
-
-   // trueX = 0.0;
-   CGSolver pcg(MPI_COMM_WORLD);
-   pcg.SetAbsTol(atol);
-   pcg.SetRelTol(rtol);
-   pcg.SetMaxIter(maxit);
-   pcg.SetOperator(*LS_Maxwellop);
-   pcg.SetPrintLevel(1);
-   chrono.Clear();
-   chrono.Start();
-   BlockMGSolver * precMG = new BlockMGSolver(LS_Maxwellop->Height(),
-                                              LS_Maxwellop->Width(), blockA, blockAcoef, P);
-   //precMG->SetTheta(0.5);
-   // // int lv_coarse = min(ref_levels,ref_levels-1);
-   // // int levels = ref_levels - lv_coarse;
-   // // BlkParSchwarzSmoother * precAS = new BlkParSchwarzSmoother(fespaces[lv_coarse]->GetParMesh(),levels,fespaces[ref_levels],LS_Maxwellop);
-   chrono.Stop();
-   if (myid == 0)
-   {
-      cout << "MG Setup time: " << chrono.RealTime() << endl;
-   }
-
-   // Randomize H RHS
-   //trueRhs = 1.0;
-   /*
-   for (int i=block_trueOffsets[1]; i<block_trueOffsets[2]; ++i)
-     trueRhs[i] = i % 53;
-   */
-
-   chrono.Clear();
-   chrono.Start();
-   pcg.SetPreconditioner(*precMG);
-   // // pcg.SetPreconditioner(*precAS);
-   pcg.Mult(trueRhs, trueX);
-   chrono.Stop();
-   delete precMG;
-   // // delete precAS;
-
-   // // trueX = 0.0;
-   // // invA->Mult(trueRhs,trueX);
-
-   if (myid == 0)
-   {
-      cout << "MG Solution time time: " << chrono.RealTime() << endl;
-   }
-   // // cin.get();
-   // //  if(myid == 0)
-   // //    cout << "MG prec Solution time: " << chrono.RealTime() << endl;
-
-
-   // // chrono.Clear();
-   // // chrono.Start();
-   // // Block_AMSSolver * precAMS = new Block_AMSSolver(block_trueOffsets,fespaces);
-   // // precAMS->SetSmootherType(Block_AMS::BlkSmootherType::SCHWARZ);
-   // // precAMS->SetSmootherType(Block_AMS::BlkSmootherType::HYPRE);
-   // // precAMS->SetOperator(LS_Maxwellop);
-   // // precAMS->SetTheta(1.0/5.0);
-   // // // 0-Smoother, 1-Grad, 2,3,4-Pix,Piy,Piz
-   // // precAMS->SetCycleType("023414320");
-   // // precAMS->SetNumberofCycles(1);
-   // // chrono.Stop();
-   // // if(myid == 0)
-   // //    cout << "BlkAMS Setup time: " << chrono.RealTime() << endl;
-
-   // // // resolve with block AMS
-   // // trueX = 0;
-   // // chrono.Clear();
-   // // chrono.Start();
-   // // pcg.SetPreconditioner(*precAMS);
-   // // pcg.Mult(trueRhs, trueX);
-   // // chrono.Stop();
-   // // delete precAMS;
-
-   // // if(myid == 0)
-   // //    cout << "BlockAMS Solution time: " << chrono.RealTime() << endl;
-
-
-   a_EE->RecoverFEMSolution(trueX.GetBlock(0), *b_E, *E_gf);
-   a_HH->RecoverFEMSolution(trueX.GetBlock(1), *b_H, *H_gf);
-
-
-   int order_quad = max(2, 2*order+1);
-   const IntegrationRule *irs[Geometry::NumGeom];
-   for (int i=0; i < Geometry::NumGeom; ++i)
-   {
-      irs[i] = &(IntRules.Get(i, order_quad));
-   }
-
-   double Error_E = E_gf->ComputeL2Error(Eex, irs);
-   double norm_E = ComputeGlobalLpNorm(2, Eex, *pmesh, irs);
-
-   double Error_H = H_gf->ComputeL2Error(Hex, irs);
-   double norm_H = ComputeGlobalLpNorm(2, Hex , *pmesh, irs);
-
-   if (myid == 0)
-   {
-      cout << "|| E_h - E || = " << Error_E  << "\n";
-      cout << "|| E_h - E ||/||E|| = " << Error_E/norm_E  << "\n";
-      cout << "|| H_h - H || = " << Error_H  << "\n";
-      cout << "|| H_h - H ||/||H|| = " << Error_H/norm_H  << "\n";
-      cout << "Total error = " << setprecision(15) << sqrt(Error_H*Error_H
-                                                           +Error_E*Error_E) << "\n";
-   }
-
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      E_gf->Save(sol_ofs);
-   }
-
-   //ParGridFunction ExactE(fespace);
-   /*
-   if (visualization)
-     {
-       // 8. Connect to GLVis.
-       char vishost[] = "localhost";
-       int  visport   = 19916;
-       // socketstream E_sock(vishost, visport);
-       // E_sock << "parallel " << num_procs << " " << myid << "\n";
-       // E_sock.precision(8);
-       // E_sock << "solution\n" << *pmesh << *E_gf << "window_title 'Electric field'" << endl;
-       socketstream Exact_sock(vishost, visport);
-       Exact_sock << "parallel " << num_procs << " " << myid << "\n";
-       Exact_sock.precision(8);
-       Exact_sock << "solution\n" << *pmesh << *Exact_gf << "window_title 'Electric field'" << endl;
-
-       // MPI_Barrier(pmesh->GetComm());
-       // socketstream Eex_sock(vishost, visport);
-       // Eex_sock << "parallel " << num_procs << " " << myid << "\n";
-       // Eex_sock.precision(8);
-       // Eex_sock << "solution\n" << *pmesh << *Exact_gf << "window_title 'Exact Electric field'" << endl;
-     }
-   */
-   // delete A_EE;
-   // delete A_HE;
-   // delete A_EH;
-   // delete A_HH;
-   // delete LS_Maxwellop;
-   // delete a_EE;
-   // delete a_HE;
-   // delete a_HH;
-   // delete b_E;
-   // delete b_H;
-   delete E_gf;
-   delete Exact_gf;
-   for (auto p: ParMeshes) { delete p; }
-   for (auto p: fespaces) { delete p; }
-   for (auto p: P) { delete p; }
-   ParMeshes.clear();
-   fespaces.clear();
-   P.clear();
-   delete fec;
-   delete fespace;
-   delete pmesh;
-
-   // cout << "Freed memory: " << endl;
-   // cin.get();
-   MPI_Finalize();
-   return 0;
-}
-
-//define exact solution
-void E_exact(const Vector &x, Vector &E)
-{
-   double curlE[3], curl2E[3];
-   get_maxwell_solution(x, E, curlE, curl2E);
-}
-
-void H_exact(const Vector &x, Vector &H)
-{
-   double E[3], curlE[3], curl2E[3];
-   get_maxwell_solution(x, E, curlE, curl2E);
-   for (int i = 0; i<3; i++) { H(i) = curlE[i]/omega; }
-}
-
-
-void f_exact_H(const Vector &x, Vector &f)
-{
-   // curl H - omega E = f
-   // = curl (curl E / omega) - omega E
-   f = 0.0;
-   if (sol !=4)
-   {
-      double E[3], curlE[3], curl2E[3];
-      get_maxwell_solution(x, E, curlE, curl2E);
-      f(0) = curl2E[0] / omega - omega * E[0];
-      f(1) = curl2E[1] / omega - omega * E[1];
-      f(2) = curl2E[2] / omega - omega * E[2];
-   }
-}
-
-void get_maxwell_solution(const Vector &X, double E[], double curlE[],
-                          double curl2E[])
-{
-   double x = X[0];
-   double y = X[1];
-   double z = X[2];
-
-
-
-   if (sol ==-1)
-   {
-      E[0] = y * z * (1.0 - y) * (1.0 - z);
-      E[1] = x * y * z * (1.0 - x) * (1.0 - z);
-      E[2] = x * y * (1.0 - x) * (1.0 - y);
-
-      curlE[0] = -(x-1.0) * x * (y*(2.0*z-3.0)+1.0);
-      curlE[1] = -2.0*(y-1.0)*y*(x-z);
-      curlE[2] = (z-1)*z*(1.0+y*(2.0*x-3.0));
-
-      curl2E[0] = 2.0 * y * (1.0 - y) - (2.0 * x - 3.0) * z * (1 - z);
-      curl2E[1] = 2.0 * y * (x * (1.0 - x) + (1.0 - z) * z);
-      curl2E[2] = 2.0 * y * (1.0 - y) + x * (3.0 - 2.0 * z) * (1.0 - x);
-   }
-   else if (sol == 0) // polynomial
-   {
-      // Polynomial vanishing on the boundary
-      E[0] = y * z * (1.0 - y) * (1.0 - z);
-      E[1] = (1.0 - x) * x * y * (1.0 - z) * z;
-      E[2] = (1.0 - x) * x * (1.0 - y) * y;
-      //
-      curlE[0] = -(-1.0 + x) * x * (1.0 + y * (-3.0 + 2.0 * z));
-      curlE[1] = -2.0 * (-1.0 + y) * y * (x - z);
-      curlE[2] = (1.0 + (-3.0 + 2.0 * x) * y) * (-1.0 + z) * z;
-
-      curl2E[0] = -2.0 * (-1.0 + y) * y + (-3.0 + 2.0 * x) * (-1.0 + z) * z;
-      curl2E[1] = -2.0 * y * (-x + x * x + (-1.0 + z) * z);
-      curl2E[2] = -2.0 * (-1.0 + y) * y + (-1.0 + x) * x * (-3.0 + 2.0 * z);
-   }
-   else if (sol == 1) // sinusoidal
-   {
-      E[0] = sin(omega * y);
-      E[1] = sin(omega * z);
-      E[2] = sin(omega * x);
-
-      curlE[0] = -omega * cos(omega * z);
-      curlE[1] = -omega * cos(omega * x);
-      curlE[2] = -omega * cos(omega * y);
-
-      curl2E[0] = omega * omega * E[0];
-      curl2E[1] = omega * omega * E[1];
-      curl2E[2] = omega * omega * E[2];
-   }
-   else if (sol == 2) // point source
-   {
-      // shift to avoid singularity
-      double x0 = x + 0.1;
-      double x1 = y + 0.1;
-      double x2 = z + 0.1;
-      //
-      double r = sqrt(x0 * x0 + x1 * x1 + x2 * x2);
-
-      E[0] = cos(omega * r);
-      E[1] = 0.0;
-      E[2] = 0.0;
-
-      double r_x = x0 / r;
-      double r_y = x1 / r;
-      double r_z = x2 / r;
-      double r_xy = -(r_x / r) * r_y;
-      double r_xz = -(r_x / r) * r_z;
-      double r_yx = r_xy;
-      double r_yy = (1.0 / r) * (1.0 - r_y * r_y);
-      double r_zx = r_xz;
-      double r_zz = (1.0 / r) * (1.0 - r_z * r_z);
-
-      curlE[0] = 0.0;
-      curlE[1] = -omega * r_z * sin(omega * r);
-      curlE[2] =  omega * r_y * sin(omega * r);
-
-      curl2E[0] = omega * ((r_yy + r_zz) * sin(omega * r) +
-                           (omega * r_y * r_y + omega * r_z * r_z) * cos(omega * r));
-      curl2E[1] = -omega * (r_yx * sin(omega * r) + omega * r_y * r_x * cos(
-                               omega * r));
-      curl2E[2] = -omega * (r_zx * sin(omega * r) + omega * r_z * r_x * cos(
-                               omega * r));
-   }
-   else if (sol == 3) // plane wave
-   {
-      double coeff = omega / sqrt(3.0);
-      E[0] = cos(coeff * (x + y + z));
-      E[1] = 0.0;
-      E[2] = 0.0;
-
-      curlE[0] = 0.0;
-      curlE[1] = -coeff * sin(coeff * (x + y + z));
-      curlE[2] = coeff * sin(coeff * (x + y + z));
-
-      curl2E[0] = 2.0 * coeff * coeff * E[0];
-      curl2E[1] = -coeff * coeff * E[0];
-      curl2E[2] = -coeff * coeff * E[0];
-   }
-   else if (sol == -1)
-   {
-      E[0] = cos(omega * y);
-      E[1] = 0.0;
-
-      curlE[0] = 0.0;
-      curlE[1] = 0.0;
-      curlE[2] = -omega * sin(omega * y);
-
-      curl2E[0] = omega*omega * cos(omega*y);
-      curl2E[1] = 0.0;
-      curl2E[2] = 0.0;
-   }
-   else if (sol == 4) // Airy function
-   {
-      E[0] = 0;
-      E[1] = 0;
-      // double b = -pow(omega/4.0,2.0/3.0)*(4.0*x(0)-1.0);
-      double b = -pow(omega/4.0,2.0/3.0)*(4.0*x-1.0);
-      //E[2] = boost::math::airy_ai(b);
-      E[2] = gsl_sf_airy_Ai(b, GSL_PREC_DOUBLE);
-
-      curlE[0] = 0.0;
-      curlE[1] = 4.0 * pow(omega/4.0,2.0/3.0) * gsl_sf_airy_Ai_deriv(b,
-                                                                     GSL_PREC_DOUBLE);
-      curlE[2] = 0.0;
-
-      //  not used
-      curl2E[0] = 0.0;
-      curl2E[1] = 0.0;
-      curl2E[2] = 0.0;
-   }
-}
-
-
-void epsilon_func(const Vector &x, DenseMatrix &M)
-{
-   M.SetSize(3);
-
-   M = 0.0;
-   M(0,0) = 1.0;
-   M(1,1) = 1.0;
-   if (sol != 4)
-   {
-      M(2,2) = 1.0;
-   }
-   else
-   {
-      M(2,2) = 4.0*x(0)-1.0;
-      // M(2,2) = 2.0;
-   }
-}
-
-void epsilon2_func(const Vector &x, DenseMatrix &M)
-{
-   M.SetSize(3);
-
-   M = 0.0;
-   M(0,0) = 1.0;
-   M(1,1) = 1.0;
-   if (sol != 4)
-   {
-      M(2,2) = 1.0;
-   }
-   else
-   {
-      M(2,2) = (4.0*x(0)-1.0) * (4.0*x(0)-1.0);
-      // M(2,2) = 4.0;
-   }
-}

examples/ex3p_sp.cpp

@@ -1,915 +0,0 @@
-//                       MFEM Example 3 - Parallel Version
-//
-// Compile with: make ex3p
-//
-// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
-//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
-//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/escher.mesh
-//               mpirun -np 4 ex3p -m ../data/escher.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/fichera.mesh
-//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
-//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
-//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
-//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
-//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
-//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
-//
-// Description:  This example code solves a simple electromagnetic diffusion
-//               problem corresponding to the second order definite Maxwell
-//               equation curl curl E + E = f with boundary condition
-//               E x n = <given tangential field>. Here, we use a given exact
-//               solution E and compute the corresponding r.h.s. f.
-//               We discretize with Nedelec finite elements in 2D or 3D.
-//
-//               The example demonstrates the use of H(curl) finite element
-//               spaces with the curl-curl and the (vector finite element) mass
-//               bilinear form, as well as the computation of discretization
-//               error when the exact solution is known. Static condensation is
-//               also illustrated.
-//
-//               We recommend viewing examples 1-2 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Exact solution, E, and r.h.s., f. See below for implementation.
-void E_exact(const Vector &, Vector &);
-void f_exact(const Vector &, Vector &);
-double freq = 1.0, kappa;
-int dim;
-
-#define SIGMAVAL -250.0
-//#define FORM_DEFINITE
-//#define SOLVE_A2
-//#define ITER_A2
-
-//#define USE_CSL
-
-//#define USE_HELMHOLTZ
-
-//#define TEST_MULTIPLE_SP
-
-#ifdef USE_HELMHOLTZ
-void GetHelmholtzMatrix(ParMesh *pmesh, const int dir, HypreParMatrix *A)
-{
-   const int order = 1;
-   FiniteElementCollection *fec;
-   fec = new H1_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-
-   Array<int> ess_tdof_list;
-
-   const bool homogeneousBCeverywhere = false;
-   if (homogeneousBCeverywhere)
-   {
-      if (pmesh->bdr_attributes.Size())
-      {
-         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-         ess_bdr = 1;
-         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   else
-   {
-      // Set boundary conditions, depending on dir.
-      MFEM_VERIFY(dim == 3, "");
-      for (int i=0; i<pmesh->GetNBE(); ++i)
-      {
-         Element *elem = pmesh->GetBdrElement(i);
-         MFEM_VERIFY(elem->GetNVertices() >= 3, "");
-         const int *vertices = elem->GetVertices();
-         double *v[3];
-         for (int j=0; j<3; ++j)
-         {
-            v[j] = pmesh->GetVertex(vertices[j]);
-         }
-
-         double u[3];
-         double w[3];
-         for (int j=0; j<3; ++j)
-         {
-            u[j] = v[1][j] - v[0][j];  // An edge tangent
-            w[j] = v[2][j] - v[1][j];  // Another edge tangent, not parallel to u.
-         }
-
-         double n[3];  // normal vector, taken as the cross product u x v
-         n[0] = (u[1]*w[2]) - (u[2]*w[1]);
-         n[1] = (u[2]*w[0]) - (u[0]*w[2]);
-         n[2] = (u[0]*w[1]) - (u[1]*w[0]);
-
-         double t = sqrt((n[0]*n[0]) + (n[1]*n[1]) + (n[2]*n[2]));
-
-         int d = -1;
-         for (int j=0; j<3; ++j)
-         {
-            n[j] /= t;  // normalize
-            if (fabs(fabs(n[j]) - 1.0) < 1.0e-8)
-            {
-               d = j;
-            }
-         }
-
-         MFEM_VERIFY(d >= 0, "");
-
-         if (d != dir)  // face has essential BC at all DOF's.
-         {
-            elem->SetAttribute(1);
-         }
-         else
-         {
-            elem->SetAttribute(0);
-         }
-      }
-
-      Array<int> ess_bdr(2);
-      ess_bdr = 0;
-      ess_bdr[1] = 1;
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient neg(SIGMAVAL);
-
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   a->AddDomainIntegrator(new MassIntegrator(neg));
-
-   ParLinearForm *b = new ParLinearForm(fespace);
-   ConstantCoefficient zero(0.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(zero));
-   b->Assemble();
-
-   bool static_cond = false;
-
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   ParGridFunction x(fespace);
-   x = 0.0;
-
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, *A, X, B);
-}
-#endif
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../data/beam-tet.mesh";
-   int order = 2;
-   bool static_cond = false;
-   bool visualization = 1;
-#ifdef MFEM_USE_STRUMPACK
-   bool use_strumpack = false;
-#endif
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
-                  " solution.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-#ifdef MFEM_USE_STRUMPACK
-   args.AddOption(&use_strumpack, "-strumpack", "--strumpack-solver",
-                  "-no-strumpack", "--no-strumpack-solver",
-                  "Use STRUMPACK's double complex linear solver.");
-#endif
-
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-   kappa = freq * M_PI;
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
-      //(int)floor(log(100000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh->UniformRefinement();
-      }
-   }
-   pmesh->ReorientTetMesh();
-
-   {
-      double minsize = pmesh->GetElementSize(0);
-      double maxsize = minsize;
-      for (int i=1; i<pmesh->GetNE(); ++i)
-      {
-         const double size_i = pmesh->GetElementSize(i);
-         minsize = std::min(minsize, size_i);
-         maxsize = std::max(maxsize, size_i);
-      }
-
-      cout << myid << ": Element size range: (" << minsize << ", " << maxsize << ")"
-           << endl;
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use the Nedelec finite elements of the specified order.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   long globalNE = pmesh->GetGlobalNE();
-   if (myid == 0)
-   {
-      cout << "Number of mesh elements: " << globalNE << endl;
-      cout << "Number of finite element unknowns: " << size << endl;
-      cout << "Root local number of finite element unknowns: " << fespace->TrueVSize()
-           << endl;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh->bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
-   //    basis functions in the finite element fespace.
-   VectorFunctionCoefficient f(sdim, f_exact);
-   ParLinearForm *b = new ParLinearForm(fespace);
-   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x by projecting the exact
-   //    solution. Note that only values from the boundary edges will be used
-   //    when eliminating the non-homogeneous boundary condition to modify the
-   //    r.h.s. vector b.
-   ParGridFunction x(fespace);
-   VectorFunctionCoefficient E(sdim, E_exact);
-   x.ProjectCoefficient(E);
-
-   // 10. Set up the parallel bilinear form corresponding to the EM diffusion
-   //     operator curl muinv curl + sigma I, by adding the curl-curl and the
-   //     mass domain integrators.
-   Coefficient *muinv = new ConstantCoefficient(1.0);
-   Coefficient *sigma = new ConstantCoefficient(SIGMAVAL);
-   Coefficient *sigmaAbs = new ConstantCoefficient(fabs(SIGMAVAL));
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
-   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
-
-   //cout << myid << ": NBE " << pmesh->GetNBE() << endl;
-
-#ifdef FORM_DEFINITE
-   ParBilinearForm *adef = new ParBilinearForm(fespace);
-   adef->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
-   adef->AddDomainIntegrator(new VectorFEMassIntegrator(*sigmaAbs));
-
-   if (static_cond) { adef->EnableStaticCondensation(); }
-   adef->Assemble();
-
-   HypreParMatrix Adef;
-   Vector Bdef, Xdef;
-   adef->FormLinearSystem(ess_tdof_list, x, *b, Adef, Xdef, Bdef);
-#endif
-
-#ifdef USE_CSL
-   Vector Bdef, Xdef;
-
-   ParBilinearForm *Mform =  new ParBilinearForm(fespace);
-   Mform->AddDomainIntegrator(new VectorFEMassIntegrator(*sigmaAbs));
-   Mform->Assemble();
-
-   // Mform->Finalize();
-
-   HypreParMatrix Mmat, Smat, Mcopy;
-   Mform->FormLinearSystem(ess_tdof_list, x, *b, Mmat, Xdef, Bdef);
-   Mform->FormLinearSystem(ess_tdof_list, x, *b, Mcopy, Xdef,
-                           Bdef);  // There must be a better way than creating two identical matrices.
-
-   ParBilinearForm *Sform =  new ParBilinearForm(fespace);
-   Sform->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
-   Sform->Assemble();
-
-   Sform->FormLinearSystem(ess_tdof_list, x, *b, Smat, Xdef, Bdef);
-
-   ParBilinearForm *agrad = new ParBilinearForm(fespace);
-   //agrad->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
-   agrad->AddDomainIntegrator(new VectorFEMassIntegrator(*muinv));
-
-   if (static_cond) { agrad->EnableStaticCondensation(); }
-   agrad->Assemble();
-   HypreParMatrix Agrad;
-   agrad->FormLinearSystem(ess_tdof_list, x, *b, Agrad, Xdef, Bdef);
-#endif
-
-#ifdef ITER_A2
-   Vector Bdef, Xdef;
-
-   ParBilinearForm *Mform =  new ParBilinearForm(fespace);
-   Mform->AddDomainIntegrator(new VectorFEMassIntegrator(*sigmaAbs));
-   Mform->Assemble();
-
-   Mform->Finalize();
-
-   HypreParMatrix Mmat, Mcopy;
-   Mform->FormLinearSystem(ess_tdof_list, x, *b, Mmat, Xdef, Bdef);
-   Mform->FormLinearSystem(ess_tdof_list, x, *b, Mcopy, Xdef,
-                           Bdef);  // There must be a better way to implement M^2.
-
-   /*
-   HypreParMatrix *Mmat = Mform->ParallelAssemble();
-   HypreParMatrix *Mcopy = Mform->ParallelAssemble();  // There must be a better way to implement M^2.
-   */
-
-   ParBilinearForm *Sform =  new ParBilinearForm(fespace);
-   Sform->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
-   Sform->Assemble();
-
-   HypreParMatrix Smat, Scopy;
-   Sform->FormLinearSystem(ess_tdof_list, x, *b, Smat, Xdef, Bdef);
-   Sform->FormLinearSystem(ess_tdof_list, x, *b, Scopy, Xdef, Bdef);
-#endif
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   HypreParMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-#ifdef SOLVE_A2
-   HypreParMatrix Acopy;
-   {
-      Vector Bdum, Xdum;
-      a->FormLinearSystem(ess_tdof_list, x, *b, Acopy, Xdum, Bdum);
-   }
-#endif
-
-   if (myid == 0)
-   {
-      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
-   }
-
-   StopWatch chrono;
-   chrono.Clear();
-   chrono.Start();
-
-   //A.Print("maxwell1000_2");
-
-#ifdef MFEM_USE_STRUMPACK
-   if (use_strumpack)
-   {
-      const bool fullDirect = true;
-
-#ifdef USE_CSL
-      const double beta1 = 1.0;
-      const double beta2 = 0.5;
-
-      Mmat *= -beta1;
-
-      // HypreParMatrix *cslRe = Add(1.0, Smat, -beta1, Mmat);
-      HypreParMatrix * cslRe = ParAdd(&Smat, &Mmat);
-
-      Mcopy *= beta2;
-
-      //ComplexHypreParMatrix chpm(cslRe, &Mcopy, false, false);
-      ComplexHypreParMatrix chpm(&A, &Mcopy, false,
-                                 false);  // For the case beta1 = 1.
-
-      HypreParMatrix *cSysMat = chpm.GetSystemMatrix();
-
-      Array<int> block_offsets(3); // number of variables + 1
-      block_offsets[0] = 0;
-      block_offsets[1] = fespace->GetVSize();
-      block_offsets[2] = fespace->GetVSize();
-      block_offsets.PartialSum();
-
-      Array<int> block_trueOffsets(3); // number of variables + 1
-      block_trueOffsets[0] = 0;
-      block_trueOffsets[1] = fespace->TrueVSize();
-      block_trueOffsets[2] = fespace->TrueVSize();
-      block_trueOffsets.PartialSum();
-
-      //cout << myid << ": V size " << fespace->GetVSize() << ", true " << fespace->TrueVSize() << ", global true " << size << ", B size "
-      //<< B.Size() << ", X size " << X.Size() << endl;
-
-      // Note that B is of true size.
-      BlockVector trueY(block_trueOffsets), trueX(block_trueOffsets),
-                  trueRhs(block_trueOffsets);
-
-      trueRhs.GetBlock(0) = B;
-      trueRhs.GetBlock(1) = 0.0;
-
-      Operator * Arow = new STRUMPACKRowLocMatrix(*cSysMat);
-
-      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-      strumpack->SetPrintFactorStatistics(true);
-      strumpack->SetPrintSolveStatistics(false);
-      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
-      // strumpack->SetSymmetricPattern(true);
-      strumpack->SetOperator(*Arow);
-      strumpack->SetFromCommandLine();
-#endif
-
-      if (fullDirect)
-      {
-#ifdef USE_CSL
-         //Solver * precond = strumpack;
-
-         // strumpack->Mult(B, X);
-
-         BlockOperator blockDiagA(block_trueOffsets);
-         for (int i=0; i<2; ++i)
-         {
-            blockDiagA.SetDiagonalBlock(i, &A);
-         }
-
-         ParFiniteElementSpace *prec_fespace =
-            (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
-         HypreSolver *amsgrad = new HypreAMS(Agrad, prec_fespace);
-
-#ifdef HYPRE_DYLAN
-         {
-            Vector Xtmp(X);
-            amsgrad->Mult(B,
-                          Xtmp);  // Just a hack to get ams to run its setup function. There should be a better way.
-         }
-
-         HypreAMSG *amsg = new HypreAMSG((HypreAMS*) amsgrad, argc, argv);
-         BlockOperator blockDiagP(block_trueOffsets);
-         for (int i=0; i<2; ++i)
-         {
-            blockDiagP.SetDiagonalBlock(i, amsg);
-         }
-
-         TripleProductOperator strumpackProj(&blockDiagP, strumpack, &blockDiagP, false,
-                                             false, false);
-         ProductOperator prod(&blockDiagA, &strumpackProj, false, false);
-#else
-         ProductOperator prod(&blockDiagA, strumpack, false, false);
-#endif
-
-         GMRESSolver *gmres = new GMRESSolver(fespace->GetComm());
-         //BiCGSTABSolver *gmres = new BiCGSTABSolver(fespace->GetComm());
-
-         gmres->SetOperator(prod);
-         gmres->SetRelTol(1e-12);
-         gmres->SetMaxIter(1000);
-         gmres->SetPrintLevel(1);
-
-         gmres->Mult(trueRhs, trueY);
-         strumpack->Mult(trueY, trueX);
-
-         X = trueX.GetBlock(0);
-         double xim2 = trueX.GetBlock(1).Norml2();
-         xim2 *= xim2;
-         double sumxim2 = 0.0;
-
-         MPI_Allreduce(&xim2, &sumxim2, 1, MPI_DOUBLE, MPI_SUM, fespace->GetComm());
-
-         if (myid == 0)
-         {
-            cout << myid << ": norm of Xim " << trueX.GetBlock(1).Norml2() << ", global " <<
-                 sqrt(sumxim2) << endl;
-         }
-
-         delete gmres;
-         delete strumpack;
-         delete Arow;
-#else
-         cout << "Solving with STRUMPACK" << endl;
-
-#ifdef TEST_MULTIPLE_SP
-         const int Ns = 2;
-         std::vector<Operator*> Arows(Ns);
-         std::vector<STRUMPACKSolver*> strumpacks(Ns);
-
-         //Operator * Arow = new STRUMPACKRowLocMatrix(A);
-
-         for (int m=0; m<Ns; ++m)
-         {
-            Arows[m] = new STRUMPACKRowLocMatrix(A);
-
-            //STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-            strumpacks[m] = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-            strumpacks[m]->SetPrintFactorStatistics(true);
-            strumpacks[m]->SetPrintSolveStatistics(false);
-            strumpacks[m]->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-            strumpacks[m]->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-            // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
-            // strumpack->SetSymmetricPattern(true);
-            strumpacks[m]->SetOperator(*Arows[m]);
-            strumpacks[m]->SetFromCommandLine();
-            //Solver * precond = strumpack;
-
-            strumpacks[m]->Mult(B, X);
-
-            //delete strumpack;
-            //delete Arow;
-         }
-#else
-         Operator * Arow = new STRUMPACKRowLocMatrix(A);
-
-         STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-         strumpack->SetPrintFactorStatistics(true);
-         strumpack->SetPrintSolveStatistics(false);
-         strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-         strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-         // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
-         // strumpack->SetSymmetricPattern(true);
-         strumpack->SetOperator(*Arow);
-         strumpack->SetFromCommandLine();
-         //Solver * precond = strumpack;
-
-         cout << "Solving with strumpack one time" << endl;
-
-         strumpack->Mult(B, X);
-
-         delete strumpack;
-         delete Arow;
-#endif
-#endif
-      }
-      else
-      {
-         ParFiniteElementSpace *prec_fespace =
-            (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
-         HypreSolver *ams = new HypreAMS(A, prec_fespace);
-
-#ifdef HYPRE_DYLAN
-         {
-            Vector Xtmp(X);
-            ams->Mult(B,
-                      Xtmp);  // Just a hack to get ams to run its setup function. There should be a better way.
-         }
-
-         HypreParMatrix H[3];
-#ifdef USE_HELMHOLTZ
-         for (int i=0; i<3; ++i)
-         {
-            GetHelmholtzMatrix(pmesh, i, &(H[i]));
-         }
-#endif
-
-#ifdef USE_CSL
-         HypreIAMS *iams = new HypreIAMS(A, H, strumpack, &trueX, &trueY,
-                                         (HypreAMS*) ams, argc, argv);
-#else
-         HypreIAMS *iams = new HypreIAMS(A, H, strumpack, NULL, NULL, (HypreAMS*) ams,
-                                         argc, argv);
-#endif
-
-         GMRESSolver *gmres = new GMRESSolver(fespace->GetComm());
-         //FGMRESSolver *gmres = new FGMRESSolver(fespace->GetComm());
-         //BiCGSTABSolver *gmres = new BiCGSTABSolver(fespace->GetComm());
-         //MINRESSolver *gmres = new MINRESSolver(fespace->GetComm());
-
-         gmres->SetOperator(A);
-         gmres->SetRelTol(1e-16);
-         gmres->SetMaxIter(1000);
-         gmres->SetPrintLevel(1);
-
-#ifdef SOLVE_A2
-         {
-            StopWatch chronoA2;
-            chronoA2.Clear();
-            chronoA2.Start();
-
-            HypreParMatrix * A2 = ParMult(&A, &Acopy);
-
-            chronoA2.Stop();
-            cout << "A2 setup time " << chronoA2.RealTime() << endl;
-
-            Vector AB(B);
-            A.Mult(B, AB);
-            gmres->SetOperator(*A2);
-
-            HypreSolver *ams2 = new HypreAMS(A, prec_fespace);
-            {
-               Vector Xtmp(X);
-               ams2->Mult(B,
-                          Xtmp);  // Just a hack to get ams to run its setup function. There should be a better way.
-            }
-
-#ifdef ITER_A2
-            // Iteratively solve 0.5 (A^2 + S^2 + M^2) u^{k+1} = 0.5 (SM + MS) u^k + Ab
-
-            StopWatch chronoIterA2;
-            chronoIterA2.Clear();
-            chronoIterA2.Start();
-
-            HypreParMatrix * M2 = ParMult(&Mmat, &Mcopy);
-            HypreParMatrix * S2 = ParMult(&Smat, &Scopy);
-
-            HypreParMatrix * MS = ParMult(&Mmat, &Scopy);
-            HypreParMatrix * SM = ParMult(&Smat, &Mcopy);
-
-            HypreParMatrix * Bmat = ParAdd(SM, MS);
-            (*Bmat) *= 0.5;
-
-            // TODO: there must be a better way to form a sum of three matrices. Of course, we could define an operator that does 3 mat-vecs.
-            //HypreParMatrix * S2M2 = ParAdd(S2, M2);
-            //HypreParMatrix * iterMat = ParAdd(A2, S2M2);
-            HypreParMatrix * iterMat = ParAdd(A2, Bmat);
-
-            chronoIterA2.Stop();
-            cout << "Iter A2 setup time " << chronoIterA2.RealTime() << endl;
-
-            /*
-            HypreSolver *ams3 = new HypreAMS(*iterMat, prec_fespace);
-            {
-              Vector Xtmp(X);
-              ams3->Mult(B, Xtmp);  // Just a hack to get ams to run its setup function. There should be a better way.
-            }
-            */
-
-            /*
-            // GMRES
-            gmres->SetOperator(*iterMat);
-            gmres->SetPreconditioner(*ams2);
-            */
-
-
-            //HypreBoomerAMG *amg = new HypreBoomerAMG(*iterMat);
-            HypreBoomerAMG *amg = new HypreBoomerAMG(*A2);
-
-            // PCG
-            HyprePCG *pcg = new HyprePCG(*iterMat);
-            //HyprePCG *pcg = new HyprePCG(*A2);
-            pcg->SetTol(1e-12);
-            pcg->SetMaxIter(10);
-            pcg->SetPrintLevel(2);
-            pcg->SetPreconditioner(*amg);
-
-            /*
-            // Strumpack linear solver
-            Operator * Arow = new STRUMPACKRowLocMatrix(*iterMat);
-
-            STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-            strumpack->SetPrintFactorStatistics(true);
-            strumpack->SetPrintSolveStatistics(false);
-            strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-            strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-            strumpack->SetOperator(*Arow);
-            strumpack->SetFromCommandLine();
-            */
-
-            Vector iterRHS(AB);
-            Vector iterU(AB);
-            Vector iterU0(AB);
-
-            iterU = 0.0;
-            iterU0 = 0.0;
-
-            bool iterate = true;
-            int numIter = 0;
-            while (iterate)
-            {
-               iterRHS = iterU;
-               iterRHS.Add(-1.0, iterU0);
-
-               cout << "Iteration " << numIter + 1 << ": diff norm " << iterRHS.Norml2() <<
-                    endl;
-
-               iterU0 = iterU;
-
-               Bmat->Mult(iterU0, iterRHS);
-               //iterRHS.Add(2.0, AB);
-               iterRHS.Add(1.0, AB);
-
-               //gmres->Mult(iterRHS, iterU);
-               pcg->Mult(iterRHS, iterU);
-               //strumpack->Mult(iterRHS, iterU);
-
-               numIter++;
-
-               if (numIter > 100)
-               {
-                  iterate = false;
-               }
-            }
-
-            //delete strumpack;
-            //delete Arow;
-
-            delete pcg;
-
-            X = iterU;
-#else
-            //HypreIAMS *iams2 = new HypreIAMS(*A2, (HypreAMS*) ams2, argc, argv);
-            //gmres->SetPreconditioner(*iams2);
-            cout << myid << ": Solving" << endl;
-            gmres->SetPreconditioner(*ams2);
-            gmres->Mult(AB, X);
-            cout << myid << ": Solved" << endl;
-            return 3;
-#endif
-         }
-#else
-         gmres->SetPreconditioner(*iams);
-         gmres->Mult(B, X);
-#endif
-#else
-         HypreGMRES *gmres = new HypreGMRES(A);
-         gmres->SetTol(1e-12);
-         gmres->SetMaxIter(100);
-         gmres->SetPrintLevel(10);
-
-#ifdef FORM_DEFINITE
-         HypreSolver *amsdef = new HypreAMS(Adef, prec_fespace);
-         gmres->SetPreconditioner(*amsdef);
-#else
-         gmres->SetPreconditioner(*ams);
-#endif
-         gmres->Mult(B, X);
-#endif
-
-         delete gmres;
-         //delete iams;
-         //delete ams;
-      }
-   }
-   else
-#endif
-   {
-      // 12. Define and apply a parallel PCG solver for AX=B with the AMS
-      //     preconditioner from hypre.
-      ParFiniteElementSpace *prec_fespace =
-         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
-      HypreSolver *ams = new HypreAMS(A, prec_fespace);
-      HyprePCG *pcg = new HyprePCG(A);
-      pcg->SetTol(1e-12);
-      pcg->SetMaxIter(500);
-      pcg->SetPrintLevel(2);
-      pcg->SetPreconditioner(*ams);
-      pcg->Mult(B, X);
-
-      delete pcg;
-      delete ams;
-   }
-
-   chrono.Stop();
-   cout << myid << ": Solver time " << chrono.RealTime() << endl;
-
-   // 13. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-
-   // 14. Compute and print the L^2 norm of the error.
-   {
-      double err = x.ComputeL2Error(E);
-      Vector zeroVec(3);
-      zeroVec = 0.0;
-      VectorConstantCoefficient vzero(zeroVec);
-      ParGridFunction zerogf(fespace);
-      zerogf = 0.0;
-      double normE = zerogf.ComputeL2Error(E);
-      double normX = x.ComputeL2Error(vzero);
-      if (myid == 0)
-      {
-         cout << "|| E_h - E ||_{L^2} = " << err << endl;
-         cout << "|| E_h ||_{L^2} = " << normX << endl;
-         cout << "|| E ||_{L^2} = " << normE << endl;
-      }
-   }
-
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   delete a;
-   delete sigma;
-   delete muinv;
-   delete b;
-   delete fespace;
-   delete fec;
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}
-
-
-void E_exact(const Vector &x, Vector &E)
-{
-   if (dim == 3)
-   {
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(2));
-      E(2) = sin(kappa * x(0));
-   }
-   else
-   {
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(0));
-      if (x.Size() == 3) { E(2) = 0.0; }
-   }
-}
-
-void f_exact(const Vector &x, Vector &f)
-{
-   if (dim == 3)
-   {
-      f(0) = (SIGMAVAL + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (SIGMAVAL + kappa * kappa) * sin(kappa * x(2));
-      f(2) = (SIGMAVAL + kappa * kappa) * sin(kappa * x(0));
-   }
-   else
-   {
-      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
-      if (x.Size() == 3) { f(2) = 0.0; }
-   }
-}

examples/ex6.cpp

@@ -108,11 +108,7 @@ int main(int argc, char *argv[])
    //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
    //    problem yet, this will be done in the main loop.
    BilinearForm a(&fespace);
-   if (pa)
-   {
-      a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      a.SetDiagonalPolicy(Operator::DIAG_ONE);
-   }
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
    LinearForm b(&fespace);
 
    ConstantCoefficient one(1.0);
@@ -203,10 +199,9 @@ int main(int argc, char *argv[])
          umf_solver.Mult(B, X);
 #endif
       }
-      else // Diagonal preconditioning in partial assembly mode.
+      else // No preconditioning for now in partial assembly mode.
       {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
-         PCG(*A, M, B, X, 3, 2000, 1e-12, 0.0);
+         CG(*A, B, X, 3, 2000, 1e-12, 0.0);
       }
 
       // 18. After solving the linear system, reconstruct the solution as a

examples/ex6p.cpp

@@ -129,11 +129,7 @@ int main(int argc, char *argv[])
    //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
    //    problem yet, this will be done in the main loop.
    ParBilinearForm a(&fespace);
-   if (pa)
-   {
-      a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      a.SetDiagonalPolicy(Operator::DIAG_ONE);
-   }
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
    ParLinearForm b(&fespace);
 
    ConstantCoefficient one(1.0);
@@ -224,26 +220,17 @@ int main(int argc, char *argv[])
 
       // 17. Solve the linear system A X = B.
       //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-      //     * With partial assembly, use a diagonal preconditioner.
-      Solver *M = NULL;
-      if (pa)
-      {
-         M = new OperatorJacobiSmoother(a, ess_tdof_list);
-      }
-      else
-      {
-         HypreBoomerAMG *amg = new HypreBoomerAMG;
-         amg->SetPrintLevel(0);
-         M = amg;
-      }
+      //     * With partial assembly, use no preconditioner, for now.
+      HypreBoomerAMG *amg = NULL;
+      if (!pa) { amg = new HypreBoomerAMG; amg->SetPrintLevel(0); }
       CGSolver cg(MPI_COMM_WORLD);
       cg.SetRelTol(1e-6);
       cg.SetMaxIter(2000);
       cg.SetPrintLevel(3); // print the first and the last iterations only
-      cg.SetPreconditioner(*M);
+      if (amg) { cg.SetPreconditioner(*amg); }
       cg.SetOperator(*A);
       cg.Mult(B, X);
-      delete M;
+      delete amg;
 
       // 18. Switch back to the host and extract the parallel grid function
       //     corresponding to the finite element approximation X. This is the

examples/multigrid.hpp

@@ -1,517 +0,0 @@
-#pragma once
-
-#include "mfem.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-STRUMPACKSolver* CreateStrumpackSolver(Operator *Arow, MPI_Comm comm)
-{
-   //STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, comm);
-   STRUMPACKSolver * strumpack = new STRUMPACKSolver(0, NULL, comm);
-   strumpack->SetPrintFactorStatistics(true);
-   strumpack->SetPrintSolveStatistics(false);
-   strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-   strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-   strumpack->SetOperator(*Arow);
-   strumpack->SetFromCommandLine();
-   return strumpack;
-}
-
-hypre_CSRMatrix* GetHypreParMatrixData(const HypreParMatrix & hypParMat)
-{
-   // First cast the parameter to a hypre_ParCSRMatrix
-   hypre_ParCSRMatrix * parcsr_op =
-      (hypre_ParCSRMatrix *)const_cast<HypreParMatrix&>(hypParMat);
-
-   MFEM_ASSERT(parcsr_op != NULL,"STRUMPACK: const_cast failed in SetOperator");
-
-   // Create the CSRMatrixMPI A_ by borrowing the internal data from a hypre_CSRMatrix.
-   return hypre_MergeDiagAndOffd(parcsr_op);
-}
-
-// Row and column offsets are assumed to be the same, for each process.
-// Array offsets stores process-local offsets with respect to the blocks. Process offsets are not included.
-HypreParMatrix* CreateHypreParMatrixFromBlocks(MPI_Comm comm,
-                                               Array<int> const& offsets, Array2D<HypreParMatrix*> const& blocks,
-                                               Array2D<SparseMatrix*> const& blocksSp,
-                                               Array2D<double> const& coefficient,
-                                               std::vector<std::vector<int> > const& blockProcOffsets,
-                                               std::vector<std::vector<int> > const& all_block_num_loc_rows)
-{
-   const int numBlocks = offsets.Size() - 1;
-   const int num_loc_rows = offsets[numBlocks];
-
-   int nprocs, rank;
-   MPI_Comm_rank(comm, &rank);
-   MPI_Comm_size(comm, &nprocs);
-
-   std::vector<int> all_num_loc_rows(nprocs);
-   std::vector<int> procOffsets(nprocs);
-   std::vector<std::vector<int> > procBlockOffsets(nprocs);
-
-   MPI_Allgather(&num_loc_rows, 1, MPI_INT, all_num_loc_rows.data(), 1, MPI_INT,
-                 comm);
-
-   int first_loc_row = 0;
-   int glob_nrows = 0;
-   procOffsets[0] = 0;
-   for (int i=0; i<nprocs; ++i)
-   {
-      glob_nrows += all_num_loc_rows[i];
-      if (i < rank)
-      {
-         first_loc_row += all_num_loc_rows[i];
-      }
-
-      if (i < nprocs-1)
-      {
-         procOffsets[i+1] = procOffsets[i] + all_num_loc_rows[i];
-      }
-
-      if (numBlocks > 0)
-      {
-         procBlockOffsets[i].resize(numBlocks);
-         procBlockOffsets[i][0] = 0;
-      }
-
-      for (int j=1; j<numBlocks; ++j)
-      {
-         procBlockOffsets[i][j] = procBlockOffsets[i][j-1] + all_block_num_loc_rows[j
-                                                                                    -1][i];
-      }
-   }
-
-   const int glob_ncols = glob_nrows;
-
-   std::vector<int> opI(num_loc_rows+1);
-   std::vector<int> cnt(num_loc_rows);
-
-   for (int i=0; i<num_loc_rows; ++i)
-   {
-      opI[i] = 0;
-      cnt[i] = 0;
-   }
-
-   opI[num_loc_rows] = 0;
-
-   Array2D<hypre_CSRMatrix*> csr_blocks(numBlocks, numBlocks);
-
-   // Loop over all blocks, to determine nnz for each row.
-   for (int i=0; i<numBlocks; ++i)
-   {
-      for (int j=0; j<numBlocks; ++j)
-      {
-         if (blocks(i, j) == NULL)
-         {
-            csr_blocks(i, j) = NULL;
-
-            if (blocksSp(i, j) != NULL)
-            {
-               const int nrows = blocksSp(i, j)->Height();
-               for (int k=0; k<nrows; ++k)
-               {
-                  const int rowg = offsets[i] + k;
-                  opI[rowg + 1] += blocksSp(i, j)->GetI()[k+1] - blocksSp(i, j)->GetI()[k];
-               }
-            }
-         }
-         else
-         {
-            MFEM_VERIFY(blocksSp(i, j) == NULL, "");
-
-            csr_blocks(i, j) = GetHypreParMatrixData(*(blocks(i, j)));
-
-            const int nrows = csr_blocks(i, j)->num_rows;
-
-            for (int k=0; k<nrows; ++k)
-            {
-               const int rowg = offsets[i] + k;
-               //(*(leftInjection(i, j)))[k]
-               opI[rowg + 1] += csr_blocks(i, j)->i[k+1] - csr_blocks(i, j)->i[k];
-            }
-         }
-      }
-   }
-
-   // Now opI[i] is nnz for row i-1. Do a partial sum to get offsets.
-   for (int i=0; i<num_loc_rows; ++i)
-   {
-      opI[i+1] += opI[i];
-   }
-
-   const int nnz = opI[num_loc_rows];
-
-   std::vector<HYPRE_Int> opJ(nnz);
-   std::vector<double> data(nnz);
-
-   // Loop over all blocks, to set matrix data.
-   for (int i=0; i<numBlocks; ++i)
-   {
-      for (int j=0; j<numBlocks; ++j)
-      {
-         if (csr_blocks(i, j) != NULL || blocksSp(i, j) != NULL)
-         {
-            const bool useCSR = (csr_blocks(i, j) != NULL);
-
-            const int nrows = useCSR ? csr_blocks(i, j)->num_rows : blocksSp(i,
-                                                                             j)->Height();
-            const double coef = coefficient(i, j);
-
-            int *Iarray = useCSR ? csr_blocks(i, j)->i : blocksSp(i, j)->GetI();
-
-            //const bool failure = (nrows != offsets[i+1] - offsets[i]);
-
-            MFEM_VERIFY(nrows == offsets[i+1] - offsets[i], "");
-
-            for (int k=0; k<nrows; ++k)
-            {
-               const int rowg = offsets[i] + k;  // process-local row
-               const int nnz_k = Iarray[k+1] - Iarray[k];
-               const int osk = Iarray[k];
-
-               for (int l=0; l<nnz_k; ++l)
-               {
-                  // Find the column process offset for the block.
-                  const int bcol = useCSR ? csr_blocks(i, j)->j[osk + l] : blocksSp(i,
-                                                                                    j)->GetJ()[osk + l];
-                  int bcolproc = 0;
-
-                  for (int p=1; p<nprocs; ++p)
-                  {
-                     if (blockProcOffsets[j][p] > bcol)
-                     {
-                        bcolproc = p-1;
-                        break;
-                     }
-                  }
-
-                  if (blockProcOffsets[j][nprocs - 1] <= bcol)
-                  {
-                     bcolproc = nprocs - 1;
-                  }
-
-                  const int colg = procOffsets[bcolproc] + procBlockOffsets[bcolproc][j] +
-                                   (bcol - blockProcOffsets[j][bcolproc]);
-
-                  if (colg < 0)
-                  {
-                     cout << "BUG, negative global column index" << endl;
-                  }
-
-                  opJ[opI[rowg] + cnt[rowg]] = colg;
-                  data[opI[rowg] + cnt[rowg]] = useCSR ? coef * csr_blocks(i,
-                                                                           j)->data[osk + l] : coef * blocksSp(i, j)->GetData()[osk + l];
-                  cnt[rowg]++;
-               }
-            }
-         }
-      }
-   }
-
-   bool cntCheck = true;
-   for (int i=0; i<num_loc_rows; ++i)
-   {
-      if (cnt[i] != opI[i+1] - opI[i])
-      {
-         cntCheck = false;
-      }
-   }
-
-   MFEM_VERIFY(cntCheck, "");
-
-   for (int i=0; i<numBlocks; ++i)
-   {
-      for (int j=0; j<numBlocks; ++j)
-      {
-         if (csr_blocks(i, j) != NULL)
-         {
-            hypre_CSRMatrixDestroy(csr_blocks(i, j));
-         }
-      }
-   }
-
-   std::vector<HYPRE_Int> rowStarts2(2);
-   rowStarts2[0] = first_loc_row;
-   rowStarts2[1] = first_loc_row + all_num_loc_rows[rank];
-
-   if (nnz > 0)
-   {
-      HYPRE_Int minJ = opJ[0];
-      HYPRE_Int maxJ = opJ[0];
-      for (int i=0; i<nnz; ++i)
-      {
-         minJ = std::min(minJ, opJ[i]);
-         maxJ = std::max(maxJ, opJ[i]);
-
-         if (opJ[i] >= glob_ncols)
-         {
-            cout << "Column indices out of range" << endl;
-         }
-      }
-   }
-
-   HypreParMatrix *hmat = new HypreParMatrix(comm, num_loc_rows, glob_nrows,
-                                             glob_ncols, (int*) opI.data(), (HYPRE_Int*) opJ.data(), (double*) data.data(),
-                                             (HYPRE_Int*) rowStarts2.data(), (HYPRE_Int*) rowStarts2.data());
-
-   return hmat;
-}
-
-class BlockMGSolver : public Solver
-{
-private:
-   /// The linear system matrix
-   Array2D<HypreParMatrix *>&
-   Af;  // TODO: remove this, as it is used only in the constructor
-   Array2D<double>&
-   Acoef;  // TODO: remove this, as it is used only in the constructor
-   vector<Array<int>> Aoffsets;
-   vector<Array<int>> Poffsets_i;
-   vector<Array<int>> Poffsets_j;
-   std::vector<Array2D<HypreParMatrix *>> A;
-   std::vector<HypreParMatrix *>& P;
-   std::vector<BlockOperator *> BlkP;
-   std::vector<BlockOperator *> BlkA;
-   std::vector<BlockOperator *> S;
-   HypreParMatrix * Ac;
-   int numGrids, numBlocks;
-   STRUMPACKSolver *invAc = nullptr;
-   double theta = 0.5;
-
-public:
-   BlockMGSolver(const int height, const int width, Array2D<HypreParMatrix *>& Af_,
-                 Array2D<double>& Acoef_, std::vector<HypreParMatrix *>& P_);
-
-   virtual void SetOperator(const Operator &op) {}
-
-   virtual void SetTheta(const double a) { theta = a; }
-
-   virtual void Mult(const Vector &r, Vector &z) const;
-   virtual ~BlockMGSolver();
-};
-
-BlockMGSolver::BlockMGSolver(const int height, const int width,
-                             Array2D<HypreParMatrix *>& Af_, Array2D<double>& Acoef_,
-                             std::vector<HypreParMatrix *>& P_)
-   : Solver(height, width), Af(Af_), Acoef(Acoef_), P(P_)
-{
-   numBlocks = Af.NumRows();
-   MFEM_VERIFY(Af.NumCols() == numBlocks, "");
-   numGrids = P.size();
-   BlkP.resize(numGrids);
-   BlkA.resize(numGrids+1);
-   S.resize(numGrids);
-   A.resize(numGrids + 1);
-   A[numGrids] = Af;
-   Aoffsets.resize(numGrids+1);
-   Poffsets_i.resize(numGrids);
-   Poffsets_j.resize(numGrids);
-   // Construct Bilinear form Matrices on each level
-   for (int k = numGrids ; k > 0; k--)
-   {
-      A[k - 1].SetSize(numBlocks,numBlocks);
-      Aoffsets[k].SetSize(numBlocks+1); Aoffsets[k][0] = 0;
-      for (int i=0; i<numBlocks; i++)
-      {
-         Aoffsets[k][i+1] = A[k](i,i)->Height();
-      }
-
-      Aoffsets[k].PartialSum();
-      BlkA[k] = new BlockOperator(Aoffsets[k]);
-      S[k-1] = new BlockOperator(Aoffsets[k]);  // Smoother
-
-      for (int i=0; i<numBlocks; i++)
-      {
-         for (int j=0; j<numBlocks; j++)
-         {
-            if (A[k](i,j) == NULL)
-            {
-               A[k - 1](i,j) = NULL;
-            }
-            else
-            {
-               A[k - 1](i,j) = RAP(A[k](i,j), P[k - 1]);
-               BlkA[k]->SetBlock(i, j, A[k](i,j), Acoef(i,j));
-            }
-         }
-
-         HypreSmoother *S_i = new HypreSmoother;
-         S_i->SetType(HypreSmoother::Jacobi);
-         S_i->SetOperator(*(A[k](i,i)));
-
-         S[k - 1]->SetBlock(i,i,S_i);
-      }
-
-      Poffsets_i[k-1].SetSize(numBlocks+1); Poffsets_i[k-1][0] = 0;
-      Poffsets_j[k-1].SetSize(numBlocks+1); Poffsets_j[k-1][0] = 0;
-      for (int i=0; i<numBlocks; i++)
-      {
-         Poffsets_i[k-1][i+1] = P[k-1]->Height();
-         Poffsets_j[k-1][i+1] = P[k-1]->Width();
-      }
-      Poffsets_i[k-1].PartialSum();
-      Poffsets_j[k-1].PartialSum();
-
-      BlkP[k-1] = new BlockOperator(Poffsets_i[k-1],Poffsets_j[k-1]);
-      for (int i=0; i<numBlocks; i++)
-      {
-         BlkP[k-1]->SetBlock(i,i,P[k-1]);
-      }
-   }
-   // Set up coarse solve operator
-   // Convert the coarse grid blockmatrix to a HypreParMatrix
-   Array<int> offsets(numBlocks+1);
-   offsets[0]=0;
-   for (int i=0; i<numBlocks; i++)
-   {
-      offsets[i+1]=A[0](i,i)->Height();
-   }
-
-   offsets.PartialSum();
-
-   BlkA[0] = new BlockOperator(offsets);
-
-   Array2D<SparseMatrix*> Asp;
-   //Array2D<double> Acoef;
-   Asp.SetSize(numBlocks,numBlocks);
-   //Acoef.SetSize(numBlocks,numBlocks);
-   for (int i=0; i<numBlocks; i++)
-   {
-      for (int j=0; j<numBlocks; j++)
-      {
-         if (A[0](i,j) != NULL)
-         {
-            BlkA[0]->SetBlock(i, j, A[0](i,j), Acoef(i,j));
-         }
-
-         Asp(i,j) = NULL;
-         //Acoef(i,j) = 1.0;
-      }
-   }
-
-   // Convert to HypreParMatrix
-   HypreParMatrix * Ac;
-
-   std::vector<std::vector<int> > blockProcOffsets(numBlocks);
-   std::vector<std::vector<int> > all_block_num_loc_rows(numBlocks);
-
-   {
-      int nprocs, rank;
-      MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-      MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-
-      std::vector<int> allnumrows(nprocs);
-      const int blockNumRows = A[0](0,0)->Height();
-      MPI_Allgather(&blockNumRows, 1, MPI_INT, allnumrows.data(), 1, MPI_INT,
-                    MPI_COMM_WORLD);
-
-      for (int b=0; b<numBlocks; ++b)
-      {
-         blockProcOffsets[b].resize(nprocs);
-         all_block_num_loc_rows[b].resize(nprocs);
-      }
-
-      blockProcOffsets[0][0] = 0;
-      for (int i=0; i<nprocs-1; ++i)
-      {
-         blockProcOffsets[0][i+1] = blockProcOffsets[0][i] + allnumrows[i];
-      }
-
-      for (int i=0; i<nprocs; ++i)
-      {
-         for (int b=0; b<numBlocks; ++b)
-         {
-            all_block_num_loc_rows[b][i] = allnumrows[i];
-         }
-
-         for (int b=1; b<numBlocks; ++b)
-         {
-            blockProcOffsets[b][i] = blockProcOffsets[0][i];
-         }
-      }
-   }
-
-   Ac = CreateHypreParMatrixFromBlocks(MPI_COMM_WORLD, offsets, A[0], Asp,
-                                       Acoef, blockProcOffsets, all_block_num_loc_rows);
-
-   invAc = CreateStrumpackSolver(new STRUMPACKRowLocMatrix(*Ac), MPI_COMM_WORLD);
-
-   delete Ac;
-}
-
-void BlockMGSolver::Mult(const Vector &r, Vector &z) const
-{
-   // Residual vectors
-   std::vector<Vector> rv(numGrids + 1);
-   // correction vectors
-   std::vector<Vector> zv(numGrids + 1);
-   // allocation
-   for (int i = 0; i <= numGrids ; i++)
-   {
-      int n = (i==0) ? invAc->Height(): BlkA[i]->Width();
-      rv[i].SetSize(n);
-      zv[i].SetSize(n);
-   }
-   // Initial residual
-   rv[numGrids] = r;
-
-   // smooth and update residuals down to the coarsest level
-   for (int i = numGrids; i > 0 ; i--)
-   {
-      // Pre smooth
-      S[i - 1]->Mult(rv[i], zv[i]); zv[i] *= theta;
-      // compute residual
-      int n = BlkA[i]->Width();
-      Vector w(n);
-      BlkA[i]->Mult(zv[i], w);
-      rv[i] -= w;
-      // Restrict
-      BlkP[i - 1]->MultTranspose(rv[i], rv[i - 1]);
-   }
-
-   // Coarse grid Solve
-   invAc->Mult(rv[0], zv[0]);
-   //
-   for (int i = 1; i <= numGrids ; i++)
-   {
-      // Prolong correction
-      Vector u(BlkP[i - 1]->Height());
-      BlkP[i - 1]->Mult(zv[i - 1], u);
-      // Update correction
-      zv[i] += u;
-      // Update residual
-      Vector v(BlkA[i]->Height());
-      BlkA[i]->Mult(u, v); rv[i] -= v;
-      // Post smooth
-      S[i - 1]->Mult(rv[i], v); v *= theta;
-      // Update correction
-      zv[i] += v;
-   }
-   z = zv[numGrids];
-}
-
-BlockMGSolver::~BlockMGSolver()
-{
-   for (int i = numGrids - 1; i >= 0 ; i--)
-   {
-      delete S[i];
-      delete BlkP[i];
-      delete BlkA[i];
-      for (int j=0; j<numBlocks; j++)
-      {
-         for (int k=0; k<numBlocks; k++)
-         {
-            delete A[i](j,k);
-         }
-      }
-      A[i].DeleteAll();
-   }
-   delete BlkA[numGrids];
-   delete invAc;
-   A.clear();
-}
-
-} // namespace mfem

examples/multigriddd.hpp

@@ -1,869 +0,0 @@
-#ifndef BGMULTIGRID
-#define BGMULTIGRID
-
-#include "mfem.hpp"
-
-using namespace std;
-
-//namespace mfem {
-namespace blockgmg
-{
-
-// Row and column offsets are assumed to be the same, for each process.
-// Array offsets stores process-local offsets with respect to the blocks. Process offsets are not included.
-HypreParMatrix* CreateHypreParMatrixFromBlocks2(MPI_Comm comm,
-                                                Array<int> const& offsets, Array2D<HypreParMatrix*> const& blocks,
-                                                Array2D<SparseMatrix*> const& blocksSp,
-                                                Array2D<double> const& coefficient,
-                                                std::vector<std::vector<int> > const& blockProcOffsets,
-                                                std::vector<std::vector<int> > const& all_block_num_loc_rows);
-
-class BlockMGSolver : public Solver
-{
-private:
-   /// The linear system matrix
-   Array2D<HypreParMatrix *>&
-   Af;  // TODO: remove this, as it is used only in the constructor
-   Array2D<double>&
-   Acoef;  // TODO: remove this, as it is used only in the constructor
-   vector<Array<int>> Aoffsets;
-   vector<Array<int>> Poffsets_i;
-   vector<Array<int>> Poffsets_j;
-   std::vector<Array2D<HypreParMatrix *>> A;
-   std::vector<HypreParMatrix *> const& P;
-   std::vector<BlockOperator *> BlkP;
-   std::vector<BlockOperator *> BlkA;
-   std::vector<BlockOperator *> S;
-   HypreParMatrix * Ac;
-   SparseMatrix AcSp;
-   int numGrids, numBlocks;
-   //STRUMPACKSolver *invAc = nullptr;
-   Operator *invAc = nullptr;
-   double theta = 0.5;
-
-public:
-   BlockMGSolver(MPI_Comm comm, const int height, const int width,
-                 Array2D<HypreParMatrix *>& Af_,
-                 Array2D<double>& Acoef_, std::vector<HypreParMatrix *> const& P_)
-      : Solver(height, width), Af(Af_), Acoef(Acoef_), P(P_)
-   {
-      numBlocks = Af.NumRows();
-      MFEM_VERIFY(Af.NumCols() == numBlocks, "");
-      numGrids = P.size();
-      BlkP.resize(numGrids);
-      BlkA.resize(numGrids+1);
-      S.resize(numGrids);
-      A.resize(numGrids + 1);
-      A[numGrids] = Af;
-      Aoffsets.resize(numGrids+1);
-      Poffsets_i.resize(numGrids);
-      Poffsets_j.resize(numGrids);
-      // Construct Bilinear form Matrices on each level
-      for (int k = numGrids ; k > 0; k--)
-      {
-         A[k - 1].SetSize(numBlocks,numBlocks);
-         Aoffsets[k].SetSize(numBlocks+1); Aoffsets[k][0] = 0;
-         for (int i=0; i<numBlocks; i++)
-         {
-            Aoffsets[k][i+1] = A[k](i,i)->Height();
-         }
-
-         Aoffsets[k].PartialSum();
-         BlkA[k] = new BlockOperator(Aoffsets[k]);
-         S[k-1] = new BlockOperator(Aoffsets[k]);  // Smoother
-
-         for (int i=0; i<numBlocks; i++)
-         {
-            for (int j=0; j<numBlocks; j++)
-            {
-               if (A[k](i,j) == NULL)
-               {
-                  A[k - 1](i,j) = NULL;
-               }
-               else
-               {
-                  A[k - 1](i,j) = RAP(A[k](i,j), P[k - 1]);
-                  BlkA[k]->SetBlock(i, j, A[k](i,j), Acoef(i,j));
-               }
-            }
-
-            HypreSmoother *S_i = new HypreSmoother;
-            S_i->SetType(HypreSmoother::Jacobi);
-            S_i->SetOperator(*(A[k](i,i)));
-
-            S[k - 1]->SetBlock(i,i,S_i);
-         }
-
-         Poffsets_i[k-1].SetSize(numBlocks+1); Poffsets_i[k-1][0] = 0;
-         Poffsets_j[k-1].SetSize(numBlocks+1); Poffsets_j[k-1][0] = 0;
-         for (int i=0; i<numBlocks; i++)
-         {
-            Poffsets_i[k-1][i+1] = P[k-1]->Height();
-            Poffsets_j[k-1][i+1] = P[k-1]->Width();
-         }
-         Poffsets_i[k-1].PartialSum();
-         Poffsets_j[k-1].PartialSum();
-
-         BlkP[k-1] = new BlockOperator(Poffsets_i[k-1],Poffsets_j[k-1]);
-         for (int i=0; i<numBlocks; i++)
-         {
-            BlkP[k-1]->SetBlock(i,i,P[k-1]);
-         }
-      }
-      // Set up coarse solve operator
-      // Convert the coarse grid blockmatrix to a HypreParMatrix
-      Array<int> offsets(numBlocks+1);
-      offsets[0]=0;
-      for (int i=0; i<numBlocks; i++)
-      {
-         offsets[i+1]=A[0](i,i)->Height();
-      }
-
-      offsets.PartialSum();
-
-      BlkA[0] = new BlockOperator(offsets);
-
-      Array2D<SparseMatrix*> Asp;
-      //Array2D<double> Acoef;
-      Asp.SetSize(numBlocks,numBlocks);
-      //Acoef.SetSize(numBlocks,numBlocks);
-      for (int i=0; i<numBlocks; i++)
-      {
-         for (int j=0; j<numBlocks; j++)
-         {
-            if (A[0](i,j) != NULL)
-            {
-               BlkA[0]->SetBlock(i, j, A[0](i,j), Acoef(i,j));
-            }
-
-            Asp(i,j) = NULL;
-            //Acoef(i,j) = 1.0;
-         }
-      }
-
-      // Convert to HypreParMatrix
-      //HypreParMatrix * Ac;
-
-      std::vector<std::vector<int> > blockProcOffsets(numBlocks);
-      std::vector<std::vector<int> > all_block_num_loc_rows(numBlocks);
-
-      {
-         int nprocs, rank;
-         MPI_Comm_rank(comm, &rank);
-         MPI_Comm_size(comm, &nprocs);
-
-         std::vector<int> allnumrows(nprocs);
-         const int blockNumRows = A[0](0,
-                                       0)->Height();    // TODO: Not valid if blocks are of different size
-         MPI_Allgather(&blockNumRows, 1, MPI_INT, allnumrows.data(), 1, MPI_INT, comm);
-
-         for (int b=0; b<numBlocks; ++b)
-         {
-            blockProcOffsets[b].resize(nprocs);
-            all_block_num_loc_rows[b].resize(nprocs);
-         }
-
-         blockProcOffsets[0][0] = 0;
-         for (int i=0; i<nprocs-1; ++i)
-         {
-            blockProcOffsets[0][i+1] = blockProcOffsets[0][i] + allnumrows[i];
-         }
-
-         for (int i=0; i<nprocs; ++i)
-         {
-            for (int b=0; b<numBlocks; ++b)
-            {
-               all_block_num_loc_rows[b][i] = allnumrows[i];
-            }
-
-            for (int b=1; b<numBlocks; ++b)
-            {
-               blockProcOffsets[b][i] = blockProcOffsets[0][i];
-            }
-         }
-      }
-
-      Ac = CreateHypreParMatrixFromBlocks2(comm, offsets, A[0], Asp,
-                                           Acoef, blockProcOffsets, all_block_num_loc_rows);
-
-#ifdef MFEM_USE_STRUMPACK
-      invAc = CreateStrumpackSolver(new STRUMPACKRowLocMatrix(*Ac), comm);
-      delete Ac;
-#else
-      Ac->GetDiag(AcSp);  // AcSp does not own the data
-      UMFPackSolver *umf_solver = new UMFPackSolver();
-      umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver->SetOperator(AcSp);
-      invAc = umf_solver;
-#endif
-   }
-
-   virtual void SetOperator(const Operator &op) {}
-
-   virtual void SetTheta(const double a) { theta = a; }
-
-   virtual void Mult(const Vector &r, Vector &z) const
-   {
-      // Residual vectors
-      std::vector<Vector> rv(numGrids + 1);
-      // correction vectors
-      std::vector<Vector> zv(numGrids + 1);
-      // allocation
-      for (int i = 0; i <= numGrids ; i++)
-      {
-         int n = (i==0) ? invAc->Height(): BlkA[i]->Width();
-         rv[i].SetSize(n);
-         zv[i].SetSize(n);
-      }
-      // Initial residual
-      rv[numGrids] = r;
-
-      // smooth and update residuals down to the coarsest level
-      for (int i = numGrids; i > 0 ; i--)
-      {
-         // Pre smooth
-         S[i - 1]->Mult(rv[i], zv[i]); zv[i] *= theta;
-         // compute residual
-         int n = BlkA[i]->Width();
-         Vector w(n);
-         BlkA[i]->Mult(zv[i], w);
-         rv[i] -= w;
-         // Restrict
-         BlkP[i - 1]->MultTranspose(rv[i], rv[i - 1]);
-      }
-
-      // Coarse grid Solve
-      invAc->Mult(rv[0], zv[0]);
-      //
-      for (int i = 1; i <= numGrids ; i++)
-      {
-         // Prolong correction
-         Vector u(BlkP[i - 1]->Height());
-         BlkP[i - 1]->Mult(zv[i - 1], u);
-         // Update correction
-         zv[i] += u;
-         // Update residual
-         Vector v(BlkA[i]->Height());
-         BlkA[i]->Mult(u, v); rv[i] -= v;
-         // Post smooth
-         S[i - 1]->Mult(rv[i], v); v *= theta;
-         // Update correction
-         zv[i] += v;
-      }
-      z = zv[numGrids];
-   }
-
-   virtual ~BlockMGSolver()
-   {
-      for (int i = numGrids - 1; i >= 0 ; i--)
-      {
-         delete S[i];
-         delete BlkP[i];
-         delete BlkA[i];
-         for (int j=0; j<numBlocks; j++)
-         {
-            for (int k=0; k<numBlocks; k++)
-            {
-               delete A[i](j,k);
-            }
-         }
-         A[i].DeleteAll();
-      }
-      delete BlkA[numGrids];
-      delete invAc;
-      A.clear();
-   }
-
-#ifdef MFEM_USE_STRUMPACK
-   STRUMPACKSolver* CreateStrumpackSolver(Operator *Arow, MPI_Comm comm)
-   {
-      //STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, comm);
-      STRUMPACKSolver * strumpack = new STRUMPACKSolver(0, NULL, comm);
-      strumpack->SetPrintFactorStatistics(true);
-      strumpack->SetPrintSolveStatistics(false);
-      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-      strumpack->SetOperator(*Arow);
-      strumpack->SetFromCommandLine();
-      return strumpack;
-   }
-#endif
-
-};
-
-#define SWTIMING
-#define SERIAL_PROLONGATION
-#define ITERATIVE_COARSE_SOLVE
-#define SPARSE_JACOBI
-//#define SPARSE_ICHOLESKY
-//#define SPARSE_ILU
-//#define COARSE_PA
-//#define BLOCK_DIAG
-
-//#define COARSE_AMS
-
-class BlockMGPASolver : public Solver
-{
-private:
-   /// The linear system matrix
-   Array2D<Operator*>&
-   Af;  // TODO: remove this, as it is used only in the constructor
-   Array2D<double>&
-   Acoef;  // TODO: remove this, as it is used only in the constructor
-   vector<Array<int>> Aoffsets;
-   vector<Array<int>> Poffsets_i;
-   vector<Array<int>> Poffsets_j;
-   Array<int> Coffsets;
-
-   std::vector<Array2D<Operator*>> A;
-#ifdef SERIAL_PROLONGATION
-   std::vector<SparseMatrix *> P;
-#else
-   std::vector<HypreParMatrix *> const& P;
-#endif
-   std::vector<Vector*> const& diag;
-   std::vector<Operator*> Pt;
-   std::vector<BlockOperator *> BlkP;
-   std::vector<BlockOperator *> BlkA;
-   std::vector<BlockOperator *> S;
-   std::vector<OperatorJacobiSmoother*> Jacobi;
-
-   HypreParMatrix * Ac;
-   SparseMatrix AcSp;
-   int numGrids, numBlocks;
-   //STRUMPACKSolver *invAc = nullptr;
-   Operator *invAc = nullptr;
-   double theta = 0.5;
-
-   mutable std::vector<Vector> rv, zv;
-   mutable Vector u, v, w;
-
-   Vector diagAc;
-   Array<int> emptyEssDof;
-   OperatorJacobiSmoother *JacobiAc;
-#ifdef SPARSE_ICHOLESKY
-   Solver *iCholAc;
-#endif
-#ifdef SPARSE_ILU
-   Solver *iluAc;
-#endif
-
-#ifdef BLOCK_DIAG
-   HypreParMatrix *AcDiag;
-   SparseMatrix AcDiagSp;
-#endif
-
-public:
-   BlockMGPASolver(MPI_Comm comm, const int height, const int width,
-                   Array2D<Operator*>& Af_, Array2D<double>& Acoef_,
-#ifdef COARSE_PA
-                   BlockOperator *BlkAc,
-#else
-                   Array2D<HypreParMatrix*> const& BlkAc,
-#endif
-#ifdef COARSE_AMS
-                   ParFiniteElementSpace* coarseFespace,
-#endif
-                   std::vector<HypreParMatrix *> const& P_, std::vector<Vector*> const& diag_,
-                   Array<int>& ess_tdof_list)
-#ifdef SERIAL_PROLONGATION
-      : Solver(height, width), Af(Af_), Acoef(Acoef_), diag(diag_)
-#else
-      : Solver(height, width), Af(Af_), Acoef(Acoef_), P(P_), diag(diag_)
-#endif
-   {
-      timeMult = 0.0;
-      timeMultAc = 0.0;
-      timeMultPresmooth = 0.0;
-      timeMultResidual = 0.0;
-      timeMultRestrict = 0.0;
-
-#ifdef SERIAL_PROLONGATION
-      P.resize(P_.size());
-      for (int i=0; i<P_.size(); ++i)
-      {
-         P[i] = new SparseMatrix();
-         P_[i]->GetDiag(*P[i]);
-         P[i]->BuildTranspose();
-      }
-#endif
-
-      numBlocks = Af.NumRows();
-      MFEM_VERIFY(Af.NumCols() == numBlocks, "");
-#ifndef COARSE_PA
-      MFEM_VERIFY(BlkAc.NumCols() == numBlocks && BlkAc.NumRows() == numBlocks, "");
-#endif
-      numGrids = P.size();
-      MFEM_VERIFY(diag.size() == numBlocks, "");
-      BlkP.resize(numGrids);
-      BlkA.resize(numGrids+1);
-      Pt.resize(numGrids);
-      S.resize(numGrids);
-      A.resize(numGrids + 1);
-      A[numGrids] = Af;
-      Aoffsets.resize(numGrids+1);
-      Poffsets_i.resize(numGrids);
-      Poffsets_j.resize(numGrids);
-
-      Jacobi.resize(numBlocks);
-      for (int i=0; i<numBlocks; ++i)
-      {
-         Jacobi[i] = new OperatorJacobiSmoother(*(diag[i]), ess_tdof_list, 1.0);
-      }
-
-      // Construct Bilinear form Matrices on each level
-      for (int k = numGrids ; k > 0; k--)
-      {
-         A[k - 1].SetSize(numBlocks,numBlocks);
-         Aoffsets[k].SetSize(numBlocks+1); Aoffsets[k][0] = 0;
-         for (int i=0; i<numBlocks; i++)
-         {
-            Aoffsets[k][i+1] = A[k](i,i)->Height();
-         }
-
-         Aoffsets[k].PartialSum();
-         BlkA[k] = new BlockOperator(Aoffsets[k]);
-         S[k-1] = new BlockOperator(Aoffsets[k]);  // Smoother
-         Pt[k - 1] = new TransposeOperator(P[k - 1]);
-
-         for (int i=0; i<numBlocks; i++)
-         {
-            for (int j=0; j<numBlocks; j++)
-            {
-               if (A[k](i,j) == NULL)
-               {
-                  A[k - 1](i,j) = NULL;
-               }
-               else
-               {
-                  //A[k - 1](i,j) = RAP(A[k](i,j), P[k - 1]);
-                  A[k - 1](i,j) = new TripleProductOperator(Pt[k - 1], A[k](i,j), P[k - 1], false,
-                                                            false, false);
-                  BlkA[k]->SetBlock(i, j, A[k](i,j), Acoef(i,j));
-               }
-            }
-
-            if (k == numGrids)  // finest level
-            {
-               S[k - 1]->SetBlock(i,i,Jacobi[i]);
-            }
-            else
-            {
-               S[k - 1]->SetBlock(i,i, new TripleProductOperator(Pt[k], &(S[k]->GetBlock(i,i)),
-                                                                 P[k], false,
-                                                                 false, false));   // TODO: get a diagonal on each level
-            }
-         }
-
-         Poffsets_i[k-1].SetSize(numBlocks+1); Poffsets_i[k-1][0] = 0;
-         Poffsets_j[k-1].SetSize(numBlocks+1); Poffsets_j[k-1][0] = 0;
-         for (int i=0; i<numBlocks; i++)
-         {
-            Poffsets_i[k-1][i+1] = P[k-1]->Height();
-            Poffsets_j[k-1][i+1] = P[k-1]->Width();
-         }
-         Poffsets_i[k-1].PartialSum();
-         Poffsets_j[k-1].PartialSum();
-
-         BlkP[k-1] = new BlockOperator(Poffsets_i[k-1],Poffsets_j[k-1]);
-         for (int i=0; i<numBlocks; i++)
-         {
-            BlkP[k-1]->SetBlock(i,i,P[k-1]);
-         }
-      }
-      // Set up coarse solve operator
-      // Convert the coarse grid blockmatrix to a HypreParMatrix
-      Coffsets.SetSize(numBlocks+1);
-      Coffsets[0]=0;
-      for (int i=0; i<numBlocks; i++)
-      {
-         Coffsets[i+1]=A[0](i,i)->Height();
-#ifndef COARSE_PA
-         MFEM_VERIFY(BlkAc(i,i)->Height() == A[0](i,i)->Height(), "");
-         MFEM_VERIFY(BlkAc(i,i)->Width() == A[0](i,i)->Width(), "");
-#endif
-      }
-      Coffsets.PartialSum();
-
-#ifdef COARSE_PA
-      BlkA[0] = BlkAc;
-#else
-      BlkA[0] = new BlockOperator(Coffsets);
-#endif
-
-      Array2D<SparseMatrix*> Asp;
-      //Array2D<double> Acoef;
-      Asp.SetSize(numBlocks,numBlocks);
-      //Acoef.SetSize(numBlocks,numBlocks);
-      for (int i=0; i<numBlocks; i++)
-      {
-         for (int j=0; j<numBlocks; j++)
-         {
-#ifndef COARSE_PA
-            MFEM_VERIFY((A[0](i,j) == NULL) == (BlkAc(i,j) == NULL), "");
-
-            if (BlkAc(i,j) != NULL)
-            {
-               BlkA[0]->SetBlock(i, j, BlkAc(i,j), Acoef(i,j));
-            }
-#endif
-
-            Asp(i,j) = NULL;
-            //Acoef(i,j) = 1.0;
-         }
-      }
-
-#ifdef COARSE_PA
-      CGSolver *cg_solver = new CGSolver();
-      cg_solver->SetAbsTol(1.0e-6);
-      cg_solver->SetRelTol(1.0e-6);
-      cg_solver->SetMaxIter(1000);
-      cg_solver->SetOperator(*BlkAc);
-      cg_solver->SetPrintLevel(0);
-      cg_solver->iterative_mode = false;
-      invAc = cg_solver;
-#else
-      // Convert to HypreParMatrix
-      HypreParMatrix * Ac;
-
-      std::vector<std::vector<int> > blockProcOffsets(numBlocks);
-      std::vector<std::vector<int> > all_block_num_loc_rows(numBlocks);
-
-      {
-         int nprocs, rank;
-         MPI_Comm_rank(comm, &rank);
-         MPI_Comm_size(comm, &nprocs);
-
-         std::vector<int> allnumrows(nprocs);
-         // TODO: Not valid if blocks are of different size
-         const int blockNumRows = BlkAc(0,0)->Height();
-
-         MPI_Allgather(&blockNumRows, 1, MPI_INT, allnumrows.data(), 1, MPI_INT, comm);
-
-         for (int b=0; b<numBlocks; ++b)
-         {
-            blockProcOffsets[b].resize(nprocs);
-            all_block_num_loc_rows[b].resize(nprocs);
-         }
-
-         blockProcOffsets[0][0] = 0;
-         for (int i=0; i<nprocs-1; ++i)
-         {
-            blockProcOffsets[0][i+1] = blockProcOffsets[0][i] + allnumrows[i];
-         }
-
-         for (int i=0; i<nprocs; ++i)
-         {
-            for (int b=0; b<numBlocks; ++b)
-            {
-               all_block_num_loc_rows[b][i] = allnumrows[i];
-            }
-
-            for (int b=1; b<numBlocks; ++b)
-            {
-               blockProcOffsets[b][i] = blockProcOffsets[0][i];
-            }
-         }
-      }
-
-      Ac = CreateHypreParMatrixFromBlocks2(comm, Coffsets, BlkAc, Asp,
-                                           Acoef, blockProcOffsets, all_block_num_loc_rows);
-
-#ifdef BLOCK_DIAG
-      {
-         Array2D<HypreParMatrix*> BlkAcDiag(numBlocks, numBlocks);
-         Array2D<double> DiagCoef(numBlocks, numBlocks);
-         DiagCoef = 0.0;
-         for (int i=0; i<numBlocks; ++i)
-         {
-            DiagCoef(i,i) = Acoef(i,i);
-            for (int j=0; j<numBlocks; ++j)
-            {
-               BlkAcDiag(i,j) = NULL;
-            }
-
-            BlkAcDiag(i,i) = BlkAc(i,i);
-         }
-
-         AcDiag = CreateHypreParMatrixFromBlocks2(comm, Coffsets, BlkAcDiag, Asp,
-                                                  DiagCoef, blockProcOffsets, all_block_num_loc_rows);
-
-         AcDiag->GetDiag(AcDiagSp);
-         //delete AcDiag;
-      }
-#endif
-
-#ifdef MFEM_USE_STRUMPACK
-      invAc = CreateStrumpackSolver(new STRUMPACKRowLocMatrix(*Ac), comm);
-      delete Ac;
-#else
-      Ac->GetDiag(AcSp);  // AcSp does not own the data
-      delete Ac;
-
-#ifdef ITERATIVE_COARSE_SOLVE
-      //CGSolver *cg_solver = new CGSolver(comm);
-      CGSolver *cg_solver = new CGSolver();
-      cg_solver->SetAbsTol(1.0e-6);
-      cg_solver->SetRelTol(1.0e-6);
-      cg_solver->SetMaxIter(1000);
-      cg_solver->SetOperator(AcSp);
-      cg_solver->SetPrintLevel(0);
-      cg_solver->iterative_mode = false;
-#ifdef SPARSE_JACOBI
-      AcSp.GetDiag(diagAc);
-      JacobiAc = new OperatorJacobiSmoother(diagAc, emptyEssDof);
-      cg_solver->SetPreconditioner(*JacobiAc);
-#endif
-#ifdef SPARSE_ICHOLESKY
-      {
-         Vector tmpX(AcSp.Height());
-         Vector tmpY(AcSp.Height());
-         tmpX = 1.0;
-         tmpY = 0.0;
-#ifdef BLOCK_DIAG
-         AcDiagSp.Finalize();
-         AcDiagSp.SortColumnIndices();
-
-         AcDiagSp.Mult(tmpX, tmpY);
-#else
-         AcSp.Finalize();
-         AcSp.SortColumnIndices();
-
-         AcSp.Mult(tmpX, tmpY);
-#endif
-      }
-#ifdef BLOCK_DIAG
-      iCholAc = new IncompleteCholesky(AcDiagSp);
-#else
-      iCholAc = new IncompleteCholesky(AcSp);
-#endif
-      cg_solver->SetPreconditioner(*iCholAc);
-      cg_solver->SetPrintLevel(-1);
-#endif // SPARSE_ICHOLESKY
-
-#ifdef SPARSE_ILU
-      {
-         Vector tmpX(AcSp.Height());
-         Vector tmpY(AcSp.Height());
-         tmpX = 1.0;
-         tmpY = 0.0;
-#ifdef BLOCK_DIAG
-         AcDiagSp.Finalize();
-         AcDiagSp.SortColumnIndices();
-
-         AcDiagSp.Mult(tmpX, tmpY);
-#else
-         AcSp.Finalize();
-         AcSp.SortColumnIndices();
-
-         AcSp.Mult(tmpX, tmpY);
-#endif
-      }
-#ifdef BLOCK_DIAG
-      iluAc = new ILUcusparse(AcDiagSp);
-#else
-      iluAc = new ILUcusparse(AcSp);
-#endif
-      cg_solver->SetPreconditioner(*iluAc);
-      cg_solver->SetPrintLevel(0);
-#endif // SPARSE_ILU
-
-#ifdef COARSE_AMS
-#ifndef COARSE_PA
-      MFEM_VERIFY(numBlocks == 4, "");
-      // TODO: just set 2 AMS solvers for E and H.
-
-      if (coarseFespace != NULL)
-      {
-         BlockDiagonalPreconditioner *blkAMS = new BlockDiagonalPreconditioner(Coffsets);
-
-         for (int i=0; i<numBlocks; ++i)
-         {
-            HypreAMS *ams = new HypreAMS(*BlkAc(i,i), coarseFespace);
-            blkAMS->SetDiagonalBlock(i, ams);
-         }
-
-         cg_solver->SetPreconditioner(*blkAMS);
-      }
-#endif
-#endif
-
-      invAc = cg_solver;
-#else
-      UMFPackSolver *umf_solver = new UMFPackSolver();
-      umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver->SetOperator(AcSp);
-      invAc = umf_solver;
-#endif
-#endif
-#endif  // COARSE_PA
-
-      // Residual vectors
-      rv.resize(numGrids + 1);
-      // correction vectors
-      zv.resize(numGrids + 1);
-      // allocation
-      for (int i = 0; i <= numGrids ; i++)
-      {
-         int n = (i==0) ? invAc->Height(): BlkA[i]->Width();
-
-         rv[i].SetSize(n);
-         zv[i].SetSize(n);
-         rv[i].UseDevice(true);
-         zv[i].UseDevice(true);
-      }
-   }
-
-   virtual void SetOperator(const Operator &op) {}
-
-   virtual void SetTheta(const double a) { theta = a; }
-
-   virtual void Mult(const Vector &r, Vector &z) const
-   {
-#ifdef SWTIMING
-      StopWatch sw;
-      sw.Clear();
-      sw.Start();
-#endif
-
-      // Initial residual
-      rv[numGrids] = r;
-
-      // smooth and update residuals down to the coarsest level
-      for (int i = numGrids; i > 0 ; i--)
-      {
-         // Pre smooth
-#ifdef SWTIMING
-         StopWatch sws;
-         sws.Clear();
-         sws.Start();
-#endif
-
-         S[i - 1]->Mult(rv[i], zv[i]); zv[i] *= theta;
-
-#ifdef SWTIMING
-         sws.Stop();
-         timeMultPresmooth += sws.RealTime();
-#endif
-
-         // compute residual
-         int n = BlkA[i]->Width();
-         w.SetSize(n);
-         w.UseDevice(true);
-#ifdef SWTIMING
-         StopWatch swop;
-         swop.Clear();
-         swop.Start();
-#endif
-         BlkA[i]->Mult(zv[i], w);
-         rv[i] -= w;
-#ifdef SWTIMING
-         swop.Stop();
-         timeMultResidual += swop.RealTime();
-#endif
-         // Restrict
-#ifdef SWTIMING
-         StopWatch swr;
-         swr.Clear();
-         swr.Start();
-#endif
-         BlkP[i - 1]->MultTranspose(rv[i], rv[i - 1]);
-#ifdef SWTIMING
-         swr.Stop();
-         timeMultRestrict += swr.RealTime();
-#endif
-      }
-
-#ifdef SWTIMING
-      StopWatch swAc;
-      swAc.Clear();
-      swAc.Start();
-#endif
-
-      // Coarse grid Solve
-      invAc->Mult(rv[0], zv[0]);
-      //
-
-#ifdef SWTIMING
-      swAc.Stop();
-      timeMultAc += swAc.RealTime();
-#endif
-
-      for (int i = 1; i <= numGrids ; i++)
-      {
-         // Prolong correction
-         u.SetSize(BlkP[i - 1]->Height());
-         u.UseDevice(true);
-
-         BlkP[i - 1]->Mult(zv[i - 1], u);
-         // Update correction
-         zv[i] += u;
-         // Update residual
-         v.SetSize(BlkA[i]->Height());
-         v.UseDevice(true);
-
-         BlkA[i]->Mult(u, v); rv[i] -= v;
-         // Post smooth
-         S[i - 1]->Mult(rv[i], v); v *= theta;
-         // Update correction
-         zv[i] += v;
-      }
-      z = zv[numGrids];
-
-#ifdef SWTIMING
-      sw.Stop();
-      timeMult += sw.RealTime();
-#endif
-   }
-
-   virtual ~BlockMGPASolver()
-   {
-      for (int i = numGrids - 1; i >= 0 ; i--)
-      {
-         delete S[i];
-         delete BlkP[i];
-         delete BlkA[i];
-         for (int j=0; j<numBlocks; j++)
-         {
-            for (int k=0; k<numBlocks; k++)
-            {
-               delete A[i](j,k);
-            }
-         }
-         A[i].DeleteAll();
-      }
-      delete BlkA[numGrids];
-      delete invAc;
-      A.clear();
-
-#ifdef SPARSE_ICHOLESKY
-      delete iCholAc;
-#endif
-#ifdef SPARSE_ILU
-      delete iluAc;
-#endif
-   }
-
-#ifdef MFEM_USE_STRUMPACK
-   STRUMPACKSolver* CreateStrumpackSolver(Operator *Arow, MPI_Comm comm)
-   {
-      //STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, comm);
-      STRUMPACKSolver * strumpack = new STRUMPACKSolver(0, NULL, comm);
-      strumpack->SetPrintFactorStatistics(true);
-      strumpack->SetPrintSolveStatistics(false);
-      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-      strumpack->SetOperator(*Arow);
-      strumpack->SetFromCommandLine();
-      return strumpack;
-   }
-#endif
-
-   mutable double timeMult, timeMultAc, timeMultPresmooth, timeMultResidual,
-   timeMultRestrict;
-};
-
-}
-//} // namespace mfem
-
-#endif // BGMULTIGRID

examples/remapBlockMatrix.cpp

@@ -1,363 +0,0 @@
-#include <iostream>
-#include <fstream>
-#include <string>
-#include <vector>
-#include <cstdlib>
-#include <cmath>
-
-using namespace std;
-
-class HypreMat
-{
-private:
-   std::vector<int> I, J;
-   std::vector<double> d;
-   int size;
-
-public:
-   HypreMat() : size(0)
-   {
-
-   }
-
-   int Size() const { return size; }
-
-   void ReadMatrixParallel(std::string filename, const int np)
-   {
-      int lastRow = 0;
-      int count = 0;
-
-      std::vector<int> R;
-
-      for (int p=0; p<np; ++p)
-      {
-         std::ifstream f(filename.c_str() + std::to_string(p), std::ifstream::in);
-
-         int rowFirst, rowLast, colFirst, colLast, row, col;
-         double v;
-
-         f >> rowFirst >> rowLast >> colFirst >> colLast;
-
-         I.resize(rowLast+2);
-
-         for (row=rowFirst+1; row<=rowLast+1; ++row)
-         {
-            I[row] = 0;
-         }
-
-         while (f.good())
-         {
-            f >> row >> col >> v;
-
-            if (row >= 2485)
-            {
-               abort();
-            }
-
-            //if (row < lastRow)  // verify ascending order of rows
-            //  abort();
-
-            lastRow = row;
-
-            I[row+1]++;
-            R.push_back(row);
-            J.push_back(col);
-            d.push_back(v);
-
-            count++;
-         }
-
-         f.close();
-      }
-
-      // Partial sum of I
-
-      size = I.size() - 1;
-
-      I[0] = 0;
-      for (int j=1; j<size; ++j)
-      {
-         I[j+1] += I[j];
-      }
-
-      //int *Idata = I.data();
-
-      if (I[size] != J.size() || I[size] != d.size())
-      {
-         abort();
-      }
-
-      {
-         // Reorder J and d
-         const int nnz = J.size();
-         if (nnz != d.size() || nnz != R.size())
-         {
-            abort();
-         }
-
-         std::vector<int> JJ(nnz);
-         std::vector<double> dd(nnz);
-         std::vector<int> cnt;
-
-         JJ = J;
-         dd = d;
-
-         cnt.assign(I.size(), 0); // slightly larger than necessary
-
-         for (int j=0; j<nnz; ++j)
-         {
-            J[I[R[j]] + cnt[R[j]]] = JJ[j];
-            d[I[R[j]] + cnt[R[j]]] = dd[j];
-            cnt[R[j]]++;
-         }
-      }
-   }
-
-   void CopyReorder(HypreMat const& A, std::vector<int> const& permRow,
-                    std::vector<int> const& permCol)
-   {
-      if (permRow.size() != A.Size() || permCol.size() != A.Size())
-      {
-         abort();
-      }
-
-      size = A.Size();
-      I.resize(size+1);
-
-      I[0] = 0;
-
-      for (int j=0; j<size; ++j)
-      {
-         const std::size_t nnz_j = A.I[j+1] - A.I[j];
-         I[permRow[j]+1] = nnz_j;
-      }
-
-      for (int j=1; j<size; ++j)
-      {
-         I[j+1] += I[j];
-      }
-
-      const std::size_t nnz = I[size];
-      J.resize(nnz);
-      d.resize(nnz);
-
-      std::vector<std::size_t> cnt;
-      cnt.assign(size, 0);
-
-      for (int j=0; j<size; ++j)
-      {
-         const int pj = permRow[j];
-         const std::size_t nnz_j = A.I[j+1] - A.I[j];
-
-         for (int k=0; k<nnz_j; ++k, cnt[pj]++)
-         {
-            J[I[pj] + cnt[pj]] = permCol[A.J[A.I[j] + k]];
-            d[I[pj] + cnt[pj]] = A.d[A.I[j] + k];
-         }
-      }
-   }
-
-   void Print(std::string filename)
-   {
-      std::ofstream f(filename.c_str());
-
-      for (int j=0; j<size; ++j)
-      {
-         for (int k=I[j]; k<I[j+1]; ++k)
-         {
-            f << j << " " << J[k] << " " << d[k] << endl;
-         }
-      }
-
-      f.close();
-   }
-
-   void Compare(HypreMat const& A)
-   {
-      if (size != A.Size())
-      {
-         abort();
-      }
-
-      for (int j=0; j<size; ++j)
-      {
-         for (int k=I[j]; k<I[j+1]; ++k)
-         {
-            int m = -1;
-
-            for (int l=A.I[j]; l<A.I[j+1]; ++l)
-            {
-               if (A.J[l] == J[k])
-               {
-                  m = l;
-               }
-            }
-
-            if (m < 0)
-            {
-               cout << "row " << j << " column " << J[k] << " not found" << endl;
-            }
-            //else if (fabs(A.d[m] - d[k]) > 1.0e-8)
-            else if (fabs(fabs(A.d[m]) - fabs(d[k])) > 0.0)
-            {
-               cout << "entry (" << j << ", " << J[k] << ") " << d[k] << " != " << A.d[m] <<
-                    endl;
-            }
-         }
-      }
-   }
-};
-
-void ReadCrd(std::string filename, std::vector<double>& crd)
-{
-   std::ifstream f(filename.c_str(), std::ifstream::in);
-
-   double c;
-
-   while (f.good())
-   {
-      f >> c;
-      crd.push_back(c);
-   }
-
-   // for some reason, the last value gets pushed twice
-   crd.pop_back();
-
-   f.close();
-}
-
-void ReadCrdParallel(std::string filenamebase, const int np,
-                     std::vector<double>& crd)
-{
-   for (int p=0; p<np; ++p)
-   {
-      ReadCrd(filenamebase + std::to_string(p),
-              crd);  // results get concatenated in crd
-   }
-}
-
-// crd is the DOF coordinates in serial, crdp in parallel.
-void SerialToParallelDOFMap(std::vector<double> const& crd,
-                            std::vector<double> const& crdp,
-                            std::vector<int>& sp)
-{
-   if (crdp.size() != crd.size())
-   {
-      abort();
-   }
-
-   const int dim = 3;
-   const int n = crd.size() / dim;  // number of DOF's
-
-   sp.resize(n);
-
-   const double tol = 1.0e-8;
-
-   // Simple O(n^2) search
-
-   for (int i=0; i<n; ++i)
-   {
-      sp[i] = -1;
-
-      for (int j=0; j<n; ++j)
-      {
-         bool eq = true;
-         for (int l=0; l<dim; ++l)
-         {
-            if (fabs(crd[(dim*i)+l] - crdp[(dim*j)+l]) > tol)
-            {
-               eq = false;
-            }
-         }
-
-         if (eq)
-         {
-            if (sp[i] != -1)
-            {
-               abort();
-            }
-
-            sp[i] = j;
-         }
-      }
-
-      if (sp[i] < 0)
-      {
-         abort();
-      }
-   }
-}
-
-int main()
-{
-   const int np = 2;
-
-   std::vector<double> crd0, crdp0, crd1, crdp1;
-
-   ReadCrd("dofcrd0Ser", crd0);
-   ReadCrd("dofcrd1Ser", crd1);
-
-   const int numCrd = crd0.size() / 3;
-
-   if (3*numCrd != crd0.size())
-   {
-      abort();
-   }
-
-   //for (int i=7440; i<crd.size(); ++i)
-   //cout << "crd[" << i << "] " << crd[i] << endl;
-
-   ReadCrdParallel("dofcrd0Par", np, crdp0);
-   ReadCrdParallel("dofcrd1Par", np, crdp1);
-
-   if (3*numCrd != crdp0.size())
-   {
-      abort();
-   }
-
-   std::vector<int> sp0, sp1;  // serial to parallel DOF map for a subdomain
-   SerialToParallelDOFMap(crd0, crdp0, sp0);
-   SerialToParallelDOFMap(crd1, crdp1, sp1);
-
-   /*
-   for (int i=0; i<sp.size(); ++i)
-     cout << i << " " << sp[i] << endl;
-   */
-
-   cout << "sp0" << endl;
-   for (int i=0; i<sp0.size(); ++i)
-   {
-      cout << i << " " << sp0[i] << endl;
-   }
-
-   cout << "sp1" << endl;
-   for (int i=0; i<sp1.size(); ++i)
-   {
-      cout << i << " " << sp1[i] << endl;
-   }
-
-   HypreMat Aser, Apar, AserToPar;
-
-   /*
-   Aser.ReadMatrixParallel("HypreAsdComplexIm1_Serial.0000", 1);
-   Apar.ReadMatrixParallel("HypreAsdComplexIm1_Par5.0000", np);
-   */
-
-   Aser.ReadMatrixParallel("ifopSer.0000", 1);
-   Apar.ReadMatrixParallel("ifopPar.0000", np);
-
-   //AserToPar.CopyReorder(Aser, sp, sp);
-   AserToPar.CopyReorder(Aser, sp0, sp1);
-
-   /*
-   Aser.Print("rbSer");
-   AserToPar.Print("rbSerToPar");
-   Apar.Print("rbPar");
-   */
-
-   cout << "Comparing " << endl;
-
-   AserToPar.Compare(Apar);
-
-   return 0;
-}

examples/testStrumpack.hpp

@@ -1,105 +0,0 @@
-#ifndef TESTSTRUMPACK_HPP
-#define TESTSTRUMPACK_HPP
-
-#include "mfem.hpp"
-
-using namespace mfem;
-using namespace std;
-
-
-void TestStrumpackConstructor()
-{
-   int num_procs, rank;
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-   const int num_loc_rows = 100;
-   const int first_loc_row = num_loc_rows * rank;
-   const int glob_nrows = num_loc_rows * num_procs;
-   const int glob_ncols = glob_nrows;
-
-   int *opI = new int[num_loc_rows+1];
-
-   for (int i=0; i<num_loc_rows+1; ++i)
-   {
-      opI[i] = 0;
-   }
-
-   for (int i=0; i<num_loc_rows; ++i)
-   {
-      int nnz_i = 3;
-
-      if ((first_loc_row + i) == 0 ||
-          (first_loc_row + i) == glob_nrows-1)  // if first or last row
-      {
-         nnz_i = 2;
-      }
-
-      opI[i+1] = opI[i] + nnz_i;
-   }
-
-   const int nnz = opI[num_loc_rows];
-
-   int *opJ = new int[nnz];
-   double *data = new double[nnz];
-
-   int cnt = 0;
-   for (int i=0; i<num_loc_rows; ++i)
-   {
-      const int globalRow = first_loc_row + i;
-
-      // Diagonal entry
-
-      opJ[cnt] = first_loc_row + i;
-
-      if (globalRow == 0 || globalRow == glob_nrows-1)
-      {
-         data[cnt] = 2.0;
-      }
-      else
-      {
-         data[cnt] = 3.0;
-      }
-
-      cnt++;
-
-      // Upper diagonal
-      if (globalRow < glob_nrows-1)
-      {
-         opJ[cnt] = first_loc_row + i + 1;
-         data[cnt] = -1.0;
-         cnt++;
-      }
-
-      // Lower diagonal
-      if (globalRow > 0)
-      {
-         opJ[cnt] = first_loc_row + i - 1;
-         data[cnt] = -1.0;
-         cnt++;
-      }
-   }
-
-   Operator *op = new STRUMPACKRowLocMatrix(MPI_COMM_WORLD, num_loc_rows,
-                                            first_loc_row, glob_nrows, glob_ncols, opI, opJ, data);
-
-   STRUMPACKSolver * strumpack = new STRUMPACKSolver(0, NULL, MPI_COMM_WORLD);
-   strumpack->SetPrintFactorStatistics(true);
-   strumpack->SetPrintSolveStatistics(false);
-   strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-   strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-   strumpack->SetOperator(*op);
-   strumpack->SetFromCommandLine();
-
-   Vector x(num_loc_rows);
-   Vector y(num_loc_rows);
-
-   x = 1.0;
-   strumpack->Mult(x, y);
-
-   delete opI;
-   delete opJ;
-   delete data;
-}
-
-#endif // TESTSTRUMPACK_HPP

fem/CMakeLists.txt

@@ -31,7 +31,6 @@ set(SRCS
   bilininteg_vecmass.cpp
   coefficient.cpp
   complex_fem.cpp
-  convergence.cpp
   datacollection.cpp
   eltrans.cpp
   estimators.cpp
@@ -66,7 +65,6 @@ set(HDRS
   bilininteg.hpp
   coefficient.hpp
   complex_fem.hpp
-  convergence.hpp
   datacollection.hpp
   eltrans.hpp
   estimators.hpp

fem/bilinearform.cpp

@@ -1771,40 +1771,9 @@ MixedBilinearForm::~MixedBilinearForm()
    delete ext;
 }
 
-void DiscreteLinearOperator::SetAssemblyLevel(AssemblyLevel assembly_level)
-{
-   if (ext)
-   {
-      MFEM_ABORT("the assembly level has already been set!");
-   }
-   assembly = assembly_level;
-   switch (assembly)
-   {
-      case AssemblyLevel::FULL:
-         // Use the original BilinearForm implementation for now
-         break;
-      case AssemblyLevel::ELEMENT:
-         mfem_error("Element assembly not supported yet... stay tuned!");
-         break;
-      case AssemblyLevel::PARTIAL:
-         ext = new PADiscreteLinearOperatorExtension(this);
-         break;
-      case AssemblyLevel::NONE:
-         mfem_error("Matrix-free action not supported yet... stay tuned!");
-         break;
-      default:
-         mfem_error("Unknown assembly level");
-   }
-}
 
 void DiscreteLinearOperator::Assemble(int skip_zeros)
 {
-   if (ext)
-   {
-      ext->Assemble();
-      return;
-   }
-
    Array<int> dom_vdofs, ran_vdofs;
    ElementTransformation *T;
    const FiniteElement *dom_fe, *ran_fe;

fem/bilinearform.hpp

@@ -375,9 +375,6 @@ public:
    /// Get the output finite element space prolongation matrix
    virtual const Operator *GetOutputProlongation() const
    { return GetProlongation(); }
-   /// Get the output finite element space prolongation matrix (local diagonal)
-   virtual const Operator *GetLocalOutputProlongation() const
-   { return GetOutputProlongation(); }
    /// Get the output finite element space restriction matrix
    virtual const Operator *GetOutputRestriction() const
    { return GetRestriction(); }
@@ -594,18 +591,6 @@ public:
    /// Indicate that integrators are not owned by the BilinearForm
    void UseExternalIntegrators() { extern_bfs = 1; };
 
-   void GetTimings(double& t1, double& t2)
-   {
-      t1 = 0.0;
-      t2 = 0.0;
-      PABilinearFormExtension *paext = dynamic_cast<PABilinearFormExtension*>(ext);
-      if (paext != NULL)
-      {
-         t1 = paext->timingDomain;
-         t2 = paext->timingBoundary;
-      }
-   }
-
    /// Destroys bilinear form.
    virtual ~BilinearForm();
 };
@@ -999,17 +984,9 @@ public:
    /// Access all interpolators added with AddDomainInterpolator().
    Array<BilinearFormIntegrator*> *GetDI() { return &dbfi; }
 
-   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
-   /** This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level);
-
    /** @brief Construct the internal matrix representation of the discrete
        linear operator. */
    virtual void Assemble(int skip_zeros = 1);
-
-   /// Get the output finite element space prolongation matrix (local diagonal)
-   virtual const Operator *GetLocalOutputProlongation() const
-   { return test_fes->GetLocalProlongationMatrix(); }
 };
 
 }

fem/bilinearform_ext.cpp

@@ -17,8 +17,6 @@
 #include "libceed/ceed.hpp"
 #include "pgridfunc.hpp"
 
-#include "../general/tic_toc.hpp"
-
 namespace mfem
 {
 
@@ -48,9 +46,6 @@ PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
    elem_restrict = NULL;
    int_face_restrict_lex = NULL;
    bdr_face_restrict_lex = NULL;
-
-   timingDomain = 0.0;
-   timingBoundary = 0.0;
 }
 
 void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
@@ -101,12 +96,8 @@ void PABilinearFormExtension::Assemble()
       integrators[i]->AssemblePA(*a->FESpace());
    }
 
-   Array<BilinearFormIntegrator*> &bdryIntegrators = *a->GetBBFI();
-   const int bdryIntegratorCount = bdryIntegrators.Size();
-   for (int i = 0; i < bdryIntegratorCount; ++i)
-   {
-      bdryIntegrators[i]->AssemblePA(*a->FESpace());
-   }
+   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
+               "Partial assembly does not support AddBoundaryIntegrator yet.");
 
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
    const int intFaceIntegratorCount = intFaceIntegrators.Size();
@@ -121,96 +112,6 @@ void PABilinearFormExtension::Assemble()
    {
       bdrFaceIntegrators[i]->AssemblePABoundaryFaces(*a->FESpace());
    }
-
-   timingDomain = 0.0;
-   timingBoundary = 0.0;
-
-   if (bdryIntegratorCount > 0)
-   {
-      FiniteElementSpace *fes = a->GetFES();
-      const int nbe = fes->GetNBE();
-      const int bedofs = nbe > 0 ? nbe * fes->GetVDim() * fes->GetBE(0)->GetDof() : 0;
-      gatherMap.SetSize(bedofs);
-      indices.SetSize(bedofs);
-
-      ndofs = fes->GetNDofs();
-      offsets.SetSize(ndofs+1);
-      for (int i = 0; i <= ndofs; ++i)
-      {
-         offsets[i] = 0;
-      }
-
-      for (int i = 0; i < nbe; i++)
-      {
-         const FiniteElement &be = *fes->GetBE(i);
-         Array<int> vdofs;
-         fes -> GetBdrElementVDofs (i, vdofs);
-
-         const TensorBasisElement* el =
-            dynamic_cast<const TensorBasisElement*>(&be);
-
-         MFEM_VERIFY(el != NULL, "");
-
-         const Array<int> &fe_dof_map = el->GetDofMap();
-
-         MFEM_VERIFY(fe_dof_map.Size() == fes->GetBE(i)->GetDof(), "");
-         MFEM_VERIFY(vdofs.Size() == fes->GetBE(i)->GetDof(), "");
-
-         for (int j=0; j<vdofs.Size(); ++j)
-         {
-            const int sidj = fe_dof_map[j];
-            const int idj = sidj >= 0 ? sidj : -1 - sidj;
-            const int dof_j = vdofs[idj];
-            const int d = dof_j >= 0 ? dof_j : -1-dof_j;
-            offsets[d+1]++;
-         }
-      }
-
-      for (int i = 1; i <= ndofs; ++i)
-      {
-         // Partial sum
-         offsets[i] += offsets[i - 1];
-      }
-
-      int os = 0;
-      Array<int> cnt(ndofs);
-      cnt = 0;
-      indices = 0;
-
-      for (int i = 0; i < nbe; i++)
-      {
-         const FiniteElement &be = *fes->GetBE(i);
-         Array<int> vdofs;
-         fes -> GetBdrElementVDofs (i, vdofs);
-
-         const TensorBasisElement* el =
-            dynamic_cast<const TensorBasisElement*>(&be);
-
-         MFEM_VERIFY(el != NULL, "");
-
-         const Array<int> &fe_dof_map = el->GetDofMap();
-
-         MFEM_VERIFY(fe_dof_map.Size() == fes->GetBE(i)->GetDof(), "");
-         MFEM_VERIFY(vdofs.Size() == fes->GetBE(i)->GetDof(), "");
-
-         for (int j=0; j<vdofs.Size(); ++j)
-         {
-            const int sidj = fe_dof_map[j];
-            const int idj = sidj >= 0 ? sidj : -1 - sidj;
-            const int dof_j = vdofs[idj];
-            const bool plus = (sidj >= 0 && dof_j >= 0) || (sidj < 0 && dof_j < 0);
-            const int d = dof_j >= 0 ? dof_j : -1-dof_j;
-            const int lid = os + j;
-            gatherMap[lid] = plus ? d : -1-d;
-            indices[offsets[d] + cnt[d]] = plus ? lid : -1-lid;
-            cnt[d]++;
-         }
-
-         os += vdofs.Size();
-      }
-
-      MFEM_VERIFY(os == bedofs, "");
-   }
 }
 
 void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
@@ -278,18 +179,10 @@ void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
    A.Reset(oper); // A will own oper
 }
 
-//#define SWTIMING
-
 void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
 {
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
 
-#ifdef SWTIMING
-   StopWatch swd;
-   swd.Clear();
-   swd.Start();
-#endif
-
    const int iSz = integrators.Size();
    if (DeviceCanUseCeed() || !elem_restrict)
    {
@@ -311,93 +204,6 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
       elem_restrict->MultTranspose(localY, y);
    }
 
-#ifdef SWTIMING
-   swd.Stop();
-   timingDomain += swd.RealTime();
-
-   StopWatch swb;
-   swb.Clear();
-   swb.Start();
-#endif
-
-   Array<BilinearFormIntegrator*> &bdryIntegrators = *a->GetBBFI();
-
-   const int biSz = bdryIntegrators.Size();
-   if (DeviceCanUseCeed() || !elem_restrict)
-   {
-      MFEM_ABORT("Not implemented");
-   }
-   else if (biSz > 0)
-   {
-      FiniteElementSpace *fes = a->GetFES();
-
-      const int nbe = fes->GetNBE();
-      const int bedofs = nbe > 0 ? nbe * fes->GetVDim() * fes->GetBE(0)->GetDof() : 0;
-      bdryX.SetSize(bedofs);
-      bdryY.SetSize(bedofs);
-      bdryX.UseDevice(true);
-      bdryY.UseDevice(true);
-      bdryX = 0.0;
-      bdryY = 0.0;
-
-      {
-         MFEM_VERIFY(gatherMap.Size() == bedofs, "");
-         auto d_gatherMap = gatherMap.Read();
-         auto d_x = x.Read();
-         auto d_bdryX = bdryX.Write();
-
-         MFEM_FORALL(i, bedofs,
-         {
-            const int gid = d_gatherMap[i];
-            const bool plus = gid >= 0;
-            const int j = plus ? gid : -1-gid;
-
-            d_bdryX[i] = plus ? d_x[j] : -d_x[j];
-         });
-      }
-
-      for (int i = 0; i < biSz; ++i)
-      {
-         bdryIntegrators[i]->AddMultPA(bdryX, bdryY);
-      }
-      //elem_restrict->MultTranspose(bdryY, y);
-
-      // bdryY contains quantities on all boundary elements. Now add them to y.
-
-      {
-         const int xsize = x.Size();
-         MFEM_VERIFY(y.Size() == xsize, "");
-         MFEM_VERIFY(gatherMap.Size() == bedofs, "");
-         auto d_offsets = offsets.Read();
-         auto d_indices = indices.Read();
-         auto d_y = y.ReadWrite();
-         auto d_bdryY = bdryY.Read();
-
-         MFEM_FORALL(i, ndofs,
-         {
-            const int offset = d_offsets[i];
-            const int nextOffset = d_offsets[i + 1];
-
-            double val = 0.0;
-            for (int j = offset; j < nextOffset; ++j)
-            {
-               const int id = d_indices[j];
-               const bool plus = id >= 0;
-               const int idj = plus ? id : -1-id;
-               const double yval = d_bdryY[idj];
-               val += plus ? yval : -yval;
-            }
-
-            d_y[i] += val;
-         });
-      }
-   }
-
-#ifdef SWTIMING
-   swb.Stop();
-   timingBoundary += swb.RealTime();
-#endif
-
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
    const int iFISz = intFaceIntegrators.Size();
    if (int_face_restrict_lex && iFISz>0)
@@ -455,10 +261,6 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
       }
    }
 
-   Array<BilinearFormIntegrator*> &bdryIntegrators = *a->GetBBFI();
-   const int biSz = bdryIntegrators.Size();
-   MFEM_VERIFY(biSz == 0, "TODO");
-
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
    const int iFISz = intFaceIntegrators.Size();
    if (int_face_restrict_lex && iFISz>0)
@@ -508,12 +310,13 @@ void EABilinearFormExtension::Assemble()
 
    ea_data.SetSize(ne*elemDofs*elemDofs, Device::GetMemoryType());
    ea_data.UseDevice(true);
+   ea_data = 0.0;
 
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
    const int integratorCount = integrators.Size();
    for (int i = 0; i < integratorCount; ++i)
    {
-      integrators[i]->AssembleEA(*a->FESpace(), ea_data, i);
+      integrators[i]->AssembleEA(*a->FESpace(), ea_data);
    }
 
    faceDofs = trialFes ->
@@ -530,13 +333,14 @@ void EABilinearFormExtension::Assemble()
       nf_int = trialFes->GetNFbyType(FaceType::Interior);
       ea_data_int.SetSize(2*nf_int*faceDofs*faceDofs, Device::GetMemoryType());
       ea_data_ext.SetSize(2*nf_int*faceDofs*faceDofs, Device::GetMemoryType());
+      ea_data_int = 0.0;
+      ea_data_ext = 0.0;
    }
    for (int i = 0; i < intFaceIntegratorCount; ++i)
    {
       intFaceIntegrators[i]->AssembleEAInteriorFaces(*a->FESpace(),
                                                      ea_data_int,
-                                                     ea_data_ext,
-                                                     i);
+                                                     ea_data_ext);
    }
 
    Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
@@ -549,7 +353,7 @@ void EABilinearFormExtension::Assemble()
    }
    for (int i = 0; i < boundFaceIntegratorCount; ++i)
    {
-      bdrFaceIntegrators[i]->AssembleEABoundaryFaces(*a->FESpace(),ea_data_bdr,i);
+      bdrFaceIntegrators[i]->AssembleEABoundaryFaces(*a->FESpace(),ea_data_bdr);
    }
 
    if (factorize_face_terms && int_face_restrict_lex)
@@ -1019,6 +823,7 @@ void PAMixedBilinearFormExtension::Update()
       localTrial.UseDevice(true);
       localTrial.SetSize(elem_restrict_trial->Height(),
                          Device::GetMemoryType());
+
    }
    if (elem_restrict_test)
    {
@@ -1218,133 +1023,4 @@ void PAMixedBilinearFormExtension::AssembleDiagonal_ADAt(const Vector &D,
    }
 }
 
-PADiscreteLinearOperatorExtension::PADiscreteLinearOperatorExtension(
-   DiscreteLinearOperator *linop) :
-   PAMixedBilinearFormExtension(linop)
-{
-}
-
-const Operator *PADiscreteLinearOperatorExtension::GetLocalOutputProlongation()
-const
-{
-   return a->GetLocalOutputProlongation();
-}
-
-void PADiscreteLinearOperatorExtension::Assemble()
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int integratorCount = integrators.Size();
-   for (int i = 0; i < integratorCount; ++i)
-   {
-      integrators[i]->AssemblePA(*trialFes, *testFes);
-   }
-
-   test_multiplicity.UseDevice(true);
-   test_multiplicity.SetSize(elem_restrict_test->Width()); // l-vector
-   Vector ones(elem_restrict_test->Height()); // e-vector
-   ones = 1.0;
-
-   const ElementRestriction* elem_restrict =
-      dynamic_cast<const ElementRestriction*>(elem_restrict_test);
-   if (elem_restrict)
-   {
-      elem_restrict->MultTransposeUnsigned(ones, test_multiplicity);
-   }
-   else
-   {
-      mfem_error("A real ElementRestriction is required in this setting!");
-   }
-
-   auto tm = test_multiplicity.ReadWrite();
-   MFEM_FORALL(i, test_multiplicity.Size(),
-   {
-      tm[i] = 1.0 / tm[i];
-   });
-}
-
-void PADiscreteLinearOperatorExtension::AddMult(
-   const Vector &x, Vector &y, const double c) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int iSz = integrators.Size();
-
-   // * G operation
-   SetupMultInputs(elem_restrict_trial, x, localTrial,
-                   elem_restrict_test, y, localTest, c);
-
-   // * B^TDB operation
-   for (int i = 0; i < iSz; ++i)
-   {
-      integrators[i]->AddMultPA(localTrial, localTest);
-   }
-
-   // do a kind of "set" rather than "add" in the below
-   // operation as compared to the BilinearForm case
-   // * G^T operation (kind of...)
-   const ElementRestriction* elem_restrict =
-      dynamic_cast<const ElementRestriction*>(elem_restrict_test);
-   if (elem_restrict)
-   {
-      tempY.SetSize(y.Size());
-      elem_restrict->MultLeftInverse(localTest, tempY);
-      y += tempY;
-   }
-   else
-   {
-      mfem_error("In this setting you need a real ElementRestriction!");
-   }
-}
-
-void PADiscreteLinearOperatorExtension::AddMultTranspose(
-   const Vector &x, Vector &y, const double c) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int iSz = integrators.Size();
-
-   // do a kind of "set" rather than "add" in the below
-   // operation as compared to the BilinearForm case
-   // * G operation (kinda)
-   Vector xscaled(x);
-   MFEM_VERIFY(x.Size() == test_multiplicity.Size(), "Input vector of wrong size");
-   auto xs = xscaled.ReadWrite();
-   auto tm = test_multiplicity.Read();
-   MFEM_FORALL(i, x.Size(),
-   {
-      xs[i] *= tm[i];
-   });
-   SetupMultInputs(elem_restrict_test, xscaled, localTest,
-                   elem_restrict_trial, y, localTrial, c);
-
-   // * B^TD^TB operation
-   for (int i = 0; i < iSz; ++i)
-   {
-      integrators[i]->AddMultTransposePA(localTest, localTrial);
-   }
-
-   // * G^T operation
-   if (elem_restrict_trial)
-   {
-      tempY.SetSize(y.Size());
-      elem_restrict_trial->MultTranspose(localTrial, tempY);
-      y += tempY;
-   }
-   else
-   {
-      mfem_error("Trial ElementRestriction not defined");
-   }
-}
-
-void PADiscreteLinearOperatorExtension::FormRectangularSystemOperator(
-   const Array<int>& ess1, const Array<int>& ess2, OperatorHandle &A)
-{
-   const Operator *Pi = this->GetProlongation();
-   const Operator *Po = this->GetLocalOutputProlongation();
-   Operator *rap = SetupRAP(Pi, Po);
-
-   RectangularConstrainedOperator *Arco
-      = new RectangularConstrainedOperator(rap, ess1, ess2, rap != this);
-
-   A.Reset(Arco);
-}
-
 } // namespace mfem

fem/bilinearform_ext.hpp

@@ -21,7 +21,6 @@ namespace mfem
 
 class BilinearForm;
 class MixedBilinearForm;
-class DiscreteLinearOperator;
 
 /// Class extending the BilinearForm class to support different AssemblyLevels.
 /**  FA - Full Assembly
@@ -69,16 +68,11 @@ class PABilinearFormExtension : public BilinearFormExtension
 protected:
    const FiniteElementSpace *trialFes, *testFes; // Not owned
    mutable Vector localX, localY;
-   mutable Vector bdryX, bdryY;
    mutable Vector faceIntX, faceIntY;
    mutable Vector faceBdrX, faceBdrY;
    const Operator *elem_restrict; // Not owned
    const Operator *int_face_restrict_lex; // Not owned
    const Operator *bdr_face_restrict_lex; // Not owned
-   Array<int> gatherMap;
-   Array<int> indices;
-   Array<int> offsets;
-   int ndofs;
 
 public:
    PABilinearFormExtension(BilinearForm*);
@@ -94,8 +88,6 @@ public:
    void MultTranspose(const Vector &x, Vector &y) const;
    void Update();
 
-   mutable double timingDomain, timingBoundary;
-
 protected:
    void SetupRestrictionOperators(const L2FaceValues m);
 };
@@ -213,7 +205,7 @@ protected:
    mutable Vector localTrial, localTest, tempY;
    const Operator *elem_restrict_trial; // Not owned
    const Operator *elem_restrict_test;  // Not owned
-
+private:
    /// Helper function to set up inputs/outputs for Mult or MultTranspose
    void SetupMultInputs(const Operator *elem_restrict_x,
                         const Vector &x, Vector &localX,
@@ -259,33 +251,6 @@ public:
    void Update();
 };
 
-
-/**
-   This acts very much like PAMixedBilinearFormExtension, but its
-   FormRectangularSystemOperator implementation emulates 'Set' rather than
-   'Add' in the assembly case.
-*/
-class PADiscreteLinearOperatorExtension : public PAMixedBilinearFormExtension
-{
-public:
-   PADiscreteLinearOperatorExtension(DiscreteLinearOperator *linop);
-
-   /// Partial assembly of all internal integrators
-   void Assemble();
-
-   void AddMult(const Vector &x, Vector &y, const double c) const;
-
-   void AddMultTranspose(const Vector &x, Vector &y, const double c=1.0) const;
-
-   void FormRectangularSystemOperator(const Array<int>&, const Array<int>&,
-                                      OperatorHandle& A);
-
-   const Operator * GetLocalOutputProlongation() const;
-
-private:
-   Vector test_multiplicity;
-};
-
 }
 
 #endif

fem/bilininteg.cpp

@@ -22,14 +22,14 @@ namespace mfem
 
 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePA(fes)\n"
+   mfem_error ("BilinearFormIntegrator::AssemblePA(...)\n"
                "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&,
                                         const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePA(fes, fes)\n"
+   mfem_error ("BilinearFormIntegrator::AssemblePA(...)\n"
                "   is not implemented for this class.");
 }
 
@@ -52,8 +52,7 @@ void BilinearFormIntegrator::AssembleDiagonalPA(Vector &)
 }
 
 void BilinearFormIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                        Vector &emat,
-                                        const bool add)
+                                        Vector &emat)
 {
    mfem_error ("BilinearFormIntegrator::AssembleEA(...)\n"
                "   is not implemented for this class.");
@@ -62,8 +61,7 @@ void BilinearFormIntegrator::AssembleEA(const FiniteElementSpace &fes,
 void BilinearFormIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace
                                                      &fes,
                                                      Vector &ea_data_int,
-                                                     Vector &ea_data_ext,
-                                                     const bool add)
+                                                     Vector &ea_data_ext)
 {
    mfem_error ("BilinearFormIntegrator::AssembleEAInteriorFaces(...)\n"
                "   is not implemented for this class.");
@@ -71,8 +69,7 @@ void BilinearFormIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace
 
 void BilinearFormIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace
                                                      &fes,
-                                                     Vector &ea_data_bdr,
-                                                     const bool add)
+                                                     Vector &ea_data_bdr)
 {
    mfem_error ("BilinearFormIntegrator::AssembleEABoundaryFaces(...)\n"
                "   is not implemented for this class.");
@@ -92,7 +89,7 @@ void BilinearFormIntegrator::AddMultPA(const Vector &, Vector &) const
 
 void BilinearFormIntegrator::AddMultTransposePA(const Vector &, Vector &) const
 {
-   mfem_error ("BilinearFormIntegrator::AddMultTransposePA(...)\n"
+   mfem_error ("BilinearFormIntegrator::MultAssembledTranspose(...)\n"
                "   is not implemented for this class.");
 }
 
@@ -900,44 +897,6 @@ const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
    return IntRules.Get(trial_fe.GetGeomType(), order);
 }
 
-/// alpha (n x u, v)
-void VectorFEBoundaryTangentIntegrator::AssembleElementMatrix
-(const FiniteElement &el, ElementTransformation &Trans,
- DenseMatrix &elmat)
-{
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, Trans);
-
-   const int nd1 = el.GetDof();
-
-   DenseMatrix vshape(nd1, 3);
-   DenseMatrix vshapeRotated(nd1, 3);
-
-   elmat.SetSize(nd1);
-
-   elmat = 0.0;
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-      Trans.SetIntPoint (&ip);
-
-      Vector n(3);
-      CalcOrtho(Trans.Jacobian(), n);
-
-      el.CalcVShape(Trans, vshape);
-
-      for (int j=0; j<nd1; ++j)
-      {
-         // Set vshapeRotated(j) = n x vshape
-         vshapeRotated(j, 0) = (n[1] * vshape(j, 2)) - (n[2] * vshape(j, 1));
-         vshapeRotated(j, 1) = (n[2] * vshape(j, 0)) - (n[0] * vshape(j, 2));
-         vshapeRotated(j, 2) = (n[0] * vshape(j, 1)) - (n[1] * vshape(j, 0));
-      }
-
-      const double w = alpha * ip.weight; // Trans.Weight() is included in n
-
-      AddMult_a_ABt(w, vshape, vshapeRotated, elmat);
-   }
-}
 
 void BoundaryMassIntegrator::AssembleFaceMatrix(
    const FiniteElement &el1, const FiniteElement &el2,
@@ -1563,7 +1522,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
    double w;
 
 #ifdef MFEM_THREAD_SAFE
-   Vector D;
    DenseMatrix curlshape(nd,dimc), curlshape_dFt(nd,dimc), M;
 #else
    curlshape.SetSize(nd,dimc);
@@ -1571,7 +1529,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
 #endif
    elmat.SetSize(nd);
    if (MQ) { M.SetSize(dimc); }
-   if (DQ) { D.SetSize(dimc); }
 
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
@@ -1615,12 +1572,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
          Mult(curlshape_dFt, M, curlshape);
          AddMultABt(curlshape, curlshape_dFt, elmat);
       }
-      else if (DQ)
-      {
-         DQ->Eval(D, Trans, ip);
-         D *= w;
-         AddMultADAt(curlshape_dFt, D, elmat);
-      }
       else if (Q)
       {
          w *= Q->Eval(Trans, ip);

fem/bilininteg.hpp

@@ -17,8 +17,6 @@
 #include "fespace.hpp"
 #include "libceed/ceed.hpp"
 
-//#define SETUPONHOST
-
 namespace mfem
 {
 
@@ -88,10 +86,9 @@ public:
    virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
 
    /// Method defining element assembly.
-   /** The result of the element assembly is added to the @a emat Vector if
-       @a add is true. Otherwise, if @a add is false, we set @a emat. */
-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
-                           const bool add = true);
+   /** The result of the element assembly is added and stored in the @a emat
+       Vector. */
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);
    /** Used with BilinearFormIntegrators that have different spaces. */
    // virtual void AssembleEA(const FiniteElementSpace &trial_fes,
    //                         const FiniteElementSpace &test_fes,
@@ -99,12 +96,10 @@ public:
 
    virtual void AssembleEAInteriorFaces(const FiniteElementSpace &fes,
                                         Vector &ea_data_int,
-                                        Vector &ea_data_ext,
-                                        const bool add = true);
+                                        Vector &ea_data_ext);
 
    virtual void AssembleEABoundaryFaces(const FiniteElementSpace &fes,
-                                        Vector &ea_data_bdr,
-                                        const bool add = true);
+                                        Vector &ea_data_bdr);
 
    /// Given a particular Finite Element computes the element matrix elmat.
    virtual void AssembleElementMatrix(const FiniteElement &el,
@@ -209,9 +204,6 @@ public:
    { return 0.0; }
 
    virtual ~BilinearFormIntegrator() { }
-
-   bool isBdryInteg = false;
-   Array<int> *el_marker = NULL;
 };
 
 /** Wraps a given @a BilinearFormIntegrator and transposes the resulting element
@@ -270,17 +262,14 @@ public:
       bfi->AddMultTransposePA(x, y);
    }
 
-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
-                           const bool add);
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);
 
    virtual void AssembleEAInteriorFaces(const FiniteElementSpace &fes,
                                         Vector &ea_data_int,
-                                        Vector &ea_data_ext,
-                                        const bool add);
+                                        Vector &ea_data_ext);
 
    virtual void AssembleEABoundaryFaces(const FiniteElementSpace &fes,
-                                        Vector &ea_data_bdr,
-                                        const bool add);
+                                        Vector &ea_data_bdr);
 
    virtual ~TransposeIntegrator() { if (own_bfi) { delete bfi; } }
 };
@@ -1814,10 +1803,8 @@ protected:
 };
 
 /** Class for integrating the bilinear form a(u,v) := (Q grad u, v) where Q is a
-    scalar coefficient, and v is a vector with components v_i in the same (H1) space
-    as u.
-
-    See also MixedVectorGradientIntegrator when v is in H(curl). */
+    scalar coefficient, and v is a vector with components v_i in the same space
+    as u. */
 class GradientIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -1965,8 +1952,7 @@ public:
 
    virtual void AssemblePA(const FiniteElementSpace &fes);
 
-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
-                           const bool add);
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);
 
    virtual void AssembleDiagonalPA(Vector &diag);
 
@@ -2041,8 +2027,7 @@ public:
 
    virtual void AssemblePA(const FiniteElementSpace &fes);
 
-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
-                           const bool add);
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);
 
    virtual void AssembleDiagonalPA(Vector &diag);
 
@@ -2069,20 +2054,6 @@ public:
                                    DenseMatrix &elmat);
 };
 
-/// alpha (n x u, v)
-class VectorFEBoundaryTangentIntegrator : public MassIntegrator
-{
-private:
-   double alpha;
-
-public:
-   VectorFEBoundaryTangentIntegrator(double a = 1.0) : alpha(a)
-   { }
-
-   void AssembleElementMatrix(const FiniteElement &el,
-                              ElementTransformation &Trans, DenseMatrix &elmat);
-};
-
 /// alpha (q . grad u, v)
 class ConvectionIntegrator : public BilinearFormIntegrator
 {
@@ -2112,8 +2083,7 @@ public:
 
    virtual void AssemblePA(const FiniteElementSpace&);
 
-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
-                           const bool add);
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);
 
    virtual void AddMultPA(const Vector&, Vector&) const;
 
@@ -2330,14 +2300,12 @@ class CurlCurlIntegrator: public BilinearFormIntegrator
 private:
    Vector vec, pointflux;
 #ifndef MFEM_THREAD_SAFE
-   Vector D;
    DenseMatrix curlshape, curlshape_dFt, M;
    DenseMatrix vshape, projcurl;
 #endif
 
 protected:
    Coefficient *Q;
-   VectorCoefficient *DQ;
    MatrixCoefficient *MQ;
 
    // PA extension
@@ -2346,17 +2314,12 @@ protected:
    const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
    const GeometricFactors *geom;   ///< Not owned
    int dim, ne, nq, dofs1D, quad1D;
-   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
 
 public:
-   CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; }
+   CurlCurlIntegrator() { Q = NULL; MQ = NULL; }
    /// Construct a bilinear form integrator for Nedelec elements
-   CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), Q(&q) { DQ = NULL; MQ = NULL; }
-   CurlCurlIntegrator(VectorCoefficient &dq, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), DQ(&dq) { Q = NULL; MQ = NULL; }
-   CurlCurlIntegrator(MatrixCoefficient &mq, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), MQ(&mq) { Q = NULL; DQ = NULL; }
+   CurlCurlIntegrator(Coefficient &q) : Q(&q) { MQ = NULL; }
+   CurlCurlIntegrator(MatrixCoefficient &m) : MQ(&m) { Q = NULL; }
 
    /* Given a particular Finite Element, compute the
       element curl-curl matrix elmat */
@@ -2690,12 +2653,10 @@ public:
 
    virtual void AssembleEAInteriorFaces(const FiniteElementSpace& fes,
                                         Vector &ea_data_int,
-                                        Vector &ea_data_ext,
-                                        const bool add);
+                                        Vector &ea_data_ext);
 
    virtual void AssembleEABoundaryFaces(const FiniteElementSpace& fes,
-                                        Vector &ea_data_bdr,
-                                        const bool add);
+                                        Vector &ea_data_bdr);
 
    static const IntegrationRule &GetRule(Geometry::Type geom, int order,
                                          FaceElementTransformations &T);
@@ -2899,33 +2860,11 @@ class DiscreteInterpolator : public BilinearFormIntegrator { };
 class GradientInterpolator : public DiscreteInterpolator
 {
 public:
-   GradientInterpolator() : fake_fe(NULL) { }
-   virtual ~GradientInterpolator() { delete fake_fe; }
-
    virtual void AssembleElementMatrix2(const FiniteElement &h1_fe,
                                        const FiniteElement &nd_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat)
    { nd_fe.ProjectGrad(h1_fe, Trans, elmat); }
-
-   using BilinearFormIntegrator::AssemblePA;
-   /**
-      trial_fes should be H1 Lagrange
-      test_fes should be Nedelec
-   */
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
-
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
-   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
-
-private:
-   /// 1D finit element that generates and owns the 1D DofToQuad maps below
-   FiniteElement * fake_fe;
-
-   const DofToQuad *maps_C_C; // one-d map with Lobatto rows, Lobatto columns
-   const DofToQuad *maps_O_C; // one-d map with Legendre rows, Lobatto columns
-   int dim, ne, o_dofs1D, c_dofs1D;
 };
 
 
@@ -2940,24 +2879,6 @@ public:
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat)
    { ran_fe.Project(dom_fe, Trans, elmat); }
-
-   using BilinearFormIntegrator::AssemblePA;
-
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
-
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
-   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
-
-private:
-   /// 1D finit element that generates and owns the 1D DofToQuad maps below
-   FiniteElement * fake_fe;
-
-   const DofToQuad *maps_C_C; // one-d map with Lobatto rows, Lobatto columns
-   const DofToQuad *maps_O_C; // one-d map with Legendre rows, Lobatto columns
-   int dim, ne, o_dofs1D, c_dofs1D;
-
-   Vector pa_data;
 };
 
 

fem/bilininteg_convection_ea.cpp

@@ -22,7 +22,6 @@ static void EAConvectionAssemble1D(const int NE,
                                    const Array<double> &g,
                                    const Vector &padata,
                                    Vector &eadata,
-                                   const bool add,
                                    const int d1d = 0,
                                    const int q1d = 0)
 {
@@ -55,14 +54,7 @@ static void EAConvectionAssemble1D(const int NE,
             {
                val += r_Bj[k1] * D(k1, e) * r_Gi[k1];
             }
-            if (add)
-            {
-               A(i1, j1, e) += val;
-            }
-            else
-            {
-               A(i1, j1, e) = val;
-            }
+            A(i1, j1, e) += val;
          }
       }
    });
@@ -74,7 +66,6 @@ static void EAConvectionAssemble2D(const int NE,
                                    const Array<double> &g,
                                    const Vector &padata,
                                    Vector &eadata,
-                                   const bool add,
                                    const int d1d = 0,
                                    const int q1d = 0)
 {
@@ -130,14 +121,7 @@ static void EAConvectionAssemble2D(const int NE,
                                * r_B[k1][j1]* r_B[k2][j2];
                      }
                   }
-                  if (add)
-                  {
-                     A(i1, i2, j1, j2, e) += val;
-                  }
-                  else
-                  {
-                     A(i1, i2, j1, j2, e) = val;
-                  }
+                  A(i1, i2, j1, j2, e) += val;
                }
             }
          }
@@ -151,7 +135,6 @@ static void EAConvectionAssemble3D(const int NE,
                                    const Array<double> &g,
                                    const Vector &padata,
                                    Vector &eadata,
-                                   const bool add,
                                    const int d1d = 0,
                                    const int q1d = 0)
 {
@@ -208,14 +191,7 @@ static void EAConvectionAssemble3D(const int NE,
                               }
                            }
                         }
-                        if (add)
-                        {
-                           A(i1, i2, i3, j1, j2, j3, e) += val;
-                        }
-                        else
-                        {
-                           A(i1, i2, i3, j1, j2, j3, e) = val;
-                        }
+                        A(i1, i2, i3, j1, j2, j3, e) += val;
                      }
                   }
                }
@@ -226,8 +202,7 @@ static void EAConvectionAssemble3D(const int NE,
 }
 
 void ConvectionIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                      Vector &ea_data,
-                                      const bool add)
+                                      Vector &ea_data)
 {
    AssemblePA(fes);
    const int ne = fes.GetMesh()->GetNE();
@@ -237,47 +212,44 @@ void ConvectionIntegrator::AssembleEA(const FiniteElementSpace &fes,
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x22: return EAConvectionAssemble1D<2,2>(ne,B,G,pa_data,ea_data,add);
-         case 0x33: return EAConvectionAssemble1D<3,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x44: return EAConvectionAssemble1D<4,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x55: return EAConvectionAssemble1D<5,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x66: return EAConvectionAssemble1D<6,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x77: return EAConvectionAssemble1D<7,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x88: return EAConvectionAssemble1D<8,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x99: return EAConvectionAssemble1D<9,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EAConvectionAssemble1D(ne,B,G,pa_data,ea_data,add,
-                                                     dofs1D,quad1D);
+         case 0x22: return EAConvectionAssemble1D<2,2>(ne,B,G,pa_data,ea_data);
+         case 0x33: return EAConvectionAssemble1D<3,3>(ne,B,G,pa_data,ea_data);
+         case 0x44: return EAConvectionAssemble1D<4,4>(ne,B,G,pa_data,ea_data);
+         case 0x55: return EAConvectionAssemble1D<5,5>(ne,B,G,pa_data,ea_data);
+         case 0x66: return EAConvectionAssemble1D<6,6>(ne,B,G,pa_data,ea_data);
+         case 0x77: return EAConvectionAssemble1D<7,7>(ne,B,G,pa_data,ea_data);
+         case 0x88: return EAConvectionAssemble1D<8,8>(ne,B,G,pa_data,ea_data);
+         case 0x99: return EAConvectionAssemble1D<9,9>(ne,B,G,pa_data,ea_data);
+         default:   return EAConvectionAssemble1D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
       }
    }
    else if (dim == 2)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x22: return EAConvectionAssemble2D<2,2>(ne,B,G,pa_data,ea_data,add);
-         case 0x33: return EAConvectionAssemble2D<3,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x44: return EAConvectionAssemble2D<4,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x55: return EAConvectionAssemble2D<5,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x66: return EAConvectionAssemble2D<6,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x77: return EAConvectionAssemble2D<7,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x88: return EAConvectionAssemble2D<8,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x99: return EAConvectionAssemble2D<9,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EAConvectionAssemble2D(ne,B,G,pa_data,ea_data,add,
-                                                     dofs1D,quad1D);
+         case 0x22: return EAConvectionAssemble2D<2,2>(ne,B,G,pa_data,ea_data);
+         case 0x33: return EAConvectionAssemble2D<3,3>(ne,B,G,pa_data,ea_data);
+         case 0x44: return EAConvectionAssemble2D<4,4>(ne,B,G,pa_data,ea_data);
+         case 0x55: return EAConvectionAssemble2D<5,5>(ne,B,G,pa_data,ea_data);
+         case 0x66: return EAConvectionAssemble2D<6,6>(ne,B,G,pa_data,ea_data);
+         case 0x77: return EAConvectionAssemble2D<7,7>(ne,B,G,pa_data,ea_data);
+         case 0x88: return EAConvectionAssemble2D<8,8>(ne,B,G,pa_data,ea_data);
+         case 0x99: return EAConvectionAssemble2D<9,9>(ne,B,G,pa_data,ea_data);
+         default:   return EAConvectionAssemble2D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
       }
    }
    else if (dim == 3)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x23: return EAConvectionAssemble3D<2,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x34: return EAConvectionAssemble3D<3,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x45: return EAConvectionAssemble3D<4,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x56: return EAConvectionAssemble3D<5,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x67: return EAConvectionAssemble3D<6,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x78: return EAConvectionAssemble3D<7,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x89: return EAConvectionAssemble3D<8,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EAConvectionAssemble3D(ne,B,G,pa_data,ea_data,add,
-                                                     dofs1D,quad1D);
+         case 0x23: return EAConvectionAssemble3D<2,3>(ne,B,G,pa_data,ea_data);
+         case 0x34: return EAConvectionAssemble3D<3,4>(ne,B,G,pa_data,ea_data);
+         case 0x45: return EAConvectionAssemble3D<4,5>(ne,B,G,pa_data,ea_data);
+         case 0x56: return EAConvectionAssemble3D<5,6>(ne,B,G,pa_data,ea_data);
+         case 0x67: return EAConvectionAssemble3D<6,7>(ne,B,G,pa_data,ea_data);
+         case 0x78: return EAConvectionAssemble3D<7,8>(ne,B,G,pa_data,ea_data);
+         case 0x89: return EAConvectionAssemble3D<8,9>(ne,B,G,pa_data,ea_data);
+         default:   return EAConvectionAssemble3D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
       }
    }
    MFEM_ABORT("Unknown kernel.");

fem/bilininteg_convection_pa.cpp

@@ -806,16 +806,16 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    {
       vel.SetSize(dim * nq * ne);
       auto C = Reshape(vel.HostWrite(), dim, nq, ne);
-      DenseMatrix Q_ir;
+      Vector Vq(dim);
       for (int e = 0; e < ne; ++e)
       {
          ElementTransformation& T = *fes.GetElementTransformation(e);
-         Q->Eval(Q_ir, T, *ir);
          for (int q = 0; q < nq; ++q)
          {
+            Q->Eval(Vq, T, ir->IntPoint(q));
             for (int i = 0; i < dim; ++i)
             {
-               C(i,q,e) = Q_ir(i,q);
+               C(i,q,e) = Vq(i);
             }
          }
       }

fem/bilininteg_dgtrace_ea.cpp

@@ -20,8 +20,7 @@ static void EADGTraceAssemble1DInt(const int NF,
                                    const Array<double> &basis,
                                    const Vector &padata,
                                    Vector &eadata_int,
-                                   Vector &eadata_ext,
-                                   const bool add)
+                                   Vector &eadata_ext)
 {
    auto D = Reshape(padata.Read(), 2, 2, NF);
    auto A_int = Reshape(eadata_int.ReadWrite(), 2, NF);
@@ -33,41 +32,23 @@ static void EADGTraceAssemble1DInt(const int NF,
       val_ext10 = D(1, 0, f);
       val_ext01 = D(0, 1, f);
       val_int1  = D(1, 1, f);
-      if (add)
-      {
-         A_int(0, f) += val_int0;
-         A_int(1, f) += val_int1;
-         A_ext(0, f) += val_ext01;
-         A_ext(1, f) += val_ext10;
-      }
-      else
-      {
-         A_int(0, f) = val_int0;
-         A_int(1, f) = val_int1;
-         A_ext(0, f) = val_ext01;
-         A_ext(1, f) = val_ext10;
-      }
+      A_int(0, f) += val_int0;
+      A_int(1, f) += val_int1;
+      A_ext(0, f) += val_ext01;
+      A_ext(1, f) += val_ext10;
    });
 }
 
 static void EADGTraceAssemble1DBdr(const int NF,
                                    const Array<double> &basis,
                                    const Vector &padata,
-                                   Vector &eadata_bdr,
-                                   const bool add)
+                                   Vector &eadata_bdr)
 {
    auto D = Reshape(padata.Read(), 2, 2, NF);
    auto A_bdr = Reshape(eadata_bdr.ReadWrite(), NF);
    MFEM_FORALL(f, NF,
    {
-      if (add)
-      {
-         A_bdr(f) += D(0, 0, f);
-      }
-      else
-      {
-         A_bdr(f) = D(0, 0, f);
-      }
+      A_bdr(f) += D(0, 0, f);
    });
 }
 
@@ -77,7 +58,6 @@ static void EADGTraceAssemble2DInt(const int NF,
                                    const Vector &padata,
                                    Vector &eadata_int,
                                    Vector &eadata_ext,
-                                   const bool add,
                                    const int d1d = 0,
                                    const int q1d = 0)
 {
@@ -108,20 +88,10 @@ static void EADGTraceAssemble2DInt(const int NF,
                val_ext10 += B(k1,i1) * B(k1,j1) * D(k1, 1, 0, f);
                val_int1  += B(k1,i1) * B(k1,j1) * D(k1, 1, 1, f);
             }
-            if (add)
-            {
-               A_int(i1, j1, 0, f) += val_int0;
-               A_int(i1, j1, 1, f) += val_int1;
-               A_ext(i1, j1, 0, f) += val_ext01;
-               A_ext(i1, j1, 1, f) += val_ext10;
-            }
-            else
-            {
-               A_int(i1, j1, 0, f) = val_int0;
-               A_int(i1, j1, 1, f) = val_int1;
-               A_ext(i1, j1, 0, f) = val_ext01;
-               A_ext(i1, j1, 1, f) = val_ext10;
-            }
+            A_int(i1, j1, 0, f) += val_int0;
+            A_int(i1, j1, 1, f) += val_int1;
+            A_ext(i1, j1, 0, f) += val_ext01;
+            A_ext(i1, j1, 1, f) += val_ext10;
          }
       }
    });
@@ -132,7 +102,6 @@ static void EADGTraceAssemble2DBdr(const int NF,
                                    const Array<double> &basis,
                                    const Vector &padata,
                                    Vector &eadata_bdr,
-                                   const bool add,
                                    const int d1d = 0,
                                    const int q1d = 0)
 {
@@ -156,14 +125,7 @@ static void EADGTraceAssemble2DBdr(const int NF,
             {
                val_bdr  += B(k1,i1) * B(k1,j1) * D(k1, 0, 0, f);
             }
-            if (add)
-            {
-               A_bdr(i1, j1, f) += val_bdr;
-            }
-            else
-            {
-               A_bdr(i1, j1, f) = val_bdr;
-            }
+            A_bdr(i1, j1, f) += val_bdr;
          }
       }
    });
@@ -175,7 +137,6 @@ static void EADGTraceAssemble3DInt(const int NF,
                                    const Vector &padata,
                                    Vector &eadata_int,
                                    Vector &eadata_ext,
-                                   const bool add,
                                    const int d1d = 0,
                                    const int q1d = 0)
 {
@@ -246,20 +207,10 @@ static void EADGTraceAssemble3DInt(const int NF,
                                     * s_D[k1][k2][1][0];
                      }
                   }
-                  if (add)
-                  {
-                     A_int(i1, i2, j1, j2, 0, f) += val_int0;
-                     A_int(i1, i2, j1, j2, 1, f) += val_int1;
-                     A_ext(i1, i2, j1, j2, 0, f) += val_ext01;
-                     A_ext(i1, i2, j1, j2, 1, f) += val_ext10;
-                  }
-                  else
-                  {
-                     A_int(i1, i2, j1, j2, 0, f) = val_int0;
-                     A_int(i1, i2, j1, j2, 1, f) = val_int1;
-                     A_ext(i1, i2, j1, j2, 0, f) = val_ext01;
-                     A_ext(i1, i2, j1, j2, 1, f) = val_ext10;
-                  }
+                  A_int(i1, i2, j1, j2, 0, f) += val_int0;
+                  A_int(i1, i2, j1, j2, 1, f) += val_int1;
+                  A_ext(i1, i2, j1, j2, 0, f) += val_ext01;
+                  A_ext(i1, i2, j1, j2, 1, f) += val_ext10;
                }
             }
          }
@@ -272,7 +223,6 @@ static void EADGTraceAssemble3DBdr(const int NF,
                                    const Array<double> &basis,
                                    const Vector &padata,
                                    Vector &eadata_bdr,
-                                   const bool add,
                                    const int d1d = 0,
                                    const int q1d = 0)
 {
@@ -330,14 +280,7 @@ static void EADGTraceAssemble3DBdr(const int NF,
                                    * s_D[k1][k2][0][0];
                      }
                   }
-                  if (add)
-                  {
-                     A_bdr(i1, i2, j1, j2, f) += val_bdr;
-                  }
-                  else
-                  {
-                     A_bdr(i1, i2, j1, j2, f) = val_bdr;
-                  }
+                  A_bdr(i1, i2, j1, j2, f) += val_bdr;
                }
             }
          }
@@ -347,8 +290,7 @@ static void EADGTraceAssemble3DBdr(const int NF,
 
 void DGTraceIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
                                                 Vector &ea_data_int,
-                                                Vector &ea_data_ext,
-                                                const bool add)
+                                                Vector &ea_data_ext)
 {
    SetupPA(fes, FaceType::Interior);
    nf = fes.GetNFbyType(FaceType::Interior);
@@ -356,7 +298,7 @@ void DGTraceIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
    const Array<double> &B = maps->B;
    if (dim == 1)
    {
-      return EADGTraceAssemble1DInt(nf,B,pa_data,ea_data_int,ea_data_ext,add);
+      return EADGTraceAssemble1DInt(nf,B,pa_data,ea_data_int,ea_data_ext);
    }
    else if (dim == 2)
    {
@@ -364,31 +306,31 @@ void DGTraceIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
       {
          case 0x22:
             return EADGTraceAssemble2DInt<2,2>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
          case 0x33:
             return EADGTraceAssemble2DInt<3,3>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
          case 0x44:
             return EADGTraceAssemble2DInt<4,4>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
          case 0x55:
             return EADGTraceAssemble2DInt<5,5>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
          case 0x66:
             return EADGTraceAssemble2DInt<6,6>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
          case 0x77:
             return EADGTraceAssemble2DInt<7,7>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
          case 0x88:
             return EADGTraceAssemble2DInt<8,8>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
          case 0x99:
             return EADGTraceAssemble2DInt<9,9>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
          default:
             return EADGTraceAssemble2DInt(nf,B,pa_data,ea_data_int,
-                                          ea_data_ext,add,dofs1D,quad1D);
+                                          ea_data_ext,dofs1D,quad1D);
       }
    }
    else if (dim == 3)
@@ -397,36 +339,35 @@ void DGTraceIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
       {
          case 0x23:
             return EADGTraceAssemble3DInt<2,3>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
          case 0x34:
             return EADGTraceAssemble3DInt<3,4>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
          case 0x45:
             return EADGTraceAssemble3DInt<4,5>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
          case 0x56:
             return EADGTraceAssemble3DInt<5,6>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
          case 0x67:
             return EADGTraceAssemble3DInt<6,7>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
          case 0x78:
             return EADGTraceAssemble3DInt<7,8>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
          case 0x89:
             return EADGTraceAssemble3DInt<8,9>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
          default:
             return EADGTraceAssemble3DInt(nf,B,pa_data,ea_data_int,
-                                          ea_data_ext,add,dofs1D,quad1D);
+                                          ea_data_ext,dofs1D,quad1D);
       }
    }
    MFEM_ABORT("Unknown kernel.");
 }
 
 void DGTraceIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
-                                                Vector &ea_data_bdr,
-                                                const bool add)
+                                                Vector &ea_data_bdr)
 {
    SetupPA(fes, FaceType::Boundary);
    nf = fes.GetNFbyType(FaceType::Boundary);
@@ -434,37 +375,37 @@ void DGTraceIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
    const Array<double> &B = maps->B;
    if (dim == 1)
    {
-      return EADGTraceAssemble1DBdr(nf,B,pa_data,ea_data_bdr,add);
+      return EADGTraceAssemble1DBdr(nf,B,pa_data,ea_data_bdr);
    }
    else if (dim == 2)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x22: return EADGTraceAssemble2DBdr<2,2>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x33: return EADGTraceAssemble2DBdr<3,3>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x44: return EADGTraceAssemble2DBdr<4,4>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x55: return EADGTraceAssemble2DBdr<5,5>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x66: return EADGTraceAssemble2DBdr<6,6>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x77: return EADGTraceAssemble2DBdr<7,7>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x88: return EADGTraceAssemble2DBdr<8,8>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x99: return EADGTraceAssemble2DBdr<9,9>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x22: return EADGTraceAssemble2DBdr<2,2>(nf,B,pa_data,ea_data_bdr);
+         case 0x33: return EADGTraceAssemble2DBdr<3,3>(nf,B,pa_data,ea_data_bdr);
+         case 0x44: return EADGTraceAssemble2DBdr<4,4>(nf,B,pa_data,ea_data_bdr);
+         case 0x55: return EADGTraceAssemble2DBdr<5,5>(nf,B,pa_data,ea_data_bdr);
+         case 0x66: return EADGTraceAssemble2DBdr<6,6>(nf,B,pa_data,ea_data_bdr);
+         case 0x77: return EADGTraceAssemble2DBdr<7,7>(nf,B,pa_data,ea_data_bdr);
+         case 0x88: return EADGTraceAssemble2DBdr<8,8>(nf,B,pa_data,ea_data_bdr);
+         case 0x99: return EADGTraceAssemble2DBdr<9,9>(nf,B,pa_data,ea_data_bdr);
          default:
-            return EADGTraceAssemble2DBdr(nf,B,pa_data,ea_data_bdr,add,dofs1D,quad1D);
+            return EADGTraceAssemble2DBdr(nf,B,pa_data,ea_data_bdr,dofs1D,quad1D);
       }
    }
    else if (dim == 3)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x23: return EADGTraceAssemble3DBdr<2,3>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x34: return EADGTraceAssemble3DBdr<3,4>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x45: return EADGTraceAssemble3DBdr<4,5>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x56: return EADGTraceAssemble3DBdr<5,6>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x67: return EADGTraceAssemble3DBdr<6,7>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x78: return EADGTraceAssemble3DBdr<7,8>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x89: return EADGTraceAssemble3DBdr<8,9>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x23: return EADGTraceAssemble3DBdr<2,3>(nf,B,pa_data,ea_data_bdr);
+         case 0x34: return EADGTraceAssemble3DBdr<3,4>(nf,B,pa_data,ea_data_bdr);
+         case 0x45: return EADGTraceAssemble3DBdr<4,5>(nf,B,pa_data,ea_data_bdr);
+         case 0x56: return EADGTraceAssemble3DBdr<5,6>(nf,B,pa_data,ea_data_bdr);
+         case 0x67: return EADGTraceAssemble3DBdr<6,7>(nf,B,pa_data,ea_data_bdr);
+         case 0x78: return EADGTraceAssemble3DBdr<7,8>(nf,B,pa_data,ea_data_bdr);
+         case 0x89: return EADGTraceAssemble3DBdr<8,9>(nf,B,pa_data,ea_data_bdr);
          default:
-            return EADGTraceAssemble3DBdr(nf,B,pa_data,ea_data_bdr,add,dofs1D,quad1D);
+            return EADGTraceAssemble3DBdr(nf,B,pa_data,ea_data_bdr,dofs1D,quad1D);
       }
    }
    MFEM_ABORT("Unknown kernel.");

fem/bilininteg_dgtrace_pa.cpp

@@ -43,7 +43,7 @@ static void PADGTraceSetup2D(const int Q1D,
    auto W = w.Read();
    auto qd = Reshape(op.Write(), Q1D, 2, 2, NF);
 
-   MFEM_FORALL(f, NF, // can be optimized with Q1D thread for NF blocks
+   MFEM_FORALL(f, NF,//can be optimized with Q1D thread for NF blocks
    {
       for (int q = 0; q < Q1D; ++q)
       {
@@ -85,7 +85,7 @@ static void PADGTraceSetup3D(const int Q1D,
    auto W = w.Read();
    auto qd = Reshape(op.Write(), Q1D, Q1D, 2, 2, NF);
 
-   MFEM_FORALL(f, NF, // can be optimized with Q1D*Q1D threads for NF blocks
+   MFEM_FORALL(f, NF,//can be optimized with Q1D*Q1D threads for NF blocks
    {
       for (int q1 = 0; q1 < Q1D; ++q1)
       {
@@ -156,6 +156,57 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
    pa_data.SetSize(symmDims * nq * nf, Device::GetMemoryType());
+   Vector r;
+   if (rho==nullptr)
+   {
+      r.SetSize(1);
+      r(0) = 1.0;
+   }
+   else if (ConstantCoefficient *c_rho = dynamic_cast<ConstantCoefficient*>(rho))
+   {
+      r.SetSize(1);
+      r(0) = c_rho->constant;
+   }
+   else if (QuadratureFunctionCoefficient* c_rho =
+               dynamic_cast<QuadratureFunctionCoefficient*>(rho))
+   {
+      const QuadratureFunction &qFun = c_rho->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == nq * nf,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      r.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
+   else
+   {
+      r.SetSize(nq * nf);
+      auto C = Reshape(r.HostWrite(), nq, nf);
+      int f_ind = 0;
+      for (int f = 0; f < fes.GetNF(); ++f)
+      {
+         int e1, e2;
+         int inf1, inf2;
+         fes.GetMesh()->GetFaceElements(f, &e1, &e2);
+         fes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
+         int face_id = inf1 / 64;
+         if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
+             (type==FaceType::Boundary && e2<0 && inf2<0) )
+         {
+            ElementTransformation& T = *fes.GetMesh()->GetFaceTransformation(f);
+            for (int q = 0; q < nq; ++q)
+            {
+               // Convert to lexicographic ordering
+               int iq = ToLexOrdering(dim, face_id, quad1D, q);
+               C(iq,f_ind) = rho->Eval(T, ir->IntPoint(q));
+            }
+            f_ind++;
+         }
+      }
+      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
+   }
    Vector vel;
    if (VectorConstantCoefficient *c_u = dynamic_cast<VectorConstantCoefficient*>
                                         (u))
@@ -192,15 +243,12 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
          if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
              (type==FaceType::Boundary && e2<0 && inf2<0) )
          {
-            FaceElementTransformations &T =
-               *fes.GetMesh()->GetFaceElementTransformations(f);
+            ElementTransformation& T = *fes.GetMesh()->GetFaceTransformation(f);
             for (int q = 0; q < nq; ++q)
             {
                // Convert to lexicographic ordering
                int iq = ToLexOrdering(dim, face_id, quad1D, q);
-               T.SetAllIntPoints(&ir->IntPoint(q));
-               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
-               u->Eval(Vq, *T.Elem1, eip1);
+               u->Eval(Vq, T, ir->IntPoint(q));
                for (int i = 0; i < dim; ++i)
                {
                   C(i,iq,f_ind) = Vq(i);
@@ -211,80 +259,6 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
       }
       MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
    }
-   Vector r;
-   if (rho==nullptr)
-   {
-      r.SetSize(1);
-      r(0) = 1.0;
-   }
-   else if (ConstantCoefficient *c_rho = dynamic_cast<ConstantCoefficient*>(rho))
-   {
-      r.SetSize(1);
-      r(0) = c_rho->constant;
-   }
-   else if (QuadratureFunctionCoefficient* c_rho =
-               dynamic_cast<QuadratureFunctionCoefficient*>(rho))
-   {
-      const QuadratureFunction &qFun = c_rho->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == nq * nf,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      r.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      r.SetSize(nq * nf);
-      auto C_vel = Reshape(vel.HostRead(), dim, nq, nf);
-      auto n = Reshape(geom->normal.HostRead(), nq, dim, nf);
-      auto C = Reshape(r.HostWrite(), nq, nf);
-      int f_ind = 0;
-      for (int f = 0; f < fes.GetNF(); ++f)
-      {
-         int e1, e2;
-         int inf1, inf2;
-         fes.GetMesh()->GetFaceElements(f, &e1, &e2);
-         fes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
-         int face_id = inf1 / 64;
-         if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
-             (type==FaceType::Boundary && e2<0 && inf2<0) )
-         {
-            FaceElementTransformations &T =
-               *fes.GetMesh()->GetFaceElementTransformations(f);
-            for (int q = 0; q < nq; ++q)
-            {
-               // Convert to lexicographic ordering
-               int iq = ToLexOrdering(dim, face_id, quad1D, q);
-
-               T.SetAllIntPoints(&ir->IntPoint(q));
-               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
-               const IntegrationPoint &eip2 = T.GetElement2IntPoint();
-               double r;
-
-               if (inf2 < 0)
-               {
-                  r = rho->Eval(*T.Elem1, eip1);
-               }
-               else
-               {
-                  double udotn = 0.0;
-                  for (int d=0; d<dim; ++d)
-                  {
-                     udotn += C_vel(d,iq,f_ind)*n(iq,d,f_ind);
-                  }
-                  if (udotn >= 0.0) { r = rho->Eval(*T.Elem2, eip2); }
-                  else { r = rho->Eval(*T.Elem1, eip1); }
-               }
-               C(iq,f_ind) = r;
-            }
-            f_ind++;
-         }
-      }
-      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
-   }
    PADGTraceSetup(dim, dofs1D, quad1D, nf, ir->GetWeights(),
                   geom->detJ, geom->normal, r, vel,
                   alpha, beta, pa_data);

fem/bilininteg_diffusion_ea.cpp

@@ -22,7 +22,6 @@ static void EADiffusionAssemble1D(const int NE,
                                   const Array<double> &g,
                                   const Vector &padata,
                                   Vector &eadata,
-                                  const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -54,14 +53,7 @@ static void EADiffusionAssemble1D(const int NE,
             {
                val += r_Gj[k1] * D(k1, e) * r_Gi[k1];
             }
-            if (add)
-            {
-               A(i1, j1, e) += val;
-            }
-            else
-            {
-               A(i1, j1, e) = val;
-            }
+            A(i1, j1, e) += val;
          }
       }
    });
@@ -73,7 +65,6 @@ static void EADiffusionAssemble2D(const int NE,
                                   const Array<double> &g,
                                   const Vector &padata,
                                   Vector &eadata,
-                                  const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -129,14 +120,7 @@ static void EADiffusionAssemble2D(const int NE,
                                + gbi * D11 * gbj;
                      }
                   }
-                  if (add)
-                  {
-                     A(i1, i2, j1, j2, e) += val;
-                  }
-                  else
-                  {
-                     A(i1, i2, j1, j2, e) = val;
-                  }
+                  A(i1, i2, j1, j2, e) += val;
                }
             }
          }
@@ -150,7 +134,6 @@ static void EADiffusionAssemble3D(const int NE,
                                   const Array<double> &g,
                                   const Vector &padata,
                                   Vector &eadata,
-                                  const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -225,14 +208,7 @@ static void EADiffusionAssemble3D(const int NE,
                               }
                            }
                         }
-                        if (add)
-                        {
-                           A(i1, i2, i3, j1, j2, j3, e) += val;
-                        }
-                        else
-                        {
-                           A(i1, i2, i3, j1, j2, j3, e) = val;
-                        }
+                        A(i1, i2, i3, j1, j2, j3, e) += val;
                      }
                   }
                }
@@ -243,8 +219,7 @@ static void EADiffusionAssemble3D(const int NE,
 }
 
 void DiffusionIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                     Vector &ea_data,
-                                     const bool add)
+                                     Vector &ea_data)
 {
    AssemblePA(fes);
    const int ne = fes.GetMesh()->GetNE();
@@ -254,47 +229,44 @@ void DiffusionIntegrator::AssembleEA(const FiniteElementSpace &fes,
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x22: return EADiffusionAssemble1D<2,2>(ne,B,G,pa_data,ea_data,add);
-         case 0x33: return EADiffusionAssemble1D<3,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x44: return EADiffusionAssemble1D<4,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x55: return EADiffusionAssemble1D<5,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x66: return EADiffusionAssemble1D<6,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x77: return EADiffusionAssemble1D<7,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x88: return EADiffusionAssemble1D<8,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x99: return EADiffusionAssemble1D<9,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EADiffusionAssemble1D(ne,B,G,pa_data,ea_data,add,
-                                                    dofs1D,quad1D);
+         case 0x22: return EADiffusionAssemble1D<2,2>(ne,B,G,pa_data,ea_data);
+         case 0x33: return EADiffusionAssemble1D<3,3>(ne,B,G,pa_data,ea_data);
+         case 0x44: return EADiffusionAssemble1D<4,4>(ne,B,G,pa_data,ea_data);
+         case 0x55: return EADiffusionAssemble1D<5,5>(ne,B,G,pa_data,ea_data);
+         case 0x66: return EADiffusionAssemble1D<6,6>(ne,B,G,pa_data,ea_data);
+         case 0x77: return EADiffusionAssemble1D<7,7>(ne,B,G,pa_data,ea_data);
+         case 0x88: return EADiffusionAssemble1D<8,8>(ne,B,G,pa_data,ea_data);
+         case 0x99: return EADiffusionAssemble1D<9,9>(ne,B,G,pa_data,ea_data);
+         default:   return EADiffusionAssemble1D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
       }
    }
    else if (dim == 2)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x22: return EADiffusionAssemble2D<2,2>(ne,B,G,pa_data,ea_data,add);
-         case 0x33: return EADiffusionAssemble2D<3,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x44: return EADiffusionAssemble2D<4,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x55: return EADiffusionAssemble2D<5,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x66: return EADiffusionAssemble2D<6,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x77: return EADiffusionAssemble2D<7,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x88: return EADiffusionAssemble2D<8,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x99: return EADiffusionAssemble2D<9,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EADiffusionAssemble2D(ne,B,G,pa_data,ea_data,add,
-                                                    dofs1D,quad1D);
+         case 0x22: return EADiffusionAssemble2D<2,2>(ne,B,G,pa_data,ea_data);
+         case 0x33: return EADiffusionAssemble2D<3,3>(ne,B,G,pa_data,ea_data);
+         case 0x44: return EADiffusionAssemble2D<4,4>(ne,B,G,pa_data,ea_data);
+         case 0x55: return EADiffusionAssemble2D<5,5>(ne,B,G,pa_data,ea_data);
+         case 0x66: return EADiffusionAssemble2D<6,6>(ne,B,G,pa_data,ea_data);
+         case 0x77: return EADiffusionAssemble2D<7,7>(ne,B,G,pa_data,ea_data);
+         case 0x88: return EADiffusionAssemble2D<8,8>(ne,B,G,pa_data,ea_data);
+         case 0x99: return EADiffusionAssemble2D<9,9>(ne,B,G,pa_data,ea_data);
+         default:   return EADiffusionAssemble2D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
       }
    }
    else if (dim == 3)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x23: return EADiffusionAssemble3D<2,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x34: return EADiffusionAssemble3D<3,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x45: return EADiffusionAssemble3D<4,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x56: return EADiffusionAssemble3D<5,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x67: return EADiffusionAssemble3D<6,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x78: return EADiffusionAssemble3D<7,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x89: return EADiffusionAssemble3D<8,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EADiffusionAssemble3D(ne,B,G,pa_data,ea_data,add,
-                                                    dofs1D,quad1D);
+         case 0x23: return EADiffusionAssemble3D<2,3>(ne,B,G,pa_data,ea_data);
+         case 0x34: return EADiffusionAssemble3D<3,4>(ne,B,G,pa_data,ea_data);
+         case 0x45: return EADiffusionAssemble3D<4,5>(ne,B,G,pa_data,ea_data);
+         case 0x56: return EADiffusionAssemble3D<5,6>(ne,B,G,pa_data,ea_data);
+         case 0x67: return EADiffusionAssemble3D<6,7>(ne,B,G,pa_data,ea_data);
+         case 0x78: return EADiffusionAssemble3D<7,8>(ne,B,G,pa_data,ea_data);
+         case 0x89: return EADiffusionAssemble3D<8,9>(ne,B,G,pa_data,ea_data);
+         default:   return EADiffusionAssemble3D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
       }
    }
    MFEM_ABORT("Unknown kernel.");

fem/bilininteg_diffusion_pa.cpp

@@ -1680,7 +1680,7 @@ static void PADiffusionApply(const int dim,
       MFEM_ABORT("OCCA PADiffusionApply unknown kernel!");
    }
 #endif // MFEM_USE_OCCA
-   const int ID = (D1D << 4) | Q1D;
+   const int ID = (D1D << 4 ) | Q1D;
 
    if (dim == 2)
    {

fem/bilininteg_gradient.cpp

@@ -334,7 +334,7 @@ static void PAGradientApplyTranspose2D(const int NE,
                                        const int q1d = 0)
 {
    // TODO
-   MFEM_ASSERT(false, "PAGradientApplyTranspose2D not implemented.");
+   MFEM_ASSERT(false, "GradientPAApplyTranspose 3D not implemented.");
 }
 
 // PA Gradient Apply 3D kernel

fem/bilininteg_mass_ea.cpp

@@ -21,7 +21,6 @@ static void EAMassAssemble1D(const int NE,
                              const Array<double> &basis,
                              const Vector &padata,
                              Vector &eadata,
-                             const bool add,
                              const int d1d = 0,
                              const int q1d = 0)
 {
@@ -53,14 +52,7 @@ static void EAMassAssemble1D(const int NE,
             {
                val += r_Bi[k1] * r_Bj[k1] * D(k1, e);
             }
-            if (add)
-            {
-               M(i1, j1, e) += val;
-            }
-            else
-            {
-               M(i1, j1, e) = val;
-            }
+            M(i1, j1, e) += val;
          }
       }
    });
@@ -71,7 +63,6 @@ static void EAMassAssemble2D(const int NE,
                              const Array<double> &basis,
                              const Vector &padata,
                              Vector &eadata,
-                             const bool add,
                              const int d1d = 0,
                              const int q1d = 0)
 {
@@ -123,14 +114,7 @@ static void EAMassAssemble2D(const int NE,
                                * s_D[k1][k2];
                      }
                   }
-                  if (add)
-                  {
-                     M(i1, i2, j1, j2, e) += val;
-                  }
-                  else
-                  {
-                     M(i1, i2, j1, j2, e) = val;
-                  }
+                  M(i1, i2, j1, j2, e) += val;
                }
             }
          }
@@ -143,7 +127,6 @@ static void EAMassAssemble3D(const int NE,
                              const Array<double> &basis,
                              const Vector &padata,
                              Vector &eadata,
-                             const bool add,
                              const int d1d = 0,
                              const int q1d = 0)
 {
@@ -206,14 +189,7 @@ static void EAMassAssemble3D(const int NE,
                               }
                            }
                         }
-                        if (add)
-                        {
-                           M(i1, i2, i3, j1, j2, j3, e) += val;
-                        }
-                        else
-                        {
-                           M(i1, i2, i3, j1, j2, j3, e) = val;
-                        }
+                        M(i1, i2, i3, j1, j2, j3, e) += val;
                      }
                   }
                }
@@ -224,8 +200,7 @@ static void EAMassAssemble3D(const int NE,
 }
 
 void MassIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                Vector &ea_data,
-                                const bool add)
+                                Vector &ea_data)
 {
    AssemblePA(fes);
    const int ne = fes.GetMesh()->GetNE();
@@ -234,47 +209,44 @@ void MassIntegrator::AssembleEA(const FiniteElementSpace &fes,
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x22: return EAMassAssemble1D<2,2>(ne,B,pa_data,ea_data,add);
-         case 0x33: return EAMassAssemble1D<3,3>(ne,B,pa_data,ea_data,add);
-         case 0x44: return EAMassAssemble1D<4,4>(ne,B,pa_data,ea_data,add);
-         case 0x55: return EAMassAssemble1D<5,5>(ne,B,pa_data,ea_data,add);
-         case 0x66: return EAMassAssemble1D<6,6>(ne,B,pa_data,ea_data,add);
-         case 0x77: return EAMassAssemble1D<7,7>(ne,B,pa_data,ea_data,add);
-         case 0x88: return EAMassAssemble1D<8,8>(ne,B,pa_data,ea_data,add);
-         case 0x99: return EAMassAssemble1D<9,9>(ne,B,pa_data,ea_data,add);
-         default:   return EAMassAssemble1D(ne,B,pa_data,ea_data,add,
-                                               dofs1D,quad1D);
+         case 0x22: return EAMassAssemble1D<2,2>(ne,B,pa_data,ea_data);
+         case 0x33: return EAMassAssemble1D<3,3>(ne,B,pa_data,ea_data);
+         case 0x44: return EAMassAssemble1D<4,4>(ne,B,pa_data,ea_data);
+         case 0x55: return EAMassAssemble1D<5,5>(ne,B,pa_data,ea_data);
+         case 0x66: return EAMassAssemble1D<6,6>(ne,B,pa_data,ea_data);
+         case 0x77: return EAMassAssemble1D<7,7>(ne,B,pa_data,ea_data);
+         case 0x88: return EAMassAssemble1D<8,8>(ne,B,pa_data,ea_data);
+         case 0x99: return EAMassAssemble1D<9,9>(ne,B,pa_data,ea_data);
+         default:   return EAMassAssemble1D(ne,B,pa_data,ea_data,dofs1D,quad1D);
       }
    }
    else if (dim == 2)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x22: return EAMassAssemble2D<2,2>(ne,B,pa_data,ea_data,add);
-         case 0x33: return EAMassAssemble2D<3,3>(ne,B,pa_data,ea_data,add);
-         case 0x44: return EAMassAssemble2D<4,4>(ne,B,pa_data,ea_data,add);
-         case 0x55: return EAMassAssemble2D<5,5>(ne,B,pa_data,ea_data,add);
-         case 0x66: return EAMassAssemble2D<6,6>(ne,B,pa_data,ea_data,add);
-         case 0x77: return EAMassAssemble2D<7,7>(ne,B,pa_data,ea_data,add);
-         case 0x88: return EAMassAssemble2D<8,8>(ne,B,pa_data,ea_data,add);
-         case 0x99: return EAMassAssemble2D<9,9>(ne,B,pa_data,ea_data,add);
-         default:   return EAMassAssemble2D(ne,B,pa_data,ea_data,add,
-                                               dofs1D,quad1D);
+         case 0x22: return EAMassAssemble2D<2,2>(ne,B,pa_data,ea_data);
+         case 0x33: return EAMassAssemble2D<3,3>(ne,B,pa_data,ea_data);
+         case 0x44: return EAMassAssemble2D<4,4>(ne,B,pa_data,ea_data);
+         case 0x55: return EAMassAssemble2D<5,5>(ne,B,pa_data,ea_data);
+         case 0x66: return EAMassAssemble2D<6,6>(ne,B,pa_data,ea_data);
+         case 0x77: return EAMassAssemble2D<7,7>(ne,B,pa_data,ea_data);
+         case 0x88: return EAMassAssemble2D<8,8>(ne,B,pa_data,ea_data);
+         case 0x99: return EAMassAssemble2D<9,9>(ne,B,pa_data,ea_data);
+         default:   return EAMassAssemble2D(ne,B,pa_data,ea_data,dofs1D,quad1D);
       }
    }
    else if (dim == 3)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x23: return EAMassAssemble3D<2,3>(ne,B,pa_data,ea_data,add);
-         case 0x34: return EAMassAssemble3D<3,4>(ne,B,pa_data,ea_data,add);
-         case 0x45: return EAMassAssemble3D<4,5>(ne,B,pa_data,ea_data,add);
-         case 0x56: return EAMassAssemble3D<5,6>(ne,B,pa_data,ea_data,add);
-         case 0x67: return EAMassAssemble3D<6,7>(ne,B,pa_data,ea_data,add);
-         case 0x78: return EAMassAssemble3D<7,8>(ne,B,pa_data,ea_data,add);
-         case 0x89: return EAMassAssemble3D<8,9>(ne,B,pa_data,ea_data,add);
-         default:   return EAMassAssemble3D(ne,B,pa_data,ea_data,add,
-                                               dofs1D,quad1D);
+         case 0x23: return EAMassAssemble3D<2,3>(ne,B,pa_data,ea_data);
+         case 0x34: return EAMassAssemble3D<3,4>(ne,B,pa_data,ea_data);
+         case 0x45: return EAMassAssemble3D<4,5>(ne,B,pa_data,ea_data);
+         case 0x56: return EAMassAssemble3D<5,6>(ne,B,pa_data,ea_data);
+         case 0x67: return EAMassAssemble3D<6,7>(ne,B,pa_data,ea_data);
+         case 0x78: return EAMassAssemble3D<7,8>(ne,B,pa_data,ea_data);
+         case 0x89: return EAMassAssemble3D<8,9>(ne,B,pa_data,ea_data);
+         default:   return EAMassAssemble3D(ne,B,pa_data,ea_data,dofs1D,quad1D);
       }
    }
    MFEM_ABORT("Unknown kernel.");

fem/bilininteg_mass_pa.cpp

@@ -1,13 +1,13 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
 // This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
+// availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
 
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
@@ -50,7 +50,7 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
    maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
-   pa_data.SetSize(ne*nq, Device::GetMemoryType());
+   pa_data.SetSize(ne*nq, Device::GetDeviceMemoryType());
    Vector coeff;
    if (Q == nullptr)
    {
@@ -950,9 +950,8 @@ static void SmemPAMassApply3D(const int NE,
    auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
    {
-      const int tidz = MFEM_THREAD_ID(z);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
@@ -969,130 +968,188 @@ static void SmemPAMassApply3D(const int NE,
       double (*QQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm1;
       double (*QQD)[MQ1][MD1] = (double (*)[MQ1][MD1]) sm0;
       double (*QDD)[MD1][MD1] = (double (*)[MD1][MD1]) sm1;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD(dy,y,D1D)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD(dx,x,D1D)
          {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
             {
                X[dz][dy][dx] = x(dx,dy,dz,e);
             }
          }
-      }
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
+         MFEM_FOREACH_THREAD(dx,x,Q1D)
          {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
+            B[dx][dy] = b(dx,dy);
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD(dy,y,D1D)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
          {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            double u[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++)
             {
-               double u = 0.0;
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  u += X[dz][dy][dx] * B[qx][dx];
-               }
-               DDQ[dz][dy][qx] = u;
+               u[dz] = 0;
             }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            MFEM_UNROLL(MD1)
+            for (int dx = 0; dx < D1D; ++dx)
             {
-               double u = 0.0;
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  u += DDQ[dz][dy][qx] * B[qy][dy];
-               }
-               DQQ[dz][qy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
+               MFEM_UNROLL(MD1)
                for (int dz = 0; dz < D1D; ++dz)
                {
-                  u += DQQ[dz][qy][qx] * B[qz][dz];
+                  u[dz] += X[dz][dy][dx] * B[qx][dx];
                }
-               QQQ[qz][qy][qx] = u * d(qx,qy,qz,e);
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               DDQ[dz][dy][qx] = u[dz];
             }
          }
       }
       MFEM_SYNC_THREAD;
-      if (tidz == 0)
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
       {
-         MFEM_FOREACH_THREAD(d,y,D1D)
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
          {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
+            double u[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++)
             {
-               Bt[d][q] = b(q,d);
+               u[dz] = 0;
             }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            MFEM_UNROLL(MD1)
+            for (int dy = 0; dy < D1D; ++dy)
             {
-               double u = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < D1D; dz++)
                {
-                  u += QQQ[qz][qy][qx] * Bt[dx][qx];
+                  u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
                }
-               QQD[qz][qy][dx] = u;
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               DQQ[dz][qy][qx] = u[dz];
             }
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
          {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            double u[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; qz++)
             {
-               double u = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; qz++)
                {
-                  u += QQD[qz][qy][dx] * Bt[dy][qy];
+                  u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
                }
-               QDD[qz][dy][dx] = u;
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; qz++)
+            {
+               QQQ[qz][qy][qx] = u[qz] * d(qx,qy,qz,e);
             }
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD(d,y,D1D)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD(q,x,Q1D)
          {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            Bt[d][q] = b(q,d);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
             {
-               double u = 0.0;
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               MFEM_UNROLL(MQ1)
                for (int qz = 0; qz < Q1D; ++qz)
                {
-                  u += QDD[qz][dy][dx] * Bt[dz][qz];
+                  u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
                }
-               y(dx,dy,dz,e) += u;
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               QQD[qz][qy][dx] = u[qz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
+               }
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               QDD[qz][dy][dx] = u[qz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               u[dz] = 0;
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
+               }
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               y(dx,dy,dz,e) += u[dz];
             }
          }
       }
@@ -1123,14 +1180,20 @@ static void PAMassApply(const int dim,
       MFEM_ABORT("OCCA PA Mass Apply unknown kernel!");
    }
 #endif // MFEM_USE_OCCA
+   const int id = (D1D << 4) | Q1D;
    if (dim == 2)
    {
-      switch ((D1D << 4) | Q1D)
+      switch (id)
       {
          case 0x22: return SmemPAMassApply2D<2,2,16>(NE,B,Bt,D,X,Y);
+         case 0x24: return SmemPAMassApply2D<2,4,16>(NE,B,Bt,D,X,Y);
          case 0x33: return SmemPAMassApply2D<3,3,16>(NE,B,Bt,D,X,Y);
+         case 0x34: return SmemPAMassApply2D<3,4,16>(NE,B,Bt,D,X,Y);
+         case 0x36: return SmemPAMassApply2D<3,6,16>(NE,B,Bt,D,X,Y);
          case 0x44: return SmemPAMassApply2D<4,4,8>(NE,B,Bt,D,X,Y);
+         case 0x48: return SmemPAMassApply2D<4,8,4>(NE,B,Bt,D,X,Y);
          case 0x55: return SmemPAMassApply2D<5,5,8>(NE,B,Bt,D,X,Y);
+         case 0x58: return SmemPAMassApply2D<5,8,2>(NE,B,Bt,D,X,Y);
          case 0x66: return SmemPAMassApply2D<6,6,4>(NE,B,Bt,D,X,Y);
          case 0x77: return SmemPAMassApply2D<7,7,4>(NE,B,Bt,D,X,Y);
          case 0x88: return SmemPAMassApply2D<8,8,2>(NE,B,Bt,D,X,Y);
@@ -1140,18 +1203,25 @@ static void PAMassApply(const int dim,
    }
    else if (dim == 3)
    {
-      switch ((D1D << 4) | Q1D)
+      switch (id)
       {
          case 0x23: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
+         case 0x24: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
          case 0x34: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
+         case 0x36: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
          case 0x45: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
+         case 0x46: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
+         case 0x48: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
          case 0x56: return SmemPAMassApply3D<5,6>(NE,B,Bt,D,X,Y);
+         case 0x58: return SmemPAMassApply3D<5,8>(NE,B,Bt,D,X,Y);
          case 0x67: return SmemPAMassApply3D<6,7>(NE,B,Bt,D,X,Y);
          case 0x78: return SmemPAMassApply3D<7,8>(NE,B,Bt,D,X,Y);
          case 0x89: return SmemPAMassApply3D<8,9>(NE,B,Bt,D,X,Y);
+         case 0x9A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
          default:   return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
       }
    }
+   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
    MFEM_ABORT("Unknown kernel.");
 }
 

fem/bilininteg_transpose_ea.cpp

@@ -16,171 +16,88 @@ namespace mfem
 {
 
 void TransposeIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                     Vector &ea_data, const bool add)
+                                     Vector &ea_data)
 {
-   if (add)
+   Vector ea_data_tmp(ea_data.Size());
+   ea_data_tmp = 0.0;
+   bfi->AssembleEA(fes, ea_data_tmp);
+   const int ne = fes.GetNE();
+   if (ne == 0) { return; }
+   const int dofs = fes.GetFE(0)->GetDof();
+   auto A = Reshape(ea_data_tmp.Write(), dofs, dofs, ne);
+   auto AT = Reshape(ea_data.ReadWrite(), dofs, dofs, ne);
+   MFEM_FORALL(e, ne,
    {
-      Vector ea_data_tmp(ea_data.Size());
-      bfi->AssembleEA(fes, ea_data_tmp, false);
-      const int ne = fes.GetNE();
-      if (ne == 0) { return; }
-      const int dofs = fes.GetFE(0)->GetDof();
-      auto A = Reshape(ea_data_tmp.Read(), dofs, dofs, ne);
-      auto AT = Reshape(ea_data.ReadWrite(), dofs, dofs, ne);
-      MFEM_FORALL(e, ne,
+      for (int i = 0; i < dofs; i++)
       {
-         for (int i = 0; i < dofs; i++)
+         for (int j = 0; j < dofs; j++)
          {
-            for (int j = 0; j < dofs; j++)
-            {
-               const double a = A(i, j, e);
-               AT(j, i, e) += a;
-            }
+            const double a = A(i, j, e);
+            AT(j, i, e) += a;
          }
-      });
-   }
-   else
-   {
-      bfi->AssembleEA(fes, ea_data, false);
-      const int ne = fes.GetNE();
-      if (ne == 0) { return; }
-      const int dofs = fes.GetFE(0)->GetDof();
-      auto A = Reshape(ea_data.ReadWrite(), dofs, dofs, ne);
-      MFEM_FORALL(e, ne,
-      {
-         for (int i = 0; i < dofs; i++)
-         {
-            for (int j = i+1; j < dofs; j++)
-            {
-               const double aij = A(i, j, e);
-               const double aji = A(j, i, e);
-               A(j, i, e) = aij;
-               A(i, j, e) = aji;
-            }
-         }
-      });
-   }
+      }
+   });
 }
 
 void TransposeIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
                                                   Vector &ea_data_int,
-                                                  Vector &ea_data_ext,
-                                                  const bool add)
+                                                  Vector &ea_data_ext)
 {
    const int nf = fes.GetNFbyType(FaceType::Interior);
    if (nf == 0) { return; }
-   if (add)
+   Vector ea_data_int_tmp(ea_data_int.Size());
+   Vector ea_data_ext_tmp(ea_data_ext.Size());
+   ea_data_int_tmp = 0.0;
+   ea_data_ext_tmp = 0.0;
+   bfi->AssembleEAInteriorFaces(fes, ea_data_int_tmp, ea_data_ext_tmp);
+   const int faceDofs = fes.GetTraceElement(0,
+                                            fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
+   auto A_int = Reshape(ea_data_int_tmp.Read(), faceDofs, faceDofs, 2, nf);
+   auto A_ext = Reshape(ea_data_ext_tmp.Read(), faceDofs, faceDofs, 2, nf);
+   auto AT_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
+   auto AT_ext = Reshape(ea_data_ext.ReadWrite(), faceDofs, faceDofs, 2, nf);
+   MFEM_FORALL(f, nf,
    {
-      Vector ea_data_int_tmp(ea_data_int.Size());
-      Vector ea_data_ext_tmp(ea_data_ext.Size());
-      bfi->AssembleEAInteriorFaces(fes, ea_data_int_tmp, ea_data_ext_tmp, false);
-      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
-      auto A_int = Reshape(ea_data_int_tmp.Read(), faceDofs, faceDofs, 2, nf);
-      auto A_ext = Reshape(ea_data_ext_tmp.Read(), faceDofs, faceDofs, 2, nf);
-      auto AT_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
-      auto AT_ext = Reshape(ea_data_ext.ReadWrite(), faceDofs, faceDofs, 2, nf);
-      MFEM_FORALL(f, nf,
+      for (int i = 0; i < faceDofs; i++)
       {
-         for (int i = 0; i < faceDofs; i++)
+         for (int j = 0; j < faceDofs; j++)
          {
-            for (int j = 0; j < faceDofs; j++)
-            {
-               const double a_int0 = A_int(i, j, 0, f);
-               const double a_int1 = A_int(i, j, 1, f);
-               const double a_ext0 = A_ext(i, j, 0, f);
-               const double a_ext1 = A_ext(i, j, 1, f);
-               AT_int(j, i, 0, f) += a_int0;
-               AT_int(j, i, 1, f) += a_int1;
-               AT_ext(j, i, 0, f) += a_ext1;
-               AT_ext(j, i, 1, f) += a_ext0;
-            }
+            const double a_int0 = A_int(i, j, 0, f);
+            const double a_int1 = A_int(i, j, 1, f);
+            const double a_ext0 = A_ext(i, j, 0, f);
+            const double a_ext1 = A_ext(i, j, 1, f);
+            AT_int(j, i, 0, f) += a_int0;
+            AT_int(j, i, 1, f) += a_int1;
+            AT_ext(j, i, 0, f) += a_ext1;
+            AT_ext(j, i, 1, f) += a_ext0;
          }
-      });
-   }
-   else
-   {
-      bfi->AssembleEAInteriorFaces(fes, ea_data_int, ea_data_ext, false);
-      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
-      auto A_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
-      auto A_ext = Reshape(ea_data_ext.ReadWrite(), faceDofs, faceDofs, 2, nf);
-      MFEM_FORALL(f, nf,
-      {
-         for (int i = 0; i < faceDofs; i++)
-         {
-            for (int j = i+1; j < faceDofs; j++)
-            {
-               const double aij_int0 = A_int(i, j, 0, f);
-               const double aij_int1 = A_int(i, j, 1, f);
-               const double aji_int0 = A_int(j, i, 0, f);
-               const double aji_int1 = A_int(j, i, 1, f);
-               A_int(j, i, 0, f) = aij_int0;
-               A_int(j, i, 1, f) = aij_int1;
-               A_int(i, j, 0, f) = aji_int0;
-               A_int(i, j, 1, f) = aji_int1;
-            }
-         }
-         for (int i = 0; i < faceDofs; i++)
-         {
-            for (int j = 0; j < faceDofs; j++)
-            {
-               const double aij_ext0 = A_ext(i, j, 0, f);
-               const double aji_ext1 = A_ext(j, i, 1, f);
-               A_ext(j, i, 1, f) = aij_ext0;
-               A_ext(i, j, 0, f) = aji_ext1;
-            }
-         }
-      });
-   }
+      }
+   });
 }
 
 void TransposeIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
-                                                  Vector &ea_data_bdr,
-                                                  const bool add)
+                                                  Vector &ea_data_bdr)
 {
    const int nf = fes.GetNFbyType(FaceType::Boundary);
    if (nf == 0) { return; }
-   if (add)
+   Vector ea_data_bdr_tmp(ea_data_bdr.Size());
+   ea_data_bdr_tmp = 0.0;
+   bfi->AssembleEABoundaryFaces(fes, ea_data_bdr_tmp);
+   const int faceDofs = fes.GetTraceElement(0,
+                                            fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
+   auto A_bdr = Reshape(ea_data_bdr_tmp.Read(), faceDofs, faceDofs, nf);
+   auto AT_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
+   MFEM_FORALL(f, nf,
    {
-      Vector ea_data_bdr_tmp(ea_data_bdr.Size());
-      bfi->AssembleEABoundaryFaces(fes, ea_data_bdr_tmp, false);
-      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
-      auto A_bdr = Reshape(ea_data_bdr_tmp.Read(), faceDofs, faceDofs, nf);
-      auto AT_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
-      MFEM_FORALL(f, nf,
+      for (int i = 0; i < faceDofs; i++)
       {
-         for (int i = 0; i < faceDofs; i++)
+         for (int j = 0; j < faceDofs; j++)
          {
-            for (int j = 0; j < faceDofs; j++)
-            {
-               const double a_bdr = A_bdr(i, j, f);
-               AT_bdr(j, i, f) += a_bdr;
-            }
+            const double a_bdr = A_bdr(i, j, f);
+            AT_bdr(j, i, f) += a_bdr;
          }
-      });
-   }
-   else
-   {
-      bfi->AssembleEABoundaryFaces(fes, ea_data_bdr, false);
-      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
-      auto A_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
-      MFEM_FORALL(f, nf,
-      {
-         for (int i = 0; i < faceDofs; i++)
-         {
-            for (int j = i+1; j < faceDofs; j++)
-            {
-               const double aij_bdr = A_bdr(i, j, f);
-               const double aji_bdr = A_bdr(j, i, f);
-               A_bdr(j, i, f) = aij_bdr;
-               A_bdr(i, j, f) = aji_bdr;
-            }
-         }
-      });
-   }
+      }
+   });
 }
 
 }

fem/bilininteg_vectorfe.cpp

@@ -20,96 +20,58 @@ void PAHcurlSetup2D(const int Q1D,
                     const int NE,
                     const Array<double> &w,
                     const Vector &j,
-                    Vector &coeff,
+                    Vector &_coeff,
                     Vector &op);
 
-void PAHcurlSetup2Din3D(const int Q1D,
-                        const int NE,
-                        const Array<double> &w,
-                        const Vector &j,
-                        Vector &_coeff,
-                        Vector &op);
-
 void PAHcurlSetup3D(const int Q1D,
                     const int coeffDim,
                     const int NE,
                     const Array<double> &w,
                     const Vector &j,
-                    Vector &coeff,
+                    Vector &_coeff,
                     Vector &op);
 
 void PAHcurlMassAssembleDiagonal2D(const int D1D,
                                    const int Q1D,
                                    const int NE,
                                    const bool symmetric,
-                                   const Array<double> &bo,
-                                   const Array<double> &bc,
-                                   const Vector &pa_data,
-                                   Vector &diag);
+                                   const Array<double> &_Bo,
+                                   const Array<double> &_Bc,
+                                   const Vector &_op,
+                                   Vector &_diag);
 
 void PAHcurlMassAssembleDiagonal3D(const int D1D,
                                    const int Q1D,
                                    const int NE,
                                    const bool symmetric,
-                                   const Array<double> &bo,
-                                   const Array<double> &bc,
-                                   const Vector &pa_data,
-                                   Vector &diag);
-
-void PAHcurlMassAssembleDiagonal3DHost(const int D1D,
-                                       const int Q1D,
-                                       const int NE,
-                                       const Array<double> &_Bo,
-                                       const Array<double> &_Bc,
-                                       const Vector &_op,
-                                       Vector &_diag);
-
-template<int T_D1D = 0, int T_Q1D = 0>
-void SmemPAHcurlMassAssembleDiagonal3D(const int D1D,
-                                       const int Q1D,
-                                       const int NE,
-                                       const bool symmetric,
-                                       const Array<double> &bo,
-                                       const Array<double> &bc,
-                                       const Vector &pa_data,
-                                       Vector &diag);
+                                   const Array<double> &_Bo,
+                                   const Array<double> &_Bc,
+                                   const Vector &_op,
+                                   Vector &_diag);
 
 void PAHcurlMassApply2D(const int D1D,
                         const int Q1D,
                         const int NE,
                         const bool symmetric,
-                        const Array<double> &bo,
-                        const Array<double> &bc,
-                        const Array<double> &bot,
-                        const Array<double> &bct,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y);
+                        const Array<double> &_Bo,
+                        const Array<double> &_Bc,
+                        const Array<double> &_Bot,
+                        const Array<double> &_Bct,
+                        const Vector &_op,
+                        const Vector &_x,
+                        Vector &_y);
 
 void PAHcurlMassApply3D(const int D1D,
                         const int Q1D,
                         const int NE,
                         const bool symmetric,
-                        const Array<double> &bo,
-                        const Array<double> &bc,
-                        const Array<double> &bot,
-                        const Array<double> &bct,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y);
-
-template<int T_D1D = 0, int T_Q1D = 0>
-void SmemPAHcurlMassApply3D(const int D1D,
-                            const int Q1D,
-                            const int NE,
-                            const bool symmetric,
-                            const Array<double> &bo,
-                            const Array<double> &bc,
-                            const Array<double> &bot,
-                            const Array<double> &bct,
-                            const Vector &pa_data,
-                            const Vector &x,
-                            Vector &y);
+                        const Array<double> &_Bo,
+                        const Array<double> &_Bc,
+                        const Array<double> &_Bot,
+                        const Array<double> &_Bct,
+                        const Vector &_op,
+                        const Vector &_x,
+                        Vector &_y);
 
 void PAHdivSetup2D(const int Q1D,
                    const int NE,
@@ -128,24 +90,24 @@ void PAHdivSetup3D(const int Q1D,
 void PAHcurlH1Apply2D(const int D1D,
                       const int Q1D,
                       const int NE,
-                      const Array<double> &bc,
-                      const Array<double> &gc,
-                      const Array<double> &bot,
-                      const Array<double> &bct,
-                      const Vector &pa_data,
-                      const Vector &x,
-                      Vector &y);
+                      const Array<double> &_Bc,
+                      const Array<double> &_Gc,
+                      const Array<double> &_Bot,
+                      const Array<double> &_Bct,
+                      const Vector &_op,
+                      const Vector &_x,
+                      Vector &_y);
 
 void PAHcurlH1Apply3D(const int D1D,
                       const int Q1D,
                       const int NE,
-                      const Array<double> &bc,
-                      const Array<double> &gc,
-                      const Array<double> &bot,
-                      const Array<double> &bct,
-                      const Vector &pa_data,
-                      const Vector &x,
-                      Vector &y);
+                      const Array<double> &_Bc,
+                      const Array<double> &_Gc,
+                      const Array<double> &_Bot,
+                      const Array<double> &_Bct,
+                      const Vector &_op,
+                      const Vector &_x,
+                      Vector &_y);
 
 void PAHdivMassAssembleDiagonal2D(const int D1D,
                                   const int Q1D,
@@ -737,14 +699,8 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 {
    // Assumes tensor-product elements
    Mesh *mesh = trial_fes.GetMesh();
-   dim = mesh->Dimension();
-   const bool bdryInteg = isBdryInteg;
-
-   // TODO: this implementation applies boundary markers just by setting coeff to zero on boundary elements. It would be better to skip those elements in AddMultPA.
-
-   const FiniteElement *trial_fel = bdryInteg ? trial_fes.GetBE(
-                                       0) : trial_fes.GetFE(0);
 
+   const FiniteElement *trial_fel = trial_fes.GetFE(0);
    const VectorTensorFiniteElement *trial_el =
       dynamic_cast<const VectorTensorFiniteElement*>(trial_fel);
    MFEM_VERIFY(trial_el != NULL, "Only VectorTensorFiniteElement is supported!");
@@ -754,38 +710,21 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
       dynamic_cast<const VectorTensorFiniteElement*>(test_fel);
    MFEM_VERIFY(test_el != NULL, "Only VectorTensorFiniteElement is supported!");
 
-   if (bdryInteg)
-   {
-      MFEM_VERIFY(trial_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3 &&
-                  mesh->SpaceDimension() == 3 && trial_el->GetDim() == 2, "");
-   }
-
    const IntegrationRule *ir
       = IntRule ? IntRule : &MassIntegrator::GetRule(*trial_el, *trial_el,
-                                                     bdryInteg ? *mesh->GetBdrElementTransformation(0) :
                                                      *mesh->GetElementTransformation(0));
    const int dims = trial_el->GetDim();
-
    MFEM_VERIFY(dims == 2 || dims == 3, "");
 
    const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
    const int nq = ir->GetNPoints();
+   dim = mesh->Dimension();
    MFEM_VERIFY(dim == 2 || dim == 3, "");
 
-   ne = bdryInteg ? trial_fes.GetNBE() : trial_fes.GetNE();
-   Vector bdryJac;
-   if (bdryInteg)
-   {
-      bdryJac.SetSize(ne*nq*6);
-   }
-   else
-   {
-      geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
-   }
-
-   MFEM_VERIFY(ne == (bdryInteg ? test_fes.GetNBE() : test_fes.GetNE()),
+   ne = trial_fes.GetNE();
+   MFEM_VERIFY(ne == test_fes.GetNE(),
                "Different meshes for test and trial spaces");
-
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
    mapsC = &trial_el->GetDofToQuad(*ir, DofToQuad::TENSOR);
    mapsO = &trial_el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
    dofs1D = mapsC->ndof;
@@ -819,12 +758,7 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
       pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne,
                       Device::GetMemoryType());
 
-   if (bdryInteg)
-   {
-      MFEM_VERIFY(coeffDim == 1 && dims == 2, "");
-   }
-
-   Vector coeff(coeffDim * nq * ne);
+   Vector coeff(coeffDim * ne * nq);
    coeff = 1.0;
    auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
    if (Q || VQ || MQ)
@@ -856,8 +790,7 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 
       for (int e=0; e<ne; ++e)
       {
-         ElementTransformation *tr = bdryInteg ? mesh->GetBdrElementTransformation(
-                                        e) : mesh->GetElementTransformation(e);
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
          for (int p=0; p<nq; ++p)
          {
             if (MQ)
@@ -898,72 +831,10 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
       }
    }
 
-   if (el_marker)
+   if (trial_curl && test_curl && dim == 3)
    {
-      MFEM_VERIFY(bdryInteg, "");
-      MFEM_VERIFY(el_marker->Size() == mesh->bdr_attributes.Max(), "");
-
-      for (int e=0; e<ne; ++e)
-      {
-         const int el_attr = (bdryInteg) ? mesh->GetBdrAttribute(e) : -1;
-         MFEM_VERIFY(el_attr > 0, "");
-
-         if ((*el_marker)[el_attr-1] == 0)
-         {
-            for (int p=0; p<nq; ++p)
-            {
-               for (int i=0; i<coeffDim; ++i)
-               {
-                  coeffh(i, p, e) = 0.0;
-               }
-            }
-         }
-      }
-   }
-
-   if (isBdryInteg || (trial_el->GetDerivType() == mfem::FiniteElement::CURL &&
-                       dim == 2 && mesh->SpaceDimension() == 3))
-   {
-      MFEM_VERIFY(coeffDim == 1,
-                  "");  // Vector coefficient not implemented in this case
-
-      if (isBdryInteg)
-      {
-         for (int e=0; e<ne; ++e)
-         {
-            ElementTransformation *tr = mesh->GetBdrElementTransformation(e);
-            for (int q = 0; q < nq; ++q)
-            {
-               tr->SetIntPoint(&(ir->IntPoint(q)));
-               const DenseMatrix& jac = tr->Jacobian();
-               for (int i=0; i<3; ++i)
-                  for (int j=0; j<2; ++j)
-                  {
-                     bdryJac[(6*nq*e) + (3*nq*j) + (i*nq) + q] = jac(i,j);
-                  }
-            }
-         }
-      }
-      else
-      {
-         // J is allocated as 3x3, but entries (i,j) are set only for 0 <= i < 3, 0 <= j < 2.
-         // That is, on each element J is 3x2 with 6 entries, not 9, so the last third of geom->J is unused.
-         MFEM_VERIFY(geom->J.Size() == 6 * ne * nq, "");
-      }
-
-      PAHcurlSetup2Din3D(quad1D, ne, ir->GetWeights(),
-                         isBdryInteg ? bdryJac : geom->J,
-                         coeff, pa_data);
-   }
-   else if (trial_curl && test_curl && dim == 3)
-   {
-#ifdef SETUPONHOST
-      PAHcurlSetup3DHost(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
-                         coeff, pa_data);
-#else
       PAHcurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
                      coeff, pa_data);
-#endif
    }
    else if (trial_curl && test_curl && dim == 2)
    {
@@ -1010,30 +881,8 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
    {
       if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
       {
-         if (Device::Allows(Backend::DEVICE_MASK))
-         {
-            const int ID = (dofs1D << 4) | quad1D;
-            switch (ID)
-            {
-               case 0x23: return SmemPAHcurlMassAssembleDiagonal3D<2,3>(dofs1D, quad1D, ne,
-                                                                           symmetric,
-                                                                           mapsO->B, mapsC->B, pa_data, diag);
-               case 0x34: return SmemPAHcurlMassAssembleDiagonal3D<3,4>(dofs1D, quad1D, ne,
-                                                                           symmetric,
-                                                                           mapsO->B, mapsC->B, pa_data, diag);
-               case 0x45: return SmemPAHcurlMassAssembleDiagonal3D<4,5>(dofs1D, quad1D, ne,
-                                                                           symmetric,
-                                                                           mapsO->B, mapsC->B, pa_data, diag);
-               case 0x56: return SmemPAHcurlMassAssembleDiagonal3D<5,6>(dofs1D, quad1D, ne,
-                                                                           symmetric,
-                                                                           mapsO->B, mapsC->B, pa_data, diag);
-               default: return SmemPAHcurlMassAssembleDiagonal3D(dofs1D, quad1D, ne, symmetric,
-                                                                    mapsO->B, mapsC->B, pa_data, diag);
-            }
-         }
-         else
-            PAHcurlMassAssembleDiagonal3D(dofs1D, quad1D, ne, symmetric,
-                                          mapsO->B, mapsC->B, pa_data, diag);
+         PAHcurlMassAssembleDiagonal3D(dofs1D, quad1D, ne, symmetric,
+                                       mapsO->B, mapsC->B, pa_data, diag);
       }
       else if (trial_fetype == mfem::FiniteElement::DIV &&
                test_fetype == trial_fetype)
@@ -1073,39 +922,12 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
    const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
    const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
 
-   if (dim == 3 && !isBdryInteg)
+   if (dim == 3)
    {
       if (trial_curl && test_curl)
       {
-         if (Device::Allows(Backend::DEVICE_MASK))
-         {
-            const int ID = (dofs1D << 4) | quad1D;
-            switch (ID)
-            {
-               case 0x23: return SmemPAHcurlMassApply3D<2,3>(dofs1D, quad1D, ne, symmetric,
-                                                                mapsO->B,
-                                                                mapsC->B, mapsO->Bt,
-                                                                mapsC->Bt, pa_data, x, y);
-               case 0x34: return SmemPAHcurlMassApply3D<3,4>(dofs1D, quad1D, ne, symmetric,
-                                                                mapsO->B,
-                                                                mapsC->B, mapsO->Bt,
-                                                                mapsC->Bt, pa_data, x, y);
-               case 0x45: return SmemPAHcurlMassApply3D<4,5>(dofs1D, quad1D, ne, symmetric,
-                                                                mapsO->B,
-                                                                mapsC->B, mapsO->Bt,
-                                                                mapsC->Bt, pa_data, x, y);
-               case 0x56: return SmemPAHcurlMassApply3D<5,6>(dofs1D, quad1D, ne, symmetric,
-                                                                mapsO->B,
-                                                                mapsC->B, mapsO->Bt,
-                                                                mapsC->Bt, pa_data, x, y);
-               default: return SmemPAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B,
-                                                         mapsC->B,
-                                                         mapsO->Bt, mapsC->Bt, pa_data, x, y);
-            }
-         }
-         else
-            PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B, mapsO->Bt,
-                               mapsC->Bt, pa_data, x, y);
+         PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                            mapsO->Bt, mapsC->Bt, pa_data, x, y);
       }
       else if (trial_div && test_div)
       {

fem/complex_fem.cpp

@@ -10,7 +10,6 @@
 // CONTRIBUTING.md for details.
 
 #include "complex_fem.hpp"
-#include "../general/forall.hpp"
 
 using namespace std;
 
@@ -20,21 +19,16 @@ namespace mfem
 ComplexGridFunction::ComplexGridFunction(FiniteElementSpace *fes)
    : Vector(2*(fes->GetVSize()))
 {
-   UseDevice(true);
-   this->Vector::operator=(0.0);
-
-   gfr = new GridFunction();
-   gfr->MakeRef(fes, *this, 0);
-
-   gfi = new GridFunction();
-   gfi->MakeRef(fes, *this, fes->GetVSize());
+   gfr = new GridFunction(fes, data);
+   gfi = new GridFunction(fes, &data[fes->GetVSize()]);
 }
 
 void
 ComplexGridFunction::Update()
 {
-   FiniteElementSpace *fes = gfr->FESpace();
-   const int vsize = fes->GetVSize();
+   FiniteElementSpace * fes = gfr->FESpace();
+
+   int vsize = fes->GetVSize();
 
    const Operator *T = fes->GetUpdateOperator();
    if (T)
@@ -46,36 +40,30 @@ ComplexGridFunction::Update()
 
       // Our data array now contains old data as well as being the wrong size so
       // reallocate it.
-      UseDevice(true);
       this->SetSize(2 * vsize);
-      this->Vector::operator=(0.0);
 
       // Create temporary vectors which point to the new data array
-      Vector gf_r; gf_r.MakeRef(*this, 0, vsize);
-      Vector gf_i; gf_i.MakeRef(*this, vsize, vsize);
+      Vector gf_r(data, vsize);
+      Vector gf_i((data) ? &data[vsize] : data, vsize);
 
       // Copy the updated GridFunctions into the new data array
       gf_r = *gfr;
       gf_i = *gfi;
-      gf_r.SyncAliasMemory(*this);
-      gf_i.SyncAliasMemory(*this);
 
       // Replace the individual data arrays with pointers into the new data
       // array
-      gfr->MakeRef(*this, 0, vsize);
-      gfi->MakeRef(*this, vsize, vsize);
+      gfr->NewDataAndSize(data, vsize);
+      gfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
    }
    else
    {
       // The existing data will not be transferred to the new GridFunctions so
-      // delete it and allocate a new array
-      UseDevice(true);
+      // delete it a allocate a new array
       this->SetSize(2 * vsize);
-      this->Vector::operator=(0.0);
 
       // Point the individual GridFunctions to the new data array
-      gfr->MakeRef(*this, 0, vsize);
-      gfi->MakeRef(*this, vsize, vsize);
+      gfr->NewDataAndSize(data, vsize);
+      gfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
 
       // These updates will only set the proper 'sequence' value within the
       // individual GridFunction objects because their sizes are already correct
@@ -88,24 +76,16 @@ void
 ComplexGridFunction::ProjectCoefficient(Coefficient &real_coeff,
                                         Coefficient &imag_coeff)
 {
-   gfr->SyncMemory(*this);
-   gfi->SyncMemory(*this);
    gfr->ProjectCoefficient(real_coeff);
    gfi->ProjectCoefficient(imag_coeff);
-   gfr->SyncAliasMemory(*this);
-   gfi->SyncAliasMemory(*this);
 }
 
 void
 ComplexGridFunction::ProjectCoefficient(VectorCoefficient &real_vcoeff,
                                         VectorCoefficient &imag_vcoeff)
 {
-   gfr->SyncMemory(*this);
-   gfi->SyncMemory(*this);
    gfr->ProjectCoefficient(real_vcoeff);
    gfi->ProjectCoefficient(imag_vcoeff);
-   gfr->SyncAliasMemory(*this);
-   gfi->SyncAliasMemory(*this);
 }
 
 void
@@ -113,12 +93,8 @@ ComplexGridFunction::ProjectBdrCoefficient(Coefficient &real_coeff,
                                            Coefficient &imag_coeff,
                                            Array<int> &attr)
 {
-   gfr->SyncMemory(*this);
-   gfi->SyncMemory(*this);
    gfr->ProjectBdrCoefficient(real_coeff, attr);
    gfi->ProjectBdrCoefficient(imag_coeff, attr);
-   gfr->SyncAliasMemory(*this);
-   gfi->SyncAliasMemory(*this);
 }
 
 void
@@ -126,12 +102,8 @@ ComplexGridFunction::ProjectBdrCoefficientNormal(VectorCoefficient &real_vcoeff,
                                                  VectorCoefficient &imag_vcoeff,
                                                  Array<int> &attr)
 {
-   gfr->SyncMemory(*this);
-   gfi->SyncMemory(*this);
    gfr->ProjectBdrCoefficientNormal(real_vcoeff, attr);
    gfi->ProjectBdrCoefficientNormal(imag_vcoeff, attr);
-   gfr->SyncAliasMemory(*this);
-   gfi->SyncAliasMemory(*this);
 }
 
 void
@@ -141,28 +113,18 @@ ComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient
                                                   &imag_vcoeff,
                                                   Array<int> &attr)
 {
-   gfr->SyncMemory(*this);
-   gfi->SyncMemory(*this);
    gfr->ProjectBdrCoefficientTangent(real_vcoeff, attr);
    gfi->ProjectBdrCoefficientTangent(imag_vcoeff, attr);
-   gfr->SyncAliasMemory(*this);
-   gfi->SyncAliasMemory(*this);
 }
 
 
-ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *fes,
+ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *f,
                                      ComplexOperator::Convention convention)
-   : Vector(2*(fes->GetVSize())),
+   : Vector(2*(f->GetVSize())),
      conv(convention)
 {
-   UseDevice(true);
-   this->Vector::operator=(0.0);
-
-   lfr = new LinearForm();
-   lfr->MakeRef(fes, *this, 0);
-
-   lfi = new LinearForm();
-   lfi->MakeRef(fes, *this, fes->GetVSize());
+   lfr = new LinearForm(f, data);
+   lfi = new LinearForm(f, &data[f->GetVSize()]);
 }
 
 ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *fes,
@@ -171,14 +133,8 @@ ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *fes,
    : Vector(2*(fes->GetVSize())),
      conv(convention)
 {
-   UseDevice(true);
-   this->Vector::operator=(0.0);
-
-   lfr = new LinearForm(fes, lf_r);
-   lfi = new LinearForm(fes, lf_i);
-
-   lfr->MakeRef(fes, *this, 0);
-   lfi->MakeRef(fes, *this, fes->GetVSize());
+   lfr = new LinearForm(fes, lf_r);  lfr->SetData(data);
+   lfi = new LinearForm(fes, lf_i);  lfi->SetData(&data[fes->GetVSize()]);
 }
 
 ComplexLinearForm::~ComplexLinearForm()
@@ -233,43 +189,42 @@ void
 ComplexLinearForm::Update()
 {
    FiniteElementSpace *fes = lfr->FESpace();
+
    this->Update(fes);
 }
 
 void
 ComplexLinearForm::Update(FiniteElementSpace *fes)
 {
-   UseDevice(true);
-   SetSize(2 * fes->GetVSize());
-   this->Vector::operator=(0.0);
+   int vsize = fes->GetVSize();
+   SetSize(2 * vsize);
 
-   lfr->MakeRef(fes, *this, 0);
-   lfi->MakeRef(fes, *this, fes->GetVSize());
+   Vector vlfr(data, vsize);
+   Vector vlfi((data) ? &data[vsize] : data, vsize);
+
+   lfr->Update(fes, vlfr, 0);
+   lfi->Update(fes, vlfi, 0);
 }
 
 void
 ComplexLinearForm::Assemble()
 {
-   lfr->SyncMemory(*this);
-   lfi->SyncMemory(*this);
    lfr->Assemble();
    lfi->Assemble();
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC) { *lfi *= -1.0; }
-   lfr->SyncAliasMemory(*this);
-   lfi->SyncAliasMemory(*this);
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
+   {
+      *lfi *= -1.0;
+   }
 }
 
 complex<double>
 ComplexLinearForm::operator()(const ComplexGridFunction &gf) const
 {
-   double s = (conv == ComplexOperator::HERMITIAN) ? 1.0 : -1.0;
-   lfr->SyncMemory(*this);
-   lfi->SyncMemory(*this);
+   double s = (conv == ComplexOperator::HERMITIAN)?1.0:-1.0;
    return complex<double>((*lfr)(gf.real()) - s * (*lfi)(gf.imag()),
                           (*lfr)(gf.imag()) + s * (*lfi)(gf.real()));
 }
 
-
 bool SesquilinearForm::RealInteg()
 {
    int nint = blfr->GetFBFI()->Size() + blfr->GetDBFI()->Size() +
@@ -386,45 +341,34 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
                                    Vector &X, Vector &B,
                                    int ci)
 {
-   FiniteElementSpace *fes = blfr->FESpace();
-   const int vsize = fes->GetVSize();
+   FiniteElementSpace * fes = blfr->FESpace();
+   int vsize  = fes->GetVSize();
 
-   // Allocate temporary vector
-   Vector b_0;
-   b_0.UseDevice(true);
-   b_0.SetSize(vsize);
-   b_0 = 0.0;
+   // Allocate temporary vectors
+   Vector b_0(vsize); b_0 = 0.0;
 
    // Extract the real and imaginary parts of the input vectors
    MFEM_ASSERT(x.Size() == 2 * vsize, "Input GridFunction of incorrect size!");
-   x.Read();
-   Vector x_r; x_r.MakeRef(x, 0, vsize);
-   Vector x_i; x_i.MakeRef(x, vsize, vsize);
+   Vector x_r(x.GetData(), vsize);
+   Vector x_i(&(x.GetData())[vsize], vsize);
 
    MFEM_ASSERT(b.Size() == 2 * vsize, "Input LinearForm of incorrect size!");
-   b.Read();
-   Vector b_r; b_r.MakeRef(b, 0, vsize);
-   Vector b_i; b_i.MakeRef(b, vsize, vsize);
+   Vector b_r(b.GetData(), vsize);
+   Vector b_i(&(b.GetData())[vsize], vsize);
 
    if (conv == ComplexOperator::BLOCK_SYMMETRIC) { b_i *= -1.0; }
 
-   const int tvsize = fes->GetTrueVSize();
+   int tvsize = fes->GetTrueVSize();
    OperatorHandle A_r, A_i;
 
-   X.UseDevice(true);
    X.SetSize(2 * tvsize);
-   X = 0.0;
-
-   B.UseDevice(true);
    B.SetSize(2 * tvsize);
-   B = 0.0;
 
-   Vector X_r; X_r.MakeRef(X, 0, tvsize);
-   Vector X_i; X_i.MakeRef(X, tvsize, tvsize);
-   Vector B_r; B_r.MakeRef(B, 0, tvsize);
-   Vector B_i; B_i.MakeRef(B, tvsize, tvsize);
-
-   Vector X_0, B_0;
+   Vector X_0(tvsize), B_0(tvsize);
+   Vector X_r(X.GetData(),tvsize);
+   Vector X_i(&(X.GetData())[tvsize], tvsize);
+   Vector B_r(B.GetData(), tvsize);
+   Vector B_i(&(B.GetData())[tvsize], tvsize);
 
    if (RealInteg())
    {
@@ -474,18 +418,13 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
       // conform with standard essential BC treatment
       if (A_i.Is<ConstrainedOperator>())
       {
-         const int n = ess_tdof_list.Size();
-         auto d_B_r = B_r.Write();
-         auto d_B_i = B_i.Write();
-         auto d_X_r = X_r.Read();
-         auto d_X_i = X_i.Read();
-         auto d_idx = ess_tdof_list.Read();
-         MFEM_FORALL(i, n,
+         int n = ess_tdof_list.Size();
+         for (int k = 0; k < n; k++)
          {
-            const int j = d_idx[i];
-            d_B_r[j] = d_X_r[j];
-            d_B_i[j] = d_X_i[j];
-         });
+            int j = ess_tdof_list[k];
+            B_r(j) = X_r(j);
+            B_i(j) = X_i(j);
+         }
          A_i.As<ConstrainedOperator>()->SetDiagonalPolicy
          (mfem::Operator::DiagonalPolicy::DIAG_ZERO);
       }
@@ -497,16 +436,6 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
       b_i *= -1.0;
    }
 
-   x_r.SyncAliasMemory(x);
-   x_i.SyncAliasMemory(x);
-   b_r.SyncAliasMemory(b);
-   b_i.SyncAliasMemory(b);
-
-   X_r.SyncAliasMemory(X);
-   X_i.SyncAliasMemory(X);
-   B_r.SyncAliasMemory(B);
-   B_i.SyncAliasMemory(B);
-
    // A = A_r + i A_i
    A.Clear();
    if ( A_r.Type() == Operator::MFEM_SPARSEMAT ||
@@ -599,32 +528,29 @@ void
 SesquilinearForm::RecoverFEMSolution(const Vector &X, const Vector &b,
                                      Vector &x)
 {
-   FiniteElementSpace *fes = blfr->FESpace();
+   FiniteElementSpace * fes = blfr->FESpace();
 
    const SparseMatrix *P = fes->GetConformingProlongation();
+
+   int vsize  = fes->GetVSize();
+   int tvsize = X.Size() / 2;
+
+   Vector X_r(X.GetData(), tvsize);
+   Vector X_i(&(X.GetData())[tvsize], tvsize);
+
+   Vector x_r(x.GetData(), vsize);
+   Vector x_i(&(x.GetData())[vsize], vsize);
+
    if (!P)
    {
       x = X;
-      return;
    }
-
-   const int vsize  = fes->GetVSize();
-   const int tvsize = X.Size() / 2;
-
-   X.Read();
-   Vector X_r; X_r.MakeRef(const_cast<Vector&>(X), 0, tvsize);
-   Vector X_i; X_i.MakeRef(const_cast<Vector&>(X), tvsize, tvsize);
-
-   x.Write();
-   Vector x_r; x_r.MakeRef(x, 0, vsize);
-   Vector x_i; x_i.MakeRef(x, vsize, vsize);
-
-   // Apply conforming prolongation
-   P->Mult(X_r, x_r);
-   P->Mult(X_i, x_i);
-
-   x_r.SyncAliasMemory(x);
-   x_i.SyncAliasMemory(x);
+   else
+   {
+      // Apply conforming prolongation
+      P->Mult(X_r, x_r);
+      P->Mult(X_i, x_i);
+   }
 }
 
 void
@@ -640,21 +566,16 @@ SesquilinearForm::Update(FiniteElementSpace *nfes)
 ParComplexGridFunction::ParComplexGridFunction(ParFiniteElementSpace *pfes)
    : Vector(2*(pfes->GetVSize()))
 {
-   UseDevice(true);
-   this->Vector::operator=(0.0);
-
-   pgfr = new ParGridFunction();
-   pgfr->MakeRef(pfes, *this, 0);
-
-   pgfi = new ParGridFunction();
-   pgfi->MakeRef(pfes, *this, pfes->GetVSize());
+   pgfr = new ParGridFunction(pfes, data);
+   pgfi = new ParGridFunction(pfes, (data) ? &data[pfes->GetVSize()]:data);
 }
 
 void
 ParComplexGridFunction::Update()
 {
-   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
-   const int vsize = pfes->GetVSize();
+   ParFiniteElementSpace * pfes = pgfr->ParFESpace();
+
+   int vsize = pfes->GetVSize();
 
    const Operator *T = pfes->GetUpdateOperator();
    if (T)
@@ -666,34 +587,30 @@ ParComplexGridFunction::Update()
 
       // Our data array now contains old data as well as being the wrong size so
       // reallocate it.
-      UseDevice(true);
       this->SetSize(2 * vsize);
-      this->Vector::operator=(0.0);
 
       // Create temporary vectors which point to the new data array
-      Vector gf_r; gf_r.MakeRef(*this, 0, vsize);
-      Vector gf_i; gf_i.MakeRef(*this, vsize, vsize);
+      Vector gf_r(data, vsize);
+      Vector gf_i((data) ? &data[vsize] : data, vsize);
 
       // Copy the updated GridFunctions into the new data array
-      gf_r = *pgfr; gf_r.SyncAliasMemory(*this);
-      gf_i = *pgfi; gf_i.SyncAliasMemory(*this);
+      gf_r = *pgfr;
+      gf_i = *pgfi;
 
       // Replace the individual data arrays with pointers into the new data
       // array
-      pgfr->MakeRef(*this, 0, vsize);
-      pgfi->MakeRef(*this, vsize, vsize);
+      pgfr->NewDataAndSize(data, vsize);
+      pgfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
    }
    else
    {
       // The existing data will not be transferred to the new GridFunctions so
-      // delete it and allocate a new array
-      UseDevice(true);
+      // delete it a allocate a new array
       this->SetSize(2 * vsize);
-      this->Vector::operator=(0.0);
 
       // Point the individual GridFunctions to the new data array
-      pgfr->MakeRef(*this, 0, vsize);
-      pgfi->MakeRef(*this, vsize, vsize);
+      pgfr->NewDataAndSize(data, vsize);
+      pgfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
 
       // These updates will only set the proper 'sequence' value within the
       // individual GridFunction objects because their sizes are already correct
@@ -706,24 +623,16 @@ void
 ParComplexGridFunction::ProjectCoefficient(Coefficient &real_coeff,
                                            Coefficient &imag_coeff)
 {
-   pgfr->SyncMemory(*this);
-   pgfi->SyncMemory(*this);
    pgfr->ProjectCoefficient(real_coeff);
    pgfi->ProjectCoefficient(imag_coeff);
-   pgfr->SyncAliasMemory(*this);
-   pgfi->SyncAliasMemory(*this);
 }
 
 void
 ParComplexGridFunction::ProjectCoefficient(VectorCoefficient &real_vcoeff,
                                            VectorCoefficient &imag_vcoeff)
 {
-   pgfr->SyncMemory(*this);
-   pgfi->SyncMemory(*this);
    pgfr->ProjectCoefficient(real_vcoeff);
    pgfi->ProjectCoefficient(imag_vcoeff);
-   pgfr->SyncAliasMemory(*this);
-   pgfi->SyncAliasMemory(*this);
 }
 
 void
@@ -731,12 +640,8 @@ ParComplexGridFunction::ProjectBdrCoefficient(Coefficient &real_coeff,
                                               Coefficient &imag_coeff,
                                               Array<int> &attr)
 {
-   pgfr->SyncMemory(*this);
-   pgfi->SyncMemory(*this);
    pgfr->ProjectBdrCoefficient(real_coeff, attr);
    pgfi->ProjectBdrCoefficient(imag_coeff, attr);
-   pgfr->SyncAliasMemory(*this);
-   pgfi->SyncAliasMemory(*this);
 }
 
 void
@@ -746,12 +651,8 @@ ParComplexGridFunction::ProjectBdrCoefficientNormal(VectorCoefficient
                                                     &imag_vcoeff,
                                                     Array<int> &attr)
 {
-   pgfr->SyncMemory(*this);
-   pgfi->SyncMemory(*this);
    pgfr->ProjectBdrCoefficientNormal(real_vcoeff, attr);
    pgfi->ProjectBdrCoefficientNormal(imag_vcoeff, attr);
-   pgfr->SyncAliasMemory(*this);
-   pgfi->SyncAliasMemory(*this);
 }
 
 void
@@ -761,51 +662,36 @@ ParComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient
                                                      &imag_vcoeff,
                                                      Array<int> &attr)
 {
-   pgfr->SyncMemory(*this);
-   pgfi->SyncMemory(*this);
    pgfr->ProjectBdrCoefficientTangent(real_vcoeff, attr);
    pgfi->ProjectBdrCoefficientTangent(imag_vcoeff, attr);
-   pgfr->SyncAliasMemory(*this);
-   pgfi->SyncAliasMemory(*this);
 }
 
 void
 ParComplexGridFunction::Distribute(const Vector *tv)
 {
-   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
-   const int tvsize = pfes->GetTrueVSize();
+   ParFiniteElementSpace * pfes = pgfr->ParFESpace();
+   HYPRE_Int size = pfes->GetTrueVSize();
 
-   tv->Read();
-   Vector tvr; tvr.MakeRef(const_cast<Vector&>(*tv), 0, tvsize);
-   Vector tvi; tvi.MakeRef(const_cast<Vector&>(*tv), tvsize, tvsize);
+   double * tvd = tv->GetData();
+   Vector tvr(tvd, size);
+   Vector tvi((tvd) ? &tvd[size] : tvd, size);
 
-   pgfr->SyncMemory(*this);
-   pgfi->SyncMemory(*this);
    pgfr->Distribute(tvr);
    pgfi->Distribute(tvi);
-   pgfr->SyncAliasMemory(*this);
-   pgfi->SyncAliasMemory(*this);
 }
 
 void
 ParComplexGridFunction::ParallelProject(Vector &tv) const
 {
-   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
-   const int tvsize = pfes->GetTrueVSize();
+   ParFiniteElementSpace * pfes = pgfr->ParFESpace();
+   HYPRE_Int size = pfes->GetTrueVSize();
 
-   tv.Write();
-   Vector tvr; tvr.MakeRef(tv, 0, tvsize);
-   Vector tvi; tvi.MakeRef(tv, tvsize, tvsize);
+   double * tvd = tv.GetData();
+   Vector tvr(tvd, size);
+   Vector tvi((tvd) ? &tvd[size] : tvd, size);
 
-   pgfr->SyncMemory(*this);
-   pgfi->SyncMemory(*this);
    pgfr->ParallelProject(tvr);
    pgfi->ParallelProject(tvi);
-   pgfr->SyncAliasMemory(*this);
-   pgfi->SyncAliasMemory(*this);
-
-   tvr.SyncAliasMemory(tv);
-   tvi.SyncAliasMemory(tv);
 }
 
 
@@ -815,16 +701,10 @@ ParComplexLinearForm::ParComplexLinearForm(ParFiniteElementSpace *pfes,
    : Vector(2*(pfes->GetVSize())),
      conv(convention)
 {
-   UseDevice(true);
-   this->Vector::operator=(0.0);
+   plfr = new ParLinearForm(pfes, data);
+   plfi = new ParLinearForm(pfes, (data) ? &data[pfes->GetVSize()]:data);
 
-   plfr = new ParLinearForm();
-   plfr->MakeRef(pfes, *this, 0);
-
-   plfi = new ParLinearForm();
-   plfi->MakeRef(pfes, *this, pfes->GetVSize());
-
-   HYPRE_Int *tdof_offsets_fes = pfes->GetTrueDofOffsets();
+   HYPRE_Int * tdof_offsets_fes = pfes->GetTrueDofOffsets();
 
    int n = (HYPRE_AssumedPartitionCheck()) ? 2 : pfes->GetNRanks();
    tdof_offsets = new HYPRE_Int[n+1];
@@ -844,16 +724,12 @@ ParComplexLinearForm::ParComplexLinearForm(ParFiniteElementSpace *pfes,
    : Vector(2*(pfes->GetVSize())),
      conv(convention)
 {
-   UseDevice(true);
-   this->Vector::operator=(0.0);
-
    plfr = new ParLinearForm(pfes, plf_r);
+   plfr->SetData(data);
    plfi = new ParLinearForm(pfes, plf_i);
+   plfi->SetData((data) ? &data[pfes->GetVSize()]:data);
 
-   plfr->MakeRef(pfes, *this, 0);
-   plfi->MakeRef(pfes, *this, pfes->GetVSize());
-
-   HYPRE_Int *tdof_offsets_fes = pfes->GetTrueDofOffsets();
+   HYPRE_Int * tdof_offsets_fes = pfes->GetTrueDofOffsets();
 
    int n = (HYPRE_AssumedPartitionCheck()) ? 2 : pfes->GetNRanks();
    tdof_offsets = new HYPRE_Int[n+1];
@@ -916,71 +792,58 @@ ParComplexLinearForm::AddBdrFaceIntegrator(LinearFormIntegrator *lfi_real,
 void
 ParComplexLinearForm::Update(ParFiniteElementSpace *pf)
 {
-   ParFiniteElementSpace *pfes = (pf != NULL) ? pf : plfr->ParFESpace();
+   ParFiniteElementSpace *pfes = (pf!=NULL)?pf:plfr->ParFESpace();
+   int vsize = pfes->GetVSize();
+   SetSize(2 * vsize);
 
-   UseDevice(true);
-   SetSize(2 * pfes->GetVSize());
-   this->Vector::operator=(0.0);
+   Vector vplfr(data, vsize);
+   Vector vplfi((data) ? &data[vsize] : data, vsize);
 
-   plfr->MakeRef(pfes, *this, 0);
-   plfi->MakeRef(pfes, *this, pfes->GetVSize());
+   plfr->Update(pfes, vplfr, 0);
+   plfi->Update(pfes, vplfi, 0);
 }
 
 void
 ParComplexLinearForm::Assemble()
 {
-   plfr->SyncMemory(*this);
-   plfi->SyncMemory(*this);
    plfr->Assemble();
    plfi->Assemble();
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC) { *plfi *= -1.0; }
-   plfr->SyncAliasMemory(*this);
-   plfi->SyncAliasMemory(*this);
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
+   {
+      *plfi *= -1.0;
+   }
 }
 
 void
 ParComplexLinearForm::ParallelAssemble(Vector &tv)
 {
-   const int tvsize = plfr->ParFESpace()->GetTrueVSize();
+   HYPRE_Int size = plfr->ParFESpace()->GetTrueVSize();
 
-   tv.Write();
-   Vector tvr; tvr.MakeRef(tv, 0, tvsize);
-   Vector tvi; tvi.MakeRef(tv, tvsize, tvsize);
+   double * tvd = tv.GetData();
+   Vector tvr(tvd, size);
+   Vector tvi((tvd) ? &tvd[size] : tvd, size);
 
-   plfr->SyncMemory(*this);
-   plfi->SyncMemory(*this);
    plfr->ParallelAssemble(tvr);
    plfi->ParallelAssemble(tvi);
-   plfr->SyncAliasMemory(*this);
-   plfi->SyncAliasMemory(*this);
-
-   tvr.SyncAliasMemory(tv);
-   tvi.SyncAliasMemory(tv);
 }
 
 HypreParVector *
 ParComplexLinearForm::ParallelAssemble()
 {
-   const ParFiniteElementSpace *pfes = plfr->ParFESpace();
-   const int tvsize = pfes->GetTrueVSize();
+   const ParFiniteElementSpace * pfes = plfr->ParFESpace();
 
-   HypreParVector *tv = new HypreParVector(pfes->GetComm(),
-                                           2*(pfes->GlobalTrueVSize()),
-                                           tdof_offsets);
+   HypreParVector * tv = new HypreParVector(pfes->GetComm(),
+                                            2*(pfes->GlobalTrueVSize()),
+                                            tdof_offsets);
 
-   ((Vector*) tv)->Write(true);
-   Vector tvr; tvr.MakeRef(*tv, 0, tvsize);
-   Vector tvi; tvi.MakeRef(*tv, tvsize, tvsize);
+   HYPRE_Int size = pfes->GetTrueVSize();
+
+   double * tvd = tv->GetData();
+   Vector tvr(tvd, size);
+   Vector tvi((tvd) ? &tvd[size] : tvd, size);
 
-   plfr->SyncMemory(*this);
-   plfi->SyncMemory(*this);
    plfr->ParallelAssemble(tvr);
    plfi->ParallelAssemble(tvi);
-   plfr->SyncAliasMemory(*this);
-   plfi->SyncAliasMemory(*this);
-
-   tvr.SyncAliasMemory(*tv);
-   tvi.SyncAliasMemory(*tv);
 
    return tv;
 }
@@ -988,14 +851,13 @@ ParComplexLinearForm::ParallelAssemble()
 complex<double>
 ParComplexLinearForm::operator()(const ParComplexGridFunction &gf) const
 {
-   plfr->SyncMemory(*this);
-   plfi->SyncMemory(*this);
-   double s = (conv == ComplexOperator::HERMITIAN) ? 1.0 : -1.0;
+   double s = (conv == ComplexOperator::HERMITIAN)?1.0:-1.0;
    return complex<double>((*plfr)(gf.real()) - s * (*plfi)(gf.imag()),
                           (*plfr)(gf.imag()) + s * (*plfi)(gf.real()));
 }
 
 
+
 bool ParSesquilinearForm::RealInteg()
 {
    int nint = pblfr->GetFBFI()->Size() + pblfr->GetDBFI()->Size() +
@@ -1102,6 +964,7 @@ ParSesquilinearForm::ParallelAssemble()
    return new ComplexHypreParMatrix(pblfr->ParallelAssemble(),
                                     pblfi->ParallelAssemble(),
                                     true, true, conv);
+
 }
 
 void
@@ -1111,45 +974,35 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
                                       Vector &X, Vector &B,
                                       int ci)
 {
-   ParFiniteElementSpace *pfes = pblfr->ParFESpace();
-   const int vsize = pfes->GetVSize();
+   ParFiniteElementSpace * pfes = pblfr->ParFESpace();
+   int vsize = pfes->GetVSize();
 
-   // Allocate temporary vector
-   Vector b_0;
-   b_0.UseDevice(true);
-   b_0.SetSize(vsize);
-   b_0 = 0.0;
+   // Allocate temporary vectors
+   Vector b_0(vsize); b_0 = 0.0;
 
    // Extract the real and imaginary parts of the input vectors
    MFEM_ASSERT(x.Size() == 2 * vsize, "Input GridFunction of incorrect size!");
-   x.Read();
-   Vector x_r; x_r.MakeRef(x, 0, vsize);
-   Vector x_i; x_i.MakeRef(x, vsize, vsize);
+   Vector x_r(x.GetData(), vsize);
+   Vector x_i(&(x.GetData())[vsize], vsize);
 
    MFEM_ASSERT(b.Size() == 2 * vsize, "Input LinearForm of incorrect size!");
-   b.Read();
-   Vector b_r; b_r.MakeRef(b, 0, vsize);
-   Vector b_i; b_i.MakeRef(b, vsize, vsize);
+   Vector b_r(b.GetData(), vsize);
+   Vector b_i(&(b.GetData())[vsize], vsize);
 
    if (conv == ComplexOperator::BLOCK_SYMMETRIC) { b_i *= -1.0; }
 
-   const int tvsize = pfes->GetTrueVSize();
+   int tvsize = pfes->GetTrueVSize();
+
    OperatorHandle A_r, A_i;
 
-   X.UseDevice(true);
    X.SetSize(2 * tvsize);
-   X = 0.0;
-
-   B.UseDevice(true);
    B.SetSize(2 * tvsize);
-   B = 0.0;
 
-   Vector X_r; X_r.MakeRef(X, 0, tvsize);
-   Vector X_i; X_i.MakeRef(X, tvsize, tvsize);
-   Vector B_r; B_r.MakeRef(B, 0, tvsize);
-   Vector B_i; B_i.MakeRef(B, tvsize, tvsize);
-
-   Vector X_0, B_0;
+   Vector X_0(tvsize), B_0(tvsize);
+   Vector X_r(X.GetData(),tvsize);
+   Vector X_i(&(X.GetData())[tvsize], tvsize);
+   Vector B_r(B.GetData(), tvsize);
+   Vector B_i(&(B.GetData())[tvsize], tvsize);
 
    if (RealInteg())
    {
@@ -1189,29 +1042,24 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
 
    if (RealInteg() && ImagInteg())
    {
+      int n = ess_tdof_list.Size();
       // Modify RHS to conform with standard essential BC treatment
-      const int n = ess_tdof_list.Size();
-      auto d_B_r = B_r.Write();
-      auto d_B_i = B_i.Write();
-      auto d_X_r = X_r.Read();
-      auto d_X_i = X_i.Read();
-      auto d_idx = ess_tdof_list.Read();
-      MFEM_FORALL(i, n,
+      for (int k = 0; k < n; k++)
       {
-         const int j = d_idx[i];
-         d_B_r[j] = d_X_r[j];
-         d_B_i[j] = d_X_i[j];
-      });
+         int j=ess_tdof_list[k];
+         B_r(j) = X_r(j);
+         B_i(j) = X_i(j);
+      }
       // Modify offdiagonal blocks (imaginary parts of the matrix) to conform
       // with standard essential BC treatment
-      if (A_i.Type() == Operator::Hypre_ParCSR)
+      if ( A_i.Type() == Operator::Hypre_ParCSR )
       {
          HypreParMatrix * Ah;
          A_i.Get(Ah);
          hypre_ParCSRMatrix *Aih = *Ah;
          for (int k = 0; k < n; k++)
          {
-            const int j = ess_tdof_list[k];
+            int j = ess_tdof_list[k];
             Aih->diag->data[Aih->diag->i[j]] = 0.0;
          }
       }
@@ -1228,16 +1076,6 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
       b_i *= -1.0;
    }
 
-   x_r.SyncAliasMemory(x);
-   x_i.SyncAliasMemory(x);
-   b_r.SyncAliasMemory(b);
-   b_i.SyncAliasMemory(b);
-
-   X_r.SyncAliasMemory(X);
-   X_i.SyncAliasMemory(X);
-   B_r.SyncAliasMemory(B);
-   B_i.SyncAliasMemory(B);
-
    // A = A_r + i A_i
    A.Clear();
    if ( A_r.Type() == Operator::Hypre_ParCSR ||
@@ -1337,27 +1175,22 @@ void
 ParSesquilinearForm::RecoverFEMSolution(const Vector &X, const Vector &b,
                                         Vector &x)
 {
-   ParFiniteElementSpace *pfes = pblfr->ParFESpace();
+   ParFiniteElementSpace * pfes = pblfr->ParFESpace();
 
    const Operator &P = *pfes->GetProlongationMatrix();
 
-   const int vsize  = pfes->GetVSize();
-   const int tvsize = X.Size() / 2;
+   int vsize  = pfes->GetVSize();
+   int tvsize = X.Size() / 2;
 
-   X.Read();
-   Vector X_r; X_r.MakeRef(const_cast<Vector&>(X), 0, tvsize);
-   Vector X_i; X_i.MakeRef(const_cast<Vector&>(X), tvsize, tvsize);
+   Vector X_r(X.GetData(), tvsize);
+   Vector X_i(&(X.GetData())[tvsize], tvsize);
 
-   x.Write();
-   Vector x_r; x_r.MakeRef(x, 0, vsize);
-   Vector x_i; x_i.MakeRef(x, vsize, vsize);
+   Vector x_r(x.GetData(), vsize);
+   Vector x_i(&(x.GetData())[vsize], vsize);
 
    // Apply conforming prolongation
    P.Mult(X_r, x_r);
    P.Mult(X_i, x_i);
-
-   x_r.SyncAliasMemory(x);
-   x_i.SyncAliasMemory(x);
 }
 
 void

fem/complex_fem.hpp

@@ -38,8 +38,8 @@ protected:
    void Destroy() { delete gfr; delete gfi; }
 
 public:
-   /** @brief Construct a ComplexGridFunction associated with the
-       FiniteElementSpace @a *f. */
+   /* @brief Construct a ComplexGridFunction associated with the
+      FiniteElementSpace @a *f. */
    ComplexGridFunction(FiniteElementSpace *f);
 
    void Update();
@@ -71,14 +71,6 @@ public:
    const GridFunction & real() const { return *gfr; }
    const GridFunction & imag() const { return *gfi; }
 
-   /// Update the memory location of the real and imaginary GridFunction @a gfr
-   /// and @a gfi to match the ComplexGridFunction.
-   void Sync() { gfr->SyncMemory(*this); gfi->SyncMemory(*this); }
-
-   /// Update the alias memory location of the real and imaginary GridFunction
-   /// @a gfr and @a gfi to match the ComplexGridFunction.
-   void SyncAlias() { gfr->SyncAliasMemory(*this); gfi->SyncAliasMemory(*this); }
-
    /// Destroys the grid function.
    virtual ~ComplexGridFunction() { Destroy(); }
 
@@ -107,8 +99,8 @@ public:
                      ComplexOperator::Convention
                      convention = ComplexOperator::HERMITIAN);
 
-   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a fes, using
-       the same integrators as the LinearForms @a lf_r (real) and @a lf_i (imag).
+   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a f, using
+       the same integrators as the LinearForms @a lfr (real) and @a lfi (imag) .
 
        The pointer @a fes is not owned by the newly constructed object.
 
@@ -165,14 +157,6 @@ public:
    const LinearForm & real() const { return *lfr; }
    const LinearForm & imag() const { return *lfi; }
 
-   /// Update the memory location of the real and imaginary LinearForm @a lfr
-   /// and @a lfi to match the ComplexLinearForm.
-   void Sync() { lfr->SyncMemory(*this); lfi->SyncMemory(*this); }
-
-   /// Update the alias memory location of the real and imaginary LinearForm @a
-   /// lfr and @a lfi to match the ComplexLinearForm.
-   void SyncAlias() { lfr->SyncAliasMemory(*this); lfi->SyncAliasMemory(*this); }
-
    void Update();
    void Update(FiniteElementSpace *f);
 
@@ -211,8 +195,8 @@ private:
    BilinearForm *blfr;
    BilinearForm *blfi;
 
-   /* These methods check if the real/imag parts of the sesquilinear form are
-      not empty */
+   /* These methods check if the real/imag parts of the sesqulinear form are not
+      empty */
    bool RealInteg();
    bool ImagInteg();
 
@@ -220,7 +204,7 @@ public:
    SesquilinearForm(FiniteElementSpace *fes,
                     ComplexOperator::Convention
                     convention = ComplexOperator::HERMITIAN);
-   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a fes, using
+   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a f, using
        the same integrators as the BilinearForms @a bfr and @a bfi .
 
        The pointer @a fes is not owned by the newly constructed object.
@@ -339,8 +323,8 @@ protected:
 
 public:
 
-   /** @brief Construct a ParComplexGridFunction associated with the
-       ParFiniteElementSpace @a *pf. */
+   /* @brief Construct a ParComplexGridFunction associated with the
+      ParFiniteElementSpace @a *f. */
    ParComplexGridFunction(ParFiniteElementSpace *pf);
 
    void Update();
@@ -381,15 +365,6 @@ public:
    const ParGridFunction & real() const { return *pgfr; }
    const ParGridFunction & imag() const { return *pgfi; }
 
-   /// Update the memory location of the real and imaginary ParGridFunction @a
-   /// pgfr and @a pgfi to match the ParComplexGridFunction.
-   void Sync() { pgfr->SyncMemory(*this); pgfi->SyncMemory(*this); }
-
-   /// Update the alias memory location of the real and imaginary
-   /// ParGridFunction @a pgfr and @a pgfi to match the ParComplexGridFunction.
-   void SyncAlias() { pgfr->SyncAliasMemory(*this); pgfi->SyncAliasMemory(*this); }
-
-
    virtual double ComputeL2Error(Coefficient &exsolr, Coefficient &exsoli,
                                  const IntegrationRule *irs[] = NULL) const
    {
@@ -441,8 +416,8 @@ public:
                         convention = ComplexOperator::HERMITIAN);
 
    /** @brief Create a ParComplexLinearForm on the ParFiniteElementSpace @a pf,
-       using the same integrators as the LinearForms @a plf_r (real) and
-       @a plf_i (imag).
+       using the same integrators as the LinearForms @a plfr (real) and @a plfi
+       (imag) .
 
       The pointer @a fes is not owned by the newly constructed object.
 
@@ -500,14 +475,6 @@ public:
    const ParLinearForm & real() const { return *plfr; }
    const ParLinearForm & imag() const { return *plfi; }
 
-   /// Update the memory location of the real and imaginary ParLinearForm @a lfr
-   /// and @a lfi to match the ParComplexLinearForm.
-   void Sync() { plfr->SyncMemory(*this); plfi->SyncMemory(*this); }
-
-   /// Update the alias memory location of the real and imaginary ParLinearForm
-   /// @a plfr and @a plfi to match the ParComplexLinearForm.
-   void SyncAlias() { plfr->SyncAliasMemory(*this); plfi->SyncAliasMemory(*this); }
-
    void Update(ParFiniteElementSpace *pf = NULL);
 
    /// Assembles the linear form i.e. sums over all domain/bdr integrators.

fem/convergence.cpp

@@ -1,297 +0,0 @@
-#include "convergence.hpp"
-
-using namespace std;
-
-
-namespace mfem
-{
-
-void ConvergenceStudy::Reset()
-{
-   counter=0;
-   dcounter=0;
-   fcounter=0;
-   cont_type=-1;
-   print_flag=1;
-   L2Errors.SetSize(0);
-   L2Rates.SetSize(0);
-   DErrors.SetSize(0);
-   DRates.SetSize(0);
-   EnErrors.SetSize(0);
-   EnRates.SetSize(0);
-   DGFaceErrors.SetSize(0);
-   DGFaceRates.SetSize(0);
-   ndofs.SetSize(0);
-}
-
-double ConvergenceStudy::GetNorm(GridFunction *gf, Coefficient *scalar_u,
-                                 VectorCoefficient *vector_u)
-{
-   bool norm_set = false;
-   double norm=0.0;
-   int order = gf->FESpace()->GetOrder(0);
-   int order_quad = std::max(2, 2*order+1);
-   const IntegrationRule *irs[Geometry::NumGeom];
-   for (int i=0; i < Geometry::NumGeom; ++i)
-   {
-      irs[i] = &(IntRules.Get(i, order_quad));
-   }
-#ifdef MFEM_USE_MPI
-   ParGridFunction *pgf = dynamic_cast<ParGridFunction *>(gf);
-   if (pgf)
-   {
-      ParMesh *pmesh = pgf->ParFESpace()->GetParMesh();
-      if (scalar_u)
-      {
-         norm = ComputeGlobalLpNorm(2.0,*scalar_u,*pmesh,irs);
-      }
-      else if (vector_u)
-      {
-         norm = ComputeGlobalLpNorm(2.0,*vector_u,*pmesh,irs);
-      }
-      norm_set = true;
-   }
-#endif
-   if (!norm_set)
-   {
-      Mesh *mesh = gf->FESpace()->GetMesh();
-      if (scalar_u)
-      {
-         norm = ComputeLpNorm(2.0,*scalar_u,*mesh,irs);
-      }
-      else if (vector_u)
-      {
-         norm = ComputeLpNorm(2.0,*vector_u,*mesh,irs);
-      }
-   }
-   return norm;
-}
-
-void ConvergenceStudy::AddL2Error(GridFunction *gf,
-                                  Coefficient *scalar_u, VectorCoefficient *vector_u)
-{
-   int tdofs=0;
-#ifdef MFEM_USE_MPI
-   ParGridFunction *pgf = dynamic_cast<ParGridFunction *>(gf);
-   if (pgf)
-   {
-      MPI_Comm comm = pgf->ParFESpace()->GetComm();
-      int rank;
-      MPI_Comm_rank(comm, &rank);
-      print_flag = 0;
-      if (rank==0) { print_flag = 1; }
-      tdofs = pgf->ParFESpace()->GlobalTrueVSize();
-   }
-#endif
-   if (!tdofs) { tdofs = gf->FESpace()->GetTrueVSize(); }
-   ndofs.Append(tdofs);
-   double L2Err;
-   if (scalar_u)
-   {
-      L2Err = gf->ComputeL2Error(*scalar_u);
-      CoeffNorm = GetNorm(gf,scalar_u,nullptr);
-   }
-   else if (vector_u)
-   {
-      L2Err = gf->ComputeL2Error(*vector_u);
-      CoeffNorm = GetNorm(gf,nullptr,vector_u);
-   }
-   else
-   {
-      MFEM_ABORT("Exact Solution Coefficient pointer is NULL");
-   }
-   L2Errors.Append(L2Err);
-   // Compute the rate of convergence by:
-   // rate = log (||u - u_h|| / ||u - u_{h/2}||)/log(2)
-   double val = (counter) ? log(L2Errors[counter-1]/L2Err)/log(2.0) : 0.0;
-   L2Rates.Append(val);
-   counter++;
-}
-
-void ConvergenceStudy::AddGf(GridFunction *gf, Coefficient *scalar_u,
-                             VectorCoefficient *grad,
-                             Coefficient *ell_coeff, double Nu)
-{
-   cont_type = gf->FESpace()->FEColl()->GetContType();
-
-   MFEM_VERIFY((cont_type == mfem::FiniteElementCollection::CONTINUOUS) ||
-               (cont_type == mfem::FiniteElementCollection::DISCONTINUOUS),
-               "This constructor is intended for H1 or L2 Elements")
-
-   AddL2Error(gf,scalar_u, nullptr);
-
-   if (grad)
-   {
-      double GradErr = gf->ComputeGradError(grad);
-      DErrors.Append(GradErr);
-      double err = sqrt(L2Errors[counter-1]*L2Errors[counter-1]+GradErr*GradErr);
-      EnErrors.Append(err);
-      // Compute the rate of convergence by:
-      // rate = log (||u - u_h|| / ||u - u_{h/2}||)/log(2)
-      double val = (dcounter) ? log(DErrors[dcounter-1]/GradErr)/log(2.0) : 0.0;
-      double eval = (dcounter) ? log(EnErrors[dcounter-1]/err)/log(2.0) : 0.0;
-      DRates.Append(val);
-      EnRates.Append(eval);
-      CoeffDNorm = GetNorm(gf,nullptr,grad);
-      dcounter++;
-      MFEM_VERIFY(counter == dcounter,
-                  "Number of added solutions and derivatives do not match")
-   }
-
-   if (cont_type == mfem::FiniteElementCollection::DISCONTINUOUS && ell_coeff)
-   {
-      double DGErr = gf->ComputeDGFaceJumpError(scalar_u,ell_coeff,Nu);
-      DGFaceErrors.Append(DGErr);
-      // Compute the rate of convergence by:
-      // rate = log (||u - u_h|| / ||u - u_{h/2}||)/log(2)
-      double val=(fcounter) ? log(DGFaceErrors[fcounter-1]/DGErr)/log(2.0):0.;
-      DGFaceRates.Append(val);
-      fcounter++;
-      MFEM_VERIFY(fcounter == counter, "Number of added solutions mismatch");
-   }
-}
-
-void ConvergenceStudy::AddGf(GridFunction *gf, VectorCoefficient *vector_u,
-                             VectorCoefficient *curl, Coefficient *div)
-{
-   cont_type = gf->FESpace()->FEColl()->GetContType();
-
-   AddL2Error(gf,nullptr,vector_u);
-   double DErr = 0.0;
-   bool derivative = false;
-   if (curl)
-   {
-      DErr = gf->ComputeCurlError(curl);
-      CoeffDNorm = GetNorm(gf,nullptr,curl);
-      derivative = true;
-   }
-   else if (div)
-   {
-      DErr = gf->ComputeDivError(div);
-      // update coefficient norm
-      CoeffDNorm = GetNorm(gf,div,nullptr);
-      derivative = true;
-   }
-   if (derivative)
-   {
-      double err = sqrt(L2Errors[counter-1]*L2Errors[counter-1] + DErr*DErr);
-      DErrors.Append(DErr);
-      EnErrors.Append(err);
-      // Compute the rate of convergence by:
-      // rate = log (||u - u_h|| / ||u - u_{h/2}||)/log(2)
-      double val = (dcounter) ? log(DErrors[dcounter-1]/DErr)/log(2.0) : 0.0;
-      double eval = (dcounter) ? log(EnErrors[dcounter-1]/err)/log(2.0) : 0.0;
-      DRates.Append(val);
-      EnRates.Append(eval);
-      dcounter++;
-      MFEM_VERIFY(counter == dcounter,
-                  "Number of added solutions and derivatives do not match")
-   }
-}
-
-void ConvergenceStudy::Print(bool relative, std::ostream &out)
-{
-   if (print_flag)
-   {
-      std::string title = (relative) ? "Relative " : "Absolute ";
-      out << "\n";
-      out << " -------------------------------------------" << "\n";
-      out <<  std::setw(21) << title << "L2 Error " << "\n";
-      out << " -------------------------------------------"
-          << "\n";
-      out << std::right<< std::setw(11)<< "DOFs "<< std::setw(13) << "Error ";
-      out <<  std::setw(15) << "Rate " << "\n";
-      out << " -------------------------------------------"
-          << "\n";
-      out << std::setprecision(4);
-      double d = (relative) ? CoeffNorm : 1.0;
-      for (int i =0; i<counter; i++)
-      {
-         out << std::right << std::setw(10)<< ndofs[i] << std::setw(16)
-             << std::scientific << L2Errors[i]/d << std::setw(13)
-             << std::fixed << L2Rates[i] << "\n";
-      }
-      out << "\n";
-      if (dcounter == counter)
-      {
-         std::string dname;
-         switch (cont_type)
-         {
-            case 0: dname = "Grad"; break;
-            case 1: dname = "Curl"; break;
-            case 2: dname = "Div";  break;
-            case 3: dname = "DG Grad";  break;
-            default: break;
-         }
-         out << " -------------------------------------------" << "\n";
-         out <<  std::setw(21) << title << dname << " Error  " << "\n";
-         out << " -------------------------------------------" << "\n";
-         out << std::right<<std::setw(11)<< "DOFs "<< std::setw(13) << "Error";
-         out << std::setw(15) << "Rate " << "\n";
-         out << " -------------------------------------------"
-             << "\n";
-         out << std::setprecision(4);
-         d = (relative) ? CoeffDNorm : 1.0;
-         for (int i =0; i<dcounter; i++)
-         {
-            out << std::right << std::setw(10)<< ndofs[i] << std::setw(16)
-                << std::scientific << DErrors[i]/d << std::setw(13)
-                << std::fixed << DRates[i] << "\n";
-         }
-         out << "\n";
-         switch (cont_type)
-         {
-            case 0: dname = "H1"; break;
-            case 1: dname = "H(Curl)"; break;
-            case 2: dname = "H(Div)";  break;
-            case 3: dname = "DG H1";  break;
-            default: break;
-         }
-
-         if (dcounter)
-         {
-            d = (relative) ?
-                sqrt(CoeffNorm*CoeffNorm + CoeffDNorm*CoeffDNorm):1.0;
-
-            out << " -------------------------------------------" << "\n";
-            out << std::setw(21) << title << dname << " Error   " << "\n";
-            out << " -------------------------------------------" << "\n";
-            out << std::right<< std::setw(11)<< "DOFs "<< std::setw(13);
-            out << "Error ";
-            out << std::setw(15) << "Rate " << "\n";
-            out << " -------------------------------------------"
-                << "\n";
-            out << std::setprecision(4);
-            for (int i =0; i<dcounter; i++)
-            {
-               out << std::right << std::setw(10)<< ndofs[i] << std::setw(16)
-                   << std::scientific << EnErrors[i]/d << std::setw(13)
-                   << std::fixed << EnRates[i] << "\n";
-            }
-            out << "\n";
-         }
-         if (cont_type == 3 && fcounter)
-         {
-            out << " -------------------------------------------" << "\n";
-            out << "            DG Face Jump Error          " << "\n";
-            out << " -------------------------------------------"
-                << "\n";
-            out << std::right<< std::setw(11)<< "DOFs "<< std::setw(13);
-            out << "Error ";
-            out << std::setw(15) << "Rate " << "\n";
-            out << " -------------------------------------------"
-                << "\n";
-            out << std::setprecision(4);
-            for (int i =0; i<fcounter; i++)
-            {
-               out << std::right << std::setw(10)<< ndofs[i] << std::setw(16)
-                   << std::scientific << DGFaceErrors[i] << std::setw(13)
-                   << std::fixed << DGFaceRates[i] << "\n";
-            }
-            out << "\n";
-         }
-      }
-   }
-}
-
-} // namespace mfem

fem/convergence.hpp

@@ -1,149 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_CONVERGENCE
-#define MFEM_CONVERGENCE
-
-#include "../linalg/linalg.hpp"
-#include "gridfunc.hpp"
-#ifdef MFEM_USE_MPI
-#include "pgridfunc.hpp"
-#endif
-
-namespace mfem
-{
-
-/** @brief Class to compute error and convergence rates.
-    It supports H1, H(curl) (ND elements), H(div) (RT elements) and L2 (DG).
-
-    For "smooth enough" solutions the Galerkin error measured in the appropriate
-    norm satisfies || u - u_h || ~ h^k
-
-    Here, k is called the asymptotic rate of convergence
-
-    For successive uniform h-refinements the rate can be estimated by
-    k = log(||u - u_h|| / ||u - u_{h/2}||)/log(2)
-*/
-class ConvergenceStudy
-{
-private:
-   // counters for solutions/derivatives
-   int counter=0;
-   int dcounter=0;
-   int fcounter=0;
-
-   // space continuity type
-   int cont_type=-1;
-
-   // printing flag for helpful for MPI calls
-   int print_flag=1;
-
-   // exact solution and derivatives
-   double CoeffNorm;
-   double CoeffDNorm;
-
-   // Arrays to store error/rates
-   Array<double> L2Errors, DGFaceErrors, DErrors, EnErrors;
-   Array<double> L2Rates, DGFaceRates, DRates, EnRates;
-   Array<int> ndofs;
-
-   void AddL2Error(GridFunction *gf, Coefficient *scalar_u,
-                   VectorCoefficient *vector_u);
-   void AddGf(GridFunction *gf, Coefficient *scalar_u,
-              VectorCoefficient *grad=nullptr,
-              Coefficient *ell_coeff=nullptr, double Nu=1.0);
-   void AddGf(GridFunction *gf, VectorCoefficient *vector_u,
-              VectorCoefficient *curl, Coefficient *div);
-   // returns the L2-norm of scalar_u or vector_u
-   double GetNorm(GridFunction *gf, Coefficient *scalar_u,
-                  VectorCoefficient *vector_u);
-
-public:
-
-   /// Clear any internal data
-   void Reset();
-
-   /// Add L2 GridFunction, the exact solution and possibly its gradient and/or
-   /// DG face jumps parameters
-   void AddL2GridFunction(GridFunction *gf, Coefficient *scalar_u,
-                          VectorCoefficient *grad=nullptr,
-                          Coefficient *ell_coeff=nullptr, double Nu=1.0)
-   {
-      AddGf(gf, scalar_u, grad, ell_coeff, Nu);
-   }
-
-   /// Add H1 GridFunction, the exact solution and possibly its gradient
-   void AddH1GridFunction(GridFunction *gf, Coefficient *scalar_u,
-                          VectorCoefficient *grad=nullptr)
-   {
-      AddGf(gf, scalar_u, grad);
-   }
-
-   /// Add H(curl) GridFunction, the exact solution and possibly its curl
-   void AddHcurlGridFunction(GridFunction *gf, VectorCoefficient *vector_u,
-                             VectorCoefficient *curl=nullptr)
-   {
-      AddGf(gf, vector_u, curl, nullptr);
-   }
-
-   /// Add H(div) GridFunction, the exact solution and possibly its div
-   void AddHdivGridFunction(GridFunction *gf, VectorCoefficient *vector_u,
-                            Coefficient *div=nullptr)
-   {
-      AddGf(gf,vector_u, nullptr, div);
-   }
-
-   /// Get the L2 error at step n
-   double GetL2Error(int n)
-   {
-      MFEM_VERIFY( n <= counter,"Step out of bounds")
-      return L2Errors[n];
-   }
-
-   /// Get all L2 errors
-   void GetL2Errors(Array<double> & L2Errors_)
-   {
-      L2Errors_ = L2Errors;
-   }
-
-   /// Get the Grad/Curl/Div error at step n
-   double GetDError(int n)
-   {
-      MFEM_VERIFY(n <= dcounter,"Step out of bounds")
-      return DErrors[n];
-   }
-
-   /// Get all Grad/Curl/Div errors
-   void GetDErrors(Array<double> & DErrors_)
-   {
-      DErrors_ = DErrors;
-   }
-
-   /// Get the DGFaceJumps error at step n
-   double GetDGFaceJumpsError(int n)
-   {
-      MFEM_VERIFY(n<= fcounter,"Step out of bounds")
-      return DGFaceErrors[n];
-   }
-
-   /// Get all DGFaceJumps errors
-   void GetDGFaceJumpsErrors(Array<double> & DGFaceErrors_)
-   {
-      DGFaceErrors_ = DGFaceErrors;
-   }
-
-   /// Print rates and errors
-   void Print(bool relative = false, std::ostream &out = mfem::out);
-};
-
-} // namespace mfem
-
-#endif // MFEM_CONVERGENCE

fem/datacollection.cpp

@@ -563,8 +563,6 @@ void VisItDataCollection::LoadVisItRootFile(const std::string& root_name)
 
 void VisItDataCollection::LoadMesh()
 {
-   // GetMeshFileName() uses 'serial', so we need to set it in advance.
-   serial = (format == SERIAL_FORMAT);
    std::string mesh_fname = GetMeshFileName();
    named_ifgzstream file(mesh_fname);
    // TODO: in parallel, check for errors on all processors

fem/fe.cpp

@@ -139,12 +139,6 @@ void FiniteElement::Project (
    mfem_error ("FiniteElement::Project (...) (vector) is not overloaded !");
 }
 
-void FiniteElement::ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                     Vector &dofs) const
-{
-   mfem_error ("FiniteElement::ProjectFromNodes() (vector) is not overloaded!");
-}
-
 void FiniteElement::ProjectMatrixCoefficient(
    MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
 {
@@ -931,23 +925,6 @@ void VectorFiniteElement::Project_RT(
    }
 }
 
-void VectorFiniteElement::Project_RT(
-   const double *nk, const Array<int> &d2n,
-   Vector &vc, ElementTransformation &Trans, Vector &dofs) const
-{
-   const int sdim = Trans.GetSpaceDim();
-   const bool square_J = (dim == sdim);
-
-   for (int k = 0; k < dof; k++)
-   {
-      Trans.SetIntPoint(&Nodes.IntPoint(k));
-      // dof_k = nk^t adj(J) xk
-      Vector vk(vc.GetData()+k*sdim, sdim);
-      dofs(k) = Trans.AdjugateJacobian().InnerProduct(vk, nk + d2n[k]*dim);
-      if (!square_J) { dofs(k) /= Trans.Weight(); }
-   }
-}
-
 void VectorFiniteElement::ProjectMatrixCoefficient_RT(
    const double *nk, const Array<int> &d2n,
    MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
@@ -1124,19 +1101,6 @@ void VectorFiniteElement::Project_ND(
    }
 }
 
-void VectorFiniteElement::Project_ND(
-   const double *tk, const Array<int> &d2t,
-   Vector &vc, ElementTransformation &Trans, Vector &dofs) const
-{
-   for (int k = 0; k < dof; k++)
-   {
-      Trans.SetIntPoint(&Nodes.IntPoint(k));
-      Vector vk(vc.GetData()+k*dim, dim);
-      // dof_k = xk^t J tk
-      dofs(k) = Trans.Jacobian().InnerProduct(tk + d2t[k]*dim, vk);
-   }
-}
-
 void VectorFiniteElement::ProjectMatrixCoefficient_ND(
    const double *tk, const Array<int> &d2t,
    MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
@@ -8031,7 +7995,7 @@ void H1_HexahedronElement::CalcHessian(const IntegrationPoint &ip,
 #ifdef MFEM_THREAD_SAFE
    Vector shape_x(p+1),  shape_y(p+1),  shape_z(p+1);
    Vector dshape_x(p+1), dshape_y(p+1), dshape_z(p+1);
-   Vector d2shape_x(p+1), d2shape_y(p+1), d2shape_z(p+1);
+   Vector d2shape_x(p+1), d2shape_y(p+1), ds2hape_z(p+1);
 #endif
 
    basis1d.Eval(ip.x, shape_x, dshape_x, d2shape_x);

fem/fe.hpp

@@ -504,21 +504,14 @@ public:
    /** @brief Given a coefficient and a transformation, compute its projection
        (approximation) in the local finite dimensional space in terms
        of the degrees of freedom. */
-   virtual void Project(Coefficient &coeff,
-                        ElementTransformation &Trans, Vector &dofs) const;
+   virtual void Project (Coefficient &coeff,
+                         ElementTransformation &Trans, Vector &dofs) const;
 
    /** @brief Given a vector coefficient and a transformation, compute its
        projection (approximation) in the local finite dimensional space
        in terms of the degrees of freedom. (VectorFiniteElements) */
-   virtual void Project(VectorCoefficient &vc,
-                        ElementTransformation &Trans, Vector &dofs) const;
-
-   /** @brief Given a vector of values at the finite element nodes and a
-       transformation, compute its projection (approximation) in the local
-       finite dimensional space in terms of the degrees of freedom. Valid for
-       VectorFiniteElements. */
-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const;
+   virtual void Project (VectorCoefficient &vc,
+                         ElementTransformation &Trans, Vector &dofs) const;
 
    /** @brief Given a matrix coefficient and a transformation, compute an
        approximation ("projection") in the local finite dimensional space in
@@ -804,12 +797,7 @@ protected:
                    VectorCoefficient &vc, ElementTransformation &Trans,
                    Vector &dofs) const;
 
-   /// Projects the vector of values given at FE nodes to RT space
-   void Project_RT(const double *nk, const Array<int> &d2n,
-                   Vector &vc, ElementTransformation &Trans,
-                   Vector &dofs) const;
-
-   /// Project the rows of the matrix coefficient in an RT space
+   // project the rows of the matrix coefficient in an RT space
    void ProjectMatrixCoefficient_RT(
       const double *nk, const Array<int> &d2n,
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const;
@@ -837,12 +825,7 @@ protected:
                    VectorCoefficient &vc, ElementTransformation &Trans,
                    Vector &dofs) const;
 
-   /// Projects the vector of values given at FE nodes to ND space
-   void Project_ND(const double *tk, const Array<int> &d2t,
-                   Vector &vc, ElementTransformation &Trans,
-                   Vector &dofs) const;
-
-   /// Project the rows of the matrix coefficient in an ND space
+   /// project the rows of the matrix coefficient in an ND space
    void ProjectMatrixCoefficient_ND(
       const double *tk, const Array<int> &d2t,
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const;
@@ -2706,9 +2689,6 @@ public:
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_RT(nk, dof2nk, vc, Trans, dofs); }
-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const
-   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
    virtual void ProjectMatrixCoefficient(
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
    { ProjectMatrixCoefficient_RT(nk, dof2nk, mc, T, dofs); }
@@ -2767,9 +2747,6 @@ public:
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_RT(nk, dof2nk, vc, Trans, dofs); }
-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const
-   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
    virtual void ProjectMatrixCoefficient(
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
    { ProjectMatrixCoefficient_RT(nk, dof2nk, mc, T, dofs); }
@@ -2821,9 +2798,6 @@ public:
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_RT(nk, dof2nk, vc, Trans, dofs); }
-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const
-   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
    virtual void ProjectMatrixCoefficient(
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
    { ProjectMatrixCoefficient_RT(nk, dof2nk, mc, T, dofs); }
@@ -2881,9 +2855,6 @@ public:
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_RT(nk, dof2nk, vc, Trans, dofs); }
-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const
-   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
    virtual void ProjectMatrixCoefficient(
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
    { ProjectMatrixCoefficient_RT(nk, dof2nk, mc, T, dofs); }
@@ -2943,10 +2914,6 @@ public:
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_ND(tk, dof2tk, vc, Trans, dofs); }
 
-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const
-   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
-
    virtual void ProjectMatrixCoefficient(
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
    { ProjectMatrixCoefficient_ND(tk, dof2tk, mc, T, dofs); }
@@ -3006,9 +2973,6 @@ public:
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_ND(tk, dof2tk, vc, Trans, dofs); }
-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const
-   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
    virtual void ProjectMatrixCoefficient(
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
    { ProjectMatrixCoefficient_ND(tk, dof2tk, mc, T, dofs); }
@@ -3060,9 +3024,6 @@ public:
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_ND(tk, dof2tk, vc, Trans, dofs); }
-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const
-   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
    virtual void ProjectMatrixCoefficient(
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
    { ProjectMatrixCoefficient_ND(tk, dof2tk, mc, T, dofs); }
@@ -3119,9 +3080,6 @@ public:
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_ND(tk, dof2tk, vc, Trans, dofs); }
-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const
-   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
    virtual void ProjectMatrixCoefficient(
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
    { ProjectMatrixCoefficient_ND(tk, dof2tk, mc, T, dofs); }

fem/fem.hpp

@@ -19,7 +19,6 @@
 #include "eltrans.hpp"
 #include "coefficient.hpp"
 #include "complex_fem.hpp"
-#include "convergence.hpp"
 #include "lininteg.hpp"
 #include "nonlininteg.hpp"
 #include "bilininteg.hpp"

fem/fespace.cpp

@@ -440,7 +440,6 @@ void FiniteElementSpace::MarkerToList(const Array<int> &marker,
       if (marker[i]) { num_marked++; }
    }
    list.SetSize(0);
-   list.HostWrite();
    list.Reserve(num_marked);
    for (int i = 0; i < marker.Size(); i++)
    {
@@ -452,9 +451,7 @@ void FiniteElementSpace::MarkerToList(const Array<int> &marker,
 void FiniteElementSpace::ListToMarker(const Array<int> &list, int marker_size,
                                       Array<int> &marker, int mark_val)
 {
-   list.HostRead(); // make sure we can read the array on host
    marker.SetSize(marker_size);
-   marker.HostWrite();
    marker = 0;
    for (int i = 0; i < list.Size(); i++)
    {

fem/fespace.hpp

@@ -330,11 +330,6 @@ public:
    virtual const Operator *GetProlongationMatrix() const
    { return GetConformingProlongation(); }
 
-   /// Return a prolongation operator that only acts on local dofs
-   /** The returned operator is owned by the FiniteElementSpace. */
-   virtual const Operator *GetLocalProlongationMatrix() const
-   { return GetConformingProlongation(); }
-
    /// The returned SparseMatrix is owned by the FiniteElementSpace.
    virtual const SparseMatrix *GetRestrictionMatrix() const
    { return GetConformingRestriction(); }
@@ -989,6 +984,7 @@ inline bool UsesTensorBasis(const FiniteElementSpace& fes)
 {
    return dynamic_cast<const mfem::TensorBasisElement *>(fes.GetFE(0))!=nullptr;
 }
+
 }
 
 #endif

fem/gridfunc.cpp

@@ -199,7 +199,8 @@ void GridFunction::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
    if (f != fes) { Destroy(); }
    fes = f;
    v.UseDevice(true);
-   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
+   NewMemoryAndSize(Memory<double>(v.GetMemory(), v_offset, fes->GetVSize()),
+                    fes->GetVSize(), true);
    sequence = fes->GetSequence();
 }
 
@@ -1833,19 +1834,6 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
    ImposeBounds(i, weights, minv, maxv);
 }
 
-void GridFunction::RestrictConforming()
-{
-   const SparseMatrix *R = fes->GetRestrictionMatrix();
-   const Operator *P = fes->GetProlongationMatrix();
-
-   if (P && R)
-   {
-      Vector tmp(R->Height());
-      R->Mult(*this, tmp);
-      P->Mult(tmp, *this);
-   }
-}
-
 void GridFunction::GetNodalValues(Vector &nval, int vdim) const
 {
    int i, j;
@@ -2614,7 +2602,11 @@ double GridFunction::ComputeL2Error(
       }
    }
 
-   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
+   if (error < 0.0)
+   {
+      return -sqrt(-error);
+   }
+   return sqrt(error);
 }
 
 double GridFunction::ComputeL2Error(
@@ -2655,199 +2647,94 @@ double GridFunction::ComputeL2Error(
       }
    }
 
-   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
-}
-
-double GridFunction::ComputeGradError(VectorCoefficient *exgrad,
-                                      const IntegrationRule *irs[]) const
-{
-   double error = 0.0;
-   const FiniteElement *fe;
-   ElementTransformation *Tr;
-   Array<int> dofs;
-   Vector grad;
-   int intorder;
-   int dim = fes->GetMesh()->SpaceDimension();
-   Vector vec(dim);
-
-   for (int i = 0; i < fes->GetNE(); i++)
+   if (error < 0.0)
    {
-      fe = fes->GetFE(i);
-      Tr = fes->GetElementTransformation(i);
-      intorder = 2*fe->GetOrder() + 3; // <--------
-      const IntegrationRule *ir;
-      if (irs)
-      {
-         ir = irs[fe->GetGeomType()];
-      }
-      else
-      {
-         ir = &(IntRules.Get(fe->GetGeomType(), intorder));
-      }
-      fes->GetElementDofs(i, dofs);
-      for (int j = 0; j < ir->GetNPoints(); j++)
-      {
-         const IntegrationPoint &ip = ir->IntPoint(j);
-         Tr->SetIntPoint(&ip);
-         GetGradient(*Tr,grad);
-         exgrad->Eval(vec,*Tr,ip);
-         vec-=grad;
-         error += ip.weight * Tr->Weight() * (vec * vec);
-      }
+      return -sqrt(-error);
    }
-   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
+   return sqrt(error);
 }
 
-double GridFunction::ComputeCurlError(VectorCoefficient *excurl,
-                                      const IntegrationRule *irs[]) const
+double GridFunction::ComputeH1Error(
+   Coefficient *exsol, VectorCoefficient *exgrad,
+   Coefficient *ell_coeff, double Nu, int norm_type) const
 {
-   double error = 0.0;
-   const FiniteElement *fe;
-   ElementTransformation *Tr;
-   Array<int> dofs;
-   Vector curl;
-   int intorder;
-   int dim = fes->GetMesh()->SpaceDimension();
-   int n = (dim == 3) ? dim : 1;
-   Vector vec(n);
-
-   for (int i = 0; i < fes->GetNE(); i++)
-   {
-      fe = fes->GetFE(i);
-      Tr = fes->GetElementTransformation(i);
-      intorder = 2*fe->GetOrder() + 3;
-      const IntegrationRule *ir;
-      if (irs)
-      {
-         ir = irs[fe->GetGeomType()];
-      }
-      else
-      {
-         ir = &(IntRules.Get(fe->GetGeomType(), intorder));
-      }
-      fes->GetElementDofs(i, dofs);
-      for (int j = 0; j < ir->GetNPoints(); j++)
-      {
-         const IntegrationPoint &ip = ir->IntPoint(j);
-         Tr->SetIntPoint(&ip);
-         GetCurl(*Tr,curl);
-         excurl->Eval(vec,*Tr,ip);
-         vec-=curl;
-         error += ip.weight * Tr->Weight() * ( vec * vec );
-      }
-   }
-
-   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
-}
-
-double GridFunction::ComputeDivError(
-   Coefficient *exdiv, const IntegrationRule *irs[]) const
-{
-   double error = 0.0, a;
-   const FiniteElement *fe;
-   ElementTransformation *Tr;
-   Array<int> dofs;
-   int intorder;
-
-   for (int i = 0; i < fes->GetNE(); i++)
-   {
-      fe = fes->GetFE(i);
-      Tr = fes->GetElementTransformation(i);
-      intorder = 2*fe->GetOrder() + 3;
-      const IntegrationRule *ir;
-      if (irs)
-      {
-         ir = irs[fe->GetGeomType()];
-      }
-      else
-      {
-         ir = &(IntRules.Get(fe->GetGeomType(), intorder));
-      }
-      fes->GetElementDofs(i, dofs);
-      for (int j = 0; j < ir->GetNPoints(); j++)
-      {
-         const IntegrationPoint &ip = ir->IntPoint(j);
-         Tr->SetIntPoint (&ip);
-         a = GetDivergence(*Tr) - exdiv->Eval(*Tr, ip);
-         error += ip.weight * Tr->Weight() * a * a;
-      }
-   }
-
-   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
-}
-
-double GridFunction::ComputeDGFaceJumpError(Coefficient *exsol,
-                                            Coefficient *ell_coeff, double Nu,
-                                            const IntegrationRule *irs[])  const
-{
-   int fdof, dim, intorder, k;
+   // assuming vdim is 1
+   int i, fdof, dim, intorder, j, k;
    Mesh *mesh;
    const FiniteElement *fe;
    ElementTransformation *transf;
    FaceElementTransformations *face_elem_transf;
-   Vector shape, el_dofs, err_val, ell_coeff_val;
+   Vector e_grad, a_grad, shape, el_dofs, err_val, ell_coeff_val;
+   DenseMatrix dshape, dshapet, Jinv;
    Array<int> vdofs;
    IntegrationPoint eip;
    double error = 0.0;
 
    mesh = fes->GetMesh();
    dim = mesh->Dimension();
+   e_grad.SetSize(dim);
+   a_grad.SetSize(dim);
+   Jinv.SetSize(dim);
 
-   for (int i = 0; i < mesh->GetNumFaces(); i++)
-   {
-      face_elem_transf = mesh->GetFaceElementTransformations(i, 5);
-      int i1 = face_elem_transf->Elem1No;
-      int i2 = face_elem_transf->Elem2No;
-      intorder = fes->GetFE(i1)->GetOrder();
-      if (i2 >= 0)
-         if ( (k = fes->GetFE(i2)->GetOrder()) > intorder )
-         {
-            intorder = k;
-         }
-      intorder = 2 * intorder;  // <-------------
-      const IntegrationRule *ir;
-      if (irs)
+   if (norm_type & 1)
+      for (i = 0; i < mesh->GetNE(); i++)
       {
-         ir = irs[face_elem_transf->GetGeometryType()];
-      }
-      else
-      {
-         ir = &(IntRules.Get(face_elem_transf->GetGeometryType(), intorder));
-      }
-      err_val.SetSize(ir->GetNPoints());
-      ell_coeff_val.SetSize(ir->GetNPoints());
-      // side 1
-      transf = face_elem_transf->Elem1;
-      fe = fes->GetFE(i1);
-      fdof = fe->GetDof();
-      fes->GetElementVDofs(i1, vdofs);
-      shape.SetSize(fdof);
-      el_dofs.SetSize(fdof);
-      for (k = 0; k < fdof; k++)
-         if (vdofs[k] >= 0)
-         {
-            el_dofs(k) =   (*this)(vdofs[k]);
-         }
-         else
-         {
-            el_dofs(k) = - (*this)(-1-vdofs[k]);
-         }
-      for (int j = 0; j < ir->GetNPoints(); j++)
-      {
-         face_elem_transf->Loc1.Transform(ir->IntPoint(j), eip);
-         fe->CalcShape(eip, shape);
-         transf->SetIntPoint(&eip);
-         ell_coeff_val(j) = ell_coeff->Eval(*transf, eip);
-         err_val(j) = exsol->Eval(*transf, eip) - (shape * el_dofs);
-      }
-      if (i2 >= 0)
-      {
-         // side 2
-         face_elem_transf = mesh->GetFaceElementTransformations(i, 10);
-         transf = face_elem_transf->Elem2;
-         fe = fes->GetFE(i2);
+         fe = fes->GetFE(i);
          fdof = fe->GetDof();
-         fes->GetElementVDofs(i2, vdofs);
+         transf = mesh->GetElementTransformation(i);
+         el_dofs.SetSize(fdof);
+         dshape.SetSize(fdof, dim);
+         dshapet.SetSize(fdof, dim);
+         intorder = 2 * fe->GetOrder(); // <----------
+         const IntegrationRule &ir = IntRules.Get(fe->GetGeomType(), intorder);
+         fes->GetElementVDofs(i, vdofs);
+         for (k = 0; k < fdof; k++)
+            if (vdofs[k] >= 0)
+            {
+               el_dofs(k) =   (*this)(vdofs[k]);
+            }
+            else
+            {
+               el_dofs(k) = - (*this)(-1-vdofs[k]);
+            }
+         for (j = 0; j < ir.GetNPoints(); j++)
+         {
+            const IntegrationPoint &ip = ir.IntPoint(j);
+            fe->CalcDShape(ip, dshape);
+            transf->SetIntPoint(&ip);
+            exgrad->Eval(e_grad, *transf, ip);
+            CalcInverse(transf->Jacobian(), Jinv);
+            Mult(dshape, Jinv, dshapet);
+            dshapet.MultTranspose(el_dofs, a_grad);
+            e_grad -= a_grad;
+            error += (ip.weight * transf->Weight() *
+                      ell_coeff->Eval(*transf, ip) *
+                      (e_grad * e_grad));
+         }
+      }
+
+   if (norm_type & 2)
+      for (i = 0; i < mesh->GetNFaces(); i++)
+      {
+         face_elem_transf = mesh->GetFaceElementTransformations(i, 5);
+         int i1 = face_elem_transf->Elem1No;
+         int i2 = face_elem_transf->Elem2No;
+         intorder = fes->GetFE(i1)->GetOrder();
+         if (i2 >= 0)
+            if ( (k = fes->GetFE(i2)->GetOrder()) > intorder )
+            {
+               intorder = k;
+            }
+         intorder = 2 * intorder;  // <-------------
+         const IntegrationRule &ir =
+            IntRules.Get(face_elem_transf->GetGeometryType(), intorder);
+         err_val.SetSize(ir.GetNPoints());
+         ell_coeff_val.SetSize(ir.GetNPoints());
+         // side 1
+         transf = face_elem_transf->Elem1;
+         fe = fes->GetFE(i1);
+         fdof = fe->GetDof();
+         fes->GetElementVDofs(i1, vdofs);
          shape.SetSize(fdof);
          el_dofs.SetSize(fdof);
          for (k = 0; k < fdof; k++)
@@ -2859,69 +2746,60 @@ double GridFunction::ComputeDGFaceJumpError(Coefficient *exsol,
             {
                el_dofs(k) = - (*this)(-1-vdofs[k]);
             }
-         for (int j = 0; j < ir->GetNPoints(); j++)
+         for (j = 0; j < ir.GetNPoints(); j++)
          {
-            face_elem_transf->Loc2.Transform(ir->IntPoint(j), eip);
+            face_elem_transf->Loc1.Transform(ir.IntPoint(j), eip);
             fe->CalcShape(eip, shape);
             transf->SetIntPoint(&eip);
-            ell_coeff_val(j) += ell_coeff->Eval(*transf, eip);
-            ell_coeff_val(j) *= 0.5;
-            err_val(j) -= (exsol->Eval(*transf, eip) - (shape * el_dofs));
+            ell_coeff_val(j) = ell_coeff->Eval(*transf, eip);
+            err_val(j) = exsol->Eval(*transf, eip) - (shape * el_dofs);
+         }
+         if (i2 >= 0)
+         {
+            // side 2
+            face_elem_transf = mesh->GetFaceElementTransformations(i, 10);
+            transf = face_elem_transf->Elem2;
+            fe = fes->GetFE(i2);
+            fdof = fe->GetDof();
+            fes->GetElementVDofs(i2, vdofs);
+            shape.SetSize(fdof);
+            el_dofs.SetSize(fdof);
+            for (k = 0; k < fdof; k++)
+               if (vdofs[k] >= 0)
+               {
+                  el_dofs(k) =   (*this)(vdofs[k]);
+               }
+               else
+               {
+                  el_dofs(k) = - (*this)(-1-vdofs[k]);
+               }
+            for (j = 0; j < ir.GetNPoints(); j++)
+            {
+               face_elem_transf->Loc2.Transform(ir.IntPoint(j), eip);
+               fe->CalcShape(eip, shape);
+               transf->SetIntPoint(&eip);
+               ell_coeff_val(j) += ell_coeff->Eval(*transf, eip);
+               ell_coeff_val(j) *= 0.5;
+               err_val(j) -= (exsol->Eval(*transf, eip) - (shape * el_dofs));
+            }
+         }
+         face_elem_transf = mesh->GetFaceElementTransformations(i, 16);
+         transf = face_elem_transf;
+         for (j = 0; j < ir.GetNPoints(); j++)
+         {
+            const IntegrationPoint &ip = ir.IntPoint(j);
+            transf->SetIntPoint(&ip);
+            error += (ip.weight * Nu * ell_coeff_val(j) *
+                      pow(transf->Weight(), 1.0-1.0/(dim-1)) *
+                      err_val(j) * err_val(j));
          }
       }
-      face_elem_transf = mesh->GetFaceElementTransformations(i, 16);
-      transf = face_elem_transf;
-      for (int j = 0; j < ir->GetNPoints(); j++)
-      {
-         const IntegrationPoint &ip = ir->IntPoint(j);
-         transf->SetIntPoint(&ip);
-         error += (ip.weight * Nu * ell_coeff_val(j) *
-                   pow(transf->Weight(), 1.0-1.0/(dim-1)) *
-                   err_val(j) * err_val(j));
-      }
+
+   if (error < 0.0)
+   {
+      return -sqrt(-error);
    }
-
-   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
-}
-
-double GridFunction::ComputeH1Error(Coefficient *exsol,
-                                    VectorCoefficient *exgrad,
-                                    Coefficient *ell_coef, double Nu,
-                                    int norm_type) const
-{
-   double error1 = 0.0;
-   double error2 = 0.0;
-   if (norm_type & 1) { error1 = GridFunction::ComputeGradError(exgrad); }
-   if (norm_type & 2) { error2 = GridFunction::ComputeDGFaceJumpError(exsol,ell_coef,Nu); }
-
-   return sqrt(error1 * error1 + error2 * error2);
-}
-
-double GridFunction::ComputeH1Error(Coefficient *exsol,
-                                    VectorCoefficient *exgrad,
-                                    const IntegrationRule *irs[]) const
-{
-   double L2error = GridFunction::ComputeLpError(2.0,*exsol,NULL,irs);
-   double GradError = ComputeGradError(exgrad,irs);
-   return sqrt(L2error*L2error + GradError*GradError);
-}
-
-double GridFunction::ComputeHDivError(VectorCoefficient *exsol,
-                                      Coefficient *exdiv,
-                                      const IntegrationRule *irs[]) const
-{
-   double L2error = GridFunction::ComputeLpError(2.0,*exsol,NULL,NULL,irs);
-   double DivError = ComputeDivError(exdiv,irs);
-   return sqrt(L2error*L2error + DivError*DivError);
-}
-
-double GridFunction::ComputeHCurlError(VectorCoefficient *exsol,
-                                       VectorCoefficient *excurl,
-                                       const IntegrationRule *irs[]) const
-{
-   double L2error = GridFunction::ComputeLpError(2.0,*exsol,NULL,NULL,irs);
-   double CurlError = ComputeCurlError(excurl,irs);
-   return sqrt(L2error*L2error + CurlError*CurlError);
+   return sqrt(error);
 }
 
 double GridFunction::ComputeMaxError(
@@ -2977,6 +2855,7 @@ double GridFunction::ComputeMaxError(
          }
       }
    }
+
    return error;
 }
 

fem/gridfunc.hpp

@@ -334,11 +334,6 @@ public:
    void ImposeBounds(int i, const Vector &weights,
                      double _min = 0.0, double _max = infinity());
 
-   /** On a non-conforming mesh, make sure the function lies in the conforming
-       space by multiplying with R and then with P, the conforming restriction
-       and prolongation matrices of the space, respectively. */
-   void RestrictConforming();
-
    /** @brief Project the @a src GridFunction to @a this GridFunction, both of
        which must be on the same mesh. */
    /** The current implementation assumes that all elements use the same
@@ -427,7 +422,6 @@ public:
    virtual void ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
                                              Array<int> &bdr_attr);
 
-
    virtual double ComputeL2Error(Coefficient &exsol,
                                  const IntegrationRule *irs[] = NULL) const
    { return ComputeLpError(2.0, exsol, NULL, irs); }
@@ -439,50 +433,10 @@ public:
                                  const IntegrationRule *irs[] = NULL,
                                  Array<int> *elems = NULL) const;
 
-   /// Returns ||grad u_ex - grad u_h||_L2 for H1 or L2 elements
-   virtual double ComputeGradError(VectorCoefficient *exgrad,
-                                   const IntegrationRule *irs[] = NULL) const;
-
-   /// Returns ||curl u_ex - curl u_h||_L2 for ND elements
-   virtual double ComputeCurlError(VectorCoefficient *excurl,
-                                   const IntegrationRule *irs[] = NULL) const;
-
-   /// Returns ||div u_ex - div u_h||_L2 for RT elements
-   virtual double ComputeDivError(Coefficient *exdiv,
-                                  const IntegrationRule *irs[] = NULL) const;
-
-   /// Returns the Face Jumps error for L2 elements
-   virtual double ComputeDGFaceJumpError(Coefficient *exsol,
-                                         Coefficient *ell_coeff,
-                                         double Nu,
-                                         const IntegrationRule *irs[] = NULL)
-   const;
-
-   /** This method is kept for backward compatibility.
-
-       Returns either the H1-seminorm, or the DG face jumps error, or both
-       depending on norm_type = 1, 2, 3. Additional arguments for the DG face
-       jumps norm: ell_coeff: mesh-depended coefficient (weight) Nu: scalar
-       constant weight */
    virtual double ComputeH1Error(Coefficient *exsol, VectorCoefficient *exgrad,
                                  Coefficient *ell_coef, double Nu,
                                  int norm_type) const;
 
-   /// Returns the error measured in H1-norm for H1 elements or in "broken"
-   /// H1-norm for L2 elements
-   virtual double ComputeH1Error(Coefficient *exsol, VectorCoefficient *exgrad,
-                                 const IntegrationRule *irs[] = NULL) const;
-
-   /// Returns the error measured in H(div)-norm for RT elements
-   virtual double ComputeHDivError(VectorCoefficient *exsol,
-                                   Coefficient *exdiv,
-                                   const IntegrationRule *irs[] = NULL) const;
-
-   /// Returns the error measured in H(curl)-norm for ND elements
-   virtual double ComputeHCurlError(VectorCoefficient *exsol,
-                                    VectorCoefficient *excurl,
-                                    const IntegrationRule *irs[] = NULL) const;
-
    virtual double ComputeMaxError(Coefficient &exsol,
                                   const IntegrationRule *irs[] = NULL) const
    {

fem/gslib.cpp

@@ -29,13 +29,10 @@ namespace mfem
 {
 
 FindPointsGSLIB::FindPointsGSLIB()
-   : mesh(NULL), meshsplit(NULL), ir_simplex(NULL),
-     fdata2D(NULL), fdata3D(NULL), cr(NULL), gsl_comm(NULL),
-     dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
-     avgtype(AvgType::ARITHMETIC)
+   : mesh(NULL), ir_simplex(NULL), fdata2D(NULL), fdata3D(NULL),
+     dim(-1), gsl_mesh(), gsl_ref(), gsl_dist(), setupflag(false)
 {
    gsl_comm = new comm;
-   cr       = new crystal;
 #ifdef MFEM_USE_MPI
    int initialized;
    MPI_Initialized(&initialized);
@@ -50,20 +47,15 @@ FindPointsGSLIB::FindPointsGSLIB()
 FindPointsGSLIB::~FindPointsGSLIB()
 {
    delete gsl_comm;
-   delete cr;
    delete ir_simplex;
-   delete meshsplit;
 }
 
 #ifdef MFEM_USE_MPI
 FindPointsGSLIB::FindPointsGSLIB(MPI_Comm _comm)
-   : mesh(NULL), meshsplit(NULL), ir_simplex(NULL),
-     fdata2D(NULL), fdata3D(NULL), cr(NULL), gsl_comm(NULL),
-     dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
-     avgtype(AvgType::ARITHMETIC)
+   : mesh(NULL), ir_simplex(NULL), fdata2D(NULL), fdata3D(NULL),
+     dim(-1), gsl_mesh(), gsl_ref(), gsl_dist(), setupflag(false)
 {
    gsl_comm = new comm;
-   cr      = new crystal;
    comm_init(gsl_comm, _comm);
 }
 #endif
@@ -78,7 +70,6 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
    // call FreeData if FindPointsGSLIB::Setup has been called already
    if (setupflag) { FreeData(); }
 
-   crystal_init(cr, gsl_comm);
    mesh = &m;
    dim  = mesh->Dimension();
    const FiniteElement *fe = mesh->GetNodalFESpace()->GetFE(0);
@@ -122,16 +113,14 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
    setupflag = true;
 }
 
-void FindPointsGSLIB::FindPoints(const Vector &point_pos)
+void FindPointsGSLIB::FindPoints(const Vector &point_pos,
+                                 Array<unsigned int> &codes,
+                                 Array<unsigned int> &proc_ids,
+                                 Array<unsigned int> &elem_ids,
+                                 Vector &ref_pos, Vector &dist)
 {
    MFEM_VERIFY(setupflag, "Use FindPointsGSLIB::Setup before finding points.");
-   points_cnt = point_pos.Size() / dim;
-   gsl_code.SetSize(points_cnt);
-   gsl_proc.SetSize(points_cnt);
-   gsl_elem.SetSize(points_cnt);
-   gsl_ref.SetSize(points_cnt * dim);
-   gsl_dist.SetSize(points_cnt);
-
+   const int points_cnt = point_pos.Size() / dim;
    if (dim == 2)
    {
       const double *xv_base[2];
@@ -140,11 +129,11 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos)
       unsigned xv_stride[2];
       xv_stride[0] = sizeof(double);
       xv_stride[1] = sizeof(double);
-      findpts_2(gsl_code.GetData(), sizeof(unsigned int),
-                gsl_proc.GetData(), sizeof(unsigned int),
-                gsl_elem.GetData(), sizeof(unsigned int),
-                gsl_ref.GetData(),  sizeof(double) * dim,
-                gsl_dist.GetData(), sizeof(double),
+      findpts_2(codes.GetData(), sizeof(unsigned int),
+                proc_ids.GetData(), sizeof(unsigned int),
+                elem_ids.GetData(), sizeof(unsigned int),
+                ref_pos.GetData(), sizeof(double) * dim,
+                dist.GetData(), sizeof(double),
                 xv_base, xv_stride, points_cnt, fdata2D);
    }
    else
@@ -157,27 +146,25 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos)
       xv_stride[0] = sizeof(double);
       xv_stride[1] = sizeof(double);
       xv_stride[2] = sizeof(double);
-      findpts_3(gsl_code.GetData(), sizeof(unsigned int),
-                gsl_proc.GetData(), sizeof(unsigned int),
-                gsl_elem.GetData(), sizeof(unsigned int),
-                gsl_ref.GetData(),  sizeof(double) * dim,
-                gsl_dist.GetData(), sizeof(double),
+      findpts_3(codes.GetData(), sizeof(unsigned int),
+                proc_ids.GetData(), sizeof(unsigned int),
+                elem_ids.GetData(), sizeof(unsigned int),
+                ref_pos.GetData(), sizeof(double) * dim,
+                dist.GetData(), sizeof(double),
                 xv_base, xv_stride, points_cnt, fdata3D);
    }
+}
 
-   // Set the element number and reference position to 0 for points not found
-   for (int i = 0; i < points_cnt; i++)
-   {
-      if (gsl_code[i] == 2)
-      {
-         gsl_elem[i] = 0;
-         for (int d = 0; d < dim; d++) { gsl_ref(i*dim + d) = -1.; }
-      }
-   }
+void FindPointsGSLIB::FindPoints(const Vector &point_pos)
+{
+   const int points_cnt = point_pos.Size() / dim;
+   gsl_code.SetSize(points_cnt);
+   gsl_proc.SetSize(points_cnt);
+   gsl_elem.SetSize(points_cnt);
+   gsl_ref.SetSize(points_cnt * dim);
+   gsl_dist.SetSize(points_cnt);
 
-   // Map element number for simplices, and ref_pos from [-1,1] to [0,1] for
-   // both simplices and quads.
-   MapRefPosAndElemIndices();
+   FindPoints(point_pos, gsl_code, gsl_proc, gsl_elem, gsl_ref, gsl_dist);
 }
 
 void FindPointsGSLIB::FindPoints(Mesh &m, const Vector &point_pos,
@@ -191,24 +178,72 @@ void FindPointsGSLIB::FindPoints(Mesh &m, const Vector &point_pos,
    FindPoints(point_pos);
 }
 
+void FindPointsGSLIB::Interpolate(Array<unsigned int> &codes,
+                                  Array<unsigned int> &proc_ids,
+                                  Array<unsigned int> &elem_ids,
+                                  Vector &ref_pos, const GridFunction &field_in,
+                                  Vector &field_out)
+{
+
+   FiniteElementSpace ind_fes(mesh, field_in.FESpace()->FEColl());
+   GridFunction field_in_scalar(&ind_fes);
+   Vector node_vals;
+
+   const int ncomp      = field_in.FESpace()->GetVDim(),
+             points_fld = field_in.Size() / ncomp,
+             points_cnt = codes.Size();
+   field_out.SetSize(points_cnt*ncomp);
+
+   for (int i = 0; i < ncomp; i++)
+   {
+      const int dataptrin  = i*points_fld,
+                dataptrout = i*points_cnt;
+      field_in_scalar.NewDataAndSize(field_in.GetData()+dataptrin, points_fld);
+      GetNodeValues(field_in_scalar, node_vals);
+
+      if (dim==2)
+      {
+         findpts_eval_2(field_out.GetData()+dataptrout, sizeof(double),
+                        codes.GetData(),       sizeof(unsigned int),
+                        proc_ids.GetData(),    sizeof(unsigned int),
+                        elem_ids.GetData(),    sizeof(unsigned int),
+                        ref_pos.GetData(),     sizeof(double) * dim,
+                        points_cnt, node_vals.GetData(), fdata2D);
+      }
+      else
+      {
+         findpts_eval_3(field_out.GetData()+dataptrout, sizeof(double),
+                        codes.GetData(),       sizeof(unsigned int),
+                        proc_ids.GetData(),    sizeof(unsigned int),
+                        elem_ids.GetData(),    sizeof(unsigned int),
+                        ref_pos.GetData(),     sizeof(double) * dim,
+                        points_cnt, node_vals.GetData(), fdata3D);
+      }
+   }
+}
+
+void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
+                                  Vector &field_out)
+{
+   Interpolate(gsl_code, gsl_proc, gsl_elem, gsl_ref, field_in, field_out);
+}
+
 void FindPointsGSLIB::Interpolate(const Vector &point_pos,
                                   const GridFunction &field_in, Vector &field_out)
 {
    FindPoints(point_pos);
-   Interpolate(field_in, field_out);
+   Interpolate(gsl_code, gsl_proc, gsl_elem, gsl_ref, field_in, field_out);
 }
 
 void FindPointsGSLIB::Interpolate(Mesh &m, const Vector &point_pos,
                                   const GridFunction &field_in, Vector &field_out)
 {
    FindPoints(m, point_pos);
-   Interpolate(field_in, field_out);
+   Interpolate(gsl_code, gsl_proc, gsl_elem, gsl_ref, field_in, field_out);
 }
 
 void FindPointsGSLIB::FreeData()
 {
-   if (!setupflag) { return; }
-   crystal_free(cr);
    if (dim == 2)
    {
       findpts_free_2(fdata2D);
@@ -217,13 +252,13 @@ void FindPointsGSLIB::FreeData()
    {
       findpts_free_3(fdata3D);
    }
+   setupflag = false;
    gsl_code.DeleteAll();
    gsl_proc.DeleteAll();
    gsl_elem.DeleteAll();
    gsl_mesh.Destroy();
    gsl_ref.Destroy();
    gsl_dist.Destroy();
-   setupflag = false;
 }
 
 void FindPointsGSLIB::GetNodeValues(const GridFunction &gf_in,
@@ -323,8 +358,9 @@ void FindPointsGSLIB::GetSimplexNodalCoordinates()
    const FiniteElement *fe   = mesh->GetNodalFESpace()->GetFE(0);
    const Geometry::Type gt   = fe->GetGeomType();
    const GridFunction *nodes = mesh->GetNodes();
+   Mesh *meshsplit           = NULL;
    const int NE              = mesh->GetNE();
-   int NEsplit = 0;
+   int NEsplit = -1;
 
    // Split the reference element into a reference submesh of quads or hexes.
    if (gt == Geometry::TRIANGLE)
@@ -480,361 +516,8 @@ void FindPointsGSLIB::GetSimplexNodalCoordinates()
          pt_id++;
       }
    }
-}
 
-void FindPointsGSLIB::MapRefPosAndElemIndices()
-{
-   gsl_mfem_ref = gsl_ref;
-   gsl_mfem_elem = gsl_elem;
-   const FiniteElement *fe = mesh->GetNodalFESpace()->GetFE(0);
-   const Geometry::Type gt = fe->GetGeomType();
-   int NEsplit = 0;
-
-   gsl_mfem_ref -= -1.;  // map  [-1, 1] to
-   gsl_mfem_ref *= 0.5;  //      [0,  1]
-   if (gt == Geometry::SQUARE || gt == Geometry::CUBE) { return; }
-
-   H1_FECollection feclin(1, dim);
-   FiniteElementSpace nodal_fes_lin(meshsplit, &feclin, dim);
-   GridFunction gf_lin(&nodal_fes_lin);
-
-   if (gt == Geometry::TRIANGLE)
-   {
-      const double quad_v[7][2] =
-      {
-         {0, 0}, {0.5, 0}, {1, 0}, {0, 0.5},
-         {1./3., 1./3.}, {0.5, 0.5}, {0, 1}
-      };
-      for (int k = 0; k < dim; k++)
-      {
-         for (int j = 0; j < gf_lin.Size()/dim; j++)
-         {
-            gf_lin(j+k*gf_lin.Size()/dim) = quad_v[j][k];
-         }
-      }
-      NEsplit = 3;
-   }
-   else if (gt == Geometry::TETRAHEDRON)
-   {
-      const double hex_v[15][3] =
-      {
-         {0, 0, 0.}, {1, 0., 0.}, {0., 1., 0.}, {0, 0., 1.},
-         {0.5, 0., 0.}, {0.5, 0.5, 0.}, {0., 0.5, 0.},
-         {0., 0., 0.5}, {0.5, 0., 0.5}, {0., 0.5, 0.5},
-         {1./3., 0., 1./3.}, {1./3., 1./3., 1./3.}, {0, 1./3., 1./3.},
-         {1./3., 1./3., 0}, {0.25, 0.25, 0.25}
-      };
-      for (int k = 0; k < dim; k++)
-      {
-         for (int j = 0; j < gf_lin.Size()/dim; j++)
-         {
-            gf_lin(j+k*gf_lin.Size()/dim) = hex_v[j][k];
-         }
-      }
-      NEsplit = 4;
-   }
-   else if (gt == Geometry::PRISM)
-   {
-      const double hex_v[14][3] =
-      {
-         {0, 0, 0}, {0.5, 0, 0}, {1, 0, 0}, {0, 0.5, 0},
-         {1./3., 1./3., 0}, {0.5, 0.5, 0}, {0, 1, 0},
-         {0, 0, 1}, {0.5, 0, 1}, {1, 0, 1}, {0, 0.5, 1},
-         {1./3., 1./3., 1}, {0.5, 0.5, 1}, {0, 1, 1}
-      };
-      for (int k = 0; k < dim; k++)
-      {
-         for (int j = 0; j < gf_lin.Size()/dim; j++)
-         {
-            gf_lin(j+k*gf_lin.Size()/dim) = hex_v[j][k];
-         }
-      }
-      NEsplit = 3;
-   }
-   else
-   {
-      MFEM_ABORT("Element type not currently supported.");
-   }
-
-   // Simplices are split into quads/hexes for GSLIB. For MFEM, we need to find
-   // the original element number and map the rst from micro to macro element.
-   for (int i = 0; i < points_cnt; i++)
-   {
-      if (gsl_code[i] == 2) { continue; }
-      int local_elem   = gsl_elem[i]%NEsplit;
-      gsl_mfem_elem[i] = (gsl_elem[i] - local_elem)/NEsplit; // macro element number
-
-      IntegrationPoint ip;
-      Vector mfem_ref(gsl_mfem_ref.GetData()+i*dim, dim);
-      ip.Set2(mfem_ref.GetData());
-      if (dim == 3) { ip.z = mfem_ref(2); }
-      gf_lin.GetVectorValue(local_elem, ip, mfem_ref); // map to rst of macro element
-   }
-}
-
-void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
-                                  Vector &field_out)
-{
-   const int  gf_order   = field_in.FESpace()->GetFE(0)->GetOrder(),
-              mesh_order = mesh->GetNodalFESpace()->GetFE(0)->GetOrder();
-
-   const FiniteElementCollection *fec_in =  field_in.FESpace()->FEColl();
-   const H1_FECollection *fec_h1 = dynamic_cast<const H1_FECollection *>(fec_in);
-   const L2_FECollection *fec_l2 = dynamic_cast<const L2_FECollection *>(fec_in);
-
-   if (fec_h1 && gf_order == mesh_order &&
-       fec_h1->GetBasisType() == BasisType::GaussLobatto)
-   {
-      InterpolateH1(field_in, field_out);
-      return;
-   }
-   else
-   {
-      InterpolateGeneral(field_in, field_out);
-      if (!fec_l2 || avgtype == AvgType::NONE) { return; }
-   }
-
-   // For points on element borders, project the L2 GridFunction to H1 and
-   // re-interpolate.
-   if (fec_l2)
-   {
-      Array<int> indl2;
-      for (int i = 0; i < points_cnt; i++)
-      {
-         if (gsl_code[i] == 1) { indl2.Append(i); }
-      }
-      if (indl2.Size() == 0) { return; } // no points on element borders
-
-      Vector field_out_l2(field_out.Size());
-      VectorGridFunctionCoefficient field_in_dg(&field_in);
-      int gf_order_h1 = std::max(gf_order, 1); // H1 should be at least order 1
-      H1_FECollection fec(gf_order_h1, dim);
-      const int ncomp = field_in.FESpace()->GetVDim();
-      FiniteElementSpace fes(mesh, &fec, ncomp);
-      GridFunction field_in_h1(&fes);
-
-      if (avgtype == AvgType::ARITHMETIC)
-      {
-         field_in_h1.ProjectDiscCoefficient(field_in_dg, GridFunction::ARITHMETIC);
-      }
-      else if (avgtype == AvgType::HARMONIC)
-      {
-         field_in_h1.ProjectDiscCoefficient(field_in_dg, GridFunction::HARMONIC);
-      }
-      else
-      {
-         MFEM_ABORT("Invalid averaging type.");
-      }
-
-      if (gf_order_h1 == mesh_order) // basis is GaussLobatto by default
-      {
-         InterpolateH1(field_in_h1, field_out_l2);
-      }
-      else
-      {
-         InterpolateGeneral(field_in_h1, field_out_l2);
-      }
-
-      // Copy interpolated values for the points on element border
-      for (int j = 0; j < ncomp; j++)
-      {
-         for (int i = 0; i < indl2.Size(); i++)
-         {
-            int idx = indl2[i] + j*points_cnt;
-            field_out(idx) = field_out_l2(idx);
-         }
-      }
-   }
-}
-
-void FindPointsGSLIB::InterpolateH1(const GridFunction &field_in,
-                                    Vector &field_out)
-{
-   FiniteElementSpace ind_fes(mesh, field_in.FESpace()->FEColl());
-   GridFunction field_in_scalar(&ind_fes);
-   Vector node_vals;
-
-   const int ncomp      = field_in.FESpace()->GetVDim(),
-             points_fld = field_in.Size() / ncomp,
-             points_cnt = gsl_code.Size();
-
-   field_out.SetSize(points_cnt*ncomp);
-   field_out = default_interp_value;
-
-   for (int i = 0; i < ncomp; i++)
-   {
-      const int dataptrin  = i*points_fld,
-                dataptrout = i*points_cnt;
-      field_in_scalar.NewDataAndSize(field_in.GetData()+dataptrin, points_fld);
-      GetNodeValues(field_in_scalar, node_vals);
-
-      if (dim==2)
-      {
-         findpts_eval_2(field_out.GetData()+dataptrout, sizeof(double),
-                        gsl_code.GetData(),       sizeof(unsigned int),
-                        gsl_proc.GetData(),    sizeof(unsigned int),
-                        gsl_elem.GetData(),    sizeof(unsigned int),
-                        gsl_ref.GetData(),     sizeof(double) * dim,
-                        points_cnt, node_vals.GetData(), fdata2D);
-      }
-      else
-      {
-         findpts_eval_3(field_out.GetData()+dataptrout, sizeof(double),
-                        gsl_code.GetData(),       sizeof(unsigned int),
-                        gsl_proc.GetData(),    sizeof(unsigned int),
-                        gsl_elem.GetData(),    sizeof(unsigned int),
-                        gsl_ref.GetData(),     sizeof(double) * dim,
-                        points_cnt, node_vals.GetData(), fdata3D);
-      }
-   }
-}
-
-void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
-                                         Vector &field_out)
-{
-   int ncomp   = field_in.VectorDim(),
-       nptorig = points_cnt,
-       npt     = points_cnt;
-
-   field_out.SetSize(points_cnt*ncomp);
-   field_out = default_interp_value;
-
-   if (gsl_comm->np == 1) // serial
-   {
-      for (int index = 0; index < npt; index++)
-      {
-         if (gsl_code[index] == 2) { continue; }
-         IntegrationPoint ip;
-         ip.Set2(gsl_mfem_ref.GetData()+index*dim);
-         if (dim == 3) { ip.z = gsl_mfem_ref(index*dim + 2); }
-         Vector localval(ncomp);
-         field_in.GetVectorValue(gsl_mfem_elem[index], ip, localval);
-         for (int i = 0; i < ncomp; i++)
-         {
-            field_out(index + i*npt) = localval(i);
-         }
-      }
-   }
-   else // parallel
-   {
-      // Determine number of points to be sent
-      int nptsend = 0;
-      for (int index = 0; index < npt; index++)
-      {
-         if (gsl_code[index] != 2) { nptsend +=1; }
-      }
-
-      // Pack data to send via crystal router
-      struct array *outpt = new array;
-      struct out_pt { double r[3], ival; uint index, el, proc; };
-      struct out_pt *pt;
-      array_init(struct out_pt, outpt, nptsend);
-      outpt->n=nptsend;
-      pt = (struct out_pt *)outpt->ptr;
-      for (int index = 0; index < npt; index++)
-      {
-         if (gsl_code[index] == 2) { continue; }
-         for (int d = 0; d < dim; ++d) { pt->r[d]= gsl_mfem_ref(index*dim + d); }
-         pt->index = index;
-         pt->proc  = gsl_proc[index];
-         pt->el    = gsl_mfem_elem[index];
-         ++pt;
-      }
-
-      // Transfer data to target MPI ranks
-      sarray_transfer(struct out_pt, outpt, proc, 1, cr);
-
-      if (ncomp == 1)
-      {
-         // Interpolate the grid function
-         npt = outpt->n;
-         pt = (struct out_pt *)outpt->ptr;
-         for (int index = 0; index < npt; index++)
-         {
-            IntegrationPoint ip;
-            ip.Set3(&pt->r[0]);
-            pt->ival = field_in.GetValue(pt->el, ip, 1);
-            ++pt;
-         }
-
-         // Transfer data back to source MPI rank
-         sarray_transfer(struct out_pt, outpt, proc, 1, cr);
-         npt = outpt->n;
-         pt = (struct out_pt *)outpt->ptr;
-         for (int index = 0; index < npt; index++)
-         {
-            field_out(pt->index) = pt->ival;
-            ++pt;
-         }
-         array_free(outpt);
-         delete outpt;
-      }
-      else // ncomp > 1
-      {
-         // Interpolate data and store in a Vector
-         npt = outpt->n;
-         pt = (struct out_pt *)outpt->ptr;
-         Vector vec_int_vals(npt*ncomp);
-         for (int index = 0; index < npt; index++)
-         {
-            IntegrationPoint ip;
-            ip.Set3(&pt->r[0]);
-            Vector localval(vec_int_vals.GetData()+index*ncomp, ncomp);
-            field_in.GetVectorValue(pt->el, ip, localval);
-            ++pt;
-         }
-
-         // Save index and proc data in a struct
-         struct array *savpt = new array;
-         struct sav_pt { uint index, proc; };
-         struct sav_pt *spt;
-         array_init(struct sav_pt, savpt, npt);
-         savpt->n=npt;
-         spt = (struct sav_pt *)savpt->ptr;
-         pt  = (struct out_pt *)outpt->ptr;
-         for (int index = 0; index < npt; index++)
-         {
-            spt->index = pt->index;
-            spt->proc  = pt->proc;
-            ++pt; ++spt;
-         }
-
-         array_free(outpt);
-         delete outpt;
-
-         // Copy data from save struct to send struct and send component wise
-         struct array *sendpt = new array;
-         struct send_pt { double ival; uint index, proc; };
-         struct send_pt *sdpt;
-         for (int j = 0; j < ncomp; j++)
-         {
-            array_init(struct send_pt, sendpt, npt);
-            sendpt->n=npt;
-            spt  = (struct sav_pt *)savpt->ptr;
-            sdpt = (struct send_pt *)sendpt->ptr;
-            for (int index = 0; index < npt; index++)
-            {
-               sdpt->index = spt->index;
-               sdpt->proc  = spt->proc;
-               sdpt->ival  = vec_int_vals(j + index*ncomp);
-               ++sdpt; ++spt;
-            }
-
-            sarray_transfer(struct send_pt, sendpt, proc, 1, cr);
-            sdpt = (struct send_pt *)sendpt->ptr;
-            for (int index = 0; index < nptorig; index++)
-            {
-               int idx = sdpt->index + j*nptorig;
-               field_out(idx) = sdpt->ival;
-               ++sdpt;
-            }
-            array_free(sendpt);
-         }
-         array_free(savpt);
-         delete sendpt;
-         delete savpt;
-      } // ncomp > 1
-   } // parallel
+   delete meshsplit;
 }
 
 } // namespace mfem

fem/gslib.hpp

@@ -20,66 +20,28 @@
 struct comm;
 struct findpts_data_2;
 struct findpts_data_3;
-struct array;
-struct crystal;
 
 namespace mfem
 {
 
-/** \brief FindPointsGSLIB can robustly evaluate a GridFunction on an arbitrary
- *  collection of points. There are three key functions in FindPointsGSLIB:
- *
- *  1. Setup - constructs the internal data structures of gslib.
- *
- *  2. FindPoints - for any given arbitrary set of points in physical space,
- *     gslib finds the element number, MPI rank, and the reference space
- *     coordinates inside the element that each point is located in. gslib also
- *     returns a code that indicates whether the point was found inside an
- *     element, on element border, or not found in the domain.
- *
- *  3. Interpolate - Interpolates any grid function at the points found using 2.
- *
- *  FindPointsGSLIB provides interface to use these functions individually or
- *  using a single call.
- */
 class FindPointsGSLIB
 {
-public:
-   enum AvgType {NONE, ARITHMETIC, HARMONIC}; // Average type for L2 functions
-
 protected:
-   Mesh *mesh, *meshsplit;
-   IntegrationRule *ir_simplex;    // IntegrationRule to split quads/hex -> simplex
-   struct findpts_data_2 *fdata2D; // gslib's internal data
-   struct findpts_data_3 *fdata3D; // gslib's internal data
-   struct crystal *cr;             // gslib's internal data
-   struct comm *gsl_comm;          // gslib's internal data
-   int dim, points_cnt;
-   Array<unsigned int> gsl_code, gsl_proc, gsl_elem, gsl_mfem_elem;
-   Vector gsl_mesh, gsl_ref, gsl_dist, gsl_mfem_ref;
-   bool setupflag;              // flag to indicate whether gslib data has been setup
-   double default_interp_value; // used for points that are not found in the mesh
-   AvgType avgtype;             // average type used for L2 functions
+   Mesh *mesh;
+   IntegrationRule *ir_simplex;
+   struct findpts_data_2 *fdata2D;
+   struct findpts_data_3 *fdata3D;
+   int dim;
+   Array<unsigned int> gsl_code, gsl_proc, gsl_elem;
+   Vector gsl_mesh, gsl_ref, gsl_dist;
+   bool setupflag;
+
+   struct comm *gsl_comm;
 
-   /// Get GridFunction from MFEM format to GSLIB format
    void GetNodeValues(const GridFunction &gf_in, Vector &node_vals);
-   /// Get nodal coordinates from mesh to the format expected by GSLIB for quads
-   /// and hexes
    void GetQuadHexNodalCoordinates();
-   /// Convert simplices to quad/hexes and then get nodal coordinates for each
-   /// split element into format expected by GSLIB
    void GetSimplexNodalCoordinates();
 
-   /// Use GSLIB for communication and interpolation
-   void InterpolateH1(const GridFunction &field_in, Vector &field_out);
-   /// Uses GSLIB Crystal Router for communication followed by MFEM's
-   /// interpolation functions
-   void InterpolateGeneral(const GridFunction &field_in, Vector &field_out);
-   /// Map {r,s,t} coordinates from [-1,1] to [0,1] for MFEM. For simplices mesh
-   /// find the original element number (that was split into micro quads/hexes
-   /// by GetSimplexNodalCoordinates())
-   void MapRefPosAndElemIndices();
-
 public:
    FindPointsGSLIB();
 
@@ -102,37 +64,45 @@ public:
    void Setup(Mesh &m, const double bb_t = 0.1, const double newt_tol = 1.0e-12,
               const int npt_max = 256);
 
-   /** Searches positions given in physical space by @a point_pos. These positions
-       must by ordered by nodes: (XXX...,YYY...,ZZZ).
-       This function populates the following member variables:
-       #gsl_code        Return codes for each point: inside element (0),
-                        element boundary (1), not found (2).
-       #gsl_proc        MPI proc ids where the points were found.
-       #gsl_elem        Element ids where the points were found.
-                        Defaults to 0 for points that were not found.
-       #gsl_mfem_elem   Element ids corresponding to MFEM-mesh where the points
-                        were found. #gsl_mfem_elem != #gsl_elem for simplices
-                        Defaults to 0 for points that were not found.
-       #gsl_ref         Reference coordinates of the found point.
-                        Ordered by vdim (XYZ,XYZ,XYZ...). Defaults to -1 for
-                        points that were not found. Note: the gslib reference
-                        frame is [-1,1].
-       #gsl_mfem_ref    Reference coordinates #gsl_ref mapped to [0,1].
-                        Defaults to 0 for points that were not found.
-       #gsl_dist        Distance between the sought and the found point
-                        in physical space. */
+   /** Searches positions given in physical space by @a point_pos. All output
+       Arrays and Vectors are expected to have the correct size.
+
+       @param[in]  point_pos  Positions to be found. Must by ordered by nodes
+                              (XXX...,YYY...,ZZZ).
+       @param[out] codes      Return codes for each point: inside element (0),
+                              element boundary (1), not found (2).
+       @param[out] proc_ids   MPI proc ids where the points were found.
+       @param[out] elem_ids   Element ids where the points were found.
+       @param[out] ref_pos    Reference coordinates of the found point. Ordered
+                              by vdim (XYZ,XYZ,XYZ...).
+                              Note: the gslib reference frame is [-1,1].
+       @param[out] dist       Distance between the sought and the found point
+                              in physical space. */
+   void FindPoints(const Vector &point_pos, Array<unsigned int> &codes,
+                   Array<unsigned int> &proc_ids, Array<unsigned int> &elem_ids,
+                   Vector &ref_pos, Vector &dist);
    void FindPoints(const Vector &point_pos);
    /// Setup FindPoints and search positions
    void FindPoints(Mesh &m, const Vector &point_pos, const double bb_t = 0.1,
                    const double newt_tol = 1.0e-12,  const int npt_max = 256);
 
    /** Interpolation of field values at prescribed reference space positions.
+
+       @param[in] codes       Return codes for each point: inside element (0),
+                              element boundary (1), not found (2).
+       @param[in] proc_ids    MPI proc ids where the points were found.
+       @param[in] elem_ids    Element ids where the points were found.
+       @param[in] ref_pos     Reference coordinates of the found point. Ordered
+                              by vdim (XYZ,XYZ,XYZ...).
+                              Note: the gslib reference frame is [-1,1].
        @param[in] field_in    Function values that will be interpolated on the
                               reference positions. Note: it is assumed that
                               @a field_in is in H1 and in the same space as the
                               mesh that was given to Setup().
-       @param[out] field_out  Interpolated values. For points that are not found
-                              the value is set to #default_interp_value. */
+       @param[out] field_out  Interpolated values. */
+   void Interpolate(Array<unsigned int> &codes, Array<unsigned int> &proc_ids,
+                    Array<unsigned int> &elem_ids, Vector &ref_pos,
+                    const GridFunction &field_in, Vector &field_out);
    void Interpolate(const GridFunction &field_in, Vector &field_out);
    /** Search positions and interpolate */
    void Interpolate(const Vector &point_pos, const GridFunction &field_in,
@@ -141,45 +111,27 @@ public:
    void Interpolate(Mesh &m, const Vector &point_pos,
                     const GridFunction &field_in, Vector &field_out);
 
-   /// Average type to be used for L2 functions in-case a point is located at
-   /// an element boundary where the function might be multi-valued.
-   void SetL2AvgType(AvgType avgtype_) { avgtype = avgtype_; }
-
-   /// Set the default interpolation value for points that are not found in the
-   /// mesh.
-   void SetDefaultInterpolationValue(double interp_value_)
-   {
-      default_interp_value = interp_value_;
-   }
-
    /** Cleans up memory allocated internally by gslib.
-       Note that in parallel, this must be called before MPI_Finalize(), as it
-       calls MPI_Comm_free() for internal gslib communicators. */
+       Note that in parallel, this must be called before MPI_Finalize(), as
+       it calls MPI_Comm_free() for internal gslib communicators. */
    void FreeData();
 
    /// Return code for each point searched by FindPoints: inside element (0), on
    /// element boundary (1), or not found (2).
    const Array<unsigned int> &GetCode() const { return gsl_code; }
    /// Return element number for each point found by FindPoints.
-   const Array<unsigned int> &GetElem() const { return gsl_mfem_elem; }
+   const Array<unsigned int> &GetElem() const { return gsl_elem; }
    /// Return MPI rank on which each point was found by FindPoints.
    const Array<unsigned int> &GetProc() const { return gsl_proc; }
    /// Return reference coordinates for each point found by FindPoints.
-   const Vector &GetReferencePosition() const { return gsl_mfem_ref;  }
+   const Vector &GetReferencePosition() const { return gsl_ref;  }
    /// Return distance Distance between the sought and the found point
    /// in physical space, for each point found by FindPoints.
    const Vector &GetDist()              const { return gsl_dist; }
-
-   /// Return element number for each point found by FindPoints corresponding to
-   /// GSLIB mesh. gsl_mfem_elem != gsl_elem for mesh with simplices.
-   const Array<unsigned int> &GetGSLIBElem() const { return gsl_elem; }
-   /// Return reference coordinates in [-1,1] (internal range in GSLIB) for each
-   /// point found by FindPoints.
-   const Vector &GetGSLIBReferencePosition() const { return gsl_ref; }
 };
 
 } // namespace mfem
 
-#endif // MFEM_USE_GSLIB
+#endif //MFEM_USE_GSLIB
 
-#endif // MFEM_GSLIB
+#endif //MFEM_GSLIB guard

fem/libceed/ceed.cpp

@@ -35,9 +35,6 @@ extern Ceed ceed;
 
 std::string ceed_path;
 
-extern CeedBasisMap ceed_basis_map;
-extern CeedRestrMap ceed_restr_map;
-
 }
 
 void InitCeedCoeff(Coefficient* Q, CeedData* ptr)
@@ -84,9 +81,10 @@ static CeedElemTopology GetCeedTopology(Geometry::Type geom)
    }
 }
 
-static void InitCeedNonTensorBasis(const FiniteElementSpace &fes,
-                                   const IntegrationRule &ir,
-                                   Ceed ceed, CeedBasis *basis)
+static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
+                                                 const IntegrationRule &ir,
+                                                 Ceed ceed, CeedBasis *basis,
+                                                 CeedElemRestriction *restr)
 {
    Mesh *mesh = fes.GetMesh();
    const FiniteElement *fe = fes.GetFE(0);
@@ -99,73 +97,7 @@ static void InitCeedNonTensorBasis(const FiniteElementSpace &fes,
    Vector qweight(Q);
    Vector shape_i(P);
    DenseMatrix grad_i(P, dim);
-   const Table &el_dof = fes.GetElementToDofTable();
-   Array<int> tp_el_dof(el_dof.Size_of_connections());
-   const TensorBasisElement * tfe =
-      dynamic_cast<const TensorBasisElement *>(fe);
-   if (tfe) // Lexicographic ordering using dof_map
-   {
-      const Array<int>& dof_map = tfe->GetDofMap();
-      for (int i = 0; i < Q; i++)
-      {
-         const IntegrationPoint &ip = ir.IntPoint(i);
-         qref(0,i) = ip.x;
-         if (dim>1) { qref(1,i) = ip.y; }
-         if (dim>2) { qref(2,i) = ip.z; }
-         qweight(i) = ip.weight;
-         fe->CalcShape(ip, shape_i);
-         fe->CalcDShape(ip, grad_i);
-         for (int j = 0; j < P; j++)
-         {
-            shape(j, i) = shape_i(dof_map[j]);
-            for (int d = 0; d < dim; ++d)
-            {
-               grad(j+i*P+d*Q*P) = grad_i(dof_map[j], d);
-            }
-         }
-      }
-   }
-   else  // Native ordering
-   {
-      for (int i = 0; i < Q; i++)
-      {
-         const IntegrationPoint &ip = ir.IntPoint(i);
-         qref(0,i) = ip.x;
-         if (dim>1) { qref(1,i) = ip.y; }
-         if (dim>2) { qref(2,i) = ip.z; }
-         qweight(i) = ip.weight;
-         fe->CalcShape(ip, shape_i);
-         fe->CalcDShape(ip, grad_i);
-         for (int j = 0; j < P; j++)
-         {
-            shape(j, i) = shape_i(j);
-            for (int d = 0; d < dim; ++d)
-            {
-               grad(j+i*P+d*Q*P) = grad_i(j, d);
-            }
-         }
-      }
-   }
-   CeedBasisCreateH1(ceed, GetCeedTopology(fe->GetGeomType()), fes.GetVDim(),
-                     fe->GetDof(), ir.GetNPoints(), shape.GetData(),
-                     grad.GetData(), qref.GetData(), qweight.GetData(), basis);
-}
 
-static void InitCeedNonTensorRestriction(const FiniteElementSpace &fes,
-                                         const IntegrationRule &ir,
-                                         Ceed ceed, CeedElemRestriction *restr)
-{
-   Mesh *mesh = fes.GetMesh();
-   const FiniteElement *fe = fes.GetFE(0);
-   const int dim = mesh->Dimension();
-   const int P = fe->GetDof();
-   const int Q = ir.GetNPoints();
-   DenseMatrix shape(P, Q);
-   Vector grad(P*dim*Q);
-   DenseMatrix qref(dim, Q);
-   Vector qweight(Q);
-   Vector shape_i(P);
-   DenseMatrix grad_i(P, dim);
    CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
    const Table &el_dof = fes.GetElementToDofTable();
    Array<int> tp_el_dof(el_dof.Size_of_connections());
@@ -192,6 +124,7 @@ static void InitCeedNonTensorRestriction(const FiniteElementSpace &fes,
             }
          }
       }
+
       for (int i = 0; i < mesh->GetNE(); i++)
       {
          const int el_offset = fe->GetDof() * i;
@@ -229,6 +162,7 @@ static void InitCeedNonTensorRestriction(const FiniteElementSpace &fes,
             }
          }
       }
+
       for (int e = 0; e < mesh->GetNE(); e++)
       {
          for (int i = 0; i < P; i++)
@@ -244,15 +178,19 @@ static void InitCeedNonTensorRestriction(const FiniteElementSpace &fes,
          }
       }
    }
+   CeedBasisCreateH1(ceed, GetCeedTopology(fe->GetGeomType()), fes.GetVDim(),
+                     fe->GetDof(), ir.GetNPoints(), shape.GetData(),
+                     grad.GetData(), qref.GetData(), qweight.GetData(), basis);
    CeedElemRestrictionCreate(ceed, mesh->GetNE(), fe->GetDof(), fes.GetVDim(),
                              compstride, (fes.GetVDim())*(fes.GetNDofs()),
                              CEED_MEM_HOST, CEED_COPY_VALUES,
                              tp_el_dof.GetData(), restr);
 }
 
-static void InitCeedTensorBasis(const FiniteElementSpace &fes,
-                                const IntegrationRule &ir,
-                                Ceed ceed, CeedBasis *basis)
+static void InitCeedTensorBasisAndRestriction(const FiniteElementSpace &fes,
+                                              const IntegrationRule &ir,
+                                              Ceed ceed, CeedBasis *basis,
+                                              CeedElemRestriction *restr)
 {
    Mesh *mesh = fes.GetMesh();
    const FiniteElement *fe = fes.GetFE(0);
@@ -260,6 +198,7 @@ static void InitCeedTensorBasis(const FiniteElementSpace &fes,
    const TensorBasisElement * tfe =
       dynamic_cast<const TensorBasisElement *>(fe);
    MFEM_VERIFY(tfe, "invalid FE");
+   const Array<int>& dof_map = tfe->GetDofMap();
    const FiniteElement *fe1d =
       fes.FEColl()->FiniteElementForGeometry(Geometry::SEGMENT);
    DenseMatrix shape1d(fe1d->GetDof(), ir.GetNPoints());
@@ -288,28 +227,6 @@ static void InitCeedTensorBasis(const FiniteElementSpace &fes,
                            ir.GetNPoints(), shape1d.GetData(),
                            grad1d.GetData(), qref1d.GetData(),
                            qweight1d.GetData(), basis);
-}
-
-static void InitCeedTensorRestriction(const FiniteElementSpace &fes,
-                                      const IntegrationRule &ir,
-                                      Ceed ceed, CeedElemRestriction *restr)
-{
-   Mesh *mesh = fes.GetMesh();
-   const FiniteElement *fe = fes.GetFE(0);
-   const TensorBasisElement * tfe =
-      dynamic_cast<const TensorBasisElement *>(fe);
-   MFEM_VERIFY(tfe, "invalid FE");
-   const Array<int>& dof_map = tfe->GetDofMap();
-   const FiniteElement *fe1d =
-      fes.FEColl()->FiniteElementForGeometry(Geometry::SEGMENT);
-   DenseMatrix shape1d(fe1d->GetDof(), ir.GetNPoints());
-   DenseMatrix grad1d(fe1d->GetDof(), ir.GetNPoints());
-   Vector qref1d(ir.GetNPoints()), qweight1d(ir.GetNPoints());
-   Vector shape_i(shape1d.Height());
-   DenseMatrix grad_i(grad1d.Height(), 1);
-   const H1_SegmentElement *h1_fe1d =
-      dynamic_cast<const H1_SegmentElement *>(fe1d);
-   MFEM_VERIFY(h1_fe1d, "invalid FE");
 
    CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
    const Table &el_dof = fes.GetElementToDofTable();
@@ -341,52 +258,14 @@ void InitCeedBasisAndRestriction(const FiniteElementSpace &fes,
                                  Ceed ceed, CeedBasis *basis,
                                  CeedElemRestriction *restr)
 {
-   // Check for FES -> basis, restriction in hash tables
-   const Mesh *mesh = fes.GetMesh();
-   const FiniteElement *fe = fes.GetFE(0);
-   const int P = fe->GetDof();
-   const int Q = irm.GetNPoints();
-   const int nelem = mesh->GetNE();
-   const int ncomp = fes.GetVDim();
-   CeedBasisKey basis_key(&fes, &irm, ncomp, P, Q);
-   auto basis_itr = internal::ceed_basis_map.find(basis_key);
-   CeedRestrKey restr_key(&fes, nelem, P, ncomp);
-   auto restr_itr = internal::ceed_restr_map.find(restr_key);
-
-   // Init or retreive key values
-   if (basis_itr == internal::ceed_basis_map.end())
+   if (UsesTensorBasis(fes))
    {
-      if (UsesTensorBasis(fes))
-      {
-         const IntegrationRule &ir = IntRules.Get(Geometry::SEGMENT, irm.GetOrder());
-         InitCeedTensorBasis(fes, ir, ceed, basis);
-      }
-      else
-      {
-         InitCeedNonTensorBasis(fes, irm, ceed, basis);
-      }
-      internal::ceed_basis_map[basis_key] = *basis;
+      const IntegrationRule &ir = IntRules.Get(Geometry::SEGMENT, irm.GetOrder());
+      InitCeedTensorBasisAndRestriction(fes, ir, ceed, basis, restr);
    }
    else
    {
-      *basis = basis_itr->second;
-   }
-   if (restr_itr == internal::ceed_restr_map.end())
-   {
-      if (UsesTensorBasis(fes))
-      {
-         const IntegrationRule &ir = IntRules.Get(Geometry::SEGMENT, irm.GetOrder());
-         InitCeedTensorRestriction(fes, ir, ceed, restr);
-      }
-      else
-      {
-         InitCeedNonTensorRestriction(fes, irm, ceed, restr);
-      }
-      internal::ceed_restr_map[restr_key] = *restr;
-   }
-   else
-   {
-      *restr = restr_itr->second;
+      InitCeedNonTensorBasisAndRestriction(fes, irm, ceed, basis, restr);
    }
 }
 
@@ -448,8 +327,8 @@ void CeedPAAssemble(const CeedPAOperator& op,
    CeedVectorCreate(ceed, nelem * nqpts * qdatasize, &ceedData.rho);
 
    // Context data to be passed to the 'f_build_diff' Q-function.
-   ceedData.build_ctx_data.dim = mesh->Dimension();
-   ceedData.build_ctx_data.space_dim = mesh->SpaceDimension();
+   ceedData.build_ctx.dim = mesh->Dimension();
+   ceedData.build_ctx.space_dim = mesh->SpaceDimension();
 
    std::string qf_file = GetCeedPath() + op.header;
    std::string qf;
@@ -463,7 +342,7 @@ void CeedPAAssemble(const CeedPAOperator& op,
          CeedQFunctionCreateInterior(ceed, 1, op.const_qf,
                                      qf.c_str(),
                                      &ceedData.build_qfunc);
-         ceedData.build_ctx_data.coeff = ((CeedConstCoeff*)ceedData.coeff)->val;
+         ceedData.build_ctx.coeff = ((CeedConstCoeff*)ceedData.coeff)->val;
          break;
       case CeedCoeff::Grid:
          qf = qf_file + op.grid_func;
@@ -479,12 +358,8 @@ void CeedPAAssemble(const CeedPAOperator& op,
    CeedQFunctionAddInput(ceedData.build_qfunc, "weights", 1, CEED_EVAL_WEIGHT);
    CeedQFunctionAddOutput(ceedData.build_qfunc, "qdata", qdatasize,
                           CEED_EVAL_NONE);
-
-   CeedQFunctionContextCreate(ceed, &ceedData.build_ctx);
-   CeedQFunctionContextSetData(ceedData.build_ctx, CEED_MEM_HOST, CEED_USE_POINTER,
-                               sizeof(ceedData.build_ctx_data),
-                               &ceedData.build_ctx_data);
-   CeedQFunctionSetContext(ceedData.build_qfunc, ceedData.build_ctx);
+   CeedQFunctionSetContext(ceedData.build_qfunc, &ceedData.build_ctx,
+                           sizeof(ceedData.build_ctx));
 
    // Create the operator that builds the quadrature data for the operator.
    CeedOperatorCreate(ceed, ceedData.build_qfunc, NULL, NULL,
@@ -524,7 +399,8 @@ void CeedPAAssemble(const CeedPAOperator& op,
    CeedQFunctionAddInput(ceedData.apply_qfunc, "qdata", qdatasize,
                          CEED_EVAL_NONE);
    CeedQFunctionAddOutput(ceedData.apply_qfunc, "v", dimV, op.test_op);
-   CeedQFunctionSetContext(ceedData.apply_qfunc, ceedData.build_ctx);
+   CeedQFunctionSetContext(ceedData.apply_qfunc, &ceedData.build_ctx,
+                           sizeof(ceedData.build_ctx));
 
    // Create the diff operator.
    CeedOperatorCreate(ceed, ceedData.apply_qfunc, NULL, NULL, &ceedData.oper);

fem/libceed/ceed.hpp

@@ -18,9 +18,6 @@
 #include "../../general/device.hpp"
 #include "../../linalg/vector.hpp"
 #include <ceed.h>
-#include <ceed-hash.h>
-#include <tuple>
-#include <unordered_map>
 
 namespace mfem
 {
@@ -30,47 +27,7 @@ class GridFunction;
 class IntegrationRule;
 class Coefficient;
 
-// Hash table for CeedBasis
-using CeedBasisKey =
-   std::tuple<const FiniteElementSpace*, const IntegrationRule*, int, int, int>;
-struct CeedBasisHash
-{
-   std::size_t operator()(const CeedBasisKey& k) const
-   {
-      return CeedHashCombine(CeedHashCombine(CeedHashInt(
-                                                reinterpret_cast<CeedHash64_t>(std::get<0>(k))),
-                                             CeedHashInt(
-                                                reinterpret_cast<CeedHash64_t>(std::get<1>(k)))),
-                             CeedHashCombine(CeedHashCombine(CeedHashInt(std::get<2>(k)),
-                                                             CeedHashInt(std::get<3>(k))),
-                                             CeedHashInt(std::get<4>(k))));
-   }
-};
-using CeedBasisMap =
-   std::unordered_map<const CeedBasisKey, CeedBasis, CeedBasisHash>;
-
-// Hash table for CeedElemRestriction
-using CeedRestrKey = std::tuple<const FiniteElementSpace*, int, int, int>;
-struct CeedRestrHash
-{
-   std::size_t operator()(const CeedRestrKey& k) const
-   {
-      return CeedHashCombine(CeedHashCombine(CeedHashInt(
-                                                reinterpret_cast<CeedHash64_t>(std::get<0>(k))),
-                                             CeedHashInt(std::get<1>(k))),
-                             CeedHashCombine(CeedHashInt(std::get<2>(k)),
-                                             CeedHashInt(std::get<3>(k))));
-   }
-};
-using CeedRestrMap =
-   std::unordered_map<const CeedRestrKey, CeedElemRestriction, CeedRestrHash>;
-
-namespace internal
-{
-extern Ceed ceed; // defined in device.cpp
-extern CeedBasisMap basis_map;
-extern CeedRestrMap restr_map;
-}
+namespace internal { extern Ceed ceed; } // defined in device.cpp
 
 /// A structure used to pass additional data to f_build_diff and f_apply_diff
 struct BuildContext { CeedInt dim, space_dim; CeedScalar coeff; };
@@ -99,8 +56,7 @@ struct CeedData
    CeedVector node_coords, rho;
    CeedCoeff coeff_type;
    void* coeff;
-   CeedQFunctionContext build_ctx;
-   BuildContext build_ctx_data;
+   BuildContext build_ctx;
 
    CeedVector u, v;
 
@@ -108,6 +64,10 @@ struct CeedData
    {
       CeedOperatorDestroy(&build_oper);
       CeedOperatorDestroy(&oper);
+      CeedBasisDestroy(&basis);
+      CeedBasisDestroy(&mesh_basis);
+      CeedElemRestrictionDestroy(&restr);
+      CeedElemRestrictionDestroy(&mesh_restr);
       CeedElemRestrictionDestroy(&restr_i);
       CeedElemRestrictionDestroy(&mesh_restr_i);
       CeedQFunctionDestroy(&apply_qfunc);
@@ -117,6 +77,8 @@ struct CeedData
       if (coeff_type==CeedCoeff::Grid)
       {
          CeedGridCoeff* c = (CeedGridCoeff*)coeff;
+         CeedBasisDestroy(&c->basis);
+         CeedElemRestrictionDestroy(&c->restr);
          CeedVectorDestroy(&c->coeffVector);
          delete c;
       }

fem/linearform.cpp

@@ -204,14 +204,6 @@ void LinearForm::Update(FiniteElementSpace *f, Vector &v, int v_offset)
    ResetDeltaLocations();
 }
 
-void LinearForm::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
-{
-   MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
-   fes = f;
-   v.UseDevice(true);
-   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
-}
-
 void LinearForm::AssembleDelta()
 {
    if (dlfi_delta.Size() == 0) { return; }

fem/linearform.hpp

@@ -26,7 +26,7 @@ protected:
    /// FE space on which the LinearForm lives. Not owned.
    FiniteElementSpace *fes;
 
-   /** @brief Indicates the LinearFormIntegrator%s stored in #dlfi, #dlfi_delta,
+   /** @brief Indicates the LinerFormIntegrator%s stored in #dlfi, #dlfi_delta,
        #blfi, and #flfi are owned by another LinearForm. */
    int extern_lfs;
 
@@ -175,16 +175,6 @@ public:
        @note This method does not perform assembly. */
    void Update(FiniteElementSpace *f, Vector &v, int v_offset);
 
-   /** @brief Make the LinearForm reference external data on a new
-       FiniteElementSpace. */
-   /** This method changes the FiniteElementSpace associated with the LinearForm
-       @a *f and sets the data of the Vector @a v (plus the @a v_offset) as
-       external data in the LinearForm.
-
-       @note This version of the method will also perform bounds checks when the
-       build option MFEM_DEBUG is enabled. */
-   virtual void MakeRef(FiniteElementSpace *f, Vector &v, int v_offset);
-
    /// Return the action of the LinearForm as a linear mapping.
    /** Linear forms are linear functionals which map GridFunctions to
        the real numbers.  This method performs this mapping which in

fem/lininteg.cpp

@@ -457,8 +457,20 @@ void VectorFEDomainLFCurlIntegrator::AssembleRHSElementVect(
 
       Tr.SetIntPoint (&ip);
       el.CalcPhysCurlShape(Tr, curlshape);
-      QF->Eval(vec, Tr, ip);
 
+      switch (spaceDim)
+      {
+         case 3:
+            MFEM_VERIFY(QF, "VectorFunctionCoefficient not provided");
+            QF->Eval(vec, Tr, ip);
+            break;
+         case 2:
+            MFEM_VERIFY(Q, "FunctionCoefficient (Scalar) not provided");
+            vec[0] = Q->Eval(Tr, ip);
+            break;
+         default:
+            break; // This should be unreachable
+      }
       vec *= ip.weight * Tr.Weight();
       curlshape.AddMult (vec, elvect);
    }
@@ -468,17 +480,38 @@ void VectorFEDomainLFCurlIntegrator::AssembleDeltaElementVect(
    const FiniteElement &fe, ElementTransformation &Trans, Vector &elvect)
 {
    int spaceDim = Trans.GetSpaceDim();
-   MFEM_ASSERT(vec_delta != NULL,
-               "coefficient must be VectorDeltaCoefficient");
+   switch (spaceDim)
+   {
+      case 3:
+         MFEM_ASSERT(vec_delta != NULL,
+                     "coefficient must be VectorDeltaCoefficient");
+         break;
+      case 2:
+         MFEM_ASSERT(delta != NULL,
+                     "coefficient must be DeltaCoefficient");
+         break;
+      default:
+         break; // This should be unreachable
+   }
    int dof = fe.GetDof();
    int n=(spaceDim == 3)? spaceDim : 1;
-   vec.SetSize(n);
    curlshape.SetSize(dof, n);
    elvect.SetSize(dof);
    fe.CalcPhysCurlShape(Trans, curlshape);
 
-   vec_delta->EvalDelta(vec, Trans, Trans.GetIntPoint());
-   curlshape.Mult(vec, elvect);
+   switch (spaceDim)
+   {
+      case 3:
+         vec_delta->EvalDelta(vec, Trans, Trans.GetIntPoint());
+         curlshape.Mult(vec, elvect);
+         break;
+      case 2:
+         curlshape.GetColumn(0,elvect);
+         elvect *= delta->EvalDelta(Trans, Trans.GetIntPoint());
+         break;
+      default:
+         break; // This should be unreachable
+   }
 }
 
 void VectorFEDomainLFDivIntegrator::AssembleRHSElementVect(

fem/lininteg.hpp

@@ -284,6 +284,7 @@ class VectorFEDomainLFCurlIntegrator : public DeltaLFIntegrator
 {
 private:
    VectorCoefficient *QF=nullptr;
+   Coefficient *Q=nullptr;
    DenseMatrix curlshape;
    Vector vec;
 
@@ -291,6 +292,8 @@ public:
    /// Constructs the domain integrator (Q, curl v)
    VectorFEDomainLFCurlIntegrator(VectorCoefficient &F)
       : DeltaLFIntegrator(F), QF(&F) { }
+   VectorFEDomainLFCurlIntegrator(Coefficient &F)
+      : DeltaLFIntegrator(F), Q(&F) { }
 
    virtual void AssembleRHSElementVect(const FiniteElement &el,
                                        ElementTransformation &Tr,

fem/nonlinearform.cpp

@@ -581,13 +581,6 @@ double BlockNonlinearForm::GetEnergyBlocked(const BlockVector &bx) const
          }
       }
 
-   // free the allocated memory
-   for (int i = 0; i < fes.Size(); ++i)
-   {
-      delete el_x[i];
-      delete vdofs[i];
-   }
-
    if (fnfi.Size())
    {
       MFEM_ABORT("TODO: add energy contribution from interior face terms");

fem/pbilinearform.cpp

@@ -565,53 +565,6 @@ HypreParMatrix* ParDiscreteLinearOperator::ParallelAssemble() const
    return RAP;
 }
 
-void ParDiscreteLinearOperator::ParallelAssemble(OperatorHandle &A)
-{
-   // construct the rectangular block-diagonal matrix dA
-   OperatorHandle dA(A.Type());
-   dA.MakeRectangularBlockDiag(domain_fes->GetComm(),
-                               range_fes->GlobalVSize(),
-                               domain_fes->GlobalVSize(),
-                               range_fes->GetDofOffsets(),
-                               domain_fes->GetDofOffsets(),
-                               mat);
-
-   OperatorHandle P_test(A.Type()), P_trial(A.Type());
-
-   // TODO - construct the Dof_TrueDof_Matrix directly in the required format.
-   P_test.ConvertFrom(range_fes->Dof_TrueDof_Matrix());
-   P_trial.ConvertFrom(domain_fes->Dof_TrueDof_Matrix());
-
-   A.MakeRAP(P_test, dA, P_trial);
-}
-
-//// @todo copied from ParMixedBilinearForm, should be some inheritance?
-void ParDiscreteLinearOperator::FormRectangularSystemMatrix(OperatorHandle &A)
-{
-   if (ext)
-   {
-      Array<int> empty;
-      ext->FormRectangularSystemOperator(empty, empty, A);
-      return;
-   }
-
-   /*
-   if (mat)
-   {
-      Finalize();
-      ParallelAssemble(p_mat);
-      delete mat;
-      mat = NULL;
-      delete mat_e;
-      mat_e = NULL;
-      p_mat_e = NULL;
-   }
-   A = p_mat;
-   */
-
-   mfem_error("not implemented!");
-}
-
 void ParDiscreteLinearOperator::GetParBlocks(Array2D<HypreParMatrix *> &blocks)
 const
 {

fem/pbilinearform.hpp

@@ -160,9 +160,6 @@ public:
    /// Get the parallel finite element space prolongation matrix
    virtual const Operator *GetProlongation() const
    { return pfes->GetProlongationMatrix(); }
-   /// Get the diagonal part of the parallel finite element space prolongation matrix
-   virtual const Operator *GetLocalProlongation() const
-   { return pfes->GetLocalProlongationMatrix(); }
    /// Get the parallel finite element space restriction matrix
    virtual const Operator *GetRestriction() const
    { return pfes->GetRestrictionMatrix(); }
@@ -304,18 +301,10 @@ public:
    /// Returns the matrix "assembled" on the true dofs
    HypreParMatrix *ParallelAssemble() const;
 
-   /** @brief Returns the matrix assembled on the true dofs, i.e.
-       @a A = P_test^t A_local P_trial, in the format (type id) specified by
-       @a A. */
-   void ParallelAssemble(OperatorHandle &A);
-
    /** Extract the parallel blocks corresponding to the vector dimensions of the
        domain and range parallel finite element spaces */
    void GetParBlocks(Array2D<HypreParMatrix *> &blocks) const;
 
-   /** @brief Return in @a A a parallel (on truedofs) version of this operator. */
-   virtual void FormRectangularSystemMatrix(OperatorHandle &A);
-
    virtual ~ParDiscreteLinearOperator() { }
 };
 

fem/pfespace.cpp

@@ -101,7 +101,6 @@ void ParFiniteElementSpace::ParInit(ParMesh *pm)
 
    P = NULL;
    Pconf = NULL;
-   Pconf_local = NULL;
    R = NULL;
 
    num_face_nbr_dofs = -1;
@@ -922,39 +921,6 @@ const Operator *ParFiniteElementSpace::GetProlongationMatrix() const
    }
 }
 
-const Operator *ParFiniteElementSpace::GetLocalProlongationMatrix() const
-{
-   if (Conforming())
-   {
-      if (Pconf_local) { return Pconf_local; }
-
-      if (NRanks == 1)
-      {
-         Pconf_local = new IdentityOperator(GetTrueVSize());
-      }
-      else
-      {
-         if (!Device::Allows(Backend::DEVICE_MASK))
-         {
-            Pconf_local = new ConformingProlongationOperator(*this, true);
-         }
-         else
-         {
-            // Pconf = new DeviceConformingProlongationOperator(*this);
-            mfem_error("Not implemented!");
-         }
-      }
-      return Pconf_local;
-   }
-   else
-   {
-      // return Dof_TrueDof_Matrix();
-      // just need diagonal portion, not too hard
-      mfem_error("Not implemented!");
-      return NULL;
-   }
-}
-
 void ParFiniteElementSpace::ExchangeFaceNbrData()
 {
    if (num_face_nbr_dofs >= 0) { return; }
@@ -2868,7 +2834,6 @@ void ParFiniteElementSpace::Destroy()
 
    delete P; P = NULL;
    delete Pconf; Pconf = NULL;
-   delete Pconf_local; Pconf_local = NULL;
    delete R; R = NULL;
 
    delete gcomm; gcomm = NULL;
@@ -2994,12 +2959,12 @@ void ParFiniteElementSpace::Update(bool want_transform)
    }
 }
 
+
 ConformingProlongationOperator::ConformingProlongationOperator(
-   const ParFiniteElementSpace &pfes, bool local_)
+   const ParFiniteElementSpace &pfes)
    : Operator(pfes.GetVSize(), pfes.GetTrueVSize()),
      external_ldofs(),
-     gc(pfes.GroupComm()),
-     local(local_)
+     gc(pfes.GroupComm())
 {
    MFEM_VERIFY(pfes.Conforming(), "");
    const Table &group_ldof = gc.GroupLDofTable();
@@ -3048,14 +3013,7 @@ void ConformingProlongationOperator::Mult(const Vector &x, Vector &y) const
    const int m = external_ldofs.Size();
 
    const int in_layout = 2; // 2 - input is ltdofs array
-   if (local)
-   {
-      y = 0.0;
-   }
-   else
-   {
-      gc.BcastBegin(const_cast<double*>(xdata), in_layout);
-   }
+   gc.BcastBegin(const_cast<double*>(xdata), in_layout);
 
    int j = 0;
    for (int i = 0; i < m; i++)
@@ -3067,10 +3025,7 @@ void ConformingProlongationOperator::Mult(const Vector &x, Vector &y) const
    std::copy(xdata+j-m, xdata+Width(), ydata+j);
 
    const int out_layout = 0; // 0 - output is ldofs array
-   if (!local)
-   {
-      gc.BcastEnd(ydata, out_layout);
-   }
+   gc.BcastEnd(ydata, out_layout);
 }
 
 void ConformingProlongationOperator::MultTranspose(
@@ -3083,10 +3038,7 @@ void ConformingProlongationOperator::MultTranspose(
    double *ydata = y.HostWrite();
    const int m = external_ldofs.Size();
 
-   if (!local)
-   {
-      gc.ReduceBegin(xdata);
-   }
+   gc.ReduceBegin(xdata);
 
    int j = 0;
    for (int i = 0; i < m; i++)
@@ -3098,10 +3050,7 @@ void ConformingProlongationOperator::MultTranspose(
    std::copy(xdata+j, xdata+Height(), ydata+j-m);
 
    const int out_layout = 2; // 2 - output is an array on all ltdofs
-   if (!local)
-   {
-      gc.ReduceEnd<double>(ydata, out_layout, GroupCommunicator::Sum);
-   }
+   gc.ReduceEnd<double>(ydata, out_layout, GroupCommunicator::Sum);
 }
 
 DeviceConformingProlongationOperator::DeviceConformingProlongationOperator(

fem/pfespace.hpp

@@ -72,9 +72,6 @@ private:
    mutable HypreParMatrix *P;
    /// Optimized action-only prolongation operator for conforming meshes. Owned.
    mutable Operator *Pconf;
-   /// Optimized action-only diagonal prolongation operator on ldofs for
-   /// conforming meshes. Owned.
-   mutable Operator *Pconf_local;
 
    /// The (block-diagonal) matrix R (restriction of dof to true dof). Owned.
    mutable SparseMatrix *R;
@@ -338,7 +335,6 @@ public:
    HYPRE_Int GetMyTDofOffset() const;
 
    virtual const Operator *GetProlongationMatrix() const;
-   virtual const Operator *GetLocalProlongationMatrix() const;
    /// Get the R matrix which restricts a local dof vector to true dof vector.
    virtual const SparseMatrix *GetRestrictionMatrix() const
    { Dof_TrueDof_Matrix(); return R; }
@@ -393,11 +389,9 @@ class ConformingProlongationOperator : public Operator
 protected:
    Array<int> external_ldofs;
    const GroupCommunicator &gc;
-   bool local;
 
 public:
-   ConformingProlongationOperator(const ParFiniteElementSpace &pfes,
-                                  bool local_=false);
+   ConformingProlongationOperator(const ParFiniteElementSpace &pfes);
 
    virtual void Mult(const Vector &x, Vector &y) const;
 

fem/pgridfunc.cpp

@@ -655,167 +655,6 @@ void ParGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
 #endif
 }
 
-double ParGridFunction::ComputeDGFaceJumpError(Coefficient *exsol,
-                                               Coefficient *ell_coeff,
-                                               double Nu,
-                                               const IntegrationRule *irs[]) const
-{
-   const_cast<ParGridFunction *>(this)->ExchangeFaceNbrData();
-
-   int fdof, dim, intorder, k;
-   ElementTransformation *transf;
-   Vector shape, el_dofs, err_val, ell_coeff_val;
-   Array<int> vdofs;
-   IntegrationPoint eip;
-   double error = 0.0;
-
-   ParMesh *mesh = pfes->GetParMesh();
-   dim = mesh->Dimension();
-
-   std::map<int,int> local_to_shared;
-   for (int i = 0; i < mesh->GetNSharedFaces(); ++i)
-   {
-      int i_local = mesh->GetSharedFace(i);
-      local_to_shared[i_local] = i;
-   }
-
-   for (int i = 0; i < mesh->GetNumFaces(); i++)
-   {
-      double shared_face_factor = 1.0;
-      bool shared_face = false;
-      int iel1, iel2, info1, info2;
-      mesh->GetFaceElements(i, &iel1, &iel2);
-      mesh->GetFaceInfos(i, &info1, &info2);
-
-      intorder = fes->GetFE(iel1)->GetOrder();
-
-      FaceElementTransformations *face_elem_transf;
-      const FiniteElement *fe1, *fe2;
-      if (info2 >= 0 && iel2 < 0)
-      {
-         int ishared = local_to_shared[i];
-         face_elem_transf = mesh->GetSharedFaceTransformations(ishared);
-         iel2 = face_elem_transf->Elem2No - mesh->GetNE();
-         fe2 = pfes->GetFaceNbrFE(iel2);
-         if ( (k = fe2->GetOrder()) > intorder )
-         {
-            intorder = k;
-         }
-         shared_face = true;
-         shared_face_factor = 0.5;
-      }
-      else
-      {
-         face_elem_transf = mesh->GetFaceElementTransformations(i);
-
-         if (iel2 >= 0)
-         {
-            fe2 = pfes->GetFE(iel2);
-            if ( (k = fe2->GetOrder()) > intorder )
-            {
-               intorder = k;
-            }
-         }
-         else
-         {
-            fe2 = NULL;
-         }
-      }
-
-      intorder = 2 * intorder;  // <-------------
-      const IntegrationRule *ir;
-      if (irs)
-      {
-         ir = irs[face_elem_transf->GetGeometryType()];
-      }
-      else
-      {
-         ir = &(IntRules.Get(face_elem_transf->GetGeometryType(), intorder));
-      }
-      err_val.SetSize(ir->GetNPoints());
-      ell_coeff_val.SetSize(ir->GetNPoints());
-      // side 1
-      transf = face_elem_transf->Elem1;
-      fe1 = fes->GetFE(iel1);
-      fdof = fe1->GetDof();
-      fes->GetElementVDofs(iel1, vdofs);
-      shape.SetSize(fdof);
-      el_dofs.SetSize(fdof);
-      for (k = 0; k < fdof; k++)
-         if (vdofs[k] >= 0)
-         {
-            el_dofs(k) =   (*this)(vdofs[k]);
-         }
-         else
-         {
-            el_dofs(k) = - (*this)(-1-vdofs[k]);
-         }
-      for (int j = 0; j < ir->GetNPoints(); j++)
-      {
-         face_elem_transf->Loc1.Transform(ir->IntPoint(j), eip);
-         fe1->CalcShape(eip, shape);
-         transf->SetIntPoint(&eip);
-         ell_coeff_val(j) = ell_coeff->Eval(*transf, eip);
-         err_val(j) = exsol->Eval(*transf, eip) - (shape * el_dofs);
-      }
-      if (fe2 != NULL)
-      {
-         // side 2
-         transf = face_elem_transf->Elem2;
-         fdof = fe2->GetDof();
-         shape.SetSize(fdof);
-         el_dofs.SetSize(fdof);
-         if (shared_face)
-         {
-            pfes->GetFaceNbrElementVDofs(iel2, vdofs);
-            for (k = 0; k < fdof; k++)
-               if (vdofs[k] >= 0)
-               {
-                  el_dofs(k) = face_nbr_data[vdofs[k]];
-               }
-               else
-               {
-                  el_dofs(k) = - face_nbr_data[-1-vdofs[k]];
-               }
-         }
-         else
-         {
-            pfes->GetElementVDofs(iel2, vdofs);
-            for (k = 0; k < fdof; k++)
-               if (vdofs[k] >= 0)
-               {
-                  el_dofs(k) = (*this)(vdofs[k]);
-               }
-               else
-               {
-                  el_dofs(k) = - (*this)(-1 - vdofs[k]);
-               }
-         }
-         for (int j = 0; j < ir->GetNPoints(); j++)
-         {
-            face_elem_transf->Loc2.Transform(ir->IntPoint(j), eip);
-            fe2->CalcShape(eip, shape);
-            transf->SetIntPoint(&eip);
-            ell_coeff_val(j) += ell_coeff->Eval(*transf, eip);
-            ell_coeff_val(j) *= 0.5;
-            err_val(j) -= (exsol->Eval(*transf, eip) - (shape * el_dofs));
-         }
-      }
-      transf = face_elem_transf;
-      for (int j = 0; j < ir->GetNPoints(); j++)
-      {
-         const IntegrationPoint &ip = ir->IntPoint(j);
-         transf->SetIntPoint(&ip);
-         error += shared_face_factor*(ip.weight * Nu * ell_coeff_val(j) *
-                                      pow(transf->Weight(), 1.0-1.0/(dim-1)) *
-                                      err_val(j) * err_val(j));
-      }
-   }
-
-   error = (error < 0.0) ? -sqrt(-error) : sqrt(error);
-   return GlobalLpNorm(2.0, error, pfes->GetComm());
-}
-
 void ParGridFunction::Save(std::ostream &out) const
 {
    double *data_  = const_cast<double*>(HostRead());
@@ -1021,6 +860,7 @@ double GlobalLpNorm(const double p, double loc_norm, MPI_Comm comm)
    return glob_norm;
 }
 
+
 void ParGridFunction::ComputeFlux(
    BilinearFormIntegrator &blfi,
    GridFunction &flux, bool wcoef, int subdomain)
@@ -1161,6 +1001,6 @@ double L2ZZErrorEstimator(BilinearFormIntegrator &flux_integrator,
    return pow(glob_error, 1.0/norm_p);
 }
 
-} // namespace mfem
+}
 
 #endif // MFEM_USE_MPI

fem/pgridfunc.hpp

@@ -283,77 +283,6 @@ public:
                           pfes->GetComm());
    }
 
-   /// Returns ||grad u_ex - grad u_h||_L2 for H1 or L2 elements
-   virtual double ComputeGradError(VectorCoefficient *exgrad,
-                                   const IntegrationRule *irs[] = NULL) const
-   {
-      return GlobalLpNorm(2.0, GridFunction::ComputeGradError(exgrad,irs),
-                          pfes->GetComm());
-   }
-
-   /// Returns ||curl u_ex - curl u_h||_L2 for ND elements
-   virtual double ComputeCurlError(VectorCoefficient *excurl,
-                                   const IntegrationRule *irs[] = NULL) const
-   {
-      return GlobalLpNorm(2.0, GridFunction::ComputeCurlError(excurl,irs),
-                          pfes->GetComm());
-   }
-
-   /// Returns ||div u_ex - div u_h||_L2 for RT elements
-   virtual double ComputeDivError(Coefficient *exdiv,
-                                  const IntegrationRule *irs[] = NULL) const
-   {
-      return GlobalLpNorm(2.0, GridFunction::ComputeDivError(exdiv,irs),
-                          pfes->GetComm());
-   }
-
-   /// Returns the Face Jumps error for L2 elements
-   virtual double ComputeDGFaceJumpError(Coefficient *exsol,
-                                         Coefficient *ell_coeff,
-                                         double Nu,
-                                         const IntegrationRule *irs[]=NULL)
-   const;
-
-   /// Returns either the H1-seminorm or the DG Face Jumps error or both
-   /// depending on norm_type = 1, 2, 3
-   virtual double ComputeH1Error(Coefficient *exsol, VectorCoefficient *exgrad,
-                                 Coefficient *ell_coef, double Nu,
-                                 int norm_type) const
-   {
-      return GlobalLpNorm(2.0,
-                          GridFunction::ComputeH1Error(exsol,exgrad,ell_coef,
-                                                       Nu, norm_type),
-                          pfes->GetComm());
-   }
-
-   /// Returns the error measured in H1-norm for H1 elements or in "broken"
-   /// H1-norm for L2 elements
-   virtual double ComputeH1Error(Coefficient *exsol, VectorCoefficient *exgrad,
-                                 const IntegrationRule *irs[] = NULL) const
-   {
-      return GlobalLpNorm(2.0, GridFunction::ComputeH1Error(exsol,exgrad,irs),
-                          pfes->GetComm());
-   }
-
-   /// Returns the error measured H(div)-norm for RT elements
-   virtual double ComputeHDivError(VectorCoefficient *exsol,
-                                   Coefficient *exdiv,
-                                   const IntegrationRule *irs[] = NULL) const
-   {
-      return GlobalLpNorm(2.0, GridFunction::ComputeHDivError(exsol,exdiv,irs),
-                          pfes->GetComm());
-   }
-
-   /// Returns the error measured H(curl)-norm for ND elements
-   virtual double ComputeHCurlError(VectorCoefficient *exsol,
-                                    VectorCoefficient *excurl,
-                                    const IntegrationRule *irs[] = NULL) const
-   {
-      return GlobalLpNorm(2.0,
-                          GridFunction::ComputeHCurlError(exsol,excurl,irs),
-                          pfes->GetComm());
-   }
-
    virtual double ComputeMaxError(Coefficient *exsol[],
                                   const IntegrationRule *irs[] = NULL) const
    {

fem/plinearform.cpp

@@ -21,6 +21,7 @@ namespace mfem
 void ParLinearForm::Update(ParFiniteElementSpace *pf)
 {
    if (pf) { pfes = pf; }
+
    LinearForm::Update(pfes);
 }
 
@@ -30,19 +31,6 @@ void ParLinearForm::Update(ParFiniteElementSpace *pf, Vector &v, int v_offset)
    LinearForm::Update(pf,v,v_offset);
 }
 
-void ParLinearForm::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
-{
-   LinearForm::MakeRef(f, v, v_offset);
-   pfes = dynamic_cast<ParFiniteElementSpace*>(f);
-   MFEM_ASSERT(pfes != NULL, "not a ParFiniteElementSpace");
-}
-
-void ParLinearForm::MakeRef(ParFiniteElementSpace *pf, Vector &v, int v_offset)
-{
-   LinearForm::MakeRef(pf, v, v_offset);
-   pfes = pf;
-}
-
 void ParLinearForm::ParallelAssemble(Vector &tv)
 {
    const Operator* prolong = pfes->GetProlongationMatrix();

fem/plinearform.hpp

@@ -92,27 +92,6 @@ public:
        @note This method does not perform assembly. */
    void Update(ParFiniteElementSpace *pf, Vector &v, int v_offset);
 
-
-   /** @brief Make the ParLinearForm reference external data on a new
-       FiniteElementSpace. */
-   /** This method changes the FiniteElementSpace associated with the
-       ParLinearForm to @a *f and sets the data of the Vector @a v (plus the @a
-       v_offset) as external data in the ParLinearForm.
-
-       @note This version of the method will also perform bounds checks when the
-       build option MFEM_DEBUG is enabled. */
-   virtual void MakeRef(FiniteElementSpace *f, Vector &v, int v_offset);
-
-   /** @brief Make the ParLinearForm reference external data on a new
-       ParFiniteElementSpace. */
-   /** This method changes the ParFiniteElementSpace associated with the
-       ParLinearForm to @a *pf and sets the data of the Vector @a v (plus the @a
-       v_offset) as external data in the ParLinearForm.
-
-       @note This version of the method will also perform bounds checks when the
-       build option MFEM_DEBUG is enabled. */
-   void MakeRef(ParFiniteElementSpace *pf, Vector &v, int v_offset);
-
    /// Assemble the vector on the true dofs, i.e. P^t v.
    void ParallelAssemble(Vector &tv);
 
@@ -120,10 +99,10 @@ public:
    HypreParVector *ParallelAssemble();
 
    /// Return the action of the ParLinearForm as a linear mapping.
-   /** Linear forms are linear functionals which map ParGridFunction%s to the
-       real numbers. This method performs this mapping which in this case is
-       equivalent as an inner product of the ParLinearForm and
-       ParGridFunction. */
+   /** Linear forms are linear functionals which map ParGridFunction%s to
+       the real numbers.  This method performs this mapping which in
+       this case is equivalent as an inner product of the ParLinearForm
+       and ParGridFunction. */
    double operator()(const ParGridFunction &gf) const
    {
       return InnerProduct(pfes->GetComm(), *this, gf);

fem/quadinterpolator.cpp

@@ -403,19 +403,6 @@ void QuadratureInterpolator::Mult(
          }
       }
    }
-   else if (vdim == 3 && dim == 2)
-   {
-      switch (100*nd + nq)
-      {
-         // Q1
-         case 404: eval_func = &Eval2D<3,4,4>; break;
-         case 409: eval_func = &Eval2D<3,4,9>; break;
-         // Q2
-         case 416: eval_func = &Eval2D<3,4,16>; break;
-         // Q3
-         case 425: eval_func = &Eval2D<3,4,25>; break;
-      }
-   }
    if (eval_func)
    {
       eval_func(ne, vdim, maps, e_vec, q_val, q_der, q_det, eval_flags);

fem/quadinterpolator_face.hpp

@@ -42,7 +42,7 @@ protected:
 
    static const int MAX_NQ2D = 100;
    static const int MAX_ND2D = 100;
-   static const int MAX_VDIM2D = 3;
+   static const int MAX_VDIM2D = 2;
 
    static const int MAX_NQ3D = 1000;
    static const int MAX_ND3D = 1000;

fem/restriction.cpp

@@ -17,90 +17,6 @@
 namespace mfem
 {
 
-/*
-// TODO: implement this based on PABilinearFormExtension::FormLinearSystem
-BoundaryElementRestriction::BoundaryElementRestriction(const FiniteElementSpace &f,
-                                     ElementDofOrdering e_ordering)
- : fes(f),
-   nbe(fes.GetNBE()),
-   vdim(fes.GetVDim()),
-   byvdim(fes.GetOrdering() == Ordering::byVDIM),
-   ndofs(fes.GetNDofs()),
-   dof(nbe > 0 ? fes.GetBE(0)->GetDof() : 0),
-   nbedofs(nbe*dof),
-   offsets(ndofs+1),
-   indices(nbe*dof),
-   gatherMap(nbe*dof)
-{
- // Assuming all finite elements are the same.
- height = vdim*nbe*dof;
- width = fes.GetVSize();
- const bool dof_reorder = (e_ordering == ElementDofOrdering::LEXICOGRAPHIC);
- const int *dof_map = NULL;
- if (dof_reorder && nbe > 0)
- {
-    for (int e = 0; e < nbe; ++e)
-    {
-       const FiniteElement *fe = fes.GetBE(e);
-       const TensorBasisElement* el =
-          dynamic_cast<const TensorBasisElement*>(fe);
-       if (el) { continue; }
-       mfem_error("Finite element not suitable for lexicographic ordering");
-    }
-    const FiniteElement *fe = fes.GetBE(0);
-    const TensorBasisElement* el =
-       dynamic_cast<const TensorBasisElement*>(fe);
-    const Array<int> &fe_dof_map = el->GetDofMap();
-    MFEM_VERIFY(fe_dof_map.Size() > 0, "invalid dof map");
-    dof_map = fe_dof_map.GetData();
- }
- const Table& e2dTable = fes.GetElementToDofTable();
- const int* elementMap = e2dTable.GetJ();
- // We will be keeping a count of how many local nodes point to its global dof
- for (int i = 0; i <= ndofs; ++i)
- {
-    offsets[i] = 0;
- }
- for (int e = 0; e < ne; ++e)
- {
-    for (int d = 0; d < dof; ++d)
-    {
-       const int sgid = elementMap[dof*e + d];  // signed
-       const int gid = (sgid >= 0) ? sgid : -1 - sgid;
-       ++offsets[gid + 1];
-    }
- }
- // Aggregate to find offsets for each global dof
- for (int i = 1; i <= ndofs; ++i)
- {
-    offsets[i] += offsets[i - 1];
- }
- // For each global dof, fill in all local nodes that point to it
- for (int e = 0; e < ne; ++e)
- {
-    for (int d = 0; d < dof; ++d)
-    {
-       const int sdid = dof_reorder ? dof_map[d] : 0;  // signed
-       const int did = (!dof_reorder)?d:(sdid >= 0 ? sdid : -1-sdid);
-       const int sgid = elementMap[dof*e + did];  // signed
-       const int gid = (sgid >= 0) ? sgid : -1-sgid;
-       const int lid = dof*e + d;
-       const bool plus = (sgid >= 0 && sdid >= 0) || (sgid < 0 && sdid < 0);
-       gatherMap[lid] = plus ? gid : -1-gid;
-       indices[offsets[gid]++] = plus ? lid : -1-lid;
-    }
- }
- // We shifted the offsets vector by 1 by using it as a counter.
- // Now we shift it back.
- for (int i = ndofs; i > 0; --i)
- {
-    offsets[i] = offsets[i - 1];
- }
- offsets[0] = 0;
-
-}
-*/
-
 ElementRestriction::ElementRestriction(const FiniteElementSpace &f,
                                        ElementDofOrdering e_ordering)
    : fes(f),
@@ -279,32 +195,6 @@ void ElementRestriction::MultTransposeUnsigned(const Vector& x, Vector& y) const
    });
 }
 
-void ElementRestriction::MultLeftInverse(const Vector& x, Vector& y) const
-{
-   // Assumes all elements have the same number of dofs
-   const int nd = dof;
-   const int vd = vdim;
-   const bool t = byvdim;
-   auto d_offsets = offsets.Read();
-   auto d_indices = indices.Read();
-   auto d_x = Reshape(x.Read(), nd, vd, ne);
-   auto d_y = Reshape(y.Write(), t?vd:ndofs, t?ndofs:vd);
-   MFEM_FORALL(i, ndofs,
-   {
-      const int nextOffset = d_offsets[i + 1];
-      for (int c = 0; c < vd; ++c)
-      {
-         double dofValue = 0;
-         const int j = nextOffset - 1;
-         const int idx_j = (d_indices[j] >= 0) ? d_indices[j] : -1 - d_indices[j];
-         dofValue =
-         (d_indices[j] >= 0) ?
-         d_x(idx_j % nd, c, idx_j / nd) : -d_x(idx_j % nd, c, idx_j / nd);
-         d_y(t?c:i,t?i:c) = dofValue;
-      }
-   });
-}
-
 void ElementRestriction::BooleanMask(Vector& y) const
 {
    // Assumes all elements have the same number of dofs

fem/restriction.hpp

@@ -57,10 +57,6 @@ public:
    /// Compute MultTranspose without applying signs based on DOF orientations.
    void MultTransposeUnsigned(const Vector &x, Vector &y) const;
 
-   /// Compute MultTranspose by setting (rather than adding) element
-   /// contributions; this is a left inverse of the Mult() operation
-   void MultLeftInverse(const Vector &x, Vector &y) const;
-
    /// @brief Fills the E-vector y with `boolean` values 0.0 and 1.0 such that each
    /// each entry of the L-vector is uniquely represented in `y`.
    /** This means, the sum of the E-vector `y` is equal to the sum of the

fem/tmop_tools.cpp

@@ -298,12 +298,34 @@ void InterpolatorFP::SetInitialField(const Vector &init_nodes,
    field0_gf = init_field;
 
    dim = f->GetFE(0)->GetDim();
+   const int pts_cnt = init_nodes.Size() / dim;
+   el_id_out.SetSize(pts_cnt);
+   code_out.SetSize(pts_cnt);
+   task_id_out.SetSize(pts_cnt);
+   pos_r_out.SetSize(pts_cnt*dim);
+   dist_p_out.SetSize(pts_cnt);
 }
 
 void InterpolatorFP::ComputeAtNewPosition(const Vector &new_nodes,
                                           Vector &new_field)
 {
-   finder->Interpolate(new_nodes, field0_gf, new_field);
+   const int pts_cnt = new_nodes.Size() / dim;
+
+   // The sizes may change between calls due to AMR.
+   if (el_id_out.Size() != pts_cnt)
+   {
+      el_id_out.SetSize(pts_cnt);
+      code_out.SetSize(pts_cnt);
+      task_id_out.SetSize(pts_cnt);
+      pos_r_out.SetSize(pts_cnt*dim);
+      dist_p_out(pts_cnt);
+   }
+
+   // Interpolate FE function values on the found points.
+   finder->FindPoints(new_nodes, code_out, task_id_out,
+                      el_id_out, pos_r_out, dist_p_out);
+   finder->Interpolate(code_out, task_id_out, el_id_out,
+                       pos_r_out, field0_gf, new_field);
 }
 
 #endif

fem/tmop_tools.hpp

@@ -49,6 +49,8 @@ private:
    Vector nodes0;
    GridFunction field0_gf;
    FindPointsGSLIB *finder;
+   Array<uint> el_id_out, code_out, task_id_out;
+   Vector pos_r_out, dist_p_out;
    int dim;
 public:
    InterpolatorFP() : finder(NULL) { }

general/adios2stream.cpp

@@ -235,8 +235,8 @@ void adios2stream::Print(const Mesh& mesh, const mode print_mode)
       }
 
       // format info
-      SafeDefineAttribute<std::string>(io, "format", "MFEM ADIOS2 BP v0.2" );
-      SafeDefineAttribute<std::string>(io, "format/version", "0.2" );
+      SafeDefineAttribute<std::string>(io, "format", "MFEM ADIOS2 BP v0.1" );
+      SafeDefineAttribute<std::string>(io, "format/version", "0.1" );
       std::string mesh_type = "Unknown";
       std::vector<std::string> viz_tools;
       viz_tools.reserve(2); //for now
@@ -298,7 +298,6 @@ void adios2stream::Print(const Mesh& mesh, const mode print_mode)
          element_nvertices = static_cast<size_t>(mesh.elements[0]->GetNVertices());
       }
       SafeDefineVariable<uint64_t>(io, "connectivity", {}, {}, {nelements, element_nvertices+1});
-      SafeDefineVariable<int32_t>(io, "material", {}, {}, {nelements});
 
       // vertices
       SafeDefineVariable<uint32_t>(io,"NumOfVertices", {adios2::LocalValueDim});
@@ -349,15 +348,8 @@ void adios2stream::Print(const Mesh& mesh, const mode print_mode)
          io.InquireVariable<uint64_t>("connectivity");
       adios2::Variable<uint64_t>::Span span_connectivity = engine.Put<uint64_t>
                                                            (var_connectivity);
-
-      adios2::Variable<int32_t> var_element_attribute =
-         io.InquireVariable<int32_t>("material");
-      adios2::Variable<int32_t>::Span span_element_attribute = engine.Put<int32_t>
-                                                               (var_element_attribute);
-
       size_t span_vertices_offset = 0;
       size_t span_connectivity_offset = 0;
-      size_t span_element_attribute_offset = 0;
       // use for setting absolute node id for each element
       size_t point_id = 0;
       DenseMatrix pmatrix;
@@ -378,9 +370,6 @@ void adios2stream::Print(const Mesh& mesh, const mode print_mode)
          }
          span_vertices_offset += static_cast<size_t>(pmatrix.Width()*pmatrix.Height());
 
-         // element attribute
-         const int element_attribute = mesh.GetAttribute(e);
-
          // connectivity
          const int nv = Geometries.GetVertices(type)->GetNPoints();
          const Array<int> &element_vertices = refined_geometry->RefGeoms;
@@ -390,10 +379,6 @@ void adios2stream::Print(const Mesh& mesh, const mode print_mode)
             span_connectivity[span_connectivity_offset] = static_cast<uint64_t>(nv);
             ++span_connectivity_offset;
 
-            span_element_attribute[span_element_attribute_offset] = static_cast<int32_t>
-                                                                    (element_attribute);
-            ++span_element_attribute_offset;
-
             for (int k =0; k < nv; k++, v++ )
             {
                span_connectivity[span_connectivity_offset] = static_cast<uint64_t>
@@ -434,17 +419,9 @@ void adios2stream::Print(const Mesh& mesh, const mode print_mode)
       adios2::Variable<uint64_t>::Span spanConnectivity =
          engine.Put<uint64_t>(varConnectivity);
 
-      adios2::Variable<int32_t> varElementAttribute =
-         io.InquireVariable<int32_t>("material");
-      // zero-copy access to adios2 buffer to put non-contiguous to contiguous memory
-      adios2::Variable<int32_t>::Span spanElementAttribute =
-         engine.Put<int32_t>(varElementAttribute);
-
       size_t elementPosition = 0;
       for (int e = 0; e < mesh.GetNE(); ++e)
       {
-         spanElementAttribute[e] = static_cast<int32_t>(mesh.GetAttribute(e));
-
          const int nVertices = mesh.elements[e]->GetNVertices();
          spanConnectivity[elementPosition] = nVertices;
          for (int v = 0; v < nVertices; ++v)
@@ -711,7 +688,7 @@ std::string adios2stream::VTKSchema() const noexcept
 {
    std::string vtkSchema = R"(
 <?xml version="1.0"?>
-<VTKFile type="UnstructuredGrid" version="0.2" byte_order="LittleEndian">
+<VTKFile type="UnstructuredGrid" version="0.1" byte_order="LittleEndian">
   <UnstructuredGrid>
     <Piece NumberOfPoints="NumOfVertices" NumberOfCells="NumOfElements">
       <Points>
@@ -719,9 +696,6 @@ std::string adios2stream::VTKSchema() const noexcept
 
    vtkSchema += R"(
       </Points>
-	  <CellData>
-        <DataArray Name="material" />
-	  </CellData>
       <Cells>
         <DataArray Name="connectivity" />
         <DataArray Name="types" />

general/array.hpp

@@ -108,8 +108,6 @@ public:
    /// Return the device flag of the Memory object used by the Array
    bool UseDevice() const { return data.UseDevice(); }
 
-   inline MemoryType GetMemoryType() const { return data.GetMemoryType(); }
-
    /// Return true if the data will be deleted by the array
    inline bool OwnsData() const { return data.OwnsHostPtr(); }