Adding coef-fact etc to .gitignore

Fix mem leak in BlockNonlinearForm

.gitignore

@@ -175,6 +175,7 @@ miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
 miniapps/meshing/minimal-surface
 miniapps/meshing/pminimal-surface
+miniapps/meshing/polar-nc
 
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
@@ -187,6 +188,7 @@ miniapps/meshing/extruder.mesh
 miniapps/meshing/trimmer.mesh
 miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
+miniapps/meshing/polar-nc.mesh
 
 miniapps/performance/ex1
 miniapps/performance/ex1p
@@ -197,10 +199,13 @@ miniapps/performance/sol.*
 
 miniapps/tools/display-basis
 miniapps/tools/load-dc
+miniapps/tools/coef-fact
 miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
 miniapps/tools/get-values
 
+miniapps/tools/coef-fact.inp
+
 miniapps/toys/automata
 miniapps/toys/life
 miniapps/toys/mandel
@@ -233,6 +238,7 @@ miniapps/nurbs/mode_*
 miniapps/nurbs/Example1*
 
 miniapps/gslib/field-diff
+miniapps/gslib/field-interp
 miniapps/gslib/findpts
 miniapps/gslib/pfindpts
 
@@ -259,5 +265,10 @@ tests/scripts/*.err
 tests/scripts/*.out
 tests/scripts/*.msg
 
+# Other tests
+tests/convergence/rates
+tests/convergence/prates
+tests/par-mesh-format/ex1p
+
 # VPATH builds
 build-*/*

.gitlab-ci.yml

@@ -71,6 +71,8 @@ stages:
   - build
   - test
   - deallocate
+  - lassen_build
+  - lassen_test
   - baseline_check
   - baseline_publish
 
@@ -79,7 +81,11 @@ stages:
 # TODO: updating tests and tpls is not necessary anymore since pipelines are
 # now using unique directories so repo are never shared with another pipeline.
 # This is not memory efficient (we keep a lot of data), hence this reminder.
-.setup:
+# Setup
+setup:
+  tags:
+    - shell
+    - quartz
   stage: setup
   variables:
     GIT_STRATEGY: none
@@ -100,6 +106,15 @@ stages:
   before_script:
     - module load gcc/6.1.0
 
+# On lassen
+.with_gcc_8_3_1:
+  variables:
+    TOOLCHAIN: gcc_8_3_1
+    CXX: g++
+    CC: gcc
+  before_script:
+    - module load gcc/8.3.1
+
 .with_gcc_4_9_3:
   variables:
     TOOLCHAIN: gcc_4_9_3
@@ -290,3 +305,4 @@ stages:
 # The list on jobs is defined in machine-specific files.
 include:
   - local: .gitlab/quartz.yml
+  - local: .gitlab/lassen.yml

.gitlab/lassen.yml

@@ -0,0 +1,57 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipelines configurations for the Lassen machine at LLNL
+
+.on_lassen:
+  tags:
+    - shell
+    - lassen
+  variables:
+    PLAT: lassen
+
+# Build MFEM
+build_mfem_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  needs: [setup]
+  stage: lassen_build
+  script:
+    - mkdir -p ${BUILD_PATH}
+    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
+    - lalloc 1 -W 5 -q pdebug make -j cuda CUDA_ARCH=sm_70
+
+build_mfem_debug_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  needs: [setup]
+  stage: lassen_build
+  script:
+    - mkdir -p ${BUILD_PATH}
+    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
+    - lalloc 1 -W 5 -q pdebug make -j cuda MFEM_DEBUG="YES" CUDA_ARCH=sm_70
+
+# Sanity check
+sanitycheck_mfem_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  stage: lassen_test
+  needs: [build_mfem_ser_lassen]
+  script:
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
+    - lalloc 1 -W 15 -q pdebug make -j test
+
+sanitycheck_mfem_debug_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  stage: lassen_test
+  needs: [build_mfem_debug_ser_lassen]
+  script:
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
+    - lalloc 1 -W 30 -q pdebug make -j test

.gitlab/quartz.yml

@@ -22,10 +22,6 @@
     MAKE_PAR: 6
     BASELINE_PAR: 18
 
-# Setup
-setup_quartz:
-  extends: [.setup, .on_quartz]
-
 # Allocate
 allocate_quartz:
   variables:

CHANGELOG

@@ -38,6 +38,11 @@ Meshing improvements
 - Added complete action of the TMOP Integrator to account for the spatial
   derivatives of discrete and analytic targets.
 
+- Added support for initialization of (serial) non-conforming meshes. Hanging
+  nodes can be marked with Mesh::AddVertexParents when building the mesh with
+  the "init" constructor. The usage is demonstrated in a new meshing miniapp
+  (polar-nc) which generates meshes that are non-conforming from the start.
+
 Performance improvements
 ------------------------
 - Added support for explicit vectorization in the high-performance templated
@@ -64,6 +69,10 @@ Improved GPU capabilities
 
 - Added support for BlockOperator on GPU. See the updated Example 5.
 
+- Added partial assembly and GPU support for complex operators, including the
+  classes ComplexOperator, [Par]ComplexGridFunction, [Par]ComplexLinearForm, and
+  [Par]SesquilinearForm. See the updated Example 22.
+
 Discretization improvements
 ---------------------------
 - Added support for matrix-free interpolation and restriction operators between
@@ -92,6 +101,14 @@ Discretization improvements
 
 - Added support face integrals on the boundaries of NURBS meshes.
 
+- Added support for interpolation of functions in L2, H(div) and H(curl)
+  spaces using GSLIB-FindPoints.
+
+- Added support for computing asymptotic error estimates and convergence rates
+  for the whole de Rham sequence based on the new class ConvergenceStudy and new
+  member methods in GridFunction and ParGridFunction. See the rates.cpp file in
+  the tests/convergence directory for sample usage.
+
 Linear and nonlinear solvers
 ----------------------------
 - Added power method to iteratively estimate the largest eigenvalue and the
@@ -117,6 +134,9 @@ Linear and nonlinear solvers
 
 - Added support for the SLEPc eigensolver package.
 
+- Added partially assembled convergent diagonal preconditioner for adaptively
+  refined meshes (i.e. non-conforming finite element spaces), see Example 6/6p.
+
 New and updated examples and miniapps
 -------------------------------------
 - Added a new example, Example 25/25p, to demonstrate the use of a Perfectly
@@ -153,6 +173,9 @@ New and updated examples and miniapps
 - Added a new meshing miniapp, Minimal Surface, which solves Plateau's problem:
   the Dirichlet problem for the minimal surface equation.
 
+- Added a new meshing miniapp, Polar NC, which demonstrates the construction of
+  polar non-conforming meshes.
+
 - Added partial assembly support to Example 4/4p and Example 5/5p, with diagonal
   preconditioning.
 
@@ -167,30 +190,47 @@ New and updated examples and miniapps
   mesh based on element attributes. Any newly exposed boundary elements are
   assigned attribute numbers related to the trimmed element attributes.
 
+- Added a new miniapp (field-interp) that demonstrates transfer of grid function
+  between different meshes using GSLIB-FindPoints.
+
+- Added diagonal preconditioner in Example 6/6p for partial assembly with AMR.
+
 - Added device support in Example 5/5p.
 
+- Added partial assembly and device support to Example 22/22p, with diagonal
+  preconditioning.
+
 - Added the option to plot a function in Mesh Explorer.
 
 Improved testing
 ----------------
+- Upgraded the Catch unit test framework from version 1.6.1 to version 2.13.0.
+
 - Added a GitLab pipeline that automates PR testing on supercomputing systems
   and Linux clusters at Lawrence Livermore National Lab (LLNL). This can be
   triggered only by LLNL developers, see .gitlab-ci.yml, the .gitlab directory
   and the updated CONTRIBUTING.md file.
 
+- Added testing of the parallel mesh format in tests/par-mesh-format.
+
 Miscellaneous
 -------------
 - Added support for ADIOS2 for parallel I/O with ParaView visualization. The
   classes adios2stream and ADIOS2DataCollection are introduced in mfem as the
   interfaces to generate ADIOS2 Binary Pack (BP4) directory datasets for the
-  entire spatial and temporal data. In addition, ADIOS2 allows for setting a
-  user-defined number of data substreams/subfiles. See examples 5, 9, 12, 16.
+  entire spatial and temporal node data. Cell centered data is accessible by
+  ADIOS2 data readers (e.g. Python), but currently not yet implement as of
+  ParaView v5.8.1. In addition, ADIOS2 allows for setting a user-defined number
+  of data substreams/subfiles at scale. See examples 5, 9, 12, 16.
 
 - The integration order used in the ComputeLpError and ComputeElementLpError
   methods of class GridFunction has been increased.
 
 - Various other simplifications, extensions, and bugfixes in the code.
 
+- Renamed "Backend::DEBUG" to "Backend::DEBUG_DEVICE" to avoid conflicts,
+  as DEBUG is sometimes used as a macro.
+
 
 Version 4.1, released on March 10, 2020
 =======================================

INSTALL

@@ -663,7 +663,7 @@ The specific libraries and their options are:
   URL: https://github.com/CEED/libCEED
        https://ceed.exascaleproject.org/libceed
   Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED > 0.6, git-hash fe5822c.
+  Versions: libCEED > 0.6, git-hash bdfed75.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
   Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.

config/cmake/modules/FindADIOS2.cmake

@@ -38,7 +38,19 @@ if(NOT ADIOS2_FOUND)
   endif()
 
   find_path(ADIOS2_INCLUDE_DIR adios2.h ${ADIOS2_INCLUDE_OPTS})
-  find_library(ADIOS2_LIBRARY NAMES adios2 ${ADIOS2_LIBRARY_OPTS}) 
+  
+  # adios2 version 2.5.0 
+  find_library(ADIOS2_LIBRARY NAMES adios2 ${ADIOS2_LIBRARY_OPTS})
+  
+  # adios2 version 2.6.0 and onwards
+  if(NOT ADIOS2_LIBRARY)
+  	find_library(ADIOS2_CXX11_MPI_LIBRARY NAMES adios2_cxx11_mpi ${ADIOS2_LIBRARY_OPTS})
+  	find_library(ADIOS2_CXX11_LIBRARY NAMES adios2_cxx11 ${ADIOS2_LIBRARY_OPTS})
+  	set(ADIOS2_LIBRARY ${ADIOS2_CXX11_MPI_LIBRARY} ${ADIOS2_CXX11_LIBRARY})
+  	if(MFEM_USE_MPI)
+  	  add_definitions(-DADIOS2_USE_MPI)
+  	endif()
+  endif() 
 
   include(FindPackageHandleStandardArgs)
   find_package_handle_standard_args(ADIOS2

config/sample-runs.sh

@@ -78,6 +78,14 @@ groups_parallel=(
    "miniapps/electromagnetics"
    "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
+'"convergence"
+   "Convergence tests:"
+   "tests/convergence"
+   "diffusion.cpp"'
+'"par-mesh-format"
+   "Parallel mesh tests:"
+   "tests/par-mesh-format"
+   "ex1p.cpp"'
 )
 # All groups serial + parallel runs mixed in the same group:
 groups_all=(
@@ -107,6 +115,14 @@ groups_all=(
    "miniapps/electromagnetics"
    "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
+'"convergence"
+   "Convergence tests:"
+   "tests/convergence"
+   "diffusion.cpp"'
+'"par-mesh-format"
+   "Parallel mesh tests:"
+   "tests/par-mesh-format"
+   "ex1p.cpp"'
 )
 make_all="all"
 base_timeformat=$'real: %3Rs  user: %3Us  sys: %3Ss  %%cpu: %P'
@@ -380,10 +396,15 @@ function timed_run()
 # This function is used to execute the sample runs
 function go()
 {
-   local cmd=("$@")
+   # Strip leading and trailing spaces from $1 and store the result in cmd_line
+   shopt -s extglob
+   local cmd_line="${1##+( )}"
+   cmd_line="${cmd_line%%+( )}"
+   shopt -u extglob
+   eval local cmd=(${cmd_line})
    local res=""
    echo $sep
-   echo "<${group}>" "${cmd[@]}"
+   echo "<${group}>" "${cmd_line}"
    echo $sep
    if [ "${timing}" == "yes" ]; then
       timed_run "${cmd[@]}"
@@ -395,15 +416,15 @@ function go()
    else
       res="${red}FAILED${none}"
    fi
-   printf "[${res}] <${group}> ${cmd[*]}\n"
+   printf "[${res}] <${group}> ${cmd_line}\n"
    if [ "${timing}" == "yes" ]; then
       printf "Run time: %s\n" "${timer}"
       timer=(${timer})
       timer="${timer[1]}"
-      printf -v line "[$res](%8s) ${cmd[*]}" "$timer"
+      printf -v line "[$res](%8s) ${cmd_line}" "$timer"
       summary=("${summary[@]}" "$line")
    else
-      summary=("${summary[@]}" "[${res}] ${cmd[*]}")
+      summary=("${summary[@]}" "[${res}] ${cmd_line}")
    fi
    echo $sep
 }
@@ -438,7 +459,7 @@ function go_group()
       fi
       for run in "${runs[@]}"; do
          if [ "${run}" == "" ]; then continue; fi
-         eval go \${run_prefix} \${run} \${run_suffix} $output
+         eval go \"\${run_prefix} \${run} \${run_suffix}\" $output
       done
    done
    ${make} clean-exec
@@ -504,7 +525,7 @@ function echo_run()
 {
    echo "   $@"
    { echo "   $@"; echo "$sep";
-     "$@"
+     eval "$@"
      echo "$sep"; } >> "$echo_log" 2>&1
 }
 
@@ -524,6 +545,28 @@ function build_all()
    echo_run ${make} config ${mfem_config} || exit 1
    echo_run ${make} ${make_j} || exit 1
    echo_run ${make} ${make_all} ${make_j} || exit 1
+   # Build groups in directories other than the directories built by 'make all':
+   for group_params in "${groups[@]}"; do
+      eval params=(${group_params})
+      group_dir="${params[2]}"
+      case "$group_dir" in
+         (examples*|miniapps*)
+            # Built by 'make all'
+            ;;
+         (*)
+            if [ "${mfem_dir}" != "${mfem_build_dir}" ]; then
+               echo_run mkdir -p "${group_dir}" || exit 1
+               echo_run cd "${group_dir}" || exit 1
+               echo_run cp -af "${mfem_dir}/${group_dir}/makefile" . || exit 1
+            else
+               echo_run cd "${group_dir}" || exit 1
+            fi
+            echo_run ${make} clean || exit 1
+            echo_run ${make} MFEM_DIR="${mfem_dir}" ${make_j} || exit 1
+            echo_run cd "${mfem_build_dir}" || exit 1
+            ;;
+      esac
+   done
 }
 
 # Function that runs all sample runs, given by the array variable "groups".

doc/CodeDocumentation.dox

@@ -149,6 +149,7 @@ namespace mfem {
  * - <a class="el" href="toroid_8cpp_source.html">Toroid</a>: generate simple toroidal meshes
  * - <a class="el" href="twist_8cpp_source.html">Twist</a>: generate simple periodic meshes
  * - <a class="el" href="minimal-surface_8cpp_source.html">Minimal Surface</a>: compute minimal surfaces, <a class="el" href="minimal-surface_8cpp_source.html">serial</a> and <a class="el" href="pminimal-surface_8cpp_source.html">parallel</a> versions
+ * - <a class="el" href="polar-nc_8cpp_source.html">Polar NC</a>: generate polar non-conforming meshes
  * - <a class="el" href="shaper_8cpp_source.html">Shaper</a>: resolve material interfaces by mesh refinement
  * - <a class="el" href="extruder_8cpp_source.html">Extruder</a>: extrude a low-dimensional mesh into a higher dimension
  * - <a class="el" href="mesh-explorer_8cpp_source.html">Mesh Explorer</a>: visualize and manipulate meshes
@@ -161,6 +162,7 @@ namespace mfem {
  * - <a class="el" href="lor-transfer_8cpp_source.html">LOR Transfer</a>: map functions between high-order and low-order refined spaces
  * - <a class="el" href="findpts_8cpp_source.html">Find Points</a>: evaluate grid function in physical space, <a class="el" href="findpts_8cpp_source.html">serial</a> and <a class="el" href="pfindpts_8cpp_source.html">parallel</a> versions
  * - <a class="el" href="field-diff_8cpp_source.html">Field Diff</a>: compare grid functions on different meshes
+ * - <a class="el" href="field-interp_8cpp_source.html">Field Interp</a>: transfer a grid functions betwen meshes
  * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
  * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
  *

examples/ex22.cpp

@@ -6,17 +6,19 @@
 //               ex22 -m ../data/inline-tri.mesh -o 3
 //               ex22 -m ../data/inline-quad.mesh -o 3
 //               ex22 -m ../data/inline-quad.mesh -o 3 -p 1
+//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa
 //               ex22 -m ../data/inline-quad.mesh -o 3 -p 2
 //               ex22 -m ../data/inline-tet.mesh -o 2
 //               ex22 -m ../data/inline-hex.mesh -o 2
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 1
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2
+//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa
 //               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0
 //
-// With partial assembly:
-//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa
-//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa
-//               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0 -pa
+// Device sample runs:
+//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa -d cuda
+//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa -d cuda
+//               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0 -pa -d cuda
 //
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
@@ -82,6 +84,7 @@ int main(int argc, char *argv[])
    bool herm_conv = true;
    bool exact_sol = true;
    bool pa = false;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -114,6 +117,8 @@ int main(int argc, char *argv[])
                   "Enable or disable GLVis visualization.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -143,13 +148,18 @@ int main(int argc, char *argv[])
    ComplexOperator::Convention conv =
       herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
 
-   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes
    //    with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 3. Refine the mesh to increase resolution. In this example we do
+   // 4. Refine the mesh to increase resolution. In this example we do
    //    'ref_levels' of uniform refinement where the user specifies
    //    the number of levels with the '-r' option.
    for (int l = 0; l < ref_levels; l++)
@@ -157,7 +167,7 @@ int main(int argc, char *argv[])
       mesh->UniformRefinement();
    }
 
-   // 4. Define a finite element space on the mesh. Here we use continuous
+   // 5. Define a finite element space on the mesh. Here we use continuous
    //    Lagrange, Nedelec, or Raviart-Thomas finite elements of the specified
    //    order.
    if (dim == 1 && prob != 0 )
@@ -179,7 +189,7 @@ int main(int argc, char *argv[])
    cout << "Number of finite element unknowns: " << fespace->GetTrueVSize()
         << endl;
 
-   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
    //    In this example, the boundary conditions are defined based on the type
    //    of mesh and the problem type.
    Array<int> ess_tdof_list;
@@ -191,12 +201,12 @@ int main(int argc, char *argv[])
       fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
 
-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
    //    the FEM linear system.
    ComplexLinearForm b(fespace, conv);
    b.Vector::operator=(0.0);
 
-   // 7. Define the solution vector u as a complex finite element grid function
+   // 8. Define the solution vector u as a complex finite element grid function
    //    corresponding to fespace. Initialize u with initial guess of 1+0i or
    //    the exact solution if it is known.
    ComplexGridFunction u(fespace);
@@ -218,7 +228,6 @@ int main(int argc, char *argv[])
    VectorConstantCoefficient zeroVecCoef(zeroVec);
    VectorConstantCoefficient oneVecCoef(oneVec);
 
-   u = 0.0;
    switch (prob)
    {
       case 0:
@@ -271,7 +280,7 @@ int main(int argc, char *argv[])
                  << "window_title 'Exact: Imaginary Part'" << flush;
    }
 
-   // 8. Set up the sesquilinear form a(.,.) on the finite element space
+   // 9. Set up the sesquilinear form a(.,.) on the finite element space
    //    corresponding to the damped harmonic oscillator operator of the
    //    appropriate type:
    //
@@ -314,7 +323,7 @@ int main(int argc, char *argv[])
       default: break; // This should be unreachable
    }
 
-   // 8a. Set up the bilinear form for the preconditioner corresponding to the
+   // 9a. Set up the bilinear form for the preconditioner corresponding to the
    //     appropriate operator
    //
    //      0) A scalar H1 field
@@ -349,9 +358,9 @@ int main(int argc, char *argv[])
       default: break; // This should be unreachable
    }
 
-   // 9. Assemble the form and the corresponding linear system, applying any
-   //    necessary transformations such as: assembly, eliminating boundary
-   //    conditions, conforming constraints for non-conforming AMR, etc.
+   // 10. Assemble the form and the corresponding linear system, applying any
+   //     necessary transformations such as: assembly, eliminating boundary
+   //     conditions, conforming constraints for non-conforming AMR, etc.
    a->Assemble();
    pcOp->Assemble();
 
@@ -362,7 +371,7 @@ int main(int argc, char *argv[])
 
    cout << "Size of linear system: " << A->Width() << endl << endl;
 
-   // 10. Define and apply a GMRES solver for AU=B with a block diagonal
+   // 11. Define and apply a GMRES solver for AU=B with a block diagonal
    //     preconditioner based on the appropriate sparse smoother.
    {
       Array<int> blockOffsets;
@@ -419,7 +428,7 @@ int main(int argc, char *argv[])
       gmres.Mult(B, U);
    }
 
-   // 11. Recover the solution as a finite element grid function and compute the
+   // 12. Recover the solution as a finite element grid function and compute the
    //     errors if the exact solution is known.
    a->RecoverFEMSolution(U, b, u);
 
@@ -451,7 +460,7 @@ int main(int argc, char *argv[])
       cout << endl;
    }
 
-   // 12. Save the refined mesh and the solution. This output can be viewed
+   // 13. Save the refined mesh and the solution. This output can be viewed
    //     later using GLVis: "glvis -m mesh -g sol".
    {
       ofstream mesh_ofs("refined.mesh");
@@ -466,7 +475,7 @@ int main(int argc, char *argv[])
       u.imag().Save(sol_i_ofs);
    }
 
-   // 13. Send the solution by socket to a GLVis server.
+   // 14. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -525,7 +534,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 14. Free the used memory.
+   // 15. Free the used memory.
    delete a;
    delete u_exact;
    delete pcOp;

examples/ex22p.cpp

@@ -7,16 +7,18 @@
 //               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3
 //               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3 -p 1
 //               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3 -p 2
+//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa
 //               mpirun -np 4 ex22p -m ../data/inline-tet.mesh -o 2
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 1
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 2
+//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa
 //               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
 //
-// With partial assembly:
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa
-//               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0 -pa
+// Device sample runs:
+//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa -d cuda
+//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa -d cuda
+//               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0 -pa -d cuda
 //
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
@@ -46,7 +48,6 @@
 //               We recommend viewing examples 1, 3 and 4 before viewing this
 //               example.
 
-
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
@@ -90,6 +91,7 @@ int main(int argc, char *argv[])
    bool herm_conv = true;
    bool exact_sol = true;
    bool pa = false;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -124,6 +126,8 @@ int main(int argc, char *argv[])
                   "Enable or disable GLVis visualization.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -160,19 +164,24 @@ int main(int argc, char *argv[])
    ComplexOperator::Convention conv =
       herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
 
-   // 3. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors. We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 4. Refine the serial mesh on all processors to increase the resolution.
+   // 5. Refine the serial mesh on all processors to increase the resolution.
    for (int l = 0; l < ser_ref_levels; l++)
    {
       mesh->UniformRefinement();
    }
 
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -182,7 +191,7 @@ int main(int argc, char *argv[])
       pmesh->UniformRefinement();
    }
 
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
    //    use continuous Lagrange, Nedelec, or Raviart-Thomas finite elements of
    //    the specified order.
    if (dim == 1 && prob != 0 )
@@ -210,7 +219,7 @@ int main(int argc, char *argv[])
       cout << "Number of finite element unknowns: " << size << endl;
    }
 
-   // 7. Determine the list of true (i.e. parallel conforming) essential
+   // 8. Determine the list of true (i.e. parallel conforming) essential
    //    boundary dofs. In this example, the boundary conditions are defined
    //    based on the type of mesh and the problem type.
    Array<int> ess_tdof_list;
@@ -222,14 +231,14 @@ int main(int argc, char *argv[])
       fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
 
-   // 8. Set up the parallel linear form b(.) which corresponds to the
+   // 9. Set up the parallel linear form b(.) which corresponds to the
    //    right-hand side of the FEM linear system.
    ParComplexLinearForm b(fespace, conv);
    b.Vector::operator=(0.0);
 
-   // 9. Define the solution vector u as a parallel complex finite element grid
-   //    function corresponding to fespace. Initialize u with initial guess of
-   //    1+0i or the exact solution if it is known.
+   // 10. Define the solution vector u as a parallel complex finite element grid
+   //     function corresponding to fespace. Initialize u with initial guess of
+   //     1+0i or the exact solution if it is known.
    ParComplexGridFunction u(fespace);
    ParComplexGridFunction * u_exact = NULL;
    if (exact_sol) { u_exact = new ParComplexGridFunction(fespace); }
@@ -249,7 +258,6 @@ int main(int argc, char *argv[])
    VectorConstantCoefficient zeroVecCoef(zeroVec);
    VectorConstantCoefficient oneVecCoef(oneVec);
 
-   u = 0.0;
    switch (prob)
    {
       case 0:
@@ -304,7 +312,7 @@ int main(int argc, char *argv[])
                  << "window_title 'Exact: Imaginary Part'" << flush;
    }
 
-   // 10. Set up the parallel sesquilinear form a(.,.) on the finite element
+   // 11. Set up the parallel sesquilinear form a(.,.) on the finite element
    //     space corresponding to the damped harmonic oscillator operator of the
    //     appropriate type:
    //
@@ -347,7 +355,7 @@ int main(int argc, char *argv[])
       default: break; // This should be unreachable
    }
 
-   // 10a. Set up the parallel bilinear form for the preconditioner
+   // 11a. Set up the parallel bilinear form for the preconditioner
    //      corresponding to the appropriate operator
    //
    //      0) A scalar H1 field
@@ -381,7 +389,7 @@ int main(int argc, char *argv[])
       default: break; // This should be unreachable
    }
 
-   // 11. Assemble the parallel bilinear form and the corresponding linear
+   // 12. Assemble the parallel bilinear form and the corresponding linear
    //     system, applying any necessary transformations such as: parallel
    //     assembly, eliminating boundary conditions, applying conforming
    //     constraints for non-conforming AMR, etc.
@@ -399,7 +407,7 @@ int main(int argc, char *argv[])
            << 2 * fespace->GlobalTrueVSize() << endl << endl;
    }
 
-   // 12. Define and apply a parallel FGMRES solver for AU=B with a block
+   // 13. Define and apply a parallel FGMRES solver for AU=B with a block
    //     diagonal preconditioner based on the appropriate multigrid
    //     preconditioner from hypre.
    {
@@ -460,7 +468,7 @@ int main(int argc, char *argv[])
       fgmres.SetPrintLevel(1);
       fgmres.Mult(B, U);
    }
-   // 13. Recover the parallel grid function corresponding to U. This is the
+   // 14. Recover the parallel grid function corresponding to U. This is the
    //     local finite element solution on each processor.
    a->RecoverFEMSolution(U, b, u);
 
@@ -495,7 +503,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 14. Save the refined mesh and the solution in parallel. This output can be
+   // 15. Save the refined mesh and the solution in parallel. This output can be
    //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
    {
       ostringstream mesh_name, sol_r_name, sol_i_name;
@@ -515,7 +523,7 @@ int main(int argc, char *argv[])
       u.imag().Save(sol_i_ofs);
    }
 
-   // 15. Send the solution by socket to a GLVis server.
+   // 16. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -580,7 +588,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 16. Free the used memory.
+   // 17. Free the used memory.
    delete a;
    delete u_exact;
    delete pcOp;

examples/ex25.cpp

@@ -82,24 +82,24 @@ public:
 };
 
 // Class for returning the PML coefficients of the bilinear form
-class PMLMatrixCoefficient : public MatrixCoefficient
+class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
    CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
+   void (*Function)(const Vector &, CartesianPML * , Vector &);
 public:
-   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                          DenseMatrix &),
-                        CartesianPML * pml_)
-      : MatrixCoefficient(dim), pml(pml_), Function(F)
+   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                              Vector &),
+                            CartesianPML * pml_)
+      : VectorCoefficient(dim), pml(pml_), Function(F)
    {}
-   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+   virtual void Eval(Vector &K, ElementTransformation &T,
                      const IntegrationPoint &ip)
    {
       double x[3];
       Vector transip(x, 3);
       T.Transform(ip, transip);
-      K.SetSize(height, width);
+      K.SetSize(vdim);
       (*Function)(transip, pml, K);
    }
 };
@@ -116,13 +116,13 @@ void source(const Vector &x, Vector & f);
 
 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D);
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D);
 
 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -365,19 +365,19 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);
 
    int cdim = (dim == 2) ? 1 : dim;
-   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
-   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
+   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
 
-   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
-   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
+   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
 
    // Integrators inside the PML region
    a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -419,13 +419,13 @@ int main(int argc, char *argv[])
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));
 
-      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
-      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
+      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
 
-      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
 
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));
@@ -763,7 +763,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
    }
 }
 
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -774,14 +774,13 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = (det / pow(dxs[i], 2)).real();
+      D(i) = (det / pow(dxs[i], 2)).real();
    }
 }
 
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -792,14 +791,13 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = (det / pow(dxs[i], 2)).imag();
+      D(i) = (det / pow(dxs[i], 2)).imag();
    }
 }
 
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -810,14 +808,13 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = abs(det / pow(dxs[i], 2));
+      D(i) = abs(det / pow(dxs[i], 2));
    }
 }
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -831,19 +828,18 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
    // in the 2D case the coefficient is scalar 1/det(J)
    if (dim == 2)
    {
-      M = (1.0 / det).real();
+      D = (1.0 / det).real();
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = (pow(dxs[i], 2) / det).real();
+         D(i) = (pow(dxs[i], 2) / det).real();
       }
    }
 }
 
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -856,19 +852,18 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 
    if (dim == 2)
    {
-      M = (1.0 / det).imag();
+      D = (1.0 / det).imag();
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = (pow(dxs[i], 2) / det).imag();
+         D(i) = (pow(dxs[i], 2) / det).imag();
       }
    }
 }
 
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -881,14 +876,13 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 
    if (dim == 2)
    {
-      M = abs(1.0 / det);
+      D = abs(1.0 / det);
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = abs(pow(dxs[i], 2) / det);
+         D(i) = abs(pow(dxs[i], 2) / det);
       }
    }
 }

examples/ex25p.cpp

@@ -82,24 +82,24 @@ public:
 };
 
 // Class for returning the PML coefficients of the bilinear form
-class PMLMatrixCoefficient : public MatrixCoefficient
+class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
    CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
+   void (*Function)(const Vector &, CartesianPML * , Vector &);
 public:
-   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                          DenseMatrix &),
-                        CartesianPML * pml_)
-      : MatrixCoefficient(dim), pml(pml_), Function(F)
+   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                              Vector &),
+                            CartesianPML * pml_)
+      : VectorCoefficient(dim), pml(pml_), Function(F)
    {}
-   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+   virtual void Eval(Vector &K, ElementTransformation &T,
                      const IntegrationPoint &ip)
    {
       double x[3];
       Vector transip(x, 3);
       T.Transform(ip, transip);
-      K.SetSize(height, width);
+      K.SetSize(vdim);
       (*Function)(transip, pml, K);
    }
 };
@@ -116,13 +116,13 @@ void source(const Vector &x, Vector & f);
 
 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D);
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D);
 
 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -393,19 +393,19 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);
 
    int cdim = (dim == 2) ? 1 : dim;
-   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
-   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
+   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
 
-   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
-   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
+   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
 
    // Integrators inside the PML region
    a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -453,13 +453,13 @@ int main(int argc, char *argv[])
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));
 
-      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
-      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
+      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
 
-      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
 
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));
@@ -819,7 +819,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
    }
 }
 
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -830,14 +830,13 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = (det / pow(dxs[i], 2)).real();
+      D(i) = (det / pow(dxs[i], 2)).real();
    }
 }
 
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -848,14 +847,13 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = (det / pow(dxs[i], 2)).imag();
+      D(i) = (det / pow(dxs[i], 2)).imag();
    }
 }
 
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -866,14 +864,13 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = abs(det / pow(dxs[i], 2));
+      D(i) = abs(det / pow(dxs[i], 2));
    }
 }
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -887,19 +884,18 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
    // in the 2D case the coefficient is scalar 1/det(J)
    if (dim == 2)
    {
-      M = (1.0 / det).real();
+      D = (1.0 / det).real();
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = (pow(dxs[i], 2) / det).real();
+         D(i) = (pow(dxs[i], 2) / det).real();
       }
    }
 }
 
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -912,19 +908,18 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 
    if (dim == 2)
    {
-      M = (1.0 / det).imag();
+      D = (1.0 / det).imag();
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = (pow(dxs[i], 2) / det).imag();
+         D(i) = (pow(dxs[i], 2) / det).imag();
       }
    }
 }
 
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -937,14 +932,13 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 
    if (dim == 2)
    {
-      M = abs(1.0 / det);
+      D = abs(1.0 / det);
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = abs(pow(dxs[i], 2) / det);
+         D(i) = abs(pow(dxs[i], 2) / det);
       }
    }
 }

examples/ex2p.cpp

@@ -60,6 +60,7 @@ int main(int argc, char *argv[])
    bool static_cond = false;
    bool visualization = 1;
    bool amg_elast = 0;
+   bool reorder_space = false;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -75,6 +76,8 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
+   args.AddOption(&reorder_space, "-nodes", "--by-nodes", "-vdim", "--by-vdim",
+                  "Use byNODES ordering of vector space instead of byVDIM");
    args.Parse();
    if (!args.Good())
    {
@@ -156,7 +159,14 @@ int main(int argc, char *argv[])
    else
    {
       fec = new H1_FECollection(order, dim);
-      fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
+      if (reorder_space)
+      {
+         fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byNODES);
+      }
+      else
+      {
+         fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
+      }
    }
    HYPRE_Int size = fespace->GlobalTrueVSize();
    if (myid == 0)
@@ -249,7 +259,7 @@ int main(int argc, char *argv[])
    }
    else
    {
-      amg->SetSystemsOptions(dim);
+      amg->SetSystemsOptions(dim, reorder_space);
    }
    HyprePCG *pcg = new HyprePCG(A);
    pcg->SetTol(1e-8);

examples/ex6.cpp

@@ -108,7 +108,11 @@ int main(int argc, char *argv[])
    //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
    //    problem yet, this will be done in the main loop.
    BilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   if (pa)
+   {
+      a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      a.SetDiagonalPolicy(Operator::DIAG_ONE);
+   }
    LinearForm b(&fespace);
 
    ConstantCoefficient one(1.0);
@@ -199,9 +203,10 @@ int main(int argc, char *argv[])
          umf_solver.Mult(B, X);
 #endif
       }
-      else // No preconditioning for now in partial assembly mode.
+      else // Diagonal preconditioning in partial assembly mode.
       {
-         CG(*A, B, X, 3, 2000, 1e-12, 0.0);
+         OperatorJacobiSmoother M(a, ess_tdof_list);
+         PCG(*A, M, B, X, 3, 2000, 1e-12, 0.0);
       }
 
       // 18. After solving the linear system, reconstruct the solution as a

examples/ex6p.cpp

@@ -129,7 +129,11 @@ int main(int argc, char *argv[])
    //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
    //    problem yet, this will be done in the main loop.
    ParBilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   if (pa)
+   {
+      a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      a.SetDiagonalPolicy(Operator::DIAG_ONE);
+   }
    ParLinearForm b(&fespace);
 
    ConstantCoefficient one(1.0);
@@ -220,17 +224,26 @@ int main(int argc, char *argv[])
 
       // 17. Solve the linear system A X = B.
       //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-      //     * With partial assembly, use no preconditioner, for now.
-      HypreBoomerAMG *amg = NULL;
-      if (!pa) { amg = new HypreBoomerAMG; amg->SetPrintLevel(0); }
+      //     * With partial assembly, use a diagonal preconditioner.
+      Solver *M = NULL;
+      if (pa)
+      {
+         M = new OperatorJacobiSmoother(a, ess_tdof_list);
+      }
+      else
+      {
+         HypreBoomerAMG *amg = new HypreBoomerAMG;
+         amg->SetPrintLevel(0);
+         M = amg;
+      }
       CGSolver cg(MPI_COMM_WORLD);
       cg.SetRelTol(1e-6);
       cg.SetMaxIter(2000);
       cg.SetPrintLevel(3); // print the first and the last iterations only
-      if (amg) { cg.SetPreconditioner(*amg); }
+      cg.SetPreconditioner(*M);
       cg.SetOperator(*A);
       cg.Mult(B, X);
-      delete amg;
+      delete M;
 
       // 18. Switch back to the host and extract the parallel grid function
       //     corresponding to the finite element approximation X. This is the

fem/CMakeLists.txt

@@ -31,6 +31,7 @@ set(SRCS
   bilininteg_vecmass.cpp
   coefficient.cpp
   complex_fem.cpp
+  convergence.cpp
   datacollection.cpp
   eltrans.cpp
   estimators.cpp
@@ -65,6 +66,7 @@ set(HDRS
   bilininteg.hpp
   coefficient.hpp
   complex_fem.hpp
+  convergence.hpp
   datacollection.hpp
   eltrans.hpp
   estimators.hpp

fem/bilinearform_ext.cpp

@@ -310,13 +310,12 @@ void EABilinearFormExtension::Assemble()
 
    ea_data.SetSize(ne*elemDofs*elemDofs, Device::GetMemoryType());
    ea_data.UseDevice(true);
-   ea_data = 0.0;
 
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
    const int integratorCount = integrators.Size();
    for (int i = 0; i < integratorCount; ++i)
    {
-      integrators[i]->AssembleEA(*a->FESpace(), ea_data);
+      integrators[i]->AssembleEA(*a->FESpace(), ea_data, i);
    }
 
    faceDofs = trialFes ->
@@ -333,14 +332,13 @@ void EABilinearFormExtension::Assemble()
       nf_int = trialFes->GetNFbyType(FaceType::Interior);
       ea_data_int.SetSize(2*nf_int*faceDofs*faceDofs, Device::GetMemoryType());
       ea_data_ext.SetSize(2*nf_int*faceDofs*faceDofs, Device::GetMemoryType());
-      ea_data_int = 0.0;
-      ea_data_ext = 0.0;
    }
    for (int i = 0; i < intFaceIntegratorCount; ++i)
    {
       intFaceIntegrators[i]->AssembleEAInteriorFaces(*a->FESpace(),
                                                      ea_data_int,
-                                                     ea_data_ext);
+                                                     ea_data_ext,
+                                                     i);
    }
 
    Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
@@ -353,7 +351,7 @@ void EABilinearFormExtension::Assemble()
    }
    for (int i = 0; i < boundFaceIntegratorCount; ++i)
    {
-      bdrFaceIntegrators[i]->AssembleEABoundaryFaces(*a->FESpace(),ea_data_bdr);
+      bdrFaceIntegrators[i]->AssembleEABoundaryFaces(*a->FESpace(),ea_data_bdr,i);
    }
 
    if (factorize_face_terms && int_face_restrict_lex)

fem/bilininteg.cpp

@@ -52,7 +52,8 @@ void BilinearFormIntegrator::AssembleDiagonalPA(Vector &)
 }
 
 void BilinearFormIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                        Vector &emat)
+                                        Vector &emat,
+                                        const bool add)
 {
    mfem_error ("BilinearFormIntegrator::AssembleEA(...)\n"
                "   is not implemented for this class.");
@@ -61,7 +62,8 @@ void BilinearFormIntegrator::AssembleEA(const FiniteElementSpace &fes,
 void BilinearFormIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace
                                                      &fes,
                                                      Vector &ea_data_int,
-                                                     Vector &ea_data_ext)
+                                                     Vector &ea_data_ext,
+                                                     const bool add)
 {
    mfem_error ("BilinearFormIntegrator::AssembleEAInteriorFaces(...)\n"
                "   is not implemented for this class.");
@@ -69,7 +71,8 @@ void BilinearFormIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace
 
 void BilinearFormIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace
                                                      &fes,
-                                                     Vector &ea_data_bdr)
+                                                     Vector &ea_data_bdr,
+                                                     const bool add)
 {
    mfem_error ("BilinearFormIntegrator::AssembleEABoundaryFaces(...)\n"
                "   is not implemented for this class.");
@@ -1522,6 +1525,7 @@ void CurlCurlIntegrator::AssembleElementMatrix
    double w;
 
 #ifdef MFEM_THREAD_SAFE
+   Vector D;
    DenseMatrix curlshape(nd,dimc), curlshape_dFt(nd,dimc), M;
 #else
    curlshape.SetSize(nd,dimc);
@@ -1529,6 +1533,7 @@ void CurlCurlIntegrator::AssembleElementMatrix
 #endif
    elmat.SetSize(nd);
    if (MQ) { M.SetSize(dimc); }
+   if (DQ) { D.SetSize(dimc); }
 
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
@@ -1572,6 +1577,12 @@ void CurlCurlIntegrator::AssembleElementMatrix
          Mult(curlshape_dFt, M, curlshape);
          AddMultABt(curlshape, curlshape_dFt, elmat);
       }
+      else if (DQ)
+      {
+         DQ->Eval(D, Trans, ip);
+         D *= w;
+         AddMultADAt(curlshape_dFt, D, elmat);
+      }
       else if (Q)
       {
          w *= Q->Eval(Trans, ip);

fem/bilininteg.hpp

@@ -86,9 +86,10 @@ public:
    virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
 
    /// Method defining element assembly.
-   /** The result of the element assembly is added and stored in the @a emat
-       Vector. */
-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);
+   /** The result of the element assembly is added to the @a emat Vector if
+       @a add is true. Otherwise, if @a add is false, we set @a emat. */
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
+                           const bool add = true);
    /** Used with BilinearFormIntegrators that have different spaces. */
    // virtual void AssembleEA(const FiniteElementSpace &trial_fes,
    //                         const FiniteElementSpace &test_fes,
@@ -96,10 +97,12 @@ public:
 
    virtual void AssembleEAInteriorFaces(const FiniteElementSpace &fes,
                                         Vector &ea_data_int,
-                                        Vector &ea_data_ext);
+                                        Vector &ea_data_ext,
+                                        const bool add = true);
 
    virtual void AssembleEABoundaryFaces(const FiniteElementSpace &fes,
-                                        Vector &ea_data_bdr);
+                                        Vector &ea_data_bdr,
+                                        const bool add = true);
 
    /// Given a particular Finite Element computes the element matrix elmat.
    virtual void AssembleElementMatrix(const FiniteElement &el,
@@ -262,14 +265,17 @@ public:
       bfi->AddMultTransposePA(x, y);
    }
 
-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
+                           const bool add);
 
    virtual void AssembleEAInteriorFaces(const FiniteElementSpace &fes,
                                         Vector &ea_data_int,
-                                        Vector &ea_data_ext);
+                                        Vector &ea_data_ext,
+                                        const bool add);
 
    virtual void AssembleEABoundaryFaces(const FiniteElementSpace &fes,
-                                        Vector &ea_data_bdr);
+                                        Vector &ea_data_bdr,
+                                        const bool add);
 
    virtual ~TransposeIntegrator() { if (own_bfi) { delete bfi; } }
 };
@@ -1952,7 +1958,8 @@ public:
 
    virtual void AssemblePA(const FiniteElementSpace &fes);
 
-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
+                           const bool add);
 
    virtual void AssembleDiagonalPA(Vector &diag);
 
@@ -2027,7 +2034,8 @@ public:
 
    virtual void AssemblePA(const FiniteElementSpace &fes);
 
-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
+                           const bool add);
 
    virtual void AssembleDiagonalPA(Vector &diag);
 
@@ -2083,7 +2091,8 @@ public:
 
    virtual void AssemblePA(const FiniteElementSpace&);
 
-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
+                           const bool add);
 
    virtual void AddMultPA(const Vector&, Vector&) const;
 
@@ -2300,12 +2309,14 @@ class CurlCurlIntegrator: public BilinearFormIntegrator
 private:
    Vector vec, pointflux;
 #ifndef MFEM_THREAD_SAFE
+   Vector D;
    DenseMatrix curlshape, curlshape_dFt, M;
    DenseMatrix vshape, projcurl;
 #endif
 
 protected:
    Coefficient *Q;
+   VectorCoefficient *DQ;
    MatrixCoefficient *MQ;
 
    // PA extension
@@ -2314,12 +2325,17 @@ protected:
    const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
    const GeometricFactors *geom;   ///< Not owned
    int dim, ne, nq, dofs1D, quad1D;
+   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
 
 public:
-   CurlCurlIntegrator() { Q = NULL; MQ = NULL; }
+   CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; }
    /// Construct a bilinear form integrator for Nedelec elements
-   CurlCurlIntegrator(Coefficient &q) : Q(&q) { MQ = NULL; }
-   CurlCurlIntegrator(MatrixCoefficient &m) : MQ(&m) { Q = NULL; }
+   CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), Q(&q) { DQ = NULL; MQ = NULL; }
+   CurlCurlIntegrator(VectorCoefficient &dq, const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), DQ(&dq) { Q = NULL; MQ = NULL; }
+   CurlCurlIntegrator(MatrixCoefficient &mq, const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), MQ(&mq) { Q = NULL; DQ = NULL; }
 
    /* Given a particular Finite Element, compute the
       element curl-curl matrix elmat */
@@ -2653,10 +2669,12 @@ public:
 
    virtual void AssembleEAInteriorFaces(const FiniteElementSpace& fes,
                                         Vector &ea_data_int,
-                                        Vector &ea_data_ext);
+                                        Vector &ea_data_ext,
+                                        const bool add);
 
    virtual void AssembleEABoundaryFaces(const FiniteElementSpace& fes,
-                                        Vector &ea_data_bdr);
+                                        Vector &ea_data_bdr,
+                                        const bool add);
 
    static const IntegrationRule &GetRule(Geometry::Type geom, int order,
                                          FaceElementTransformations &T);

fem/bilininteg_convection_ea.cpp

@@ -22,6 +22,7 @@ static void EAConvectionAssemble1D(const int NE,
                                    const Array<double> &g,
                                    const Vector &padata,
                                    Vector &eadata,
+                                   const bool add,
                                    const int d1d = 0,
                                    const int q1d = 0)
 {
@@ -54,7 +55,14 @@ static void EAConvectionAssemble1D(const int NE,
             {
                val += r_Bj[k1] * D(k1, e) * r_Gi[k1];
             }
-            A(i1, j1, e) += val;
+            if (add)
+            {
+               A(i1, j1, e) += val;
+            }
+            else
+            {
+               A(i1, j1, e) = val;
+            }
          }
       }
    });
@@ -66,6 +74,7 @@ static void EAConvectionAssemble2D(const int NE,
                                    const Array<double> &g,
                                    const Vector &padata,
                                    Vector &eadata,
+                                   const bool add,
                                    const int d1d = 0,
                                    const int q1d = 0)
 {
@@ -121,7 +130,14 @@ static void EAConvectionAssemble2D(const int NE,
                                * r_B[k1][j1]* r_B[k2][j2];
                      }
                   }
-                  A(i1, i2, j1, j2, e) += val;
+                  if (add)
+                  {
+                     A(i1, i2, j1, j2, e) += val;
+                  }
+                  else
+                  {
+                     A(i1, i2, j1, j2, e) = val;
+                  }
                }
             }
          }
@@ -135,6 +151,7 @@ static void EAConvectionAssemble3D(const int NE,
                                    const Array<double> &g,
                                    const Vector &padata,
                                    Vector &eadata,
+                                   const bool add,
                                    const int d1d = 0,
                                    const int q1d = 0)
 {
@@ -191,7 +208,14 @@ static void EAConvectionAssemble3D(const int NE,
                               }
                            }
                         }
-                        A(i1, i2, i3, j1, j2, j3, e) += val;
+                        if (add)
+                        {
+                           A(i1, i2, i3, j1, j2, j3, e) += val;
+                        }
+                        else
+                        {
+                           A(i1, i2, i3, j1, j2, j3, e) = val;
+                        }
                      }
                   }
                }
@@ -202,7 +226,8 @@ static void EAConvectionAssemble3D(const int NE,
 }
 
 void ConvectionIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                      Vector &ea_data)
+                                      Vector &ea_data,
+                                      const bool add)
 {
    AssemblePA(fes);
    const int ne = fes.GetMesh()->GetNE();
@@ -212,44 +237,47 @@ void ConvectionIntegrator::AssembleEA(const FiniteElementSpace &fes,
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x22: return EAConvectionAssemble1D<2,2>(ne,B,G,pa_data,ea_data);
-         case 0x33: return EAConvectionAssemble1D<3,3>(ne,B,G,pa_data,ea_data);
-         case 0x44: return EAConvectionAssemble1D<4,4>(ne,B,G,pa_data,ea_data);
-         case 0x55: return EAConvectionAssemble1D<5,5>(ne,B,G,pa_data,ea_data);
-         case 0x66: return EAConvectionAssemble1D<6,6>(ne,B,G,pa_data,ea_data);
-         case 0x77: return EAConvectionAssemble1D<7,7>(ne,B,G,pa_data,ea_data);
-         case 0x88: return EAConvectionAssemble1D<8,8>(ne,B,G,pa_data,ea_data);
-         case 0x99: return EAConvectionAssemble1D<9,9>(ne,B,G,pa_data,ea_data);
-         default:   return EAConvectionAssemble1D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
+         case 0x22: return EAConvectionAssemble1D<2,2>(ne,B,G,pa_data,ea_data,add);
+         case 0x33: return EAConvectionAssemble1D<3,3>(ne,B,G,pa_data,ea_data,add);
+         case 0x44: return EAConvectionAssemble1D<4,4>(ne,B,G,pa_data,ea_data,add);
+         case 0x55: return EAConvectionAssemble1D<5,5>(ne,B,G,pa_data,ea_data,add);
+         case 0x66: return EAConvectionAssemble1D<6,6>(ne,B,G,pa_data,ea_data,add);
+         case 0x77: return EAConvectionAssemble1D<7,7>(ne,B,G,pa_data,ea_data,add);
+         case 0x88: return EAConvectionAssemble1D<8,8>(ne,B,G,pa_data,ea_data,add);
+         case 0x99: return EAConvectionAssemble1D<9,9>(ne,B,G,pa_data,ea_data,add);
+         default:   return EAConvectionAssemble1D(ne,B,G,pa_data,ea_data,add,
+                                                     dofs1D,quad1D);
       }
    }
    else if (dim == 2)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x22: return EAConvectionAssemble2D<2,2>(ne,B,G,pa_data,ea_data);
-         case 0x33: return EAConvectionAssemble2D<3,3>(ne,B,G,pa_data,ea_data);
-         case 0x44: return EAConvectionAssemble2D<4,4>(ne,B,G,pa_data,ea_data);
-         case 0x55: return EAConvectionAssemble2D<5,5>(ne,B,G,pa_data,ea_data);
-         case 0x66: return EAConvectionAssemble2D<6,6>(ne,B,G,pa_data,ea_data);
-         case 0x77: return EAConvectionAssemble2D<7,7>(ne,B,G,pa_data,ea_data);
-         case 0x88: return EAConvectionAssemble2D<8,8>(ne,B,G,pa_data,ea_data);
-         case 0x99: return EAConvectionAssemble2D<9,9>(ne,B,G,pa_data,ea_data);
-         default:   return EAConvectionAssemble2D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
+         case 0x22: return EAConvectionAssemble2D<2,2>(ne,B,G,pa_data,ea_data,add);
+         case 0x33: return EAConvectionAssemble2D<3,3>(ne,B,G,pa_data,ea_data,add);
+         case 0x44: return EAConvectionAssemble2D<4,4>(ne,B,G,pa_data,ea_data,add);
+         case 0x55: return EAConvectionAssemble2D<5,5>(ne,B,G,pa_data,ea_data,add);
+         case 0x66: return EAConvectionAssemble2D<6,6>(ne,B,G,pa_data,ea_data,add);
+         case 0x77: return EAConvectionAssemble2D<7,7>(ne,B,G,pa_data,ea_data,add);
+         case 0x88: return EAConvectionAssemble2D<8,8>(ne,B,G,pa_data,ea_data,add);
+         case 0x99: return EAConvectionAssemble2D<9,9>(ne,B,G,pa_data,ea_data,add);
+         default:   return EAConvectionAssemble2D(ne,B,G,pa_data,ea_data,add,
+                                                     dofs1D,quad1D);
       }
    }
    else if (dim == 3)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x23: return EAConvectionAssemble3D<2,3>(ne,B,G,pa_data,ea_data);
-         case 0x34: return EAConvectionAssemble3D<3,4>(ne,B,G,pa_data,ea_data);
-         case 0x45: return EAConvectionAssemble3D<4,5>(ne,B,G,pa_data,ea_data);
-         case 0x56: return EAConvectionAssemble3D<5,6>(ne,B,G,pa_data,ea_data);
-         case 0x67: return EAConvectionAssemble3D<6,7>(ne,B,G,pa_data,ea_data);
-         case 0x78: return EAConvectionAssemble3D<7,8>(ne,B,G,pa_data,ea_data);
-         case 0x89: return EAConvectionAssemble3D<8,9>(ne,B,G,pa_data,ea_data);
-         default:   return EAConvectionAssemble3D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
+         case 0x23: return EAConvectionAssemble3D<2,3>(ne,B,G,pa_data,ea_data,add);
+         case 0x34: return EAConvectionAssemble3D<3,4>(ne,B,G,pa_data,ea_data,add);
+         case 0x45: return EAConvectionAssemble3D<4,5>(ne,B,G,pa_data,ea_data,add);
+         case 0x56: return EAConvectionAssemble3D<5,6>(ne,B,G,pa_data,ea_data,add);
+         case 0x67: return EAConvectionAssemble3D<6,7>(ne,B,G,pa_data,ea_data,add);
+         case 0x78: return EAConvectionAssemble3D<7,8>(ne,B,G,pa_data,ea_data,add);
+         case 0x89: return EAConvectionAssemble3D<8,9>(ne,B,G,pa_data,ea_data,add);
+         default:   return EAConvectionAssemble3D(ne,B,G,pa_data,ea_data,add,
+                                                     dofs1D,quad1D);
       }
    }
    MFEM_ABORT("Unknown kernel.");

fem/bilininteg_convection_pa.cpp

@@ -806,16 +806,16 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    {
       vel.SetSize(dim * nq * ne);
       auto C = Reshape(vel.HostWrite(), dim, nq, ne);
-      Vector Vq(dim);
+      DenseMatrix Q_ir;
       for (int e = 0; e < ne; ++e)
       {
          ElementTransformation& T = *fes.GetElementTransformation(e);
+         Q->Eval(Q_ir, T, *ir);
          for (int q = 0; q < nq; ++q)
          {
-            Q->Eval(Vq, T, ir->IntPoint(q));
             for (int i = 0; i < dim; ++i)
             {
-               C(i,q,e) = Vq(i);
+               C(i,q,e) = Q_ir(i,q);
             }
          }
       }

fem/bilininteg_dgtrace_ea.cpp

@@ -20,7 +20,8 @@ static void EADGTraceAssemble1DInt(const int NF,
                                    const Array<double> &basis,
                                    const Vector &padata,
                                    Vector &eadata_int,
-                                   Vector &eadata_ext)
+                                   Vector &eadata_ext,
+                                   const bool add)
 {
    auto D = Reshape(padata.Read(), 2, 2, NF);
    auto A_int = Reshape(eadata_int.ReadWrite(), 2, NF);
@@ -32,23 +33,41 @@ static void EADGTraceAssemble1DInt(const int NF,
       val_ext10 = D(1, 0, f);
       val_ext01 = D(0, 1, f);
       val_int1  = D(1, 1, f);
-      A_int(0, f) += val_int0;
-      A_int(1, f) += val_int1;
-      A_ext(0, f) += val_ext01;
-      A_ext(1, f) += val_ext10;
+      if (add)
+      {
+         A_int(0, f) += val_int0;
+         A_int(1, f) += val_int1;
+         A_ext(0, f) += val_ext01;
+         A_ext(1, f) += val_ext10;
+      }
+      else
+      {
+         A_int(0, f) = val_int0;
+         A_int(1, f) = val_int1;
+         A_ext(0, f) = val_ext01;
+         A_ext(1, f) = val_ext10;
+      }
    });
 }
 
 static void EADGTraceAssemble1DBdr(const int NF,
                                    const Array<double> &basis,
                                    const Vector &padata,
-                                   Vector &eadata_bdr)
+                                   Vector &eadata_bdr,
+                                   const bool add)
 {
    auto D = Reshape(padata.Read(), 2, 2, NF);
    auto A_bdr = Reshape(eadata_bdr.ReadWrite(), NF);
    MFEM_FORALL(f, NF,
    {
-      A_bdr(f) += D(0, 0, f);
+      if (add)
+      {
+         A_bdr(f) += D(0, 0, f);
+      }
+      else
+      {
+         A_bdr(f) = D(0, 0, f);
+      }
    });
 }
 
@@ -58,6 +77,7 @@ static void EADGTraceAssemble2DInt(const int NF,
                                    const Vector &padata,
                                    Vector &eadata_int,
                                    Vector &eadata_ext,
+                                   const bool add,
                                    const int d1d = 0,
                                    const int q1d = 0)
 {
@@ -88,10 +108,20 @@ static void EADGTraceAssemble2DInt(const int NF,
                val_ext10 += B(k1,i1) * B(k1,j1) * D(k1, 1, 0, f);
                val_int1  += B(k1,i1) * B(k1,j1) * D(k1, 1, 1, f);
             }
-            A_int(i1, j1, 0, f) += val_int0;
-            A_int(i1, j1, 1, f) += val_int1;
-            A_ext(i1, j1, 0, f) += val_ext01;
-            A_ext(i1, j1, 1, f) += val_ext10;
+            if (add)
+            {
+               A_int(i1, j1, 0, f) += val_int0;
+               A_int(i1, j1, 1, f) += val_int1;
+               A_ext(i1, j1, 0, f) += val_ext01;
+               A_ext(i1, j1, 1, f) += val_ext10;
+            }
+            else
+            {
+               A_int(i1, j1, 0, f) = val_int0;
+               A_int(i1, j1, 1, f) = val_int1;
+               A_ext(i1, j1, 0, f) = val_ext01;
+               A_ext(i1, j1, 1, f) = val_ext10;
+            }
          }
       }
    });
@@ -102,6 +132,7 @@ static void EADGTraceAssemble2DBdr(const int NF,
                                    const Array<double> &basis,
                                    const Vector &padata,
                                    Vector &eadata_bdr,
+                                   const bool add,
                                    const int d1d = 0,
                                    const int q1d = 0)
 {
@@ -125,7 +156,14 @@ static void EADGTraceAssemble2DBdr(const int NF,
             {
                val_bdr  += B(k1,i1) * B(k1,j1) * D(k1, 0, 0, f);
             }
-            A_bdr(i1, j1, f) += val_bdr;
+            if (add)
+            {
+               A_bdr(i1, j1, f) += val_bdr;
+            }
+            else
+            {
+               A_bdr(i1, j1, f) = val_bdr;
+            }
          }
       }
    });
@@ -137,6 +175,7 @@ static void EADGTraceAssemble3DInt(const int NF,
                                    const Vector &padata,
                                    Vector &eadata_int,
                                    Vector &eadata_ext,
+                                   const bool add,
                                    const int d1d = 0,
                                    const int q1d = 0)
 {
@@ -207,10 +246,20 @@ static void EADGTraceAssemble3DInt(const int NF,
                                     * s_D[k1][k2][1][0];
                      }
                   }
-                  A_int(i1, i2, j1, j2, 0, f) += val_int0;
-                  A_int(i1, i2, j1, j2, 1, f) += val_int1;
-                  A_ext(i1, i2, j1, j2, 0, f) += val_ext01;
-                  A_ext(i1, i2, j1, j2, 1, f) += val_ext10;
+                  if (add)
+                  {
+                     A_int(i1, i2, j1, j2, 0, f) += val_int0;
+                     A_int(i1, i2, j1, j2, 1, f) += val_int1;
+                     A_ext(i1, i2, j1, j2, 0, f) += val_ext01;
+                     A_ext(i1, i2, j1, j2, 1, f) += val_ext10;
+                  }
+                  else
+                  {
+                     A_int(i1, i2, j1, j2, 0, f) = val_int0;
+                     A_int(i1, i2, j1, j2, 1, f) = val_int1;
+                     A_ext(i1, i2, j1, j2, 0, f) = val_ext01;
+                     A_ext(i1, i2, j1, j2, 1, f) = val_ext10;
+                  }
                }
             }
          }
@@ -223,6 +272,7 @@ static void EADGTraceAssemble3DBdr(const int NF,
                                    const Array<double> &basis,
                                    const Vector &padata,
                                    Vector &eadata_bdr,
+                                   const bool add,
                                    const int d1d = 0,
                                    const int q1d = 0)
 {
@@ -280,7 +330,14 @@ static void EADGTraceAssemble3DBdr(const int NF,
                                    * s_D[k1][k2][0][0];
                      }
                   }
-                  A_bdr(i1, i2, j1, j2, f) += val_bdr;
+                  if (add)
+                  {
+                     A_bdr(i1, i2, j1, j2, f) += val_bdr;
+                  }
+                  else
+                  {
+                     A_bdr(i1, i2, j1, j2, f) = val_bdr;
+                  }
                }
             }
          }
@@ -290,7 +347,8 @@ static void EADGTraceAssemble3DBdr(const int NF,
 
 void DGTraceIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
                                                 Vector &ea_data_int,
-                                                Vector &ea_data_ext)
+                                                Vector &ea_data_ext,
+                                                const bool add)
 {
    SetupPA(fes, FaceType::Interior);
    nf = fes.GetNFbyType(FaceType::Interior);
@@ -298,7 +356,7 @@ void DGTraceIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
    const Array<double> &B = maps->B;
    if (dim == 1)
    {
-      return EADGTraceAssemble1DInt(nf,B,pa_data,ea_data_int,ea_data_ext);
+      return EADGTraceAssemble1DInt(nf,B,pa_data,ea_data_int,ea_data_ext,add);
    }
    else if (dim == 2)
    {
@@ -306,31 +364,31 @@ void DGTraceIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
       {
          case 0x22:
             return EADGTraceAssemble2DInt<2,2>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
+                                               ea_data_ext,add);
          case 0x33:
             return EADGTraceAssemble2DInt<3,3>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
+                                               ea_data_ext,add);
          case 0x44:
             return EADGTraceAssemble2DInt<4,4>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
+                                               ea_data_ext,add);
          case 0x55:
             return EADGTraceAssemble2DInt<5,5>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
+                                               ea_data_ext,add);
          case 0x66:
             return EADGTraceAssemble2DInt<6,6>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
+                                               ea_data_ext,add);
          case 0x77:
             return EADGTraceAssemble2DInt<7,7>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
+                                               ea_data_ext,add);
          case 0x88:
             return EADGTraceAssemble2DInt<8,8>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
+                                               ea_data_ext,add);
          case 0x99:
             return EADGTraceAssemble2DInt<9,9>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
+                                               ea_data_ext,add);
          default:
             return EADGTraceAssemble2DInt(nf,B,pa_data,ea_data_int,
-                                          ea_data_ext,dofs1D,quad1D);
+                                          ea_data_ext,add,dofs1D,quad1D);
       }
    }
    else if (dim == 3)
@@ -339,35 +397,36 @@ void DGTraceIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
       {
          case 0x23:
             return EADGTraceAssemble3DInt<2,3>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
+                                               ea_data_ext,add);
          case 0x34:
             return EADGTraceAssemble3DInt<3,4>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
+                                               ea_data_ext,add);
          case 0x45:
             return EADGTraceAssemble3DInt<4,5>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
+                                               ea_data_ext,add);
          case 0x56:
             return EADGTraceAssemble3DInt<5,6>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
+                                               ea_data_ext,add);
          case 0x67:
             return EADGTraceAssemble3DInt<6,7>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
+                                               ea_data_ext,add);
          case 0x78:
             return EADGTraceAssemble3DInt<7,8>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
+                                               ea_data_ext,add);
          case 0x89:
             return EADGTraceAssemble3DInt<8,9>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
+                                               ea_data_ext,add);
          default:
             return EADGTraceAssemble3DInt(nf,B,pa_data,ea_data_int,
-                                          ea_data_ext,dofs1D,quad1D);
+                                          ea_data_ext,add,dofs1D,quad1D);
       }
    }
    MFEM_ABORT("Unknown kernel.");
 }
 
 void DGTraceIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
-                                                Vector &ea_data_bdr)
+                                                Vector &ea_data_bdr,
+                                                const bool add)
 {
    SetupPA(fes, FaceType::Boundary);
    nf = fes.GetNFbyType(FaceType::Boundary);
@@ -375,37 +434,37 @@ void DGTraceIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
    const Array<double> &B = maps->B;
    if (dim == 1)
    {
-      return EADGTraceAssemble1DBdr(nf,B,pa_data,ea_data_bdr);
+      return EADGTraceAssemble1DBdr(nf,B,pa_data,ea_data_bdr,add);
    }
    else if (dim == 2)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x22: return EADGTraceAssemble2DBdr<2,2>(nf,B,pa_data,ea_data_bdr);
-         case 0x33: return EADGTraceAssemble2DBdr<3,3>(nf,B,pa_data,ea_data_bdr);
-         case 0x44: return EADGTraceAssemble2DBdr<4,4>(nf,B,pa_data,ea_data_bdr);
-         case 0x55: return EADGTraceAssemble2DBdr<5,5>(nf,B,pa_data,ea_data_bdr);
-         case 0x66: return EADGTraceAssemble2DBdr<6,6>(nf,B,pa_data,ea_data_bdr);
-         case 0x77: return EADGTraceAssemble2DBdr<7,7>(nf,B,pa_data,ea_data_bdr);
-         case 0x88: return EADGTraceAssemble2DBdr<8,8>(nf,B,pa_data,ea_data_bdr);
-         case 0x99: return EADGTraceAssemble2DBdr<9,9>(nf,B,pa_data,ea_data_bdr);
+         case 0x22: return EADGTraceAssemble2DBdr<2,2>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x33: return EADGTraceAssemble2DBdr<3,3>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x44: return EADGTraceAssemble2DBdr<4,4>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x55: return EADGTraceAssemble2DBdr<5,5>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x66: return EADGTraceAssemble2DBdr<6,6>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x77: return EADGTraceAssemble2DBdr<7,7>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x88: return EADGTraceAssemble2DBdr<8,8>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x99: return EADGTraceAssemble2DBdr<9,9>(nf,B,pa_data,ea_data_bdr,add);
          default:
-            return EADGTraceAssemble2DBdr(nf,B,pa_data,ea_data_bdr,dofs1D,quad1D);
+            return EADGTraceAssemble2DBdr(nf,B,pa_data,ea_data_bdr,add,dofs1D,quad1D);
       }
    }
    else if (dim == 3)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x23: return EADGTraceAssemble3DBdr<2,3>(nf,B,pa_data,ea_data_bdr);
-         case 0x34: return EADGTraceAssemble3DBdr<3,4>(nf,B,pa_data,ea_data_bdr);
-         case 0x45: return EADGTraceAssemble3DBdr<4,5>(nf,B,pa_data,ea_data_bdr);
-         case 0x56: return EADGTraceAssemble3DBdr<5,6>(nf,B,pa_data,ea_data_bdr);
-         case 0x67: return EADGTraceAssemble3DBdr<6,7>(nf,B,pa_data,ea_data_bdr);
-         case 0x78: return EADGTraceAssemble3DBdr<7,8>(nf,B,pa_data,ea_data_bdr);
-         case 0x89: return EADGTraceAssemble3DBdr<8,9>(nf,B,pa_data,ea_data_bdr);
+         case 0x23: return EADGTraceAssemble3DBdr<2,3>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x34: return EADGTraceAssemble3DBdr<3,4>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x45: return EADGTraceAssemble3DBdr<4,5>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x56: return EADGTraceAssemble3DBdr<5,6>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x67: return EADGTraceAssemble3DBdr<6,7>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x78: return EADGTraceAssemble3DBdr<7,8>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x89: return EADGTraceAssemble3DBdr<8,9>(nf,B,pa_data,ea_data_bdr,add);
          default:
-            return EADGTraceAssemble3DBdr(nf,B,pa_data,ea_data_bdr,dofs1D,quad1D);
+            return EADGTraceAssemble3DBdr(nf,B,pa_data,ea_data_bdr,add,dofs1D,quad1D);
       }
    }
    MFEM_ABORT("Unknown kernel.");

fem/bilininteg_dgtrace_pa.cpp

@@ -43,7 +43,7 @@ static void PADGTraceSetup2D(const int Q1D,
    auto W = w.Read();
    auto qd = Reshape(op.Write(), Q1D, 2, 2, NF);
 
-   MFEM_FORALL(f, NF,//can be optimized with Q1D thread for NF blocks
+   MFEM_FORALL(f, NF, // can be optimized with Q1D thread for NF blocks
    {
       for (int q = 0; q < Q1D; ++q)
       {
@@ -85,7 +85,7 @@ static void PADGTraceSetup3D(const int Q1D,
    auto W = w.Read();
    auto qd = Reshape(op.Write(), Q1D, Q1D, 2, 2, NF);
 
-   MFEM_FORALL(f, NF,//can be optimized with Q1D*Q1D threads for NF blocks
+   MFEM_FORALL(f, NF, // can be optimized with Q1D*Q1D threads for NF blocks
    {
       for (int q1 = 0; q1 < Q1D; ++q1)
       {
@@ -156,57 +156,6 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
    pa_data.SetSize(symmDims * nq * nf, Device::GetMemoryType());
-   Vector r;
-   if (rho==nullptr)
-   {
-      r.SetSize(1);
-      r(0) = 1.0;
-   }
-   else if (ConstantCoefficient *c_rho = dynamic_cast<ConstantCoefficient*>(rho))
-   {
-      r.SetSize(1);
-      r(0) = c_rho->constant;
-   }
-   else if (QuadratureFunctionCoefficient* c_rho =
-               dynamic_cast<QuadratureFunctionCoefficient*>(rho))
-   {
-      const QuadratureFunction &qFun = c_rho->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == nq * nf,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      r.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      r.SetSize(nq * nf);
-      auto C = Reshape(r.HostWrite(), nq, nf);
-      int f_ind = 0;
-      for (int f = 0; f < fes.GetNF(); ++f)
-      {
-         int e1, e2;
-         int inf1, inf2;
-         fes.GetMesh()->GetFaceElements(f, &e1, &e2);
-         fes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
-         int face_id = inf1 / 64;
-         if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
-             (type==FaceType::Boundary && e2<0 && inf2<0) )
-         {
-            ElementTransformation& T = *fes.GetMesh()->GetFaceTransformation(f);
-            for (int q = 0; q < nq; ++q)
-            {
-               // Convert to lexicographic ordering
-               int iq = ToLexOrdering(dim, face_id, quad1D, q);
-               C(iq,f_ind) = rho->Eval(T, ir->IntPoint(q));
-            }
-            f_ind++;
-         }
-      }
-      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
-   }
    Vector vel;
    if (VectorConstantCoefficient *c_u = dynamic_cast<VectorConstantCoefficient*>
                                         (u))
@@ -243,12 +192,15 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
          if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
              (type==FaceType::Boundary && e2<0 && inf2<0) )
          {
-            ElementTransformation& T = *fes.GetMesh()->GetFaceTransformation(f);
+            FaceElementTransformations &T =
+               *fes.GetMesh()->GetFaceElementTransformations(f);
             for (int q = 0; q < nq; ++q)
             {
                // Convert to lexicographic ordering
                int iq = ToLexOrdering(dim, face_id, quad1D, q);
-               u->Eval(Vq, T, ir->IntPoint(q));
+               T.SetAllIntPoints(&ir->IntPoint(q));
+               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
+               u->Eval(Vq, *T.Elem1, eip1);
                for (int i = 0; i < dim; ++i)
                {
                   C(i,iq,f_ind) = Vq(i);
@@ -259,6 +211,80 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
       }
       MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
    }
+   Vector r;
+   if (rho==nullptr)
+   {
+      r.SetSize(1);
+      r(0) = 1.0;
+   }
+   else if (ConstantCoefficient *c_rho = dynamic_cast<ConstantCoefficient*>(rho))
+   {
+      r.SetSize(1);
+      r(0) = c_rho->constant;
+   }
+   else if (QuadratureFunctionCoefficient* c_rho =
+               dynamic_cast<QuadratureFunctionCoefficient*>(rho))
+   {
+      const QuadratureFunction &qFun = c_rho->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == nq * nf,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      r.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
+   else
+   {
+      r.SetSize(nq * nf);
+      auto C_vel = Reshape(vel.HostRead(), dim, nq, nf);
+      auto n = Reshape(geom->normal.HostRead(), nq, dim, nf);
+      auto C = Reshape(r.HostWrite(), nq, nf);
+      int f_ind = 0;
+      for (int f = 0; f < fes.GetNF(); ++f)
+      {
+         int e1, e2;
+         int inf1, inf2;
+         fes.GetMesh()->GetFaceElements(f, &e1, &e2);
+         fes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
+         int face_id = inf1 / 64;
+         if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
+             (type==FaceType::Boundary && e2<0 && inf2<0) )
+         {
+            FaceElementTransformations &T =
+               *fes.GetMesh()->GetFaceElementTransformations(f);
+            for (int q = 0; q < nq; ++q)
+            {
+               // Convert to lexicographic ordering
+               int iq = ToLexOrdering(dim, face_id, quad1D, q);
+
+               T.SetAllIntPoints(&ir->IntPoint(q));
+               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
+               const IntegrationPoint &eip2 = T.GetElement2IntPoint();
+               double r;
+
+               if (inf2 < 0)
+               {
+                  r = rho->Eval(*T.Elem1, eip1);
+               }
+               else
+               {
+                  double udotn = 0.0;
+                  for (int d=0; d<dim; ++d)
+                  {
+                     udotn += C_vel(d,iq,f_ind)*n(iq,d,f_ind);
+                  }
+                  if (udotn >= 0.0) { r = rho->Eval(*T.Elem2, eip2); }
+                  else { r = rho->Eval(*T.Elem1, eip1); }
+               }
+               C(iq,f_ind) = r;
+            }
+            f_ind++;
+         }
+      }
+      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
+   }
    PADGTraceSetup(dim, dofs1D, quad1D, nf, ir->GetWeights(),
                   geom->detJ, geom->normal, r, vel,
                   alpha, beta, pa_data);

fem/bilininteg_diffusion_ea.cpp

@@ -22,6 +22,7 @@ static void EADiffusionAssemble1D(const int NE,
                                   const Array<double> &g,
                                   const Vector &padata,
                                   Vector &eadata,
+                                  const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -53,7 +54,14 @@ static void EADiffusionAssemble1D(const int NE,
             {
                val += r_Gj[k1] * D(k1, e) * r_Gi[k1];
             }
-            A(i1, j1, e) += val;
+            if (add)
+            {
+               A(i1, j1, e) += val;
+            }
+            else
+            {
+               A(i1, j1, e) = val;
+            }
          }
       }
    });
@@ -65,6 +73,7 @@ static void EADiffusionAssemble2D(const int NE,
                                   const Array<double> &g,
                                   const Vector &padata,
                                   Vector &eadata,
+                                  const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -120,7 +129,14 @@ static void EADiffusionAssemble2D(const int NE,
                                + gbi * D11 * gbj;
                      }
                   }
-                  A(i1, i2, j1, j2, e) += val;
+                  if (add)
+                  {
+                     A(i1, i2, j1, j2, e) += val;
+                  }
+                  else
+                  {
+                     A(i1, i2, j1, j2, e) = val;
+                  }
                }
             }
          }
@@ -134,6 +150,7 @@ static void EADiffusionAssemble3D(const int NE,
                                   const Array<double> &g,
                                   const Vector &padata,
                                   Vector &eadata,
+                                  const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -208,7 +225,14 @@ static void EADiffusionAssemble3D(const int NE,
                               }
                            }
                         }
-                        A(i1, i2, i3, j1, j2, j3, e) += val;
+                        if (add)
+                        {
+                           A(i1, i2, i3, j1, j2, j3, e) += val;
+                        }
+                        else
+                        {
+                           A(i1, i2, i3, j1, j2, j3, e) = val;
+                        }
                      }
                   }
                }
@@ -219,7 +243,8 @@ static void EADiffusionAssemble3D(const int NE,
 }
 
 void DiffusionIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                     Vector &ea_data)
+                                     Vector &ea_data,
+                                     const bool add)
 {
    AssemblePA(fes);
    const int ne = fes.GetMesh()->GetNE();
@@ -229,44 +254,47 @@ void DiffusionIntegrator::AssembleEA(const FiniteElementSpace &fes,
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x22: return EADiffusionAssemble1D<2,2>(ne,B,G,pa_data,ea_data);
-         case 0x33: return EADiffusionAssemble1D<3,3>(ne,B,G,pa_data,ea_data);
-         case 0x44: return EADiffusionAssemble1D<4,4>(ne,B,G,pa_data,ea_data);
-         case 0x55: return EADiffusionAssemble1D<5,5>(ne,B,G,pa_data,ea_data);
-         case 0x66: return EADiffusionAssemble1D<6,6>(ne,B,G,pa_data,ea_data);
-         case 0x77: return EADiffusionAssemble1D<7,7>(ne,B,G,pa_data,ea_data);
-         case 0x88: return EADiffusionAssemble1D<8,8>(ne,B,G,pa_data,ea_data);
-         case 0x99: return EADiffusionAssemble1D<9,9>(ne,B,G,pa_data,ea_data);
-         default:   return EADiffusionAssemble1D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
+         case 0x22: return EADiffusionAssemble1D<2,2>(ne,B,G,pa_data,ea_data,add);
+         case 0x33: return EADiffusionAssemble1D<3,3>(ne,B,G,pa_data,ea_data,add);
+         case 0x44: return EADiffusionAssemble1D<4,4>(ne,B,G,pa_data,ea_data,add);
+         case 0x55: return EADiffusionAssemble1D<5,5>(ne,B,G,pa_data,ea_data,add);
+         case 0x66: return EADiffusionAssemble1D<6,6>(ne,B,G,pa_data,ea_data,add);
+         case 0x77: return EADiffusionAssemble1D<7,7>(ne,B,G,pa_data,ea_data,add);
+         case 0x88: return EADiffusionAssemble1D<8,8>(ne,B,G,pa_data,ea_data,add);
+         case 0x99: return EADiffusionAssemble1D<9,9>(ne,B,G,pa_data,ea_data,add);
+         default:   return EADiffusionAssemble1D(ne,B,G,pa_data,ea_data,add,
+                                                    dofs1D,quad1D);
       }
    }
    else if (dim == 2)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x22: return EADiffusionAssemble2D<2,2>(ne,B,G,pa_data,ea_data);
-         case 0x33: return EADiffusionAssemble2D<3,3>(ne,B,G,pa_data,ea_data);
-         case 0x44: return EADiffusionAssemble2D<4,4>(ne,B,G,pa_data,ea_data);
-         case 0x55: return EADiffusionAssemble2D<5,5>(ne,B,G,pa_data,ea_data);
-         case 0x66: return EADiffusionAssemble2D<6,6>(ne,B,G,pa_data,ea_data);
-         case 0x77: return EADiffusionAssemble2D<7,7>(ne,B,G,pa_data,ea_data);
-         case 0x88: return EADiffusionAssemble2D<8,8>(ne,B,G,pa_data,ea_data);
-         case 0x99: return EADiffusionAssemble2D<9,9>(ne,B,G,pa_data,ea_data);
-         default:   return EADiffusionAssemble2D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
+         case 0x22: return EADiffusionAssemble2D<2,2>(ne,B,G,pa_data,ea_data,add);
+         case 0x33: return EADiffusionAssemble2D<3,3>(ne,B,G,pa_data,ea_data,add);
+         case 0x44: return EADiffusionAssemble2D<4,4>(ne,B,G,pa_data,ea_data,add);
+         case 0x55: return EADiffusionAssemble2D<5,5>(ne,B,G,pa_data,ea_data,add);
+         case 0x66: return EADiffusionAssemble2D<6,6>(ne,B,G,pa_data,ea_data,add);
+         case 0x77: return EADiffusionAssemble2D<7,7>(ne,B,G,pa_data,ea_data,add);
+         case 0x88: return EADiffusionAssemble2D<8,8>(ne,B,G,pa_data,ea_data,add);
+         case 0x99: return EADiffusionAssemble2D<9,9>(ne,B,G,pa_data,ea_data,add);
+         default:   return EADiffusionAssemble2D(ne,B,G,pa_data,ea_data,add,
+                                                    dofs1D,quad1D);
       }
    }
    else if (dim == 3)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x23: return EADiffusionAssemble3D<2,3>(ne,B,G,pa_data,ea_data);
-         case 0x34: return EADiffusionAssemble3D<3,4>(ne,B,G,pa_data,ea_data);
-         case 0x45: return EADiffusionAssemble3D<4,5>(ne,B,G,pa_data,ea_data);
-         case 0x56: return EADiffusionAssemble3D<5,6>(ne,B,G,pa_data,ea_data);
-         case 0x67: return EADiffusionAssemble3D<6,7>(ne,B,G,pa_data,ea_data);
-         case 0x78: return EADiffusionAssemble3D<7,8>(ne,B,G,pa_data,ea_data);
-         case 0x89: return EADiffusionAssemble3D<8,9>(ne,B,G,pa_data,ea_data);
-         default:   return EADiffusionAssemble3D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
+         case 0x23: return EADiffusionAssemble3D<2,3>(ne,B,G,pa_data,ea_data,add);
+         case 0x34: return EADiffusionAssemble3D<3,4>(ne,B,G,pa_data,ea_data,add);
+         case 0x45: return EADiffusionAssemble3D<4,5>(ne,B,G,pa_data,ea_data,add);
+         case 0x56: return EADiffusionAssemble3D<5,6>(ne,B,G,pa_data,ea_data,add);
+         case 0x67: return EADiffusionAssemble3D<6,7>(ne,B,G,pa_data,ea_data,add);
+         case 0x78: return EADiffusionAssemble3D<7,8>(ne,B,G,pa_data,ea_data,add);
+         case 0x89: return EADiffusionAssemble3D<8,9>(ne,B,G,pa_data,ea_data,add);
+         default:   return EADiffusionAssemble3D(ne,B,G,pa_data,ea_data,add,
+                                                    dofs1D,quad1D);
       }
    }
    MFEM_ABORT("Unknown kernel.");

fem/bilininteg_diffusion_pa.cpp

@@ -1680,7 +1680,7 @@ static void PADiffusionApply(const int dim,
       MFEM_ABORT("OCCA PADiffusionApply unknown kernel!");
    }
 #endif // MFEM_USE_OCCA
-   const int ID = (D1D << 4 ) | Q1D;
+   const int ID = (D1D << 4) | Q1D;
 
    if (dim == 2)
    {

fem/bilininteg_mass_ea.cpp

@@ -21,6 +21,7 @@ static void EAMassAssemble1D(const int NE,
                              const Array<double> &basis,
                              const Vector &padata,
                              Vector &eadata,
+                             const bool add,
                              const int d1d = 0,
                              const int q1d = 0)
 {
@@ -52,7 +53,14 @@ static void EAMassAssemble1D(const int NE,
             {
                val += r_Bi[k1] * r_Bj[k1] * D(k1, e);
             }
-            M(i1, j1, e) += val;
+            if (add)
+            {
+               M(i1, j1, e) += val;
+            }
+            else
+            {
+               M(i1, j1, e) = val;
+            }
          }
       }
    });
@@ -63,6 +71,7 @@ static void EAMassAssemble2D(const int NE,
                              const Array<double> &basis,
                              const Vector &padata,
                              Vector &eadata,
+                             const bool add,
                              const int d1d = 0,
                              const int q1d = 0)
 {
@@ -114,7 +123,14 @@ static void EAMassAssemble2D(const int NE,
                                * s_D[k1][k2];
                      }
                   }
-                  M(i1, i2, j1, j2, e) += val;
+                  if (add)
+                  {
+                     M(i1, i2, j1, j2, e) += val;
+                  }
+                  else
+                  {
+                     M(i1, i2, j1, j2, e) = val;
+                  }
                }
             }
          }
@@ -127,6 +143,7 @@ static void EAMassAssemble3D(const int NE,
                              const Array<double> &basis,
                              const Vector &padata,
                              Vector &eadata,
+                             const bool add,
                              const int d1d = 0,
                              const int q1d = 0)
 {
@@ -189,7 +206,14 @@ static void EAMassAssemble3D(const int NE,
                               }
                            }
                         }
-                        M(i1, i2, i3, j1, j2, j3, e) += val;
+                        if (add)
+                        {
+                           M(i1, i2, i3, j1, j2, j3, e) += val;
+                        }
+                        else
+                        {
+                           M(i1, i2, i3, j1, j2, j3, e) = val;
+                        }
                      }
                   }
                }
@@ -200,7 +224,8 @@ static void EAMassAssemble3D(const int NE,
 }
 
 void MassIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                Vector &ea_data)
+                                Vector &ea_data,
+                                const bool add)
 {
    AssemblePA(fes);
    const int ne = fes.GetMesh()->GetNE();
@@ -209,44 +234,47 @@ void MassIntegrator::AssembleEA(const FiniteElementSpace &fes,
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x22: return EAMassAssemble1D<2,2>(ne,B,pa_data,ea_data);
-         case 0x33: return EAMassAssemble1D<3,3>(ne,B,pa_data,ea_data);
-         case 0x44: return EAMassAssemble1D<4,4>(ne,B,pa_data,ea_data);
-         case 0x55: return EAMassAssemble1D<5,5>(ne,B,pa_data,ea_data);
-         case 0x66: return EAMassAssemble1D<6,6>(ne,B,pa_data,ea_data);
-         case 0x77: return EAMassAssemble1D<7,7>(ne,B,pa_data,ea_data);
-         case 0x88: return EAMassAssemble1D<8,8>(ne,B,pa_data,ea_data);
-         case 0x99: return EAMassAssemble1D<9,9>(ne,B,pa_data,ea_data);
-         default:   return EAMassAssemble1D(ne,B,pa_data,ea_data,dofs1D,quad1D);
+         case 0x22: return EAMassAssemble1D<2,2>(ne,B,pa_data,ea_data,add);
+         case 0x33: return EAMassAssemble1D<3,3>(ne,B,pa_data,ea_data,add);
+         case 0x44: return EAMassAssemble1D<4,4>(ne,B,pa_data,ea_data,add);
+         case 0x55: return EAMassAssemble1D<5,5>(ne,B,pa_data,ea_data,add);
+         case 0x66: return EAMassAssemble1D<6,6>(ne,B,pa_data,ea_data,add);
+         case 0x77: return EAMassAssemble1D<7,7>(ne,B,pa_data,ea_data,add);
+         case 0x88: return EAMassAssemble1D<8,8>(ne,B,pa_data,ea_data,add);
+         case 0x99: return EAMassAssemble1D<9,9>(ne,B,pa_data,ea_data,add);
+         default:   return EAMassAssemble1D(ne,B,pa_data,ea_data,add,
+                                               dofs1D,quad1D);
       }
    }
    else if (dim == 2)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x22: return EAMassAssemble2D<2,2>(ne,B,pa_data,ea_data);
-         case 0x33: return EAMassAssemble2D<3,3>(ne,B,pa_data,ea_data);
-         case 0x44: return EAMassAssemble2D<4,4>(ne,B,pa_data,ea_data);
-         case 0x55: return EAMassAssemble2D<5,5>(ne,B,pa_data,ea_data);
-         case 0x66: return EAMassAssemble2D<6,6>(ne,B,pa_data,ea_data);
-         case 0x77: return EAMassAssemble2D<7,7>(ne,B,pa_data,ea_data);
-         case 0x88: return EAMassAssemble2D<8,8>(ne,B,pa_data,ea_data);
-         case 0x99: return EAMassAssemble2D<9,9>(ne,B,pa_data,ea_data);
-         default:   return EAMassAssemble2D(ne,B,pa_data,ea_data,dofs1D,quad1D);
+         case 0x22: return EAMassAssemble2D<2,2>(ne,B,pa_data,ea_data,add);
+         case 0x33: return EAMassAssemble2D<3,3>(ne,B,pa_data,ea_data,add);
+         case 0x44: return EAMassAssemble2D<4,4>(ne,B,pa_data,ea_data,add);
+         case 0x55: return EAMassAssemble2D<5,5>(ne,B,pa_data,ea_data,add);
+         case 0x66: return EAMassAssemble2D<6,6>(ne,B,pa_data,ea_data,add);
+         case 0x77: return EAMassAssemble2D<7,7>(ne,B,pa_data,ea_data,add);
+         case 0x88: return EAMassAssemble2D<8,8>(ne,B,pa_data,ea_data,add);
+         case 0x99: return EAMassAssemble2D<9,9>(ne,B,pa_data,ea_data,add);
+         default:   return EAMassAssemble2D(ne,B,pa_data,ea_data,add,
+                                               dofs1D,quad1D);
       }
    }
    else if (dim == 3)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x23: return EAMassAssemble3D<2,3>(ne,B,pa_data,ea_data);
-         case 0x34: return EAMassAssemble3D<3,4>(ne,B,pa_data,ea_data);
-         case 0x45: return EAMassAssemble3D<4,5>(ne,B,pa_data,ea_data);
-         case 0x56: return EAMassAssemble3D<5,6>(ne,B,pa_data,ea_data);
-         case 0x67: return EAMassAssemble3D<6,7>(ne,B,pa_data,ea_data);
-         case 0x78: return EAMassAssemble3D<7,8>(ne,B,pa_data,ea_data);
-         case 0x89: return EAMassAssemble3D<8,9>(ne,B,pa_data,ea_data);
-         default:   return EAMassAssemble3D(ne,B,pa_data,ea_data,dofs1D,quad1D);
+         case 0x23: return EAMassAssemble3D<2,3>(ne,B,pa_data,ea_data,add);
+         case 0x34: return EAMassAssemble3D<3,4>(ne,B,pa_data,ea_data,add);
+         case 0x45: return EAMassAssemble3D<4,5>(ne,B,pa_data,ea_data,add);
+         case 0x56: return EAMassAssemble3D<5,6>(ne,B,pa_data,ea_data,add);
+         case 0x67: return EAMassAssemble3D<6,7>(ne,B,pa_data,ea_data,add);
+         case 0x78: return EAMassAssemble3D<7,8>(ne,B,pa_data,ea_data,add);
+         case 0x89: return EAMassAssemble3D<8,9>(ne,B,pa_data,ea_data,add);
+         default:   return EAMassAssemble3D(ne,B,pa_data,ea_data,add,
+                                               dofs1D,quad1D);
       }
    }
    MFEM_ABORT("Unknown kernel.");

fem/bilininteg_transpose_ea.cpp

@@ -16,88 +16,171 @@ namespace mfem
 {
 
 void TransposeIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                     Vector &ea_data)
+                                     Vector &ea_data, const bool add)
 {
-   Vector ea_data_tmp(ea_data.Size());
-   ea_data_tmp = 0.0;
-   bfi->AssembleEA(fes, ea_data_tmp);
-   const int ne = fes.GetNE();
-   if (ne == 0) { return; }
-   const int dofs = fes.GetFE(0)->GetDof();
-   auto A = Reshape(ea_data_tmp.Write(), dofs, dofs, ne);
-   auto AT = Reshape(ea_data.ReadWrite(), dofs, dofs, ne);
-   MFEM_FORALL(e, ne,
+   if (add)
    {
-      for (int i = 0; i < dofs; i++)
+      Vector ea_data_tmp(ea_data.Size());
+      bfi->AssembleEA(fes, ea_data_tmp, false);
+      const int ne = fes.GetNE();
+      if (ne == 0) { return; }
+      const int dofs = fes.GetFE(0)->GetDof();
+      auto A = Reshape(ea_data_tmp.Read(), dofs, dofs, ne);
+      auto AT = Reshape(ea_data.ReadWrite(), dofs, dofs, ne);
+      MFEM_FORALL(e, ne,
       {
-         for (int j = 0; j < dofs; j++)
+         for (int i = 0; i < dofs; i++)
          {
-            const double a = A(i, j, e);
-            AT(j, i, e) += a;
+            for (int j = 0; j < dofs; j++)
+            {
+               const double a = A(i, j, e);
+               AT(j, i, e) += a;
+            }
          }
-      }
-   });
+      });
+   }
+   else
+   {
+      bfi->AssembleEA(fes, ea_data, false);
+      const int ne = fes.GetNE();
+      if (ne == 0) { return; }
+      const int dofs = fes.GetFE(0)->GetDof();
+      auto A = Reshape(ea_data.ReadWrite(), dofs, dofs, ne);
+      MFEM_FORALL(e, ne,
+      {
+         for (int i = 0; i < dofs; i++)
+         {
+            for (int j = i+1; j < dofs; j++)
+            {
+               const double aij = A(i, j, e);
+               const double aji = A(j, i, e);
+               A(j, i, e) = aij;
+               A(i, j, e) = aji;
+            }
+         }
+      });
+   }
 }
 
 void TransposeIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
                                                   Vector &ea_data_int,
-                                                  Vector &ea_data_ext)
+                                                  Vector &ea_data_ext,
+                                                  const bool add)
 {
    const int nf = fes.GetNFbyType(FaceType::Interior);
    if (nf == 0) { return; }
-   Vector ea_data_int_tmp(ea_data_int.Size());
-   Vector ea_data_ext_tmp(ea_data_ext.Size());
-   ea_data_int_tmp = 0.0;
-   ea_data_ext_tmp = 0.0;
-   bfi->AssembleEAInteriorFaces(fes, ea_data_int_tmp, ea_data_ext_tmp);
-   const int faceDofs = fes.GetTraceElement(0,
-                                            fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
-   auto A_int = Reshape(ea_data_int_tmp.Read(), faceDofs, faceDofs, 2, nf);
-   auto A_ext = Reshape(ea_data_ext_tmp.Read(), faceDofs, faceDofs, 2, nf);
-   auto AT_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
-   auto AT_ext = Reshape(ea_data_ext.ReadWrite(), faceDofs, faceDofs, 2, nf);
-   MFEM_FORALL(f, nf,
+   if (add)
    {
-      for (int i = 0; i < faceDofs; i++)
+      Vector ea_data_int_tmp(ea_data_int.Size());
+      Vector ea_data_ext_tmp(ea_data_ext.Size());
+      bfi->AssembleEAInteriorFaces(fes, ea_data_int_tmp, ea_data_ext_tmp, false);
+      const int faceDofs = fes.GetTraceElement(0,
+                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
+      auto A_int = Reshape(ea_data_int_tmp.Read(), faceDofs, faceDofs, 2, nf);
+      auto A_ext = Reshape(ea_data_ext_tmp.Read(), faceDofs, faceDofs, 2, nf);
+      auto AT_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
+      auto AT_ext = Reshape(ea_data_ext.ReadWrite(), faceDofs, faceDofs, 2, nf);
+      MFEM_FORALL(f, nf,
       {
-         for (int j = 0; j < faceDofs; j++)
+         for (int i = 0; i < faceDofs; i++)
          {
-            const double a_int0 = A_int(i, j, 0, f);
-            const double a_int1 = A_int(i, j, 1, f);
-            const double a_ext0 = A_ext(i, j, 0, f);
-            const double a_ext1 = A_ext(i, j, 1, f);
-            AT_int(j, i, 0, f) += a_int0;
-            AT_int(j, i, 1, f) += a_int1;
-            AT_ext(j, i, 0, f) += a_ext1;
-            AT_ext(j, i, 1, f) += a_ext0;
+            for (int j = 0; j < faceDofs; j++)
+            {
+               const double a_int0 = A_int(i, j, 0, f);
+               const double a_int1 = A_int(i, j, 1, f);
+               const double a_ext0 = A_ext(i, j, 0, f);
+               const double a_ext1 = A_ext(i, j, 1, f);
+               AT_int(j, i, 0, f) += a_int0;
+               AT_int(j, i, 1, f) += a_int1;
+               AT_ext(j, i, 0, f) += a_ext1;
+               AT_ext(j, i, 1, f) += a_ext0;
+            }
          }
-      }
-   });
+      });
+   }
+   else
+   {
+      bfi->AssembleEAInteriorFaces(fes, ea_data_int, ea_data_ext, false);
+      const int faceDofs = fes.GetTraceElement(0,
+                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
+      auto A_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
+      auto A_ext = Reshape(ea_data_ext.ReadWrite(), faceDofs, faceDofs, 2, nf);
+      MFEM_FORALL(f, nf,
+      {
+         for (int i = 0; i < faceDofs; i++)
+         {
+            for (int j = i+1; j < faceDofs; j++)
+            {
+               const double aij_int0 = A_int(i, j, 0, f);
+               const double aij_int1 = A_int(i, j, 1, f);
+               const double aji_int0 = A_int(j, i, 0, f);
+               const double aji_int1 = A_int(j, i, 1, f);
+               A_int(j, i, 0, f) = aij_int0;
+               A_int(j, i, 1, f) = aij_int1;
+               A_int(i, j, 0, f) = aji_int0;
+               A_int(i, j, 1, f) = aji_int1;
+            }
+         }
+         for (int i = 0; i < faceDofs; i++)
+         {
+            for (int j = 0; j < faceDofs; j++)
+            {
+               const double aij_ext0 = A_ext(i, j, 0, f);
+               const double aji_ext1 = A_ext(j, i, 1, f);
+               A_ext(j, i, 1, f) = aij_ext0;
+               A_ext(i, j, 0, f) = aji_ext1;
+            }
+         }
+      });
+   }
 }
 
 void TransposeIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
-                                                  Vector &ea_data_bdr)
+                                                  Vector &ea_data_bdr,
+                                                  const bool add)
 {
    const int nf = fes.GetNFbyType(FaceType::Boundary);
    if (nf == 0) { return; }
-   Vector ea_data_bdr_tmp(ea_data_bdr.Size());
-   ea_data_bdr_tmp = 0.0;
-   bfi->AssembleEABoundaryFaces(fes, ea_data_bdr_tmp);
-   const int faceDofs = fes.GetTraceElement(0,
-                                            fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
-   auto A_bdr = Reshape(ea_data_bdr_tmp.Read(), faceDofs, faceDofs, nf);
-   auto AT_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
-   MFEM_FORALL(f, nf,
+   if (add)
    {
-      for (int i = 0; i < faceDofs; i++)
+      Vector ea_data_bdr_tmp(ea_data_bdr.Size());
+      bfi->AssembleEABoundaryFaces(fes, ea_data_bdr_tmp, false);
+      const int faceDofs = fes.GetTraceElement(0,
+                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
+      auto A_bdr = Reshape(ea_data_bdr_tmp.Read(), faceDofs, faceDofs, nf);
+      auto AT_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
+      MFEM_FORALL(f, nf,
       {
-         for (int j = 0; j < faceDofs; j++)
+         for (int i = 0; i < faceDofs; i++)
          {
-            const double a_bdr = A_bdr(i, j, f);
-            AT_bdr(j, i, f) += a_bdr;
+            for (int j = 0; j < faceDofs; j++)
+            {
+               const double a_bdr = A_bdr(i, j, f);
+               AT_bdr(j, i, f) += a_bdr;
+            }
          }
-      }
-   });
+      });
+   }
+   else
+   {
+      bfi->AssembleEABoundaryFaces(fes, ea_data_bdr, false);
+      const int faceDofs = fes.GetTraceElement(0,
+                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
+      auto A_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
+      MFEM_FORALL(f, nf,
+      {
+         for (int i = 0; i < faceDofs; i++)
+         {
+            for (int j = i+1; j < faceDofs; j++)
+            {
+               const double aij_bdr = A_bdr(i, j, f);
+               const double aji_bdr = A_bdr(j, i, f);
+               A_bdr(j, i, f) = aij_bdr;
+               A_bdr(i, j, f) = aji_bdr;
+            }
+         }
+      });
+   }
 }
 
 }

fem/bilininteg_vectorfe.cpp

@@ -20,7 +20,7 @@ void PAHcurlSetup2D(const int Q1D,
                     const int NE,
                     const Array<double> &w,
                     const Vector &j,
-                    Vector &_coeff,
+                    Vector &coeff,
                     Vector &op);
 
 void PAHcurlSetup3D(const int Q1D,
@@ -28,50 +28,73 @@ void PAHcurlSetup3D(const int Q1D,
                     const int NE,
                     const Array<double> &w,
                     const Vector &j,
-                    Vector &_coeff,
+                    Vector &coeff,
                     Vector &op);
 
 void PAHcurlMassAssembleDiagonal2D(const int D1D,
                                    const int Q1D,
                                    const int NE,
                                    const bool symmetric,
-                                   const Array<double> &_Bo,
-                                   const Array<double> &_Bc,
-                                   const Vector &_op,
-                                   Vector &_diag);
+                                   const Array<double> &bo,
+                                   const Array<double> &bc,
+                                   const Vector &pa_data,
+                                   Vector &diag);
 
 void PAHcurlMassAssembleDiagonal3D(const int D1D,
                                    const int Q1D,
                                    const int NE,
                                    const bool symmetric,
-                                   const Array<double> &_Bo,
-                                   const Array<double> &_Bc,
-                                   const Vector &_op,
-                                   Vector &_diag);
+                                   const Array<double> &bo,
+                                   const Array<double> &bc,
+                                   const Vector &pa_data,
+                                   Vector &diag);
+
+template<int T_D1D = 0, int T_Q1D = 0>
+void SmemPAHcurlMassAssembleDiagonal3D(const int D1D,
+                                       const int Q1D,
+                                       const int NE,
+                                       const bool symmetric,
+                                       const Array<double> &bo,
+                                       const Array<double> &bc,
+                                       const Vector &pa_data,
+                                       Vector &diag);
 
 void PAHcurlMassApply2D(const int D1D,
                         const int Q1D,
                         const int NE,
                         const bool symmetric,
-                        const Array<double> &_Bo,
-                        const Array<double> &_Bc,
-                        const Array<double> &_Bot,
-                        const Array<double> &_Bct,
-                        const Vector &_op,
-                        const Vector &_x,
-                        Vector &_y);
+                        const Array<double> &bo,
+                        const Array<double> &bc,
+                        const Array<double> &bot,
+                        const Array<double> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y);
 
 void PAHcurlMassApply3D(const int D1D,
                         const int Q1D,
                         const int NE,
                         const bool symmetric,
-                        const Array<double> &_Bo,
-                        const Array<double> &_Bc,
-                        const Array<double> &_Bot,
-                        const Array<double> &_Bct,
-                        const Vector &_op,
-                        const Vector &_x,
-                        Vector &_y);
+                        const Array<double> &bo,
+                        const Array<double> &bc,
+                        const Array<double> &bot,
+                        const Array<double> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y);
+
+template<int T_D1D = 0, int T_Q1D = 0>
+void SmemPAHcurlMassApply3D(const int D1D,
+                            const int Q1D,
+                            const int NE,
+                            const bool symmetric,
+                            const Array<double> &bo,
+                            const Array<double> &bc,
+                            const Array<double> &bot,
+                            const Array<double> &bct,
+                            const Vector &pa_data,
+                            const Vector &x,
+                            Vector &y);
 
 void PAHdivSetup2D(const int Q1D,
                    const int NE,
@@ -90,24 +113,24 @@ void PAHdivSetup3D(const int Q1D,
 void PAHcurlH1Apply2D(const int D1D,
                       const int Q1D,
                       const int NE,
-                      const Array<double> &_Bc,
-                      const Array<double> &_Gc,
-                      const Array<double> &_Bot,
-                      const Array<double> &_Bct,
-                      const Vector &_op,
-                      const Vector &_x,
-                      Vector &_y);
+                      const Array<double> &bc,
+                      const Array<double> &gc,
+                      const Array<double> &bot,
+                      const Array<double> &bct,
+                      const Vector &pa_data,
+                      const Vector &x,
+                      Vector &y);
 
 void PAHcurlH1Apply3D(const int D1D,
                       const int Q1D,
                       const int NE,
-                      const Array<double> &_Bc,
-                      const Array<double> &_Gc,
-                      const Array<double> &_Bot,
-                      const Array<double> &_Bct,
-                      const Vector &_op,
-                      const Vector &_x,
-                      Vector &_y);
+                      const Array<double> &bc,
+                      const Array<double> &gc,
+                      const Array<double> &bot,
+                      const Array<double> &bct,
+                      const Vector &pa_data,
+                      const Vector &x,
+                      Vector &y);
 
 void PAHdivMassAssembleDiagonal2D(const int D1D,
                                   const int Q1D,
@@ -881,8 +904,30 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
    {
       if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
       {
-         PAHcurlMassAssembleDiagonal3D(dofs1D, quad1D, ne, symmetric,
-                                       mapsO->B, mapsC->B, pa_data, diag);
+         if (Device::Allows(Backend::DEVICE_MASK))
+         {
+            const int ID = (dofs1D << 4) | quad1D;
+            switch (ID)
+            {
+               case 0x23: return SmemPAHcurlMassAssembleDiagonal3D<2,3>(dofs1D, quad1D, ne,
+                                                                           symmetric,
+                                                                           mapsO->B, mapsC->B, pa_data, diag);
+               case 0x34: return SmemPAHcurlMassAssembleDiagonal3D<3,4>(dofs1D, quad1D, ne,
+                                                                           symmetric,
+                                                                           mapsO->B, mapsC->B, pa_data, diag);
+               case 0x45: return SmemPAHcurlMassAssembleDiagonal3D<4,5>(dofs1D, quad1D, ne,
+                                                                           symmetric,
+                                                                           mapsO->B, mapsC->B, pa_data, diag);
+               case 0x56: return SmemPAHcurlMassAssembleDiagonal3D<5,6>(dofs1D, quad1D, ne,
+                                                                           symmetric,
+                                                                           mapsO->B, mapsC->B, pa_data, diag);
+               default: return SmemPAHcurlMassAssembleDiagonal3D(dofs1D, quad1D, ne, symmetric,
+                                                                    mapsO->B, mapsC->B, pa_data, diag);
+            }
+         }
+         else
+            PAHcurlMassAssembleDiagonal3D(dofs1D, quad1D, ne, symmetric,
+                                          mapsO->B, mapsC->B, pa_data, diag);
       }
       else if (trial_fetype == mfem::FiniteElement::DIV &&
                test_fetype == trial_fetype)
@@ -926,8 +971,35 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
    {
       if (trial_curl && test_curl)
       {
-         PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
-                            mapsO->Bt, mapsC->Bt, pa_data, x, y);
+         if (Device::Allows(Backend::DEVICE_MASK))
+         {
+            const int ID = (dofs1D << 4) | quad1D;
+            switch (ID)
+            {
+               case 0x23: return SmemPAHcurlMassApply3D<2,3>(dofs1D, quad1D, ne, symmetric,
+                                                                mapsO->B,
+                                                                mapsC->B, mapsO->Bt,
+                                                                mapsC->Bt, pa_data, x, y);
+               case 0x34: return SmemPAHcurlMassApply3D<3,4>(dofs1D, quad1D, ne, symmetric,
+                                                                mapsO->B,
+                                                                mapsC->B, mapsO->Bt,
+                                                                mapsC->Bt, pa_data, x, y);
+               case 0x45: return SmemPAHcurlMassApply3D<4,5>(dofs1D, quad1D, ne, symmetric,
+                                                                mapsO->B,
+                                                                mapsC->B, mapsO->Bt,
+                                                                mapsC->Bt, pa_data, x, y);
+               case 0x56: return SmemPAHcurlMassApply3D<5,6>(dofs1D, quad1D, ne, symmetric,
+                                                                mapsO->B,
+                                                                mapsC->B, mapsO->Bt,
+                                                                mapsC->Bt, pa_data, x, y);
+               default: return SmemPAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B,
+                                                         mapsC->B,
+                                                         mapsO->Bt, mapsC->Bt, pa_data, x, y);
+            }
+         }
+         else
+            PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B, mapsO->Bt,
+                               mapsC->Bt, pa_data, x, y);
       }
       else if (trial_div && test_div)
       {

fem/complex_fem.cpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "complex_fem.hpp"
+#include "../general/forall.hpp"
 
 using namespace std;
 
@@ -19,16 +20,21 @@ namespace mfem
 ComplexGridFunction::ComplexGridFunction(FiniteElementSpace *fes)
    : Vector(2*(fes->GetVSize()))
 {
-   gfr = new GridFunction(fes, data);
-   gfi = new GridFunction(fes, &data[fes->GetVSize()]);
+   UseDevice(true);
+   this->Vector::operator=(0.0);
+
+   gfr = new GridFunction();
+   gfr->MakeRef(fes, *this, 0);
+
+   gfi = new GridFunction();
+   gfi->MakeRef(fes, *this, fes->GetVSize());
 }
 
 void
 ComplexGridFunction::Update()
 {
-   FiniteElementSpace * fes = gfr->FESpace();
-
-   int vsize = fes->GetVSize();
+   FiniteElementSpace *fes = gfr->FESpace();
+   const int vsize = fes->GetVSize();
 
    const Operator *T = fes->GetUpdateOperator();
    if (T)
@@ -40,30 +46,36 @@ ComplexGridFunction::Update()
 
       // Our data array now contains old data as well as being the wrong size so
       // reallocate it.
+      UseDevice(true);
       this->SetSize(2 * vsize);
+      this->Vector::operator=(0.0);
 
       // Create temporary vectors which point to the new data array
-      Vector gf_r(data, vsize);
-      Vector gf_i((data) ? &data[vsize] : data, vsize);
+      Vector gf_r; gf_r.MakeRef(*this, 0, vsize);
+      Vector gf_i; gf_i.MakeRef(*this, vsize, vsize);
 
       // Copy the updated GridFunctions into the new data array
       gf_r = *gfr;
       gf_i = *gfi;
+      gf_r.SyncAliasMemory(*this);
+      gf_i.SyncAliasMemory(*this);
 
       // Replace the individual data arrays with pointers into the new data
       // array
-      gfr->NewDataAndSize(data, vsize);
-      gfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
+      gfr->MakeRef(*this, 0, vsize);
+      gfi->MakeRef(*this, vsize, vsize);
    }
    else
    {
       // The existing data will not be transferred to the new GridFunctions so
-      // delete it a allocate a new array
+      // delete it and allocate a new array
+      UseDevice(true);
       this->SetSize(2 * vsize);
+      this->Vector::operator=(0.0);
 
       // Point the individual GridFunctions to the new data array
-      gfr->NewDataAndSize(data, vsize);
-      gfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
+      gfr->MakeRef(*this, 0, vsize);
+      gfi->MakeRef(*this, vsize, vsize);
 
       // These updates will only set the proper 'sequence' value within the
       // individual GridFunction objects because their sizes are already correct
@@ -76,16 +88,24 @@ void
 ComplexGridFunction::ProjectCoefficient(Coefficient &real_coeff,
                                         Coefficient &imag_coeff)
 {
+   gfr->SyncMemory(*this);
+   gfi->SyncMemory(*this);
    gfr->ProjectCoefficient(real_coeff);
    gfi->ProjectCoefficient(imag_coeff);
+   gfr->SyncAliasMemory(*this);
+   gfi->SyncAliasMemory(*this);
 }
 
 void
 ComplexGridFunction::ProjectCoefficient(VectorCoefficient &real_vcoeff,
                                         VectorCoefficient &imag_vcoeff)
 {
+   gfr->SyncMemory(*this);
+   gfi->SyncMemory(*this);
    gfr->ProjectCoefficient(real_vcoeff);
    gfi->ProjectCoefficient(imag_vcoeff);
+   gfr->SyncAliasMemory(*this);
+   gfi->SyncAliasMemory(*this);
 }
 
 void
@@ -93,8 +113,12 @@ ComplexGridFunction::ProjectBdrCoefficient(Coefficient &real_coeff,
                                            Coefficient &imag_coeff,
                                            Array<int> &attr)
 {
+   gfr->SyncMemory(*this);
+   gfi->SyncMemory(*this);
    gfr->ProjectBdrCoefficient(real_coeff, attr);
    gfi->ProjectBdrCoefficient(imag_coeff, attr);
+   gfr->SyncAliasMemory(*this);
+   gfi->SyncAliasMemory(*this);
 }
 
 void
@@ -102,8 +126,12 @@ ComplexGridFunction::ProjectBdrCoefficientNormal(VectorCoefficient &real_vcoeff,
                                                  VectorCoefficient &imag_vcoeff,
                                                  Array<int> &attr)
 {
+   gfr->SyncMemory(*this);
+   gfi->SyncMemory(*this);
    gfr->ProjectBdrCoefficientNormal(real_vcoeff, attr);
    gfi->ProjectBdrCoefficientNormal(imag_vcoeff, attr);
+   gfr->SyncAliasMemory(*this);
+   gfi->SyncAliasMemory(*this);
 }
 
 void
@@ -113,18 +141,28 @@ ComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient
                                                   &imag_vcoeff,
                                                   Array<int> &attr)
 {
+   gfr->SyncMemory(*this);
+   gfi->SyncMemory(*this);
    gfr->ProjectBdrCoefficientTangent(real_vcoeff, attr);
    gfi->ProjectBdrCoefficientTangent(imag_vcoeff, attr);
+   gfr->SyncAliasMemory(*this);
+   gfi->SyncAliasMemory(*this);
 }
 
 
-ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *f,
+ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *fes,
                                      ComplexOperator::Convention convention)
-   : Vector(2*(f->GetVSize())),
+   : Vector(2*(fes->GetVSize())),
      conv(convention)
 {
-   lfr = new LinearForm(f, data);
-   lfi = new LinearForm(f, &data[f->GetVSize()]);
+   UseDevice(true);
+   this->Vector::operator=(0.0);
+
+   lfr = new LinearForm();
+   lfr->MakeRef(fes, *this, 0);
+
+   lfi = new LinearForm();
+   lfi->MakeRef(fes, *this, fes->GetVSize());
 }
 
 ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *fes,
@@ -133,8 +171,14 @@ ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *fes,
    : Vector(2*(fes->GetVSize())),
      conv(convention)
 {
-   lfr = new LinearForm(fes, lf_r);  lfr->SetData(data);
-   lfi = new LinearForm(fes, lf_i);  lfi->SetData(&data[fes->GetVSize()]);
+   UseDevice(true);
+   this->Vector::operator=(0.0);
+
+   lfr = new LinearForm(fes, lf_r);
+   lfi = new LinearForm(fes, lf_i);
+
+   lfr->MakeRef(fes, *this, 0);
+   lfi->MakeRef(fes, *this, fes->GetVSize());
 }
 
 ComplexLinearForm::~ComplexLinearForm()
@@ -189,42 +233,43 @@ void
 ComplexLinearForm::Update()
 {
    FiniteElementSpace *fes = lfr->FESpace();
-
    this->Update(fes);
 }
 
 void
 ComplexLinearForm::Update(FiniteElementSpace *fes)
 {
-   int vsize = fes->GetVSize();
-   SetSize(2 * vsize);
+   UseDevice(true);
+   SetSize(2 * fes->GetVSize());
+   this->Vector::operator=(0.0);
 
-   Vector vlfr(data, vsize);
-   Vector vlfi((data) ? &data[vsize] : data, vsize);
-
-   lfr->Update(fes, vlfr, 0);
-   lfi->Update(fes, vlfi, 0);
+   lfr->MakeRef(fes, *this, 0);
+   lfi->MakeRef(fes, *this, fes->GetVSize());
 }
 
 void
 ComplexLinearForm::Assemble()
 {
+   lfr->SyncMemory(*this);
+   lfi->SyncMemory(*this);
    lfr->Assemble();
    lfi->Assemble();
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
-   {
-      *lfi *= -1.0;
-   }
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC) { *lfi *= -1.0; }
+   lfr->SyncAliasMemory(*this);
+   lfi->SyncAliasMemory(*this);
 }
 
 complex<double>
 ComplexLinearForm::operator()(const ComplexGridFunction &gf) const
 {
-   double s = (conv == ComplexOperator::HERMITIAN)?1.0:-1.0;
+   double s = (conv == ComplexOperator::HERMITIAN) ? 1.0 : -1.0;
+   lfr->SyncMemory(*this);
+   lfi->SyncMemory(*this);
    return complex<double>((*lfr)(gf.real()) - s * (*lfi)(gf.imag()),
                           (*lfr)(gf.imag()) + s * (*lfi)(gf.real()));
 }
 
+
 bool SesquilinearForm::RealInteg()
 {
    int nint = blfr->GetFBFI()->Size() + blfr->GetDBFI()->Size() +
@@ -341,34 +386,45 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
                                    Vector &X, Vector &B,
                                    int ci)
 {
-   FiniteElementSpace * fes = blfr->FESpace();
-   int vsize  = fes->GetVSize();
+   FiniteElementSpace *fes = blfr->FESpace();
+   const int vsize = fes->GetVSize();
 
-   // Allocate temporary vectors
-   Vector b_0(vsize); b_0 = 0.0;
+   // Allocate temporary vector
+   Vector b_0;
+   b_0.UseDevice(true);
+   b_0.SetSize(vsize);
+   b_0 = 0.0;
 
    // Extract the real and imaginary parts of the input vectors
    MFEM_ASSERT(x.Size() == 2 * vsize, "Input GridFunction of incorrect size!");
-   Vector x_r(x.GetData(), vsize);
-   Vector x_i(&(x.GetData())[vsize], vsize);
+   x.Read();
+   Vector x_r; x_r.MakeRef(x, 0, vsize);
+   Vector x_i; x_i.MakeRef(x, vsize, vsize);
 
    MFEM_ASSERT(b.Size() == 2 * vsize, "Input LinearForm of incorrect size!");
-   Vector b_r(b.GetData(), vsize);
-   Vector b_i(&(b.GetData())[vsize], vsize);
+   b.Read();
+   Vector b_r; b_r.MakeRef(b, 0, vsize);
+   Vector b_i; b_i.MakeRef(b, vsize, vsize);
 
    if (conv == ComplexOperator::BLOCK_SYMMETRIC) { b_i *= -1.0; }
 
-   int tvsize = fes->GetTrueVSize();
+   const int tvsize = fes->GetTrueVSize();
    OperatorHandle A_r, A_i;
 
+   X.UseDevice(true);
    X.SetSize(2 * tvsize);
-   B.SetSize(2 * tvsize);
+   X = 0.0;
 
-   Vector X_0(tvsize), B_0(tvsize);
-   Vector X_r(X.GetData(),tvsize);
-   Vector X_i(&(X.GetData())[tvsize], tvsize);
-   Vector B_r(B.GetData(), tvsize);
-   Vector B_i(&(B.GetData())[tvsize], tvsize);
+   B.UseDevice(true);
+   B.SetSize(2 * tvsize);
+   B = 0.0;
+
+   Vector X_r; X_r.MakeRef(X, 0, tvsize);
+   Vector X_i; X_i.MakeRef(X, tvsize, tvsize);
+   Vector B_r; B_r.MakeRef(B, 0, tvsize);
+   Vector B_i; B_i.MakeRef(B, tvsize, tvsize);
+
+   Vector X_0, B_0;
 
    if (RealInteg())
    {
@@ -418,13 +474,18 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
       // conform with standard essential BC treatment
       if (A_i.Is<ConstrainedOperator>())
       {
-         int n = ess_tdof_list.Size();
-         for (int k = 0; k < n; k++)
+         const int n = ess_tdof_list.Size();
+         auto d_B_r = B_r.Write();
+         auto d_B_i = B_i.Write();
+         auto d_X_r = X_r.Read();
+         auto d_X_i = X_i.Read();
+         auto d_idx = ess_tdof_list.Read();
+         MFEM_FORALL(i, n,
          {
-            int j = ess_tdof_list[k];
-            B_r(j) = X_r(j);
-            B_i(j) = X_i(j);
-         }
+            const int j = d_idx[i];
+            d_B_r[j] = d_X_r[j];
+            d_B_i[j] = d_X_i[j];
+         });
          A_i.As<ConstrainedOperator>()->SetDiagonalPolicy
          (mfem::Operator::DiagonalPolicy::DIAG_ZERO);
       }
@@ -436,6 +497,16 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
       b_i *= -1.0;
    }
 
+   x_r.SyncAliasMemory(x);
+   x_i.SyncAliasMemory(x);
+   b_r.SyncAliasMemory(b);
+   b_i.SyncAliasMemory(b);
+
+   X_r.SyncAliasMemory(X);
+   X_i.SyncAliasMemory(X);
+   B_r.SyncAliasMemory(B);
+   B_i.SyncAliasMemory(B);
+
    // A = A_r + i A_i
    A.Clear();
    if ( A_r.Type() == Operator::MFEM_SPARSEMAT ||
@@ -528,29 +599,32 @@ void
 SesquilinearForm::RecoverFEMSolution(const Vector &X, const Vector &b,
                                      Vector &x)
 {
-   FiniteElementSpace * fes = blfr->FESpace();
+   FiniteElementSpace *fes = blfr->FESpace();
 
    const SparseMatrix *P = fes->GetConformingProlongation();
-
-   int vsize  = fes->GetVSize();
-   int tvsize = X.Size() / 2;
-
-   Vector X_r(X.GetData(), tvsize);
-   Vector X_i(&(X.GetData())[tvsize], tvsize);
-
-   Vector x_r(x.GetData(), vsize);
-   Vector x_i(&(x.GetData())[vsize], vsize);
-
    if (!P)
    {
       x = X;
+      return;
    }
-   else
-   {
-      // Apply conforming prolongation
-      P->Mult(X_r, x_r);
-      P->Mult(X_i, x_i);
-   }
+
+   const int vsize  = fes->GetVSize();
+   const int tvsize = X.Size() / 2;
+
+   X.Read();
+   Vector X_r; X_r.MakeRef(const_cast<Vector&>(X), 0, tvsize);
+   Vector X_i; X_i.MakeRef(const_cast<Vector&>(X), tvsize, tvsize);
+
+   x.Write();
+   Vector x_r; x_r.MakeRef(x, 0, vsize);
+   Vector x_i; x_i.MakeRef(x, vsize, vsize);
+
+   // Apply conforming prolongation
+   P->Mult(X_r, x_r);
+   P->Mult(X_i, x_i);
+
+   x_r.SyncAliasMemory(x);
+   x_i.SyncAliasMemory(x);
 }
 
 void
@@ -566,16 +640,21 @@ SesquilinearForm::Update(FiniteElementSpace *nfes)
 ParComplexGridFunction::ParComplexGridFunction(ParFiniteElementSpace *pfes)
    : Vector(2*(pfes->GetVSize()))
 {
-   pgfr = new ParGridFunction(pfes, data);
-   pgfi = new ParGridFunction(pfes, (data) ? &data[pfes->GetVSize()]:data);
+   UseDevice(true);
+   this->Vector::operator=(0.0);
+
+   pgfr = new ParGridFunction();
+   pgfr->MakeRef(pfes, *this, 0);
+
+   pgfi = new ParGridFunction();
+   pgfi->MakeRef(pfes, *this, pfes->GetVSize());
 }
 
 void
 ParComplexGridFunction::Update()
 {
-   ParFiniteElementSpace * pfes = pgfr->ParFESpace();
-
-   int vsize = pfes->GetVSize();
+   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
+   const int vsize = pfes->GetVSize();
 
    const Operator *T = pfes->GetUpdateOperator();
    if (T)
@@ -587,30 +666,34 @@ ParComplexGridFunction::Update()
 
       // Our data array now contains old data as well as being the wrong size so
       // reallocate it.
+      UseDevice(true);
       this->SetSize(2 * vsize);
+      this->Vector::operator=(0.0);
 
       // Create temporary vectors which point to the new data array
-      Vector gf_r(data, vsize);
-      Vector gf_i((data) ? &data[vsize] : data, vsize);
+      Vector gf_r; gf_r.MakeRef(*this, 0, vsize);
+      Vector gf_i; gf_i.MakeRef(*this, vsize, vsize);
 
       // Copy the updated GridFunctions into the new data array
-      gf_r = *pgfr;
-      gf_i = *pgfi;
+      gf_r = *pgfr; gf_r.SyncAliasMemory(*this);
+      gf_i = *pgfi; gf_i.SyncAliasMemory(*this);
 
       // Replace the individual data arrays with pointers into the new data
       // array
-      pgfr->NewDataAndSize(data, vsize);
-      pgfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
+      pgfr->MakeRef(*this, 0, vsize);
+      pgfi->MakeRef(*this, vsize, vsize);
    }
    else
    {
       // The existing data will not be transferred to the new GridFunctions so
-      // delete it a allocate a new array
+      // delete it and allocate a new array
+      UseDevice(true);
       this->SetSize(2 * vsize);
+      this->Vector::operator=(0.0);
 
       // Point the individual GridFunctions to the new data array
-      pgfr->NewDataAndSize(data, vsize);
-      pgfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
+      pgfr->MakeRef(*this, 0, vsize);
+      pgfi->MakeRef(*this, vsize, vsize);
 
       // These updates will only set the proper 'sequence' value within the
       // individual GridFunction objects because their sizes are already correct
@@ -623,16 +706,24 @@ void
 ParComplexGridFunction::ProjectCoefficient(Coefficient &real_coeff,
                                            Coefficient &imag_coeff)
 {
+   pgfr->SyncMemory(*this);
+   pgfi->SyncMemory(*this);
    pgfr->ProjectCoefficient(real_coeff);
    pgfi->ProjectCoefficient(imag_coeff);
+   pgfr->SyncAliasMemory(*this);
+   pgfi->SyncAliasMemory(*this);
 }
 
 void
 ParComplexGridFunction::ProjectCoefficient(VectorCoefficient &real_vcoeff,
                                            VectorCoefficient &imag_vcoeff)
 {
+   pgfr->SyncMemory(*this);
+   pgfi->SyncMemory(*this);
    pgfr->ProjectCoefficient(real_vcoeff);
    pgfi->ProjectCoefficient(imag_vcoeff);
+   pgfr->SyncAliasMemory(*this);
+   pgfi->SyncAliasMemory(*this);
 }
 
 void
@@ -640,8 +731,12 @@ ParComplexGridFunction::ProjectBdrCoefficient(Coefficient &real_coeff,
                                               Coefficient &imag_coeff,
                                               Array<int> &attr)
 {
+   pgfr->SyncMemory(*this);
+   pgfi->SyncMemory(*this);
    pgfr->ProjectBdrCoefficient(real_coeff, attr);
    pgfi->ProjectBdrCoefficient(imag_coeff, attr);
+   pgfr->SyncAliasMemory(*this);
+   pgfi->SyncAliasMemory(*this);
 }
 
 void
@@ -651,8 +746,12 @@ ParComplexGridFunction::ProjectBdrCoefficientNormal(VectorCoefficient
                                                     &imag_vcoeff,
                                                     Array<int> &attr)
 {
+   pgfr->SyncMemory(*this);
+   pgfi->SyncMemory(*this);
    pgfr->ProjectBdrCoefficientNormal(real_vcoeff, attr);
    pgfi->ProjectBdrCoefficientNormal(imag_vcoeff, attr);
+   pgfr->SyncAliasMemory(*this);
+   pgfi->SyncAliasMemory(*this);
 }
 
 void
@@ -662,36 +761,51 @@ ParComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient
                                                      &imag_vcoeff,
                                                      Array<int> &attr)
 {
+   pgfr->SyncMemory(*this);
+   pgfi->SyncMemory(*this);
    pgfr->ProjectBdrCoefficientTangent(real_vcoeff, attr);
    pgfi->ProjectBdrCoefficientTangent(imag_vcoeff, attr);
+   pgfr->SyncAliasMemory(*this);
+   pgfi->SyncAliasMemory(*this);
 }
 
 void
 ParComplexGridFunction::Distribute(const Vector *tv)
 {
-   ParFiniteElementSpace * pfes = pgfr->ParFESpace();
-   HYPRE_Int size = pfes->GetTrueVSize();
+   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
+   const int tvsize = pfes->GetTrueVSize();
 
-   double * tvd = tv->GetData();
-   Vector tvr(tvd, size);
-   Vector tvi((tvd) ? &tvd[size] : tvd, size);
+   tv->Read();
+   Vector tvr; tvr.MakeRef(const_cast<Vector&>(*tv), 0, tvsize);
+   Vector tvi; tvi.MakeRef(const_cast<Vector&>(*tv), tvsize, tvsize);
 
+   pgfr->SyncMemory(*this);
+   pgfi->SyncMemory(*this);
    pgfr->Distribute(tvr);
    pgfi->Distribute(tvi);
+   pgfr->SyncAliasMemory(*this);
+   pgfi->SyncAliasMemory(*this);
 }
 
 void
 ParComplexGridFunction::ParallelProject(Vector &tv) const
 {
-   ParFiniteElementSpace * pfes = pgfr->ParFESpace();
-   HYPRE_Int size = pfes->GetTrueVSize();
+   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
+   const int tvsize = pfes->GetTrueVSize();
 
-   double * tvd = tv.GetData();
-   Vector tvr(tvd, size);
-   Vector tvi((tvd) ? &tvd[size] : tvd, size);
+   tv.Write();
+   Vector tvr; tvr.MakeRef(tv, 0, tvsize);
+   Vector tvi; tvi.MakeRef(tv, tvsize, tvsize);
 
+   pgfr->SyncMemory(*this);
+   pgfi->SyncMemory(*this);
    pgfr->ParallelProject(tvr);
    pgfi->ParallelProject(tvi);
+   pgfr->SyncAliasMemory(*this);
+   pgfi->SyncAliasMemory(*this);
+
+   tvr.SyncAliasMemory(tv);
+   tvi.SyncAliasMemory(tv);
 }
 
 
@@ -701,10 +815,16 @@ ParComplexLinearForm::ParComplexLinearForm(ParFiniteElementSpace *pfes,
    : Vector(2*(pfes->GetVSize())),
      conv(convention)
 {
-   plfr = new ParLinearForm(pfes, data);
-   plfi = new ParLinearForm(pfes, (data) ? &data[pfes->GetVSize()]:data);
+   UseDevice(true);
+   this->Vector::operator=(0.0);
 
-   HYPRE_Int * tdof_offsets_fes = pfes->GetTrueDofOffsets();
+   plfr = new ParLinearForm();
+   plfr->MakeRef(pfes, *this, 0);
+
+   plfi = new ParLinearForm();
+   plfi->MakeRef(pfes, *this, pfes->GetVSize());
+
+   HYPRE_Int *tdof_offsets_fes = pfes->GetTrueDofOffsets();
 
    int n = (HYPRE_AssumedPartitionCheck()) ? 2 : pfes->GetNRanks();
    tdof_offsets = new HYPRE_Int[n+1];
@@ -724,12 +844,16 @@ ParComplexLinearForm::ParComplexLinearForm(ParFiniteElementSpace *pfes,
    : Vector(2*(pfes->GetVSize())),
      conv(convention)
 {
-   plfr = new ParLinearForm(pfes, plf_r);
-   plfr->SetData(data);
-   plfi = new ParLinearForm(pfes, plf_i);
-   plfi->SetData((data) ? &data[pfes->GetVSize()]:data);
+   UseDevice(true);
+   this->Vector::operator=(0.0);
 
-   HYPRE_Int * tdof_offsets_fes = pfes->GetTrueDofOffsets();
+   plfr = new ParLinearForm(pfes, plf_r);
+   plfi = new ParLinearForm(pfes, plf_i);
+
+   plfr->MakeRef(pfes, *this, 0);
+   plfi->MakeRef(pfes, *this, pfes->GetVSize());
+
+   HYPRE_Int *tdof_offsets_fes = pfes->GetTrueDofOffsets();
 
    int n = (HYPRE_AssumedPartitionCheck()) ? 2 : pfes->GetNRanks();
    tdof_offsets = new HYPRE_Int[n+1];
@@ -792,58 +916,71 @@ ParComplexLinearForm::AddBdrFaceIntegrator(LinearFormIntegrator *lfi_real,
 void
 ParComplexLinearForm::Update(ParFiniteElementSpace *pf)
 {
-   ParFiniteElementSpace *pfes = (pf!=NULL)?pf:plfr->ParFESpace();
-   int vsize = pfes->GetVSize();
-   SetSize(2 * vsize);
+   ParFiniteElementSpace *pfes = (pf != NULL) ? pf : plfr->ParFESpace();
 
-   Vector vplfr(data, vsize);
-   Vector vplfi((data) ? &data[vsize] : data, vsize);
+   UseDevice(true);
+   SetSize(2 * pfes->GetVSize());
+   this->Vector::operator=(0.0);
 
-   plfr->Update(pfes, vplfr, 0);
-   plfi->Update(pfes, vplfi, 0);
+   plfr->MakeRef(pfes, *this, 0);
+   plfi->MakeRef(pfes, *this, pfes->GetVSize());
 }
 
 void
 ParComplexLinearForm::Assemble()
 {
+   plfr->SyncMemory(*this);
+   plfi->SyncMemory(*this);
    plfr->Assemble();
    plfi->Assemble();
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
-   {
-      *plfi *= -1.0;
-   }
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC) { *plfi *= -1.0; }
+   plfr->SyncAliasMemory(*this);
+   plfi->SyncAliasMemory(*this);
 }
 
 void
 ParComplexLinearForm::ParallelAssemble(Vector &tv)
 {
-   HYPRE_Int size = plfr->ParFESpace()->GetTrueVSize();
+   const int tvsize = plfr->ParFESpace()->GetTrueVSize();
 
-   double * tvd = tv.GetData();
-   Vector tvr(tvd, size);
-   Vector tvi((tvd) ? &tvd[size] : tvd, size);
+   tv.Write();
+   Vector tvr; tvr.MakeRef(tv, 0, tvsize);
+   Vector tvi; tvi.MakeRef(tv, tvsize, tvsize);
 
+   plfr->SyncMemory(*this);
+   plfi->SyncMemory(*this);
    plfr->ParallelAssemble(tvr);
    plfi->ParallelAssemble(tvi);
+   plfr->SyncAliasMemory(*this);
+   plfi->SyncAliasMemory(*this);
+
+   tvr.SyncAliasMemory(tv);
+   tvi.SyncAliasMemory(tv);
 }
 
 HypreParVector *
 ParComplexLinearForm::ParallelAssemble()
 {
-   const ParFiniteElementSpace * pfes = plfr->ParFESpace();
+   const ParFiniteElementSpace *pfes = plfr->ParFESpace();
+   const int tvsize = pfes->GetTrueVSize();
 
-   HypreParVector * tv = new HypreParVector(pfes->GetComm(),
-                                            2*(pfes->GlobalTrueVSize()),
-                                            tdof_offsets);
+   HypreParVector *tv = new HypreParVector(pfes->GetComm(),
+                                           2*(pfes->GlobalTrueVSize()),
+                                           tdof_offsets);
 
-   HYPRE_Int size = pfes->GetTrueVSize();
-
-   double * tvd = tv->GetData();
-   Vector tvr(tvd, size);
-   Vector tvi((tvd) ? &tvd[size] : tvd, size);
+   tv->Write();
+   Vector tvr; tvr.MakeRef(*tv, 0, tvsize);
+   Vector tvi; tvi.MakeRef(*tv, tvsize, tvsize);
 
+   plfr->SyncMemory(*this);
+   plfi->SyncMemory(*this);
    plfr->ParallelAssemble(tvr);
    plfi->ParallelAssemble(tvi);
+   plfr->SyncAliasMemory(*this);
+   plfi->SyncAliasMemory(*this);
+
+   tvr.SyncAliasMemory(*tv);
+   tvi.SyncAliasMemory(*tv);
 
    return tv;
 }
@@ -851,13 +988,14 @@ ParComplexLinearForm::ParallelAssemble()
 complex<double>
 ParComplexLinearForm::operator()(const ParComplexGridFunction &gf) const
 {
-   double s = (conv == ComplexOperator::HERMITIAN)?1.0:-1.0;
+   plfr->SyncMemory(*this);
+   plfi->SyncMemory(*this);
+   double s = (conv == ComplexOperator::HERMITIAN) ? 1.0 : -1.0;
    return complex<double>((*plfr)(gf.real()) - s * (*plfi)(gf.imag()),
                           (*plfr)(gf.imag()) + s * (*plfi)(gf.real()));
 }
 
 
-
 bool ParSesquilinearForm::RealInteg()
 {
    int nint = pblfr->GetFBFI()->Size() + pblfr->GetDBFI()->Size() +
@@ -964,7 +1102,6 @@ ParSesquilinearForm::ParallelAssemble()
    return new ComplexHypreParMatrix(pblfr->ParallelAssemble(),
                                     pblfi->ParallelAssemble(),
                                     true, true, conv);
-
 }
 
 void
@@ -974,35 +1111,45 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
                                       Vector &X, Vector &B,
                                       int ci)
 {
-   ParFiniteElementSpace * pfes = pblfr->ParFESpace();
-   int vsize = pfes->GetVSize();
+   ParFiniteElementSpace *pfes = pblfr->ParFESpace();
+   const int vsize = pfes->GetVSize();
 
-   // Allocate temporary vectors
-   Vector b_0(vsize); b_0 = 0.0;
+   // Allocate temporary vector
+   Vector b_0;
+   b_0.UseDevice(true);
+   b_0.SetSize(vsize);
+   b_0 = 0.0;
 
    // Extract the real and imaginary parts of the input vectors
    MFEM_ASSERT(x.Size() == 2 * vsize, "Input GridFunction of incorrect size!");
-   Vector x_r(x.GetData(), vsize);
-   Vector x_i(&(x.GetData())[vsize], vsize);
+   x.Read();
+   Vector x_r; x_r.MakeRef(x, 0, vsize);
+   Vector x_i; x_i.MakeRef(x, vsize, vsize);
 
    MFEM_ASSERT(b.Size() == 2 * vsize, "Input LinearForm of incorrect size!");
-   Vector b_r(b.GetData(), vsize);
-   Vector b_i(&(b.GetData())[vsize], vsize);
+   b.Read();
+   Vector b_r; b_r.MakeRef(b, 0, vsize);
+   Vector b_i; b_i.MakeRef(b, vsize, vsize);
 
    if (conv == ComplexOperator::BLOCK_SYMMETRIC) { b_i *= -1.0; }
 
-   int tvsize = pfes->GetTrueVSize();
-
+   const int tvsize = pfes->GetTrueVSize();
    OperatorHandle A_r, A_i;
 
+   X.UseDevice(true);
    X.SetSize(2 * tvsize);
-   B.SetSize(2 * tvsize);
+   X = 0.0;
 
-   Vector X_0(tvsize), B_0(tvsize);
-   Vector X_r(X.GetData(),tvsize);
-   Vector X_i(&(X.GetData())[tvsize], tvsize);
-   Vector B_r(B.GetData(), tvsize);
-   Vector B_i(&(B.GetData())[tvsize], tvsize);
+   B.UseDevice(true);
+   B.SetSize(2 * tvsize);
+   B = 0.0;
+
+   Vector X_r; X_r.MakeRef(X, 0, tvsize);
+   Vector X_i; X_i.MakeRef(X, tvsize, tvsize);
+   Vector B_r; B_r.MakeRef(B, 0, tvsize);
+   Vector B_i; B_i.MakeRef(B, tvsize, tvsize);
+
+   Vector X_0, B_0;
 
    if (RealInteg())
    {
@@ -1042,24 +1189,29 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
 
    if (RealInteg() && ImagInteg())
    {
-      int n = ess_tdof_list.Size();
       // Modify RHS to conform with standard essential BC treatment
-      for (int k = 0; k < n; k++)
+      const int n = ess_tdof_list.Size();
+      auto d_B_r = B_r.Write();
+      auto d_B_i = B_i.Write();
+      auto d_X_r = X_r.Read();
+      auto d_X_i = X_i.Read();
+      auto d_idx = ess_tdof_list.Read();
+      MFEM_FORALL(i, n,
       {
-         int j=ess_tdof_list[k];
-         B_r(j) = X_r(j);
-         B_i(j) = X_i(j);
-      }
+         const int j = d_idx[i];
+         d_B_r[j] = d_X_r[j];
+         d_B_i[j] = d_X_i[j];
+      });
       // Modify offdiagonal blocks (imaginary parts of the matrix) to conform
       // with standard essential BC treatment
-      if ( A_i.Type() == Operator::Hypre_ParCSR )
+      if (A_i.Type() == Operator::Hypre_ParCSR)
       {
          HypreParMatrix * Ah;
          A_i.Get(Ah);
          hypre_ParCSRMatrix *Aih = *Ah;
          for (int k = 0; k < n; k++)
          {
-            int j = ess_tdof_list[k];
+            const int j = ess_tdof_list[k];
             Aih->diag->data[Aih->diag->i[j]] = 0.0;
          }
       }
@@ -1076,6 +1228,16 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
       b_i *= -1.0;
    }
 
+   x_r.SyncAliasMemory(x);
+   x_i.SyncAliasMemory(x);
+   b_r.SyncAliasMemory(b);
+   b_i.SyncAliasMemory(b);
+
+   X_r.SyncAliasMemory(X);
+   X_i.SyncAliasMemory(X);
+   B_r.SyncAliasMemory(B);
+   B_i.SyncAliasMemory(B);
+
    // A = A_r + i A_i
    A.Clear();
    if ( A_r.Type() == Operator::Hypre_ParCSR ||
@@ -1175,22 +1337,27 @@ void
 ParSesquilinearForm::RecoverFEMSolution(const Vector &X, const Vector &b,
                                         Vector &x)
 {
-   ParFiniteElementSpace * pfes = pblfr->ParFESpace();
+   ParFiniteElementSpace *pfes = pblfr->ParFESpace();
 
    const Operator &P = *pfes->GetProlongationMatrix();
 
-   int vsize  = pfes->GetVSize();
-   int tvsize = X.Size() / 2;
+   const int vsize  = pfes->GetVSize();
+   const int tvsize = X.Size() / 2;
 
-   Vector X_r(X.GetData(), tvsize);
-   Vector X_i(&(X.GetData())[tvsize], tvsize);
+   X.Read();
+   Vector X_r; X_r.MakeRef(const_cast<Vector&>(X), 0, tvsize);
+   Vector X_i; X_i.MakeRef(const_cast<Vector&>(X), tvsize, tvsize);
 
-   Vector x_r(x.GetData(), vsize);
-   Vector x_i(&(x.GetData())[vsize], vsize);
+   x.Write();
+   Vector x_r; x_r.MakeRef(x, 0, vsize);
+   Vector x_i; x_i.MakeRef(x, vsize, vsize);
 
    // Apply conforming prolongation
    P.Mult(X_r, x_r);
    P.Mult(X_i, x_i);
+
+   x_r.SyncAliasMemory(x);
+   x_i.SyncAliasMemory(x);
 }
 
 void

fem/complex_fem.hpp

@@ -38,8 +38,8 @@ protected:
    void Destroy() { delete gfr; delete gfi; }
 
 public:
-   /* @brief Construct a ComplexGridFunction associated with the
-      FiniteElementSpace @a *f. */
+   /** @brief Construct a ComplexGridFunction associated with the
+       FiniteElementSpace @a *f. */
    ComplexGridFunction(FiniteElementSpace *f);
 
    void Update();
@@ -71,6 +71,14 @@ public:
    const GridFunction & real() const { return *gfr; }
    const GridFunction & imag() const { return *gfi; }
 
+   /// Update the memory location of the real and imaginary GridFunction @a gfr
+   /// and @a gfi to match the ComplexGridFunction.
+   void Sync() { gfr->SyncMemory(*this); gfi->SyncMemory(*this); }
+
+   /// Update the alias memory location of the real and imaginary GridFunction
+   /// @a gfr and @a gfi to match the ComplexGridFunction.
+   void SyncAlias() { gfr->SyncAliasMemory(*this); gfi->SyncAliasMemory(*this); }
+
    /// Destroys the grid function.
    virtual ~ComplexGridFunction() { Destroy(); }
 
@@ -99,8 +107,8 @@ public:
                      ComplexOperator::Convention
                      convention = ComplexOperator::HERMITIAN);
 
-   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a f, using
-       the same integrators as the LinearForms @a lfr (real) and @a lfi (imag) .
+   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a fes, using
+       the same integrators as the LinearForms @a lf_r (real) and @a lf_i (imag).
 
        The pointer @a fes is not owned by the newly constructed object.
 
@@ -157,6 +165,14 @@ public:
    const LinearForm & real() const { return *lfr; }
    const LinearForm & imag() const { return *lfi; }
 
+   /// Update the memory location of the real and imaginary LinearForm @a lfr
+   /// and @a lfi to match the ComplexLinearForm.
+   void Sync() { lfr->SyncMemory(*this); lfi->SyncMemory(*this); }
+
+   /// Update the alias memory location of the real and imaginary LinearForm @a
+   /// lfr and @a lfi to match the ComplexLinearForm.
+   void SyncAlias() { lfr->SyncAliasMemory(*this); lfi->SyncAliasMemory(*this); }
+
    void Update();
    void Update(FiniteElementSpace *f);
 
@@ -195,8 +211,8 @@ private:
    BilinearForm *blfr;
    BilinearForm *blfi;
 
-   /* These methods check if the real/imag parts of the sesqulinear form are not
-      empty */
+   /* These methods check if the real/imag parts of the sesquilinear form are
+      not empty */
    bool RealInteg();
    bool ImagInteg();
 
@@ -204,7 +220,7 @@ public:
    SesquilinearForm(FiniteElementSpace *fes,
                     ComplexOperator::Convention
                     convention = ComplexOperator::HERMITIAN);
-   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a f, using
+   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a fes, using
        the same integrators as the BilinearForms @a bfr and @a bfi .
 
        The pointer @a fes is not owned by the newly constructed object.
@@ -323,8 +339,8 @@ protected:
 
 public:
 
-   /* @brief Construct a ParComplexGridFunction associated with the
-      ParFiniteElementSpace @a *f. */
+   /** @brief Construct a ParComplexGridFunction associated with the
+       ParFiniteElementSpace @a *pf. */
    ParComplexGridFunction(ParFiniteElementSpace *pf);
 
    void Update();
@@ -365,6 +381,15 @@ public:
    const ParGridFunction & real() const { return *pgfr; }
    const ParGridFunction & imag() const { return *pgfi; }
 
+   /// Update the memory location of the real and imaginary ParGridFunction @a
+   /// pgfr and @a pgfi to match the ParComplexGridFunction.
+   void Sync() { pgfr->SyncMemory(*this); pgfi->SyncMemory(*this); }
+
+   /// Update the alias memory location of the real and imaginary
+   /// ParGridFunction @a pgfr and @a pgfi to match the ParComplexGridFunction.
+   void SyncAlias() { pgfr->SyncAliasMemory(*this); pgfi->SyncAliasMemory(*this); }
+
+
    virtual double ComputeL2Error(Coefficient &exsolr, Coefficient &exsoli,
                                  const IntegrationRule *irs[] = NULL) const
    {
@@ -416,8 +441,8 @@ public:
                         convention = ComplexOperator::HERMITIAN);
 
    /** @brief Create a ParComplexLinearForm on the ParFiniteElementSpace @a pf,
-       using the same integrators as the LinearForms @a plfr (real) and @a plfi
-       (imag) .
+       using the same integrators as the LinearForms @a plf_r (real) and
+       @a plf_i (imag).
 
       The pointer @a fes is not owned by the newly constructed object.
 
@@ -475,6 +500,14 @@ public:
    const ParLinearForm & real() const { return *plfr; }
    const ParLinearForm & imag() const { return *plfi; }
 
+   /// Update the memory location of the real and imaginary ParLinearForm @a lfr
+   /// and @a lfi to match the ParComplexLinearForm.
+   void Sync() { plfr->SyncMemory(*this); plfi->SyncMemory(*this); }
+
+   /// Update the alias memory location of the real and imaginary ParLinearForm
+   /// @a plfr and @a plfi to match the ParComplexLinearForm.
+   void SyncAlias() { plfr->SyncAliasMemory(*this); plfi->SyncAliasMemory(*this); }
+
    void Update(ParFiniteElementSpace *pf = NULL);
 
    /// Assembles the linear form i.e. sums over all domain/bdr integrators.

fem/convergence.cpp

@@ -0,0 +1,297 @@
+#include "convergence.hpp"
+
+using namespace std;
+
+
+namespace mfem
+{
+
+void ConvergenceStudy::Reset()
+{
+   counter=0;
+   dcounter=0;
+   fcounter=0;
+   cont_type=-1;
+   print_flag=1;
+   L2Errors.SetSize(0);
+   L2Rates.SetSize(0);
+   DErrors.SetSize(0);
+   DRates.SetSize(0);
+   EnErrors.SetSize(0);
+   EnRates.SetSize(0);
+   DGFaceErrors.SetSize(0);
+   DGFaceRates.SetSize(0);
+   ndofs.SetSize(0);
+}
+
+double ConvergenceStudy::GetNorm(GridFunction *gf, Coefficient *scalar_u,
+                                 VectorCoefficient *vector_u)
+{
+   bool norm_set = false;
+   double norm=0.0;
+   int order = gf->FESpace()->GetOrder(0);
+   int order_quad = std::max(2, 2*order+1);
+   const IntegrationRule *irs[Geometry::NumGeom];
+   for (int i=0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+#ifdef MFEM_USE_MPI
+   ParGridFunction *pgf = dynamic_cast<ParGridFunction *>(gf);
+   if (pgf)
+   {
+      ParMesh *pmesh = pgf->ParFESpace()->GetParMesh();
+      if (scalar_u)
+      {
+         norm = ComputeGlobalLpNorm(2.0,*scalar_u,*pmesh,irs);
+      }
+      else if (vector_u)
+      {
+         norm = ComputeGlobalLpNorm(2.0,*vector_u,*pmesh,irs);
+      }
+      norm_set = true;
+   }
+#endif
+   if (!norm_set)
+   {
+      Mesh *mesh = gf->FESpace()->GetMesh();
+      if (scalar_u)
+      {
+         norm = ComputeLpNorm(2.0,*scalar_u,*mesh,irs);
+      }
+      else if (vector_u)
+      {
+         norm = ComputeLpNorm(2.0,*vector_u,*mesh,irs);
+      }
+   }
+   return norm;
+}
+
+void ConvergenceStudy::AddL2Error(GridFunction *gf,
+                                  Coefficient *scalar_u, VectorCoefficient *vector_u)
+{
+   int tdofs=0;
+#ifdef MFEM_USE_MPI
+   ParGridFunction *pgf = dynamic_cast<ParGridFunction *>(gf);
+   if (pgf)
+   {
+      MPI_Comm comm = pgf->ParFESpace()->GetComm();
+      int rank;
+      MPI_Comm_rank(comm, &rank);
+      print_flag = 0;
+      if (rank==0) { print_flag = 1; }
+      tdofs = pgf->ParFESpace()->GlobalTrueVSize();
+   }
+#endif
+   if (!tdofs) { tdofs = gf->FESpace()->GetTrueVSize(); }
+   ndofs.Append(tdofs);
+   double L2Err;
+   if (scalar_u)
+   {
+      L2Err = gf->ComputeL2Error(*scalar_u);
+      CoeffNorm = GetNorm(gf,scalar_u,nullptr);
+   }
+   else if (vector_u)
+   {
+      L2Err = gf->ComputeL2Error(*vector_u);
+      CoeffNorm = GetNorm(gf,nullptr,vector_u);
+   }
+   else
+   {
+      MFEM_ABORT("Exact Solution Coefficient pointer is NULL");
+   }
+   L2Errors.Append(L2Err);
+   // Compute the rate of convergence by:
+   // rate = log (||u - u_h|| / ||u - u_{h/2}||)/log(2)
+   double val = (counter) ? log(L2Errors[counter-1]/L2Err)/log(2.0) : 0.0;
+   L2Rates.Append(val);
+   counter++;
+}
+
+void ConvergenceStudy::AddGf(GridFunction *gf, Coefficient *scalar_u,
+                             VectorCoefficient *grad,
+                             Coefficient *ell_coeff, double Nu)
+{
+   cont_type = gf->FESpace()->FEColl()->GetContType();
+
+   MFEM_VERIFY((cont_type == mfem::FiniteElementCollection::CONTINUOUS) ||
+               (cont_type == mfem::FiniteElementCollection::DISCONTINUOUS),
+               "This constructor is intended for H1 or L2 Elements")
+
+   AddL2Error(gf,scalar_u, nullptr);
+
+   if (grad)
+   {
+      double GradErr = gf->ComputeGradError(grad);
+      DErrors.Append(GradErr);
+      double err = sqrt(L2Errors[counter-1]*L2Errors[counter-1]+GradErr*GradErr);
+      EnErrors.Append(err);
+      // Compute the rate of convergence by:
+      // rate = log (||u - u_h|| / ||u - u_{h/2}||)/log(2)
+      double val = (dcounter) ? log(DErrors[dcounter-1]/GradErr)/log(2.0) : 0.0;
+      double eval = (dcounter) ? log(EnErrors[dcounter-1]/err)/log(2.0) : 0.0;
+      DRates.Append(val);
+      EnRates.Append(eval);
+      CoeffDNorm = GetNorm(gf,nullptr,grad);
+      dcounter++;
+      MFEM_VERIFY(counter == dcounter,
+                  "Number of added solutions and derivatives do not match")
+   }
+
+   if (cont_type == mfem::FiniteElementCollection::DISCONTINUOUS && ell_coeff)
+   {
+      double DGErr = gf->ComputeDGFaceJumpError(scalar_u,ell_coeff,Nu);
+      DGFaceErrors.Append(DGErr);
+      // Compute the rate of convergence by:
+      // rate = log (||u - u_h|| / ||u - u_{h/2}||)/log(2)
+      double val=(fcounter) ? log(DGFaceErrors[fcounter-1]/DGErr)/log(2.0):0.;
+      DGFaceRates.Append(val);
+      fcounter++;
+      MFEM_VERIFY(fcounter == counter, "Number of added solutions mismatch");
+   }
+}
+
+void ConvergenceStudy::AddGf(GridFunction *gf, VectorCoefficient *vector_u,
+                             VectorCoefficient *curl, Coefficient *div)
+{
+   cont_type = gf->FESpace()->FEColl()->GetContType();
+
+   AddL2Error(gf,nullptr,vector_u);
+   double DErr = 0.0;
+   bool derivative = false;
+   if (curl)
+   {
+      DErr = gf->ComputeCurlError(curl);
+      CoeffDNorm = GetNorm(gf,nullptr,curl);
+      derivative = true;
+   }
+   else if (div)
+   {
+      DErr = gf->ComputeDivError(div);
+      // update coefficient norm
+      CoeffDNorm = GetNorm(gf,div,nullptr);
+      derivative = true;
+   }
+   if (derivative)
+   {
+      double err = sqrt(L2Errors[counter-1]*L2Errors[counter-1] + DErr*DErr);
+      DErrors.Append(DErr);
+      EnErrors.Append(err);
+      // Compute the rate of convergence by:
+      // rate = log (||u - u_h|| / ||u - u_{h/2}||)/log(2)
+      double val = (dcounter) ? log(DErrors[dcounter-1]/DErr)/log(2.0) : 0.0;
+      double eval = (dcounter) ? log(EnErrors[dcounter-1]/err)/log(2.0) : 0.0;
+      DRates.Append(val);
+      EnRates.Append(eval);
+      dcounter++;
+      MFEM_VERIFY(counter == dcounter,
+                  "Number of added solutions and derivatives do not match")
+   }
+}
+
+void ConvergenceStudy::Print(bool relative, std::ostream &out)
+{
+   if (print_flag)
+   {
+      std::string title = (relative) ? "Relative " : "Absolute ";
+      out << "\n";
+      out << " -------------------------------------------" << "\n";
+      out <<  std::setw(21) << title << "L2 Error " << "\n";
+      out << " -------------------------------------------"
+          << "\n";
+      out << std::right<< std::setw(11)<< "DOFs "<< std::setw(13) << "Error ";
+      out <<  std::setw(15) << "Rate " << "\n";
+      out << " -------------------------------------------"
+          << "\n";
+      out << std::setprecision(4);
+      double d = (relative) ? CoeffNorm : 1.0;
+      for (int i =0; i<counter; i++)
+      {
+         out << std::right << std::setw(10)<< ndofs[i] << std::setw(16)
+             << std::scientific << L2Errors[i]/d << std::setw(13)
+             << std::fixed << L2Rates[i] << "\n";
+      }
+      out << "\n";
+      if (dcounter == counter)
+      {
+         std::string dname;
+         switch (cont_type)
+         {
+            case 0: dname = "Grad"; break;
+            case 1: dname = "Curl"; break;
+            case 2: dname = "Div";  break;
+            case 3: dname = "DG Grad";  break;
+            default: break;
+         }
+         out << " -------------------------------------------" << "\n";
+         out <<  std::setw(21) << title << dname << " Error  " << "\n";
+         out << " -------------------------------------------" << "\n";
+         out << std::right<<std::setw(11)<< "DOFs "<< std::setw(13) << "Error";
+         out << std::setw(15) << "Rate " << "\n";
+         out << " -------------------------------------------"
+             << "\n";
+         out << std::setprecision(4);
+         d = (relative) ? CoeffDNorm : 1.0;
+         for (int i =0; i<dcounter; i++)
+         {
+            out << std::right << std::setw(10)<< ndofs[i] << std::setw(16)
+                << std::scientific << DErrors[i]/d << std::setw(13)
+                << std::fixed << DRates[i] << "\n";
+         }
+         out << "\n";
+         switch (cont_type)
+         {
+            case 0: dname = "H1"; break;
+            case 1: dname = "H(Curl)"; break;
+            case 2: dname = "H(Div)";  break;
+            case 3: dname = "DG H1";  break;
+            default: break;
+         }
+
+         if (dcounter)
+         {
+            d = (relative) ?
+                sqrt(CoeffNorm*CoeffNorm + CoeffDNorm*CoeffDNorm):1.0;
+
+            out << " -------------------------------------------" << "\n";
+            out << std::setw(21) << title << dname << " Error   " << "\n";
+            out << " -------------------------------------------" << "\n";
+            out << std::right<< std::setw(11)<< "DOFs "<< std::setw(13);
+            out << "Error ";
+            out << std::setw(15) << "Rate " << "\n";
+            out << " -------------------------------------------"
+                << "\n";
+            out << std::setprecision(4);
+            for (int i =0; i<dcounter; i++)
+            {
+               out << std::right << std::setw(10)<< ndofs[i] << std::setw(16)
+                   << std::scientific << EnErrors[i]/d << std::setw(13)
+                   << std::fixed << EnRates[i] << "\n";
+            }
+            out << "\n";
+         }
+         if (cont_type == 3 && fcounter)
+         {
+            out << " -------------------------------------------" << "\n";
+            out << "            DG Face Jump Error          " << "\n";
+            out << " -------------------------------------------"
+                << "\n";
+            out << std::right<< std::setw(11)<< "DOFs "<< std::setw(13);
+            out << "Error ";
+            out << std::setw(15) << "Rate " << "\n";
+            out << " -------------------------------------------"
+                << "\n";
+            out << std::setprecision(4);
+            for (int i =0; i<fcounter; i++)
+            {
+               out << std::right << std::setw(10)<< ndofs[i] << std::setw(16)
+                   << std::scientific << DGFaceErrors[i] << std::setw(13)
+                   << std::fixed << DGFaceRates[i] << "\n";
+            }
+            out << "\n";
+         }
+      }
+   }
+}
+
+} // namespace mfem

fem/convergence.hpp

@@ -0,0 +1,149 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_CONVERGENCE
+#define MFEM_CONVERGENCE
+
+#include "../linalg/linalg.hpp"
+#include "gridfunc.hpp"
+#ifdef MFEM_USE_MPI
+#include "pgridfunc.hpp"
+#endif
+
+namespace mfem
+{
+
+/** @brief Class to compute error and convergence rates.
+    It supports H1, H(curl) (ND elements), H(div) (RT elements) and L2 (DG).
+
+    For "smooth enough" solutions the Galerkin error measured in the appropriate
+    norm satisfies || u - u_h || ~ h^k
+
+    Here, k is called the asymptotic rate of convergence
+
+    For successive uniform h-refinements the rate can be estimated by
+    k = log(||u - u_h|| / ||u - u_{h/2}||)/log(2)
+*/
+class ConvergenceStudy
+{
+private:
+   // counters for solutions/derivatives
+   int counter=0;
+   int dcounter=0;
+   int fcounter=0;
+
+   // space continuity type
+   int cont_type=-1;
+
+   // printing flag for helpful for MPI calls
+   int print_flag=1;
+
+   // exact solution and derivatives
+   double CoeffNorm;
+   double CoeffDNorm;
+
+   // Arrays to store error/rates
+   Array<double> L2Errors, DGFaceErrors, DErrors, EnErrors;
+   Array<double> L2Rates, DGFaceRates, DRates, EnRates;
+   Array<int> ndofs;
+
+   void AddL2Error(GridFunction *gf, Coefficient *scalar_u,
+                   VectorCoefficient *vector_u);
+   void AddGf(GridFunction *gf, Coefficient *scalar_u,
+              VectorCoefficient *grad=nullptr,
+              Coefficient *ell_coeff=nullptr, double Nu=1.0);
+   void AddGf(GridFunction *gf, VectorCoefficient *vector_u,
+              VectorCoefficient *curl, Coefficient *div);
+   // returns the L2-norm of scalar_u or vector_u
+   double GetNorm(GridFunction *gf, Coefficient *scalar_u,
+                  VectorCoefficient *vector_u);
+
+public:
+
+   /// Clear any internal data
+   void Reset();
+
+   /// Add L2 GridFunction, the exact solution and possibly its gradient and/or
+   /// DG face jumps parameters
+   void AddL2GridFunction(GridFunction *gf, Coefficient *scalar_u,
+                          VectorCoefficient *grad=nullptr,
+                          Coefficient *ell_coeff=nullptr, double Nu=1.0)
+   {
+      AddGf(gf, scalar_u, grad, ell_coeff, Nu);
+   }
+
+   /// Add H1 GridFunction, the exact solution and possibly its gradient
+   void AddH1GridFunction(GridFunction *gf, Coefficient *scalar_u,
+                          VectorCoefficient *grad=nullptr)
+   {
+      AddGf(gf, scalar_u, grad);
+   }
+
+   /// Add H(curl) GridFunction, the exact solution and possibly its curl
+   void AddHcurlGridFunction(GridFunction *gf, VectorCoefficient *vector_u,
+                             VectorCoefficient *curl=nullptr)
+   {
+      AddGf(gf, vector_u, curl, nullptr);
+   }
+
+   /// Add H(div) GridFunction, the exact solution and possibly its div
+   void AddHdivGridFunction(GridFunction *gf, VectorCoefficient *vector_u,
+                            Coefficient *div=nullptr)
+   {
+      AddGf(gf,vector_u, nullptr, div);
+   }
+
+   /// Get the L2 error at step n
+   double GetL2Error(int n)
+   {
+      MFEM_VERIFY( n <= counter,"Step out of bounds")
+      return L2Errors[n];
+   }
+
+   /// Get all L2 errors
+   void GetL2Errors(Array<double> & L2Errors_)
+   {
+      L2Errors_ = L2Errors;
+   }
+
+   /// Get the Grad/Curl/Div error at step n
+   double GetDError(int n)
+   {
+      MFEM_VERIFY(n <= dcounter,"Step out of bounds")
+      return DErrors[n];
+   }
+
+   /// Get all Grad/Curl/Div errors
+   void GetDErrors(Array<double> & DErrors_)
+   {
+      DErrors_ = DErrors;
+   }
+
+   /// Get the DGFaceJumps error at step n
+   double GetDGFaceJumpsError(int n)
+   {
+      MFEM_VERIFY(n<= fcounter,"Step out of bounds")
+      return DGFaceErrors[n];
+   }
+
+   /// Get all DGFaceJumps errors
+   void GetDGFaceJumpsErrors(Array<double> & DGFaceErrors_)
+   {
+      DGFaceErrors_ = DGFaceErrors;
+   }
+
+   /// Print rates and errors
+   void Print(bool relative = false, std::ostream &out = mfem::out);
+};
+
+} // namespace mfem
+
+#endif // MFEM_CONVERGENCE

fem/datacollection.cpp

@@ -563,6 +563,8 @@ void VisItDataCollection::LoadVisItRootFile(const std::string& root_name)
 
 void VisItDataCollection::LoadMesh()
 {
+   // GetMeshFileName() uses 'serial', so we need to set it in advance.
+   serial = (format == SERIAL_FORMAT);
    std::string mesh_fname = GetMeshFileName();
    named_ifgzstream file(mesh_fname);
    // TODO: in parallel, check for errors on all processors

fem/fe.cpp

@@ -139,6 +139,12 @@ void FiniteElement::Project (
    mfem_error ("FiniteElement::Project (...) (vector) is not overloaded !");
 }
 
+void FiniteElement::ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                     Vector &dofs) const
+{
+   mfem_error ("FiniteElement::ProjectFromNodes() (vector) is not overloaded!");
+}
+
 void FiniteElement::ProjectMatrixCoefficient(
    MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
 {
@@ -925,6 +931,23 @@ void VectorFiniteElement::Project_RT(
    }
 }
 
+void VectorFiniteElement::Project_RT(
+   const double *nk, const Array<int> &d2n,
+   Vector &vc, ElementTransformation &Trans, Vector &dofs) const
+{
+   const int sdim = Trans.GetSpaceDim();
+   const bool square_J = (dim == sdim);
+
+   for (int k = 0; k < dof; k++)
+   {
+      Trans.SetIntPoint(&Nodes.IntPoint(k));
+      // dof_k = nk^t adj(J) xk
+      Vector vk(vc.GetData()+k*sdim, sdim);
+      dofs(k) = Trans.AdjugateJacobian().InnerProduct(vk, nk + d2n[k]*dim);
+      if (!square_J) { dofs(k) /= Trans.Weight(); }
+   }
+}
+
 void VectorFiniteElement::ProjectMatrixCoefficient_RT(
    const double *nk, const Array<int> &d2n,
    MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
@@ -1101,6 +1124,19 @@ void VectorFiniteElement::Project_ND(
    }
 }
 
+void VectorFiniteElement::Project_ND(
+   const double *tk, const Array<int> &d2t,
+   Vector &vc, ElementTransformation &Trans, Vector &dofs) const
+{
+   for (int k = 0; k < dof; k++)
+   {
+      Trans.SetIntPoint(&Nodes.IntPoint(k));
+      Vector vk(vc.GetData()+k*dim, dim);
+      // dof_k = xk^t J tk
+      dofs(k) = Trans.Jacobian().InnerProduct(tk + d2t[k]*dim, vk);
+   }
+}
+
 void VectorFiniteElement::ProjectMatrixCoefficient_ND(
    const double *tk, const Array<int> &d2t,
    MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
@@ -7995,7 +8031,7 @@ void H1_HexahedronElement::CalcHessian(const IntegrationPoint &ip,
 #ifdef MFEM_THREAD_SAFE
    Vector shape_x(p+1),  shape_y(p+1),  shape_z(p+1);
    Vector dshape_x(p+1), dshape_y(p+1), dshape_z(p+1);
-   Vector d2shape_x(p+1), d2shape_y(p+1), ds2hape_z(p+1);
+   Vector d2shape_x(p+1), d2shape_y(p+1), d2shape_z(p+1);
 #endif
 
    basis1d.Eval(ip.x, shape_x, dshape_x, d2shape_x);

fem/fe.hpp

@@ -504,14 +504,21 @@ public:
    /** @brief Given a coefficient and a transformation, compute its projection
        (approximation) in the local finite dimensional space in terms
        of the degrees of freedom. */
-   virtual void Project (Coefficient &coeff,
-                         ElementTransformation &Trans, Vector &dofs) const;
+   virtual void Project(Coefficient &coeff,
+                        ElementTransformation &Trans, Vector &dofs) const;
 
    /** @brief Given a vector coefficient and a transformation, compute its
        projection (approximation) in the local finite dimensional space
        in terms of the degrees of freedom. (VectorFiniteElements) */
-   virtual void Project (VectorCoefficient &vc,
-                         ElementTransformation &Trans, Vector &dofs) const;
+   virtual void Project(VectorCoefficient &vc,
+                        ElementTransformation &Trans, Vector &dofs) const;
+
+   /** @brief Given a vector of values at the finite element nodes and a
+       transformation, compute its projection (approximation) in the local
+       finite dimensional space in terms of the degrees of freedom. Valid for
+       VectorFiniteElements. */
+   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                 Vector &dofs) const;
 
    /** @brief Given a matrix coefficient and a transformation, compute an
        approximation ("projection") in the local finite dimensional space in
@@ -797,7 +804,12 @@ protected:
                    VectorCoefficient &vc, ElementTransformation &Trans,
                    Vector &dofs) const;
 
-   // project the rows of the matrix coefficient in an RT space
+   /// Projects the vector of values given at FE nodes to RT space
+   void Project_RT(const double *nk, const Array<int> &d2n,
+                   Vector &vc, ElementTransformation &Trans,
+                   Vector &dofs) const;
+
+   /// Project the rows of the matrix coefficient in an RT space
    void ProjectMatrixCoefficient_RT(
       const double *nk, const Array<int> &d2n,
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const;
@@ -825,7 +837,12 @@ protected:
                    VectorCoefficient &vc, ElementTransformation &Trans,
                    Vector &dofs) const;
 
-   /// project the rows of the matrix coefficient in an ND space
+   /// Projects the vector of values given at FE nodes to ND space
+   void Project_ND(const double *tk, const Array<int> &d2t,
+                   Vector &vc, ElementTransformation &Trans,
+                   Vector &dofs) const;
+
+   /// Project the rows of the matrix coefficient in an ND space
    void ProjectMatrixCoefficient_ND(
       const double *tk, const Array<int> &d2t,
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const;
@@ -2689,6 +2706,9 @@ public:
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_RT(nk, dof2nk, vc, Trans, dofs); }
+   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                 Vector &dofs) const
+   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
    virtual void ProjectMatrixCoefficient(
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
    { ProjectMatrixCoefficient_RT(nk, dof2nk, mc, T, dofs); }
@@ -2747,6 +2767,9 @@ public:
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_RT(nk, dof2nk, vc, Trans, dofs); }
+   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                 Vector &dofs) const
+   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
    virtual void ProjectMatrixCoefficient(
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
    { ProjectMatrixCoefficient_RT(nk, dof2nk, mc, T, dofs); }
@@ -2798,6 +2821,9 @@ public:
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_RT(nk, dof2nk, vc, Trans, dofs); }
+   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                 Vector &dofs) const
+   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
    virtual void ProjectMatrixCoefficient(
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
    { ProjectMatrixCoefficient_RT(nk, dof2nk, mc, T, dofs); }
@@ -2855,6 +2881,9 @@ public:
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_RT(nk, dof2nk, vc, Trans, dofs); }
+   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                 Vector &dofs) const
+   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
    virtual void ProjectMatrixCoefficient(
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
    { ProjectMatrixCoefficient_RT(nk, dof2nk, mc, T, dofs); }
@@ -2914,6 +2943,10 @@ public:
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_ND(tk, dof2tk, vc, Trans, dofs); }
 
+   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                 Vector &dofs) const
+   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+
    virtual void ProjectMatrixCoefficient(
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
    { ProjectMatrixCoefficient_ND(tk, dof2tk, mc, T, dofs); }
@@ -2973,6 +3006,9 @@ public:
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                 Vector &dofs) const
+   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
    virtual void ProjectMatrixCoefficient(
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
    { ProjectMatrixCoefficient_ND(tk, dof2tk, mc, T, dofs); }
@@ -3024,6 +3060,9 @@ public:
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                 Vector &dofs) const
+   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
    virtual void ProjectMatrixCoefficient(
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
    { ProjectMatrixCoefficient_ND(tk, dof2tk, mc, T, dofs); }
@@ -3080,6 +3119,9 @@ public:
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
+                                 Vector &dofs) const
+   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
    virtual void ProjectMatrixCoefficient(
       MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
    { ProjectMatrixCoefficient_ND(tk, dof2tk, mc, T, dofs); }

fem/fem.hpp

@@ -19,6 +19,7 @@
 #include "eltrans.hpp"
 #include "coefficient.hpp"
 #include "complex_fem.hpp"
+#include "convergence.hpp"
 #include "lininteg.hpp"
 #include "nonlininteg.hpp"
 #include "bilininteg.hpp"

fem/fespace.cpp

@@ -440,6 +440,7 @@ void FiniteElementSpace::MarkerToList(const Array<int> &marker,
       if (marker[i]) { num_marked++; }
    }
    list.SetSize(0);
+   list.HostWrite();
    list.Reserve(num_marked);
    for (int i = 0; i < marker.Size(); i++)
    {
@@ -451,7 +452,9 @@ void FiniteElementSpace::MarkerToList(const Array<int> &marker,
 void FiniteElementSpace::ListToMarker(const Array<int> &list, int marker_size,
                                       Array<int> &marker, int mark_val)
 {
+   list.HostRead(); // make sure we can read the array on host
    marker.SetSize(marker_size);
+   marker.HostWrite();
    marker = 0;
    for (int i = 0; i < list.Size(); i++)
    {

fem/gridfunc.cpp

@@ -199,8 +199,7 @@ void GridFunction::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
    if (f != fes) { Destroy(); }
    fes = f;
    v.UseDevice(true);
-   NewMemoryAndSize(Memory<double>(v.GetMemory(), v_offset, fes->GetVSize()),
-                    fes->GetVSize(), true);
+   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
    sequence = fes->GetSequence();
 }
 
@@ -1834,6 +1833,19 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
    ImposeBounds(i, weights, minv, maxv);
 }
 
+void GridFunction::RestrictConforming()
+{
+   const SparseMatrix *R = fes->GetRestrictionMatrix();
+   const Operator *P = fes->GetProlongationMatrix();
+
+   if (P && R)
+   {
+      Vector tmp(R->Height());
+      R->Mult(*this, tmp);
+      P->Mult(tmp, *this);
+   }
+}
+
 void GridFunction::GetNodalValues(Vector &nval, int vdim) const
 {
    int i, j;
@@ -2602,11 +2614,7 @@ double GridFunction::ComputeL2Error(
       }
    }
 
-   if (error < 0.0)
-   {
-      return -sqrt(-error);
-   }
-   return sqrt(error);
+   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
 }
 
 double GridFunction::ComputeL2Error(
@@ -2647,94 +2655,199 @@ double GridFunction::ComputeL2Error(
       }
    }
 
-   if (error < 0.0)
-   {
-      return -sqrt(-error);
-   }
-   return sqrt(error);
+   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
 }
 
-double GridFunction::ComputeH1Error(
-   Coefficient *exsol, VectorCoefficient *exgrad,
-   Coefficient *ell_coeff, double Nu, int norm_type) const
+double GridFunction::ComputeGradError(VectorCoefficient *exgrad,
+                                      const IntegrationRule *irs[]) const
 {
-   // assuming vdim is 1
-   int i, fdof, dim, intorder, j, k;
+   double error = 0.0;
+   const FiniteElement *fe;
+   ElementTransformation *Tr;
+   Array<int> dofs;
+   Vector grad;
+   int intorder;
+   int dim = fes->GetMesh()->SpaceDimension();
+   Vector vec(dim);
+
+   for (int i = 0; i < fes->GetNE(); i++)
+   {
+      fe = fes->GetFE(i);
+      Tr = fes->GetElementTransformation(i);
+      intorder = 2*fe->GetOrder() + 3; // <--------
+      const IntegrationRule *ir;
+      if (irs)
+      {
+         ir = irs[fe->GetGeomType()];
+      }
+      else
+      {
+         ir = &(IntRules.Get(fe->GetGeomType(), intorder));
+      }
+      fes->GetElementDofs(i, dofs);
+      for (int j = 0; j < ir->GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(j);
+         Tr->SetIntPoint(&ip);
+         GetGradient(*Tr,grad);
+         exgrad->Eval(vec,*Tr,ip);
+         vec-=grad;
+         error += ip.weight * Tr->Weight() * (vec * vec);
+      }
+   }
+   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
+}
+
+double GridFunction::ComputeCurlError(VectorCoefficient *excurl,
+                                      const IntegrationRule *irs[]) const
+{
+   double error = 0.0;
+   const FiniteElement *fe;
+   ElementTransformation *Tr;
+   Array<int> dofs;
+   Vector curl;
+   int intorder;
+   int dim = fes->GetMesh()->SpaceDimension();
+   int n = (dim == 3) ? dim : 1;
+   Vector vec(n);
+
+   for (int i = 0; i < fes->GetNE(); i++)
+   {
+      fe = fes->GetFE(i);
+      Tr = fes->GetElementTransformation(i);
+      intorder = 2*fe->GetOrder() + 3;
+      const IntegrationRule *ir;
+      if (irs)
+      {
+         ir = irs[fe->GetGeomType()];
+      }
+      else
+      {
+         ir = &(IntRules.Get(fe->GetGeomType(), intorder));
+      }
+      fes->GetElementDofs(i, dofs);
+      for (int j = 0; j < ir->GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(j);
+         Tr->SetIntPoint(&ip);
+         GetCurl(*Tr,curl);
+         excurl->Eval(vec,*Tr,ip);
+         vec-=curl;
+         error += ip.weight * Tr->Weight() * ( vec * vec );
+      }
+   }
+
+   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
+}
+
+double GridFunction::ComputeDivError(
+   Coefficient *exdiv, const IntegrationRule *irs[]) const
+{
+   double error = 0.0, a;
+   const FiniteElement *fe;
+   ElementTransformation *Tr;
+   Array<int> dofs;
+   int intorder;
+
+   for (int i = 0; i < fes->GetNE(); i++)
+   {
+      fe = fes->GetFE(i);
+      Tr = fes->GetElementTransformation(i);
+      intorder = 2*fe->GetOrder() + 3;
+      const IntegrationRule *ir;
+      if (irs)
+      {
+         ir = irs[fe->GetGeomType()];
+      }
+      else
+      {
+         ir = &(IntRules.Get(fe->GetGeomType(), intorder));
+      }
+      fes->GetElementDofs(i, dofs);
+      for (int j = 0; j < ir->GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(j);
+         Tr->SetIntPoint (&ip);
+         a = GetDivergence(*Tr) - exdiv->Eval(*Tr, ip);
+         error += ip.weight * Tr->Weight() * a * a;
+      }
+   }
+
+   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
+}
+
+double GridFunction::ComputeDGFaceJumpError(Coefficient *exsol,
+                                            Coefficient *ell_coeff, double Nu,
+                                            const IntegrationRule *irs[])  const
+{
+   int fdof, dim, intorder, k;
    Mesh *mesh;
    const FiniteElement *fe;
    ElementTransformation *transf;
    FaceElementTransformations *face_elem_transf;
-   Vector e_grad, a_grad, shape, el_dofs, err_val, ell_coeff_val;
-   DenseMatrix dshape, dshapet, Jinv;
+   Vector shape, el_dofs, err_val, ell_coeff_val;
    Array<int> vdofs;
    IntegrationPoint eip;
    double error = 0.0;
 
    mesh = fes->GetMesh();
    dim = mesh->Dimension();
-   e_grad.SetSize(dim);
-   a_grad.SetSize(dim);
-   Jinv.SetSize(dim);
 
-   if (norm_type & 1)
-      for (i = 0; i < mesh->GetNE(); i++)
-      {
-         fe = fes->GetFE(i);
-         fdof = fe->GetDof();
-         transf = mesh->GetElementTransformation(i);
-         el_dofs.SetSize(fdof);
-         dshape.SetSize(fdof, dim);
-         dshapet.SetSize(fdof, dim);
-         intorder = 2 * fe->GetOrder(); // <----------
-         const IntegrationRule &ir = IntRules.Get(fe->GetGeomType(), intorder);
-         fes->GetElementVDofs(i, vdofs);
-         for (k = 0; k < fdof; k++)
-            if (vdofs[k] >= 0)
-            {
-               el_dofs(k) =   (*this)(vdofs[k]);
-            }
-            else
-            {
-               el_dofs(k) = - (*this)(-1-vdofs[k]);
-            }
-         for (j = 0; j < ir.GetNPoints(); j++)
+   for (int i = 0; i < mesh->GetNumFaces(); i++)
+   {
+      face_elem_transf = mesh->GetFaceElementTransformations(i, 5);
+      int i1 = face_elem_transf->Elem1No;
+      int i2 = face_elem_transf->Elem2No;
+      intorder = fes->GetFE(i1)->GetOrder();
+      if (i2 >= 0)
+         if ( (k = fes->GetFE(i2)->GetOrder()) > intorder )
          {
-            const IntegrationPoint &ip = ir.IntPoint(j);
-            fe->CalcDShape(ip, dshape);
-            transf->SetIntPoint(&ip);
-            exgrad->Eval(e_grad, *transf, ip);
-            CalcInverse(transf->Jacobian(), Jinv);
-            Mult(dshape, Jinv, dshapet);
-            dshapet.MultTranspose(el_dofs, a_grad);
-            e_grad -= a_grad;
-            error += (ip.weight * transf->Weight() *
-                      ell_coeff->Eval(*transf, ip) *
-                      (e_grad * e_grad));
+            intorder = k;
          }
-      }
-
-   if (norm_type & 2)
-      for (i = 0; i < mesh->GetNFaces(); i++)
+      intorder = 2 * intorder;  // <-------------
+      const IntegrationRule *ir;
+      if (irs)
       {
-         face_elem_transf = mesh->GetFaceElementTransformations(i, 5);
-         int i1 = face_elem_transf->Elem1No;
-         int i2 = face_elem_transf->Elem2No;
-         intorder = fes->GetFE(i1)->GetOrder();
-         if (i2 >= 0)
-            if ( (k = fes->GetFE(i2)->GetOrder()) > intorder )
-            {
-               intorder = k;
-            }
-         intorder = 2 * intorder;  // <-------------
-         const IntegrationRule &ir =
-            IntRules.Get(face_elem_transf->GetGeometryType(), intorder);
-         err_val.SetSize(ir.GetNPoints());
-         ell_coeff_val.SetSize(ir.GetNPoints());
-         // side 1
-         transf = face_elem_transf->Elem1;
-         fe = fes->GetFE(i1);
+         ir = irs[face_elem_transf->GetGeometryType()];
+      }
+      else
+      {
+         ir = &(IntRules.Get(face_elem_transf->GetGeometryType(), intorder));
+      }
+      err_val.SetSize(ir->GetNPoints());
+      ell_coeff_val.SetSize(ir->GetNPoints());
+      // side 1
+      transf = face_elem_transf->Elem1;
+      fe = fes->GetFE(i1);
+      fdof = fe->GetDof();
+      fes->GetElementVDofs(i1, vdofs);
+      shape.SetSize(fdof);
+      el_dofs.SetSize(fdof);
+      for (k = 0; k < fdof; k++)
+         if (vdofs[k] >= 0)
+         {
+            el_dofs(k) =   (*this)(vdofs[k]);
+         }
+         else
+         {
+            el_dofs(k) = - (*this)(-1-vdofs[k]);
+         }
+      for (int j = 0; j < ir->GetNPoints(); j++)
+      {
+         face_elem_transf->Loc1.Transform(ir->IntPoint(j), eip);
+         fe->CalcShape(eip, shape);
+         transf->SetIntPoint(&eip);
+         ell_coeff_val(j) = ell_coeff->Eval(*transf, eip);
+         err_val(j) = exsol->Eval(*transf, eip) - (shape * el_dofs);
+      }
+      if (i2 >= 0)
+      {
+         // side 2
+         face_elem_transf = mesh->GetFaceElementTransformations(i, 10);
+         transf = face_elem_transf->Elem2;
+         fe = fes->GetFE(i2);
          fdof = fe->GetDof();
-         fes->GetElementVDofs(i1, vdofs);
+         fes->GetElementVDofs(i2, vdofs);
          shape.SetSize(fdof);
          el_dofs.SetSize(fdof);
          for (k = 0; k < fdof; k++)
@@ -2746,60 +2859,69 @@ double GridFunction::ComputeH1Error(
             {
                el_dofs(k) = - (*this)(-1-vdofs[k]);
             }
-         for (j = 0; j < ir.GetNPoints(); j++)
+         for (int j = 0; j < ir->GetNPoints(); j++)
          {
-            face_elem_transf->Loc1.Transform(ir.IntPoint(j), eip);
+            face_elem_transf->Loc2.Transform(ir->IntPoint(j), eip);
             fe->CalcShape(eip, shape);
             transf->SetIntPoint(&eip);
-            ell_coeff_val(j) = ell_coeff->Eval(*transf, eip);
-            err_val(j) = exsol->Eval(*transf, eip) - (shape * el_dofs);
-         }
-         if (i2 >= 0)
-         {
-            // side 2
-            face_elem_transf = mesh->GetFaceElementTransformations(i, 10);
-            transf = face_elem_transf->Elem2;
-            fe = fes->GetFE(i2);
-            fdof = fe->GetDof();
-            fes->GetElementVDofs(i2, vdofs);
-            shape.SetSize(fdof);
-            el_dofs.SetSize(fdof);
-            for (k = 0; k < fdof; k++)
-               if (vdofs[k] >= 0)
-               {
-                  el_dofs(k) =   (*this)(vdofs[k]);
-               }
-               else
-               {
-                  el_dofs(k) = - (*this)(-1-vdofs[k]);
-               }
-            for (j = 0; j < ir.GetNPoints(); j++)
-            {
-               face_elem_transf->Loc2.Transform(ir.IntPoint(j), eip);
-               fe->CalcShape(eip, shape);
-               transf->SetIntPoint(&eip);
-               ell_coeff_val(j) += ell_coeff->Eval(*transf, eip);
-               ell_coeff_val(j) *= 0.5;
-               err_val(j) -= (exsol->Eval(*transf, eip) - (shape * el_dofs));
-            }
-         }
-         face_elem_transf = mesh->GetFaceElementTransformations(i, 16);
-         transf = face_elem_transf;
-         for (j = 0; j < ir.GetNPoints(); j++)
-         {
-            const IntegrationPoint &ip = ir.IntPoint(j);
-            transf->SetIntPoint(&ip);
-            error += (ip.weight * Nu * ell_coeff_val(j) *
-                      pow(transf->Weight(), 1.0-1.0/(dim-1)) *
-                      err_val(j) * err_val(j));
+            ell_coeff_val(j) += ell_coeff->Eval(*transf, eip);
+            ell_coeff_val(j) *= 0.5;
+            err_val(j) -= (exsol->Eval(*transf, eip) - (shape * el_dofs));
          }
       }
-
-   if (error < 0.0)
-   {
-      return -sqrt(-error);
+      face_elem_transf = mesh->GetFaceElementTransformations(i, 16);
+      transf = face_elem_transf;
+      for (int j = 0; j < ir->GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(j);
+         transf->SetIntPoint(&ip);
+         error += (ip.weight * Nu * ell_coeff_val(j) *
+                   pow(transf->Weight(), 1.0-1.0/(dim-1)) *
+                   err_val(j) * err_val(j));
+      }
    }
-   return sqrt(error);
+
+   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
+}
+
+double GridFunction::ComputeH1Error(Coefficient *exsol,
+                                    VectorCoefficient *exgrad,
+                                    Coefficient *ell_coef, double Nu,
+                                    int norm_type) const
+{
+   double error1 = 0.0;
+   double error2 = 0.0;
+   if (norm_type & 1) { error1 = GridFunction::ComputeGradError(exgrad); }
+   if (norm_type & 2) { error2 = GridFunction::ComputeDGFaceJumpError(exsol,ell_coef,Nu); }
+
+   return sqrt(error1 * error1 + error2 * error2);
+}
+
+double GridFunction::ComputeH1Error(Coefficient *exsol,
+                                    VectorCoefficient *exgrad,
+                                    const IntegrationRule *irs[]) const
+{
+   double L2error = GridFunction::ComputeLpError(2.0,*exsol,NULL,irs);
+   double GradError = ComputeGradError(exgrad,irs);
+   return sqrt(L2error*L2error + GradError*GradError);
+}
+
+double GridFunction::ComputeHDivError(VectorCoefficient *exsol,
+                                      Coefficient *exdiv,
+                                      const IntegrationRule *irs[]) const
+{
+   double L2error = GridFunction::ComputeLpError(2.0,*exsol,NULL,NULL,irs);
+   double DivError = ComputeDivError(exdiv,irs);
+   return sqrt(L2error*L2error + DivError*DivError);
+}
+
+double GridFunction::ComputeHCurlError(VectorCoefficient *exsol,
+                                       VectorCoefficient *excurl,
+                                       const IntegrationRule *irs[]) const
+{
+   double L2error = GridFunction::ComputeLpError(2.0,*exsol,NULL,NULL,irs);
+   double CurlError = ComputeCurlError(excurl,irs);
+   return sqrt(L2error*L2error + CurlError*CurlError);
 }
 
 double GridFunction::ComputeMaxError(
@@ -2855,7 +2977,6 @@ double GridFunction::ComputeMaxError(
          }
       }
    }
-
    return error;
 }
 

fem/gridfunc.hpp

@@ -334,6 +334,11 @@ public:
    void ImposeBounds(int i, const Vector &weights,
                      double _min = 0.0, double _max = infinity());
 
+   /** On a non-conforming mesh, make sure the function lies in the conforming
+       space by multiplying with R and then with P, the conforming restriction
+       and prolongation matrices of the space, respectively. */
+   void RestrictConforming();
+
    /** @brief Project the @a src GridFunction to @a this GridFunction, both of
        which must be on the same mesh. */
    /** The current implementation assumes that all elements use the same
@@ -422,6 +427,7 @@ public:
    virtual void ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
                                              Array<int> &bdr_attr);
 
+
    virtual double ComputeL2Error(Coefficient &exsol,
                                  const IntegrationRule *irs[] = NULL) const
    { return ComputeLpError(2.0, exsol, NULL, irs); }
@@ -433,10 +439,50 @@ public:
                                  const IntegrationRule *irs[] = NULL,
                                  Array<int> *elems = NULL) const;
 
+   /// Returns ||grad u_ex - grad u_h||_L2 for H1 or L2 elements
+   virtual double ComputeGradError(VectorCoefficient *exgrad,
+                                   const IntegrationRule *irs[] = NULL) const;
+
+   /// Returns ||curl u_ex - curl u_h||_L2 for ND elements
+   virtual double ComputeCurlError(VectorCoefficient *excurl,
+                                   const IntegrationRule *irs[] = NULL) const;
+
+   /// Returns ||div u_ex - div u_h||_L2 for RT elements
+   virtual double ComputeDivError(Coefficient *exdiv,
+                                  const IntegrationRule *irs[] = NULL) const;
+
+   /// Returns the Face Jumps error for L2 elements
+   virtual double ComputeDGFaceJumpError(Coefficient *exsol,
+                                         Coefficient *ell_coeff,
+                                         double Nu,
+                                         const IntegrationRule *irs[] = NULL)
+   const;
+
+   /** This method is kept for backward compatibility.
+
+       Returns either the H1-seminorm, or the DG face jumps error, or both
+       depending on norm_type = 1, 2, 3. Additional arguments for the DG face
+       jumps norm: ell_coeff: mesh-depended coefficient (weight) Nu: scalar
+       constant weight */
    virtual double ComputeH1Error(Coefficient *exsol, VectorCoefficient *exgrad,
                                  Coefficient *ell_coef, double Nu,
                                  int norm_type) const;
 
+   /// Returns the error measured in H1-norm for H1 elements or in "broken"
+   /// H1-norm for L2 elements
+   virtual double ComputeH1Error(Coefficient *exsol, VectorCoefficient *exgrad,
+                                 const IntegrationRule *irs[] = NULL) const;
+
+   /// Returns the error measured in H(div)-norm for RT elements
+   virtual double ComputeHDivError(VectorCoefficient *exsol,
+                                   Coefficient *exdiv,
+                                   const IntegrationRule *irs[] = NULL) const;
+
+   /// Returns the error measured in H(curl)-norm for ND elements
+   virtual double ComputeHCurlError(VectorCoefficient *exsol,
+                                    VectorCoefficient *excurl,
+                                    const IntegrationRule *irs[] = NULL) const;
+
    virtual double ComputeMaxError(Coefficient &exsol,
                                   const IntegrationRule *irs[] = NULL) const
    {

fem/gslib.cpp

@@ -29,10 +29,13 @@ namespace mfem
 {
 
 FindPointsGSLIB::FindPointsGSLIB()
-   : mesh(NULL), ir_simplex(NULL), fdata2D(NULL), fdata3D(NULL),
-     dim(-1), gsl_mesh(), gsl_ref(), gsl_dist(), setupflag(false)
+   : mesh(NULL), meshsplit(NULL), ir_simplex(NULL),
+     fdata2D(NULL), fdata3D(NULL), cr(NULL), gsl_comm(NULL),
+     dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
+     avgtype(AvgType::ARITHMETIC)
 {
    gsl_comm = new comm;
+   cr       = new crystal;
 #ifdef MFEM_USE_MPI
    int initialized;
    MPI_Initialized(&initialized);
@@ -47,15 +50,20 @@ FindPointsGSLIB::FindPointsGSLIB()
 FindPointsGSLIB::~FindPointsGSLIB()
 {
    delete gsl_comm;
+   delete cr;
    delete ir_simplex;
+   delete meshsplit;
 }
 
 #ifdef MFEM_USE_MPI
 FindPointsGSLIB::FindPointsGSLIB(MPI_Comm _comm)
-   : mesh(NULL), ir_simplex(NULL), fdata2D(NULL), fdata3D(NULL),
-     dim(-1), gsl_mesh(), gsl_ref(), gsl_dist(), setupflag(false)
+   : mesh(NULL), meshsplit(NULL), ir_simplex(NULL),
+     fdata2D(NULL), fdata3D(NULL), cr(NULL), gsl_comm(NULL),
+     dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
+     avgtype(AvgType::ARITHMETIC)
 {
    gsl_comm = new comm;
+   cr      = new crystal;
    comm_init(gsl_comm, _comm);
 }
 #endif
@@ -70,6 +78,7 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
    // call FreeData if FindPointsGSLIB::Setup has been called already
    if (setupflag) { FreeData(); }
 
+   crystal_init(cr, gsl_comm);
    mesh = &m;
    dim  = mesh->Dimension();
    const FiniteElement *fe = mesh->GetNodalFESpace()->GetFE(0);
@@ -113,14 +122,16 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
    setupflag = true;
 }
 
-void FindPointsGSLIB::FindPoints(const Vector &point_pos,
-                                 Array<unsigned int> &codes,
-                                 Array<unsigned int> &proc_ids,
-                                 Array<unsigned int> &elem_ids,
-                                 Vector &ref_pos, Vector &dist)
+void FindPointsGSLIB::FindPoints(const Vector &point_pos)
 {
    MFEM_VERIFY(setupflag, "Use FindPointsGSLIB::Setup before finding points.");
-   const int points_cnt = point_pos.Size() / dim;
+   points_cnt = point_pos.Size() / dim;
+   gsl_code.SetSize(points_cnt);
+   gsl_proc.SetSize(points_cnt);
+   gsl_elem.SetSize(points_cnt);
+   gsl_ref.SetSize(points_cnt * dim);
+   gsl_dist.SetSize(points_cnt);
+
    if (dim == 2)
    {
       const double *xv_base[2];
@@ -129,11 +140,11 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos,
       unsigned xv_stride[2];
       xv_stride[0] = sizeof(double);
       xv_stride[1] = sizeof(double);
-      findpts_2(codes.GetData(), sizeof(unsigned int),
-                proc_ids.GetData(), sizeof(unsigned int),
-                elem_ids.GetData(), sizeof(unsigned int),
-                ref_pos.GetData(), sizeof(double) * dim,
-                dist.GetData(), sizeof(double),
+      findpts_2(gsl_code.GetData(), sizeof(unsigned int),
+                gsl_proc.GetData(), sizeof(unsigned int),
+                gsl_elem.GetData(), sizeof(unsigned int),
+                gsl_ref.GetData(),  sizeof(double) * dim,
+                gsl_dist.GetData(), sizeof(double),
                 xv_base, xv_stride, points_cnt, fdata2D);
    }
    else
@@ -146,25 +157,27 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos,
       xv_stride[0] = sizeof(double);
       xv_stride[1] = sizeof(double);
       xv_stride[2] = sizeof(double);
-      findpts_3(codes.GetData(), sizeof(unsigned int),
-                proc_ids.GetData(), sizeof(unsigned int),
-                elem_ids.GetData(), sizeof(unsigned int),
-                ref_pos.GetData(), sizeof(double) * dim,
-                dist.GetData(), sizeof(double),
+      findpts_3(gsl_code.GetData(), sizeof(unsigned int),
+                gsl_proc.GetData(), sizeof(unsigned int),
+                gsl_elem.GetData(), sizeof(unsigned int),
+                gsl_ref.GetData(),  sizeof(double) * dim,
+                gsl_dist.GetData(), sizeof(double),
                 xv_base, xv_stride, points_cnt, fdata3D);
    }
-}
 
-void FindPointsGSLIB::FindPoints(const Vector &point_pos)
-{
-   const int points_cnt = point_pos.Size() / dim;
-   gsl_code.SetSize(points_cnt);
-   gsl_proc.SetSize(points_cnt);
-   gsl_elem.SetSize(points_cnt);
-   gsl_ref.SetSize(points_cnt * dim);
-   gsl_dist.SetSize(points_cnt);
+   // Set the element number and reference position to 0 for points not found
+   for (int i = 0; i < points_cnt; i++)
+   {
+      if (gsl_code[i] == 2)
+      {
+         gsl_elem[i] = 0;
+         for (int d = 0; d < dim; d++) { gsl_ref(i*dim + d) = -1.; }
+      }
+   }
 
-   FindPoints(point_pos, gsl_code, gsl_proc, gsl_elem, gsl_ref, gsl_dist);
+   // Map element number for simplices, and ref_pos from [-1,1] to [0,1] for
+   // both simplices and quads.
+   MapRefPosAndElemIndices();
 }
 
 void FindPointsGSLIB::FindPoints(Mesh &m, const Vector &point_pos,
@@ -178,72 +191,24 @@ void FindPointsGSLIB::FindPoints(Mesh &m, const Vector &point_pos,
    FindPoints(point_pos);
 }
 
-void FindPointsGSLIB::Interpolate(Array<unsigned int> &codes,
-                                  Array<unsigned int> &proc_ids,
-                                  Array<unsigned int> &elem_ids,
-                                  Vector &ref_pos, const GridFunction &field_in,
-                                  Vector &field_out)
-{
-
-   FiniteElementSpace ind_fes(mesh, field_in.FESpace()->FEColl());
-   GridFunction field_in_scalar(&ind_fes);
-   Vector node_vals;
-
-   const int ncomp      = field_in.FESpace()->GetVDim(),
-             points_fld = field_in.Size() / ncomp,
-             points_cnt = codes.Size();
-   field_out.SetSize(points_cnt*ncomp);
-
-   for (int i = 0; i < ncomp; i++)
-   {
-      const int dataptrin  = i*points_fld,
-                dataptrout = i*points_cnt;
-      field_in_scalar.NewDataAndSize(field_in.GetData()+dataptrin, points_fld);
-      GetNodeValues(field_in_scalar, node_vals);
-
-      if (dim==2)
-      {
-         findpts_eval_2(field_out.GetData()+dataptrout, sizeof(double),
-                        codes.GetData(),       sizeof(unsigned int),
-                        proc_ids.GetData(),    sizeof(unsigned int),
-                        elem_ids.GetData(),    sizeof(unsigned int),
-                        ref_pos.GetData(),     sizeof(double) * dim,
-                        points_cnt, node_vals.GetData(), fdata2D);
-      }
-      else
-      {
-         findpts_eval_3(field_out.GetData()+dataptrout, sizeof(double),
-                        codes.GetData(),       sizeof(unsigned int),
-                        proc_ids.GetData(),    sizeof(unsigned int),
-                        elem_ids.GetData(),    sizeof(unsigned int),
-                        ref_pos.GetData(),     sizeof(double) * dim,
-                        points_cnt, node_vals.GetData(), fdata3D);
-      }
-   }
-}
-
-void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
-                                  Vector &field_out)
-{
-   Interpolate(gsl_code, gsl_proc, gsl_elem, gsl_ref, field_in, field_out);
-}
-
 void FindPointsGSLIB::Interpolate(const Vector &point_pos,
                                   const GridFunction &field_in, Vector &field_out)
 {
    FindPoints(point_pos);
-   Interpolate(gsl_code, gsl_proc, gsl_elem, gsl_ref, field_in, field_out);
+   Interpolate(field_in, field_out);
 }
 
 void FindPointsGSLIB::Interpolate(Mesh &m, const Vector &point_pos,
                                   const GridFunction &field_in, Vector &field_out)
 {
    FindPoints(m, point_pos);
-   Interpolate(gsl_code, gsl_proc, gsl_elem, gsl_ref, field_in, field_out);
+   Interpolate(field_in, field_out);
 }
 
 void FindPointsGSLIB::FreeData()
 {
+   if (!setupflag) { return; }
+   crystal_free(cr);
    if (dim == 2)
    {
       findpts_free_2(fdata2D);
@@ -252,13 +217,13 @@ void FindPointsGSLIB::FreeData()
    {
       findpts_free_3(fdata3D);
    }
-   setupflag = false;
    gsl_code.DeleteAll();
    gsl_proc.DeleteAll();
    gsl_elem.DeleteAll();
    gsl_mesh.Destroy();
    gsl_ref.Destroy();
    gsl_dist.Destroy();
+   setupflag = false;
 }
 
 void FindPointsGSLIB::GetNodeValues(const GridFunction &gf_in,
@@ -358,9 +323,8 @@ void FindPointsGSLIB::GetSimplexNodalCoordinates()
    const FiniteElement *fe   = mesh->GetNodalFESpace()->GetFE(0);
    const Geometry::Type gt   = fe->GetGeomType();
    const GridFunction *nodes = mesh->GetNodes();
-   Mesh *meshsplit           = NULL;
    const int NE              = mesh->GetNE();
-   int NEsplit = -1;
+   int NEsplit = 0;
 
    // Split the reference element into a reference submesh of quads or hexes.
    if (gt == Geometry::TRIANGLE)
@@ -516,8 +480,361 @@ void FindPointsGSLIB::GetSimplexNodalCoordinates()
          pt_id++;
       }
    }
+}
 
-   delete meshsplit;
+void FindPointsGSLIB::MapRefPosAndElemIndices()
+{
+   gsl_mfem_ref = gsl_ref;
+   gsl_mfem_elem = gsl_elem;
+   const FiniteElement *fe = mesh->GetNodalFESpace()->GetFE(0);
+   const Geometry::Type gt = fe->GetGeomType();
+   int NEsplit = 0;
+
+   gsl_mfem_ref -= -1.;  // map  [-1, 1] to
+   gsl_mfem_ref *= 0.5;  //      [0,  1]
+   if (gt == Geometry::SQUARE || gt == Geometry::CUBE) { return; }
+
+   H1_FECollection feclin(1, dim);
+   FiniteElementSpace nodal_fes_lin(meshsplit, &feclin, dim);
+   GridFunction gf_lin(&nodal_fes_lin);
+
+   if (gt == Geometry::TRIANGLE)
+   {
+      const double quad_v[7][2] =
+      {
+         {0, 0}, {0.5, 0}, {1, 0}, {0, 0.5},
+         {1./3., 1./3.}, {0.5, 0.5}, {0, 1}
+      };
+      for (int k = 0; k < dim; k++)
+      {
+         for (int j = 0; j < gf_lin.Size()/dim; j++)
+         {
+            gf_lin(j+k*gf_lin.Size()/dim) = quad_v[j][k];
+         }
+      }
+      NEsplit = 3;
+   }
+   else if (gt == Geometry::TETRAHEDRON)
+   {
+      const double hex_v[15][3] =
+      {
+         {0, 0, 0.}, {1, 0., 0.}, {0., 1., 0.}, {0, 0., 1.},
+         {0.5, 0., 0.}, {0.5, 0.5, 0.}, {0., 0.5, 0.},
+         {0., 0., 0.5}, {0.5, 0., 0.5}, {0., 0.5, 0.5},
+         {1./3., 0., 1./3.}, {1./3., 1./3., 1./3.}, {0, 1./3., 1./3.},
+         {1./3., 1./3., 0}, {0.25, 0.25, 0.25}
+      };
+      for (int k = 0; k < dim; k++)
+      {
+         for (int j = 0; j < gf_lin.Size()/dim; j++)
+         {
+            gf_lin(j+k*gf_lin.Size()/dim) = hex_v[j][k];
+         }
+      }
+      NEsplit = 4;
+   }
+   else if (gt == Geometry::PRISM)
+   {
+      const double hex_v[14][3] =
+      {
+         {0, 0, 0}, {0.5, 0, 0}, {1, 0, 0}, {0, 0.5, 0},
+         {1./3., 1./3., 0}, {0.5, 0.5, 0}, {0, 1, 0},
+         {0, 0, 1}, {0.5, 0, 1}, {1, 0, 1}, {0, 0.5, 1},
+         {1./3., 1./3., 1}, {0.5, 0.5, 1}, {0, 1, 1}
+      };
+      for (int k = 0; k < dim; k++)
+      {
+         for (int j = 0; j < gf_lin.Size()/dim; j++)
+         {
+            gf_lin(j+k*gf_lin.Size()/dim) = hex_v[j][k];
+         }
+      }
+      NEsplit = 3;
+   }
+   else
+   {
+      MFEM_ABORT("Element type not currently supported.");
+   }
+
+   // Simplices are split into quads/hexes for GSLIB. For MFEM, we need to find
+   // the original element number and map the rst from micro to macro element.
+   for (int i = 0; i < points_cnt; i++)
+   {
+      if (gsl_code[i] == 2) { continue; }
+      int local_elem   = gsl_elem[i]%NEsplit;
+      gsl_mfem_elem[i] = (gsl_elem[i] - local_elem)/NEsplit; // macro element number
+
+      IntegrationPoint ip;
+      Vector mfem_ref(gsl_mfem_ref.GetData()+i*dim, dim);
+      ip.Set2(mfem_ref.GetData());
+      if (dim == 3) { ip.z = mfem_ref(2); }
+      gf_lin.GetVectorValue(local_elem, ip, mfem_ref); // map to rst of macro element
+   }
+}
+
+void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
+                                  Vector &field_out)
+{
+   const int  gf_order   = field_in.FESpace()->GetFE(0)->GetOrder(),
+              mesh_order = mesh->GetNodalFESpace()->GetFE(0)->GetOrder();
+
+   const FiniteElementCollection *fec_in =  field_in.FESpace()->FEColl();
+   const H1_FECollection *fec_h1 = dynamic_cast<const H1_FECollection *>(fec_in);
+   const L2_FECollection *fec_l2 = dynamic_cast<const L2_FECollection *>(fec_in);
+
+   if (fec_h1 && gf_order == mesh_order &&
+       fec_h1->GetBasisType() == BasisType::GaussLobatto)
+   {
+      InterpolateH1(field_in, field_out);
+      return;
+   }
+   else
+   {
+      InterpolateGeneral(field_in, field_out);
+      if (!fec_l2 || avgtype == AvgType::NONE) { return; }
+   }
+
+   // For points on element borders, project the L2 GridFunction to H1 and
+   // re-interpolate.
+   if (fec_l2)
+   {
+      Array<int> indl2;
+      for (int i = 0; i < points_cnt; i++)
+      {
+         if (gsl_code[i] == 1) { indl2.Append(i); }
+      }
+      if (indl2.Size() == 0) { return; } // no points on element borders
+
+      Vector field_out_l2(field_out.Size());
+      VectorGridFunctionCoefficient field_in_dg(&field_in);
+      int gf_order_h1 = std::max(gf_order, 1); // H1 should be at least order 1
+      H1_FECollection fec(gf_order_h1, dim);
+      const int ncomp = field_in.FESpace()->GetVDim();
+      FiniteElementSpace fes(mesh, &fec, ncomp);
+      GridFunction field_in_h1(&fes);
+
+      if (avgtype == AvgType::ARITHMETIC)
+      {
+         field_in_h1.ProjectDiscCoefficient(field_in_dg, GridFunction::ARITHMETIC);
+      }
+      else if (avgtype == AvgType::HARMONIC)
+      {
+         field_in_h1.ProjectDiscCoefficient(field_in_dg, GridFunction::HARMONIC);
+      }
+      else
+      {
+         MFEM_ABORT("Invalid averaging type.");
+      }
+
+      if (gf_order_h1 == mesh_order) // basis is GaussLobatto by default
+      {
+         InterpolateH1(field_in_h1, field_out_l2);
+      }
+      else
+      {
+         InterpolateGeneral(field_in_h1, field_out_l2);
+      }
+
+      // Copy interpolated values for the points on element border
+      for (int j = 0; j < ncomp; j++)
+      {
+         for (int i = 0; i < indl2.Size(); i++)
+         {
+            int idx = indl2[i] + j*points_cnt;
+            field_out(idx) = field_out_l2(idx);
+         }
+      }
+   }
+}
+
+void FindPointsGSLIB::InterpolateH1(const GridFunction &field_in,
+                                    Vector &field_out)
+{
+   FiniteElementSpace ind_fes(mesh, field_in.FESpace()->FEColl());
+   GridFunction field_in_scalar(&ind_fes);
+   Vector node_vals;
+
+   const int ncomp      = field_in.FESpace()->GetVDim(),
+             points_fld = field_in.Size() / ncomp,
+             points_cnt = gsl_code.Size();
+
+   field_out.SetSize(points_cnt*ncomp);
+   field_out = default_interp_value;
+
+   for (int i = 0; i < ncomp; i++)
+   {
+      const int dataptrin  = i*points_fld,
+                dataptrout = i*points_cnt;
+      field_in_scalar.NewDataAndSize(field_in.GetData()+dataptrin, points_fld);
+      GetNodeValues(field_in_scalar, node_vals);
+
+      if (dim==2)
+      {
+         findpts_eval_2(field_out.GetData()+dataptrout, sizeof(double),
+                        gsl_code.GetData(),       sizeof(unsigned int),
+                        gsl_proc.GetData(),    sizeof(unsigned int),
+                        gsl_elem.GetData(),    sizeof(unsigned int),
+                        gsl_ref.GetData(),     sizeof(double) * dim,
+                        points_cnt, node_vals.GetData(), fdata2D);
+      }
+      else
+      {
+         findpts_eval_3(field_out.GetData()+dataptrout, sizeof(double),
+                        gsl_code.GetData(),       sizeof(unsigned int),
+                        gsl_proc.GetData(),    sizeof(unsigned int),
+                        gsl_elem.GetData(),    sizeof(unsigned int),
+                        gsl_ref.GetData(),     sizeof(double) * dim,
+                        points_cnt, node_vals.GetData(), fdata3D);
+      }
+   }
+}
+
+void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
+                                         Vector &field_out)
+{
+   int ncomp   = field_in.VectorDim(),
+       nptorig = points_cnt,
+       npt     = points_cnt;
+
+   field_out.SetSize(points_cnt*ncomp);
+   field_out = default_interp_value;
+
+   if (gsl_comm->np == 1) // serial
+   {
+      for (int index = 0; index < npt; index++)
+      {
+         if (gsl_code[index] == 2) { continue; }
+         IntegrationPoint ip;
+         ip.Set2(gsl_mfem_ref.GetData()+index*dim);
+         if (dim == 3) { ip.z = gsl_mfem_ref(index*dim + 2); }
+         Vector localval(ncomp);
+         field_in.GetVectorValue(gsl_mfem_elem[index], ip, localval);
+         for (int i = 0; i < ncomp; i++)
+         {
+            field_out(index + i*npt) = localval(i);
+         }
+      }
+   }
+   else // parallel
+   {
+      // Determine number of points to be sent
+      int nptsend = 0;
+      for (int index = 0; index < npt; index++)
+      {
+         if (gsl_code[index] != 2) { nptsend +=1; }
+      }
+
+      // Pack data to send via crystal router
+      struct array *outpt = new array;
+      struct out_pt { double r[3], ival; uint index, el, proc; };
+      struct out_pt *pt;
+      array_init(struct out_pt, outpt, nptsend);
+      outpt->n=nptsend;
+      pt = (struct out_pt *)outpt->ptr;
+      for (int index = 0; index < npt; index++)
+      {
+         if (gsl_code[index] == 2) { continue; }
+         for (int d = 0; d < dim; ++d) { pt->r[d]= gsl_mfem_ref(index*dim + d); }
+         pt->index = index;
+         pt->proc  = gsl_proc[index];
+         pt->el    = gsl_mfem_elem[index];
+         ++pt;
+      }
+
+      // Transfer data to target MPI ranks
+      sarray_transfer(struct out_pt, outpt, proc, 1, cr);
+
+      if (ncomp == 1)
+      {
+         // Interpolate the grid function
+         npt = outpt->n;
+         pt = (struct out_pt *)outpt->ptr;
+         for (int index = 0; index < npt; index++)
+         {
+            IntegrationPoint ip;
+            ip.Set3(&pt->r[0]);
+            pt->ival = field_in.GetValue(pt->el, ip, 1);
+            ++pt;
+         }
+
+         // Transfer data back to source MPI rank
+         sarray_transfer(struct out_pt, outpt, proc, 1, cr);
+         npt = outpt->n;
+         pt = (struct out_pt *)outpt->ptr;
+         for (int index = 0; index < npt; index++)
+         {
+            field_out(pt->index) = pt->ival;
+            ++pt;
+         }
+         array_free(outpt);
+         delete outpt;
+      }
+      else // ncomp > 1
+      {
+         // Interpolate data and store in a Vector
+         npt = outpt->n;
+         pt = (struct out_pt *)outpt->ptr;
+         Vector vec_int_vals(npt*ncomp);
+         for (int index = 0; index < npt; index++)
+         {
+            IntegrationPoint ip;
+            ip.Set3(&pt->r[0]);
+            Vector localval(vec_int_vals.GetData()+index*ncomp, ncomp);
+            field_in.GetVectorValue(pt->el, ip, localval);
+            ++pt;
+         }
+
+         // Save index and proc data in a struct
+         struct array *savpt = new array;
+         struct sav_pt { uint index, proc; };
+         struct sav_pt *spt;
+         array_init(struct sav_pt, savpt, npt);
+         savpt->n=npt;
+         spt = (struct sav_pt *)savpt->ptr;
+         pt  = (struct out_pt *)outpt->ptr;
+         for (int index = 0; index < npt; index++)
+         {
+            spt->index = pt->index;
+            spt->proc  = pt->proc;
+            ++pt; ++spt;
+         }
+
+         array_free(outpt);
+         delete outpt;
+
+         // Copy data from save struct to send struct and send component wise
+         struct array *sendpt = new array;
+         struct send_pt { double ival; uint index, proc; };
+         struct send_pt *sdpt;
+         for (int j = 0; j < ncomp; j++)
+         {
+            array_init(struct send_pt, sendpt, npt);
+            sendpt->n=npt;
+            spt  = (struct sav_pt *)savpt->ptr;
+            sdpt = (struct send_pt *)sendpt->ptr;
+            for (int index = 0; index < npt; index++)
+            {
+               sdpt->index = spt->index;
+               sdpt->proc  = spt->proc;
+               sdpt->ival  = vec_int_vals(j + index*ncomp);
+               ++sdpt; ++spt;
+            }
+
+            sarray_transfer(struct send_pt, sendpt, proc, 1, cr);
+            sdpt = (struct send_pt *)sendpt->ptr;
+            for (int index = 0; index < nptorig; index++)
+            {
+               int idx = sdpt->index + j*nptorig;
+               field_out(idx) = sdpt->ival;
+               ++sdpt;
+            }
+            array_free(sendpt);
+         }
+         array_free(savpt);
+         delete sendpt;
+         delete savpt;
+      } // ncomp > 1
+   } // parallel
 }
 
 } // namespace mfem

fem/gslib.hpp

@@ -20,28 +20,66 @@
 struct comm;
 struct findpts_data_2;
 struct findpts_data_3;
+struct array;
+struct crystal;
 
 namespace mfem
 {
 
+/** \brief FindPointsGSLIB can robustly evaluate a GridFunction on an arbitrary
+ *  collection of points. There are three key functions in FindPointsGSLIB:
+ *
+ *  1. Setup - constructs the internal data structures of gslib.
+ *
+ *  2. FindPoints - for any given arbitrary set of points in physical space,
+ *     gslib finds the element number, MPI rank, and the reference space
+ *     coordinates inside the element that each point is located in. gslib also
+ *     returns a code that indicates whether the point was found inside an
+ *     element, on element border, or not found in the domain.
+ *
+ *  3. Interpolate - Interpolates any grid function at the points found using 2.
+ *
+ *  FindPointsGSLIB provides interface to use these functions individually or
+ *  using a single call.
+ */
 class FindPointsGSLIB
 {
+public:
+   enum AvgType {NONE, ARITHMETIC, HARMONIC}; // Average type for L2 functions
+
 protected:
-   Mesh *mesh;
-   IntegrationRule *ir_simplex;
-   struct findpts_data_2 *fdata2D;
-   struct findpts_data_3 *fdata3D;
-   int dim;
-   Array<unsigned int> gsl_code, gsl_proc, gsl_elem;
-   Vector gsl_mesh, gsl_ref, gsl_dist;
-   bool setupflag;
-
-   struct comm *gsl_comm;
+   Mesh *mesh, *meshsplit;
+   IntegrationRule *ir_simplex;    // IntegrationRule to split quads/hex -> simplex
+   struct findpts_data_2 *fdata2D; // gslib's internal data
+   struct findpts_data_3 *fdata3D; // gslib's internal data
+   struct crystal *cr;             // gslib's internal data
+   struct comm *gsl_comm;          // gslib's internal data
+   int dim, points_cnt;
+   Array<unsigned int> gsl_code, gsl_proc, gsl_elem, gsl_mfem_elem;
+   Vector gsl_mesh, gsl_ref, gsl_dist, gsl_mfem_ref;
+   bool setupflag;              // flag to indicate whether gslib data has been setup
+   double default_interp_value; // used for points that are not found in the mesh
+   AvgType avgtype;             // average type used for L2 functions
 
+   /// Get GridFunction from MFEM format to GSLIB format
    void GetNodeValues(const GridFunction &gf_in, Vector &node_vals);
+   /// Get nodal coordinates from mesh to the format expected by GSLIB for quads
+   /// and hexes
    void GetQuadHexNodalCoordinates();
+   /// Convert simplices to quad/hexes and then get nodal coordinates for each
+   /// split element into format expected by GSLIB
    void GetSimplexNodalCoordinates();
 
+   /// Use GSLIB for communication and interpolation
+   void InterpolateH1(const GridFunction &field_in, Vector &field_out);
+   /// Uses GSLIB Crystal Router for communication followed by MFEM's
+   /// interpolation functions
+   void InterpolateGeneral(const GridFunction &field_in, Vector &field_out);
+   /// Map {r,s,t} coordinates from [-1,1] to [0,1] for MFEM. For simplices mesh
+   /// find the original element number (that was split into micro quads/hexes
+   /// by GetSimplexNodalCoordinates())
+   void MapRefPosAndElemIndices();
+
 public:
    FindPointsGSLIB();
 
@@ -64,45 +102,37 @@ public:
    void Setup(Mesh &m, const double bb_t = 0.1, const double newt_tol = 1.0e-12,
               const int npt_max = 256);
 
-   /** Searches positions given in physical space by @a point_pos. All output
-       Arrays and Vectors are expected to have the correct size.
-
-       @param[in]  point_pos  Positions to be found. Must by ordered by nodes
-                              (XXX...,YYY...,ZZZ).
-       @param[out] codes      Return codes for each point: inside element (0),
-                              element boundary (1), not found (2).
-       @param[out] proc_ids   MPI proc ids where the points were found.
-       @param[out] elem_ids   Element ids where the points were found.
-       @param[out] ref_pos    Reference coordinates of the found point. Ordered
-                              by vdim (XYZ,XYZ,XYZ...).
-                              Note: the gslib reference frame is [-1,1].
-       @param[out] dist       Distance between the sought and the found point
-                              in physical space. */
-   void FindPoints(const Vector &point_pos, Array<unsigned int> &codes,
-                   Array<unsigned int> &proc_ids, Array<unsigned int> &elem_ids,
-                   Vector &ref_pos, Vector &dist);
+   /** Searches positions given in physical space by @a point_pos. These positions
+       must by ordered by nodes: (XXX...,YYY...,ZZZ).
+       This function populates the following member variables:
+       #gsl_code        Return codes for each point: inside element (0),
+                        element boundary (1), not found (2).
+       #gsl_proc        MPI proc ids where the points were found.
+       #gsl_elem        Element ids where the points were found.
+                        Defaults to 0 for points that were not found.
+       #gsl_mfem_elem   Element ids corresponding to MFEM-mesh where the points
+                        were found. #gsl_mfem_elem != #gsl_elem for simplices
+                        Defaults to 0 for points that were not found.
+       #gsl_ref         Reference coordinates of the found point.
+                        Ordered by vdim (XYZ,XYZ,XYZ...). Defaults to -1 for
+                        points that were not found. Note: the gslib reference
+                        frame is [-1,1].
+       #gsl_mfem_ref    Reference coordinates #gsl_ref mapped to [0,1].
+                        Defaults to 0 for points that were not found.
+       #gsl_dist        Distance between the sought and the found point
+                        in physical space. */
    void FindPoints(const Vector &point_pos);
    /// Setup FindPoints and search positions
    void FindPoints(Mesh &m, const Vector &point_pos, const double bb_t = 0.1,
                    const double newt_tol = 1.0e-12,  const int npt_max = 256);
 
    /** Interpolation of field values at prescribed reference space positions.
-
-       @param[in] codes       Return codes for each point: inside element (0),
-                              element boundary (1), not found (2).
-       @param[in] proc_ids    MPI proc ids where the points were found.
-       @param[in] elem_ids    Element ids where the points were found.
-       @param[in] ref_pos     Reference coordinates of the found point. Ordered
-                              by vdim (XYZ,XYZ,XYZ...).
-                              Note: the gslib reference frame is [-1,1].
        @param[in] field_in    Function values that will be interpolated on the
                               reference positions. Note: it is assumed that
                               @a field_in is in H1 and in the same space as the
                               mesh that was given to Setup().
-       @param[out] field_out  Interpolated values. */
-   void Interpolate(Array<unsigned int> &codes, Array<unsigned int> &proc_ids,
-                    Array<unsigned int> &elem_ids, Vector &ref_pos,
-                    const GridFunction &field_in, Vector &field_out);
+       @param[out] field_out  Interpolated values. For points that are not found
+                              the value is set to #default_interp_value. */
    void Interpolate(const GridFunction &field_in, Vector &field_out);
    /** Search positions and interpolate */
    void Interpolate(const Vector &point_pos, const GridFunction &field_in,
@@ -111,27 +141,45 @@ public:
    void Interpolate(Mesh &m, const Vector &point_pos,
                     const GridFunction &field_in, Vector &field_out);
 
+   /// Average type to be used for L2 functions in-case a point is located at
+   /// an element boundary where the function might be multi-valued.
+   void SetL2AvgType(AvgType avgtype_) { avgtype = avgtype_; }
+
+   /// Set the default interpolation value for points that are not found in the
+   /// mesh.
+   void SetDefaultInterpolationValue(double interp_value_)
+   {
+      default_interp_value = interp_value_;
+   }
+
    /** Cleans up memory allocated internally by gslib.
-       Note that in parallel, this must be called before MPI_Finalize(), as
-       it calls MPI_Comm_free() for internal gslib communicators. */
+       Note that in parallel, this must be called before MPI_Finalize(), as it
+       calls MPI_Comm_free() for internal gslib communicators. */
    void FreeData();
 
    /// Return code for each point searched by FindPoints: inside element (0), on
    /// element boundary (1), or not found (2).
    const Array<unsigned int> &GetCode() const { return gsl_code; }
    /// Return element number for each point found by FindPoints.
-   const Array<unsigned int> &GetElem() const { return gsl_elem; }
+   const Array<unsigned int> &GetElem() const { return gsl_mfem_elem; }
    /// Return MPI rank on which each point was found by FindPoints.
    const Array<unsigned int> &GetProc() const { return gsl_proc; }
    /// Return reference coordinates for each point found by FindPoints.
-   const Vector &GetReferencePosition() const { return gsl_ref;  }
+   const Vector &GetReferencePosition() const { return gsl_mfem_ref;  }
    /// Return distance Distance between the sought and the found point
    /// in physical space, for each point found by FindPoints.
    const Vector &GetDist()              const { return gsl_dist; }
+
+   /// Return element number for each point found by FindPoints corresponding to
+   /// GSLIB mesh. gsl_mfem_elem != gsl_elem for mesh with simplices.
+   const Array<unsigned int> &GetGSLIBElem() const { return gsl_elem; }
+   /// Return reference coordinates in [-1,1] (internal range in GSLIB) for each
+   /// point found by FindPoints.
+   const Vector &GetGSLIBReferencePosition() const { return gsl_ref; }
 };
 
 } // namespace mfem
 
-#endif //MFEM_USE_GSLIB
+#endif // MFEM_USE_GSLIB
 
-#endif //MFEM_GSLIB guard
+#endif // MFEM_GSLIB

fem/libceed/ceed.cpp

@@ -35,6 +35,9 @@ extern Ceed ceed;
 
 std::string ceed_path;
 
+extern CeedBasisMap ceed_basis_map;
+extern CeedRestrMap ceed_restr_map;
+
 }
 
 void InitCeedCoeff(Coefficient* Q, CeedData* ptr)
@@ -81,10 +84,9 @@ static CeedElemTopology GetCeedTopology(Geometry::Type geom)
    }
 }
 
-static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
-                                                 const IntegrationRule &ir,
-                                                 Ceed ceed, CeedBasis *basis,
-                                                 CeedElemRestriction *restr)
+static void InitCeedNonTensorBasis(const FiniteElementSpace &fes,
+                                   const IntegrationRule &ir,
+                                   Ceed ceed, CeedBasis *basis)
 {
    Mesh *mesh = fes.GetMesh();
    const FiniteElement *fe = fes.GetFE(0);
@@ -97,7 +99,73 @@ static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
    Vector qweight(Q);
    Vector shape_i(P);
    DenseMatrix grad_i(P, dim);
+   const Table &el_dof = fes.GetElementToDofTable();
+   Array<int> tp_el_dof(el_dof.Size_of_connections());
+   const TensorBasisElement * tfe =
+      dynamic_cast<const TensorBasisElement *>(fe);
+   if (tfe) // Lexicographic ordering using dof_map
+   {
+      const Array<int>& dof_map = tfe->GetDofMap();
+      for (int i = 0; i < Q; i++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(i);
+         qref(0,i) = ip.x;
+         if (dim>1) { qref(1,i) = ip.y; }
+         if (dim>2) { qref(2,i) = ip.z; }
+         qweight(i) = ip.weight;
+         fe->CalcShape(ip, shape_i);
+         fe->CalcDShape(ip, grad_i);
+         for (int j = 0; j < P; j++)
+         {
+            shape(j, i) = shape_i(dof_map[j]);
+            for (int d = 0; d < dim; ++d)
+            {
+               grad(j+i*P+d*Q*P) = grad_i(dof_map[j], d);
+            }
+         }
+      }
+   }
+   else  // Native ordering
+   {
+      for (int i = 0; i < Q; i++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(i);
+         qref(0,i) = ip.x;
+         if (dim>1) { qref(1,i) = ip.y; }
+         if (dim>2) { qref(2,i) = ip.z; }
+         qweight(i) = ip.weight;
+         fe->CalcShape(ip, shape_i);
+         fe->CalcDShape(ip, grad_i);
+         for (int j = 0; j < P; j++)
+         {
+            shape(j, i) = shape_i(j);
+            for (int d = 0; d < dim; ++d)
+            {
+               grad(j+i*P+d*Q*P) = grad_i(j, d);
+            }
+         }
+      }
+   }
+   CeedBasisCreateH1(ceed, GetCeedTopology(fe->GetGeomType()), fes.GetVDim(),
+                     fe->GetDof(), ir.GetNPoints(), shape.GetData(),
+                     grad.GetData(), qref.GetData(), qweight.GetData(), basis);
+}
 
+static void InitCeedNonTensorRestriction(const FiniteElementSpace &fes,
+                                         const IntegrationRule &ir,
+                                         Ceed ceed, CeedElemRestriction *restr)
+{
+   Mesh *mesh = fes.GetMesh();
+   const FiniteElement *fe = fes.GetFE(0);
+   const int dim = mesh->Dimension();
+   const int P = fe->GetDof();
+   const int Q = ir.GetNPoints();
+   DenseMatrix shape(P, Q);
+   Vector grad(P*dim*Q);
+   DenseMatrix qref(dim, Q);
+   Vector qweight(Q);
+   Vector shape_i(P);
+   DenseMatrix grad_i(P, dim);
    CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
    const Table &el_dof = fes.GetElementToDofTable();
    Array<int> tp_el_dof(el_dof.Size_of_connections());
@@ -124,7 +192,6 @@ static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
             }
          }
       }
-
       for (int i = 0; i < mesh->GetNE(); i++)
       {
          const int el_offset = fe->GetDof() * i;
@@ -162,7 +229,6 @@ static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
             }
          }
       }
-
       for (int e = 0; e < mesh->GetNE(); e++)
       {
          for (int i = 0; i < P; i++)
@@ -178,19 +244,15 @@ static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
          }
       }
    }
-   CeedBasisCreateH1(ceed, GetCeedTopology(fe->GetGeomType()), fes.GetVDim(),
-                     fe->GetDof(), ir.GetNPoints(), shape.GetData(),
-                     grad.GetData(), qref.GetData(), qweight.GetData(), basis);
    CeedElemRestrictionCreate(ceed, mesh->GetNE(), fe->GetDof(), fes.GetVDim(),
                              compstride, (fes.GetVDim())*(fes.GetNDofs()),
                              CEED_MEM_HOST, CEED_COPY_VALUES,
                              tp_el_dof.GetData(), restr);
 }
 
-static void InitCeedTensorBasisAndRestriction(const FiniteElementSpace &fes,
-                                              const IntegrationRule &ir,
-                                              Ceed ceed, CeedBasis *basis,
-                                              CeedElemRestriction *restr)
+static void InitCeedTensorBasis(const FiniteElementSpace &fes,
+                                const IntegrationRule &ir,
+                                Ceed ceed, CeedBasis *basis)
 {
    Mesh *mesh = fes.GetMesh();
    const FiniteElement *fe = fes.GetFE(0);
@@ -198,7 +260,6 @@ static void InitCeedTensorBasisAndRestriction(const FiniteElementSpace &fes,
    const TensorBasisElement * tfe =
       dynamic_cast<const TensorBasisElement *>(fe);
    MFEM_VERIFY(tfe, "invalid FE");
-   const Array<int>& dof_map = tfe->GetDofMap();
    const FiniteElement *fe1d =
       fes.FEColl()->FiniteElementForGeometry(Geometry::SEGMENT);
    DenseMatrix shape1d(fe1d->GetDof(), ir.GetNPoints());
@@ -227,6 +288,28 @@ static void InitCeedTensorBasisAndRestriction(const FiniteElementSpace &fes,
                            ir.GetNPoints(), shape1d.GetData(),
                            grad1d.GetData(), qref1d.GetData(),
                            qweight1d.GetData(), basis);
+}
+
+static void InitCeedTensorRestriction(const FiniteElementSpace &fes,
+                                      const IntegrationRule &ir,
+                                      Ceed ceed, CeedElemRestriction *restr)
+{
+   Mesh *mesh = fes.GetMesh();
+   const FiniteElement *fe = fes.GetFE(0);
+   const TensorBasisElement * tfe =
+      dynamic_cast<const TensorBasisElement *>(fe);
+   MFEM_VERIFY(tfe, "invalid FE");
+   const Array<int>& dof_map = tfe->GetDofMap();
+   const FiniteElement *fe1d =
+      fes.FEColl()->FiniteElementForGeometry(Geometry::SEGMENT);
+   DenseMatrix shape1d(fe1d->GetDof(), ir.GetNPoints());
+   DenseMatrix grad1d(fe1d->GetDof(), ir.GetNPoints());
+   Vector qref1d(ir.GetNPoints()), qweight1d(ir.GetNPoints());
+   Vector shape_i(shape1d.Height());
+   DenseMatrix grad_i(grad1d.Height(), 1);
+   const H1_SegmentElement *h1_fe1d =
+      dynamic_cast<const H1_SegmentElement *>(fe1d);
+   MFEM_VERIFY(h1_fe1d, "invalid FE");
 
    CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
    const Table &el_dof = fes.GetElementToDofTable();
@@ -258,14 +341,52 @@ void InitCeedBasisAndRestriction(const FiniteElementSpace &fes,
                                  Ceed ceed, CeedBasis *basis,
                                  CeedElemRestriction *restr)
 {
-   if (UsesTensorBasis(fes))
+   // Check for FES -> basis, restriction in hash tables
+   const Mesh *mesh = fes.GetMesh();
+   const FiniteElement *fe = fes.GetFE(0);
+   const int P = fe->GetDof();
+   const int Q = irm.GetNPoints();
+   const int nelem = mesh->GetNE();
+   const int ncomp = fes.GetVDim();
+   CeedBasisKey basis_key(&fes, &irm, ncomp, P, Q);
+   auto basis_itr = internal::ceed_basis_map.find(basis_key);
+   CeedRestrKey restr_key(&fes, nelem, P, ncomp);
+   auto restr_itr = internal::ceed_restr_map.find(restr_key);
+
+   // Init or retreive key values
+   if (basis_itr == internal::ceed_basis_map.end())
    {
-      const IntegrationRule &ir = IntRules.Get(Geometry::SEGMENT, irm.GetOrder());
-      InitCeedTensorBasisAndRestriction(fes, ir, ceed, basis, restr);
+      if (UsesTensorBasis(fes))
+      {
+         const IntegrationRule &ir = IntRules.Get(Geometry::SEGMENT, irm.GetOrder());
+         InitCeedTensorBasis(fes, ir, ceed, basis);
+      }
+      else
+      {
+         InitCeedNonTensorBasis(fes, irm, ceed, basis);
+      }
+      internal::ceed_basis_map[basis_key] = *basis;
    }
    else
    {
-      InitCeedNonTensorBasisAndRestriction(fes, irm, ceed, basis, restr);
+      *basis = basis_itr->second;
+   }
+   if (restr_itr == internal::ceed_restr_map.end())
+   {
+      if (UsesTensorBasis(fes))
+      {
+         const IntegrationRule &ir = IntRules.Get(Geometry::SEGMENT, irm.GetOrder());
+         InitCeedTensorRestriction(fes, ir, ceed, restr);
+      }
+      else
+      {
+         InitCeedNonTensorRestriction(fes, irm, ceed, restr);
+      }
+      internal::ceed_restr_map[restr_key] = *restr;
+   }
+   else
+   {
+      *restr = restr_itr->second;
    }
 }
 
@@ -327,8 +448,8 @@ void CeedPAAssemble(const CeedPAOperator& op,
    CeedVectorCreate(ceed, nelem * nqpts * qdatasize, &ceedData.rho);
 
    // Context data to be passed to the 'f_build_diff' Q-function.
-   ceedData.build_ctx.dim = mesh->Dimension();
-   ceedData.build_ctx.space_dim = mesh->SpaceDimension();
+   ceedData.build_ctx_data.dim = mesh->Dimension();
+   ceedData.build_ctx_data.space_dim = mesh->SpaceDimension();
 
    std::string qf_file = GetCeedPath() + op.header;
    std::string qf;
@@ -342,7 +463,7 @@ void CeedPAAssemble(const CeedPAOperator& op,
          CeedQFunctionCreateInterior(ceed, 1, op.const_qf,
                                      qf.c_str(),
                                      &ceedData.build_qfunc);
-         ceedData.build_ctx.coeff = ((CeedConstCoeff*)ceedData.coeff)->val;
+         ceedData.build_ctx_data.coeff = ((CeedConstCoeff*)ceedData.coeff)->val;
          break;
       case CeedCoeff::Grid:
          qf = qf_file + op.grid_func;
@@ -358,8 +479,12 @@ void CeedPAAssemble(const CeedPAOperator& op,
    CeedQFunctionAddInput(ceedData.build_qfunc, "weights", 1, CEED_EVAL_WEIGHT);
    CeedQFunctionAddOutput(ceedData.build_qfunc, "qdata", qdatasize,
                           CEED_EVAL_NONE);
-   CeedQFunctionSetContext(ceedData.build_qfunc, &ceedData.build_ctx,
-                           sizeof(ceedData.build_ctx));
+
+   CeedQFunctionContextCreate(ceed, &ceedData.build_ctx);
+   CeedQFunctionContextSetData(ceedData.build_ctx, CEED_MEM_HOST, CEED_USE_POINTER,
+                               sizeof(ceedData.build_ctx_data),
+                               &ceedData.build_ctx_data);
+   CeedQFunctionSetContext(ceedData.build_qfunc, ceedData.build_ctx);
 
    // Create the operator that builds the quadrature data for the operator.
    CeedOperatorCreate(ceed, ceedData.build_qfunc, NULL, NULL,
@@ -399,8 +524,7 @@ void CeedPAAssemble(const CeedPAOperator& op,
    CeedQFunctionAddInput(ceedData.apply_qfunc, "qdata", qdatasize,
                          CEED_EVAL_NONE);
    CeedQFunctionAddOutput(ceedData.apply_qfunc, "v", dimV, op.test_op);
-   CeedQFunctionSetContext(ceedData.apply_qfunc, &ceedData.build_ctx,
-                           sizeof(ceedData.build_ctx));
+   CeedQFunctionSetContext(ceedData.apply_qfunc, ceedData.build_ctx);
 
    // Create the diff operator.
    CeedOperatorCreate(ceed, ceedData.apply_qfunc, NULL, NULL, &ceedData.oper);

fem/libceed/ceed.hpp

@@ -18,6 +18,9 @@
 #include "../../general/device.hpp"
 #include "../../linalg/vector.hpp"
 #include <ceed.h>
+#include <ceed-hash.h>
+#include <tuple>
+#include <unordered_map>
 
 namespace mfem
 {
@@ -27,7 +30,47 @@ class GridFunction;
 class IntegrationRule;
 class Coefficient;
 
-namespace internal { extern Ceed ceed; } // defined in device.cpp
+// Hash table for CeedBasis
+using CeedBasisKey =
+   std::tuple<const FiniteElementSpace*, const IntegrationRule*, int, int, int>;
+struct CeedBasisHash
+{
+   std::size_t operator()(const CeedBasisKey& k) const
+   {
+      return CeedHashCombine(CeedHashCombine(CeedHashInt(
+                                                reinterpret_cast<CeedHash64_t>(std::get<0>(k))),
+                                             CeedHashInt(
+                                                reinterpret_cast<CeedHash64_t>(std::get<1>(k)))),
+                             CeedHashCombine(CeedHashCombine(CeedHashInt(std::get<2>(k)),
+                                                             CeedHashInt(std::get<3>(k))),
+                                             CeedHashInt(std::get<4>(k))));
+   }
+};
+using CeedBasisMap =
+   std::unordered_map<const CeedBasisKey, CeedBasis, CeedBasisHash>;
+
+// Hash table for CeedElemRestriction
+using CeedRestrKey = std::tuple<const FiniteElementSpace*, int, int, int>;
+struct CeedRestrHash
+{
+   std::size_t operator()(const CeedRestrKey& k) const
+   {
+      return CeedHashCombine(CeedHashCombine(CeedHashInt(
+                                                reinterpret_cast<CeedHash64_t>(std::get<0>(k))),
+                                             CeedHashInt(std::get<1>(k))),
+                             CeedHashCombine(CeedHashInt(std::get<2>(k)),
+                                             CeedHashInt(std::get<3>(k))));
+   }
+};
+using CeedRestrMap =
+   std::unordered_map<const CeedRestrKey, CeedElemRestriction, CeedRestrHash>;
+
+namespace internal
+{
+extern Ceed ceed; // defined in device.cpp
+extern CeedBasisMap basis_map;
+extern CeedRestrMap restr_map;
+}
 
 /// A structure used to pass additional data to f_build_diff and f_apply_diff
 struct BuildContext { CeedInt dim, space_dim; CeedScalar coeff; };
@@ -56,7 +99,8 @@ struct CeedData
    CeedVector node_coords, rho;
    CeedCoeff coeff_type;
    void* coeff;
-   BuildContext build_ctx;
+   CeedQFunctionContext build_ctx;
+   BuildContext build_ctx_data;
 
    CeedVector u, v;
 
@@ -64,10 +108,6 @@ struct CeedData
    {
       CeedOperatorDestroy(&build_oper);
       CeedOperatorDestroy(&oper);
-      CeedBasisDestroy(&basis);
-      CeedBasisDestroy(&mesh_basis);
-      CeedElemRestrictionDestroy(&restr);
-      CeedElemRestrictionDestroy(&mesh_restr);
       CeedElemRestrictionDestroy(&restr_i);
       CeedElemRestrictionDestroy(&mesh_restr_i);
       CeedQFunctionDestroy(&apply_qfunc);
@@ -77,8 +117,6 @@ struct CeedData
       if (coeff_type==CeedCoeff::Grid)
       {
          CeedGridCoeff* c = (CeedGridCoeff*)coeff;
-         CeedBasisDestroy(&c->basis);
-         CeedElemRestrictionDestroy(&c->restr);
          CeedVectorDestroy(&c->coeffVector);
          delete c;
       }

fem/linearform.cpp

@@ -204,6 +204,14 @@ void LinearForm::Update(FiniteElementSpace *f, Vector &v, int v_offset)
    ResetDeltaLocations();
 }
 
+void LinearForm::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
+{
+   MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
+   fes = f;
+   v.UseDevice(true);
+   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
+}
+
 void LinearForm::AssembleDelta()
 {
    if (dlfi_delta.Size() == 0) { return; }

fem/linearform.hpp

@@ -26,7 +26,7 @@ protected:
    /// FE space on which the LinearForm lives. Not owned.
    FiniteElementSpace *fes;
 
-   /** @brief Indicates the LinerFormIntegrator%s stored in #dlfi, #dlfi_delta,
+   /** @brief Indicates the LinearFormIntegrator%s stored in #dlfi, #dlfi_delta,
        #blfi, and #flfi are owned by another LinearForm. */
    int extern_lfs;
 
@@ -175,6 +175,16 @@ public:
        @note This method does not perform assembly. */
    void Update(FiniteElementSpace *f, Vector &v, int v_offset);
 
+   /** @brief Make the LinearForm reference external data on a new
+       FiniteElementSpace. */
+   /** This method changes the FiniteElementSpace associated with the LinearForm
+       @a *f and sets the data of the Vector @a v (plus the @a v_offset) as
+       external data in the LinearForm.
+
+       @note This version of the method will also perform bounds checks when the
+       build option MFEM_DEBUG is enabled. */
+   virtual void MakeRef(FiniteElementSpace *f, Vector &v, int v_offset);
+
    /// Return the action of the LinearForm as a linear mapping.
    /** Linear forms are linear functionals which map GridFunctions to
        the real numbers.  This method performs this mapping which in

fem/lininteg.cpp

@@ -457,20 +457,8 @@ void VectorFEDomainLFCurlIntegrator::AssembleRHSElementVect(
 
       Tr.SetIntPoint (&ip);
       el.CalcPhysCurlShape(Tr, curlshape);
+      QF->Eval(vec, Tr, ip);
 
-      switch (spaceDim)
-      {
-         case 3:
-            MFEM_VERIFY(QF, "VectorFunctionCoefficient not provided");
-            QF->Eval(vec, Tr, ip);
-            break;
-         case 2:
-            MFEM_VERIFY(Q, "FunctionCoefficient (Scalar) not provided");
-            vec[0] = Q->Eval(Tr, ip);
-            break;
-         default:
-            break; // This should be unreachable
-      }
       vec *= ip.weight * Tr.Weight();
       curlshape.AddMult (vec, elvect);
    }
@@ -480,38 +468,17 @@ void VectorFEDomainLFCurlIntegrator::AssembleDeltaElementVect(
    const FiniteElement &fe, ElementTransformation &Trans, Vector &elvect)
 {
    int spaceDim = Trans.GetSpaceDim();
-   switch (spaceDim)
-   {
-      case 3:
-         MFEM_ASSERT(vec_delta != NULL,
-                     "coefficient must be VectorDeltaCoefficient");
-         break;
-      case 2:
-         MFEM_ASSERT(delta != NULL,
-                     "coefficient must be DeltaCoefficient");
-         break;
-      default:
-         break; // This should be unreachable
-   }
+   MFEM_ASSERT(vec_delta != NULL,
+               "coefficient must be VectorDeltaCoefficient");
    int dof = fe.GetDof();
    int n=(spaceDim == 3)? spaceDim : 1;
+   vec.SetSize(n);
    curlshape.SetSize(dof, n);
    elvect.SetSize(dof);
    fe.CalcPhysCurlShape(Trans, curlshape);
 
-   switch (spaceDim)
-   {
-      case 3:
-         vec_delta->EvalDelta(vec, Trans, Trans.GetIntPoint());
-         curlshape.Mult(vec, elvect);
-         break;
-      case 2:
-         curlshape.GetColumn(0,elvect);
-         elvect *= delta->EvalDelta(Trans, Trans.GetIntPoint());
-         break;
-      default:
-         break; // This should be unreachable
-   }
+   vec_delta->EvalDelta(vec, Trans, Trans.GetIntPoint());
+   curlshape.Mult(vec, elvect);
 }
 
 void VectorFEDomainLFDivIntegrator::AssembleRHSElementVect(

fem/lininteg.hpp

@@ -284,7 +284,6 @@ class VectorFEDomainLFCurlIntegrator : public DeltaLFIntegrator
 {
 private:
    VectorCoefficient *QF=nullptr;
-   Coefficient *Q=nullptr;
    DenseMatrix curlshape;
    Vector vec;
 
@@ -292,8 +291,6 @@ public:
    /// Constructs the domain integrator (Q, curl v)
    VectorFEDomainLFCurlIntegrator(VectorCoefficient &F)
       : DeltaLFIntegrator(F), QF(&F) { }
-   VectorFEDomainLFCurlIntegrator(Coefficient &F)
-      : DeltaLFIntegrator(F), Q(&F) { }
 
    virtual void AssembleRHSElementVect(const FiniteElement &el,
                                        ElementTransformation &Tr,

fem/nonlinearform.cpp

@@ -581,6 +581,13 @@ double BlockNonlinearForm::GetEnergyBlocked(const BlockVector &bx) const
          }
       }
 
+   // free the allocated memory
+   for (int i = 0; i < fes.Size(); ++i)
+   {
+      delete el_x[i];
+      delete vdofs[i];
+   }
+
    if (fnfi.Size())
    {
       MFEM_ABORT("TODO: add energy contribution from interior face terms");

fem/pgridfunc.cpp

@@ -655,6 +655,167 @@ void ParGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
 #endif
 }
 
+double ParGridFunction::ComputeDGFaceJumpError(Coefficient *exsol,
+                                               Coefficient *ell_coeff,
+                                               double Nu,
+                                               const IntegrationRule *irs[]) const
+{
+   const_cast<ParGridFunction *>(this)->ExchangeFaceNbrData();
+
+   int fdof, dim, intorder, k;
+   ElementTransformation *transf;
+   Vector shape, el_dofs, err_val, ell_coeff_val;
+   Array<int> vdofs;
+   IntegrationPoint eip;
+   double error = 0.0;
+
+   ParMesh *mesh = pfes->GetParMesh();
+   dim = mesh->Dimension();
+
+   std::map<int,int> local_to_shared;
+   for (int i = 0; i < mesh->GetNSharedFaces(); ++i)
+   {
+      int i_local = mesh->GetSharedFace(i);
+      local_to_shared[i_local] = i;
+   }
+
+   for (int i = 0; i < mesh->GetNumFaces(); i++)
+   {
+      double shared_face_factor = 1.0;
+      bool shared_face = false;
+      int iel1, iel2, info1, info2;
+      mesh->GetFaceElements(i, &iel1, &iel2);
+      mesh->GetFaceInfos(i, &info1, &info2);
+
+      intorder = fes->GetFE(iel1)->GetOrder();
+
+      FaceElementTransformations *face_elem_transf;
+      const FiniteElement *fe1, *fe2;
+      if (info2 >= 0 && iel2 < 0)
+      {
+         int ishared = local_to_shared[i];
+         face_elem_transf = mesh->GetSharedFaceTransformations(ishared);
+         iel2 = face_elem_transf->Elem2No - mesh->GetNE();
+         fe2 = pfes->GetFaceNbrFE(iel2);
+         if ( (k = fe2->GetOrder()) > intorder )
+         {
+            intorder = k;
+         }
+         shared_face = true;
+         shared_face_factor = 0.5;
+      }
+      else
+      {
+         face_elem_transf = mesh->GetFaceElementTransformations(i);
+
+         if (iel2 >= 0)
+         {
+            fe2 = pfes->GetFE(iel2);
+            if ( (k = fe2->GetOrder()) > intorder )
+            {
+               intorder = k;
+            }
+         }
+         else
+         {
+            fe2 = NULL;
+         }
+      }
+
+      intorder = 2 * intorder;  // <-------------
+      const IntegrationRule *ir;
+      if (irs)
+      {
+         ir = irs[face_elem_transf->GetGeometryType()];
+      }
+      else
+      {
+         ir = &(IntRules.Get(face_elem_transf->GetGeometryType(), intorder));
+      }
+      err_val.SetSize(ir->GetNPoints());
+      ell_coeff_val.SetSize(ir->GetNPoints());
+      // side 1
+      transf = face_elem_transf->Elem1;
+      fe1 = fes->GetFE(iel1);
+      fdof = fe1->GetDof();
+      fes->GetElementVDofs(iel1, vdofs);
+      shape.SetSize(fdof);
+      el_dofs.SetSize(fdof);
+      for (k = 0; k < fdof; k++)
+         if (vdofs[k] >= 0)
+         {
+            el_dofs(k) =   (*this)(vdofs[k]);
+         }
+         else
+         {
+            el_dofs(k) = - (*this)(-1-vdofs[k]);
+         }
+      for (int j = 0; j < ir->GetNPoints(); j++)
+      {
+         face_elem_transf->Loc1.Transform(ir->IntPoint(j), eip);
+         fe1->CalcShape(eip, shape);
+         transf->SetIntPoint(&eip);
+         ell_coeff_val(j) = ell_coeff->Eval(*transf, eip);
+         err_val(j) = exsol->Eval(*transf, eip) - (shape * el_dofs);
+      }
+      if (fe2 != NULL)
+      {
+         // side 2
+         transf = face_elem_transf->Elem2;
+         fdof = fe2->GetDof();
+         shape.SetSize(fdof);
+         el_dofs.SetSize(fdof);
+         if (shared_face)
+         {
+            pfes->GetFaceNbrElementVDofs(iel2, vdofs);
+            for (k = 0; k < fdof; k++)
+               if (vdofs[k] >= 0)
+               {
+                  el_dofs(k) = face_nbr_data[vdofs[k]];
+               }
+               else
+               {
+                  el_dofs(k) = - face_nbr_data[-1-vdofs[k]];
+               }
+         }
+         else
+         {
+            pfes->GetElementVDofs(iel2, vdofs);
+            for (k = 0; k < fdof; k++)
+               if (vdofs[k] >= 0)
+               {
+                  el_dofs(k) = (*this)(vdofs[k]);
+               }
+               else
+               {
+                  el_dofs(k) = - (*this)(-1 - vdofs[k]);
+               }
+         }
+         for (int j = 0; j < ir->GetNPoints(); j++)
+         {
+            face_elem_transf->Loc2.Transform(ir->IntPoint(j), eip);
+            fe2->CalcShape(eip, shape);
+            transf->SetIntPoint(&eip);
+            ell_coeff_val(j) += ell_coeff->Eval(*transf, eip);
+            ell_coeff_val(j) *= 0.5;
+            err_val(j) -= (exsol->Eval(*transf, eip) - (shape * el_dofs));
+         }
+      }
+      transf = face_elem_transf;
+      for (int j = 0; j < ir->GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(j);
+         transf->SetIntPoint(&ip);
+         error += shared_face_factor*(ip.weight * Nu * ell_coeff_val(j) *
+                                      pow(transf->Weight(), 1.0-1.0/(dim-1)) *
+                                      err_val(j) * err_val(j));
+      }
+   }
+
+   error = (error < 0.0) ? -sqrt(-error) : sqrt(error);
+   return GlobalLpNorm(2.0, error, pfes->GetComm());
+}
+
 void ParGridFunction::Save(std::ostream &out) const
 {
    double *data_  = const_cast<double*>(HostRead());
@@ -860,7 +1021,6 @@ double GlobalLpNorm(const double p, double loc_norm, MPI_Comm comm)
    return glob_norm;
 }
 
-
 void ParGridFunction::ComputeFlux(
    BilinearFormIntegrator &blfi,
    GridFunction &flux, bool wcoef, int subdomain)
@@ -1001,6 +1161,6 @@ double L2ZZErrorEstimator(BilinearFormIntegrator &flux_integrator,
    return pow(glob_error, 1.0/norm_p);
 }
 
-}
+} // namespace mfem
 
 #endif // MFEM_USE_MPI

fem/pgridfunc.hpp

@@ -283,6 +283,77 @@ public:
                           pfes->GetComm());
    }
 
+   /// Returns ||grad u_ex - grad u_h||_L2 for H1 or L2 elements
+   virtual double ComputeGradError(VectorCoefficient *exgrad,
+                                   const IntegrationRule *irs[] = NULL) const
+   {
+      return GlobalLpNorm(2.0, GridFunction::ComputeGradError(exgrad,irs),
+                          pfes->GetComm());
+   }
+
+   /// Returns ||curl u_ex - curl u_h||_L2 for ND elements
+   virtual double ComputeCurlError(VectorCoefficient *excurl,
+                                   const IntegrationRule *irs[] = NULL) const
+   {
+      return GlobalLpNorm(2.0, GridFunction::ComputeCurlError(excurl,irs),
+                          pfes->GetComm());
+   }
+
+   /// Returns ||div u_ex - div u_h||_L2 for RT elements
+   virtual double ComputeDivError(Coefficient *exdiv,
+                                  const IntegrationRule *irs[] = NULL) const
+   {
+      return GlobalLpNorm(2.0, GridFunction::ComputeDivError(exdiv,irs),
+                          pfes->GetComm());
+   }
+
+   /// Returns the Face Jumps error for L2 elements
+   virtual double ComputeDGFaceJumpError(Coefficient *exsol,
+                                         Coefficient *ell_coeff,
+                                         double Nu,
+                                         const IntegrationRule *irs[]=NULL)
+   const;
+
+   /// Returns either the H1-seminorm or the DG Face Jumps error or both
+   /// depending on norm_type = 1, 2, 3
+   virtual double ComputeH1Error(Coefficient *exsol, VectorCoefficient *exgrad,
+                                 Coefficient *ell_coef, double Nu,
+                                 int norm_type) const
+   {
+      return GlobalLpNorm(2.0,
+                          GridFunction::ComputeH1Error(exsol,exgrad,ell_coef,
+                                                       Nu, norm_type),
+                          pfes->GetComm());
+   }
+
+   /// Returns the error measured in H1-norm for H1 elements or in "broken"
+   /// H1-norm for L2 elements
+   virtual double ComputeH1Error(Coefficient *exsol, VectorCoefficient *exgrad,
+                                 const IntegrationRule *irs[] = NULL) const
+   {
+      return GlobalLpNorm(2.0, GridFunction::ComputeH1Error(exsol,exgrad,irs),
+                          pfes->GetComm());
+   }
+
+   /// Returns the error measured H(div)-norm for RT elements
+   virtual double ComputeHDivError(VectorCoefficient *exsol,
+                                   Coefficient *exdiv,
+                                   const IntegrationRule *irs[] = NULL) const
+   {
+      return GlobalLpNorm(2.0, GridFunction::ComputeHDivError(exsol,exdiv,irs),
+                          pfes->GetComm());
+   }
+
+   /// Returns the error measured H(curl)-norm for ND elements
+   virtual double ComputeHCurlError(VectorCoefficient *exsol,
+                                    VectorCoefficient *excurl,
+                                    const IntegrationRule *irs[] = NULL) const
+   {
+      return GlobalLpNorm(2.0,
+                          GridFunction::ComputeHCurlError(exsol,excurl,irs),
+                          pfes->GetComm());
+   }
+
    virtual double ComputeMaxError(Coefficient *exsol[],
                                   const IntegrationRule *irs[] = NULL) const
    {

fem/plinearform.cpp

@@ -21,7 +21,6 @@ namespace mfem
 void ParLinearForm::Update(ParFiniteElementSpace *pf)
 {
    if (pf) { pfes = pf; }
-
    LinearForm::Update(pfes);
 }
 
@@ -31,6 +30,19 @@ void ParLinearForm::Update(ParFiniteElementSpace *pf, Vector &v, int v_offset)
    LinearForm::Update(pf,v,v_offset);
 }
 
+void ParLinearForm::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
+{
+   LinearForm::MakeRef(f, v, v_offset);
+   pfes = dynamic_cast<ParFiniteElementSpace*>(f);
+   MFEM_ASSERT(pfes != NULL, "not a ParFiniteElementSpace");
+}
+
+void ParLinearForm::MakeRef(ParFiniteElementSpace *pf, Vector &v, int v_offset)
+{
+   LinearForm::MakeRef(pf, v, v_offset);
+   pfes = pf;
+}
+
 void ParLinearForm::ParallelAssemble(Vector &tv)
 {
    const Operator* prolong = pfes->GetProlongationMatrix();

fem/plinearform.hpp

@@ -92,6 +92,27 @@ public:
        @note This method does not perform assembly. */
    void Update(ParFiniteElementSpace *pf, Vector &v, int v_offset);
 
+
+   /** @brief Make the ParLinearForm reference external data on a new
+       FiniteElementSpace. */
+   /** This method changes the FiniteElementSpace associated with the
+       ParLinearForm to @a *f and sets the data of the Vector @a v (plus the @a
+       v_offset) as external data in the ParLinearForm.
+
+       @note This version of the method will also perform bounds checks when the
+       build option MFEM_DEBUG is enabled. */
+   virtual void MakeRef(FiniteElementSpace *f, Vector &v, int v_offset);
+
+   /** @brief Make the ParLinearForm reference external data on a new
+       ParFiniteElementSpace. */
+   /** This method changes the ParFiniteElementSpace associated with the
+       ParLinearForm to @a *pf and sets the data of the Vector @a v (plus the @a
+       v_offset) as external data in the ParLinearForm.
+
+       @note This version of the method will also perform bounds checks when the
+       build option MFEM_DEBUG is enabled. */
+   void MakeRef(ParFiniteElementSpace *pf, Vector &v, int v_offset);
+
    /// Assemble the vector on the true dofs, i.e. P^t v.
    void ParallelAssemble(Vector &tv);
 
@@ -99,10 +120,10 @@ public:
    HypreParVector *ParallelAssemble();
 
    /// Return the action of the ParLinearForm as a linear mapping.
-   /** Linear forms are linear functionals which map ParGridFunction%s to
-       the real numbers.  This method performs this mapping which in
-       this case is equivalent as an inner product of the ParLinearForm
-       and ParGridFunction. */
+   /** Linear forms are linear functionals which map ParGridFunction%s to the
+       real numbers. This method performs this mapping which in this case is
+       equivalent as an inner product of the ParLinearForm and
+       ParGridFunction. */
    double operator()(const ParGridFunction &gf) const
    {
       return InnerProduct(pfes->GetComm(), *this, gf);

fem/tmop_tools.cpp

@@ -298,34 +298,12 @@ void InterpolatorFP::SetInitialField(const Vector &init_nodes,
    field0_gf = init_field;
 
    dim = f->GetFE(0)->GetDim();
-   const int pts_cnt = init_nodes.Size() / dim;
-   el_id_out.SetSize(pts_cnt);
-   code_out.SetSize(pts_cnt);
-   task_id_out.SetSize(pts_cnt);
-   pos_r_out.SetSize(pts_cnt*dim);
-   dist_p_out.SetSize(pts_cnt);
 }
 
 void InterpolatorFP::ComputeAtNewPosition(const Vector &new_nodes,
                                           Vector &new_field)
 {
-   const int pts_cnt = new_nodes.Size() / dim;
-
-   // The sizes may change between calls due to AMR.
-   if (el_id_out.Size() != pts_cnt)
-   {
-      el_id_out.SetSize(pts_cnt);
-      code_out.SetSize(pts_cnt);
-      task_id_out.SetSize(pts_cnt);
-      pos_r_out.SetSize(pts_cnt*dim);
-      dist_p_out(pts_cnt);
-   }
-
-   // Interpolate FE function values on the found points.
-   finder->FindPoints(new_nodes, code_out, task_id_out,
-                      el_id_out, pos_r_out, dist_p_out);
-   finder->Interpolate(code_out, task_id_out, el_id_out,
-                       pos_r_out, field0_gf, new_field);
+   finder->Interpolate(new_nodes, field0_gf, new_field);
 }
 
 #endif

fem/tmop_tools.hpp

@@ -49,8 +49,6 @@ private:
    Vector nodes0;
    GridFunction field0_gf;
    FindPointsGSLIB *finder;
-   Array<uint> el_id_out, code_out, task_id_out;
-   Vector pos_r_out, dist_p_out;
    int dim;
 public:
    InterpolatorFP() : finder(NULL) { }

general/adios2stream.cpp

@@ -235,8 +235,8 @@ void adios2stream::Print(const Mesh& mesh, const mode print_mode)
       }
 
       // format info
-      SafeDefineAttribute<std::string>(io, "format", "MFEM ADIOS2 BP v0.1" );
-      SafeDefineAttribute<std::string>(io, "format/version", "0.1" );
+      SafeDefineAttribute<std::string>(io, "format", "MFEM ADIOS2 BP v0.2" );
+      SafeDefineAttribute<std::string>(io, "format/version", "0.2" );
       std::string mesh_type = "Unknown";
       std::vector<std::string> viz_tools;
       viz_tools.reserve(2); //for now
@@ -298,6 +298,7 @@ void adios2stream::Print(const Mesh& mesh, const mode print_mode)
          element_nvertices = static_cast<size_t>(mesh.elements[0]->GetNVertices());
       }
       SafeDefineVariable<uint64_t>(io, "connectivity", {}, {}, {nelements, element_nvertices+1});
+      SafeDefineVariable<int32_t>(io, "material", {}, {}, {nelements});
 
       // vertices
       SafeDefineVariable<uint32_t>(io,"NumOfVertices", {adios2::LocalValueDim});
@@ -348,8 +349,15 @@ void adios2stream::Print(const Mesh& mesh, const mode print_mode)
          io.InquireVariable<uint64_t>("connectivity");
       adios2::Variable<uint64_t>::Span span_connectivity = engine.Put<uint64_t>
                                                            (var_connectivity);
+
+      adios2::Variable<int32_t> var_element_attribute =
+         io.InquireVariable<int32_t>("material");
+      adios2::Variable<int32_t>::Span span_element_attribute = engine.Put<int32_t>
+                                                               (var_element_attribute);
+
       size_t span_vertices_offset = 0;
       size_t span_connectivity_offset = 0;
+      size_t span_element_attribute_offset = 0;
       // use for setting absolute node id for each element
       size_t point_id = 0;
       DenseMatrix pmatrix;
@@ -370,6 +378,9 @@ void adios2stream::Print(const Mesh& mesh, const mode print_mode)
          }
          span_vertices_offset += static_cast<size_t>(pmatrix.Width()*pmatrix.Height());
 
+         // element attribute
+         const int element_attribute = mesh.GetAttribute(e);
+
          // connectivity
          const int nv = Geometries.GetVertices(type)->GetNPoints();
          const Array<int> &element_vertices = refined_geometry->RefGeoms;
@@ -379,6 +390,10 @@ void adios2stream::Print(const Mesh& mesh, const mode print_mode)
             span_connectivity[span_connectivity_offset] = static_cast<uint64_t>(nv);
             ++span_connectivity_offset;
 
+            span_element_attribute[span_element_attribute_offset] = static_cast<int32_t>
+                                                                    (element_attribute);
+            ++span_element_attribute_offset;
+
             for (int k =0; k < nv; k++, v++ )
             {
                span_connectivity[span_connectivity_offset] = static_cast<uint64_t>
@@ -419,9 +434,17 @@ void adios2stream::Print(const Mesh& mesh, const mode print_mode)
       adios2::Variable<uint64_t>::Span spanConnectivity =
          engine.Put<uint64_t>(varConnectivity);
 
+      adios2::Variable<int32_t> varElementAttribute =
+         io.InquireVariable<int32_t>("material");
+      // zero-copy access to adios2 buffer to put non-contiguous to contiguous memory
+      adios2::Variable<int32_t>::Span spanElementAttribute =
+         engine.Put<int32_t>(varElementAttribute);
+
       size_t elementPosition = 0;
       for (int e = 0; e < mesh.GetNE(); ++e)
       {
+         spanElementAttribute[e] = static_cast<int32_t>(mesh.GetAttribute(e));
+
          const int nVertices = mesh.elements[e]->GetNVertices();
          spanConnectivity[elementPosition] = nVertices;
          for (int v = 0; v < nVertices; ++v)
@@ -688,7 +711,7 @@ std::string adios2stream::VTKSchema() const noexcept
 {
    std::string vtkSchema = R"(
 <?xml version="1.0"?>
-<VTKFile type="UnstructuredGrid" version="0.1" byte_order="LittleEndian">
+<VTKFile type="UnstructuredGrid" version="0.2" byte_order="LittleEndian">
   <UnstructuredGrid>
     <Piece NumberOfPoints="NumOfVertices" NumberOfCells="NumOfElements">
       <Points>
@@ -696,6 +719,9 @@ std::string adios2stream::VTKSchema() const noexcept
 
    vtkSchema += R"(
       </Points>
+	  <CellData>
+        <DataArray Name="material" />
+	  </CellData>
       <Cells>
         <DataArray Name="connectivity" />
         <DataArray Name="types" />

general/device.cpp

@@ -12,9 +12,10 @@
 #include "forall.hpp"
 #include "occa.hpp"
 #ifdef MFEM_USE_CEED
-#include <ceed.h>
+#include "../fem/libceed/ceed.hpp"
 #endif
 
+#include <unordered_map>
 #include <string>
 #include <map>
 
@@ -33,13 +34,16 @@ occa::device occaDevice;
 
 #ifdef MFEM_USE_CEED
 Ceed ceed = NULL;
+
+CeedBasisMap ceed_basis_map;
+CeedRestrMap ceed_restr_map;
 #endif
 
 // Backends listed by priority, high to low:
 static const Backend::Id backend_list[Backend::NUM_BACKENDS] =
 {
    Backend::CEED_CUDA, Backend::OCCA_CUDA, Backend::RAJA_CUDA, Backend::CUDA,
-   Backend::HIP, Backend::DEBUG,
+   Backend::HIP, Backend::DEBUG_DEVICE,
    Backend::OCCA_OMP, Backend::RAJA_OMP, Backend::OMP,
    Backend::CEED_CPU, Backend::OCCA_CPU, Backend::RAJA_CPU, Backend::CPU
 };
@@ -154,6 +158,16 @@ Device::~Device()
    {
       free(device_option);
 #ifdef MFEM_USE_CEED
+      // Destroy FES -> CeedBasis, CeedElemRestriction hash table contents
+      for (auto entry : internal::ceed_basis_map)
+      {
+         CeedBasisDestroy(&entry.second);
+      }
+      for (auto entry : internal::ceed_restr_map)
+      {
+         CeedElemRestrictionDestroy(&entry.second);
+      }
+      // Destroy Ceed context
       CeedDestroy(&internal::ceed);
 #endif
       mm.Destroy();
@@ -266,7 +280,7 @@ void Device::Print(std::ostream &out)
 
 void Device::UpdateMemoryTypeAndClass()
 {
-   const bool debug = Device::Allows(Backend::DEBUG);
+   const bool debug = Device::Allows(Backend::DEBUG_DEVICE);
 
    const bool device = Device::Allows(Backend::DEVICE_MASK);
 
@@ -504,7 +518,7 @@ void Device::Setup(const int device)
          CeedDeviceSetup(device_option);
       }
    }
-   if (Allows(Backend::DEBUG)) { ngpu = 1; }
+   if (Allows(Backend::DEBUG_DEVICE)) { ngpu = 1; }
 }
 
 } // mfem

general/device.hpp

@@ -64,8 +64,9 @@ struct Backend
       /** @brief [device] Debug backend: host memory is READ/WRITE protected
           while a device is in use. It allows to test the "device" code-path
           (using separate host/device memory pools and host <-> device
-          transfers) without any GPU hardware. */
-      DEBUG = 1 << 12
+          transfers) without any GPU hardware. As 'DEBUG' is sometimes used
+          as a macro, `_DEVICE` has been added to avoid conflicts. */
+      DEBUG_DEVICE = 1 << 12
    };
 
    /** @brief Additional useful constants. For example, the *_MASK constants can
@@ -86,7 +87,7 @@ struct Backend
       /// Bitwise-OR of all CEED backends
       CEED_MASK = CEED_CPU | CEED_CUDA,
       /// Biwise-OR of all device backends
-      DEVICE_MASK = CUDA_MASK | HIP_MASK | DEBUG,
+      DEVICE_MASK = CUDA_MASK | HIP_MASK | DEBUG_DEVICE,
 
       /// Biwise-OR of all RAJA backends
       RAJA_MASK = RAJA_CPU | RAJA_OMP | RAJA_CUDA,
@@ -193,7 +194,8 @@ public:
        * The available backends are described by the Backend class.
        * The string name of a backend is the lowercase version of the
          Backend::Id enumeration constant with '_' replaced by '-', e.g. the
-         string name of 'RAJA_CPU' is 'raja-cpu'.
+         string name of 'RAJA_CPU' is 'raja-cpu'. The string name of the debug
+         backend (Backend::Id 'DEBUG_DEVICE') is exceptionally set to 'debug'.
        * The 'cpu' backend is always enabled with lowest priority.
        * The current backend priority from highest to lowest is:
          'ceed-cuda', 'occa-cuda', 'raja-cuda', 'cuda', 'hip', 'debug',

general/forall.hpp

@@ -343,7 +343,7 @@ inline void ForallWrap(const bool use_dev, const int N,
    { return HipWrap3D(N, d_body, X, Y, Z); }
 #endif
 
-   if (Device::Allows(Backend::DEBUG)) { goto backend_cpu; }
+   if (Device::Allows(Backend::DEBUG_DEVICE)) { goto backend_cpu; }
 
 #if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_OPENMP)
    // Handle all allowed OpenMP backends except Backend::OMP

general/mem_manager.cpp

@@ -136,8 +136,10 @@ struct Memory
    void *d_ptr;
    const size_t bytes;
    const MemoryType h_mt, d_mt;
+   mutable bool h_rw, d_rw;
    Memory(void *p, size_t b, MemoryType h, MemoryType d):
-      h_ptr(p), d_ptr(nullptr), bytes(b), h_mt(h), d_mt(d) { }
+      h_ptr(p), d_ptr(nullptr), bytes(b), h_mt(h), d_mt(d),
+      h_rw(true), d_rw(true) { }
 };
 
 /// Alias class that holds the base memory region and the offset
@@ -173,8 +175,8 @@ public:
    virtual ~HostMemorySpace() { }
    virtual void Alloc(void **ptr, size_t bytes) { *ptr = std::malloc(bytes); }
    virtual void Dealloc(void *ptr) { std::free(ptr); }
-   virtual void Protect(const void*, size_t) { }
-   virtual void Unprotect(const void*, size_t) { }
+   virtual void Protect(const Memory&, size_t) { }
+   virtual void Unprotect(const Memory&, size_t) { }
    virtual void AliasProtect(const void*, size_t) { }
    virtual void AliasUnprotect(const void*, size_t) { }
 };
@@ -352,8 +354,10 @@ public:
    MmuHostMemorySpace(): HostMemorySpace() { MmuInit(); }
    void Alloc(void **ptr, size_t bytes) { MmuAlloc(ptr, bytes); }
    void Dealloc(void *ptr) { MmuDealloc(ptr, maps->memories.at(ptr).bytes); }
-   void Protect(const void *ptr, size_t bytes) { MmuProtect(ptr, bytes); }
-   void Unprotect(const void *ptr, size_t bytes) { MmuAllow(ptr, bytes); }
+   void Protect(const Memory& mem, size_t bytes)
+   { if (mem.h_rw) { mem.h_rw = false; MmuProtect(mem.h_ptr, bytes); } }
+   void Unprotect(const Memory &mem, size_t bytes)
+   { if (!mem.h_rw) { mem.h_rw = true; MmuAllow(mem.h_ptr, bytes); } }
    /// Aliases need to be restricted during protection
    void AliasProtect(const void *ptr, size_t bytes)
    { MmuProtect(MmuAddrR(ptr), MmuLengthR(ptr, bytes)); }
@@ -442,8 +446,10 @@ public:
    MmuDeviceMemorySpace(): DeviceMemorySpace() { }
    void Alloc(Memory &m) { MmuAlloc(&m.d_ptr, m.bytes); }
    void Dealloc(Memory &m) { MmuDealloc(m.d_ptr, m.bytes); }
-   void Protect(const Memory &m) { MmuProtect(m.d_ptr, m.bytes); }
-   void Unprotect(const Memory &m) { MmuAllow(m.d_ptr, m.bytes); }
+   void Protect(const Memory &m)
+   { if (m.d_rw) { m.d_rw = false; MmuProtect(m.d_ptr, m.bytes); } }
+   void Unprotect(const Memory &m)
+   { if (!m.d_rw) { m.d_rw = true; MmuAllow(m.d_ptr, m.bytes); } }
    /// Aliases need to be restricted during protection
    void AliasProtect(const void *ptr, size_t bytes)
    { MmuProtect(MmuAddrR(ptr), MmuLengthR(ptr, bytes)); }
@@ -969,11 +975,8 @@ void MemoryManager::Copy_(void *dst_h_ptr, const void *src_h_ptr,
       {
          if (dst_h_ptr != src_d_ptr && bytes != 0)
          {
-            internal::Memory &dst_h_base = maps->memories.at(dst_h_ptr);
             internal::Memory &src_d_base = maps->memories.at(src_d_ptr);
-            MemoryType dst_h_mt = dst_h_base.h_mt;
             MemoryType src_d_mt = src_d_base.d_mt;
-            ctrl->Host(dst_h_mt)->Unprotect(dst_h_ptr, bytes);
             ctrl->Device(src_d_mt)->DtoH(dst_h_ptr, src_d_ptr, bytes);
          }
       }
@@ -1174,13 +1177,14 @@ void *MemoryManager::GetDevicePtr(const void *h_ptr, size_t bytes,
    const MemoryType &d_mt = mem.d_mt;
    MFEM_VERIFY_TYPES(h_mt, d_mt);
    if (!mem.d_ptr) { ctrl->Device(d_mt)->Alloc(mem); }
+   // Aliases might have done some protections
    ctrl->Device(d_mt)->Unprotect(mem);
    if (copy_data)
    {
       MFEM_ASSERT(bytes <= mem.bytes, "invalid copy size");
       ctrl->Device(d_mt)->HtoD(mem.d_ptr, h_ptr, bytes);
    }
-   ctrl->Host(h_mt)->Protect(h_ptr, bytes);
+   ctrl->Host(h_mt)->Protect(mem, bytes);
    return mem.d_ptr;
 }
 
@@ -1206,6 +1210,7 @@ void *MemoryManager::GetAliasDevicePtr(const void *alias_ptr, size_t bytes,
    void *alias_d_ptr = static_cast<char*>(mem.d_ptr) + offset;
    MFEM_ASSERT(alias_h_ptr == alias_ptr, "internal error");
    MFEM_ASSERT(bytes <= alias.bytes, "internal error");
+   mem.d_rw = false;
    ctrl->Device(d_mt)->AliasUnprotect(alias_d_ptr, bytes);
    ctrl->Host(h_mt)->AliasUnprotect(alias_ptr, bytes);
    if (copy) { ctrl->Device(d_mt)->HtoD(alias_d_ptr, alias_h_ptr, bytes); }
@@ -1221,8 +1226,8 @@ void *MemoryManager::GetHostPtr(const void *ptr, size_t bytes, bool copy)
    const MemoryType &h_mt = mem.h_mt;
    const MemoryType &d_mt = mem.d_mt;
    MFEM_VERIFY_TYPES(h_mt, d_mt);
-   ctrl->Host(h_mt)->Unprotect(mem.h_ptr, bytes);
    // Aliases might have done some protections
+   ctrl->Host(h_mt)->Unprotect(mem, bytes);
    if (mem.d_ptr) { ctrl->Device(d_mt)->Unprotect(mem); }
    if (copy && mem.d_ptr) { ctrl->Device(d_mt)->DtoH(mem.h_ptr, mem.d_ptr, bytes); }
    if (mem.d_ptr) { ctrl->Device(d_mt)->Protect(mem); }
@@ -1240,6 +1245,7 @@ void *MemoryManager::GetAliasHostPtr(const void *ptr, size_t bytes,
    void *alias_h_ptr = static_cast<char*>(mem->h_ptr) + alias.offset;
    void *alias_d_ptr = static_cast<char*>(mem->d_ptr) + alias.offset;
    MFEM_ASSERT(alias_h_ptr == ptr,  "internal error");
+   mem->h_rw = false;
    ctrl->Host(h_mt)->AliasUnprotect(alias_h_ptr, bytes);
    if (mem->d_ptr) { ctrl->Device(d_mt)->AliasUnprotect(alias_d_ptr, bytes); }
    if (copy_data && mem->d_ptr)

linalg/complex_operator.cpp

@@ -26,10 +26,10 @@ ComplexOperator::ComplexOperator(Operator * Op_Real, Operator * Op_Imag,
    , ownReal_(ownReal)
    , ownImag_(ownImag)
    , convention_(convention)
-   , x_r_(NULL, width / 2)
-   , x_i_(NULL, width / 2)
-   , y_r_(NULL, height / 2)
-   , y_i_(NULL, height / 2)
+   , x_r_()
+   , x_i_()
+   , y_r_()
+   , y_i_()
    , u_(NULL)
    , v_(NULL)
 {}
@@ -68,14 +68,26 @@ const Operator & ComplexOperator::imag() const
 
 void ComplexOperator::Mult(const Vector &x, Vector &y) const
 {
-   double * x_data = x.GetData();
-   x_r_.SetData(x_data);
-   x_i_.SetData(&x_data[width / 2]);
+   x.Read();
+   y.UseDevice(true); y = 0.0;
 
-   y_r_.SetData(&y[0]);
-   y_i_.SetData(&y[height / 2]);
+   x_r_.MakeRef(const_cast<Vector&>(x), 0, width/2);
+   x_i_.MakeRef(const_cast<Vector&>(x), width/2, width/2);
+
+   y_r_.MakeRef(y, 0, height/2);
+   y_i_.MakeRef(y, height/2, height/2);
 
    this->Mult(x_r_, x_i_, y_r_, y_i_);
+
+   y_r_.SyncAliasMemory(y);
+   y_i_.SyncAliasMemory(y);
+
+   // Destroy alias vectors to prevent dangling aliases when the base vectors
+   // are deleted
+   x_r_.Destroy();
+   x_i_.Destroy();
+   y_r_.Destroy();
+   y_i_.Destroy();
 }
 
 void ComplexOperator::Mult(const Vector &x_r, const Vector &x_i,
@@ -91,31 +103,47 @@ void ComplexOperator::Mult(const Vector &x_r, const Vector &x_i,
       y_r = 0.0;
       y_i = 0.0;
    }
+
    if (Op_Imag_)
    {
-      if (!v_) { v_ = new Vector(Op_Imag_->Height()); }
+      if (!v_) { v_ = new Vector(); }
+      v_->UseDevice(true);
+      v_->SetSize(Op_Imag_->Height());
+
       Op_Imag_->Mult(x_i, *v_);
-      y_r_ -= *v_;
+      y_r.Add(-1.0, *v_);
       Op_Imag_->Mult(x_r, *v_);
-      y_i_ += *v_;
+      y_i.Add(1.0, *v_);
    }
 
    if (convention_ == BLOCK_SYMMETRIC)
    {
-      y_i_ *= -1.0;
+      y_i *= -1.0;
    }
 }
 
 void ComplexOperator::MultTranspose(const Vector &x, Vector &y) const
 {
-   double * x_data = x.GetData();
-   y_r_.SetData(x_data);
-   y_i_.SetData(&x_data[height / 2]);
+   x.Read();
+   y.UseDevice(true); y = 0.0;
 
-   x_r_.SetData(&y[0]);
-   x_i_.SetData(&y[width / 2]);
+   x_r_.MakeRef(const_cast<Vector&>(x), 0, height/2);
+   x_i_.MakeRef(const_cast<Vector&>(x), height/2, height/2);
 
-   this->MultTranspose(y_r_, y_i_, x_r_, x_i_);
+   y_r_.MakeRef(y, 0, width/2);
+   y_i_.MakeRef(y, width/2, width/2);
+
+   this->MultTranspose(x_r_, x_i_, y_r_, y_i_);
+
+   y_r_.SyncAliasMemory(y);
+   y_i_.SyncAliasMemory(y);
+
+   // Destroy alias vectors to prevent dangling aliases when the base vectors
+   // are deleted
+   x_r_.Destroy();
+   x_i_.Destroy();
+   y_r_.Destroy();
+   y_i_.Destroy();
 }
 
 void ComplexOperator::MultTranspose(const Vector &x_r, const Vector &x_i,
@@ -136,13 +164,17 @@ void ComplexOperator::MultTranspose(const Vector &x_r, const Vector &x_i,
       y_r = 0.0;
       y_i = 0.0;
    }
+
    if (Op_Imag_)
    {
-      if (!u_) { u_ = new Vector(Op_Imag_->Width()); }
+      if (!u_) { u_ = new Vector(); }
+      u_->UseDevice(true);
+      u_->SetSize(Op_Imag_->Width());
+
       Op_Imag_->MultTranspose(x_i, *u_);
-      y_r_.Add(convention_ == BLOCK_SYMMETRIC ? -1.0 : 1.0, *u_);
+      y_r.Add(convention_ == BLOCK_SYMMETRIC ? -1.0 : 1.0, *u_);
       Op_Imag_->MultTranspose(x_r, *u_);
-      y_i_ -= *u_;
+      y_i.Add(-1.0, *u_);
    }
 }
 

linalg/complex_operator.hpp

@@ -100,7 +100,7 @@ public:
    /** @brief Real or imaginary part accessor methods
 
        The following accessor methods should only be called if the requested
-       part of the opertor is known to exist.  This can be checked with
+       part of the operator is known to exist. This can be checked with
        hasRealPart() or hasImagPart().
    */
    virtual Operator & real();
@@ -166,7 +166,7 @@ public:
    /** Combine the blocks making up this complex operator into a single
        SparseMatrix. The resulting matrix can be passed to solvers which require
        access to the matrix entries themselves, such as sparse direct solvers,
-       rather than simply the action of the opertor. Note that this combined
+       rather than simply the action of the operator. Note that this combined
        operator requires roughly twice the memory of the block structured
        operator. */
    SparseMatrix * GetSystemMatrix() const;
@@ -269,7 +269,7 @@ public:
        HypreParMatrix. The resulting matrix can be passed to solvers which
        require access to the matrix entries themselves, such as sparse direct
        solvers or Hypre preconditioners, rather than simply the action of the
-       opertor. Note that this combined operator requires roughly twice the
+       operator. Note that this combined operator requires roughly twice the
        memory of the block structured operator. */
    HypreParMatrix * GetSystemMatrix() const;
 

linalg/densemat.cpp

@@ -373,7 +373,7 @@ void DenseMatrix::SymmetricScaling(const Vector & s)
 {
    if (height != width || s.Size() != height)
    {
-      mfem_error("DenseMatrix::SymmetricScaling");
+      mfem_error("DenseMatrix::SymmetricScaling: dimension mismatch");
    }
 
    double * ss = new double[width];
@@ -401,7 +401,7 @@ void DenseMatrix::InvSymmetricScaling(const Vector & s)
 {
    if (height != width || s.Size() != width)
    {
-      mfem_error("DenseMatrix::SymmetricScaling");
+      mfem_error("DenseMatrix::InvSymmetricScaling: dimension mismatch");
    }
 
    double * ss = new double[width];
@@ -528,7 +528,7 @@ double DenseMatrix::Weight() const
       double F = d[0] * d[3] + d[1] * d[4] + d[2] * d[5];
       return sqrt(E * G - F * F);
    }
-   mfem_error("DenseMatrix::Weight()");
+   mfem_error("DenseMatrix::Weight(): mismatched or unsupported dimensions");
    return 0.0;
 }
 
@@ -639,7 +639,7 @@ void DenseMatrix::Invert()
 #ifdef MFEM_DEBUG
    if (Height() <= 0 || Height() != Width())
    {
-      mfem_error("DenseMatrix::Invert()");
+      mfem_error("DenseMatrix::Invert(): dimension mismatch");
    }
 #endif
 
@@ -1083,7 +1083,7 @@ void DenseMatrix::Eigensystem(Vector &ev, DenseMatrix *evect)
 
    MFEM_CONTRACT_VAR(ev);
    MFEM_CONTRACT_VAR(evect);
-   mfem_error("DenseMatrix::Eigensystem");
+   mfem_error("DenseMatrix::Eigensystem: Compiled without LAPACK");
 
 #endif
 }
@@ -1164,7 +1164,7 @@ void DenseMatrix::Eigensystem(DenseMatrix &b, Vector &ev,
    MFEM_CONTRACT_VAR(b);
    MFEM_CONTRACT_VAR(ev);
    MFEM_CONTRACT_VAR(evect);
-   mfem_error("DenseMatrix::Eigensystem for generalized eigenvalues");
+   mfem_error("DenseMatrix::Eigensystem(generalized): Compiled without LAPACK");
 #endif
 }
 
@@ -1204,7 +1204,7 @@ void DenseMatrix::SingularValues(Vector &sv) const
 #else
    MFEM_CONTRACT_VAR(sv);
    // compiling without lapack
-   mfem_error("DenseMatrix::SingularValues");
+   mfem_error("DenseMatrix::SingularValues: Compiled without LAPACK");
 #endif
 }
 
@@ -1441,7 +1441,7 @@ void DenseMatrix::GradToCurl(DenseMatrix &curl)
    if ((Width() != 2 || curl.Width() != 1 || 2*n != curl.Height()) &&
        (Width() != 3 || curl.Width() != 3 || 3*n != curl.Height()))
    {
-      mfem_error("DenseMatrix::GradToCurl(...)");
+      mfem_error("DenseMatrix::GradToCurl(...): dimension mismatch");
    }
 #endif
 
@@ -1676,7 +1676,7 @@ void DenseMatrix::AddMatrix(DenseMatrix &A, int ro, int co)
 #ifdef MFEM_DEBUG
    if (co+aw > Width() || ro+ah > h)
    {
-      mfem_error("DenseMatrix::AddMatrix(...) 1");
+      mfem_error("DenseMatrix::AddMatrix(...) 1 : dimension mismatch");
    }
 #endif
 
@@ -1706,7 +1706,7 @@ void DenseMatrix::AddMatrix(double a, const DenseMatrix &A, int ro, int co)
 #ifdef MFEM_DEBUG
    if (co+aw > Width() || ro+ah > h)
    {
-      mfem_error("DenseMatrix::AddMatrix(...) 2");
+      mfem_error("DenseMatrix::AddMatrix(...) 2 : dimension mismatch");
    }
 #endif
 
@@ -1753,7 +1753,7 @@ void DenseMatrix::AdjustDofDirection(Array<int> &dofs)
 #ifdef MFEM_DEBUG
    if (dofs.Size() != n || Width() != n)
    {
-      mfem_error("DenseMatrix::AdjustDofDirection(...)");
+      mfem_error("DenseMatrix::AdjustDofDirection(...): dimension mismatch");
    }
 #endif
 
@@ -2093,11 +2093,11 @@ void CalcAdjugate(const DenseMatrix &a, DenseMatrix &adja)
 #ifdef MFEM_DEBUG
    if (a.Width() > a.Height() || a.Width() < 1 || a.Height() > 3)
    {
-      mfem_error("CalcAdjugate(...)");
+      mfem_error("CalcAdjugate(...): unsupported dimensions");
    }
    if (a.Width() != adja.Height() || a.Height() != adja.Width())
    {
-      mfem_error("CalcAdjugate(...)");
+      mfem_error("CalcAdjugate(...): dimension mismatch");
    }
 #endif
 
@@ -2166,7 +2166,7 @@ void CalcAdjugateTranspose(const DenseMatrix &a, DenseMatrix &adjat)
    if (a.Height() != a.Width() || adjat.Height() != adjat.Width() ||
        a.Width() != adjat.Width() || a.Width() < 1 || a.Width() > 3)
    {
-      mfem_error("CalcAdjugateTranspose(...)");
+      mfem_error("CalcAdjugateTranspose(...): dimension mismatch");
    }
 #endif
    if (a.Width() == 1)
@@ -2269,7 +2269,7 @@ void CalcInverseTranspose(const DenseMatrix &a, DenseMatrix &inva)
    if ( (a.Width() != a.Height()) || ( (a.Height()!= 1) && (a.Height()!= 2)
                                        && (a.Height()!= 3) ) )
    {
-      mfem_error("CalcInverseTranspose(...)");
+      mfem_error("CalcInverseTranspose(...): dimension mismatch");
    }
 #endif
 
@@ -2396,7 +2396,7 @@ void MultABt(const DenseMatrix &A, const DenseMatrix &B, DenseMatrix &ABt)
    if (A.Height() != ABt.Height() || B.Height() != ABt.Width() ||
        A.Width() != B.Width())
    {
-      mfem_error("MultABt(...)");
+      mfem_error("MultABt(...): dimension mismatch");
    }
 #endif
 
@@ -2462,7 +2462,7 @@ void MultADBt(const DenseMatrix &A, const Vector &D,
    if (A.Height() != ADBt.Height() || B.Height() != ADBt.Width() ||
        A.Width() != B.Width() || A.Width() != D.Size())
    {
-      mfem_error("MultADBt(...)");
+      mfem_error("MultADBt(...): dimension mismatch");
    }
 #endif
 
@@ -2501,7 +2501,7 @@ void AddMultABt(const DenseMatrix &A, const DenseMatrix &B, DenseMatrix &ABt)
    if (A.Height() != ABt.Height() || B.Height() != ABt.Width() ||
        A.Width() != B.Width())
    {
-      mfem_error("AddMultABt(...)");
+      mfem_error("AddMultABt(...): dimension mismatch");
    }
 #endif
 
@@ -2559,7 +2559,7 @@ void AddMultADBt(const DenseMatrix &A, const Vector &D,
    if (A.Height() != ADBt.Height() || B.Height() != ADBt.Width() ||
        A.Width() != B.Width() || A.Width() != D.Size())
    {
-      mfem_error("AddMultADBt(...)");
+      mfem_error("AddMultADBt(...): dimension mismatch");
    }
 #endif
 
@@ -2595,7 +2595,7 @@ void AddMult_a_ABt(double a, const DenseMatrix &A, const DenseMatrix &B,
    if (A.Height() != ABt.Height() || B.Height() != ABt.Width() ||
        A.Width() != B.Width())
    {
-      mfem_error("AddMult_a_ABt(...)");
+      mfem_error("AddMult_a_ABt(...): dimension mismatch");
    }
 #endif
 
@@ -2653,7 +2653,7 @@ void MultAtB(const DenseMatrix &A, const DenseMatrix &B, DenseMatrix &AtB)
    if (A.Width() != AtB.Height() || B.Width() != AtB.Width() ||
        A.Height() != B.Height())
    {
-      mfem_error("MultAtB(...)");
+      mfem_error("MultAtB(...): dimension mismatch");
    }
 #endif
 
@@ -2761,7 +2761,7 @@ void MultVWt(const Vector &v, const Vector &w, DenseMatrix &VWt)
 #ifdef MFEM_DEBUG
    if (v.Size() != VWt.Height() || w.Size() != VWt.Width())
    {
-      mfem_error("MultVWt(...)");
+      mfem_error("MultVWt(...): dimension mismatch");
    }
 #endif
 
@@ -2782,7 +2782,7 @@ void AddMultVWt(const Vector &v, const Vector &w, DenseMatrix &VWt)
 #ifdef MFEM_DEBUG
    if (VWt.Height() != m || VWt.Width() != n)
    {
-      mfem_error("AddMultVWt(...)");
+      mfem_error("AddMultVWt(...): dimension mismatch");
    }
 #endif
 
@@ -2803,7 +2803,7 @@ void AddMultVVt(const Vector &v, DenseMatrix &VVt)
 #ifdef MFEM_DEBUG
    if (VVt.Height() != n || VVt.Width() != n)
    {
-      mfem_error("AddMultVVt(...)");
+      mfem_error("AddMultVVt(...): dimension mismatch");
    }
 #endif
 
@@ -2828,7 +2828,7 @@ void AddMult_a_VWt(const double a, const Vector &v, const Vector &w,
 #ifdef MFEM_DEBUG
    if (VWt.Height() != m || VWt.Width() != n)
    {
-      mfem_error("AddMult_a_VWt(...)");
+      mfem_error("AddMult_a_VWt(...): dimension mismatch");
    }
 #endif
 
@@ -3353,7 +3353,7 @@ void DenseMatrixEigensystem::Eval()
 #ifdef MFEM_DEBUG
    if (mat.Width() != n)
    {
-      mfem_error("DenseMatrixEigensystem::Eval()");
+      mfem_error("DenseMatrixEigensystem::Eval(): dimension mismatch");
    }
 #endif
 

linalg/hypre.cpp

@@ -3215,10 +3215,31 @@ void HypreBoomerAMG::SetOperator(const Operator &op)
    B = X = NULL;
 }
 
-void HypreBoomerAMG::SetSystemsOptions(int dim)
+void HypreBoomerAMG::SetSystemsOptions(int dim, bool order_bynodes)
 {
    HYPRE_BoomerAMGSetNumFunctions(amg_precond, dim);
 
+   // The default "system" ordering in hypre is Ordering::byVDIM. When we are
+   // using Ordering::byNODES, we have to specify the ordering explicitly with
+   // HYPRE_BoomerAMGSetDofFunc as in the following code.
+   if (order_bynodes)
+   {
+      // hypre actually deletes the following pointer in HYPRE_BoomerAMGDestroy,
+      // so we don't need to track it
+      HYPRE_Int *mapping = mfem_hypre_CTAlloc(HYPRE_Int, height);
+      int h_nnodes = height / dim; // nodes owned in linear algebra (not fem)
+      MFEM_VERIFY(height % dim == 0, "Ordering does not work as claimed!");
+      int k = 0;
+      for (int i = 0; i < dim; ++i)
+      {
+         for (int j = 0; j < h_nnodes; ++j)
+         {
+            mapping[k++] = i;
+         }
+      }
+      HYPRE_BoomerAMGSetDofFunc(amg_precond, mapping);
+   }
+
    // More robust options with respect to convergence
    HYPRE_BoomerAMGSetAggNumLevels(amg_precond, 0);
    HYPRE_BoomerAMGSetStrongThreshold(amg_precond, 0.5);

linalg/hypre.hpp

@@ -992,16 +992,15 @@ public:
 
    virtual void SetOperator(const Operator &op);
 
-   /** More robust options for systems, such as elasticity. Note that BoomerAMG
-       assumes Ordering::byVDIM in the finite element space used to generate the
-       matrix A. */
-   void SetSystemsOptions(int dim);
+   /** More robust options for systems, such as elasticity. */
+   void SetSystemsOptions(int dim, bool order_bynodes=false);
 
    /** A special elasticity version of BoomerAMG that takes advantage of
        geometric rigid body modes and could perform better on some problems, see
        "Improving algebraic multigrid interpolation operators for linear
        elasticity problems", Baker, Kolev, Yang, NLAA 2009, DOI:10.1002/nla.688.
-       As with SetSystemsOptions(), this solver assumes Ordering::byVDIM. */
+       This solver assumes Ordering::byVDIM in the FiniteElementSpace used to
+       construct A. */
    void SetElasticityOptions(ParFiniteElementSpace *fespace);
 
    void SetPrintLevel(int print_level)

linalg/superlu.cpp

@@ -537,7 +537,29 @@ void SuperLUSolver::Mult( const Vector & x, Vector & y ) const
 
    if ( info != 0 )
    {
-      if ( info <= A->ncol )
+      if ( info < 0 )
+      {
+         switch (-info)
+         {
+            case 1:
+               MFEM_ABORT("SuperLU:  SuperLU options are invalid.");
+               break;
+            case 2:
+               MFEM_ABORT("SuperLU:  Matrix A (in Ax=b) is invalid.");
+               break;
+            case 5:
+               MFEM_ABORT("SuperLU:  Vector b dimension (in Ax=b) is invalid.");
+               break;
+            case 6:
+               MFEM_ABORT("SuperLU:  Number of right-hand sides is invalid.");
+               break;
+            default:
+               MFEM_ABORT("SuperLU:  Parameter with index "
+                          << -info << "invalid. (1-indexed)");
+               break;
+         }
+      }
+      else if ( info <= A->ncol )
       {
          MFEM_ABORT("SuperLU:  Found a singular matrix, U("
                     << info << "," << info << ") is exactly zero.");

linalg/vector.cpp

@@ -1071,7 +1071,7 @@ double Vector::operator*(const Vector &v) const
       return prod;
    }
 #endif
-   if (Device::Allows(Backend::DEBUG))
+   if (Device::Allows(Backend::DEBUG_DEVICE))
    {
       const int N = size;
       auto v_data = v.Read();
@@ -1131,7 +1131,7 @@ double Vector::Min() const
    }
 #endif
 
-   if (Device::Allows(Backend::DEBUG))
+   if (Device::Allows(Backend::DEBUG_DEVICE))
    {
       const int N = size;
       auto m_data = Read();

mesh/mesh.cpp

@@ -1223,58 +1223,136 @@ void Mesh::InitMesh(int _Dim, int _spaceDim, int NVert, int NElem, int NBdrElem)
    boundary.SetSize(NBdrElem);  // just allocate space for Element *
 }
 
-void Mesh::AddVertex(const double *x)
+template<typename T>
+static void CheckEnlarge(Array<T> &array, int size)
 {
-   double *y = vertices[NumOfVertices]();
+   if (size >= array.Size()) { array.SetSize(size + 1); }
+}
 
-   for (int i = 0; i < spaceDim; i++)
+int Mesh::AddVertex(double x, double y, double z)
+{
+   CheckEnlarge(vertices, NumOfVertices);
+   double *v = vertices[NumOfVertices]();
+   v[0] = x;
+   v[1] = y;
+   v[2] = z;
+   return NumOfVertices++;
+}
+
+int Mesh::AddVertex(const double *coords)
+{
+   CheckEnlarge(vertices, NumOfVertices);
+   vertices[NumOfVertices].SetCoords(spaceDim, coords);
+   return NumOfVertices++;
+}
+
+void Mesh::AddVertexParents(int i, int p1, int p2)
+{
+   tmp_vertex_parents.Append(Triple<int, int, int>(i, p1, p2));
+
+   // if vertex coordinates are defined, make sure the hanging vertex has the
+   // correct position
+   if (i < vertices.Size())
    {
-      y[i] = x[i];
+      double *vi = vertices[i](), *vp1 = vertices[p1](), *vp2 = vertices[p2]();
+      for (int j = 0; j < 3; j++)
+      {
+         vi[j] = (vp1[j] + vp2[j]) * 0.5;
+      }
    }
-   NumOfVertices++;
 }
 
-void Mesh::AddSegment(const int *vi, int attr)
+int Mesh::AddSegment(int v1, int v2, int attr)
 {
-   elements[NumOfElements++] = new Segment(vi, attr);
+   CheckEnlarge(elements, NumOfElements);
+   elements[NumOfElements] = new Segment(v1, v2, attr);
+   return NumOfElements++;
 }
 
-void Mesh::AddTri(const int *vi, int attr)
+int Mesh::AddSegment(const int *vi, int attr)
 {
-   elements[NumOfElements++] = new Triangle(vi, attr);
+   CheckEnlarge(elements, NumOfElements);
+   elements[NumOfElements] = new Segment(vi, attr);
+   return NumOfElements++;
 }
 
-void Mesh::AddTriangle(const int *vi, int attr)
+int Mesh::AddTriangle(int v1, int v2, int v3, int attr)
 {
-   elements[NumOfElements++] = new Triangle(vi, attr);
+   CheckEnlarge(elements, NumOfElements);
+   elements[NumOfElements] = new Triangle(v1, v2, v3, attr);
+   return NumOfElements++;
 }
 
-void Mesh::AddQuad(const int *vi, int attr)
+int Mesh::AddTriangle(const int *vi, int attr)
 {
-   elements[NumOfElements++] = new Quadrilateral(vi, attr);
+   CheckEnlarge(elements, NumOfElements);
+   elements[NumOfElements] = new Triangle(vi, attr);
+   return NumOfElements++;
 }
 
-void Mesh::AddTet(const int *vi, int attr)
+int Mesh::AddQuad(int v1, int v2, int v3, int v4, int attr)
 {
+   CheckEnlarge(elements, NumOfElements);
+   elements[NumOfElements] = new Quadrilateral(v1, v2, v3, v4, attr);
+   return NumOfElements++;
+}
+
+int Mesh::AddQuad(const int *vi, int attr)
+{
+   CheckEnlarge(elements, NumOfElements);
+   elements[NumOfElements] = new Quadrilateral(vi, attr);
+   return NumOfElements++;
+}
+
+int Mesh::AddTet(int v1, int v2, int v3, int v4, int attr)
+{
+   int vi[4] = {v1, v2, v3, v4};
+   return AddTet(vi, attr);
+}
+
+int Mesh::AddTet(const int *vi, int attr)
+{
+   CheckEnlarge(elements, NumOfElements);
 #ifdef MFEM_USE_MEMALLOC
    Tetrahedron *tet;
    tet = TetMemory.Alloc();
    tet->SetVertices(vi);
    tet->SetAttribute(attr);
-   elements[NumOfElements++] = tet;
+   elements[NumOfElements] = tet;
 #else
-   elements[NumOfElements++] = new Tetrahedron(vi, attr);
+   elements[NumOfElements] = new Tetrahedron(vi, attr);
 #endif
+   return NumOfElements++;
 }
 
-void Mesh::AddWedge(const int *vi, int attr)
+int Mesh::AddWedge(int v1, int v2, int v3, int v4, int v5, int v6, int attr)
 {
-   elements[NumOfElements++] = new Wedge(vi, attr);
+   CheckEnlarge(elements, NumOfElements);
+   elements[NumOfElements] = new Wedge(v1, v2, v3, v4, v5, v6, attr);
+   return NumOfElements++;
 }
 
-void Mesh::AddHex(const int *vi, int attr)
+int Mesh::AddWedge(const int *vi, int attr)
 {
-   elements[NumOfElements++] = new Hexahedron(vi, attr);
+   CheckEnlarge(elements, NumOfElements);
+   elements[NumOfElements] = new Wedge(vi, attr);
+   return NumOfElements++;
+}
+
+int Mesh::AddHex(int v1, int v2, int v3, int v4, int v5, int v6, int v7, int v8,
+                 int attr)
+{
+   CheckEnlarge(elements, NumOfElements);
+   elements[NumOfElements] =
+      new Hexahedron(v1, v2, v3, v4, v5, v6, v7, v8, attr);
+   return NumOfElements++;
+}
+
+int Mesh::AddHex(const int *vi, int attr)
+{
+   CheckEnlarge(elements, NumOfElements);
+   elements[NumOfElements] = new Hexahedron(vi, attr);
+   return NumOfElements++;
 }
 
 void Mesh::AddHexAsTets(const int *vi, int attr)
@@ -1314,19 +1392,60 @@ void Mesh::AddHexAsWedges(const int *vi, int attr)
    }
 }
 
-void Mesh::AddBdrSegment(const int *vi, int attr)
+int Mesh::AddElement(Element *elem)
 {
-   boundary[NumOfBdrElements++] = new Segment(vi, attr);
+   CheckEnlarge(elements, NumOfElements);
+   elements[NumOfElements] = elem;
+   return NumOfElements++;
 }
 
-void Mesh::AddBdrTriangle(const int *vi, int attr)
+int Mesh::AddBdrElement(Element *elem)
 {
-   boundary[NumOfBdrElements++] = new Triangle(vi, attr);
+   CheckEnlarge(boundary, NumOfBdrElements);
+   boundary[NumOfBdrElements] = elem;
+   return NumOfBdrElements++;
 }
 
-void Mesh::AddBdrQuad(const int *vi, int attr)
+int Mesh::AddBdrSegment(int v1, int v2, int attr)
 {
-   boundary[NumOfBdrElements++] = new Quadrilateral(vi, attr);
+   CheckEnlarge(boundary, NumOfBdrElements);
+   boundary[NumOfBdrElements] = new Segment(v1, v2, attr);
+   return NumOfBdrElements++;
+}
+
+int Mesh::AddBdrSegment(const int *vi, int attr)
+{
+   CheckEnlarge(boundary, NumOfBdrElements);
+   boundary[NumOfBdrElements] = new Segment(vi, attr);
+   return NumOfBdrElements++;
+}
+
+int Mesh::AddBdrTriangle(int v1, int v2, int v3, int attr)
+{
+   CheckEnlarge(boundary, NumOfBdrElements);
+   boundary[NumOfBdrElements] = new Triangle(v1, v2, v3, attr);
+   return NumOfBdrElements++;
+}
+
+int Mesh::AddBdrTriangle(const int *vi, int attr)
+{
+   CheckEnlarge(boundary, NumOfBdrElements);
+   boundary[NumOfBdrElements] = new Triangle(vi, attr);
+   return NumOfBdrElements++;
+}
+
+int Mesh::AddBdrQuad(int v1, int v2, int v3, int v4, int attr)
+{
+   CheckEnlarge(boundary, NumOfBdrElements);
+   boundary[NumOfBdrElements] = new Quadrilateral(v1, v2, v3, v4, attr);
+   return NumOfBdrElements++;
+}
+
+int Mesh::AddBdrQuad(const int *vi, int attr)
+{
+   CheckEnlarge(boundary, NumOfBdrElements);
+   boundary[NumOfBdrElements] = new Quadrilateral(vi, attr);
+   return NumOfBdrElements++;
 }
 
 void Mesh::AddBdrQuadAsTriangles(const int *vi, int attr)
@@ -2419,6 +2538,15 @@ void Mesh::FinalizeTopology(bool generate_bdr)
 
    // generate the arrays 'attributes' and 'bdr_attributes'
    SetAttributes();
+
+   // if the user defined any hanging nodes (see AddVertexParent),
+   // initialize the NC mesh now
+   if (tmp_vertex_parents.Size())
+   {
+      MFEM_VERIFY(ncmesh == NULL, "");
+      EnsureNCMesh(true);
+      tmp_vertex_parents.DeleteAll();
+   }
 }
 
 void Mesh::Finalize(bool refine, bool fix_orientation)

mesh/mesh.hpp

@@ -206,6 +206,9 @@ public:
    Array<FaceGeometricFactors*>
    face_geom_factors; ///< Optional face geometric factors.
 
+   /// Used during initialization only.
+   Array<Triple<int, int, int> > tmp_vertex_parents;
+
    // Global parameter that can be used to control the removal of unused
    // vertices performed when reading a mesh in MFEM format. The default value
    // (true) is set in mesh_readers.cpp.
@@ -499,10 +502,7 @@ public:
        @brief _Init_ constructor: begin the construction of a Mesh object. */
    Mesh(int _Dim, int NVert, int NElem, int NBdrElem = 0, int _spaceDim = -1)
    {
-      if (_spaceDim == -1)
-      {
-         _spaceDim = _Dim;
-      }
+      if (_spaceDim == -1) { _spaceDim = _Dim; }
       InitMesh(_Dim, _spaceDim, NVert, NElem, NBdrElem);
    }
 
@@ -514,22 +514,45 @@ public:
 
    Element *NewElement(int geom);
 
-   void AddVertex(const double *);
-   void AddSegment(const int *vi, int attr = 1);
-   void AddTri(const int *vi, int attr = 1);
-   void AddTriangle(const int *vi, int attr = 1);
-   void AddQuad(const int *vi, int attr = 1);
-   void AddTet(const int *vi, int attr = 1);
-   void AddWedge(const int *vi, int attr = 1);
-   void AddHex(const int *vi, int attr = 1);
+   int AddVertex(double x, double y = 0.0, double z = 0.0);
+   int AddVertex(const double *coords);
+   /// Mark vertex @a i as non-conforming, with parent vertices @a p1 and @a p2.
+   void AddVertexParents(int i, int p1, int p2);
+
+   int AddSegment(int v1, int v2, int attr = 1);
+   int AddSegment(const int *vi, int attr = 1);
+
+   int AddTriangle(int v1, int v2, int v3, int attr = 1);
+   int AddTriangle(const int *vi, int attr = 1);
+   int AddTri(const int *vi, int attr = 1) { return AddTriangle(vi, attr); }
+
+   int AddQuad(int v1, int v2, int v3, int v4, int attr = 1);
+   int AddQuad(const int *vi, int attr = 1);
+
+   int AddTet(int v1, int v2, int v3, int v4, int attr = 1);
+   int AddTet(const int *vi, int attr = 1);
+
+   int AddWedge(int v1, int v2, int v3, int v4, int v5, int v6, int attr = 1);
+   int AddWedge(const int *vi, int attr = 1);
+
+   int AddHex(int v1, int v2, int v3, int v4, int v5, int v6, int v7, int v8,
+              int attr = 1);
+   int AddHex(const int *vi, int attr = 1);
    void AddHexAsTets(const int *vi, int attr = 1);
    void AddHexAsWedges(const int *vi, int attr = 1);
+
    /// The parameter @a elem should be allocated using the NewElement() method
-   void AddElement(Element *elem)     { elements[NumOfElements++] = elem; }
-   void AddBdrElement(Element *elem)  { boundary[NumOfBdrElements++] = elem; }
-   void AddBdrSegment(const int *vi, int attr = 1);
-   void AddBdrTriangle(const int *vi, int attr = 1);
-   void AddBdrQuad(const int *vi, int attr = 1);
+   int AddElement(Element *elem);
+   int AddBdrElement(Element *elem);
+
+   int AddBdrSegment(int v1, int v2, int attr = 1);
+   int AddBdrSegment(const int *vi, int attr = 1);
+
+   int AddBdrTriangle(int v1, int v2, int v3, int attr = 1);
+   int AddBdrTriangle(const int *vi, int attr = 1);
+
+   int AddBdrQuad(int v1, int v2, int v3, int v4, int attr = 1);
+   int AddBdrQuad(const int *vi, int attr = 1);
    void AddBdrQuadAsTriangles(const int *vi, int attr = 1);
 
    void GenerateBoundaryElements();

mesh/ncmesh.cpp

@@ -104,7 +104,16 @@ NCMesh::NCMesh(const Mesh *mesh, std::istream *vertex_parents)
    {
       LoadVertexParents(*vertex_parents);
    }
-   else
+   // alternatively, the user might have initialized hanging nodes with
+   // Mesh::AddVertexParents; copy the hierarchy now
+   else if (mesh->tmp_vertex_parents.Size())
+   {
+      for (const auto &triple : mesh->tmp_vertex_parents)
+      {
+         nodes.Reparent(triple.one, triple.two, triple.three);
+      }
+   }
+   else // otherwise we just assume a standard conforming coarse mesh
    {
       top_vertex_pos.SetSize(3*mesh->GetNV());
       for (int i = 0; i < mesh->GetNV(); i++)

mesh/pncmesh.cpp

@@ -1066,9 +1066,9 @@ void ParNCMesh::GetFaceNeighbors(ParMesh &pmesh)
       for (int j = mf.slaves_begin; j < mf.slaves_end; j++)
       {
          const Slave &sf = full_list.slaves[j];
-         if (sf.index < 0) { continue; }
+         if (sf.element < 0) { continue; }
 
-         MFEM_ASSERT(mf.element >= 0 && sf.element >= 0, "");
+         MFEM_ASSERT(mf.element >= 0, "");
          Element* e[2] = { &elements[mf.element], &elements[sf.element] };
 
          bool loc0 = (e[0]->rank == MyRank);
@@ -1224,9 +1224,9 @@ void ParNCMesh::GetFaceNeighbors(ParMesh &pmesh)
          for (int j = mf.slaves_begin; j < mf.slaves_end; j++)
          {
             const Slave &sf = full_list.slaves[j];
-            if (sf.index < 0) { continue; }
+            if (sf.element < 0) { continue; }
 
-            MFEM_ASSERT(sf.element >= 0 && mf.element >= 0, "");
+            MFEM_ASSERT(mf.element >= 0, "");
             Element &sfe = elements[sf.element];
             Element &mfe = elements[mf.element];
 

miniapps/common/fem_extras.cpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "fem_extras.hpp"
+#include "../../general/text.hpp"
 
 using namespace std;
 
@@ -56,6 +57,345 @@ RT_FESpace::~RT_FESpace()
    delete FEC_;
 }
 
+CoefFactory::~CoefFactory()
+{
+   for (int i=0; i<sCoefs.Size(); i++)
+   {
+      delete sCoefs[i];
+   }
+   for (int i=0; i<vCoefs.Size(); i++)
+   {
+      delete vCoefs[i];
+   }
+   for (int i=0; i<mCoefs.Size(); i++)
+   {
+      delete mCoefs[i];
+   }
+}
+
+Coefficient * CoefFactory::GetScalarCoef(std::istream &input)
+{
+   string buff;
+
+   skip_comment_lines(input, '#');
+   input >> buff;
+
+   return this->GetScalarCoef(buff, input);
+}
+
+VectorCoefficient * CoefFactory::GetVectorCoef(std::istream &input)
+{
+   string buff;
+
+   skip_comment_lines(input, '#');
+   input >> buff;
+
+   return this->GetVectorCoef(buff, input);
+}
+
+MatrixCoefficient * CoefFactory::GetMatrixCoef(std::istream &input)
+{
+   string buff;
+
+   skip_comment_lines(input, '#');
+   input >> buff;
+
+   return this->GetMatrixCoef(buff, input);
+}
+
+Coefficient * CoefFactory::GetScalarCoef(std::string &name,
+                                         std::istream &input)
+{
+   int c = -1;
+   if (name == "ConstantCoefficient")
+   {
+      double val;
+      input >> val;
+      c = sCoefs.Append(new ConstantCoefficient(val));
+   }
+   else if (name == "PWConstCoefficient")
+   {
+      int nvals;
+      input >> nvals;
+      Vector vals(nvals);
+      for (int i=0; i<nvals; i++)
+      {
+         input >> vals[i];
+      }
+      c = sCoefs.Append(new PWConstCoefficient(vals));
+   }
+   else if (name == "FunctionCoefficient")
+   {
+      int type, index;
+      input >> type >> index;
+      MFEM_VERIFY(type >=0 && type <= 1,
+                  "Invalid Function type read by CoefFactory");
+      if (type == 0)
+      {
+         MFEM_VERIFY(index >=0 && index < ext_sfn.Size(),
+                     "Invalid Function index read by CoefFactory");
+      }
+      else
+      {
+         MFEM_VERIFY(index >=0 && index < ext_stfn.Size(),
+                     "Invalid Time dependent Function index "
+                     "read by CoefFactory");
+      }
+      c = sCoefs.Append((type == 0) ?
+                        new FunctionCoefficient(ext_sfn[index]) :
+                        new FunctionCoefficient(ext_stfn[index]));
+   }
+   else if (name == "GridFunctionCoefficient")
+   {
+      int index, comp;
+      input >> index >> comp;
+      MFEM_VERIFY(index >=0 && index < ext_gf.Size(),
+                  "Invalid GridFunction index read by CoefFactory");
+      c = sCoefs.Append(new GridFunctionCoefficient(ext_gf[index], comp));
+   }
+   else if (name == "DivergenceGridFunctionCoefficient")
+   {
+      int index;
+      input >> index;
+      MFEM_VERIFY(index >=0 && index < ext_gf.Size(),
+                  "Invalid GridFunction index for "
+                  "DivergenceGridFunctionCoefficient read by CoefFactory");
+      c = sCoefs.Append(new DivergenceGridFunctionCoefficient(ext_gf[index]));
+   }
+   else if (name == "DeltaCofficient")
+   {
+      int dim;
+      input >> dim;
+      MFEM_VERIFY(dim >=1 && dim <= 3,
+                  "Invalid dimension for DeltaCoefficient "
+                  "read by CoefFactory");
+      double x, y, z, s;
+      input >> x;
+      if (dim > 1) { input >> y; }
+      if (dim > 2) { input >> z; }
+      input >> s;
+      if (dim == 1)
+      {
+         c = sCoefs.Append(new DeltaCoefficient(x, s));
+      }
+      else if (dim == 2)
+      {
+         c = sCoefs.Append(new DeltaCoefficient(x, y, s));
+      }
+      else
+      {
+         c = sCoefs.Append(new DeltaCoefficient(x, y, z, s));
+      }
+   }
+   else if (name == "RestrictedCoefficient")
+   {
+      Coefficient * rc = this->GetScalarCoef(input);
+      int nattr;
+      input >> nattr;
+      Array<int> attr(nattr);
+      for (int i=0; i<nattr; i++)
+      {
+         input >> attr[i];
+      }
+      c = sCoefs.Append(new RestrictedCoefficient(*rc, attr));
+   }
+   else
+   {
+      return NULL;
+   }
+   return sCoefs[--c];
+}
+
+VectorCoefficient * CoefFactory::GetVectorCoef(std::string &name,
+                                               std::istream &input)
+{
+   int c = -1;
+   if (name == "VectorConstantCoefficient")
+   {
+      int dim;
+      input >> dim;
+      Vector val(dim);
+      for (int i=0; i<dim; i++) { input >> val[i]; }
+      c = vCoefs.Append(new VectorConstantCoefficient(val));
+   }
+   else if (name == "VectorFunctionCoefficient")
+   {
+      int dim, type, index;
+      input >> dim >> type >> index;
+      MFEM_VERIFY(type >=0 && type <= 1,
+                  "Invalid Function type read by VecCoefFactory");
+      if (type == 0)
+      {
+         MFEM_VERIFY(index >=0 && index < ext_vfn.Size(),
+                     "Invalid Vector Function index read by CoefFactory");
+      }
+      else
+      {
+         MFEM_VERIFY(index >=0 && index < ext_vtfn.Size(),
+                     "Invalid Time dependent Vector Function index "
+                     "read by CoefFactory");
+      }
+      c = vCoefs.Append((type==0) ?
+                        new VectorFunctionCoefficient(dim, ext_vfn[index]) :
+                        new VectorFunctionCoefficient(dim, ext_vtfn[index]));
+   }
+   else if (name == "VectorArrayCoefficient")
+   {
+      int dim;
+      input >> dim;
+      MFEM_VERIFY(dim > 0,
+                  "Invalid dimension for VectorArrayCoefficient "
+                  "read by CoefFactory");
+      VectorArrayCoefficient * vCoef = new VectorArrayCoefficient(dim);
+      for (int i=0; i<dim; i++)
+      {
+         Coefficient *sCoef = this->GetScalarCoef(input);
+         vCoef->Set(i, sCoef, false);
+      }
+      c = vCoefs.Append(vCoef);
+   }
+   else if (name == "VectorGridFunctionCoefficient")
+   {
+      int index;
+      input >> index;
+      MFEM_VERIFY(index >=0 && index < ext_gf.Size(),
+                  "Invalid GridFunction index read by CoefFactory");
+      c = vCoefs.Append(new VectorGridFunctionCoefficient(ext_gf[index]));
+   }
+   else if (name == "GradientGridFunctionCoefficient")
+   {
+      int index;
+      input >> index;
+      MFEM_VERIFY(index >=0 && index < ext_gf.Size(),
+                  "Invalid GridFunction index for "
+                  "GradientGridFunctionCoefficient read by CoefFactory");
+      c = vCoefs.Append(new GradientGridFunctionCoefficient(ext_gf[index]));
+   }
+   else if (name == "CurlGridFunctionCoefficient")
+   {
+      int index;
+      input >> index;
+      MFEM_VERIFY(index >=0 && index < ext_gf.Size(),
+                  "Invalid GridFunction index for "
+                  "CurlGridFunctionCoefficient read by CoefFactory");
+      c = vCoefs.Append(new CurlGridFunctionCoefficient(ext_gf[index]));
+   }
+   else if (name == "VectorDeltaCofficient")
+   {
+      int dim;
+      input >> dim;
+      MFEM_VERIFY(dim >=1 && dim <= 3,
+                  "Invalid dimension for DeltaCoefficient "
+                  "read by CoefFactory");
+      Vector dir(dim);
+      for (int i=0; i<dim; i++) { input >> dir[i]; }
+      double x, y, z, s;
+      input >> x;
+      if (dim > 1) { input >> y; }
+      if (dim > 2) { input >> z; }
+      input >> s;
+      if (dim == 1)
+      {
+         c = vCoefs.Append(new VectorDeltaCoefficient(dir, x, s));
+      }
+      else if (dim == 2)
+      {
+         c = vCoefs.Append(new VectorDeltaCoefficient(dir, x, y, s));
+      }
+      else
+      {
+         c = vCoefs.Append(new VectorDeltaCoefficient(dir, x, y, z, s));
+      }
+   }
+   else if (name == "VectorRestrictedCoefficient")
+   {
+      VectorCoefficient * rc = this->GetVectorCoef(input);
+      int nattr;
+      input >> nattr;
+      Array<int> attr(nattr);
+      for (int i=0; i<nattr; i++)
+      {
+         input >> attr[i];
+      }
+      c = vCoefs.Append(new VectorRestrictedCoefficient(*rc, attr));
+   }
+   else
+   {
+      return NULL;
+   }
+   return vCoefs[--c];
+}
+
+MatrixCoefficient * CoefFactory::GetMatrixCoef(std::string &name,
+                                               std::istream &input)
+{
+   int c = -1;
+   if (name == "MatrixConstantCoefficient")
+   {
+      int h, w;
+      input >> h >> w;
+      DenseMatrix val(h, w);
+      for (int i=0; i<h; i++)
+         for (int j=0; j<w; j++)
+         { input >> val(i, j); }
+      c = mCoefs.Append(new MatrixConstantCoefficient(val));
+   }
+   else if (name == "MatrixFunctionCoefficient")
+   {
+      int type;
+      input >> type;
+      MFEM_VERIFY(type >=0 && type <= 2,
+                  "Invalid Function type read by MatCoefFactory");
+      if (type < 2)
+      {
+         int dim, index;
+         input >> dim >> index;
+         if (type == 0)
+         {
+            MFEM_VERIFY(index >=0 && index < ext_mfn.Size(),
+                        "Invalid Matrix Function index read by MatCoefFactory");
+         }
+         else
+         {
+            MFEM_VERIFY(index >=0 && index < ext_mtfn.Size(),
+                        "Invalid Time dependent Matrix Function index "
+                        "read by MatCoefFactory");
+         }
+         c = mCoefs.Append((type==0) ?
+                           new MatrixFunctionCoefficient(dim, ext_mfn[index]) :
+                           new MatrixFunctionCoefficient(dim, ext_mtfn[index]));
+      }
+      else
+      {
+         int h, w;
+         input >> h >> w;
+         DenseMatrix val(h, w);
+         for (int i=0; i<h; i++)
+            for (int j=0; j<w; j++)
+            { input >> val(i, j); }
+         Coefficient * sCoef = this->GetScalarCoef(input);
+         c = mCoefs.Append(new MatrixFunctionCoefficient(val, *sCoef));
+      }
+   }
+   else if (name == "MatrixRestrictedCoefficient")
+   {
+      MatrixCoefficient * rc = this->GetMatrixCoef(input);
+      int nattr;
+      input >> nattr;
+      Array<int> attr(nattr);
+      for (int i=0; i<nattr; i++)
+      {
+         input >> attr[i];
+      }
+      c = mCoefs.Append(new MatrixRestrictedCoefficient(*rc, attr));
+   }
+   else
+   {
+      return NULL;
+   }
+   return mCoefs[--c];
+}
+
 void VisualizeMesh(socketstream &sock, const char *vishost, int visport,
                    Mesh &mesh, const char *title,
                    int x, int y, int w, int h, const char * keys, bool vec)

miniapps/common/fem_extras.hpp

@@ -66,6 +66,61 @@ private:
 };
 
 
+class CoefFactory
+{
+protected:
+   Array<Coefficient*>       sCoefs; ///< Owned
+   Array<VectorCoefficient*> vCoefs; ///< Owned
+   Array<MatrixCoefficient*> mCoefs; ///< Owned
+
+   Array<GridFunction*>      ext_gf; ///< Not owned
+
+   Array<double (*)(const Vector &)> ext_sfn;          ///< Not owned
+   Array<double (*)(const Vector &, double)> ext_stfn; ///< Not owned
+
+   Array<void (*)(const Vector &, Vector &)> ext_vfn;          ///< Not owned
+   Array<void (*)(const Vector &, double, Vector &)> ext_vtfn; ///< Not owned
+
+   Array<void (*)(const Vector &, DenseMatrix &)> ext_mfn;     ///< Not owned
+   Array<void (*)(const Vector &, double, DenseMatrix &)> ext_mtfn;
+
+public:
+   CoefFactory() {}
+
+   virtual ~CoefFactory();
+
+   int AddExternalGridFunction(GridFunction &gf) { return ext_gf.Append(&gf); }
+
+   int AddExternalFunction(double (*fn)(const Vector &))
+   { return ext_sfn.Append(fn); }
+
+   int AddExternalFunction(double (*fn)(const Vector &, double))
+   { return ext_stfn.Append(fn); }
+
+   int AddExternalFunction(void (*fn)(const Vector &, Vector &))
+   { return ext_vfn.Append(fn); }
+
+   int AddExternalFunction(void (*fn)(const Vector &, double, Vector &))
+   { return ext_vtfn.Append(fn); }
+
+   int AddExternalFunction(void (*fn)(const Vector &, DenseMatrix &))
+   { return ext_mfn.Append(fn); }
+
+   int AddExternalFunction(void (*fn)(const Vector &, double, DenseMatrix &))
+   { return ext_mtfn.Append(fn); }
+
+   virtual Coefficient * GetScalarCoef(std::istream &input);
+   virtual Coefficient * GetScalarCoef(std::string &coef_name,
+                                       std::istream &input);
+   virtual VectorCoefficient * GetVectorCoef(std::istream &input);
+   virtual VectorCoefficient * GetVectorCoef(std::string &coef_name,
+                                             std::istream &input);
+   virtual MatrixCoefficient * GetMatrixCoef(std::istream &input);
+   virtual MatrixCoefficient * GetMatrixCoef(std::string &coef_name,
+                                             std::istream &input);
+};
+
+
 /// Visualize the given mesh object, using a GLVis server on the
 /// specified host and port. Set the visualization window title, and optionally,
 /// its geometry.

miniapps/gslib/field-interp.cpp

@@ -0,0 +1,388 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//      -------------------------------------------------------------
+//      Field Interp Miniapp: Transfer a grid function between meshes
+//      -------------------------------------------------------------
+//
+// This miniapp provides the capability to transfer a grid function (H1, L2,
+// H(div), and H(curl)) from one mesh onto another using GSLIB-FindPoints. Using
+// FindPoints, we identify the nodal positions of the target mesh with respect
+// to the source mesh and then interpolate the source grid function. The
+// interpolated values are then projected onto the desired finite element space
+// on the target mesh. Finally, the transferred solution is visualized using
+// GLVis. Note that the source grid function can be a user-defined vector
+// function or a grid function file that is compatible with the source mesh.
+//
+// Compile with: make field-interp
+//
+// Sample runs:
+//   field-interp
+//   field-interp -fts 3 -ft 0
+//   field-interp -m1 triple-pt-1.mesh -s1 triple-pt-1.gf -m2 triple-pt-2.mesh -ft 1
+//   field-interp -m2 ../meshing/amr-quad-q2.mesh -ft 0 -r 1
+
+#include "mfem.hpp"
+#include <fstream>
+
+using namespace mfem;
+using namespace std;
+
+// Scalar function to project
+double scalar_func(const Vector &x)
+{
+   const int dim = x.Size();
+   double res = 0.0;
+   for (int d = 0; d < dim; d++) { res += x(d) * x(d); }
+   return res;
+}
+
+void vector_func(const Vector &p, Vector &F)
+{
+   F(0) = scalar_func(p);
+   for (int i = 1; i < F.Size(); i++) { F(i) = (i+1)*pow(-1, i)*F(0); }
+}
+
+int main (int argc, char *argv[])
+{
+   // Set the method's default parameters.
+   const char *src_mesh_file = "../meshing/square01.mesh";
+   const char *tar_mesh_file = "../../data/inline-tri.mesh";
+   const char *src_sltn_file = "must_be_provided_by_the_user.gf";
+   int src_fieldtype  = 0;
+   int src_ncomp      = 1;
+   int ref_levels     = 0;
+   int fieldtype      = -1;
+   int order          = 3;
+   bool visualization = true;
+
+   // Parse command-line options.
+   OptionsParser args(argc, argv);
+   args.AddOption(&src_mesh_file, "-m1", "--mesh1",
+                  "Mesh file for the starting solution.");
+   args.AddOption(&tar_mesh_file, "-m2", "--mesh2",
+                  "Mesh file for interpolation.");
+   args.AddOption(&src_sltn_file, "-s1", "--solution1",
+                  "(optional) GridFunction file compatible with src_mesh_file."
+                  "Set src_fieldtype to -1 if this option is used.");
+   args.AddOption(&src_fieldtype, "-fts", "--field-type-src",
+                  "Source GridFunction type:"
+                  "0 - H1 (default), 1 - L2, 2 - H(div), 3 - H(curl).");
+   args.AddOption(&src_ncomp, "-nc", "--ncomp",
+                  "Number of components for H1 or L2 GridFunctions.");
+   args.AddOption(&ref_levels, "-r", "--refine",
+                  "Number of refinements of the interpolation mesh.");
+   args.AddOption(&fieldtype, "-ft", "--field-type",
+                  "Target GridFunction type: -1 - source GridFunction type (default),"
+                  "0 - H1, 1 - L2, 2 - H(div), 3 - H(curl).");
+   args.AddOption(&order, "-o", "--order",
+                  "Order of the interpolated solution.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // Input meshes.
+   Mesh mesh_1(src_mesh_file, 1, 1, false);
+   Mesh mesh_2(tar_mesh_file, 1, 1, false);
+   const int dim = mesh_1.Dimension();
+   MFEM_ASSERT(dim == mesh_2.Dimension(), "Source and target meshes "
+               "must be in the same dimension.");
+   MFEM_VERIFY(dim > 1, "GSLIB requires a 2D or a 3D mesh" );
+
+   for (int lev = 0; lev < ref_levels; lev++)
+   {
+      mesh_2.UniformRefinement();
+   }
+
+   if (mesh_1.GetNodes() == NULL) { mesh_1.SetCurvature(1); }
+   if (mesh_2.GetNodes() == NULL) { mesh_2.SetCurvature(1); }
+   const int mesh_poly_deg = mesh_2.GetNodes()->FESpace()->GetOrder(0);
+   cout << "Source mesh curvature: "
+        << mesh_1.GetNodes()->OwnFEC()->Name() << endl
+        << "Target mesh curvature: "
+        << mesh_2.GetNodes()->OwnFEC()->Name() << endl;
+
+   int src_vdim = src_ncomp;
+   FiniteElementCollection *src_fec = NULL;
+   FiniteElementSpace *src_fes = NULL;
+   GridFunction *func_source = NULL;
+   if (src_fieldtype < 0) // use src_sltn_file
+   {
+      ifstream mat_stream_1(src_sltn_file);
+      func_source = new GridFunction(&mesh_1, mat_stream_1);
+      src_vdim = func_source->FESpace()->GetVDim();
+   }
+   else if (src_fieldtype == 0)
+   {
+      src_fec = new H1_FECollection(order, dim);
+   }
+   else if (src_fieldtype == 1)
+   {
+      src_fec = new L2_FECollection(order, dim);
+   }
+   else if (src_fieldtype == 2)
+   {
+      src_fec = new RT_FECollection(order, dim);
+      src_ncomp = 1;
+      src_vdim = dim;
+   }
+   else if (src_fieldtype == 3)
+   {
+      src_fec = new ND_FECollection(order, dim);
+      src_ncomp = 1;
+      src_vdim = dim;
+   }
+   else
+   {
+      MFEM_ABORT("Invalid FECollection type.");
+   }
+
+   if (src_fieldtype > -1)
+   {
+      src_fes = new FiniteElementSpace(&mesh_1, src_fec, src_ncomp);
+      func_source = new GridFunction(src_fes);
+      // Project the grid function using VectorFunctionCoefficient.
+      VectorFunctionCoefficient F(src_vdim, vector_func);
+      func_source->ProjectCoefficient(F);
+   }
+
+   // Display the starting mesh and the field.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sout1;
+      sout1.open(vishost, visport);
+      if (!sout1)
+      {
+         cout << "Unable to connect to GLVis server at "
+              << vishost << ':' << visport << endl;
+      }
+      else
+      {
+         sout1.precision(8);
+         sout1 << "solution\n" << mesh_1 << *func_source
+               << "window_title 'Source mesh and solution'"
+               << "window_geometry 0 0 600 600";
+         if (dim == 2) { sout1 << "keys RmjAc"; }
+         if (dim == 3) { sout1 << "keys mA\n"; }
+         sout1 << flush;
+      }
+   }
+
+   const Geometry::Type gt = mesh_2.GetNodalFESpace()->GetFE(0)->GetGeomType();
+   MFEM_VERIFY(gt != Geometry::PRISM, "Wedge elements are not currently "
+               "supported.");
+   MFEM_VERIFY(mesh_2.GetNumGeometries(mesh_2.Dimension()) == 1, "Mixed meshes"
+               "are not currently supported.");
+
+   // Ensure the source grid function can be transferred using GSLIB-FindPoints.
+   const FiniteElementCollection *fec_in = func_source->FESpace()->FEColl();
+   std::cout << "Source FE collection: " << fec_in->Name() << std::endl;
+
+   if (src_fieldtype < 0)
+   {
+      const H1_FECollection *fec_h1 = dynamic_cast<const H1_FECollection *>(fec_in);
+      const L2_FECollection *fec_l2 = dynamic_cast<const L2_FECollection *>(fec_in);
+      const RT_FECollection *fec_rt = dynamic_cast<const RT_FECollection *>(fec_in);
+      const ND_FECollection *fec_nd = dynamic_cast<const ND_FECollection *>(fec_in);
+      if (fec_h1)      { src_fieldtype = 0; }
+      else if (fec_l2) { src_fieldtype = 1; }
+      else if (fec_rt) { src_fieldtype = 2; }
+      else if (fec_nd) { src_fieldtype = 3; }
+      else { MFEM_ABORT("GridFunction type not supported yet."); }
+   }
+   if (fieldtype < 0) { fieldtype = src_fieldtype; }
+
+   // Setup the FiniteElementSpace and GridFunction on the target mesh.
+   FiniteElementCollection *tar_fec = NULL;
+   FiniteElementSpace *tar_fes = NULL;
+
+   int tar_vdim = src_vdim;
+   if (fieldtype == 0)
+   {
+      tar_fec = new H1_FECollection(order, dim);
+      tar_vdim = (src_fieldtype > 1) ? dim : src_vdim;
+   }
+   else if (fieldtype == 1)
+   {
+      tar_fec = new L2_FECollection(order, dim);
+      tar_vdim = (src_fieldtype > 1) ? dim : src_vdim;
+   }
+   else if (fieldtype == 2)
+   {
+      tar_fec = new RT_FECollection(order, dim);
+      tar_vdim = 1;
+      MFEM_VERIFY(src_fieldtype > 1, "Cannot interpolate a scalar "
+                  "grid function to a vector");
+
+   }
+   else if (fieldtype == 3)
+   {
+      tar_fec = new ND_FECollection(order, dim);
+      tar_vdim = 1;
+      MFEM_VERIFY(src_fieldtype > 1, "Cannot interpolate a scalar "
+                  "grid function to a vector");
+   }
+   else
+   {
+      MFEM_ABORT("GridFunction type not supported.");
+   }
+   std::cout << "Target FE collection: " << tar_fec->Name() << std::endl;
+   tar_fes = new FiniteElementSpace(&mesh_2, tar_fec, tar_vdim);
+   GridFunction func_target(tar_fes);
+
+   const int NE = mesh_2.GetNE(),
+             nsp = tar_fes->GetFE(0)->GetNodes().GetNPoints(),
+             tar_ncomp = func_target.VectorDim();
+
+   // Generate list of points where the grid function will be evaluated.
+   Vector vxyz;
+   if (fieldtype == 0 && order == mesh_poly_deg)
+   {
+      vxyz = *mesh_2.GetNodes();
+   }
+   else
+   {
+      vxyz.SetSize(nsp*NE*dim);
+      for (int i = 0; i < NE; i++)
+      {
+         const FiniteElement *fe = tar_fes->GetFE(i);
+         const IntegrationRule ir = fe->GetNodes();
+         ElementTransformation *et = tar_fes->GetElementTransformation(i);
+
+         DenseMatrix pos;
+         et->Transform(ir, pos);
+         Vector rowx(vxyz.GetData() + i*nsp, nsp),
+                rowy(vxyz.GetData() + i*nsp + NE*nsp, nsp),
+                rowz;
+         if (dim == 3)
+         {
+            rowz.SetDataAndSize(vxyz.GetData() + i*nsp + 2*NE*nsp, nsp);
+         }
+         pos.GetRow(0, rowx);
+         pos.GetRow(1, rowy);
+         if (dim == 3) { pos.GetRow(2, rowz); }
+      }
+   }
+   const int nodes_cnt = vxyz.Size() / dim;
+
+   // Evaluate source grid function.
+   Vector interp_vals(nodes_cnt*tar_ncomp);
+   FindPointsGSLIB finder;
+   finder.Setup(mesh_1);
+   finder.Interpolate(vxyz, *func_source, interp_vals);
+
+   // Project the interpolated values to the target FiniteElementSpace.
+   if (fieldtype <= 1) // H1 or L2
+   {
+      if ((fieldtype == 0 && order == mesh_poly_deg) || fieldtype == 1)
+      {
+         func_target = interp_vals;
+      }
+      else // H1 - but mesh order != GridFunction order
+      {
+         Array<int> vdofs;
+         Vector vals;
+         Vector elem_dof_vals(nsp*tar_ncomp);
+
+         for (int i = 0; i < mesh_2.GetNE(); i++)
+         {
+            tar_fes->GetElementVDofs(i, vdofs);
+            vals.SetSize(vdofs.Size());
+            for (int j = 0; j < nsp; j++)
+            {
+               for (int d = 0; d < tar_ncomp; d++)
+               {
+                  // Arrange values byNodes
+                  elem_dof_vals(j+d*nsp) = interp_vals(d*nsp*NE + i*nsp + j);
+               }
+            }
+            func_target.SetSubVector(vdofs, elem_dof_vals);
+         }
+      }
+   }
+   else // H(div) or H(curl)
+   {
+      Array<int> vdofs;
+      Vector vals;
+      Vector elem_dof_vals(nsp*tar_ncomp);
+
+      for (int i = 0; i < mesh_2.GetNE(); i++)
+      {
+         tar_fes->GetElementVDofs(i, vdofs);
+         vals.SetSize(vdofs.Size());
+         for (int j = 0; j < nsp; j++)
+         {
+            for (int d = 0; d < tar_ncomp; d++)
+            {
+               // Arrange values byVDim
+               elem_dof_vals(j*tar_ncomp+d) = interp_vals(d*nsp*NE + i*nsp + j);
+            }
+         }
+         tar_fes->GetFE(i)->ProjectFromNodes(elem_dof_vals,
+                                             *tar_fes->GetElementTransformation(i),
+                                             vals);
+         func_target.SetSubVector(vdofs, vals);
+      }
+   }
+
+   // Visualize the transferred solution.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sout1;
+      sout1.open(vishost, visport);
+      if (!sout1)
+      {
+         cout << "Unable to connect to GLVis server at "
+              << vishost << ':' << visport << endl;
+      }
+      else
+      {
+         sout1.precision(8);
+         sout1 << "solution\n" << mesh_2 << func_target
+               << "window_title 'Target mesh and solution'"
+               << "window_geometry 600 0 600 600";
+         if (dim == 2) { sout1 << "keys RmjAc"; }
+         if (dim == 3) { sout1 << "keys mA\n"; }
+         sout1 << flush;
+      }
+   }
+
+   // Output the target mesh with the interpolated solution.
+   ostringstream rho_name;
+   rho_name  << "interpolated.gf";
+   ofstream rho_ofs(rho_name.str().c_str());
+   rho_ofs.precision(8);
+   func_target.Save(rho_ofs);
+   rho_ofs.close();
+
+   // Free the internal gslib data.
+   finder.FreeData();
+
+   // Delete remaining memory.
+   delete func_source;
+   delete src_fes;
+   delete src_fec;
+   delete tar_fes;
+   delete tar_fec;
+
+   return 0;
+}

miniapps/gslib/findpts.cpp

@@ -27,7 +27,6 @@
 // Compile with: make findpts
 //
 // Sample runs:
-//    findpts -m ../../data/rt-2d-q3.mesh -o 3
 //    findpts -m ../../data/rt-2d-p4-tri.mesh -o 4
 //    findpts -m ../../data/inline-tri.mesh -o 3
 //    findpts -m ../../data/inline-quad.mesh -o 3
@@ -35,6 +34,7 @@
 //    findpts -m ../../data/inline-hex.mesh -o 3
 //    findpts -m ../../data/inline-wedge.mesh -o 3
 //    findpts -m ../../data/amr-quad.mesh -o 2
+//    findpts -m ../../data/rt-2d-q3.mesh -o 3 -mo 4 -ft 2
 
 #include "mfem.hpp"
 
@@ -50,22 +50,37 @@ double field_func(const Vector &x)
    return res;
 }
 
+void F_exact(const Vector &p, Vector &F)
+{
+   F(0) = field_func(p);
+   for (int i = 1; i < F.Size(); i++) { F(i) = (i+1)*F(0); }
+}
+
 int main (int argc, char *argv[])
 {
    // Set the method's default parameters.
    const char *mesh_file = "../../data/rt-2d-q3.mesh";
+   int order             = 3;
    int mesh_poly_deg     = 3;
    int rs_levels         = 0;
    bool visualization    = true;
+   int fieldtype         = 0;
+   int ncomp             = 1;
 
    // Parse command-line options.
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
-   args.AddOption(&mesh_poly_deg, "-o", "--mesh-order",
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&mesh_poly_deg, "-mo", "--mesh-order",
                   "Polynomial degree of mesh finite element space.");
    args.AddOption(&rs_levels, "-rs", "--refine-serial",
                   "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&fieldtype, "-ft", "--field-type",
+                  "Field type: 0 - H1, 1 - L2, 2 - H(div), 3 - H(curl).");
+   args.AddOption(&ncomp, "-nc", "--ncomp",
+                  "Number of components for H1 or L2 GridFunctions");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -99,16 +114,48 @@ int main (int argc, char *argv[])
    }
 
    // Curve the mesh based on the chosen polynomial degree.
-   H1_FECollection fec(mesh_poly_deg, dim);
-   FiniteElementSpace fespace(&mesh, &fec, dim);
+   H1_FECollection fecm(mesh_poly_deg, dim);
+   FiniteElementSpace fespace(&mesh, &fecm, dim);
    mesh.SetNodalFESpace(&fespace);
-   cout << "Mesh curvature of the curved mesh: " << fec.Name() << endl;
+   cout << "Mesh curvature of the curved mesh: " << fecm.Name() << endl;
 
-   // Define a scalar function on the mesh.
-   FiniteElementSpace sc_fes(&mesh, &fec, 1);
+   MFEM_VERIFY(ncomp > 0, "Invalid number of components.");
+   int vec_dim = ncomp;
+   FiniteElementCollection *fec = NULL;
+   if (fieldtype == 0)
+   {
+      fec = new H1_FECollection(order, dim);
+      cout << "H1-GridFunction\n";
+   }
+   else if (fieldtype == 1)
+   {
+      fec = new L2_FECollection(order, dim);
+      cout << "L2-GridFunction\n";
+   }
+   else if (fieldtype == 2)
+   {
+      fec = new RT_FECollection(order, dim);
+      ncomp = 1;
+      vec_dim = dim;
+      cout << "H(div)-GridFunction\n";
+   }
+   else if (fieldtype == 3)
+   {
+      fec = new ND_FECollection(order, dim);
+      ncomp = 1;
+      vec_dim = dim;
+      cout << "H(curl)-GridFunction\n";
+   }
+   else
+   {
+      MFEM_ABORT("Invalid field type.");
+   }
+   FiniteElementSpace sc_fes(&mesh, fec, ncomp);
    GridFunction field_vals(&sc_fes);
-   FunctionCoefficient fc(field_func);
-   field_vals.ProjectCoefficient(fc);
+
+   // Project the GridFunction using VectorFunctionCoefficient.
+   VectorFunctionCoefficient F(vec_dim, F_exact);
+   field_vals.ProjectCoefficient(F);
 
    // Display the mesh and the field through glvis.
    if (visualization)
@@ -135,8 +182,8 @@ int main (int argc, char *argv[])
    // Generate equidistant points in physical coordinates over the whole mesh.
    // Note that some points might be outside, if the mesh is not a box. Note
    // also that all tasks search the same points (not mandatory).
-   const int pts_cnt_1D = 5;
-   const int pts_cnt = pow(pts_cnt_1D, dim);
+   const int pts_cnt_1D = 25;
+   int pts_cnt = pow(pts_cnt_1D, dim);
    Vector vxyz(pts_cnt * dim);
    if (dim == 2)
    {
@@ -145,8 +192,8 @@ int main (int argc, char *argv[])
       for (int i = 0; i < ir.GetNPoints(); i++)
       {
          const IntegrationPoint &ip = ir.IntPoint(i);
-         vxyz(i)           = pos_min(0) + ip.x * (pos_max(0)-pos_min(0));
-         vxyz(pts_cnt + i) = pos_min(1) + ip.y * (pos_max(1)-pos_min(1));
+         vxyz(i)           = 100*pos_min(0) + ip.x * (pos_max(0)-pos_min(0));
+         vxyz(pts_cnt + i) = 100*pos_min(1) + ip.y * (pos_max(1)-pos_min(1));
       }
    }
    else
@@ -163,32 +210,35 @@ int main (int argc, char *argv[])
    }
 
    // Find and Interpolate FE function values on the desired points.
-   Vector interp_vals(pts_cnt);
-   // FindPoints using GSLIB and interpolate
+   Vector interp_vals(pts_cnt*vec_dim);
    FindPointsGSLIB finder;
-   finder.Interpolate(mesh, vxyz, field_vals, interp_vals);
-   Array<unsigned int> code_out = finder.GetCode();
+   finder.Setup(mesh);
+   finder.SetL2AvgType(FindPointsGSLIB::NONE);
+   finder.Interpolate(vxyz, field_vals, interp_vals);
+   Array<unsigned int> code_out    = finder.GetCode();
    Vector dist_p_out = finder.GetDist();
 
-   // Free the internal gslib data.
-   finder.FreeData();
-
    int face_pts = 0, not_found = 0, found = 0;
    double max_err = 0.0, max_dist = 0.0;
    Vector pos(dim);
-   for (int i = 0; i < pts_cnt; i++)
+   int npt = 0;
+   for (int j = 0; j < vec_dim; j++)
    {
-      if (code_out[i] < 2)
+      for (int i = 0; i < pts_cnt; i++)
       {
-         found++;
-         for (int d = 0; d < dim; d++) { pos(d) = vxyz(d * pts_cnt + i); }
-         const double exact_val = field_func(pos);
-
-         max_err  = std::max(max_err, fabs(exact_val - interp_vals[i]));
-         max_dist = std::max(max_dist, dist_p_out(i));
-         if (code_out[i] == 1) { face_pts++; }
+         if (code_out[i] < 2)
+         {
+            if (j == 0) { found++; }
+            for (int d = 0; d < dim; d++) { pos(d) = vxyz(d * pts_cnt + i); }
+            Vector exact_val(vec_dim);
+            F_exact(pos, exact_val);
+            max_err  = std::max(max_err, fabs(exact_val(j) - interp_vals[npt]));
+            max_dist = std::max(max_dist, dist_p_out(i));
+            if (code_out[i] == 1 && j == 0) { face_pts++; }
+         }
+         else { if (j == 0) { not_found++; } }
+         npt++;
       }
-      else { not_found++; }
    }
 
    cout << setprecision(16)
@@ -199,5 +249,10 @@ int main (int argc, char *argv[])
         << "\nPoints not found:    " << not_found
         << "\nPoints on faces:     " << face_pts << endl;
 
+   // Free the internal gslib data.
+   finder.FreeData();
+
+   delete fec;
+
    return 0;
 }

miniapps/gslib/makefile

@@ -22,7 +22,7 @@ MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
 ifeq ($(MFEM_USE_GSLIB),YES)
-   SEQ_MINIAPPS = findpts field-diff
+   SEQ_MINIAPPS = findpts field-diff field-interp
    PAR_MINIAPPS = pfindpts
 else
    SEQ_MINIAPPS =

miniapps/gslib/pfindpts.cpp

@@ -27,7 +27,6 @@
 // Compile with: make pfindpts
 //
 // Sample runs:
-//    mpirun -np 2 pfindpts -m ../../data/rt-2d-q3.mesh -o 3
 //    mpirun -np 2 pfindpts -m ../../data/rt-2d-p4-tri.mesh -o 4
 //    mpirun -np 2 pfindpts -m ../../data/inline-tri.mesh -o 3
 //    mpirun -np 2 pfindpts -m ../../data/inline-quad.mesh -o 3
@@ -35,6 +34,7 @@
 //    mpirun -np 2 pfindpts -m ../../data/inline-hex.mesh -o 3
 //    mpirun -np 2 pfindpts -m ../../data/inline-wedge.mesh -o 3
 //    mpirun -np 2 pfindpts -m ../../data/amr-quad.mesh -o 2
+//    mpirun -np 2 pfindpts -m ../../data/rt-2d-q3.mesh -o 3 -mo 4 -ft 2
 
 
 #include "mfem.hpp"
@@ -51,6 +51,12 @@ double field_func(const Vector &x)
    return res;
 }
 
+void F_exact(const Vector &p, Vector &F)
+{
+   F(0) = field_func(p);
+   for (int i = 1; i < F.Size(); i++) { F(i) = (i+1)*F(0); }
+}
+
 int main (int argc, char *argv[])
 {
    // Initialize MPI.
@@ -61,21 +67,30 @@ int main (int argc, char *argv[])
 
    // Set the method's default parameters.
    const char *mesh_file = "../../data/rt-2d-q3.mesh";
+   int order             = 3;
    int mesh_poly_deg     = 3;
    int rs_levels         = 0;
    int rp_levels         = 0;
    bool visualization    = true;
+   int fieldtype         = 0;
+   int ncomp             = 1;
 
    // Parse command-line options.
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
-   args.AddOption(&mesh_poly_deg, "-o", "--mesh-order",
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&mesh_poly_deg, "-mo", "--mesh-order",
                   "Polynomial degree of mesh finite element space.");
    args.AddOption(&rs_levels, "-rs", "--refine-serial",
                   "Number of times to refine the mesh uniformly in serial.");
    args.AddOption(&rp_levels, "-rp", "--refine-parallel",
                   "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&fieldtype, "-ft", "--field-type",
+                  "Field type: 0 - H1, 1 - L2, 2 - H(div), 3 - H(curl).");
+   args.AddOption(&ncomp, "-nc", "--ncomp",
+                  "Number of components for H1 or L2 GridFunctions");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -120,19 +135,51 @@ int main (int argc, char *argv[])
    for (int lev = 0; lev < rp_levels; lev++) { pmesh.UniformRefinement(); }
 
    // Curve the mesh based on the chosen polynomial degree.
-   H1_FECollection fec(mesh_poly_deg, dim);
-   ParFiniteElementSpace pfespace(&pmesh, &fec, dim);
+   H1_FECollection fecm(mesh_poly_deg, dim);
+   ParFiniteElementSpace pfespace(&pmesh, &fecm, dim);
    pmesh.SetNodalFESpace(&pfespace);
    if (myid == 0)
    {
-      cout << "Mesh curvature of the curved mesh: " << fec.Name() << endl;
+      cout << "Mesh curvature of the curved mesh: " << fecm.Name() << endl;
    }
 
-   // Define a scalar function on the mesh.
-   ParFiniteElementSpace sc_fes(&pmesh, &fec, 1);
-   GridFunction field_vals(&sc_fes);
-   FunctionCoefficient fc(field_func);
-   field_vals.ProjectCoefficient(fc);
+   MFEM_VERIFY(ncomp > 0, "Invalid number of components.");
+   int vec_dim = ncomp;
+   FiniteElementCollection *fec = NULL;
+   if (fieldtype == 0)
+   {
+      fec = new H1_FECollection(order, dim);
+      if (myid == 0) { cout << "H1-GridFunction\n"; }
+   }
+   else if (fieldtype == 1)
+   {
+      fec = new L2_FECollection(order, dim);
+      if (myid == 0) { cout << "L2-GridFunction\n"; }
+   }
+   else if (fieldtype == 2)
+   {
+      fec = new RT_FECollection(order, dim);
+      ncomp = 1;
+      vec_dim = dim;
+      if (myid == 0) { cout << "H(div)-GridFunction\n"; }
+   }
+   else if (fieldtype == 3)
+   {
+      fec = new ND_FECollection(order, dim);
+      ncomp = 1;
+      vec_dim = dim;
+      if (myid == 0) { cout << "H(curl)-GridFunction\n"; }
+   }
+   else
+   {
+      if (myid == 0) { MFEM_ABORT("Invalid FECollection type."); }
+   }
+   ParFiniteElementSpace sc_fes(&pmesh, fec, ncomp);
+   ParGridFunction field_vals(&sc_fes);
+
+   // Project the GridFunction using VectorFunctionCoefficient.
+   VectorFunctionCoefficient F(vec_dim, F_exact);
+   field_vals.ProjectCoefficient(F);
 
    // Display the mesh and the field through glvis.
    if (visualization)
@@ -163,7 +210,7 @@ int main (int argc, char *argv[])
    // Generate equidistant points in physical coordinates over the whole mesh.
    // Note that some points might be outside, if the mesh is not a box. Note
    // also that all tasks search the same points (not mandatory).
-   const int pts_cnt_1D = 5;
+   const int pts_cnt_1D = 10;
    const int pts_cnt = pow(pts_cnt_1D, dim);
    Vector vxyz(pts_cnt * dim);
    if (dim == 2)
@@ -191,10 +238,10 @@ int main (int argc, char *argv[])
    }
 
    // Find and Interpolate FE function values on the desired points.
-   Vector interp_vals(pts_cnt);
-   // FindPoints using GSLIB and interpolate
+   Vector interp_vals(pts_cnt*vec_dim);
    FindPointsGSLIB finder(MPI_COMM_WORLD);
-   finder.Interpolate(pmesh, vxyz, field_vals, interp_vals);
+   finder.Setup(pmesh);
+   finder.Interpolate(vxyz, field_vals, interp_vals);
    Array<unsigned int> code_out    = finder.GetCode();
    Array<unsigned int> task_id_out = finder.GetProc();
    Vector dist_p_out = finder.GetDist();
@@ -202,28 +249,35 @@ int main (int argc, char *argv[])
    int face_pts = 0, not_found = 0, found_loc = 0, found_away = 0;
    double max_err = 0.0, max_dist = 0.0;
    Vector pos(dim);
-   for (int i = 0; i < pts_cnt; i++)
+   int npt = 0;
+   for (int j = 0; j < vec_dim; j++)
    {
-      (task_id_out[i] == (unsigned)myid) ? found_loc++ : found_away++;
-
-      if (code_out[i] < 2)
+      for (int i = 0; i < pts_cnt; i++)
       {
-         for (int d = 0; d < dim; d++) { pos(d) = vxyz(d * pts_cnt + i); }
-         const double exact_val = field_func(pos);
+         if (j == 0)
+         {
+            (task_id_out[i] == (unsigned)myid) ? found_loc++ : found_away++;
+         }
 
-         max_err  = std::max(max_err, fabs(exact_val - interp_vals[i]));
-         max_dist = std::max(max_dist, dist_p_out(i));
-         if (code_out[i] == 1) { face_pts++; }
+         if (code_out[i] < 2)
+         {
+            for (int d = 0; d < dim; d++) { pos(d) = vxyz(d * pts_cnt + i); }
+            Vector exact_val(vec_dim);
+            F_exact(pos, exact_val);
+            max_err  = std::max(max_err, fabs(exact_val(j) - interp_vals(npt)));
+            max_dist = std::max(max_dist, dist_p_out(i));
+            if (code_out[i] == 1 && j == 0) { face_pts++; }
+         }
+         else { if (j == 0) { not_found++; } }
+         npt++;
       }
-      else { not_found++; }
    }
 
-   // We print only the task 0 result (other tasks should be identical except
-   // the number of points found locally).
+   // Print the results for task 0 since all tasks have the same set of points.
    if (myid == 0)
    {
-      cout << setprecision(16) << "--- Task " << myid << ": "
-           << "\nSearched points:      " << pts_cnt
+      cout << setprecision(16)
+           << "Searched unique points: " << pts_cnt
            << "\nFound on local mesh:  " << found_loc
            << "\nFound on other tasks: " << found_away
            << "\nMax interp error:     " << max_err
@@ -234,6 +288,9 @@ int main (int argc, char *argv[])
 
    // Free the internal gslib data.
    finder.FreeData();
+
+   delete fec;
+
    MPI_Finalize();
    return 0;
 }

miniapps/meshing/CMakeLists.txt

@@ -50,6 +50,10 @@ add_mfem_miniapp(twist
   MAIN twist.cpp
   LIBRARIES mfem)
 
+add_mfem_miniapp(polar-nc
+  MAIN polar-nc.cpp
+  LIBRARIES mfem)
+
 # Add serial tests.
 add_test(NAME mesh-optimizer
   COMMAND mesh-optimizer -no-vis -m ${CMAKE_CURRENT_SOURCE_DIR}/icf.mesh)

miniapps/meshing/makefile

@@ -25,9 +25,8 @@ include $(DEFAULTS_MK)
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
-SEQ_MINIAPPS = mobius-strip klein-bottle toroid trimmer twist \
-	mesh-explorer shaper extruder mesh-optimizer \
-	minimal-surface
+SEQ_MINIAPPS = mobius-strip klein-bottle toroid trimmer twist mesh-explorer\
+	shaper extruder mesh-optimizer minimal-surface polar-nc
 PAR_MINIAPPS = pmesh-optimizer pminimal-surface
 ifeq ($(MFEM_USE_MPI),NO)
    MINIAPPS = $(SEQ_MINIAPPS)
@@ -107,7 +106,7 @@ clean: clean-build clean-exec
 clean-build:
 	rm -f *.o *~ mobius-strip klein-bottle toroid twist
 	rm -f mesh-explorer shaper extruder trimmer
-	rm -f mesh-optimizer pmesh-optimizer
+	rm -f mesh-optimizer pmesh-optimizer polar-nc
 	rm -f minimal-surface pminimal-surface
 	rm -rf *.dSYM *.TVD.*breakpoints
 
@@ -115,4 +114,4 @@ clean-exec:
 	@rm -f mobius-strip.mesh klein-bottle.mesh mesh-explorer.mesh
 	@rm -f toroid-*.mesh twist-*.mesh trimmer.mesh
 	@rm -f partitioning.txt shaper.mesh extruder.mesh
-	@rm -f optimized* perturbed*
+	@rm -f optimized* perturbed* polar-nc.mesh

miniapps/meshing/polar-nc.cpp

@@ -0,0 +1,560 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//             ----------------------------------------------
+//             Polar NC: Generate polar non-conforming meshes
+//             ----------------------------------------------
+//
+// This miniapp generates a circular sector mesh that consist of quadrilaterals
+// and triangles of similar sizes. The 3D version of the mesh is made of prisms
+// and tetrahedra. The mesh is non-conforming by design, and can optionally be
+// made curvilinear. The elements are ordered along a space-filling curve by
+// default, which makes the mesh ready for parallel non-conforming AMR in MFEM.
+//
+// The implementation also demonstrates how to initialize a non-conforming mesh
+// on the fly by marking hanging nodes with Mesh::AddVertexParents.
+//
+// Compile with: make polar-nc
+//
+// Sample runs:  polar-nc --radius 1 --nsteps 10
+//               polar-nc --aspect 2
+//               polar-nc --dim 3 --order 4
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace mfem;
+using namespace std;
+
+
+struct Params2
+{
+   double r, dr;
+   double a, da;
+
+   Params2() = default;
+   Params2(double r0, double r1, double a0, double a1)
+      : r(r0), dr(r1 - r0), a(a0), da(a1 - a0) {}
+};
+
+Mesh* Make2D(int nsteps, double rstep, double phi, double aspect, int order,
+             bool sfc)
+{
+   Mesh *mesh = new Mesh(2, 0, 0);
+
+   int origin = mesh->AddVertex(0.0, 0.0);
+
+   // n is the number of steps in the polar direction
+   int n = 1;
+   while (phi * rstep/2 / n * aspect > rstep) { n++; }
+
+   double r = rstep;
+   int first = mesh->AddVertex(r, 0.0);
+
+   Array<Params2> params;
+   Array<Pair<int, int>> blocks;
+
+   // create triangles around the origin
+   double prev_alpha = 0.0;
+   for (int i = 0; i < n; i++)
+   {
+      double alpha = phi * (i+1) / n;
+      mesh->AddVertex(r*cos(alpha), r*sin(alpha));
+      mesh->AddTriangle(origin, first+i, first+i+1);
+
+      params.Append(Params2(0, r, prev_alpha, alpha));
+      prev_alpha = alpha;
+   }
+
+   mesh->AddBdrSegment(origin, first, 1);
+   mesh->AddBdrSegment(first+n, origin, 2);
+
+   for (int k = 1; k < nsteps; k++)
+   {
+      // m is the number of polar steps of the previous row
+      int m = n;
+      int prev_first = first;
+
+      double prev_r = r;
+      r += rstep;
+
+      if (phi * (r + prev_r)/2 / n * aspect < rstep * sqrt(2))
+      {
+         if (k == 1) { blocks.Append(Pair<int, int>(mesh->GetNE(), n)); }
+
+         first = mesh->AddVertex(r, 0.0);
+         mesh->AddBdrSegment(prev_first, first, 1);
+
+         // create a row of quads, same number as in previous row
+         prev_alpha = 0.0;
+         for (int i = 0; i < n; i++)
+         {
+            double alpha = phi * (i+1) / n;
+            mesh->AddVertex(r*cos(alpha), r*sin(alpha));
+            mesh->AddQuad(prev_first+i, first+i, first+i+1, prev_first+i+1);
+
+            params.Append(Params2(prev_r, r, prev_alpha, alpha));
+            prev_alpha = alpha;
+         }
+
+         mesh->AddBdrSegment(first+n, prev_first+n, 2);
+      }
+      else // we need to double the number of elements per row
+      {
+         n *= 2;
+
+         blocks.Append(Pair<int, int>(mesh->GetNE(), n));
+
+         // first create hanging vertices
+         int hang;
+         for (int i = 0; i < m; i++)
+         {
+            double alpha = phi * (2*i+1) / n;
+            int index = mesh->AddVertex(prev_r*cos(alpha), prev_r*sin(alpha));
+            mesh->AddVertexParents(index, prev_first+i, prev_first+i+1);
+            if (!i) { hang = index; }
+         }
+
+         first = mesh->AddVertex(r, 0.0);
+         int a = prev_first, b = first;
+
+         mesh->AddBdrSegment(a, b, 1);
+
+         // create a row of quad pairs
+         prev_alpha = 0.0;
+         for (int i = 0; i < m; i++)
+         {
+            int c = hang+i, e = a+1;
+
+            double alpha_half = phi * (2*i+1) / n;
+            int d = mesh->AddVertex(r*cos(alpha_half), r*sin(alpha_half));
+
+            double alpha = phi * (2*i+2) / n;
+            int f = mesh->AddVertex(r*cos(alpha), r*sin(alpha));
+
+            mesh->AddQuad(a, b, d, c);
+            mesh->AddQuad(c, d, f, e);
+
+            a = e, b = f;
+
+            params.Append(Params2(prev_r, r, prev_alpha, alpha_half));
+            params.Append(Params2(prev_r, r, alpha_half, alpha));
+            prev_alpha = alpha;
+         }
+
+         mesh->AddBdrSegment(b, a, 2);
+      }
+   }
+
+   for (int i = 0; i < n; i++)
+   {
+      mesh->AddBdrSegment(first+i, first+i+1, 3);
+   }
+
+   // reorder blocks of elements with Grid SFC ordering
+   if (sfc)
+   {
+      blocks.Append(Pair<int, int>(mesh->GetNE(), 0));
+
+      Array<Params2> new_params(params.Size());
+
+      Array<int> ordering(mesh->GetNE());
+      for (int i = 0; i < blocks[0].one; i++)
+      {
+         ordering[i] = i;
+         new_params[i] = params[i];
+      }
+
+      Array<int> coords;
+      for (int i = 0; i < blocks.Size()-1; i++)
+      {
+         int beg = blocks[i].one;
+         int width = blocks[i].two;
+         int height = (blocks[i+1].one - blocks[i].one) / width;
+
+         NCMesh::GridSfcOrdering2D(width, height, coords);
+
+         for (int j = 0, k = 0; j < coords.Size(); k++, j += 2)
+         {
+            int sfc = ((i & 1) ? coords[j] : (width-1 - coords[j]))
+                      + coords[j+1]*width;
+            int old_index = beg + sfc;
+
+            ordering[old_index] = beg + k;
+            new_params[beg + k] = params[old_index];
+         }
+      }
+
+      mesh->ReorderElements(ordering, false);
+
+      mfem::Swap(params, new_params);
+   }
+
+   // create high-order curvature
+   if (order > 1)
+   {
+      mesh->SetCurvature(order);
+
+      GridFunction *nodes = mesh->GetNodes();
+      const FiniteElementSpace *fes = mesh->GetNodalFESpace();
+
+      Array<int> dofs;
+      MFEM_ASSERT(params.Size() == mesh->GetNE(), "");
+
+      for (int i = 0; i < mesh->GetNE(); i++)
+      {
+         const Params2 &par = params[i];
+         const IntegrationRule &ir = fes->GetFE(i)->GetNodes();
+         Geometry::Type geom = mesh->GetElementBaseGeometry(i);
+         fes->GetElementDofs(i, dofs);
+
+         for (int j = 0; j < dofs.Size(); j++)
+         {
+            double r, a;
+            if (geom == Geometry::SQUARE)
+            {
+               r = par.r + ir[j].x * par.dr;
+               a = par.a + ir[j].y * par.da;
+            }
+            else
+            {
+               double rr = ir[j].x + ir[j].y;
+               if (std::abs(rr) < 1e-12) { continue; }
+               r = par.r + rr * par.dr;
+               a = par.a + ir[j].y/rr * par.da;
+            }
+            (*nodes)(fes->DofToVDof(dofs[j], 0)) = r*cos(a);
+            (*nodes)(fes->DofToVDof(dofs[j], 1)) = r*sin(a);
+         }
+      }
+
+      nodes->RestrictConforming();
+   }
+
+   mesh->FinalizeMesh();
+
+   return mesh;
+}
+
+
+const double pi2 = M_PI / 2;
+
+struct Params3
+{
+   double r, dr;
+   double u1, u2, u3;
+   double v1, v2, v3;
+
+   Params3() = default;
+   Params3(double r0, double r1,
+           double u1, double v1, double u2, double v2, double u3, double v3)
+      : r(r0), dr(r1 - r0), u1(u1), u2(u2), u3(u3), v1(v1), v2(v2), v3(v3) {}
+};
+
+struct Vert : public Hashed2
+{
+   int id;
+};
+
+int GetMidVertex(int v1, int v2, double r, double u, double v, bool hanging,
+                 Mesh *mesh, HashTable<Vert> &hash)
+{
+   int vmid = hash.FindId(v1, v2);
+   if (vmid < 0)
+   {
+      vmid = hash.GetId(v1, v2);
+
+      double w = 1.0 - u - v;
+      double q = r / sqrt(u*u + v*v + w*w);
+      int index = mesh->AddVertex(u*q, v*q, w*q);
+
+      if (hanging) { mesh->AddVertexParents(index, v1, v2); }
+
+      hash[vmid].id = index;
+   }
+   return hash[vmid].id;
+}
+
+void MakeLayer(int vx1, int vy1, int vz1, int vx2, int vy2, int vz2, int level,
+               double r1, double r2, double u1, double v1, double u2, double v2,
+               double u3, double v3, bool bnd1, bool bnd2, bool bnd3, bool bnd4,
+               Mesh *mesh, HashTable<Vert> &hash, Array<Params3> &params)
+{
+   if (!level)
+   {
+      mesh->AddWedge(vx1, vy1, vz1, vx2, vy2, vz2);
+
+      if (bnd1) { mesh->AddBdrQuad(vx1, vy1, vy2, vx2, 1); }
+      if (bnd2) { mesh->AddBdrQuad(vy1, vz1, vz2, vy2, 2); }
+      if (bnd3) { mesh->AddBdrQuad(vz1, vx1, vx2, vz2, 3); }
+      if (bnd4) { mesh->AddBdrTriangle(vx2, vy2, vz2, 4); }
+
+      params.Append(Params3(r1, r2, u1, v1, u2, v2, u3, v3));
+   }
+   else
+   {
+      double u12 = (u1+u2)/2, v12 = (v1+v2)/2;
+      double u23 = (u2+u3)/2, v23 = (v2+v3)/2;
+      double u31 = (u3+u1)/2, v31 = (v3+v1)/2;
+
+      bool hang = (level == 1);
+
+      int vxy1 = GetMidVertex(vx1, vy1, r1, u12, v12, hang, mesh, hash);
+      int vyz1 = GetMidVertex(vy1, vz1, r1, u23, v23, hang, mesh, hash);
+      int vxz1 = GetMidVertex(vx1, vz1, r1, u31, v31, hang, mesh, hash);
+      int vxy2 = GetMidVertex(vx2, vy2, r2, u12, v12, false, mesh, hash);
+      int vyz2 = GetMidVertex(vy2, vz2, r2, u23, v23, false, mesh, hash);
+      int vxz2 = GetMidVertex(vx2, vz2, r2, u31, v31, false, mesh, hash);
+
+      MakeLayer(vx1, vxy1, vxz1, vx2, vxy2, vxz2, level-1,
+                r1, r2, u1, v1, u12, v12, u31, v31,
+                bnd1, false, bnd3, bnd4, mesh, hash, params);
+      MakeLayer(vxy1, vy1, vyz1, vxy2, vy2, vyz2, level-1,
+                r1, r2, u12, v12, u2, v2, u23, v23,
+                bnd1, bnd2, false, bnd4, mesh, hash, params);
+      MakeLayer(vxz1, vyz1, vz1, vxz2, vyz2, vz2, level-1,
+                r1, r2, u31, v31, u23, v23, u3, v3,
+                false, bnd2, bnd3, bnd4, mesh, hash, params);
+      MakeLayer(vyz1, vxz1, vxy1, vyz2, vxz2, vxy2, level-1,
+                r1, r2, u23, v23, u31, v31, u12, v12,
+                false, false, false, bnd4, mesh, hash, params);
+   }
+}
+
+void MakeCenter(int origin, int vx, int vy, int vz, int level, double r,
+                double u1, double v1, double u2, double v2, double u3, double v3,
+                bool bnd1, bool bnd2, bool bnd3, bool bnd4,
+                Mesh *mesh, HashTable<Vert> &hash, Array<Params3> &params)
+{
+   if (!level)
+   {
+      mesh->AddTet(origin, vx, vy, vz);
+
+      if (bnd1) { mesh->AddBdrTriangle(0, vy, vx, 1); }
+      if (bnd2) { mesh->AddBdrTriangle(0, vz, vy, 2); }
+      if (bnd3) { mesh->AddBdrTriangle(0, vx, vz, 3); }
+      if (bnd4) { mesh->AddBdrTriangle(vx, vy, vz, 4); }
+
+      params.Append(Params3(0, r, u1, v1, u2, v2, u3, v3));
+   }
+   else
+   {
+      double u12 = (u1+u2)/2, v12 = (v1+v2)/2;
+      double u23 = (u2+u3)/2, v23 = (v2+v3)/2;
+      double u31 = (u3+u1)/2, v31 = (v3+v1)/2;
+
+      int vxy = GetMidVertex(vx, vy, r, u12, v12, false, mesh, hash);
+      int vyz = GetMidVertex(vy, vz, r, u23, v23, false, mesh, hash);
+      int vxz = GetMidVertex(vx, vz, r, u31, v31, false, mesh, hash);
+
+      MakeCenter(origin, vx, vxy, vxz, level-1, r, u1, v1, u12, v12, u31, v31,
+                 bnd1, false, bnd3, bnd4, mesh, hash, params);
+      MakeCenter(origin, vxy, vy, vyz, level-1, r, u12, v12, u2, v2, u23, v23,
+                 bnd1, bnd2, false, bnd4, mesh, hash, params);
+      MakeCenter(origin, vxz, vyz, vz, level-1, r, u31, v31, u23, v23, u3, v3,
+                 false, bnd2, bnd3, bnd4, mesh, hash, params);
+      MakeCenter(origin, vyz, vxz, vxy, level-1, r, u23, v23, u31, v31, u12, v12,
+                 false, false, false, bnd4, mesh, hash, params);
+   }
+}
+
+Mesh* Make3D(int nsteps, double rstep, double aspect, int order, bool sfc)
+{
+   Mesh *mesh = new Mesh(3, 0, 0);
+
+   HashTable<Vert> hash;
+   Array<Params3> params;
+
+   int origin = mesh->AddVertex(0, 0, 0);
+
+   double r = rstep;
+   int a = mesh->AddVertex(r, 0, 0);
+   int b = mesh->AddVertex(0, r, 0);
+   int c = mesh->AddVertex(0, 0, r);
+
+   int levels = 0;
+   while (pi2 * rstep / (1 << levels) * aspect > rstep) { levels++; }
+
+   MakeCenter(origin, a, b, c, levels, r, 1, 0, 0, 1, 0, 0,
+              true, true, true, (nsteps == 1), mesh, hash, params);
+
+   for (int k = 1; k < nsteps; k++)
+   {
+      double prev_r = r;
+      r += rstep;
+
+      if ((prev_r + rstep/2) * pi2 * aspect / (1 << levels) > rstep * sqrt(2))
+      {
+         levels++;
+      }
+
+      int d = mesh->AddVertex(r, 0, 0);
+      int e = mesh->AddVertex(0, r, 0);
+      int f = mesh->AddVertex(0, 0, r);
+
+      MakeLayer(a, b, c, d, e, f, levels, prev_r, r,
+                1, 0, 0, 1, 0, 0, true, true, true, (k == nsteps-1),
+                mesh, hash, params);
+
+      a = d;
+      b = e;
+      c = f;
+   }
+
+   // reorder mesh with Hilbert spatial sort
+   if (sfc)
+   {
+      Array<int> ordering;
+      mesh->GetHilbertElementOrdering(ordering);
+      mesh->ReorderElements(ordering, false);
+
+      Array<Params3> new_params(params.Size());
+      for (int i = 0; i < ordering.Size(); i++)
+      {
+         new_params[ordering[i]] = params[i];
+      }
+      mfem::Swap(params, new_params);
+   }
+
+   mesh->FinalizeMesh();
+
+   // create high-order curvature
+   if (order > 1)
+   {
+      mesh->SetCurvature(order);
+
+      GridFunction *nodes = mesh->GetNodes();
+      const FiniteElementSpace *fes = mesh->GetNodalFESpace();
+
+      Array<int> dofs;
+      MFEM_ASSERT(params.Size() == mesh->GetNE(), "");
+
+      for (int i = 0; i < mesh->GetNE(); i++)
+      {
+         const Params3 &par = params[i];
+         const IntegrationRule &ir = fes->GetFE(i)->GetNodes();
+         Geometry::Type geom = mesh->GetElementBaseGeometry(i);
+         fes->GetElementDofs(i, dofs);
+
+         for (int j = 0; j < dofs.Size(); j++)
+         {
+            const IntegrationPoint &ip = ir[j];
+
+            double u, v, w, r;
+            if (geom == Geometry::PRISM)
+            {
+               double l1 = 1.0 - ip.x - ip.y;
+               double l2 = ip.x, l3 = ip.y;
+               u = l1 * par.u1 + l2 * par.u2 + l3 * par.u3;
+               v = l1 * par.v1 + l2 * par.v2 + l3 * par.v3;
+               w = 1.0 - u - v;
+               r = par.r + ip.z * par.dr;
+            }
+            else
+            {
+               u = ip.x * par.u1 + ip.y * par.u2 + ip.z * par.u3;
+               v = ip.x * par.v1 + ip.y * par.v2 + ip.z * par.v3;
+               double rr = ip.x + ip.y + ip.z;
+               if (std::abs(rr) < 1e-12) { continue; }
+               w = rr - u - v;
+               r = par.r + rr * par.dr;
+            }
+
+            double q = r / sqrt(u*u + v*v + w*w);
+            (*nodes)(fes->DofToVDof(dofs[j], 0)) = u*q;
+            (*nodes)(fes->DofToVDof(dofs[j], 1)) = v*q;
+            (*nodes)(fes->DofToVDof(dofs[j], 2)) = w*q;
+         }
+      }
+
+      nodes->RestrictConforming();
+   }
+
+   return mesh;
+}
+
+
+int main(int argc, char *argv[])
+{
+   int dim = 2;
+   double radius = 1.0;
+   int nsteps = 10;
+   double angle = 90;
+   double aspect = 1.0;
+   int order = 2;
+   bool sfc = true;
+   bool visualization = true;
+
+   // parse command line
+   OptionsParser args(argc, argv);
+   args.AddOption(&dim, "-d", "--dim", "Mesh dimension (2 or 3).");
+   args.AddOption(&radius, "-r", "--radius", "Radius of the domain.");
+   args.AddOption(&nsteps, "-n", "--nsteps",
+                  "Number of elements along the radial direction");
+   args.AddOption(&aspect, "-a", "--aspect",
+                  "Target aspect ratio of the elements.");
+   args.AddOption(&angle, "-phi", "--phi", "Angular range (2D only).");
+   args.AddOption(&order, "-o", "--order",
+                  "Polynomial degree of mesh curvature.");
+   args.AddOption(&sfc, "-sfc", "--sfc", "-no-sfc", "--no-sfc",
+                  "Try to order elements along a space-filling curve.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return EXIT_FAILURE;
+   }
+   args.PrintOptions(cout);
+
+   // validate options
+   MFEM_VERIFY(radius > 0, "");
+   MFEM_VERIFY(aspect > 0, "");
+   MFEM_VERIFY(dim >= 2 && dim <= 3, "");
+   MFEM_VERIFY(angle > 0 && angle < 360, "");
+   MFEM_VERIFY(nsteps > 0, "");
+
+   double phi = angle * M_PI / 180;
+
+   // generate
+   Mesh *mesh;
+   if (dim == 2)
+   {
+      mesh = Make2D(nsteps, radius/nsteps, phi, aspect, order, sfc);
+   }
+   else
+   {
+      mesh = Make3D(nsteps, radius/nsteps, aspect, order, sfc);
+   }
+
+   // save the final mesh
+   ofstream ofs("polar-nc.mesh");
+   ofs.precision(8);
+   mesh->Print(ofs);
+
+   // output the mesh to GLVis
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "mesh\n" << *mesh << flush;
+   }
+
+   delete mesh;
+
+   return EXIT_SUCCESS;
+}
+

miniapps/tools/CMakeLists.txt

@@ -14,6 +14,11 @@ add_mfem_miniapp(display-basis
   ${MFEM_MINIAPPS_COMMON_HEADERS}
   LIBRARIES mfem mfem-common)
 
+add_mfem_miniapp(coef-fact
+  MAIN coef-fact.cpp
+  ${MFEM_MINIAPPS_COMMON_HEADERS}
+  LIBRARIES mfem mfem-common)
+
 add_mfem_miniapp(get-values
   MAIN get-values.cpp
   ${MFEM_MINIAPPS_COMMON_HEADERS}

miniapps/tools/coef-fact.cpp

@@ -0,0 +1,234 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//    -------------------------------------------------------------------
+//    Coef Fact Miniapp:  Visualize Coefficient fields
+//    -------------------------------------------------------------------
+//
+//  ./coef-fact
+//  ./coef-fact -c coef-fact.inp
+//
+#include "mfem.hpp"
+#include "../common/fem_extras.hpp"
+#include "../../general/text.hpp"
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+double MyScalarFunc(const Vector &x)
+{
+   return x * x;
+}
+
+void MyVectorFunc(const Vector &x, Vector &v)
+{
+   v.SetSize(x.Size());
+   v.Set(-2.0, x);
+}
+
+class MyCoefficient : public Coefficient
+{
+private:
+   Vector k;
+   mutable Vector x;
+
+public:
+   MyCoefficient(const Vector & _k) : k(_k), x(_k.Size()) {}
+
+   double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+   { T.Transform(ip, x); return sin(k * x); }
+};
+
+class MyVectorCoefficient : public VectorCoefficient
+{
+private:
+   double a;
+   Vector b;
+   mutable Vector x;
+
+public:
+   MyVectorCoefficient(double _a, const Vector & _b)
+      : VectorCoefficient(_b.Size()), a(_a), b(_b), x(_b.Size()) {}
+
+   void Eval(Vector & v, ElementTransformation &T, const IntegrationPoint &ip)
+   { T.Transform(ip, x); v = b; v.Add(a, x); }
+};
+
+class MyCoefFactory : public CoefFactory
+{
+public:
+   MyCoefFactory() {}
+
+   using CoefFactory::GetScalarCoef;
+   using CoefFactory::GetVectorCoef;
+   using CoefFactory::GetMatrixCoef;
+
+   Coefficient * GetScalarCoef(string &name, istream &input)
+   {
+      int c = -1;
+      if (name == "MyCoefficient")
+      {
+         int dim;
+         input >> dim;
+         MFEM_VERIFY(dim >=1 && dim <= 3,
+                     "Invalid dimension for MyCoefficient "
+                     "read by MyCoefFactory");
+         Vector val(dim);
+         for (int i=0; i<dim; i++) { input >> val[i]; }
+         c = sCoefs.Append(new MyCoefficient(val));
+      }
+      else
+      {
+         return CoefFactory::GetScalarCoef(name, input);
+      }
+      return sCoefs[--c];
+   }
+
+   VectorCoefficient * GetVectorCoef(string &name, istream &input)
+   {
+      int c = -1;
+      if (name == "MyVectorCoefficient")
+      {
+         int dim;
+         input >> dim;
+         MFEM_VERIFY(dim >=1 && dim <= 3,
+                     "Invalid dimension for MyVectorCoefficient "
+                     "read by MyCoefFactory");
+         double a;
+         input >> a;
+         Vector val(dim);
+         for (int i=0; i<dim; i++) { input >> val[i]; }
+         c = vCoefs.Append(new MyVectorCoefficient(a, val));
+      }
+      else
+      {
+         return CoefFactory::GetVectorCoef(name, input);
+      }
+      return vCoefs[--c];
+   }
+};
+
+const char coef_str[] =
+   "scalar_coef\nConstantCoefficient\n3.14\nvector_coef\nVectorConstantCoefficient\n2 2.0 1.0\nscalar_coef\nFunctionCoefficient\n0 0\nvector_coef\nVectorFunctionCoefficient\n2 0 0\nscalar_coef\nMyCoefficient\n2 2.0 1.0\nvector_coef\nMyVectorCoefficient\n2 3.0 2.0 1.0\n";
+
+int main(int argc, char *argv[])
+{
+#ifdef MFEM_USE_MPI
+   MPI_Session mpi;
+   if (!mpi.Root()) { mfem::out.Disable(); mfem::err.Disable(); }
+#endif
+
+   // Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   const char *coef_file = "";
+   int order = 1;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&coef_file, "-c", "--coef-file",
+                  "Set the coefficient file name.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(mfem::out);
+      return 1;
+   }
+   args.PrintOptions(mfem::out);
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 10,000 elements.
+   {
+      int ref_levels =
+         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh.UniformRefinement();
+      }
+   }
+
+   H1_FESpace fespace_h1(&mesh, order, mesh.Dimension());
+   ND_FESpace fespace_nd(&mesh, order, mesh.Dimension());
+
+   GridFunction sgf(&fespace_h1);
+   GridFunction vgf(&fespace_nd);
+
+   MyCoefFactory coefFact;
+   coefFact.AddExternalFunction(MyScalarFunc);
+   coefFact.AddExternalFunction(MyVectorFunc);
+
+   istream * iss = NULL;
+   if (strncmp(coef_file,"",1) != 0)
+   {
+      iss = new ifstream(coef_file);
+   }
+   else
+   {
+      iss = new istringstream(coef_str);
+   }
+   skip_comment_lines(*iss, '#');
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream s_sock, v_sock;
+
+   string buff;
+   while (*iss >> buff)
+   {
+      if (buff == "scalar_coef")
+      {
+         Coefficient * sc = coefFact.GetScalarCoef(*iss);
+         sgf.ProjectCoefficient(*sc);
+         if (visualization)
+         {
+            VisualizeField(s_sock, vishost, visport, sgf, "Scalar Coef",
+                           0, 0, 275, 250);
+         }
+      }
+      else if (buff == "vector_coef")
+      {
+         VectorCoefficient * vc = coefFact.GetVectorCoef(*iss);
+         vgf.ProjectCoefficient(*vc);
+         if (visualization)
+         {
+            VisualizeField(v_sock, vishost, visport, vgf, "Vector Coef",
+                           275 + 3, 0, 275, 250);
+         }
+      }
+      skip_comment_lines(*iss, '#');
+
+      char c;
+      cout << "press (q)uit or (c)ontinue --> " << flush;
+      cin >> c;
+
+      if (c != 'c')
+      {
+         break;
+      }
+   }
+
+   return 0;
+}

miniapps/tools/coef-fact.inp

@@ -0,0 +1,53 @@
+##############################################################################
+# For the ConstantCoefficient the argument is the constant value
+##############################################################################
+scalar_coef
+ConstantCoefficient
+3.14
+
+vector_coef
+##############################################################################
+# For the VectorConstantCoefficient the arguments are the dimension of the
+# vector and it components.
+##############################################################################
+VectorConstantCoefficient
+2 2.0 1.0
+
+scalar_coef
+##############################################################################
+# For the FunctionCoefficient the first integer specifies the type of
+# constructor to be used (based on a simple function of position in this case)
+# and the second is the index into an array of simple functions provided
+# in the calling application via CoefFactory::AddExternalFunction.
+##############################################################################
+FunctionCoefficient
+0 0
+
+vector_coef
+##############################################################################
+# For the VectorFunctionCoefficient the first integer argument is the
+# dimension of the vector and the second specifies the type of
+# constructor to be used (based on a simple function of position in this case)
+# and the second is the index into an array of simple vector-valued functions
+# provided in the calling application via CoefFactory::AddExternalFunction.
+##############################################################################
+VectorFunctionCoefficient
+2 0 0
+
+scalar_coef
+##############################################################################
+# Application specific (or user supplied) Coefficients can be added by
+# implementing a Factory class which derives from CoefFactory and recognizes
+# the new Coefficient types.
+##############################################################################
+MyCoefficient
+2 2.0 1.0
+
+vector_coef
+##############################################################################
+# Application specific (or user supplied) VectorCoefficients can be added by
+# implementing a Factory class which derives from CoefFactory and recognizes
+# the new VectorCoefficient types.
+##############################################################################
+MyVectorCoefficient
+2 3.0 2.0 1.0

miniapps/tools/makefile

@@ -25,7 +25,7 @@ include $(DEFAULTS_MK)
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
-SEQ_MINIAPPS = display-basis load-dc convert-dc get-values lor-transfer
+SEQ_MINIAPPS = display-basis load-dc coef-fact convert-dc get-values lor-transfer
 PAR_MINIAPPS =
 ifeq ($(MFEM_USE_MPI),NO)
    MINIAPPS = $(SEQ_MINIAPPS)
@@ -60,6 +60,10 @@ display-basis: %: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK) | lib-common
 	$(MFEM_CXX) $(MFEM_FLAGS) -c $(<)
 	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $@.o $(COMMON_LIB) $(MFEM_LIBS)
 
+coef-fact: %: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK) | lib-common
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $(<)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $@.o $(COMMON_LIB) $(MFEM_LIBS)
+
 # Rule for building lib-common
 lib-common:
 	$(MAKE) -C $(MFEM_BUILD_DIR)/miniapps/common
@@ -75,8 +79,8 @@ RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
 	@$(call mfem-test,$<,, Tools miniapp)
 
 # Testing: Specific execution options
-# Do not test: display-basis, load-dc, convert-dc, get-values, lor-transfer
-NO_TEST_APPS = display-basis load-dc convert-dc get-values lor-transfer
+# Do not test: display-basis, load-dc, coef-fact, convert-dc, get-values, lor-transfer
+NO_TEST_APPS = display-basis load-dc coef-fact convert-dc get-values lor-transfer
 $(foreach app,$(NO_TEST_APPS),$(app)-test-seq $(app)-test-par):
 	@true
 

tests/convergence/makefile

@@ -0,0 +1,62 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/tests/convergence/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_TESTS = rates
+PAR_TESTS = prates
+ifeq ($(MFEM_USE_MPI),NO)
+   TESTS = $(SEQ_TESTS)
+else
+   TESTS = $(PAR_TESTS) $(SEQ_TESTS)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean
+.PRECIOUS: %.o
+
+# Remove built-in rules
+%: %.cpp
+%.o: %.cpp
+
+all: $(TESTS)
+
+# Rules for building the TESTS
+
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+# Rules for compiling miniapp dependencies
+$($(TESTS)): \
+%.o: $(SRC)%.cpp $(SRC)%.hpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $(<) -o $(@)
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(SEQ_TESTS) $(PAR_TESTS)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -f refined.mesh sol.gf
+

tests/convergence/prates.cpp

@@ -0,0 +1,408 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//                     ---------------------------------
+//                     Convergence Rates Test (Parallel)
+//                     ---------------------------------
+//
+// Compile with: make prates
+//
+// Sample runs:  mpirun -np 4 prates -m ../../data/inline-segment.mesh -sr 1 -pr 4 -prob 0 -o 1
+//               mpirun -np 4 prates -m ../../data/inline-quad.mesh -sr 1 -pr 3 -prob 0 -o 2
+//               mpirun -np 4 prates -m ../../data/inline-quad.mesh -sr 1 -pr 3 -prob 1 -o 2
+//               mpirun -np 4 prates -m ../../data/inline-quad.mesh -sr 1 -pr 3 -prob 2 -o 2
+//               mpirun -np 4 prates -m ../../data/inline-tri.mesh -sr 1 -pr 3 -prob 2 -o 3
+//               mpirun -np 4 prates -m ../../data/star.mesh -sr 1 -pr 2 -prob 1 -o 4
+//               mpirun -np 4 prates -m ../../data/fichera.mesh -sr 1 -pr 2 -prob 2 -o 2
+//               mpirun -np 4 prates -m ../../data/inline-wedge.mesh -sr 0 -pr 2 -prob 0 -o 2
+//               mpirun -np 4 prates -m ../../data/inline-hex.mesh -sr 0 -pr 1 -prob 1 -o 3
+//               mpirun -np 4 prates -m ../../data/square-disc.mesh -sr 1 -pr 2 -prob 1 -o 2
+//               mpirun -np 4 prates -m ../../data/star.mesh -sr 1 -pr 2 -prob 3 -o 2
+//               mpirun -np 4 prates -m ../../data/inline-hex.mesh -sr 1 -pr 1 -prob 3 -o 2
+//
+// Description:  This example code demonstrates the use of MFEM to define and
+//               solve finite element problem for various discretizations and
+//               provide convergence rates in parallel.
+//
+//               prob 0: H1 projection:
+//                       (grad u, grad v) + (u,v) = (grad u_exact, grad v) + (u_exact, v)
+//               prob 1: H(curl) projection
+//                       (curl u, curl v) + (u,v) = (curl u_exact, curl v) + (u_exact, v)
+//               prob 2: H(div) projection
+//                       (div  u, div  v) + (u,v) = (div  u_exact, div  v) + (u_exact, v)
+//               prob 3: DG discretization for the Poisson problem
+//                       -Delta u = f
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+using namespace std;
+using namespace mfem;
+
+// Exact solution parameters:
+double sol_s[3] = { -0.32, 0.15, 0.24 };
+double sol_k[3] = { 1.21, 1.45, 1.37 };
+
+// H1
+double scalar_u_exact(const Vector &x);
+double rhs_func(const Vector &x);
+void gradu_exact(const Vector &x, Vector &gradu);
+
+// Vector FE
+void vector_u_exact(const Vector &x, Vector & vector_u);
+// H(curl)
+void curlu_exact(const Vector &x, Vector &curlu);
+// H(div)
+double divu_exact(const Vector &x);
+
+int dim;
+int prob=0;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = 1;
+   int sr = 1;
+   int pr = 1;
+   double sigma = -1.0;
+   double kappa = -1.0;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&prob, "-prob", "--problem",
+                  "Problem kind: 0: H1, 1: H(curl), 2: H(div), 3: DG ");
+   args.AddOption(&sigma, "-s", "--sigma",
+                  "One of the two DG penalty parameters, typically +1/-1."
+                  " See the documentation of class DGDiffusionIntegrator.");
+   args.AddOption(&kappa, "-k", "--kappa",
+                  "One of the two DG penalty parameters, should be positive."
+                  " Negative values are replaced with (order+1)^2.");
+   args.AddOption(&sr, "-sr", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&pr, "-pr", "--parallel_ref",
+                  "Number of parallel refinements.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (prob >3 || prob <0) prob = 0; // default problem = H1
+   if (prob == 3)
+   {
+      if (kappa < 0)
+      {
+         kappa = (order+1)*(order+1);
+      }
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution.
+   for (int i = 0; i < sr; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   // 6. Define a parallel finite element space on the parallel mesh.
+   FiniteElementCollection *fec=nullptr;
+   switch (prob)
+   {
+      case 0: fec = new H1_FECollection(order,dim);   break;
+      case 1: fec = new ND_FECollection(order,dim);   break;
+      case 2: fec = new RT_FECollection(order-1,dim); break;
+      case 3: fec = new DG_FECollection(order,dim); break;
+      default: break;
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+
+   // 7. Define the solution vector x as a parallel finite element grid function
+   //     corresponding to fespace.
+   ParGridFunction x(fespace);
+   x = 0.0;
+
+   // 8. Set up the parallel linear form b(.) and the parallel bilinear form
+   //    a(.,.).
+   FunctionCoefficient *f=nullptr;
+   FunctionCoefficient *scalar_u=nullptr;
+   FunctionCoefficient *divu=nullptr;
+   VectorFunctionCoefficient *vector_u=nullptr;
+   VectorFunctionCoefficient *gradu=nullptr;
+   VectorFunctionCoefficient *curlu=nullptr;
+
+   ConstantCoefficient one(1.0);
+   ParLinearForm b(fespace);
+   ParBilinearForm a(fespace);
+
+   switch (prob)
+   {
+      case 0:
+         //(grad u_ex, grad v) + (u_ex,v)
+         scalar_u = new FunctionCoefficient(scalar_u_exact);
+         gradu = new VectorFunctionCoefficient(dim,gradu_exact);
+         b.AddDomainIntegrator(new DomainLFGradIntegrator(*gradu));
+         b.AddDomainIntegrator(new DomainLFIntegrator(*scalar_u));
+
+         // (grad u, grad v) + (u,v)
+         a.AddDomainIntegrator(new DiffusionIntegrator(one));
+         a.AddDomainIntegrator(new MassIntegrator(one));
+
+         break;
+      case 1:
+         //(curl u_ex, curl v) + (u_ex,v)
+         vector_u = new VectorFunctionCoefficient(dim,vector_u_exact);
+         curlu = new VectorFunctionCoefficient((dim==3)?dim:1,curlu_exact);
+         b.AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(*curlu));
+         b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(*vector_u));
+
+         // (curl u, curl v) + (u,v)
+         a.AddDomainIntegrator(new CurlCurlIntegrator(one));
+         a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
+         break;
+
+      case 2:
+         //(div u_ex, div v) + (u_ex,v)
+         vector_u = new VectorFunctionCoefficient(dim,vector_u_exact);
+         divu = new FunctionCoefficient(divu_exact);
+         b.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(*divu));
+         b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(*vector_u));
+
+         // (div u, div v) + (u,v)
+         a.AddDomainIntegrator(new DivDivIntegrator(one));
+         a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
+         break;
+
+      case 3:
+         scalar_u = new FunctionCoefficient(scalar_u_exact);
+         f = new FunctionCoefficient(rhs_func);
+         gradu = new VectorFunctionCoefficient(dim,gradu_exact);
+         b.AddDomainIntegrator(new DomainLFIntegrator(*f));
+         b.AddBdrFaceIntegrator(
+      new DGDirichletLFIntegrator(*scalar_u, one, sigma, kappa));
+         a.AddDomainIntegrator(new DiffusionIntegrator(one));
+         a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+         a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+         break;
+
+      default:
+         break;
+   }
+
+   // 9. Perform successive parallel refinements, compute the L2 error and the
+   //    corresponding rate of convergence.
+   ConvergenceStudy rates;
+   for (int l = 0; l <= pr; l++)
+   {
+      b.Assemble();
+      a.Assemble();
+      a.Finalize();
+
+      HypreParMatrix *A = a.ParallelAssemble();
+      HypreParVector *B = b.ParallelAssemble();
+      HypreParVector *X = x.ParallelProject();
+
+      Solver *prec = nullptr;
+      IterativeSolver *solver = nullptr;
+      switch (prob)
+      {
+         case 0:
+         case 3:
+            prec = new HypreBoomerAMG(*A);
+            dynamic_cast<HypreBoomerAMG *>(prec)->SetPrintLevel(0);
+            break;
+         case 1:
+            prec = new HypreAMS(*A, fespace);
+            dynamic_cast<HypreAMS *>(prec)->SetPrintLevel(0);
+            break;
+         case 2:
+            if (dim == 2)
+            {
+               prec = new HypreAMS(*A, fespace);
+               dynamic_cast<HypreAMS *>(prec)->SetPrintLevel(0);
+            }
+            else
+            {
+               prec = new HypreADS(*A, fespace);
+               dynamic_cast<HypreADS *>(prec)->SetPrintLevel(0);
+            }
+            break;
+         default:
+            break;
+      }
+      if (prob==3 && sigma !=-1.0)
+      {
+         solver = new GMRESSolver(MPI_COMM_WORLD);
+      }
+      else
+      {
+         solver = new CGSolver(MPI_COMM_WORLD);
+      }
+      solver->SetRelTol(1e-12);
+      solver->SetMaxIter(2000);
+      solver->SetPrintLevel(0);
+      solver->SetPreconditioner(*prec);
+      solver->SetOperator(*A);
+      solver->Mult(*B, *X);
+      delete prec;
+      delete solver;
+
+      x = *X;
+      switch (prob)
+      {
+         case 0: rates.AddH1GridFunction(&x,scalar_u,gradu); break;
+         case 1: rates.AddHcurlGridFunction(&x,vector_u,curlu); break;
+         case 2: rates.AddHdivGridFunction(&x,vector_u,divu);  break;
+         case 3: rates.AddL2GridFunction(&x,scalar_u,gradu,&one); break;
+      }
+
+      delete X;
+      delete B;
+      delete A;
+
+      if (l==pr) break;
+
+      pmesh->UniformRefinement();
+      fespace->Update();
+      a.Update();
+      b.Update();
+      x.Update();
+   }
+   rates.Print();
+
+   // 10. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x <<
+               "window_title 'Numerical Pressure (real part)' "
+               << flush;
+   }
+
+   // 10. Free the used memory.
+   delete scalar_u;
+   delete divu;
+   delete vector_u;
+   delete gradu;
+   delete curlu;
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+double rhs_func(const Vector &x)
+{
+   double val = 1.0, lap = 0.0;
+   for (int d = 0; d < x.Size(); d++)
+   {
+      const double f = sin(M_PI*(sol_s[d]+sol_k[d]*x(d)));
+      val *= f;
+      lap = lap*f + val*M_PI*M_PI*sol_k[d]*sol_k[d];
+   }
+   return lap;
+}
+
+double scalar_u_exact(const Vector &x)
+{
+   double val = 1.0;
+   for (int d = 0; d < x.Size(); d++)
+   {
+      val *= sin(M_PI*(sol_s[d]+sol_k[d]*x(d)));
+   }
+   return val;
+}
+
+void gradu_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   double *g = grad.GetData();
+   double val = 1.0;
+   for (int d = 0; d < x.Size(); d++)
+   {
+      const double y = M_PI*(sol_s[d]+sol_k[d]*x(d));
+      const double f = sin(y);
+      for (int j = 0; j < d; j++) { g[j] *= f; }
+      g[d] = val*M_PI*sol_k[d]*cos(y);
+      val *= f;
+   }
+}
+
+void vector_u_exact(const Vector &x, Vector & vector_u)
+{
+   vector_u.SetSize(x.Size());
+   vector_u=0.0;
+   vector_u[0] = scalar_u_exact(x);
+}
+
+// H(curl)
+void curlu_exact(const Vector &x, Vector &curlu)
+{
+   Vector grad;
+   gradu_exact(x,grad);
+   int n = (x.Size()==3)?3:1;
+   curlu.SetSize(n);
+   if (x.Size()==3)
+   {
+      curlu[0] = 0.0;
+      curlu[1] = grad[2];
+      curlu[2] = -grad[1];
+   }
+   else if (x.Size()==2)
+   {
+      curlu[0] = -grad[1];
+   }
+}
+
+// H(div)
+double divu_exact(const Vector &x)
+{
+   Vector grad;
+   gradu_exact(x,grad);
+
+   return grad[0];
+}

tests/convergence/rates.cpp

@@ -0,0 +1,334 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//                      -------------------------------
+//                      Convergence Rates Test (Serial)
+//                      -------------------------------
+//
+// Compile with: make rates
+//
+// Sample runs:  rates -m ../../data/inline-segment.mesh -sr 4 -prob 0 -o 1
+//               rates -m ../../data/inline-quad.mesh -sr 3 -prob 0 -o 2
+//               rates -m ../../data/inline-quad.mesh -sr 3 -prob 1 -o 2
+//               rates -m ../../data/inline-quad.mesh -sr 3 -prob 2 -o 2
+//               rates -m ../../data/inline-tri.mesh -sr 2 -prob 2 -o 3
+//               rates -m ../../data/star.mesh -sr 2 -prob 1 -o 4
+//               rates -m ../../data/fichera.mesh -sr 3 -prob 2 -o 1
+//               rates -m ../../data/inline-wedge.mesh -sr 1 -prob 0 -o 2
+//               rates -m ../../data/inline-hex.mesh -sr 1 -prob 1 -o 2
+//               rates -m ../../data/square-disc.mesh -sr 2 -prob 1 -o 1
+//               rates -m ../../data/star.mesh -sr 2 -prob 3 -o 2
+//               rates -m ../../data/inline-hex.mesh -sr 1 -prob 3 -o 1
+//
+// Description:  This example code demonstrates the use of MFEM to define and
+//               solve finite element problem for various discretizations and
+//               provide convergence rates in serial.
+//
+//               prob 0: H1 projection:
+//                       (grad u, grad v) + (u,v) = (grad u_exact, grad v) + (u_exact, v)
+//               prob 1: H(curl) projection
+//                       (curl u, curl v) + (u,v) = (curl u_exact, curl v) + (u_exact, v)
+//               prob 2: H(div) projection
+//                       (div  u, div  v) + (u,v) = (div  u_exact, div  v) + (u_exact, v)
+//               prob 3: DG discretization for the Poisson problem
+//                       -Delta u = f
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+using namespace std;
+using namespace mfem;
+
+// Exact solution parameters:
+double sol_s[3] = { -0.32, 0.15, 0.24 };
+double sol_k[3] = { 1.21, 1.45, 1.37 };
+
+// H1
+double scalar_u_exact(const Vector &x);
+double rhs_func(const Vector &x);
+void gradu_exact(const Vector &x, Vector &gradu);
+
+// Vector FE
+void vector_u_exact(const Vector &x, Vector & vector_u);
+// H(curl)
+void curlu_exact(const Vector &x, Vector &curlu);
+// H(div)
+double divu_exact(const Vector &x);
+
+int dim;
+int prob=0;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = 1;
+   int sr = 1;
+   double sigma = -1.0;
+   double kappa = -1.0;
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&prob, "-prob", "--problem",
+                  "Problem kind: 0: H1, 1: H(curl), 2: H(div), 3: DG ");
+   args.AddOption(&sigma, "-s", "--sigma",
+                  "One of the two DG penalty parameters, typically +1/-1."
+                  " See the documentation of class DGDiffusionIntegrator.");
+   args.AddOption(&kappa, "-k", "--kappa",
+                  "One of the two DG penalty parameters, should be positive."
+                  " Negative values are replaced with (order+1)^2.");
+   args.AddOption(&sr, "-sr", "--serial_ref",
+                  "Number of serial refinements.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   if (prob >3 || prob <0) prob = 0; // default problem = H1
+   if (prob == 3)
+   {
+      if (kappa < 0)
+      {
+         kappa = (order+1)*(order+1);
+      }
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the (serial) mesh from the given mesh file.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+
+   // 3. Refine the serial mesh on all processors to increase the resolution.
+   mesh->UniformRefinement();
+
+   // 4. Define a finite element space on the parallel mesh.
+   FiniteElementCollection *fec=nullptr;
+   switch (prob)
+   {
+      case 0: fec = new H1_FECollection(order,dim);   break;
+      case 1: fec = new ND_FECollection(order,dim);   break;
+      case 2: fec = new RT_FECollection(order-1,dim); break;
+      case 3: fec = new DG_FECollection(order,dim); break;
+      default: break;
+   }
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+
+   // 5. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace.
+   GridFunction x(fespace);
+   x = 0.0;
+
+   // 6. Set up the linear form b(.) and the bilinear form a(.,.).
+   FunctionCoefficient *f=nullptr;
+   FunctionCoefficient *scalar_u=nullptr;
+   FunctionCoefficient *divu=nullptr;
+   VectorFunctionCoefficient *vector_u=nullptr;
+   VectorFunctionCoefficient *gradu=nullptr;
+   VectorFunctionCoefficient *curlu=nullptr;
+
+   ConstantCoefficient one(1.0);
+   LinearForm b(fespace);
+   BilinearForm a(fespace);
+
+   switch (prob)
+   {
+      case 0:
+         //(grad u_ex, grad v) + (u_ex,v)
+         scalar_u = new FunctionCoefficient(scalar_u_exact);
+         gradu = new VectorFunctionCoefficient(dim,gradu_exact);
+         b.AddDomainIntegrator(new DomainLFGradIntegrator(*gradu));
+         b.AddDomainIntegrator(new DomainLFIntegrator(*scalar_u));
+
+         // (grad u, grad v) + (u,v)
+         a.AddDomainIntegrator(new DiffusionIntegrator(one));
+         a.AddDomainIntegrator(new MassIntegrator(one));
+
+         break;
+      case 1:
+         //(curl u_ex, curl v) + (u_ex,v)
+         vector_u = new VectorFunctionCoefficient(dim,vector_u_exact);
+         curlu = new VectorFunctionCoefficient((dim==3)?dim:1,curlu_exact);
+         b.AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(*curlu));
+         b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(*vector_u));
+
+         // (curl u, curl v) + (u,v)
+         a.AddDomainIntegrator(new CurlCurlIntegrator(one));
+         a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
+         break;
+
+      case 2:
+         //(div u_ex, div v) + (u_ex,v)
+         vector_u = new VectorFunctionCoefficient(dim,vector_u_exact);
+         divu = new FunctionCoefficient(divu_exact);
+         b.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(*divu));
+         b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(*vector_u));
+
+         // (div u, div v) + (u,v)
+         a.AddDomainIntegrator(new DivDivIntegrator(one));
+         a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
+         break;
+
+      case 3:
+         scalar_u = new FunctionCoefficient(scalar_u_exact);
+         f = new FunctionCoefficient(rhs_func);
+         gradu = new VectorFunctionCoefficient(dim,gradu_exact);
+         b.AddDomainIntegrator(new DomainLFIntegrator(*f));
+         b.AddBdrFaceIntegrator(
+      new DGDirichletLFIntegrator(*scalar_u, one, sigma, kappa));
+         a.AddDomainIntegrator(new DiffusionIntegrator(one));
+         a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+         a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+         break;
+
+      default:
+         break;
+   }
+
+   // 7. Perform successive refinements, compute the errors and the
+   //    corresponding rates of convergence.
+   ConvergenceStudy rates;
+   for (int l = 0; l <= sr; l++)
+   {
+      b.Assemble();
+      a.Assemble();
+      a.Finalize();
+      const SparseMatrix &A = a.SpMat();
+      GSSmoother M(A);
+      if (prob == 3 && sigma != -1.0)
+      {
+         GMRES(A, M, b, x, 0, 500, 10, 1e-12, 0.0);
+      }
+      else
+      {
+         PCG(A, M, b, x, 0, 500, 1e-12, 0.0);
+      }
+
+      switch (prob)
+      {
+         case 0: rates.AddH1GridFunction(&x,scalar_u,gradu); break;
+         case 1: rates.AddHcurlGridFunction(&x,vector_u,curlu); break;
+         case 2: rates.AddHdivGridFunction(&x,vector_u,divu);  break;
+         case 3: rates.AddL2GridFunction(&x,scalar_u,gradu,&one); break;
+      }
+
+      if (l==sr) break;
+
+      mesh->UniformRefinement();
+      fespace->Update();
+      a.Update();
+      b.Update();
+      x.Update();
+   }
+   rates.Print();
+
+   // 8. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *mesh << x <<
+               "window_title 'Numerical Pressure (real part)' "
+               << flush;
+   }
+
+   // 9. Free the used memory.
+   delete f;
+   delete scalar_u;
+   delete divu;
+   delete vector_u;
+   delete gradu;
+   delete curlu;
+   delete fespace;
+   delete fec;
+   delete mesh;
+   return 0;
+}
+
+double rhs_func(const Vector &x)
+{
+   double val = 1.0, lap = 0.0;
+   for (int d = 0; d < x.Size(); d++)
+   {
+      const double f = sin(M_PI*(sol_s[d]+sol_k[d]*x(d)));
+      val *= f;
+      lap = lap*f + val*M_PI*M_PI*sol_k[d]*sol_k[d];
+   }
+   return lap;
+}
+
+double scalar_u_exact(const Vector &x)
+{
+   double val = 1.0;
+   for (int d = 0; d < x.Size(); d++)
+   {
+      val *= sin(M_PI*(sol_s[d]+sol_k[d]*x(d)));
+   }
+   return val;
+}
+
+void gradu_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   double *g = grad.GetData();
+   double val = 1.0;
+   for (int d = 0; d < x.Size(); d++)
+   {
+      const double y = M_PI*(sol_s[d]+sol_k[d]*x(d));
+      const double f = sin(y);
+      for (int j = 0; j < d; j++) { g[j] *= f; }
+      g[d] = val*M_PI*sol_k[d]*cos(y);
+      val *= f;
+   }
+}
+
+void vector_u_exact(const Vector &x, Vector & vector_u)
+{
+   vector_u.SetSize(x.Size());
+   vector_u=0.0;
+   vector_u[0] = scalar_u_exact(x);
+}
+
+// H(curl)
+void curlu_exact(const Vector &x, Vector &curlu)
+{
+   Vector grad;
+   gradu_exact(x,grad);
+   int n = (x.Size()==3)?3:1;
+   curlu.SetSize(n);
+   if (x.Size()==3)
+   {
+      curlu[0] = 0.0;
+      curlu[1] = grad[2];
+      curlu[2] = -grad[1];
+   }
+   else if (x.Size()==2)
+   {
+      curlu[0] = -grad[1];
+   }
+}
+
+// H(div)
+double divu_exact(const Vector &x)
+{
+   Vector grad;
+   gradu_exact(x,grad);
+
+   return grad[0];
+}

tests/par-mesh-format/ex1p.cpp

@@ -0,0 +1,269 @@
+//            MFEM test - mesh I/O using the parallel mesh format
+//
+// Compile with: make ex1p
+//
+// Sample runs:
+//
+//    The following sample runs alternate between the following two modes:
+//
+//    1. (serial mode) Read a serial mesh, refine it (before and after parallel
+//       partitioning), solve a diffusion problem, and write the result, mesh
+//       and solution, using a VisItDataCollection using the parallel format for
+//       the mesh.
+//    2. (parallel mode) Read a VisItDataCollection saved using the parallel
+//       mesh format, then solve the same diffusion problem as above and compare
+//       the result to the saved solution.
+//
+//    (This sequence is used to support testing with the script sample-runs.sh)
+//
+//       mpirun -np 4 ex1p -m ../../data/star.mesh
+//       mpirun -np 4 ex1p
+//       mpirun -np 4 ex1p -m ../../data/square-disc.mesh
+//       mpirun -np 4 ex1p
+//       mpirun -np 4 ex1p -m ../../data/star-mixed.mesh
+//       mpirun -np 4 ex1p
+//       mpirun -np 4 ex1p -m ../../data/escher.mesh
+//       mpirun -np 4 ex1p
+//       mpirun -np 4 ex1p -m ../../data/fichera.mesh
+//       mpirun -np 4 ex1p
+//       mpirun -np 4 ex1p -m ../../data/fichera-mixed.mesh
+//       mpirun -np 4 ex1p
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *not_set = "(not set)";
+   const char *mesh_file = not_set;
+   const char *coll_name = "ex1p-dc";
+   int order = 1;
+   int serial_ref_levels = 1;
+   int parallel_ref_levels = 2;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&coll_name, "-n", "--collection-name",
+                  "Set the data collection name to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&serial_ref_levels, "-rs", "--refine-serial",
+                  "Number of uniform refinements of the mesh before parallel"
+                  " decomposition.");
+   args.AddOption(&parallel_ref_levels, "-rp", "--refine-parallel",
+                  "Number of uniform refinements to perform after parallel"
+                  " decomposition.\n\t" "This is the number of levels used for"
+                  " the convergence study.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   const bool serial_mode = (mesh_file != not_set);
+
+   ParMesh *pmesh;
+   VisItDataCollection visit_dc(MPI_COMM_WORLD, coll_name);
+
+   if (serial_mode)
+   {
+      // 3. Read the serial mesh on all processors, refine it in serial, then
+      //    partition it across all processors and refine it in parallel.
+      Mesh *mesh = new Mesh(mesh_file, 1, 1);
+
+      for (int l = 0; l < serial_ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+
+      pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+      delete mesh;
+
+      for (int l = 0; l < parallel_ref_levels; l++)
+      {
+         pmesh->UniformRefinement();
+      }
+      visit_dc.SetMesh(pmesh);
+   }
+   else
+   {
+      // 4. Read the given data collection.
+      visit_dc.Load();
+      if (visit_dc.Error())
+      {
+         if (myid == 0)
+         {
+            cout << "Error loading data collection: " << coll_name << endl;
+         }
+         return 1;
+      }
+      pmesh = dynamic_cast<ParMesh*>(visit_dc.GetMesh());
+      if (pmesh == NULL)
+      {
+         if (myid == 0)
+         {
+            cout << "The given data collection does not have a parallel mesh."
+                 << endl;
+         }
+         return 2;
+      }
+   }
+   int dim = pmesh->Dimension();
+
+   // 5. Solve a simple diffusion problem on the parallel mesh.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+      if (myid == 0)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ParLinearForm *b = new ParLinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   ParGridFunction x(fespace);
+   x = 0.0;
+
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   a->Assemble();
+
+   HypreParMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+   }
+
+   HypreBoomerAMG *amg = new HypreBoomerAMG;
+   amg->SetPrintLevel(0);
+   amg->SetOperator(A);
+   HyprePCG *pcg = new HyprePCG(A);
+   pcg->SetTol(1e-12);
+   pcg->SetMaxIter(200);
+   pcg->SetPrintLevel(5);
+   pcg->SetPreconditioner(*amg);
+   pcg->Mult(B, X);
+
+   a->RecoverFEMSolution(X, *b, x);
+
+   if (serial_mode)
+   {
+      // 6. Save the parallel mesh and the solution using the data collection.
+      visit_dc.RegisterField("temperature", &x);
+      visit_dc.SetFormat(DataCollection::PARALLEL_FORMAT);
+      visit_dc.SetPrecision(16);
+      if (myid == 0)
+      {
+         cout << "\nSaving data collection '" << coll_name << "' ..." << flush;
+      }
+      visit_dc.Save();
+      if (myid == 0)
+      {
+         cout << " done.\n" << endl;
+      }
+   }
+   else
+   {
+      ParGridFunction *saved_x = visit_dc.GetParField("temperature");
+      if (!saved_x)
+      {
+         if (myid == 0)
+         {
+            cout << "The given data collection has no 'temperature' field."
+                 << endl;
+         }
+      }
+      else
+      {
+         ParGridFunction err(fespace);
+         subtract(x, *saved_x, err);
+         ConstantCoefficient zero(0.0);
+         double err_norm = err.ComputeL2Error(zero);
+         if (myid == 0)
+         {
+            cout << "\n|| x - x_saved ||_L2 = " << err_norm << '\n' << endl;
+         }
+      }
+   }
+
+   // 7. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x << flush;
+   }
+
+   // 8. Free the used memory.
+   delete pcg;
+   delete amg;
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   if (serial_mode) { delete pmesh; }
+
+   MPI_Finalize();
+
+   return 0;
+}

tests/par-mesh-format/makefile

@@ -0,0 +1,54 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/tests/par-mesh-format/,)
+CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
+   $(wildcard $(MFEM_BUILD_DIR)/share/mfem/config.mk))
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_TESTS =
+PAR_TESTS = ex1p
+ifeq ($(MFEM_USE_MPI),NO)
+   TESTS = $(SEQ_TESTS)
+else
+   TESTS = $(PAR_TESTS) $(SEQ_TESTS)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(TESTS)
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	$(RM) $(SEQ_TESTS) $(PAR_TESTS)
+	$(RM) -r *.dSYM
+
+clean-exec:
+	@$(RM) -r ex1p-dc*

tests/unit/CMakeLists.txt

@@ -67,6 +67,18 @@ add_executable(unit_tests unit_test_main.cpp ${UNIT_TESTS_SRCS})
 add_dependencies(unit_tests copy_data)
 target_link_libraries(unit_tests mfem)
 
+if (MFEM_USE_CUDA)
+  set(CUNIT_TESTS_SRCS
+    cunit_test_main.cpp
+  )
+  set_property(SOURCE ${CUNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
+  add_executable(cunit_tests ${CUNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS})
+  add_dependencies(cunit_tests copy_data)
+  target_link_libraries(cunit_tests mfem)
+
+  add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} cunit_tests)
+endif()
+
 # All device unit tests are built into another executable, in order to be able
 # to change the device.
 set(SEDOV_TESTS_SRCS
@@ -122,6 +134,7 @@ add_test(NAME sedov_tests_debug COMMAND sedov_tests_debug)
 
 # Additional CUDA unit tests
 if (MFEM_USE_CUDA)
+   add_test(NAME cunit_tests COMMAND cunit_tests)
    add_test(NAME sedov_tests_cuda COMMAND sedov_tests_cuda)
    add_test(NAME sedov_tests_cuda_uvm COMMAND sedov_tests_cuda_uvm)
 endif()

tests/unit/cunit_test_main.cpp

@@ -0,0 +1,45 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#define CATCH_CONFIG_RUNNER
+#include "mfem.hpp"
+#include "catch.hpp"
+
+int main(int argc, char *argv[])
+{
+   mfem::Device device("cuda");
+
+   // There must be exactly one instance.
+   Catch::Session session;
+
+   // Apply provided command line arguments.
+   int r = session.applyCommandLine(argc, argv);
+   if (r != 0)
+   {
+      return r;
+   }
+
+   auto cfg = session.configData();
+
+   cfg.testsOrTags.push_back("[CUDA]");
+
+#ifdef MFEM_USE_MPI
+   // Exclude tests marked as Parallel in a serial run, even when compiled with
+   // MPI. This is done because there is no MPI session initialized.
+   cfg.testsOrTags.push_back("~[Parallel]");
+#endif
+
+   session.useConfigData(cfg);
+
+   int result = session.run();
+
+   return result;
+}

tests/unit/fem/test_1d_bilininteg.cpp

@@ -10,7 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "mfem.hpp"
-#include "catch.hpp"
+#include "unit_tests.hpp"
 
 using namespace mfem;
 

tests/unit/fem/test_2d_bilininteg.cpp

@@ -10,7 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "mfem.hpp"
-#include "catch.hpp"
+#include "unit_tests.hpp"
 
 using namespace mfem;
 

tests/unit/fem/test_3d_bilininteg.cpp

@@ -10,7 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "mfem.hpp"
-#include "catch.hpp"
+#include "unit_tests.hpp"
 
 using namespace mfem;
 

tests/unit/fem/test_assemblediagonalpa.cpp

@@ -10,7 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "mfem.hpp"
-#include "catch.hpp"
+#include "unit_tests.hpp"
 
 using namespace mfem;
 
@@ -256,19 +256,19 @@ TEST_CASE("Vector Mass Diagonal PA", "[PartialAssembly], [AssembleDiagonal]")
    SECTION("2D")
    {
       REQUIRE(test_vdiagpa<VectorMassIntegrator>(2,
-                                                 2) == Approx(0.0));
+                                                 2) == MFEM_Approx(0.0));
 
       REQUIRE(test_vdiagpa<VectorMassIntegrator>(2,
-                                                 3) == Approx(0.0));
+                                                 3) == MFEM_Approx(0.0));
    }
 
    SECTION("3D")
    {
       REQUIRE(test_vdiagpa<VectorMassIntegrator>(3,
-                                                 2) == Approx(0.0));
+                                                 2) == MFEM_Approx(0.0));
 
       REQUIRE(test_vdiagpa<VectorMassIntegrator>(3,
-                                                 3) == Approx(0.0));
+                                                 3) == MFEM_Approx(0.0));
    }
 }
 
@@ -279,30 +279,30 @@ TEST_CASE("Vector Diffusion Diagonal PA",
    {
       REQUIRE(
          test_vdiagpa<VectorDiffusionIntegrator>(2,
-                                                 2) == Approx(0.0));
+                                                 2) == MFEM_Approx(0.0));
 
       REQUIRE(test_vdiagpa<VectorDiffusionIntegrator>(2,
-                                                      3) == Approx(0.0));
+                                                      3) == MFEM_Approx(0.0));
    }
 
    SECTION("3D")
    {
       REQUIRE(test_vdiagpa<VectorDiffusionIntegrator>(3,
-                                                      2) == Approx(0.0));
+                                                      2) == MFEM_Approx(0.0));
 
       REQUIRE(test_vdiagpa<VectorDiffusionIntegrator>(3,
-                                                      3) == Approx(0.0));
+                                                      3) == MFEM_Approx(0.0));
    }
 }
 
-TEST_CASE("Hcurl/Hdiv diagonal PA")
+TEST_CASE("Hcurl/Hdiv diagonal PA",
+          "[CUDA]")
 {
    for (dimension = 2; dimension < 4; ++dimension)
    {
       for (int coeffType = 0; coeffType < 5; ++coeffType)
       {
          const int numSpaces = (coeffType == 0) ? 2 : 1;
-         const int numIntegrators = (coeffType == 0) ? 2 : 1;
 
          Coefficient* coeff = nullptr;
          VectorCoefficient* vcoeff = nullptr;
@@ -335,13 +335,16 @@ TEST_CASE("Hcurl/Hdiv diagonal PA")
                                                     &asymmetricMatrixCoeffFunction);
          }
 
+         enum Spaces {Hcurl, Hdiv};
+
          for (int spaceType = 0; spaceType < numSpaces; ++spaceType)
          {
+            const int numIntegrators = (dimension == 3 || coeffType < 2) ? 2 : 1;
             for (int integrator = 0; integrator < numIntegrators; ++integrator)
             {
                for (int ne = 1; ne < 3; ++ne)
                {
-                  if (spaceType == 0)
+                  if (spaceType == Hcurl)
                      std::cout << "Testing " << dimension <<
                                "D partial assembly H(curl) diagonal for integrator " << integrator
                                << " and coeffType " << coeffType << ": "
@@ -364,7 +367,7 @@ TEST_CASE("Hcurl/Hdiv diagonal PA")
                         mesh = new Mesh(ne, ne, ne, Element::HEXAHEDRON, 1, 1.0, 1.0, 1.0);
                      }
 
-                     FiniteElementCollection* fec = (spaceType == 0) ?
+                     FiniteElementCollection* fec = (spaceType == Hcurl) ?
                                                     (FiniteElementCollection*) new ND_FECollection(order, dimension) :
                                                     (FiniteElementCollection*) new RT_FECollection(order, dimension);
 
@@ -392,10 +395,27 @@ TEST_CASE("Hcurl/Hdiv diagonal PA")
                      }
                      else
                      {
-                        if (spaceType == 0)
+                        if (spaceType == Hcurl)
                         {
-                           paform.AddDomainIntegrator(new CurlCurlIntegrator(*coeff));
-                           faform.AddDomainIntegrator(new CurlCurlIntegrator(*coeff));
+                           const FiniteElement *fel = fespace.GetFE(0);
+                           const IntegrationRule *intRule = &MassIntegrator::GetRule(*fel, *fel,
+                                                                                     *mesh->GetElementTransformation(0));
+
+                           if (coeffType >= 3)
+                           {
+                              paform.AddDomainIntegrator(new CurlCurlIntegrator(*smcoeff, intRule));
+                              faform.AddDomainIntegrator(new CurlCurlIntegrator(*mcoeff, intRule));
+                           }
+                           else if (coeffType == 2)
+                           {
+                              paform.AddDomainIntegrator(new CurlCurlIntegrator(*vcoeff, intRule));
+                              faform.AddDomainIntegrator(new CurlCurlIntegrator(*vcoeff, intRule));
+                           }
+                           else
+                           {
+                              paform.AddDomainIntegrator(new CurlCurlIntegrator(*coeff, intRule));
+                              faform.AddDomainIntegrator(new CurlCurlIntegrator(*coeff, intRule));
+                           }
                         }
                         else
                         {

tests/unit/fem/test_assembly_levels.cpp

@@ -9,7 +9,7 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "catch.hpp"
+#include "unit_tests.hpp"
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
@@ -44,9 +44,12 @@ void AddConvectionIntegrators(BilinearForm &k, VectorCoefficient &velocity,
    }
 }
 
-void test_assembly_level(Mesh &&mesh, int order, bool dg, const int pb,
+void test_assembly_level(const char *meshname, int order, bool dg, const int pb,
                          const AssemblyLevel assembly)
 {
+   INFO("mesh=" << meshname << ", order=" << order << ", DG=" << dg
+        << ", pb=" << pb << ", assembly=" << int(assembly));
+   Mesh mesh(meshname, 1, 1);
    mesh.EnsureNodes();
    int dim = mesh.Dimension();
 
@@ -104,109 +107,43 @@ void test_assembly_level(Mesh &&mesh, int order, bool dg, const int pb,
    delete fec;
 }
 
-TEST_CASE("Assembly Levels", "[AssemblyLevel]")
+TEST_CASE("Assembly Levels", "[AssemblyLevel], [PartialAssembly]")
 {
-   SECTION("Continuous Galerkin")
+   auto assembly = GENERATE(AssemblyLevel::PARTIAL, AssemblyLevel::ELEMENT,
+                            AssemblyLevel::FULL);
+   auto pb = GENERATE(0, 1, 2);
+   auto dg = GENERATE(true, false);
+   auto order_2d = GENERATE(2, 3, 4);
+   auto order_3d = GENERATE(2);
+
+   SECTION("2D")
    {
-      const bool dg = false;
-      SECTION("2D")
-      {
-         for (AssemblyLevel assembly : {AssemblyLevel::PARTIAL,AssemblyLevel::ELEMENT,AssemblyLevel::FULL})
-         {
-            for (int pb : {0, 1, 2})
-            {
-               for (int order : {2, 3, 4})
-               {
-                  test_assembly_level(Mesh("../../data/inline-quad.mesh", 1, 1),
-                                      order, dg, pb, assembly);
-                  test_assembly_level(Mesh("../../data/periodic-hexagon.mesh", 1, 1),
-                                      order, dg, pb, assembly);
-                  test_assembly_level(Mesh("../../data/star-q3.mesh", 1, 1),
-                                      order, dg, pb, assembly);
-               }
-            }
-         }
-      }
-      SECTION("3D")
-      {
-         for (AssemblyLevel assembly : {AssemblyLevel::PARTIAL,AssemblyLevel::ELEMENT,AssemblyLevel::FULL})
-         {
-            for (int pb : {0, 1, 2})
-            {
-               int order = 2;
-               test_assembly_level(Mesh("../../data/inline-hex.mesh", 1, 1),
-                                   order, dg, pb, assembly);
-               test_assembly_level(Mesh("../../data/fichera-q3.mesh", 1, 1),
-                                   order, dg, pb, assembly);
-            }
-         }
-      }
-      SECTION("AMR 2D")
-      {
-         for (AssemblyLevel assembly : {AssemblyLevel::PARTIAL,AssemblyLevel::ELEMENT,AssemblyLevel::FULL})
-         {
-            for (int pb : {0, 1, 2})
-            {
-               for (int order : {2, 3, 4})
-               {
-                  test_assembly_level(Mesh("../../data/amr-quad.mesh", 1, 1),
-                                      order, false, 0, assembly);
-               }
-            }
-         }
-      }
-      SECTION("AMR 3D")
-      {
-         for (AssemblyLevel assembly : {AssemblyLevel::PARTIAL,AssemblyLevel::ELEMENT,AssemblyLevel::FULL})
-         {
-            for (int pb : {0, 1, 2})
-            {
-               int order = 2;
-               test_assembly_level(Mesh("../../data/fichera-amr.mesh", 1, 1),
-                                   order, false, 0, assembly);
-            }
-         }
-      }
+      test_assembly_level("../../data/periodic-square.mesh",
+                          order_2d, dg, pb, assembly);
+      test_assembly_level("../../data/periodic-hexagon.mesh",
+                          order_2d, dg, pb, assembly);
+      test_assembly_level("../../data/star-q3.mesh",
+                          order_2d, dg, pb, assembly);
    }
 
-   SECTION("Discontinuous Galerkin")
+   SECTION("3D")
    {
-      const bool dg = true;
-      SECTION("2D")
-      {
-         for (AssemblyLevel assembly : {AssemblyLevel::PARTIAL,AssemblyLevel::ELEMENT,AssemblyLevel::FULL})
-         {
-            for (int pb : {0, 1, 2})
-            {
-               for (int order : {2, 3, 4})
-               {
-                  test_assembly_level(Mesh("../../data/periodic-square.mesh", 1, 1),
-                                      order, dg, pb, assembly);
-                  test_assembly_level(Mesh("../../data/periodic-hexagon.mesh", 1, 1),
-                                      order, dg, pb, assembly);
-                  test_assembly_level(Mesh("../../data/star-q3.mesh", 1, 1),
-                                      order, dg, pb, assembly);
-               }
-            }
-         }
-      }
-      SECTION("3D")
-      {
-         for (AssemblyLevel assembly : {AssemblyLevel::PARTIAL,AssemblyLevel::ELEMENT,AssemblyLevel::FULL})
-         {
-            for (int pb : {0, 1, 2})
-            {
-               for (bool dg : {true, false})
-               {
-                  int order = 2;
-                  test_assembly_level(Mesh("../../data/periodic-cube.mesh", 1, 1),
-                                      order, dg, pb, assembly);
-                  test_assembly_level(Mesh("../../data/fichera-q3.mesh", 1, 1),
-                                      order, dg, pb, assembly);
-               }
-            }
-         }
-      }
+      test_assembly_level("../../data/periodic-cube.mesh",
+                          order_3d, dg, pb, assembly);
+      test_assembly_level("../../data/fichera-q3.mesh",
+                          order_3d, dg, pb, assembly);
+   }
+
+   // Test AMR cases (DG not implemented)
+   SECTION("AMR 2D")
+   {
+      test_assembly_level("../../data/amr-quad.mesh",
+                          order_2d, false, 0, assembly);
+   }
+   SECTION("AMR 3D")
+   {
+      test_assembly_level("../../data/fichera-amr.mesh",
+                          order_3d, false, 0, assembly);
    }
 } // test case
 

tests/unit/fem/test_bilinearform.cpp

@@ -10,7 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "mfem.hpp"
-#include "catch.hpp"
+#include "unit_tests.hpp"
 
 #include <iostream>
 
@@ -60,7 +60,7 @@ TEST_CASE("Test order of boundary integrators",
 
       SparseMatrix *D = Add(1.0, A1234, -1.0, A4321);
 
-      REQUIRE(D->MaxNorm() == Approx(0.0));
+      REQUIRE(D->MaxNorm() == MFEM_Approx(0.0));
 
       delete D;
    }

tests/unit/fem/test_calcshape.cpp

@@ -10,7 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "mfem.hpp"
-#include "catch.hpp"
+#include "unit_tests.hpp"
 
 #include <iostream>
 #include <cmath>
@@ -102,7 +102,7 @@ void TestCalcShape(FiniteElement* fe, int res)
       {
          IntegrationPoint& ip = ipArr[j];
          fe->CalcShape(ip, weights);
-         REQUIRE( weights.Sum() == Approx(1.) );
+         REQUIRE( weights.Sum() == MFEM_Approx(1.) );
       }
    }
 }