minor

Support for SLEPc eigensolver [slepc-dev]

.gitignore

@@ -122,7 +122,7 @@ examples/sundials/ex16-final.*
 examples/sundials/Example16*
 
 examples/petsc/ex[1-69]p
-examples/petsc/ex10p
+examples/petsc/ex1[0-1]p
 
 examples/petsc/mesh.*
 examples/petsc/sol.*
@@ -137,6 +137,7 @@ examples/petsc/Example9*
 examples/petsc/deformed.*
 examples/petsc/velocity.*
 examples/petsc/elastic_energy.*
+examples/petsc/mode_*
 
 examples/pumi/ex1
 examples/pumi/ex[126]p
@@ -162,6 +163,20 @@ miniapps/electromagnetics/Tesla-AMR*
 miniapps/electromagnetics/Maxwell-Parallel*
 miniapps/electromagnetics/Joule_*
 
+miniapps/hypsys/build
+miniapps/hypsys/errors.txt
+miniapps/hypsys/grid*
+miniapps/hypsys/hypsys
+miniapps/hypsys/initial*
+miniapps/hypsys/output
+miniapps/hypsys/phypsys
+miniapps/hypsys/pressure*
+miniapps/hypsys/results
+miniapps/hypsys/scripts/gridfunc-scatter
+miniapps/hypsys/ultimate*
+miniapps/hypsys/velocity*
+miniapps/hypsys/various
+
 miniapps/meshing/mobius-strip
 miniapps/meshing/klein-bottle
 miniapps/meshing/toroid

CHANGELOG

@@ -95,6 +95,8 @@ Linear and nonlinear solvers
 - In SLISolver, changed the residual inner product from (Br,r) to (Br,Br) so the
   solver can work with non-SPD preconditioner B.
 
+- Added support for the SLEPc eigensolver package.
+
 New and updated examples and miniapps
 -------------------------------------
 - Added a new example, Example 25/25p, to demonstrate the use of a Perfectly
@@ -109,6 +111,13 @@ New and updated examples and miniapps
   for applying Dirichlet, Neumann (both homogeneous and inhomogeneous), Robin,
   and periodic boundary conditions with either H1 or DG discretizations.
 
+- Added a new miniapp, Navier, that solves the time-dependent Navier-Stokes
+  equations of incompressible fluid dynamics. See the miniapps/navier directory
+  for more details.
+
+- Ported Example 11p to SLEPc, to demonstrate solving the Laplace eigenvalue
+  equation with the shift-and-invert spectral transformation method.
+
 - Added a simple meshing miniapp, Twist, which demonstrates MFEM's strategy of
   stitching together opposite surfaces of a mesh to create a topologically
   periodic mesh.

CMakeLists.txt

@@ -149,9 +149,13 @@ if (MFEM_USE_MPI)
       message(FATAL_ERROR "PETSc version >= 3.8.0 is required")
     endif()
     set(PETSC_INCLUDE_DIRS ${PETSC_INCLUDES})
+    if (MFEM_USE_SLEPC)
+      find_package(SLEPc REQUIRED config)
+      message(STATUS "Found SLEPc version ${SLEPC_VERSION}")
+    endif()
   endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU PETSC STRUMPACK PUMI)
+  set(PKGS_NEED_MPI SUPERLU PETSC SLEPC STRUMPACK PUMI)
   foreach(PKG IN LISTS PKGS_NEED_MPI)
     if (MFEM_USE_${PKG})
       message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -352,7 +356,7 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
-    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+    SLEPC MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
     MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")

INSTALL

@@ -383,6 +383,10 @@ MFEM_USE_PETSC = YES/NO
    and other features based on the PETSc package. When enabled, this option uses
    the PETSC_* library options, see below.
 
+MFEM_USE_SLEPC = YES/NO
+   Enable MFEM eigensolvers based on the SLEPc package. When enabled, this
+   option uses the SLEPC_* library options, see below.
+
 MFEM_USE_MPFR = YES/NO
    MPFR is a library for multiple-precision floating-point computations. This
    option enables the use of MPFR in MFEM, e.g. for precise computation of 1D
@@ -597,6 +601,12 @@ The specific libraries and their options are:
   Options: PETSC_OPT, PETSC_LIB.
   Versions: PETSc >= 3.8.0.
 
+- SLEPc (optional), used when MFEM_USE_SLEPC = YES. SLEPc depends on PETSc and
+  uses some of the PETSc options when compiled.
+  URL: https://slepc.upv.es/
+  Options: SLEPC_OPT, SLEPC_LIB.
+  Versions: SLEPc >= 3.8.0.
+
 - Sidre (optional), part of LLNL's axom project, used when MFEM_USE_SIDRE = YES.
   Starting with MFEM v4.1, Axom version 0.3.1 or later is required.
   URL: https://github.com/LLNL/axom
@@ -649,12 +659,11 @@ The specific libraries and their options are:
   Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.
   Versions: OCCA >= 1.0.9.
 
-- libCEED (optional), used when MFEM_USE_CEED = YES. Requires libCEED v0.6
-  or later version, specifically, git-hash 3d05795 or later.
+- libCEED (optional), used when MFEM_USE_CEED = YES.
   URL: https://github.com/CEED/libCEED
        https://ceed.exascaleproject.org/libceed
   Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.6.
+  Versions: libCEED >= 0.6, git-hash a970f63.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
   Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.

config/XSDKDefaults.cmake

@@ -244,6 +244,10 @@ IF (DEFINED TPL_ENABLE_PETSC)
   SET(MFEM_USE_PETSC ${TPL_ENABLE_PETSC} CACHE BOOL "Enable PETSc support." FORCE)
 ENDIF()
 
+IF (DEFINED TPL_ENABLE_SLEPC)
+  SET(MFEM_USE_SLEPC ${TPL_ENABLE_SLEPC} CACHE BOOL "Enable SLEPc support." FORCE)
+ENDIF()
+
 IF (DEFINED TPL_ENABLE_MPFR)
   SET(MFEM_USE_MPFR ${TPL_ENABLE_MPFR} CACHE BOOL "Enable MPFR usage." FORCE)
 ENDIF()

config/cmake/MFEMConfig.cmake.in

@@ -38,6 +38,7 @@ set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)
 set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
+set(MFEM_USE_SLEPC @MFEM_USE_SLEPC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)

config/cmake/config.hpp.in

@@ -104,6 +104,9 @@
 // Enable MFEM functionality based on the PETSc library
 #cmakedefine MFEM_USE_PETSC
 
+// Enable MFEM functionality based on the SLEPc library
+#cmakedefine MFEM_USE_SLEPC
+
 // Enable MFEM functionality based on the Sidre library
 #cmakedefine MFEM_USE_SIDRE
 

config/cmake/modules/FindSLEPc.cmake

@@ -0,0 +1,44 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Sets the following variables:
+#   - SLEPC_FOUND
+#   - SLEPC_INCLUDE_DIRS
+#   - SLEPC_LIBRARIES
+
+set(SLEPc_REQUIRED_PACKAGES "PETSC" CACHE STRING
+  "Additional packages required by SLEPc")
+
+include(MfemCmakeUtilities)
+mfem_find_package(SLEPc SLEPC SLEPC_DIR
+  "include" "slepceps.h"
+  "${PETSC_ARCH}/lib" "slepc" # add NAMES_PER_DIR?
+  "Paths to headers required by SLEPc."
+  "Libraries required by SLEPc."
+  ADD_COMPONENT "config" "${PETSC_ARCH}/include" "slepcconf.h" "" ""
+  CHECK_BUILD SLEPC_VERSION_OK TRUE
+"
+#include \"petsc.h\"
+#include \"slepceps.h\"
+int main()
+{
+  PetscErrorCode ierr;
+  int argc = 0;
+  char** argv = NULL;
+  ierr = SlepcInitialize(&argc, &argv, PETSC_NULL, PETSC_NULL);
+  EPS eps;
+  ierr = EPSCreate(PETSC_COMM_SELF, &eps); CHKERRQ(ierr);
+  ierr = EPSDestroy(&eps); CHKERRQ(ierr);
+  ierr = SlepcFinalize(); CHKERRQ(ierr);
+  return 0;
+}
+"
+  )

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -731,7 +731,7 @@ function(mfem_export_mk_files)
       MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
       MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
       MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GNUTLS
-      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_MPFR MFEM_USE_SIDRE
+      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE
       MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA
       MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2)
   foreach(var ${CONFIG_MK_BOOL_VARS})

config/config.hpp

@@ -48,6 +48,9 @@
 #ifdef MFEM_USE_PETSC
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
+#ifdef MFEM_USE_SLEPC
+#error Building with SLEPc (MFEM_USE_SLEPC=YES) requires MPI (MFEM_USE_MPI=YES)
+#endif
 #ifdef MFEM_USE_PUMI
 #error Building with PUMI (MFEM_USE_PUMI=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif

config/config.hpp.in

@@ -118,6 +118,9 @@
 // Enable functionality based on the PETSc library
 // #define MFEM_USE_PETSC
 
+// Enable functionality based on the SLEPc library
+// #define MFEM_USE_SLEPC
+
 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR
 

config/config.mk.in

@@ -37,6 +37,7 @@ MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
 MFEM_USE_PETSC         = @MFEM_USE_PETSC@
+MFEM_USE_SLEPC         = @MFEM_USE_SLEPC@
 MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@

config/defaults.cmake

@@ -39,6 +39,7 @@ option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
 option(MFEM_USE_GSLIB "Enable GSLIB usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
 option(MFEM_USE_PETSC "Enable PETSc support." OFF)
+option(MFEM_USE_SLEPC "Enable SLEPc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
@@ -155,6 +156,10 @@ set(PETSC_DIR "${MFEM_DIR}/../petsc" CACHE PATH
     "Path to the PETSc main directory.")
 set(PETSC_ARCH "arch-linux2-c-debug" CACHE STRING "PETSc build architecture.")
 
+set(SLEPC_DIR "${MFEM_DIR}/../slepc" CACHE PATH
+    "Path to the SLEPc main directory.")
+set(SLEPC_ARCH "arch-linux2-c-debug" CACHE STRING "SLEPC build architecture.")
+
 set(MPFR_DIR "" CACHE PATH "Path to the MPFR library.")
 
 set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH

config/defaults.mk

@@ -339,9 +339,9 @@ GSLIB_DIR = @MFEM_DIR@/../gslib/build
 GSLIB_OPT = -I$(GSLIB_DIR)/include
 GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs
 
-# CUDA library configuration
+# CUDA library configuration (currently not needed)
 CUDA_OPT =
-CUDA_LIB = -lcusparse
+CUDA_LIB =
 
 # HIP library configuration (currently not needed)
 HIP_OPT =

examples/CMakeLists.txt

@@ -91,7 +91,7 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
     add_test(NAME ${TEST_NAME}_ser
       COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
   else()
-    add_test(NAME ${TEST_NAME}_np=4
+    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
       COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
       ${MPIEXEC_PREFLAGS}
       $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
@@ -101,13 +101,22 @@ endforeach()
 
 # If STRUMPACK is enabled, add a test run that uses it.
 if (MFEM_USE_STRUMPACK)
-  add_test(NAME ex11p_strumpack_np=4
+  add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
     COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
     ${MPIEXEC_PREFLAGS}
     $<TARGET_FILE:ex11p> "-no-vis" "--strumpack"
     ${MPIEXEC_POSTFLAGS})
 endif()
 
+# If SuperLU_DIST is enabled, add a test run that uses it.
+if (MFEM_USE_SUPERLU)
+  add_test(NAME ex11p_superlu_np=${MFEM_MPI_NP}
+    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+    ${MPIEXEC_PREFLAGS}
+    $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
+    ${MPIEXEC_POSTFLAGS})
+endif()
+
 # Include the examples/sundials directory if SUNDIALS is enabled.
 if (MFEM_USE_SUNDIALS)
   add_subdirectory(sundials)

examples/ex1.cpp

@@ -34,7 +34,8 @@
 //               ex1 -pa -d raja-omp
 //               ex1 -pa -d occa-omp
 //               ex1 -pa -d ceed-cpu
-//               ex1 -pa -d ceed-cuda
+//             * ex1 -pa -d ceed-cuda
+//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
 //               ex1 -m ../data/beam-hex.mesh -pa -d cuda
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref

examples/ex18.hpp

@@ -88,8 +88,6 @@ private:
    Vector funval2;
    Vector nor;
    Vector fluxN;
-   IntegrationPoint eip1;
-   IntegrationPoint eip2;
 
 public:
    FaceIntegrator(RiemannSolver &rsolver_, const int dim);
@@ -424,19 +422,16 @@ void FaceIntegrator::AssembleFaceVector(const FiniteElement &el1,
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
 
-      Tr.Loc1.Transform(ip, eip1);
-      Tr.Loc2.Transform(ip, eip2);
+      Tr.SetAllIntPoints(&ip); // set face and element int. points
 
       // Calculate basis functions on both elements at the face
-      el1.CalcShape(eip1, shape1);
-      el2.CalcShape(eip2, shape2);
+      el1.CalcShape(Tr.GetElement1IntPoint(), shape1);
+      el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
 
       // Interpolate elfun at the point
       elfun1_mat.MultTranspose(shape1, funval1);
       elfun2_mat.MultTranspose(shape2, funval2);
 
-      Tr.SetIntPoint(&ip);
-
       // Get the normal vector and the flux on the face
       CalcOrtho(Tr.Jacobian(), nor);
       const double mcs = rsolver.Eval(funval1, funval2, nor, fluxN);

examples/ex1p.cpp

@@ -32,7 +32,8 @@
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
-//               mpirun -np 4 ex1p -pa -d ceed-cuda
+//             * mpirun -np 4 ex1p -pa -d ceed-cuda
+//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
 //               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //
 // Description:  This example code demonstrates the use of MFEM to define a

examples/ex6.cpp

@@ -20,7 +20,7 @@
 //               ex6 -pa -d occa-cuda
 //               ex6 -pa -d raja-omp
 //               ex6 -pa -d ceed-cpu
-//               * ex6 -pa -d ceed-cuda
+//             * ex6 -pa -d ceed-cuda
 //               ex6 -pa -d ceed-cuda:/gpu/cuda/shared
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh

examples/ex6p.cpp

@@ -20,7 +20,7 @@
 //               mpirun -np 4 ex6p -pa -d occa-cuda
 //               mpirun -np 4 ex6p -pa -d raja-omp
 //               mpirun -np 4 ex6p -pa -d ceed-cpu
-//               * mpirun -np 4 ex6p -pa -d ceed-cuda
+//             * mpirun -np 4 ex6p -pa -d ceed-cuda
 //               mpirun -np 4 ex6p -pa -d ceed-cuda:/gpu/cuda/shared
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh

examples/ex9.cpp

@@ -254,7 +254,7 @@ int main(int argc, char *argv[])
 
    // 5. Define the discontinuous DG finite element space of the given
    //    polynomial order on the refined mesh.
-   DG_FECollection fec(order, dim, BasisType::GaussLobatto);
+   DG_FECollection fec(order, dim, BasisType::Positive);
    FiniteElementSpace fes(&mesh, &fec);
 
    cout << "Number of unknowns: " << fes.GetVSize() << endl;
@@ -378,6 +378,10 @@ int main(int argc, char *argv[])
    //    iterations, ti, with a time-step dt).
    FE_Evolution adv(m, k, b);
 
+   Vector masses(u.Size());
+   m.SpMat().Mult(u, masses);
+   double mass = masses.Sum();
+
    double t = 0.0;
    adv.SetTime(t);
    ode_solver->Init(adv);
@@ -424,6 +428,9 @@ int main(int argc, char *argv[])
       u.Save(osol);
    }
 
+   m.SpMat().Mult(u, masses);
+   cout << "Mass difference:" << abs(mass - masses.Sum()) << endl;
+
    // 10. Free the used memory.
    delete ode_solver;
    delete pd;

examples/makefile

@@ -114,6 +114,11 @@ ex11p-test-strumpack: ex11p
 	@$(call mfem-test,$<, $(RUN_MPI), STRUMPACK example,--strumpack)
 test-par-YES: ex11p-test-strumpack
 endif
+ifeq ($(MFEM_USE_SUPERLU),YES)
+ex11p-test-superlu: ex11p
+	@$(call mfem-test,$<, $(RUN_MPI), SuperLU_DIST example,--superlu)
+test-par-YES: ex11p-test-superlu
+endif
 
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk
 

examples/petsc/CMakeLists.txt

@@ -34,6 +34,15 @@ if (MFEM_USE_MPI)
     )
 endif()
 
+if (MFEM_USE_SLEPC)
+  list(APPEND PETSC_EXAMPLES_SRCS
+    ex11p.cpp
+    )
+  list(APPEND PETSC_RC_FILES
+    rc_ex11p_lobpcg rc_ex11p_gd
+    )
+endif()
+
 # Include the source directory where mfem.hpp and mfem-performance.hpp are.
 include_directories(BEFORE ${PROJECT_BINARY_DIR})
 
@@ -78,12 +87,22 @@ set(EX9_E_ARGS       -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts
 set(EX9_ES_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step)
 set(EX9_IS_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5)
 set(EX10_ARGS        -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3)
+if (MFEM_USE_SLEPC)
+  set(EX11_ARGS_SINV   -m ../../data/star.mesh --useslepc)
+  set(EX11_ARGS_LOBPCG -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg)
+  set(EX11_ARGS_GD -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd)
+endif()
 
 # Add the tests: one test per command-line-variable.
-foreach(TEST_OPTIONS_VAR
-        EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
-        EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
-        EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
+set(TEST_OPTIONS_VARS
+    EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
+    EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
+    EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
+if (MFEM_USE_SLEPC)
+    list(APPEND TEST_OPTIONS_VARS EX11_ARGS_SINV EX11_ARGS_LOBPCG EX11_ARGS_GD)
+endif()
+
+foreach(TEST_OPTIONS_VAR ${TEST_OPTIONS_VARS})
   string(REGEX REPLACE "^(.+)_ARGS" "\\1" TEST_NAME_UC ${TEST_OPTIONS_VAR})
   string(REGEX REPLACE "^([^_]+)" "\\1P" TEST_NAME_UC ${TEST_NAME_UC})
   string(TOLOWER ${TEST_NAME_UC} TEST_NAME_FULL)

examples/petsc/ex11p.cpp

@@ -0,0 +1,440 @@
+//                       MFEM Example 11 - Parallel Version
+//                              PETSc Modification
+//
+// Compile with: make ex11p
+//
+// Sample runs:  mpirun -np 4 ex11p -m ../../data/star.mesh
+//               mpirun -np 4 ex11p -m ../../data/star.mesh --slepcopts rc_ex11p_lobpcg
+//               mpirun -np 4 ex11p -m ../../data/star.mesh --slepcopts rc_ex11p_gd
+//
+// Description:  This example code demonstrates the use of MFEM to solve the
+//               eigenvalue problem -Delta u = lambda u with homogeneous
+//               Dirichlet boundary conditions.
+//
+//               We compute a number of the lowest eigenmodes by discretizing
+//               the Laplacian and Mass operators using a FE space of the
+//               specified order, or an isoparametric/isogeometric space if
+//               order < 1 (quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example demonstrates the use of the SLEPc eigensolver as an
+//               alternative to the LOBPCG eigenvalue solver. The shift and
+//               invert spectral transformation is used to help the convergence
+//               to the smaller eigenvalues. Alternative solver parameters can
+//               be passed in a file with "-slepcopts".
+//
+//               Reusing a single GLVis visualization window for multiple
+//               eigenfunctions is also illustrated.
+//
+//               We recommend viewing Example 1 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#ifndef MFEM_USE_SLEPC
+#error This examples requires that MFEM is build with MFEM_USE_SLEPC=YES
+#endif
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int ser_ref_levels = 2;
+   int par_ref_levels = 1;
+   int order = 1;
+   int nev = 5;
+   int seed = 75;
+   bool slu_solver  = false;
+   bool sp_solver = false;
+   bool visualization = 1;
+   bool use_slepc = true;
+   const char *slepcrc_file = "";
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nev, "-n", "--num-eigs",
+                  "Number of desired eigenmodes.");
+   args.AddOption(&seed, "-s", "--seed",
+                  "Random seed used to initialize LOBPCG.");
+#ifdef MFEM_USE_SUPERLU
+   args.AddOption(&slu_solver, "-slu", "--superlu", "-no-slu",
+                  "--no-superlu", "Use the SuperLU Solver.");
+#endif
+#ifdef MFEM_USE_STRUMPACK
+   args.AddOption(&sp_solver, "-sp", "--strumpack", "-no-sp",
+                  "--no-strumpack", "Use the STRUMPACK Solver.");
+#endif
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&use_slepc, "-useslepc","--useslepc","-no-slepc",
+                  "--no-slepc","Use or not SLEPc to solve the eigenvalue problem");
+   args.AddOption(&slepcrc_file, "-slepcopts", "--slepcopts",
+                  "SlepcOptions file to use.");
+   args.Parse();
+   if (slu_solver && sp_solver)
+   {
+      if (myid == 0)
+         cout << "WARNING: Both SuperLU and STRUMPACK have been selected,"
+              << " please choose either one." << endl
+              << "         Defaulting to SuperLU." << endl;
+      sp_solver = false;
+   }
+   // The command line options are also passed to the STRUMPACK
+   // solver. So do not exit if some options are not recognized.
+   if (!sp_solver)
+   {
+      if (!args.Good())
+      {
+         if (myid == 0)
+         {
+            args.PrintUsage(cout);
+         }
+         MPI_Finalize();
+         return 1;
+      }
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 2b. We initialize SLEPc. This internally initializes PETSc as well.
+   MFEMInitializeSlepc(NULL,NULL,slepcrc_file,NULL);
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors. We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
+   //    specified on the command line with -rs).
+   for (int lev = 0; lev < ser_ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution (1 time by
+   //    default, or specified on the command line with -rp). Once the parallel
+   //    mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   for (int lev = 0; lev < par_ref_levels; lev++)
+   {
+      pmesh->UniformRefinement();
+   }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of unknowns: " << size << endl;
+   }
+
+   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   //    element space. The first corresponds to the Laplacian operator -Delta,
+   //    while the second is a simple mass matrix needed on the right hand side
+   //    of the generalized eigenvalue problem below. The boundary conditions
+   //    are implemented by elimination with special values on the diagonal to
+   //    shift the Dirichlet eigenvalues out of the computational range. After
+   //    serial and parallel assembly we extract the corresponding parallel
+   //    matrices A and M.
+   ConstantCoefficient one(1.0);
+   Array<int> ess_bdr;
+   if (pmesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   if (pmesh->bdr_attributes.Size() == 0)
+   {
+      // Add a mass term if the mesh has no boundary, e.g. periodic mesh or
+      // closed surface.
+      a->AddDomainIntegrator(new MassIntegrator(one));
+   }
+   a->Assemble();
+   a->EliminateEssentialBCDiag(ess_bdr, 1.0);
+   a->Finalize();
+
+   ParBilinearForm *m = new ParBilinearForm(fespace);
+   m->AddDomainIntegrator(new MassIntegrator(one));
+   m->Assemble();
+   // shift the eigenvalue corresponding to eliminated dofs to a large value
+   m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
+   m->Finalize();
+
+   PetscParMatrix *pA = NULL, *pM = NULL;
+   HypreParMatrix *A = NULL, *M = NULL;
+   Operator::Type tid =
+      !use_slepc ? Operator::Hypre_ParCSR : Operator::PETSC_MATAIJ;
+   OperatorHandle Ah(tid), Mh(tid);
+
+   a->ParallelAssemble(Ah);
+   if (!use_slepc) { Ah.Get(A); }
+   else { Ah.Get(pA); }
+   Ah.SetOperatorOwner(false);
+
+   m->ParallelAssemble(Mh);
+   if (!use_slepc) {Mh.Get(M); }
+   else {Mh.Get(pM); }
+   Mh.SetOperatorOwner(false);
+
+#if defined(MFEM_USE_SUPERLU) || defined(MFEM_USE_STRUMPACK)
+   Operator * Arow = NULL;
+#ifdef MFEM_USE_SUPERLU
+   if (slu_solver)
+   {
+      Arow = new SuperLURowLocMatrix(*A);
+   }
+#endif
+#ifdef MFEM_USE_STRUMPACK
+   if (sp_solver)
+   {
+      Arow = new STRUMPACKRowLocMatrix(*A);
+   }
+#endif
+#endif
+
+   delete a;
+   delete m;
+
+   // 8. Define and configure the LOBPCG eigensolver and the BoomerAMG
+   //    preconditioner for A to be used within the solver. Set the matrices
+   //    which define the generalized eigenproblem A x = lambda M x.
+   Solver * precond = NULL;
+   if (!use_slepc)
+   {
+      if (!slu_solver && !sp_solver)
+      {
+         HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
+         amg->SetPrintLevel(0);
+         precond = amg;
+      }
+      else
+      {
+#ifdef MFEM_USE_SUPERLU
+         if (slu_solver)
+         {
+            SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
+            superlu->SetPrintStatistics(false);
+            superlu->SetSymmetricPattern(true);
+            superlu->SetColumnPermutation(superlu::PARMETIS);
+            superlu->SetOperator(*Arow);
+            precond = superlu;
+         }
+#endif
+#ifdef MFEM_USE_STRUMPACK
+         if (sp_solver)
+         {
+            STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+            strumpack->SetPrintFactorStatistics(true);
+            strumpack->SetPrintSolveStatistics(false);
+            strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+            strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+            strumpack->DisableMatching();
+            strumpack->SetOperator(*Arow);
+            strumpack->SetFromCommandLine();
+            precond = strumpack;
+         }
+#endif
+      }
+   }
+
+   HypreLOBPCG * lobpcg = NULL;
+   SlepcEigenSolver * slepc = NULL;
+   if (!use_slepc)
+   {
+
+      lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
+      lobpcg->SetNumModes(nev);
+      lobpcg->SetRandomSeed(seed);
+      lobpcg->SetPreconditioner(*precond);
+      lobpcg->SetMaxIter(200);
+      lobpcg->SetTol(1e-8);
+      lobpcg->SetPrecondUsageMode(1);
+      lobpcg->SetPrintLevel(1);
+      lobpcg->SetMassMatrix(*M);
+      lobpcg->SetOperator(*A);
+   }
+   else
+   {
+      slepc = new SlepcEigenSolver(MPI_COMM_WORLD);
+      slepc->SetNumModes(nev);
+      slepc->SetWhichEigenpairs(SlepcEigenSolver::TARGET_REAL);
+      slepc->SetTarget(0.0);
+      slepc->SetSpectralTransformation(SlepcEigenSolver::SHIFT_INVERT);
+      slepc->SetOperators(*pA,*pM);
+   }
+
+   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
+   //    parallel grid function to represent each of the eigenmodes returned by
+   //    the solver.
+   Array<double> eigenvalues;
+   if (!use_slepc)
+   {
+      lobpcg->Solve();
+      lobpcg->GetEigenvalues(eigenvalues);
+   }
+   else
+   {
+      slepc->Solve();
+      eigenvalues.SetSize(nev);
+      for (int i=0; i<nev; i++)
+      {
+         slepc->GetEigenvalue(i,eigenvalues[i]);
+      }
+   }
+   Vector temp(fespace->GetTrueVSize());
+   ParGridFunction x(fespace);
+
+   // 10. Save the refined mesh and the modes in parallel. This output can be
+   //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
+   {
+      ostringstream mesh_name, mode_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      for (int i=0; i<nev; i++)
+      {
+         // convert eigenvector from HypreParVector to ParGridFunction
+         if (!use_slepc)
+         {
+            x = lobpcg->GetEigenvector(i);
+         }
+         else
+         {
+            slepc->GetEigenvector(i,temp);
+            x.Distribute(temp);
+
+         }
+
+         mode_name << "mode_" << setfill('0') << setw(2) << i << "."
+                   << setfill('0') << setw(6) << myid;
+
+         ofstream mode_ofs(mode_name.str().c_str());
+         mode_ofs.precision(8);
+         x.Save(mode_ofs);
+         mode_name.str("");
+      }
+   }
+
+   // 11. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream mode_sock(vishost, visport);
+      mode_sock.precision(8);
+
+      for (int i=0; i<nev; i++)
+      {
+         if ( myid == 0 )
+         {
+            cout << "Eigenmode " << i+1 << '/' << nev
+                 << ", Lambda = " << eigenvalues[i] << endl;
+         }
+
+         // convert eigenvector from HypreParVector to ParGridFunction
+         if (!use_slepc)
+         {
+            x = lobpcg->GetEigenvector(i);
+         }
+         else
+         {
+            slepc->GetEigenvector(i,temp);
+            x.Distribute(temp);
+         }
+
+         mode_sock << "parallel " << num_procs << " " << myid << "\n"
+                   << "solution\n" << *pmesh << x << flush
+                   << "window_title 'Eigenmode " << i+1 << '/' << nev
+                   << ", Lambda = " << eigenvalues[i] << "'" << endl;
+
+         char c;
+         if (myid == 0)
+         {
+            cout << "press (q)uit or (c)ontinue --> " << flush;
+            cin >> c;
+         }
+         MPI_Bcast(&c, 1, MPI_CHAR, 0, MPI_COMM_WORLD);
+
+         if (c != 'c')
+         {
+            break;
+         }
+      }
+      mode_sock.close();
+   }
+
+   // 12. Free the used memory.
+   if (!use_slepc)
+   {
+      delete lobpcg;
+   }
+   else
+   {
+      delete slepc;
+   }
+   delete precond;
+   delete M;
+   delete A;
+#if defined(MFEM_USE_SUPERLU) || defined(MFEM_USE_STRUMPACK)
+   delete Arow;
+#endif
+
+   delete fespace;
+   if (order > 0)
+   {
+      delete fec;
+   }
+   delete pmesh;
+
+   // We finalize SLEPc
+   MFEMFinalizeSlepc();
+   MPI_Finalize();
+
+   return 0;
+}

examples/petsc/makefile

@@ -23,6 +23,9 @@ MFEM_LIB_FILE = mfem_is_not_built
 
 SEQ_EXAMPLES =
 PAR_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex9p ex10p
+ifeq ($(MFEM_USE_SLEPC),YES)
+   PAR_EXAMPLES += ex11p
+endif
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
 else
@@ -87,6 +90,9 @@ EX10_ARGS        := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p
 EX10_MF_ARGS     := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mf -tf 6 -s 3 -rs 0 -dt 3
 EX10_MFOP_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mfop -tf 6 -s 3 -rs 0 -dt 3
 EX10_JFNK_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_jfnk --jfnk -tf 6 -s 3 -rs 0 -dt 3
+EX11_ARGS_SINV   := -m ../../data/star.mesh --useslepc
+EX11_ARGS_LOBPCG := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg
+EX11_ARGS_GD     := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd
 
 ex1p-test-par: ex1p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_W))
@@ -114,6 +120,12 @@ ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MF_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MFOP_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_JFNK_ARGS))
+ifeq ($(MFEM_USE_SLEPC),YES)
+ex11p-test-par: ex11p
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_SINV))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_LOBPCG))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_GD))
+endif
 
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk
 

examples/petsc/rc_ex11p_gd

@@ -0,0 +1,6 @@
+# Options for the eigenvalue solver
+-eps_view
+-eps_converged_reason
+-eps_type gd
+# Options for the spectral transform
+-st_type precond

examples/petsc/rc_ex11p_lobpcg

@@ -0,0 +1,11 @@
+# Options for the eigenvalue solver
+-eps_monitor
+-eps_converged_reason
+-eps_view_values
+-eps_type lobpcg
+-eps_gen_hermitian
+-eps_smallest_real
+-eps_lobpcg_blocksize 5
+# Options for the spectral transform
+-st_type precond
+-st_pc_type gamg

fem/bilininteg.cpp

@@ -926,11 +926,14 @@ void BoundaryMassIntegrator::AssembleFaceMatrix(
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
-      IntegrationPoint eip;
-      Trans.Loc1.Transform(ip, eip);
+
+      // Set the integration point in the face and the neighboring element
+      Trans.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Trans.GetElement1IntPoint();
       el1.CalcShape(eip, shape);
 
-      Trans.SetIntPoint(&ip);
       w = Trans.Weight() * ip.weight;
       if (Q)
       {
@@ -2571,15 +2574,16 @@ void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
    for (int p = 0; p < ir->GetNPoints(); p++)
    {
       const IntegrationPoint &ip = ir->IntPoint(p);
-      IntegrationPoint eip1, eip2;
-      Trans.Loc1.Transform(ip, eip1);
-      if (ndof2)
-      {
-         Trans.Loc2.Transform(ip, eip2);
-      }
-      el1.CalcShape(eip1, shape1);
 
-      Trans.SetIntPoint(&ip);
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+
+      // Access the neighboring elements' integration points
+      // Note: eip2 will only contain valid data if Elem2 exists
+      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
+      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
+
+      el1.CalcShape(eip1, shape1);
 
       u->Eval(vu, *Trans.Elem1, eip1);
 
@@ -2727,10 +2731,15 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
    for (int p = 0; p < ir->GetNPoints(); p++)
    {
       const IntegrationPoint &ip = ir->IntPoint(p);
-      IntegrationPoint eip1, eip2;
 
-      Trans.Loc1.Transform(ip, eip1);
-      Trans.SetIntPoint(&ip);
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+
+      // Access the neighboring elements' integration points
+      // Note: eip2 will only contain valid data if Elem2 exists
+      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
+      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
+
       if (dim == 1)
       {
          nor(0) = 2*eip1.x - 1.0;
@@ -2787,7 +2796,6 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
 
       if (ndof2)
       {
-         Trans.Loc2.Transform(ip, eip2);
          el2.CalcShape(eip2, shape2);
          el2.CalcDShape(eip2, dshape2);
          w = ip.weight/2/Trans.Elem2->Weight();
@@ -3005,9 +3013,14 @@ void DGElasticityIntegrator::AssembleFaceMatrix(
    for (int pind = 0; pind < ir->GetNPoints(); ++pind)
    {
       const IntegrationPoint &ip = ir->IntPoint(pind);
-      IntegrationPoint eip1, eip2; // integration point in the reference space
-      Trans.Loc1.Transform(ip, eip1);
-      Trans.SetIntPoint(&ip);
+
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+
+      // Access the neighboring elements' integration points
+      // Note: eip2 will only contain valid data if Elem2 exists
+      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
+      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
 
       el1.CalcShape(eip1, shape1);
       el1.CalcDShape(eip1, dshape1);
@@ -3027,7 +3040,6 @@ void DGElasticityIntegrator::AssembleFaceMatrix(
       double w, wLM;
       if (ndofs2)
       {
-         Trans.Loc2.Transform(ip, eip2);
          el2.CalcShape(eip2, shape2);
          el2.CalcDShape(eip2, dshape2);
          CalcAdjugate(Trans.Elem2->Jacobian(), adjJ);
@@ -3165,17 +3177,22 @@ void TraceJumpIntegrator::AssembleFaceMatrix(
    for (int p = 0; p < ir->GetNPoints(); p++)
    {
       const IntegrationPoint &ip = ir->IntPoint(p);
-      IntegrationPoint eip1, eip2;
+
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+
+      // Access the neighboring elements' integration points
+      // Note: eip2 will only contain valid data if Elem2 exists
+      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
+      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
+
       // Trace finite element shape function
-      Trans.SetIntPoint(&ip);
       trial_face_fe.CalcShape(ip, face_shape);
       // Side 1 finite element shape function
-      Trans.Loc1.Transform(ip, eip1);
       test_fe1.CalcShape(eip1, shape1);
       if (ndof2)
       {
          // Side 2 finite element shape function
-         Trans.Loc2.Transform(ip, eip2);
          test_fe2.CalcShape(eip2, shape2);
       }
       w = ip.weight;

fem/bilininteg_diffusion_pa.cpp

@@ -176,41 +176,42 @@ static void PADiffusionSetup3D(const int Q1D,
    auto J = Reshape(j.Read(), NQ, 3, 3, NE);
    auto C = const_c ? Reshape(c.Read(), 1, 1) : Reshape(c.Read(), NQ, NE);
    auto D = Reshape(d.Write(), NQ, 6, NE);
-   MFEM_FORALL(eq, NE*NQ,
+   MFEM_FORALL(e, NE,
    {
-      const int e = eq / NQ;
-      const int q = eq % NQ;
-      const double J11 = J(q,0,0,e);
-      const double J21 = J(q,1,0,e);
-      const double J31 = J(q,2,0,e);
-      const double J12 = J(q,0,1,e);
-      const double J22 = J(q,1,1,e);
-      const double J32 = J(q,2,1,e);
-      const double J13 = J(q,0,2,e);
-      const double J23 = J(q,1,2,e);
-      const double J33 = J(q,2,2,e);
-      const double detJ = J11 * (J22 * J33 - J32 * J23) -
-      /* */               J21 * (J12 * J33 - J32 * J13) +
-      /* */               J31 * (J12 * J23 - J22 * J13);
-      const double coeff = const_c ? C(0,0) : C(q,e);
-      const double c_detJ = W[q] * coeff / detJ;
-      // adj(J)
-      const double A11 = (J22 * J33) - (J23 * J32);
-      const double A12 = (J32 * J13) - (J12 * J33);
-      const double A13 = (J12 * J23) - (J22 * J13);
-      const double A21 = (J31 * J23) - (J21 * J33);
-      const double A22 = (J11 * J33) - (J13 * J31);
-      const double A23 = (J21 * J13) - (J11 * J23);
-      const double A31 = (J21 * J32) - (J31 * J22);
-      const double A32 = (J31 * J12) - (J11 * J32);
-      const double A33 = (J11 * J22) - (J12 * J21);
-      // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
-      D(q,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
-      D(q,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
-      D(q,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
-      D(q,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
-      D(q,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
-      D(q,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
+      for (int q = 0; q < NQ; ++q)
+      {
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J31 = J(q,2,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double J32 = J(q,2,1,e);
+         const double J13 = J(q,0,2,e);
+         const double J23 = J(q,1,2,e);
+         const double J33 = J(q,2,2,e);
+         const double detJ = J11 * (J22 * J33 - J32 * J23) -
+         /* */               J21 * (J12 * J33 - J32 * J13) +
+         /* */               J31 * (J12 * J23 - J22 * J13);
+         const double coeff = const_c ? C(0,0) : C(q,e);
+         const double c_detJ = W[q] * coeff / detJ;
+         // adj(J)
+         const double A11 = (J22 * J33) - (J23 * J32);
+         const double A12 = (J32 * J13) - (J12 * J33);
+         const double A13 = (J12 * J23) - (J22 * J13);
+         const double A21 = (J31 * J23) - (J21 * J33);
+         const double A22 = (J11 * J33) - (J13 * J31);
+         const double A23 = (J21 * J13) - (J11 * J23);
+         const double A31 = (J21 * J32) - (J31 * J22);
+         const double A32 = (J31 * J12) - (J11 * J32);
+         const double A33 = (J11 * J22) - (J12 * J21);
+         // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
+         D(q,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
+         D(q,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
+         D(q,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
+         D(q,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
+         D(q,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
+         D(q,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
+      }
    });
 }
 

fem/bilininteg_mass_pa.cpp

@@ -25,7 +25,6 @@ namespace mfem
 
 void MassIntegrator::SetupPA(const FiniteElementSpace &fes, const bool force)
 {
-
    // Assuming the same element type
    fespace = &fes;
    Mesh *mesh = fes.GetMesh();
@@ -52,30 +51,21 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes, const bool force)
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
    pa_data.SetSize(ne*nq, Device::GetDeviceMemoryType());
-   Vector *coeff{nullptr};
-   bool own_coeff{true};
+   Vector coeff;
    if (Q == nullptr)
    {
-      coeff = new Vector;
-      coeff->SetSize(1);
-      (*coeff)(0) = 1.0;
+      coeff.SetSize(1);
+      coeff(0) = 1.0;
    }
    else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
    {
-      coeff = new Vector;
-      coeff->SetSize(1);
-      (*coeff)(0) = 1.0;
-   }
-   else if (QuadratureCoefficient* cQ = dynamic_cast<QuadratureCoefficient*>(Q))
-   {
-      coeff = cQ->Data();
-      own_coeff = false;
+      coeff.SetSize(1);
+      coeff(0) = cQ->constant;
    }
    else
    {
-      coeff = new Vector;
-      coeff->SetSize(nq * ne);
-      auto C = Reshape(coeff->HostWrite(), nq, ne);
+      coeff.SetSize(nq * ne);
+      auto C = Reshape(coeff.HostWrite(), nq, ne);
       for (int e = 0; e < ne; ++e)
       {
          ElementTransformation& T = *fes.GetElementTransformation(e);
@@ -90,11 +80,11 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes, const bool force)
    {
       const int NE = ne;
       const int NQ = nq;
-      const bool const_c = coeff->Size() == 1;
+      const bool const_c = coeff.Size() == 1;
       auto w = ir->GetWeights().Read();
       auto J = Reshape(geom->J.Read(), NQ,2,2,NE);
       auto C =
-         const_c ? Reshape(coeff->Read(), 1,1) : Reshape(coeff->Read(), NQ,NE);
+         const_c ? Reshape(coeff.Read(), 1,1) : Reshape(coeff.Read(), NQ,NE);
       auto v = Reshape(pa_data.Write(), NQ, NE);
       MFEM_FORALL(e, NE,
       {
@@ -114,11 +104,11 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes, const bool force)
    {
       const int NE = ne;
       const int NQ = nq;
-      const bool const_c = coeff->Size() == 1;
+      const bool const_c = coeff.Size() == 1;
       auto W = ir->GetWeights().Read();
       auto J = Reshape(geom->J.Read(), NQ,3,3,NE);
       auto C =
-         const_c ? Reshape(coeff->Read(), 1,1) : Reshape(coeff->Read(), NQ,NE);
+         const_c ? Reshape(coeff.Read(), 1,1) : Reshape(coeff.Read(), NQ,NE);
       auto v = Reshape(pa_data.Write(), NQ,NE);
       MFEM_FORALL(e, NE,
       {
@@ -135,8 +125,6 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes, const bool force)
          }
       });
    }
-
-   if (own_coeff) { delete coeff; }
 }
 
 void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)

fem/coefficient.cpp

@@ -12,7 +12,6 @@
 // Implementation of Coefficient class
 
 #include "fem.hpp"
-#include "../linalg/dtensor.hpp"
 
 #include <cmath>
 #include <limits>
@@ -22,13 +21,6 @@ namespace mfem
 
 using namespace std;
 
-double QuadratureCoefficient::Eval(ElementTransformation & T,
-                                   const IntegrationPoint & ip)
-{
-   auto coeff = mfem::Reshape(qData->HostRead(), nip, NE);
-   return coeff(ip.index, T.ElementNo);
-}
-
 double PWConstCoefficient::Eval(ElementTransformation & T,
                                 const IntegrationPoint & ip)
 {
@@ -217,18 +209,24 @@ void GradientGridFunctionCoefficient::Eval(
    GridFunc->GetGradients(T, ir, M);
 }
 
-CurlGridFunctionCoefficient::CurlGridFunctionCoefficient (
+CurlGridFunctionCoefficient::CurlGridFunctionCoefficient(
    const GridFunction *gf)
-   : VectorCoefficient ((gf) ?
-                        gf -> FESpace() -> GetMesh() -> SpaceDimension() : 0)
+   : VectorCoefficient(0)
 {
-   GridFunc = gf;
+   SetGridFunction(gf);
 }
 
 void CurlGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 {
-   GridFunc = gf; vdim = (gf) ?
-                         gf -> FESpace() -> GetMesh() -> SpaceDimension() : 0;
+   if (gf)
+   {
+      int sdim = gf -> FESpace() -> GetMesh() -> SpaceDimension();
+      MFEM_VERIFY(sdim == 2 || sdim == 3,
+                  "CurlGridFunctionCoefficient "
+                  "only defind for spaces of dimension 2 or 3.");
+   }
+   GridFunc = gf;
+   vdim = (gf) ? (2 * gf -> FESpace() -> GetMesh() -> SpaceDimension() - 3) : 0;
 }
 
 void CurlGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,

fem/coefficient.hpp

@@ -84,33 +84,6 @@ public:
    { return (constant); }
 };
 
-
-/// class for quadrature coefficient
-class QuadratureCoefficient : public Coefficient
-{
-
-private:
-   const int nip;
-   const int NE;
-public:
-   Vector *qData{nullptr};
-
-   //Set external data
-   QuadratureCoefficient(Vector *Data, int in_nip, int in_NE)
-      : qData(Data), nip(in_nip), NE(in_NE)
-   { }
-
-   virtual double Eval(ElementTransformation &T,
-                       const IntegrationPoint &ip);
-
-   Vector *Data()
-   {
-      return qData;
-   }
-
-};
-
-/// class for piecewise constant coefficient
 /** @brief A piecewise constant coefficient with the constants keyed
     off the element attribute numbers. */
 class PWConstCoefficient : public Coefficient

fem/eltrans.cpp

@@ -552,26 +552,32 @@ void IntegrationPointTransformation::Transform (const IntegrationRule &ir1,
    }
 }
 
-void FaceElementTransformations::SetIntPoint(const IntegrationPoint *ip)
+void FaceElementTransformations::SetIntPoint(const IntegrationPoint *face_ip)
 {
-   IsoparametricTransformation::SetIntPoint(ip);
+   IsoparametricTransformation::SetIntPoint(face_ip);
 
-   if (Elem1)
+   if (mask & 4)
    {
-      Loc1.Transform(*ip, eip1);
-      Elem1->SetIntPoint(&eip1);
+      Loc1.Transform(*face_ip, eip1);
+      if (Elem1)
+      {
+         Elem1->SetIntPoint(&eip1);
+      }
    }
-   if (Elem2)
+   if (mask & 8)
    {
-      Loc2.Transform(*ip, eip2);
-      Elem2->SetIntPoint(&eip2);
+      Loc2.Transform(*face_ip, eip2);
+      if (Elem2)
+      {
+         Elem2->SetIntPoint(&eip2);
+      }
    }
 }
 
 ElementTransformation &
 FaceElementTransformations::GetElement1Transformation()
 {
-   MFEM_VERIFY(mask & 1 && Elem1 != NULL, "The ElementTransformation "
+   MFEM_VERIFY(mask & HAVE_ELEM1 && Elem1 != NULL, "The ElementTransformation "
                "for the element has not been configured for side 1.");
    return *Elem1;
 }
@@ -579,7 +585,7 @@ FaceElementTransformations::GetElement1Transformation()
 ElementTransformation &
 FaceElementTransformations::GetElement2Transformation()
 {
-   MFEM_VERIFY(mask & 2 && Elem2 != NULL, "The ElementTransformation "
+   MFEM_VERIFY(mask & HAVE_ELEM2 && Elem2 != NULL, "The ElementTransformation "
                "for the element has not been configured for side 2.");
    return *Elem2;
 }
@@ -587,7 +593,7 @@ FaceElementTransformations::GetElement2Transformation()
 IntegrationPointTransformation &
 FaceElementTransformations::GetIntPoint1Transformation()
 {
-   MFEM_VERIFY(mask & 4, "The IntegrationPointTransformation "
+   MFEM_VERIFY(mask & HAVE_LOC1, "The IntegrationPointTransformation "
                "for the element has not been configured for side 1.");
    return Loc1;
 }
@@ -595,7 +601,7 @@ FaceElementTransformations::GetIntPoint1Transformation()
 IntegrationPointTransformation &
 FaceElementTransformations::GetIntPoint2Transformation()
 {
-   MFEM_VERIFY(mask & 8, "The IntegrationPointTransformation "
+   MFEM_VERIFY(mask & HAVE_LOC2, "The IntegrationPointTransformation "
                "for the element has not been configured for side 2.");
    return Loc2;
 }
@@ -603,7 +609,7 @@ FaceElementTransformations::GetIntPoint2Transformation()
 void FaceElementTransformations::Transform(const IntegrationPoint &ip,
                                            Vector &trans)
 {
-   MFEM_VERIFY(mask & 16, "The ElementTransformation "
+   MFEM_VERIFY(mask & HAVE_FACE, "The ElementTransformation "
                "for the face has not been configured.");
    IsoparametricTransformation::Transform(ip, trans);
 }
@@ -611,7 +617,7 @@ void FaceElementTransformations::Transform(const IntegrationPoint &ip,
 void FaceElementTransformations::Transform(const IntegrationRule &ir,
                                            DenseMatrix &tr)
 {
-   MFEM_VERIFY(mask & 16, "The ElementTransformation "
+   MFEM_VERIFY(mask & HAVE_FACE, "The ElementTransformation "
                "for the face has not been configured.");
    IsoparametricTransformation::Transform(ir, tr);
 }
@@ -619,7 +625,7 @@ void FaceElementTransformations::Transform(const IntegrationRule &ir,
 void FaceElementTransformations::Transform(const DenseMatrix &matrix,
                                            DenseMatrix &result)
 {
-   MFEM_VERIFY(mask & 16, "The ElementTransformation "
+   MFEM_VERIFY(mask & HAVE_FACE, "The ElementTransformation "
                "for the face has not been configured.");
    IsoparametricTransformation::Transform(matrix, result);
 }

fem/eltrans.hpp

@@ -439,15 +439,57 @@ public:
    void Transform (const IntegrationRule  &, IntegrationRule  &);
 };
 
+/** @brief A specialized ElementTransformation class representing a face and
+    its two neighboring elements.
 
+    This class can be used as a container for the element transformation data
+    needed for integrating discontinuous fields on element interfaces in a
+    Discontinuous Galerkin (DG) context.
+
+    The secondary purpose of this class is to enable the
+    GridFunction::GetValue function, and various related functions, to properly
+    evaluate fields with limited continuity on boundary elements.
+*/
 class FaceElementTransformations : public IsoparametricTransformation
 {
 private:
+
+   // Bitwise OR of ConfigMasks
    int mask;
 
    IntegrationPoint eip1, eip2;
 
+protected: // interface for Mesh to be able to configure this object.
+
+   friend class Mesh;
+#ifdef MFEM_USE_MPI
+   friend class ParMesh;
+#endif
+
+   /// Set the mask indicating which portions of the object have been setup
+   /** The argument @a m is a bitmask used in
+       Mesh::GetFaceElementTransformations to indicate which portions of the
+       FaceElementTransformations object have been configured.
+
+       mask &  1: Elem1 is configured
+       mask &  2: Elem2 is configured
+       mask &  4: Loc1 is configured
+       mask &  8: Loc2 is configured
+       mask & 16: The Face transformation itself is configured
+   */
+   void SetConfigurationMask(int m) { mask = m; }
+
 public:
+
+   enum ConfigMasks
+   {
+      HAVE_ELEM1 =  1, ///< Element on side 1 is configured
+      HAVE_ELEM2 =  2, ///< Element on side 2 is configured
+      HAVE_LOC1  =  4, ///< Point transformation for side 1 is configured
+      HAVE_LOC2  =  8, ///< Point transformation for side 2 is configured
+      HAVE_FACE  = 16  ///< Face transformation is configured
+   };
+
    int Elem1No, Elem2No;
    Geometry::Type &FaceGeom; ///< @deprecated Use GetGeometryType instead
    ElementTransformation *Elem1, *Elem2;
@@ -466,10 +508,10 @@ public:
    */
    void SetGeometryType(Geometry::Type g) { geom = g; }
 
-   /// Set the mask indicating which portions of the object have been setup
-   /** The argument @a m is a bitmask used in
-       Mesh::GetFaceElementTransformations to indicate which portions of the
-       FaceElement Transformations object have been configured.
+   /** @brief Return the mask defining the configuration state.
+
+       The mask value indicates which portions of FaceElementTransformations
+       object have been configured.
 
        mask &  1: Elem1 is configured
        mask &  2: Elem2 is configured
@@ -477,12 +519,45 @@ public:
        mask &  8: Loc2 is configured
        mask & 16: The Face transformation itself is configured
    */
-   void SetConfigurationMask(int m) { mask = m; }
-   int  GetConfigurationMask() const { return mask; }
+   int GetConfigurationMask() const { return mask; }
 
    /** @brief Set the integration point in the Face and the two neighboring
-       elements, if present. */
-   void SetIntPoint(const IntegrationPoint *ip);
+       elements, if present.
+
+       The point @a face_ip must be in the reference coordinate system of the
+       face.
+   */
+   void SetIntPoint(const IntegrationPoint *face_ip);
+
+   /** @brief Set the integration point in the Face and the two neighboring
+       elements, if present.
+
+       This is a more expressive member function name than SetIntPoint, which
+       in this special case, does the same thing. This function can be used for
+       greater code clarity.
+   */
+   inline void SetAllIntPoints(const IntegrationPoint *face_ip)
+   { FaceElementTransformations::SetIntPoint(face_ip); }
+
+   /** @brief Get a const reference to the integration point in neighboring
+       element 1 corresponding to the currently set integration point on the
+       face.
+
+       This IntegrationPoint object will only contain up-to-date data if
+       SetIntPoint or SetAllIntPoints has been called with the latest
+       integration point for the face and the appropriate point transformation
+       has been configured. */
+   const IntegrationPoint &GetElement1IntPoint() { return eip1; }
+
+   /** @brief Get a const reference to the integration point in neighboring
+       element 2 corresponding to the currently set integration point on the
+       face.
+
+       This IntegrationPoint object will only contain up-to-date data if
+       SetIntPoint or SetAllIntPoints has been called with the latest
+       integration point for the face and the appropriate point transformation
+       has been configured. */
+   const IntegrationPoint &GetElement2IntPoint() { return eip2; }
 
    virtual void Transform(const IntegrationPoint &, Vector &);
    virtual void Transform(const IntegrationRule &, DenseMatrix &);

fem/gridfunc.cpp

@@ -397,8 +397,16 @@ const
    fes->DofsToVDofs(vdim-1, dofs);
    Vector DofVal(dofs.Size()), LocVec;
    const FiniteElement *fe = fes->GetFE(i);
-   MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE, "invalid FE map type");
-   fe->CalcShape(ip, DofVal);
+   if (fe->GetMapType() == FiniteElement::VALUE)
+   {
+      fe->CalcShape(ip, DofVal);
+   }
+   else
+   {
+      ElementTransformation *Tr = fes->GetElementTransformation(i);
+      Tr->SetIntPoint(&ip);
+      fe->CalcPhysShape(*Tr, DofVal);
+   }
    GetSubVector(dofs, LocVec);
 
    return (DofVal * LocVec);
@@ -415,10 +423,17 @@ void GridFunction::GetVectorValue(int i, const IntegrationPoint &ip,
    GetSubVector(vdofs, loc_data);
    if (FElem->GetRangeType() == FiniteElement::SCALAR)
    {
-      MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
-                  "invalid FE map type");
       Vector shape(dof);
-      FElem->CalcShape(ip, shape);
+      if (FElem->GetMapType() == FiniteElement::VALUE)
+      {
+         FElem->CalcShape(ip, shape);
+      }
+      else
+      {
+         ElementTransformation *Tr = fes->GetElementTransformation(i);
+         Tr->SetIntPoint(&ip);
+         FElem->CalcPhysShape(*Tr, shape);
+      }
       int vdim = fes->GetVDim();
       val.SetSize(vdim);
       for (int k = 0; k < vdim; k++)
@@ -752,19 +767,21 @@ double GridFunction::GetValue(ElementTransformation &T,
             IntegrationPoint fip;
             be_to_bfe(FET->GetGeometryType(), o, ip, fip);
 
-            FET->SetIntPoint(&fip);
+            // Compute and set the point in element 1 from fip
+            FET->SetAllIntPoints(&fip);
             ElementTransformation & T1 = FET->GetElement1Transformation();
             return GetValue(T1, T1.GetIntPoint(), comp);
          }
-         break;
       }
+      break;
       case ElementTransformation::BDR_FACE:
       {
          FaceElementTransformations * FET =
             dynamic_cast<FaceElementTransformations *>(&T);
 
          // Evaluate in neighboring element for both continuous and
-         // discontinuous fields.
+         // discontinuous fields (the integration point in T1 should have
+         // already been set).
          ElementTransformation & T1 = FET->GetElement1Transformation();
          return GetValue(T1, T1.GetIntPoint(), comp);
       }
@@ -888,19 +905,21 @@ void GridFunction::GetVectorValue(ElementTransformation &T,
             IntegrationPoint fip;
             be_to_bfe(FET->GetGeometryType(), o, ip, fip);
 
-            FET->SetIntPoint(&fip);
+            // Compute and set the point in element 1 from fip
+            FET->SetAllIntPoints(&fip);
             ElementTransformation & T1 = FET->GetElement1Transformation();
             return GetVectorValue(T1, T1.GetIntPoint(), val);
          }
-         break;
       }
+      break;
       case ElementTransformation::BDR_FACE:
       {
          FaceElementTransformations * FET =
             dynamic_cast<FaceElementTransformations *>(&T);
 
          // Evaluate in neighboring element for both continuous and
-         // discontinuous fields.
+         // discontinuous fields (the integration point in T1 should have
+         // already been set).
          ElementTransformation & T1 = FET->GetElement1Transformation();
          return GetVectorValue(T1, T1.GetIntPoint(), val);
       }
@@ -1338,107 +1357,262 @@ void GridFunction::GetVectorGradientHat(
    MultAtB(loc_data_mat, dshape, gh);
 }
 
-double GridFunction::GetDivergence(ElementTransformation &tr) const
+double GridFunction::GetDivergence(ElementTransformation &T) const
 {
-   double div_v;
-   int elNo = tr.ElementNo;
-   const FiniteElement *FElem = fes->GetFE(elNo);
-   if (FElem->GetRangeType() == FiniteElement::SCALAR)
+   switch (T.ElementType)
    {
-      MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
-                  "invalid FE map type");
-      DenseMatrix grad_hat;
-      GetVectorGradientHat(tr, grad_hat);
-      const DenseMatrix &Jinv = tr.InverseJacobian();
-      div_v = 0.0;
-      for (int i = 0; i < Jinv.Width(); i++)
+      case ElementTransformation::ELEMENT:
       {
-         for (int j = 0; j < Jinv.Height(); j++)
+         int elNo = T.ElementNo;
+         const FiniteElement *fe = fes->GetFE(elNo);
+         if (fe->GetRangeType() == FiniteElement::SCALAR)
          {
-            div_v += grad_hat(i, j) * Jinv(j, i);
+            MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE,
+                        "invalid FE map type");
+            DenseMatrix grad_hat;
+            GetVectorGradientHat(T, grad_hat);
+            const DenseMatrix &Jinv = T.InverseJacobian();
+            double div_v = 0.0;
+            for (int i = 0; i < Jinv.Width(); i++)
+            {
+               for (int j = 0; j < Jinv.Height(); j++)
+               {
+                  div_v += grad_hat(i, j) * Jinv(j, i);
+               }
+            }
+            return div_v;
+         }
+         else
+         {
+            // Assuming RT-type space
+            Array<int> dofs;
+            fes->GetElementDofs(elNo, dofs);
+            Vector loc_data, divshape(fe->GetDof());
+            GetSubVector(dofs, loc_data);
+            fe->CalcDivShape(T.GetIntPoint(), divshape);
+            return (loc_data * divshape) / T.Weight();
          }
       }
+      break;
+      case ElementTransformation::BDR_ELEMENT:
+      {
+         // In order to properly capture the derivative of the normal component
+         // of the field (as well as the transverse divergence of the
+         // tangential compoents) we must evaluate it in the neighboring
+         // element.
+         FaceElementTransformations * FET =
+            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
+
+         // Boundary elements and Boundary Faces may have different
+         // orientations so adjust the integration point if necessary.
+         int o = 0;
+         if (fes->GetMesh()->Dimension() == 3)
+         {
+            int f;
+            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+         }
+
+         IntegrationPoint fip;
+         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);
+
+         // Compute and set the point in element 1 from fip
+         FET->SetAllIntPoints(&fip);
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+
+         return GetDivergence(T1);
+      }
+      break;
+      case ElementTransformation::BDR_FACE:
+      {
+         // This must be a DG context so this dynamic cast must succeed.
+         FaceElementTransformations * FET =
+            dynamic_cast<FaceElementTransformations *>(&T);
+
+         // Evaluate in neighboring element (the integration point in T1 should
+         // have already been set).
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+         return GetDivergence(T1);
+      }
+      break;
+      default:
+      {
+         MFEM_ABORT("GridFunction::GetDivergence: Unsupported element type \""
+                    << T.ElementType << "\"");
+      }
    }
-   else
-   {
-      // Assuming RT-type space
-      Array<int> dofs;
-      fes->GetElementDofs(elNo, dofs);
-      Vector loc_data, divshape(FElem->GetDof());
-      GetSubVector(dofs, loc_data);
-      FElem->CalcDivShape(tr.GetIntPoint(), divshape);
-      div_v = (loc_data * divshape) / tr.Weight();
-   }
-   return div_v;
+   return 0.0; // never reached
 }
 
-void GridFunction::GetCurl(ElementTransformation &tr, Vector &curl) const
+void GridFunction::GetCurl(ElementTransformation &T, Vector &curl) const
 {
-   int elNo = tr.ElementNo;
-   const FiniteElement *FElem = fes->GetFE(elNo);
-   if (FElem->GetRangeType() == FiniteElement::SCALAR)
+   switch (T.ElementType)
    {
-      MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
-                  "invalid FE map type");
-      DenseMatrix grad_hat;
-      GetVectorGradientHat(tr, grad_hat);
-      const DenseMatrix &Jinv = tr.InverseJacobian();
-      DenseMatrix grad(grad_hat.Height(), Jinv.Width()); // vdim x FElem->Dim
-      Mult(grad_hat, Jinv, grad);
-      MFEM_ASSERT(grad.Height() == grad.Width(), "");
-      if (grad.Height() == 3)
+      case ElementTransformation::ELEMENT:
       {
-         curl.SetSize(3);
-         curl(0) = grad(2,1) - grad(1,2);
-         curl(1) = grad(0,2) - grad(2,0);
-         curl(2) = grad(1,0) - grad(0,1);
+         int elNo = T.ElementNo;
+         const FiniteElement *fe = fes->GetFE(elNo);
+         if (fe->GetRangeType() == FiniteElement::SCALAR)
+         {
+            MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE,
+                        "invalid FE map type");
+            DenseMatrix grad_hat;
+            GetVectorGradientHat(T, grad_hat);
+            const DenseMatrix &Jinv = T.InverseJacobian();
+            // Dimensions of grad are vdim x FElem->Dim
+            DenseMatrix grad(grad_hat.Height(), Jinv.Width());
+            Mult(grad_hat, Jinv, grad);
+            MFEM_ASSERT(grad.Height() == grad.Width(), "");
+            if (grad.Height() == 3)
+            {
+               curl.SetSize(3);
+               curl(0) = grad(2,1) - grad(1,2);
+               curl(1) = grad(0,2) - grad(2,0);
+               curl(2) = grad(1,0) - grad(0,1);
+            }
+            else if (grad.Height() == 2)
+            {
+               curl.SetSize(1);
+               curl(0) = grad(1,0) - grad(0,1);
+            }
+         }
+         else
+         {
+            // Assuming ND-type space
+            Array<int> dofs;
+            fes->GetElementDofs(elNo, dofs);
+            Vector loc_data;
+            GetSubVector(dofs, loc_data);
+            DenseMatrix curl_shape(fe->GetDof(), fe->GetDim() == 3 ? 3 : 1);
+            fe->CalcCurlShape(T.GetIntPoint(), curl_shape);
+            curl.SetSize(curl_shape.Width());
+            if (curl_shape.Width() == 3)
+            {
+               double curl_hat[3];
+               curl_shape.MultTranspose(loc_data, curl_hat);
+               T.Jacobian().Mult(curl_hat, curl);
+            }
+            else
+            {
+               curl_shape.MultTranspose(loc_data, curl);
+            }
+            curl /= T.Weight();
+         }
       }
-      else if (grad.Height() == 2)
+      break;
+      case ElementTransformation::BDR_ELEMENT:
       {
-         curl.SetSize(1);
-         curl(0) = grad(1,0) - grad(0,1);
+         // In order to capture the tangential components of the curl we
+         // must evaluate it in the neighboring element.
+         FaceElementTransformations * FET =
+            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
+
+         // Boundary elements and Boundary Faces may have different
+         // orientations so adjust the integration point if necessary.
+         int o = 0;
+         if (fes->GetMesh()->Dimension() == 3)
+         {
+            int f;
+            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+         }
+
+         IntegrationPoint fip;
+         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);
+
+         // Compute and set the point in element 1 from fip
+         FET->SetAllIntPoints(&fip);
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+
+         GetCurl(T1, curl);
       }
-   }
-   else
-   {
-      // Assuming ND-type space
-      Array<int> dofs;
-      fes->GetElementDofs(elNo, dofs);
-      Vector loc_data;
-      GetSubVector(dofs, loc_data);
-      DenseMatrix curl_shape(FElem->GetDof(), FElem->GetDim() == 3 ? 3 : 1);
-      FElem->CalcCurlShape(tr.GetIntPoint(), curl_shape);
-      curl.SetSize(curl_shape.Width());
-      if (curl_shape.Width() == 3)
+      break;
+      case ElementTransformation::BDR_FACE:
       {
-         double curl_hat[3];
-         curl_shape.MultTranspose(loc_data, curl_hat);
-         tr.Jacobian().Mult(curl_hat, curl);
+         // This must be a DG context so this dynamic cast must succeed.
+         FaceElementTransformations * FET =
+            dynamic_cast<FaceElementTransformations *>(&T);
+
+         // Evaluate in neighboring element (the integration point in T1 should
+         // have already been set).
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+         GetCurl(T1, curl);
       }
-      else
+      break;
+      default:
       {
-         curl_shape.MultTranspose(loc_data, curl);
+         MFEM_ABORT("GridFunction::GetCurl: Unsupported element type \""
+                    << T.ElementType << "\"");
       }
-      curl /= tr.Weight();
    }
 }
 
-void GridFunction::GetGradient(ElementTransformation &tr, Vector &grad) const
+void GridFunction::GetGradient(ElementTransformation &T, Vector &grad) const
 {
-   int elNo = tr.ElementNo;
-   const FiniteElement *fe = fes->GetFE(elNo);
-   MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE, "invalid FE map type");
-   int dim = fe->GetDim(), dof = fe->GetDof();
-   DenseMatrix dshape(dof, dim);
-   Vector lval, gh(dim);
-   Array<int> dofs;
+   switch (T.ElementType)
+   {
+      case ElementTransformation::ELEMENT:
+      {
+         const FiniteElement * fe = fes->GetFE(T.ElementNo);
+         MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE,
+                     "invalid FE map type");
+         int spaceDim = fes->GetMesh()->SpaceDimension();
+         int dim = fe->GetDim(), dof = fe->GetDof();
+         DenseMatrix dshape(dof, dim);
+         Vector lval, gh(dim);
+         Array<int> dofs;
 
-   grad.SetSize(dim);
-   fes->GetElementDofs(elNo, dofs);
-   GetSubVector(dofs, lval);
-   fe->CalcDShape(tr.GetIntPoint(), dshape);
-   dshape.MultTranspose(lval, gh);
-   tr.InverseJacobian().MultTranspose(gh, grad);
+         grad.SetSize(spaceDim);
+         fes->GetElementDofs(T.ElementNo, dofs);
+         GetSubVector(dofs, lval);
+         fe->CalcDShape(T.GetIntPoint(), dshape);
+         dshape.MultTranspose(lval, gh);
+         T.InverseJacobian().MultTranspose(gh, grad);
+      }
+      break;
+      case ElementTransformation::BDR_ELEMENT:
+      {
+         // In order to properly capture the normal component of the gradient
+         // as well as its tangential components we must evaluate it in the
+         // neighboring element.
+         FaceElementTransformations * FET =
+            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
+
+         // Boundary elements and Boundary Faces may have different
+         // orientations so adjust the integration point if necessary.
+         int o = 0;
+         if (fes->GetMesh()->Dimension() == 3)
+         {
+            int f;
+            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+         }
+
+         IntegrationPoint fip;
+         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);
+
+         // Compute and set the point in element 1 from fip
+         FET->SetAllIntPoints(&fip);
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+
+         GetGradient(T1, grad);
+      }
+      break;
+      case ElementTransformation::BDR_FACE:
+      {
+         // This must be a DG context so this dynamic cast must succeed.
+         FaceElementTransformations * FET =
+            dynamic_cast<FaceElementTransformations *>(&T);
+
+         // Evaluate in neighboring element (the integration point in T1 should
+         // have already been set).
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+         GetGradient(T1, grad);
+      }
+      break;
+      default:
+      {
+         MFEM_ABORT("GridFunction::GetGradient: Unsupported element type \""
+                    << T.ElementType << "\"");
+      }
+   }
 }
 
 void GridFunction::GetGradients(ElementTransformation &tr,
@@ -1467,15 +1641,65 @@ void GridFunction::GetGradients(ElementTransformation &tr,
 }
 
 void GridFunction::GetVectorGradient(
-   ElementTransformation &tr, DenseMatrix &grad) const
+   ElementTransformation &T, DenseMatrix &grad) const
 {
-   MFEM_ASSERT(fes->GetFE(tr.ElementNo)->GetMapType() == FiniteElement::VALUE,
-               "invalid FE map type");
-   DenseMatrix grad_hat;
-   GetVectorGradientHat(tr, grad_hat);
-   const DenseMatrix &Jinv = tr.InverseJacobian();
-   grad.SetSize(grad_hat.Height(), Jinv.Width());
-   Mult(grad_hat, Jinv, grad);
+   switch (T.ElementType)
+   {
+      case ElementTransformation::ELEMENT:
+      {
+         MFEM_ASSERT(fes->GetFE(T.ElementNo)->GetMapType() ==
+                     FiniteElement::VALUE, "invalid FE map type");
+         DenseMatrix grad_hat;
+         GetVectorGradientHat(T, grad_hat);
+         const DenseMatrix &Jinv = T.InverseJacobian();
+         grad.SetSize(grad_hat.Height(), Jinv.Width());
+         Mult(grad_hat, Jinv, grad);
+      }
+      break;
+      case ElementTransformation::BDR_ELEMENT:
+      {
+         // In order to capture the normal component of the gradient we
+         // must evaluate it in the neighboring element.
+         FaceElementTransformations * FET =
+            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
+
+         // Boundary elements and Boundary Faces may have different
+         // orientations so adjust the integration point if necessary.
+         int o = 0;
+         if (fes->GetMesh()->Dimension() == 3)
+         {
+            int f;
+            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+         }
+
+         IntegrationPoint fip;
+         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);
+
+         // Compute and set the point in element 1 from fip
+         FET->SetAllIntPoints(&fip);
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+
+         GetVectorGradient(T1, grad);
+      }
+      break;
+      case ElementTransformation::BDR_FACE:
+      {
+         // This must be a DG context so this dynamic cast must succeed.
+         FaceElementTransformations * FET =
+            dynamic_cast<FaceElementTransformations *>(&T);
+
+         // Evaluate in neighboring element (the integration point in T1 should
+         // have already been set).
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+         GetVectorGradient(T1, grad);
+      }
+      break;
+      default:
+      {
+         MFEM_ABORT("GridFunction::GetVectorGradient: "
+                    "Unsupported element type \"" << T.ElementType << "\"");
+      }
+   }
 }
 
 void GridFunction::GetElementAverages(GridFunction &avgs) const
@@ -2760,7 +2984,7 @@ double GridFunction::ComputeLpError(const double p, Coefficient &exsol,
       }
       else
       {
-         int intorder = 2*fe->GetOrder() + 3; // <----------
+         int intorder = 2*fe->GetOrder() + 1; // <----------
          ir = &(IntRules.Get(fe->GetGeomType(), intorder));
       }
       GetValues(i, *ir, vals);
@@ -2893,7 +3117,7 @@ double GridFunction::ComputeLpError(const double p, VectorCoefficient &exsol,
       }
       else
       {
-         int intorder = 2*fe->GetOrder() + 3; // <----------
+         int intorder = 2*fe->GetOrder() + 1; // <----------
          ir = &(IntRules.Get(fe->GetGeomType(), intorder));
       }
       T = fes->GetElementTransformation(i);

fem/gridfunc.hpp

@@ -162,7 +162,8 @@ public:
                            int vdim = 1) const;
 
    /** Return a vector value from within the given element. */
-   void GetVectorValue(int i, const IntegrationPoint &ip, Vector &val) const;
+   virtual void GetVectorValue(int i, const IntegrationPoint &ip,
+                               Vector &val) const;
    ///@}
 
    /** @name Element Index Get Values Methods
@@ -208,13 +209,14 @@ public:
    ///@{
    /** Return a scalar value from within the element indicated by the
        ElementTransformation Object. */
-   double GetValue(ElementTransformation &T, const IntegrationPoint &ip,
-                   int comp = 0, Vector *tr = NULL) const;
+   virtual double GetValue(ElementTransformation &T, const IntegrationPoint &ip,
+                           int comp = 0, Vector *tr = NULL) const;
 
    /** Return a vector value from within the element indicated by the
        ElementTransformation Object. */
-   void GetVectorValue(ElementTransformation &T, const IntegrationPoint &ip,
-                       Vector &val, Vector *tr = NULL) const;
+   virtual void GetVectorValue(ElementTransformation &T,
+                               const IntegrationPoint &ip,
+                               Vector &val, Vector *tr = NULL) const;
    ///@}
 
    /** @name ElementTransformation Get Values Methods

fem/gslib.cpp

@@ -192,6 +192,7 @@ void FindPointsGSLIB::Interpolate(Array<unsigned int> &codes,
    const int ncomp      = field_in.FESpace()->GetVDim(),
              points_fld = field_in.Size() / ncomp,
              points_cnt = codes.Size();
+   field_out.SetSize(points_cnt*ncomp);
 
    for (int i = 0; i < ncomp; i++)
    {

fem/libceed/ceed.cpp

@@ -97,6 +97,8 @@ static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
    Vector qweight(Q);
    Vector shape_i(P);
    DenseMatrix grad_i(P, dim);
+
+   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
    const Table &el_dof = fes.GetElementToDofTable();
    Array<int> tp_el_dof(el_dof.Size_of_connections());
    const TensorBasisElement * tfe =
@@ -128,7 +130,15 @@ static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
          const int el_offset = fe->GetDof() * i;
          for (int j = 0; j < fe->GetDof(); j++)
          {
-            tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
+            if (compstride == 1)
+            {
+               tp_el_dof[j + el_offset] = fes.GetVDim()*
+                                          el_dof.GetJ()[dof_map[j] + el_offset];
+            }
+            else
+            {
+               tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
+            }
          }
       }
    }
@@ -157,20 +167,23 @@ static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
       {
          for (int i = 0; i < P; i++)
          {
-            tp_el_dof[i + e*P] = el_dof.GetJ()[i + e*P];
+            if (compstride == 1)
+            {
+               tp_el_dof[i + e*P] = fes.GetVDim()*el_dof.GetJ()[i + e*P];
+            }
+            else
+            {
+               tp_el_dof[i + e*P] = el_dof.GetJ()[i + e*P];
+            }
          }
       }
    }
    CeedBasisCreateH1(ceed, GetCeedTopology(fe->GetGeomType()), fes.GetVDim(),
                      fe->GetDof(), ir.GetNPoints(), shape.GetData(),
                      grad.GetData(), qref.GetData(), qweight.GetData(), basis);
-   CeedInterlaceMode imode = CEED_NONINTERLACED;
-   if (fes.GetOrdering()==Ordering::byVDIM)
-   {
-      imode = CEED_INTERLACED;
-   }
-   CeedElemRestrictionCreate(ceed, imode, mesh->GetNE(), fe->GetDof(),
-                             fes.GetNDofs(), fes.GetVDim(), CEED_MEM_HOST, CEED_COPY_VALUES,
+   CeedElemRestrictionCreate(ceed, mesh->GetNE(), fe->GetDof(), fes.GetVDim(),
+                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
+                             CEED_MEM_HOST, CEED_COPY_VALUES,
                              tp_el_dof.GetData(), restr);
 }
 
@@ -215,6 +228,7 @@ static void InitCeedTensorBasisAndRestriction(const FiniteElementSpace &fes,
                            grad1d.GetData(), qref1d.GetData(),
                            qweight1d.GetData(), basis);
 
+   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
    const Table &el_dof = fes.GetElementToDofTable();
    Array<int> tp_el_dof(el_dof.Size_of_connections());
    for (int i = 0; i < mesh->GetNE(); i++)
@@ -222,16 +236,20 @@ static void InitCeedTensorBasisAndRestriction(const FiniteElementSpace &fes,
       const int el_offset = fe->GetDof() * i;
       for (int j = 0; j < fe->GetDof(); j++)
       {
-         tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
+         if (compstride == 1)
+         {
+            tp_el_dof[j + el_offset] = fes.GetVDim()*
+                                       el_dof.GetJ()[dof_map[j] + el_offset];
+         }
+         else
+         {
+            tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
+         }
       }
    }
-   CeedInterlaceMode imode = CEED_NONINTERLACED;
-   if (fes.GetOrdering()==Ordering::byVDIM)
-   {
-      imode = CEED_INTERLACED;
-   }
-   CeedElemRestrictionCreate(ceed, imode, mesh->GetNE(), fe->GetDof(),
-                             fes.GetNDofs(), fes.GetVDim(), CEED_MEM_HOST, CEED_COPY_VALUES,
+   CeedElemRestrictionCreate(ceed, mesh->GetNE(), fe->GetDof(), fes.GetVDim(),
+                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
+                             CEED_MEM_HOST, CEED_COPY_VALUES,
                              tp_el_dof.GetData(), restr);
 }
 
@@ -298,8 +316,9 @@ void CeedPAAssemble(const CeedPAOperator& op,
    CeedBasisGetNumQuadraturePoints(ceedData.basis, &nqpts);
 
    const int qdatasize = op.qdatasize;
-   CeedElemRestrictionCreateStrided(ceed, nelem, nqpts, nelem*nqpts, qdatasize,
-                                    CEED_STRIDES_BACKEND, &ceedData.restr_i);
+   CeedElemRestrictionCreateStrided(ceed, nelem, nqpts, qdatasize,
+                                    nelem*nqpts*qdatasize, CEED_STRIDES_BACKEND,
+                                    &ceedData.restr_i);
 
    CeedVectorCreate(ceed, mesh->GetNodes()->Size(), &ceedData.node_coords);
    CeedVectorSetArray(ceedData.node_coords, CEED_MEM_HOST, CEED_USE_POINTER,

fem/lininteg.cpp

@@ -159,10 +159,13 @@ void BoundaryLFIntegrator::AssembleRHSElementVect(
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
-      IntegrationPoint eip;
-      Tr.Loc1.Transform(ip, eip);
 
-      Tr.Face->SetIntPoint (&ip);
+      // Set the integration point in the face and the neighboring element
+      Tr.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
+
       double val = Tr.Face->Weight() * ip.weight * Q.Eval(*Tr.Face, ip);
 
       el.CalcShape(eip, shape);
@@ -359,10 +362,12 @@ void VectorBoundaryLFIntegrator::AssembleRHSElementVect(
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
-      IntegrationPoint eip;
-      Tr.Loc1.Transform(ip, eip);
 
-      Tr.SetIntPoint(&ip);
+      // Set the integration point in the face and the neighboring element
+      Tr.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
 
       // Use Tr transformation in case Q depends on boundary attribute
       Q.Eval(vec, Tr, ip);
@@ -683,11 +688,13 @@ void BoundaryFlowIntegrator::AssembleRHSElementVect(
    for (int p = 0; p < ir->GetNPoints(); p++)
    {
       const IntegrationPoint &ip = ir->IntPoint(p);
-      IntegrationPoint eip;
-      Tr.Loc1.Transform(ip, eip);
-      el.CalcShape(eip, shape);
 
-      Tr.SetIntPoint(&ip);
+      // Set the integration point in the face and the neighboring element
+      Tr.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
+      el.CalcShape(eip, shape);
 
       // Use Tr.Elem1 transformation for u so that it matches the coefficient
       // used with the ConvectionIntegrator and/or the DGTraceIntegrator.
@@ -752,10 +759,13 @@ void DGDirichletLFIntegrator::AssembleRHSElementVect(
    for (int p = 0; p < ir->GetNPoints(); p++)
    {
       const IntegrationPoint &ip = ir->IntPoint(p);
-      IntegrationPoint eip;
 
-      Tr.Loc1.Transform(ip, eip);
-      Tr.SetIntPoint(&ip);
+      // Set the integration point in the face and the neighboring element
+      Tr.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
+
       if (dim == 1)
       {
          nor(0) = 2*eip.x - 1.0;
@@ -774,14 +784,14 @@ void DGDirichletLFIntegrator::AssembleRHSElementVect(
       {
          if (Q)
          {
-            w *= Q->Eval(Tr, ip);
+            w *= Q->Eval(*Tr.Elem1, eip);
          }
          ni.Set(w, nor);
       }
       else
       {
          nh.Set(w, nor);
-         MQ->Eval(mq, Tr, ip);
+         MQ->Eval(mq, *Tr.Elem1, eip);
          mq.MultTranspose(nh, ni);
       }
       CalcAdjugate(Tr.Elem1->Jacobian(), adjJ);
@@ -845,9 +855,12 @@ void DGElasticityDirichletLFIntegrator::AssembleRHSElementVect(
    for (int pi = 0; pi < ir->GetNPoints(); ++pi)
    {
       const IntegrationPoint &ip = ir->IntPoint(pi);
-      IntegrationPoint eip;
-      Tr.Loc1.Transform(ip, eip);
-      Tr.SetIntPoint(&ip);
+
+      // Set the integration point in the face and the neighboring element
+      Tr.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
 
       // Evaluate the Dirichlet b.c. using the face transformation.
       uD.Eval(u_dir, Tr, ip);

fem/pbilinearform.cpp

@@ -198,8 +198,9 @@ void ParBilinearForm::AssembleSharedFaces(int skip_zeros)
    for (int i = 0; i < nfaces; i++)
    {
       T = pmesh->GetSharedFaceTransformations(i);
+      int Elem2NbrNo = T->Elem2No - pmesh->GetNE();
       pfes->GetElementVDofs(T->Elem1No, vdofs1);
-      pfes->GetFaceNbrElementVDofs(T->Elem2No, vdofs2);
+      pfes->GetFaceNbrElementVDofs(Elem2NbrNo, vdofs2);
       vdofs1.Copy(vdofs_all);
       for (int j = 0; j < vdofs2.Size(); j++)
       {
@@ -216,7 +217,7 @@ void ParBilinearForm::AssembleSharedFaces(int skip_zeros)
       for (int k = 0; k < fbfi.Size(); k++)
       {
          fbfi[k]->AssembleFaceMatrix(*pfes->GetFE(T->Elem1No),
-                                     *pfes->GetFaceNbrFE(T->Elem2No),
+                                     *pfes->GetFaceNbrFE(Elem2NbrNo),
                                      *T, elemmat);
          if (keep_nbr_block)
          {

fem/pfespace.hpp

@@ -347,6 +347,8 @@ public:
    const FiniteElement *GetFaceNbrFE(int i) const;
    const FiniteElement *GetFaceNbrFaceFE(int i) const;
    const HYPRE_Int *GetFaceNbrGlobalDofMap() { return face_nbr_glob_dof_map; }
+   ElementTransformation *GetFaceNbrElementTransformation(int i) const
+   { return pmesh->GetFaceNbrElementTransformation(i); }
 
    void Lose_Dof_TrueDof_Matrix();
    void LoseDofOffsets() { dof_offsets.LoseData(); }

fem/pgridfunc.cpp

@@ -271,6 +271,7 @@ const
    {
       int fes_vdim = pfes->GetVDim();
       pfes->GetFaceNbrElementVDofs(nbr_el_no, dofs);
+      const FiniteElement *fe = pfes->GetFaceNbrFE(nbr_el_no);
       if (fes_vdim > 1)
       {
          int s = dofs.Size()/fes_vdim;
@@ -283,7 +284,17 @@ const
          face_nbr_data.GetSubVector(dofs, LocVec);
          DofVal.SetSize(dofs.Size());
       }
-      pfes->GetFaceNbrFE(nbr_el_no)->CalcShape(ip, DofVal);
+      if (fe->GetMapType() == FiniteElement::VALUE)
+      {
+         fe->CalcShape(ip, DofVal);
+      }
+      else
+      {
+         ElementTransformation *Tr =
+            pfes->GetFaceNbrElementTransformation(nbr_el_no);
+         Tr->SetIntPoint(&ip);
+         fe->CalcPhysShape(*Tr, DofVal);
+      }
    }
    else
    {
@@ -291,14 +302,175 @@ const
       fes->DofsToVDofs(vdim-1, dofs);
       DofVal.SetSize(dofs.Size());
       const FiniteElement *fe = fes->GetFE(i);
-      MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE, "invalid FE map type");
-      fe->CalcShape(ip, DofVal);
+      if (fe->GetMapType() == FiniteElement::VALUE)
+      {
+         fe->CalcShape(ip, DofVal);
+      }
+      else
+      {
+         ElementTransformation *Tr = fes->GetElementTransformation(i);
+         Tr->SetIntPoint(&ip);
+         fe->CalcPhysShape(*Tr, DofVal);
+      }
       GetSubVector(dofs, LocVec);
    }
 
    return (DofVal * LocVec);
 }
 
+void ParGridFunction::GetVectorValue(int i, const IntegrationPoint &ip,
+                                     Vector &val) const
+{
+   int nbr_el_no = i - pfes->GetParMesh()->GetNE();
+   if (nbr_el_no >= 0)
+   {
+      Array<int> dofs;
+      pfes->GetFaceNbrElementVDofs(nbr_el_no, dofs);
+      Vector loc_data;
+      face_nbr_data.GetSubVector(dofs, loc_data);
+      const FiniteElement *FElem = pfes->GetFaceNbrFE(nbr_el_no);
+      int dof = FElem->GetDof();
+      if (FElem->GetRangeType() == FiniteElement::SCALAR)
+      {
+         Vector shape(dof);
+         if (FElem->GetMapType() == FiniteElement::VALUE)
+         {
+            FElem->CalcShape(ip, shape);
+         }
+         else
+         {
+            ElementTransformation *Tr =
+               pfes->GetParMesh()->GetFaceNbrElementTransformation(nbr_el_no);
+            Tr->SetIntPoint(&ip);
+            FElem->CalcPhysShape(*Tr, shape);
+         }
+         int vdim = fes->GetVDim();
+         val.SetSize(vdim);
+         for (int k = 0; k < vdim; k++)
+         {
+            val(k) = shape * ((const double *)loc_data + dof * k);
+         }
+      }
+      else
+      {
+         int spaceDim = fes->GetMesh()->SpaceDimension();
+         DenseMatrix vshape(dof, spaceDim);
+         ElementTransformation *Tr =
+            pfes->GetParMesh()->GetFaceNbrElementTransformation(nbr_el_no);
+         Tr->SetIntPoint(&ip);
+         FElem->CalcVShape(*Tr, vshape);
+         val.SetSize(spaceDim);
+         vshape.MultTranspose(loc_data, val);
+      }
+   }
+   else
+   {
+      GridFunction::GetVectorValue(i, ip, val);
+   }
+}
+
+double ParGridFunction::GetValue(ElementTransformation &T,
+                                 const IntegrationPoint &ip,
+                                 int comp, Vector *tr) const
+{
+   // We can assume faces and edges are local
+   if (T.ElementType != ElementTransformation::ELEMENT)
+   {
+      return GridFunction::GetValue(T, ip, comp, tr);
+   }
+
+   // Check for evaluation in a local element
+   int nbr_el_no = T.ElementNo - pfes->GetParMesh()->GetNE();
+   if (nbr_el_no < 0)
+   {
+      return GridFunction::GetValue(T, ip, comp, tr);
+   }
+
+   // Evaluate using DoFs from a neighboring element
+   if (tr)
+   {
+      T.SetIntPoint(&ip);
+      T.Transform(ip, *tr);
+   }
+
+   Array<int> dofs;
+   const FiniteElement * fe = pfes->GetFaceNbrFE(nbr_el_no);
+   pfes->GetFaceNbrElementVDofs(nbr_el_no, dofs);
+
+   pfes->DofsToVDofs(comp-1, dofs);
+   Vector DofVal(dofs.Size()), LocVec;
+   if (fe->GetMapType() == FiniteElement::VALUE)
+   {
+      fe->CalcShape(ip, DofVal);
+   }
+   else
+   {
+      fe->CalcPhysShape(T, DofVal);
+   }
+   face_nbr_data.GetSubVector(dofs, LocVec);
+
+   return (DofVal * LocVec);
+}
+
+void ParGridFunction::GetVectorValue(ElementTransformation &T,
+                                     const IntegrationPoint &ip,
+                                     Vector &val, Vector *tr) const
+{
+   // We can assume faces and edges are local
+   if (T.ElementType != ElementTransformation::ELEMENT)
+   {
+      return GridFunction::GetVectorValue(T, ip, val, tr);
+   }
+
+   // Check for evaluation in a local element
+   int nbr_el_no = T.ElementNo - pfes->GetParMesh()->GetNE();
+   if (nbr_el_no < 0)
+   {
+      return GridFunction::GetVectorValue(T, ip, val, tr);
+   }
+
+   // Evaluate using DoFs from a neighboring element
+   if (tr)
+   {
+      T.SetIntPoint(&ip);
+      T.Transform(ip, *tr);
+   }
+
+   Array<int> vdofs;
+   pfes->GetFaceNbrElementVDofs(nbr_el_no, vdofs);
+   const FiniteElement *fe = pfes->GetFaceNbrFE(nbr_el_no);
+
+   int dof = fe->GetDof();
+   Vector loc_data;
+   face_nbr_data.GetSubVector(vdofs, loc_data);
+   if (fe->GetRangeType() == FiniteElement::SCALAR)
+   {
+      Vector shape(dof);
+      if (fe->GetMapType() == FiniteElement::VALUE)
+      {
+         fe->CalcShape(ip, shape);
+      }
+      else
+      {
+         fe->CalcPhysShape(T, shape);
+      }
+      int vdim = pfes->GetVDim();
+      val.SetSize(vdim);
+      for (int k = 0; k < vdim; k++)
+      {
+         val(k) = shape * ((const double *)loc_data + dof * k);
+      }
+   }
+   else
+   {
+      int spaceDim = pfes->GetMesh()->SpaceDimension();
+      DenseMatrix vshape(dof, spaceDim);
+      fe->CalcVShape(T, vshape);
+      val.SetSize(spaceDim);
+      vshape.MultTranspose(loc_data, val);
+   }
+}
+
 void ParGridFunction::ProjectCoefficient(Coefficient &coeff)
 {
    DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);

fem/pgridfunc.hpp

@@ -39,7 +39,7 @@ protected:
    Vector face_nbr_data;
 
    /** @brief Vector used as an MPI buffer to send face-neighbor data
-        in ExchangeFaceNbrData() to neighboring processors. */
+       in ExchangeFaceNbrData() to neighboring processors. */
    //TODO: Use temporary memory to avoid CUDA malloc allocation cost.
    Vector send_data;
 
@@ -209,6 +209,18 @@ public:
    double GetValue(ElementTransformation &T)
    { return GetValue(T.ElementNo, T.GetIntPoint()); }
 
+   // Redefine to handle the case when T describes a face-neighbor element
+   virtual double GetValue(ElementTransformation &T, const IntegrationPoint &ip,
+                           int comp = 0, Vector *tr = NULL) const;
+
+   virtual void GetVectorValue(int i, const IntegrationPoint &ip,
+                               Vector &val) const;
+
+   // Redefine to handle the case when T describes a face-neighbor element
+   virtual void GetVectorValue(ElementTransformation &T,
+                               const IntegrationPoint &ip,
+                               Vector &val, Vector *tr = NULL) const;
+
    using GridFunction::ProjectCoefficient;
    virtual void ProjectCoefficient(Coefficient &coeff);
 

fem/pnonlinearform.cpp

@@ -64,12 +64,13 @@ void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
       for (int i = 0; i < n_shared_faces; i++)
       {
          tr = pmesh->GetSharedFaceTransformations(i, true);
+         int Elem2NbrNo = tr->Elem2No - pmesh->GetNE();
 
          fe1 = pfes->GetFE(tr->Elem1No);
-         fe2 = pfes->GetFaceNbrFE(tr->Elem2No);
+         fe2 = pfes->GetFaceNbrFE(Elem2NbrNo);
 
          pfes->GetElementVDofs(tr->Elem1No, vdofs1);
-         pfes->GetFaceNbrElementVDofs(tr->Elem2No, vdofs2);
+         pfes->GetFaceNbrElementVDofs(Elem2NbrNo, vdofs2);
 
          el_x.SetSize(vdofs1.Size() + vdofs2.Size());
          X.GetSubVector(vdofs1, el_x.GetData());

fem/quadinterpolator.cpp

@@ -150,7 +150,6 @@ void QuadratureInterpolator::Eval3D(
    const int nq = maps.nqpt;
    const int ND = T_ND ? T_ND : nd;
    const int NQ = T_NQ ? T_NQ : nq;
-   const int NMAX = NQ > ND ? NQ : ND;
    const int VDIM = T_VDIM ? T_VDIM : vdim;
    MFEM_VERIFY(ND <= MAX_ND3D, "");
    MFEM_VERIFY(NQ <= MAX_NQ3D, "");
@@ -161,24 +160,22 @@ void QuadratureInterpolator::Eval3D(
    auto val = Reshape(q_val.Write(), NQ, VDIM, NE);
    auto der = Reshape(q_der.Write(), NQ, VDIM, 3, NE);
    auto det = Reshape(q_det.Write(), NQ, NE);
-   MFEM_FORALL_2D(e, NE, NMAX, 1, 1,
+   MFEM_FORALL(e, NE,
    {
       const int ND = T_ND ? T_ND : nd;
       const int NQ = T_NQ ? T_NQ : nq;
       const int VDIM = T_VDIM ? T_VDIM : vdim;
       constexpr int max_ND = T_ND ? T_ND : MAX_ND3D;
       constexpr int max_VDIM = T_VDIM ? T_VDIM : MAX_VDIM3D;
-      MFEM_SHARED double s_E[max_VDIM*max_ND];
-      MFEM_FOREACH_THREAD(d, x, ND)
+      double s_E[max_VDIM*max_ND];
+      for (int d = 0; d < ND; d++)
       {
          for (int c = 0; c < VDIM; c++)
          {
             s_E[c+d*VDIM] = E(d,c,e);
          }
       }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(q, x, NQ)
+      for (int q = 0; q < NQ; ++q)
       {
          if (eval_flags & VALUES)
          {

fem/quadinterpolator_face.cpp

@@ -495,9 +495,8 @@ void FaceQuadratureInterpolator::Mult(
    }
 }
 
-
-void FaceQuadratureInterpolator::Values(const Vector &e_vec,
-                                        Vector &q_val) const
+void FaceQuadratureInterpolator::Values(
+   const Vector &e_vec, Vector &q_val) const
 {
    Vector q_der, q_det, q_nor;
    Mult(e_vec, VALUES, q_val, q_der, q_det, q_nor);

fem/tmop.cpp

@@ -177,24 +177,6 @@ double TMOP_Metric_SSA2D::EvalW(const DenseMatrix &Jpt) const
    return Mat.FNorm2();
 }
 
-// mu_85 = |T-T'|^2, where T'= |T|*I/sqrt(2)
-double TMOP_Metric_SS2D::EvalW(const DenseMatrix &Jpt) const
-{
-   MFEM_VERIFY(Jtr != NULL,
-               "Requires a target Jacobian, use SetTargetJacobian().");
-
-   DenseMatrix Id(2,2);
-   DenseMatrix Mat(2,2);
-   Mat = Jpt;
-
-   Id(0,0) = 1; Id(0,1) = 0;
-   Id(1,0) = 0; Id(1,1) = 1;
-   Id *= Mat.FNorm()/pow(2,0.5);
-
-   Mat.Add(-1.,Id);
-   return Mat.FNorm2();
-}
-
 double TMOP_Metric_002::EvalW(const DenseMatrix &Jpt) const
 {
    ie.SetJacobian(Jpt.GetData());
@@ -484,6 +466,24 @@ void TMOP_Metric_077::AssembleH(const DenseMatrix &Jpt,
    ie.Assemble_TProd(weight * I2inv_sq / I2, ie.Get_dI2(), A.GetData());
 }
 
+// mu_85 = |T-T'|^2, where T'= |T|*I/sqrt(2)
+double TMOP_Metric_085::EvalW(const DenseMatrix &Jpt) const
+{
+   MFEM_VERIFY(Jtr != NULL,
+               "Requires a target Jacobian, use SetTargetJacobian().");
+
+   DenseMatrix Id(2,2);
+   DenseMatrix Mat(2,2);
+   Mat = Jpt;
+
+   Id(0,0) = 1; Id(0,1) = 0;
+   Id(1,0) = 0; Id(1,1) = 1;
+   Id *= Mat.FNorm()/pow(2,0.5);
+
+   Mat.Add(-1.,Id);
+   return Mat.FNorm2();
+}
+
 double TMOP_Metric_211::EvalW(const DenseMatrix &Jpt) const
 {
    // mu_211 = (det(J) - 1)^2 - det(J) + (det(J)^2 + eps)^{1/2}
@@ -1058,7 +1058,6 @@ void DiscreteAdaptTC::SetDiscreteTargetBase(const GridFunction &tspec_)
    // make a copy of tspec->tspec_temp, increase its size, and
    // copy data from tspec_temp -> tspec, then add new entries
    Vector tspec_temp = tspec;
-   tspec.UseDevice(true);
    tspec.SetSize(ncomp*dof_cnt);
 
    for (int i = 0; i < tspec_temp.Size(); i++)
@@ -1215,7 +1214,6 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
          Array<int> dofs;
          DenseMatrix D_rho(dim), Q_phi(dim), R_theta(dim);
          tspec_fesv->GetElementVDofs(e_id, dofs);
-         tspec.UseDevice(true);
          tspec.GetSubVector(dofs, tspec_vals);
 
          for (int i = 0; i < ir.GetNPoints(); i++)

fem/tmop.hpp

@@ -162,21 +162,6 @@ public:
    { MFEM_ABORT("Not implemented"); }
 };
 
-/// Shape+Size metric, 2D.
-class TMOP_Metric_SS2D : public TMOP_QualityMetric
-{
-public:
-   // W = 0.5 (1 - cos(theta_Jpr - theta_Jtr)).
-   virtual double EvalW(const DenseMatrix &Jpt) const;
-
-   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
-   { MFEM_ABORT("Not implemented"); }
-
-   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
-                          const double weight, DenseMatrix &A) const
-   { MFEM_ABORT("Not implemented"); }
-};
-
 /// Shape, ideal barrier metric, 2D
 class TMOP_Metric_002 : public TMOP_QualityMetric
 {
@@ -331,6 +316,21 @@ public:
 
 };
 
+/// Shape & orientation metric, 2D.
+class TMOP_Metric_085 : public TMOP_QualityMetric
+{
+public:
+   // W = |T-T'|^2, where T'= |T|*I/sqrt(2).
+   virtual double EvalW(const DenseMatrix &Jpt) const;
+
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+   { MFEM_ABORT("Not implemented"); }
+
+   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                          const double weight, DenseMatrix &A) const
+   { MFEM_ABORT("Not implemented"); }
+};
+
 /// Untangling metric, 2D
 class TMOP_Metric_211 : public TMOP_QualityMetric
 {

fem/tmop_tools.cpp

@@ -33,11 +33,10 @@ void AdvectorCG::ComputeAtNewPosition(const Vector &new_nodes,
    const int pnt_cnt = new_field.Size()/ncomp;
 
    new_field = field0;
-   new_field.HostReadWrite();
-   Vector new_field_temp;
+
    for (int i = 0; i < ncomp; i++)
    {
-      new_field_temp.MakeRef(new_field, i*pnt_cnt, pnt_cnt);
+      Vector new_field_temp(new_field.GetData()+i*pnt_cnt, pnt_cnt);
       ComputeAtNewPositionScalar(new_nodes, new_field_temp);
    }
 
@@ -95,7 +94,6 @@ void AdvectorCG::ComputeAtNewPositionScalar(const Vector &new_nodes,
    double v_max = 0.0;
    const int s = new_field.Size();
 
-   u.HostReadWrite();
    for (int i = 0; i < s; i++)
    {
       double vel = 0.;
@@ -151,7 +149,6 @@ void AdvectorCG::ComputeAtNewPositionScalar(const Vector &new_nodes,
 #endif
 
    // Trim the overshoots and undershoots.
-   new_field.HostReadWrite();
    for (int i = 0; i < s; i++)
    {
       if (new_field(i) < glob_minv) { new_field(i) = glob_minv; }
@@ -356,20 +353,59 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       energy_in = nlf->GetEnergy(x);
    }
 
-   const bool have_b = (b.Size() == Height());
-
    const int NE = fes->GetMesh()->GetNE(), dim = fes->GetFE(0)->GetDim(),
              dof = fes->GetFE(0)->GetDof(), nsp = ir.GetNPoints();
    Array<int> xdofs(dof * dim);
    DenseMatrix Jpr(dim), dshape(dof, dim), pos(dof, dim);
    Vector posV(pos.Data(), dof * dim);
+   Vector x_out_loc(fes->GetVSize());
 
-   Vector x_out(x.Size()), x_out_loc(fes->GetVSize());
+   if (serial)
+   {
+      const SparseMatrix *cP = fes->GetConformingProlongation();
+      if (!cP) { x_out_loc = x; }
+      else     { cP->Mult(x, x_out_loc); }
+   }
+#ifdef MFEM_USE_MPI
+   else
+   {
+      fes->GetProlongationMatrix()->Mult(x, x_out_loc);
+   }
+#endif
+
+   double min_detJ = infinity();
+   for (int i = 0; i < NE; i++)
+   {
+      fes->GetElementVDofs(i, xdofs);
+      x_out_loc.GetSubVector(xdofs, posV);
+
+      for (int j = 0; j < nsp; j++)
+      {
+         fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
+         MultAtB(pos, dshape, Jpr);
+         min_detJ = std::min(min_detJ, Jpr.Det());
+      }
+   }
+   double min_detJ_all = min_detJ;
+#ifdef MFEM_USE_MPI
+   if (parallel)
+   {
+      MPI_Allreduce(&min_detJ, &min_detJ_all, 1, MPI_DOUBLE, MPI_MIN,
+                    p_nlf->ParFESpace()->GetComm());
+   }
+#endif
+   bool untangling = false;
+   if (min_detJ_all <= 0) { untangling = true; }
+
+   const bool have_b = (b.Size() == Height());
+
+   Vector x_out(x.Size());
    bool x_out_ok = false;
    double scale = 1.0, energy_out = 0.0;
    double norm0 = Norm(r);
 
-   // Decreases the scaling of the update until the new mesh is valid.
+   const double detJ_factor = (solver_type == 1) ? 0.25 : 0.5;
+
    for (int i = 0; i < 12; i++)
    {
       add(x, -scale, c, x_out);
@@ -387,35 +423,39 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       }
 #endif
 
-      int jac_ok = 1;
-      for (int i = 0; i < NE; i++)
+      // Check det(Jpr) > 0.
+      if (!untangling)
       {
-         fes->GetElementVDofs(i, xdofs);
-         x_out_loc.GetSubVector(xdofs, posV);
-         for (int j = 0; j < nsp; j++)
+         int jac_ok = 1;
+         for (int i = 0; i < NE; i++)
          {
-            fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
-            MultAtB(pos, dshape, Jpr);
-            if (Jpr.Det() <= 0.0) { jac_ok = 0; goto break2; }
+            fes->GetElementVDofs(i, xdofs);
+            x_out_loc.GetSubVector(xdofs, posV);
+            for (int j = 0; j < nsp; j++)
+            {
+               fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
+               MultAtB(pos, dshape, Jpr);
+               if (Jpr.Det() <= 0.0) { jac_ok = 0; goto break2; }
+            }
          }
-      }
 
-   break2:
-      int jac_ok_all = jac_ok;
+      break2:
+         int jac_ok_all = jac_ok;
 #ifdef MFEM_USE_MPI
-      if (parallel)
-      {
-         MPI_Allreduce(&jac_ok, &jac_ok_all, 1, MPI_INT, MPI_LAND,
-                       p_nlf->ParFESpace()->GetComm());
-      }
+         if (parallel)
+         {
+            MPI_Allreduce(&jac_ok, &jac_ok_all, 1, MPI_INT, MPI_LAND,
+                          p_nlf->ParFESpace()->GetComm());
+         }
 #endif
 
-      if (jac_ok_all == 0)
-      {
-         if (print_level >= 0)
-         { mfem::out << "Scale = " << scale << " Neg det(J) found.\n"; }
-         scale *= 0.5; continue;
-      }
+         if (jac_ok_all == 0)
+         {
+            if (print_level >= 0)
+            { mfem::out << "Scale = " << scale << " Neg det(J) found.\n"; }
+            scale *= detJ_factor; continue;
+         }
+      } // endif(!untangling)
 
       ProcessNewState(x_out);
       if (serial)
@@ -428,25 +468,37 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
          energy_out = p_nlf->GetParGridFunctionEnergy(x_out_loc);
       }
 #endif
-      if (energy_out > 1.2*energy_in || std::isnan(energy_out) != 0)
-      {
-         if (print_level >= 0)
-         { mfem::out << "Scale = " << scale << " Increasing energy.\n"; }
-         scale *= 0.5; continue;
-      }
 
-      oper->Mult(x_out, r);
-      if (have_b) { r -= b; }
-      double norm = Norm(r);
-
-      if (norm > 1.2*norm0)
+      if (untangling)
       {
-         if (print_level >= 0)
-         { mfem::out << "Scale = " << scale << " Norm increased.\n"; }
-         scale *= 0.5; continue;
+         if (energy_out > energy_in || std::isnan(energy_out) != 0)
+         {
+            scale *= 0.5;
+         }
+         else { x_out_ok = true; break; }
       }
-      else { x_out_ok = true; break; }
-   }
+      else
+      {
+         if (energy_out > 1.2*energy_in || std::isnan(energy_out) != 0)
+         {
+            if (print_level >= 0)
+            { mfem::out << "Scale = " << scale << " Increasing energy.\n"; }
+            scale *= 0.5; continue;
+         }
+
+         oper->Mult(x_out, r);
+         if (have_b) { r -= b; }
+         double norm = Norm(r);
+
+         if (norm > 1.2*norm0)
+         {
+            if (print_level >= 0)
+            { mfem::out << "Scale = " << scale << " Norm increased.\n"; }
+            scale *= 0.5; continue;
+         }
+         else { x_out_ok = true; break; }
+      } // endif (untangling)
+   } // enddo (i)
 
    if (print_level >= 0)
    {
@@ -573,105 +625,6 @@ void TMOPNewtonSolver::UpdateDiscreteTC(const TMOP_Integrator &ti,
    }
 }
 
-double TMOPDescentNewtonSolver::ComputeScalingFactor(const Vector &x,
-                                                     const Vector &b) const
-{
-   const FiniteElementSpace *fes = NULL;
-   double energy_in = 0.0;
-#ifdef MFEM_USE_MPI
-   const ParNonlinearForm *p_nlf = dynamic_cast<const ParNonlinearForm *>(oper);
-   MFEM_VERIFY(!(parallel && p_nlf == NULL), "Invalid Operator subclass.");
-   if (parallel)
-   {
-      fes = p_nlf->FESpace();
-      energy_in = p_nlf->GetEnergy(x);
-   }
-#endif
-   const bool serial = !parallel;
-   const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
-   MFEM_VERIFY(!(serial && nlf == NULL), "Invalid Operator subclass.");
-   if (serial)
-   {
-      fes = nlf->FESpace();
-      energy_in = nlf->GetEnergy(x);
-   }
-
-   const int NE = fes->GetMesh()->GetNE(), dim = fes->GetFE(0)->GetDim(),
-             dof = fes->GetFE(0)->GetDof(), nsp = ir.GetNPoints();
-   Array<int> xdofs(dof * dim);
-   DenseMatrix Jpr(dim), dshape(dof, dim), pos(dof, dim);
-   Vector posV(pos.Data(), dof * dim);
-   Vector x_loc(fes->GetVSize());
-
-   double min_detJ = infinity();
-   for (int i = 0; i < NE; i++)
-   {
-      fes->GetElementVDofs(i, xdofs);
-      // TODO x_loc doesn't have valid values here!
-      MFEM_ABORT("This function has to be fixed!");
-      x_loc.GetSubVector(xdofs, posV);
-
-      for (int j = 0; j < nsp; j++)
-      {
-         fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
-         MultAtB(pos, dshape, Jpr);
-         min_detJ = std::min(min_detJ, Jpr.Det());
-      }
-   }
-   double min_detJ_all = min_detJ;
-#ifdef MFEM_USE_MPI
-   if (parallel)
-   {
-      MPI_Allreduce(&min_detJ, &min_detJ_all, 1, MPI_DOUBLE, MPI_MIN,
-                    p_nlf->ParFESpace()->GetComm());
-   }
-#endif
-   if (print_level >= 0)
-   {
-      mfem::out << "Minimum det(J) = " << min_detJ_all << '\n';
-   }
-
-   Vector x_out(x.Size());
-   bool x_out_ok = false;
-   double scale = 1.0, energy_out = 0.0;
-
-   for (int i = 0; i < 7; i++)
-   {
-      add(x, -scale, c, x_out);
-      if (serial)
-      {
-         const SparseMatrix *cP = fes->GetConformingProlongation();
-         if (!cP) { x_loc = x_out; }
-         else     { cP->Mult(x_out,x_loc); }
-         energy_out = nlf->GetGridFunctionEnergy(x_loc);
-      }
-#ifdef MFEM_USE_MPI
-      else
-      {
-         fes->GetProlongationMatrix()->Mult(x_out, x_loc);
-         energy_out = p_nlf->GetParGridFunctionEnergy(x_loc);
-      }
-#endif
-
-      if (energy_out > energy_in || std::isnan(energy_out) != 0)
-      {
-         scale *= 0.5;
-      }
-      else { x_out_ok = true; break; }
-   }
-
-   if (print_level >= 0)
-   {
-      mfem::out << "Energy decrease: "
-                << (energy_in - energy_out) / energy_in * 100.0
-                << "% with " << scale << " scaling.\n";
-   }
-
-   if (x_out_ok == false) { return 0.0; }
-
-   return scale;
-}
-
 #ifdef MFEM_USE_MPI
 // Metric values are visualized by creating an L2 finite element functions and
 // computing the metric values at the nodes.

fem/tmop_tools.hpp

@@ -109,9 +109,11 @@ public:
 };
 #endif
 
-class TMOPNewtonSolver : public NewtonSolver
+class TMOPNewtonSolver : public LBFGSSolver
 {
 protected:
+   // 0 - Newton, 1 - LBFGS.
+   int solver_type;
    bool parallel;
 
    // Quadrature points that are checked for negative Jacobians etc.
@@ -121,29 +123,42 @@ protected:
 
 public:
 #ifdef MFEM_USE_MPI
-   TMOPNewtonSolver(MPI_Comm comm, const IntegrationRule &irule)
-      : NewtonSolver(comm), parallel(true), ir(irule) { }
+   TMOPNewtonSolver(MPI_Comm comm, const IntegrationRule &irule, int type = 0)
+      : LBFGSSolver(comm), solver_type(type), parallel(true), ir(irule) { }
 #endif
-   TMOPNewtonSolver(const IntegrationRule &irule)
-      : NewtonSolver(), parallel(false), ir(irule) { }
+   TMOPNewtonSolver(const IntegrationRule &irule, int type = 0)
+      : LBFGSSolver(), solver_type(type), parallel(false), ir(irule) { }
 
    virtual double ComputeScalingFactor(const Vector &x, const Vector &b) const;
 
    virtual void ProcessNewState(const Vector &x) const;
-};
 
-/// Allows negative Jacobians. Used for untangling.
-class TMOPDescentNewtonSolver : public TMOPNewtonSolver
-{
-public:
-#ifdef MFEM_USE_MPI
-   TMOPDescentNewtonSolver(MPI_Comm comm, const IntegrationRule &irule)
-      : TMOPNewtonSolver(comm, irule) { }
-#endif
-   TMOPDescentNewtonSolver(const IntegrationRule &irule)
-      : TMOPNewtonSolver(irule) { }
+   virtual void Mult(const Vector &b, Vector &x) const
+   {
+      if (solver_type == 0)
+      {
+         NewtonSolver::Mult(b, x);
+      }
+      else if (solver_type == 1)
+      {
+         LBFGSSolver::Mult(b, x);
+      }
+      else { MFEM_ABORT("Invalid type"); }
+   }
 
-   virtual double ComputeScalingFactor(const Vector &x, const Vector &b) const;
+   virtual void SetSolver(Solver &solver)
+   {
+      if (solver_type == 0)
+      {
+         NewtonSolver::SetSolver(solver);
+      }
+      else if (solver_type == 1)
+      {
+         LBFGSSolver::SetSolver(solver);
+      }
+      else { MFEM_ABORT("Invalid type"); }
+   }
+   virtual void SetPreconditioner(Solver &pr) { SetSolver(pr); }
 };
 
 void vis_tmop_metric_s(int order, TMOP_QualityMetric &qm,

general/device.cpp

@@ -61,11 +61,15 @@ Device Device::device_singleton;
 bool Device::device_env = false;
 bool Device::mem_host_env = false;
 bool Device::mem_device_env = false;
-#ifdef MFEM_USE_UMPIRE
-bool Device::use_umpire = true;
-#endif
 
-Device::Device()
+Device::Device() : mode(Device::SEQUENTIAL),
+   backends(Backend::CPU),
+   destroy_mm(false),
+   mpi_gpu_aware(false),
+   host_mem_type(MemoryType::HOST),
+   host_mem_class(MemoryClass::HOST),
+   device_mem_type(MemoryType::HOST),
+   device_mem_class(MemoryClass::HOST)
 {
    if (getenv("MFEM_MEMORY") && !mem_host_env && !mem_device_env)
    {
@@ -131,7 +135,7 @@ Device::Device()
       {
          MFEM_ABORT("Unknown memory backend!");
       }
-      mm.Configure(host_mem_type, device_mem_type, device_mem_type);
+      mm.Configure(host_mem_type, device_mem_type);
    }
 
    if (getenv("MFEM_DEVICE"))
@@ -161,8 +165,6 @@ Device::~Device()
    Get().host_mem_class = MemoryClass::HOST;
    Get().device_mem_type = MemoryType::HOST;
    Get().device_mem_class = MemoryClass::HOST;
-   Get().device_temp_mem_type = MemoryType::HOST;
-   Get().device_temp_mem_class = MemoryClass::HOST;
 }
 
 void Device::Configure(const std::string &device, const int dev)
@@ -258,10 +260,6 @@ void Device::Print(std::ostream &out)
    if (Device::Allows(Backend::DEVICE_MASK))
    {
       out << ',' << MemoryTypeName[static_cast<int>(device_mem_type)];
-      if (device_temp_mem_type != device_mem_type)
-      {
-         out << ',' << MemoryTypeName[static_cast<int>(device_temp_mem_type)];
-      }
    }
    out << std::endl;
 }
@@ -274,8 +272,7 @@ void Device::UpdateMemoryTypeAndClass()
 
 #ifdef MFEM_USE_UMPIRE
    // If MFEM has been compiled with Umpire support, use it as the default
-   // TODO TMS: temporary
-   //if (!mem_host_env && use_umpire) { host_mem_type = MemoryType::HOST_UMPIRE; }
+   if (!mem_host_env) { host_mem_type = MemoryType::HOST_UMPIRE; }
 #endif
 
    // Enable the device memory type
@@ -299,16 +296,11 @@ void Device::UpdateMemoryTypeAndClass()
          }
          else
          {
-#ifdef MFEM_USE_UMPIRE
-            if (use_umpire)
-            {
-               device_mem_type = MemoryType::DEVICE_UMPIRE;
-            }
-            else
+#ifndef MFEM_USE_UMPIRE
+            device_mem_type = MemoryType::DEVICE;
+#else
+            device_mem_type = MemoryType::DEVICE_UMPIRE;
 #endif
-            {
-               device_mem_type = MemoryType::DEVICE;
-            }
          }
       }
       device_mem_class = MemoryClass::DEVICE;
@@ -328,20 +320,8 @@ void Device::UpdateMemoryTypeAndClass()
       device_mem_type = MemoryType::DEVICE_DEBUG;
    }
 
-   // Setup device_temp_mem_{type,class}
-   switch (device_mem_type)
-   {
-      case MemoryType::DEVICE_UMPIRE:
-         device_temp_mem_type = MemoryType::DEVICE_TEMP_UMPIRE;
-         break;
-      default:
-         device_temp_mem_type = device_mem_type;
-         break;
-   }
-   device_temp_mem_class = device_mem_class;
-
    // Update the memory manager with the new settings
-   mm.Configure(host_mem_type, device_mem_type, device_temp_mem_type);
+   mm.Configure(host_mem_type, device_mem_type);
 }
 
 void Device::Enable()

general/device.hpp

@@ -119,26 +119,20 @@ private:
 
    static bool device_env, mem_host_env, mem_device_env;
    static Device device_singleton;
-#ifdef MFEM_USE_UMPIRE
-   static bool use_umpire;
-#endif
 
-   MODES mode{Device::SEQUENTIAL};
+   MODES mode;
    int dev = 0; ///< Device ID of the configured device.
    int ngpu = -1; ///< Number of detected devices; -1: not initialized.
-   unsigned long backends{Backend::CPU}; ///< Bitwise-OR of all configured backends.
+   unsigned long backends; ///< Bitwise-OR of all configured backends.
    /// Set to true during configuration, except in 'device_singleton'.
-   bool destroy_mm{false};
-   bool mpi_gpu_aware{false};
+   bool destroy_mm;
+   bool mpi_gpu_aware;
 
-   MemoryType host_mem_type{MemoryType::HOST};      ///< Current Host MemoryType
-   MemoryClass host_mem_class{MemoryClass::HOST};    ///< Current Host MemoryClass
+   MemoryType host_mem_type;      ///< Current Host MemoryType
+   MemoryClass host_mem_class;    ///< Current Host MemoryClass
 
-   MemoryType device_mem_type{MemoryType::HOST};    ///< Current Device MemoryType
-   MemoryClass device_mem_class{MemoryClass::HOST};  ///< Current Device MemoryClass
-
-   MemoryType device_temp_mem_type{MemoryType::HOST};    ///< Current Device MemoryType
-   MemoryClass device_temp_mem_class{MemoryClass::HOST};  ///< Current Device MemoryClass
+   MemoryType device_mem_type;    ///< Current Device MemoryType
+   MemoryClass device_mem_class;  ///< Current Device MemoryClass
 
    char *device_option = NULL;
    Device(Device const&);
@@ -179,6 +173,14 @@ public:
        @note This object should be destroyed after all other MFEM objects that
        use the Device are destroyed. */
    Device(const std::string &device, const int dev = 0)
+      : mode(Device::SEQUENTIAL),
+        backends(Backend::CPU),
+        destroy_mm(false),
+        mpi_gpu_aware(false),
+        host_mem_type(MemoryType::HOST),
+        host_mem_class(MemoryClass::HOST),
+        device_mem_type(MemoryType::HOST),
+        device_mem_class(MemoryClass::HOST)
    { Configure(device, dev); }
 
    /// Destructor.
@@ -258,24 +260,10 @@ public:
    /** @deprecated Use GetDeviceMemoryClass() instead. */
    static inline MemoryClass GetMemoryClass() { return Get().device_mem_class; }
 
-   /** @brief Get the current Device Temporary MemoryType. This is the MemoryType used by
-       MFEM classes when allocating temporary memory to be used with device kernels.
-   */
-   static inline MemoryType GetDeviceTempMemoryType() { return Get().device_temp_mem_type; }
-
-   /** @brief Get the current Device Temporary MemoryClass. This is the MemoryClass used
-       by MFEM device kernels when they need to access temporary Memory objects. */
-   static inline MemoryClass GetDeviceTempMemoryClass() { return Get().device_temp_mem_class; }
-
    static void SetGPUAwareMPI(const bool force = true)
    { Get().mpi_gpu_aware = force; }
 
    static bool GetGPUAwareMPI() { return Get().mpi_gpu_aware; }
-
-#ifdef MFEM_USE_UMPIRE
-   static bool UseUmpire() { return Get().use_umpire; }
-   static void UseUmpire(bool use) { Get().use_umpire = use; }
-#endif
 };
 
 

general/mem_manager.cpp

@@ -67,19 +67,15 @@ MemoryType MemoryManager::GetDualMemoryType_(MemoryType mt)
 {
    switch (mt)
    {
-      // TODO TMS: temporary
-      case MemoryType::HOST:           return MemoryType::DEVICE_UMPIRE;
+      case MemoryType::HOST:           return MemoryType::DEVICE;
       case MemoryType::HOST_32:        return MemoryType::DEVICE;
       case MemoryType::HOST_64:        return MemoryType::DEVICE;
       case MemoryType::HOST_DEBUG:     return MemoryType::DEVICE_DEBUG;
-      //case MemoryType::HOST_UMPIRE:    return MemoryType::DEVICE_UMPIRE;
+      case MemoryType::HOST_UMPIRE:    return MemoryType::DEVICE_UMPIRE;
       case MemoryType::MANAGED:        return MemoryType::MANAGED;
       case MemoryType::DEVICE:         return MemoryType::HOST;
       case MemoryType::DEVICE_DEBUG:   return MemoryType::HOST_DEBUG;
-      //case MemoryType::DEVICE_UMPIRE:  return MemoryType::HOST_UMPIRE;
-      case MemoryType::DEVICE_UMPIRE:  return MemoryType::HOST;
-      //case MemoryType::DEVICE_TEMP_UMPIRE:  return MemoryType::HOST_UMPIRE;
-      case MemoryType::DEVICE_TEMP_UMPIRE:  return MemoryType::HOST;
+      case MemoryType::DEVICE_UMPIRE:  return MemoryType::HOST_UMPIRE;
       default: mfem_error("Unknown memory type!");
    }
    MFEM_VERIFY(false,"");
@@ -92,9 +88,6 @@ static void MFEM_VERIFY_TYPES(const MemoryType h_mt, const MemoryType d_mt)
    MFEM_ASSERT(IsDeviceMemory(d_mt),"");
    const bool sync =
       (h_mt == MemoryType::HOST_UMPIRE && d_mt == MemoryType::DEVICE_UMPIRE) ||
-      (h_mt == MemoryType::HOST_UMPIRE && d_mt == MemoryType::DEVICE_TEMP_UMPIRE) ||
-      (h_mt == MemoryType::HOST && d_mt == MemoryType::DEVICE_UMPIRE) ||
-      (h_mt == MemoryType::HOST && d_mt == MemoryType::DEVICE_TEMP_UMPIRE) ||
       (h_mt == MemoryType::HOST_DEBUG && d_mt == MemoryType::DEVICE_DEBUG) ||
       (h_mt == MemoryType::MANAGED && d_mt == MemoryType::MANAGED) ||
       (h_mt == MemoryType::HOST_64 && d_mt == MemoryType::DEVICE) ||
@@ -468,96 +461,48 @@ public:
 #ifndef MFEM_USE_UMPIRE
 class UmpireHostMemorySpace : public NoHostMemorySpace { };
 class UmpireDeviceMemorySpace : public NoDeviceMemorySpace { };
-class UmpireDeviceTempMemorySpace : public NoDeviceMemorySpace { };
 #else
-
-// TODO TMS: replace with um.hasAllocatorId(int) when it exists
-bool UmpireHasId(const umpire::ResourceManager & rm, int id)
-{
-   const auto & ids = rm.getAllocatorIds();
-   return std::find(ids.begin(), ids.end(), id) != ids.end();
-}
-
 /// The Umpire host memory space
 class UmpireHostMemorySpace : public HostMemorySpace
 {
 private:
+   const char *name;
    umpire::ResourceManager &rm;
    umpire::Allocator h_allocator;
-   bool owns_allocator{false};
+   umpire::strategy::AllocationStrategy *strat;
 public:
-   // TODO: this only releases unused memory
-   ~UmpireHostMemorySpace() { if (owns_allocator) { h_allocator.release(); } }
-   UmpireHostMemorySpace(): HostMemorySpace(),
-      rm(umpire::ResourceManager::getInstance())
-   {
-      const int id = MemoryManager::GetUmpireHostAllocatorId();
-      if (!UmpireHasId(rm, id))
-      {
-         h_allocator = rm.makeAllocator<umpire::strategy::DynamicPool>("MFEM_HOST",
-                                                                       rm.getAllocator("HOST"));
-         owns_allocator = true;
-      }
-      else
-      {
-         h_allocator = rm.getAllocator(id);
-      }
-      MemoryManager::SetUmpireHostAllocatorId(id);
-   }
+   ~UmpireHostMemorySpace() { h_allocator.release(); }
+   UmpireHostMemorySpace():
+      HostMemorySpace(),
+      name(mm.GetUmpireAllocatorHostName()),
+      rm(umpire::ResourceManager::getInstance()),
+      h_allocator(rm.isAllocator(name)? rm.getAllocator(name):
+                  rm.makeAllocator<umpire::strategy::DynamicPool>
+                  (name, rm.getAllocator("HOST"))),
+      strat(h_allocator.getAllocationStrategy()) { }
    void Alloc(void **ptr, size_t bytes) { *ptr = h_allocator.allocate(bytes); }
    void Dealloc(void *ptr) { h_allocator.deallocate(ptr); }
    void Insert(void *ptr, size_t bytes)
-   { mfem_error("UmpireHostMemorySpace::Insert is unsupported"); }
+   { rm.registerAllocation(ptr, {ptr, bytes, strat}); }
 };
 
 /// The Umpire device memory space
 #ifdef MFEM_USE_CUDA
-class UmpireDeviceMemorySpaceImpl : public DeviceMemorySpace
+class UmpireDeviceMemorySpace : public DeviceMemorySpace
 {
-public:
-   enum class AllocatorType { TEMPORARY, PERMANENT };
 private:
+   const char *name;
    umpire::ResourceManager &rm;
    umpire::Allocator d_allocator;
-   bool owns_allocator{false};
-
-   int SetupAllocator(int possible_id, const char * allocator_name)
-   {
-      if (!UmpireHasId(rm, possible_id))
-      {
-         d_allocator = rm.makeAllocator<umpire::strategy::DynamicPool>(allocator_name,
-                                                                       rm.getAllocator("DEVICE"));
-         owns_allocator = true;
-      }
-      else
-      {
-         d_allocator = rm.getAllocator(possible_id);
-      }
-
-      return d_allocator.getId();
-   }
 public:
-   // TODO: this only releases unused memory
-   ~UmpireDeviceMemorySpaceImpl() { if (owns_allocator) { d_allocator.release(); } }
-   UmpireDeviceMemorySpaceImpl(AllocatorType t): DeviceMemorySpace(),
-      rm(umpire::ResourceManager::getInstance())
-   {
-      switch (t)
-      {
-         case AllocatorType::PERMANENT:
-            MemoryManager::SetUmpireDeviceAllocatorId(SetupAllocator(
-                                                         MemoryManager::GetUmpireDeviceAllocatorId(),
-                                                         "MFEM_DEVICE"));
-            break;
-         case AllocatorType::TEMPORARY:
-            MemoryManager::SetUmpireDeviceTempAllocatorId(SetupAllocator(
-                                                             MemoryManager::GetUmpireDeviceTempAllocatorId(),
-                                                             "MFEM_DEVICE_TEMPORARY"));
-            break;
-         default:
-            mfem_error("Unknown Umpire AllocatorType");
-      }
-   }
+   ~UmpireDeviceMemorySpace() { d_allocator.release(); }
+   UmpireDeviceMemorySpace():
+      DeviceMemorySpace(),
+      name(mm.GetUmpireAllocatorDeviceName()),
+      rm(umpire::ResourceManager::getInstance()),
+      d_allocator(rm.isAllocator(name)? rm.getAllocator(name):
+                  rm.makeAllocator<umpire::strategy::DynamicPool>
+                  (name, rm.getAllocator("DEVICE"))) { }
    void Alloc(Memory &base) { base.d_ptr = d_allocator.allocate(base.bytes); }
    void Dealloc(Memory &base) { d_allocator.deallocate(base.d_ptr); }
    void *HtoD(void *dst, const void *src, size_t bytes)
@@ -591,23 +536,8 @@ public:
       //rm.copy(dst, const_cast<void*>(src), bytes); return dst;
    }
 };
-
-class UmpireDeviceMemorySpace : public UmpireDeviceMemorySpaceImpl
-{
-public:
-   UmpireDeviceMemorySpace() : UmpireDeviceMemorySpaceImpl(
-         AllocatorType::PERMANENT) {}
-};
-
-class UmpireDeviceTempMemorySpace : public UmpireDeviceMemorySpaceImpl
-{
-public:
-   UmpireDeviceTempMemorySpace() : UmpireDeviceMemorySpaceImpl(
-         AllocatorType::TEMPORARY) {}
-};
 #else
 class UmpireDeviceMemorySpace : public NoDeviceMemorySpace { };
-class UmpireDeviceTempMemorySpace : public NoDeviceMemorySpace { };
 #endif // MFEM_USE_CUDA
 #endif // MFEM_USE_UMPIRE
 
@@ -638,7 +568,7 @@ public:
       host[static_cast<int>(MT::HOST_64)] = new Aligned64HostMemorySpace();
       // HOST_DEBUG is delayed, as it reroutes signals
       host[static_cast<int>(MT::HOST_DEBUG)] = nullptr;
-      host[static_cast<int>(MT::HOST_UMPIRE)] = nullptr;
+      host[static_cast<int>(MT::HOST_UMPIRE)] = new UmpireHostMemorySpace();
       host[static_cast<int>(MT::MANAGED)] = new UvmHostMemorySpace();
 
       // Filling the device memory backends, shifting with the device size
@@ -680,12 +610,8 @@ public:
 private:
    HostMemorySpace* NewHostCtrl(const MemoryType mt)
    {
-      switch (mt)
-      {
-         case MT::HOST_DEBUG: return new MmuHostMemorySpace();
-         case MT::HOST_UMPIRE: return new UmpireHostMemorySpace();
-         default: MFEM_ABORT("Unknown host memory controller!");
-      }
+      if (mt == MT::HOST_DEBUG) { return new MmuHostMemorySpace(); }
+      MFEM_ABORT("Unknown host memory controller!");
       return nullptr;
    }
 
@@ -694,7 +620,6 @@ private:
       switch (mt)
       {
          case MT::DEVICE_UMPIRE: return new UmpireDeviceMemorySpace();
-         case MT::DEVICE_TEMP_UMPIRE: return new UmpireDeviceTempMemorySpace();
          case MT::DEVICE_DEBUG: return new MmuDeviceMemorySpace();
          case MT::DEVICE:
          {
@@ -835,7 +760,7 @@ bool MemoryManager::MemoryClassCheck_(MemoryClass mc, void *h_ptr,
    const bool known = mm.IsKnown(h_ptr);
    const bool alias = mm.IsAlias(h_ptr);
    const bool check = known || ((flags & Mem::ALIAS) && alias);
-   MFEM_VERIFY(check,"Unknown host pointer: " << h_ptr);
+   MFEM_VERIFY(check,"");
    const internal::Memory &mem =
       (flags & Mem::ALIAS) ?
       *maps->aliases.at(h_ptr).mem : maps->memories.at(h_ptr);
@@ -858,7 +783,6 @@ bool MemoryManager::MemoryClassCheck_(MemoryClass mc, void *h_ptr,
          MFEM_VERIFY(d_mt == MemoryType::DEVICE ||
                      d_mt == MemoryType::DEVICE_DEBUG ||
                      d_mt == MemoryType::DEVICE_UMPIRE ||
-                     d_mt == MemoryType::DEVICE_TEMP_UMPIRE ||
                      d_mt == MemoryType::MANAGED,"");
          return true;
       }
@@ -1338,15 +1262,22 @@ MemoryManager::MemoryManager() { Init(); }
 MemoryManager::~MemoryManager() { if (exists) { Destroy(); } }
 
 void MemoryManager::Configure(const MemoryType host_mt,
-                              const MemoryType device_mt,
-                              const MemoryType device_tmt)
+                              const MemoryType device_mt)
 {
    Init();
    host_mem_type = host_mt;
    device_mem_type = device_mt;
-   device_temp_mem_type = device_tmt;
 }
 
+#ifdef MFEM_USE_UMPIRE
+void MemoryManager::SetUmpireAllocatorNames(const char *h_name,
+                                            const char *d_name)
+{
+   h_umpire_name = h_name;
+   d_umpire_name = d_name;
+}
+#endif
+
 void MemoryManager::Destroy()
 {
    MFEM_VERIFY(exists, "MemoryManager has already been destroyed!");
@@ -1450,14 +1381,12 @@ MemoryManager mm;
 bool MemoryManager::exists = false;
 
 #ifdef MFEM_USE_UMPIRE
-int MemoryManager::h_umpire_id = -1;
-int MemoryManager::d_umpire_id = -1;
-int MemoryManager::d_umpire_temp_id = -1;
+const char* MemoryManager::h_umpire_name = "HOST";
+const char* MemoryManager::d_umpire_name = "DEVICE";
 #endif
 
 MemoryType MemoryManager::host_mem_type = MemoryType::HOST;
 MemoryType MemoryManager::device_mem_type = MemoryType::HOST;
-MemoryType MemoryManager::device_temp_mem_type = MemoryType::HOST;
 
 const char *MemoryTypeName[MemoryTypeSize] =
 {
@@ -1474,14 +1403,11 @@ const char *MemoryTypeName[MemoryTypeSize] =
 #endif
    "device-debug",
 #if defined(MFEM_USE_CUDA)
-   "cuda-umpire",
-   "cuda-umpire-temp"
+   "cuda-umpire"
 #elif defined(MFEM_USE_HIP)
-   "hip-umpire",
-   "hip-umpire-temp"
+   "hip-umpire"
 #else
-   "device-umpire",
-   "device-umpire-temp"
+   "device-umpire"
 #endif
 };
 

general/mem_manager.hpp

@@ -27,19 +27,18 @@ namespace mfem
 /// Memory types supported by MFEM.
 enum class MemoryType
 {
-   HOST,                ///< Host memory; using new[] and delete[]
-   HOST_32,             ///< Host memory; aligned at 32 bytes
-   HOST_64,             ///< Host memory; aligned at 64 bytes
-   HOST_DEBUG,          ///< Host memory; allocated from a "host-debug" pool
-   HOST_UMPIRE,         ///< Host memory; using Umpire
-   MANAGED,             /**< Managed memory; using CUDA or HIP *MallocManaged
-                             and *Free */
-   DEVICE,              ///< Device memory; using CUDA or HIP *Malloc and *Free
-   DEVICE_DEBUG,        /**< Pseudo-device memory; allocated on host from a
-                             "device-debug" pool */
-   DEVICE_UMPIRE,       ///< Device memory; using Umpire
-   DEVICE_TEMP_UMPIRE,  ///< Temporary Device memory; using Umpire
-   SIZE                 ///< Number of host and device memory types
+   HOST,           ///< Host memory; using new[] and delete[]
+   HOST_32,        ///< Host memory; aligned at 32 bytes
+   HOST_64,        ///< Host memory; aligned at 64 bytes
+   HOST_DEBUG,     ///< Host memory; allocated from a "host-debug" pool
+   HOST_UMPIRE,    ///< Host memory; using Umpire
+   MANAGED,        /**< Managed memory; using CUDA or HIP *MallocManaged
+                        and *Free */
+   DEVICE,         ///< Device memory; using CUDA or HIP *Malloc and *Free
+   DEVICE_DEBUG,   /**< Pseudo-device memory; allocated on host from a
+                        "device-debug" pool */
+   DEVICE_UMPIRE,  ///< Device memory; using Umpire
+   SIZE            ///< Number of host and device memory types
 };
 
 /// Static casts to 'int' and sizes of some useful memory types.
@@ -62,7 +61,7 @@ enum class MemoryClass
                                  HOST_UMPIRE, MANAGED } */
    HOST_32, ///< Memory types: { HOST_32, HOST_64, HOST_DEBUG }
    HOST_64, ///< Memory types: { HOST_64, HOST_DEBUG }
-   DEVICE,  ///< Memory types: { DEVICE, DEVICE_DEBUG, DEVICE_UMPIRE, DEVICE_TEMP_UMPIRE, MANAGED }
+   DEVICE,  ///< Memory types: { DEVICE, DEVICE_DEBUG, DEVICE_UMPIRE, MANAGED }
    MANAGED  ///< Memory types: { MANAGED }
 };
 
@@ -490,9 +489,6 @@ private:
    /// Device memory type set during the Setup.
    static MemoryType device_mem_type;
 
-   /// Device temporary memory type set during the Setup.
-   static MemoryType device_temp_mem_type;
-
    /// Allow to detect if a global memory manager instance exists.
    static bool exists;
 
@@ -501,9 +497,8 @@ private:
 
    /// Host and device allocator names for Umpire.
 #ifdef MFEM_USE_UMPIRE
-   static int h_umpire_id;
-   static int d_umpire_id;
-   static int d_umpire_temp_id;
+   static const char *h_umpire_name;
+   static const char *d_umpire_name;
 #endif
 
 private: // Static methods used by the Memory<T> class
@@ -626,21 +621,15 @@ public:
    /// Initialize the memory manager.
    void Init();
 
-   /// Configure the Memory manager with given default host, device, and device temporary types
+   /// Configure the Memory manager with given default host and device types
    /// This method will be called when configuring a device.
-   void Configure(const MemoryType h_mt, const MemoryType d_mt,
-                  const MemoryType d_tmt);
+   void Configure(const MemoryType h_mt, const MemoryType d_mt);
 
 #ifdef MFEM_USE_UMPIRE
-   /// Set the host and device Umpire allocator ids
-   static void SetUmpireHostAllocatorId(int h_id) { h_umpire_id = h_id; }
-   static void SetUmpireDeviceAllocatorId(int d_id) { d_umpire_id = d_id; }
-   static void SetUmpireDeviceTempAllocatorId(int d_id) { d_umpire_temp_id = d_id; }
-
-   /// Get the host and device Umpire allocator ids
-   static int GetUmpireHostAllocatorId() { return h_umpire_id; }
-   static int GetUmpireDeviceAllocatorId() { return d_umpire_id; }
-   static int GetUmpireDeviceTempAllocatorId() { return d_umpire_temp_id; }
+   /// Set the host and device UMpire allocator names
+   void SetUmpireAllocatorNames(const char *h_name, const char *d_name);
+   const char *GetUmpireAllocatorHostName() { return h_umpire_name; }
+   const char *GetUmpireAllocatorDeviceName() { return d_umpire_name; }
 #endif
 
    /// Free all the device memories
@@ -665,7 +654,6 @@ public:
 
    static MemoryType GetHostMemoryType() { return host_mem_type; }
    static MemoryType GetDeviceMemoryType() { return device_mem_type; }
-   static MemoryType GetDeviceTempMemoryType() { return device_temp_mem_type; }
 };
 
 

linalg/CMakeLists.txt

@@ -62,6 +62,12 @@ if (MFEM_USE_MPI)
       petsc.cpp)
     list(APPEND HDRS
       petsc.hpp)
+    if (MFEM_USE_SLEPC)
+      list(APPEND SRCS
+        slepc.cpp)
+      list(APPEND HDRS
+        slepc.hpp)
+    endif()
   endif()
 endif()
 

linalg/densemat.cpp

@@ -3595,114 +3595,4 @@ void BatchLUSolve(const DenseTensor &Mlu, const Array<int> &P, Vector &X)
 
 }
 
-void BatchLUFactor(Vector &Minv,const int m,const int NE, Array<int> &P)
-{
-   P.SetSize(m*NE);
-   auto data_all = mfem::Reshape(Minv.ReadWrite(), m, m, NE);
-   auto piv_all = mfem::Reshape(P.Write(), m, NE);
-
-   MFEM_FORALL(e, NE,
-   {
-
-      double *data = &data_all(0,0,e);
-      int *ipiv = &piv_all(0,e);
-      for (int i = 0; i < m; i++)
-      {
-
-         // pivoting
-         {
-            int piv = i;
-            double a = fabs(data[piv+i*m]);
-            for (int j = i+1; j < m; j++)
-            {
-               const double b = fabs(data[j+i*m]);
-               if (b > a)
-               {
-                  a = b;
-                  piv = j;
-               }
-            }
-            ipiv[i] = piv;
-            if (piv != i)
-            {
-               // swap rows i and piv in both L and U parts
-               for (int j = 0; j < m; j++)
-               {
-                  mfem::kernels::internal::Swap<double>(data[i+j*m], data[piv+j*m]);
-               }
-            }
-         }//pivot end
-
-         //Q: How to check for errors?
-         //if (abs(data[i + i*m]) <= TOL)
-         //{
-         //return false; // failed
-         //}
-
-         const double a_ii_inv = 1.0 / data[i+i*m];
-         for (int j = i+1; j < m; j++)
-         {
-            data[j+i*m] *= a_ii_inv;
-         }
-
-         for (int k = i+1; k < m; k++)
-         {
-            const double a_ik = data[i+k*m];
-            for (int j = i+1; j < m; j++)
-            {
-               data[j+k*m] -= a_ik * data[j+i*m];
-            }
-         }
-
-      }//m loop
-
-   });
-
-}
-
-void BatchLUSolve(Vector &Minv, int m, int NE,
-                  Array<int> &P, Vector &X)
-{
-
-   auto data_all = mfem::Reshape(Minv.Read(), m, m, NE);
-   auto piv_all = mfem::Reshape(P.Read(), m, NE);
-   auto x_all = mfem::Reshape(X.ReadWrite(), m, NE);
-
-   MFEM_FORALL(e, NE,
-   {
-
-      const double *data = &data_all(0,0,e);
-      const int *ipiv = &piv_all(0,e);
-      double *x = &x_all(0,e);
-
-      // X <- P X
-      for (int i = 0; i < m; i++)
-      {
-         mfem::kernels::internal::Swap<double>(x[i], x[ipiv[i]]);
-      }
-
-      // X <- L^{-1} X
-      for (int j = 0; j < m; j++)
-      {
-         const double x_j = x[j];
-         for (int i = j+1; i < m; i++)
-         {
-            x[i] -= data[i+j*m] * x_j;
-         }
-      }
-
-      // X <- U^{-1} X
-      for (int j = m-1; j >= 0; j--)
-      {
-         const double x_j = ( x[j] /= data[j+j*m] );
-         for (int i = 0; i < j; i++)
-         {
-            x[i] -= data[i+j*m] * x_j;
-         }
-      }
-   });
-
-}
-
-
 } // namespace mfem

linalg/densemat.hpp

@@ -884,10 +884,6 @@ void BatchLUFactor(DenseTensor &Mlu, Array<int> &P, const double TOL = 0.0);
     dimension m x n. */
 void BatchLUSolve(const DenseTensor &Mlu, const Array<int> &P, Vector &X);
 
-void BatchLUFactor(Vector &Minv,int m,int NE, Array<int> &P);
-
-void BatchLUSolve(Vector &Minv, int m, int NE,
-                  Array<int> &P, Vector &X);
 
 // Inline methods
 

linalg/hypre.cpp

@@ -1667,12 +1667,12 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A,
 // Helper function for HypreParMatrixFromBlocks. Note that scalability to
 // extremely large processor counts is limited by the use of MPI_Allgather.
 void GatherBlockOffsetData(MPI_Comm comm, const int rank, const int nprocs,
-                           const int num_loc, Array<int> &offsets,
+                           const int num_loc, const Array<int> &offsets,
                            std::vector<int> &all_num_loc, const int numBlocks,
-                           std::vector<std::vector<int>> &blockProcOffsets,
-                           std::vector<int> &procOffsets,
+                           std::vector<std::vector<HYPRE_Int>> &blockProcOffsets,
+                           std::vector<HYPRE_Int> &procOffsets,
                            std::vector<std::vector<int>> &procBlockOffsets,
-                           int &firstLocal, int &globalNum)
+                           HYPRE_Int &firstLocal, HYPRE_Int &globalNum)
 {
    std::vector<std::vector<int>> all_block_num_loc(numBlocks);
 
@@ -1700,6 +1700,10 @@ void GatherBlockOffsetData(MPI_Comm comm, const int rank, const int nprocs,
    for (int i = 0; i < nprocs; ++i)
    {
       globalNum += all_num_loc[i];
+      if (rank == 0)
+      {
+         MFEM_VERIFY(globalNum >= 0, "overflow in global size");
+      }
       if (i < rank)
       {
          firstLocal += all_num_loc[i];
@@ -1808,14 +1812,14 @@ HypreParMatrix * HypreParMatrixFromBlocks(Array2D<HypreParMatrix*> &blocks,
 
    std::vector<int> all_num_loc_rows(nprocs);
    std::vector<int> all_num_loc_cols(nprocs);
-   std::vector<int> procRowOffsets(nprocs);
-   std::vector<int> procColOffsets(nprocs);
-   std::vector<std::vector<int>> blockRowProcOffsets(numBlockRows);
-   std::vector<std::vector<int>> blockColProcOffsets(numBlockCols);
+   std::vector<HYPRE_Int> procRowOffsets(nprocs);
+   std::vector<HYPRE_Int> procColOffsets(nprocs);
+   std::vector<std::vector<HYPRE_Int>> blockRowProcOffsets(numBlockRows);
+   std::vector<std::vector<HYPRE_Int>> blockColProcOffsets(numBlockCols);
    std::vector<std::vector<int>> procBlockRowOffsets(nprocs);
    std::vector<std::vector<int>> procBlockColOffsets(nprocs);
 
-   int first_loc_row, glob_nrows, first_loc_col, glob_ncols;
+   HYPRE_Int first_loc_row, glob_nrows, first_loc_col, glob_ncols;
    GatherBlockOffsetData(comm, rank, nprocs, num_loc_rows, rowOffsets,
                          all_num_loc_rows, numBlockRows, blockRowProcOffsets,
                          procRowOffsets, procBlockRowOffsets, first_loc_row,
@@ -1850,13 +1854,7 @@ HypreParMatrix * HypreParMatrixFromBlocks(Array2D<HypreParMatrix*> &blocks,
          }
          else
          {
-            {
-               hypre_ParCSRMatrix *parcsr_op = (hypre_ParCSRMatrix*)
-                                               const_cast<HypreParMatrix&>
-                                               (*(blocks(i, j)));
-               MFEM_ASSERT(parcsr_op != NULL, "const_cast failed");
-               csr_blocks(i, j) = hypre_MergeDiagAndOffd(parcsr_op);
-            }
+            csr_blocks(i, j) = hypre_MergeDiagAndOffd(*blocks(i, j));
 
             for (int k = 0; k < csr_blocks(i, j)->num_rows; ++k)
             {
@@ -1887,6 +1885,9 @@ HypreParMatrix * HypreParMatrixFromBlocks(Array2D<HypreParMatrix*> &blocks,
          {
             const int nrows = csr_blocks(i, j)->num_rows;
             const double cij = blockCoeff ? (*blockCoeff)(i, j) : 1.0;
+#if MFEM_HYPRE_VERSION >= 21600
+            const bool usingBigJ = (csr_blocks(i, j)->big_j != NULL);
+#endif
 
             for (int k = 0; k < nrows; ++k)
             {
@@ -1897,21 +1898,19 @@ HypreParMatrix * HypreParMatrixFromBlocks(Array2D<HypreParMatrix*> &blocks,
                for (int l = 0; l < nnz_k; ++l)
                {
                   // Find the column process offset for the block.
-                  const int bcol = csr_blocks(i, j)->j[osk + l];
-                  int bcolproc = 0;
+#if MFEM_HYPRE_VERSION >= 21600
+                  const HYPRE_Int bcol = usingBigJ ?
+                                         csr_blocks(i, j)->big_j[osk + l] :
+                                         csr_blocks(i, j)->j[osk + l];
+#else
+                  const HYPRE_Int bcol = csr_blocks(i, j)->j[osk + l];
+#endif
 
-                  for (int p = 1; p < nprocs; ++p)
-                  {
-                     if (blockColProcOffsets[j][p] > bcol)
-                     {
-                        bcolproc = p - 1;
-                        break;
-                     }
-                  }
-                  if (blockColProcOffsets[j][nprocs - 1] <= bcol)
-                  {
-                     bcolproc = nprocs - 1;
-                  }
+                  // find the processor 'bcolproc' that holds column 'bcol':
+                  const auto &offs = blockColProcOffsets[j];
+                  const int bcolproc =
+                     std::upper_bound(offs.begin() + 1, offs.end(), bcol)
+                     - offs.begin() - 1;
 
                   opJ[opI[rowg] + cnt[rowg]] = procColOffsets[bcolproc] +
                                                procBlockColOffsets[bcolproc][j]
@@ -1944,11 +1943,12 @@ HypreParMatrix * HypreParMatrixFromBlocks(Array2D<HypreParMatrix*> &blocks,
    colStarts2[0] = first_loc_col;
    colStarts2[1] = first_loc_col + all_num_loc_cols[rank];
 
+   MFEM_VERIFY(HYPRE_AssumedPartitionCheck(),
+               "only 'assumed partition' mode is supported");
+
    return new HypreParMatrix(comm, num_loc_rows, glob_nrows, glob_ncols,
-                             (int *)opI.data(), (HYPRE_Int *)opJ.data(),
-                             (double *)data.data(),
-                             (HYPRE_Int *)rowStarts2.data(),
-                             (HYPRE_Int *)colStarts2.data());
+                             opI.data(), opJ.data(), data.data(),
+                             rowStarts2.data(), colStarts2.data());
 }
 
 void EliminateBC(HypreParMatrix &A, HypreParMatrix &Ae,

linalg/hypre.hpp

@@ -577,7 +577,7 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A,
     each process remain on that process in the resulting matrix. Some blocks can
     be NULL. Each block and the entire system can be rectangular. Scalability to
     extremely large processor counts is limited by global MPI communication, see
-    GatherBlockOffsetData in hypre.cpp. */
+    GatherBlockOffsetData() in hypre.cpp. */
 HypreParMatrix * HypreParMatrixFromBlocks(Array2D<HypreParMatrix*> &blocks,
                                           Array2D<double> *blockCoeff=NULL);
 

linalg/linalg.hpp

@@ -49,6 +49,10 @@
 #include "petsc.hpp"
 #endif
 
+#ifdef MFEM_USE_SLEPC
+#include "slepc.hpp"
+#endif
+
 #ifdef MFEM_USE_SUPERLU
 #include "superlu.hpp"
 #endif

linalg/petsc.cpp

@@ -37,25 +37,7 @@
 
 // Note: there are additional #include statements below.
 
-// Error handling
-// Prints PETSc's stacktrace and then calls MFEM_ABORT
-// We cannot use PETSc's CHKERRQ since it returns a PetscErrorCode
-#define PCHKERRQ(obj,err) do {                                                   \
-     if ((err))                                                                  \
-     {                                                                           \
-        PetscError(PetscObjectComm((PetscObject)(obj)),__LINE__,_MFEM_FUNC_NAME, \
-                   __FILE__,(err),PETSC_ERROR_REPEAT,NULL);                      \
-        MFEM_ABORT("Error in PETSc. See stacktrace above.");                     \
-     }                                                                           \
-  } while(0);
-#define CCHKERRQ(comm,err) do {                                \
-     if ((err))                                                \
-     {                                                         \
-        PetscError(comm,__LINE__,_MFEM_FUNC_NAME,              \
-                   __FILE__,(err),PETSC_ERROR_REPEAT,NULL);    \
-        MFEM_ABORT("Error in PETSc. See stacktrace above.");   \
-     }                                                         \
-  } while(0);
+#include "petscinternals.hpp"
 
 // Callback functions: these functions will be called by PETSc
 static PetscErrorCode __mfem_ts_monitor(TS,PetscInt,PetscReal,Vec,void*);

linalg/petscinternals.hpp

@@ -0,0 +1,38 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_PETSCINTERNALS
+#define MFEM_PETSCINTERNALS
+
+#include "../general/error.hpp"
+#include "petsc.h"
+
+// Error handling
+// Prints PETSc's stacktrace and then calls MFEM_ABORT
+// We cannot use PETSc's CHKERRQ since it returns a PetscErrorCode
+#define PCHKERRQ(obj,err) do {                                                   \
+     if ((err))                                                                  \
+     {                                                                           \
+        PetscError(PetscObjectComm((PetscObject)(obj)),__LINE__,_MFEM_FUNC_NAME, \
+                   __FILE__,(err),PETSC_ERROR_REPEAT,NULL);                      \
+        MFEM_ABORT("Error in PETSc. See stacktrace above.");                     \
+     }                                                                           \
+  } while(0);
+#define CCHKERRQ(comm,err) do {                                \
+     if ((err))                                                \
+     {                                                         \
+        PetscError(comm,__LINE__,_MFEM_FUNC_NAME,              \
+                   __FILE__,(err),PETSC_ERROR_REPEAT,NULL);    \
+        MFEM_ABORT("Error in PETSc. See stacktrace above.");   \
+     }                                                         \
+  } while(0);
+
+#endif

linalg/slepc.cpp

@@ -0,0 +1,248 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_MPI
+#ifdef MFEM_USE_PETSC
+#ifdef MFEM_USE_SLEPC
+
+#include "linalg.hpp"
+
+#include "slepc.h"
+
+#include "petscinternals.hpp"
+
+static PetscErrorCode ierr;
+
+namespace mfem
+{
+
+void MFEMInitializeSlepc()
+{
+   MFEMInitializeSlepc(NULL,NULL,NULL,NULL);
+}
+
+void MFEMInitializeSlepc(int *argc,char*** argv)
+{
+   MFEMInitializeSlepc(argc,argv,NULL,NULL);
+}
+
+void MFEMInitializeSlepc(int *argc,char ***argv,const char rc_file[],
+                         const char help[])
+{
+   ierr = SlepcInitialize(argc,argv,rc_file,help);
+   MFEM_VERIFY(!ierr,"Unable to initialize SLEPc");
+}
+
+void MFEMFinalizeSlepc()
+{
+   ierr = SlepcFinalize();
+   MFEM_VERIFY(!ierr,"Unable to finalize SLEPc");
+}
+
+
+SlepcEigenSolver::SlepcEigenSolver(MPI_Comm comm, const std::string &prefix)
+{
+   clcustom = false;
+   VR = NULL;
+   VC = NULL;
+
+   ierr = EPSCreate(comm,&eps); CCHKERRQ(comm,ierr);
+   ierr = EPSSetOptionsPrefix(eps, prefix.c_str()); PCHKERRQ(eps, ierr);
+}
+
+SlepcEigenSolver::~SlepcEigenSolver()
+{
+   MPI_Comm comm;
+   ierr = PetscObjectGetComm((PetscObject)eps,&comm); PCHKERRQ(eps,ierr);
+   ierr = EPSDestroy(&eps); CCHKERRQ(comm,ierr);
+}
+
+
+void SlepcEigenSolver::SetOperator(const PetscParMatrix &op)
+{
+   delete VR;
+   delete VC;
+   VR = VC = NULL;
+
+   ierr = EPSSetOperators(eps,op,NULL); PCHKERRQ(eps, ierr);
+
+   VR = new PetscParVector(op, true, false);
+   VC = new PetscParVector(op, true, false);
+
+}
+
+void SlepcEigenSolver::SetOperators(const PetscParMatrix &op,
+                                    const PetscParMatrix&opB)
+{
+   delete VR;
+   delete VC;
+   VR = VC = NULL;
+
+   ierr = EPSSetOperators(eps,op,opB); PCHKERRQ(eps,ierr);
+
+   VR = new PetscParVector(op, true, false);
+   VC = new PetscParVector(op, true, false);
+}
+
+void SlepcEigenSolver::SetTol(double tol)
+{
+   int max_its;
+
+   ierr = EPSGetTolerances(eps,NULL,&max_its); PCHKERRQ(eps,ierr);
+   // Work around uninitialized maximum iterations
+   if (max_its==0) { max_its = PETSC_DECIDE; }
+   ierr = EPSSetTolerances(eps,tol,max_its); PCHKERRQ(eps,ierr);
+}
+
+void SlepcEigenSolver::SetMaxIter(int max_its)
+{
+   double tol;
+
+   ierr = EPSGetTolerances(eps,&tol,NULL); PCHKERRQ(eps,ierr);
+   ierr = EPSSetTolerances(eps,tol,max_its); PCHKERRQ(eps,ierr);
+}
+
+void SlepcEigenSolver::SetNumModes(int num_eigs)
+{
+   ierr = EPSSetDimensions(eps,num_eigs,PETSC_DECIDE,PETSC_DECIDE);
+   PCHKERRQ(eps,ierr);
+}
+
+void SlepcEigenSolver::Solve()
+{
+   Customize();
+
+   ierr = EPSSolve(eps); PCHKERRQ(eps,ierr);
+}
+
+void SlepcEigenSolver::Customize(bool customize) const
+{
+   if (!customize) {clcustom = true; }
+   if (!clcustom)
+   {
+      ierr = EPSSetFromOptions(eps); PCHKERRQ(eps,ierr);
+   }
+   clcustom = true;
+}
+
+void SlepcEigenSolver::GetEigenvalue(unsigned int i, double & lr) const
+{
+   ierr = EPSGetEigenvalue(eps,i,&lr,NULL); PCHKERRQ(eps,ierr);
+}
+
+void SlepcEigenSolver::GetEigenvalue(unsigned int i, double & lr,
+                                     double & lc) const
+{
+   ierr = EPSGetEigenvalue(eps,i,&lr,&lc); PCHKERRQ(eps,ierr);
+}
+
+void SlepcEigenSolver::GetEigenvector(unsigned int i, Vector & vr) const
+{
+   MFEM_VERIFY(VR,"Missing real vector");
+
+   MFEM_ASSERT(vr.Size() == VR->Size(), "invalid vr.Size() = " << vr.Size()
+               << ", expected size = " << VR->Size());
+
+   VR->PlaceArray(vr.GetData());
+   ierr = EPSGetEigenvector(eps,i,*VR,NULL); PCHKERRQ(eps,ierr);
+   VR->ResetArray();
+
+}
+
+void SlepcEigenSolver::GetEigenvector(unsigned int i, Vector & vr,
+                                      Vector & vc) const
+{
+   MFEM_VERIFY(VR,"Missing real vector");
+   MFEM_VERIFY(VC,"Missing imaginary vector");
+   MFEM_ASSERT(vr.Size() == VR->Size(), "invalid vr.Size() = " << vr.Size()
+               << ", expected size = " << VR->Size());
+   MFEM_ASSERT(vc.Size() == VC->Size(), "invalid vc.Size() = " << vc.Size()
+               << ", expected size = " << VC->Size());
+
+   VR->PlaceArray(vr.GetData());
+   VC->PlaceArray(vc.GetData());
+   ierr = EPSGetEigenvector(eps,i,*VR,*VC); PCHKERRQ(eps,ierr);
+   VR->ResetArray();
+   VC->ResetArray();
+}
+
+int SlepcEigenSolver::GetNumConverged()
+{
+   int num_conv;
+   ierr = EPSGetConverged(eps,&num_conv); PCHKERRQ(eps,ierr);
+   return num_conv;
+}
+
+void SlepcEigenSolver::SetWhichEigenpairs(SlepcEigenSolver::Which which)
+{
+   switch (which)
+   {
+      case SlepcEigenSolver::LARGEST_MAGNITUDE:
+         ierr = EPSSetWhichEigenpairs(eps,EPS_LARGEST_MAGNITUDE); PCHKERRQ(eps,ierr);
+         break;
+      case SlepcEigenSolver::SMALLEST_MAGNITUDE:
+         ierr = EPSSetWhichEigenpairs(eps,EPS_SMALLEST_MAGNITUDE); PCHKERRQ(eps,ierr);
+         break;
+      case SlepcEigenSolver::LARGEST_REAL:
+         ierr = EPSSetWhichEigenpairs(eps,EPS_LARGEST_REAL); PCHKERRQ(eps,ierr);
+         break;
+      case SlepcEigenSolver::SMALLEST_REAL:
+         ierr = EPSSetWhichEigenpairs(eps,EPS_SMALLEST_REAL); PCHKERRQ(eps,ierr);
+         break;
+      case SlepcEigenSolver::LARGEST_IMAGINARY:
+         ierr = EPSSetWhichEigenpairs(eps,EPS_LARGEST_IMAGINARY); PCHKERRQ(eps,ierr);
+         break;
+      case SlepcEigenSolver::SMALLEST_IMAGINARY:
+         ierr = EPSSetWhichEigenpairs(eps,EPS_SMALLEST_IMAGINARY); PCHKERRQ(eps,ierr);
+         break;
+      case SlepcEigenSolver::TARGET_MAGNITUDE:
+         ierr = EPSSetWhichEigenpairs(eps,EPS_TARGET_MAGNITUDE); PCHKERRQ(eps,ierr);
+         break;
+      case SlepcEigenSolver::TARGET_REAL:
+         ierr = EPSSetWhichEigenpairs(eps,EPS_TARGET_REAL); PCHKERRQ(eps,ierr);
+         break;
+      default:
+         MFEM_ABORT("Which eigenpair not implemented!");
+         break;
+   }
+}
+
+void SlepcEigenSolver::SetTarget(double target)
+{
+   ierr = EPSSetTarget(eps,target); PCHKERRQ(eps,ierr);
+}
+
+void SlepcEigenSolver::SetSpectralTransformation(
+   SlepcEigenSolver::SpectralTransformation transformation)
+{
+   ST st;
+   ierr = EPSGetST(eps,&st); PCHKERRQ(eps,ierr);
+   switch (transformation)
+   {
+      case SlepcEigenSolver::SHIFT:
+         ierr = STSetType(st,STSHIFT); PCHKERRQ(eps,ierr);
+         break;
+      case SlepcEigenSolver::SHIFT_INVERT:
+         ierr = STSetType(st,STSINVERT); PCHKERRQ(eps,ierr);
+         break;
+      default:
+         MFEM_ABORT("Spectral transformation not implemented!");
+         break;
+   }
+}
+
+}
+
+#endif  // MFEM_USE_SLEPC
+#endif  // MFEM_USE_PETSC
+#endif  // MFEM_USE_MPI

linalg/slepc.hpp

@@ -0,0 +1,116 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_SLEPC
+#define MFEM_SLEPC
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_SLEPC
+#ifdef MFEM_USE_MPI
+
+#include "petsc.hpp"
+
+// Forward declarations
+typedef struct _p_EPS *EPS;
+
+namespace mfem
+{
+
+void MFEMInitializeSlepc();
+void MFEMInitializeSlepc(int*,char***);
+void MFEMInitializeSlepc(int*,char***,const char[],const char[]);
+void MFEMFinalizeSlepc();
+
+class SlepcEigenSolver
+{
+private:
+   /// Boolean to handle SetFromOptions calls
+   mutable bool clcustom;
+
+   /// SLEPc linear eigensolver object
+   EPS eps;
+
+   /// Real and imaginary part of eigenvector
+   mutable PetscParVector *VR, *VC;
+
+public:
+   /// Constructors
+   SlepcEigenSolver(MPI_Comm comm, const std::string &prefix = std::string());
+
+   virtual ~SlepcEigenSolver();
+
+   /// Set solver tolerance
+   void SetTol(double tol);
+
+   /// Set maximum number of iterations
+   void SetMaxIter(int max_iter);
+   /// Set the number of required eigenmodes
+   void SetNumModes(int num_eigs);
+   /// Set operator for standard eigenvalue problem
+   void SetOperator(const PetscParMatrix &op);
+   /// Set operator for generalized eigenvalue problem
+   void SetOperators(const PetscParMatrix &op, const PetscParMatrix &opB);
+
+   /// Customize object with options set
+   void Customize(bool customize = true) const;
+
+   /// Solve the eigenvalue problem for the specified number of eigenvalues
+   void Solve();
+
+   /// Get the number of converged eigenvalues
+   int GetNumConverged();
+
+   /// Get the corresponding eigenvalue
+   void GetEigenvalue(unsigned int i, double & lr) const;
+   void GetEigenvalue(unsigned int i, double & lr, double & lc) const;
+
+   /// Get the corresponding eigenvector
+   void GetEigenvector(unsigned int i, Vector & vr) const;
+   void GetEigenvector(unsigned int i, Vector & vr, Vector & vc) const;
+
+   /// Target spectrum for the eigensolver. Target imaginary is not supported
+   /// without complex support in SLEPc, and intervals are not implemented.
+   enum Which
+   {
+      LARGEST_MAGNITUDE,
+      SMALLEST_MAGNITUDE,
+      LARGEST_REAL,
+      SMALLEST_REAL,
+      LARGEST_IMAGINARY,
+      SMALLEST_IMAGINARY,
+      TARGET_MAGNITUDE,
+      TARGET_REAL
+   };
+
+   enum SpectralTransformation
+   {
+      SHIFT,
+      SHIFT_INVERT
+   };
+
+   void SetWhichEigenpairs(Which which);
+   void SetTarget(double target);
+   void SetSpectralTransformation(SpectralTransformation transformation);
+
+   /// Conversion function to SLEPc's EPS type.
+   operator EPS() const { return eps; }
+
+   /// Conversion function to PetscObject
+   operator PetscObject() const {return (PetscObject)eps; }
+};
+
+}
+
+#endif // MFEM_USE_MPI
+#endif // MFEM_USE_SLEPC
+
+#endif // MFEM_SLEPC

linalg/solvers.cpp

@@ -1641,6 +1641,138 @@ void NewtonSolver::Mult(const Vector &b, Vector &x) const
    final_norm = norm;
 }
 
+void LBFGSSolver::Mult(const Vector &b, Vector &x) const
+{
+   MFEM_VERIFY(oper != NULL, "the Operator is not set (use SetOperator).");
+
+   // Quadrature points that are checked for negative Jacobians etc.
+   Vector sk, rk, yk, rho, alpha;
+   DenseMatrix skM(width, m), ykM(width, m);
+
+   //r - r_{k+1}, c - descent direction
+   sk.SetSize(width);    //x_{k+1}-x_k
+   rk.SetSize(width);    //nabla(f(x_{k}))
+   yk.SetSize(width);    //r_{k+1}-r_{k}
+   rho.SetSize(m);       //1/(dot(yk,sk)
+   alpha.SetSize(m);    //rhok*sk'*c
+   int last_saved_id = -1;
+
+   int it;
+   double norm0, norm, norm_goal;
+   const bool have_b = (b.Size() == Height());
+
+   if (!iterative_mode)
+   {
+      x = 0.0;
+   }
+
+   // r = F(x)-b
+   oper->Mult(x, r);
+   if (have_b) { r -= b; }
+
+   c = r;           // initial descent direction
+
+   norm0 = norm = Norm(r);
+   norm_goal = std::max(rel_tol*norm, abs_tol);
+   for (it = 0; true; it++)
+   {
+      MFEM_ASSERT(IsFinite(norm), "norm = " << norm);
+      if (print_level >= 0)
+      {
+         mfem::out << "LBFGS iteration " <<  it
+                   << " : ||r|| = " << norm;
+         if (it > 0)
+         {
+            mfem::out << ", ||r||/||r_0|| = " << norm/norm0;
+         }
+         mfem::out << '\n';
+      }
+
+      if (norm <= norm_goal)
+      {
+         converged = 1;
+         break;
+      }
+
+      if (it >= max_iter)
+      {
+         converged = 0;
+         break;
+      }
+
+      rk = r;
+      const double c_scale = ComputeScalingFactor(x, b);
+      if (c_scale == 0.0)
+      {
+         converged = 0;
+         break;
+      }
+      add(x, -c_scale, c, x); //x_{k+1} = x_k - c_scale*c
+
+      ProcessNewState(x);
+
+      oper->Mult(x, r);
+      if (have_b)
+      {
+         r -= b;
+      }
+
+      //    LBFGS - construct descent direction
+      subtract(r, rk, yk);   // yk = r_{k+1} - r_{k}
+      sk = c; sk *= -c_scale; //sk = x_{k+1} - x_{k} = -c_scale*c
+      const double gamma = Dot(sk, yk)/Dot(yk, yk);
+
+      //  Save last m vectors
+      last_saved_id = (last_saved_id == m-1) ? 0 : last_saved_id+1;
+      skM.SetCol(last_saved_id, sk);
+      ykM.SetCol(last_saved_id, yk);
+
+      c = r;
+      for (int i = last_saved_id; i > -1; i--)
+      {
+         skM.GetColumn(i, sk);
+         ykM.GetColumn(i, yk);
+         rho(i) = 1./Dot(sk, yk);
+         alpha(i) = rho(i)*Dot(sk,c);
+         add(c, -alpha(i), yk, c);
+      }
+      if (it > m-1)
+      {
+         for (int i = m-1; i > last_saved_id; i--)
+         {
+            skM.GetColumn(i, sk);
+            ykM.GetColumn(i, yk);
+            rho(i) = 1./Dot(sk, yk);
+            alpha(i) = rho(i)*Dot(sk,c);
+            add(c, -alpha(i), yk, c);
+         }
+      }
+
+      c *= gamma;   // scale search direction
+      if (it > m-1)
+      {
+         for (int i = last_saved_id+1; i < m ; i++)
+         {
+            skM.GetColumn(i,sk);
+            ykM.GetColumn(i,yk);
+            double betai = rho(i)*Dot(yk, c);
+            add(c, alpha(i)-betai, sk, c);
+         }
+      }
+      for (int i = 0; i < last_saved_id+1 ; i++)
+      {
+         skM.GetColumn(i,sk);
+         ykM.GetColumn(i,yk);
+         double betai = rho(i)*Dot(yk, c);
+         add(c, alpha(i)-betai, sk, c);
+      }
+
+      norm = Norm(r);
+   }
+
+   final_iter = it;
+   final_norm = norm;
+}
 
 int aGMRES(const Operator &A, Vector &x, const Vector &b,
            const Operator &M, int &max_iter,

linalg/solvers.hpp

@@ -416,6 +416,32 @@ public:
    virtual void ProcessNewState(const Vector &x) const { }
 };
 
+/** L-BFGS method for solving F(x)=b for a given operator F, by minimizing
+    the norm of F(x) - b. Requires only the action of the operator F. */
+class LBFGSSolver : public NewtonSolver
+{
+protected:
+   int m = 10;
+
+public:
+   LBFGSSolver() : NewtonSolver() { }
+
+#ifdef MFEM_USE_MPI
+   LBFGSSolver(MPI_Comm _comm) : NewtonSolver(_comm) { }
+#endif
+
+   void SetHistorySize(int dim) { m = dim; }
+
+   /// Solve the nonlinear system with right-hand side @a b.
+   /** If `b.Size() != Height()`, then @a b is assumed to be zero. */
+   virtual void Mult(const Vector &b, Vector &x) const;
+
+   virtual void SetPreconditioner(Solver &pr)
+   { MFEM_WARNING("L-BFGS won't use the given preconditioner."); }
+   virtual void SetSolver(Solver &solver)
+   { MFEM_WARNING("L-BFGS won't use the given solver."); }
+};
+
 /** Adaptive restarted GMRES.
     m_max and m_min(=1) are the maximal and minimal restart parameters.
     m_step(=1) is the step to use for going from m_max and m_min.

linalg/sparsemat.cpp

@@ -28,25 +28,6 @@ namespace mfem
 
 using namespace std;
 
-#ifdef MFEM_USE_CUDA
-int SparseMatrix::SparseMatrixCount = 0;
-cusparseHandle_t SparseMatrix::handle;
-size_t SparseMatrix::bufferSize = 0;
-void * SparseMatrix::dBuffer = nullptr;
-#endif
-
-void SparseMatrix::InitCuSparse()
-{
-   /* Initialize CuSparse library */
-#ifdef MFEM_USE_CUDA
-   SparseMatrixCount++;
-   if (SparseMatrixCount == 1 && Device::Allows(Backend::CUDA_MASK))
-   {
-      cusparseCreate(&handle);
-   }
-#endif
-}
-
 SparseMatrix::SparseMatrix(int nrows, int ncols)
    : AbstractSparseMatrix(nrows, (ncols >= 0) ? ncols : nrows),
      Rows(new RowNode *[nrows]),
@@ -69,8 +50,6 @@ SparseMatrix::SparseMatrix(int nrows, int ncols)
 #ifdef MFEM_USE_MEMALLOC
    NodesMem = new RowNodeAlloc;
 #endif
-
-   InitCuSparse();
 }
 
 SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n)
@@ -88,8 +67,6 @@ SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n)
 #ifdef MFEM_USE_MEMALLOC
    NodesMem = NULL;
 #endif
-
-   InitCuSparse();
 }
 
 SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n,
@@ -121,8 +98,6 @@ SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n,
          A[i] = 0.0;
       }
    }
-
-   InitCuSparse();
 }
 
 SparseMatrix::SparseMatrix(int nrows, int ncols, int rowsize)
@@ -144,11 +119,9 @@ SparseMatrix::SparseMatrix(int nrows, int ncols, int rowsize)
    {
       I[i] = i * rowsize;
    }
-
-   InitCuSparse();
 }
 
-SparseMatrix::SparseMatrix(const SparseMatrix &mat, bool copy_graph, MemoryType mt)
+SparseMatrix::SparseMatrix(const SparseMatrix &mat, bool copy_graph)
    : AbstractSparseMatrix(mat.Height(), mat.Width())
 {
    if (mat.Finalized())
@@ -156,8 +129,8 @@ SparseMatrix::SparseMatrix(const SparseMatrix &mat, bool copy_graph, MemoryType
       const int nnz = mat.I[height];
       if (copy_graph)
       {
-         I.New(height+1, mt == MemoryType::SIZE ? mat.I.GetMemoryType() : mt);
-         J.New(nnz, mt == MemoryType::SIZE ? mat.J.GetMemoryType() : mt);
+         I.New(height+1, mat.I.GetMemoryType());
+         J.New(nnz, mat.J.GetMemoryType());
          I.CopyFrom(mat.I, height+1);
          J.CopyFrom(mat.J, nnz);
       }
@@ -168,7 +141,7 @@ SparseMatrix::SparseMatrix(const SparseMatrix &mat, bool copy_graph, MemoryType
          I.ClearOwnerFlags();
          J.ClearOwnerFlags();
       }
-      A.New(nnz, mt == MemoryType::SIZE ? mat.A.GetMemoryType() : mt);
+      A.New(nnz, mat.A.GetMemoryType());
       A.CopyFrom(mat.A, nnz);
 
       Rows = NULL;
@@ -211,8 +184,6 @@ SparseMatrix::SparseMatrix(const SparseMatrix &mat, bool copy_graph, MemoryType
    ColPtrNode = NULL;
    At = NULL;
    isSorted = mat.isSorted;
-
-   InitCuSparse();
 }
 
 SparseMatrix::SparseMatrix(const Vector &v)
@@ -240,8 +211,6 @@ SparseMatrix::SparseMatrix(const Vector &v)
       J[r] = r;
       A[r] = v[r];
    }
-
-   InitCuSparse();
 }
 
 SparseMatrix& SparseMatrix::operator=(const SparseMatrix &rhs)
@@ -281,16 +250,6 @@ void SparseMatrix::SetEmpty()
    NodesMem = NULL;
 #endif
    isSorted = false;
-
-#ifdef MFEM_USE_CUDA
-   if (initBuffers)
-   {
-      cusparseDestroySpMat(matA_descr);
-      cusparseDestroyDnVec(vecX_descr);
-      cusparseDestroyDnVec(vecY_descr);
-      initBuffers = false;
-   }
-#endif
 }
 
 int SparseMatrix::RowSize(const int i) const
@@ -535,29 +494,24 @@ void SparseMatrix::GetDiag(Vector & d) const
 
    d.SetSize(height);
 
-   auto I = this->ReadI();
-   auto J = this->ReadJ();
-   auto A = this->ReadData();
-   auto dd = d.Write();
-
-   MFEM_FORALL(i, height,
+   int j, end;
+   for (int i = 0; i < height; i++)
    {
-      const int begin = I[i];
-      const int end = I[i+1];
-      int j;
-      for (j = begin; j < end; j++)
+
+      end = I[i+1];
+      for (j = I[i]; j < end; j++)
       {
          if (J[j] == i)
          {
-            dd[i] = A[j];
+            d[i] = A[j];
             break;
          }
       }
       if (j == end)
       {
-         dd[i] = 0.;
+         d[i] = 0.;
       }
-   });
+   }
 }
 
 /// Produces a DenseMatrix from a SparseMatrix
@@ -633,72 +587,16 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const double a) const
    auto d_A = Read(A, nnz);
    auto d_x = x.Read();
    auto d_y = y.ReadWrite();
-
-   //Skip if matrix has no non-zeros
-   if (nnz == 0) {return;}
-   if (Device::Allows(Backend::CUDA_MASK) && useCuSparse)
+   MFEM_FORALL(i, height,
    {
-#ifdef MFEM_USE_CUDA
-      const double alpha = a;
-      const double beta  = 1.0;
-
-      //Setup descriptors
-      if (!initBuffers)
+      double d = 0.0;
+      const int end = d_I[i+1];
+      for (int j = d_I[i]; j < end; j++)
       {
-         /* Setup matrix descriptor */
-         cusparseCreateCsr(&matA_descr,Height(), Width(), J.Capacity(),
-                           const_cast<int *>(d_I),
-                           const_cast<int *>(d_J), const_cast<double *>(d_A), CUSPARSE_INDEX_32I,
-                           CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_64F);
-
-         /*Create handles for input/output vectors */
-         cusparseCreateDnVec(&vecX_descr, x.Size(), const_cast<double *>(d_x),
-                             CUDA_R_64F);
-         cusparseCreateDnVec(&vecY_descr, y.Size(), d_y, CUDA_R_64F);
-
-         initBuffers = true;
+         d += d_A[j] * d_x[d_J[j]];
       }
-
-      /*Allocate space for kernel. Buffer is shared between different sparsemats */
-      size_t newBufferSize = 0;
-      cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha,
-                              matA_descr,
-                              vecX_descr, &beta, vecY_descr, CUDA_R_64F,
-                              CUSPARSE_CSRMV_ALG1, &newBufferSize);
-
-      //Check if need to resize
-      if (newBufferSize > bufferSize)
-      {
-         bufferSize = newBufferSize;
-         if (dBuffer != NULL) { CuMemFree(dBuffer); }
-         CuMemAlloc(&dBuffer, bufferSize);
-      }
-
-      //Update input/output vectors
-      cusparseDnVecSetValues(vecX_descr, const_cast<double *>(d_x));
-      cusparseDnVecSetValues(vecY_descr, d_y);
-
-      // Y = alpha A * X + beta * Y
-      cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA_descr,
-                   vecX_descr, &beta, vecY_descr, CUDA_R_64F, CUSPARSE_CSRMV_ALG1, dBuffer);
-#endif
-   }
-   else
-   {
-      //Native version
-      MFEM_FORALL(i, height,
-      {
-         double d = 0.0;
-         const int end = d_I[i+1];
-         for (int j = d_I[i]; j < end; j++)
-         {
-            d += d_A[j] * d_x[d_J[j]];
-         }
-         d_y[i] += a * d;
-      });
-
-   }
-
+      d_y[i] += a * d;
+   });
 #else
    const double *Ap = A, *xp = x.GetData();
    double *yp = y.GetData();
@@ -2247,46 +2145,31 @@ void SparseMatrix::DiagScale(const Vector &b, Vector &x, double sc) const
 {
    MFEM_VERIFY(Finalized(), "Matrix must be finalized.");
 
-   const int nnz = J.Capacity();
-
-   const bool use_dev = b.UseDevice() || x.UseDevice();
-
-   auto bp = b.Read(use_dev);
-   auto xp = x.Write(use_dev);
-
-   auto Ap = Read(A, nnz);
-   auto Ip = Read(I, height+1);
-   auto Jp = Read(J, nnz);
-
    bool scale = (sc != 1.0);
-   MFEM_FORALL(i, height,
+   for (int i = 0, j = 0; i < height; i++)
    {
-      int end = Ip[i+1];
-      for (int j = Ip[i]; true; j++)
+      int end = I[i+1];
+      for ( ; true; j++)
       {
-         if (j == end)
+         MFEM_VERIFY(j != end, "Couldn't find diagonal in row. i = " << i
+                     << ", j = " << j
+                     << ", I[i+1] = " << end );
+         if (J[j] == i)
          {
-            //MFEM_ABORT_KERNEL("Diagonal not found in SparseMatrix::DiagScale");
-         }
-         if (Jp[j] == i)
-         {
-            if (!(std::abs(Ap[j]) > 0.0))
-            {
-               //MFEM_ABORT_KERNEL("Zero diagonal in SparseMatrix::DiagScale");
-            }
-
+            MFEM_VERIFY(std::abs(A[j]) > 0.0, "Diagonal " << j << " must be nonzero");
             if (scale)
             {
-               xp[i] = sc * bp[i] / Ap[j];
+               x(i) = sc * b(i) / A[j];
             }
             else
             {
-               xp[i] = bp[i] / Ap[j];
+               x(i) = b(i) / A[j];
             }
             break;
          }
       }
-   });
+      j = end;
+   }
    return;
 }
 
@@ -2866,10 +2749,6 @@ void SparseMatrix::Print(std::ostream & out, int _width) const
       return;
    }
 
-   // HostRead forces synchronization
-   HostReadI();
-   HostReadJ();
-   HostReadData();
    for (i = 0; i < height; i++)
    {
       out << "[row " << i << "]\n";
@@ -3059,16 +2938,6 @@ void SparseMatrix::Destroy()
    delete NodesMem;
 #endif
    delete At;
-
-#ifdef MFEM_USE_CUDA
-   if (initBuffers)
-   {
-      cusparseDestroySpMat(matA_descr);
-      cusparseDestroyDnVec(vecX_descr);
-      cusparseDestroyDnVec(vecY_descr);
-      initBuffers = false;
-   }
-#endif
 }
 
 int SparseMatrix::ActualWidth() const

linalg/sparsemat.hpp

@@ -21,12 +21,6 @@
 #include "../general/globals.hpp"
 #include "densemat.hpp"
 
-#ifdef MFEM_USE_CUDA
-#include <cusparse.h>
-#include <library_types.h>
-#include "../general/cuda.hpp"
-#endif
-
 namespace mfem
 {
 
@@ -86,33 +80,9 @@ protected:
    void Destroy();   // Delete all owned data
    void SetEmpty();  // Init all entries with empty values
 
-   bool useCuSparse{true}; //Use CuSparse if available
-
-   // Initialize CuSparse
-   void InitCuSparse();
-
-#ifdef MFEM_USE_CUDA
-   cusparseStatus_t status;
-   static cusparseHandle_t handle;
-   cusparseMatDescr_t descr=0;
-   static size_t bufferSize;
-   static void *dBuffer;
-   mutable bool initBuffers{false};
-
-   static int SparseMatrixCount;
-   mutable cusparseSpMatDescr_t matA_descr;
-   mutable cusparseDnVecDescr_t vecX_descr;
-   mutable cusparseDnVecDescr_t vecY_descr;
-#endif
-
 public:
    /// Create an empty SparseMatrix.
-   SparseMatrix()
-   {
-      SetEmpty();
-
-      InitCuSparse();
-   }
+   SparseMatrix() { SetEmpty(); }
 
    /** @brief Create a sparse matrix with flexible sparsity structure using a
        row-wise linked list (LIL) format. */
@@ -143,14 +113,11 @@ public:
    /** If @a mat is finalized and @a copy_graph is false, the #I and #J arrays
        will use a shallow copy (copy the pointers only) without transferring
        ownership. */
-   SparseMatrix(const SparseMatrix &mat, bool copy_graph = true, MemoryType mt = MemoryType::SIZE);
+   SparseMatrix(const SparseMatrix &mat, bool copy_graph = true);
 
    /// Create a SparseMatrix with diagonal @a v, i.e. A = Diag(v)
    SparseMatrix(const Vector & v);
 
-   // Runtime option to use CuSparse
-   // Only valid when using a CUDA backend
-   void UseCuSparse(bool _useCuSparse = true) { useCuSparse = _useCuSparse;}
 
    /// Assignment operator: deep copy
    SparseMatrix& operator=(const SparseMatrix &rhs);
@@ -606,28 +573,11 @@ public:
    void Swap(SparseMatrix &other);
 
    /// Destroys sparse matrix.
-   virtual ~SparseMatrix()
-   {
-      Destroy();
-#ifdef MFEM_USE_CUDA
-      if (handle && SparseMatrixCount==1 && Device::Allows(Backend::CUDA_MASK))
-      {
-         cusparseDestroy(handle);
-         CuMemFree(dBuffer);
-      }
-      SparseMatrixCount--;
-#endif
-   }
+   virtual ~SparseMatrix() { Destroy(); }
 
    Type GetType() const { return MFEM_SPARSEMAT; }
 };
 
-inline std::ostream& operator<<(std::ostream& os, SparseMatrix const& mat)
-{
-   mat.Print(os);
-   return os;
-}
-
 /// Applies f() to each element of the matrix (after it is finalized).
 void SparseMatrixFunction(SparseMatrix &S, double (*f)(double));
 

linalg/superlu.cpp

@@ -24,6 +24,11 @@
 #error "SuperLUDist has been built with 64bit integers. This is not supported"
 #endif
 
+// For now, it is assumed that HYPRE_Int is int.
+#ifdef HYPRE_BIGINT
+#error "SuperLUDist support requires HYPRE_Int == int, for now."
+#endif
+
 #if SUPERLU_DIST_MAJOR_VERSION > 6 ||                                   \
   (SUPERLU_DIST_MAJOR_VERSION == 6 && SUPERLU_DIST_MINOR_VERSION > 2)
 #define ScalePermstruct_t dScalePermstruct_t
@@ -147,8 +152,10 @@ SuperLURowLocMatrix::SuperLURowLocMatrix( const HypreParMatrix & hypParMat )
    hypre_CSRMatrix * csr_op = hypre_MergeDiagAndOffd(parcsr_op);
    hypre_CSRMatrixSetDataOwner(csr_op,0);
 #if MFEM_HYPRE_VERSION >= 21600
-   MFEM_VERIFY(csr_op->num_rows < INT_MAX,"SuperLU: number of local rows "
-               "is too large to store as an integer.");
+   // For now, this method assumes that HYPRE_Int is int. Also, csr_op->num_cols
+   // is of type HYPRE_Int, so if we want to check for big indices in
+   // csr_op->big_j, we'll have to check all entries and that check will only be
+   // necessary in HYPRE_MIXEDINT mode which is not supported at the moment.
    hypre_CSRMatrixBigJtoJ(csr_op);
 #endif
 

makefile

@@ -204,7 +204,7 @@ CXXFLAGS ?= $(OPTIM_FLAGS)
 # MPI configuration
 ifneq ($(MFEM_USE_MPI),YES)
    MFEM_HOST_CXX = $(CXX)
-   PKGS_NEED_MPI = SUPERLU STRUMPACK PETSC PUMI
+   PKGS_NEED_MPI = SUPERLU STRUMPACK PETSC PUMI SLEPC
    $(foreach mpidep,$(PKGS_NEED_MPI),$(if $(MFEM_USE_$(mpidep):NO=),\
      $(warning *** [MPI is OFF] setting MFEM_USE_$(mpidep) = NO)\
      $(eval override MFEM_USE_$(mpidep)=NO),))
@@ -260,9 +260,10 @@ endif
 
 # List of MFEM dependencies, that require the *_LIB variable to be non-empty
 MFEM_REQ_LIB_DEPS = SUPERLU METIS CONDUIT SIDRE LAPACK SUNDIALS MESQUITE\
- SUITESPARSE STRUMPACK GINKGO GNUTLS NETCDF PETSC MPFR PUMI HIOP GSLIB\
+ SUITESPARSE STRUMPACK GINKGO GNUTLS NETCDF PETSC SLEPC MPFR PUMI HIOP GSLIB\
  OCCA CEED RAJA UMPIRE
 PETSC_ERROR_MSG = $(if $(PETSC_FOUND),,. PETSC config not found: $(PETSC_VARS))
+SLEPC_ERROR_MSG = $(if $(SLEPC_FOUND),,. SLEPC config not found: $(SLEPC_VARS))
 
 define mfem_check_dependency
 ifeq ($$(MFEM_USE_$(1)),YES)
@@ -321,7 +322,7 @@ MFEM_DEFINES = MFEM_VERSION MFEM_VERSION_STRING MFEM_GIT_STRING MFEM_USE_MPI\
  MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP MFEM_USE_MEMALLOC MFEM_TIMER_TYPE\
  MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE MFEM_USE_GINKGO\
  MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GNUTLS\
- MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT\
+ MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT\
  MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA MFEM_USE_HIP\
  MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD\
  MFEM_USE_ADIOS2 MFEM_SOURCE_DIR MFEM_INSTALL_DIR
@@ -631,6 +632,7 @@ status info:
 	$(info MFEM_USE_GNUTLS        = $(MFEM_USE_GNUTLS))
 	$(info MFEM_USE_NETCDF        = $(MFEM_USE_NETCDF))
 	$(info MFEM_USE_PETSC         = $(MFEM_USE_PETSC))
+	$(info MFEM_USE_SLEPC         = $(MFEM_USE_SLEPC))
 	$(info MFEM_USE_MPFR          = $(MFEM_USE_MPFR))
 	$(info MFEM_USE_SIDRE         = $(MFEM_USE_SIDRE))
 	$(info MFEM_USE_CONDUIT       = $(MFEM_USE_CONDUIT))

mesh/mesh.cpp

@@ -859,35 +859,41 @@ FaceElementTransformations *Mesh::GetFaceElementTransformations(int FaceNo,
 {
    FaceInfo &face_info = faces_info[FaceNo];
 
-   FaceElemTr.SetConfigurationMask(0);
+   int cmask = 0;
+   FaceElemTr.SetConfigurationMask(cmask);
    FaceElemTr.Elem1 = NULL;
    FaceElemTr.Elem2 = NULL;
 
    // setup the transformation for the first element
    FaceElemTr.Elem1No = face_info.Elem1No;
-   if (mask & 1)
+   if (mask & FaceElementTransformations::HAVE_ELEM1)
    {
       GetElementTransformation(FaceElemTr.Elem1No, &Transformation);
       FaceElemTr.Elem1 = &Transformation;
+      cmask |= 1;
    }
 
    //  setup the transformation for the second element
    //     return NULL in the Elem2 field if there's no second element, i.e.
    //     the face is on the "boundary"
    FaceElemTr.Elem2No = face_info.Elem2No;
-   if ((mask & 2) && FaceElemTr.Elem2No >= 0)
+   if ((mask & FaceElementTransformations::HAVE_ELEM2) &&
+       FaceElemTr.Elem2No >= 0)
    {
 #ifdef MFEM_DEBUG
-      if (NURBSext && (mask & 1)) { MFEM_ABORT("NURBS mesh not supported!"); }
+      if (NURBSext && (mask & FaceElementTransformations::HAVE_ELEM1))
+      { MFEM_ABORT("NURBS mesh not supported!"); }
 #endif
       GetElementTransformation(FaceElemTr.Elem2No, &Transformation2);
       FaceElemTr.Elem2 = &Transformation2;
+      cmask |= 2;
    }
 
    // setup the face transformation
-   if (mask & 16)
+   if (mask & FaceElementTransformations::HAVE_FACE)
    {
       GetFaceTransformation(FaceNo, &FaceElemTr);
+      cmask |= 16;
    }
    else
    {
@@ -896,13 +902,15 @@ FaceElementTransformations *Mesh::GetFaceElementTransformations(int FaceNo,
 
    // setup Loc1 & Loc2
    int face_type = GetFaceElementType(FaceNo);
-   if (mask & 4)
+   if (mask & FaceElementTransformations::HAVE_LOC1)
    {
       int elem_type = GetElementType(face_info.Elem1No);
       GetLocalFaceTransformation(face_type, elem_type,
                                  FaceElemTr.Loc1.Transf, face_info.Elem1Inf);
+      cmask |= 4;
    }
-   if ((mask & 8) && FaceElemTr.Elem2No >= 0)
+   if ((mask & FaceElementTransformations::HAVE_LOC2) &&
+       FaceElemTr.Elem2No >= 0)
    {
       int elem_type = GetElementType(face_info.Elem2No);
       GetLocalFaceTransformation(face_type, elem_type,
@@ -913,9 +921,10 @@ FaceElementTransformations *Mesh::GetFaceElementTransformations(int FaceNo,
       {
          ApplyLocalSlaveTransformation(FaceElemTr, face_info, false);
       }
+      cmask |= 8;
    }
 
-   FaceElemTr.SetConfigurationMask(mask);
+   FaceElemTr.SetConfigurationMask(cmask);
 
    // This check can be useful for internal debugging, however it will fail on
    // periodic boundary faces, so we keep it disabled in general.
@@ -1000,7 +1009,7 @@ FaceElementTransformations *Mesh::GetBdrFaceTransformations(int BdrElemNo)
    {
       return NULL;
    }
-   tr = GetFaceElementTransformations(fn);
+   tr = GetFaceElementTransformations(fn, 21);
    tr->Attribute = boundary[BdrElemNo]->GetAttribute();
    tr->ElementNo = BdrElemNo;
    tr->ElementType = ElementTransformation::BDR_FACE;
@@ -10524,17 +10533,17 @@ GeometricFactors::GeometricFactors(const Mesh *mesh, const IntegrationRule &ir,
    unsigned eval_flags = 0;
    if (flags & GeometricFactors::COORDINATES)
    {
-      X.SetSize(vdim*NQ*NE, Device::GetDeviceTempMemoryType());
+      X.SetSize(vdim*NQ*NE);
       eval_flags |= QuadratureInterpolator::VALUES;
    }
    if (flags & GeometricFactors::JACOBIANS)
    {
-      J.SetSize(dim*vdim*NQ*NE, Device::GetDeviceTempMemoryType());
+      J.SetSize(dim*vdim*NQ*NE);
       eval_flags |= QuadratureInterpolator::DERIVATIVES;
    }
    if (flags & GeometricFactors::DETERMINANTS)
    {
-      detJ.SetSize(NQ*NE, Device::GetDeviceTempMemoryType());
+      detJ.SetSize(NQ*NE);
       eval_flags |= QuadratureInterpolator::DETERMINANTS;
    }
 
@@ -10544,7 +10553,7 @@ GeometricFactors::GeometricFactors(const Mesh *mesh, const IntegrationRule &ir,
    qi->SetOutputLayout(QVectorLayout::byNODES);
    if (elem_restr)
    {
-      Vector Enodes(vdim*ND*NE, Device::GetDeviceTempMemoryType());
+      Vector Enodes(vdim*ND*NE);
       elem_restr->Mult(*nodes, Enodes);
       qi->Mult(Enodes, eval_flags, X, J, detJ);
    }

mesh/mesh.hpp

@@ -961,7 +961,7 @@ public:
    /// Returns the transformation defining the given face element
    ElementTransformation *GetEdgeTransformation(int EdgeNo);
 
-   /// Returns (a pointer to a structure containing) the following data:
+   /// Returns (a pointer to an object containing) the following data:
    ///
    /// 1) Elem1No - the index of the first element that contains this face this
    ///    is the element that has the same outward unit normal vector as the
@@ -989,6 +989,8 @@ public:
    /// The mask specifies which fields in the structure to return:
    ///    mask & 1 - Elem1, mask & 2 - Elem2
    ///    mask & 4 - Loc1, mask & 8 - Loc2, mask & 16 - Face.
+   /// These mask values are defined in the ConfigMasks enum type as part of the
+   /// FaceElementTransformations class in fem/eltrans.hpp.
    FaceElementTransformations *GetFaceElementTransformations(int FaceNo,
                                                              int mask = 31);
 

mesh/pmesh.cpp

@@ -1690,6 +1690,7 @@ void ParMesh::GetFaceNbrElementTransformation(
 
    ElTr->Attribute = elem->GetAttribute();
    ElTr->ElementNo = NumOfElements + i;
+   ElTr->ElementType = ElementTransformation::ELEMENT;
 
    if (Nodes == NULL)
    {
@@ -2393,24 +2394,20 @@ void ParMesh::GetGhostFaceTransformation(
 }
 
 FaceElementTransformations *ParMesh::
-GetSharedFaceTransformations(int sf, bool fill2, bool direct)
+GetSharedFaceTransformations(int sf, bool fill2)
 {
-   //int FaceNo = GetSharedFace(sf);
-   int FaceNo;
-   if (direct)
-   {
-      FaceNo = sf;
-   }
-   else
-   {
-      FaceNo = GetSharedFace(sf);
-   }
+   int FaceNo = GetSharedFace(sf);
 
    FaceInfo &face_info = faces_info[FaceNo];
 
    bool is_slave = Nonconforming() && IsSlaveFace(face_info);
    bool is_ghost = Nonconforming() && FaceNo >= GetNumFaces();
 
+   int mask = 0;
+   FaceElemTr.SetConfigurationMask(0);
+   FaceElemTr.Elem1 = NULL;
+   FaceElemTr.Elem2 = NULL;
+
    NCFaceInfo* nc_info = NULL;
    if (is_slave) { nc_info = &nc_faces_info[face_info.NCFace]; }
 
@@ -2422,13 +2419,21 @@ GetSharedFaceTransformations(int sf, bool fill2, bool direct)
    FaceElemTr.Elem1No = face_info.Elem1No;
    GetElementTransformation(FaceElemTr.Elem1No, &Transformation);
    FaceElemTr.Elem1 = &Transformation;
+   mask |= FaceElementTransformations::HAVE_ELEM1;
 
    // setup the transformation for the second (neighbor) element
+   int Elem2NbrNo;
    if (fill2)
    {
-      FaceElemTr.Elem2No = -1 - face_info.Elem2No;
-      GetFaceNbrElementTransformation(FaceElemTr.Elem2No, &Transformation2);
+      Elem2NbrNo = -1 - face_info.Elem2No;
+      // Store the "shifted index" for element 2 in FaceElemTr.Elem2No.
+      // `Elem2NbrNo` is the index of the face neighbor (starting from 0),
+      // and `FaceElemTr.Elem2No` will be offset by the number of (local)
+      // elements in the mesh.
+      FaceElemTr.Elem2No = NumOfElements + Elem2NbrNo;
+      GetFaceNbrElementTransformation(Elem2NbrNo, &Transformation2);
       FaceElemTr.Elem2 = &Transformation2;
+      mask |= FaceElementTransformations::HAVE_ELEM2;
    }
    else
    {
@@ -2440,6 +2445,7 @@ GetSharedFaceTransformations(int sf, bool fill2, bool direct)
    {
       GetFaceTransformation(FaceNo, &FaceElemTr);
       // NOTE: The above call overwrites FaceElemTr.Loc1
+      mask |= FaceElementTransformations::HAVE_FACE;
    }
    else
    {
@@ -2450,12 +2456,14 @@ GetSharedFaceTransformations(int sf, bool fill2, bool direct)
    int elem_type = GetElementType(face_info.Elem1No);
    GetLocalFaceTransformation(face_type, elem_type, FaceElemTr.Loc1.Transf,
                               face_info.Elem1Inf);
+   mask |= FaceElementTransformations::HAVE_LOC1;
 
    if (fill2)
    {
-      elem_type = face_nbr_elements[FaceElemTr.Elem2No]->GetType();
+      elem_type = face_nbr_elements[Elem2NbrNo]->GetType();
       GetLocalFaceTransformation(face_type, elem_type, FaceElemTr.Loc2.Transf,
                                  face_info.Elem2Inf);
+      mask |= FaceElementTransformations::HAVE_LOC2;
    }
 
    // adjust Loc1 or Loc2 of the master face if this is a slave face
@@ -2472,9 +2480,10 @@ GetSharedFaceTransformations(int sf, bool fill2, bool direct)
    if (is_ghost)
    {
       GetGhostFaceTransformation(&FaceElemTr, face_type, face_geom);
+      mask |= FaceElementTransformations::HAVE_FACE;
    }
 
-   FaceElemTr.SetConfigurationMask(fill2 ? 31 : 21);
+   FaceElemTr.SetConfigurationMask(mask);
 
    // This check can be useful for internal debugging, however it will fail on
    // periodic boundary faces, so we keep it disabled in general.

mesh/pmesh.hpp

@@ -78,6 +78,8 @@ protected:
    // sface ids: all triangles first, then all quads
    Array<int> sface_lface;
 
+   IsoparametricTransformation FaceNbrTransformation;
+
    // glob_elem_offset + local element number defines a global element numbering
    mutable long glob_elem_offset, glob_offset_sequence;
    void ComputeGlobalElementOffset() const;
@@ -291,9 +293,17 @@ public:
 
    /** Get the FaceElementTransformations for the given shared face (edge 2D).
        In the returned object, 1 and 2 refer to the local and the neighbor
-       elements, respectively. Use direct if sf is the face number */
+       elements, respectively. */
    FaceElementTransformations *
-   GetSharedFaceTransformations(int sf, bool fill2 = true, bool direct = false);
+   GetSharedFaceTransformations(int sf, bool fill2 = true);
+
+   ElementTransformation *
+   GetFaceNbrElementTransformation(int i)
+   {
+      GetFaceNbrElementTransformation(i, &FaceNbrTransformation);
+
+      return &FaceNbrTransformation;
+   }
 
    /// Return the number of shared faces (3D), edges (2D), vertices (1D)
    int GetNSharedFaces() const;

miniapps/electromagnetics/joule_solver.cpp

@@ -906,7 +906,7 @@ double JouleHeatingCoefficient::Eval(ElementTransformation &T,
 {
    Vector E;
    double thisSigma;
-   E_gf.GetVectorValue(T.ElementNo, ip, E);
+   E_gf.GetVectorValue(T, ip, E);
    thisSigma = sigma.Eval(T, ip);
    return thisSigma*(E*E);
 }

miniapps/hypsys/apps/advection.cpp

@@ -0,0 +1,371 @@
+#include "advection.hpp"
+
+Configuration ConfigAdv;
+
+void AnalyticalSolutionAdv(const Vector &x, double t, Vector &u);
+void InitialConditionAdv(const Vector &x, Vector &u);
+void InflowFunctionAdv(const Vector &x, double t, Vector &u);
+void VelocityFunctionAdv(const Vector &x, Vector &v);
+
+Advection::Advection(FiniteElementSpace *fes_, BlockVector &u_block,
+                     Configuration &config_, bool NodalQuadRule)
+   : HyperbolicSystem(fes_, u_block, 1, config_,
+                      VectorFunctionCoefficient (1, InflowFunctionAdv))
+{
+   DiscreteUpwinding = true;
+
+   ConfigAdv = config_;
+   VectorFunctionCoefficient ic(NumEq, InitialConditionAdv);
+
+   switch (ConfigAdv.ConfigNum)
+   {
+      case 0:
+      {
+         ProblemName = "Advection - Smooth Circular Convection";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = true;
+         TimeDepBC = false;
+         ProjType = 0;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 1:
+      {
+         ProblemName = "Advection - Solid Body Rotation";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 2:
+      {
+         ProblemName = "Advection - Step function";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 3:
+      {
+         ProblemName = "Advection - Smooth profile";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 0;
+         L2_Projection(ic, u0);;
+         break;
+      }
+      case 4:
+      {
+         ProblemName = "Advection - Discontinuous and Smooth profile";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 5:
+      {
+         ProblemName = "Advection - C1 curve";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      default:
+         MFEM_ABORT("No such test case implemented.");
+   }
+
+   // The following computes and stores all necessary evaluations of the time-independent velocity.
+   Mesh *mesh = fes->GetMesh();
+   DofInfo dofs(fes);
+   const int ne = fes->GetNE();
+   const IntegrationRule *IntRuleElem = GetElementIntegrationRule(fes);
+   const IntegrationRule *IntRuleFace = GetFaceIntegrationRule(fes);
+   const IntegrationRule *nodes = GetElementIntegrationRule(fes, true);
+
+   const int nqe = IntRuleElem->GetNPoints();
+   nqf = IntRuleFace->GetNPoints();
+   Vector vec, vval;
+   VelocityVector.SetSize(dim);
+   DenseMatrix VelEval, mat(dim, nqe);
+
+   VelElem.SetSize(dim, nqe, ne);
+   VelFace.SetSize(dim, dofs.NumBdrs, ne*nqf);
+   VelNode.SetSize(dim, nd, ne);
+   VectorFunctionCoefficient velocity(dim, VelocityFunctionAdv);
+
+   Array<int> bdrs, orientation;
+   Array<IntegrationPoint> eip(nqf*dofs.NumBdrs);
+
+   if (dim==1)      { mesh->GetElementVertices(0, bdrs); }
+   else if (dim==2) { mesh->GetElementEdges(0, bdrs, orientation); }
+   else if (dim==3) { mesh->GetElementFaces(0, bdrs, orientation); }
+
+   for (int i = 0; i < dofs.NumBdrs; i++)
+   {
+      FaceElementTransformations *help
+         = mesh->GetFaceElementTransformations(bdrs[i]);
+
+      if (help->Elem1No != 0)
+      {
+         // NOTE: If this error ever occurs, use neighbor element to
+         // obtain the correct quadrature points and weight.
+         MFEM_ABORT("First element has inward pointing normal.");
+      }
+      for (int k = 0; k < nqf; k++)
+      {
+         const IntegrationPoint &ip = IntRuleFace->IntPoint(k);
+         help->Loc1.Transform(ip, eip[i*nqf + k]);
+      }
+   }
+
+   for (int e = 0; e < ne; e++)
+   {
+      ElementTransformation *eltrans = fes->GetElementTransformation(e);
+      velocity.Eval(VelEval, *eltrans, *IntRuleElem);
+
+      for (int k = 0; k < nqe; k++)
+      {
+         VelEval.GetColumnReference(k, vec);
+         mat.SetCol(k, vec);
+      }
+
+      VelElem(e) = mat;
+
+      for (int i = 0; i < nd; i++)
+      {
+         const IntegrationPoint ip = nodes->IntPoint(i);
+         velocity.Eval(vec, *eltrans, ip);
+         VelNode(e).SetCol(i, vec);
+      }
+
+      if (dim==1)      { mesh->GetElementVertices(e, bdrs); }
+      else if (dim==2) { mesh->GetElementEdges(e, bdrs, orientation); }
+      else if (dim==3) { mesh->GetElementFaces(e, bdrs, orientation); }
+
+      for (int i = 0; i < dofs.NumBdrs; i++)
+      {
+         FaceElementTransformations *facetrans
+            = mesh->GetFaceElementTransformations(bdrs[i]);
+
+         for (int k = 0; k < nqf; k++)
+         {
+            if (facetrans->Elem1No != e)
+            {
+               velocity.Eval(vval, *facetrans->Elem2, eip[i * nqf + k]);
+            }
+            else
+            {
+               velocity.Eval(vval, *facetrans->Elem1, eip[i * nqf + k]);
+            }
+
+            for (int l = 0; l < dim; l++)
+            {
+               VelFace(l, i, e * nqf + k) = vval(l);
+            }
+         }
+      }
+   }
+}
+
+void Advection::EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i) const
+{
+   Vector x(dim), v(dim);
+   VelocityFunctionAdv(x, v);
+   v *= u(0);
+   FluxEval.SetRow(0, v);
+   if (i == -1) // Element terms
+   {
+      VelocityVector = VelElem(e).GetColumn(k);
+      VelocityVector *= u(0);
+      FluxEval.SetRow(0, VelocityVector);
+   }
+   else
+   {
+      VelocityVector = VelFace(e*nqf+k).GetColumn(i);
+      VelocityVector *= u(0);
+      FluxEval.SetRow(0, VelocityVector);
+   }
+}
+
+double Advection::GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i) const
+{
+   if (i == -1) // Element terms
+   {
+      VelocityVector = VelElem(e).GetColumn(k);
+   }
+   else
+   {
+      VelocityVector = VelFace(e*nqf+k).GetColumn(i);
+   }
+
+   return abs(VelocityVector * n);
+}
+
+void Advection::ComputeErrors(Array<double> &errors, const GridFunction &u,
+                              double DomainSize, double t) const
+{
+   errors.SetSize(3);
+   VectorFunctionCoefficient uAnalytic(NumEq, AnalyticalSolutionAdv);
+   uAnalytic.SetTime(t);
+   errors[0] = u.ComputeLpError(1., uAnalytic) / DomainSize;
+   errors[1] = u.ComputeLpError(2., uAnalytic) / DomainSize;
+   errors[2] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic);
+}
+
+
+void VelocityFunctionAdv(const Vector &x, Vector &v)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+   double s = 1.0;
+
+   for (int i = 0; i < dim; i++)
+   {
+      switch (ConfigAdv.ConfigNum)
+      {
+         case 0:
+         case 1:
+         case 4:
+         case 5: // Map to the reference domain [0,1]^d.
+         {
+            X(i) = (x(i) - ConfigAdv.bbMin(i)) / (ConfigAdv.bbMax(i) - ConfigAdv.bbMin(i));
+            s *= ConfigAdv.bbMax(i) - ConfigAdv.bbMin(i);
+            break;
+         }
+         case 2:
+         case 3: // Map to the reference domain [-1,1]^d.
+         {
+            double center = 0.5 * (ConfigAdv.bbMin(i) + ConfigAdv.bbMax(i));
+            X(i) = 2. * (x(i) - center) / (ConfigAdv.bbMax(i) - ConfigAdv.bbMin(i));
+            s *= ConfigAdv.bbMax(i) - ConfigAdv.bbMin(i);
+            break;
+         }
+      }
+   }
+
+   // Scale to be normed to a full revolution.
+   s = pow(s, 1./dim);
+
+   switch (ConfigAdv.ConfigNum)
+   {
+      case 0: // Rotation around corner.
+      {
+         switch (dim)
+         {
+            case 1: v(0) = s; break;
+            case 2: s *= 2.0 * M_PI; v(0) = s*X(1); v(1) = -s*X(0); break;
+            case 3: s *= 2.0 * M_PI; v(0) = s*X(1); v(1) = -s*X(0); v(2) = 0.0; break;
+         }
+         break;
+      }
+      case 1: // Rotation around center.
+      {
+         switch (dim)
+         {
+            case 1: v(0) = s; break;
+            case 2: s *= 2.0 * M_PI; v(0) = s * (0.5-X(1)); v(1) = s*(X(0)-0.5); break;
+            case 3: s *= 2.0 * M_PI; v(0) = s * (0.5-X(1)); v(1) = s*(X(0)-0.5); v(2) = 0.0;
+               break;
+         }
+         break;
+      }
+      case 2:
+      case 3:
+      case 4:
+      case 5:
+      {
+         switch (dim)
+         {
+            case 1: v(0) = s; break;
+            case 2: v(0) = s; v(1) = -0.5*s; break;
+            case 3: v(0) = s; v(1) = -0.5*s; v(2) = 0.25*s; break;
+         }
+         break;
+      }
+   }
+}
+
+void AnalyticalSolutionAdv(const Vector &x, double t, Vector &u)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+
+   for (int i = 0; i < dim; i++)
+   {
+      switch (ConfigAdv.ConfigNum)
+      {
+         case 0:
+         case 1:
+         case 4:
+         case 5: // Map to the reference domain [0,1]^d.
+         {
+            X(i) = (x(i) - ConfigAdv.bbMin(i)) / (ConfigAdv.bbMax(i) - ConfigAdv.bbMin(i));
+            break;
+         }
+         case 2:
+         case 3: // Map to the reference domain [-1,1]^d.
+         {
+            double center = 0.5 * (ConfigAdv.bbMin(i) + ConfigAdv.bbMax(i));
+            X(i) = 2.0 * (x(i) - center) / (ConfigAdv.bbMax(i) - ConfigAdv.bbMin(i));
+            break;
+         }
+      }
+   }
+
+   double r = X.Norml2();
+
+   switch (ConfigAdv.ConfigNum)
+   {
+      case 0:
+      {
+         double a = 0.5, b = 0.03, c = 0.1;
+         u(0) = 0.25 * (1. + tanh((r+c-a)/b)) * (1. - tanh((r-c-a)/b));
+         break;
+      }
+      case 1:
+      {
+         if (dim==1) { MFEM_ABORT("Test case not implemented in 1D."); }
+
+         double s = 0.15;
+         double cone = sqrt(pow(X(0)-0.5, 2.) + pow(X(1)-0.25, 2.));
+         double hump = sqrt(pow(X(0)-0.25, 2.) + pow(X(1)-0.5, 2.));
+
+         u(0) = (1. - cone / s) * (cone <= s) +
+                0.25 * (1. + cos(M_PI*hump / s)) * (hump <= s) +
+                ( ( sqrt(pow(X(0)-0.5, 2.) + pow(X(1)-0.75, 2.)) <= s ) &&
+                  ( abs(X(0)-0.5) >= 0.025 || (X(1) >= 0.85) ) ? 1. : 0. );
+         break;
+      }
+      case 2: { u(0) = r < 0.2 ? 1. : 0.; break; }
+      case 3: { u(0) = exp(-25.0 * r*r); break; }
+      case 4: { u(0) = abs(r - 0.3) < 0.1 ? 1. : ( (abs(r-0.7) < 0.2) ? (exp(10.)*exp(-1./(r-0.5))*exp(1./(r-0.9))) : 0. ); break; }
+      case 5: { u(0) = abs(r-0.25) <= 0.15 ? 0.5*(1.+cos(M_PI*(r-0.25)/0.15)) : 0.; break; }
+   }
+}
+
+void InitialConditionAdv(const Vector &x, Vector &u)
+{
+   AnalyticalSolutionAdv(x, 0.0, u);
+}
+
+void InflowFunctionAdv(const Vector &x, double t, Vector &u)
+{
+   AnalyticalSolutionAdv(x, t, u);
+}

miniapps/hypsys/apps/advection.hpp

@@ -0,0 +1,26 @@
+#ifndef HYPSYS_ADVECTION
+#define HYPSYS_ADVECTION
+
+#include "hyperbolic_system.hpp"
+#include "../lib/dofs.hpp"
+
+class Advection : public HyperbolicSystem
+{
+public:
+   explicit Advection(FiniteElementSpace *fes_, BlockVector &u_block,
+                      Configuration &config_, bool NodalQuadRule);
+   ~Advection() { };
+
+   virtual void EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i = -1) const;
+   virtual double GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i) const;
+   virtual void ComputeErrors(Array<double> &errors, const GridFunction &u,
+                              double DomainSize, double t) const override;
+
+   int nqf;
+   DenseTensor VelElem, VelFace;
+   mutable Vector VelocityVector;
+};
+
+#endif

miniapps/hypsys/apps/buckley_leverett.cpp

@@ -0,0 +1,118 @@
+#include "buckley_leverett.hpp"
+
+Configuration ConfigBL;
+double BLConst;
+
+void InitialConditionBuckleyLeverett(const Vector &x, Vector &u);
+void InflowFunctionBuckleyLeverett(const Vector &x, double t, Vector &u);
+
+BuckleyLeverett::BuckleyLeverett(FiniteElementSpace *fes_, BlockVector &u_block,
+                                 Configuration &config_)
+   : HyperbolicSystem(fes_, u_block, 1, config_,
+                      VectorFunctionCoefficient(1, InflowFunctionBuckleyLeverett))
+{
+   ConfigBL = config_;
+   VectorFunctionCoefficient ic(NumEq, InitialConditionBuckleyLeverett);
+
+   switch (ConfigBL.ConfigNum)
+   {
+      case 1:
+      {
+         ProblemName = "Buckley-Leverett - 1D";
+         glvis_scale = "on";
+         BLConst = 0.5;
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 0;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 2:
+      {
+         ProblemName = "Buckley-Leverett - 2D";
+         glvis_scale = "on";
+         BLConst = 1.0;
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      default:
+         MFEM_ABORT("No such test case implemented.");
+   }
+}
+
+void BuckleyLeverett::EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                                   int e, int k, int i) const
+{
+   double coef = u(0)*u(0) / (u(0)*u(0) + BLConst * (1.0-u(0))*(1.0-u(0)));
+   FluxEval(0,0) = coef;
+   if (dim > 1)
+   {
+      FluxEval(0,1) = coef * (1.0 - 5.0 * (1.0-u(0))*(1.0-u(0)));
+   }
+   if (dim > 2) { MFEM_ABORT("Not implemented."); }
+}
+
+double BuckleyLeverett::GetWaveSpeed(const Vector &u, const Vector n, int e,
+                                     int k,
+                                     int i) const
+{
+   if (dim == 1)
+   {
+      return abs( 2.0 * BLConst * u(0) * (1.0-u(0)) / pow(u(0)*u(0) + BLConst *
+                                                          (1.0-u(0))*(1.0-u(0)), 2.0) );
+   }
+   else if (dim == 2)
+   {
+      return 3.4;
+   }
+   else { MFEM_ABORT("Not implemented."); }
+}
+
+void InitialConditionBuckleyLeverett(const Vector &x, Vector &u)
+{
+   const int dim = x.Size();
+
+   // Map to the reference domain [-1,1]^d.
+   Vector X(dim);
+   for (int i = 0; i < dim; i++)
+   {
+      double center = 0.5 * (ConfigBL.bbMin(i) + ConfigBL.bbMax(i));
+      X(i) = 2. * (x(i) - center) / (ConfigBL.bbMax(i) - ConfigBL.bbMin(i));
+   }
+
+   switch (ConfigBL.ConfigNum)
+   {
+      case 1:
+      {
+         u(0) =  X(0) < 0.0 ? -3.0 : 3.0;
+         break;
+      }
+      case 2:
+      {
+         u(0) = X.Norml2()*X.Norml2() < 2.0 / 9.0 ? 1.0 : 0.0;
+         break;
+      }
+   }
+}
+
+void InflowFunctionBuckleyLeverett(const Vector &x, double t, Vector &u)
+{
+   switch (ConfigBL.ConfigNum)
+   {
+      case 1:
+      {
+         u(0) = x(0) < 0.0 ? 3.0 : -3.0;
+         break;
+      }
+      case 2:
+      {
+         u(0) = 0.0;
+         break;
+      }
+   }
+}

miniapps/hypsys/apps/buckley_leverett.hpp

@@ -0,0 +1,19 @@
+#ifndef HYPSYS_BUCKLEYLEVERETT
+#define HYPSYS_BUCKLEYLEVERETT
+
+#include "hyperbolic_system.hpp"
+
+class BuckleyLeverett : public HyperbolicSystem
+{
+public:
+   explicit BuckleyLeverett(FiniteElementSpace *fes_, BlockVector &u_block,
+                            Configuration &config_);
+   ~BuckleyLeverett() { };
+
+   virtual void EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i = -1) const;
+   virtual double GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i) const;
+};
+
+#endif

miniapps/hypsys/apps/burgers.cpp

@@ -0,0 +1,175 @@
+#include "burgers.hpp"
+
+Configuration ConfigBurgers;
+
+void AnalyticalSolutionBurgers(const Vector &x, double t, Vector &u);
+void InitialConditionBurgers(const Vector &x, Vector &u);
+void InflowFunctionBurgers(const Vector &x, double t, Vector &u);
+
+Burgers::Burgers(FiniteElementSpace *fes_, BlockVector &u_block,
+                 Configuration &config_)
+   : HyperbolicSystem(fes_, u_block, 1, config_,
+                      VectorFunctionCoefficient (1, InflowFunctionBurgers))
+{
+   ConfigBurgers = config_;
+   VectorFunctionCoefficient ic(NumEq, InitialConditionBurgers);
+
+   switch (ConfigBurgers.ConfigNum)
+   {
+      case 0:
+      {
+         ProblemName = "Burgers Equation - 1D";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 1:
+      {
+         ProblemName = "Burgers Equation - Riemann Problem";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = true;
+         ProjType = 1;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 2:
+      {
+         ProblemName = "Burgers Equation - Steady State";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = true;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      default:
+         MFEM_ABORT("No such test case implemented.");
+   }
+}
+
+void Burgers::EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                           int e, int k, int i) const
+{
+   FluxEval = 0.5 * u(0) * u(0);
+}
+
+double Burgers::GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                             int i) const
+{
+   return abs(u(0) * double(n.Size()));
+}
+
+void Burgers::ComputeErrors(Array<double> &errors, const GridFunction &u,
+                            double DomainSize, double t) const
+{
+   errors.SetSize(3);
+   VectorFunctionCoefficient uAnalytic(NumEq, AnalyticalSolutionBurgers);
+   uAnalytic.SetTime(t);
+   errors[0] = u.ComputeLpError(1., uAnalytic) / DomainSize;
+   errors[1] = u.ComputeLpError(2., uAnalytic) / DomainSize;
+   errors[2] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic);
+}
+
+
+void AnalyticalSolutionBurgers(const Vector &x, double t, Vector &u)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+
+   // Map to the reference domain [0,1]^d.
+   for (int i = 0; i < dim; i++)
+   {
+      double factor = 1.0 / ( ConfigBurgers.bbMax(i) - ConfigBurgers.bbMin(i));
+      X(i) = factor * (x(i) - ConfigBurgers.bbMin(i));
+      t *= pow(factor, 1.0 / (double(dim)));
+   }
+
+   switch (ConfigBurgers.ConfigNum)
+   {
+      case 0:
+      {
+         if (dim != 1) { MFEM_ABORT("Test case only implemented in 1D."); }
+
+         double un = sin(2.0*M_PI*X(0));
+         double fn, fpn;
+         double tol = 1.E-15;
+         double error = 1.0;
+         int iter = 0, maxiter = 100;
+
+         while (error > tol)
+         {
+            // Do not trust this solution at a time later than t = 0.1.
+            if (iter == maxiter) { break; }
+
+            fn = sin(2.0*M_PI*(X(0)-un*t))-un;
+            fpn = -2.0*M_PI*t*cos(2.*M_PI*(X(0)-un*t))-1.0;
+            un -= fn/fpn;
+            error = abs(sin(2.*M_PI*(X(0)-un*t))-un);
+            iter++;
+         }
+
+         u(0) = un;
+         break;
+      }
+      case 1:
+      {
+         if (dim != 2) { MFEM_ABORT("Test case only implemented in 2D."); }
+
+         if (X(0) <= 0.5 - 0.6 * t)
+         {
+            u(0) = X(1) >= 0.5 + 0.15 * t ? -0.2 : 0.5;
+         }
+         else if (X(0) < 0.5 - 0.25 * t)
+         {
+            u(0) = X(1) > -8. / 7. * X(0) + 15. / 14. - 15. / 28. * t ? -1. : 0.5;
+         }
+         else if (X(0) < 0.5 + 0.5 * t)
+         {
+            u(0) = X(1) > X(0) / 6. + 5. / 12. - 5. / 24. * t ? -1. : 0.5;
+         }
+         else if (X(0) < 0.5 + 0.8 * t)
+         {
+            u(0) = X(1) > X(0) - 5. / (18. * t) * (X(0) + t - 0.5)
+                   * (X(0) + t - 0.5) ? -1. : (2. * X(0) - 1.) / (2 * t);
+         }
+         else
+         {
+            u(0) = X(1) >= 0.5 - 0.1 * t ? -1 : 0.8;
+         }
+
+         break;
+      }
+      case 2:
+      {
+         u(0) = X.Sum() < 0.5 ? 1.0 : -1.0;
+         break;
+      }
+   }
+}
+
+void InitialConditionBurgers(const Vector &x,Vector &u)
+{
+   switch (ConfigBurgers.ConfigNum)
+   {
+      case 0:
+      case 1:
+      case 2: { AnalyticalSolutionBurgers(x, 0.0, u); break; }
+   }
+}
+
+void InflowFunctionBurgers(const Vector &x, double t, Vector &u)
+{
+   switch (ConfigBurgers.ConfigNum)
+   {
+      case 0:
+      case 1:
+      case 2: { AnalyticalSolutionBurgers(x, t, u); break; }
+   }
+}

miniapps/hypsys/apps/burgers.hpp

@@ -0,0 +1,21 @@
+#ifndef HYPSYS_BURGERS
+#define HYPSYS_BURGERS
+
+#include "hyperbolic_system.hpp"
+
+class Burgers : public HyperbolicSystem
+{
+public:
+   explicit Burgers(FiniteElementSpace *fes_, BlockVector &u_block,
+                    Configuration &config_);
+   ~Burgers() { };
+
+   virtual void EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i = -1) const;
+   virtual double GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i) const;
+   virtual void ComputeErrors(Array<double> &errors, const GridFunction &u,
+                              double DomainSize, double t) const override;
+};
+
+#endif

miniapps/hypsys/apps/euler.cpp

@@ -0,0 +1,647 @@
+#include "euler.hpp"
+
+Configuration ConfigEuler;
+
+double SpHeatRatio;
+
+void AnalyticalSolutionEuler(const Vector &x, double t, Vector &u);
+void InitialConditionEuler(const Vector &x, Vector &u);
+void InflowFunctionEuler(const Vector &x, double t, Vector &u);
+
+Euler::Euler(FiniteElementSpace *fes_, BlockVector &u_block,
+             Configuration &config_)
+   : HyperbolicSystem(fes_, u_block, fes_->GetMesh()->Dimension() + 2, config_,
+                      VectorFunctionCoefficient(fes_->GetMesh()->Dimension() + 2,
+                                                InflowFunctionEuler))
+{
+   ConfigEuler = config_;
+   VectorFunctionCoefficient ic(NumEq, InitialConditionEuler);
+
+   switch (ConfigEuler.ConfigNum)
+   {
+      case 0:
+      {
+         // Periodic meshes must be used for this problem.
+         ProblemName = "Euler Equations of Gas dynamics - Smooth Vortex";
+         glvis_scale = "on";
+         SpHeatRatio = 1.4;
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 0;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 1:
+      {
+         ProblemName = "Euler Equations of Gas dynamics - SOD Shock Tube";
+         glvis_scale = "on";
+         SpHeatRatio = 1.4;
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 2:
+      {
+         ProblemName = "Euler Equations of Gas dynamics - Woodward Colella";
+         glvis_scale = "on";
+         SpHeatRatio = 1.4;
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 3:
+      {
+         ProblemName = "Euler Equations of Gas dynamics - Double Mach Reflection";
+         glvis_scale = "on";
+         SpHeatRatio = 1.4;
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = true;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 4:
+      {
+         ProblemName = "Euler Equations of Gas dynamics - Sedov Blast";
+         glvis_scale = "on";
+         SpHeatRatio = 5.0 / 3.0;
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = true;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 5:
+      {
+         ProblemName = "Euler Equations of Gas dynamics - Noh Problem";
+         glvis_scale = "on";
+         SpHeatRatio = 5.0 / 3.0;
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = true;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 6:
+      {
+         ProblemName = "Euler Equations of Gas dynamics - Gresho Vortex";
+         glvis_scale = "on";
+         SpHeatRatio = 1.4;
+         SolutionKnown = true;
+         SteadyState = true;
+         TimeDepBC = false;
+         ProjType = 0;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 7:
+      {
+         ProblemName = "Euler Equations of Gas dynamics - Constricted Channel";
+         glvis_scale = "on";
+         SpHeatRatio = 1.4;
+         SolutionKnown = false;
+         SteadyState = true;
+         TimeDepBC = false;
+         ProjType = 0;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      default:
+         MFEM_ABORT("No such test case implemented.");
+   }
+}
+
+double Euler::EvaluatePressure(const Vector &u) const
+{
+   double aux = 0.0;
+   for (int l = 0; l < dim; l++)
+   {
+      aux += u(1+l) * u(1+l);
+   }
+   double pressure = (SpHeatRatio - 1.0) * (u(dim+1) - 0.5 * aux / u(0));
+   if (pressure < 0.)
+   {
+      ostringstream press_str;
+      press_str << pressure;
+      string err_msg = "Negative pressure p = ";
+      MFEM_ABORT(err_msg << press_str.str());
+   }
+   return pressure;
+}
+
+void Euler::EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                         int e, int k, int i) const
+{
+   double pressure = EvaluatePressure(u);
+   CheckAdmissibility(u);
+
+   switch (dim)
+   {
+      case 1:
+      {
+         double vx = u(1) / u(0);
+         FluxEval(0,0) = u(1);
+         FluxEval(1,0) = u(1) * vx + pressure;
+         FluxEval(2,0) = (u(2) + pressure) * vx;
+         break;
+      }
+      case 2:
+      {
+         double vx = u(1) / u(0);
+         double vy = u(2) / u(0);
+         double energy = u(3) + pressure;
+
+         FluxEval(0,0) = u(1);
+         FluxEval(0,1) = u(2);
+
+         FluxEval(1,0) = u(1) * vx + pressure;
+         FluxEval(1,1) = u(1) * vy;
+
+         FluxEval(2,0) = u(2) * vx;
+         FluxEval(2,1) = u(2) * vy + pressure;
+
+         FluxEval(3,0) = energy * vx;
+         FluxEval(3,1) = energy * vy;
+         break;
+      }
+      case 3:
+      {
+         double vx = u(1) / u(0);
+         double vy = u(2) / u(0);
+         double vz = u(3) / u(0);
+         double energy = u(4) + pressure;
+
+         FluxEval(0,0) = u(1);
+         FluxEval(0,1) = u(2);
+         FluxEval(0,2) = u(3);
+
+         FluxEval(1,0) = u(1) * vx + pressure;
+         FluxEval(1,1) = u(1) * vy;
+         FluxEval(1,2) = u(1) * vz;
+
+         FluxEval(2,0) = u(2) * vx;
+         FluxEval(2,1) = u(2) * vy + pressure;
+         FluxEval(2,2) = u(2) * vz;
+
+         FluxEval(3,0) = u(3) * vx;
+         FluxEval(3,1) = u(3) * vy;
+         FluxEval(3,2) = u(3) * vz + pressure;
+
+         FluxEval(4,0) = energy * vx;
+         FluxEval(4,1) = energy * vy;
+         FluxEval(4,2) = energy * vz;
+         break;
+      }
+      default:
+         MFEM_ABORT("Invalid space dimension.");
+   }
+}
+
+double Euler::GetGMS(const Vector &uL, const Vector &uR,
+                     const Vector &normal) const
+{
+   CheckAdmissibility(uL);
+   CheckAdmissibility(uR);
+   double pL = EvaluatePressure(uL);
+   double pR = EvaluatePressure(uR);
+   double aL = sqrt(SpHeatRatio * pL / uL(0));
+   double aR = sqrt(SpHeatRatio * pR / uR(0));
+   double vL = uL(1)/uL(0) * normal(0);
+   double vR = uR(1)/uR(0) * normal(0);
+
+   double p = pow( (aL+aR-0.5*(SpHeatRatio-1.)*(vR-vL)) / (aL*pow(pL,
+                                                                  (1.-SpHeatRatio)/(2.*SpHeatRatio)) + aR*pow(pR,
+                                                                        (1.-SpHeatRatio)/(2.*SpHeatRatio)) ), 2.*SpHeatRatio/(SpHeatRatio-1.) );
+
+   double lambda1 = vL - aL * sqrt( 1. + (SpHeatRatio+1.)/(2.*SpHeatRatio) * max(
+                                       0., (p-pL)/pL) );
+   double lambda3 = vR + aR * sqrt( 1. + (SpHeatRatio+1.)/(2.*SpHeatRatio) * max(
+                                       0., (p-pR)/pR) );
+   return max(abs(lambda1), abs(lambda3));
+}
+
+double Euler::GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                           int i) const
+{
+   CheckAdmissibility(u);
+   switch (u.Size())
+   {
+      case 3:
+         return abs( u(1)*n(0) / u(0) ) + sqrt(SpHeatRatio * EvaluatePressure(u) / u(0));
+      case 4:
+         return abs( (u(1)*n(0) + u(2)*n(1)) / u(0) )
+                + sqrt(SpHeatRatio * EvaluatePressure(u) / u(0));
+      case 5:
+         return abs( (u(1)*n(0) + u(2)*n(1) + u(3)*n(2)) / u(0) )
+                + sqrt(SpHeatRatio * EvaluatePressure(u) / u(0));
+   }
+}
+
+void Euler::CheckAdmissibility(const Vector &u) const
+{
+   double RhoMin = 1.e-12;
+
+   if (u.Size() != NumEq) { MFEM_ABORT("Invalid solution vector."); }
+
+   if (u(0) < RhoMin)
+   {
+      ostringstream rho_str;
+      rho_str << u(0);
+      string err_msg = "Density too small rho = ";
+      MFEM_ABORT(err_msg << rho_str.str());
+   }
+}
+
+void Euler::SetBdrCond(const Vector &y1, Vector &y2, const Vector &normal,
+                       int attr) const
+{
+   switch (attr)
+   {
+      case -1: // wall boundary
+      {
+         if (dim == 1)
+         {
+            y2(0) = y1(0);
+            y2(1) = -y1(1);
+            y2(2) = y1(2);
+         }
+         else if (dim == 2)
+         {
+            double MomTimesNorm = y1(1) * normal(0) + y1(2) * normal(1);
+            y2(0) = y1(0);
+            y2(1) = y1(1) - 2. * MomTimesNorm * normal(0);
+            y2(2) = y1(2) - 2. * MomTimesNorm * normal(1);
+            y2(3) = y1(3);
+         }
+         else
+         {
+            double MomTimesNorm = y1(1) * normal(0) + y1(2) * normal(1) + y1(3) * normal(2);
+            y2(0) = y1(0);
+            y2(1) = y1(1) - 2. * MomTimesNorm * normal(0);
+            y2(2) = y1(2) - 2. * MomTimesNorm * normal(1);
+            y2(3) = y1(3) - 2. * MomTimesNorm * normal(2);
+            y2(4) = y1(4);
+         }
+         break;
+      }
+      case -2: // supersonic outlet
+      {
+         y2 = y1;
+         break;
+      }
+      case -3: // supersonic inlet
+      {
+         break;
+      }
+      // TODO subsonic in- and outlet
+      default:
+         MFEM_ABORT("Invalid boundary attribute.");
+   }
+}
+
+void Euler::ComputeDerivedQuantities(const GridFunction &u, GridFunction &d1,
+                                     GridFunction &d2) const
+{
+   double density, momentum;
+   const IntegrationRule ir = u.FESpace()->GetFE(0)->GetNodes();
+
+   for (int e = 0; e < ne; e++)
+   {
+      for (int i = 0; i < nd; i++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(i);
+         density = u.GetValue(e, ip, 1);
+         momentum = u.GetValue(e, ip, 2);
+         d1(e*nd + i) = pow(momentum / density, 2.0);
+
+         if (dim > 1)
+         {
+            momentum = u.GetValue(e, ip, 3);
+            d1(e*nd + i) += pow(momentum / density, 2.0);
+         }
+         if (dim > 2)
+         {
+            momentum = u.GetValue(e, ip, 4);
+            d1(e*nd + i) += pow(momentum / density, 2.0);
+         }
+
+         d2(e*nd + i) = (SpHeatRatio - 1.0) * (u.GetValue(e, ip,
+                                                          dim+2) - 0.5 * density * d1(e*nd + i));
+         d1(e*nd + i) = sqrt(d1(e*nd + i));
+      }
+   }
+}
+
+void Euler::ComputeErrors(Array<double> & errors, const GridFunction &u,
+                          double DomainSize, double t) const
+{
+   errors.SetSize(NumEq*3);
+   Vector component(dim+2);
+   VectorFunctionCoefficient uAnalytic(NumEq, AnalyticalSolutionEuler);
+
+   if (ConfigEuler.ConfigNum == 0) { uAnalytic.SetTime(0);  }
+   else { uAnalytic.SetTime(t); }
+
+   component = 0.0;
+   component(0) = 1.0;
+   VectorConstantCoefficient weight1(component);
+   errors[0] = u.ComputeLpError(1.0, uAnalytic, NULL, &weight1) / DomainSize;
+   errors[1] = u.ComputeLpError(2.0, uAnalytic, NULL, &weight1) / DomainSize;
+   errors[2] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic,
+                                NULL, &weight1);
+
+   component = 0.0;
+   component(1) = 1.0;
+   VectorConstantCoefficient weight2(component);
+   errors[3] = u.ComputeLpError(1.0, uAnalytic, NULL, &weight2) / DomainSize;
+   errors[4] = u.ComputeLpError(2.0, uAnalytic, NULL, &weight2) / DomainSize;
+   errors[5] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic,
+                                NULL, &weight2);
+
+   component = 0.0;
+   component(2) = 1.0;
+   VectorConstantCoefficient weight3(component);
+   errors[6] = u.ComputeLpError(1.0, uAnalytic, NULL, &weight3) / DomainSize;
+   errors[7] = u.ComputeLpError(2.0, uAnalytic, NULL, &weight3) / DomainSize;
+   errors[8] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic,
+                                NULL, &weight3);
+
+   if (dim > 1)
+   {
+      component = 0.0;
+      component(3) = 1.0;
+      VectorConstantCoefficient weight4(component);
+      errors[9] = u.ComputeLpError(1.0, uAnalytic, NULL, &weight4) / DomainSize;
+      errors[10] = u.ComputeLpError(2.0, uAnalytic, NULL, &weight4) / DomainSize;
+      errors[11] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic,
+                                    NULL, &weight4);
+   }
+
+   if (dim > 2)
+   {
+      component = 0.0;
+      component(4) = 1.0;
+      VectorConstantCoefficient weight5(component);
+      errors[12] = u.ComputeLpError(1.0, uAnalytic, NULL, &weight5) / DomainSize;
+      errors[13] = u.ComputeLpError(2.0, uAnalytic, NULL, &weight5) / DomainSize;
+      errors[14] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic,
+                                    NULL, &weight5);
+   }
+}
+
+void EvaluateEnergy(Vector &u, const double &pressure)
+{
+   const int dim = u.Size() - 2;
+   double aux = 0.0;
+   for (int l = 0; l < dim; l++)
+   {
+      aux += u(1+l)*u(1+l);
+   }
+   u(dim+1) = pressure / (SpHeatRatio - 1.0) + 0.5 * aux / u(0);
+}
+
+void AnalyticalSolutionEuler(const Vector &x, double t, Vector &u)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+
+   for (int i = 0; i < dim; i++)
+   {
+      switch (ConfigEuler.ConfigNum)
+      {
+         case 0:
+         case 5:
+         case 6: // Map to the reference domain [-1,1]^d.
+         {
+            double center = 0.5 * (ConfigEuler.bbMin(i) + ConfigEuler.bbMax(i));
+            double factor = 2.0 / (ConfigEuler.bbMax(i) - ConfigEuler.bbMin(i));
+            X(i) = factor * (x(i) - center);
+            t *= pow(factor, 1.0 / (double(dim)));
+            break;
+         }
+         case 3: // Map to the reference domain [0,1]^d.
+         {
+            double factor = 1.0 / (ConfigEuler.bbMax(i) - ConfigEuler.bbMin(i));
+            X(i) = factor * (x(i) - ConfigEuler.bbMin(i));
+            t *= pow(factor, 1.0 / (double(dim)));
+            break;
+         }
+      }
+   }
+
+   switch (ConfigEuler.ConfigNum)
+   {
+      case 0:
+      {
+         if (dim != 2) { MFEM_ABORT("Test case works only in 2D."); }
+
+         // Map to test case specific domain [-5,5]^d.
+         X *= 5.0;
+         t *= 5.0;
+
+         double beta = 5.0;
+         double r = X.Norml2();
+         double T0 = 1.0 - (SpHeatRatio - 1.0) * beta * beta
+                     / (8.0 * SpHeatRatio * M_PI * M_PI) * exp(1.0 - r*r);
+
+         u(0) = pow(T0, 1.0 / (SpHeatRatio - 1.0));
+         u(1) = (1.0 - beta / (2.0 * M_PI) * exp(0.5 * (1.0 - r*r)) * X(1)) * u(0);
+         u(2) = (1.0 + beta / (2.0 * M_PI) * exp(0.5 * (1.0 - r*r)) * X(0)) * u(0);
+         EvaluateEnergy(u, u(0) * T0);
+         break;
+      }
+      case 3:
+      {
+         if (dim != 2) { MFEM_ABORT("Test case works only in 2D."); }
+
+         // Map to test case specific domain [0,4] x [0,1].
+         X(0) = 4.0 * X(0);
+         t *= 2.0;
+
+         bool PostShock = X(0) < 1.0/6.0 + (X(1) + 20.0*t) / sqrt(3.0);
+
+         if (PostShock)
+         {
+            u(0) = 8.0;
+            u(1) =  66.0 * cos(M_PI / 6.0);
+            u(2) = -66.0 * sin(M_PI / 6.0);
+            EvaluateEnergy(u, 116.5);
+         }
+         else
+         {
+            u = 0.0;
+            u(0) = 1.4;
+            EvaluateEnergy(u, 1.0);
+         }
+
+         break;
+      }
+      case 5:
+      {
+         double r = X.Norml2();
+
+         if (r > t / 3.)
+         {
+            u(0) = 1.0 + t / r;
+            for (int l = 0; l < dim; l++) { u(l+1) = -X(l) / r * u(0); }
+            EvaluateEnergy(u, 1.0E-6);
+         }
+         else
+         {
+            u(0) = 1.0;
+            for (int l = 0; l < dim; l++) { u(l+1) = 0.0; }
+            EvaluateEnergy(u, 16.0 / 3.0);
+         }
+
+         break;
+      }
+      case 6:
+      {
+         if (dim != 2) { MFEM_ABORT("Test case works only in 2D."); }
+
+         double pressure = 3.0 + 4.0*log(2.0);
+         double r = X.Norml2();
+
+         u = 0.0;
+         u(0) = 1.0;
+
+         if (r < 0.2)
+         {
+            u(1) = -5.0 * X(1);
+            u(2) =  5.0 * X(0);
+            pressure = 5.0 + 12.5*r*r;
+         }
+         else if (r < 0.4)
+         {
+            u(1) = -(2.0 / r - 5.0) * X(1);
+            u(2) =  (2.0 / r - 5.0) * X(0);
+            pressure = 9.0 + 4.0 * (log(r) - log(0.2)) + 12.5*r*r - 20.0*r;
+         }
+
+         EvaluateEnergy(u, pressure);
+         break;
+      }
+      default:
+         MFEM_ABORT("Analytical solution not known.");
+   }
+}
+
+void InitialConditionEuler(const Vector &x, Vector &u)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+
+   for (int i = 0; i < dim; i++)
+   {
+      switch (ConfigEuler.ConfigNum)
+      {
+         case 4: // Map to the reference domain [-1,1]^d.
+         {
+            double center = 0.5 * (ConfigEuler.bbMin(i) + ConfigEuler.bbMax(i));
+            double factor = 2.0 / (ConfigEuler.bbMax(i) - ConfigEuler.bbMin(i));
+            X(i) = factor * (x(i) - center);
+            break;
+         }
+         case 1:
+         case 2: // Map to the reference domain [0,1]^d.
+         {
+            double factor = 1.0 / (ConfigEuler.bbMax(i) - ConfigEuler.bbMin(i));
+            X(i) = factor * (x(i) - ConfigEuler.bbMin(i));
+            break;
+         }
+      }
+   }
+
+   switch (ConfigEuler.ConfigNum)
+   {
+      case 0:
+      case 3:
+      case 5:
+      case 6:
+      {
+         AnalyticalSolutionEuler(x, 0.0, u);
+         break;
+      }
+      case 1:
+      {
+         if (dim != 1) { MFEM_ABORT("Test case works only in 1D."); }
+
+         u = 0.0;
+         u(0) = X.Norml2() < 0.5 ? 1.0 : 0.125;
+         EvaluateEnergy(u, X.Norml2() < 0.5 ? 1.0 : 0.1);
+         break;
+      }
+      case 2:
+      {
+         if (dim != 1) { MFEM_ABORT("Test case works only in 1D."); }
+
+         u = 0.0;
+         u(0) = 1.0;
+         if (X(0) < 0.1)
+         {
+            EvaluateEnergy(u, 1000.);
+         }
+         else if (X(0) < 0.9)
+         {
+            EvaluateEnergy(u, 0.01);
+         }
+         else
+         {
+            EvaluateEnergy(u, 100.);
+         }
+         break;
+      }
+      case 4:
+      {
+         // Map to test case specific domain [-5,5]^d.
+         X *= 5.0;
+
+         u = 0.0;
+         u(0) = 1.0;
+         // TODO make sure that energy is essentially a delta distribution.
+         u(dim+1) = X.Norml2() < 1.0E-1 ? 1000.0 : 1.0E-8;
+         break;
+      }
+      case 7:
+      {
+         u = 0.0;
+         u(0) = 1.0;
+         u(1) = 1.0;
+         EvaluateEnergy(u, 0.1);
+         break;
+      }
+   }
+}
+
+void InflowFunctionEuler(const Vector &x, double t, Vector &u)
+{
+   switch (ConfigEuler.ConfigNum)
+   {
+      case 0:
+      case 3:
+      case 5:
+      case 6:
+      {
+         AnalyticalSolutionEuler(x, t, u);
+         break;
+      }
+      case 7:
+      {
+         InitialConditionEuler(x, u);
+         break;
+      }
+      case 1:
+      case 2:
+      case 4: break; // No boundary conditions needed.
+   }
+}

miniapps/hypsys/apps/euler.hpp

@@ -0,0 +1,30 @@
+#ifndef HYPSYS_EULER
+#define HYPSYS_EULER
+
+#include "hyperbolic_system.hpp"
+
+class Euler : public HyperbolicSystem
+{
+public:
+   explicit Euler(FiniteElementSpace *fes_, BlockVector &u_block,
+                  Configuration &config_);
+   ~Euler() { };
+
+   virtual double EvaluatePressure(const Vector &u) const;
+
+   virtual void EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i = -1) const;
+   virtual double GetGMS(const Vector &uL, const Vector &uR,
+                         const Vector &normal) const override;
+   virtual double GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i) const;
+   virtual void CheckAdmissibility(const Vector &u) const override;
+   virtual void SetBdrCond(const Vector &y1, Vector &y2, const Vector &normal,
+                           int attr) const override;
+   virtual void ComputeDerivedQuantities(const GridFunction &u, GridFunction &d1,
+                                         GridFunction &d2) const override;
+   virtual void ComputeErrors(Array<double> &errors, const GridFunction &u,
+                              double DomainSize, double t) const override;
+};
+
+#endif

miniapps/hypsys/apps/hyperbolic_system.hpp

@@ -0,0 +1,140 @@
+#ifndef HYPSYS_HYPERBOLIC_SYSTEM
+#define HYPSYS_HYPERBOLIC_SYSTEM
+
+#include "../lib/tools.hpp"
+
+struct Configuration
+{
+   int ConfigNum;
+   double tFinal;
+   Vector bbMin, bbMax;
+};
+
+class HyperbolicSystem
+{
+public:
+   explicit HyperbolicSystem(FiniteElementSpace *fes_, BlockVector &u_block,
+                             int NumEq_, Configuration &config_,
+                             VectorFunctionCoefficient BdrCond_) : fes(fes_), u0(fes_, u_block),
+      NumEq(NumEq_), BdrCond(BdrCond_)
+   {
+      ne = fes->GetNE();
+      nd = fes->GetFE(0)->GetDof();
+      dim = fes->GetMesh()->Dimension();
+
+      l2_fec = new L2_FECollection(fes->GetFE(0)->GetOrder(),
+                                   fes->GetMesh()->Dimension());
+      l2_fes = new FiniteElementSpace(fes->GetMesh(), l2_fec, NumEq,
+                                      Ordering::byNODES);
+      l2_proj = new GridFunction(l2_fes);
+   }
+
+   virtual ~HyperbolicSystem()
+   {
+      delete l2_proj;
+      delete l2_fes;
+      delete l2_fec;
+   }
+
+   virtual void EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i = -1) const = 0;
+   virtual double GetGMS(const Vector &uL, const Vector &uR,
+                         const Vector &normal) const { } // TODO: if used "= 0"
+   virtual double GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i = -1) const = 0;
+   virtual void CheckAdmissibility(const Vector &u) const { };
+   virtual void SetBdrCond(const Vector &y1, Vector &y2, const Vector &normal,
+                           int attr) const  { };
+   virtual void ComputeDerivedQuantities(const GridFunction &u, GridFunction &d1,
+                                         GridFunction &d2) const { };
+   virtual void ComputeErrors(Array<double> &errors, const GridFunction &u,
+                              double DomainSize, double t) const { };
+
+   virtual void WriteErrors(const Array<double> &errors) const
+   {
+      ofstream file("errors.txt", ios_base::app);
+
+      if (!file)
+      {
+         MFEM_ABORT("Error opening file.");
+      }
+      else
+      {
+         ostringstream strs;
+         for (int i = 0; i < errors.Size(); i++)
+         {
+            strs << errors[i] << " ";
+         }
+         strs << "\n";
+         string str = strs.str();
+         file << str;
+         file.close();
+      }
+   }
+
+   // L2 projection for scalar problems.
+   void L2_Projection(FunctionCoefficient fun, GridFunction &proj) const
+   {
+      l2_proj->ProjectCoefficient(fun);
+      proj.ProjectGridFunction(*l2_proj);
+   }
+
+   // L2 projection for systems.
+   void L2_Projection(VectorFunctionCoefficient fun, GridFunction &proj) const
+   {
+      l2_proj->ProjectCoefficient(fun);
+      proj.ProjectGridFunction(*l2_proj);
+   }
+
+   // Lumped L2 projection for general problems.
+   void LumpedL2_Projection(VectorFunctionCoefficient fun, GridFunction &proj) const
+   {
+      Vector LumpedMassMat;
+
+      Vector aux_vec(NumEq);
+      aux_vec = 1.0;
+      VectorConstantCoefficient ones(aux_vec);
+      BilinearForm ml(fes);
+      ml.AddDomainIntegrator(new LumpedIntegrator(new VectorMassIntegrator(ones)));
+      ml.Assemble();
+      ml.Finalize();
+      ml.SpMat().GetDiag(LumpedMassMat);
+
+      LinearForm rhs(fes);
+      rhs.AddDomainIntegrator(new VectorDomainLFIntegrator(fun));
+      rhs.Assemble();
+
+      for (int i = 0; i < LumpedMassMat.Size(); i++)
+      {
+         proj(i) = rhs.Elem(i) / LumpedMassMat(i);
+      }
+   }
+
+   int ne, nd, dim;
+
+   // 0: L2 projection,
+   // 1: Nodal values as GridFunction coefficients (only second order accurate).
+   int ProjType;
+   const int NumEq;
+
+   FiniteElementSpace *fes;
+   GridFunction u0;
+
+   // Auxiliary data needed for L2 projections
+   L2_FECollection *l2_fec;
+   FiniteElementSpace *l2_fes;
+   GridFunction *l2_proj;
+
+   mutable VectorFunctionCoefficient BdrCond;
+
+   string ProblemName, glvis_scale;
+   bool SolutionKnown;
+   bool SteadyState;
+   bool TimeDepBC;
+
+   // Currently only true for advection, due to spatially dependent flux.
+   bool DiscreteUpwinding = false;
+   DenseTensor VelNode;
+};
+
+#endif

miniapps/hypsys/apps/kpp.cpp

@@ -0,0 +1,121 @@
+#include "kpp.hpp"
+
+Configuration ConfigKPP;
+
+void InitialConditionKPP(const Vector &x, Vector &u);
+void InflowFunctionKPP(const Vector &x, double t, Vector &u);
+
+KPP::KPP(FiniteElementSpace *fes_, BlockVector &u_block,
+         Configuration &config_)
+   : HyperbolicSystem(fes_, u_block, 1, config_,
+                      VectorFunctionCoefficient(1, InflowFunctionKPP))
+{
+   ConfigKPP = config_;
+   VectorFunctionCoefficient ic(NumEq, InitialConditionKPP);
+
+   switch (ConfigKPP.ConfigNum)
+   {
+      case 1:
+      {
+         ProblemName = "KPP Equation - 2D Spiral";
+         glvis_scale = "on";
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 2:
+      {
+         ProblemName = "KPP Equation - 1D";
+         glvis_scale = "on";
+         SolutionKnown =
+            false; // There is a solution, but I don't have an analytical expression.
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      default:
+         MFEM_ABORT("No such test case implemented.");
+   }
+}
+
+void KPP::EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                       int e, int k, int i) const
+{
+   if (dim==1)
+   {
+      double coef = u(0)*(1.0-u(0));
+      FluxEval(0,0) = u(0) < 0.5 ? (0.25*coef) : (0.1875 - 0.5*coef);
+   }
+   else if (dim==2)
+   {
+      FluxEval(0,0) = sin(u(0));
+      FluxEval(0,1) = cos(u(0));
+   }
+   else { MFEM_ABORT("Not implemented."); }
+}
+
+double KPP::GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                         int i) const
+{
+   return 1.0; // Tighter bound exists.
+}
+
+void InitialConditionKPP(const Vector &x, Vector &u)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+
+   for (int i = 0; i < dim; i++)
+   {
+      switch (ConfigKPP.ConfigNum)
+      {
+         case 1: // Map to the reference domain [-1,1]^d.
+         {
+            double center = 0.5 * (ConfigKPP.bbMin(i) + ConfigKPP.bbMax(i));
+            X(i) = 2.0 * (x(i) - center) / (ConfigKPP.bbMax(i) - ConfigKPP.bbMin(i));
+            break;
+         }
+         case 2: // Map to the reference domain [0,1]^d.
+         {
+            X(i) = (x(i) - ConfigKPP.bbMin(i)) / (ConfigKPP.bbMax(i) - ConfigKPP.bbMin(i));
+            break;
+         }
+      }
+   }
+
+   switch (ConfigKPP.ConfigNum)
+   {
+      case 1:
+      {
+         // Map to test case specific domain [-2,2] x [-2.5,1.5].
+         X *= 2.0;
+         X(1) -= 0.5;
+
+         u(0) = X.Norml2() <= 1. ? 3.5 * M_PI : 0.25 * M_PI;
+         break;
+      }
+      case 2:
+      {
+         u(0) = X.Norml2() <= 0.25 ? 0.0 :
+                1.0; // According to the original KPP paper, not Ern and Guermond.
+         break;
+      }
+   }
+}
+
+void InflowFunctionKPP(const Vector &x, double t, Vector &u)
+{
+   switch (ConfigKPP.ConfigNum)
+   {
+      case 1: { u(0) = 0.25 * M_PI;  break; }
+
+      // This definition is consistent with the problem and assures correct
+      // handling of inflow (left) and outflow (right) boundaries.
+      case 2: { u(0) = x(0); break; }
+   }
+}

miniapps/hypsys/apps/kpp.hpp

@@ -0,0 +1,19 @@
+#ifndef HYPSYS_KPP
+#define HYPSYS_KPP
+
+#include "hyperbolic_system.hpp"
+
+class KPP : public HyperbolicSystem
+{
+public:
+   explicit KPP(FiniteElementSpace *fes_, BlockVector &u_block,
+                Configuration &config_);
+   ~KPP() { };
+
+   virtual void EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i = -1) const;
+   virtual double GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i) const;
+};
+
+#endif

miniapps/hypsys/apps/lib.hpp

@@ -0,0 +1,11 @@
+#ifndef HYPSYS_APPS_LIB
+#define HYPSYS_APPS_LIB
+
+#include "advection.hpp"
+#include "burgers.hpp"
+#include "kpp.hpp"
+#include "buckley_leverett.hpp"
+#include "shallowwater.hpp"
+#include "euler.hpp"
+
+#endif

miniapps/hypsys/apps/shallowwater.cpp

@@ -0,0 +1,463 @@
+#include "shallowwater.hpp"
+
+Configuration ConfigSWE;
+
+double GravConst;
+double Depth;
+
+void AnalyticalSolutionSWE(const Vector &x, double t, Vector &u);
+void InitialConditionSWE(const Vector &x, Vector &u);
+void InflowFunctionSWE(const Vector &x, double t, Vector &u);
+
+ShallowWater::ShallowWater(FiniteElementSpace *fes_, BlockVector &u_block,
+                           Configuration &config_)
+   : HyperbolicSystem(fes_, u_block, fes_->GetMesh()->Dimension() + 1, config_,
+                      VectorFunctionCoefficient(fes_->GetMesh()->Dimension() + 1,
+                                                InflowFunctionSWE))
+{
+   ConfigSWE = config_;
+   VectorFunctionCoefficient ic(NumEq, InitialConditionSWE);
+
+   switch (ConfigSWE.ConfigNum)
+   {
+      case 0:
+      {
+         // Periodic meshes must be used for this problem.
+         ProblemName = "Shallow Water Equations - Vorticity Advection";
+         glvis_scale = "on";
+         GravConst = 1.0;
+         Depth = 1.0;
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 0;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 1:
+      {
+         ProblemName = "Shallow Water Equations - Dam Break";
+         glvis_scale = "on";
+         GravConst = 9.81;
+         Depth = 1.0;
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 2:
+      {
+         ProblemName = "Shallow Water Equations - Radial Dam Break";
+         glvis_scale = "off valuerange 0.1 1";
+         GravConst = 9.81;
+         Depth = 0.1;
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 3:
+      {
+         ProblemName = "Shallow Water Equations - Constricted Channel";
+         glvis_scale = "on";
+         GravConst = 0.16;
+         Depth = 1.0;
+         SolutionKnown = false;
+         SteadyState = true;
+         TimeDepBC = false;
+         ProjType = 0;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 4:
+      {
+         ProblemName = "Shallow Water Equations - MoST Gimmick";
+         glvis_scale = "on";
+         GravConst = 1.0;
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+
+         Mesh *mesh = fes->GetMesh();
+         const int nd = fes->GetFE(0)->GetDof();
+         const int ne = fes->GetNE();
+         if (mesh->Dimension() != 2) { MFEM_ABORT("Test case works only in 2D."); }
+         u0 = 0.;
+
+         for (int e = 0; e < ne; e++)
+         {
+            int id = mesh->GetElement(e)->GetAttribute();
+            for (int j = 0; j < nd; j++)
+            {
+               switch (id)
+               {
+                  case 1:
+                  {
+                     u0(e*nd+j) = 1.;
+                     break;
+                  }
+                  case 2:
+                  case 3:
+                  case 4:
+                  {
+                     u0(e*nd+j) = 0.125;
+                     break;
+                  }
+                  default:
+                     MFEM_ABORT("Too many element IDs.");
+               }
+            }
+         }
+
+         break;
+      }
+      default:
+         MFEM_ABORT("No such test case implemented.");
+   }
+}
+
+void ShallowWater::EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                                int e, int k, int i) const
+{
+   CheckAdmissibility(u);
+   switch (dim)
+   {
+      case 1:
+      {
+         FluxEval(0,0) = u(1);
+         FluxEval(1,0) = u(1) * u(1) / u(0) + 0.5 * GravConst * u(0) * u(0);
+         break;
+      }
+      case 2:
+      {
+         double vx = u(1) / u(0);
+         double vy = u(2) / u(0);
+         double gravitation = 0.5 * GravConst * u(0) * u(0);
+
+         FluxEval(0,0) = u(1);
+         FluxEval(0,1) = u(2);
+
+         FluxEval(1,0) = u(1) * vx + gravitation;
+         FluxEval(1,1) = u(1) * vy;
+
+         FluxEval(2,0) = u(2) * vx;
+         FluxEval(2,1) = u(2) * vy + gravitation;
+         break;
+      }
+      default:
+         MFEM_ABORT("Invalid space dimension.");
+   }
+}
+
+double ShallowWater::GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                                  int i) const
+{
+   CheckAdmissibility(u);
+   switch (u.Size())
+   {
+      case 2:
+         return abs( u(1)*n(0) / u(0) ) + sqrt(GravConst * u(0));
+      case 3:
+         return abs( (u(1)*n(0) + u(2)*n(1)) / u(0) ) + sqrt(GravConst * u(0));
+   }
+}
+
+void ShallowWater::CheckAdmissibility(const Vector &u) const
+{
+   double HMin = 1.e-12;
+
+   if (u.Size() != NumEq) { MFEM_ABORT("Invalid solution vector."); }
+
+   if (u(0) < HMin)
+   {
+      ostringstream height_str;
+      height_str << u(0);
+      string err_msg = "Water height too small H = ";
+      MFEM_ABORT(err_msg << height_str.str() << ".");
+   }
+}
+
+void ShallowWater::SetBdrCond(const Vector &y1, Vector &y2,
+                              const Vector &normal, int attr) const
+{
+   switch (attr)
+   {
+      case -1: // Land boundary
+      {
+         if (normal.Size() == 1)
+         {
+            y2(0) = y1(0);
+            y2(1) = -y1(1);
+         }
+         else
+         {
+            double MomTimesNor = y1(1) * normal(0) + y1(2) * normal(1);
+            y2(0) = y1(0);
+            y2(1) = y1(1) - 2. * MomTimesNor * normal(0);
+            y2(2) = y1(2) - 2. * MomTimesNor * normal(1);
+         }
+         break;
+      }
+      case -2: // Radiation boundary
+      {
+         y2 = y1;
+         break;
+      }
+      case -3: // River boundary
+      {
+         break;
+      }
+      case -4: // Open sea boundary
+      {
+         double tmp = y2(0);
+         y2 = y1;
+         y2(0) = tmp;
+         break;
+      }
+      default:
+         MFEM_ABORT("Invalid boundary attribute.");
+   }
+}
+
+void ShallowWater::ComputeDerivedQuantities(const GridFunction &u,
+                                            GridFunction &d1, GridFunction &d2) const
+{
+   double height, momentum;
+   const IntegrationRule ir = u.FESpace()->GetFE(0)->GetNodes();
+
+   for (int e = 0; e < ne; e++)
+   {
+      for (int i = 0; i < nd; i++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(i);
+         height = u.GetValue(e, ip, 1);
+         momentum = u.GetValue(e, ip, 2);
+         d1(e*nd + i) = pow(momentum / height, 2.0);
+
+         if (dim==2)
+         {
+            momentum = u.GetValue(e, ip, 3);
+            d1(e*nd + i) += pow(momentum / height, 2.0);
+         }
+         d1(e*nd + i) = sqrt(d1(e*nd + i));
+      }
+   }
+}
+
+void ShallowWater::ComputeErrors(Array<double> &errors, const GridFunction &u,
+                                 double DomainSize, double t) const
+{
+   errors.SetSize(NumEq*3);
+   Vector component(dim+1);
+   VectorFunctionCoefficient uAnalytic(NumEq, AnalyticalSolutionSWE);
+
+   if (ConfigSWE.ConfigNum == 0) { uAnalytic.SetTime(0); }
+   else { uAnalytic.SetTime(t); }
+
+   component = 0.0;
+   component(0) = 1.0;
+   VectorConstantCoefficient weight1(component);
+   errors[0] = u.ComputeLpError(1.0, uAnalytic, NULL, &weight1) / DomainSize;
+   errors[1] = u.ComputeLpError(2.0, uAnalytic, NULL, &weight1) / DomainSize;
+   errors[2] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic,
+                                NULL, &weight1);
+
+   component = 0.0;
+   component(1) = 1.0;
+   VectorConstantCoefficient weight2(component);
+   errors[3] = u.ComputeLpError(1.0, uAnalytic, NULL, &weight2) / DomainSize;
+   errors[4] = u.ComputeLpError(2.0, uAnalytic, NULL, &weight2) / DomainSize;
+   errors[5] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic,
+                                NULL, &weight2);
+
+   if (dim ==  2)
+   {
+      component = 0.0;
+      component(2) = 1.0;
+      VectorConstantCoefficient weight3(component);
+      errors[6] = u.ComputeLpError(1.0, uAnalytic, NULL, &weight3) / DomainSize;
+      errors[7] = u.ComputeLpError(2.0, uAnalytic, NULL, &weight3) / DomainSize;
+      errors[8] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic,
+                                   NULL, &weight3);
+   }
+}
+
+
+void AnalyticalSolutionSWE(const Vector &x, double t, Vector &u)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+
+   for (int i = 0; i < dim; i++)
+   {
+      switch (ConfigSWE.ConfigNum)
+      {
+         case 0: // Map to the reference domain [-1,1]^d.
+         {
+            double center = 0.5 * (ConfigSWE.bbMin(i) + ConfigSWE.bbMax(i));
+            double factor = 2.0 / (ConfigSWE.bbMax(i) - ConfigSWE.bbMin(i));
+            X(i) = factor * (x(i) - center);
+            t *= pow(factor, 1.0 / (double(dim)));
+            break;
+         }
+         case 1: // Map to the reference domain [0,1]^d.
+         {
+            double factor = 1.0 / (ConfigSWE.bbMax(i) - ConfigSWE.bbMin(i));
+            X(i) = factor * (x(i) - ConfigSWE.bbMin(i));
+            t *= pow(factor, 1.0 / (double(dim)));
+            break;
+         }
+      }
+   }
+
+   switch (ConfigSWE.ConfigNum)
+   {
+      case 0:
+      {
+         if (dim != 2) { MFEM_ABORT("Test case works only in 2D."); }
+
+         // Map to test case specific domain [-50,50].
+         X *= 50.;
+         t *= 50.;
+
+         double M = sqrt(2);
+         double c1 = -0.1;
+         double c2 = 0.005;
+         double a = M_PI / 4.0;
+         double x0 = 0.0;
+         double y0 = 0.0;
+
+         double f = -c2 * ( pow(X(0) - x0 - M*t*cos(a), 2.0)
+                            + pow(X(1) - y0 - M*t*sin(a), 2.0) );
+
+         u(0) = 1.0;
+         u(1) = M*cos(a) + c1 * (X(1) - y0 - M*t*sin(a)) * exp(f);
+         u(2) = M*sin(a) - c1 * (X(0) - x0 - M*t*cos(a)) * exp(f);
+         u *= Depth - c1*c1 / (4.0*c2*GravConst) * exp(2.0*f);
+
+         break;
+      }
+      case 1:
+      {
+         // Map to test case specific domain [0,1000]^d.
+         X *= 1000;
+         t *= 1000;
+
+         double r =  X(0);
+         u = 0.;
+
+         if (t==0)
+         {
+            u(0) = r < 500.0 ? Depth + 9.0 : Depth;
+            return;
+         }
+
+         double cm = 6.23416;
+         double aux = sqrt(10.0 * GravConst);
+         double xA = 500.0 - t*aux;
+         double xB = 500.0 + t*(2.0*aux - 3.0*cm);
+         double xC = 500.0 + t*(2.0*cm*cm*(aux - cm))/(cm*cm - GravConst);
+
+         u(0) = 9.0 * (r<xA) + (4.0/(9.0*GravConst) * pow( aux - (r-500.0)/(2.0*t),
+                                                           2.0 ) - 1.0) * (r>=xA) * (r<xB)
+                + (cm*cm/GravConst - 1.) * (r>=xB) * (r<xC);
+         u(1) = 2.0/3.0 * ((r-500.0)/t + aux) * (r >= xA) * (r < xB) + 2.0 *
+                (aux - cm) * (r >= xB) * (r < xC);
+         u(0) += Depth;
+         u(1) *= u(0);
+
+         break;
+      }
+      case 3:
+      {
+         if (dim != 2) { MFEM_ABORT("Test case works only in 2D."); }
+
+         const double x1[2]= {-10., 0.}, x2[2]= {-10., 40.},
+                                                x3[2]= {53.8622, 5.5872}, x4[2]= {53.8622, 34.4128},
+                                                                                 slope0=0.53886, slope1=0.79893;
+         int sign_top, sign_bot;
+
+         if (x(0)>x3[0])
+         {
+            sign_top = -(x(0)-x4[0])*slope1-(x(1)-x4[1])>0 ? 1 : -1;
+            sign_bot = (x(0)-x3[0])*slope1-(x(1)-x3[1])>0 ? 1 : -1;
+            u(0) = sign_top*sign_bot>0. ? 0.8350436: 0.5273361;
+         }
+         else
+         {
+            sign_top = -(x(0)-x2[0])*slope0-(x(1)-x2[1])>0 ? 1 : -1;
+            sign_bot = (x(0)-x1[0])*slope0-(x(1)-x1[1])>0 ? 1 : -1;
+            u(0) = sign_top*sign_bot>0. ? 0.250133 : (sign_top>0 ? 1. : 0.5273361);
+         }
+
+         u(0) += Depth;
+         break;
+      }
+   }
+}
+
+void InitialConditionSWE(const Vector &x, Vector &u)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+
+   // Map to the reference domain [-1,1]^d.
+   for (int i = 0; i < dim; i++)
+   {
+      double center = 0.5 * (ConfigSWE.bbMin(i) + ConfigSWE.bbMax(i));
+      double factor = 2.0 / (ConfigSWE.bbMax(i) - ConfigSWE.bbMin(i));
+      X(i) = factor * (x(i) - center);
+   }
+
+   switch (ConfigSWE.ConfigNum)
+   {
+      case 0:
+      case 1:
+      {
+         AnalyticalSolutionSWE(x, 0., u);
+         break;
+      }
+      case 2:
+      {
+         u = 0.0;
+         u(0) = X.Norml2() < 0.5 ? 0.9 : 0.0;
+         u(0) += Depth;
+         break;
+      }
+      case 3:
+      {
+         u(0) = Depth;
+         u(1) = Depth;
+         u(2) = 0.;
+         break;
+      }
+   }
+}
+
+void InflowFunctionSWE(const Vector &x, double t, Vector &u)
+{
+   switch (ConfigSWE.ConfigNum)
+   {
+      case 0:
+      case 2:
+      {
+         // Do not impose inflow values in this setup.
+         break;
+      }
+      case 1:
+      {
+         AnalyticalSolutionSWE(x, 0., u);
+         break;
+      }
+      case 3:
+      {
+         InitialConditionSWE(x, u);
+         break;
+      }
+   }
+}

miniapps/hypsys/apps/shallowwater.hpp

@@ -0,0 +1,26 @@
+#ifndef HYPSYS_SHALLOWWATER
+#define HYPSYS_SHALLOWWATER
+
+#include "hyperbolic_system.hpp"
+
+class ShallowWater : public HyperbolicSystem
+{
+public:
+   explicit ShallowWater(FiniteElementSpace *fes_, BlockVector &u_block,
+                         Configuration &config_);
+   ~ShallowWater() { };
+
+   virtual void EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i = -1) const;
+   virtual double GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i) const;
+   virtual void CheckAdmissibility(const Vector &u) const override;
+   virtual void SetBdrCond(const Vector &y1, Vector &y2, const Vector &normal,
+                           int attr) const override;
+   virtual void ComputeDerivedQuantities(const GridFunction &u, GridFunction &d1,
+                                         GridFunction &d2) const override;
+   virtual void ComputeErrors(Array<double> &errors, const GridFunction &u,
+                              double DomainSize, double t) const override;
+};
+
+#endif

miniapps/hypsys/apps/template.cpp

@@ -0,0 +1,115 @@
+#include "template.hpp"
+
+Configuration ConfigTEMPLATE;
+
+void AnalyticalSolutionTEMPLATE(const Vector &x, double t, Vector &u);
+void InitialConditionTEMPLATE(const Vector &x, Vector &u);
+void InflowFunctionTEMPLATE(const Vector &x, double t, Vector &u);
+
+TEMPLATE::TEMPLATE(FiniteElementSpace *fes_, BlockVector &u_block,
+                   Configuration &config_)
+   : HyperbolicSystem(fes_, u_block, NUMEQ, config_,
+                      VectorFunctionCoefficient(NUMEQ, InflowFunctionTEMPLATE))
+{
+   ConfigTEMPLATE = config_;
+   VectorFunctionCoefficient ic(NumEq, InitialConditionTEMPLATE);
+
+   switch (ConfigTEMPLATE.ConfigNum)
+   {
+      case 0:
+      {
+         ProblemName = "TEMPLATE - ";
+         glvis_scale = "on";
+         SolutionKnown = ;
+         SteadyState = ;
+         TimeDepBC = ;
+         ProjType = 0;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 1:
+      {
+         ProblemName = "TEMPLATE - ";
+         glvis_scale = "off valuerange 0 1";
+         SolutionKnown = ;
+         SteadyState = ;
+         TimeDepBC = ;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      default:
+         MFEM_ABORT("No such test case implemented.");
+   }
+}
+
+void TEMPLATE::EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                            int e, int k, int i) const
+{
+   // TODO
+}
+
+double TEMPLATE::GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                              int i) const
+{
+   //TODO
+   return 0.;
+}
+
+void TEMPLATE::SetBdrCond(const Vector &y1, Vector &y2, const Vector &normal,
+                          int attr) const
+{
+   //TODO
+}
+
+void TEMPLATE::ComputeErrors(Array<double> &errors, const GridFunction &u,
+                             double DomainSize, double t) const
+{
+   errors.SetSize(3);
+   VectorFunctionCoefficient uAnalytic(NumEq, AnalyticalSolutionTEMPLATE);
+   uAnalytic.SetTime(t);
+   errors[0] = u.ComputeLpError(1., uAnalytic) / DomainSize;
+   errors[1] = u.ComputeLpError(2., uAnalytic) / DomainSize;
+   errors[2] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic);
+}
+
+
+void AnalyticalSolutionTEMPLATE(const Vector &x, double t, Vector &u)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+
+   for (int i = 0; i < dim; i++)
+   {
+      switch (ConfigTEMPLATE.ConfigNum)
+      {
+         case /* TODO */: // Map to the reference domain [-1,1]^d.
+         {
+            double center = 0.5 * (ConfigTEMPLATE.bbMin(i) + ConfigTEMPLATE.bbMax(i));
+            double factor = 2.0 / (ConfigTEMPLATE.bbMax(i) - ConfigTEMPLATE.bbMin(i));
+            X(i) = factor * (x(i) - center);
+            t *= pow(factor, 1.0 / (double(dim)));
+            break;
+         }
+         case /* TODO */: // Map to the reference domain [0,1]^d.
+         {
+            double factor = 1.0 / (ConfigTEMPLATE.bbMax(i) - ConfigTEMPLATE.bbMin(i));
+            X(i) = factor * (x(i) - ConfigTEMPLATE.bbMin(i));
+            t *= pow(factor, 1.0 / (double(dim)));
+            break;
+         }
+      }
+   }
+
+   // TODO
+}
+
+void InitialConditionTEMPLATE(const Vector &x, Vector &u)
+{
+   // TODO
+}
+
+void InflowFunctionTEMPLATE(const Vector &x, double t, Vector &u)
+{
+   // TODO
+}

miniapps/hypsys/apps/template.hpp

@@ -0,0 +1,24 @@
+#ifndef HYPSYS_TEMPLATE
+#define HYPSYS_TEMPLATE
+
+#include "hyperbolic_system.hpp"
+
+class TEMPLATE : public HyperbolicSystem
+{
+public:
+   explicit TEMPLATE(FiniteElementSpace *fes_, BlockVector &u_block,
+                     Configuration &config_);
+   ~TEMPLATE() { };
+
+   virtual void EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i = -1) const;
+   virtual double GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i) const;
+   virtual void SetBdrCond(const Vector &y1, Vector &y2, const Vector &normal,
+                           int attr) const;
+   virtual void ComputeDerivedQuantities(const Vector &u) const { };
+   virtual void ComputeErrors(Array<double> &errors, const GridFunction &u,
+                              double DomainSize, double t) const;
+};
+
+#endif

miniapps/hypsys/config/advection.sh

@@ -0,0 +1,32 @@
+EXEC="mpirun -np 7 phypsys"
+
+SCHEME=1
+MESH="data/periodic-4segment.mesh"
+MESH0="data/periodic-3segment.mesh"
+CONFIG="-p 0 -c 4 -vf 100 -tf 1 -s 3 -dt 0.001 -m $MESH"
+
+$EXEC $CONFIG -es $SCHEME -o 1 -r 5
+$EXEC $CONFIG -es $SCHEME -o 3 -r 4
+$EXEC $CONFIG -es $SCHEME -o 7 -r 3
+
+
+# GRID CONVERGENCE TEST
+ORDER=1
+DT=0.0004
+# ORDER=2
+# DT=0.00025
+# ORDER=3
+# DT=0.0001
+# ORDER=4
+# DT=0.000025
+CONFIG0="-p 0 -c 3 -vf 1000 -tf 1 -s 3 -dt $DT -m $MESH0 -o $ORDER"
+CONFIG1="-p 0 -c 3 -vf 1000 -tf 1 -s 3 -dt $DT -m $MESH -o $ORDER"
+
+# rm errors.txt
+# $EXEC $CONFIG0 -r 4 -es $SCHEME
+# $EXEC $CONFIG1 -r 4 -es $SCHEME
+# $EXEC $CONFIG0 -r 5 -es $SCHEME
+# $EXEC $CONFIG1 -r 5 -es $SCHEME
+# $EXEC $CONFIG0 -r 6 -es $SCHEME
+# $EXEC $CONFIG1 -r 6 -es $SCHEME
+# $EXEC $CONFIG0 -r 7 -es $SCHEME

miniapps/hypsys/config/burgers.sh

@@ -0,0 +1,43 @@
+EXEC="mpirun -np 7 phypsys"
+
+SCHEME=0
+MESH=data/inline-4quad.mesh
+CONFIG="-p 1 -c 1 -vf 100 -tf 0.5 -s 3 -dt 0.0005 -m $MESH"
+
+$EXEC $CONFIG -es $SCHEME -o 0 -r 5
+$EXEC $CONFIG -es $SCHEME -o 1 -r 4
+
+SCHEME=1
+
+$EXEC $CONFIG -es $SCHEME -o  1 -r 4
+$EXEC $CONFIG -es $SCHEME -o  3 -r 3
+$EXEC $CONFIG -es $SCHEME -o  7 -r 2
+$EXEC $CONFIG -es $SCHEME -o 15 -r 1
+
+
+# # GRID CONVERGENCE TEST
+# SCHEME=1
+ORDER=0
+DT=0.002
+# ORDER=1
+# DT=0.0004
+# # ORDER=2
+# # DT=0.00016
+# # ORDER=3
+# # DT=-dt 0.0001
+# # ORDER=4
+# # DT=0.00005
+# MESH="data/periodic-4segment.mesh"
+# MESH0="data/periodic-3segment.mesh"
+# CONFIG0="-p 1 -c 0 -vf 1000 -tf 0.1 -s 3 -dt $DT -m $MESH0 -o $ORDER"
+# CONFIG1="-p 1 -c 0 -vf 1000 -tf 0.1 -s 3 -dt $DT -m $MESH -o $ORDER"
+
+# rm errors.txt
+# $EXEC $CONFIG0 -r 4 -es $SCHEME
+# $EXEC $CONFIG1 -r 4 -es $SCHEME
+# $EXEC $CONFIG0 -r 5 -es $SCHEME
+# $EXEC $CONFIG1 -r 5 -es $SCHEME
+# $EXEC $CONFIG0 -r 6 -es $SCHEME
+# $EXEC $CONFIG1 -r 6 -es $SCHEME
+# $EXEC $CONFIG0 -r 7 -es $SCHEME
+# $EXEC $CONFIG1 -r 7 -es $SCHEME

miniapps/hypsys/config/debug.sh

@@ -0,0 +1,7 @@
+MESH=data/wall-bdr-100segment.mesh
+CONFIG="-p 3 -c 1 -vf 1000 -tf 0.02 -s 1 -m $MESH -r 0 -es 1"
+
+./hypsys $CONFIG -o 3 -dt 0.00005
+mv ultimate.gf cmp.gf
+mpirun -np 3 phypsys $CONFIG -o 3 -dt 0.00005
+meld ultimate.gf cmp.gf

miniapps/hypsys/config/euler.sh

@@ -0,0 +1,55 @@
+EXEC="mpirun -np 7 phypsys"
+
+SCHEME=1
+MESH=data/wall-bdr-4segment.mesh
+
+## SOD Shock tube
+CONFIG="-p 5 -c 1 -vf 1000 -tf 0.231 -s 3 -m $MESH -es $SCHEME"
+
+# # h-refinement
+# $EXEC $CONFIG -o 1 -r 5 -dt 0.00064
+# $EXEC $CONFIG -o 1 -r 6 -dt 0.00032
+# $EXEC $CONFIG -o 1 -r 7 -dt 0.00016
+
+# # p-refinement & h-coarsening
+# $EXEC $CONFIG -r 5 -o 1 -dt 0.0004
+# $EXEC $CONFIG -r 4 -o 3 -dt 0.0004
+# $EXEC $CONFIG -r 3 -o 7 -dt 0.0004
+# $EXEC $CONFIG -r 2 -o 15 -dt 0.0004
+# $EXEC $CONFIG -r 1 -o 31 -dt 0.0004
+
+
+## Woodward Colella
+MESH=data/wall-bdr-100segment.mesh
+CONFIG="-p 5 -c 2 -vf 1000 -tf 0.038 -s 3 -m $MESH -es $SCHEME"
+$EXEC $CONFIG -o 1 -r 2 -dt 1e-6
+
+
+# ## Double Mach reflection
+# MESH=data/double-mach-quad.mesh
+# CONFIG="-p 5 -c 3 -vf 1000 -tf 0.2 -s 3 -m $MESH -es $SCHEME"
+# $EXEC $CONFIG -o 1 -r 3 -dt 5e-5
+
+
+# ## Vortex advection
+# SCHEME=0
+# MESH3=data/periodic-3quad.mesh
+# MESH4=data/periodic-4quad.mesh
+
+# ORDER=1
+# ODESOLVER=2
+# DT=0.000625
+# # ORDER=2
+# # ODESOLVER=3
+# # DT=0.0002
+# # ORDER=3
+# # ODESOLVER=3
+# # DT=0.00032
+# CONFIG="-p 5 -c 0 -vf 100 -tf 1 -s $ODESOLVER -dt $DT -o $ORDER -es $SCHEME"
+
+# rm errors.txt
+# $EXEC $CONFIG -m $MESH3 -r 3
+# $EXEC $CONFIG -m $MESH4 -r 3
+# $EXEC $CONFIG -m $MESH3 -r 4
+# $EXEC $CONFIG -m $MESH4 -r 4
+# $EXEC $CONFIG -m $MESH3 -r 5

miniapps/hypsys/config/openMPI-test.sh

@@ -0,0 +1,47 @@
+SCHEME=0
+
+## Advection
+
+# Solid Body Roatation
+# ./hypsys -r 0 -es $SCHEME
+# ./phypsys -r 0 -es $SCHEME
+# mpirun -np 4 ./phypsys -r 0 -es $SCHEME
+
+# Steady Circular Convection
+# ./hypsys -vf 1000 -m data/inline-4quad.mesh -dt 0.0001 -o 2 -s 1 -r 3 -c 0 -es $SCHEME
+# ./phypsys -vf 1000 -m data/inline-4quad.mesh -dt 0.0001 -o 2 -s 1 -r 3 -c 0 -es $SCHEME
+# mpirun -np 4 ./phypsys -vf 1000 -m data/inline-4quad.mesh -dt 0.0001 -o 2 -s 1 -r 3 -c 0 -es $SCHEME
+
+# Translation
+CONFIG="-p 0 -c 2 -vf 100 -tf 0.4 -s 3 -dt 0.002 -m data/periodic-3tri.mesh -o 2 -r 3 -es $SCHEME"
+./hypsys $CONFIG
+./phypsys $CONFIG
+mpirun -np 4 phypsys $CONFIG
+
+## Burgers
+CONFIG="-p 1 -c 1 -vf 100 -tf 0.5 -s 3 -dt 0.004 -m data/inline-3quad.mesh -o 1 -r 3 -es $SCHEME"
+./hypsys $CONFIG
+./phypsys $CONFIG
+mpirun -np 7 phypsys $CONFIG
+
+## KPP
+CONFIG="-p 2 -c 1 -vf 50 -tf 0.25 -s 3 -dt 0.005 -m data/inline-4tri.mesh -o 0 -r 4 -es 0"
+./hypsys $CONFIG
+./phypsys $CONFIG
+mpirun -np 1 phypsys $CONFIG
+
+## Shallow-Water
+
+# Dam break
+CONFIG="-p 4 -c 2 -vf 20 -tf 0.1 -s 3 -dt 0.0005 -m data/wall-bdr-4tri.mesh -o 1 -r 3 -es $SCHEME"
+./hypsys $CONFIG
+./phypsys $CONFIG
+mpirun -np 2 phypsys $CONFIG
+
+## Euler
+
+# Smooth vortex
+CONFIG="-p 5 -c 0 -vf 100 -tf 1 -s 3 -dt 0.00125 -m data/periodic-3quad.mesh -r 2 -es $SCHEME"
+./hypsys $CONFIG
+./phypsys $CONFIG
+mpirun -np 4 phypsys $CONFIG

miniapps/hypsys/config/rotating-wave-problems.sh

@@ -0,0 +1,12 @@
+EXEC="mpirun -np 7 phypsys"
+MESH=data/inline-4quad.mesh
+
+# KPP
+# $EXEC -p 2 -c 1 -vf 100 -tf 0.25 -s 3 -dt 0.00025 -r 9 -o 0 -m $MESH -es 0
+$EXEC -p 2 -c 1 -vf 100 -tf 0.25 -s 3 -dt 0.002 -r 6 -o 0 -m $MESH -es 0
+$EXEC -p 2 -c 1 -vf 100 -tf 0.25 -s 3 -dt 0.0008 -r 5 -o 1 -m $MESH -es 0
+$EXEC -p 2 -c 1 -vf 100 -tf 0.25 -s 3 -dt 0.0008 -r 5 -o 1 -m $MESH -es 1
+
+# Buckley-Leverett
+$EXEC -p 3 -c 2 -vf 100 -tf 0.1666667 -s 3 -dt 0.0005 -r 5 -o 0 -m $MESH -es 0
+$EXEC -p 3 -c 2 -vf 100 -tf 0.1666667 -s 3 -dt 0.0005 -r 4 -o 1 -m $MESH -es 1

miniapps/hypsys/config/shallow-water.sh

@@ -0,0 +1,54 @@
+EXEC="mpirun -np 7 phypsys"
+
+## 1D Dam break
+SCHEME=0
+MESH=data/wall-bdr-4segment.mesh
+CONFIG="-p 4 -c 1 -vf 1000 -tf 0.02 -s 3 -m $MESH -es $SCHEME"
+
+# # h-refinement
+# $EXEC $CONFIG -o 1 -r 5 -dt 0.0001
+# $EXEC $CONFIG -o 1 -r 6 -dt 0.00005
+# $EXEC $CONFIG -o 1 -r 7 -dt 0.000025
+# $EXEC $CONFIG -o 1 -r 8 -dt 0.0000125
+
+# # p-refinement & h-coarsening
+# $EXEC $CONFIG -o 1 -r 5 -dt 0.000025
+# $EXEC $CONFIG -o 3 -r 4 -dt 0.000025
+# $EXEC $CONFIG -o 7 -r 3 -dt 0.000025
+
+# ## Radial dambreak
+# MESH=data/outflow-bdr-4quad.mesh
+# CONFIG="-p 4 -c 2 -vf 50 -tf 0.06 -s 3 -m $MESH -es 1"
+
+# # p-refinement & h-coarsening
+# $EXEC $CONFIG -o 1 -r 5 -dt 0.0001
+# $EXEC $CONFIG -o 3 -r 4 -dt 0.0001
+# $EXEC $CONFIG -o 7 -r 3 -dt 0.0001
+
+## Constricted channel
+MESH=data/constricted-channel.mesh
+CONFIG="-p 4 -c 3 -vf 100 -tf 1000 -s 1 -m $MESH -es $SCHEME"
+$EXEC $CONFIG -r 1 -o 1 -dt 0.025
+
+# ## Vortex advection
+# SCHEME=0
+# MESH3=data/periodic-3quad.mesh
+# MESH4=data/periodic-4quad.mesh
+
+# ORDER=1
+# ODESOLVER=2
+# DT=0.00064
+# # ORDER=2
+# # ODESOLVER=3
+# # DT=0.0004
+# # ORDER=3
+# # ODESOLVER=3
+# # DT=0.00025
+# CONFIG="-p 4 -c 0 -vf 100 -tf 1 -s $ODESOLVER -dt $DT -o $ORDER -es $SCHEME"
+
+# rm errors.txt
+# $EXEC $CONFIG -m $MESH3 -r 3
+# $EXEC $CONFIG -m $MESH4 -r 3
+# $EXEC $CONFIG -m $MESH3 -r 4
+# $EXEC $CONFIG -m $MESH4 -r 4
+# $EXEC $CONFIG -m $MESH3 -r 5

miniapps/hypsys/config/solid-body-rotation.sh

@@ -0,0 +1,6 @@
+EXEC="mpirun -np 7 phypsys"
+
+MESH=data/periodic-4quad.mesh
+SCHEME=1
+
+$EXEC -p 0 -c 1 -vf 200 -tf 1 -s 3 -dt 0.00032 -m $MESH -r 4 -o 2 -es $SCHEME

miniapps/hypsys/data/beam-hex.mesh

@@ -0,0 +1,103 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+#
+
+dimension
+3
+
+elements
+8
+1 5 0 1 10 9 18 19 28 27
+1 5 1 2 11 10 19 20 29 28
+1 5 2 3 12 11 20 21 30 29
+1 5 3 4 13 12 21 22 31 30
+2 5 4 5 14 13 22 23 32 31
+2 5 5 6 15 14 23 24 33 32
+2 5 6 7 16 15 24 25 34 33
+2 5 7 8 17 16 25 26 35 34
+
+boundary
+34
+3 3 9 10 1 0
+3 3 0 1 19 18
+3 3 10 9 27 28
+1 3 9 0 18 27
+3 3 18 19 28 27
+3 3 10 11 2 1
+3 3 1 2 20 19
+3 3 11 10 28 29
+3 3 19 20 29 28
+3 3 11 12 3 2
+3 3 2 3 21 20
+3 3 12 11 29 30
+3 3 20 21 30 29
+3 3 12 13 4 3
+3 3 3 4 22 21
+3 3 13 12 30 31
+3 3 21 22 31 30
+3 3 13 14 5 4
+3 3 4 5 23 22
+3 3 14 13 31 32
+3 3 22 23 32 31
+3 3 14 15 6 5
+3 3 5 6 24 23
+3 3 15 14 32 33
+3 3 23 24 33 32
+3 3 15 16 7 6
+3 3 6 7 25 24
+3 3 16 15 33 34
+3 3 24 25 34 33
+3 3 16 17 8 7
+3 3 7 8 26 25
+2 3 8 17 35 26
+3 3 17 16 34 35
+3 3 25 26 35 34
+
+vertices
+36
+3
+0 0 0
+1 0 0
+2 0 0
+3 0 0
+4 0 0
+5 0 0
+6 0 0
+7 0 0
+8 0 0
+0 1 0
+1 1 0
+2 1 0
+3 1 0
+4 1 0
+5 1 0
+6 1 0
+7 1 0
+8 1 0
+0 0 1
+1 0 1
+2 0 1
+3 0 1
+4 0 1
+5 0 1
+6 0 1
+7 0 1
+8 0 1
+0 1 1
+1 1 1
+2 1 1
+3 1 1
+4 1 1
+5 1 1
+6 1 1
+7 1 1
+8 1 1