minor

Support for SLEPc eigensolver [slepc-dev]

.gitignore

@@ -122,7 +122,7 @@ examples/sundials/ex16-final.*
 examples/sundials/Example16*
 
 examples/petsc/ex[1-69]p
-examples/petsc/ex10p
+examples/petsc/ex1[0-1]p
 
 examples/petsc/mesh.*
 examples/petsc/sol.*
@@ -137,6 +137,7 @@ examples/petsc/Example9*
 examples/petsc/deformed.*
 examples/petsc/velocity.*
 examples/petsc/elastic_energy.*
+examples/petsc/mode_*
 
 examples/pumi/ex1
 examples/pumi/ex[126]p
@@ -162,6 +163,20 @@ miniapps/electromagnetics/Tesla-AMR*
 miniapps/electromagnetics/Maxwell-Parallel*
 miniapps/electromagnetics/Joule_*
 
+miniapps/hypsys/build
+miniapps/hypsys/errors.txt
+miniapps/hypsys/grid*
+miniapps/hypsys/hypsys
+miniapps/hypsys/initial*
+miniapps/hypsys/output
+miniapps/hypsys/phypsys
+miniapps/hypsys/pressure*
+miniapps/hypsys/results
+miniapps/hypsys/scripts/gridfunc-scatter
+miniapps/hypsys/ultimate*
+miniapps/hypsys/velocity*
+miniapps/hypsys/various
+
 miniapps/meshing/mobius-strip
 miniapps/meshing/klein-bottle
 miniapps/meshing/toroid

CHANGELOG

@@ -95,6 +95,8 @@ Linear and nonlinear solvers
 - In SLISolver, changed the residual inner product from (Br,r) to (Br,Br) so the
   solver can work with non-SPD preconditioner B.
 
+- Added support for the SLEPc eigensolver package.
+
 New and updated examples and miniapps
 -------------------------------------
 - Added a new example, Example 25/25p, to demonstrate the use of a Perfectly
@@ -109,6 +111,13 @@ New and updated examples and miniapps
   for applying Dirichlet, Neumann (both homogeneous and inhomogeneous), Robin,
   and periodic boundary conditions with either H1 or DG discretizations.
 
+- Added a new miniapp, Navier, that solves the time-dependent Navier-Stokes
+  equations of incompressible fluid dynamics. See the miniapps/navier directory
+  for more details.
+
+- Ported Example 11p to SLEPc, to demonstrate solving the Laplace eigenvalue
+  equation with the shift-and-invert spectral transformation method.
+
 - Added a simple meshing miniapp, Twist, which demonstrates MFEM's strategy of
   stitching together opposite surfaces of a mesh to create a topologically
   periodic mesh.

CMakeLists.txt

@@ -149,9 +149,13 @@ if (MFEM_USE_MPI)
       message(FATAL_ERROR "PETSc version >= 3.8.0 is required")
     endif()
     set(PETSC_INCLUDE_DIRS ${PETSC_INCLUDES})
+    if (MFEM_USE_SLEPC)
+      find_package(SLEPc REQUIRED config)
+      message(STATUS "Found SLEPc version ${SLEPC_VERSION}")
+    endif()
   endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU PETSC STRUMPACK PUMI)
+  set(PKGS_NEED_MPI SUPERLU PETSC SLEPC STRUMPACK PUMI)
   foreach(PKG IN LISTS PKGS_NEED_MPI)
     if (MFEM_USE_${PKG})
       message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -352,7 +356,7 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
-    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+    SLEPC MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
     MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")

INSTALL

@@ -383,6 +383,10 @@ MFEM_USE_PETSC = YES/NO
    and other features based on the PETSc package. When enabled, this option uses
    the PETSC_* library options, see below.
 
+MFEM_USE_SLEPC = YES/NO
+   Enable MFEM eigensolvers based on the SLEPc package. When enabled, this
+   option uses the SLEPC_* library options, see below.
+
 MFEM_USE_MPFR = YES/NO
    MPFR is a library for multiple-precision floating-point computations. This
    option enables the use of MPFR in MFEM, e.g. for precise computation of 1D
@@ -597,6 +601,12 @@ The specific libraries and their options are:
   Options: PETSC_OPT, PETSC_LIB.
   Versions: PETSc >= 3.8.0.
 
+- SLEPc (optional), used when MFEM_USE_SLEPC = YES. SLEPc depends on PETSc and
+  uses some of the PETSc options when compiled.
+  URL: https://slepc.upv.es/
+  Options: SLEPC_OPT, SLEPC_LIB.
+  Versions: SLEPc >= 3.8.0.
+
 - Sidre (optional), part of LLNL's axom project, used when MFEM_USE_SIDRE = YES.
   Starting with MFEM v4.1, Axom version 0.3.1 or later is required.
   URL: https://github.com/LLNL/axom
@@ -649,12 +659,11 @@ The specific libraries and their options are:
   Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.
   Versions: OCCA >= 1.0.9.
 
-- libCEED (optional), used when MFEM_USE_CEED = YES. Requires libCEED v0.6
-  or later version, specifically, git-hash 3d05795 or later.
+- libCEED (optional), used when MFEM_USE_CEED = YES.
   URL: https://github.com/CEED/libCEED
        https://ceed.exascaleproject.org/libceed
   Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.6.
+  Versions: libCEED >= 0.6, git-hash a970f63.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
   Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.

config/XSDKDefaults.cmake

@@ -244,6 +244,10 @@ IF (DEFINED TPL_ENABLE_PETSC)
   SET(MFEM_USE_PETSC ${TPL_ENABLE_PETSC} CACHE BOOL "Enable PETSc support." FORCE)
 ENDIF()
 
+IF (DEFINED TPL_ENABLE_SLEPC)
+  SET(MFEM_USE_SLEPC ${TPL_ENABLE_SLEPC} CACHE BOOL "Enable SLEPc support." FORCE)
+ENDIF()
+
 IF (DEFINED TPL_ENABLE_MPFR)
   SET(MFEM_USE_MPFR ${TPL_ENABLE_MPFR} CACHE BOOL "Enable MPFR usage." FORCE)
 ENDIF()

config/cmake/MFEMConfig.cmake.in

@@ -38,6 +38,7 @@ set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)
 set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
+set(MFEM_USE_SLEPC @MFEM_USE_SLEPC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)

config/cmake/config.hpp.in

@@ -104,6 +104,9 @@
 // Enable MFEM functionality based on the PETSc library
 #cmakedefine MFEM_USE_PETSC
 
+// Enable MFEM functionality based on the SLEPc library
+#cmakedefine MFEM_USE_SLEPC
+
 // Enable MFEM functionality based on the Sidre library
 #cmakedefine MFEM_USE_SIDRE
 

config/cmake/modules/FindSLEPc.cmake

@@ -0,0 +1,44 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Sets the following variables:
+#   - SLEPC_FOUND
+#   - SLEPC_INCLUDE_DIRS
+#   - SLEPC_LIBRARIES
+
+set(SLEPc_REQUIRED_PACKAGES "PETSC" CACHE STRING
+  "Additional packages required by SLEPc")
+
+include(MfemCmakeUtilities)
+mfem_find_package(SLEPc SLEPC SLEPC_DIR
+  "include" "slepceps.h"
+  "${PETSC_ARCH}/lib" "slepc" # add NAMES_PER_DIR?
+  "Paths to headers required by SLEPc."
+  "Libraries required by SLEPc."
+  ADD_COMPONENT "config" "${PETSC_ARCH}/include" "slepcconf.h" "" ""
+  CHECK_BUILD SLEPC_VERSION_OK TRUE
+"
+#include \"petsc.h\"
+#include \"slepceps.h\"
+int main()
+{
+  PetscErrorCode ierr;
+  int argc = 0;
+  char** argv = NULL;
+  ierr = SlepcInitialize(&argc, &argv, PETSC_NULL, PETSC_NULL);
+  EPS eps;
+  ierr = EPSCreate(PETSC_COMM_SELF, &eps); CHKERRQ(ierr);
+  ierr = EPSDestroy(&eps); CHKERRQ(ierr);
+  ierr = SlepcFinalize(); CHKERRQ(ierr);
+  return 0;
+}
+"
+  )

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -731,7 +731,7 @@ function(mfem_export_mk_files)
       MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
       MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
       MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GNUTLS
-      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_MPFR MFEM_USE_SIDRE
+      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE
       MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA
       MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2)
   foreach(var ${CONFIG_MK_BOOL_VARS})

config/config.hpp

@@ -48,6 +48,9 @@
 #ifdef MFEM_USE_PETSC
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
+#ifdef MFEM_USE_SLEPC
+#error Building with SLEPc (MFEM_USE_SLEPC=YES) requires MPI (MFEM_USE_MPI=YES)
+#endif
 #ifdef MFEM_USE_PUMI
 #error Building with PUMI (MFEM_USE_PUMI=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif

config/config.hpp.in

@@ -118,6 +118,9 @@
 // Enable functionality based on the PETSc library
 // #define MFEM_USE_PETSC
 
+// Enable functionality based on the SLEPc library
+// #define MFEM_USE_SLEPC
+
 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR
 

config/config.mk.in

@@ -37,6 +37,7 @@ MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
 MFEM_USE_PETSC         = @MFEM_USE_PETSC@
+MFEM_USE_SLEPC         = @MFEM_USE_SLEPC@
 MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@

config/defaults.cmake

@@ -39,6 +39,7 @@ option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
 option(MFEM_USE_GSLIB "Enable GSLIB usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
 option(MFEM_USE_PETSC "Enable PETSc support." OFF)
+option(MFEM_USE_SLEPC "Enable SLEPc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
@@ -155,6 +156,10 @@ set(PETSC_DIR "${MFEM_DIR}/../petsc" CACHE PATH
     "Path to the PETSc main directory.")
 set(PETSC_ARCH "arch-linux2-c-debug" CACHE STRING "PETSc build architecture.")
 
+set(SLEPC_DIR "${MFEM_DIR}/../slepc" CACHE PATH
+    "Path to the SLEPc main directory.")
+set(SLEPC_ARCH "arch-linux2-c-debug" CACHE STRING "SLEPC build architecture.")
+
 set(MPFR_DIR "" CACHE PATH "Path to the MPFR library.")
 
 set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH

config/defaults.mk

@@ -125,6 +125,7 @@ MFEM_USE_GINKGO        = NO
 MFEM_USE_GNUTLS        = NO
 MFEM_USE_NETCDF        = NO
 MFEM_USE_PETSC         = NO
+MFEM_USE_SLEPC         = NO
 MFEM_USE_MPFR          = NO
 MFEM_USE_SIDRE         = NO
 MFEM_USE_CONDUIT       = NO
@@ -276,6 +277,20 @@ ifeq ($(PETSC_FOUND),YES)
       -L$(abspath $(PETSC_DIR))/lib -lpetsc $(PETSC_LIB)
 endif
 
+SLEPC_DIR := $(MFEM_DIR)/../slepc
+SLEPC_VARS := $(SLEPC_DIR)/lib/slepc/conf/slepc_variables
+SLEPC_FOUND := $(if $(wildcard $(SLEPC_VARS)),YES,)
+SLEPC_INC_VAR = SLEPC_INCLUDE
+SLEPC_LIB_VAR = SLEPC_EXTERNAL_LIB
+ifeq ($(SLEPC_FOUND),YES)
+   SLEPC_OPT := $(shell sed -n "s/$(SLEPC_INC_VAR) *= *//p" $(SLEPC_VARS))
+   # Some additional external libraries might be defined in this file
+   -include ${SLEPC_DIR}/${PETSC_ARCH}/lib/slepc/conf/slepcvariables
+   SLEPC_LIB := $(shell sed -n "s/$(SLEPC_LIB_VAR) *= *//p" $(SLEPC_VARS))
+   SLEPC_LIB := -Wl,-rpath,$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib\
+      -L$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib -lslepc $(SLEPC_LIB)
+endif
+
 # MPFR library configuration
 MPFR_OPT =
 MPFR_LIB = -lmpfr

examples/CMakeLists.txt

@@ -91,7 +91,7 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
     add_test(NAME ${TEST_NAME}_ser
       COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
   else()
-    add_test(NAME ${TEST_NAME}_np=4
+    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
       COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
       ${MPIEXEC_PREFLAGS}
       $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
@@ -101,13 +101,22 @@ endforeach()
 
 # If STRUMPACK is enabled, add a test run that uses it.
 if (MFEM_USE_STRUMPACK)
-  add_test(NAME ex11p_strumpack_np=4
+  add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
     COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
     ${MPIEXEC_PREFLAGS}
     $<TARGET_FILE:ex11p> "-no-vis" "--strumpack"
     ${MPIEXEC_POSTFLAGS})
 endif()
 
+# If SuperLU_DIST is enabled, add a test run that uses it.
+if (MFEM_USE_SUPERLU)
+  add_test(NAME ex11p_superlu_np=${MFEM_MPI_NP}
+    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+    ${MPIEXEC_PREFLAGS}
+    $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
+    ${MPIEXEC_POSTFLAGS})
+endif()
+
 # Include the examples/sundials directory if SUNDIALS is enabled.
 if (MFEM_USE_SUNDIALS)
   add_subdirectory(sundials)

examples/ex1.cpp

@@ -34,7 +34,8 @@
 //               ex1 -pa -d raja-omp
 //               ex1 -pa -d occa-omp
 //               ex1 -pa -d ceed-cpu
-//               ex1 -pa -d ceed-cuda
+//             * ex1 -pa -d ceed-cuda
+//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
 //               ex1 -m ../data/beam-hex.mesh -pa -d cuda
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref

examples/ex18.hpp

@@ -88,8 +88,6 @@ private:
    Vector funval2;
    Vector nor;
    Vector fluxN;
-   IntegrationPoint eip1;
-   IntegrationPoint eip2;
 
 public:
    FaceIntegrator(RiemannSolver &rsolver_, const int dim);
@@ -424,19 +422,16 @@ void FaceIntegrator::AssembleFaceVector(const FiniteElement &el1,
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
 
-      Tr.Loc1.Transform(ip, eip1);
-      Tr.Loc2.Transform(ip, eip2);
+      Tr.SetAllIntPoints(&ip); // set face and element int. points
 
       // Calculate basis functions on both elements at the face
-      el1.CalcShape(eip1, shape1);
-      el2.CalcShape(eip2, shape2);
+      el1.CalcShape(Tr.GetElement1IntPoint(), shape1);
+      el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
 
       // Interpolate elfun at the point
       elfun1_mat.MultTranspose(shape1, funval1);
       elfun2_mat.MultTranspose(shape2, funval2);
 
-      Tr.SetIntPoint(&ip);
-
       // Get the normal vector and the flux on the face
       CalcOrtho(Tr.Jacobian(), nor);
       const double mcs = rsolver.Eval(funval1, funval2, nor, fluxN);

examples/ex1p.cpp

@@ -32,7 +32,8 @@
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
-//               mpirun -np 4 ex1p -pa -d ceed-cuda
+//             * mpirun -np 4 ex1p -pa -d ceed-cuda
+//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
 //               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //
 // Description:  This example code demonstrates the use of MFEM to define a

examples/ex6.cpp

@@ -20,7 +20,7 @@
 //               ex6 -pa -d occa-cuda
 //               ex6 -pa -d raja-omp
 //               ex6 -pa -d ceed-cpu
-//               * ex6 -pa -d ceed-cuda
+//             * ex6 -pa -d ceed-cuda
 //               ex6 -pa -d ceed-cuda:/gpu/cuda/shared
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh

examples/ex6p.cpp

@@ -20,7 +20,7 @@
 //               mpirun -np 4 ex6p -pa -d occa-cuda
 //               mpirun -np 4 ex6p -pa -d raja-omp
 //               mpirun -np 4 ex6p -pa -d ceed-cpu
-//               * mpirun -np 4 ex6p -pa -d ceed-cuda
+//             * mpirun -np 4 ex6p -pa -d ceed-cuda
 //               mpirun -np 4 ex6p -pa -d ceed-cuda:/gpu/cuda/shared
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh

examples/ex9.cpp

@@ -254,7 +254,7 @@ int main(int argc, char *argv[])
 
    // 5. Define the discontinuous DG finite element space of the given
    //    polynomial order on the refined mesh.
-   DG_FECollection fec(order, dim, BasisType::GaussLobatto);
+   DG_FECollection fec(order, dim, BasisType::Positive);
    FiniteElementSpace fes(&mesh, &fec);
 
    cout << "Number of unknowns: " << fes.GetVSize() << endl;
@@ -378,6 +378,10 @@ int main(int argc, char *argv[])
    //    iterations, ti, with a time-step dt).
    FE_Evolution adv(m, k, b);
 
+   Vector masses(u.Size());
+   m.SpMat().Mult(u, masses);
+   double mass = masses.Sum();
+
    double t = 0.0;
    adv.SetTime(t);
    ode_solver->Init(adv);
@@ -424,6 +428,9 @@ int main(int argc, char *argv[])
       u.Save(osol);
    }
 
+   m.SpMat().Mult(u, masses);
+   cout << "Mass difference:" << abs(mass - masses.Sum()) << endl;
+
    // 10. Free the used memory.
    delete ode_solver;
    delete pd;

examples/makefile

@@ -114,6 +114,11 @@ ex11p-test-strumpack: ex11p
 	@$(call mfem-test,$<, $(RUN_MPI), STRUMPACK example,--strumpack)
 test-par-YES: ex11p-test-strumpack
 endif
+ifeq ($(MFEM_USE_SUPERLU),YES)
+ex11p-test-superlu: ex11p
+	@$(call mfem-test,$<, $(RUN_MPI), SuperLU_DIST example,--superlu)
+test-par-YES: ex11p-test-superlu
+endif
 
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk
 

examples/petsc/CMakeLists.txt

@@ -34,6 +34,15 @@ if (MFEM_USE_MPI)
     )
 endif()
 
+if (MFEM_USE_SLEPC)
+  list(APPEND PETSC_EXAMPLES_SRCS
+    ex11p.cpp
+    )
+  list(APPEND PETSC_RC_FILES
+    rc_ex11p_lobpcg rc_ex11p_gd
+    )
+endif()
+
 # Include the source directory where mfem.hpp and mfem-performance.hpp are.
 include_directories(BEFORE ${PROJECT_BINARY_DIR})
 
@@ -78,12 +87,22 @@ set(EX9_E_ARGS       -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts
 set(EX9_ES_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step)
 set(EX9_IS_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5)
 set(EX10_ARGS        -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3)
+if (MFEM_USE_SLEPC)
+  set(EX11_ARGS_SINV   -m ../../data/star.mesh --useslepc)
+  set(EX11_ARGS_LOBPCG -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg)
+  set(EX11_ARGS_GD -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd)
+endif()
 
 # Add the tests: one test per command-line-variable.
-foreach(TEST_OPTIONS_VAR
-        EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
-        EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
-        EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
+set(TEST_OPTIONS_VARS
+    EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
+    EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
+    EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
+if (MFEM_USE_SLEPC)
+    list(APPEND TEST_OPTIONS_VARS EX11_ARGS_SINV EX11_ARGS_LOBPCG EX11_ARGS_GD)
+endif()
+
+foreach(TEST_OPTIONS_VAR ${TEST_OPTIONS_VARS})
   string(REGEX REPLACE "^(.+)_ARGS" "\\1" TEST_NAME_UC ${TEST_OPTIONS_VAR})
   string(REGEX REPLACE "^([^_]+)" "\\1P" TEST_NAME_UC ${TEST_NAME_UC})
   string(TOLOWER ${TEST_NAME_UC} TEST_NAME_FULL)

examples/petsc/ex11p.cpp

@@ -0,0 +1,440 @@
+//                       MFEM Example 11 - Parallel Version
+//                              PETSc Modification
+//
+// Compile with: make ex11p
+//
+// Sample runs:  mpirun -np 4 ex11p -m ../../data/star.mesh
+//               mpirun -np 4 ex11p -m ../../data/star.mesh --slepcopts rc_ex11p_lobpcg
+//               mpirun -np 4 ex11p -m ../../data/star.mesh --slepcopts rc_ex11p_gd
+//
+// Description:  This example code demonstrates the use of MFEM to solve the
+//               eigenvalue problem -Delta u = lambda u with homogeneous
+//               Dirichlet boundary conditions.
+//
+//               We compute a number of the lowest eigenmodes by discretizing
+//               the Laplacian and Mass operators using a FE space of the
+//               specified order, or an isoparametric/isogeometric space if
+//               order < 1 (quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example demonstrates the use of the SLEPc eigensolver as an
+//               alternative to the LOBPCG eigenvalue solver. The shift and
+//               invert spectral transformation is used to help the convergence
+//               to the smaller eigenvalues. Alternative solver parameters can
+//               be passed in a file with "-slepcopts".
+//
+//               Reusing a single GLVis visualization window for multiple
+//               eigenfunctions is also illustrated.
+//
+//               We recommend viewing Example 1 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#ifndef MFEM_USE_SLEPC
+#error This examples requires that MFEM is build with MFEM_USE_SLEPC=YES
+#endif
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int ser_ref_levels = 2;
+   int par_ref_levels = 1;
+   int order = 1;
+   int nev = 5;
+   int seed = 75;
+   bool slu_solver  = false;
+   bool sp_solver = false;
+   bool visualization = 1;
+   bool use_slepc = true;
+   const char *slepcrc_file = "";
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nev, "-n", "--num-eigs",
+                  "Number of desired eigenmodes.");
+   args.AddOption(&seed, "-s", "--seed",
+                  "Random seed used to initialize LOBPCG.");
+#ifdef MFEM_USE_SUPERLU
+   args.AddOption(&slu_solver, "-slu", "--superlu", "-no-slu",
+                  "--no-superlu", "Use the SuperLU Solver.");
+#endif
+#ifdef MFEM_USE_STRUMPACK
+   args.AddOption(&sp_solver, "-sp", "--strumpack", "-no-sp",
+                  "--no-strumpack", "Use the STRUMPACK Solver.");
+#endif
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&use_slepc, "-useslepc","--useslepc","-no-slepc",
+                  "--no-slepc","Use or not SLEPc to solve the eigenvalue problem");
+   args.AddOption(&slepcrc_file, "-slepcopts", "--slepcopts",
+                  "SlepcOptions file to use.");
+   args.Parse();
+   if (slu_solver && sp_solver)
+   {
+      if (myid == 0)
+         cout << "WARNING: Both SuperLU and STRUMPACK have been selected,"
+              << " please choose either one." << endl
+              << "         Defaulting to SuperLU." << endl;
+      sp_solver = false;
+   }
+   // The command line options are also passed to the STRUMPACK
+   // solver. So do not exit if some options are not recognized.
+   if (!sp_solver)
+   {
+      if (!args.Good())
+      {
+         if (myid == 0)
+         {
+            args.PrintUsage(cout);
+         }
+         MPI_Finalize();
+         return 1;
+      }
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 2b. We initialize SLEPc. This internally initializes PETSc as well.
+   MFEMInitializeSlepc(NULL,NULL,slepcrc_file,NULL);
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors. We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
+   //    specified on the command line with -rs).
+   for (int lev = 0; lev < ser_ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution (1 time by
+   //    default, or specified on the command line with -rp). Once the parallel
+   //    mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   for (int lev = 0; lev < par_ref_levels; lev++)
+   {
+      pmesh->UniformRefinement();
+   }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of unknowns: " << size << endl;
+   }
+
+   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   //    element space. The first corresponds to the Laplacian operator -Delta,
+   //    while the second is a simple mass matrix needed on the right hand side
+   //    of the generalized eigenvalue problem below. The boundary conditions
+   //    are implemented by elimination with special values on the diagonal to
+   //    shift the Dirichlet eigenvalues out of the computational range. After
+   //    serial and parallel assembly we extract the corresponding parallel
+   //    matrices A and M.
+   ConstantCoefficient one(1.0);
+   Array<int> ess_bdr;
+   if (pmesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   if (pmesh->bdr_attributes.Size() == 0)
+   {
+      // Add a mass term if the mesh has no boundary, e.g. periodic mesh or
+      // closed surface.
+      a->AddDomainIntegrator(new MassIntegrator(one));
+   }
+   a->Assemble();
+   a->EliminateEssentialBCDiag(ess_bdr, 1.0);
+   a->Finalize();
+
+   ParBilinearForm *m = new ParBilinearForm(fespace);
+   m->AddDomainIntegrator(new MassIntegrator(one));
+   m->Assemble();
+   // shift the eigenvalue corresponding to eliminated dofs to a large value
+   m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
+   m->Finalize();
+
+   PetscParMatrix *pA = NULL, *pM = NULL;
+   HypreParMatrix *A = NULL, *M = NULL;
+   Operator::Type tid =
+      !use_slepc ? Operator::Hypre_ParCSR : Operator::PETSC_MATAIJ;
+   OperatorHandle Ah(tid), Mh(tid);
+
+   a->ParallelAssemble(Ah);
+   if (!use_slepc) { Ah.Get(A); }
+   else { Ah.Get(pA); }
+   Ah.SetOperatorOwner(false);
+
+   m->ParallelAssemble(Mh);
+   if (!use_slepc) {Mh.Get(M); }
+   else {Mh.Get(pM); }
+   Mh.SetOperatorOwner(false);
+
+#if defined(MFEM_USE_SUPERLU) || defined(MFEM_USE_STRUMPACK)
+   Operator * Arow = NULL;
+#ifdef MFEM_USE_SUPERLU
+   if (slu_solver)
+   {
+      Arow = new SuperLURowLocMatrix(*A);
+   }
+#endif
+#ifdef MFEM_USE_STRUMPACK
+   if (sp_solver)
+   {
+      Arow = new STRUMPACKRowLocMatrix(*A);
+   }
+#endif
+#endif
+
+   delete a;
+   delete m;
+
+   // 8. Define and configure the LOBPCG eigensolver and the BoomerAMG
+   //    preconditioner for A to be used within the solver. Set the matrices
+   //    which define the generalized eigenproblem A x = lambda M x.
+   Solver * precond = NULL;
+   if (!use_slepc)
+   {
+      if (!slu_solver && !sp_solver)
+      {
+         HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
+         amg->SetPrintLevel(0);
+         precond = amg;
+      }
+      else
+      {
+#ifdef MFEM_USE_SUPERLU
+         if (slu_solver)
+         {
+            SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
+            superlu->SetPrintStatistics(false);
+            superlu->SetSymmetricPattern(true);
+            superlu->SetColumnPermutation(superlu::PARMETIS);
+            superlu->SetOperator(*Arow);
+            precond = superlu;
+         }
+#endif
+#ifdef MFEM_USE_STRUMPACK
+         if (sp_solver)
+         {
+            STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+            strumpack->SetPrintFactorStatistics(true);
+            strumpack->SetPrintSolveStatistics(false);
+            strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+            strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+            strumpack->DisableMatching();
+            strumpack->SetOperator(*Arow);
+            strumpack->SetFromCommandLine();
+            precond = strumpack;
+         }
+#endif
+      }
+   }
+
+   HypreLOBPCG * lobpcg = NULL;
+   SlepcEigenSolver * slepc = NULL;
+   if (!use_slepc)
+   {
+
+      lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
+      lobpcg->SetNumModes(nev);
+      lobpcg->SetRandomSeed(seed);
+      lobpcg->SetPreconditioner(*precond);
+      lobpcg->SetMaxIter(200);
+      lobpcg->SetTol(1e-8);
+      lobpcg->SetPrecondUsageMode(1);
+      lobpcg->SetPrintLevel(1);
+      lobpcg->SetMassMatrix(*M);
+      lobpcg->SetOperator(*A);
+   }
+   else
+   {
+      slepc = new SlepcEigenSolver(MPI_COMM_WORLD);
+      slepc->SetNumModes(nev);
+      slepc->SetWhichEigenpairs(SlepcEigenSolver::TARGET_REAL);
+      slepc->SetTarget(0.0);
+      slepc->SetSpectralTransformation(SlepcEigenSolver::SHIFT_INVERT);
+      slepc->SetOperators(*pA,*pM);
+   }
+
+   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
+   //    parallel grid function to represent each of the eigenmodes returned by
+   //    the solver.
+   Array<double> eigenvalues;
+   if (!use_slepc)
+   {
+      lobpcg->Solve();
+      lobpcg->GetEigenvalues(eigenvalues);
+   }
+   else
+   {
+      slepc->Solve();
+      eigenvalues.SetSize(nev);
+      for (int i=0; i<nev; i++)
+      {
+         slepc->GetEigenvalue(i,eigenvalues[i]);
+      }
+   }
+   Vector temp(fespace->GetTrueVSize());
+   ParGridFunction x(fespace);
+
+   // 10. Save the refined mesh and the modes in parallel. This output can be
+   //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
+   {
+      ostringstream mesh_name, mode_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      for (int i=0; i<nev; i++)
+      {
+         // convert eigenvector from HypreParVector to ParGridFunction
+         if (!use_slepc)
+         {
+            x = lobpcg->GetEigenvector(i);
+         }
+         else
+         {
+            slepc->GetEigenvector(i,temp);
+            x.Distribute(temp);
+
+         }
+
+         mode_name << "mode_" << setfill('0') << setw(2) << i << "."
+                   << setfill('0') << setw(6) << myid;
+
+         ofstream mode_ofs(mode_name.str().c_str());
+         mode_ofs.precision(8);
+         x.Save(mode_ofs);
+         mode_name.str("");
+      }
+   }
+
+   // 11. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream mode_sock(vishost, visport);
+      mode_sock.precision(8);
+
+      for (int i=0; i<nev; i++)
+      {
+         if ( myid == 0 )
+         {
+            cout << "Eigenmode " << i+1 << '/' << nev
+                 << ", Lambda = " << eigenvalues[i] << endl;
+         }
+
+         // convert eigenvector from HypreParVector to ParGridFunction
+         if (!use_slepc)
+         {
+            x = lobpcg->GetEigenvector(i);
+         }
+         else
+         {
+            slepc->GetEigenvector(i,temp);
+            x.Distribute(temp);
+         }
+
+         mode_sock << "parallel " << num_procs << " " << myid << "\n"
+                   << "solution\n" << *pmesh << x << flush
+                   << "window_title 'Eigenmode " << i+1 << '/' << nev
+                   << ", Lambda = " << eigenvalues[i] << "'" << endl;
+
+         char c;
+         if (myid == 0)
+         {
+            cout << "press (q)uit or (c)ontinue --> " << flush;
+            cin >> c;
+         }
+         MPI_Bcast(&c, 1, MPI_CHAR, 0, MPI_COMM_WORLD);
+
+         if (c != 'c')
+         {
+            break;
+         }
+      }
+      mode_sock.close();
+   }
+
+   // 12. Free the used memory.
+   if (!use_slepc)
+   {
+      delete lobpcg;
+   }
+   else
+   {
+      delete slepc;
+   }
+   delete precond;
+   delete M;
+   delete A;
+#if defined(MFEM_USE_SUPERLU) || defined(MFEM_USE_STRUMPACK)
+   delete Arow;
+#endif
+
+   delete fespace;
+   if (order > 0)
+   {
+      delete fec;
+   }
+   delete pmesh;
+
+   // We finalize SLEPc
+   MFEMFinalizeSlepc();
+   MPI_Finalize();
+
+   return 0;
+}

examples/petsc/makefile

@@ -23,6 +23,9 @@ MFEM_LIB_FILE = mfem_is_not_built
 
 SEQ_EXAMPLES =
 PAR_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex9p ex10p
+ifeq ($(MFEM_USE_SLEPC),YES)
+   PAR_EXAMPLES += ex11p
+endif
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
 else
@@ -87,6 +90,9 @@ EX10_ARGS        := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p
 EX10_MF_ARGS     := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mf -tf 6 -s 3 -rs 0 -dt 3
 EX10_MFOP_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mfop -tf 6 -s 3 -rs 0 -dt 3
 EX10_JFNK_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_jfnk --jfnk -tf 6 -s 3 -rs 0 -dt 3
+EX11_ARGS_SINV   := -m ../../data/star.mesh --useslepc
+EX11_ARGS_LOBPCG := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg
+EX11_ARGS_GD     := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd
 
 ex1p-test-par: ex1p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_W))
@@ -114,6 +120,12 @@ ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MF_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MFOP_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_JFNK_ARGS))
+ifeq ($(MFEM_USE_SLEPC),YES)
+ex11p-test-par: ex11p
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_SINV))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_LOBPCG))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_GD))
+endif
 
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk
 

examples/petsc/rc_ex11p_gd

@@ -0,0 +1,6 @@
+# Options for the eigenvalue solver
+-eps_view
+-eps_converged_reason
+-eps_type gd
+# Options for the spectral transform
+-st_type precond

examples/petsc/rc_ex11p_lobpcg

@@ -0,0 +1,11 @@
+# Options for the eigenvalue solver
+-eps_monitor
+-eps_converged_reason
+-eps_view_values
+-eps_type lobpcg
+-eps_gen_hermitian
+-eps_smallest_real
+-eps_lobpcg_blocksize 5
+# Options for the spectral transform
+-st_type precond
+-st_pc_type gamg

fem/bilininteg.cpp

@@ -926,11 +926,14 @@ void BoundaryMassIntegrator::AssembleFaceMatrix(
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
-      IntegrationPoint eip;
-      Trans.Loc1.Transform(ip, eip);
+
+      // Set the integration point in the face and the neighboring element
+      Trans.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Trans.GetElement1IntPoint();
       el1.CalcShape(eip, shape);
 
-      Trans.SetIntPoint(&ip);
       w = Trans.Weight() * ip.weight;
       if (Q)
       {
@@ -2571,15 +2574,16 @@ void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
    for (int p = 0; p < ir->GetNPoints(); p++)
    {
       const IntegrationPoint &ip = ir->IntPoint(p);
-      IntegrationPoint eip1, eip2;
-      Trans.Loc1.Transform(ip, eip1);
-      if (ndof2)
-      {
-         Trans.Loc2.Transform(ip, eip2);
-      }
-      el1.CalcShape(eip1, shape1);
 
-      Trans.SetIntPoint(&ip);
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+
+      // Access the neighboring elements' integration points
+      // Note: eip2 will only contain valid data if Elem2 exists
+      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
+      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
+
+      el1.CalcShape(eip1, shape1);
 
       u->Eval(vu, *Trans.Elem1, eip1);
 
@@ -2727,10 +2731,15 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
    for (int p = 0; p < ir->GetNPoints(); p++)
    {
       const IntegrationPoint &ip = ir->IntPoint(p);
-      IntegrationPoint eip1, eip2;
 
-      Trans.Loc1.Transform(ip, eip1);
-      Trans.SetIntPoint(&ip);
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+
+      // Access the neighboring elements' integration points
+      // Note: eip2 will only contain valid data if Elem2 exists
+      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
+      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
+
       if (dim == 1)
       {
          nor(0) = 2*eip1.x - 1.0;
@@ -2787,7 +2796,6 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
 
       if (ndof2)
       {
-         Trans.Loc2.Transform(ip, eip2);
          el2.CalcShape(eip2, shape2);
          el2.CalcDShape(eip2, dshape2);
          w = ip.weight/2/Trans.Elem2->Weight();
@@ -3005,9 +3013,14 @@ void DGElasticityIntegrator::AssembleFaceMatrix(
    for (int pind = 0; pind < ir->GetNPoints(); ++pind)
    {
       const IntegrationPoint &ip = ir->IntPoint(pind);
-      IntegrationPoint eip1, eip2; // integration point in the reference space
-      Trans.Loc1.Transform(ip, eip1);
-      Trans.SetIntPoint(&ip);
+
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+
+      // Access the neighboring elements' integration points
+      // Note: eip2 will only contain valid data if Elem2 exists
+      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
+      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
 
       el1.CalcShape(eip1, shape1);
       el1.CalcDShape(eip1, dshape1);
@@ -3027,7 +3040,6 @@ void DGElasticityIntegrator::AssembleFaceMatrix(
       double w, wLM;
       if (ndofs2)
       {
-         Trans.Loc2.Transform(ip, eip2);
          el2.CalcShape(eip2, shape2);
          el2.CalcDShape(eip2, dshape2);
          CalcAdjugate(Trans.Elem2->Jacobian(), adjJ);
@@ -3165,17 +3177,22 @@ void TraceJumpIntegrator::AssembleFaceMatrix(
    for (int p = 0; p < ir->GetNPoints(); p++)
    {
       const IntegrationPoint &ip = ir->IntPoint(p);
-      IntegrationPoint eip1, eip2;
+
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+
+      // Access the neighboring elements' integration points
+      // Note: eip2 will only contain valid data if Elem2 exists
+      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
+      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
+
       // Trace finite element shape function
-      Trans.SetIntPoint(&ip);
       trial_face_fe.CalcShape(ip, face_shape);
       // Side 1 finite element shape function
-      Trans.Loc1.Transform(ip, eip1);
       test_fe1.CalcShape(eip1, shape1);
       if (ndof2)
       {
          // Side 2 finite element shape function
-         Trans.Loc2.Transform(ip, eip2);
          test_fe2.CalcShape(eip2, shape2);
       }
       w = ip.weight;

fem/coefficient.cpp

@@ -209,18 +209,24 @@ void GradientGridFunctionCoefficient::Eval(
    GridFunc->GetGradients(T, ir, M);
 }
 
-CurlGridFunctionCoefficient::CurlGridFunctionCoefficient (
+CurlGridFunctionCoefficient::CurlGridFunctionCoefficient(
    const GridFunction *gf)
-   : VectorCoefficient ((gf) ?
-                        gf -> FESpace() -> GetMesh() -> SpaceDimension() : 0)
+   : VectorCoefficient(0)
 {
-   GridFunc = gf;
+   SetGridFunction(gf);
 }
 
 void CurlGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 {
-   GridFunc = gf; vdim = (gf) ?
-                         gf -> FESpace() -> GetMesh() -> SpaceDimension() : 0;
+   if (gf)
+   {
+      int sdim = gf -> FESpace() -> GetMesh() -> SpaceDimension();
+      MFEM_VERIFY(sdim == 2 || sdim == 3,
+                  "CurlGridFunctionCoefficient "
+                  "only defind for spaces of dimension 2 or 3.");
+   }
+   GridFunc = gf;
+   vdim = (gf) ? (2 * gf -> FESpace() -> GetMesh() -> SpaceDimension() - 3) : 0;
 }
 
 void CurlGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,

fem/eltrans.cpp

@@ -552,26 +552,32 @@ void IntegrationPointTransformation::Transform (const IntegrationRule &ir1,
    }
 }
 
-void FaceElementTransformations::SetIntPoint(const IntegrationPoint *ip)
+void FaceElementTransformations::SetIntPoint(const IntegrationPoint *face_ip)
 {
-   IsoparametricTransformation::SetIntPoint(ip);
+   IsoparametricTransformation::SetIntPoint(face_ip);
 
-   if (Elem1)
+   if (mask & 4)
    {
-      Loc1.Transform(*ip, eip1);
-      Elem1->SetIntPoint(&eip1);
+      Loc1.Transform(*face_ip, eip1);
+      if (Elem1)
+      {
+         Elem1->SetIntPoint(&eip1);
+      }
    }
-   if (Elem2)
+   if (mask & 8)
    {
-      Loc2.Transform(*ip, eip2);
-      Elem2->SetIntPoint(&eip2);
+      Loc2.Transform(*face_ip, eip2);
+      if (Elem2)
+      {
+         Elem2->SetIntPoint(&eip2);
+      }
    }
 }
 
 ElementTransformation &
 FaceElementTransformations::GetElement1Transformation()
 {
-   MFEM_VERIFY(mask & 1 && Elem1 != NULL, "The ElementTransformation "
+   MFEM_VERIFY(mask & HAVE_ELEM1 && Elem1 != NULL, "The ElementTransformation "
                "for the element has not been configured for side 1.");
    return *Elem1;
 }
@@ -579,7 +585,7 @@ FaceElementTransformations::GetElement1Transformation()
 ElementTransformation &
 FaceElementTransformations::GetElement2Transformation()
 {
-   MFEM_VERIFY(mask & 2 && Elem2 != NULL, "The ElementTransformation "
+   MFEM_VERIFY(mask & HAVE_ELEM2 && Elem2 != NULL, "The ElementTransformation "
                "for the element has not been configured for side 2.");
    return *Elem2;
 }
@@ -587,7 +593,7 @@ FaceElementTransformations::GetElement2Transformation()
 IntegrationPointTransformation &
 FaceElementTransformations::GetIntPoint1Transformation()
 {
-   MFEM_VERIFY(mask & 4, "The IntegrationPointTransformation "
+   MFEM_VERIFY(mask & HAVE_LOC1, "The IntegrationPointTransformation "
                "for the element has not been configured for side 1.");
    return Loc1;
 }
@@ -595,7 +601,7 @@ FaceElementTransformations::GetIntPoint1Transformation()
 IntegrationPointTransformation &
 FaceElementTransformations::GetIntPoint2Transformation()
 {
-   MFEM_VERIFY(mask & 8, "The IntegrationPointTransformation "
+   MFEM_VERIFY(mask & HAVE_LOC2, "The IntegrationPointTransformation "
                "for the element has not been configured for side 2.");
    return Loc2;
 }
@@ -603,7 +609,7 @@ FaceElementTransformations::GetIntPoint2Transformation()
 void FaceElementTransformations::Transform(const IntegrationPoint &ip,
                                            Vector &trans)
 {
-   MFEM_VERIFY(mask & 16, "The ElementTransformation "
+   MFEM_VERIFY(mask & HAVE_FACE, "The ElementTransformation "
                "for the face has not been configured.");
    IsoparametricTransformation::Transform(ip, trans);
 }
@@ -611,7 +617,7 @@ void FaceElementTransformations::Transform(const IntegrationPoint &ip,
 void FaceElementTransformations::Transform(const IntegrationRule &ir,
                                            DenseMatrix &tr)
 {
-   MFEM_VERIFY(mask & 16, "The ElementTransformation "
+   MFEM_VERIFY(mask & HAVE_FACE, "The ElementTransformation "
                "for the face has not been configured.");
    IsoparametricTransformation::Transform(ir, tr);
 }
@@ -619,7 +625,7 @@ void FaceElementTransformations::Transform(const IntegrationRule &ir,
 void FaceElementTransformations::Transform(const DenseMatrix &matrix,
                                            DenseMatrix &result)
 {
-   MFEM_VERIFY(mask & 16, "The ElementTransformation "
+   MFEM_VERIFY(mask & HAVE_FACE, "The ElementTransformation "
                "for the face has not been configured.");
    IsoparametricTransformation::Transform(matrix, result);
 }

fem/eltrans.hpp

@@ -439,15 +439,57 @@ public:
    void Transform (const IntegrationRule  &, IntegrationRule  &);
 };
 
+/** @brief A specialized ElementTransformation class representing a face and
+    its two neighboring elements.
 
+    This class can be used as a container for the element transformation data
+    needed for integrating discontinuous fields on element interfaces in a
+    Discontinuous Galerkin (DG) context.
+
+    The secondary purpose of this class is to enable the
+    GridFunction::GetValue function, and various related functions, to properly
+    evaluate fields with limited continuity on boundary elements.
+*/
 class FaceElementTransformations : public IsoparametricTransformation
 {
 private:
+
+   // Bitwise OR of ConfigMasks
    int mask;
 
    IntegrationPoint eip1, eip2;
 
+protected: // interface for Mesh to be able to configure this object.
+
+   friend class Mesh;
+#ifdef MFEM_USE_MPI
+   friend class ParMesh;
+#endif
+
+   /// Set the mask indicating which portions of the object have been setup
+   /** The argument @a m is a bitmask used in
+       Mesh::GetFaceElementTransformations to indicate which portions of the
+       FaceElementTransformations object have been configured.
+
+       mask &  1: Elem1 is configured
+       mask &  2: Elem2 is configured
+       mask &  4: Loc1 is configured
+       mask &  8: Loc2 is configured
+       mask & 16: The Face transformation itself is configured
+   */
+   void SetConfigurationMask(int m) { mask = m; }
+
 public:
+
+   enum ConfigMasks
+   {
+      HAVE_ELEM1 =  1, ///< Element on side 1 is configured
+      HAVE_ELEM2 =  2, ///< Element on side 2 is configured
+      HAVE_LOC1  =  4, ///< Point transformation for side 1 is configured
+      HAVE_LOC2  =  8, ///< Point transformation for side 2 is configured
+      HAVE_FACE  = 16  ///< Face transformation is configured
+   };
+
    int Elem1No, Elem2No;
    Geometry::Type &FaceGeom; ///< @deprecated Use GetGeometryType instead
    ElementTransformation *Elem1, *Elem2;
@@ -466,10 +508,10 @@ public:
    */
    void SetGeometryType(Geometry::Type g) { geom = g; }
 
-   /// Set the mask indicating which portions of the object have been setup
-   /** The argument @a m is a bitmask used in
-       Mesh::GetFaceElementTransformations to indicate which portions of the
-       FaceElement Transformations object have been configured.
+   /** @brief Return the mask defining the configuration state.
+
+       The mask value indicates which portions of FaceElementTransformations
+       object have been configured.
 
        mask &  1: Elem1 is configured
        mask &  2: Elem2 is configured
@@ -477,12 +519,45 @@ public:
        mask &  8: Loc2 is configured
        mask & 16: The Face transformation itself is configured
    */
-   void SetConfigurationMask(int m) { mask = m; }
-   int  GetConfigurationMask() const { return mask; }
+   int GetConfigurationMask() const { return mask; }
 
    /** @brief Set the integration point in the Face and the two neighboring
-       elements, if present. */
-   void SetIntPoint(const IntegrationPoint *ip);
+       elements, if present.
+
+       The point @a face_ip must be in the reference coordinate system of the
+       face.
+   */
+   void SetIntPoint(const IntegrationPoint *face_ip);
+
+   /** @brief Set the integration point in the Face and the two neighboring
+       elements, if present.
+
+       This is a more expressive member function name than SetIntPoint, which
+       in this special case, does the same thing. This function can be used for
+       greater code clarity.
+   */
+   inline void SetAllIntPoints(const IntegrationPoint *face_ip)
+   { FaceElementTransformations::SetIntPoint(face_ip); }
+
+   /** @brief Get a const reference to the integration point in neighboring
+       element 1 corresponding to the currently set integration point on the
+       face.
+
+       This IntegrationPoint object will only contain up-to-date data if
+       SetIntPoint or SetAllIntPoints has been called with the latest
+       integration point for the face and the appropriate point transformation
+       has been configured. */
+   const IntegrationPoint &GetElement1IntPoint() { return eip1; }
+
+   /** @brief Get a const reference to the integration point in neighboring
+       element 2 corresponding to the currently set integration point on the
+       face.
+
+       This IntegrationPoint object will only contain up-to-date data if
+       SetIntPoint or SetAllIntPoints has been called with the latest
+       integration point for the face and the appropriate point transformation
+       has been configured. */
+   const IntegrationPoint &GetElement2IntPoint() { return eip2; }
 
    virtual void Transform(const IntegrationPoint &, Vector &);
    virtual void Transform(const IntegrationRule &, DenseMatrix &);

fem/gridfunc.cpp

@@ -397,8 +397,16 @@ const
    fes->DofsToVDofs(vdim-1, dofs);
    Vector DofVal(dofs.Size()), LocVec;
    const FiniteElement *fe = fes->GetFE(i);
-   MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE, "invalid FE map type");
-   fe->CalcShape(ip, DofVal);
+   if (fe->GetMapType() == FiniteElement::VALUE)
+   {
+      fe->CalcShape(ip, DofVal);
+   }
+   else
+   {
+      ElementTransformation *Tr = fes->GetElementTransformation(i);
+      Tr->SetIntPoint(&ip);
+      fe->CalcPhysShape(*Tr, DofVal);
+   }
    GetSubVector(dofs, LocVec);
 
    return (DofVal * LocVec);
@@ -415,10 +423,17 @@ void GridFunction::GetVectorValue(int i, const IntegrationPoint &ip,
    GetSubVector(vdofs, loc_data);
    if (FElem->GetRangeType() == FiniteElement::SCALAR)
    {
-      MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
-                  "invalid FE map type");
       Vector shape(dof);
-      FElem->CalcShape(ip, shape);
+      if (FElem->GetMapType() == FiniteElement::VALUE)
+      {
+         FElem->CalcShape(ip, shape);
+      }
+      else
+      {
+         ElementTransformation *Tr = fes->GetElementTransformation(i);
+         Tr->SetIntPoint(&ip);
+         FElem->CalcPhysShape(*Tr, shape);
+      }
       int vdim = fes->GetVDim();
       val.SetSize(vdim);
       for (int k = 0; k < vdim; k++)
@@ -752,19 +767,21 @@ double GridFunction::GetValue(ElementTransformation &T,
             IntegrationPoint fip;
             be_to_bfe(FET->GetGeometryType(), o, ip, fip);
 
-            FET->SetIntPoint(&fip);
+            // Compute and set the point in element 1 from fip
+            FET->SetAllIntPoints(&fip);
             ElementTransformation & T1 = FET->GetElement1Transformation();
             return GetValue(T1, T1.GetIntPoint(), comp);
          }
-         break;
       }
+      break;
       case ElementTransformation::BDR_FACE:
       {
          FaceElementTransformations * FET =
             dynamic_cast<FaceElementTransformations *>(&T);
 
          // Evaluate in neighboring element for both continuous and
-         // discontinuous fields.
+         // discontinuous fields (the integration point in T1 should have
+         // already been set).
          ElementTransformation & T1 = FET->GetElement1Transformation();
          return GetValue(T1, T1.GetIntPoint(), comp);
       }
@@ -888,19 +905,21 @@ void GridFunction::GetVectorValue(ElementTransformation &T,
             IntegrationPoint fip;
             be_to_bfe(FET->GetGeometryType(), o, ip, fip);
 
-            FET->SetIntPoint(&fip);
+            // Compute and set the point in element 1 from fip
+            FET->SetAllIntPoints(&fip);
             ElementTransformation & T1 = FET->GetElement1Transformation();
             return GetVectorValue(T1, T1.GetIntPoint(), val);
          }
-         break;
       }
+      break;
       case ElementTransformation::BDR_FACE:
       {
          FaceElementTransformations * FET =
             dynamic_cast<FaceElementTransformations *>(&T);
 
          // Evaluate in neighboring element for both continuous and
-         // discontinuous fields.
+         // discontinuous fields (the integration point in T1 should have
+         // already been set).
          ElementTransformation & T1 = FET->GetElement1Transformation();
          return GetVectorValue(T1, T1.GetIntPoint(), val);
       }
@@ -1338,107 +1357,262 @@ void GridFunction::GetVectorGradientHat(
    MultAtB(loc_data_mat, dshape, gh);
 }
 
-double GridFunction::GetDivergence(ElementTransformation &tr) const
+double GridFunction::GetDivergence(ElementTransformation &T) const
 {
-   double div_v;
-   int elNo = tr.ElementNo;
-   const FiniteElement *FElem = fes->GetFE(elNo);
-   if (FElem->GetRangeType() == FiniteElement::SCALAR)
+   switch (T.ElementType)
    {
-      MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
-                  "invalid FE map type");
-      DenseMatrix grad_hat;
-      GetVectorGradientHat(tr, grad_hat);
-      const DenseMatrix &Jinv = tr.InverseJacobian();
-      div_v = 0.0;
-      for (int i = 0; i < Jinv.Width(); i++)
+      case ElementTransformation::ELEMENT:
       {
-         for (int j = 0; j < Jinv.Height(); j++)
+         int elNo = T.ElementNo;
+         const FiniteElement *fe = fes->GetFE(elNo);
+         if (fe->GetRangeType() == FiniteElement::SCALAR)
          {
-            div_v += grad_hat(i, j) * Jinv(j, i);
+            MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE,
+                        "invalid FE map type");
+            DenseMatrix grad_hat;
+            GetVectorGradientHat(T, grad_hat);
+            const DenseMatrix &Jinv = T.InverseJacobian();
+            double div_v = 0.0;
+            for (int i = 0; i < Jinv.Width(); i++)
+            {
+               for (int j = 0; j < Jinv.Height(); j++)
+               {
+                  div_v += grad_hat(i, j) * Jinv(j, i);
+               }
+            }
+            return div_v;
+         }
+         else
+         {
+            // Assuming RT-type space
+            Array<int> dofs;
+            fes->GetElementDofs(elNo, dofs);
+            Vector loc_data, divshape(fe->GetDof());
+            GetSubVector(dofs, loc_data);
+            fe->CalcDivShape(T.GetIntPoint(), divshape);
+            return (loc_data * divshape) / T.Weight();
          }
       }
+      break;
+      case ElementTransformation::BDR_ELEMENT:
+      {
+         // In order to properly capture the derivative of the normal component
+         // of the field (as well as the transverse divergence of the
+         // tangential compoents) we must evaluate it in the neighboring
+         // element.
+         FaceElementTransformations * FET =
+            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
+
+         // Boundary elements and Boundary Faces may have different
+         // orientations so adjust the integration point if necessary.
+         int o = 0;
+         if (fes->GetMesh()->Dimension() == 3)
+         {
+            int f;
+            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+         }
+
+         IntegrationPoint fip;
+         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);
+
+         // Compute and set the point in element 1 from fip
+         FET->SetAllIntPoints(&fip);
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+
+         return GetDivergence(T1);
+      }
+      break;
+      case ElementTransformation::BDR_FACE:
+      {
+         // This must be a DG context so this dynamic cast must succeed.
+         FaceElementTransformations * FET =
+            dynamic_cast<FaceElementTransformations *>(&T);
+
+         // Evaluate in neighboring element (the integration point in T1 should
+         // have already been set).
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+         return GetDivergence(T1);
+      }
+      break;
+      default:
+      {
+         MFEM_ABORT("GridFunction::GetDivergence: Unsupported element type \""
+                    << T.ElementType << "\"");
+      }
    }
-   else
-   {
-      // Assuming RT-type space
-      Array<int> dofs;
-      fes->GetElementDofs(elNo, dofs);
-      Vector loc_data, divshape(FElem->GetDof());
-      GetSubVector(dofs, loc_data);
-      FElem->CalcDivShape(tr.GetIntPoint(), divshape);
-      div_v = (loc_data * divshape) / tr.Weight();
-   }
-   return div_v;
+   return 0.0; // never reached
 }
 
-void GridFunction::GetCurl(ElementTransformation &tr, Vector &curl) const
+void GridFunction::GetCurl(ElementTransformation &T, Vector &curl) const
 {
-   int elNo = tr.ElementNo;
-   const FiniteElement *FElem = fes->GetFE(elNo);
-   if (FElem->GetRangeType() == FiniteElement::SCALAR)
+   switch (T.ElementType)
    {
-      MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
-                  "invalid FE map type");
-      DenseMatrix grad_hat;
-      GetVectorGradientHat(tr, grad_hat);
-      const DenseMatrix &Jinv = tr.InverseJacobian();
-      DenseMatrix grad(grad_hat.Height(), Jinv.Width()); // vdim x FElem->Dim
-      Mult(grad_hat, Jinv, grad);
-      MFEM_ASSERT(grad.Height() == grad.Width(), "");
-      if (grad.Height() == 3)
+      case ElementTransformation::ELEMENT:
       {
-         curl.SetSize(3);
-         curl(0) = grad(2,1) - grad(1,2);
-         curl(1) = grad(0,2) - grad(2,0);
-         curl(2) = grad(1,0) - grad(0,1);
+         int elNo = T.ElementNo;
+         const FiniteElement *fe = fes->GetFE(elNo);
+         if (fe->GetRangeType() == FiniteElement::SCALAR)
+         {
+            MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE,
+                        "invalid FE map type");
+            DenseMatrix grad_hat;
+            GetVectorGradientHat(T, grad_hat);
+            const DenseMatrix &Jinv = T.InverseJacobian();
+            // Dimensions of grad are vdim x FElem->Dim
+            DenseMatrix grad(grad_hat.Height(), Jinv.Width());
+            Mult(grad_hat, Jinv, grad);
+            MFEM_ASSERT(grad.Height() == grad.Width(), "");
+            if (grad.Height() == 3)
+            {
+               curl.SetSize(3);
+               curl(0) = grad(2,1) - grad(1,2);
+               curl(1) = grad(0,2) - grad(2,0);
+               curl(2) = grad(1,0) - grad(0,1);
+            }
+            else if (grad.Height() == 2)
+            {
+               curl.SetSize(1);
+               curl(0) = grad(1,0) - grad(0,1);
+            }
+         }
+         else
+         {
+            // Assuming ND-type space
+            Array<int> dofs;
+            fes->GetElementDofs(elNo, dofs);
+            Vector loc_data;
+            GetSubVector(dofs, loc_data);
+            DenseMatrix curl_shape(fe->GetDof(), fe->GetDim() == 3 ? 3 : 1);
+            fe->CalcCurlShape(T.GetIntPoint(), curl_shape);
+            curl.SetSize(curl_shape.Width());
+            if (curl_shape.Width() == 3)
+            {
+               double curl_hat[3];
+               curl_shape.MultTranspose(loc_data, curl_hat);
+               T.Jacobian().Mult(curl_hat, curl);
+            }
+            else
+            {
+               curl_shape.MultTranspose(loc_data, curl);
+            }
+            curl /= T.Weight();
+         }
       }
-      else if (grad.Height() == 2)
+      break;
+      case ElementTransformation::BDR_ELEMENT:
       {
-         curl.SetSize(1);
-         curl(0) = grad(1,0) - grad(0,1);
+         // In order to capture the tangential components of the curl we
+         // must evaluate it in the neighboring element.
+         FaceElementTransformations * FET =
+            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
+
+         // Boundary elements and Boundary Faces may have different
+         // orientations so adjust the integration point if necessary.
+         int o = 0;
+         if (fes->GetMesh()->Dimension() == 3)
+         {
+            int f;
+            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+         }
+
+         IntegrationPoint fip;
+         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);
+
+         // Compute and set the point in element 1 from fip
+         FET->SetAllIntPoints(&fip);
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+
+         GetCurl(T1, curl);
       }
-   }
-   else
-   {
-      // Assuming ND-type space
-      Array<int> dofs;
-      fes->GetElementDofs(elNo, dofs);
-      Vector loc_data;
-      GetSubVector(dofs, loc_data);
-      DenseMatrix curl_shape(FElem->GetDof(), FElem->GetDim() == 3 ? 3 : 1);
-      FElem->CalcCurlShape(tr.GetIntPoint(), curl_shape);
-      curl.SetSize(curl_shape.Width());
-      if (curl_shape.Width() == 3)
+      break;
+      case ElementTransformation::BDR_FACE:
       {
-         double curl_hat[3];
-         curl_shape.MultTranspose(loc_data, curl_hat);
-         tr.Jacobian().Mult(curl_hat, curl);
+         // This must be a DG context so this dynamic cast must succeed.
+         FaceElementTransformations * FET =
+            dynamic_cast<FaceElementTransformations *>(&T);
+
+         // Evaluate in neighboring element (the integration point in T1 should
+         // have already been set).
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+         GetCurl(T1, curl);
       }
-      else
+      break;
+      default:
       {
-         curl_shape.MultTranspose(loc_data, curl);
+         MFEM_ABORT("GridFunction::GetCurl: Unsupported element type \""
+                    << T.ElementType << "\"");
       }
-      curl /= tr.Weight();
    }
 }
 
-void GridFunction::GetGradient(ElementTransformation &tr, Vector &grad) const
+void GridFunction::GetGradient(ElementTransformation &T, Vector &grad) const
 {
-   int elNo = tr.ElementNo;
-   const FiniteElement *fe = fes->GetFE(elNo);
-   MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE, "invalid FE map type");
-   int dim = fe->GetDim(), dof = fe->GetDof();
-   DenseMatrix dshape(dof, dim);
-   Vector lval, gh(dim);
-   Array<int> dofs;
+   switch (T.ElementType)
+   {
+      case ElementTransformation::ELEMENT:
+      {
+         const FiniteElement * fe = fes->GetFE(T.ElementNo);
+         MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE,
+                     "invalid FE map type");
+         int spaceDim = fes->GetMesh()->SpaceDimension();
+         int dim = fe->GetDim(), dof = fe->GetDof();
+         DenseMatrix dshape(dof, dim);
+         Vector lval, gh(dim);
+         Array<int> dofs;
 
-   grad.SetSize(dim);
-   fes->GetElementDofs(elNo, dofs);
-   GetSubVector(dofs, lval);
-   fe->CalcDShape(tr.GetIntPoint(), dshape);
-   dshape.MultTranspose(lval, gh);
-   tr.InverseJacobian().MultTranspose(gh, grad);
+         grad.SetSize(spaceDim);
+         fes->GetElementDofs(T.ElementNo, dofs);
+         GetSubVector(dofs, lval);
+         fe->CalcDShape(T.GetIntPoint(), dshape);
+         dshape.MultTranspose(lval, gh);
+         T.InverseJacobian().MultTranspose(gh, grad);
+      }
+      break;
+      case ElementTransformation::BDR_ELEMENT:
+      {
+         // In order to properly capture the normal component of the gradient
+         // as well as its tangential components we must evaluate it in the
+         // neighboring element.
+         FaceElementTransformations * FET =
+            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
+
+         // Boundary elements and Boundary Faces may have different
+         // orientations so adjust the integration point if necessary.
+         int o = 0;
+         if (fes->GetMesh()->Dimension() == 3)
+         {
+            int f;
+            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+         }
+
+         IntegrationPoint fip;
+         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);
+
+         // Compute and set the point in element 1 from fip
+         FET->SetAllIntPoints(&fip);
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+
+         GetGradient(T1, grad);
+      }
+      break;
+      case ElementTransformation::BDR_FACE:
+      {
+         // This must be a DG context so this dynamic cast must succeed.
+         FaceElementTransformations * FET =
+            dynamic_cast<FaceElementTransformations *>(&T);
+
+         // Evaluate in neighboring element (the integration point in T1 should
+         // have already been set).
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+         GetGradient(T1, grad);
+      }
+      break;
+      default:
+      {
+         MFEM_ABORT("GridFunction::GetGradient: Unsupported element type \""
+                    << T.ElementType << "\"");
+      }
+   }
 }
 
 void GridFunction::GetGradients(ElementTransformation &tr,
@@ -1467,15 +1641,65 @@ void GridFunction::GetGradients(ElementTransformation &tr,
 }
 
 void GridFunction::GetVectorGradient(
-   ElementTransformation &tr, DenseMatrix &grad) const
+   ElementTransformation &T, DenseMatrix &grad) const
 {
-   MFEM_ASSERT(fes->GetFE(tr.ElementNo)->GetMapType() == FiniteElement::VALUE,
-               "invalid FE map type");
-   DenseMatrix grad_hat;
-   GetVectorGradientHat(tr, grad_hat);
-   const DenseMatrix &Jinv = tr.InverseJacobian();
-   grad.SetSize(grad_hat.Height(), Jinv.Width());
-   Mult(grad_hat, Jinv, grad);
+   switch (T.ElementType)
+   {
+      case ElementTransformation::ELEMENT:
+      {
+         MFEM_ASSERT(fes->GetFE(T.ElementNo)->GetMapType() ==
+                     FiniteElement::VALUE, "invalid FE map type");
+         DenseMatrix grad_hat;
+         GetVectorGradientHat(T, grad_hat);
+         const DenseMatrix &Jinv = T.InverseJacobian();
+         grad.SetSize(grad_hat.Height(), Jinv.Width());
+         Mult(grad_hat, Jinv, grad);
+      }
+      break;
+      case ElementTransformation::BDR_ELEMENT:
+      {
+         // In order to capture the normal component of the gradient we
+         // must evaluate it in the neighboring element.
+         FaceElementTransformations * FET =
+            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
+
+         // Boundary elements and Boundary Faces may have different
+         // orientations so adjust the integration point if necessary.
+         int o = 0;
+         if (fes->GetMesh()->Dimension() == 3)
+         {
+            int f;
+            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+         }
+
+         IntegrationPoint fip;
+         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);
+
+         // Compute and set the point in element 1 from fip
+         FET->SetAllIntPoints(&fip);
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+
+         GetVectorGradient(T1, grad);
+      }
+      break;
+      case ElementTransformation::BDR_FACE:
+      {
+         // This must be a DG context so this dynamic cast must succeed.
+         FaceElementTransformations * FET =
+            dynamic_cast<FaceElementTransformations *>(&T);
+
+         // Evaluate in neighboring element (the integration point in T1 should
+         // have already been set).
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+         GetVectorGradient(T1, grad);
+      }
+      break;
+      default:
+      {
+         MFEM_ABORT("GridFunction::GetVectorGradient: "
+                    "Unsupported element type \"" << T.ElementType << "\"");
+      }
+   }
 }
 
 void GridFunction::GetElementAverages(GridFunction &avgs) const
@@ -2760,7 +2984,7 @@ double GridFunction::ComputeLpError(const double p, Coefficient &exsol,
       }
       else
       {
-         int intorder = 2*fe->GetOrder() + 3; // <----------
+         int intorder = 2*fe->GetOrder() + 1; // <----------
          ir = &(IntRules.Get(fe->GetGeomType(), intorder));
       }
       GetValues(i, *ir, vals);
@@ -2893,7 +3117,7 @@ double GridFunction::ComputeLpError(const double p, VectorCoefficient &exsol,
       }
       else
       {
-         int intorder = 2*fe->GetOrder() + 3; // <----------
+         int intorder = 2*fe->GetOrder() + 1; // <----------
          ir = &(IntRules.Get(fe->GetGeomType(), intorder));
       }
       T = fes->GetElementTransformation(i);

fem/gridfunc.hpp

@@ -162,7 +162,8 @@ public:
                            int vdim = 1) const;
 
    /** Return a vector value from within the given element. */
-   void GetVectorValue(int i, const IntegrationPoint &ip, Vector &val) const;
+   virtual void GetVectorValue(int i, const IntegrationPoint &ip,
+                               Vector &val) const;
    ///@}
 
    /** @name Element Index Get Values Methods
@@ -208,13 +209,14 @@ public:
    ///@{
    /** Return a scalar value from within the element indicated by the
        ElementTransformation Object. */
-   double GetValue(ElementTransformation &T, const IntegrationPoint &ip,
-                   int comp = 0, Vector *tr = NULL) const;
+   virtual double GetValue(ElementTransformation &T, const IntegrationPoint &ip,
+                           int comp = 0, Vector *tr = NULL) const;
 
    /** Return a vector value from within the element indicated by the
        ElementTransformation Object. */
-   void GetVectorValue(ElementTransformation &T, const IntegrationPoint &ip,
-                       Vector &val, Vector *tr = NULL) const;
+   virtual void GetVectorValue(ElementTransformation &T,
+                               const IntegrationPoint &ip,
+                               Vector &val, Vector *tr = NULL) const;
    ///@}
 
    /** @name ElementTransformation Get Values Methods

fem/gslib.cpp

@@ -192,6 +192,7 @@ void FindPointsGSLIB::Interpolate(Array<unsigned int> &codes,
    const int ncomp      = field_in.FESpace()->GetVDim(),
              points_fld = field_in.Size() / ncomp,
              points_cnt = codes.Size();
+   field_out.SetSize(points_cnt*ncomp);
 
    for (int i = 0; i < ncomp; i++)
    {

fem/libceed/ceed.cpp

@@ -97,6 +97,8 @@ static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
    Vector qweight(Q);
    Vector shape_i(P);
    DenseMatrix grad_i(P, dim);
+
+   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
    const Table &el_dof = fes.GetElementToDofTable();
    Array<int> tp_el_dof(el_dof.Size_of_connections());
    const TensorBasisElement * tfe =
@@ -128,7 +130,15 @@ static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
          const int el_offset = fe->GetDof() * i;
          for (int j = 0; j < fe->GetDof(); j++)
          {
-            tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
+            if (compstride == 1)
+            {
+               tp_el_dof[j + el_offset] = fes.GetVDim()*
+                                          el_dof.GetJ()[dof_map[j] + el_offset];
+            }
+            else
+            {
+               tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
+            }
          }
       }
    }
@@ -157,20 +167,23 @@ static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
       {
          for (int i = 0; i < P; i++)
          {
-            tp_el_dof[i + e*P] = el_dof.GetJ()[i + e*P];
+            if (compstride == 1)
+            {
+               tp_el_dof[i + e*P] = fes.GetVDim()*el_dof.GetJ()[i + e*P];
+            }
+            else
+            {
+               tp_el_dof[i + e*P] = el_dof.GetJ()[i + e*P];
+            }
          }
       }
    }
    CeedBasisCreateH1(ceed, GetCeedTopology(fe->GetGeomType()), fes.GetVDim(),
                      fe->GetDof(), ir.GetNPoints(), shape.GetData(),
                      grad.GetData(), qref.GetData(), qweight.GetData(), basis);
-   CeedInterlaceMode imode = CEED_NONINTERLACED;
-   if (fes.GetOrdering()==Ordering::byVDIM)
-   {
-      imode = CEED_INTERLACED;
-   }
-   CeedElemRestrictionCreate(ceed, imode, mesh->GetNE(), fe->GetDof(),
-                             fes.GetNDofs(), fes.GetVDim(), CEED_MEM_HOST, CEED_COPY_VALUES,
+   CeedElemRestrictionCreate(ceed, mesh->GetNE(), fe->GetDof(), fes.GetVDim(),
+                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
+                             CEED_MEM_HOST, CEED_COPY_VALUES,
                              tp_el_dof.GetData(), restr);
 }
 
@@ -215,6 +228,7 @@ static void InitCeedTensorBasisAndRestriction(const FiniteElementSpace &fes,
                            grad1d.GetData(), qref1d.GetData(),
                            qweight1d.GetData(), basis);
 
+   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
    const Table &el_dof = fes.GetElementToDofTable();
    Array<int> tp_el_dof(el_dof.Size_of_connections());
    for (int i = 0; i < mesh->GetNE(); i++)
@@ -222,16 +236,20 @@ static void InitCeedTensorBasisAndRestriction(const FiniteElementSpace &fes,
       const int el_offset = fe->GetDof() * i;
       for (int j = 0; j < fe->GetDof(); j++)
       {
-         tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
+         if (compstride == 1)
+         {
+            tp_el_dof[j + el_offset] = fes.GetVDim()*
+                                       el_dof.GetJ()[dof_map[j] + el_offset];
+         }
+         else
+         {
+            tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
+         }
       }
    }
-   CeedInterlaceMode imode = CEED_NONINTERLACED;
-   if (fes.GetOrdering()==Ordering::byVDIM)
-   {
-      imode = CEED_INTERLACED;
-   }
-   CeedElemRestrictionCreate(ceed, imode, mesh->GetNE(), fe->GetDof(),
-                             fes.GetNDofs(), fes.GetVDim(), CEED_MEM_HOST, CEED_COPY_VALUES,
+   CeedElemRestrictionCreate(ceed, mesh->GetNE(), fe->GetDof(), fes.GetVDim(),
+                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
+                             CEED_MEM_HOST, CEED_COPY_VALUES,
                              tp_el_dof.GetData(), restr);
 }
 
@@ -298,8 +316,9 @@ void CeedPAAssemble(const CeedPAOperator& op,
    CeedBasisGetNumQuadraturePoints(ceedData.basis, &nqpts);
 
    const int qdatasize = op.qdatasize;
-   CeedElemRestrictionCreateStrided(ceed, nelem, nqpts, nelem*nqpts, qdatasize,
-                                    CEED_STRIDES_BACKEND, &ceedData.restr_i);
+   CeedElemRestrictionCreateStrided(ceed, nelem, nqpts, qdatasize,
+                                    nelem*nqpts*qdatasize, CEED_STRIDES_BACKEND,
+                                    &ceedData.restr_i);
 
    CeedVectorCreate(ceed, mesh->GetNodes()->Size(), &ceedData.node_coords);
    CeedVectorSetArray(ceedData.node_coords, CEED_MEM_HOST, CEED_USE_POINTER,

fem/lininteg.cpp

@@ -159,10 +159,13 @@ void BoundaryLFIntegrator::AssembleRHSElementVect(
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
-      IntegrationPoint eip;
-      Tr.Loc1.Transform(ip, eip);
 
-      Tr.Face->SetIntPoint (&ip);
+      // Set the integration point in the face and the neighboring element
+      Tr.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
+
       double val = Tr.Face->Weight() * ip.weight * Q.Eval(*Tr.Face, ip);
 
       el.CalcShape(eip, shape);
@@ -359,10 +362,12 @@ void VectorBoundaryLFIntegrator::AssembleRHSElementVect(
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
-      IntegrationPoint eip;
-      Tr.Loc1.Transform(ip, eip);
 
-      Tr.SetIntPoint(&ip);
+      // Set the integration point in the face and the neighboring element
+      Tr.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
 
       // Use Tr transformation in case Q depends on boundary attribute
       Q.Eval(vec, Tr, ip);
@@ -683,11 +688,13 @@ void BoundaryFlowIntegrator::AssembleRHSElementVect(
    for (int p = 0; p < ir->GetNPoints(); p++)
    {
       const IntegrationPoint &ip = ir->IntPoint(p);
-      IntegrationPoint eip;
-      Tr.Loc1.Transform(ip, eip);
-      el.CalcShape(eip, shape);
 
-      Tr.SetIntPoint(&ip);
+      // Set the integration point in the face and the neighboring element
+      Tr.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
+      el.CalcShape(eip, shape);
 
       // Use Tr.Elem1 transformation for u so that it matches the coefficient
       // used with the ConvectionIntegrator and/or the DGTraceIntegrator.
@@ -752,10 +759,13 @@ void DGDirichletLFIntegrator::AssembleRHSElementVect(
    for (int p = 0; p < ir->GetNPoints(); p++)
    {
       const IntegrationPoint &ip = ir->IntPoint(p);
-      IntegrationPoint eip;
 
-      Tr.Loc1.Transform(ip, eip);
-      Tr.SetIntPoint(&ip);
+      // Set the integration point in the face and the neighboring element
+      Tr.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
+
       if (dim == 1)
       {
          nor(0) = 2*eip.x - 1.0;
@@ -774,14 +784,14 @@ void DGDirichletLFIntegrator::AssembleRHSElementVect(
       {
          if (Q)
          {
-            w *= Q->Eval(Tr, ip);
+            w *= Q->Eval(*Tr.Elem1, eip);
          }
          ni.Set(w, nor);
       }
       else
       {
          nh.Set(w, nor);
-         MQ->Eval(mq, Tr, ip);
+         MQ->Eval(mq, *Tr.Elem1, eip);
          mq.MultTranspose(nh, ni);
       }
       CalcAdjugate(Tr.Elem1->Jacobian(), adjJ);
@@ -845,9 +855,12 @@ void DGElasticityDirichletLFIntegrator::AssembleRHSElementVect(
    for (int pi = 0; pi < ir->GetNPoints(); ++pi)
    {
       const IntegrationPoint &ip = ir->IntPoint(pi);
-      IntegrationPoint eip;
-      Tr.Loc1.Transform(ip, eip);
-      Tr.SetIntPoint(&ip);
+
+      // Set the integration point in the face and the neighboring element
+      Tr.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
 
       // Evaluate the Dirichlet b.c. using the face transformation.
       uD.Eval(u_dir, Tr, ip);

fem/pbilinearform.cpp

@@ -198,8 +198,9 @@ void ParBilinearForm::AssembleSharedFaces(int skip_zeros)
    for (int i = 0; i < nfaces; i++)
    {
       T = pmesh->GetSharedFaceTransformations(i);
+      int Elem2NbrNo = T->Elem2No - pmesh->GetNE();
       pfes->GetElementVDofs(T->Elem1No, vdofs1);
-      pfes->GetFaceNbrElementVDofs(T->Elem2No, vdofs2);
+      pfes->GetFaceNbrElementVDofs(Elem2NbrNo, vdofs2);
       vdofs1.Copy(vdofs_all);
       for (int j = 0; j < vdofs2.Size(); j++)
       {
@@ -216,7 +217,7 @@ void ParBilinearForm::AssembleSharedFaces(int skip_zeros)
       for (int k = 0; k < fbfi.Size(); k++)
       {
          fbfi[k]->AssembleFaceMatrix(*pfes->GetFE(T->Elem1No),
-                                     *pfes->GetFaceNbrFE(T->Elem2No),
+                                     *pfes->GetFaceNbrFE(Elem2NbrNo),
                                      *T, elemmat);
          if (keep_nbr_block)
          {

fem/pfespace.hpp

@@ -347,6 +347,8 @@ public:
    const FiniteElement *GetFaceNbrFE(int i) const;
    const FiniteElement *GetFaceNbrFaceFE(int i) const;
    const HYPRE_Int *GetFaceNbrGlobalDofMap() { return face_nbr_glob_dof_map; }
+   ElementTransformation *GetFaceNbrElementTransformation(int i) const
+   { return pmesh->GetFaceNbrElementTransformation(i); }
 
    void Lose_Dof_TrueDof_Matrix();
    void LoseDofOffsets() { dof_offsets.LoseData(); }

fem/pgridfunc.cpp

@@ -214,7 +214,7 @@ void ParGridFunction::ExchangeFaceNbrData()
    ParMesh *pmesh = pfes->GetParMesh();
 
    face_nbr_data.SetSize(pfes->GetFaceNbrVSize());
-   Vector send_data(pfes->send_face_nbr_ldof.Size_of_connections());
+   send_data.SetSize(pfes->send_face_nbr_ldof.Size_of_connections());
 
    int *send_offset = pfes->send_face_nbr_ldof.GetI();
    const int *d_send_ldof = mfem::Read(pfes->send_face_nbr_ldof.GetJMemory(),
@@ -271,6 +271,7 @@ const
    {
       int fes_vdim = pfes->GetVDim();
       pfes->GetFaceNbrElementVDofs(nbr_el_no, dofs);
+      const FiniteElement *fe = pfes->GetFaceNbrFE(nbr_el_no);
       if (fes_vdim > 1)
       {
          int s = dofs.Size()/fes_vdim;
@@ -283,7 +284,17 @@ const
          face_nbr_data.GetSubVector(dofs, LocVec);
          DofVal.SetSize(dofs.Size());
       }
-      pfes->GetFaceNbrFE(nbr_el_no)->CalcShape(ip, DofVal);
+      if (fe->GetMapType() == FiniteElement::VALUE)
+      {
+         fe->CalcShape(ip, DofVal);
+      }
+      else
+      {
+         ElementTransformation *Tr =
+            pfes->GetFaceNbrElementTransformation(nbr_el_no);
+         Tr->SetIntPoint(&ip);
+         fe->CalcPhysShape(*Tr, DofVal);
+      }
    }
    else
    {
@@ -291,14 +302,175 @@ const
       fes->DofsToVDofs(vdim-1, dofs);
       DofVal.SetSize(dofs.Size());
       const FiniteElement *fe = fes->GetFE(i);
-      MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE, "invalid FE map type");
-      fe->CalcShape(ip, DofVal);
+      if (fe->GetMapType() == FiniteElement::VALUE)
+      {
+         fe->CalcShape(ip, DofVal);
+      }
+      else
+      {
+         ElementTransformation *Tr = fes->GetElementTransformation(i);
+         Tr->SetIntPoint(&ip);
+         fe->CalcPhysShape(*Tr, DofVal);
+      }
       GetSubVector(dofs, LocVec);
    }
 
    return (DofVal * LocVec);
 }
 
+void ParGridFunction::GetVectorValue(int i, const IntegrationPoint &ip,
+                                     Vector &val) const
+{
+   int nbr_el_no = i - pfes->GetParMesh()->GetNE();
+   if (nbr_el_no >= 0)
+   {
+      Array<int> dofs;
+      pfes->GetFaceNbrElementVDofs(nbr_el_no, dofs);
+      Vector loc_data;
+      face_nbr_data.GetSubVector(dofs, loc_data);
+      const FiniteElement *FElem = pfes->GetFaceNbrFE(nbr_el_no);
+      int dof = FElem->GetDof();
+      if (FElem->GetRangeType() == FiniteElement::SCALAR)
+      {
+         Vector shape(dof);
+         if (FElem->GetMapType() == FiniteElement::VALUE)
+         {
+            FElem->CalcShape(ip, shape);
+         }
+         else
+         {
+            ElementTransformation *Tr =
+               pfes->GetParMesh()->GetFaceNbrElementTransformation(nbr_el_no);
+            Tr->SetIntPoint(&ip);
+            FElem->CalcPhysShape(*Tr, shape);
+         }
+         int vdim = fes->GetVDim();
+         val.SetSize(vdim);
+         for (int k = 0; k < vdim; k++)
+         {
+            val(k) = shape * ((const double *)loc_data + dof * k);
+         }
+      }
+      else
+      {
+         int spaceDim = fes->GetMesh()->SpaceDimension();
+         DenseMatrix vshape(dof, spaceDim);
+         ElementTransformation *Tr =
+            pfes->GetParMesh()->GetFaceNbrElementTransformation(nbr_el_no);
+         Tr->SetIntPoint(&ip);
+         FElem->CalcVShape(*Tr, vshape);
+         val.SetSize(spaceDim);
+         vshape.MultTranspose(loc_data, val);
+      }
+   }
+   else
+   {
+      GridFunction::GetVectorValue(i, ip, val);
+   }
+}
+
+double ParGridFunction::GetValue(ElementTransformation &T,
+                                 const IntegrationPoint &ip,
+                                 int comp, Vector *tr) const
+{
+   // We can assume faces and edges are local
+   if (T.ElementType != ElementTransformation::ELEMENT)
+   {
+      return GridFunction::GetValue(T, ip, comp, tr);
+   }
+
+   // Check for evaluation in a local element
+   int nbr_el_no = T.ElementNo - pfes->GetParMesh()->GetNE();
+   if (nbr_el_no < 0)
+   {
+      return GridFunction::GetValue(T, ip, comp, tr);
+   }
+
+   // Evaluate using DoFs from a neighboring element
+   if (tr)
+   {
+      T.SetIntPoint(&ip);
+      T.Transform(ip, *tr);
+   }
+
+   Array<int> dofs;
+   const FiniteElement * fe = pfes->GetFaceNbrFE(nbr_el_no);
+   pfes->GetFaceNbrElementVDofs(nbr_el_no, dofs);
+
+   pfes->DofsToVDofs(comp-1, dofs);
+   Vector DofVal(dofs.Size()), LocVec;
+   if (fe->GetMapType() == FiniteElement::VALUE)
+   {
+      fe->CalcShape(ip, DofVal);
+   }
+   else
+   {
+      fe->CalcPhysShape(T, DofVal);
+   }
+   face_nbr_data.GetSubVector(dofs, LocVec);
+
+   return (DofVal * LocVec);
+}
+
+void ParGridFunction::GetVectorValue(ElementTransformation &T,
+                                     const IntegrationPoint &ip,
+                                     Vector &val, Vector *tr) const
+{
+   // We can assume faces and edges are local
+   if (T.ElementType != ElementTransformation::ELEMENT)
+   {
+      return GridFunction::GetVectorValue(T, ip, val, tr);
+   }
+
+   // Check for evaluation in a local element
+   int nbr_el_no = T.ElementNo - pfes->GetParMesh()->GetNE();
+   if (nbr_el_no < 0)
+   {
+      return GridFunction::GetVectorValue(T, ip, val, tr);
+   }
+
+   // Evaluate using DoFs from a neighboring element
+   if (tr)
+   {
+      T.SetIntPoint(&ip);
+      T.Transform(ip, *tr);
+   }
+
+   Array<int> vdofs;
+   pfes->GetFaceNbrElementVDofs(nbr_el_no, vdofs);
+   const FiniteElement *fe = pfes->GetFaceNbrFE(nbr_el_no);
+
+   int dof = fe->GetDof();
+   Vector loc_data;
+   face_nbr_data.GetSubVector(vdofs, loc_data);
+   if (fe->GetRangeType() == FiniteElement::SCALAR)
+   {
+      Vector shape(dof);
+      if (fe->GetMapType() == FiniteElement::VALUE)
+      {
+         fe->CalcShape(ip, shape);
+      }
+      else
+      {
+         fe->CalcPhysShape(T, shape);
+      }
+      int vdim = pfes->GetVDim();
+      val.SetSize(vdim);
+      for (int k = 0; k < vdim; k++)
+      {
+         val(k) = shape * ((const double *)loc_data + dof * k);
+      }
+   }
+   else
+   {
+      int spaceDim = pfes->GetMesh()->SpaceDimension();
+      DenseMatrix vshape(dof, spaceDim);
+      fe->CalcVShape(T, vshape);
+      val.SetSize(spaceDim);
+      vshape.MultTranspose(loc_data, val);
+   }
+}
+
 void ParGridFunction::ProjectCoefficient(Coefficient &coeff)
 {
    DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);

fem/pgridfunc.hpp

@@ -38,6 +38,11 @@ protected:
        initialized by ExchangeFaceNbrData(). */
    Vector face_nbr_data;
 
+   /** @brief Vector used as an MPI buffer to send face-neighbor data
+       in ExchangeFaceNbrData() to neighboring processors. */
+   //TODO: Use temporary memory to avoid CUDA malloc allocation cost.
+   Vector send_data;
+
    void ProjectBdrCoefficient(Coefficient *coeff[], VectorCoefficient *vcoeff,
                               Array<int> &attr);
 
@@ -204,6 +209,18 @@ public:
    double GetValue(ElementTransformation &T)
    { return GetValue(T.ElementNo, T.GetIntPoint()); }
 
+   // Redefine to handle the case when T describes a face-neighbor element
+   virtual double GetValue(ElementTransformation &T, const IntegrationPoint &ip,
+                           int comp = 0, Vector *tr = NULL) const;
+
+   virtual void GetVectorValue(int i, const IntegrationPoint &ip,
+                               Vector &val) const;
+
+   // Redefine to handle the case when T describes a face-neighbor element
+   virtual void GetVectorValue(ElementTransformation &T,
+                               const IntegrationPoint &ip,
+                               Vector &val, Vector *tr = NULL) const;
+
    using GridFunction::ProjectCoefficient;
    virtual void ProjectCoefficient(Coefficient &coeff);
 

fem/pnonlinearform.cpp

@@ -64,12 +64,13 @@ void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
       for (int i = 0; i < n_shared_faces; i++)
       {
          tr = pmesh->GetSharedFaceTransformations(i, true);
+         int Elem2NbrNo = tr->Elem2No - pmesh->GetNE();
 
          fe1 = pfes->GetFE(tr->Elem1No);
-         fe2 = pfes->GetFaceNbrFE(tr->Elem2No);
+         fe2 = pfes->GetFaceNbrFE(Elem2NbrNo);
 
          pfes->GetElementVDofs(tr->Elem1No, vdofs1);
-         pfes->GetFaceNbrElementVDofs(tr->Elem2No, vdofs2);
+         pfes->GetFaceNbrElementVDofs(Elem2NbrNo, vdofs2);
 
          el_x.SetSize(vdofs1.Size() + vdofs2.Size());
          X.GetSubVector(vdofs1, el_x.GetData());

fem/tmop.cpp

@@ -177,24 +177,6 @@ double TMOP_Metric_SSA2D::EvalW(const DenseMatrix &Jpt) const
    return Mat.FNorm2();
 }
 
-// mu_85 = |T-T'|^2, where T'= |T|*I/sqrt(2)
-double TMOP_Metric_SS2D::EvalW(const DenseMatrix &Jpt) const
-{
-   MFEM_VERIFY(Jtr != NULL,
-               "Requires a target Jacobian, use SetTargetJacobian().");
-
-   DenseMatrix Id(2,2);
-   DenseMatrix Mat(2,2);
-   Mat = Jpt;
-
-   Id(0,0) = 1; Id(0,1) = 0;
-   Id(1,0) = 0; Id(1,1) = 1;
-   Id *= Mat.FNorm()/pow(2,0.5);
-
-   Mat.Add(-1.,Id);
-   return Mat.FNorm2();
-}
-
 double TMOP_Metric_002::EvalW(const DenseMatrix &Jpt) const
 {
    ie.SetJacobian(Jpt.GetData());
@@ -484,6 +466,24 @@ void TMOP_Metric_077::AssembleH(const DenseMatrix &Jpt,
    ie.Assemble_TProd(weight * I2inv_sq / I2, ie.Get_dI2(), A.GetData());
 }
 
+// mu_85 = |T-T'|^2, where T'= |T|*I/sqrt(2)
+double TMOP_Metric_085::EvalW(const DenseMatrix &Jpt) const
+{
+   MFEM_VERIFY(Jtr != NULL,
+               "Requires a target Jacobian, use SetTargetJacobian().");
+
+   DenseMatrix Id(2,2);
+   DenseMatrix Mat(2,2);
+   Mat = Jpt;
+
+   Id(0,0) = 1; Id(0,1) = 0;
+   Id(1,0) = 0; Id(1,1) = 1;
+   Id *= Mat.FNorm()/pow(2,0.5);
+
+   Mat.Add(-1.,Id);
+   return Mat.FNorm2();
+}
+
 double TMOP_Metric_211::EvalW(const DenseMatrix &Jpt) const
 {
    // mu_211 = (det(J) - 1)^2 - det(J) + (det(J)^2 + eps)^{1/2}

fem/tmop.hpp

@@ -162,21 +162,6 @@ public:
    { MFEM_ABORT("Not implemented"); }
 };
 
-/// Shape+Size metric, 2D.
-class TMOP_Metric_SS2D : public TMOP_QualityMetric
-{
-public:
-   // W = 0.5 (1 - cos(theta_Jpr - theta_Jtr)).
-   virtual double EvalW(const DenseMatrix &Jpt) const;
-
-   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
-   { MFEM_ABORT("Not implemented"); }
-
-   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
-                          const double weight, DenseMatrix &A) const
-   { MFEM_ABORT("Not implemented"); }
-};
-
 /// Shape, ideal barrier metric, 2D
 class TMOP_Metric_002 : public TMOP_QualityMetric
 {
@@ -331,6 +316,21 @@ public:
 
 };
 
+/// Shape & orientation metric, 2D.
+class TMOP_Metric_085 : public TMOP_QualityMetric
+{
+public:
+   // W = |T-T'|^2, where T'= |T|*I/sqrt(2).
+   virtual double EvalW(const DenseMatrix &Jpt) const;
+
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+   { MFEM_ABORT("Not implemented"); }
+
+   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                          const double weight, DenseMatrix &A) const
+   { MFEM_ABORT("Not implemented"); }
+};
+
 /// Untangling metric, 2D
 class TMOP_Metric_211 : public TMOP_QualityMetric
 {

fem/tmop_tools.cpp

@@ -353,20 +353,59 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       energy_in = nlf->GetEnergy(x);
    }
 
-   const bool have_b = (b.Size() == Height());
-
    const int NE = fes->GetMesh()->GetNE(), dim = fes->GetFE(0)->GetDim(),
              dof = fes->GetFE(0)->GetDof(), nsp = ir.GetNPoints();
    Array<int> xdofs(dof * dim);
    DenseMatrix Jpr(dim), dshape(dof, dim), pos(dof, dim);
    Vector posV(pos.Data(), dof * dim);
+   Vector x_out_loc(fes->GetVSize());
 
-   Vector x_out(x.Size()), x_out_loc(fes->GetVSize());
+   if (serial)
+   {
+      const SparseMatrix *cP = fes->GetConformingProlongation();
+      if (!cP) { x_out_loc = x; }
+      else     { cP->Mult(x, x_out_loc); }
+   }
+#ifdef MFEM_USE_MPI
+   else
+   {
+      fes->GetProlongationMatrix()->Mult(x, x_out_loc);
+   }
+#endif
+
+   double min_detJ = infinity();
+   for (int i = 0; i < NE; i++)
+   {
+      fes->GetElementVDofs(i, xdofs);
+      x_out_loc.GetSubVector(xdofs, posV);
+
+      for (int j = 0; j < nsp; j++)
+      {
+         fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
+         MultAtB(pos, dshape, Jpr);
+         min_detJ = std::min(min_detJ, Jpr.Det());
+      }
+   }
+   double min_detJ_all = min_detJ;
+#ifdef MFEM_USE_MPI
+   if (parallel)
+   {
+      MPI_Allreduce(&min_detJ, &min_detJ_all, 1, MPI_DOUBLE, MPI_MIN,
+                    p_nlf->ParFESpace()->GetComm());
+   }
+#endif
+   bool untangling = false;
+   if (min_detJ_all <= 0) { untangling = true; }
+
+   const bool have_b = (b.Size() == Height());
+
+   Vector x_out(x.Size());
    bool x_out_ok = false;
    double scale = 1.0, energy_out = 0.0;
    double norm0 = Norm(r);
 
-   // Decreases the scaling of the update until the new mesh is valid.
+   const double detJ_factor = (solver_type == 1) ? 0.25 : 0.5;
+
    for (int i = 0; i < 12; i++)
    {
       add(x, -scale, c, x_out);
@@ -384,35 +423,39 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       }
 #endif
 
-      int jac_ok = 1;
-      for (int i = 0; i < NE; i++)
+      // Check det(Jpr) > 0.
+      if (!untangling)
       {
-         fes->GetElementVDofs(i, xdofs);
-         x_out_loc.GetSubVector(xdofs, posV);
-         for (int j = 0; j < nsp; j++)
+         int jac_ok = 1;
+         for (int i = 0; i < NE; i++)
          {
-            fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
-            MultAtB(pos, dshape, Jpr);
-            if (Jpr.Det() <= 0.0) { jac_ok = 0; goto break2; }
+            fes->GetElementVDofs(i, xdofs);
+            x_out_loc.GetSubVector(xdofs, posV);
+            for (int j = 0; j < nsp; j++)
+            {
+               fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
+               MultAtB(pos, dshape, Jpr);
+               if (Jpr.Det() <= 0.0) { jac_ok = 0; goto break2; }
+            }
          }
-      }
 
-   break2:
-      int jac_ok_all = jac_ok;
+      break2:
+         int jac_ok_all = jac_ok;
 #ifdef MFEM_USE_MPI
-      if (parallel)
-      {
-         MPI_Allreduce(&jac_ok, &jac_ok_all, 1, MPI_INT, MPI_LAND,
-                       p_nlf->ParFESpace()->GetComm());
-      }
+         if (parallel)
+         {
+            MPI_Allreduce(&jac_ok, &jac_ok_all, 1, MPI_INT, MPI_LAND,
+                          p_nlf->ParFESpace()->GetComm());
+         }
 #endif
 
-      if (jac_ok_all == 0)
-      {
-         if (print_level >= 0)
-         { mfem::out << "Scale = " << scale << " Neg det(J) found.\n"; }
-         scale *= 0.5; continue;
-      }
+         if (jac_ok_all == 0)
+         {
+            if (print_level >= 0)
+            { mfem::out << "Scale = " << scale << " Neg det(J) found.\n"; }
+            scale *= detJ_factor; continue;
+         }
+      } // endif(!untangling)
 
       ProcessNewState(x_out);
       if (serial)
@@ -425,25 +468,37 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
          energy_out = p_nlf->GetParGridFunctionEnergy(x_out_loc);
       }
 #endif
-      if (energy_out > 1.2*energy_in || std::isnan(energy_out) != 0)
-      {
-         if (print_level >= 0)
-         { mfem::out << "Scale = " << scale << " Increasing energy.\n"; }
-         scale *= 0.5; continue;
-      }
 
-      oper->Mult(x_out, r);
-      if (have_b) { r -= b; }
-      double norm = Norm(r);
-
-      if (norm > 1.2*norm0)
+      if (untangling)
       {
-         if (print_level >= 0)
-         { mfem::out << "Scale = " << scale << " Norm increased.\n"; }
-         scale *= 0.5; continue;
+         if (energy_out > energy_in || std::isnan(energy_out) != 0)
+         {
+            scale *= 0.5;
+         }
+         else { x_out_ok = true; break; }
       }
-      else { x_out_ok = true; break; }
-   }
+      else
+      {
+         if (energy_out > 1.2*energy_in || std::isnan(energy_out) != 0)
+         {
+            if (print_level >= 0)
+            { mfem::out << "Scale = " << scale << " Increasing energy.\n"; }
+            scale *= 0.5; continue;
+         }
+
+         oper->Mult(x_out, r);
+         if (have_b) { r -= b; }
+         double norm = Norm(r);
+
+         if (norm > 1.2*norm0)
+         {
+            if (print_level >= 0)
+            { mfem::out << "Scale = " << scale << " Norm increased.\n"; }
+            scale *= 0.5; continue;
+         }
+         else { x_out_ok = true; break; }
+      } // endif (untangling)
+   } // enddo (i)
 
    if (print_level >= 0)
    {
@@ -570,105 +625,6 @@ void TMOPNewtonSolver::UpdateDiscreteTC(const TMOP_Integrator &ti,
    }
 }
 
-double TMOPDescentNewtonSolver::ComputeScalingFactor(const Vector &x,
-                                                     const Vector &b) const
-{
-   const FiniteElementSpace *fes = NULL;
-   double energy_in = 0.0;
-#ifdef MFEM_USE_MPI
-   const ParNonlinearForm *p_nlf = dynamic_cast<const ParNonlinearForm *>(oper);
-   MFEM_VERIFY(!(parallel && p_nlf == NULL), "Invalid Operator subclass.");
-   if (parallel)
-   {
-      fes = p_nlf->FESpace();
-      energy_in = p_nlf->GetEnergy(x);
-   }
-#endif
-   const bool serial = !parallel;
-   const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
-   MFEM_VERIFY(!(serial && nlf == NULL), "Invalid Operator subclass.");
-   if (serial)
-   {
-      fes = nlf->FESpace();
-      energy_in = nlf->GetEnergy(x);
-   }
-
-   const int NE = fes->GetMesh()->GetNE(), dim = fes->GetFE(0)->GetDim(),
-             dof = fes->GetFE(0)->GetDof(), nsp = ir.GetNPoints();
-   Array<int> xdofs(dof * dim);
-   DenseMatrix Jpr(dim), dshape(dof, dim), pos(dof, dim);
-   Vector posV(pos.Data(), dof * dim);
-   Vector x_loc(fes->GetVSize());
-
-   double min_detJ = infinity();
-   for (int i = 0; i < NE; i++)
-   {
-      fes->GetElementVDofs(i, xdofs);
-      // TODO x_loc doesn't have valid values here!
-      MFEM_ABORT("This function has to be fixed!");
-      x_loc.GetSubVector(xdofs, posV);
-
-      for (int j = 0; j < nsp; j++)
-      {
-         fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
-         MultAtB(pos, dshape, Jpr);
-         min_detJ = std::min(min_detJ, Jpr.Det());
-      }
-   }
-   double min_detJ_all = min_detJ;
-#ifdef MFEM_USE_MPI
-   if (parallel)
-   {
-      MPI_Allreduce(&min_detJ, &min_detJ_all, 1, MPI_DOUBLE, MPI_MIN,
-                    p_nlf->ParFESpace()->GetComm());
-   }
-#endif
-   if (print_level >= 0)
-   {
-      mfem::out << "Minimum det(J) = " << min_detJ_all << '\n';
-   }
-
-   Vector x_out(x.Size());
-   bool x_out_ok = false;
-   double scale = 1.0, energy_out = 0.0;
-
-   for (int i = 0; i < 7; i++)
-   {
-      add(x, -scale, c, x_out);
-      if (serial)
-      {
-         const SparseMatrix *cP = fes->GetConformingProlongation();
-         if (!cP) { x_loc = x_out; }
-         else     { cP->Mult(x_out,x_loc); }
-         energy_out = nlf->GetGridFunctionEnergy(x_loc);
-      }
-#ifdef MFEM_USE_MPI
-      else
-      {
-         fes->GetProlongationMatrix()->Mult(x_out, x_loc);
-         energy_out = p_nlf->GetParGridFunctionEnergy(x_loc);
-      }
-#endif
-
-      if (energy_out > energy_in || std::isnan(energy_out) != 0)
-      {
-         scale *= 0.5;
-      }
-      else { x_out_ok = true; break; }
-   }
-
-   if (print_level >= 0)
-   {
-      mfem::out << "Energy decrease: "
-                << (energy_in - energy_out) / energy_in * 100.0
-                << "% with " << scale << " scaling.\n";
-   }
-
-   if (x_out_ok == false) { return 0.0; }
-
-   return scale;
-}
-
 #ifdef MFEM_USE_MPI
 // Metric values are visualized by creating an L2 finite element functions and
 // computing the metric values at the nodes.

fem/tmop_tools.hpp

@@ -109,9 +109,11 @@ public:
 };
 #endif
 
-class TMOPNewtonSolver : public NewtonSolver
+class TMOPNewtonSolver : public LBFGSSolver
 {
 protected:
+   // 0 - Newton, 1 - LBFGS.
+   int solver_type;
    bool parallel;
 
    // Quadrature points that are checked for negative Jacobians etc.
@@ -121,29 +123,42 @@ protected:
 
 public:
 #ifdef MFEM_USE_MPI
-   TMOPNewtonSolver(MPI_Comm comm, const IntegrationRule &irule)
-      : NewtonSolver(comm), parallel(true), ir(irule) { }
+   TMOPNewtonSolver(MPI_Comm comm, const IntegrationRule &irule, int type = 0)
+      : LBFGSSolver(comm), solver_type(type), parallel(true), ir(irule) { }
 #endif
-   TMOPNewtonSolver(const IntegrationRule &irule)
-      : NewtonSolver(), parallel(false), ir(irule) { }
+   TMOPNewtonSolver(const IntegrationRule &irule, int type = 0)
+      : LBFGSSolver(), solver_type(type), parallel(false), ir(irule) { }
 
    virtual double ComputeScalingFactor(const Vector &x, const Vector &b) const;
 
    virtual void ProcessNewState(const Vector &x) const;
-};
 
-/// Allows negative Jacobians. Used for untangling.
-class TMOPDescentNewtonSolver : public TMOPNewtonSolver
-{
-public:
-#ifdef MFEM_USE_MPI
-   TMOPDescentNewtonSolver(MPI_Comm comm, const IntegrationRule &irule)
-      : TMOPNewtonSolver(comm, irule) { }
-#endif
-   TMOPDescentNewtonSolver(const IntegrationRule &irule)
-      : TMOPNewtonSolver(irule) { }
+   virtual void Mult(const Vector &b, Vector &x) const
+   {
+      if (solver_type == 0)
+      {
+         NewtonSolver::Mult(b, x);
+      }
+      else if (solver_type == 1)
+      {
+         LBFGSSolver::Mult(b, x);
+      }
+      else { MFEM_ABORT("Invalid type"); }
+   }
 
-   virtual double ComputeScalingFactor(const Vector &x, const Vector &b) const;
+   virtual void SetSolver(Solver &solver)
+   {
+      if (solver_type == 0)
+      {
+         NewtonSolver::SetSolver(solver);
+      }
+      else if (solver_type == 1)
+      {
+         LBFGSSolver::SetSolver(solver);
+      }
+      else { MFEM_ABORT("Invalid type"); }
+   }
+   virtual void SetPreconditioner(Solver &pr) { SetSolver(pr); }
 };
 
 void vis_tmop_metric_s(int order, TMOP_QualityMetric &qm,

general/mem_manager.hpp

@@ -450,16 +450,16 @@ private:
 
    template <std::size_t align_bytes, bool dummy = true> struct Alloc
    {
+#if __cplusplus < 201703L
       static inline T *New(std::size_t)
       {
-#if __cplusplus < 201703L
          // Generate an error in debug mode
          MFEM_ASSERT(false, "overaligned type cannot use MemoryType::HOST");
          return nullptr;
-#else
-         return new T[size];
-#endif
       }
+#else
+      static inline T *New(std::size_t size) { return new T[size]; }
+#endif
    };
 
 #if __cplusplus < 201703L

linalg/CMakeLists.txt

@@ -62,6 +62,12 @@ if (MFEM_USE_MPI)
       petsc.cpp)
     list(APPEND HDRS
       petsc.hpp)
+    if (MFEM_USE_SLEPC)
+      list(APPEND SRCS
+        slepc.cpp)
+      list(APPEND HDRS
+        slepc.hpp)
+    endif()
   endif()
 endif()
 

linalg/hypre.cpp

@@ -1667,12 +1667,12 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A,
 // Helper function for HypreParMatrixFromBlocks. Note that scalability to
 // extremely large processor counts is limited by the use of MPI_Allgather.
 void GatherBlockOffsetData(MPI_Comm comm, const int rank, const int nprocs,
-                           const int num_loc, Array<int> &offsets,
+                           const int num_loc, const Array<int> &offsets,
                            std::vector<int> &all_num_loc, const int numBlocks,
-                           std::vector<std::vector<int>> &blockProcOffsets,
-                           std::vector<int> &procOffsets,
+                           std::vector<std::vector<HYPRE_Int>> &blockProcOffsets,
+                           std::vector<HYPRE_Int> &procOffsets,
                            std::vector<std::vector<int>> &procBlockOffsets,
-                           int &firstLocal, int &globalNum)
+                           HYPRE_Int &firstLocal, HYPRE_Int &globalNum)
 {
    std::vector<std::vector<int>> all_block_num_loc(numBlocks);
 
@@ -1700,6 +1700,10 @@ void GatherBlockOffsetData(MPI_Comm comm, const int rank, const int nprocs,
    for (int i = 0; i < nprocs; ++i)
    {
       globalNum += all_num_loc[i];
+      if (rank == 0)
+      {
+         MFEM_VERIFY(globalNum >= 0, "overflow in global size");
+      }
       if (i < rank)
       {
          firstLocal += all_num_loc[i];
@@ -1808,14 +1812,14 @@ HypreParMatrix * HypreParMatrixFromBlocks(Array2D<HypreParMatrix*> &blocks,
 
    std::vector<int> all_num_loc_rows(nprocs);
    std::vector<int> all_num_loc_cols(nprocs);
-   std::vector<int> procRowOffsets(nprocs);
-   std::vector<int> procColOffsets(nprocs);
-   std::vector<std::vector<int>> blockRowProcOffsets(numBlockRows);
-   std::vector<std::vector<int>> blockColProcOffsets(numBlockCols);
+   std::vector<HYPRE_Int> procRowOffsets(nprocs);
+   std::vector<HYPRE_Int> procColOffsets(nprocs);
+   std::vector<std::vector<HYPRE_Int>> blockRowProcOffsets(numBlockRows);
+   std::vector<std::vector<HYPRE_Int>> blockColProcOffsets(numBlockCols);
    std::vector<std::vector<int>> procBlockRowOffsets(nprocs);
    std::vector<std::vector<int>> procBlockColOffsets(nprocs);
 
-   int first_loc_row, glob_nrows, first_loc_col, glob_ncols;
+   HYPRE_Int first_loc_row, glob_nrows, first_loc_col, glob_ncols;
    GatherBlockOffsetData(comm, rank, nprocs, num_loc_rows, rowOffsets,
                          all_num_loc_rows, numBlockRows, blockRowProcOffsets,
                          procRowOffsets, procBlockRowOffsets, first_loc_row,
@@ -1850,13 +1854,7 @@ HypreParMatrix * HypreParMatrixFromBlocks(Array2D<HypreParMatrix*> &blocks,
          }
          else
          {
-            {
-               hypre_ParCSRMatrix *parcsr_op = (hypre_ParCSRMatrix*)
-                                               const_cast<HypreParMatrix&>
-                                               (*(blocks(i, j)));
-               MFEM_ASSERT(parcsr_op != NULL, "const_cast failed");
-               csr_blocks(i, j) = hypre_MergeDiagAndOffd(parcsr_op);
-            }
+            csr_blocks(i, j) = hypre_MergeDiagAndOffd(*blocks(i, j));
 
             for (int k = 0; k < csr_blocks(i, j)->num_rows; ++k)
             {
@@ -1887,6 +1885,9 @@ HypreParMatrix * HypreParMatrixFromBlocks(Array2D<HypreParMatrix*> &blocks,
          {
             const int nrows = csr_blocks(i, j)->num_rows;
             const double cij = blockCoeff ? (*blockCoeff)(i, j) : 1.0;
+#if MFEM_HYPRE_VERSION >= 21600
+            const bool usingBigJ = (csr_blocks(i, j)->big_j != NULL);
+#endif
 
             for (int k = 0; k < nrows; ++k)
             {
@@ -1897,21 +1898,19 @@ HypreParMatrix * HypreParMatrixFromBlocks(Array2D<HypreParMatrix*> &blocks,
                for (int l = 0; l < nnz_k; ++l)
                {
                   // Find the column process offset for the block.
-                  const int bcol = csr_blocks(i, j)->j[osk + l];
-                  int bcolproc = 0;
+#if MFEM_HYPRE_VERSION >= 21600
+                  const HYPRE_Int bcol = usingBigJ ?
+                                         csr_blocks(i, j)->big_j[osk + l] :
+                                         csr_blocks(i, j)->j[osk + l];
+#else
+                  const HYPRE_Int bcol = csr_blocks(i, j)->j[osk + l];
+#endif
 
-                  for (int p = 1; p < nprocs; ++p)
-                  {
-                     if (blockColProcOffsets[j][p] > bcol)
-                     {
-                        bcolproc = p - 1;
-                        break;
-                     }
-                  }
-                  if (blockColProcOffsets[j][nprocs - 1] <= bcol)
-                  {
-                     bcolproc = nprocs - 1;
-                  }
+                  // find the processor 'bcolproc' that holds column 'bcol':
+                  const auto &offs = blockColProcOffsets[j];
+                  const int bcolproc =
+                     std::upper_bound(offs.begin() + 1, offs.end(), bcol)
+                     - offs.begin() - 1;
 
                   opJ[opI[rowg] + cnt[rowg]] = procColOffsets[bcolproc] +
                                                procBlockColOffsets[bcolproc][j]
@@ -1944,11 +1943,12 @@ HypreParMatrix * HypreParMatrixFromBlocks(Array2D<HypreParMatrix*> &blocks,
    colStarts2[0] = first_loc_col;
    colStarts2[1] = first_loc_col + all_num_loc_cols[rank];
 
+   MFEM_VERIFY(HYPRE_AssumedPartitionCheck(),
+               "only 'assumed partition' mode is supported");
+
    return new HypreParMatrix(comm, num_loc_rows, glob_nrows, glob_ncols,
-                             (int *)opI.data(), (HYPRE_Int *)opJ.data(),
-                             (double *)data.data(),
-                             (HYPRE_Int *)rowStarts2.data(),
-                             (HYPRE_Int *)colStarts2.data());
+                             opI.data(), opJ.data(), data.data(),
+                             rowStarts2.data(), colStarts2.data());
 }
 
 void EliminateBC(HypreParMatrix &A, HypreParMatrix &Ae,

linalg/hypre.hpp

@@ -577,7 +577,7 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A,
     each process remain on that process in the resulting matrix. Some blocks can
     be NULL. Each block and the entire system can be rectangular. Scalability to
     extremely large processor counts is limited by global MPI communication, see
-    GatherBlockOffsetData in hypre.cpp. */
+    GatherBlockOffsetData() in hypre.cpp. */
 HypreParMatrix * HypreParMatrixFromBlocks(Array2D<HypreParMatrix*> &blocks,
                                           Array2D<double> *blockCoeff=NULL);
 

linalg/linalg.hpp

@@ -49,6 +49,10 @@
 #include "petsc.hpp"
 #endif
 
+#ifdef MFEM_USE_SLEPC
+#include "slepc.hpp"
+#endif
+
 #ifdef MFEM_USE_SUPERLU
 #include "superlu.hpp"
 #endif

linalg/petsc.cpp

@@ -37,25 +37,7 @@
 
 // Note: there are additional #include statements below.
 
-// Error handling
-// Prints PETSc's stacktrace and then calls MFEM_ABORT
-// We cannot use PETSc's CHKERRQ since it returns a PetscErrorCode
-#define PCHKERRQ(obj,err) do {                                                   \
-     if ((err))                                                                  \
-     {                                                                           \
-        PetscError(PetscObjectComm((PetscObject)(obj)),__LINE__,_MFEM_FUNC_NAME, \
-                   __FILE__,(err),PETSC_ERROR_REPEAT,NULL);                      \
-        MFEM_ABORT("Error in PETSc. See stacktrace above.");                     \
-     }                                                                           \
-  } while(0);
-#define CCHKERRQ(comm,err) do {                                \
-     if ((err))                                                \
-     {                                                         \
-        PetscError(comm,__LINE__,_MFEM_FUNC_NAME,              \
-                   __FILE__,(err),PETSC_ERROR_REPEAT,NULL);    \
-        MFEM_ABORT("Error in PETSc. See stacktrace above.");   \
-     }                                                         \
-  } while(0);
+#include "petscinternals.hpp"
 
 // Callback functions: these functions will be called by PETSc
 static PetscErrorCode __mfem_ts_monitor(TS,PetscInt,PetscReal,Vec,void*);

linalg/petscinternals.hpp

@@ -0,0 +1,38 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_PETSCINTERNALS
+#define MFEM_PETSCINTERNALS
+
+#include "../general/error.hpp"
+#include "petsc.h"
+
+// Error handling
+// Prints PETSc's stacktrace and then calls MFEM_ABORT
+// We cannot use PETSc's CHKERRQ since it returns a PetscErrorCode
+#define PCHKERRQ(obj,err) do {                                                   \
+     if ((err))                                                                  \
+     {                                                                           \
+        PetscError(PetscObjectComm((PetscObject)(obj)),__LINE__,_MFEM_FUNC_NAME, \
+                   __FILE__,(err),PETSC_ERROR_REPEAT,NULL);                      \
+        MFEM_ABORT("Error in PETSc. See stacktrace above.");                     \
+     }                                                                           \
+  } while(0);
+#define CCHKERRQ(comm,err) do {                                \
+     if ((err))                                                \
+     {                                                         \
+        PetscError(comm,__LINE__,_MFEM_FUNC_NAME,              \
+                   __FILE__,(err),PETSC_ERROR_REPEAT,NULL);    \
+        MFEM_ABORT("Error in PETSc. See stacktrace above.");   \
+     }                                                         \
+  } while(0);
+
+#endif

linalg/slepc.cpp

@@ -0,0 +1,248 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_MPI
+#ifdef MFEM_USE_PETSC
+#ifdef MFEM_USE_SLEPC
+
+#include "linalg.hpp"
+
+#include "slepc.h"
+
+#include "petscinternals.hpp"
+
+static PetscErrorCode ierr;
+
+namespace mfem
+{
+
+void MFEMInitializeSlepc()
+{
+   MFEMInitializeSlepc(NULL,NULL,NULL,NULL);
+}
+
+void MFEMInitializeSlepc(int *argc,char*** argv)
+{
+   MFEMInitializeSlepc(argc,argv,NULL,NULL);
+}
+
+void MFEMInitializeSlepc(int *argc,char ***argv,const char rc_file[],
+                         const char help[])
+{
+   ierr = SlepcInitialize(argc,argv,rc_file,help);
+   MFEM_VERIFY(!ierr,"Unable to initialize SLEPc");
+}
+
+void MFEMFinalizeSlepc()
+{
+   ierr = SlepcFinalize();
+   MFEM_VERIFY(!ierr,"Unable to finalize SLEPc");
+}
+
+
+SlepcEigenSolver::SlepcEigenSolver(MPI_Comm comm, const std::string &prefix)
+{
+   clcustom = false;
+   VR = NULL;
+   VC = NULL;
+
+   ierr = EPSCreate(comm,&eps); CCHKERRQ(comm,ierr);
+   ierr = EPSSetOptionsPrefix(eps, prefix.c_str()); PCHKERRQ(eps, ierr);
+}
+
+SlepcEigenSolver::~SlepcEigenSolver()
+{
+   MPI_Comm comm;
+   ierr = PetscObjectGetComm((PetscObject)eps,&comm); PCHKERRQ(eps,ierr);
+   ierr = EPSDestroy(&eps); CCHKERRQ(comm,ierr);
+}
+
+
+void SlepcEigenSolver::SetOperator(const PetscParMatrix &op)
+{
+   delete VR;
+   delete VC;
+   VR = VC = NULL;
+
+   ierr = EPSSetOperators(eps,op,NULL); PCHKERRQ(eps, ierr);
+
+   VR = new PetscParVector(op, true, false);
+   VC = new PetscParVector(op, true, false);
+
+}
+
+void SlepcEigenSolver::SetOperators(const PetscParMatrix &op,
+                                    const PetscParMatrix&opB)
+{
+   delete VR;
+   delete VC;
+   VR = VC = NULL;
+
+   ierr = EPSSetOperators(eps,op,opB); PCHKERRQ(eps,ierr);
+
+   VR = new PetscParVector(op, true, false);
+   VC = new PetscParVector(op, true, false);
+}
+
+void SlepcEigenSolver::SetTol(double tol)
+{
+   int max_its;
+
+   ierr = EPSGetTolerances(eps,NULL,&max_its); PCHKERRQ(eps,ierr);
+   // Work around uninitialized maximum iterations
+   if (max_its==0) { max_its = PETSC_DECIDE; }
+   ierr = EPSSetTolerances(eps,tol,max_its); PCHKERRQ(eps,ierr);
+}
+
+void SlepcEigenSolver::SetMaxIter(int max_its)
+{
+   double tol;
+
+   ierr = EPSGetTolerances(eps,&tol,NULL); PCHKERRQ(eps,ierr);
+   ierr = EPSSetTolerances(eps,tol,max_its); PCHKERRQ(eps,ierr);
+}
+
+void SlepcEigenSolver::SetNumModes(int num_eigs)
+{
+   ierr = EPSSetDimensions(eps,num_eigs,PETSC_DECIDE,PETSC_DECIDE);
+   PCHKERRQ(eps,ierr);
+}
+
+void SlepcEigenSolver::Solve()
+{
+   Customize();
+
+   ierr = EPSSolve(eps); PCHKERRQ(eps,ierr);
+}
+
+void SlepcEigenSolver::Customize(bool customize) const
+{
+   if (!customize) {clcustom = true; }
+   if (!clcustom)
+   {
+      ierr = EPSSetFromOptions(eps); PCHKERRQ(eps,ierr);
+   }
+   clcustom = true;
+}
+
+void SlepcEigenSolver::GetEigenvalue(unsigned int i, double & lr) const
+{
+   ierr = EPSGetEigenvalue(eps,i,&lr,NULL); PCHKERRQ(eps,ierr);
+}
+
+void SlepcEigenSolver::GetEigenvalue(unsigned int i, double & lr,
+                                     double & lc) const
+{
+   ierr = EPSGetEigenvalue(eps,i,&lr,&lc); PCHKERRQ(eps,ierr);
+}
+
+void SlepcEigenSolver::GetEigenvector(unsigned int i, Vector & vr) const
+{
+   MFEM_VERIFY(VR,"Missing real vector");
+
+   MFEM_ASSERT(vr.Size() == VR->Size(), "invalid vr.Size() = " << vr.Size()
+               << ", expected size = " << VR->Size());
+
+   VR->PlaceArray(vr.GetData());
+   ierr = EPSGetEigenvector(eps,i,*VR,NULL); PCHKERRQ(eps,ierr);
+   VR->ResetArray();
+
+}
+
+void SlepcEigenSolver::GetEigenvector(unsigned int i, Vector & vr,
+                                      Vector & vc) const
+{
+   MFEM_VERIFY(VR,"Missing real vector");
+   MFEM_VERIFY(VC,"Missing imaginary vector");
+   MFEM_ASSERT(vr.Size() == VR->Size(), "invalid vr.Size() = " << vr.Size()
+               << ", expected size = " << VR->Size());
+   MFEM_ASSERT(vc.Size() == VC->Size(), "invalid vc.Size() = " << vc.Size()
+               << ", expected size = " << VC->Size());
+
+   VR->PlaceArray(vr.GetData());
+   VC->PlaceArray(vc.GetData());
+   ierr = EPSGetEigenvector(eps,i,*VR,*VC); PCHKERRQ(eps,ierr);
+   VR->ResetArray();
+   VC->ResetArray();
+}
+
+int SlepcEigenSolver::GetNumConverged()
+{
+   int num_conv;
+   ierr = EPSGetConverged(eps,&num_conv); PCHKERRQ(eps,ierr);
+   return num_conv;
+}
+
+void SlepcEigenSolver::SetWhichEigenpairs(SlepcEigenSolver::Which which)
+{
+   switch (which)
+   {
+      case SlepcEigenSolver::LARGEST_MAGNITUDE:
+         ierr = EPSSetWhichEigenpairs(eps,EPS_LARGEST_MAGNITUDE); PCHKERRQ(eps,ierr);
+         break;
+      case SlepcEigenSolver::SMALLEST_MAGNITUDE:
+         ierr = EPSSetWhichEigenpairs(eps,EPS_SMALLEST_MAGNITUDE); PCHKERRQ(eps,ierr);
+         break;
+      case SlepcEigenSolver::LARGEST_REAL:
+         ierr = EPSSetWhichEigenpairs(eps,EPS_LARGEST_REAL); PCHKERRQ(eps,ierr);
+         break;
+      case SlepcEigenSolver::SMALLEST_REAL:
+         ierr = EPSSetWhichEigenpairs(eps,EPS_SMALLEST_REAL); PCHKERRQ(eps,ierr);
+         break;
+      case SlepcEigenSolver::LARGEST_IMAGINARY:
+         ierr = EPSSetWhichEigenpairs(eps,EPS_LARGEST_IMAGINARY); PCHKERRQ(eps,ierr);
+         break;
+      case SlepcEigenSolver::SMALLEST_IMAGINARY:
+         ierr = EPSSetWhichEigenpairs(eps,EPS_SMALLEST_IMAGINARY); PCHKERRQ(eps,ierr);
+         break;
+      case SlepcEigenSolver::TARGET_MAGNITUDE:
+         ierr = EPSSetWhichEigenpairs(eps,EPS_TARGET_MAGNITUDE); PCHKERRQ(eps,ierr);
+         break;
+      case SlepcEigenSolver::TARGET_REAL:
+         ierr = EPSSetWhichEigenpairs(eps,EPS_TARGET_REAL); PCHKERRQ(eps,ierr);
+         break;
+      default:
+         MFEM_ABORT("Which eigenpair not implemented!");
+         break;
+   }
+}
+
+void SlepcEigenSolver::SetTarget(double target)
+{
+   ierr = EPSSetTarget(eps,target); PCHKERRQ(eps,ierr);
+}
+
+void SlepcEigenSolver::SetSpectralTransformation(
+   SlepcEigenSolver::SpectralTransformation transformation)
+{
+   ST st;
+   ierr = EPSGetST(eps,&st); PCHKERRQ(eps,ierr);
+   switch (transformation)
+   {
+      case SlepcEigenSolver::SHIFT:
+         ierr = STSetType(st,STSHIFT); PCHKERRQ(eps,ierr);
+         break;
+      case SlepcEigenSolver::SHIFT_INVERT:
+         ierr = STSetType(st,STSINVERT); PCHKERRQ(eps,ierr);
+         break;
+      default:
+         MFEM_ABORT("Spectral transformation not implemented!");
+         break;
+   }
+}
+
+}
+
+#endif  // MFEM_USE_SLEPC
+#endif  // MFEM_USE_PETSC
+#endif  // MFEM_USE_MPI

linalg/slepc.hpp

@@ -0,0 +1,116 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_SLEPC
+#define MFEM_SLEPC
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_SLEPC
+#ifdef MFEM_USE_MPI
+
+#include "petsc.hpp"
+
+// Forward declarations
+typedef struct _p_EPS *EPS;
+
+namespace mfem
+{
+
+void MFEMInitializeSlepc();
+void MFEMInitializeSlepc(int*,char***);
+void MFEMInitializeSlepc(int*,char***,const char[],const char[]);
+void MFEMFinalizeSlepc();
+
+class SlepcEigenSolver
+{
+private:
+   /// Boolean to handle SetFromOptions calls
+   mutable bool clcustom;
+
+   /// SLEPc linear eigensolver object
+   EPS eps;
+
+   /// Real and imaginary part of eigenvector
+   mutable PetscParVector *VR, *VC;
+
+public:
+   /// Constructors
+   SlepcEigenSolver(MPI_Comm comm, const std::string &prefix = std::string());
+
+   virtual ~SlepcEigenSolver();
+
+   /// Set solver tolerance
+   void SetTol(double tol);
+
+   /// Set maximum number of iterations
+   void SetMaxIter(int max_iter);
+   /// Set the number of required eigenmodes
+   void SetNumModes(int num_eigs);
+   /// Set operator for standard eigenvalue problem
+   void SetOperator(const PetscParMatrix &op);
+   /// Set operator for generalized eigenvalue problem
+   void SetOperators(const PetscParMatrix &op, const PetscParMatrix &opB);
+
+   /// Customize object with options set
+   void Customize(bool customize = true) const;
+
+   /// Solve the eigenvalue problem for the specified number of eigenvalues
+   void Solve();
+
+   /// Get the number of converged eigenvalues
+   int GetNumConverged();
+
+   /// Get the corresponding eigenvalue
+   void GetEigenvalue(unsigned int i, double & lr) const;
+   void GetEigenvalue(unsigned int i, double & lr, double & lc) const;
+
+   /// Get the corresponding eigenvector
+   void GetEigenvector(unsigned int i, Vector & vr) const;
+   void GetEigenvector(unsigned int i, Vector & vr, Vector & vc) const;
+
+   /// Target spectrum for the eigensolver. Target imaginary is not supported
+   /// without complex support in SLEPc, and intervals are not implemented.
+   enum Which
+   {
+      LARGEST_MAGNITUDE,
+      SMALLEST_MAGNITUDE,
+      LARGEST_REAL,
+      SMALLEST_REAL,
+      LARGEST_IMAGINARY,
+      SMALLEST_IMAGINARY,
+      TARGET_MAGNITUDE,
+      TARGET_REAL
+   };
+
+   enum SpectralTransformation
+   {
+      SHIFT,
+      SHIFT_INVERT
+   };
+
+   void SetWhichEigenpairs(Which which);
+   void SetTarget(double target);
+   void SetSpectralTransformation(SpectralTransformation transformation);
+
+   /// Conversion function to SLEPc's EPS type.
+   operator EPS() const { return eps; }
+
+   /// Conversion function to PetscObject
+   operator PetscObject() const {return (PetscObject)eps; }
+};
+
+}
+
+#endif // MFEM_USE_MPI
+#endif // MFEM_USE_SLEPC
+
+#endif // MFEM_SLEPC

linalg/solvers.cpp

@@ -1641,6 +1641,138 @@ void NewtonSolver::Mult(const Vector &b, Vector &x) const
    final_norm = norm;
 }
 
+void LBFGSSolver::Mult(const Vector &b, Vector &x) const
+{
+   MFEM_VERIFY(oper != NULL, "the Operator is not set (use SetOperator).");
+
+   // Quadrature points that are checked for negative Jacobians etc.
+   Vector sk, rk, yk, rho, alpha;
+   DenseMatrix skM(width, m), ykM(width, m);
+
+   //r - r_{k+1}, c - descent direction
+   sk.SetSize(width);    //x_{k+1}-x_k
+   rk.SetSize(width);    //nabla(f(x_{k}))
+   yk.SetSize(width);    //r_{k+1}-r_{k}
+   rho.SetSize(m);       //1/(dot(yk,sk)
+   alpha.SetSize(m);    //rhok*sk'*c
+   int last_saved_id = -1;
+
+   int it;
+   double norm0, norm, norm_goal;
+   const bool have_b = (b.Size() == Height());
+
+   if (!iterative_mode)
+   {
+      x = 0.0;
+   }
+
+   // r = F(x)-b
+   oper->Mult(x, r);
+   if (have_b) { r -= b; }
+
+   c = r;           // initial descent direction
+
+   norm0 = norm = Norm(r);
+   norm_goal = std::max(rel_tol*norm, abs_tol);
+   for (it = 0; true; it++)
+   {
+      MFEM_ASSERT(IsFinite(norm), "norm = " << norm);
+      if (print_level >= 0)
+      {
+         mfem::out << "LBFGS iteration " <<  it
+                   << " : ||r|| = " << norm;
+         if (it > 0)
+         {
+            mfem::out << ", ||r||/||r_0|| = " << norm/norm0;
+         }
+         mfem::out << '\n';
+      }
+
+      if (norm <= norm_goal)
+      {
+         converged = 1;
+         break;
+      }
+
+      if (it >= max_iter)
+      {
+         converged = 0;
+         break;
+      }
+
+      rk = r;
+      const double c_scale = ComputeScalingFactor(x, b);
+      if (c_scale == 0.0)
+      {
+         converged = 0;
+         break;
+      }
+      add(x, -c_scale, c, x); //x_{k+1} = x_k - c_scale*c
+
+      ProcessNewState(x);
+
+      oper->Mult(x, r);
+      if (have_b)
+      {
+         r -= b;
+      }
+
+      //    LBFGS - construct descent direction
+      subtract(r, rk, yk);   // yk = r_{k+1} - r_{k}
+      sk = c; sk *= -c_scale; //sk = x_{k+1} - x_{k} = -c_scale*c
+      const double gamma = Dot(sk, yk)/Dot(yk, yk);
+
+      //  Save last m vectors
+      last_saved_id = (last_saved_id == m-1) ? 0 : last_saved_id+1;
+      skM.SetCol(last_saved_id, sk);
+      ykM.SetCol(last_saved_id, yk);
+
+      c = r;
+      for (int i = last_saved_id; i > -1; i--)
+      {
+         skM.GetColumn(i, sk);
+         ykM.GetColumn(i, yk);
+         rho(i) = 1./Dot(sk, yk);
+         alpha(i) = rho(i)*Dot(sk,c);
+         add(c, -alpha(i), yk, c);
+      }
+      if (it > m-1)
+      {
+         for (int i = m-1; i > last_saved_id; i--)
+         {
+            skM.GetColumn(i, sk);
+            ykM.GetColumn(i, yk);
+            rho(i) = 1./Dot(sk, yk);
+            alpha(i) = rho(i)*Dot(sk,c);
+            add(c, -alpha(i), yk, c);
+         }
+      }
+
+      c *= gamma;   // scale search direction
+      if (it > m-1)
+      {
+         for (int i = last_saved_id+1; i < m ; i++)
+         {
+            skM.GetColumn(i,sk);
+            ykM.GetColumn(i,yk);
+            double betai = rho(i)*Dot(yk, c);
+            add(c, alpha(i)-betai, sk, c);
+         }
+      }
+      for (int i = 0; i < last_saved_id+1 ; i++)
+      {
+         skM.GetColumn(i,sk);
+         ykM.GetColumn(i,yk);
+         double betai = rho(i)*Dot(yk, c);
+         add(c, alpha(i)-betai, sk, c);
+      }
+
+      norm = Norm(r);
+   }
+
+   final_iter = it;
+   final_norm = norm;
+}
 
 int aGMRES(const Operator &A, Vector &x, const Vector &b,
            const Operator &M, int &max_iter,

linalg/solvers.hpp

@@ -416,6 +416,32 @@ public:
    virtual void ProcessNewState(const Vector &x) const { }
 };
 
+/** L-BFGS method for solving F(x)=b for a given operator F, by minimizing
+    the norm of F(x) - b. Requires only the action of the operator F. */
+class LBFGSSolver : public NewtonSolver
+{
+protected:
+   int m = 10;
+
+public:
+   LBFGSSolver() : NewtonSolver() { }
+
+#ifdef MFEM_USE_MPI
+   LBFGSSolver(MPI_Comm _comm) : NewtonSolver(_comm) { }
+#endif
+
+   void SetHistorySize(int dim) { m = dim; }
+
+   /// Solve the nonlinear system with right-hand side @a b.
+   /** If `b.Size() != Height()`, then @a b is assumed to be zero. */
+   virtual void Mult(const Vector &b, Vector &x) const;
+
+   virtual void SetPreconditioner(Solver &pr)
+   { MFEM_WARNING("L-BFGS won't use the given preconditioner."); }
+   virtual void SetSolver(Solver &solver)
+   { MFEM_WARNING("L-BFGS won't use the given solver."); }
+};
+
 /** Adaptive restarted GMRES.
     m_max and m_min(=1) are the maximal and minimal restart parameters.
     m_step(=1) is the step to use for going from m_max and m_min.

linalg/superlu.cpp

@@ -24,6 +24,11 @@
 #error "SuperLUDist has been built with 64bit integers. This is not supported"
 #endif
 
+// For now, it is assumed that HYPRE_Int is int.
+#ifdef HYPRE_BIGINT
+#error "SuperLUDist support requires HYPRE_Int == int, for now."
+#endif
+
 #if SUPERLU_DIST_MAJOR_VERSION > 6 ||                                   \
   (SUPERLU_DIST_MAJOR_VERSION == 6 && SUPERLU_DIST_MINOR_VERSION > 2)
 #define ScalePermstruct_t dScalePermstruct_t
@@ -147,8 +152,10 @@ SuperLURowLocMatrix::SuperLURowLocMatrix( const HypreParMatrix & hypParMat )
    hypre_CSRMatrix * csr_op = hypre_MergeDiagAndOffd(parcsr_op);
    hypre_CSRMatrixSetDataOwner(csr_op,0);
 #if MFEM_HYPRE_VERSION >= 21600
-   MFEM_VERIFY(csr_op->num_rows < INT_MAX,"SuperLU: number of local rows "
-               "is too large to store as an integer.");
+   // For now, this method assumes that HYPRE_Int is int. Also, csr_op->num_cols
+   // is of type HYPRE_Int, so if we want to check for big indices in
+   // csr_op->big_j, we'll have to check all entries and that check will only be
+   // necessary in HYPRE_MIXEDINT mode which is not supported at the moment.
    hypre_CSRMatrixBigJtoJ(csr_op);
 #endif
 

makefile

@@ -204,7 +204,7 @@ CXXFLAGS ?= $(OPTIM_FLAGS)
 # MPI configuration
 ifneq ($(MFEM_USE_MPI),YES)
    MFEM_HOST_CXX = $(CXX)
-   PKGS_NEED_MPI = SUPERLU STRUMPACK PETSC PUMI
+   PKGS_NEED_MPI = SUPERLU STRUMPACK PETSC PUMI SLEPC
    $(foreach mpidep,$(PKGS_NEED_MPI),$(if $(MFEM_USE_$(mpidep):NO=),\
      $(warning *** [MPI is OFF] setting MFEM_USE_$(mpidep) = NO)\
      $(eval override MFEM_USE_$(mpidep)=NO),))
@@ -260,9 +260,10 @@ endif
 
 # List of MFEM dependencies, that require the *_LIB variable to be non-empty
 MFEM_REQ_LIB_DEPS = SUPERLU METIS CONDUIT SIDRE LAPACK SUNDIALS MESQUITE\
- SUITESPARSE STRUMPACK GINKGO GNUTLS NETCDF PETSC MPFR PUMI HIOP GSLIB\
+ SUITESPARSE STRUMPACK GINKGO GNUTLS NETCDF PETSC SLEPC MPFR PUMI HIOP GSLIB\
  OCCA CEED RAJA UMPIRE
 PETSC_ERROR_MSG = $(if $(PETSC_FOUND),,. PETSC config not found: $(PETSC_VARS))
+SLEPC_ERROR_MSG = $(if $(SLEPC_FOUND),,. SLEPC config not found: $(SLEPC_VARS))
 
 define mfem_check_dependency
 ifeq ($$(MFEM_USE_$(1)),YES)
@@ -321,7 +322,7 @@ MFEM_DEFINES = MFEM_VERSION MFEM_VERSION_STRING MFEM_GIT_STRING MFEM_USE_MPI\
  MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP MFEM_USE_MEMALLOC MFEM_TIMER_TYPE\
  MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE MFEM_USE_GINKGO\
  MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GNUTLS\
- MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT\
+ MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT\
  MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA MFEM_USE_HIP\
  MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD\
  MFEM_USE_ADIOS2 MFEM_SOURCE_DIR MFEM_INSTALL_DIR
@@ -631,6 +632,7 @@ status info:
 	$(info MFEM_USE_GNUTLS        = $(MFEM_USE_GNUTLS))
 	$(info MFEM_USE_NETCDF        = $(MFEM_USE_NETCDF))
 	$(info MFEM_USE_PETSC         = $(MFEM_USE_PETSC))
+	$(info MFEM_USE_SLEPC         = $(MFEM_USE_SLEPC))
 	$(info MFEM_USE_MPFR          = $(MFEM_USE_MPFR))
 	$(info MFEM_USE_SIDRE         = $(MFEM_USE_SIDRE))
 	$(info MFEM_USE_CONDUIT       = $(MFEM_USE_CONDUIT))

mesh/mesh.cpp

@@ -859,35 +859,41 @@ FaceElementTransformations *Mesh::GetFaceElementTransformations(int FaceNo,
 {
    FaceInfo &face_info = faces_info[FaceNo];
 
-   FaceElemTr.SetConfigurationMask(0);
+   int cmask = 0;
+   FaceElemTr.SetConfigurationMask(cmask);
    FaceElemTr.Elem1 = NULL;
    FaceElemTr.Elem2 = NULL;
 
    // setup the transformation for the first element
    FaceElemTr.Elem1No = face_info.Elem1No;
-   if (mask & 1)
+   if (mask & FaceElementTransformations::HAVE_ELEM1)
    {
       GetElementTransformation(FaceElemTr.Elem1No, &Transformation);
       FaceElemTr.Elem1 = &Transformation;
+      cmask |= 1;
    }
 
    //  setup the transformation for the second element
    //     return NULL in the Elem2 field if there's no second element, i.e.
    //     the face is on the "boundary"
    FaceElemTr.Elem2No = face_info.Elem2No;
-   if ((mask & 2) && FaceElemTr.Elem2No >= 0)
+   if ((mask & FaceElementTransformations::HAVE_ELEM2) &&
+       FaceElemTr.Elem2No >= 0)
    {
 #ifdef MFEM_DEBUG
-      if (NURBSext && (mask & 1)) { MFEM_ABORT("NURBS mesh not supported!"); }
+      if (NURBSext && (mask & FaceElementTransformations::HAVE_ELEM1))
+      { MFEM_ABORT("NURBS mesh not supported!"); }
 #endif
       GetElementTransformation(FaceElemTr.Elem2No, &Transformation2);
       FaceElemTr.Elem2 = &Transformation2;
+      cmask |= 2;
    }
 
    // setup the face transformation
-   if (mask & 16)
+   if (mask & FaceElementTransformations::HAVE_FACE)
    {
       GetFaceTransformation(FaceNo, &FaceElemTr);
+      cmask |= 16;
    }
    else
    {
@@ -896,13 +902,15 @@ FaceElementTransformations *Mesh::GetFaceElementTransformations(int FaceNo,
 
    // setup Loc1 & Loc2
    int face_type = GetFaceElementType(FaceNo);
-   if (mask & 4)
+   if (mask & FaceElementTransformations::HAVE_LOC1)
    {
       int elem_type = GetElementType(face_info.Elem1No);
       GetLocalFaceTransformation(face_type, elem_type,
                                  FaceElemTr.Loc1.Transf, face_info.Elem1Inf);
+      cmask |= 4;
    }
-   if ((mask & 8) && FaceElemTr.Elem2No >= 0)
+   if ((mask & FaceElementTransformations::HAVE_LOC2) &&
+       FaceElemTr.Elem2No >= 0)
    {
       int elem_type = GetElementType(face_info.Elem2No);
       GetLocalFaceTransformation(face_type, elem_type,
@@ -913,9 +921,10 @@ FaceElementTransformations *Mesh::GetFaceElementTransformations(int FaceNo,
       {
          ApplyLocalSlaveTransformation(FaceElemTr, face_info, false);
       }
+      cmask |= 8;
    }
 
-   FaceElemTr.SetConfigurationMask(mask);
+   FaceElemTr.SetConfigurationMask(cmask);
 
    // This check can be useful for internal debugging, however it will fail on
    // periodic boundary faces, so we keep it disabled in general.
@@ -1000,7 +1009,7 @@ FaceElementTransformations *Mesh::GetBdrFaceTransformations(int BdrElemNo)
    {
       return NULL;
    }
-   tr = GetFaceElementTransformations(fn);
+   tr = GetFaceElementTransformations(fn, 21);
    tr->Attribute = boundary[BdrElemNo]->GetAttribute();
    tr->ElementNo = BdrElemNo;
    tr->ElementType = ElementTransformation::BDR_FACE;

mesh/mesh.hpp

@@ -961,7 +961,7 @@ public:
    /// Returns the transformation defining the given face element
    ElementTransformation *GetEdgeTransformation(int EdgeNo);
 
-   /// Returns (a pointer to a structure containing) the following data:
+   /// Returns (a pointer to an object containing) the following data:
    ///
    /// 1) Elem1No - the index of the first element that contains this face this
    ///    is the element that has the same outward unit normal vector as the
@@ -989,6 +989,8 @@ public:
    /// The mask specifies which fields in the structure to return:
    ///    mask & 1 - Elem1, mask & 2 - Elem2
    ///    mask & 4 - Loc1, mask & 8 - Loc2, mask & 16 - Face.
+   /// These mask values are defined in the ConfigMasks enum type as part of the
+   /// FaceElementTransformations class in fem/eltrans.hpp.
    FaceElementTransformations *GetFaceElementTransformations(int FaceNo,
                                                              int mask = 31);
 

mesh/pmesh.cpp

@@ -1690,6 +1690,7 @@ void ParMesh::GetFaceNbrElementTransformation(
 
    ElTr->Attribute = elem->GetAttribute();
    ElTr->ElementNo = NumOfElements + i;
+   ElTr->ElementType = ElementTransformation::ELEMENT;
 
    if (Nodes == NULL)
    {
@@ -2402,6 +2403,11 @@ GetSharedFaceTransformations(int sf, bool fill2)
    bool is_slave = Nonconforming() && IsSlaveFace(face_info);
    bool is_ghost = Nonconforming() && FaceNo >= GetNumFaces();
 
+   int mask = 0;
+   FaceElemTr.SetConfigurationMask(0);
+   FaceElemTr.Elem1 = NULL;
+   FaceElemTr.Elem2 = NULL;
+
    NCFaceInfo* nc_info = NULL;
    if (is_slave) { nc_info = &nc_faces_info[face_info.NCFace]; }
 
@@ -2413,13 +2419,21 @@ GetSharedFaceTransformations(int sf, bool fill2)
    FaceElemTr.Elem1No = face_info.Elem1No;
    GetElementTransformation(FaceElemTr.Elem1No, &Transformation);
    FaceElemTr.Elem1 = &Transformation;
+   mask |= FaceElementTransformations::HAVE_ELEM1;
 
    // setup the transformation for the second (neighbor) element
+   int Elem2NbrNo;
    if (fill2)
    {
-      FaceElemTr.Elem2No = -1 - face_info.Elem2No;
-      GetFaceNbrElementTransformation(FaceElemTr.Elem2No, &Transformation2);
+      Elem2NbrNo = -1 - face_info.Elem2No;
+      // Store the "shifted index" for element 2 in FaceElemTr.Elem2No.
+      // `Elem2NbrNo` is the index of the face neighbor (starting from 0),
+      // and `FaceElemTr.Elem2No` will be offset by the number of (local)
+      // elements in the mesh.
+      FaceElemTr.Elem2No = NumOfElements + Elem2NbrNo;
+      GetFaceNbrElementTransformation(Elem2NbrNo, &Transformation2);
       FaceElemTr.Elem2 = &Transformation2;
+      mask |= FaceElementTransformations::HAVE_ELEM2;
    }
    else
    {
@@ -2431,6 +2445,7 @@ GetSharedFaceTransformations(int sf, bool fill2)
    {
       GetFaceTransformation(FaceNo, &FaceElemTr);
       // NOTE: The above call overwrites FaceElemTr.Loc1
+      mask |= FaceElementTransformations::HAVE_FACE;
    }
    else
    {
@@ -2441,12 +2456,14 @@ GetSharedFaceTransformations(int sf, bool fill2)
    int elem_type = GetElementType(face_info.Elem1No);
    GetLocalFaceTransformation(face_type, elem_type, FaceElemTr.Loc1.Transf,
                               face_info.Elem1Inf);
+   mask |= FaceElementTransformations::HAVE_LOC1;
 
    if (fill2)
    {
-      elem_type = face_nbr_elements[FaceElemTr.Elem2No]->GetType();
+      elem_type = face_nbr_elements[Elem2NbrNo]->GetType();
       GetLocalFaceTransformation(face_type, elem_type, FaceElemTr.Loc2.Transf,
                                  face_info.Elem2Inf);
+      mask |= FaceElementTransformations::HAVE_LOC2;
    }
 
    // adjust Loc1 or Loc2 of the master face if this is a slave face
@@ -2463,9 +2480,10 @@ GetSharedFaceTransformations(int sf, bool fill2)
    if (is_ghost)
    {
       GetGhostFaceTransformation(&FaceElemTr, face_type, face_geom);
+      mask |= FaceElementTransformations::HAVE_FACE;
    }
 
-   FaceElemTr.SetConfigurationMask(fill2 ? 31 : 21);
+   FaceElemTr.SetConfigurationMask(mask);
 
    // This check can be useful for internal debugging, however it will fail on
    // periodic boundary faces, so we keep it disabled in general.

mesh/pmesh.hpp

@@ -78,6 +78,8 @@ protected:
    // sface ids: all triangles first, then all quads
    Array<int> sface_lface;
 
+   IsoparametricTransformation FaceNbrTransformation;
+
    // glob_elem_offset + local element number defines a global element numbering
    mutable long glob_elem_offset, glob_offset_sequence;
    void ComputeGlobalElementOffset() const;
@@ -295,6 +297,14 @@ public:
    FaceElementTransformations *
    GetSharedFaceTransformations(int sf, bool fill2 = true);
 
+   ElementTransformation *
+   GetFaceNbrElementTransformation(int i)
+   {
+      GetFaceNbrElementTransformation(i, &FaceNbrTransformation);
+
+      return &FaceNbrTransformation;
+   }
+
    /// Return the number of shared faces (3D), edges (2D), vertices (1D)
    int GetNSharedFaces() const;
 

miniapps/electromagnetics/joule_solver.cpp

@@ -906,7 +906,7 @@ double JouleHeatingCoefficient::Eval(ElementTransformation &T,
 {
    Vector E;
    double thisSigma;
-   E_gf.GetVectorValue(T.ElementNo, ip, E);
+   E_gf.GetVectorValue(T, ip, E);
    thisSigma = sigma.Eval(T, ip);
    return thisSigma*(E*E);
 }

miniapps/hypsys/apps/advection.cpp

@@ -0,0 +1,371 @@
+#include "advection.hpp"
+
+Configuration ConfigAdv;
+
+void AnalyticalSolutionAdv(const Vector &x, double t, Vector &u);
+void InitialConditionAdv(const Vector &x, Vector &u);
+void InflowFunctionAdv(const Vector &x, double t, Vector &u);
+void VelocityFunctionAdv(const Vector &x, Vector &v);
+
+Advection::Advection(FiniteElementSpace *fes_, BlockVector &u_block,
+                     Configuration &config_, bool NodalQuadRule)
+   : HyperbolicSystem(fes_, u_block, 1, config_,
+                      VectorFunctionCoefficient (1, InflowFunctionAdv))
+{
+   DiscreteUpwinding = true;
+
+   ConfigAdv = config_;
+   VectorFunctionCoefficient ic(NumEq, InitialConditionAdv);
+
+   switch (ConfigAdv.ConfigNum)
+   {
+      case 0:
+      {
+         ProblemName = "Advection - Smooth Circular Convection";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = true;
+         TimeDepBC = false;
+         ProjType = 0;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 1:
+      {
+         ProblemName = "Advection - Solid Body Rotation";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 2:
+      {
+         ProblemName = "Advection - Step function";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 3:
+      {
+         ProblemName = "Advection - Smooth profile";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 0;
+         L2_Projection(ic, u0);;
+         break;
+      }
+      case 4:
+      {
+         ProblemName = "Advection - Discontinuous and Smooth profile";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 5:
+      {
+         ProblemName = "Advection - C1 curve";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      default:
+         MFEM_ABORT("No such test case implemented.");
+   }
+
+   // The following computes and stores all necessary evaluations of the time-independent velocity.
+   Mesh *mesh = fes->GetMesh();
+   DofInfo dofs(fes);
+   const int ne = fes->GetNE();
+   const IntegrationRule *IntRuleElem = GetElementIntegrationRule(fes);
+   const IntegrationRule *IntRuleFace = GetFaceIntegrationRule(fes);
+   const IntegrationRule *nodes = GetElementIntegrationRule(fes, true);
+
+   const int nqe = IntRuleElem->GetNPoints();
+   nqf = IntRuleFace->GetNPoints();
+   Vector vec, vval;
+   VelocityVector.SetSize(dim);
+   DenseMatrix VelEval, mat(dim, nqe);
+
+   VelElem.SetSize(dim, nqe, ne);
+   VelFace.SetSize(dim, dofs.NumBdrs, ne*nqf);
+   VelNode.SetSize(dim, nd, ne);
+   VectorFunctionCoefficient velocity(dim, VelocityFunctionAdv);
+
+   Array<int> bdrs, orientation;
+   Array<IntegrationPoint> eip(nqf*dofs.NumBdrs);
+
+   if (dim==1)      { mesh->GetElementVertices(0, bdrs); }
+   else if (dim==2) { mesh->GetElementEdges(0, bdrs, orientation); }
+   else if (dim==3) { mesh->GetElementFaces(0, bdrs, orientation); }
+
+   for (int i = 0; i < dofs.NumBdrs; i++)
+   {
+      FaceElementTransformations *help
+         = mesh->GetFaceElementTransformations(bdrs[i]);
+
+      if (help->Elem1No != 0)
+      {
+         // NOTE: If this error ever occurs, use neighbor element to
+         // obtain the correct quadrature points and weight.
+         MFEM_ABORT("First element has inward pointing normal.");
+      }
+      for (int k = 0; k < nqf; k++)
+      {
+         const IntegrationPoint &ip = IntRuleFace->IntPoint(k);
+         help->Loc1.Transform(ip, eip[i*nqf + k]);
+      }
+   }
+
+   for (int e = 0; e < ne; e++)
+   {
+      ElementTransformation *eltrans = fes->GetElementTransformation(e);
+      velocity.Eval(VelEval, *eltrans, *IntRuleElem);
+
+      for (int k = 0; k < nqe; k++)
+      {
+         VelEval.GetColumnReference(k, vec);
+         mat.SetCol(k, vec);
+      }
+
+      VelElem(e) = mat;
+
+      for (int i = 0; i < nd; i++)
+      {
+         const IntegrationPoint ip = nodes->IntPoint(i);
+         velocity.Eval(vec, *eltrans, ip);
+         VelNode(e).SetCol(i, vec);
+      }
+
+      if (dim==1)      { mesh->GetElementVertices(e, bdrs); }
+      else if (dim==2) { mesh->GetElementEdges(e, bdrs, orientation); }
+      else if (dim==3) { mesh->GetElementFaces(e, bdrs, orientation); }
+
+      for (int i = 0; i < dofs.NumBdrs; i++)
+      {
+         FaceElementTransformations *facetrans
+            = mesh->GetFaceElementTransformations(bdrs[i]);
+
+         for (int k = 0; k < nqf; k++)
+         {
+            if (facetrans->Elem1No != e)
+            {
+               velocity.Eval(vval, *facetrans->Elem2, eip[i * nqf + k]);
+            }
+            else
+            {
+               velocity.Eval(vval, *facetrans->Elem1, eip[i * nqf + k]);
+            }
+
+            for (int l = 0; l < dim; l++)
+            {
+               VelFace(l, i, e * nqf + k) = vval(l);
+            }
+         }
+      }
+   }
+}
+
+void Advection::EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i) const
+{
+   Vector x(dim), v(dim);
+   VelocityFunctionAdv(x, v);
+   v *= u(0);
+   FluxEval.SetRow(0, v);
+   if (i == -1) // Element terms
+   {
+      VelocityVector = VelElem(e).GetColumn(k);
+      VelocityVector *= u(0);
+      FluxEval.SetRow(0, VelocityVector);
+   }
+   else
+   {
+      VelocityVector = VelFace(e*nqf+k).GetColumn(i);
+      VelocityVector *= u(0);
+      FluxEval.SetRow(0, VelocityVector);
+   }
+}
+
+double Advection::GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i) const
+{
+   if (i == -1) // Element terms
+   {
+      VelocityVector = VelElem(e).GetColumn(k);
+   }
+   else
+   {
+      VelocityVector = VelFace(e*nqf+k).GetColumn(i);
+   }
+
+   return abs(VelocityVector * n);
+}
+
+void Advection::ComputeErrors(Array<double> &errors, const GridFunction &u,
+                              double DomainSize, double t) const
+{
+   errors.SetSize(3);
+   VectorFunctionCoefficient uAnalytic(NumEq, AnalyticalSolutionAdv);
+   uAnalytic.SetTime(t);
+   errors[0] = u.ComputeLpError(1., uAnalytic) / DomainSize;
+   errors[1] = u.ComputeLpError(2., uAnalytic) / DomainSize;
+   errors[2] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic);
+}
+
+
+void VelocityFunctionAdv(const Vector &x, Vector &v)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+   double s = 1.0;
+
+   for (int i = 0; i < dim; i++)
+   {
+      switch (ConfigAdv.ConfigNum)
+      {
+         case 0:
+         case 1:
+         case 4:
+         case 5: // Map to the reference domain [0,1]^d.
+         {
+            X(i) = (x(i) - ConfigAdv.bbMin(i)) / (ConfigAdv.bbMax(i) - ConfigAdv.bbMin(i));
+            s *= ConfigAdv.bbMax(i) - ConfigAdv.bbMin(i);
+            break;
+         }
+         case 2:
+         case 3: // Map to the reference domain [-1,1]^d.
+         {
+            double center = 0.5 * (ConfigAdv.bbMin(i) + ConfigAdv.bbMax(i));
+            X(i) = 2. * (x(i) - center) / (ConfigAdv.bbMax(i) - ConfigAdv.bbMin(i));
+            s *= ConfigAdv.bbMax(i) - ConfigAdv.bbMin(i);
+            break;
+         }
+      }
+   }
+
+   // Scale to be normed to a full revolution.
+   s = pow(s, 1./dim);
+
+   switch (ConfigAdv.ConfigNum)
+   {
+      case 0: // Rotation around corner.
+      {
+         switch (dim)
+         {
+            case 1: v(0) = s; break;
+            case 2: s *= 2.0 * M_PI; v(0) = s*X(1); v(1) = -s*X(0); break;
+            case 3: s *= 2.0 * M_PI; v(0) = s*X(1); v(1) = -s*X(0); v(2) = 0.0; break;
+         }
+         break;
+      }
+      case 1: // Rotation around center.
+      {
+         switch (dim)
+         {
+            case 1: v(0) = s; break;
+            case 2: s *= 2.0 * M_PI; v(0) = s * (0.5-X(1)); v(1) = s*(X(0)-0.5); break;
+            case 3: s *= 2.0 * M_PI; v(0) = s * (0.5-X(1)); v(1) = s*(X(0)-0.5); v(2) = 0.0;
+               break;
+         }
+         break;
+      }
+      case 2:
+      case 3:
+      case 4:
+      case 5:
+      {
+         switch (dim)
+         {
+            case 1: v(0) = s; break;
+            case 2: v(0) = s; v(1) = -0.5*s; break;
+            case 3: v(0) = s; v(1) = -0.5*s; v(2) = 0.25*s; break;
+         }
+         break;
+      }
+   }
+}
+
+void AnalyticalSolutionAdv(const Vector &x, double t, Vector &u)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+
+   for (int i = 0; i < dim; i++)
+   {
+      switch (ConfigAdv.ConfigNum)
+      {
+         case 0:
+         case 1:
+         case 4:
+         case 5: // Map to the reference domain [0,1]^d.
+         {
+            X(i) = (x(i) - ConfigAdv.bbMin(i)) / (ConfigAdv.bbMax(i) - ConfigAdv.bbMin(i));
+            break;
+         }
+         case 2:
+         case 3: // Map to the reference domain [-1,1]^d.
+         {
+            double center = 0.5 * (ConfigAdv.bbMin(i) + ConfigAdv.bbMax(i));
+            X(i) = 2.0 * (x(i) - center) / (ConfigAdv.bbMax(i) - ConfigAdv.bbMin(i));
+            break;
+         }
+      }
+   }
+
+   double r = X.Norml2();
+
+   switch (ConfigAdv.ConfigNum)
+   {
+      case 0:
+      {
+         double a = 0.5, b = 0.03, c = 0.1;
+         u(0) = 0.25 * (1. + tanh((r+c-a)/b)) * (1. - tanh((r-c-a)/b));
+         break;
+      }
+      case 1:
+      {
+         if (dim==1) { MFEM_ABORT("Test case not implemented in 1D."); }
+
+         double s = 0.15;
+         double cone = sqrt(pow(X(0)-0.5, 2.) + pow(X(1)-0.25, 2.));
+         double hump = sqrt(pow(X(0)-0.25, 2.) + pow(X(1)-0.5, 2.));
+
+         u(0) = (1. - cone / s) * (cone <= s) +
+                0.25 * (1. + cos(M_PI*hump / s)) * (hump <= s) +
+                ( ( sqrt(pow(X(0)-0.5, 2.) + pow(X(1)-0.75, 2.)) <= s ) &&
+                  ( abs(X(0)-0.5) >= 0.025 || (X(1) >= 0.85) ) ? 1. : 0. );
+         break;
+      }
+      case 2: { u(0) = r < 0.2 ? 1. : 0.; break; }
+      case 3: { u(0) = exp(-25.0 * r*r); break; }
+      case 4: { u(0) = abs(r - 0.3) < 0.1 ? 1. : ( (abs(r-0.7) < 0.2) ? (exp(10.)*exp(-1./(r-0.5))*exp(1./(r-0.9))) : 0. ); break; }
+      case 5: { u(0) = abs(r-0.25) <= 0.15 ? 0.5*(1.+cos(M_PI*(r-0.25)/0.15)) : 0.; break; }
+   }
+}
+
+void InitialConditionAdv(const Vector &x, Vector &u)
+{
+   AnalyticalSolutionAdv(x, 0.0, u);
+}
+
+void InflowFunctionAdv(const Vector &x, double t, Vector &u)
+{
+   AnalyticalSolutionAdv(x, t, u);
+}

miniapps/hypsys/apps/advection.hpp

@@ -0,0 +1,26 @@
+#ifndef HYPSYS_ADVECTION
+#define HYPSYS_ADVECTION
+
+#include "hyperbolic_system.hpp"
+#include "../lib/dofs.hpp"
+
+class Advection : public HyperbolicSystem
+{
+public:
+   explicit Advection(FiniteElementSpace *fes_, BlockVector &u_block,
+                      Configuration &config_, bool NodalQuadRule);
+   ~Advection() { };
+
+   virtual void EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i = -1) const;
+   virtual double GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i) const;
+   virtual void ComputeErrors(Array<double> &errors, const GridFunction &u,
+                              double DomainSize, double t) const override;
+
+   int nqf;
+   DenseTensor VelElem, VelFace;
+   mutable Vector VelocityVector;
+};
+
+#endif

miniapps/hypsys/apps/buckley_leverett.cpp

@@ -0,0 +1,118 @@
+#include "buckley_leverett.hpp"
+
+Configuration ConfigBL;
+double BLConst;
+
+void InitialConditionBuckleyLeverett(const Vector &x, Vector &u);
+void InflowFunctionBuckleyLeverett(const Vector &x, double t, Vector &u);
+
+BuckleyLeverett::BuckleyLeverett(FiniteElementSpace *fes_, BlockVector &u_block,
+                                 Configuration &config_)
+   : HyperbolicSystem(fes_, u_block, 1, config_,
+                      VectorFunctionCoefficient(1, InflowFunctionBuckleyLeverett))
+{
+   ConfigBL = config_;
+   VectorFunctionCoefficient ic(NumEq, InitialConditionBuckleyLeverett);
+
+   switch (ConfigBL.ConfigNum)
+   {
+      case 1:
+      {
+         ProblemName = "Buckley-Leverett - 1D";
+         glvis_scale = "on";
+         BLConst = 0.5;
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 0;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 2:
+      {
+         ProblemName = "Buckley-Leverett - 2D";
+         glvis_scale = "on";
+         BLConst = 1.0;
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      default:
+         MFEM_ABORT("No such test case implemented.");
+   }
+}
+
+void BuckleyLeverett::EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                                   int e, int k, int i) const
+{
+   double coef = u(0)*u(0) / (u(0)*u(0) + BLConst * (1.0-u(0))*(1.0-u(0)));
+   FluxEval(0,0) = coef;
+   if (dim > 1)
+   {
+      FluxEval(0,1) = coef * (1.0 - 5.0 * (1.0-u(0))*(1.0-u(0)));
+   }
+   if (dim > 2) { MFEM_ABORT("Not implemented."); }
+}
+
+double BuckleyLeverett::GetWaveSpeed(const Vector &u, const Vector n, int e,
+                                     int k,
+                                     int i) const
+{
+   if (dim == 1)
+   {
+      return abs( 2.0 * BLConst * u(0) * (1.0-u(0)) / pow(u(0)*u(0) + BLConst *
+                                                          (1.0-u(0))*(1.0-u(0)), 2.0) );
+   }
+   else if (dim == 2)
+   {
+      return 3.4;
+   }
+   else { MFEM_ABORT("Not implemented."); }
+}
+
+void InitialConditionBuckleyLeverett(const Vector &x, Vector &u)
+{
+   const int dim = x.Size();
+
+   // Map to the reference domain [-1,1]^d.
+   Vector X(dim);
+   for (int i = 0; i < dim; i++)
+   {
+      double center = 0.5 * (ConfigBL.bbMin(i) + ConfigBL.bbMax(i));
+      X(i) = 2. * (x(i) - center) / (ConfigBL.bbMax(i) - ConfigBL.bbMin(i));
+   }
+
+   switch (ConfigBL.ConfigNum)
+   {
+      case 1:
+      {
+         u(0) =  X(0) < 0.0 ? -3.0 : 3.0;
+         break;
+      }
+      case 2:
+      {
+         u(0) = X.Norml2()*X.Norml2() < 2.0 / 9.0 ? 1.0 : 0.0;
+         break;
+      }
+   }
+}
+
+void InflowFunctionBuckleyLeverett(const Vector &x, double t, Vector &u)
+{
+   switch (ConfigBL.ConfigNum)
+   {
+      case 1:
+      {
+         u(0) = x(0) < 0.0 ? 3.0 : -3.0;
+         break;
+      }
+      case 2:
+      {
+         u(0) = 0.0;
+         break;
+      }
+   }
+}

miniapps/hypsys/apps/buckley_leverett.hpp

@@ -0,0 +1,19 @@
+#ifndef HYPSYS_BUCKLEYLEVERETT
+#define HYPSYS_BUCKLEYLEVERETT
+
+#include "hyperbolic_system.hpp"
+
+class BuckleyLeverett : public HyperbolicSystem
+{
+public:
+   explicit BuckleyLeverett(FiniteElementSpace *fes_, BlockVector &u_block,
+                            Configuration &config_);
+   ~BuckleyLeverett() { };
+
+   virtual void EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i = -1) const;
+   virtual double GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i) const;
+};
+
+#endif

miniapps/hypsys/apps/burgers.cpp

@@ -0,0 +1,175 @@
+#include "burgers.hpp"
+
+Configuration ConfigBurgers;
+
+void AnalyticalSolutionBurgers(const Vector &x, double t, Vector &u);
+void InitialConditionBurgers(const Vector &x, Vector &u);
+void InflowFunctionBurgers(const Vector &x, double t, Vector &u);
+
+Burgers::Burgers(FiniteElementSpace *fes_, BlockVector &u_block,
+                 Configuration &config_)
+   : HyperbolicSystem(fes_, u_block, 1, config_,
+                      VectorFunctionCoefficient (1, InflowFunctionBurgers))
+{
+   ConfigBurgers = config_;
+   VectorFunctionCoefficient ic(NumEq, InitialConditionBurgers);
+
+   switch (ConfigBurgers.ConfigNum)
+   {
+      case 0:
+      {
+         ProblemName = "Burgers Equation - 1D";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 1:
+      {
+         ProblemName = "Burgers Equation - Riemann Problem";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = true;
+         ProjType = 1;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 2:
+      {
+         ProblemName = "Burgers Equation - Steady State";
+         glvis_scale = "on";
+         SolutionKnown = true;
+         SteadyState = true;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      default:
+         MFEM_ABORT("No such test case implemented.");
+   }
+}
+
+void Burgers::EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                           int e, int k, int i) const
+{
+   FluxEval = 0.5 * u(0) * u(0);
+}
+
+double Burgers::GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                             int i) const
+{
+   return abs(u(0) * double(n.Size()));
+}
+
+void Burgers::ComputeErrors(Array<double> &errors, const GridFunction &u,
+                            double DomainSize, double t) const
+{
+   errors.SetSize(3);
+   VectorFunctionCoefficient uAnalytic(NumEq, AnalyticalSolutionBurgers);
+   uAnalytic.SetTime(t);
+   errors[0] = u.ComputeLpError(1., uAnalytic) / DomainSize;
+   errors[1] = u.ComputeLpError(2., uAnalytic) / DomainSize;
+   errors[2] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic);
+}
+
+
+void AnalyticalSolutionBurgers(const Vector &x, double t, Vector &u)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+
+   // Map to the reference domain [0,1]^d.
+   for (int i = 0; i < dim; i++)
+   {
+      double factor = 1.0 / ( ConfigBurgers.bbMax(i) - ConfigBurgers.bbMin(i));
+      X(i) = factor * (x(i) - ConfigBurgers.bbMin(i));
+      t *= pow(factor, 1.0 / (double(dim)));
+   }
+
+   switch (ConfigBurgers.ConfigNum)
+   {
+      case 0:
+      {
+         if (dim != 1) { MFEM_ABORT("Test case only implemented in 1D."); }
+
+         double un = sin(2.0*M_PI*X(0));
+         double fn, fpn;
+         double tol = 1.E-15;
+         double error = 1.0;
+         int iter = 0, maxiter = 100;
+
+         while (error > tol)
+         {
+            // Do not trust this solution at a time later than t = 0.1.
+            if (iter == maxiter) { break; }
+
+            fn = sin(2.0*M_PI*(X(0)-un*t))-un;
+            fpn = -2.0*M_PI*t*cos(2.*M_PI*(X(0)-un*t))-1.0;
+            un -= fn/fpn;
+            error = abs(sin(2.*M_PI*(X(0)-un*t))-un);
+            iter++;
+         }
+
+         u(0) = un;
+         break;
+      }
+      case 1:
+      {
+         if (dim != 2) { MFEM_ABORT("Test case only implemented in 2D."); }
+
+         if (X(0) <= 0.5 - 0.6 * t)
+         {
+            u(0) = X(1) >= 0.5 + 0.15 * t ? -0.2 : 0.5;
+         }
+         else if (X(0) < 0.5 - 0.25 * t)
+         {
+            u(0) = X(1) > -8. / 7. * X(0) + 15. / 14. - 15. / 28. * t ? -1. : 0.5;
+         }
+         else if (X(0) < 0.5 + 0.5 * t)
+         {
+            u(0) = X(1) > X(0) / 6. + 5. / 12. - 5. / 24. * t ? -1. : 0.5;
+         }
+         else if (X(0) < 0.5 + 0.8 * t)
+         {
+            u(0) = X(1) > X(0) - 5. / (18. * t) * (X(0) + t - 0.5)
+                   * (X(0) + t - 0.5) ? -1. : (2. * X(0) - 1.) / (2 * t);
+         }
+         else
+         {
+            u(0) = X(1) >= 0.5 - 0.1 * t ? -1 : 0.8;
+         }
+
+         break;
+      }
+      case 2:
+      {
+         u(0) = X.Sum() < 0.5 ? 1.0 : -1.0;
+         break;
+      }
+   }
+}
+
+void InitialConditionBurgers(const Vector &x,Vector &u)
+{
+   switch (ConfigBurgers.ConfigNum)
+   {
+      case 0:
+      case 1:
+      case 2: { AnalyticalSolutionBurgers(x, 0.0, u); break; }
+   }
+}
+
+void InflowFunctionBurgers(const Vector &x, double t, Vector &u)
+{
+   switch (ConfigBurgers.ConfigNum)
+   {
+      case 0:
+      case 1:
+      case 2: { AnalyticalSolutionBurgers(x, t, u); break; }
+   }
+}

miniapps/hypsys/apps/burgers.hpp

@@ -0,0 +1,21 @@
+#ifndef HYPSYS_BURGERS
+#define HYPSYS_BURGERS
+
+#include "hyperbolic_system.hpp"
+
+class Burgers : public HyperbolicSystem
+{
+public:
+   explicit Burgers(FiniteElementSpace *fes_, BlockVector &u_block,
+                    Configuration &config_);
+   ~Burgers() { };
+
+   virtual void EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i = -1) const;
+   virtual double GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i) const;
+   virtual void ComputeErrors(Array<double> &errors, const GridFunction &u,
+                              double DomainSize, double t) const override;
+};
+
+#endif

miniapps/hypsys/apps/euler.cpp

@@ -0,0 +1,647 @@
+#include "euler.hpp"
+
+Configuration ConfigEuler;
+
+double SpHeatRatio;
+
+void AnalyticalSolutionEuler(const Vector &x, double t, Vector &u);
+void InitialConditionEuler(const Vector &x, Vector &u);
+void InflowFunctionEuler(const Vector &x, double t, Vector &u);
+
+Euler::Euler(FiniteElementSpace *fes_, BlockVector &u_block,
+             Configuration &config_)
+   : HyperbolicSystem(fes_, u_block, fes_->GetMesh()->Dimension() + 2, config_,
+                      VectorFunctionCoefficient(fes_->GetMesh()->Dimension() + 2,
+                                                InflowFunctionEuler))
+{
+   ConfigEuler = config_;
+   VectorFunctionCoefficient ic(NumEq, InitialConditionEuler);
+
+   switch (ConfigEuler.ConfigNum)
+   {
+      case 0:
+      {
+         // Periodic meshes must be used for this problem.
+         ProblemName = "Euler Equations of Gas dynamics - Smooth Vortex";
+         glvis_scale = "on";
+         SpHeatRatio = 1.4;
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 0;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 1:
+      {
+         ProblemName = "Euler Equations of Gas dynamics - SOD Shock Tube";
+         glvis_scale = "on";
+         SpHeatRatio = 1.4;
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 2:
+      {
+         ProblemName = "Euler Equations of Gas dynamics - Woodward Colella";
+         glvis_scale = "on";
+         SpHeatRatio = 1.4;
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 3:
+      {
+         ProblemName = "Euler Equations of Gas dynamics - Double Mach Reflection";
+         glvis_scale = "on";
+         SpHeatRatio = 1.4;
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = true;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 4:
+      {
+         ProblemName = "Euler Equations of Gas dynamics - Sedov Blast";
+         glvis_scale = "on";
+         SpHeatRatio = 5.0 / 3.0;
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = true;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 5:
+      {
+         ProblemName = "Euler Equations of Gas dynamics - Noh Problem";
+         glvis_scale = "on";
+         SpHeatRatio = 5.0 / 3.0;
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = true;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 6:
+      {
+         ProblemName = "Euler Equations of Gas dynamics - Gresho Vortex";
+         glvis_scale = "on";
+         SpHeatRatio = 1.4;
+         SolutionKnown = true;
+         SteadyState = true;
+         TimeDepBC = false;
+         ProjType = 0;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 7:
+      {
+         ProblemName = "Euler Equations of Gas dynamics - Constricted Channel";
+         glvis_scale = "on";
+         SpHeatRatio = 1.4;
+         SolutionKnown = false;
+         SteadyState = true;
+         TimeDepBC = false;
+         ProjType = 0;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      default:
+         MFEM_ABORT("No such test case implemented.");
+   }
+}
+
+double Euler::EvaluatePressure(const Vector &u) const
+{
+   double aux = 0.0;
+   for (int l = 0; l < dim; l++)
+   {
+      aux += u(1+l) * u(1+l);
+   }
+   double pressure = (SpHeatRatio - 1.0) * (u(dim+1) - 0.5 * aux / u(0));
+   if (pressure < 0.)
+   {
+      ostringstream press_str;
+      press_str << pressure;
+      string err_msg = "Negative pressure p = ";
+      MFEM_ABORT(err_msg << press_str.str());
+   }
+   return pressure;
+}
+
+void Euler::EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                         int e, int k, int i) const
+{
+   double pressure = EvaluatePressure(u);
+   CheckAdmissibility(u);
+
+   switch (dim)
+   {
+      case 1:
+      {
+         double vx = u(1) / u(0);
+         FluxEval(0,0) = u(1);
+         FluxEval(1,0) = u(1) * vx + pressure;
+         FluxEval(2,0) = (u(2) + pressure) * vx;
+         break;
+      }
+      case 2:
+      {
+         double vx = u(1) / u(0);
+         double vy = u(2) / u(0);
+         double energy = u(3) + pressure;
+
+         FluxEval(0,0) = u(1);
+         FluxEval(0,1) = u(2);
+
+         FluxEval(1,0) = u(1) * vx + pressure;
+         FluxEval(1,1) = u(1) * vy;
+
+         FluxEval(2,0) = u(2) * vx;
+         FluxEval(2,1) = u(2) * vy + pressure;
+
+         FluxEval(3,0) = energy * vx;
+         FluxEval(3,1) = energy * vy;
+         break;
+      }
+      case 3:
+      {
+         double vx = u(1) / u(0);
+         double vy = u(2) / u(0);
+         double vz = u(3) / u(0);
+         double energy = u(4) + pressure;
+
+         FluxEval(0,0) = u(1);
+         FluxEval(0,1) = u(2);
+         FluxEval(0,2) = u(3);
+
+         FluxEval(1,0) = u(1) * vx + pressure;
+         FluxEval(1,1) = u(1) * vy;
+         FluxEval(1,2) = u(1) * vz;
+
+         FluxEval(2,0) = u(2) * vx;
+         FluxEval(2,1) = u(2) * vy + pressure;
+         FluxEval(2,2) = u(2) * vz;
+
+         FluxEval(3,0) = u(3) * vx;
+         FluxEval(3,1) = u(3) * vy;
+         FluxEval(3,2) = u(3) * vz + pressure;
+
+         FluxEval(4,0) = energy * vx;
+         FluxEval(4,1) = energy * vy;
+         FluxEval(4,2) = energy * vz;
+         break;
+      }
+      default:
+         MFEM_ABORT("Invalid space dimension.");
+   }
+}
+
+double Euler::GetGMS(const Vector &uL, const Vector &uR,
+                     const Vector &normal) const
+{
+   CheckAdmissibility(uL);
+   CheckAdmissibility(uR);
+   double pL = EvaluatePressure(uL);
+   double pR = EvaluatePressure(uR);
+   double aL = sqrt(SpHeatRatio * pL / uL(0));
+   double aR = sqrt(SpHeatRatio * pR / uR(0));
+   double vL = uL(1)/uL(0) * normal(0);
+   double vR = uR(1)/uR(0) * normal(0);
+
+   double p = pow( (aL+aR-0.5*(SpHeatRatio-1.)*(vR-vL)) / (aL*pow(pL,
+                                                                  (1.-SpHeatRatio)/(2.*SpHeatRatio)) + aR*pow(pR,
+                                                                        (1.-SpHeatRatio)/(2.*SpHeatRatio)) ), 2.*SpHeatRatio/(SpHeatRatio-1.) );
+
+   double lambda1 = vL - aL * sqrt( 1. + (SpHeatRatio+1.)/(2.*SpHeatRatio) * max(
+                                       0., (p-pL)/pL) );
+   double lambda3 = vR + aR * sqrt( 1. + (SpHeatRatio+1.)/(2.*SpHeatRatio) * max(
+                                       0., (p-pR)/pR) );
+   return max(abs(lambda1), abs(lambda3));
+}
+
+double Euler::GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                           int i) const
+{
+   CheckAdmissibility(u);
+   switch (u.Size())
+   {
+      case 3:
+         return abs( u(1)*n(0) / u(0) ) + sqrt(SpHeatRatio * EvaluatePressure(u) / u(0));
+      case 4:
+         return abs( (u(1)*n(0) + u(2)*n(1)) / u(0) )
+                + sqrt(SpHeatRatio * EvaluatePressure(u) / u(0));
+      case 5:
+         return abs( (u(1)*n(0) + u(2)*n(1) + u(3)*n(2)) / u(0) )
+                + sqrt(SpHeatRatio * EvaluatePressure(u) / u(0));
+   }
+}
+
+void Euler::CheckAdmissibility(const Vector &u) const
+{
+   double RhoMin = 1.e-12;
+
+   if (u.Size() != NumEq) { MFEM_ABORT("Invalid solution vector."); }
+
+   if (u(0) < RhoMin)
+   {
+      ostringstream rho_str;
+      rho_str << u(0);
+      string err_msg = "Density too small rho = ";
+      MFEM_ABORT(err_msg << rho_str.str());
+   }
+}
+
+void Euler::SetBdrCond(const Vector &y1, Vector &y2, const Vector &normal,
+                       int attr) const
+{
+   switch (attr)
+   {
+      case -1: // wall boundary
+      {
+         if (dim == 1)
+         {
+            y2(0) = y1(0);
+            y2(1) = -y1(1);
+            y2(2) = y1(2);
+         }
+         else if (dim == 2)
+         {
+            double MomTimesNorm = y1(1) * normal(0) + y1(2) * normal(1);
+            y2(0) = y1(0);
+            y2(1) = y1(1) - 2. * MomTimesNorm * normal(0);
+            y2(2) = y1(2) - 2. * MomTimesNorm * normal(1);
+            y2(3) = y1(3);
+         }
+         else
+         {
+            double MomTimesNorm = y1(1) * normal(0) + y1(2) * normal(1) + y1(3) * normal(2);
+            y2(0) = y1(0);
+            y2(1) = y1(1) - 2. * MomTimesNorm * normal(0);
+            y2(2) = y1(2) - 2. * MomTimesNorm * normal(1);
+            y2(3) = y1(3) - 2. * MomTimesNorm * normal(2);
+            y2(4) = y1(4);
+         }
+         break;
+      }
+      case -2: // supersonic outlet
+      {
+         y2 = y1;
+         break;
+      }
+      case -3: // supersonic inlet
+      {
+         break;
+      }
+      // TODO subsonic in- and outlet
+      default:
+         MFEM_ABORT("Invalid boundary attribute.");
+   }
+}
+
+void Euler::ComputeDerivedQuantities(const GridFunction &u, GridFunction &d1,
+                                     GridFunction &d2) const
+{
+   double density, momentum;
+   const IntegrationRule ir = u.FESpace()->GetFE(0)->GetNodes();
+
+   for (int e = 0; e < ne; e++)
+   {
+      for (int i = 0; i < nd; i++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(i);
+         density = u.GetValue(e, ip, 1);
+         momentum = u.GetValue(e, ip, 2);
+         d1(e*nd + i) = pow(momentum / density, 2.0);
+
+         if (dim > 1)
+         {
+            momentum = u.GetValue(e, ip, 3);
+            d1(e*nd + i) += pow(momentum / density, 2.0);
+         }
+         if (dim > 2)
+         {
+            momentum = u.GetValue(e, ip, 4);
+            d1(e*nd + i) += pow(momentum / density, 2.0);
+         }
+
+         d2(e*nd + i) = (SpHeatRatio - 1.0) * (u.GetValue(e, ip,
+                                                          dim+2) - 0.5 * density * d1(e*nd + i));
+         d1(e*nd + i) = sqrt(d1(e*nd + i));
+      }
+   }
+}
+
+void Euler::ComputeErrors(Array<double> & errors, const GridFunction &u,
+                          double DomainSize, double t) const
+{
+   errors.SetSize(NumEq*3);
+   Vector component(dim+2);
+   VectorFunctionCoefficient uAnalytic(NumEq, AnalyticalSolutionEuler);
+
+   if (ConfigEuler.ConfigNum == 0) { uAnalytic.SetTime(0);  }
+   else { uAnalytic.SetTime(t); }
+
+   component = 0.0;
+   component(0) = 1.0;
+   VectorConstantCoefficient weight1(component);
+   errors[0] = u.ComputeLpError(1.0, uAnalytic, NULL, &weight1) / DomainSize;
+   errors[1] = u.ComputeLpError(2.0, uAnalytic, NULL, &weight1) / DomainSize;
+   errors[2] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic,
+                                NULL, &weight1);
+
+   component = 0.0;
+   component(1) = 1.0;
+   VectorConstantCoefficient weight2(component);
+   errors[3] = u.ComputeLpError(1.0, uAnalytic, NULL, &weight2) / DomainSize;
+   errors[4] = u.ComputeLpError(2.0, uAnalytic, NULL, &weight2) / DomainSize;
+   errors[5] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic,
+                                NULL, &weight2);
+
+   component = 0.0;
+   component(2) = 1.0;
+   VectorConstantCoefficient weight3(component);
+   errors[6] = u.ComputeLpError(1.0, uAnalytic, NULL, &weight3) / DomainSize;
+   errors[7] = u.ComputeLpError(2.0, uAnalytic, NULL, &weight3) / DomainSize;
+   errors[8] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic,
+                                NULL, &weight3);
+
+   if (dim > 1)
+   {
+      component = 0.0;
+      component(3) = 1.0;
+      VectorConstantCoefficient weight4(component);
+      errors[9] = u.ComputeLpError(1.0, uAnalytic, NULL, &weight4) / DomainSize;
+      errors[10] = u.ComputeLpError(2.0, uAnalytic, NULL, &weight4) / DomainSize;
+      errors[11] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic,
+                                    NULL, &weight4);
+   }
+
+   if (dim > 2)
+   {
+      component = 0.0;
+      component(4) = 1.0;
+      VectorConstantCoefficient weight5(component);
+      errors[12] = u.ComputeLpError(1.0, uAnalytic, NULL, &weight5) / DomainSize;
+      errors[13] = u.ComputeLpError(2.0, uAnalytic, NULL, &weight5) / DomainSize;
+      errors[14] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic,
+                                    NULL, &weight5);
+   }
+}
+
+void EvaluateEnergy(Vector &u, const double &pressure)
+{
+   const int dim = u.Size() - 2;
+   double aux = 0.0;
+   for (int l = 0; l < dim; l++)
+   {
+      aux += u(1+l)*u(1+l);
+   }
+   u(dim+1) = pressure / (SpHeatRatio - 1.0) + 0.5 * aux / u(0);
+}
+
+void AnalyticalSolutionEuler(const Vector &x, double t, Vector &u)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+
+   for (int i = 0; i < dim; i++)
+   {
+      switch (ConfigEuler.ConfigNum)
+      {
+         case 0:
+         case 5:
+         case 6: // Map to the reference domain [-1,1]^d.
+         {
+            double center = 0.5 * (ConfigEuler.bbMin(i) + ConfigEuler.bbMax(i));
+            double factor = 2.0 / (ConfigEuler.bbMax(i) - ConfigEuler.bbMin(i));
+            X(i) = factor * (x(i) - center);
+            t *= pow(factor, 1.0 / (double(dim)));
+            break;
+         }
+         case 3: // Map to the reference domain [0,1]^d.
+         {
+            double factor = 1.0 / (ConfigEuler.bbMax(i) - ConfigEuler.bbMin(i));
+            X(i) = factor * (x(i) - ConfigEuler.bbMin(i));
+            t *= pow(factor, 1.0 / (double(dim)));
+            break;
+         }
+      }
+   }
+
+   switch (ConfigEuler.ConfigNum)
+   {
+      case 0:
+      {
+         if (dim != 2) { MFEM_ABORT("Test case works only in 2D."); }
+
+         // Map to test case specific domain [-5,5]^d.
+         X *= 5.0;
+         t *= 5.0;
+
+         double beta = 5.0;
+         double r = X.Norml2();
+         double T0 = 1.0 - (SpHeatRatio - 1.0) * beta * beta
+                     / (8.0 * SpHeatRatio * M_PI * M_PI) * exp(1.0 - r*r);
+
+         u(0) = pow(T0, 1.0 / (SpHeatRatio - 1.0));
+         u(1) = (1.0 - beta / (2.0 * M_PI) * exp(0.5 * (1.0 - r*r)) * X(1)) * u(0);
+         u(2) = (1.0 + beta / (2.0 * M_PI) * exp(0.5 * (1.0 - r*r)) * X(0)) * u(0);
+         EvaluateEnergy(u, u(0) * T0);
+         break;
+      }
+      case 3:
+      {
+         if (dim != 2) { MFEM_ABORT("Test case works only in 2D."); }
+
+         // Map to test case specific domain [0,4] x [0,1].
+         X(0) = 4.0 * X(0);
+         t *= 2.0;
+
+         bool PostShock = X(0) < 1.0/6.0 + (X(1) + 20.0*t) / sqrt(3.0);
+
+         if (PostShock)
+         {
+            u(0) = 8.0;
+            u(1) =  66.0 * cos(M_PI / 6.0);
+            u(2) = -66.0 * sin(M_PI / 6.0);
+            EvaluateEnergy(u, 116.5);
+         }
+         else
+         {
+            u = 0.0;
+            u(0) = 1.4;
+            EvaluateEnergy(u, 1.0);
+         }
+
+         break;
+      }
+      case 5:
+      {
+         double r = X.Norml2();
+
+         if (r > t / 3.)
+         {
+            u(0) = 1.0 + t / r;
+            for (int l = 0; l < dim; l++) { u(l+1) = -X(l) / r * u(0); }
+            EvaluateEnergy(u, 1.0E-6);
+         }
+         else
+         {
+            u(0) = 1.0;
+            for (int l = 0; l < dim; l++) { u(l+1) = 0.0; }
+            EvaluateEnergy(u, 16.0 / 3.0);
+         }
+
+         break;
+      }
+      case 6:
+      {
+         if (dim != 2) { MFEM_ABORT("Test case works only in 2D."); }
+
+         double pressure = 3.0 + 4.0*log(2.0);
+         double r = X.Norml2();
+
+         u = 0.0;
+         u(0) = 1.0;
+
+         if (r < 0.2)
+         {
+            u(1) = -5.0 * X(1);
+            u(2) =  5.0 * X(0);
+            pressure = 5.0 + 12.5*r*r;
+         }
+         else if (r < 0.4)
+         {
+            u(1) = -(2.0 / r - 5.0) * X(1);
+            u(2) =  (2.0 / r - 5.0) * X(0);
+            pressure = 9.0 + 4.0 * (log(r) - log(0.2)) + 12.5*r*r - 20.0*r;
+         }
+
+         EvaluateEnergy(u, pressure);
+         break;
+      }
+      default:
+         MFEM_ABORT("Analytical solution not known.");
+   }
+}
+
+void InitialConditionEuler(const Vector &x, Vector &u)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+
+   for (int i = 0; i < dim; i++)
+   {
+      switch (ConfigEuler.ConfigNum)
+      {
+         case 4: // Map to the reference domain [-1,1]^d.
+         {
+            double center = 0.5 * (ConfigEuler.bbMin(i) + ConfigEuler.bbMax(i));
+            double factor = 2.0 / (ConfigEuler.bbMax(i) - ConfigEuler.bbMin(i));
+            X(i) = factor * (x(i) - center);
+            break;
+         }
+         case 1:
+         case 2: // Map to the reference domain [0,1]^d.
+         {
+            double factor = 1.0 / (ConfigEuler.bbMax(i) - ConfigEuler.bbMin(i));
+            X(i) = factor * (x(i) - ConfigEuler.bbMin(i));
+            break;
+         }
+      }
+   }
+
+   switch (ConfigEuler.ConfigNum)
+   {
+      case 0:
+      case 3:
+      case 5:
+      case 6:
+      {
+         AnalyticalSolutionEuler(x, 0.0, u);
+         break;
+      }
+      case 1:
+      {
+         if (dim != 1) { MFEM_ABORT("Test case works only in 1D."); }
+
+         u = 0.0;
+         u(0) = X.Norml2() < 0.5 ? 1.0 : 0.125;
+         EvaluateEnergy(u, X.Norml2() < 0.5 ? 1.0 : 0.1);
+         break;
+      }
+      case 2:
+      {
+         if (dim != 1) { MFEM_ABORT("Test case works only in 1D."); }
+
+         u = 0.0;
+         u(0) = 1.0;
+         if (X(0) < 0.1)
+         {
+            EvaluateEnergy(u, 1000.);
+         }
+         else if (X(0) < 0.9)
+         {
+            EvaluateEnergy(u, 0.01);
+         }
+         else
+         {
+            EvaluateEnergy(u, 100.);
+         }
+         break;
+      }
+      case 4:
+      {
+         // Map to test case specific domain [-5,5]^d.
+         X *= 5.0;
+
+         u = 0.0;
+         u(0) = 1.0;
+         // TODO make sure that energy is essentially a delta distribution.
+         u(dim+1) = X.Norml2() < 1.0E-1 ? 1000.0 : 1.0E-8;
+         break;
+      }
+      case 7:
+      {
+         u = 0.0;
+         u(0) = 1.0;
+         u(1) = 1.0;
+         EvaluateEnergy(u, 0.1);
+         break;
+      }
+   }
+}
+
+void InflowFunctionEuler(const Vector &x, double t, Vector &u)
+{
+   switch (ConfigEuler.ConfigNum)
+   {
+      case 0:
+      case 3:
+      case 5:
+      case 6:
+      {
+         AnalyticalSolutionEuler(x, t, u);
+         break;
+      }
+      case 7:
+      {
+         InitialConditionEuler(x, u);
+         break;
+      }
+      case 1:
+      case 2:
+      case 4: break; // No boundary conditions needed.
+   }
+}

miniapps/hypsys/apps/euler.hpp

@@ -0,0 +1,30 @@
+#ifndef HYPSYS_EULER
+#define HYPSYS_EULER
+
+#include "hyperbolic_system.hpp"
+
+class Euler : public HyperbolicSystem
+{
+public:
+   explicit Euler(FiniteElementSpace *fes_, BlockVector &u_block,
+                  Configuration &config_);
+   ~Euler() { };
+
+   virtual double EvaluatePressure(const Vector &u) const;
+
+   virtual void EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i = -1) const;
+   virtual double GetGMS(const Vector &uL, const Vector &uR,
+                         const Vector &normal) const override;
+   virtual double GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i) const;
+   virtual void CheckAdmissibility(const Vector &u) const override;
+   virtual void SetBdrCond(const Vector &y1, Vector &y2, const Vector &normal,
+                           int attr) const override;
+   virtual void ComputeDerivedQuantities(const GridFunction &u, GridFunction &d1,
+                                         GridFunction &d2) const override;
+   virtual void ComputeErrors(Array<double> &errors, const GridFunction &u,
+                              double DomainSize, double t) const override;
+};
+
+#endif

miniapps/hypsys/apps/hyperbolic_system.hpp

@@ -0,0 +1,140 @@
+#ifndef HYPSYS_HYPERBOLIC_SYSTEM
+#define HYPSYS_HYPERBOLIC_SYSTEM
+
+#include "../lib/tools.hpp"
+
+struct Configuration
+{
+   int ConfigNum;
+   double tFinal;
+   Vector bbMin, bbMax;
+};
+
+class HyperbolicSystem
+{
+public:
+   explicit HyperbolicSystem(FiniteElementSpace *fes_, BlockVector &u_block,
+                             int NumEq_, Configuration &config_,
+                             VectorFunctionCoefficient BdrCond_) : fes(fes_), u0(fes_, u_block),
+      NumEq(NumEq_), BdrCond(BdrCond_)
+   {
+      ne = fes->GetNE();
+      nd = fes->GetFE(0)->GetDof();
+      dim = fes->GetMesh()->Dimension();
+
+      l2_fec = new L2_FECollection(fes->GetFE(0)->GetOrder(),
+                                   fes->GetMesh()->Dimension());
+      l2_fes = new FiniteElementSpace(fes->GetMesh(), l2_fec, NumEq,
+                                      Ordering::byNODES);
+      l2_proj = new GridFunction(l2_fes);
+   }
+
+   virtual ~HyperbolicSystem()
+   {
+      delete l2_proj;
+      delete l2_fes;
+      delete l2_fec;
+   }
+
+   virtual void EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i = -1) const = 0;
+   virtual double GetGMS(const Vector &uL, const Vector &uR,
+                         const Vector &normal) const { } // TODO: if used "= 0"
+   virtual double GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i = -1) const = 0;
+   virtual void CheckAdmissibility(const Vector &u) const { };
+   virtual void SetBdrCond(const Vector &y1, Vector &y2, const Vector &normal,
+                           int attr) const  { };
+   virtual void ComputeDerivedQuantities(const GridFunction &u, GridFunction &d1,
+                                         GridFunction &d2) const { };
+   virtual void ComputeErrors(Array<double> &errors, const GridFunction &u,
+                              double DomainSize, double t) const { };
+
+   virtual void WriteErrors(const Array<double> &errors) const
+   {
+      ofstream file("errors.txt", ios_base::app);
+
+      if (!file)
+      {
+         MFEM_ABORT("Error opening file.");
+      }
+      else
+      {
+         ostringstream strs;
+         for (int i = 0; i < errors.Size(); i++)
+         {
+            strs << errors[i] << " ";
+         }
+         strs << "\n";
+         string str = strs.str();
+         file << str;
+         file.close();
+      }
+   }
+
+   // L2 projection for scalar problems.
+   void L2_Projection(FunctionCoefficient fun, GridFunction &proj) const
+   {
+      l2_proj->ProjectCoefficient(fun);
+      proj.ProjectGridFunction(*l2_proj);
+   }
+
+   // L2 projection for systems.
+   void L2_Projection(VectorFunctionCoefficient fun, GridFunction &proj) const
+   {
+      l2_proj->ProjectCoefficient(fun);
+      proj.ProjectGridFunction(*l2_proj);
+   }
+
+   // Lumped L2 projection for general problems.
+   void LumpedL2_Projection(VectorFunctionCoefficient fun, GridFunction &proj) const
+   {
+      Vector LumpedMassMat;
+
+      Vector aux_vec(NumEq);
+      aux_vec = 1.0;
+      VectorConstantCoefficient ones(aux_vec);
+      BilinearForm ml(fes);
+      ml.AddDomainIntegrator(new LumpedIntegrator(new VectorMassIntegrator(ones)));
+      ml.Assemble();
+      ml.Finalize();
+      ml.SpMat().GetDiag(LumpedMassMat);
+
+      LinearForm rhs(fes);
+      rhs.AddDomainIntegrator(new VectorDomainLFIntegrator(fun));
+      rhs.Assemble();
+
+      for (int i = 0; i < LumpedMassMat.Size(); i++)
+      {
+         proj(i) = rhs.Elem(i) / LumpedMassMat(i);
+      }
+   }
+
+   int ne, nd, dim;
+
+   // 0: L2 projection,
+   // 1: Nodal values as GridFunction coefficients (only second order accurate).
+   int ProjType;
+   const int NumEq;
+
+   FiniteElementSpace *fes;
+   GridFunction u0;
+
+   // Auxiliary data needed for L2 projections
+   L2_FECollection *l2_fec;
+   FiniteElementSpace *l2_fes;
+   GridFunction *l2_proj;
+
+   mutable VectorFunctionCoefficient BdrCond;
+
+   string ProblemName, glvis_scale;
+   bool SolutionKnown;
+   bool SteadyState;
+   bool TimeDepBC;
+
+   // Currently only true for advection, due to spatially dependent flux.
+   bool DiscreteUpwinding = false;
+   DenseTensor VelNode;
+};
+
+#endif

miniapps/hypsys/apps/kpp.cpp

@@ -0,0 +1,121 @@
+#include "kpp.hpp"
+
+Configuration ConfigKPP;
+
+void InitialConditionKPP(const Vector &x, Vector &u);
+void InflowFunctionKPP(const Vector &x, double t, Vector &u);
+
+KPP::KPP(FiniteElementSpace *fes_, BlockVector &u_block,
+         Configuration &config_)
+   : HyperbolicSystem(fes_, u_block, 1, config_,
+                      VectorFunctionCoefficient(1, InflowFunctionKPP))
+{
+   ConfigKPP = config_;
+   VectorFunctionCoefficient ic(NumEq, InitialConditionKPP);
+
+   switch (ConfigKPP.ConfigNum)
+   {
+      case 1:
+      {
+         ProblemName = "KPP Equation - 2D Spiral";
+         glvis_scale = "on";
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 2:
+      {
+         ProblemName = "KPP Equation - 1D";
+         glvis_scale = "on";
+         SolutionKnown =
+            false; // There is a solution, but I don't have an analytical expression.
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      default:
+         MFEM_ABORT("No such test case implemented.");
+   }
+}
+
+void KPP::EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                       int e, int k, int i) const
+{
+   if (dim==1)
+   {
+      double coef = u(0)*(1.0-u(0));
+      FluxEval(0,0) = u(0) < 0.5 ? (0.25*coef) : (0.1875 - 0.5*coef);
+   }
+   else if (dim==2)
+   {
+      FluxEval(0,0) = sin(u(0));
+      FluxEval(0,1) = cos(u(0));
+   }
+   else { MFEM_ABORT("Not implemented."); }
+}
+
+double KPP::GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                         int i) const
+{
+   return 1.0; // Tighter bound exists.
+}
+
+void InitialConditionKPP(const Vector &x, Vector &u)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+
+   for (int i = 0; i < dim; i++)
+   {
+      switch (ConfigKPP.ConfigNum)
+      {
+         case 1: // Map to the reference domain [-1,1]^d.
+         {
+            double center = 0.5 * (ConfigKPP.bbMin(i) + ConfigKPP.bbMax(i));
+            X(i) = 2.0 * (x(i) - center) / (ConfigKPP.bbMax(i) - ConfigKPP.bbMin(i));
+            break;
+         }
+         case 2: // Map to the reference domain [0,1]^d.
+         {
+            X(i) = (x(i) - ConfigKPP.bbMin(i)) / (ConfigKPP.bbMax(i) - ConfigKPP.bbMin(i));
+            break;
+         }
+      }
+   }
+
+   switch (ConfigKPP.ConfigNum)
+   {
+      case 1:
+      {
+         // Map to test case specific domain [-2,2] x [-2.5,1.5].
+         X *= 2.0;
+         X(1) -= 0.5;
+
+         u(0) = X.Norml2() <= 1. ? 3.5 * M_PI : 0.25 * M_PI;
+         break;
+      }
+      case 2:
+      {
+         u(0) = X.Norml2() <= 0.25 ? 0.0 :
+                1.0; // According to the original KPP paper, not Ern and Guermond.
+         break;
+      }
+   }
+}
+
+void InflowFunctionKPP(const Vector &x, double t, Vector &u)
+{
+   switch (ConfigKPP.ConfigNum)
+   {
+      case 1: { u(0) = 0.25 * M_PI;  break; }
+
+      // This definition is consistent with the problem and assures correct
+      // handling of inflow (left) and outflow (right) boundaries.
+      case 2: { u(0) = x(0); break; }
+   }
+}

miniapps/hypsys/apps/kpp.hpp

@@ -0,0 +1,19 @@
+#ifndef HYPSYS_KPP
+#define HYPSYS_KPP
+
+#include "hyperbolic_system.hpp"
+
+class KPP : public HyperbolicSystem
+{
+public:
+   explicit KPP(FiniteElementSpace *fes_, BlockVector &u_block,
+                Configuration &config_);
+   ~KPP() { };
+
+   virtual void EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i = -1) const;
+   virtual double GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i) const;
+};
+
+#endif

miniapps/hypsys/apps/lib.hpp

@@ -0,0 +1,11 @@
+#ifndef HYPSYS_APPS_LIB
+#define HYPSYS_APPS_LIB
+
+#include "advection.hpp"
+#include "burgers.hpp"
+#include "kpp.hpp"
+#include "buckley_leverett.hpp"
+#include "shallowwater.hpp"
+#include "euler.hpp"
+
+#endif

miniapps/hypsys/apps/shallowwater.cpp

@@ -0,0 +1,463 @@
+#include "shallowwater.hpp"
+
+Configuration ConfigSWE;
+
+double GravConst;
+double Depth;
+
+void AnalyticalSolutionSWE(const Vector &x, double t, Vector &u);
+void InitialConditionSWE(const Vector &x, Vector &u);
+void InflowFunctionSWE(const Vector &x, double t, Vector &u);
+
+ShallowWater::ShallowWater(FiniteElementSpace *fes_, BlockVector &u_block,
+                           Configuration &config_)
+   : HyperbolicSystem(fes_, u_block, fes_->GetMesh()->Dimension() + 1, config_,
+                      VectorFunctionCoefficient(fes_->GetMesh()->Dimension() + 1,
+                                                InflowFunctionSWE))
+{
+   ConfigSWE = config_;
+   VectorFunctionCoefficient ic(NumEq, InitialConditionSWE);
+
+   switch (ConfigSWE.ConfigNum)
+   {
+      case 0:
+      {
+         // Periodic meshes must be used for this problem.
+         ProblemName = "Shallow Water Equations - Vorticity Advection";
+         glvis_scale = "on";
+         GravConst = 1.0;
+         Depth = 1.0;
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 0;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 1:
+      {
+         ProblemName = "Shallow Water Equations - Dam Break";
+         glvis_scale = "on";
+         GravConst = 9.81;
+         Depth = 1.0;
+         SolutionKnown = true;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 2:
+      {
+         ProblemName = "Shallow Water Equations - Radial Dam Break";
+         glvis_scale = "off valuerange 0.1 1";
+         GravConst = 9.81;
+         Depth = 0.1;
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 3:
+      {
+         ProblemName = "Shallow Water Equations - Constricted Channel";
+         glvis_scale = "on";
+         GravConst = 0.16;
+         Depth = 1.0;
+         SolutionKnown = false;
+         SteadyState = true;
+         TimeDepBC = false;
+         ProjType = 0;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      case 4:
+      {
+         ProblemName = "Shallow Water Equations - MoST Gimmick";
+         glvis_scale = "on";
+         GravConst = 1.0;
+         SolutionKnown = false;
+         SteadyState = false;
+         TimeDepBC = false;
+         ProjType = 1;
+
+         Mesh *mesh = fes->GetMesh();
+         const int nd = fes->GetFE(0)->GetDof();
+         const int ne = fes->GetNE();
+         if (mesh->Dimension() != 2) { MFEM_ABORT("Test case works only in 2D."); }
+         u0 = 0.;
+
+         for (int e = 0; e < ne; e++)
+         {
+            int id = mesh->GetElement(e)->GetAttribute();
+            for (int j = 0; j < nd; j++)
+            {
+               switch (id)
+               {
+                  case 1:
+                  {
+                     u0(e*nd+j) = 1.;
+                     break;
+                  }
+                  case 2:
+                  case 3:
+                  case 4:
+                  {
+                     u0(e*nd+j) = 0.125;
+                     break;
+                  }
+                  default:
+                     MFEM_ABORT("Too many element IDs.");
+               }
+            }
+         }
+
+         break;
+      }
+      default:
+         MFEM_ABORT("No such test case implemented.");
+   }
+}
+
+void ShallowWater::EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                                int e, int k, int i) const
+{
+   CheckAdmissibility(u);
+   switch (dim)
+   {
+      case 1:
+      {
+         FluxEval(0,0) = u(1);
+         FluxEval(1,0) = u(1) * u(1) / u(0) + 0.5 * GravConst * u(0) * u(0);
+         break;
+      }
+      case 2:
+      {
+         double vx = u(1) / u(0);
+         double vy = u(2) / u(0);
+         double gravitation = 0.5 * GravConst * u(0) * u(0);
+
+         FluxEval(0,0) = u(1);
+         FluxEval(0,1) = u(2);
+
+         FluxEval(1,0) = u(1) * vx + gravitation;
+         FluxEval(1,1) = u(1) * vy;
+
+         FluxEval(2,0) = u(2) * vx;
+         FluxEval(2,1) = u(2) * vy + gravitation;
+         break;
+      }
+      default:
+         MFEM_ABORT("Invalid space dimension.");
+   }
+}
+
+double ShallowWater::GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                                  int i) const
+{
+   CheckAdmissibility(u);
+   switch (u.Size())
+   {
+      case 2:
+         return abs( u(1)*n(0) / u(0) ) + sqrt(GravConst * u(0));
+      case 3:
+         return abs( (u(1)*n(0) + u(2)*n(1)) / u(0) ) + sqrt(GravConst * u(0));
+   }
+}
+
+void ShallowWater::CheckAdmissibility(const Vector &u) const
+{
+   double HMin = 1.e-12;
+
+   if (u.Size() != NumEq) { MFEM_ABORT("Invalid solution vector."); }
+
+   if (u(0) < HMin)
+   {
+      ostringstream height_str;
+      height_str << u(0);
+      string err_msg = "Water height too small H = ";
+      MFEM_ABORT(err_msg << height_str.str() << ".");
+   }
+}
+
+void ShallowWater::SetBdrCond(const Vector &y1, Vector &y2,
+                              const Vector &normal, int attr) const
+{
+   switch (attr)
+   {
+      case -1: // Land boundary
+      {
+         if (normal.Size() == 1)
+         {
+            y2(0) = y1(0);
+            y2(1) = -y1(1);
+         }
+         else
+         {
+            double MomTimesNor = y1(1) * normal(0) + y1(2) * normal(1);
+            y2(0) = y1(0);
+            y2(1) = y1(1) - 2. * MomTimesNor * normal(0);
+            y2(2) = y1(2) - 2. * MomTimesNor * normal(1);
+         }
+         break;
+      }
+      case -2: // Radiation boundary
+      {
+         y2 = y1;
+         break;
+      }
+      case -3: // River boundary
+      {
+         break;
+      }
+      case -4: // Open sea boundary
+      {
+         double tmp = y2(0);
+         y2 = y1;
+         y2(0) = tmp;
+         break;
+      }
+      default:
+         MFEM_ABORT("Invalid boundary attribute.");
+   }
+}
+
+void ShallowWater::ComputeDerivedQuantities(const GridFunction &u,
+                                            GridFunction &d1, GridFunction &d2) const
+{
+   double height, momentum;
+   const IntegrationRule ir = u.FESpace()->GetFE(0)->GetNodes();
+
+   for (int e = 0; e < ne; e++)
+   {
+      for (int i = 0; i < nd; i++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(i);
+         height = u.GetValue(e, ip, 1);
+         momentum = u.GetValue(e, ip, 2);
+         d1(e*nd + i) = pow(momentum / height, 2.0);
+
+         if (dim==2)
+         {
+            momentum = u.GetValue(e, ip, 3);
+            d1(e*nd + i) += pow(momentum / height, 2.0);
+         }
+         d1(e*nd + i) = sqrt(d1(e*nd + i));
+      }
+   }
+}
+
+void ShallowWater::ComputeErrors(Array<double> &errors, const GridFunction &u,
+                                 double DomainSize, double t) const
+{
+   errors.SetSize(NumEq*3);
+   Vector component(dim+1);
+   VectorFunctionCoefficient uAnalytic(NumEq, AnalyticalSolutionSWE);
+
+   if (ConfigSWE.ConfigNum == 0) { uAnalytic.SetTime(0); }
+   else { uAnalytic.SetTime(t); }
+
+   component = 0.0;
+   component(0) = 1.0;
+   VectorConstantCoefficient weight1(component);
+   errors[0] = u.ComputeLpError(1.0, uAnalytic, NULL, &weight1) / DomainSize;
+   errors[1] = u.ComputeLpError(2.0, uAnalytic, NULL, &weight1) / DomainSize;
+   errors[2] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic,
+                                NULL, &weight1);
+
+   component = 0.0;
+   component(1) = 1.0;
+   VectorConstantCoefficient weight2(component);
+   errors[3] = u.ComputeLpError(1.0, uAnalytic, NULL, &weight2) / DomainSize;
+   errors[4] = u.ComputeLpError(2.0, uAnalytic, NULL, &weight2) / DomainSize;
+   errors[5] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic,
+                                NULL, &weight2);
+
+   if (dim ==  2)
+   {
+      component = 0.0;
+      component(2) = 1.0;
+      VectorConstantCoefficient weight3(component);
+      errors[6] = u.ComputeLpError(1.0, uAnalytic, NULL, &weight3) / DomainSize;
+      errors[7] = u.ComputeLpError(2.0, uAnalytic, NULL, &weight3) / DomainSize;
+      errors[8] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic,
+                                   NULL, &weight3);
+   }
+}
+
+
+void AnalyticalSolutionSWE(const Vector &x, double t, Vector &u)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+
+   for (int i = 0; i < dim; i++)
+   {
+      switch (ConfigSWE.ConfigNum)
+      {
+         case 0: // Map to the reference domain [-1,1]^d.
+         {
+            double center = 0.5 * (ConfigSWE.bbMin(i) + ConfigSWE.bbMax(i));
+            double factor = 2.0 / (ConfigSWE.bbMax(i) - ConfigSWE.bbMin(i));
+            X(i) = factor * (x(i) - center);
+            t *= pow(factor, 1.0 / (double(dim)));
+            break;
+         }
+         case 1: // Map to the reference domain [0,1]^d.
+         {
+            double factor = 1.0 / (ConfigSWE.bbMax(i) - ConfigSWE.bbMin(i));
+            X(i) = factor * (x(i) - ConfigSWE.bbMin(i));
+            t *= pow(factor, 1.0 / (double(dim)));
+            break;
+         }
+      }
+   }
+
+   switch (ConfigSWE.ConfigNum)
+   {
+      case 0:
+      {
+         if (dim != 2) { MFEM_ABORT("Test case works only in 2D."); }
+
+         // Map to test case specific domain [-50,50].
+         X *= 50.;
+         t *= 50.;
+
+         double M = sqrt(2);
+         double c1 = -0.1;
+         double c2 = 0.005;
+         double a = M_PI / 4.0;
+         double x0 = 0.0;
+         double y0 = 0.0;
+
+         double f = -c2 * ( pow(X(0) - x0 - M*t*cos(a), 2.0)
+                            + pow(X(1) - y0 - M*t*sin(a), 2.0) );
+
+         u(0) = 1.0;
+         u(1) = M*cos(a) + c1 * (X(1) - y0 - M*t*sin(a)) * exp(f);
+         u(2) = M*sin(a) - c1 * (X(0) - x0 - M*t*cos(a)) * exp(f);
+         u *= Depth - c1*c1 / (4.0*c2*GravConst) * exp(2.0*f);
+
+         break;
+      }
+      case 1:
+      {
+         // Map to test case specific domain [0,1000]^d.
+         X *= 1000;
+         t *= 1000;
+
+         double r =  X(0);
+         u = 0.;
+
+         if (t==0)
+         {
+            u(0) = r < 500.0 ? Depth + 9.0 : Depth;
+            return;
+         }
+
+         double cm = 6.23416;
+         double aux = sqrt(10.0 * GravConst);
+         double xA = 500.0 - t*aux;
+         double xB = 500.0 + t*(2.0*aux - 3.0*cm);
+         double xC = 500.0 + t*(2.0*cm*cm*(aux - cm))/(cm*cm - GravConst);
+
+         u(0) = 9.0 * (r<xA) + (4.0/(9.0*GravConst) * pow( aux - (r-500.0)/(2.0*t),
+                                                           2.0 ) - 1.0) * (r>=xA) * (r<xB)
+                + (cm*cm/GravConst - 1.) * (r>=xB) * (r<xC);
+         u(1) = 2.0/3.0 * ((r-500.0)/t + aux) * (r >= xA) * (r < xB) + 2.0 *
+                (aux - cm) * (r >= xB) * (r < xC);
+         u(0) += Depth;
+         u(1) *= u(0);
+
+         break;
+      }
+      case 3:
+      {
+         if (dim != 2) { MFEM_ABORT("Test case works only in 2D."); }
+
+         const double x1[2]= {-10., 0.}, x2[2]= {-10., 40.},
+                                                x3[2]= {53.8622, 5.5872}, x4[2]= {53.8622, 34.4128},
+                                                                                 slope0=0.53886, slope1=0.79893;
+         int sign_top, sign_bot;
+
+         if (x(0)>x3[0])
+         {
+            sign_top = -(x(0)-x4[0])*slope1-(x(1)-x4[1])>0 ? 1 : -1;
+            sign_bot = (x(0)-x3[0])*slope1-(x(1)-x3[1])>0 ? 1 : -1;
+            u(0) = sign_top*sign_bot>0. ? 0.8350436: 0.5273361;
+         }
+         else
+         {
+            sign_top = -(x(0)-x2[0])*slope0-(x(1)-x2[1])>0 ? 1 : -1;
+            sign_bot = (x(0)-x1[0])*slope0-(x(1)-x1[1])>0 ? 1 : -1;
+            u(0) = sign_top*sign_bot>0. ? 0.250133 : (sign_top>0 ? 1. : 0.5273361);
+         }
+
+         u(0) += Depth;
+         break;
+      }
+   }
+}
+
+void InitialConditionSWE(const Vector &x, Vector &u)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+
+   // Map to the reference domain [-1,1]^d.
+   for (int i = 0; i < dim; i++)
+   {
+      double center = 0.5 * (ConfigSWE.bbMin(i) + ConfigSWE.bbMax(i));
+      double factor = 2.0 / (ConfigSWE.bbMax(i) - ConfigSWE.bbMin(i));
+      X(i) = factor * (x(i) - center);
+   }
+
+   switch (ConfigSWE.ConfigNum)
+   {
+      case 0:
+      case 1:
+      {
+         AnalyticalSolutionSWE(x, 0., u);
+         break;
+      }
+      case 2:
+      {
+         u = 0.0;
+         u(0) = X.Norml2() < 0.5 ? 0.9 : 0.0;
+         u(0) += Depth;
+         break;
+      }
+      case 3:
+      {
+         u(0) = Depth;
+         u(1) = Depth;
+         u(2) = 0.;
+         break;
+      }
+   }
+}
+
+void InflowFunctionSWE(const Vector &x, double t, Vector &u)
+{
+   switch (ConfigSWE.ConfigNum)
+   {
+      case 0:
+      case 2:
+      {
+         // Do not impose inflow values in this setup.
+         break;
+      }
+      case 1:
+      {
+         AnalyticalSolutionSWE(x, 0., u);
+         break;
+      }
+      case 3:
+      {
+         InitialConditionSWE(x, u);
+         break;
+      }
+   }
+}

miniapps/hypsys/apps/shallowwater.hpp

@@ -0,0 +1,26 @@
+#ifndef HYPSYS_SHALLOWWATER
+#define HYPSYS_SHALLOWWATER
+
+#include "hyperbolic_system.hpp"
+
+class ShallowWater : public HyperbolicSystem
+{
+public:
+   explicit ShallowWater(FiniteElementSpace *fes_, BlockVector &u_block,
+                         Configuration &config_);
+   ~ShallowWater() { };
+
+   virtual void EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i = -1) const;
+   virtual double GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i) const;
+   virtual void CheckAdmissibility(const Vector &u) const override;
+   virtual void SetBdrCond(const Vector &y1, Vector &y2, const Vector &normal,
+                           int attr) const override;
+   virtual void ComputeDerivedQuantities(const GridFunction &u, GridFunction &d1,
+                                         GridFunction &d2) const override;
+   virtual void ComputeErrors(Array<double> &errors, const GridFunction &u,
+                              double DomainSize, double t) const override;
+};
+
+#endif

miniapps/hypsys/apps/template.cpp

@@ -0,0 +1,115 @@
+#include "template.hpp"
+
+Configuration ConfigTEMPLATE;
+
+void AnalyticalSolutionTEMPLATE(const Vector &x, double t, Vector &u);
+void InitialConditionTEMPLATE(const Vector &x, Vector &u);
+void InflowFunctionTEMPLATE(const Vector &x, double t, Vector &u);
+
+TEMPLATE::TEMPLATE(FiniteElementSpace *fes_, BlockVector &u_block,
+                   Configuration &config_)
+   : HyperbolicSystem(fes_, u_block, NUMEQ, config_,
+                      VectorFunctionCoefficient(NUMEQ, InflowFunctionTEMPLATE))
+{
+   ConfigTEMPLATE = config_;
+   VectorFunctionCoefficient ic(NumEq, InitialConditionTEMPLATE);
+
+   switch (ConfigTEMPLATE.ConfigNum)
+   {
+      case 0:
+      {
+         ProblemName = "TEMPLATE - ";
+         glvis_scale = "on";
+         SolutionKnown = ;
+         SteadyState = ;
+         TimeDepBC = ;
+         ProjType = 0;
+         L2_Projection(ic, u0);
+         break;
+      }
+      case 1:
+      {
+         ProblemName = "TEMPLATE - ";
+         glvis_scale = "off valuerange 0 1";
+         SolutionKnown = ;
+         SteadyState = ;
+         TimeDepBC = ;
+         ProjType = 1;
+         u0.ProjectCoefficient(ic);
+         break;
+      }
+      default:
+         MFEM_ABORT("No such test case implemented.");
+   }
+}
+
+void TEMPLATE::EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                            int e, int k, int i) const
+{
+   // TODO
+}
+
+double TEMPLATE::GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                              int i) const
+{
+   //TODO
+   return 0.;
+}
+
+void TEMPLATE::SetBdrCond(const Vector &y1, Vector &y2, const Vector &normal,
+                          int attr) const
+{
+   //TODO
+}
+
+void TEMPLATE::ComputeErrors(Array<double> &errors, const GridFunction &u,
+                             double DomainSize, double t) const
+{
+   errors.SetSize(3);
+   VectorFunctionCoefficient uAnalytic(NumEq, AnalyticalSolutionTEMPLATE);
+   uAnalytic.SetTime(t);
+   errors[0] = u.ComputeLpError(1., uAnalytic) / DomainSize;
+   errors[1] = u.ComputeLpError(2., uAnalytic) / DomainSize;
+   errors[2] = u.ComputeLpError(numeric_limits<double>::infinity(), uAnalytic);
+}
+
+
+void AnalyticalSolutionTEMPLATE(const Vector &x, double t, Vector &u)
+{
+   const int dim = x.Size();
+   Vector X(dim);
+
+   for (int i = 0; i < dim; i++)
+   {
+      switch (ConfigTEMPLATE.ConfigNum)
+      {
+         case /* TODO */: // Map to the reference domain [-1,1]^d.
+         {
+            double center = 0.5 * (ConfigTEMPLATE.bbMin(i) + ConfigTEMPLATE.bbMax(i));
+            double factor = 2.0 / (ConfigTEMPLATE.bbMax(i) - ConfigTEMPLATE.bbMin(i));
+            X(i) = factor * (x(i) - center);
+            t *= pow(factor, 1.0 / (double(dim)));
+            break;
+         }
+         case /* TODO */: // Map to the reference domain [0,1]^d.
+         {
+            double factor = 1.0 / (ConfigTEMPLATE.bbMax(i) - ConfigTEMPLATE.bbMin(i));
+            X(i) = factor * (x(i) - ConfigTEMPLATE.bbMin(i));
+            t *= pow(factor, 1.0 / (double(dim)));
+            break;
+         }
+      }
+   }
+
+   // TODO
+}
+
+void InitialConditionTEMPLATE(const Vector &x, Vector &u)
+{
+   // TODO
+}
+
+void InflowFunctionTEMPLATE(const Vector &x, double t, Vector &u)
+{
+   // TODO
+}

miniapps/hypsys/apps/template.hpp

@@ -0,0 +1,24 @@
+#ifndef HYPSYS_TEMPLATE
+#define HYPSYS_TEMPLATE
+
+#include "hyperbolic_system.hpp"
+
+class TEMPLATE : public HyperbolicSystem
+{
+public:
+   explicit TEMPLATE(FiniteElementSpace *fes_, BlockVector &u_block,
+                     Configuration &config_);
+   ~TEMPLATE() { };
+
+   virtual void EvaluateFlux(const Vector &u, DenseMatrix &FluxEval,
+                             int e, int k, int i = -1) const;
+   virtual double GetWaveSpeed(const Vector &u, const Vector n, int e, int k,
+                               int i) const;
+   virtual void SetBdrCond(const Vector &y1, Vector &y2, const Vector &normal,
+                           int attr) const;
+   virtual void ComputeDerivedQuantities(const Vector &u) const { };
+   virtual void ComputeErrors(Array<double> &errors, const GridFunction &u,
+                              double DomainSize, double t) const;
+};
+
+#endif

miniapps/hypsys/config/advection.sh

@@ -0,0 +1,32 @@
+EXEC="mpirun -np 7 phypsys"
+
+SCHEME=1
+MESH="data/periodic-4segment.mesh"
+MESH0="data/periodic-3segment.mesh"
+CONFIG="-p 0 -c 4 -vf 100 -tf 1 -s 3 -dt 0.001 -m $MESH"
+
+$EXEC $CONFIG -es $SCHEME -o 1 -r 5
+$EXEC $CONFIG -es $SCHEME -o 3 -r 4
+$EXEC $CONFIG -es $SCHEME -o 7 -r 3
+
+
+# GRID CONVERGENCE TEST
+ORDER=1
+DT=0.0004
+# ORDER=2
+# DT=0.00025
+# ORDER=3
+# DT=0.0001
+# ORDER=4
+# DT=0.000025
+CONFIG0="-p 0 -c 3 -vf 1000 -tf 1 -s 3 -dt $DT -m $MESH0 -o $ORDER"
+CONFIG1="-p 0 -c 3 -vf 1000 -tf 1 -s 3 -dt $DT -m $MESH -o $ORDER"
+
+# rm errors.txt
+# $EXEC $CONFIG0 -r 4 -es $SCHEME
+# $EXEC $CONFIG1 -r 4 -es $SCHEME
+# $EXEC $CONFIG0 -r 5 -es $SCHEME
+# $EXEC $CONFIG1 -r 5 -es $SCHEME
+# $EXEC $CONFIG0 -r 6 -es $SCHEME
+# $EXEC $CONFIG1 -r 6 -es $SCHEME
+# $EXEC $CONFIG0 -r 7 -es $SCHEME

miniapps/hypsys/config/burgers.sh

@@ -0,0 +1,43 @@
+EXEC="mpirun -np 7 phypsys"
+
+SCHEME=0
+MESH=data/inline-4quad.mesh
+CONFIG="-p 1 -c 1 -vf 100 -tf 0.5 -s 3 -dt 0.0005 -m $MESH"
+
+$EXEC $CONFIG -es $SCHEME -o 0 -r 5
+$EXEC $CONFIG -es $SCHEME -o 1 -r 4
+
+SCHEME=1
+
+$EXEC $CONFIG -es $SCHEME -o  1 -r 4
+$EXEC $CONFIG -es $SCHEME -o  3 -r 3
+$EXEC $CONFIG -es $SCHEME -o  7 -r 2
+$EXEC $CONFIG -es $SCHEME -o 15 -r 1
+
+
+# # GRID CONVERGENCE TEST
+# SCHEME=1
+ORDER=0
+DT=0.002
+# ORDER=1
+# DT=0.0004
+# # ORDER=2
+# # DT=0.00016
+# # ORDER=3
+# # DT=-dt 0.0001
+# # ORDER=4
+# # DT=0.00005
+# MESH="data/periodic-4segment.mesh"
+# MESH0="data/periodic-3segment.mesh"
+# CONFIG0="-p 1 -c 0 -vf 1000 -tf 0.1 -s 3 -dt $DT -m $MESH0 -o $ORDER"
+# CONFIG1="-p 1 -c 0 -vf 1000 -tf 0.1 -s 3 -dt $DT -m $MESH -o $ORDER"
+
+# rm errors.txt
+# $EXEC $CONFIG0 -r 4 -es $SCHEME
+# $EXEC $CONFIG1 -r 4 -es $SCHEME
+# $EXEC $CONFIG0 -r 5 -es $SCHEME
+# $EXEC $CONFIG1 -r 5 -es $SCHEME
+# $EXEC $CONFIG0 -r 6 -es $SCHEME
+# $EXEC $CONFIG1 -r 6 -es $SCHEME
+# $EXEC $CONFIG0 -r 7 -es $SCHEME
+# $EXEC $CONFIG1 -r 7 -es $SCHEME

miniapps/hypsys/config/debug.sh

@@ -0,0 +1,7 @@
+MESH=data/wall-bdr-100segment.mesh
+CONFIG="-p 3 -c 1 -vf 1000 -tf 0.02 -s 1 -m $MESH -r 0 -es 1"
+
+./hypsys $CONFIG -o 3 -dt 0.00005
+mv ultimate.gf cmp.gf
+mpirun -np 3 phypsys $CONFIG -o 3 -dt 0.00005
+meld ultimate.gf cmp.gf

miniapps/hypsys/config/euler.sh

@@ -0,0 +1,55 @@
+EXEC="mpirun -np 7 phypsys"
+
+SCHEME=1
+MESH=data/wall-bdr-4segment.mesh
+
+## SOD Shock tube
+CONFIG="-p 5 -c 1 -vf 1000 -tf 0.231 -s 3 -m $MESH -es $SCHEME"
+
+# # h-refinement
+# $EXEC $CONFIG -o 1 -r 5 -dt 0.00064
+# $EXEC $CONFIG -o 1 -r 6 -dt 0.00032
+# $EXEC $CONFIG -o 1 -r 7 -dt 0.00016
+
+# # p-refinement & h-coarsening
+# $EXEC $CONFIG -r 5 -o 1 -dt 0.0004
+# $EXEC $CONFIG -r 4 -o 3 -dt 0.0004
+# $EXEC $CONFIG -r 3 -o 7 -dt 0.0004
+# $EXEC $CONFIG -r 2 -o 15 -dt 0.0004
+# $EXEC $CONFIG -r 1 -o 31 -dt 0.0004
+
+
+## Woodward Colella
+MESH=data/wall-bdr-100segment.mesh
+CONFIG="-p 5 -c 2 -vf 1000 -tf 0.038 -s 3 -m $MESH -es $SCHEME"
+$EXEC $CONFIG -o 1 -r 2 -dt 1e-6
+
+
+# ## Double Mach reflection
+# MESH=data/double-mach-quad.mesh
+# CONFIG="-p 5 -c 3 -vf 1000 -tf 0.2 -s 3 -m $MESH -es $SCHEME"
+# $EXEC $CONFIG -o 1 -r 3 -dt 5e-5
+
+
+# ## Vortex advection
+# SCHEME=0
+# MESH3=data/periodic-3quad.mesh
+# MESH4=data/periodic-4quad.mesh
+
+# ORDER=1
+# ODESOLVER=2
+# DT=0.000625
+# # ORDER=2
+# # ODESOLVER=3
+# # DT=0.0002
+# # ORDER=3
+# # ODESOLVER=3
+# # DT=0.00032
+# CONFIG="-p 5 -c 0 -vf 100 -tf 1 -s $ODESOLVER -dt $DT -o $ORDER -es $SCHEME"
+
+# rm errors.txt
+# $EXEC $CONFIG -m $MESH3 -r 3
+# $EXEC $CONFIG -m $MESH4 -r 3
+# $EXEC $CONFIG -m $MESH3 -r 4
+# $EXEC $CONFIG -m $MESH4 -r 4
+# $EXEC $CONFIG -m $MESH3 -r 5

miniapps/hypsys/config/openMPI-test.sh

@@ -0,0 +1,47 @@
+SCHEME=0
+
+## Advection
+
+# Solid Body Roatation
+# ./hypsys -r 0 -es $SCHEME
+# ./phypsys -r 0 -es $SCHEME
+# mpirun -np 4 ./phypsys -r 0 -es $SCHEME
+
+# Steady Circular Convection
+# ./hypsys -vf 1000 -m data/inline-4quad.mesh -dt 0.0001 -o 2 -s 1 -r 3 -c 0 -es $SCHEME
+# ./phypsys -vf 1000 -m data/inline-4quad.mesh -dt 0.0001 -o 2 -s 1 -r 3 -c 0 -es $SCHEME
+# mpirun -np 4 ./phypsys -vf 1000 -m data/inline-4quad.mesh -dt 0.0001 -o 2 -s 1 -r 3 -c 0 -es $SCHEME
+
+# Translation
+CONFIG="-p 0 -c 2 -vf 100 -tf 0.4 -s 3 -dt 0.002 -m data/periodic-3tri.mesh -o 2 -r 3 -es $SCHEME"
+./hypsys $CONFIG
+./phypsys $CONFIG
+mpirun -np 4 phypsys $CONFIG
+
+## Burgers
+CONFIG="-p 1 -c 1 -vf 100 -tf 0.5 -s 3 -dt 0.004 -m data/inline-3quad.mesh -o 1 -r 3 -es $SCHEME"
+./hypsys $CONFIG
+./phypsys $CONFIG
+mpirun -np 7 phypsys $CONFIG
+
+## KPP
+CONFIG="-p 2 -c 1 -vf 50 -tf 0.25 -s 3 -dt 0.005 -m data/inline-4tri.mesh -o 0 -r 4 -es 0"
+./hypsys $CONFIG
+./phypsys $CONFIG
+mpirun -np 1 phypsys $CONFIG
+
+## Shallow-Water
+
+# Dam break
+CONFIG="-p 4 -c 2 -vf 20 -tf 0.1 -s 3 -dt 0.0005 -m data/wall-bdr-4tri.mesh -o 1 -r 3 -es $SCHEME"
+./hypsys $CONFIG
+./phypsys $CONFIG
+mpirun -np 2 phypsys $CONFIG
+
+## Euler
+
+# Smooth vortex
+CONFIG="-p 5 -c 0 -vf 100 -tf 1 -s 3 -dt 0.00125 -m data/periodic-3quad.mesh -r 2 -es $SCHEME"
+./hypsys $CONFIG
+./phypsys $CONFIG
+mpirun -np 4 phypsys $CONFIG

miniapps/hypsys/config/rotating-wave-problems.sh

@@ -0,0 +1,12 @@
+EXEC="mpirun -np 7 phypsys"
+MESH=data/inline-4quad.mesh
+
+# KPP
+# $EXEC -p 2 -c 1 -vf 100 -tf 0.25 -s 3 -dt 0.00025 -r 9 -o 0 -m $MESH -es 0
+$EXEC -p 2 -c 1 -vf 100 -tf 0.25 -s 3 -dt 0.002 -r 6 -o 0 -m $MESH -es 0
+$EXEC -p 2 -c 1 -vf 100 -tf 0.25 -s 3 -dt 0.0008 -r 5 -o 1 -m $MESH -es 0
+$EXEC -p 2 -c 1 -vf 100 -tf 0.25 -s 3 -dt 0.0008 -r 5 -o 1 -m $MESH -es 1
+
+# Buckley-Leverett
+$EXEC -p 3 -c 2 -vf 100 -tf 0.1666667 -s 3 -dt 0.0005 -r 5 -o 0 -m $MESH -es 0
+$EXEC -p 3 -c 2 -vf 100 -tf 0.1666667 -s 3 -dt 0.0005 -r 4 -o 1 -m $MESH -es 1

miniapps/hypsys/config/shallow-water.sh

@@ -0,0 +1,54 @@
+EXEC="mpirun -np 7 phypsys"
+
+## 1D Dam break
+SCHEME=0
+MESH=data/wall-bdr-4segment.mesh
+CONFIG="-p 4 -c 1 -vf 1000 -tf 0.02 -s 3 -m $MESH -es $SCHEME"
+
+# # h-refinement
+# $EXEC $CONFIG -o 1 -r 5 -dt 0.0001
+# $EXEC $CONFIG -o 1 -r 6 -dt 0.00005
+# $EXEC $CONFIG -o 1 -r 7 -dt 0.000025
+# $EXEC $CONFIG -o 1 -r 8 -dt 0.0000125
+
+# # p-refinement & h-coarsening
+# $EXEC $CONFIG -o 1 -r 5 -dt 0.000025
+# $EXEC $CONFIG -o 3 -r 4 -dt 0.000025
+# $EXEC $CONFIG -o 7 -r 3 -dt 0.000025
+
+# ## Radial dambreak
+# MESH=data/outflow-bdr-4quad.mesh
+# CONFIG="-p 4 -c 2 -vf 50 -tf 0.06 -s 3 -m $MESH -es 1"
+
+# # p-refinement & h-coarsening
+# $EXEC $CONFIG -o 1 -r 5 -dt 0.0001
+# $EXEC $CONFIG -o 3 -r 4 -dt 0.0001
+# $EXEC $CONFIG -o 7 -r 3 -dt 0.0001
+
+## Constricted channel
+MESH=data/constricted-channel.mesh
+CONFIG="-p 4 -c 3 -vf 100 -tf 1000 -s 1 -m $MESH -es $SCHEME"
+$EXEC $CONFIG -r 1 -o 1 -dt 0.025
+
+# ## Vortex advection
+# SCHEME=0
+# MESH3=data/periodic-3quad.mesh
+# MESH4=data/periodic-4quad.mesh
+
+# ORDER=1
+# ODESOLVER=2
+# DT=0.00064
+# # ORDER=2
+# # ODESOLVER=3
+# # DT=0.0004
+# # ORDER=3
+# # ODESOLVER=3
+# # DT=0.00025
+# CONFIG="-p 4 -c 0 -vf 100 -tf 1 -s $ODESOLVER -dt $DT -o $ORDER -es $SCHEME"
+
+# rm errors.txt
+# $EXEC $CONFIG -m $MESH3 -r 3
+# $EXEC $CONFIG -m $MESH4 -r 3
+# $EXEC $CONFIG -m $MESH3 -r 4
+# $EXEC $CONFIG -m $MESH4 -r 4
+# $EXEC $CONFIG -m $MESH3 -r 5

miniapps/hypsys/config/solid-body-rotation.sh

@@ -0,0 +1,6 @@
+EXEC="mpirun -np 7 phypsys"
+
+MESH=data/periodic-4quad.mesh
+SCHEME=1
+
+$EXEC -p 0 -c 1 -vf 200 -tf 1 -s 3 -dt 0.00032 -m $MESH -r 4 -o 2 -es $SCHEME

miniapps/hypsys/data/beam-hex.mesh

@@ -0,0 +1,103 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+#
+
+dimension
+3
+
+elements
+8
+1 5 0 1 10 9 18 19 28 27
+1 5 1 2 11 10 19 20 29 28
+1 5 2 3 12 11 20 21 30 29
+1 5 3 4 13 12 21 22 31 30
+2 5 4 5 14 13 22 23 32 31
+2 5 5 6 15 14 23 24 33 32
+2 5 6 7 16 15 24 25 34 33
+2 5 7 8 17 16 25 26 35 34
+
+boundary
+34
+3 3 9 10 1 0
+3 3 0 1 19 18
+3 3 10 9 27 28
+1 3 9 0 18 27
+3 3 18 19 28 27
+3 3 10 11 2 1
+3 3 1 2 20 19
+3 3 11 10 28 29
+3 3 19 20 29 28
+3 3 11 12 3 2
+3 3 2 3 21 20
+3 3 12 11 29 30
+3 3 20 21 30 29
+3 3 12 13 4 3
+3 3 3 4 22 21
+3 3 13 12 30 31
+3 3 21 22 31 30
+3 3 13 14 5 4
+3 3 4 5 23 22
+3 3 14 13 31 32
+3 3 22 23 32 31
+3 3 14 15 6 5
+3 3 5 6 24 23
+3 3 15 14 32 33
+3 3 23 24 33 32
+3 3 15 16 7 6
+3 3 6 7 25 24
+3 3 16 15 33 34
+3 3 24 25 34 33
+3 3 16 17 8 7
+3 3 7 8 26 25
+2 3 8 17 35 26
+3 3 17 16 34 35
+3 3 25 26 35 34
+
+vertices
+36
+3
+0 0 0
+1 0 0
+2 0 0
+3 0 0
+4 0 0
+5 0 0
+6 0 0
+7 0 0
+8 0 0
+0 1 0
+1 1 0
+2 1 0
+3 1 0
+4 1 0
+5 1 0
+6 1 0
+7 1 0
+8 1 0
+0 0 1
+1 0 1
+2 0 1
+3 0 1
+4 0 1
+5 0 1
+6 0 1
+7 0 1
+8 0 1
+0 1 1
+1 1 1
+2 1 1
+3 1 1
+4 1 1
+5 1 1
+6 1 1
+7 1 1
+8 1 1

miniapps/hypsys/data/beam-quad.mesh

@@ -0,0 +1,69 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+#
+
+dimension
+2
+
+elements
+8
+1 3 0 1 10 9
+1 3 1 2 11 10
+1 3 2 3 12 11
+1 3 3 4 13 12
+2 3 4 5 14 13
+2 3 5 6 15 14
+2 3 6 7 16 15
+2 3 7 8 17 16
+
+boundary
+18
+3 1 0 1
+3 1 1 2
+3 1 2 3
+3 1 3 4
+3 1 4 5
+3 1 5 6
+3 1 6 7
+3 1 7 8
+3 1 10 9
+3 1 11 10
+3 1 12 11
+3 1 13 12
+3 1 14 13
+3 1 15 14
+3 1 16 15
+3 1 17 16
+1 1 9 0
+2 1 8 17
+
+vertices
+18
+2
+0 0
+1 0
+2 0
+3 0
+4 0
+5 0
+6 0
+7 0
+8 0
+0 1
+1 1
+2 1
+3 1
+4 1
+5 1
+6 1
+7 1
+8 1

miniapps/hypsys/data/beam-tri.mesh

@@ -0,0 +1,77 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+#
+
+dimension
+2
+
+elements
+16
+1 2 10 0 1
+1 2 0 10 9
+1 2 11 1 2
+1 2 1 11 10
+1 2 12 2 3
+1 2 2 12 11
+1 2 13 3 4
+1 2 3 13 12
+2 2 14 4 5
+2 2 4 14 13
+2 2 15 5 6
+2 2 5 15 14
+2 2 16 6 7
+2 2 6 16 15
+2 2 17 7 8
+2 2 7 17 16
+
+boundary
+18
+3 1 0 1
+3 1 1 2
+3 1 2 3
+3 1 3 4
+3 1 4 5
+3 1 5 6
+3 1 6 7
+3 1 7 8
+3 1 10 9
+3 1 11 10
+3 1 12 11
+3 1 13 12
+3 1 14 13
+3 1 15 14
+3 1 16 15
+3 1 17 16
+1 1 9 0
+2 1 8 17
+
+vertices
+18
+2
+0 0
+1 0
+2 0
+3 0
+4 0
+5 0
+6 0
+7 0
+8 0
+0 1
+1 1
+2 1
+3 1
+4 1
+5 1
+6 1
+7 1
+8 1

miniapps/hypsys/data/double-mach-quad.mesh

@@ -0,0 +1,405 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+2
+
+elements
+144
+1 3 0 1 26 25
+1 3 25 26 51 50
+1 3 26 27 52 51
+1 3 1 2 27 26
+1 3 2 3 28 27
+1 3 27 28 53 52
+1 3 52 53 78 77
+1 3 77 78 103 102
+1 3 76 77 102 101
+1 3 51 52 77 76
+1 3 50 51 76 75
+1 3 75 76 101 100
+1 3 100 101 126 125
+1 3 125 126 151 150
+1 3 126 127 152 151
+1 3 101 102 127 126
+1 3 102 103 128 127
+1 3 127 128 153 152
+1 3 128 129 154 153
+1 3 103 104 129 128
+1 3 104 105 130 129
+1 3 129 130 155 154
+1 3 130 131 156 155
+1 3 105 106 131 130
+1 3 80 81 106 105
+1 3 55 56 81 80
+1 3 54 55 80 79
+1 3 79 80 105 104
+1 3 78 79 104 103
+1 3 53 54 79 78
+1 3 28 29 54 53
+1 3 3 4 29 28
+1 3 4 5 30 29
+1 3 29 30 55 54
+1 3 30 31 56 55
+1 3 5 6 31 30
+1 3 6 7 32 31
+1 3 31 32 57 56
+1 3 32 33 58 57
+1 3 7 8 33 32
+1 3 8 9 34 33
+1 3 33 34 59 58
+1 3 58 59 84 83
+1 3 83 84 109 108
+1 3 82 83 108 107
+1 3 57 58 83 82
+1 3 56 57 82 81
+1 3 81 82 107 106
+1 3 106 107 132 131
+1 3 131 132 157 156
+1 3 132 133 158 157
+1 3 107 108 133 132
+1 3 108 109 134 133
+1 3 133 134 159 158
+1 3 134 135 160 159
+1 3 109 110 135 134
+1 3 110 111 136 135
+1 3 135 136 161 160
+1 3 136 137 162 161
+1 3 111 112 137 136
+1 3 86 87 112 111
+1 3 61 62 87 86
+1 3 60 61 86 85
+1 3 85 86 111 110
+1 3 84 85 110 109
+1 3 59 60 85 84
+1 3 34 35 60 59
+1 3 9 10 35 34
+1 3 10 11 36 35
+1 3 35 36 61 60
+1 3 36 37 62 61
+1 3 11 12 37 36
+1 3 12 13 38 37
+1 3 37 38 63 62
+1 3 38 39 64 63
+1 3 13 14 39 38
+1 3 14 15 40 39
+1 3 39 40 65 64
+1 3 64 65 90 89
+1 3 89 90 115 114
+1 3 88 89 114 113
+1 3 63 64 89 88
+1 3 62 63 88 87
+1 3 87 88 113 112
+1 3 112 113 138 137
+1 3 137 138 163 162
+1 3 138 139 164 163
+1 3 113 114 139 138
+1 3 114 115 140 139
+1 3 139 140 165 164
+1 3 140 141 166 165
+1 3 115 116 141 140
+1 3 116 117 142 141
+1 3 141 142 167 166
+1 3 142 143 168 167
+1 3 117 118 143 142
+1 3 92 93 118 117
+1 3 67 68 93 92
+1 3 66 67 92 91
+1 3 91 92 117 116
+1 3 90 91 116 115
+1 3 65 66 91 90
+1 3 40 41 66 65
+1 3 15 16 41 40
+1 3 16 17 42 41
+1 3 41 42 67 66
+1 3 42 43 68 67
+1 3 17 18 43 42
+1 3 18 19 44 43
+1 3 43 44 69 68
+1 3 44 45 70 69
+1 3 19 20 45 44
+1 3 20 21 46 45
+1 3 45 46 71 70
+1 3 70 71 96 95
+1 3 95 96 121 120
+1 3 94 95 120 119
+1 3 69 70 95 94
+1 3 68 69 94 93
+1 3 93 94 119 118
+1 3 118 119 144 143
+1 3 143 144 169 168
+1 3 144 145 170 169
+1 3 119 120 145 144
+1 3 120 121 146 145
+1 3 145 146 171 170
+1 3 146 147 172 171
+1 3 121 122 147 146
+1 3 122 123 148 147
+1 3 147 148 173 172
+1 3 148 149 174 173
+1 3 123 124 149 148
+1 3 98 99 124 123
+1 3 73 74 99 98
+1 3 72 73 98 97
+1 3 97 98 123 122
+1 3 96 97 122 121
+1 3 71 72 97 96
+1 3 46 47 72 71
+1 3 21 22 47 46
+1 3 22 23 48 47
+1 3 47 48 73 72
+1 3 48 49 74 73
+1 3 23 24 49 48
+
+boundary
+60
+3 1 0 1
+1 1 1 2
+1 1 2 3
+1 1 3 4
+1 1 4 5
+1 1 5 6
+1 1 6 7
+1 1 7 8
+1 1 8 9
+1 1 9 10
+1 1 10 11
+1 1 11 12
+1 1 12 13
+1 1 13 14
+1 1 14 15
+1 1 15 16
+1 1 16 17
+1 1 17 18
+1 1 18 19
+1 1 19 20
+1 1 20 21
+1 1 21 22
+1 1 22 23
+1 1 23 24
+3 1 151 150
+3 1 152 151
+3 1 153 152
+3 1 154 153
+3 1 155 154
+3 1 156 155
+3 1 157 156
+3 1 158 157
+3 1 159 158
+3 1 160 159
+3 1 161 160
+3 1 162 161
+3 1 163 162
+3 1 164 163
+3 1 165 164
+3 1 166 165
+3 1 167 166
+3 1 168 167
+3 1 169 168
+3 1 170 169
+3 1 171 170
+3 1 172 171
+3 1 173 172
+3 1 174 173
+3 1 25 0
+3 1 50 25
+3 1 75 50
+3 1 100 75
+3 1 125 100
+3 1 150 125
+2 1 24 49
+2 1 49 74
+2 1 74 99
+2 1 99 124
+2 1 124 149
+2 1 149 174
+
+vertices
+175
+2
+0 0
+0.16666666666666666 0
+0.33333333333333333 0
+0.5 0
+0.66666666666666666 0
+0.83333333333333333 0
+1 0
+1.1666666666666666 0
+1.3333333333333333 0
+1.5 0
+1.6666666666666666 0
+1.8333333333333333 0
+2 0
+2.1666666666666666 0
+2.3333333333333333 0
+2.5 0
+2.6666666666666666 0
+2.8333333333333333 0
+3 0
+3.1666666666666666 0
+3.3333333333333333 0
+3.5 0
+3.6666666666666666 0
+3.8333333333333333 0
+4 0
+0 0.16666666666666666
+0.16666666666666666 0.16666666666666666
+0.33333333333333333 0.16666666666666666
+0.5 0.16666666666666666
+0.66666666666666666 0.16666666666666666
+0.83333333333333333 0.16666666666666666
+1 0.16666666666666666
+1.1666666666666666 0.16666666666666666
+1.3333333333333333 0.16666666666666666
+1.5 0.16666666666666666
+1.6666666666666666 0.16666666666666666
+1.8333333333333333 0.16666666666666666
+2 0.16666666666666666
+2.1666666666666666 0.16666666666666666
+2.3333333333333333 0.16666666666666666
+2.5 0.16666666666666666
+2.6666666666666666 0.16666666666666666
+2.8333333333333333 0.16666666666666666
+3 0.16666666666666666
+3.1666666666666666 0.16666666666666666
+3.3333333333333333 0.16666666666666666
+3.5 0.16666666666666666
+3.6666666666666666 0.16666666666666666
+3.8333333333333333 0.16666666666666666
+4 0.16666666666666666
+0 0.33333333333333333
+0.16666666666666666 0.33333333333333333
+0.33333333333333333 0.33333333333333333
+0.5 0.33333333333333333
+0.66666666666666666 0.33333333333333333
+0.83333333333333333 0.33333333333333333
+1 0.33333333333333333
+1.1666666666666666 0.33333333333333333
+1.3333333333333333 0.33333333333333333
+1.5 0.33333333333333333
+1.6666666666666666 0.33333333333333333
+1.8333333333333333 0.33333333333333333
+2 0.33333333333333333
+2.1666666666666666 0.33333333333333333
+2.3333333333333333 0.33333333333333333
+2.5 0.33333333333333333
+2.6666666666666666 0.33333333333333333
+2.8333333333333333 0.33333333333333333
+3 0.33333333333333333
+3.1666666666666666 0.33333333333333333
+3.3333333333333333 0.33333333333333333
+3.5 0.33333333333333333
+3.6666666666666666 0.33333333333333333
+3.8333333333333333 0.33333333333333333
+4 0.33333333333333333
+0 0.5
+0.16666666666666666 0.5
+0.33333333333333333 0.5
+0.5 0.5
+0.66666666666666666 0.5
+0.83333333333333333 0.5
+1 0.5
+1.1666666666666666 0.5
+1.3333333333333333 0.5
+1.5 0.5
+1.6666666666666666 0.5
+1.8333333333333333 0.5
+2 0.5
+2.1666666666666666 0.5
+2.3333333333333333 0.5
+2.5 0.5
+2.6666666666666666 0.5
+2.8333333333333333 0.5
+3 0.5
+3.1666666666666666 0.5
+3.3333333333333333 0.5
+3.5 0.5
+3.6666666666666666 0.5
+3.8333333333333333 0.5
+4 0.5
+0 0.66666666666666666
+0.16666666666666666 0.66666666666666666
+0.33333333333333333 0.66666666666666666
+0.5 0.66666666666666666
+0.66666666666666666 0.66666666666666666
+0.83333333333333333 0.66666666666666666
+1 0.66666666666666666
+1.1666666666666666 0.66666666666666666
+1.3333333333333333 0.66666666666666666
+1.5 0.66666666666666666
+1.6666666666666666 0.66666666666666666
+1.8333333333333333 0.66666666666666666
+2 0.66666666666666666
+2.1666666666666666 0.66666666666666666
+2.3333333333333333 0.66666666666666666
+2.5 0.66666666666666666
+2.6666666666666666 0.66666666666666666
+2.8333333333333333 0.66666666666666666
+3 0.66666666666666666
+3.1666666666666666 0.66666666666666666
+3.3333333333333333 0.66666666666666666
+3.5 0.66666666666666666
+3.6666666666666666 0.66666666666666666
+3.8333333333333333 0.66666666666666666
+4 0.66666666666666666
+0 0.83333333333333333
+0.16666666666666666 0.83333333333333333
+0.33333333333333333 0.83333333333333333
+0.5 0.83333333333333333
+0.66666666666666666 0.83333333333333333
+0.83333333333333333 0.83333333333333333
+1 0.83333333333333333
+1.1666666666666666 0.83333333333333333
+1.3333333333333333 0.83333333333333333
+1.5 0.83333333333333333
+1.6666666666666666 0.83333333333333333
+1.8333333333333333 0.83333333333333333
+2 0.83333333333333333
+2.1666666666666666 0.83333333333333333
+2.3333333333333333 0.83333333333333333
+2.5 0.83333333333333333
+2.6666666666666666 0.83333333333333333
+2.8333333333333333 0.83333333333333333
+3 0.83333333333333333
+3.1666666666666666 0.83333333333333333
+3.3333333333333333 0.83333333333333333
+3.5 0.83333333333333333
+3.6666666666666666 0.83333333333333333
+3.8333333333333333 0.83333333333333333
+4 0.83333333333333333
+0 1
+0.16666666666666666 1
+0.33333333333333333 1
+0.5 1
+0.66666666666666666 1
+0.83333333333333333 1
+1 1
+1.1666666666666666 1
+1.3333333333333333 1
+1.5 1
+1.6666666666666666 1
+1.8333333333333333 1
+2 1
+2.1666666666666666 1
+2.3333333333333333 1
+2.5 1
+2.6666666666666666 1
+2.8333333333333333 1
+3 1
+3.1666666666666666 1
+3.3333333333333333 1
+3.5 1
+3.6666666666666666 1
+3.8333333333333333 1
+4 1

miniapps/hypsys/data/double-mach-tri.mesh

@@ -0,0 +1,549 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+2
+
+elements
+288
+1 2 0 26 25
+1 2 26 0 1
+1 2 1 27 26
+1 2 27 1 2
+1 2 2 28 27
+1 2 28 2 3
+1 2 3 29 28
+1 2 29 3 4
+1 2 4 30 29
+1 2 30 4 5
+1 2 5 31 30
+1 2 31 5 6
+1 2 6 32 31
+1 2 32 6 7
+1 2 7 33 32
+1 2 33 7 8
+1 2 8 34 33
+1 2 34 8 9
+1 2 9 35 34
+1 2 35 9 10
+1 2 10 36 35
+1 2 36 10 11
+1 2 11 37 36
+1 2 37 11 12
+1 2 12 38 37
+1 2 38 12 13
+1 2 13 39 38
+1 2 39 13 14
+1 2 14 40 39
+1 2 40 14 15
+1 2 15 41 40
+1 2 41 15 16
+1 2 16 42 41
+1 2 42 16 17
+1 2 17 43 42
+1 2 43 17 18
+1 2 18 44 43
+1 2 44 18 19
+1 2 19 45 44
+1 2 45 19 20
+1 2 20 46 45
+1 2 46 20 21
+1 2 21 47 46
+1 2 47 21 22
+1 2 22 48 47
+1 2 48 22 23
+1 2 23 49 48
+1 2 49 23 24
+1 2 25 51 50
+1 2 51 25 26
+1 2 26 52 51
+1 2 52 26 27
+1 2 27 53 52
+1 2 53 27 28
+1 2 28 54 53
+1 2 54 28 29
+1 2 29 55 54
+1 2 55 29 30
+1 2 30 56 55
+1 2 56 30 31
+1 2 31 57 56
+1 2 57 31 32
+1 2 32 58 57
+1 2 58 32 33
+1 2 33 59 58
+1 2 59 33 34
+1 2 34 60 59
+1 2 60 34 35
+1 2 35 61 60
+1 2 61 35 36
+1 2 36 62 61
+1 2 62 36 37
+1 2 37 63 62
+1 2 63 37 38
+1 2 38 64 63
+1 2 64 38 39
+1 2 39 65 64
+1 2 65 39 40
+1 2 40 66 65
+1 2 66 40 41
+1 2 41 67 66
+1 2 67 41 42
+1 2 42 68 67
+1 2 68 42 43
+1 2 43 69 68
+1 2 69 43 44
+1 2 44 70 69
+1 2 70 44 45
+1 2 45 71 70
+1 2 71 45 46
+1 2 46 72 71
+1 2 72 46 47
+1 2 47 73 72
+1 2 73 47 48
+1 2 48 74 73
+1 2 74 48 49
+1 2 50 76 75
+1 2 76 50 51
+1 2 51 77 76
+1 2 77 51 52
+1 2 52 78 77
+1 2 78 52 53
+1 2 53 79 78
+1 2 79 53 54
+1 2 54 80 79
+1 2 80 54 55
+1 2 55 81 80
+1 2 81 55 56
+1 2 56 82 81
+1 2 82 56 57
+1 2 57 83 82
+1 2 83 57 58
+1 2 58 84 83
+1 2 84 58 59
+1 2 59 85 84
+1 2 85 59 60
+1 2 60 86 85
+1 2 86 60 61
+1 2 61 87 86
+1 2 87 61 62
+1 2 62 88 87
+1 2 88 62 63
+1 2 63 89 88
+1 2 89 63 64
+1 2 64 90 89
+1 2 90 64 65
+1 2 65 91 90
+1 2 91 65 66
+1 2 66 92 91
+1 2 92 66 67
+1 2 67 93 92
+1 2 93 67 68
+1 2 68 94 93
+1 2 94 68 69
+1 2 69 95 94
+1 2 95 69 70
+1 2 70 96 95
+1 2 96 70 71
+1 2 71 97 96
+1 2 97 71 72
+1 2 72 98 97
+1 2 98 72 73
+1 2 73 99 98
+1 2 99 73 74
+1 2 75 101 100
+1 2 101 75 76
+1 2 76 102 101
+1 2 102 76 77
+1 2 77 103 102
+1 2 103 77 78
+1 2 78 104 103
+1 2 104 78 79
+1 2 79 105 104
+1 2 105 79 80
+1 2 80 106 105
+1 2 106 80 81
+1 2 81 107 106
+1 2 107 81 82
+1 2 82 108 107
+1 2 108 82 83
+1 2 83 109 108
+1 2 109 83 84
+1 2 84 110 109
+1 2 110 84 85
+1 2 85 111 110
+1 2 111 85 86
+1 2 86 112 111
+1 2 112 86 87
+1 2 87 113 112
+1 2 113 87 88
+1 2 88 114 113
+1 2 114 88 89
+1 2 89 115 114
+1 2 115 89 90
+1 2 90 116 115
+1 2 116 90 91
+1 2 91 117 116
+1 2 117 91 92
+1 2 92 118 117
+1 2 118 92 93
+1 2 93 119 118
+1 2 119 93 94
+1 2 94 120 119
+1 2 120 94 95
+1 2 95 121 120
+1 2 121 95 96
+1 2 96 122 121
+1 2 122 96 97
+1 2 97 123 122
+1 2 123 97 98
+1 2 98 124 123
+1 2 124 98 99
+1 2 100 126 125
+1 2 126 100 101
+1 2 101 127 126
+1 2 127 101 102
+1 2 102 128 127
+1 2 128 102 103
+1 2 103 129 128
+1 2 129 103 104
+1 2 104 130 129
+1 2 130 104 105
+1 2 105 131 130
+1 2 131 105 106
+1 2 106 132 131
+1 2 132 106 107
+1 2 107 133 132
+1 2 133 107 108
+1 2 108 134 133
+1 2 134 108 109
+1 2 109 135 134
+1 2 135 109 110
+1 2 110 136 135
+1 2 136 110 111
+1 2 111 137 136
+1 2 137 111 112
+1 2 112 138 137
+1 2 138 112 113
+1 2 113 139 138
+1 2 139 113 114
+1 2 114 140 139
+1 2 140 114 115
+1 2 115 141 140
+1 2 141 115 116
+1 2 116 142 141
+1 2 142 116 117
+1 2 117 143 142
+1 2 143 117 118
+1 2 118 144 143
+1 2 144 118 119
+1 2 119 145 144
+1 2 145 119 120
+1 2 120 146 145
+1 2 146 120 121
+1 2 121 147 146
+1 2 147 121 122
+1 2 122 148 147
+1 2 148 122 123
+1 2 123 149 148
+1 2 149 123 124
+1 2 125 151 150
+1 2 151 125 126
+1 2 126 152 151
+1 2 152 126 127
+1 2 127 153 152
+1 2 153 127 128
+1 2 128 154 153
+1 2 154 128 129
+1 2 129 155 154
+1 2 155 129 130
+1 2 130 156 155
+1 2 156 130 131
+1 2 131 157 156
+1 2 157 131 132
+1 2 132 158 157
+1 2 158 132 133
+1 2 133 159 158
+1 2 159 133 134
+1 2 134 160 159
+1 2 160 134 135
+1 2 135 161 160
+1 2 161 135 136
+1 2 136 162 161
+1 2 162 136 137
+1 2 137 163 162
+1 2 163 137 138
+1 2 138 164 163
+1 2 164 138 139
+1 2 139 165 164
+1 2 165 139 140
+1 2 140 166 165
+1 2 166 140 141
+1 2 141 167 166
+1 2 167 141 142
+1 2 142 168 167
+1 2 168 142 143
+1 2 143 169 168
+1 2 169 143 144
+1 2 144 170 169
+1 2 170 144 145
+1 2 145 171 170
+1 2 171 145 146
+1 2 146 172 171
+1 2 172 146 147
+1 2 147 173 172
+1 2 173 147 148
+1 2 148 174 173
+1 2 174 148 149
+
+boundary
+60
+3 1 0 1
+1 1 1 2
+1 1 2 3
+1 1 3 4
+1 1 4 5
+1 1 5 6
+1 1 6 7
+1 1 7 8
+1 1 8 9
+1 1 9 10
+1 1 10 11
+1 1 11 12
+1 1 12 13
+1 1 13 14
+1 1 14 15
+1 1 15 16
+1 1 16 17
+1 1 17 18
+1 1 18 19
+1 1 19 20
+1 1 20 21
+1 1 21 22
+1 1 22 23
+1 1 23 24
+3 1 151 150
+3 1 152 151
+3 1 153 152
+3 1 154 153
+3 1 155 154
+3 1 156 155
+3 1 157 156
+3 1 158 157
+3 1 159 158
+3 1 160 159
+3 1 161 160
+3 1 162 161
+3 1 163 162
+3 1 164 163
+3 1 165 164
+3 1 166 165
+3 1 167 166
+3 1 168 167
+3 1 169 168
+3 1 170 169
+3 1 171 170
+3 1 172 171
+3 1 173 172
+3 1 174 173
+3 1 25 0
+3 1 50 25
+3 1 75 50
+3 1 100 75
+3 1 125 100
+3 1 150 125
+2 1 24 49
+2 1 49 74
+2 1 74 99
+2 1 99 124
+2 1 124 149
+2 1 149 174
+
+vertices
+175
+2
+0 0
+0.1666666666666666 0
+0.3333333333333333 0
+0.5 0
+0.6666666666666666 0
+0.8333333333333333 0
+1 0
+1.1666667 0
+1.333333333333333 0
+1.5 0
+1.666666666666666 0
+1.8333333 0
+2 0
+2.1666667 0
+2.333333333333333 0
+2.5 0
+2.666666666666666 0
+2.8333333 0
+3 0
+3.1666667 0
+3.333333333333333 0
+3.5 0
+3.666666666666666 0
+3.8333333 0
+4 0
+0 0.1666666666666666
+0.1666666666666666 0.1666666666666666
+0.3333333333333333 0.1666666666666666
+0.5 0.1666666666666666
+0.6666666666666666 0.1666666666666666
+0.8333333333333333 0.1666666666666666
+1 0.1666666666666666
+1.1666667 0.1666666666666666
+1.333333333333333 0.1666666666666666
+1.5 0.1666666666666666
+1.666666666666666 0.1666666666666666
+1.8333333 0.1666666666666666
+2 0.1666666666666666
+2.1666667 0.1666666666666666
+2.333333333333333 0.1666666666666666
+2.5 0.1666666666666666
+2.666666666666666 0.1666666666666666
+2.8333333 0.1666666666666666
+3 0.1666666666666666
+3.1666667 0.1666666666666666
+3.333333333333333 0.1666666666666666
+3.5 0.1666666666666666
+3.666666666666666 0.1666666666666666
+3.8333333 0.1666666666666666
+4 0.1666666666666666
+0 0.3333333333333333
+0.1666666666666666 0.3333333333333333
+0.3333333333333333 0.3333333333333333
+0.5 0.3333333333333333
+0.6666666666666666 0.3333333333333333
+0.8333333333333333 0.3333333333333333
+1 0.3333333333333333
+1.1666667 0.3333333333333333
+1.333333333333333 0.3333333333333333
+1.5 0.3333333333333333
+1.666666666666666 0.3333333333333333
+1.8333333 0.3333333333333333
+2 0.3333333333333333
+2.1666667 0.3333333333333333
+2.333333333333333 0.3333333333333333
+2.5 0.3333333333333333
+2.666666666666666 0.3333333333333333
+2.8333333 0.3333333333333333
+3 0.3333333333333333
+3.1666667 0.3333333333333333
+3.333333333333333 0.3333333333333333
+3.5 0.3333333333333333
+3.666666666666666 0.3333333333333333
+3.8333333 0.3333333333333333
+4 0.3333333333333333
+0 0.5
+0.1666666666666666 0.5
+0.3333333333333333 0.5
+0.5 0.5
+0.6666666666666666 0.5
+0.8333333333333333 0.5
+1 0.5
+1.1666667 0.5
+1.333333333333333 0.5
+1.5 0.5
+1.666666666666666 0.5
+1.8333333 0.5
+2 0.5
+2.1666667 0.5
+2.333333333333333 0.5
+2.5 0.5
+2.666666666666666 0.5
+2.8333333 0.5
+3 0.5
+3.1666667 0.5
+3.333333333333333 0.5
+3.5 0.5
+3.666666666666666 0.5
+3.8333333 0.5
+4 0.5
+0 0.6666666666666666
+0.1666666666666666 0.6666666666666666
+0.3333333333333333 0.6666666666666666
+0.5 0.6666666666666666
+0.6666666666666666 0.6666666666666666
+0.8333333333333333 0.6666666666666666
+1 0.6666666666666666
+1.1666667 0.6666666666666666
+1.333333333333333 0.6666666666666666
+1.5 0.6666666666666666
+1.666666666666666 0.6666666666666666
+1.8333333 0.6666666666666666
+2 0.6666666666666666
+2.1666667 0.6666666666666666
+2.333333333333333 0.6666666666666666
+2.5 0.6666666666666666
+2.666666666666666 0.6666666666666666
+2.8333333 0.6666666666666666
+3 0.6666666666666666
+3.1666667 0.6666666666666666
+3.333333333333333 0.6666666666666666
+3.5 0.6666666666666666
+3.666666666666666 0.6666666666666666
+3.8333333 0.6666666666666666
+4 0.6666666666666666
+0 0.8333333333333333
+0.1666666666666666 0.8333333333333333
+0.3333333333333333 0.8333333333333333
+0.5 0.8333333333333333
+0.6666666666666666 0.8333333333333333
+0.8333333333333333 0.8333333333333333
+1 0.8333333333333333
+1.1666667 0.8333333333333333
+1.333333333333333 0.8333333333333333
+1.5 0.8333333333333333
+1.666666666666666 0.8333333333333333
+1.8333333 0.8333333333333333
+2 0.8333333333333333
+2.1666667 0.8333333333333333
+2.333333333333333 0.8333333333333333
+2.5 0.8333333333333333
+2.666666666666666 0.8333333333333333
+2.8333333 0.8333333333333333
+3 0.8333333333333333
+3.1666667 0.8333333333333333
+3.333333333333333 0.8333333333333333
+3.5 0.8333333333333333
+3.666666666666666 0.8333333333333333
+3.8333333 0.8333333333333333
+4 0.8333333333333333
+0 1
+0.1666666666666666 1
+0.3333333333333333 1
+0.5 1
+0.6666666666666666 1
+0.8333333333333333 1
+1 1
+1.1666667 1
+1.333333333333333 1
+1.5 1
+1.666666666666666 1
+1.8333333 1
+2 1
+2.1666667 1
+2.333333333333333 1
+2.5 1
+2.666666666666666 1
+2.8333333 1
+3 1
+3.1666667 1
+3.333333333333333 1
+3.5 1
+3.666666666666666 1
+3.8333333 1
+4 1

miniapps/hypsys/data/inline-3quad.mesh

@@ -0,0 +1,7 @@
+MFEM INLINE mesh v1.0
+
+type = quad
+nx = 3
+ny = 3
+sx = 1.0
+sy = 1.0

miniapps/hypsys/data/inline-3segment.mesh

@@ -0,0 +1,5 @@
+MFEM INLINE mesh v1.0
+
+type = segment
+nx = 3
+sx = 1.0

miniapps/hypsys/data/inline-3tri.mesh

@@ -0,0 +1,7 @@
+MFEM INLINE mesh v1.0
+
+type = tri
+nx = 3
+ny = 3
+sx = 1.0
+sy = 1.0

miniapps/hypsys/data/inline-4quad.mesh

@@ -0,0 +1,7 @@
+MFEM INLINE mesh v1.0
+
+type = quad
+nx = 4
+ny = 4
+sx = 1.0
+sy = 1.0

miniapps/hypsys/data/inline-4segment.mesh

@@ -0,0 +1,5 @@
+MFEM INLINE mesh v1.0
+
+type = segment
+nx = 4
+sx = 1.0

miniapps/hypsys/data/inline-4tri.mesh

@@ -0,0 +1,7 @@
+MFEM INLINE mesh v1.0
+
+type = tri
+nx = 4
+ny = 4
+sx = 1.0
+sy = 1.0

miniapps/hypsys/data/inline-hex.mesh

@@ -0,0 +1,9 @@
+MFEM INLINE mesh v1.0
+
+type = hex
+nx = 4
+ny = 4
+nz = 4
+sx = 1.0
+sy = 1.0
+sz = 1.0