Interior DoF for Cubic Quadrilateral Elements Fixed and currently implemented for Quadrant 1 Mesh

Add dimension checks in the Get and Set methods of IntegrationPoint

.gitignore

@@ -112,6 +112,15 @@ examples/petsc/deformed.*
 examples/petsc/velocity.*
 examples/petsc/elastic_energy.*
 
+examples/pumi/ex1
+examples/pumi/ex[126]p
+
+examples/pumi/refined.mesh
+examples/pumi/sol.gf
+examples/pumi/mesh.*
+examples/pumi/sol.*
+examples/pumi/displaced.mesh
+
 miniapps/electromagnetics/volta
 miniapps/electromagnetics/tesla
 miniapps/electromagnetics/maxwell

CHANGELOG

@@ -8,11 +8,30 @@
                                http://mfem.org
 
 
-Version 3.3.3 (development)
+Version 3.4.1 (development)
 ===========================
+- Added support for reading linear and quadratic 2D quadrilateral and triangular
+  Cubit meshes.
+
+- The tetrahedral mesh refinement algorithm in serial and in parallel now
+  follows precisely the paper:
+     D. Arnold, A. Mukherjee, and L. Pouly, "Locally Adapted Tetrahedral Meshes
+     Using Bisection", SIAM J. Sci. Comput., 22(2), 431–448.
+  This guarantees that the shape regularity of the elements will be preserved
+  under refinement.
+
+
+Version 3.4, released on May 29, 2018
+=====================================
+
+More general and efficient mesh adaptivity
+------------------------------------------
+- Added support for PUMI, the Parallel Unstructured Mesh Infrastructure from
+  https://scorec.rpi.edu/pumi. PUMI is an unstructured, distributed mesh data
+  management system that is capable of handling general non-manifold models and
+  effectively supports automated adaptive analysis. PUMI enables for the first
+  time support for parallel unstructured modifications of MFEM meshes.
 
-More efficient non-conforming adaptive mesh refinement
-------------------------------------------------------
 - Significantly reduced MPI communication in the construction of the parallel
   prolongation matrix in ParFiniteElementSpace, for much improved parallel
   scaling of non-conforming AMR on hundreds of thousands of MPI tasks. The
@@ -81,6 +100,11 @@ New and updated examples and miniapps
   NURBS meshes in the miniapps/nurbs directory. Currently the directory contains
   variable order NURBS versions of examples 1, 1p and 11p.
 
+- Added PUMI versions of examples ex1, ex1p, ex2 and ex6p in a new examples/pumi
+  directory. The new examples demonstrate the PUMI APIs for parallel and serial
+  mesh loading (ex1 and ex1p), applying BCs using classification (ex2), and
+  performing parallel mesh adaptation (ex6p).
+
 - Added two new miniapps related to DataCollection I/O in miniapps/tools:
   load-dc.cpp can be used to visualize fields saved via DataCollection classes;
   convert-dc.cpp demonstrates how to convert between MFEM's different concrete

CMakeLists.txt

@@ -45,7 +45,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 3.3.3)
+set(${PROJECT_NAME}_VERSION 3.4.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -139,7 +139,7 @@ if (MFEM_USE_MPI)
     set(PETSC_INCLUDE_DIRS ${PETSC_INCLUDES})
   endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU PETSC STRUMPACK)
+  set(PKGS_NEED_MPI SUPERLU PETSC STRUMPACK PUMI)
   foreach(PKG IN LISTS PKGS_NEED_MPI)
     if (MFEM_USE_${PKG})
       message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -246,6 +246,22 @@ if (MFEM_USE_SIDRE)
   find_package(Axom REQUIRED Sidre SLIC axom_utils)
 endif()
 
+# PUMI
+if (MFEM_USE_PUMI)
+  # If PUMI_DIR was specified, only link to that directory,
+  # i.e. don't link to another installation in /usr/lib by mistake
+  find_package(SCOREC 2.1.0 REQUIRED OPTIONAL_COMPONENTS gmi_sim
+    CONFIG PATHS ${PUMI_DIR} NO_DEFAULT_PATH)
+  if (SCOREC_FOUND)
+    # Define a header file with the MFEM_USE_SIMMETRIX preprocessor variable
+    set(MFEM_USE_SIMMETRIX ${SCOREC_gmi_sim_FOUND})
+    set(PUMI_FOUND ${SCOREC_FOUND})
+    get_target_property(PUMI_INCLUDE_DIRS
+      SCOREC::apf INTERFACE_INCLUDE_DIRECTORIES)
+    set(PUMI_LIBRARIES SCOREC::core)
+  endif()
+endif()
+
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -270,8 +286,8 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
-    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT GECKO GNUTLS NETCDF MPFR POSIXCLOCKS
-    MFEMBacktrace ZLIB)
+    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT GECKO GNUTLS NETCDF MPFR PUMI
+    POSIXCLOCKS MFEMBacktrace ZLIB)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")

CONTRIBUTING.md

@@ -1,3 +1,15 @@
+<p align="center">
+<a href="http://mfem.org/"><img alt="mfem" src="http://mfem.org/img/logo-300.png"></a>
+</p>
+
+<p align="center">
+<a href="https://github.com/mfem/mfem/blob/master/COPYRIGHT"><img alt="License" src="https://img.shields.io/badge/License-LGPL--2.1-brightgreen.svg"></a>
+<a href="https://travis-ci.org/mfem/mfem"><img alt="Build Status" src="https://travis-ci.org/mfem/mfem.svg?branch=master"></a>
+<a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
+<a href="http://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
+</p>
+
+
 # How to Contribute
 
 The MFEM team welcomes contributions at all levels: bugfixes; code
@@ -16,6 +28,7 @@ See the [Quick Summary](#quick-summary) section for the main highlights of our
 GitHub workflow. For more details, consult the following sections and refer
 back to them before issuing pull requests:
 
+- [Code Overview](#code-overview)
 - [GitHub Workflow](#github-workflow)
   - [MFEM Organization](#mfem-organization)
   - [New Feature Development](#new-feature-development)
@@ -53,6 +66,94 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
 - Don't hesitate to [contact us](#contact-information) if you have any questions.
 
 
+### Code Overview
+
+- The MFEM library uses object-orient design principles which reflect, in code,
+  the independent mathematical concepts of meshing, linear algebra and finite
+  element spaces and operators.
+
+- The MFEM source code has the following structure:
+  ```
+  .
+  ├── config
+  │   └── cmake
+  │       └── modules
+  ├── data
+  ├── doc
+  │   └── web
+  │       └── examples
+  ├── examples
+  │   ├── petsc
+  │   ├── pumi
+  │   └── sundials
+  ├── fem
+  ├── general
+  ├── linalg
+  ├── mesh
+  └── miniapps
+      ├── common
+      ├── electromagnetics
+      ├── meshing
+      ├── nurbs
+      ├── performance
+      └── tools
+  ```
+
+- The main directories are `fem/`, `mesh/` and `linalg/` containing the C++
+  classes implementing the finite element, mesh and linear algebra concepts
+  respectively.
+
+- The main mesh classes are:
+  + [`Mesh`](http://mfem.github.io/doxygen/html/classmfem_1_1Mesh.html)
+  + [`NCMesh`](http://mfem.github.io/doxygen/html/classmfem_1_1NCMesh.html)
+  + [`Element`](http://mfem.github.io/doxygen/html/classmfem_1_1Element.html)
+  + [`ElementTransformation`](http://mfem.github.io/doxygen/html/classmfem_1_1ElementTransformation.html)
+
+- The main finite element classes are:
+  + [`FiniteElement`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementCollection`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementSpace`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElementSpace.html)
+  + [`GridFunction`](http://mfem.github.io/doxygen/html/classmfem_1_1GridFunction.html)
+  + [`BilinearFormIntegrator`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html)
+  + [`LinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1MixedBilinearForm.html)
+
+- The main linear algebra classes and sources are
+  + [`Operator`](http://mfem.github.io/doxygen/html/classmfem_1_1Operator.html) and [`BilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html)
+  + [`Vector`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearForm.html)
+  + [`DenseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1SparseMatrix.html)
+  + Sparse [smoothers](http://mfem.github.io/doxygen/html/sparsesmoothers_8hpp.html) and linear [solvers](http://mfem.github.io/doxygen/html/solvers_8hpp.html)
+
+- Parallel MPI objects in MFEM inherit their serial counterparts, so a parallel
+  mesh for example is just a serial mesh on each task plus the information on
+  shared geometric entities between different tasks. The parallel source files
+  have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
+
+- The main parallel classes are
+  + [`ParMesh`](http://mfem.github.io/doxygen/html/solvers_8hpp.html)
+  + [`ParNCMesh`](http://mfem.github.io/doxygen/html/classmfem_1_1ParMesh.html)
+  + [`ParFiniteElementSpace`](http://mfem.github.io/doxygen/html/classmfem_1_1ParFiniteElementSpace.html)
+  + [`ParGridFunction`](http://mfem.github.io/doxygen/html/classmfem_1_1ParGridFunction.html)
+  + [`ParBilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1ParLinearForm.html)
+  + [`HypreParMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParVector.html)
+  + [`HypreSolver`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreSolver.html) and other [hypre classes](http://mfem.github.io/doxygen/html/hypre_8hpp.html)
+
+- The `general/` directory contains C++ classes that serve as utilities for
+  communication, error handling, arrays, (Boolean) tables, timing, etc.
+
+- The `config/` directory contains build-related files, both for the plain
+  Makefile and the CMake build options.
+
+- The `doc/` directory contains configuration for the Doxygen code documentation
+  that can either be build locally, or browsed online at
+  http://mfem.github.io/doxygen/html/index.html.
+
+- The `data/` directory contains a collection of small mesh files, that are used
+  in the simple example codes and more fully-featured mini applications in the
+  `examples/` and `miniapps/` directories.
+
+- See also the [code overview](http://mfem.org/code-overview/) section on the
+  MFEM website.
+
 ## GitHub Workflow
 
 The GitHub organization, https://github.com/mfem, is the main developer hub for
@@ -122,7 +223,7 @@ will allow us to reach you directly with project announcements.
   # Work on "feature-dev", add local commits
   # ...
 
-  # One time only) push the branch to github and setup your local
+  # (One time only) push the branch to github and setup your local
   # branch to track the github branch (for "git pull"):
   git push -u origin feature-dev
 
@@ -332,7 +433,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
     - [ ] `CHANGELOG`
     - [ ] `makefile`
     - [ ] `CMakeLists.txt`
-    - [ ] `doc/CodeDocumentation.conf`
+    - [ ] `doc/CodeDocumentation.conf.in`
 - [ ] (LLNL only) Make sure all `README.html` files in the source repo are up to date.
 - [ ] Tag the repository:
 

INSTALL

@@ -28,10 +28,13 @@ without GNU make or CMake can be found at the end of this file.
 In addition to the native build systems, MFEM packages are also available in the
 following package managers:
 
-- Spack, https://github.com/LLNL/spack
+- Spack, https://github.com/spack/spack
 - OpenHPC, http://openhpc.community
 - Homebrew/Science, https://github.com/Homebrew/homebrew-science
 
+We also recommend downloading and building the MFEM-based GLVis visualization
+tool which can be used to visualize the meshes and solution in MFEM's examples
+and miniapps. See http://glvis.org and http://mfem.org/building.
 
 Quick start with GNU make
 =========================
@@ -352,6 +355,13 @@ MFEM_USE_GZSTREAM = YES/NO
    before attempting to use it with MFEM.
    When enabled, this option uses the ZLIB_* library options, see below.
 
+MFEM_USE_PUMI = YES/NO
+   Enable the usage of PUMI (https://scorec.rpi.edu/pumi/) in MFEM. The Parallel
+   Unstructured Mesh Infrastructure (PUMI) is an unstructured, distributed mesh
+   data management system that is capable of handling general non-manifold
+   models and effectively supports automated adaptive analysis. PUMI enables
+   support for parallel unstructured mesh modifications in MFEM.
+
 MFEM_BUILD_TAG = (any value)
    An optional tag to characterize the build. Exported to config/config.mk.
    Can be used to identify the MFEM build from other makefiles.
@@ -461,6 +471,10 @@ The specific libraries and their options are:
        https://support.hdfgroup.org/HDF5 (HDF5)
   Options: CONDUIT_OPT, CONDUIT_LIB.
 
+- PUMI, used when MFEM_USE_PUMI = YES.
+  URL: https://scorec.rpi.edu/pumi
+  Options: PUMI_OPT, PUMI_LIB.
+
 - MPFR (optional), used when MFEM_USE_MPFR = YES.
   URL: http://mpfr.org, it depends on the GMP library: https://gmplib.org
   Options: MPFR_OPT, MPFR_LIB.
@@ -593,6 +607,7 @@ MFEM_USE_GNUTLS
 MFEM_USE_NETCDF
 MFEM_USE_MPFR
 MFEM_USE_GZSTREAM
+MFEM_USE_PUMI
 
 The following options are CMake specific:
 
@@ -638,6 +653,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - MPFR
  - LIBUNWIND
  - POSIXCLOCKS
+ - PUMI
 
 The following built-in CMake packages are also used:
 

README

@@ -12,11 +12,15 @@ to enable the research and development of scalable finite element discretization
 and solver algorithms through general finite element abstractions, accurate and
 flexible visualization, and tight integration with the hypre library.
 
-For building instructions, see the file INSTALL, or type "make help". Copyright
-information and licensing restrictions can be found in the file COPYRIGHT.
+* For building instructions, see the file INSTALL, or type "make help".
 
-The best starting point for new users interested in MFEM's features is the
-interactive documentation in examples/README.html.
+* Copyright and licensing information can be found in the file COPYRIGHT.
+
+* The best starting point for new users interested in MFEM's features is the
+  interactive documentation in examples/README.html.
+
+* Developers interested in contributing to the library, should read the
+  instructions and documentation in the CONTRIBUTING.md file.
 
 Conceptually, MFEM can be viewed as a finite element toolbox that provides the
 building blocks for developing finite element algorithms in a manner similar to
@@ -56,8 +60,8 @@ time integrators, etc.
 For examples of using MFEM, see the examples/ and miniapps/ directories, as well
 as the OpenGL visualization tool GLVis which is available at http://glvis.org.
 
-This project is released under the LGPL v2.1 license. See LICENSE file for full
-details.
+This project is released under the LGPL v2.1 license with static linking
+exception. See files COPYRIGHT and LICENSE file for full details.
 
 LLNL Release Number: LLNL-CODE-443211
 DOI: 10.11578/dc.20171025.1248

config/cmake/MFEMConfig.cmake.in

@@ -39,6 +39,7 @@ set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
+set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
 
 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")

config/cmake/config.hpp.in

@@ -98,6 +98,9 @@
 // Enable MFEM functionality based on Conduit
 #cmakedefine MFEM_USE_CONDUIT
 
+// Enable MFEM functionality based on the PUMI library
+#cmakedefine MFEM_USE_PUMI
+
 // Which library functions to use in class StopWatch for measuring time.
 // For a list of the available options, see INSTALL.
 // If not defined, an option is selected automatically.
@@ -113,4 +116,8 @@
 // Version of HYPRE used for building MFEM.
 #cmakedefine MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
 
+// Macro defined when PUMI is built with support for the Simmetrix SimModSuite
+// library.
+#cmakedefine MFEM_USE_SIMMETRIX
+
 #endif // MFEM_CONFIG_HEADER

config/config.hpp

@@ -36,4 +36,7 @@
 #ifdef MFEM_USE_PETSC
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
+#ifdef MFEM_USE_PUMI
+#error Building with PUMI (MFEM_USE_PUMI=YES) requires MPI (MFEM_USE_MPI=YES)
+#endif
 #endif // MFEM_USE_MPI not defined

config/config.hpp.in

@@ -109,6 +109,9 @@
 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR
 
+// Enable MFEM functionality based on the PUMI library
+// #define MFEM_USE_PUMI
+
 // Windows specific options
 #ifdef _WIN32
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)
@@ -118,4 +121,8 @@
 // Version of HYPRE used for building MFEM.
 // #define MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
 
+// Macro defined when PUMI is built with support for the Simmetrix SimModSuite
+// library.
+// #define MFEM_USE_SIMMETRIX
+
 #endif // MFEM_CONFIG_HEADER

config/config.mk.in

@@ -37,6 +37,7 @@ MFEM_USE_PETSC       = @MFEM_USE_PETSC@
 MFEM_USE_MPFR        = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE       = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT     = @MFEM_USE_CONDUIT@
+MFEM_USE_PUMI        = @MFEM_USE_PUMI@
 
 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@

config/defaults.cmake

@@ -40,6 +40,7 @@ option(MFEM_USE_PETSC "Enable PETSc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
+option(MFEM_USE_PUMI "Enable PUMI" OFF)
 
 # Allow a user to disable testing, examples, and/or miniapps at CONFIGURE TIME
 # if they don't want/need them (e.g. if MFEM is "just a dependency" and all they
@@ -145,6 +146,9 @@ set(AXOM_DIR "${MFEM_DIR}/../axom" CACHE PATH "Path to the Axom library.")
 set(Axom_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
     "Additional packages required by Axom.")
 
+set(PUMI_DIR "${MFEM_DIR}/../pumi-2.1.0" CACHE STRING
+    "Directory where PUMI is installed")
+
 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")

config/defaults.mk

@@ -106,6 +106,7 @@ MFEM_USE_PETSC       = NO
 MFEM_USE_MPFR        = NO
 MFEM_USE_SIDRE       = NO
 MFEM_USE_CONDUIT     = NO
+MFEM_USE_PUMI        = NO
 
 # Compile and link options for zlib.
 ZLIB_DIR =
@@ -271,6 +272,13 @@ SIDRE_LIB = \
    -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
    -lsidre -lslic -laxom_utils -lconduit -lconduit_relay -lhdf5 $(ZLIB_LIB) -ldl
 
+# PUMI
+# Note that PUMI_DIR is needed -- it is used to check for gmi_sim.h
+PUMI_DIR = @MFEM_DIR@/../pumi-2.1.0
+PUMI_OPT = -I$(PUMI_DIR)/include
+PUMI_LIB = -L$(PUMI_DIR)/lib -lpumi -lcrv -lma -lmds -lapf -lpcu -lgmi -lparma\
+   -llion -lmth -lapf_zoltan -lspr
+
 # If YES, enable some informational messages
 VERBOSE = NO
 

config/makefile

@@ -38,6 +38,9 @@ all: header config-mk
 MPI = $(MFEM_USE_MPI:NO=)
 GHV = get_hypre_version
 GHV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
+SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)
+SMX_PATH = $(PUMI_DIR)/include/gmi_sim.h
+SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))
 
 $(GHV): $(SRC)$(GHV).cpp
 	$(call mfem-info, Determining HYPRE version ...)
@@ -52,10 +55,16 @@ get-hypre-version: $(GHV).out
 	$(info HYPRE version: $(MFEM_HYPRE_VERSION)),\
 	$(error Unable to determine HYPRE version))
 
-header: $(if $(MPI),get-hypre-version,)
+check-smx:
+	$(call mfem-info, Checking for Simmetrix header [$(SMX_FILE)] ...)
+	$(eval MFEM_USE_SIMMETRIX:=$(if $(wildcard $(SMX_FILE)),YES,NO))
+	$(call mfem-info, MFEM_USE_SIMMETRIX = $(MFEM_USE_SIMMETRIX))
+	$(eval export MFEM_USE_SIMMETRIX)
+
+header: $(if $(MPI),get-hypre-version,) $(if $(SMX),check-smx)
 	$(call mfem-info, Writing $(CONFIG_HPP) ...)
 	@set -- && \
-	for def in $${MFEM_DEFINES} $(if $(MPI),MFEM_HYPRE_VERSION,); do \
+	for def in $${MFEM_DEFINES} $(if $(MPI),MFEM_HYPRE_VERSION) $(SMX); do \
 	   eval var=\$$$$def && \
 	   if [ "NO" != "$${var}" ]; then \
 	      set -- "$$@" -e "s|// \(#define $${def} \)|\1|" && \

config/test.mk

@@ -14,11 +14,13 @@
 # Colors used below:
 # green    '\033[0;32m'
 # red      '\033[0;31m'
+# yellow   '\033[0;33m'
 # no color '\033[0m'
 COLOR_PRINT = if [ -t 1 ]; then \
    printf $(1)$(2)'\033[0m'$(3); else printf $(2)$(3); fi
 PRINT_OK = $(call COLOR_PRINT,'\033[0;32m',OK,"  ($$1 $$2)\n")
 PRINT_FAILED = $(call COLOR_PRINT,'\033[0;31m',FAILED,"  ($$1 $$2)\n")
+PRINT_SKIP = $(call COLOR_PRINT,'\033[0;33m',SKIP,"\n")
 
 # Timing support
 define TIMECMD_detect

data/mixed-ele-meshes/annulus-trans-24.mesh

@@ -0,0 +1,218 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+74
+2 3 0 1 2 3
+2 3 1 5 6 2
+2 3 5 8 9 6
+2 3 8 11 12 9
+2 3 11 14 15 12
+2 3 14 17 18 15
+2 3 17 20 21 18
+2 3 20 23 24 21
+2 3 23 26 27 24
+2 3 26 29 30 27
+2 3 29 32 33 30
+2 3 32 35 36 33
+2 3 35 38 39 36
+2 3 38 41 42 39
+2 3 41 44 45 42
+2 3 44 47 48 45
+2 3 47 50 51 48
+2 3 50 53 54 51
+2 3 53 56 57 54
+2 3 56 59 60 57
+2 3 59 62 63 60
+2 3 62 65 66 63
+2 3 65 68 69 66
+2 3 68 71 72 69
+2 3 71 74 75 72
+1 2 2 3 4
+1 2 6 2 7
+1 2 9 6 10
+1 2 12 9 13
+1 2 15 12 16
+1 2 18 15 19
+1 2 21 18 22
+1 2 24 21 25
+1 2 27 24 28
+1 2 30 27 31
+1 2 33 30 34
+1 2 36 33 37
+1 2 39 36 40
+1 2 42 39 43
+1 2 45 42 46
+1 2 48 45 49
+1 2 51 48 52
+1 2 54 51 55
+1 2 57 54 58
+1 2 60 57 61
+1 2 63 60 64
+1 2 66 63 67
+1 2 69 66 70
+1 2 72 69 73
+1 2 75 72 76
+1 2 2 4 7
+1 2 6 7 10
+1 2 9 10 13
+1 2 12 13 16
+1 2 15 16 19
+1 2 18 19 22
+1 2 21 22 25
+1 2 24 25 28
+1 2 27 28 31
+1 2 30 31 34
+1 2 33 34 37
+1 2 36 37 40
+1 2 39 40 43
+1 2 42 43 46
+1 2 45 46 49
+1 2 48 49 52
+1 2 51 52 55
+1 2 54 55 58
+1 2 57 58 61
+1 2 60 61 64
+1 2 63 64 67
+1 2 66 67 70
+1 2 69 70 73
+1 2 72 73 76
+
+boundary
+53
+1 1 0 1
+1 1 1 5
+1 1 5 8
+1 1 8 11
+1 1 11 14
+1 1 14 17
+1 1 17 20
+1 1 20 23
+1 1 23 26
+1 1 26 29
+1 1 29 32
+1 1 32 35
+1 1 35 38
+1 1 38 41
+1 1 41 44
+1 1 44 47
+1 1 47 50
+1 1 50 53
+1 1 53 56
+1 1 56 59
+1 1 59 62
+1 1 62 65
+1 1 65 68
+1 1 68 71
+1 1 71 74
+1 1 74 75
+1 1 75 76
+1 1 76 73
+1 1 73 70
+1 1 70 67
+1 1 67 64
+1 1 64 61
+1 1 61 58
+1 1 58 55
+1 1 55 52
+1 1 52 49
+1 1 49 46
+1 1 46 43
+1 1 43 40
+1 1 40 37
+1 1 37 34
+1 1 34 31
+1 1 31 28
+1 1 28 25
+1 1 25 22
+1 1 22 19
+1 1 19 16
+1 1 16 13
+1 1 13 10
+1 1 10 7
+1 1 7 4
+1 1 4 3
+1 1 3 0
+
+vertices
+77
+2
+3.9788735773 0.0
+3.84329674785 1.02980825986
+2.88247256089 0.772356194895
+2.98415518297 0.0
+1.97241688113 0.259673608685
+3.44580559639 1.98943678865
+2.58435419729 1.49207759149
+1.83799993026 0.761324498753
+2.81348848799 2.81348848799
+2.11011636599 2.11011636599
+1.57832632157 1.21109238238
+1.98943678865 3.44580559639
+1.49207759149 2.58435419729
+1.21109238238 1.57832632157
+1.02980825986 3.84329674785
+0.772356194895 2.88247256089
+0.761324498753 1.83799993026
+2.43635739532e-16 3.9788735773
+1.82726804649e-16 2.98415518297
+0.259673608685 1.97241688113
+-1.02980825986 3.84329674785
+-0.772356194895 2.88247256089
+-0.259673608685 1.97241688113
+-1.98943678865 3.44580559639
+-1.49207759149 2.58435419729
+-0.761324498753 1.83799993026
+-2.81348848799 2.81348848799
+-2.11011636599 2.11011636599
+-1.21109238238 1.57832632157
+-3.44580559639 1.98943678865
+-2.58435419729 1.49207759149
+-1.57832632157 1.21109238238
+-3.84329674785 1.02980825986
+-2.88247256089 0.772356194895
+-1.83799993026 0.761324498753
+-3.9788735773 4.87271479065e-16
+-2.98415518297 3.65453609299e-16
+-1.97241688113 0.259673608685
+-3.84329674785 -1.02980825986
+-2.88247256089 -0.772356194895
+-1.97241688113 -0.259673608685
+-3.44580559639 -1.98943678865
+-2.58435419729 -1.49207759149
+-1.83799993026 -0.761324498753
+-2.81348848799 -2.81348848799
+-2.11011636599 -2.11011636599
+-1.57832632157 -1.21109238238
+-1.98943678865 -3.44580559639
+-1.49207759149 -2.58435419729
+-1.21109238238 -1.57832632157
+-1.02980825986 -3.84329674785
+-0.772356194895 -2.88247256089
+-0.761324498753 -1.83799993026
+-7.30907218597e-16 -3.9788735773
+-5.48180413948e-16 -2.98415518297
+-0.259673608685 -1.97241688113
+1.02980825986 -3.84329674785
+0.772356194895 -2.88247256089
+0.259673608685 -1.97241688113
+1.98943678865 -3.44580559639
+1.49207759149 -2.58435419729
+0.761324498753 -1.83799993026
+2.81348848799 -2.81348848799
+2.11011636599 -2.11011636599
+1.21109238238 -1.57832632157
+3.44580559639 -1.98943678865
+2.58435419729 -1.49207759149
+1.57832632157 -1.21109238238
+3.84329674785 -1.02980825986
+2.88247256089 -0.772356194895
+1.83799993026 -0.761324498753
+3.9788735773 -9.7454295813e-16
+2.98415518297 -7.30907218597e-16
+1.97241688113 -0.259673608685
+3.84329674785 1.02980825986
+2.88247256089 0.772356194895
+1.97241688113 0.259673608685

data/mixed-ele-meshes/annulus-trans-6.mesh

@@ -0,0 +1,74 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+20
+2 3 0 1 2 3
+2 3 1 5 6 2
+2 3 5 8 9 6
+2 3 8 11 12 9
+2 3 11 14 15 12
+2 3 14 17 18 15
+2 3 17 20 21 18
+1 2 2 3 4
+1 2 6 2 7
+1 2 9 6 10
+1 2 12 9 13
+1 2 15 12 16
+1 2 18 15 19
+1 2 21 18 22
+1 2 2 4 7
+1 2 6 7 10
+1 2 9 10 13
+1 2 12 13 16
+1 2 15 16 19
+1 2 18 19 22
+
+boundary
+17
+1 1 0 1
+1 1 1 5
+1 1 5 8
+1 1 8 11
+1 1 11 14
+1 1 14 17
+1 1 17 20
+1 1 20 21
+1 1 21 22
+1 1 22 19
+1 1 19 16
+1 1 16 13
+1 1 13 10
+1 1 10 7
+1 1 7 4
+1 1 4 3
+1 1 3 0
+
+vertices
+23
+2
+1.11408460164 0.0
+0.557042300822 0.964825566988
+0.417781725616 0.723619175241
+0.835563451232 0.0
+0.482412783494 0.278521150411
+-0.557042300822 0.964825566988
+-0.417781725616 0.723619175241
+3.41090035345e-17 0.557042300822
+-1.11408460164 1.36436014138e-16
+-0.835563451232 1.02327010604e-16
+-0.482412783494 0.278521150411
+-0.557042300822 -0.964825566988
+-0.417781725616 -0.723619175241
+-0.482412783494 -0.278521150411
+0.557042300822 -0.964825566988
+0.417781725616 -0.723619175241
+-1.02327010604e-16 -0.557042300822
+1.11408460164 -2.72872028276e-16
+0.835563451232 -2.04654021207e-16
+0.482412783494 -0.278521150411
+0.557042300822 0.964825566988
+0.417781725616 0.723619175241
+0.482412783494 0.278521150411

data/mixed-ele-meshes/circInSquare.py

@@ -0,0 +1,924 @@
+#Title:circInSquare.py
+#Author:T. M. McManus
+#Date:10-7-18
+#Purpose: Fill a circular sector with triangles and a bounding region,
+#defined by 3 nodes, with quads.  Then reflect/preserve QuadI twice to
+#create a complete disc bounded in a square.
+
+import scipy as sp
+import argparse
+import sys
+import subprocess
+import time
+
+parser=argparse.ArgumentParser(description='Fill a circular sector with triangles and a bounding region,\
+defined by 3 nodes, with quads.  Then reflect/preserve QuadI twice to create a complete disc bounded in a square.'
+,epilog='Sample run: python circInSquare.py -r 1 -e 2 -n 8 -g ../../../glvis/glvis')
+
+parser.add_argument('-r','--circRad', nargs='?',const=1, default = 1.0, type=float, help='Radius of circle')
+parser.add_argument('-e','--edgeLength', nargs='?',const=1,default=2.0,type=float,help='Edge-length of bounding square')
+parser.add_argument('-n','--numEdges',nargs='?',const=1,default=6,type=int,help='n-gon approximation of internal circle')
+parser.add_argument('-o','--outputFile',nargs='?',const=1,default='circInSquare', help='Output file name.')
+parser.add_argument('-g','--glvis',nargs='?',const=1,default='',type=str,help='Abs. or rel. path of glvis binary.')
+args=parser.parse_args()
+
+r=args.circRad 
+edgeLength=args.edgeLength
+numEdges=args.numEdges
+outputName=args.outputFile
+glvis=args.glvis
+
+visMesh=False;
+
+if glvis!='':
+    visMesh=True
+
+if r >= edgeLength:
+    print("Circle radius must be less than bounding square edge length")
+    sys.exit(1)
+
+if sp.mod(numEdges,2) != 0:
+    print("Currently this mixed element generator only supports an even numbers of edges.")
+    sys.exit(1)
+
+    
+#The basic idea:
+#1. Construct topology for regions
+#2. Combine topologies
+#3. Construct boundary
+#4. Construct geometry for regions
+#5. Combine geometries
+#6. Output
+
+def eleMatCirc(numEdges):
+    
+    nNodesSeq=sp.zeros([numEdges])
+    nNodesSeq[0]=3
+    if numEdges != 1:
+        for n in range(1,numEdges):
+            nNodesSeq[n]=nNodesSeq[n-1]+(2+n)
+            
+    numCircNodesTot =int(((numEdges+1)*(numEdges+2))/2)
+    
+    b=range(numCircNodesTot)
+    row_size=1
+    A=sp.zeros([numEdges+1,numEdges+1])
+    start=0;stop=1;
+    for m in range(numEdges+1):
+        if m==0:
+            A[m,range(m+1)]=b[0:1]
+            start=0
+            stop=1
+        else:
+            start=stop
+            stop=stop+m+1
+            A[m,range(m+1)]=b[start:stop]
+
+    M=sp.ones([numEdges**2,5])
+    m_row=0
+    for m in range(numEdges):
+        if m==0:
+            M[0,:]=[1,2,0,1,2]
+            m_row+=1
+        else:
+           holder=sp.size(sp.nonzero(A[m,:]))
+           for n in range(holder):
+               if n!=holder-1:
+                   M[m_row,:]=[1,2,A[m,n],A[m,n+1],A[m+1,n+1]]
+                   m_row+=1
+                   M[m_row,:]=[1,2,A[m,n],A[m+1,n],A[m+1,n+1]]
+                   m_row+=1
+               else:
+                   M[m_row,:]=[1,2,A[m,n],A[m+1,n],A[m+1,n+1]]
+                   m_row+=1
+            
+    return M.astype(int),numCircNodesTot
+
+def eleMatQuad(numEdges):
+    S0=numEdges*(numEdges+1)/(2.0)
+    A=sp.linspace(S0,(S0+(numEdges+1)**2)-1,(numEdges+1)**2)
+    A=A.reshape([numEdges+1,numEdges+1])
+    quadNode=sp.delete(A,-1,1)
+    quadNode=sp.delete(quadNode,-1,0)
+    quadNode=quadNode.flatten()
+    M=sp.zeros([numEdges**2,6])
+    for n in range(numEdges**2):
+        M[n,:]=[2,3,quadNode[n],quadNode[n]+1,quadNode[n]+numEdges+2,quadNode[n]+numEdges+1]
+    return M.astype(int)
+
+def boundMatTot(numEdges):
+    triS1=sp.zeros(numEdges+1)
+    triS3=sp.zeros(numEdges+1)
+    quadS1=sp.zeros(numEdges)
+    quadS2=sp.zeros(numEdges-1)
+    quadS3=sp.zeros(numEdges)
+
+    triS1[0]=0;
+    triS3[0]=0;
+    for n in range(1,numEdges+1):
+        triS1[n]=triS1[n-1]+n
+        triS3[n]=triS1[n]+n
+    ref1=triS3
+    
+    triS3=sp.flipud(triS3)
+    quadS1[0]=triS1[-1]+numEdges+1
+    quadS3[0]=triS1[-1]+2*numEdges+1
+
+    for n in range(1,numEdges):
+        quadS1[n]=quadS1[n-1]+(numEdges+1)
+        quadS3[n]=quadS3[n-1]+(numEdges+1)    
+    ref2=quadS3
+    xAxisRootRef=sp.concatenate([triS1.copy(),quadS1],axis=0)
+    quadS3=sp.flipud(quadS3)
+    quadS2=range(int(quadS1[-1]+1),int(quadS3[0]),1)
+    STOT=sp.concatenate([triS1,quadS1,quadS2,quadS3,triS3],axis=0)
+
+    filler=sp.zeros(1)
+    filler[0]=quadS3[0]
+    fillerFirst=sp.zeros(1)
+    fillerFirst[0]=quadS1[-1]
+    sTotRef=sp.concatenate([triS1,quadS1,quadS2,filler],axis=0)
+    newsTotRef=sp.concatenate([fillerFirst,quadS2,filler],axis=0)
+    boundMat=sp.zeros([STOT.size-1,4])
+    boundMatRef=sp.zeros([sTotRef.size-1,4])
+    new_boundMat_ref=sp.zeros([newsTotRef.size-1,4])
+    
+    for n in range(STOT.size-1):
+        boundMat[n,:]=[1,1,STOT[n],STOT[n+1]]
+    for n in range(sTotRef.size-1):
+        boundMatRef[n,:]=[1,1,sTotRef[n],sTotRef[n+1]]
+    for n in range(newsTotRef.size-1):
+        new_boundMat_ref[n,:]=[1,1,newsTotRef[n],newsTotRef[n+1]]
+        
+    ref=sp.concatenate([ref1,ref2],axis=0).astype(int)
+    return boundMat.astype(int),ref,boundMatRef.astype(int),xAxisRootRef.astype(int),new_boundMat_ref.astype(int)
+
+def vertMatCirc(numEdges):
+    r_o=sp.linspace(0,r,numEdges+1)
+    counter=0
+    vertMat=sp.zeros([numCircNodesTot,2])
+    for m in range(numEdges+1):
+        theta=sp.linspace(0,sp.pi/2.0,m+1)
+        for n in range(sp.size(theta)):
+            vertMat[counter,:]=[r_o[m]*sp.cos(theta[n]),r_o[m]*sp.sin(theta[n])]
+            counter+=1
+    return vertMat
+
+def vertMatQuad(numEdges):
+
+    theta=sp.linspace(0,sp.pi/2.0,numEdges+1)
+    AX=sp.zeros([numEdges+1,numEdges+1])
+    AY=sp.zeros([numEdges+1,numEdges+1])
+    AX[0,:]=r*sp.cos(theta)
+    AY[0,:]=r*sp.sin(theta)
+
+    vertLinSpace=sp.linspace(0,edgeLength,(numEdges/2)+1)
+    horzLineSpace=sp.linspace(edgeLength,0,(numEdges/2)+1)
+
+    #Assigning node locations along the boundary
+    vertCount=0
+    horzCount=1
+    for n in range(numEdges+1):
+        if n < (numEdges/2):
+            AX[-1,n]=edgeLength
+            AY[-1,n]=vertLinSpace[vertCount]
+            vertCount+=1
+        elif n == int(numEdges/2):
+            AX[-1,n]=edgeLength
+            AY[-1,n]=edgeLength
+        else:
+            AX[-1,n]=horzLineSpace[horzCount]
+            AY[-1,n]=edgeLength
+            horzCount+=1
+
+    #Linearly spacing nodes between the inner/outer boundaries
+    #One could then smooth this via r-based adaptivity
+    for col in range(numEdges+1):
+        for row in range(1,numEdges):
+            AX[row,col]=sp.linspace(AX[0,col],AX[-1,col],numEdges+1)[row]
+            AY[row,col]=sp.linspace(AY[0,col],AY[-1,col],numEdges+1)[row]
+
+    AX=sp.delete(AX,0,0)
+    AY=sp.delete(AY,0,0)
+    AX=AX.flatten()
+    AY=AY.flatten()
+    AX_reshape = AX.flatten()
+    
+    numQuadNodesTot=numEdges*(numEdges+1)
+    vertMat=sp.zeros([numQuadNodesTot,2])
+    for n in range(numQuadNodesTot):
+        vertMat[n,:]=[AX[n],AY[n]]    
+    return vertMat
+
+def orient(A):
+    aOrient=sp.zeros([A.shape[0],A.shape[1]])
+    triCounter=0
+    quadCounter=0
+    #Determine the number of triangle and quad elments in the given element matrix
+    for n in range(A.shape[0]):
+        if A[n,1]==2:
+            triCounter+=1
+        else:
+            quadCounter+=1
+    edgeMatTotal=sp.zeros([3*triCounter+4*quadCounter,2])
+    counter=0
+    for n in range(A.shape[0]):
+        detected=0
+        if A[n,1]==2:
+            for m in range(edgeMatTotal.shape[0]):
+                if detected != 1:
+                    if edgeMatTotal[m,0]==A[n,2] and edgeMatTotal[m,1]==A[n,3]:
+                        aOrient[n,:]=[1,2,A[n,2],A[n,4],A[n,3],0]
+                        detected=1
+                        #print("reorder:[{} {} {}] to [{} {} {}]".format(A[n,2],A[n,3],A[n,4],int(aOrient[n,2]),int(aOrient[n,3]),int(aOrient[n,4])))
+                    elif edgeMatTotal[m,0]==A[n,4] and edgeMatTotal[m,1]==A[n,2]:
+                        aOrient[n,:]=[1,2,A[n,2],A[n,4],A[n,3],0]
+                        detected=1
+                    else:
+                        aOrient[n,:]=A[n,:]
+
+            edgeMatTotal[counter,:]=[aOrient[n,2],aOrient[n,3]]
+            counter+=1
+            edgeMatTotal[counter,:]=[aOrient[n,3],aOrient[n,4]]
+            counter+=1
+            edgeMatTotal[counter,:]=[aOrient[n,4],aOrient[n,2]]
+            counter+=1
+        else:
+            for m in range(edgeMatTotal.shape[0]):
+                if detected != 1: 
+                    if edgeMatTotal[m,0]==A[n,2] and edgeMatTotal[m,1]==A[n,3]:
+                        aOrient[n,:]=[2,3,A[n,2],A[n,5],A[n,4],A[n,3]]
+                        detected=1
+                        #print("reorder:[{} {} {} {}] to [{} {} {} {}]".format(A[n,2],A[n,3],A[n,4],A[n,5],int(aOrient[n,2]),int(aOrient[n,3]),int(aOrient[n,4]),int(aOrient[n,5])))
+                    elif edgeMatTotal[m,0]==A[n,5] and edgeMatTotal[m,1]==A[n,2]:
+                        aOrient[n,:]=[2,3,A[n,2],A[n,5],A[n,4],A[n,3]]
+                        detected=1
+                    else:
+                        aOrient[n,:]=A[n,:]
+            edgeMatTotal[counter,:]=[aOrient[n,2],aOrient[n,3]]
+            counter+=1
+            edgeMatTotal[counter,:]=[aOrient[n,3],aOrient[n,4]]
+            counter+=1
+            edgeMatTotal[counter,:]=[aOrient[n,4],aOrient[n,5]]
+            counter+=1
+            edgeMatTotal[counter,:]=[aOrient[n,5],aOrient[n,2]]
+            counter+=1
+    return aOrient.astype(int)
+
+def gVis(_glvis,_meshFile):
+
+    if(_glvis==''):
+        print("Failure: Set glvis location via -g switch")
+        sys.exit(1)
+
+    colFuncFileName=_meshFile.replace('.mesh','.gf')
+    glvsScriptFileName=_meshFile.replace('.mesh','.glvs')
+    imageFileName=_meshFile.replace('.mesh','.png')
+    
+    #Create Coloring Function for mesh
+    _colFuncCommand=_glvis+ ' -m '+ _meshFile +' -sc -k q'
+    args=_colFuncCommand.split()
+    p=subprocess.Popen(args)#Create 'GLVis_coloring.gf'
+
+    _renameCommand='mv GLVis_coloring.gf {}'.format(colFuncFileName)
+    args=_renameCommand.split()
+    p=subprocess.Popen(args)
+
+    #Glvis script template
+    f=open(glvsScriptFileName,'w')
+    f.write('window 0 0 800 800\n'+'\n')
+    f.write('solution {} {}\n'.format(_meshFile,colFuncFileName)+'\n')
+    f.write('{\n'+'perspective off\n'+'zoom 1.5\n'+'keys gAeeRM\n'+'solution {} {} screenshot {}\n'.format(_meshFile,colFuncFileName,imageFileName)+'keys q\n'+'}\n')
+    f.close()
+
+    _runGlvisCommand=_glvis+' -run {}'.format(glvsScriptFileName)
+    args=_runGlvisCommand.split()
+    p=subprocess.Popen(args)
+    p.wait()
+    
+    return 0
+def quadInterDof(_edge,_linEleMat,_linVertMatRound):
+    _state=False
+    for n in range(_linEleMat.shape[0]):
+        if _linEleMat[n,1]==3:
+            if sp.any(_edge[0]==_linEleMat[n,2:6]) and sp.any(_edge[1]==_linEleMat[n,2:6]):
+                print("{} is possibly in {}".format(_edge,_linEleMat[n,2:6]))
+                _n1Loc=sp.where(_edge[0]==_linEleMat[n,2:6])[0][0]
+                _n2Loc=sp.where(_edge[1]==_linEleMat[n,2:6])[0][0]
+                if _n1Loc==sp.mod(_n2Loc+1,4) or _n1Loc==sp.mod(_n2Loc-1,4):
+                    _state=True
+                    xcent=(_linVertMatRound[_linEleMat[n,2],0]+_linVertMatRound[_linEleMat[n,3],0]+_linVertMatRound[_linEleMat[n,4],0]+_linVertMatRound[_linEleMat[n,5],0])/4.0
+                    ycent=(_linVertMatRound[_linEleMat[n,2],1]+_linVertMatRound[_linEleMat[n,3],1]+_linVertMatRound[_linEleMat[n,4],1]+_linVertMatRound[_linEleMat[n,5],1])/4.0
+                    _interDof=sp.zeros(2)
+                    _interDof[0]=sp.round_((_linVertMatRound[_edge[0],0]+_linVertMatRound[_edge[1],0]+xcent)/3.0,5)
+                    _interDof[1]=sp.round_((_linVertMatRound[_edge[0],1]+_linVertMatRound[_edge[1],1]+ycent)/3.0,5)
+                    print("dof loc is {},{}".format(_interDof[0],_interDof[1]))
+                    return(_state,_interDof[0],_interDof[1])
+
+    return(_state,0,0)
+
+[eleMatTriHolder,numCircNodesTot]=eleMatCirc(numEdges) #Construct tri element matrix for the region inside circular sector
+eleMatQuadHolder=eleMatQuad(numEdges) #Construct quad element matrix for region outside the circular sector
+
+#Combining eleMatTriHolder and eleMatQuadHolder
+linEleMat=sp.zeros([eleMatTriHolder.shape[0]+eleMatQuadHolder.shape[0],6])
+counter=0
+for n in range(eleMatTriHolder.shape[0]):
+    linEleMat[n,[0,1,2,3,4]]=eleMatTriHolder[n,:]
+    counter+=1
+for n in range(eleMatQuadHolder.shape[0]):
+    linEleMat[counter+n,:]=eleMatQuadHolder[n,:]
+
+linEleMat=linEleMat.astype(int)
+linBoundMat=boundMatTot(numEdges)[0] #Construct the boundary
+vertMatCircHolder = vertMatCirc(numEdges) #Construct vertex matrix for triang region
+vertMatQuadHolder = vertMatQuad(numEdges) #Construct vertex matrix for the quad region
+
+
+#Combining the two vertex matrices in Quadrant I (q1)
+linVertMat=sp.zeros([vertMatCircHolder.shape[0]+vertMatQuadHolder.shape[0],2])
+counter=0
+for n in range(vertMatCircHolder.shape[0]):
+    linVertMat[n,:]=vertMatCircHolder[n,:]
+    counter+=1
+for n in range(vertMatQuadHolder.shape[0]):
+    linVertMat[counter+n,:]=vertMatQuadHolder[n,:]
+
+#Outputting P1/Q1 mesh to a .mesh file
+g=open(outputName+'Lin.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(linEleMat.shape[0]))
+for n in range(linEleMat.shape[0]):
+    if linEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4],linEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(linBoundMat.shape[0]))
+for n in range(linBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(linBoundMat[n,0],linBoundMat[n,1],linBoundMat[n,2],linBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(linVertMat.shape[0])+'2\n')
+for n in range(linVertMat.shape[0]):
+    g.write('{} {}\n'.format(linVertMat[n,0],linVertMat[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'Lin.mesh')
+
+#Quadratic (P2/Q2) Element Generation
+
+#1.)Create Edge list from previously generated linear elements
+edgeMat=sp.zeros([3*eleMatTriHolder.shape[0]+4*eleMatQuadHolder.shape[0],2])
+linEleMat=orient(linEleMat)#Make sure that element orientation is in agreement with MFEM requirements
+
+counter=0
+for n in range(linEleMat.shape[0]):
+    if linEleMat[n,1]==2:
+        edgeMat[counter,:]=[linEleMat[n,2],linEleMat[n,3]]
+        counter+=1
+        edgeMat[counter,:]=[linEleMat[n,3],linEleMat[n,4]]
+        counter+=1
+        edgeMat[counter,:]=[linEleMat[n,4],linEleMat[n,2]]
+        counter+=1
+    else:
+        edgeMat[counter,:]=[linEleMat[n,2],linEleMat[n,3]]
+        counter+=1
+        edgeMat[counter,:]=[linEleMat[n,3],linEleMat[n,4]]
+        counter+=1
+        edgeMat[counter,:]=[linEleMat[n,4],linEleMat[n,5]]
+        counter+=1
+        edgeMat[counter,:]=[linEleMat[n,5],linEleMat[n,2]]
+        counter+=1
+
+#Remove duplicates
+holder=[]
+for n in range(edgeMat.shape[0]):
+    counter=0
+    for m in range(edgeMat.shape[0]):
+        if edgeMat[n,0]==edgeMat[m,0] and edgeMat[n,1]==edgeMat[m,1] and m!=n:
+            holder.append([n,m])
+        elif edgeMat[n,1]==edgeMat[m,0] and edgeMat[n,0]==edgeMat[m,1] and m!=n:
+            holder.append([n,m])
+
+removeIndices=sp.zeros(len(holder))
+for n in range(len(holder)):
+    if holder[n][0]>holder[n][1]:
+        removeIndices[n]=holder[n][0]
+    else:
+        removeIndices[n]=holder[n][1]
+removeIndices=sp.unique(removeIndices).astype(int)
+edgeMat=sp.delete(edgeMat,removeIndices,0)
+edgeMat=edgeMat.astype(int)
+
+edgeDofMat=sp.zeros([edgeMat.shape[0],2])#These will be the new DoFs that appear after the Element Vertices within the .mesh file
+linVertMatRound=sp.round_(linVertMat,5) 
+ 
+counter=0
+
+for n in edgeMat:
+    if linVertMatRound[n[0],1] == linVertMatRound[n[1],1]:
+        xmid=(linVertMatRound[n[0],0]+linVertMatRound[n[1],0])/2.0
+        ymid=linVertMatRound[n[0],1]
+        edgeDofMat[counter,:]=[xmid,ymid]
+    elif linVertMatRound[n[0],0] == linVertMatRound[n[1],0]:
+        xmid=linVertMatRound[n[0],0]
+        ymid=(linVertMatRound[n[0],1]+linVertMatRound[n[1],1])/2.0
+        edgeDofMat[counter,:]=[xmid,ymid]
+    else:
+        r0=sp.sqrt(linVertMatRound[n[0],0]**2+linVertMatRound[n[0],1]**2)
+        r1=sp.sqrt(linVertMatRound[n[1],0]**2+linVertMatRound[n[1],1]**2)
+        rmid = (r0+r1)/2.0 #should not be needed
+        xmidOld=(linVertMatRound[n[0],0]+linVertMatRound[n[1],0])/2.0
+        ymidOld=(linVertMatRound[n[0],1]+linVertMatRound[n[1],1])/2.0
+        midtheta=sp.arctan(ymidOld/xmidOld)
+        xmid=rmid*sp.cos(midtheta)
+        ymid=rmid*sp.sin(midtheta)
+        edgeDofMat[counter,:]=[xmid,ymid]
+    counter+=1
+edgeDofMat = sp.round_(edgeDofMat,5)
+
+#Determine midpoints of all Q1 elements:
+quadCentroidLoc=sp.zeros([eleMatQuadHolder.shape[0],2])
+for n in range(eleMatQuadHolder.shape[0]):
+    quadCentroidLoc[n,0]=(linVertMatRound[eleMatQuadHolder[n,2],0]+linVertMatRound[eleMatQuadHolder[n,3],0]+linVertMatRound[eleMatQuadHolder[n,4],0]+linVertMatRound[eleMatQuadHolder[n,5],0])/4.0
+    quadCentroidLoc[n,1]=(linVertMatRound[eleMatQuadHolder[n,2],1]+linVertMatRound[eleMatQuadHolder[n,3],1]+linVertMatRound[eleMatQuadHolder[n,4],1]+linVertMatRound[eleMatQuadHolder[n,5],1])/4.0
+
+quadCentroidLoc = sp.round_(quadCentroidLoc,5)
+
+#3.)Populate nodes section
+g=open(outputName+'Quad.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(linEleMat.shape[0]))
+for n in range(linEleMat.shape[0]):
+    if linEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4],linEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(linBoundMat.shape[0]))
+for n in range(linBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(linBoundMat[n,0],linBoundMat[n,1],linBoundMat[n,2],linBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(linVertMat.shape[0]))
+g.write('\n'+'nodes'+'\n'+'FiniteElementSpace'+'\n'+'FiniteElementCollection: H1_2D_P2'+'\n'+'VDim: 2'+'\n'+'Ordering: 1' +'\n\n')
+for n in range(linVertMatRound.shape[0]):
+    g.write('{} {}\n'.format(linVertMatRound[n,0],linVertMatRound[n,1]))
+for n in range(edgeDofMat.shape[0]):
+    g.write('{} {}\n'.format(edgeDofMat[n,0],edgeDofMat[n,1]))
+for n in range(quadCentroidLoc.shape[0]):
+    g.write('{} {}\n'.format(quadCentroidLoc[n,0],quadCentroidLoc[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'Quad.mesh')
+
+#Cubic (P3/Q3) Element Generation
+
+cubeDofMat=sp.zeros([2*edgeMat.shape[0],2])#These will be the new DoFs that appear after the Element Vertices within the .mesh file
+
+counter=0
+for n in edgeMat: #Here DoF ordering matters. 
+    if linVertMatRound[n[0],1] == linVertMatRound[n[1],1]:
+        xmid=(linVertMatRound[n[0],0]+linVertMatRound[n[1],0])/2.0
+        ymid=linVertMatRound[n[0],1]
+        xmid1=(linVertMatRound[n[0],0]+xmid)/2.0
+        ymid1=linVertMatRound[n[0],1]
+        xmid2=(linVertMatRound[n[1],0]+xmid)/2.0
+        ymid2=linVertMatRound[n[0],1]
+        if n[0] > n[1]:
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+        else:
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+            
+    elif linVertMatRound[n[0],0] == linVertMatRound[n[1],0]:
+        xmid=linVertMatRound[n[0],0]
+        ymid=(linVertMatRound[n[0],1]+linVertMatRound[n[1],1])/2.0
+        xmid1=linVertMatRound[n[0],0]
+        ymid1=(linVertMatRound[n[0],1]+ymid)/2.0
+        xmid2=linVertMatRound[n[0],0]
+        ymid2=(linVertMatRound[n[1],1]+ymid)/2.0
+        if n[0] > n[1]:
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+        else:
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+    else:
+        r0=sp.sqrt(linVertMatRound[n[0],0]**2+linVertMatRound[n[0],1]**2)
+        r1=sp.sqrt(linVertMatRound[n[1],0]**2+linVertMatRound[n[1],1]**2)
+        rmid = (r0+r1)/2.0 #should not be needed
+        xmidOld=(linVertMatRound[n[0],0]+linVertMatRound[n[1],0])/2.0
+        ymidOld=(linVertMatRound[n[0],1]+linVertMatRound[n[1],1])/2.0
+        midtheta=sp.arctan(ymidOld/xmidOld)
+        xmid=rmid*sp.cos(midtheta)
+        ymid=rmid*sp.sin(midtheta)
+        xmid1=(linVertMatRound[n[0],0]+xmid)/2.0
+        ymid1=(linVertMatRound[n[0],1]+ymid)/2.0
+        xmid2=(linVertMatRound[n[1],0]+xmid)/2.0
+        ymid2=(linVertMatRound[n[1],1]+ymid)/2.0
+        if n[0] > n[1]:
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+        else:
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+            
+cubeDofMat = sp.round_(cubeDofMat,5)
+
+triCentroidLoc=sp.zeros([eleMatTriHolder.shape[0],2])
+
+for n in range(eleMatTriHolder.shape[0]):
+    triCentroidLoc[n,0]=(linVertMatRound[eleMatTriHolder[n,2],0]+linVertMatRound[eleMatTriHolder[n,3],0]+linVertMatRound[eleMatTriHolder[n,4],0])/3.0
+    triCentroidLoc[n,1]=(linVertMatRound[eleMatTriHolder[n,2],1]+linVertMatRound[eleMatTriHolder[n,3],1]+linVertMatRound[eleMatTriHolder[n,4],1])/3.0
+
+quadCentroidLocCubic=sp.zeros([4*eleMatQuadHolder.shape[0],2])
+
+counter=0
+for n in range(eleMatQuadHolder.shape[0]):
+    xcent=quadCentroidLoc[n,0];ycent=quadCentroidLoc[n,1]
+    a=eleMatQuadHolder[n,2:6]
+    aMinIndex=sp.where(a[:]==a.min())[0][0]
+    dof0=0.5*sp.array([xcent+linVertMatRound[a[aMinIndex],0],ycent+linVertMatRound[a[aMinIndex],1]])
+    quadCentroidLocCubic[counter,:]=dof0
+    counter+=1
+    if aMinIndex==0:
+        aLeft=-1
+        aRight=1
+        aLast=2
+    else:
+        aLeft=aMinIndex-1
+        aRight=aMinIndex+1
+        aLast=sp.delete(a,[aMinIndex,aLeft,aRight])[0]
+    edge1=[a[aMinIndex], a[aLeft]]
+    edge2=[a[aMinIndex], a[aRight]]
+    edge1Index=0
+    edge2Index=0
+    edgeCounter=0
+    for edge in edgeMat:
+        if(edge[0]==edge1[0] and edge[1]==edge1[1]) or (edge[1]==edge1[0] and edge[0]==edge1[1]):
+            edge1Index=edgeCounter
+        if(edge[0]==edge2[0] and edge[1]==edge2[1]) or (edge[1]==edge2[0] and edge[0]==edge2[1]):
+            edge2Index=edgeCounter
+        edgeCounter+=1
+
+    if (edge1Index > edge2Index):
+        dof1=0.5*sp.array([xcent+linVertMatRound[a[aLeft],0],ycent+linVertMatRound[a[aLeft],1]])
+        quadCentroidLocCubic[counter,:]=dof1
+        counter+=1
+        dof2=0.5*sp.array([xcent+linVertMatRound[a[aRight],0],ycent+linVertMatRound[a[aRight],1]])
+        quadCentroidLocCubic[counter,:]=dof2
+        counter+=1
+        dof3=0.5*sp.array([xcent+linVertMatRound[a[aLast],0],ycent+linVertMatRound[a[aLast],1]])
+        quadCentroidLocCubic[counter,:]=dof3
+        counter+=1
+    else:
+        dof1=0.5*sp.array([xcent+linVertMatRound[a[aRight],0],ycent+linVertMatRound[a[aRight],1]])
+        quadCentroidLocCubic[counter,:]=dof1
+        counter+=1
+        dof2=0.5*sp.array([xcent+linVertMatRound[a[aLeft],0],ycent+linVertMatRound[a[aLeft],1]])
+        quadCentroidLocCubic[counter,:]=dof2
+        counter+=1
+        dof3=0.5*sp.array([xcent+linVertMatRound[a[aLast],0],ycent+linVertMatRound[a[aLast],1]])
+        quadCentroidLocCubic[counter,:]=dof3
+        counter+=1
+        
+truCentroidLoc=sp.round_(triCentroidLoc,5)
+    
+#3.)Populate nodes section
+g=open(outputName+'Cub.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(linEleMat.shape[0]))
+for n in range(linEleMat.shape[0]):
+    if linEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4],linEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(linBoundMat.shape[0]))
+for n in range(linBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(linBoundMat[n,0],linBoundMat[n,1],linBoundMat[n,2],linBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(linVertMat.shape[0]))
+g.write('\n'+'nodes'+'\n'+'FiniteElementSpace'+'\n'+'FiniteElementCollection: H1_2D_P3'+'\n'+'VDim: 2'+'\n'+'Ordering: 1' +'\n\n')
+for n in range(linVertMatRound.shape[0]):
+    g.write('{} {}\n'.format(linVertMatRound[n,0],linVertMatRound[n,1]))
+for n in range(cubeDofMat.shape[0]):
+    g.write('{} {}\n'.format(cubeDofMat[n,0],cubeDofMat[n,1]))
+for n in range(triCentroidLoc.shape[0]):
+    g.write('{} {}\n'.format(triCentroidLoc[n,0],triCentroidLoc[n,1]))
+for n in range(quadCentroidLocCubic.shape[0]):
+    g.write('{} {}\n'.format(quadCentroidLocCubic[n,0],quadCentroidLocCubic[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'Cub.mesh')
+#raw_input()
+#'Reflecting' topology about one of its edges and append it to itself
+upperPlaneEleMat = sp.zeros([2*linEleMat.shape[0],6])
+for n in range(linEleMat.shape[0]):
+    upperPlaneEleMat[n,:]=linEleMat[n,:]
+    
+#Create ele_mat_holder.shape[0]x2 matrix for mapping
+refEdge=boundMatTot(numEdges)[1]
+q1NumNodes=linVertMat.shape[0]
+
+mapping = sp.zeros([q1NumNodes])
+counter=0
+for n in range(q1NumNodes):
+     if (sp.any(refEdge == n)):
+         mapping[n]=n
+     else:
+         mapping[n]=counter+q1NumNodes
+         counter+=1
+
+mapping=mapping.astype(int)
+#Implement mapping
+
+counter=0
+for n in range(linEleMat.shape[0],2*linEleMat.shape[0]):
+    upperPlaneEleMat[n,0]=linEleMat[counter,0]
+    upperPlaneEleMat[n,1]=linEleMat[counter,1]
+    upperPlaneEleMat[n,2]=mapping[linEleMat[counter,2]]
+    upperPlaneEleMat[n,3]=mapping[linEleMat[counter,3]]
+    upperPlaneEleMat[n,4]=mapping[linEleMat[counter,4]]
+    upperPlaneEleMat[n,5]=mapping[linEleMat[counter,5]]
+    counter+=1
+    
+upperPlaneEleMat = upperPlaneEleMat.astype(int)
+
+#Reflecting boundary matrix
+origBound=boundMatTot(numEdges)[2]
+upperPlaneBoundMat=sp.zeros([2*origBound.shape[0],4])
+for n in range(origBound.shape[0]):
+    upperPlaneBoundMat[n,:]=origBound[n,:]
+counter=0
+newOrigBound=origBound.copy()
+newOrigBound[:,2]=sp.flipud(origBound[:,3])
+newOrigBound[:,3]=sp.flipud(origBound[:,2])
+for n in range(newOrigBound.shape[0],upperPlaneBoundMat.shape[0]):
+    upperPlaneBoundMat[n,0]=newOrigBound[counter,0]
+    upperPlaneBoundMat[n,1]=newOrigBound[counter,1]
+    upperPlaneBoundMat[n,2]=mapping[newOrigBound[counter,2]]
+    upperPlaneBoundMat[n,3]=mapping[newOrigBound[counter,3]]
+    counter+=1
+upperPlaneBoundMat=upperPlaneBoundMat.astype(int)
+
+#Reflecting vertex matrix about the y-axis and appending it to itself
+upperPlaneNumNodes=q1NumNodes+(q1NumNodes-refEdge.shape[0])
+upperPlaneVertMat = sp.zeros([upperPlaneNumNodes,2])
+for n in range(linVertMat.shape[0]):
+    upperPlaneVertMat[n,:]=linVertMat[n,:]
+counter=0
+for n in range(linVertMat.shape[0],upperPlaneNumNodes):
+        upperPlaneVertMat[n,0]=-1.0*linVertMat[sp.where(mapping==n)[0][0],0]
+        upperPlaneVertMat[n,1]=linVertMat[sp.where(mapping==n)[0][0],1]
+        counter+=1
+
+upperPlaneEleMat=orient(upperPlaneEleMat)
+g=open(outputName+'UpperPlaneLin.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(upperPlaneEleMat.shape[0]))
+for n in range(upperPlaneEleMat.shape[0]):
+    if upperPlaneEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(upperPlaneEleMat[n,0],upperPlaneEleMat[n,1],upperPlaneEleMat[n,2],upperPlaneEleMat[n,3],upperPlaneEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(upperPlaneEleMat[n,0],upperPlaneEleMat[n,1],upperPlaneEleMat[n,2],upperPlaneEleMat[n,3],upperPlaneEleMat[n,4],upperPlaneEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(upperPlaneBoundMat.shape[0]))
+for n in range(upperPlaneBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(upperPlaneBoundMat[n,0],upperPlaneBoundMat[n,1],upperPlaneBoundMat[n,2],upperPlaneBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(upperPlaneVertMat.shape[0])+'2\n')
+for n in range(upperPlaneVertMat.shape[0]):
+    g.write('{} {}\n'.format(upperPlaneVertMat[n,0],upperPlaneVertMat[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'UpperPlaneLin.mesh')
+
+#'Reflecting' topology about one of its edges and append it to itself
+wholePlaneEleMat = sp.zeros([2*upperPlaneEleMat.shape[0],6])
+for n in range(upperPlaneEleMat.shape[0]):
+    wholePlaneEleMat[n,:]=upperPlaneEleMat[n,:]
+
+quad1Edge=boundMatTot(numEdges)[3]
+newRefEdge=sp.zeros(2*quad1Edge.shape[0]-1)
+for n in range(quad1Edge.shape[0]):
+    newRefEdge[n]=quad1Edge[n]
+counter=0
+for n in range(quad1Edge.shape[0],newRefEdge.shape[0]):
+    newRefEdge[n]=mapping[quad1Edge[counter]]
+    counter+=1
+newRefEdge=sp.unique(newRefEdge)
+newRefEdge=newRefEdge.astype(int)
+newTotNumNodes=upperPlaneVertMat.shape[0]
+
+newMapping=sp.zeros([newTotNumNodes])
+counter=0
+for n in range(newTotNumNodes):
+    if (sp.any(newRefEdge == n)):
+        newMapping[n]=n
+    else:
+        newMapping[n]=counter+newTotNumNodes
+        counter+=1
+newMapping=newMapping.astype(int)
+
+counter=0
+for n in range(upperPlaneEleMat.shape[0],2*upperPlaneEleMat.shape[0]):
+    wholePlaneEleMat[n,0]=upperPlaneEleMat[counter,0]
+    wholePlaneEleMat[n,1]=upperPlaneEleMat[counter,1]
+    wholePlaneEleMat[n,2]=newMapping[upperPlaneEleMat[counter,2]]
+    wholePlaneEleMat[n,3]=newMapping[upperPlaneEleMat[counter,3]]
+    wholePlaneEleMat[n,4]=newMapping[upperPlaneEleMat[counter,4]]
+    wholePlaneEleMat[n,5]=newMapping[upperPlaneEleMat[counter,5]]
+    counter+=1
+
+wholePlaneEleMat=wholePlaneEleMat.astype(int)
+
+#Reflecting boundary matrix
+newOrigBoundQuad1=boundMatTot(numEdges)[4]
+newFirstBoundMatHolder=sp.zeros([2*newOrigBoundQuad1.shape[0],4])
+for n in range(newOrigBoundQuad1.shape[0]):
+    newFirstBoundMatHolder[n,:]=newOrigBoundQuad1[n,:]
+
+newNewOrigBoundQuad1=newOrigBoundQuad1.copy()
+newNewOrigBoundQuad1[:,2]=sp.flipud(newOrigBoundQuad1[:,3])
+newNewOrigBoundQuad1[:,3]=sp.flipud(newOrigBoundQuad1[:,2])
+counter=0
+for n in range(newOrigBoundQuad1.shape[0],newFirstBoundMatHolder.shape[0]):
+    newFirstBoundMatHolder[n,0]=newNewOrigBoundQuad1[counter,0]
+    newFirstBoundMatHolder[n,1]=newNewOrigBoundQuad1[counter,1]
+    newFirstBoundMatHolder[n,2]=mapping[newNewOrigBoundQuad1[counter,2]]
+    newFirstBoundMatHolder[n,3]=mapping[newNewOrigBoundQuad1[counter,3]]
+    counter+=1
+
+upperQuadMat=newFirstBoundMatHolder.copy()
+wholePlaneBoundMat=sp.zeros([2*upperQuadMat.shape[0],4])
+for n in range(upperQuadMat.shape[0]):
+    wholePlaneBoundMat[n,:]=upperQuadMat[n,:]
+
+counter=0
+newNewOrigBound=upperQuadMat.copy()
+newNewOrigBound[:,2]=sp.flipud(upperQuadMat[:,3])
+newNewOrigBound[:,3]=sp.flipud(upperQuadMat[:,2])
+newNewOrigBound=newNewOrigBound.astype(int)
+for n in range(newNewOrigBound.shape[0],wholePlaneBoundMat.shape[0]):
+    wholePlaneBoundMat[n,0]=newNewOrigBound[counter,0]
+    wholePlaneBoundMat[n,1]=newNewOrigBound[counter,1]
+    wholePlaneBoundMat[n,2]=newMapping[newNewOrigBound[counter,2]]
+    wholePlaneBoundMat[n,3]=newMapping[newNewOrigBound[counter,3]]
+    counter+=1
+wholePlaneBoundMat=wholePlaneBoundMat.astype(int)
+
+wholePlaneNumNodes=newTotNumNodes+(newTotNumNodes-newRefEdge.shape[0])
+wholePlaneVertMat = sp.zeros([wholePlaneNumNodes,2])
+for n in range(upperPlaneVertMat.shape[0]):
+    wholePlaneVertMat[n,:]=upperPlaneVertMat[n,:]
+counter=0
+for n in range(upperPlaneVertMat.shape[0],wholePlaneNumNodes):
+        wholePlaneVertMat[n,0]=upperPlaneVertMat[sp.where(newMapping==n)[0][0],0]
+        wholePlaneVertMat[n,1]=-1.0*upperPlaneVertMat[sp.where(newMapping==n)[0][0],1]
+        counter+=1
+
+g=open(outputName+'WholePlaneLin.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(wholePlaneEleMat.shape[0]))
+for n in range(wholePlaneEleMat.shape[0]):
+    if wholePlaneEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(wholePlaneEleMat[n,0],wholePlaneEleMat[n,1],wholePlaneEleMat[n,2],wholePlaneEleMat[n,3],wholePlaneEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(wholePlaneEleMat[n,0],wholePlaneEleMat[n,1],wholePlaneEleMat[n,2],wholePlaneEleMat[n,3],wholePlaneEleMat[n,4],wholePlaneEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(wholePlaneBoundMat.shape[0]))
+for n in range(wholePlaneBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(wholePlaneBoundMat[n,0],wholePlaneBoundMat[n,1],wholePlaneBoundMat[n,2],wholePlaneBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(wholePlaneVertMat.shape[0])+'2\n')
+for n in range(wholePlaneVertMat.shape[0]):
+    g.write('{} {}\n'.format(wholePlaneVertMat[n,0],wholePlaneVertMat[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'WholePlaneLin.mesh')
+
+#1.)Create Edge list from elements
+wholePlaneEleMat=orient(wholePlaneEleMat)
+triCounter=0;quadCounter=0;
+for n in range(wholePlaneEleMat.shape[0]):
+    if wholePlaneEleMat[n,1]==2:
+        triCounter+=1
+    else:
+        quadCounter+=1
+edgeMat=sp.zeros([3*triCounter+4*quadCounter,2])
+counter=0
+for n in range(wholePlaneEleMat.shape[0]):
+    if wholePlaneEleMat[n,1]==2:
+        edgeMat[counter,:]=[wholePlaneEleMat[n,2],wholePlaneEleMat[n,3]]
+        counter+=1
+        edgeMat[counter,:]=[wholePlaneEleMat[n,3],wholePlaneEleMat[n,4]]
+        counter+=1
+        edgeMat[counter,:]=[wholePlaneEleMat[n,4],wholePlaneEleMat[n,2]]
+        counter+=1
+    else:
+        edgeMat[counter,:]=[wholePlaneEleMat[n,2],wholePlaneEleMat[n,3]]
+        counter+=1
+        edgeMat[counter,:]=[wholePlaneEleMat[n,3],wholePlaneEleMat[n,4]]
+        counter+=1
+        edgeMat[counter,:]=[wholePlaneEleMat[n,4],wholePlaneEleMat[n,5]]
+        counter+=1
+        edgeMat[counter,:]=[wholePlaneEleMat[n,5],wholePlaneEleMat[n,2]]
+        counter+=1
+
+#Remove duplicates
+holder=[]
+for n in range(edgeMat.shape[0]):
+    counter=0
+    for m in range(edgeMat.shape[0]):
+        if edgeMat[n,0]==edgeMat[m,0] and edgeMat[n,1]==edgeMat[m,1] and m!=n:
+            holder.append([n,m])
+        elif edgeMat[n,1]==edgeMat[m,0] and edgeMat[n,0]==edgeMat[m,1] and m!=n:
+            holder.append([n,m])
+removeIndices=sp.zeros(len(holder))
+for n in range(len(holder)):
+    if holder[n][0]>holder[n][1]:
+        removeIndices[n]=holder[n][0]
+    else:
+        removeIndices[n]=holder[n][1]
+removeIndices=sp.unique(removeIndices).astype(int)
+edgeMat=sp.delete(edgeMat,removeIndices,0)
+edgeMat=edgeMat.astype(int)
+
+edgeDofMat=sp.zeros([edgeMat.shape[0],2])
+
+wholePlaneVertMatRound=sp.round_(wholePlaneVertMat,5)
+
+counter=0
+for n in edgeMat:
+    if wholePlaneVertMatRound[n[0],1] == wholePlaneVertMatRound[n[1],1]:
+        xmid=(wholePlaneVertMatRound[n[0],0]+wholePlaneVertMatRound[n[1],0])/2.0
+        ymid=wholePlaneVertMatRound[n[0],1]
+        edgeDofMat[counter,:]=[xmid,ymid]
+    elif wholePlaneVertMatRound[n[0],0] == wholePlaneVertMatRound[n[1],0]:
+        xmid=wholePlaneVertMatRound[n[0],0]
+        ymid=(wholePlaneVertMatRound[n[0],1]+wholePlaneVertMatRound[n[1],1])/2.0
+        edgeDofMat[counter,:]=[xmid,ymid]
+    else:
+        r0=sp.sqrt(wholePlaneVertMatRound[n[0],0]**2+wholePlaneVertMatRound[n[0],1]**2)
+        r1=sp.sqrt(wholePlaneVertMatRound[n[1],0]**2+wholePlaneVertMatRound[n[1],1]**2)
+        rmid = (r0+r1)/2.0 #should not be needed
+        xmidOld=(wholePlaneVertMatRound[n[0],0]+wholePlaneVertMatRound[n[1],0])/2.0
+        ymidOld=(wholePlaneVertMatRound[n[0],1]+wholePlaneVertMatRound[n[1],1])/2.0
+        midtheta=sp.arctan2(ymidOld,xmidOld)
+        xmid=rmid*sp.cos(midtheta)
+        ymid=rmid*sp.sin(midtheta)
+        edgeDofMat[counter,:]=[xmid,ymid]
+    counter+=1
+edgeDofMat = sp.round_(edgeDofMat,5)
+
+#2.)Create correct dof locations
+#Determine midpoints of all quads:
+quadCentroidLoc=sp.zeros([quadCounter,2])
+counter=0
+for n in range(wholePlaneEleMat.shape[0]):
+    if wholePlaneEleMat[n,1]==3:
+        quadCentroidLoc[counter,0]=(wholePlaneVertMatRound[wholePlaneEleMat[n,2],0]+wholePlaneVertMatRound[wholePlaneEleMat[n,3],0]+wholePlaneVertMatRound[wholePlaneEleMat[n,4],0]+wholePlaneVertMatRound[wholePlaneEleMat[n,5],0])/4.0
+        quadCentroidLoc[counter,1]=(wholePlaneVertMatRound[wholePlaneEleMat[n,2],1]+wholePlaneVertMatRound[wholePlaneEleMat[n,3],1]+wholePlaneVertMatRound[wholePlaneEleMat[n,4],1]+wholePlaneVertMatRound[wholePlaneEleMat[n,5],1])/4.0
+        counter+=1
+
+quadCentroidLoc = sp.round_(quadCentroidLoc,5)
+
+#3.)Populate nodes section
+g=open(outputName+'WholePlaneQuad.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(wholePlaneEleMat.shape[0]))
+for n in range(wholePlaneEleMat.shape[0]):
+    if wholePlaneEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(wholePlaneEleMat[n,0],wholePlaneEleMat[n,1],wholePlaneEleMat[n,2],wholePlaneEleMat[n,3],wholePlaneEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(wholePlaneEleMat[n,0],wholePlaneEleMat[n,1],wholePlaneEleMat[n,2],wholePlaneEleMat[n,3],wholePlaneEleMat[n,4],wholePlaneEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(wholePlaneBoundMat.shape[0]))
+for n in range(wholePlaneBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(wholePlaneBoundMat[n,0],wholePlaneBoundMat[n,1],wholePlaneBoundMat[n,2],wholePlaneBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(wholePlaneVertMat.shape[0]))
+g.write('\n'+'nodes'+'\n'+'FiniteElementSpace'+'\n'+'FiniteElementCollection: H1_2D_P2'+'\n'+'VDim: 2'+'\n'+'Ordering: 1' +'\n\n')
+for n in range(wholePlaneVertMatRound.shape[0]):
+    g.write('{} {}\n'.format(wholePlaneVertMatRound[n,0],wholePlaneVertMatRound[n,1]))
+for n in range(edgeDofMat.shape[0]):
+    g.write('{} {}\n'.format(edgeDofMat[n,0],edgeDofMat[n,1]))
+for n in range(quadCentroidLoc.shape[0]):
+    g.write('{} {}\n'.format(quadCentroidLoc[n,0],quadCentroidLoc[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'WholePlaneQuad.mesh')

data/mixed-ele-meshes/comp_tri_quad_4_edges.mesh

@@ -0,0 +1,264 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+128
+1 2 0 1 2
+1 2 1 2 4
+1 2 1 3 4
+1 2 2 4 5
+1 2 3 4 7
+1 2 3 6 7
+1 2 4 5 8
+1 2 4 7 8
+1 2 5 8 9
+1 2 6 7 11
+1 2 6 10 11
+1 2 7 8 12
+1 2 7 11 12
+1 2 8 9 13
+1 2 8 12 13
+1 2 9 13 14
+2 3 10 15 16 11
+2 3 11 16 17 12
+2 3 12 17 18 13
+2 3 13 18 19 14
+2 3 15 20 21 16
+2 3 16 21 22 17
+2 3 17 22 23 18
+2 3 18 23 24 19
+2 3 20 25 26 21
+2 3 21 26 27 22
+2 3 22 27 28 23
+2 3 23 28 29 24
+2 3 25 30 31 26
+2 3 26 31 32 27
+2 3 27 32 33 28
+2 3 28 33 34 29
+1 2 0 35 2
+1 2 35 2 37
+1 2 35 36 37
+1 2 2 37 5
+1 2 36 37 39
+1 2 36 38 39
+1 2 37 5 40
+1 2 37 39 40
+1 2 5 40 9
+1 2 38 39 42
+1 2 38 41 42
+1 2 39 40 43
+1 2 39 42 43
+1 2 40 9 44
+1 2 40 43 44
+1 2 9 44 14
+2 3 41 45 46 42
+2 3 42 46 47 43
+2 3 43 47 48 44
+2 3 44 48 19 14
+2 3 45 49 50 46
+2 3 46 50 51 47
+2 3 47 51 52 48
+2 3 48 52 24 19
+2 3 49 53 54 50
+2 3 50 54 55 51
+2 3 51 55 56 52
+2 3 52 56 29 24
+2 3 53 57 58 54
+2 3 54 58 59 55
+2 3 55 59 60 56
+2 3 56 60 34 29
+1 2 0 1 61
+1 2 1 61 62
+1 2 1 3 62
+1 2 61 62 63
+1 2 3 62 64
+1 2 3 6 64
+1 2 62 63 65
+1 2 62 64 65
+1 2 63 65 66
+1 2 6 64 67
+1 2 6 10 67
+1 2 64 65 68
+1 2 64 67 68
+1 2 65 66 69
+1 2 65 68 69
+1 2 66 69 70
+2 3 10 15 71 67
+2 3 67 71 72 68
+2 3 68 72 73 69
+2 3 69 73 74 70
+2 3 15 20 75 71
+2 3 71 75 76 72
+2 3 72 76 77 73
+2 3 73 77 78 74
+2 3 20 25 79 75
+2 3 75 79 80 76
+2 3 76 80 81 77
+2 3 77 81 82 78
+2 3 25 30 83 79
+2 3 79 83 84 80
+2 3 80 84 85 81
+2 3 81 85 86 82
+1 2 0 35 61
+1 2 35 61 87
+1 2 35 36 87
+1 2 61 87 63
+1 2 36 87 88
+1 2 36 38 88
+1 2 87 63 89
+1 2 87 88 89
+1 2 63 89 66
+1 2 38 88 90
+1 2 38 41 90
+1 2 88 89 91
+1 2 88 90 91
+1 2 89 66 92
+1 2 89 91 92
+1 2 66 92 70
+2 3 41 45 93 90
+2 3 90 93 94 91
+2 3 91 94 95 92
+2 3 92 95 74 70
+2 3 45 49 96 93
+2 3 93 96 97 94
+2 3 94 97 98 95
+2 3 95 98 78 74
+2 3 49 53 99 96
+2 3 96 99 100 97
+2 3 97 100 101 98
+2 3 98 101 82 78
+2 3 53 102 103 99
+2 3 99 103 104 100
+2 3 100 104 105 101
+2 3 101 105 86 82
+
+boundary
+16
+1 1 30 31
+1 1 31 32
+1 1 32 33
+1 1 33 34
+1 1 34 60
+1 1 60 59
+1 1 59 58
+1 1 58 57
+1 1 102 103
+1 1 103 104
+1 1 104 105
+1 1 105 86
+1 1 86 85
+1 1 85 84
+1 1 84 83
+1 1 83 30
+
+vertices
+106
+2
+0.0 0.0
+0.125 0.0
+7.65404249467e-18 0.125
+0.25 0.0
+0.176776695297 0.176776695297
+1.53080849893e-17 0.25
+0.375 0.0
+0.324759526419 0.1875
+0.1875 0.324759526419
+2.2962127484e-17 0.375
+0.5 0.0
+0.461939766256 0.191341716183
+0.353553390593 0.353553390593
+0.191341716183 0.461939766256
+3.06161699787e-17 0.5
+0.625 0.0
+0.596454824692 0.268506287137
+0.515165042945 0.515165042945
+0.268506287137 0.596454824692
+2.2962127484e-17 0.625
+0.75 0.0
+0.730969883128 0.345670858091
+0.676776695297 0.676776695297
+0.345670858091 0.730969883128
+1.53080849893e-17 0.75
+0.875 0.0
+0.865484941564 0.422835429046
+0.838388347648 0.838388347648
+0.422835429046 0.865484941564
+7.65404249467e-18 0.875
+1.0 0.0
+1.0 0.5
+1.0 1.0
+0.5 1.0
+0.0 1.0
+-0.125 0.0
+-0.25 0.0
+-0.176776695297 0.176776695297
+-0.375 0.0
+-0.324759526419 0.1875
+-0.1875 0.324759526419
+-0.5 0.0
+-0.461939766256 0.191341716183
+-0.353553390593 0.353553390593
+-0.191341716183 0.461939766256
+-0.625 0.0
+-0.596454824692 0.268506287137
+-0.515165042945 0.515165042945
+-0.268506287137 0.596454824692
+-0.75 0.0
+-0.730969883128 0.345670858091
+-0.676776695297 0.676776695297
+-0.345670858091 0.730969883128
+-0.875 0.0
+-0.865484941564 0.422835429046
+-0.838388347648 0.838388347648
+-0.422835429046 0.865484941564
+-1.0 0.0
+-1.0 0.5
+-1.0 1.0
+-0.5 1.0
+7.65404249467e-18 -0.125
+0.176776695297 -0.176776695297
+1.53080849893e-17 -0.25
+0.324759526419 -0.1875
+0.1875 -0.324759526419
+2.2962127484e-17 -0.375
+0.461939766256 -0.191341716183
+0.353553390593 -0.353553390593
+0.191341716183 -0.461939766256
+3.06161699787e-17 -0.5
+0.596454824692 -0.268506287137
+0.515165042945 -0.515165042945
+0.268506287137 -0.596454824692
+2.2962127484e-17 -0.625
+0.730969883128 -0.345670858091
+0.676776695297 -0.676776695297
+0.345670858091 -0.730969883128
+1.53080849893e-17 -0.75
+0.865484941564 -0.422835429046
+0.838388347648 -0.838388347648
+0.422835429046 -0.865484941564
+7.65404249467e-18 -0.875
+1.0 -0.5
+1.0 -1.0
+0.5 -1.0
+0.0 -1.0
+-0.176776695297 -0.176776695297
+-0.324759526419 -0.1875
+-0.1875 -0.324759526419
+-0.461939766256 -0.191341716183
+-0.353553390593 -0.353553390593
+-0.191341716183 -0.461939766256
+-0.596454824692 -0.268506287137
+-0.515165042945 -0.515165042945
+-0.268506287137 -0.596454824692
+-0.730969883128 -0.345670858091
+-0.676776695297 -0.676776695297
+-0.345670858091 -0.730969883128
+-0.865484941564 -0.422835429046
+-0.838388347648 -0.838388347648
+-0.422835429046 -0.865484941564
+-1.0 -0.0
+-1.0 -0.5
+-1.0 -1.0
+-0.5 -1.0

data/mixed-ele-meshes/spec-3-mixed-type.mesh

@@ -0,0 +1,72 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+#
+
+dimension
+2
+
+elements
+17
+1 2 0 1 2
+1 2 0 2 3
+1 2 0 3 4
+2 3 0 4 5 6
+2 3 0 6 7 1
+1 2 7 8 1
+1 2 1 8 9
+2 3 1 9 10 2
+1 2 2 10 11
+2 3 2 11 12 3
+1 2 3 12 13
+2 3 3 13 14 4
+1 2 4 14 15
+1 2 4 15 5
+1 2 5 16 6 
+1 2 6 16 17
+1 2 6 17 7
+
+boundary
+12
+1 1 7 8
+1 1 8 9
+1 1 9 10
+1 1 10 11
+1 1 11 12
+1 1 12 13
+1 1 13 14
+1 1 14 15
+1 1 15 5
+1 1 5 16
+1 1 16 17
+1 1 17 7
+
+vertices
+18
+2
+0 0
+1 0
+0.5 0.866025
+-0.5 0.866025
+-1 0
+-1 -1
+0 -1
+1 -1
+1.866025 -0.5
+1.866025 0.5
+1.366025 1.366025
+0.5 1.866025
+-0.5 1.866025
+-1.366025 1.366025
+-1.866025 0.5
+-1.866025 -0.5
+-0.5 -1.866025
+0.5 -1.866025

data/mixed-ele-meshes/spec-3-trans-and-ref.mesh

@@ -0,0 +1,362 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+154
+2 3 0 1 2 3
+2 3 1 5 6 2
+2 3 5 8 9 6
+2 3 8 11 12 9
+2 3 11 14 15 12
+2 3 14 17 18 15
+2 3 17 20 21 18
+2 3 20 23 24 21
+2 3 23 26 27 24
+2 3 26 29 30 27
+2 3 29 32 33 30
+2 3 32 35 36 33
+2 3 35 38 39 36
+2 3 38 41 42 39
+2 3 41 44 45 42
+2 3 44 47 48 45
+2 3 47 50 51 48
+2 3 50 53 54 51
+2 3 53 56 57 54
+2 3 56 59 60 57
+2 3 59 62 63 60
+2 3 62 65 66 63
+2 3 65 68 69 66
+2 3 68 71 72 69
+2 3 71 74 75 72
+2 3 74 77 78 75
+1 2 2 3 4
+1 2 6 2 7
+1 2 9 6 10
+1 2 12 9 13
+1 2 15 12 16
+1 2 18 15 19
+1 2 21 18 22
+1 2 24 21 25
+1 2 27 24 28
+1 2 30 27 31
+1 2 33 30 34
+1 2 36 33 37
+1 2 39 36 40
+1 2 42 39 43
+1 2 45 42 46
+1 2 48 45 49
+1 2 51 48 52
+1 2 54 51 55
+1 2 57 54 58
+1 2 60 57 61
+1 2 63 60 64
+1 2 66 63 67
+1 2 69 66 70
+1 2 72 69 73
+1 2 75 72 76
+1 2 78 75 79
+1 2 2 4 7
+1 2 6 7 10
+1 2 9 10 13
+1 2 12 13 16
+1 2 15 16 19
+1 2 18 19 22
+1 2 21 22 25
+1 2 24 25 28
+1 2 27 28 31
+1 2 30 31 34
+1 2 33 34 37
+1 2 36 37 40
+1 2 39 40 43
+1 2 42 43 46
+1 2 45 46 49
+1 2 48 49 52
+1 2 51 52 55
+1 2 54 55 58
+1 2 57 58 61
+1 2 60 61 64
+1 2 63 64 67
+1 2 66 67 70
+1 2 69 70 73
+1 2 72 73 76
+1 2 75 76 79
+2 3 80 81 82 83
+2 3 81 84 85 82
+2 3 84 86 87 85
+2 3 86 88 89 87
+2 3 88 90 91 89
+2 3 90 92 93 91
+2 3 92 94 95 93
+2 3 94 96 97 95
+2 3 96 98 99 97
+2 3 98 100 101 99
+2 3 100 102 103 101
+2 3 102 104 105 103
+2 3 104 106 107 105
+2 3 106 108 109 107
+2 3 108 110 111 109
+2 3 110 112 113 111
+2 3 112 114 115 113
+2 3 114 116 117 115
+2 3 116 118 119 117
+2 3 118 120 121 119
+2 3 120 122 123 121
+2 3 122 124 125 123
+2 3 124 126 127 125
+2 3 126 128 129 127
+2 3 128 130 131 129
+2 3 130 132 133 131
+1 2 82 83 4
+1 2 85 82 7
+1 2 87 85 10
+1 2 89 87 13
+1 2 91 89 16
+1 2 93 91 19
+1 2 95 93 22
+1 2 97 95 25
+1 2 99 97 28
+1 2 101 99 31
+1 2 103 101 34
+1 2 105 103 37
+1 2 107 105 40
+1 2 109 107 43
+1 2 111 109 46
+1 2 113 111 49
+1 2 115 113 52
+1 2 117 115 55
+1 2 119 117 58
+1 2 121 119 61
+1 2 123 121 64
+1 2 125 123 67
+1 2 127 125 70
+1 2 129 127 73
+1 2 131 129 76
+1 2 133 131 79
+1 2 82 4 7
+1 2 85 7 10
+1 2 87 10 13
+1 2 89 13 16
+1 2 91 16 19
+1 2 93 19 22
+1 2 95 22 25
+1 2 97 25 28
+1 2 99 28 31
+1 2 101 31 34
+1 2 103 34 37
+1 2 105 37 40
+1 2 107 40 43
+1 2 109 43 46
+1 2 111 46 49
+1 2 113 49 52
+1 2 115 52 55
+1 2 117 55 58
+1 2 119 58 61
+1 2 121 61 64
+1 2 123 64 67
+1 2 125 67 70
+1 2 127 70 73
+1 2 129 73 76
+1 2 131 76 79
+
+boundary
+60
+1 1 0 1
+1 1 1 5
+1 1 5 8
+1 1 8 11
+1 1 11 14
+1 1 14 17
+1 1 17 20
+1 1 20 23
+1 1 23 26
+1 1 26 29
+1 1 29 32
+1 1 32 35
+1 1 35 38
+1 1 38 41
+1 1 41 44
+1 1 44 47
+1 1 47 50
+1 1 50 53
+1 1 53 56
+1 1 56 59
+1 1 59 62
+1 1 62 65
+1 1 65 68
+1 1 68 71
+1 1 71 74
+1 1 74 77
+1 1 77 78
+1 1 78 79
+1 1 79 133
+1 1 133 132
+1 1 132 130
+1 1 130 128
+1 1 128 126
+1 1 126 124
+1 1 124 122
+1 1 122 120
+1 1 120 118
+1 1 118 116
+1 1 116 114
+1 1 114 112
+1 1 112 110
+1 1 110 108
+1 1 108 106
+1 1 106 104
+1 1 104 102
+1 1 102 100
+1 1 100 98
+1 1 98 96
+1 1 96 94
+1 1 94 92
+1 1 92 90
+1 1 90 88
+1 1 88 86
+1 1 86 84
+1 1 84 81
+1 1 81 80
+1 1 80 83
+1 1 83 4
+1 1 4 3
+1 1 3 0
+
+vertices
+134
+2
+0.0 0.0
+1.0 0.0
+1.0 1.0
+0.0 1.0
+0.5 1.86602540378
+2.0 0.0
+2.0 1.0
+1.5 1.86602540378
+3.0 0.0
+3.0 1.0
+2.5 1.86602540378
+4.0 0.0
+4.0 1.0
+3.5 1.86602540378
+5.0 0.0
+5.0 1.0
+4.5 1.86602540378
+6.0 0.0
+6.0 1.0
+5.5 1.86602540378
+7.0 0.0
+7.0 1.0
+6.5 1.86602540378
+8.0 0.0
+8.0 1.0
+7.5 1.86602540378
+9.0 0.0
+9.0 1.0
+8.5 1.86602540378
+10.0 0.0
+10.0 1.0
+9.5 1.86602540378
+11.0 0.0
+11.0 1.0
+10.5 1.86602540378
+12.0 0.0
+12.0 1.0
+11.5 1.86602540378
+13.0 0.0
+13.0 1.0
+12.5 1.86602540378
+14.0 0.0
+14.0 1.0
+13.5 1.86602540378
+15.0 0.0
+15.0 1.0
+14.5 1.86602540378
+16.0 0.0
+16.0 1.0
+15.5 1.86602540378
+17.0 0.0
+17.0 1.0
+16.5 1.86602540378
+18.0 0.0
+18.0 1.0
+17.5 1.86602540378
+19.0 0.0
+19.0 1.0
+18.5 1.86602540378
+20.0 0.0
+20.0 1.0
+19.5 1.86602540378
+21.0 0.0
+21.0 1.0
+20.5 1.86602540378
+22.0 0.0
+22.0 1.0
+21.5 1.86602540378
+23.0 0.0
+23.0 1.0
+22.5 1.86602540378
+24.0 0.0
+24.0 1.0
+23.5 1.86602540378
+25.0 0.0
+25.0 1.0
+24.5 1.86602540378
+26.0 0.0
+26.0 1.0
+25.5 1.86602540378
+0.0 3.73205080757
+1.0 3.73205080757
+1.0 2.73205080757
+0.0 2.73205080757
+2.0 3.73205080757
+2.0 2.73205080757
+3.0 3.73205080757
+3.0 2.73205080757
+4.0 3.73205080757
+4.0 2.73205080757
+5.0 3.73205080757
+5.0 2.73205080757
+6.0 3.73205080757
+6.0 2.73205080757
+7.0 3.73205080757
+7.0 2.73205080757
+8.0 3.73205080757
+8.0 2.73205080757
+9.0 3.73205080757
+9.0 2.73205080757
+10.0 3.73205080757
+10.0 2.73205080757
+11.0 3.73205080757
+11.0 2.73205080757
+12.0 3.73205080757
+12.0 2.73205080757
+13.0 3.73205080757
+13.0 2.73205080757
+14.0 3.73205080757
+14.0 2.73205080757
+15.0 3.73205080757
+15.0 2.73205080757
+16.0 3.73205080757
+16.0 2.73205080757
+17.0 3.73205080757
+17.0 2.73205080757
+18.0 3.73205080757
+18.0 2.73205080757
+19.0 3.73205080757
+19.0 2.73205080757
+20.0 3.73205080757
+20.0 2.73205080757
+21.0 3.73205080757
+21.0 2.73205080757
+22.0 3.73205080757
+22.0 2.73205080757
+23.0 3.73205080757
+23.0 2.73205080757
+24.0 3.73205080757
+24.0 2.73205080757
+25.0 3.73205080757
+25.0 2.73205080757
+26.0 3.73205080757
+26.0 2.73205080757

data/mixed-ele-meshes/spec-3-type-1-trans-5.mesh

@@ -0,0 +1,66 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+17
+2 3 0 1 2 3
+2 3 1 5 6 2
+2 3 5 8 9 6
+2 3 8 11 12 9
+2 3 11 14 15 12
+2 3 14 17 18 15
+1 2 2 3 4
+1 2 6 2 7
+1 2 9 6 10
+1 2 12 9 13
+1 2 15 12 16
+1 2 18 15 19
+1 2 2 4 7
+1 2 6 7 10
+1 2 9 10 13
+1 2 12 13 16
+1 2 15 16 19
+
+boundary
+15
+1 1 0 1
+1 1 1 5
+1 1 5 8
+1 1 8 11
+1 1 11 14
+1 1 14 17
+1 1 17 18
+1 1 18 19
+1 1 19 16
+1 1 16 13
+1 1 13 10
+1 1 10 7
+1 1 7 4
+1 1 4 3
+1 1 3 0
+
+vertices
+20
+2
+0.0 0.0
+1.0 0.0
+1.0 1.0
+0.0 1.0
+0.5 1.86602540378
+2.0 0.0
+2.0 1.0
+1.5 1.86602540378
+3.0 0.0
+3.0 1.0
+2.5 1.86602540378
+4.0 0.0
+4.0 1.0
+3.5 1.86602540378
+5.0 0.0
+5.0 1.0
+4.5 1.86602540378
+6.0 0.0
+6.0 1.0
+5.5 1.86602540378

data/mixed-ele-meshes/spec-3-type-1.mesh

@@ -0,0 +1,45 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+#
+
+dimension
+2
+
+elements
+5
+1 2 0 1 2
+1 2 0 2 3
+1 2 0 3 4
+2 3 0 4 5 6
+2 3 0 6 7 1
+
+boundary
+7
+1 1 1 2
+1 1 2 3
+1 1 3 4
+1 1 4 5
+1 1 5 6
+1 1 6 7
+1 1 7 1
+
+vertices
+8
+2
+0 0
+1 0
+0.5 0.866025
+-0.5 0.866025
+-1 0
+-1 -1
+0 -1
+1 -1

data/mixed-ele-meshes/spec-3-type-2.mesh

@@ -0,0 +1,45 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+#
+
+dimension
+2
+
+elements
+5
+1 2 0 1 2
+1 2 0 2 3
+1 2 0 5 6
+2 3 0 3 4 5
+2 3 0 6 7 1
+
+boundary
+7
+1 1 1 2
+1 1 2 3
+1 1 3 4
+1 1 4 5
+1 1 5 6
+1 1 6 7
+1 1 7 1
+
+vertices
+8
+2
+0 0
+0.866025 0.5
+0 1
+-0.866025 0.5
+-1.366025 -0.36602500
+-0.5 -0.866025
+0.5 -0.866025
+1.366025 -0.36602500

data/mixed-ele-meshes/tri_circ_sect_4.mesh

@@ -0,0 +1,57 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+16
+1 2 0 1 2
+1 2 1 2 4
+1 2 1 3 4
+1 2 2 4 5
+1 2 3 4 7
+1 2 3 6 7
+1 2 4 5 8
+1 2 4 7 8
+1 2 5 8 9
+1 2 6 7 11
+1 2 6 10 11
+1 2 7 8 12
+1 2 7 11 12
+1 2 8 9 13
+1 2 8 12 13
+1 2 9 13 14
+
+boundary
+12
+1 1 0 1
+1 1 1 3
+1 1 3 6
+1 1 6 10
+1 1 10 11
+1 1 11 12
+1 1 12 13
+1 1 13 14
+1 1 14 9
+1 1 9 5
+1 1 5 2
+1 1 2 0
+
+vertices
+15
+2
+0.0 0.0
+0.25 0.0
+1.53080849893e-17 0.25
+0.5 0.0
+0.353553390593 0.353553390593
+3.06161699787e-17 0.5
+0.75 0.0
+0.649519052838 0.375
+0.375 0.649519052838
+4.5924254968e-17 0.75
+1.0 0.0
+0.923879532511 0.382683432365
+0.707106781187 0.707106781187
+0.382683432365 0.923879532511
+6.12323399574e-17 1.0

data/mixed-ele-meshes/tri_circ_sect_8.mesh

@@ -0,0 +1,147 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+64
+1 2 0 1 2
+1 2 1 2 4
+1 2 1 3 4
+1 2 2 4 5
+1 2 3 4 7
+1 2 3 6 7
+1 2 4 5 8
+1 2 4 7 8
+1 2 5 8 9
+1 2 6 7 11
+1 2 6 10 11
+1 2 7 8 12
+1 2 7 11 12
+1 2 8 9 13
+1 2 8 12 13
+1 2 9 13 14
+1 2 10 11 16
+1 2 10 15 16
+1 2 11 12 17
+1 2 11 16 17
+1 2 12 13 18
+1 2 12 17 18
+1 2 13 14 19
+1 2 13 18 19
+1 2 14 19 20
+1 2 15 16 22
+1 2 15 21 22
+1 2 16 17 23
+1 2 16 22 23
+1 2 17 18 24
+1 2 17 23 24
+1 2 18 19 25
+1 2 18 24 25
+1 2 19 20 26
+1 2 19 25 26
+1 2 20 26 27
+1 2 21 22 29
+1 2 21 28 29
+1 2 22 23 30
+1 2 22 29 30
+1 2 23 24 31
+1 2 23 30 31
+1 2 24 25 32
+1 2 24 31 32
+1 2 25 26 33
+1 2 25 32 33
+1 2 26 27 34
+1 2 26 33 34
+1 2 27 34 35
+1 2 28 29 37
+1 2 28 36 37
+1 2 29 30 38
+1 2 29 37 38
+1 2 30 31 39
+1 2 30 38 39
+1 2 31 32 40
+1 2 31 39 40
+1 2 32 33 41
+1 2 32 40 41
+1 2 33 34 42
+1 2 33 41 42
+1 2 34 35 43
+1 2 34 42 43
+1 2 35 43 44
+
+boundary
+24
+1 1 0 1
+1 1 1 3
+1 1 3 6
+1 1 6 10
+1 1 10 15
+1 1 15 21
+1 1 21 28
+1 1 28 36
+1 1 36 37
+1 1 37 38
+1 1 38 39
+1 1 39 40
+1 1 40 41
+1 1 41 42
+1 1 42 43
+1 1 43 44
+1 1 44 35
+1 1 35 27
+1 1 27 20
+1 1 20 14
+1 1 14 9
+1 1 9 5
+1 1 5 2
+1 1 2 0
+
+vertices
+45
+2
+0.0 0.0
+0.125 0.0
+7.65404249467e-18 0.125
+0.25 0.0
+0.176776695297 0.176776695297
+1.53080849893e-17 0.25
+0.375 0.0
+0.324759526419 0.1875
+0.1875 0.324759526419
+2.2962127484e-17 0.375
+0.5 0.0
+0.461939766256 0.191341716183
+0.353553390593 0.353553390593
+0.191341716183 0.461939766256
+3.06161699787e-17 0.5
+0.625 0.0
+0.594410322684 0.193135621484
+0.505635621484 0.367365782683
+0.367365782683 0.505635621484
+0.193135621484 0.594410322684
+3.82702124734e-17 0.625
+0.75 0.0
+0.724444369717 0.194114283827
+0.649519052838 0.375
+0.53033008589 0.53033008589
+0.375 0.649519052838
+0.194114283827 0.724444369717
+4.5924254968e-17 0.75
+0.875 0.0
+0.853061923159 0.194705817212
+0.788347759415 0.379648271728
+0.68410254716 0.545553576626
+0.545553576626 0.68410254716
+0.379648271728 0.788347759415
+0.194705817212 0.853061923159
+5.35782974627e-17 0.875
+1.0 0.0
+0.980785280403 0.195090322016
+0.923879532511 0.382683432365
+0.831469612303 0.55557023302
+0.707106781187 0.707106781187
+0.55557023302 0.831469612303
+0.382683432365 0.923879532511
+0.195090322016 0.980785280403
+6.12323399574e-17 1.0

data/mixed-ele-meshes/tri_quad_2nd_order.mesh

@@ -0,0 +1,73 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+8
+1 2 0 1 2
+1 2 1 4 2
+1 2 1 3 4
+1 2 4 5 2
+2 3 3 6 7 4
+2 3 8 5 4 7
+2 3 6 9 10 7
+2 3 11 8 7 10
+
+boundary
+10
+1 1 0 1
+1 1 1 3
+1 1 3 6
+1 1 6 9
+1 1 9 10
+1 1 10 11
+1 1 11 8
+1 1 8 5
+1 1 5 2
+1 1 2 0
+
+vertices
+12
+
+nodes
+FiniteElementSpace
+FiniteElementCollection: Quadratic
+VDim: 2
+Ordering: 1
+
+0.0 0.0
+0.5 0.0
+0.0 0.5
+1.0 0.0
+0.70711 0.70711
+0.0 1.0
+1.5 0.0
+1.35355 1.35355
+0.0 1.5
+2.0 0.0
+2.0 2.0
+0.0 2.0
+0.25 0.0
+0.35355 0.35355
+0.0 0.25
+0.64714 0.37909
+0.37909 0.64714
+0.75 0.0
+0.92388 0.38269
+0.38269 0.92388
+0.0 0.75
+1.25 0.0
+1.54238 0.73161
+1.03033 1.03033
+0.0 1.25
+0.73161 1.54238
+1.75 0.0
+2.0 1.0
+1.67678 1.67678
+0.0 1.75
+1.0 2.0
+1.14017 0.51516
+0.51516 1.14017
+1.71339 0.83839
+0.83839 1.71339

data/mixed-ele-meshes/whole_plane_tri_quad_2nd_order.mesh

@@ -0,0 +1,567 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+128
+1 2 0 1 2
+1 2 1 4 2
+1 2 1 3 4
+1 2 2 4 5
+1 2 3 7 4
+1 2 3 6 7
+1 2 4 8 5
+1 2 4 7 8
+1 2 5 8 9
+1 2 6 11 7
+1 2 6 10 11
+1 2 7 12 8
+1 2 7 11 12
+1 2 8 13 9
+1 2 8 12 13
+1 2 9 13 14
+2 3 10 15 16 11
+2 3 11 16 17 12
+2 3 12 17 18 13
+2 3 13 18 19 14
+2 3 15 20 21 16
+2 3 16 21 22 17
+2 3 17 22 23 18
+2 3 18 23 24 19
+2 3 20 25 26 21
+2 3 21 26 27 22
+2 3 22 27 28 23
+2 3 23 28 29 24
+2 3 25 30 31 26
+2 3 26 31 32 27
+2 3 27 32 33 28
+2 3 28 33 34 29
+1 2 0 2 35
+1 2 35 2 37
+1 2 35 37 36
+1 2 2 5 37
+1 2 36 37 39
+1 2 36 39 38
+1 2 37 5 40
+1 2 37 40 39
+1 2 5 9 40
+1 2 38 39 42
+1 2 38 42 41
+1 2 39 40 43
+1 2 39 43 42
+1 2 40 9 44
+1 2 40 44 43
+1 2 9 14 44
+2 3 41 42 46 45
+2 3 42 43 47 46
+2 3 43 44 48 47
+2 3 44 14 19 48
+2 3 45 46 50 49
+2 3 46 47 51 50
+2 3 47 48 52 51
+2 3 48 19 24 52
+2 3 49 50 54 53
+2 3 50 51 55 54
+2 3 51 52 56 55
+2 3 52 24 29 56
+2 3 53 54 58 57
+2 3 54 55 59 58
+2 3 55 56 60 59
+2 3 56 29 34 60
+1 2 0 61 1
+1 2 1 61 62
+1 2 1 62 3
+1 2 61 63 62
+1 2 3 62 64
+1 2 3 64 6
+1 2 62 63 65
+1 2 62 65 64
+1 2 63 66 65
+1 2 6 64 67
+1 2 6 67 10
+1 2 64 65 68
+1 2 64 68 67
+1 2 65 66 69
+1 2 65 69 68
+1 2 66 70 69
+2 3 10 67 71 15
+2 3 67 68 72 71
+2 3 68 69 73 72
+2 3 69 70 74 73
+2 3 15 71 75 20
+2 3 71 72 76 75
+2 3 72 73 77 76
+2 3 73 74 78 77
+2 3 20 75 79 25
+2 3 75 76 80 79
+2 3 76 77 81 80
+2 3 77 78 82 81
+2 3 25 79 83 30
+2 3 79 80 84 83
+2 3 80 81 85 84
+2 3 81 82 86 85
+1 2 0 35 61
+1 2 35 87 61
+1 2 35 36 87
+1 2 61 87 63
+1 2 36 88 87
+1 2 36 38 88
+1 2 87 89 63
+1 2 87 88 89
+1 2 63 89 66
+1 2 38 90 88
+1 2 38 41 90
+1 2 88 91 89
+1 2 88 90 91
+1 2 89 92 66
+1 2 89 91 92
+1 2 66 92 70
+2 3 41 45 93 90
+2 3 90 93 94 91
+2 3 91 94 95 92
+2 3 92 95 74 70
+2 3 45 49 96 93
+2 3 93 96 97 94
+2 3 94 97 98 95
+2 3 95 98 78 74
+2 3 49 53 99 96
+2 3 96 99 100 97
+2 3 97 100 101 98
+2 3 98 101 82 78
+2 3 53 102 103 99
+2 3 99 103 104 100
+2 3 100 104 105 101
+2 3 101 105 86 82
+
+boundary
+16
+1 1 30 31
+1 1 31 32
+1 1 32 33
+1 1 33 34
+1 1 34 60
+1 1 60 59
+1 1 59 58
+1 1 58 57
+1 1 102 103
+1 1 103 104
+1 1 104 105
+1 1 105 86
+1 1 86 85
+1 1 85 84
+1 1 84 83
+1 1 83 30
+
+vertices
+106
+
+nodes
+FiniteElementSpace
+FiniteElementCollection: H1_2D_P2
+VDim: 2
+Ordering: 1
+
+0.0 0.0
+0.25 0.0
+0.0 0.25
+0.5 0.0
+0.35355 0.35355
+0.0 0.5
+0.75 0.0
+0.64952 0.375
+0.375 0.64952
+0.0 0.75
+1.0 0.0
+0.92388 0.38268
+0.70711 0.70711
+0.38268 0.92388
+0.0 1.0
+1.25 0.0
+1.19291 0.53701
+1.03033 1.03033
+0.53701 1.19291
+0.0 1.25
+1.5 0.0
+1.46194 0.69134
+1.35355 1.35355
+0.69134 1.46194
+0.0 1.5
+1.75 0.0
+1.73097 0.84567
+1.67678 1.67678
+0.84567 1.73097
+0.0 1.75
+2.0 0.0
+2.0 1.0
+2.0 2.0
+1.0 2.0
+0.0 2.0
+-0.25 0.0
+-0.5 0.0
+-0.35355 0.35355
+-0.75 0.0
+-0.64952 0.375
+-0.375 0.64952
+-1.0 0.0
+-0.92388 0.38268
+-0.70711 0.70711
+-0.38268 0.92388
+-1.25 0.0
+-1.19291 0.53701
+-1.03033 1.03033
+-0.53701 1.19291
+-1.5 0.0
+-1.46194 0.69134
+-1.35355 1.35355
+-0.69134 1.46194
+-1.75 0.0
+-1.73097 0.84567
+-1.67678 1.67678
+-0.84567 1.73097
+-2.0 0.0
+-2.0 1.0
+-2.0 2.0
+-1.0 2.0
+0.0 -0.25
+0.35355 -0.35355
+0.0 -0.5
+0.64952 -0.375
+0.375 -0.64952
+0.0 -0.75
+0.92388 -0.38268
+0.70711 -0.70711
+0.38268 -0.92388
+0.0 -1.0
+1.19291 -0.53701
+1.03033 -1.03033
+0.53701 -1.19291
+0.0 -1.25
+1.46194 -0.69134
+1.35355 -1.35355
+0.69134 -1.46194
+0.0 -1.5
+1.73097 -0.84567
+1.67678 -1.67678
+0.84567 -1.73097
+0.0 -1.75
+2.0 -1.0
+2.0 -2.0
+1.0 -2.0
+0.0 -2.0
+-0.35355 -0.35355
+-0.64952 -0.375
+-0.375 -0.64952
+-0.92388 -0.38268
+-0.70711 -0.70711
+-0.38268 -0.92388
+-1.19291 -0.53701
+-1.03033 -1.03033
+-0.53701 -1.19291
+-1.46194 -0.69134
+-1.35355 -1.35355
+-0.69134 -1.46194
+-1.73097 -0.84567
+-1.67678 -1.67678
+-0.84567 -1.73097
+-2.0 -0.0
+-2.0 -1.0
+-2.0 -2.0
+-1.0 -2.0
+0.125 0.0
+0.17678 0.17678
+0.0 0.125
+0.32357 0.18954
+0.18954 0.32357
+0.375 0.0
+0.46194 0.19134
+0.19134 0.46194
+0.0 0.375
+0.59418 0.19384
+0.50569 0.36729
+0.625 0.0
+0.72444 0.19411
+0.36729 0.50569
+0.19384 0.59418
+0.53033 0.53033
+0.19411 0.72444
+0.0 0.625
+0.85299 0.19501
+0.78835 0.37964
+0.875 0.0
+0.98079 0.19509
+0.68405 0.54563
+0.54563 0.68405
+0.83147 0.55557
+0.37964 0.78835
+0.19501 0.85299
+0.55557 0.83147
+0.19509 0.98079
+0.0 0.875
+1.125 0.0
+1.24928 0.27462
+1.05851 0.4599
+1.13007 0.79668
+0.86872 0.86872
+0.79668 1.13007
+0.4599 1.05851
+0.27462 1.24928
+0.0 1.125
+1.375 0.0
+1.51779 0.35426
+1.32748 0.6142
+1.42864 1.03762
+1.19194 1.19194
+1.03762 1.42864
+0.6142 1.32748
+0.35426 1.51779
+0.0 1.375
+1.625 0.0
+1.78629 0.43396
+1.59649 0.76852
+1.72721 1.2785
+1.51516 1.51516
+1.2785 1.72721
+0.76852 1.59649
+0.43396 1.78629
+0.0 1.625
+1.875 0.0
+2.0 0.5
+1.8655 0.92284
+2.0 1.5
+1.83839 1.83839
+1.5 2.0
+0.92284 1.8655
+0.5 2.0
+0.0 1.875
+-0.17678 0.17678
+-0.125 0.0
+-0.18954 0.32357
+-0.32357 0.18954
+-0.46194 0.19134
+-0.375 0.0
+-0.19134 0.46194
+-0.50569 0.36729
+-0.59418 0.19384
+-0.72444 0.19411
+-0.625 0.0
+-0.19384 0.59418
+-0.36729 0.50569
+-0.53033 0.53033
+-0.19411 0.72444
+-0.78835 0.37964
+-0.85299 0.19501
+-0.98079 0.19509
+-0.875 0.0
+-0.54563 0.68405
+-0.68405 0.54563
+-0.83147 0.55557
+-0.19501 0.85299
+-0.37964 0.78835
+-0.55557 0.83147
+-0.19509 0.98079
+-1.05851 0.4599
+-1.24928 0.27462
+-1.125 0.0
+-0.86872 0.86872
+-1.13007 0.79668
+-0.4599 1.05851
+-0.79668 1.13007
+-0.27462 1.24928
+-1.32748 0.6142
+-1.51779 0.35426
+-1.375 0.0
+-1.19194 1.19194
+-1.42864 1.03762
+-0.6142 1.32748
+-1.03762 1.42864
+-0.35426 1.51779
+-1.59649 0.76852
+-1.78629 0.43396
+-1.625 0.0
+-1.51516 1.51516
+-1.72721 1.2785
+-0.76852 1.59649
+-1.2785 1.72721
+-0.43396 1.78629
+-1.8655 0.92284
+-2.0 0.5
+-1.875 0.0
+-1.83839 1.83839
+-2.0 1.5
+-0.92284 1.8655
+-1.5 2.0
+-0.5 2.0
+0.0 -0.125
+0.17678 -0.17678
+0.18954 -0.32357
+0.32357 -0.18954
+0.46194 -0.19134
+0.0 -0.375
+0.19134 -0.46194
+0.50569 -0.36729
+0.59418 -0.19384
+0.72444 -0.19411
+0.19384 -0.59418
+0.36729 -0.50569
+0.53033 -0.53033
+0.0 -0.625
+0.19411 -0.72444
+0.78835 -0.37964
+0.85299 -0.19501
+0.98079 -0.19509
+0.54563 -0.68405
+0.68405 -0.54563
+0.83147 -0.55557
+0.19501 -0.85299
+0.37964 -0.78835
+0.55557 -0.83147
+0.0 -0.875
+0.19509 -0.98079
+1.05851 -0.4599
+1.24928 -0.27462
+0.86872 -0.86872
+1.13007 -0.79668
+0.4599 -1.05851
+0.79668 -1.13007
+0.0 -1.125
+0.27462 -1.24928
+1.32748 -0.6142
+1.51779 -0.35426
+1.19194 -1.19194
+1.42864 -1.03762
+0.6142 -1.32748
+1.03762 -1.42864
+0.0 -1.375
+0.35426 -1.51779
+1.59649 -0.76852
+1.78629 -0.43396
+1.51516 -1.51516
+1.72721 -1.2785
+0.76852 -1.59649
+1.2785 -1.72721
+0.0 -1.625
+0.43396 -1.78629
+1.8655 -0.92284
+2.0 -0.5
+1.83839 -1.83839
+2.0 -1.5
+0.92284 -1.8655
+1.5 -2.0
+0.0 -1.875
+0.5 -2.0
+-0.17678 -0.17678
+-0.32357 -0.18954
+-0.18954 -0.32357
+-0.46194 -0.19134
+-0.19134 -0.46194
+-0.59418 -0.19384
+-0.50569 -0.36729
+-0.72444 -0.19411
+-0.36729 -0.50569
+-0.19384 -0.59418
+-0.53033 -0.53033
+-0.19411 -0.72444
+-0.85299 -0.19501
+-0.78835 -0.37964
+-0.98079 -0.19509
+-0.68405 -0.54563
+-0.54563 -0.68405
+-0.83147 -0.55557
+-0.37964 -0.78835
+-0.19501 -0.85299
+-0.55557 -0.83147
+-0.19509 -0.98079
+-1.24928 -0.27462
+-1.05851 -0.4599
+-1.13007 -0.79668
+-0.86872 -0.86872
+-0.79668 -1.13007
+-0.4599 -1.05851
+-0.27462 -1.24928
+-1.51779 -0.35426
+-1.32748 -0.6142
+-1.42864 -1.03762
+-1.19194 -1.19194
+-1.03762 -1.42864
+-0.6142 -1.32748
+-0.35426 -1.51779
+-1.78629 -0.43396
+-1.59649 -0.76852
+-1.72721 -1.2785
+-1.51516 -1.51516
+-1.2785 -1.72721
+-0.76852 -1.59649
+-0.43396 -1.78629
+-1.875 0.0
+-2.0 -0.5
+-1.8655 -0.92284
+-2.0 -1.5
+-1.83839 -1.83839
+-1.5 -2.0
+-0.92284 -1.8655
+-0.5 -2.0
+1.0917 0.22992
+0.96356 0.66428
+0.66428 0.96356
+0.22992 1.0917
+1.35121 0.30709
+1.25968 0.90306
+0.90306 1.25968
+0.30709 1.35121
+1.61073 0.38425
+1.55581 1.14184
+1.14184 1.55581
+0.38425 1.61073
+1.87024 0.46142
+1.85194 1.38061
+1.38061 1.85194
+0.46142 1.87024
+-1.0917 0.22992
+-0.96356 0.66428
+-0.66428 0.96356
+-0.22992 1.0917
+-1.35121 0.30709
+-1.25968 0.90306
+-0.90306 1.25968
+-0.30709 1.35121
+-1.61073 0.38425
+-1.55581 1.14184
+-1.14184 1.55581
+-0.38425 1.61073
+-1.87024 0.46142
+-1.85194 1.38061
+-1.38061 1.85194
+-0.46142 1.87024
+1.0917 -0.22992
+0.96356 -0.66428
+0.66428 -0.96356
+0.22992 -1.0917
+1.35121 -0.30709
+1.25968 -0.90306
+0.90306 -1.25968
+0.30709 -1.35121
+1.61073 -0.38425
+1.55581 -1.14184
+1.14184 -1.55581
+0.38425 -1.61073
+1.87024 -0.46142
+1.85194 -1.38061
+1.38061 -1.85194
+0.46142 -1.87024
+-1.0917 -0.22992
+-0.96356 -0.66428
+-0.66428 -0.96356
+-0.22992 -1.0917
+-1.35121 -0.30709
+-1.25968 -0.90306
+-0.90306 -1.25968
+-0.30709 -1.35121
+-1.61073 -0.38425
+-1.55581 -1.14184
+-1.14184 -1.55581
+-0.38425 -1.61073
+-1.87024 -0.46142
+-1.85194 -1.38061
+-1.38061 -1.85194
+-0.46142 -1.87024

doc/CodeDocumentation.conf.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v3.3.3
+PROJECT_NUMBER         = v3.4.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -767,6 +767,7 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/fem \
                          @MFEM_SOURCE_DIR@/examples \
                          @MFEM_SOURCE_DIR@/examples/petsc \
+                         @MFEM_SOURCE_DIR@/examples/pumi \
                          @MFEM_SOURCE_DIR@/examples/sundials \
                          @MFEM_SOURCE_DIR@/miniapps/common \
                          @MFEM_SOURCE_DIR@/miniapps/meshing \

doc/CodeDocumentation.dox

@@ -36,8 +36,8 @@ namespace mfem {
  * - HypreSolver and other \link hypre.hpp hypre classes\endlink
  *
  * <H3>Example codes</H3>
- * - <a class="el" href="ex1_8cpp_source.html">Example 1</a>: nodal H1 FEM for the Laplace problem
- * - <a class="el" href="ex1p_8cpp_source.html">Example 1p</a>: parallel nodal H1 FEM for the Laplace problem
+ * - <a class="el" href="examples_2ex1_8cpp_source.html">Example 1</a>: nodal H1 FEM for the Laplace problem
+ * - <a class="el" href="examples_2ex1p_8cpp_source.html">Example 1p</a>: parallel nodal H1 FEM for the Laplace problem
  * - <a class="el" href="ex2_8cpp_source.html">Example 2</a>: vector FEM for linear elasticity
  * - <a class="el" href="ex2p_8cpp_source.html">Example 2p</a>: parallel vector FEM for linear elasticity
  * - <a class="el" href="ex3_8cpp_source.html">Example 3</a>: Nedelec H(curl) FEM for the definite Maxwell problem
@@ -56,7 +56,7 @@ namespace mfem {
  * - <a class="el" href="ex9p_8cpp_source.html">Example 9p</a>: parallel Discontinuous Galerkin (DG) time-dependent advection
  * - <a class="el" href="ex10_8cpp_source.html">Example 10</a>: time-dependent implicit nonlinear elasticity
  * - <a class="el" href="ex10p_8cpp_source.html">Example 10p</a>: parallel time-dependent implicit nonlinear elasticity
- * - <a class="el" href="ex11p_8cpp_source.html">Example 11p</a>: parallel Laplace eigensolver
+ * - <a class="el" href="examples_2ex11p_8cpp_source.html">Example 11p</a>: parallel Laplace eigensolver
  * - <a class="el" href="ex12p_8cpp_source.html">Example 12p</a>: parallel linear elasticity eigensolver
  * - <a class="el" href="ex13p_8cpp_source.html">Example 13p</a>: parallel Maxwell eigensolver
  * - <a class="el" href="ex14_8cpp_source.html">Example 14</a>: Discontinuous Galerkin (DG) for the Laplace problem
@@ -96,6 +96,15 @@ namespace mfem {
  *   <a class="el" href="petsc_2ex10p_8cpp_source.html">10p</a>
  *   demonstrating the use of MFEM's \link petsc.hpp PETSc classes\endlink
  *
+ * <H4>PUMI Examples</H4>
+ * - Variants of Examples
+ *   <a class="el" href="examples_2pumi_2ex1_8cpp_source.html">1</a>,
+ *   <a class="el" href="examples_2pumi_2ex1p_8cpp_source.html">1p</a>,
+ *   <a class="el" href="pumi_2ex2_8cpp_source.html">2</a>,
+ *   and
+ *   <a class="el" href="pumi_2ex6p_8cpp_source.html">6p</a>
+ *   demonstrating the use of MFEM's \link pumi.hpp PUMI classes\endlink
+ *
  * <H3>Miniapps</H3>
  * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
  * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code

examples/CMakeLists.txt

@@ -25,6 +25,7 @@ list(APPEND ALL_EXE_SRCS
   ex16.cpp
   ex17.cpp
   ex18.cpp
+  ex19.cpp
   )
 
 if (MFEM_USE_MPI)
@@ -47,6 +48,7 @@ if (MFEM_USE_MPI)
     ex16p.cpp
     ex17p.cpp
     ex18p.cpp
+    ex19p.cpp
     )
 endif()
 
@@ -61,8 +63,6 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
   get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
   string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
 
-  string(FIND ${TEST_NAME} "p" is_parallel_test)
-
   set(THIS_TEST_OPTIONS "-no-vis")
   if (${TEST_NAME} MATCHES "ex10p*")
     list(APPEND THIS_TEST_OPTIONS "-tf" "5")
@@ -70,7 +70,7 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
     list(APPEND THIS_TEST_OPTIONS "-e" "1")
   endif()
 
-  if (is_parallel_test EQUAL -1)
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
     add_test(NAME ${TEST_NAME}_ser
       COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
   else()
@@ -91,3 +91,8 @@ endif()
 if (MFEM_USE_PETSC)
   add_subdirectory(petsc)
 endif()
+
+# Include the examples/pumi directory if PUMI is enabled
+if (MFEM_USE_PUMI)
+  add_subdirectory(pumi)
+endif()

examples/ex1.cpp

@@ -80,8 +80,8 @@ int main(int argc, char *argv[])
    //    largest number that gives a final mesh with no more than 50,000
    //    elements.
    {
-      int ref_levels =
-         (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
+     int ref_levels = 0; 
+       //(int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
          mesh->UniformRefinement();

examples/ex13p.cpp

@@ -110,6 +110,7 @@ int main(int argc, char *argv[])
    {
       pmesh->UniformRefinement();
    }
+   pmesh->ReorientTetMesh();
 
    // 6. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Nedelec finite elements of the specified order.

examples/ex19.cpp

@@ -50,7 +50,7 @@ using namespace mfem;
 //
 // and K^-1 is an approximation of the inverse of the displacement part of the
 // Jacobian and S^-1 is an approximation of the inverse of the Schur
-// complement S = B K^-1 B^T. The Schur complement is approximiated using
+// complement S = B K^-1 B^T. The Schur complement is approximated using
 // a mass matrix of the pressure variables.
 class JacobianPreconditioner : public Solver
 {

examples/ex19p.cpp

@@ -50,7 +50,7 @@ using namespace mfem;
 //
 // and K^-1 is an approximation of the inverse of the displacement part of the
 // Jacobian and S^-1 is an approximation of the inverse of the Schur
-// complement S = B K^-1 B^T. The Schur complement is approximiated using
+// complement S = B K^-1 B^T. The Schur complement is approximated using
 // a mass matrix of the pressure variables.
 class JacobianPreconditioner : public Solver
 {

examples/makefile

@@ -38,6 +38,9 @@ endif
 ifeq ($(MFEM_USE_PETSC),YES)
    SUBDIRS += petsc
 endif
+ifeq ($(MFEM_USE_PUMI),YES)
+   SUBDIRS += pumi
+endif
 SUBDIRS_ALL = $(addsuffix /all,$(SUBDIRS))
 SUBDIRS_TEST = $(addsuffix /test,$(SUBDIRS))
 SUBDIRS_CLEAN = $(addsuffix /clean,$(SUBDIRS))

examples/petsc/CMakeLists.txt

@@ -63,31 +63,29 @@ add_mfem_examples(PETSC_EXAMPLES_SRCS ${PFX} copy_petsc_rc_files test_petsc)
 #   ctest -R petsc
 
 # Command line options for the tests.
-set(EX1P_ARGS -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_ex1p)
-set(EX2P_ARGS -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p)
-set(EX3P_ARGS -m ../../data/klein-bottle.mesh
-    -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping)
-set(EX4P_ARGS -m ../../data/klein-bottle.mesh
-    -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping)
-set(EX5P_BDDC_ARGS -m ../../data/star.mesh
-    --usepetsc --petscopts rc_ex5p_bddc --nonoverlapping)
-set(EX5P_FSPL_ARGS -m ../../data/beam-tet.mesh
-    --usepetsc --petscopts rc_ex5p_fieldsplit)
-set(EX6P_ARGS -m ../../data/amr-quad.mesh --usepetsc)
-set(EX9P_E_ARGS -m ../../data/periodic-hexagon.mesh
-    --usepetsc --petscopts rc_ex9p_expl -dt 0.1)
-set(EX9P_ES_ARGS -m ../../data/periodic-hexagon.mesh
-    --usepetsc --petscopts rc_ex9p_expl --no-step)
-set(EX9P_IS_ARGS -m ../../data/periodic-hexagon.mesh
-    --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5)
-set(EX10P_ARGS -m ../../data/beam-quad.mesh
-    -tf 30 -s 3 -rs 2 -dt 3 --usepetsc --petscopts rc_ex10p)
+set(EX1_ARGS_W       -m ../../data/amr-quad.mesh --usepetsc)
+set(EX1_ARGS_P       -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_ex1p)
+set(EX2_ARGS         -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p)
+set(EX3_ARGS         -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping)
+set(EX4_ARGS         -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping)
+set(EX4_HYB_ARGS     -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping --hybridization)
+set(EX5_BDDC_LB_ARGS -m ../../data/star.mesh --usepetsc -o 0 --petscopts rc_ex5p_bddc --nonoverlapping --local-bdr)
+set(EX5_BDDC_GB_ARGS -m ../../data/star.mesh --usepetsc -o 0 --petscopts rc_ex5p_bddc --nonoverlapping)
+set(EX5_FSPL_ARGS    -m ../../data/beam-tet.mesh --usepetsc -o 0 --petscopts rc_ex5p_fieldsplit)
+set(EX6_ARGS         -m ../../data/amr-quad.mesh --usepetsc)
+set(EX6_NONOVL_ARGS  -m ../../data/amr-quad.mesh --usepetsc --nonoverlapping)
+set(EX9_E_ARGS       -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl -dt 0.1)
+set(EX9_ES_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step)
+set(EX9_IS_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5)
+set(EX10_ARGS        -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3)
 
 # Add the tests: one test per command-line-variable.
 foreach(TEST_OPTIONS_VAR
-        EX1P_ARGS EX2P_ARGS EX3P_ARGS EX4P_ARGS EX5P_BDDC_ARGS EX5P_FSPL_ARGS
-        EX6P_ARGS EX9P_E_ARGS EX9P_ES_ARGS EX9P_IS_ARGS EX10P_ARGS)
+        EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
+        EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
+        EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
   string(REGEX REPLACE "^(.+)_ARGS" "\\1" TEST_NAME_UC ${TEST_OPTIONS_VAR})
+  string(REGEX REPLACE "^([^_]+)" "\\1P" TEST_NAME_UC ${TEST_NAME_UC})
   string(TOLOWER ${TEST_NAME_UC} TEST_NAME_FULL)
   string(REGEX REPLACE "^([^_]+).*" "\\1" TEST_NAME ${TEST_NAME_FULL})
   set(TEST_NAME_FULL ${PFX}${TEST_NAME_FULL})

examples/petsc/ex5p.cpp

@@ -64,6 +64,7 @@ int main(int argc, char *argv[])
    bool visualization = 1;
    bool use_petsc = true;
    bool use_nonoverlapping = false;
+   bool local_bdr_spec = false;
    const char *petscrc_file = "";
 
    OptionsParser args(argc, argv);
@@ -86,6 +87,9 @@ int main(int argc, char *argv[])
                   "-no-nonoverlapping", "--no-nonoverlapping",
                   "Use or not the block diagonal PETSc's matrix format "
                   "for non-overlapping domain decomposition.");
+   args.AddOption(&local_bdr_spec, "-local-bdr", "--local-bdr", "-no-local-bdr",
+                  "--no-local-bdr",
+                  "Specify boundary dofs in local (Vdofs) ordering.");
    args.Parse();
    if (!args.Good())
    {
@@ -306,33 +310,38 @@ int main(int argc, char *argv[])
    {
       if (use_nonoverlapping)
       {
+         PetscBDDCSolverParams opts;
+
          // For saddle point problems, we need to provide BDDC the list of
          // boundary dofs either essential or natural.
          // Since R_space is the only space that may have boundary dofs and it
          // is ordered first then W_space, we don't need any local offset when
          // specifying the dofs.
          Array<int> bdr_tdof_list;
-         bool local = false;
          if (pmesh->bdr_attributes.Size())
          {
             Array<int> bdr(pmesh->bdr_attributes.Max());
             bdr = 1;
 
-            R_space->GetEssentialTrueDofs(bdr, bdr_tdof_list);
-            local = false;
-            // Alternatively, you can also provide the list of dofs in local
-            // ordering:
-            // R_space->GetEssentialVDofs(bdr, bdr_tdof_list);
-            // bdr_tdof_list.SetSize(R_space->GetVSize()+W_space->GetVSize(),0);
-            // local = true;
+            if (!local_bdr_spec)
+            {
+               // Essential dofs in global ordering
+               R_space->GetEssentialTrueDofs(bdr, bdr_tdof_list);
+            }
+            else
+            {
+               // Alternatively, you can also provide the list of dofs in local
+               // ordering
+               R_space->GetEssentialVDofs(bdr, bdr_tdof_list);
+               bdr_tdof_list.SetSize(R_space->GetVSize()+W_space->GetVSize(),0);
+            }
+            opts.SetNatBdrDofs(&bdr_tdof_list,local_bdr_spec);
          }
          else
          {
-            MFEM_ABORT("Need to know the boundary dofs");
+            MFEM_WARNING("Missing boundary dofs. This may cause solver failures.");
          }
 
-         PetscBDDCSolverParams opts;
-         opts.SetNatBdrDofs(&bdr_tdof_list,local);
          // See also command line options rc_ex5p_bddc
          pdarcyPr = new PetscBDDCSolver(MPI_COMM_WORLD,*darcyOp,opts,"prec_");
       }

examples/petsc/makefile

@@ -69,18 +69,21 @@ TESTNAME = Parallel PETSc example
 
 
 # Testing PETSc execution options.
-EX1_ARGS_W    := -m ../../data/amr-quad.mesh --usepetsc
-EX1_ARGS_P    := -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_ex1p
-EX2_ARGS      := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p
-EX3_ARGS      := -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping
-EX4_ARGS      := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping
-EX5_BDDC_ARGS := -m ../../data/star.mesh --usepetsc -o 0 --petscopts rc_ex5p_bddc --nonoverlapping
-EX5_FSPL_ARGS := -m ../../data/beam-tet.mesh --usepetsc -o 0 --petscopts rc_ex5p_fieldsplit
-EX6_ARGS      := -m ../../data/amr-quad.mesh --usepetsc
-EX9_E_ARGS    := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl -dt 0.1
-EX9_ES_ARGS   := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step
-EX9_IS_ARGS   := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5
-EX10_ARGS     := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3
+EX1_ARGS_W       := -m ../../data/amr-quad.mesh --usepetsc
+EX1_ARGS_P       := -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_ex1p
+EX2_ARGS         := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p
+EX3_ARGS         := -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping
+EX4_ARGS         := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping
+EX4_HYB_ARGS     := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping --hybridization
+EX5_BDDC_LB_ARGS := -m ../../data/star.mesh --usepetsc -o 0 --petscopts rc_ex5p_bddc --nonoverlapping --local-bdr
+EX5_BDDC_GB_ARGS := -m ../../data/star.mesh --usepetsc -o 0 --petscopts rc_ex5p_bddc --nonoverlapping
+EX5_FSPL_ARGS    := -m ../../data/beam-tet.mesh --usepetsc -o 0 --petscopts rc_ex5p_fieldsplit
+EX6_ARGS         := -m ../../data/amr-quad.mesh --usepetsc
+EX6_NONOVL_ARGS  := -m ../../data/amr-quad.mesh --usepetsc --nonoverlapping
+EX9_E_ARGS       := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl -dt 0.1
+EX9_ES_ARGS      := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step
+EX9_IS_ARGS      := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5
+EX10_ARGS        := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3
 ex1p-test-par: ex1p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_W))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_P))
@@ -90,11 +93,14 @@ ex3p-test-par: ex3p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX3_ARGS))
 ex4p-test-par: ex4p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX4_ARGS))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX4_HYB_ARGS))
 ex5p-test-par: ex5p
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX5_BDDC_ARGS))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX5_BDDC_LB_ARGS))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX5_BDDC_GB_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX5_FSPL_ARGS))
 ex6p-test-par: ex6p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX6_ARGS))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX6_NONOVL_ARGS))
 ex9p-test-par: ex9p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX9_E_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX9_ES_ARGS))

examples/petsc/rc_ex3p

@@ -6,4 +6,7 @@
 # it needs PETSc configured with MUMPS
 
 -solver_pc_type cholesky
+# Petsc 3.9
+-solver_pc_factor_mat_solver_type mumps
+# Older versions of PETSc
 -solver_pc_factor_mat_solver_package mumps

examples/petsc/rc_ex3p_bddc

@@ -16,7 +16,10 @@
 #-pc_bddc_adaptive_threshold 10
 
 # Customization of the local solvers
-#-pc_bddc_neumann_pc_factor_mat_solver_package mumps
-#-pc_bddc_dirichlet_pc_factor_mat_solver_package mumps
+# With PETSc versions older than 3.9
+# use "mat_solver_package" instead of "mat_solver_type"
+#
+#-pc_bddc_neumann_pc_factor_mat_solver_type mumps
+#-pc_bddc_dirichlet_pc_factor_mat_solver_type mumps
 #-pc_bddc_coarse_pc_type cholesky
-#-pc_bddc_coarse_pc_factor_mat_solver_package mumps
+#-pc_bddc_coarse_pc_factor_mat_solver_type mumps

examples/petsc/rc_ex4p

@@ -2,4 +2,7 @@
 # it needs PETSc configured with MUMPS
 
 -solver_pc_type cholesky
+# Petsc 3.9
+-solver_pc_factor_mat_solver_type mumps
+# Older versions of PETSc
 -solver_pc_factor_mat_solver_package mumps

examples/petsc/rc_ex4p_bddc

@@ -13,7 +13,10 @@
 #-pc_bddc_adaptive_threshold 10
 
 # Customization of the local solvers
-#-pc_bddc_neumann_pc_factor_mat_solver_package mumps
-#-pc_bddc_dirichlet_pc_factor_mat_solver_package mumps
+# With PETSc versions older than 3.9
+# use "mat_solver_package" instead of "mat_solver_type"
+#
+#-pc_bddc_neumann_pc_factor_mat_solver_type mumps
+#-pc_bddc_dirichlet_pc_factor_mat_solver_type mumps
 #-pc_bddc_coarse_pc_type cholesky
-#-pc_bddc_coarse_pc_factor_mat_solver_package mumps
+#-pc_bddc_coarse_pc_factor_mat_solver_type mumps

examples/petsc/rc_ex5p_bddc

@@ -25,15 +25,29 @@
 # verbose output
 #-prec_pc_bddc_check_level 1
 
-# local solvers (needs PETSc compiled with support for SuiteSparse)
-# default solvers will fail
+# local solvers (default "petsc" solvers will fail)
+# needs PETSc compiled with support for MUMPS or SuiteSparse
+# use "umfpack" in place of "mumps" if you want to use
+# SuiteSparse solvers
+#
+# With PETSc versions older than 3.9
+# use "mat_solver_package" instead of "mat_solver_type"
+#
 -prec_pc_bddc_neumann_pc_type lu
--prec_pc_bddc_neumann_pc_factor_mat_solver_package umfpack
+-prec_pc_bddc_neumann_pc_factor_mat_solver_type mumps
+-prec_pc_bddc_neumann_pc_factor_mat_solver_package mumps
 -prec_pc_bddc_dirichlet_pc_type lu
--prec_pc_bddc_dirichlet_pc_factor_mat_solver_package umfpack
+-prec_pc_bddc_dirichlet_pc_factor_mat_solver_type mumps
+-prec_pc_bddc_dirichlet_pc_factor_mat_solver_package mumps
+
+#  MUMPS sometimes fails with a very annoying error
+-mat_mumps_icntl_14 500
+-prec_pc_bddc_dirichlet_mat_mumps_icntl_14 500
+-prec_pc_bddc_neumann_mat_mumps_icntl_14 500
 
 # coarse solver (needs PETSc compiled with support for MUMPS)
 # default solver may fail
+-prec_pc_bddc_coarse_pc_factor_mat_solver_type mumps
 -prec_pc_bddc_coarse_pc_factor_mat_solver_package mumps
 -prec_pc_bddc_coarse_pc_type cholesky
 

examples/pumi/CMakeLists.txt

@@ -0,0 +1,72 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+set(PUMI_EXAMPLES_SRCS)
+# All PUMI examples require MPI
+if (MFEM_USE_MPI)
+  list(APPEND PUMI_EXAMPLES_SRCS
+    ex1.cpp
+    ex1p.cpp
+    ex2.cpp
+    ex6p.cpp
+    )
+endif()
+
+# Include the source directory where mfem.hpp and mfem-performance.hpp are.
+include_directories(BEFORE ${PROJECT_BINARY_DIR})
+
+# Add "test_pumi" target, see below.
+add_custom_target(test_pumi
+  ${CMAKE_CTEST_COMMAND} -R pumi USES_TERMINAL)
+
+# Add one executable per cpp file, adding "pumi_" as prefix. Sets
+# "test_pumi" as a target that depends on the given examples.
+set(PFX pumi_)
+add_mfem_examples(PUMI_EXAMPLES_SRCS ${PFX} "" test_pumi)
+
+# Testing.
+# The PUMI tests can be run separately using the target "test_pumi"
+# which builds the examples and runs:
+#   ctest -R pumi
+
+# Command line options for the tests.
+# TODO...
+
+# Set the number of processors for the parallel examples.
+set(EX1_TEST_NP 1)
+set(EX1P_TEST_NP 8)
+set(EX2_TEST_NP 1)
+set(EX6P_TEST_NP 8)
+
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${PUMI_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})
+
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+
+  # All PUMI examples require MPI
+  if (FALSE)
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    set(TEST_NP ${${UP_TEST_NAME}_TEST_NP})
+    add_test(NAME ${TEST_NAME}_np=${TEST_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${TEST_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()

examples/pumi/README

@@ -0,0 +1,18 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               http://mfem.org
+
+This directory contains modifications of the example codes that illustrate the
+use of MFEM features based on the Parallel Unstructured Mesh Infrastructure,
+PUMI, from https://scorec.rpi.edu/pumi.
+
+To build these examples, make sure that MFEM is configured with the option
+"MFEM_USE_PUMI = YES", see the top-level INSTALL file for details.
+
+We recommend comparing the original example codes with the corresponding files
+in the current directory.

examples/pumi/ex1.cpp

@@ -0,0 +1,262 @@
+//                                MFEM Example 1
+//                              PUMI Modification
+//
+// Compile with: make ex1
+//
+// Sample runs:
+//    ex1 -m ../../data/pumi/serial/Kova.smb -p ../../data/pumi/geom/Kova.dmg
+//
+// Note:         Example models + meshes for the PUMI examples can be downloaded
+//               from github.com/mfem/data/pumi. After downloading we recommend
+//               creating a symbolic link to the above directory in ../../data.
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Laplace problem
+//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
+//               Specifically, we discretize using a FE space of the specified
+//               order, or if order < 1 using an isoparametric/isogeometric
+//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of mesh refinement, finite
+//               element grid functions, as well as linear and bilinear forms
+//               corresponding to the left-hand side and right-hand side of the
+//               discrete linear system. We also cover the explicit elimination
+//               of essential boundary conditions, static condensation, and the
+//               optional connection to the GLVis tool for visualization.
+//
+//               This PUMI modification demonstrates how PUMI's API can be used
+//               to load a PUMI mesh classified on a geometric model and then
+//               convert it to the MFEM mesh format. The inputs are a Parasolid
+//               model, "*.xmt_txt" and a SCOREC mesh "*.smb". The option "-o"
+//               is used for the Finite Element order and "-go" is used for the
+//               geometry order. Note that they can be used independently, i.e.
+//               "-o 8 -go 3" solves for 8th order FE on a third order geometry.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#ifdef MFEM_USE_SIMMETRIX
+#include <SimUtil.h>
+#include <gmi_sim.h>
+#endif
+#include <apfMDS.h>
+#include <gmi_null.h>
+#include <PCU.h>
+#include <apfConvert.h>
+#include <gmi_mesh.h>
+#include <crv.h>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI (required by PUMI).
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/pumi/serial/Kova.smb";
+#ifdef MFEM_USE_SIMMETRIX
+   const char *model_file = "../../data/pumi/geom/Kova.x_t";
+#else
+   const char *model_file = "../../data/pumi/geom/Kova.dmg";
+#endif
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+   int geom_order = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&model_file, "-p", "--parasolid",
+                  "Parasolid model to use.");
+   args.AddOption(&geom_order, "-go", "--geometry_order",
+                  "Geometric order of the model");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 3. Read the SCOREC Mesh.
+   PCU_Comm_Init();
+#ifdef MFEM_USE_SIMMETRIX
+   Sim_readLicenseFile(0);
+   gmi_sim_start();
+   gmi_register_sim();
+#endif
+   gmi_register_mesh();
+
+   apf::Mesh2* pumi_mesh;
+   pumi_mesh = apf::loadMdsMesh(model_file, mesh_file);
+
+   // 4. Increase the geometry order if necessary.
+   if (geom_order > 1)
+   {
+      crv::BezierCurver bc(pumi_mesh, geom_order, 2);
+      bc.run();
+   }
+
+   pumi_mesh->verify();
+
+   // 5. Create the MFEM mesh object from the PUMI mesh. We can handle
+   //    triangular and tetrahedral meshes. Other inputs are the same as the
+   //    MFEM default constructor.
+   Mesh *mesh = new PumiMesh(pumi_mesh, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 6. Refine the mesh to increase the resolution. In this example we do
+   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
+   //    largest number that gives a final mesh with no more than 50,000
+   //    elements.
+   {
+      int ref_levels =
+         (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 7. Define a finite element space on the mesh. Here we use continuous
+   //    Lagrange finite elements of the specified order. If order < 1, we
+   //    instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (mesh->GetNodes())
+   {
+      fec = mesh->GetNodes()->OwnFEC();
+      cout << "Using isoparametric FEs: " << fec->Name() << endl;
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   cout << "Number of finite element unknowns: "
+        << fespace->GetTrueVSize() << endl;
+
+   // 8. Determine the list of true (i.e. conforming) essential boundary dofs.
+   //    In this example, the boundary conditions are defined by marking all
+   //    the boundary attributes from the mesh as essential (Dirichlet) and
+   //    converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 9. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
+   //    the basis functions in the finite element fespace.
+   LinearForm *b = new LinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   // 10. Define the solution vector x as a finite element grid function
+   //     corresponding to fespace. Initialize x with initial guess of zero,
+   //     which satisfies the boundary conditions.
+   GridFunction x(fespace);
+   x = 0.0;
+
+   // 11. Set up the bilinear form a(.,.) on the finite element space
+   //     corresponding to the Laplacian operator -Delta, by adding the
+   //     Diffusion domain integrator.
+   BilinearForm *a = new BilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 12. Assemble the bilinear form and the corresponding linear system,
+   //     applying any necessary transformations such as: eliminating boundary
+   //     conditions, applying conforming constraints for non-conforming AMR,
+   //     static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   SparseMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   cout << "Size of linear system: " << A.Height() << endl;
+
+#ifndef MFEM_USE_SUITESPARSE
+   // 13. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   //     solve the system A X = B with PCG.
+   GSSmoother M(A);
+   PCG(A, M, B, X, 1, 200, 1e-12, 0.0);
+#else
+   // 13. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+   UMFPackSolver umf_solver;
+   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   umf_solver.SetOperator(A);
+   umf_solver.Mult(B, X);
+#endif
+
+   // 14. Recover the solution as a finite element grid function.
+   a->RecoverFEMSolution(X, *b, x);
+
+   // 15. Save the refined mesh and the solution. This output can be viewed later
+   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
+   ofstream mesh_ofs("refined.mesh");
+   mesh_ofs.precision(8);
+   mesh->Print(mesh_ofs);
+   ofstream sol_ofs("sol.gf");
+   sol_ofs.precision(8);
+   x.Save(sol_ofs);
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *mesh << x << flush;
+   }
+
+   // 17. Free the used memory.
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete mesh;
+
+   pumi_mesh->destroyNative();
+   apf::destroyMesh(pumi_mesh);
+   PCU_Comm_Free();
+#ifdef MFEM_USE_SIMMETRIX
+   gmi_sim_stop();
+   Sim_unregisterAllKeys();
+#endif
+
+   MPI_Finalize();
+   return 0;
+}

examples/pumi/ex1p.cpp

@@ -0,0 +1,292 @@
+//                       MFEM Example 1 - Parallel Version
+//                              PUMI Modification
+//
+// Compile with: make ex1p
+//
+// Sample runs:
+//    mpirun -np 8 ex1p -m ../../data/pumi/parallel/Kova/Kova100k_8.smb
+//                      -p ../../data/pumi/geom/Kova.dmg -o 1 -go 2
+//
+// Note:         Example models + meshes for the PUMI examples can be downloaded
+//               from github.com/mfem/data/pumi. After downloading we recommend
+//               creating a symbolic link to the above directory in ../../data.
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Laplace problem
+//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
+//               Specifically, we discretize using a FE space of the specified
+//               order, or if order < 1 using an isoparametric/isogeometric
+//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of mesh refinement, finite
+//               element grid functions, as well as linear and bilinear forms
+//               corresponding to the left-hand side and right-hand side of the
+//               discrete linear system. We also cover the explicit elimination
+//               of essential boundary conditions, static condensation, and the
+//               optional connection to the GLVis tool for visualization.
+//
+//               This PUMI modification demonstrates how PUMI's API can be used
+//               to load a parallel PUMI mesh classified on a geometric model
+//               and then generate the corresponding parallel MFEM mesh. The
+//               example also performs a "uniform" refinement, similar to the
+//               MFEM examples, for coarse meshes. However, the refinement is
+//               performed using the PUMI API.  The inputs are a Parasolid
+//               model, "*.xmt_txt" and SCOREC parallel meshes "*.smb". The
+//               option "-o" is used for the Finite Element order and "-go" for
+//               the geometry order. Note that they can be used independently:
+//               "-o 8 -go 3" solves for 8th order FE on third order geometry.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#ifdef MFEM_USE_SIMMETRIX
+#include <SimUtil.h>
+#include <gmi_sim.h>
+#endif
+#include <apfMDS.h>
+#include <gmi_null.h>
+#include <PCU.h>
+#include <apfConvert.h>
+#include <gmi_mesh.h>
+#include <crv.h>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/pumi/parallel/Kova/Kova100k_8.smb";
+#ifdef MFEM_USE_SIMMETRIX
+   const char *model_file = "../../data/pumi/geom/Kova.x_t";
+#else
+   const char *model_file = "../../data/pumi/geom/Kova.dmg";
+#endif
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+   int geom_order = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&model_file, "-p", "--parasolid",
+                  "Parasolid model to use.");
+   args.AddOption(&geom_order, "-go", "--geometry_order",
+                  "Geometric order of the model");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 3. Read the SCOREC Mesh
+   PCU_Comm_Init();
+#ifdef MFEM_USE_SIMMETRIX
+   Sim_readLicenseFile(0);
+   gmi_sim_start();
+   gmi_register_sim();
+#endif
+   gmi_register_mesh();
+
+   apf::Mesh2* pumi_mesh;
+   pumi_mesh = apf::loadMdsMesh(model_file, mesh_file);
+
+   // 4. Increase the geometry order and refine the mesh if necessary. Parallel
+   //    uniform refinement is performed if the total number of elements is less
+   //    than 10,000.
+   int dim = pumi_mesh->getDimension();
+   int nEle = pumi_mesh->count(dim);
+   int ref_levels = (int)floor(log(10000./nEle)/log(2.)/dim);
+
+   if (geom_order > 1)
+   {
+      crv::BezierCurver bc(pumi_mesh, geom_order, 2);
+      bc.run();
+   }
+
+   // Perform Uniform refinement
+   if (ref_levels > 1)
+   {
+      ma::Input* uniInput = ma::configureUniformRefine(pumi_mesh, ref_levels);
+
+      if (geom_order > 1)
+      {
+         crv::adapt(uniInput);
+      }
+      else
+      {
+         ma::adapt(uniInput);
+      }
+   }
+
+   pumi_mesh->verify();
+
+   // 5. Create the parallel MFEM mesh object from the parallel PUMI mesh.
+   //    We can handle triangular and tetrahedral meshes. Note that the
+   //    mesh resolution is performed on the PUMI mesh.
+   ParMesh *pmesh = new ParPumiMesh(MPI_COMM_WORLD, pumi_mesh);
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+      if (myid == 0)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (1,phi_i) where phi_i are the basis functions in fespace.
+   ParLinearForm *b = new ParLinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   ParGridFunction x(fespace);
+   x = 0.0;
+
+   // 10. Set up the parallel bilinear form a(.,.) on the finite element space
+   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //     domain integrator.
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   HypreParMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+   }
+
+   // 12. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+   //     preconditioner from hypre.
+   HypreSolver *amg = new HypreBoomerAMG(A);
+   HyprePCG *pcg = new HyprePCG(A);
+   pcg->SetTol(1e-12);
+   pcg->SetMaxIter(200);
+   pcg->SetPrintLevel(2);
+   pcg->SetPreconditioner(*amg);
+   pcg->Mult(B, X);
+
+   // 13. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a->RecoverFEMSolution(X, *b, x);
+
+   // 14. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 15. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x << flush;
+   }
+
+   // 16. Free the used memory.
+   delete pcg;
+   delete amg;
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
+   pumi_mesh->destroyNative();
+   apf::destroyMesh(pumi_mesh);
+   PCU_Comm_Free();
+
+#ifdef MFEM_USE_SIMMETRIX
+   gmi_sim_stop();
+   Sim_unregisterAllKeys();
+#endif
+
+   MPI_Finalize();
+
+   return 0;
+}

examples/pumi/ex2.cpp

@@ -0,0 +1,413 @@
+//                                MFEM Example 2
+//                              PUMI Modification
+//
+// Compile with: make ex2
+//
+// Sample runs:
+//    ex2 -m ../../data/pumi/serial/pillbox.smb -p ../../data/pumi/geom/pillbox.dmg
+//        -bf ../../data/pumi/serial/boundary.mesh
+//
+// Note:         Example models + meshes for the PUMI examples can be downloaded
+//               from github.com/mfem/data/pumi. After downloading we recommend
+//               creating a symbolic link to the above directory in ../../data.
+//
+// Description:  This example code solves a simple linear elasticity problem
+//               describing a multi-material cantilever beam.
+//
+//               Specifically, we approximate the weak form of -div(sigma(u))=0
+//               where sigma(u)=lambda*div(u)*I+mu*(grad*u+u*grad) is the stress
+//               tensor corresponding to displacement field u, and lambda and mu
+//               are the material Lame constants. The boundary conditions are
+//               u=0 on the fixed part of the boundary with attribute 1, and
+//               sigma(u).n=f on the remainder with f being a constant pull down
+//               vector on boundary elements with attribute 2, and zero
+//               otherwise. The geometry of the domain is assumed to be as
+//               follows:
+//                                        boundary
+//                                       attribute 2
+//                                       (push down)
+//                                            ||
+//                                            \/
+//                                       +----------+
+//                                       |          |
+//                                       |          |
+//                             +---------| material |----------+
+//                boundary --->| material|    2     | material |<--- boundary
+//                attribute 1  |    1    |          |    3     |     attribute 1
+//                 (fixed)     +---------+----------+----------+     (fixed)
+//
+//               The example demonstrates the use of high-order and NURBS vector
+//               finite element spaces with the linear elasticity bilinear form,
+//               meshes with curved elements, and the definition of piece-wise
+//               constant and vector coefficient objects. Static condensation is
+//               also illustrated.
+//
+//               We recommend viewing Example 1 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#include "../../general/text.hpp"
+
+#ifdef MFEM_USE_SIMMETRIX
+#include <SimUtil.h>
+#include <gmi_sim.h>
+#endif
+#include <apfMDS.h>
+#include <gmi_null.h>
+#include <PCU.h>
+#include <apfConvert.h>
+#include <gmi_mesh.h>
+#include <crv.h>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI (required by PUMI).
+   int num_proc, myId;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_proc);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myId);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/pumi/serial/pillbox.smb";
+   const char *boundary_file = "../../data/pumi/serial/boundary.mesh";
+#ifdef MFEM_USE_SIMMETRIX
+   const char *model_file = "../../data/pumi/geom/pillbox.smd";
+#else
+   const char *model_file = "../../data/pumi/geom/pillbox.dmg";
+#endif
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+   int geom_order = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&model_file, "-p", "--parasolid",
+                  "Parasolid model to use.");
+   args.AddOption(&geom_order, "-go", "--geometry_order",
+                  "Geometric order of the model");
+   args.AddOption(&boundary_file, "-bf", "--txt",
+                  "txt file containing boundary tags");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 3. Read the SCOREC Mesh.
+   PCU_Comm_Init();
+#ifdef MFEM_USE_SIMMETRIX
+   Sim_readLicenseFile(0);
+   gmi_sim_start();
+   gmi_register_sim();
+#endif
+   gmi_register_mesh();
+
+   apf::Mesh2* pumi_mesh;
+   pumi_mesh = apf::loadMdsMesh(model_file, mesh_file);
+
+   // 4. Increase the geometry order if necessary.
+   if (geom_order > 1)
+   {
+      crv::BezierCurver bc(pumi_mesh, geom_order, 0);
+      bc.run();
+   }
+   pumi_mesh->verify();
+
+   // Read boundary
+   string bdr_tags;
+   named_ifgzstream input_bdr(boundary_file);
+   input_bdr >> ws;
+   getline(input_bdr, bdr_tags);
+   filter_dos(bdr_tags);
+   cout << " the boundary tag is : " << bdr_tags << endl;
+   Array<int> Dirichlet;
+   int numOfent;
+   if (bdr_tags == "Dirichlet")
+   {
+      input_bdr >> numOfent;
+      cout << " num of Dirichlet bdr conditions : " << numOfent << endl;
+      Dirichlet.SetSize(numOfent);
+      for (int kk = 0; kk < numOfent; kk++)
+      {
+         input_bdr >> Dirichlet[kk];
+      }
+   }
+   Dirichlet.Print();
+
+   Array<int> load_bdr;
+   skip_comment_lines(input_bdr, '#');
+   input_bdr >> bdr_tags;
+   filter_dos(bdr_tags);
+   cout << " the boundary tag is : " << bdr_tags << endl;
+   if (bdr_tags == "Load")
+   {
+      input_bdr >> numOfent;
+      load_bdr.SetSize(numOfent);
+      cout << " num of load bdr conditions : " << numOfent << endl;
+      for (int kk = 0; kk < numOfent; kk++)
+      {
+         input_bdr >> load_bdr[kk];
+      }
+   }
+   load_bdr.Print();
+
+   // 5. Create the MFEM mesh object from the PUMI mesh. We can handle triangular
+   //    and tetrahedral meshes. Other inputs are the same as MFEM default
+   //    constructor.
+   Mesh *mesh = new PumiMesh(pumi_mesh, 1, 1);
+   int dim = mesh->Dimension();
+
+   //  Boundary conditions hack.
+   apf::MeshIterator* itr = pumi_mesh->begin(dim-1);
+   apf::MeshEntity* ent ;
+   int bdr_cnt = 0;
+   while ((ent = pumi_mesh->iterate(itr)))
+   {
+      apf::ModelEntity *me = pumi_mesh->toModel(ent);
+      if (pumi_mesh->getModelType(me) == (dim-1))
+      {
+         // Everywhere 3 as initial
+         (mesh->GetBdrElement(bdr_cnt))->SetAttribute(3);
+         int tag = pumi_mesh->getModelTag(me);
+         if (Dirichlet.Find(tag) != -1)
+         {
+            // Dirichlet attr -> 1
+            (mesh->GetBdrElement(bdr_cnt))->SetAttribute(1);
+         }
+         else if (load_bdr.Find(tag) != -1)
+         {
+            // Load attr -> 2
+            (mesh->GetBdrElement(bdr_cnt))->SetAttribute(2);
+         }
+         bdr_cnt++;
+      }
+   }
+   pumi_mesh->end(itr);
+
+   // Assign attributes for elements.
+   double ppt[3];
+   Vector cent(ppt, dim);
+   for (int el = 0; el < mesh->GetNE(); el++)
+   {
+      (mesh->GetElementTransformation(el))->
+      Transform(Geometries.GetCenter(mesh->GetElementBaseGeometry(el)),cent);
+      if (cent(0) <= -0.05)
+      {
+         mesh->SetAttribute(el , 1);
+      }
+      else if (cent(0) >= 0.05)
+      {
+         mesh->SetAttribute(el , 2);
+      }
+      else
+      {
+         mesh->SetAttribute(el , 3);
+      }
+   }
+   mesh->SetAttributes();
+   if (mesh->attributes.Max() < 2 || mesh->bdr_attributes.Max() < 2)
+   {
+      cerr << "\nInput mesh should have at least two materials and "
+           << "two boundary attributes! (See schematic in ex2.cpp)\n"
+           << endl;
+      return 3;
+   }
+
+   // 6. Refine the mesh to increase the resolution. In this example we do
+   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
+   //    largest number that gives a final mesh with no more than 5,000
+   //    elements.
+   {
+      int ref_levels =
+         (int)floor(log(5000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 7. Define a finite element space on the mesh. Here we use vector finite
+   //    elements, i.e. dim copies of a scalar finite element space. The vector
+   //    dimension is specified by the last argument of the FiniteElementSpace
+   //    constructor. For NURBS meshes, we use the (degree elevated) NURBS space
+   //    associated with the mesh nodes.
+   FiniteElementCollection *fec;
+   FiniteElementSpace *fespace;
+   if (mesh->NURBSext)
+   {
+      fec = NULL;
+      fespace = mesh->GetNodes()->FESpace();
+   }
+   else
+   {
+      fec = new H1_FECollection(order, dim);
+      fespace = new FiniteElementSpace(mesh, fec, dim);
+   }
+   cout << "Number of finite element unknowns: " << fespace->GetTrueVSize()
+        << endl << "Assembling: " << flush;
+
+   // 8. Determine the list of true (i.e. conforming) essential boundary dofs.
+   //    In this example, the boundary conditions are defined by marking only
+   //    boundary attribute 1 from the mesh as essential and converting it to a
+   //    list of true dofs.
+   Array<int> ess_tdof_list, ess_bdr(mesh->bdr_attributes.Max());
+   ess_bdr = 0;
+   ess_bdr[0] = 1;
+   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   // 9. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system. In this case, b_i equals the boundary integral
+   //    of f*phi_i where f represents a "pull down" force on the Neumann part
+   //    of the boundary and phi_i are the basis functions in the finite element
+   //    fespace. The force is defined by the VectorArrayCoefficient object f,
+   //    which is a vector of Coefficient objects. The fact that f is non-zero
+   //    on boundary attribute 2 is indicated by the use of piece-wise constants
+   //    coefficient for its last component.
+   VectorArrayCoefficient f(dim);
+   for (int i = 0; i < dim-1; i++)
+   {
+      f.Set(i, new ConstantCoefficient(0.0));
+   }
+   {
+      Vector pull_force(mesh->bdr_attributes.Max());
+      pull_force = 0.0;
+      pull_force(1) = -3.0e-2;
+      f.Set(dim-1, new PWConstCoefficient(pull_force));
+      f.Set(dim-2, new PWConstCoefficient(pull_force));
+   }
+
+   LinearForm *b = new LinearForm(fespace);
+   b->AddBoundaryIntegrator(new VectorBoundaryLFIntegrator(f));
+   cout << "r.h.s. ... " << flush;
+   b->Assemble();
+
+   // 10. Define the solution vector x as a finite element grid function
+   //     corresponding to fespace. Initialize x with initial guess of zero,
+   //     which satisfies the boundary conditions.
+   GridFunction x(fespace);
+   x = 0.0;
+
+   // 11. Set up the bilinear form a(.,.) on the finite element space
+   //     corresponding to the linear elasticity integrator with piece-wise
+   //     constants coefficient lambda and mu.
+   Vector lambda(mesh->attributes.Max());
+   lambda = 1.0;
+   lambda(0) = lambda(1)*10;
+   lambda(1) = lambda(1)*100;
+   PWConstCoefficient lambda_func(lambda);
+   Vector mu(mesh->attributes.Max());
+   mu = 1.0;
+   mu(0) = mu(1)*10;
+   mu(1) = mu(1)*100;
+   PWConstCoefficient mu_func(mu);
+
+   BilinearForm *a = new BilinearForm(fespace);
+   a->AddDomainIntegrator(new ElasticityIntegrator(lambda_func,mu_func));
+
+   // 12. Assemble the bilinear form and the corresponding linear system,
+   //     applying any necessary transformations such as: eliminating boundary
+   //     conditions, applying conforming constraints for non-conforming AMR,
+   //     static condensation, etc.
+   cout << "matrix ... " << flush;
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   SparseMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+   cout << "done." << endl;
+
+   cout << "Size of linear system: " << A.Height() << endl;
+
+#ifndef MFEM_USE_SUITESPARSE
+   // 13. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   //     solve the system Ax=b with PCG.
+   GSSmoother M(A);
+   PCG(A, M, B, X, 1, 500, 1e-8, 0.0);
+#else
+   // 13. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+   UMFPackSolver umf_solver;
+   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   umf_solver.SetOperator(A);
+   umf_solver.Mult(B, X);
+#endif
+
+   // 14. Recover the solution as a finite element grid function.
+   a->RecoverFEMSolution(X, *b, x);
+
+   // 15. For non-NURBS meshes, make the mesh curved based on the finite element
+   //     space. This means that we define the mesh elements through a fespace
+   //     based transformation of the reference element. This allows us to save
+   //     the displaced mesh as a curved mesh when using high-order finite
+   //     element displacement field. We assume that the initial mesh (read from
+   //     the file) is not higher order curved mesh compared to the chosen FE
+   //     space.
+   if (!mesh->NURBSext)
+   {
+      mesh->SetNodalFESpace(fespace);
+   }
+
+   // 16. Save the displaced mesh and the inverted solution (which gives the
+   //     backward displacements to the original grid). This output can be
+   //     viewed later using GLVis: "glvis -m displaced.mesh -g sol.gf".
+   {
+      GridFunction *nodes = mesh->GetNodes();
+      *nodes += x;
+      x *= -1;
+      ofstream mesh_ofs("displaced.mesh");
+      mesh_ofs.precision(8);
+      mesh->Print(mesh_ofs);
+      ofstream sol_ofs("sol.gf");
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 17. Send the above data by socket to a GLVis server. Use the "n" and "b"
+   //     keys in GLVis to visualize the displacements.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *mesh << x << flush;
+   }
+
+   // 18. Free the used memory.
+   delete a;
+   delete b;
+   if (fec)
+   {
+      delete fespace;
+      delete fec;
+   }
+   delete mesh;
+
+   pumi_mesh->destroyNative();
+   apf::destroyMesh(pumi_mesh);
+   PCU_Comm_Free();
+#ifdef MFEM_USE_SIMMETRIX
+   gmi_sim_stop();
+   Sim_unregisterAllKeys();
+#endif
+
+   MPI_Finalize();
+
+   return 0;
+}

examples/pumi/ex6p.cpp

@@ -0,0 +1,387 @@
+//                       MFEM Example 6 - Parallel Version
+//                              PUMI Modification
+//
+// Compile with: make ex1p
+//
+// Sample runs:  mpirun -np 8 ex6p
+//
+// Description:  This is a version of Example 1 with a simple adaptive mesh
+//               refinement loop. The problem being solved is again the Laplace
+//               equation -Delta u = 1 with homogeneous Dirichlet boundary
+//               conditions. The problem is solved on a sequence of meshes which
+//               are adapted in a conforming (tetrahedrons) manner according
+//               to a simple SPR ZZ error estimator.
+//
+//               This PUMI variation also performs a "uniform" refinement,
+//               similar to MFEM examples, for coarse meshes. However, the
+//               refinement is performed using the PUMI API. A new option "-ar"
+//               is added to modify the "adapt_ratio" which is the fraction of
+//               allowable error that scales the output size field of the error
+//               estimator.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#ifdef MFEM_USE_SIMMETRIX
+#include <SimUtil.h>
+#include <gmi_sim.h>
+#endif
+#include <apfMDS.h>
+#include <gmi_null.h>
+#include <PCU.h>
+#include <spr.h>
+#include <apfConvert.h>
+#include <gmi_mesh.h>
+#include <crv.h>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/pumi/parallel/Kova/Kova100k_8.smb";
+#ifdef MFEM_USE_SIMMETRIX
+   const char *model_file = "../../data/pumi/geom/Kova.x_t";
+   const char *smd_file = NULL;
+#else
+   const char *model_file = "../../data/pumi/geom/Kova.dmg";
+#endif
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+   int geom_order = 1;
+   double adapt_ratio = 0.05;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&model_file, "-p", "--model",
+                  "parasolid or .dmg model to use.");
+#ifdef MFEM_USE_SIMMETRIX
+   args.AddOption(&smd_file, "-sm", "--smd_model",
+                  "smd model file to use.");
+#endif
+   args.AddOption(&geom_order, "-go", "--geometry_order",
+                  "Geometric order of the model");
+   args.AddOption(&adapt_ratio, "-ar", "--adapt_ratio",
+                  "adaptation factor used in MeshAdapt");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 3. Read the SCOREC Mesh.
+   PCU_Comm_Init();
+#ifdef MFEM_USE_SIMMETRIX
+   Sim_readLicenseFile(0);
+   gmi_sim_start();
+   gmi_register_sim();
+#endif
+   gmi_register_mesh();
+
+   apf::Mesh2* pumi_mesh;
+#ifdef MFEM_USE_SIMMETRIX
+   if (smd_file)
+   {
+      gmi_model *mixed_model = gmi_sim_load(model_file, smd_file);
+      pumi_mesh = apf::loadMdsMesh(mixed_model, mesh_file);
+   }
+   else
+#endif
+   {
+      pumi_mesh = apf::loadMdsMesh(model_file, mesh_file);
+   }
+
+   // 4. Increase the geometry order and refine the mesh if necessary.  Parallel
+   //    uniform refinement is performed if the total number of elements is less
+   //    than 100,000.
+   int dim = pumi_mesh->getDimension();
+   int nEle = pumi_mesh->count(dim);
+   int ref_levels = (int)floor(log(100000./nEle)/log(2.)/dim);
+
+   if (geom_order > 1)
+   {
+      crv::BezierCurver bc(pumi_mesh, geom_order, 2);
+      bc.run();
+   }
+
+   // Perform Uniform refinement
+   if (myid == 1)
+   {
+      std::cout << " ref level : " <<     ref_levels << std::endl;
+   }
+
+   if (ref_levels > 1)
+   {
+      ma::Input* uniInput = ma::configureUniformRefine(pumi_mesh, ref_levels);
+
+      if ( geom_order > 1)
+      {
+         crv::adapt(uniInput);
+      }
+      else
+      {
+         ma::adapt(uniInput);
+      }
+   }
+
+   pumi_mesh->verify();
+
+   // 5. Create the parallel MFEM mesh object from the parallel PUMI mesh.  We
+   //    can handle triangular and tetrahedral meshes. Note that the mesh
+   //    resolution is performed on the PUMI mesh.
+   ParMesh *pmesh = new ParPumiMesh(MPI_COMM_WORLD, pumi_mesh);
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+      if (myid == 1)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 1)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (1,phi_i) where phi_i are the basis functions in fespace.
+   ParLinearForm *b = new ParLinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+
+   // 8. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   ParGridFunction x(fespace);
+   x = 0.0;
+
+   // 9. Connect to GLVis.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+
+   socketstream sout;
+   if (visualization)
+   {
+      sout.open(vishost, visport);
+      if (!sout)
+      {
+         if (myid == 0)
+         {
+            cout << "Unable to connect to GLVis server at "
+                 << vishost << ':' << visport << endl;
+            cout << "GLVis visualization disabled.\n";
+         }
+         visualization = false;
+      }
+
+      sout.precision(8);
+   }
+
+   // 10. Set up the parallel bilinear form a(.,.) on the finite element space
+   //     corresponding to the Laplacian operator -Delta, by adding the
+   //     Diffusion domain integrator.
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+
+   // 12. The main AMR loop. In each iteration we solve the problem on the
+   //     current mesh, visualize the solution, and adapt the mesh.
+   apf::Field* Tmag_field = 0;
+   apf::Field* temp_field = 0;
+   apf::Field* ipfield = 0;
+   apf::Field* sizefield = 0;
+   int max_iter = 3;
+
+   for (int Itr = 0; Itr < max_iter; Itr++)
+   {
+      HYPRE_Int global_dofs = fespace->GlobalTrueVSize();
+      if (myid == 1)
+      {
+         cout << "\nAMR iteration " << Itr << endl;
+         cout << "Number of unknowns: " << global_dofs << endl;
+      }
+
+      // Assemble.
+      a->Assemble();
+      b->Assemble();
+
+      // Essential boundary condition.
+      Array<int> ess_tdof_list;
+      if (pmesh->bdr_attributes.Size())
+      {
+         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+         ess_bdr = 1;
+         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      // Form linear system.
+      HypreParMatrix A;
+      Vector B, X;
+      const int copy_interior = 1;
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B, copy_interior);
+
+      // 13. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+      //     preconditioner from hypre.
+      HypreBoomerAMG amg;
+      amg.SetPrintLevel(0);
+      CGSolver pcg(A.GetComm());
+      pcg.SetPreconditioner(amg);
+      pcg.SetOperator(A);
+      pcg.SetRelTol(1e-6);
+      pcg.SetMaxIter(200);
+      pcg.SetPrintLevel(3); // print the first and the last iterations only
+      pcg.Mult(B, X);
+
+      // 14. Recover the parallel grid function corresponding to X. This is the
+      //     local finite element solution on each processor.
+      a->RecoverFEMSolution(X, *b, x);
+
+      // 15. Save in parallel the displaced mesh and the inverted solution (which
+      //     gives the backward displacements to the original grid). This output
+      //     can be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+      {
+         ostringstream mesh_name, sol_name;
+         mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+         sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+         ofstream mesh_ofs(mesh_name.str().c_str());
+         mesh_ofs.precision(8);
+         pmesh->Print(mesh_ofs);
+
+         ofstream sol_ofs(sol_name.str().c_str());
+         sol_ofs.precision(8);
+         x.Save(sol_ofs);
+      }
+
+      // 16. Send the above data by socket to a GLVis server.  Use the "n" and "b"
+      //     keys in GLVis to visualize the displacements.
+      if (visualization)
+      {
+         sout << "parallel " << num_procs << " " << myid << "\n";
+         sout << "solution\n" << *pmesh << x << flush;
+      }
+
+      // 17. Field transfer. Scalar solution field and magnitude field for error
+      //     estimation are created the PUMI mesh.
+      if (order > geom_order)
+      {
+         Tmag_field = apf::createField(pumi_mesh, "field_mag",
+                                       apf::SCALAR, apf::getLagrange(order));
+         temp_field = apf::createField(pumi_mesh, "T_field",
+                                       apf::SCALAR, apf::getLagrange(order));
+      }
+      else
+      {
+         Tmag_field = apf::createFieldOn(pumi_mesh, "field_mag",apf::SCALAR);
+         temp_field = apf::createFieldOn(pumi_mesh, "T_field", apf::SCALAR);
+      }
+
+      ParPumiMesh* pPPmesh = dynamic_cast<ParPumiMesh*>(pmesh);
+      pPPmesh->FieldMFEMtoPUMI(pumi_mesh, &x, temp_field, Tmag_field);
+
+      ipfield= spr::getGradIPField(Tmag_field, "MFEM_gradip", 2);
+      sizefield = spr::getSPRSizeField(ipfield, adapt_ratio);
+
+      apf::destroyField(Tmag_field);
+      apf::destroyField(ipfield);
+      apf::destroyNumbering(pumi_mesh->findNumbering("LocalVertexNumbering"));
+
+      // 18. Perform MesAdapt.
+      ma::Input* erinput = ma::configure(pumi_mesh, sizefield);
+      erinput->shouldFixShape = true;
+      erinput->maximumIterations = 2;
+      if ( geom_order > 1)
+      {
+         crv::adapt(erinput);
+      }
+      else
+      {
+         ma::adapt(erinput);
+      }
+
+      ParMesh* Adapmesh = new ParPumiMesh(MPI_COMM_WORLD, pumi_mesh);
+      pPPmesh->UpdateMesh(Adapmesh);
+      delete Adapmesh;
+
+      // 19. Update the FiniteElementSpace, GridFunction, and bilinear form.
+      fespace->Update();
+      x.Update();
+      x = 0.0;
+
+      pPPmesh->FieldPUMItoMFEM(pumi_mesh, temp_field, &x);
+      a->Update();
+      b->Update();
+
+      // Destroy fields.
+      apf::destroyField(temp_field);
+      apf::destroyField(sizefield);
+   }
+
+   // 20. Free the used memory.
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
+   pumi_mesh->destroyNative();
+   apf::destroyMesh(pumi_mesh);
+   PCU_Comm_Free();
+
+#ifdef MFEM_USE_SIMMETRIX
+   gmi_sim_stop();
+   Sim_unregisterAllKeys();
+#endif
+
+   MPI_Finalize();
+
+   return 0;
+}

examples/pumi/makefile

@@ -0,0 +1,94 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/pumi/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+# All PUMI examples require MPI
+SEQ_EXAMPLES =
+PAR_EXAMPLES = ex1 ex1p ex2 ex6p
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES)
+
+ifeq ($(MFEM_USE_PUMI),NO)
+$(EXAMPLES):
+	$(error MFEM is not configured with PUMI)
+endif
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+ifneq (,$(filter test%,$(MAKECMDGOALS)))
+   ifeq (,$(wildcard ../../data/pumi))
+      $(info PUMI data directory not found. The PUMI tests will be SKIPPED.)
+      mfem-test = printf "   $(3) [$(2) $(1) ... ]: "; $(PRINT_SKIP)
+   endif
+endif
+
+# Testing: Parallel vs. serial runs
+RUN_MPI_NP = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP)
+RUN_MPI = $(RUN_MPI_NP) $(MFEM_MPI_NP)
+SERIAL_NAME := Serial PUMI example
+PARALLEL_NAME := Parallel PUMI example
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
+%-test-seq: %
+	@$(call mfem-test,$<,, $(SERIAL_NAME))
+
+# Testing: Example-specific execution options:
+ex1-test-par: ex1
+	@$(call mfem-test,$<, $(RUN_MPI_NP) 1, $(PARALLEL_NAME))
+ex1p-test-par: ex1p
+	@$(call mfem-test,$<, $(RUN_MPI_NP) 8, $(PARALLEL_NAME))
+ex2-test-par: ex2
+	@$(call mfem-test,$<, $(RUN_MPI_NP) 1, $(PARALLEL_NAME))
+ex6p-test-par: ex6p
+	@$(call mfem-test,$<, $(RUN_MPI_NP) 8, $(PARALLEL_NAME))
+
+# Testing: "test" target and mfem-test* variables are defined in config/test.mk
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -f refined.mesh sol.gf mesh.* sol.* displaced.mesh

fem/bilinearform.cpp

@@ -734,7 +734,7 @@ void BilinearForm::ComputeElementMatrices()
 }
 
 void BilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                                        Vector &sol, Vector &rhs, DiagonalPolicy dpolicy)
+                                        const Vector &sol, Vector &rhs, DiagonalPolicy dpolicy)
 {
    Array<int> ess_dofs, conf_ess_dofs;
    fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);
@@ -785,7 +785,7 @@ void BilinearForm::EliminateEssentialBCDiag (const Array<int> &bdr_attr_is_ess,
 }
 
 void BilinearForm::EliminateVDofs(const Array<int> &vdofs,
-                                  Vector &sol, Vector &rhs,
+                                  const Vector &sol, Vector &rhs,
                                   DiagonalPolicy dpolicy)
 {
    for (int i = 0; i < vdofs.Size(); i++)
@@ -825,7 +825,8 @@ void BilinearForm::EliminateVDofs(const Array<int> &vdofs,
 }
 
 void BilinearForm::EliminateEssentialBCFromDofs(
-   const Array<int> &ess_dofs, Vector &sol, Vector &rhs, DiagonalPolicy dpolicy)
+   const Array<int> &ess_dofs, const Vector &sol, Vector &rhs,
+   DiagonalPolicy dpolicy)
 {
    MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
    MFEM_ASSERT(sol.Size() == height, "incorrect sol Vector size");
@@ -1118,7 +1119,7 @@ void MixedBilinearForm::ConformingAssemble()
 }
 
 void MixedBilinearForm::EliminateTrialDofs (
-   Array<int> &bdr_attr_is_ess, Vector &sol, Vector &rhs )
+   Array<int> &bdr_attr_is_ess, const Vector &sol, Vector &rhs )
 {
    int i, j, k;
    Array<int> tr_vdofs, cols_marker (trial_fes -> GetVSize());
@@ -1141,7 +1142,7 @@ void MixedBilinearForm::EliminateTrialDofs (
 }
 
 void MixedBilinearForm::EliminateEssentialBCFromTrialDofs (
-   Array<int> &marked_vdofs, Vector &sol, Vector &rhs)
+   Array<int> &marked_vdofs, const Vector &sol, Vector &rhs)
 {
    mat -> EliminateCols (marked_vdofs, &sol, &rhs);
 }

fem/bilinearform.hpp

@@ -311,7 +311,7 @@ public:
        essential DOFs is set to 1.0. This behavior is controlled by the argument
        @a dpolicy. */
    void EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                             Vector &sol, Vector &rhs,
+                             const Vector &sol, Vector &rhs,
                              DiagonalPolicy dpolicy = DIAG_ONE);
 
    /// Eliminate essential boundary DOFs from the system matrix.
@@ -322,7 +322,7 @@ public:
                                  double value);
 
    /// Eliminate the given @a vdofs. NOTE: here, @a vdofs is a list of DOFs.
-   void EliminateVDofs(const Array<int> &vdofs, Vector &sol, Vector &rhs,
+   void EliminateVDofs(const Array<int> &vdofs, const Vector &sol, Vector &rhs,
                        DiagonalPolicy dpolicy = DIAG_ONE);
 
    /// Eliminate the given @a vdofs, storing the eliminated part internally.
@@ -333,10 +333,10 @@ public:
                        DiagonalPolicy dpolicy = DIAG_ONE);
 
    /** @brief Similar to
-       EliminateVDofs(const Array<int> &, Vector &, Vector &, DiagonalPolicy)
+       EliminateVDofs(const Array<int> &, const Vector &, Vector &, DiagonalPolicy)
        but here @a ess_dofs is a marker (boolean) array on all vector-dofs
        (@a ess_dofs[i] < 0 is true). */
-   void EliminateEssentialBCFromDofs(const Array<int> &ess_dofs, Vector &sol,
+   void EliminateEssentialBCFromDofs(const Array<int> &ess_dofs, const Vector &sol,
                                      Vector &rhs, DiagonalPolicy dpolicy = DIAG_ONE);
 
    /** @brief Similar to EliminateVDofs(const Array<int> &, DiagonalPolicy) but
@@ -460,10 +460,10 @@ public:
    void ConformingAssemble();
 
    void EliminateTrialDofs(Array<int> &bdr_attr_is_ess,
-                           Vector &sol, Vector &rhs);
+                           const Vector &sol, Vector &rhs);
 
    void EliminateEssentialBCFromTrialDofs(Array<int> &marked_vdofs,
-                                          Vector &sol, Vector &rhs);
+                                          const Vector &sol, Vector &rhs);
 
    virtual void EliminateTestDofs(Array<int> &bdr_attr_is_ess);
 

fem/bilininteg.cpp

@@ -1626,6 +1626,7 @@ void VectorCurlCurlIntegrator::AssembleElementMatrix(
       ir = &IntRules.Get(el.GetGeomType(), order);
    }
 
+   elmat.SetSize(dof*dim);
    elmat = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
    {

fem/coefficient.cpp

@@ -209,7 +209,7 @@ void VectorRestrictedCoefficient::Eval(
    }
    else
    {
-      M.SetSize(vdim);
+      M.SetSize(vdim, ir.GetNPoints());
       M = 0.0;
    }
 }

fem/conduitdatacollection.cpp

@@ -304,32 +304,43 @@ ConduitDataCollection::BlueprintMeshToMesh(const Node &n_mesh,
    if ( n_mesh_topo.has_child("boundary_topology") )
    {
       std::string bndry_topo_name = n_mesh_topo["boundary_topology"].as_string();
-      const Node &n_bndry_topo    = n_mesh["topologies"][bndry_topo_name];
-      std::string bndry_ele_shape = n_bndry_topo["elements/shape"].as_string();
 
-      bndry_geo = ShapeNameToGeomType(bndry_ele_shape);
-      int num_idxs_per_bndry_ele = Geometry::NumVerts[mesh_geo];
+      // In VisIt, we encountered a case were a mesh specified a boundary
+      // topology, but the boundary topology was omitted from the blueprint
+      // index, so it's data could not be obtained.
+      //
+      // This guard prevents an error in that case, allowing the mesh to be
+      // created without boundary info
 
-      const Node &n_bndry_conn = n_bndry_topo["elements/connectivity"];
-
-      // mfem requires ints, we could have int64s, etc convert if necessary
-      if ( n_bndry_conn.dtype().is_int() &&
-           n_bndry_conn.is_compact())
+      if (n_mesh["topologies"].has_child(bndry_topo_name))
       {
-         bndry_indices = n_bndry_conn.value();
-      }
-      else
-      {
-         Node &(n_bndry_conn_conv) =
-            n_conv["topologies"][bndry_topo_name]["elements/connectivity"];
-         n_bndry_conn.to_int_array(n_bndry_conn_conv);
-         bndry_indices = (n_bndry_conn_conv).value();
+         const Node &n_bndry_topo    = n_mesh["topologies"][bndry_topo_name];
+         std::string bndry_ele_shape = n_bndry_topo["elements/shape"].as_string();
 
-      }
+         bndry_geo = ShapeNameToGeomType(bndry_ele_shape);
+         int num_idxs_per_bndry_ele = Geometry::NumVerts[mesh_geo];
 
-      num_bndry_ele =
-         n_bndry_topo["elements/connectivity"].dtype().number_of_elements();
-      num_bndry_ele = num_bndry_ele / num_idxs_per_bndry_ele;
+         const Node &n_bndry_conn = n_bndry_topo["elements/connectivity"];
+
+         // mfem requires ints, we could have int64s, etc convert if necessary
+         if ( n_bndry_conn.dtype().is_int() &&
+              n_bndry_conn.is_compact())
+         {
+            bndry_indices = n_bndry_conn.value();
+         }
+         else
+         {
+            Node &(n_bndry_conn_conv) =
+               n_conv["topologies"][bndry_topo_name]["elements/connectivity"];
+            n_bndry_conn.to_int_array(n_bndry_conn_conv);
+            bndry_indices = (n_bndry_conn_conv).value();
+
+         }
+
+         num_bndry_ele =
+            n_bndry_topo["elements/connectivity"].dtype().number_of_elements();
+         num_bndry_ele = num_bndry_ele / num_idxs_per_bndry_ele;
+      }
    }
    else
    {

fem/datacollection.cpp

@@ -10,6 +10,7 @@
 // Software Foundation) version 2.1 dated February 1999.
 
 #include "fem.hpp"
+#include "../mesh/nurbs.hpp"
 #include "../general/text.hpp"
 #include "picojson.h"
 
@@ -331,6 +332,25 @@ DataCollection::~DataCollection()
 
 // class VisItDataCollection implementation
 
+void VisItDataCollection::UpdateMeshInfo()
+{
+   if (mesh)
+   {
+      spatial_dim = mesh->SpaceDimension();
+      topo_dim = mesh->Dimension();
+      if (mesh->NURBSext)
+      {
+         visit_levels_of_detail =
+            std::max(visit_levels_of_detail, mesh->NURBSext->GetOrder());
+      }
+   }
+   else
+   {
+      spatial_dim = 0;
+      topo_dim = 0;
+   }
+}
+
 VisItDataCollection::VisItDataCollection(const std::string& collection_name,
                                          Mesh *mesh)
    : DataCollection(collection_name, mesh)
@@ -338,17 +358,10 @@ VisItDataCollection::VisItDataCollection(const std::string& collection_name,
    appendRankToFileName = true; // always include rank in file names
    cycle = 0;                   // always include cycle in directory names
 
-   if (mesh)
-   {
-      spatial_dim = mesh->SpaceDimension();
-      topo_dim = mesh->Dimension();
-   }
-   else
-   {
-      spatial_dim = 0;
-      topo_dim = 0;
-   }
+   visit_levels_of_detail = 1;
    visit_max_levels_of_detail = 32;
+
+   UpdateMeshInfo();
 }
 
 #ifdef MFEM_USE_MPI
@@ -362,9 +375,11 @@ VisItDataCollection::VisItDataCollection(MPI_Comm comm,
    MPI_Comm_size(comm, &num_procs);
    appendRankToFileName = true; // always include rank in file names
    cycle = 0;                   // always include cycle in directory names
-   spatial_dim = 0;
-   topo_dim = 0;
+
+   visit_levels_of_detail = 1;
    visit_max_levels_of_detail = 32;
+
+   UpdateMeshInfo();
 }
 #endif
 
@@ -372,8 +387,7 @@ void VisItDataCollection::SetMesh(Mesh *new_mesh)
 {
    DataCollection::SetMesh(new_mesh);
    appendRankToFileName = true;
-   spatial_dim = mesh->SpaceDimension();
-   topo_dim = mesh->Dimension();
+   UpdateMeshInfo();
 }
 
 #ifdef MFEM_USE_MPI
@@ -392,6 +406,26 @@ void VisItDataCollection::RegisterField(const std::string& name,
 {
    DataCollection::RegisterField(name, gf);
    field_info_map[name] = VisItFieldInfo("nodes", gf->VectorDim());
+
+   int LOD = 1;
+   if (gf->FESpace()->GetNURBSext())
+   {
+      LOD = gf->FESpace()->GetNURBSext()->GetOrder();
+   }
+   else
+   {
+      for (int e=0; e<gf->FESpace()->GetNE() ; e++)
+      {
+         LOD = std::max(LOD,gf->FESpace()->GetFE(e)->GetOrder());
+      }
+   }
+
+   visit_levels_of_detail = std::max(visit_levels_of_detail, LOD);
+}
+
+void VisItDataCollection::SetLevelsOfDetail(int levels_of_detail)
+{
+   visit_levels_of_detail = levels_of_detail;
 }
 
 void VisItDataCollection::SetMaxLevelsOfDetail(int max_levels_of_detail)
@@ -595,6 +629,7 @@ std::string VisItDataCollection::GetVisItRootString()
    {
       ftags["assoc"] = picojson::value((it->second).association);
       ftags["comps"] = picojson::value(to_string((it->second).num_components));
+      ftags["lod"] = picojson::value(to_string(visit_levels_of_detail));
       field["path"] = picojson::value(path_str + it->first + file_ext_format);
       field["tags"] = picojson::value(ftags);
       fields[it->first] = picojson::value(field);

fem/datacollection.hpp

@@ -398,6 +398,7 @@ protected:
    // Additional data needed in the VisIt root file, which describes the mesh
    // and all the fields in the collection
    int spatial_dim, topo_dim;
+   int visit_levels_of_detail;
    int visit_max_levels_of_detail;
    std::map<std::string, VisItFieldInfo> field_info_map;
    typedef std::map<std::string, VisItFieldInfo>::iterator FieldInfoMapIterator;
@@ -407,6 +408,8 @@ protected:
    /// Read in a VisIt root file in JSON format
    void ParseVisItRootString(const std::string& json);
 
+   void UpdateMeshInfo();
+
    // Helper functions for Load()
    void LoadVisItRootFile(const std::string& root_name);
    void LoadMesh();
@@ -437,6 +440,9 @@ public:
    /// Add a grid function to the collection and update the root file
    virtual void RegisterField(const std::string& field_name, GridFunction *gf);
 
+   /// Set VisIt parameter: default levels of detail for the MultiresControl
+   void SetLevelsOfDetail(int levels_of_detail);
+
    /// Set VisIt parameter: maximum levels of detail for the MultiresControl
    void SetMaxLevelsOfDetail(int max_levels_of_detail);
 

fem/fe.cpp

@@ -6598,6 +6598,34 @@ void Poly_1D::CalcChebyshev(const int p, const double x, double *u, double *d)
    }
 }
 
+void Poly_1D::CalcChebyshev(const int p, const double x, double *u, double *d,
+                            double *dd)
+{
+   // recursive definition, z in [-1,1]
+   // T_0(z) = 1,  T_1(z) = z
+   // T_{n+1}(z) = 2*z*T_n(z) - T_{n-1}(z)
+   // T'_n(z) = n*U_{n-1}(z)
+   // U_0(z) = 1  U_1(z) = 2*z
+   // U_{n+1}(z) = 2*z*U_n(z) - U_{n-1}(z)
+   // U_n(z) = z*U_{n-1}(z) + T_n(z) = z*T'_n(z)/n + T_n(z)
+   // T'_{n+1}(z) = (n + 1)*(z*T'_n(z)/n + T_n(z))
+   // T''_{n+1}(z) = (n + 1)*(2*(n + 1)*T'_n(z) + z*T''_n(z)) / n
+   double z;
+   u[0] = 1.;
+   d[0] = 0.;
+   dd[0]= 0.;
+   if (p == 0) { return; }
+   u[1] = z = 2.*x - 1.;
+   d[1] = 2.;
+   dd[1] = 0;
+   for (int n = 1; n < p; n++)
+   {
+      u[n+1] = 2*z*u[n] - u[n-1];
+      d[n+1] = (n + 1)*(z*d[n]/n + 2*u[n]);
+      dd[n+1] = (n + 1)*(2.*(n + 1)*d[n] + z*dd[n])/n;
+   }
+}
+
 const double *Poly_1D::GetPoints(const int p, const int btype)
 {
    BasisType::Check(btype);
@@ -7463,8 +7491,12 @@ H1_TriangleElement::H1_TriangleElement(const int p, const int btype)
    dshape_x.SetSize(p + 1);
    dshape_y.SetSize(p + 1);
    dshape_l.SetSize(p + 1);
+   ddshape_x.SetSize(p + 1);
+   ddshape_y.SetSize(p + 1);
+   ddshape_l.SetSize(p + 1);
    u.SetSize(Dof);
    du.SetSize(Dof, Dim);
+   ddu.SetSize(Dof, (Dim * (Dim + 1)) / 2 );
 #else
    Vector shape_x(p + 1), shape_y(p + 1), shape_l(p + 1);
 #endif
@@ -7568,6 +7600,38 @@ void H1_TriangleElement::CalcDShape(const IntegrationPoint &ip,
    Ti.Mult(du, dshape);
 }
 
+void H1_TriangleElement::CalcHessian(const IntegrationPoint &ip,
+                                     DenseMatrix &ddshape) const
+{
+   const int p = Order;
+#ifdef MFEM_THREAD_SAFE
+   Vector   shape_x(p + 1),   shape_y(p + 1),   shape_l(p + 1);
+   Vector  dshape_x(p + 1),  dshape_y(p + 1),  dshape_l(p + 1);
+   Vector ddshape_x(p + 1), ddshape_y(p + 1), ddshape_l(p + 1);
+   DenseMatrix ddu(Dof, Dim);
+#endif
+
+   poly1d.CalcBasis(p, ip.x, shape_x, dshape_x, ddshape_x);
+   poly1d.CalcBasis(p, ip.y, shape_y, dshape_y, ddshape_y);
+   poly1d.CalcBasis(p, 1. - ip.x - ip.y, shape_l, dshape_l, ddshape_l);
+
+   for (int o = 0, j = 0; j <= p; j++)
+      for (int i = 0; i + j <= p; i++)
+      {
+         int k = p - i - j;
+         // u_xx, u_xy, u_yy
+         ddu(o,0) = ((ddshape_x(i) * shape_l(k)) - 2. * (dshape_x(i) * dshape_l(k)) +
+                     (shape_x(i) * ddshape_l(k))) * shape_y(j);
+         ddu(o,1) = (((shape_x(i) * ddshape_l(k)) - dshape_x(i) * dshape_l(k)) * shape_y(
+                        j)) + (((dshape_x(i) * shape_l(k)) - (shape_x(i) * dshape_l(k))) * dshape_y(j));
+         ddu(o,2) = ((ddshape_y(j) * shape_l(k)) - 2. * (dshape_y(j) * dshape_l(k)) +
+                     (shape_y(j) * ddshape_l(k))) * shape_x(i);
+         o++;
+      }
+
+   Ti.Mult(ddu, ddshape);
+}
+
 
 H1_TetrahedronElement::H1_TetrahedronElement(const int p, const int btype)
    : NodalFiniteElement(3, Geometry::TETRAHEDRON, ((p + 1)*(p + 2)*(p + 3))/6,
@@ -7584,8 +7648,13 @@ H1_TetrahedronElement::H1_TetrahedronElement(const int p, const int btype)
    dshape_y.SetSize(p + 1);
    dshape_z.SetSize(p + 1);
    dshape_l.SetSize(p + 1);
+   ddshape_x.SetSize(p + 1);
+   ddshape_y.SetSize(p + 1);
+   ddshape_z.SetSize(p + 1);
+   ddshape_l.SetSize(p + 1);
    u.SetSize(Dof);
    du.SetSize(Dof, Dim);
+   ddu.SetSize(Dof, (Dim * (Dim + 1)) / 2);
 #else
    Vector shape_x(p + 1), shape_y(p + 1), shape_z(p + 1), shape_l(p + 1);
 #endif
@@ -7738,6 +7807,51 @@ void H1_TetrahedronElement::CalcDShape(const IntegrationPoint &ip,
    Ti.Mult(du, dshape);
 }
 
+void H1_TetrahedronElement::CalcHessian(const IntegrationPoint &ip,
+                                        DenseMatrix &ddshape) const
+{
+   const int p = Order;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector   shape_x(p + 1),   shape_y(p + 1),   shape_z(p + 1),   shape_l(p + 1);
+   Vector  dshape_x(p + 1),  dshape_y(p + 1),  dshape_z(p + 1),  dshape_l(p + 1);
+   Vector ddshape_x(p + 1), ddshape_y(p + 1), ddshape_z(p + 1), ddshape_l(p + 1);
+   DenseMatrix ddu(Dof, ((Dim + 1) * Dim) / 2);
+#endif
+
+   poly1d.CalcBasis(p, ip.x, shape_x, dshape_x, ddshape_x);
+   poly1d.CalcBasis(p, ip.y, shape_y, dshape_y, ddshape_y);
+   poly1d.CalcBasis(p, ip.z, shape_z, dshape_z, ddshape_z);
+   poly1d.CalcBasis(p, 1. - ip.x - ip.y - ip.z, shape_l, dshape_l, ddshape_l);
+
+   for (int o = 0, k = 0; k <= p; k++)
+      for (int j = 0; j + k <= p; j++)
+         for (int i = 0; i + j + k <= p; i++)
+         {
+            // u_xx, u_xy, u_xz, u_yy, u_yz, u_zz
+            int l = p - i - j - k;
+            ddu(o,0) = ((ddshape_x(i) * shape_l(l)) - 2. * (dshape_x(i) * dshape_l(l)) +
+                        (shape_x(i) * ddshape_l(l))) * shape_y(j) * shape_z(k);
+            ddu(o,1) = ((dshape_y(j) * ((dshape_x(i) * shape_l(l)) -
+                                        (shape_x(i) * dshape_l(l)))) +
+                        (shape_y(j) * ((ddshape_l(l) * shape_x(i)) -
+                                       (dshape_x(i) * dshape_l(l)))))* shape_z(k);
+            ddu(o,2) = ((dshape_z(k) * ((dshape_x(i) * shape_l(l)) -
+                                        (shape_x(i) * dshape_l(l)))) +
+                        (shape_z(k) * ((ddshape_l(l) * shape_x(i)) -
+                                       (dshape_x(i) * dshape_l(l)))))* shape_y(j);
+            ddu(o,3) = ((ddshape_y(j) * shape_l(l)) - 2. * (dshape_y(j) * dshape_l(l)) +
+                        (shape_y(j) * ddshape_l(l))) * shape_x(i) * shape_z(k);
+            ddu(o,4) = ((dshape_z(k) * ((dshape_y(j) * shape_l(l)) -
+                                        (shape_y(j)*dshape_l(l))) ) +
+                        (shape_z(k)* ((ddshape_l(l)*shape_y(j)) -
+                                      (dshape_y(j) * dshape_l(l)) ) ) )* shape_x(i);
+            ddu(o,5) = ((ddshape_z(k) * shape_l(l)) - 2. * (dshape_z(k) * dshape_l(l)) +
+                        (shape_z(k) * ddshape_l(l))) * shape_y(j) * shape_x(i);
+            o++;
+         }
+   Ti.Mult(ddu, ddshape);
+}
 
 H1Pos_TriangleElement::H1Pos_TriangleElement(const int p)
    : PositiveFiniteElement(2, Geometry::TRIANGLE, ((p + 1)*(p + 2))/2, p,

fem/fe.hpp

@@ -1564,6 +1564,8 @@ private:
 
    static void CalcChebyshev(const int p, const double x, double *u);
    static void CalcChebyshev(const int p, const double x, double *u, double *d);
+   static void CalcChebyshev(const int p, const double x, double *u, double *d,
+                             double *dd);
 
    QuadratureFunctions1D quad_func;
 
@@ -1618,6 +1620,14 @@ public:
    // { CalcLegendre(p, x, u, d); }
    { CalcChebyshev(p, x, u, d); }
 
+   // Evaluate the values, derivatives and second derivatives of a hierarchical 1D basis at point x
+   static void CalcBasis(const int p, const double x, double *u, double *d,
+                         double *dd)
+   // { CalcMono(p, x, u, d); }
+   // { CalcBernstein(p, x, u, d); }
+   // { CalcLegendre(p, x, u, d); }
+   { CalcChebyshev(p, x, u, d, dd); }
+
    // Evaluate a representation of a Delta function at point x
    static double CalcDelta(const int p, const double x)
    { return pow(x, (double) p); }
@@ -1820,7 +1830,8 @@ class H1_TriangleElement : public NodalFiniteElement
 private:
 #ifndef MFEM_THREAD_SAFE
    mutable Vector shape_x, shape_y, shape_l, dshape_x, dshape_y, dshape_l, u;
-   mutable DenseMatrix du;
+   mutable Vector ddshape_x, ddshape_y, ddshape_l;
+   mutable DenseMatrix du, ddu;
 #endif
    DenseMatrixInverse Ti;
 
@@ -1829,6 +1840,8 @@ public:
    virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
    virtual void CalcDShape(const IntegrationPoint &ip,
                            DenseMatrix &dshape) const;
+   virtual void CalcHessian(const IntegrationPoint &ip,
+                            DenseMatrix &ddshape) const;
 };
 
 
@@ -1838,7 +1851,8 @@ private:
 #ifndef MFEM_THREAD_SAFE
    mutable Vector shape_x, shape_y, shape_z, shape_l;
    mutable Vector dshape_x, dshape_y, dshape_z, dshape_l, u;
-   mutable DenseMatrix du;
+   mutable Vector ddshape_x, ddshape_y, ddshape_z, ddshape_l;
+   mutable DenseMatrix du, ddu;
 #endif
    DenseMatrixInverse Ti;
 
@@ -1848,6 +1862,8 @@ public:
    virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
    virtual void CalcDShape(const IntegrationPoint &ip,
                            DenseMatrix &dshape) const;
+   virtual void CalcHessian(const IntegrationPoint &ip,
+                            DenseMatrix &ddshape) const;
 };
 
 

fem/fespace.hpp

@@ -557,10 +557,10 @@ public:
    virtual ~QuadratureSpace() { delete [] element_offsets; }
 
    /// Return the total number of quadrature points.
-   int GetSize() { return size; }
+   int GetSize() const { return size; }
 
    /// Get the IntegrationRule associated with mesh element @a idx.
-   const IntegrationRule &GetElementIntRule(int idx)
+   const IntegrationRule &GetElementIntRule(int idx) const
    { return *int_rule[mesh->GetElementBaseGeometry(idx)]; }
 
    /// Write the QuadratureSpace to the stream @a out.

fem/gridfunc.cpp

@@ -2583,6 +2583,24 @@ QuadratureFunction::QuadratureFunction(Mesh *mesh, std::istream &in)
    Load(in, vdim*qspace->GetSize());
 }
 
+QuadratureFunction & QuadratureFunction::operator=(double value)
+{
+   Vector::operator=(value);
+   return *this;
+}
+
+QuadratureFunction & QuadratureFunction::operator=(const Vector &v)
+{
+   MFEM_ASSERT(qspace && v.Size() == qspace->GetSize(), "");
+   Vector::operator=(v);
+   return *this;
+}
+
+QuadratureFunction & QuadratureFunction::operator=(const QuadratureFunction &v)
+{
+   return this->operator=((const Vector &)v);
+}
+
 void QuadratureFunction::Save(std::ostream &out) const
 {
    qspace->Save(out);

fem/gridfunc.hpp

@@ -472,8 +472,21 @@ public:
    /// Set the QuadratureSpace ownership flag.
    void SetOwnsSpace(bool own) { own_qspace = own; }
 
+   /// Redefine '=' for QuadratureFunction = constant.
+   QuadratureFunction &operator=(double value);
+
+   /// Copy the data from @a v.
+   /** The size of @a v must be equal to the size of the QuadratureSpace
+       @a qspace. */
+   QuadratureFunction &operator=(const Vector &v);
+
+   /// Copy the data from @a v.
+   /** The QuadratureFunctions @a v and @a *this must have QuadratureSpaces with
+       the same size. */
+   QuadratureFunction &operator=(const QuadratureFunction &v);
+
    /// Get the IntegrationRule associated with mesh element @a idx.
-   const IntegrationRule &GetElementIntRule(int idx)
+   const IntegrationRule &GetElementIntRule(int idx) const
    { return qspace->GetElementIntRule(idx); }
 
    /// Return all values associated with mesh element @a idx in a Vector.
@@ -485,6 +498,15 @@ public:
     */
    inline void GetElementValues(int idx, Vector &values);
 
+   /// Return all values associated with mesh element @a idx in a Vector.
+   /** The result is stored in the Vector @a values as a copy of the
+       global values.
+
+       Inside the Vector @a values, the index `i+vdim*j` corresponds to the
+       `i`-th vector component at the `j`-th quadrature point.
+    */
+   inline void GetElementValues(int idx, Vector &values) const;
+
    /// Return all values associated with mesh element @a idx in a DenseMatrix.
    /** The result is stored in the DenseMatrix @a values as a reference to the
        global values.
@@ -494,6 +516,15 @@ public:
     */
    inline void GetElementValues(int idx, DenseMatrix &values);
 
+   /// Return all values associated with mesh element @a idx in a const DenseMatrix.
+   /** The result is stored in the DenseMatrix @a values as a copy of the
+       global values.
+
+       Inside the DenseMatrix @a values, the `(i,j)` entry corresponds to the
+       `i`-th vector component at the `j`-th quadrature point.
+    */
+   inline void GetElementValues(int idx, DenseMatrix &values) const;
+
    /// Write the QuadratureFunction to the stream @a out.
    void Save(std::ostream &out) const;
 };
@@ -567,6 +598,18 @@ inline void QuadratureFunction::GetElementValues(int idx, Vector &values)
    values.NewDataAndSize(data + vdim*s_offset, vdim*sl_size);
 }
 
+inline void QuadratureFunction::GetElementValues(int idx, Vector &values) const
+{
+   const int s_offset = qspace->element_offsets[idx];
+   const int sl_size = qspace->element_offsets[idx+1] - s_offset;
+   values.SetSize(vdim*sl_size);
+   double *q = data + vdim*s_offset;
+   for (int i = 0; i<values.Size(); i++)
+   {
+      values(i) = *(q++);
+   }
+}
+
 inline void QuadratureFunction::GetElementValues(int idx, DenseMatrix &values)
 {
    const int s_offset = qspace->element_offsets[idx];
@@ -574,6 +617,20 @@ inline void QuadratureFunction::GetElementValues(int idx, DenseMatrix &values)
    values.Reset(data + vdim*s_offset, vdim, sl_size);
 }
 
+inline void QuadratureFunction::GetElementValues(int idx,
+                                                 DenseMatrix &values) const
+{
+   const int s_offset = qspace->element_offsets[idx];
+   const int sl_size = qspace->element_offsets[idx+1] - s_offset;
+   values.SetSize(vdim, sl_size);
+   double *q = data + vdim*s_offset;
+   for (int j = 0; j<sl_size; j++)
+      for (int i = 0; i<vdim; i++)
+      {
+         values(i,j) = *(q++);
+      }
+}
+
 } // namespace mfem
 
 #endif

fem/intrules.hpp

@@ -31,6 +31,7 @@ public:
 
    void Set(const double *p, const int dim)
    {
+      MFEM_ASSERT(1 <= dim && dim <= 3, "invalid dim: " << dim);
       x = p[0];
       if (dim > 1)
       {
@@ -44,6 +45,7 @@ public:
 
    void Get(double *p, const int dim) const
    {
+      MFEM_ASSERT(1 <= dim && dim <= 3, "invalid dim: " << dim);
       p[0] = x;
       if (dim > 1)
       {

fem/nonlininteg.hpp

@@ -29,7 +29,7 @@ protected:
    const IntegrationRule *IntRule;
 
    NonlinearFormIntegrator(const IntegrationRule *ir = NULL)
-      : IntRule(NULL) { }
+      : IntRule(ir) { }
 
 public:
    /** @brief Prescribe a fixed IntegrationRule to use (when @a ir != NULL) or

fem/pfespace.cpp

@@ -1229,7 +1229,7 @@ void ParFiniteElementSpace::GetGhostDofs(int entity, const MeshId &id,
    }
 }
 
-void ParFiniteElementSpace::GetBareDofs(int entity, const MeshId &id,
+void ParFiniteElementSpace::GetBareDofs(int entity, int index,
                                         Array<int> &dofs) const
 {
    int ned, ghost, first;
@@ -1238,25 +1238,25 @@ void ParFiniteElementSpace::GetBareDofs(int entity, const MeshId &id,
       case 0:
          ned = fec->DofForGeometry(Geometry::POINT);
          ghost = pncmesh->GetNVertices();
-         first = (id.index < ghost)
-                 ? id.index*ned // regular vertex
-                 : ndofs + (id.index - ghost)*ned; // ghost vertex
+         first = (index < ghost)
+                 ? index*ned // regular vertex
+                 : ndofs + (index - ghost)*ned; // ghost vertex
          break;
 
       case 1:
          ned = fec->DofForGeometry(Geometry::SEGMENT);
          ghost = pncmesh->GetNEdges();
-         first = (id.index < ghost)
-                 ? nvdofs + id.index*ned // regular edge
-                 : ndofs + ngvdofs + (id.index - ghost)*ned; // ghost edge
+         first = (index < ghost)
+                 ? nvdofs + index*ned // regular edge
+                 : ndofs + ngvdofs + (index - ghost)*ned; // ghost edge
          break;
 
       default:
          ned = fec->DofForGeometry(mesh->GetFaceBaseGeometry(0));
          ghost = pncmesh->GetNFaces();
-         first = (id.index < ghost)
-                 ? nvdofs + nedofs + id.index*ned // regular face
-                 : ndofs + ngvdofs + ngedofs + (id.index - ghost)*ned; // ghost
+         first = (index < ghost)
+                 ? nvdofs + nedofs + index*ned // regular face
+                 : ndofs + ngvdofs + ngedofs + (index - ghost)*ned; // ghost
          break;
    }
 
@@ -1674,7 +1674,7 @@ void ParFiniteElementSpace::ForwardRow(const PMatrixRow &row, int dof,
 
 #ifdef MFEM_DEBUG_PMATRIX
 void ParFiniteElementSpace
-::DebugDumpDOFs(std::ofstream &os,
+::DebugDumpDOFs(std::ostream &os,
                 const SparseMatrix &deps,
                 const Array<GroupId> &dof_group,
                 const Array<GroupId> &dof_owner,
@@ -1800,9 +1800,6 @@ int ParFiniteElementSpace
          }
       }
 
-      // make sure all master DOFs are transmitted to participating slave ranks
-      pncmesh->AugmentMasterGroups();
-
       deps.Finalize();
    }
 
@@ -1820,7 +1817,7 @@ int ParFiniteElementSpace
       // initialize dof_group[], dof_owner[]
       for (int entity = 0; entity <= 2; entity++)
       {
-         const NCMesh::NCList &list = pncmesh->GetSharedList(entity);
+         const NCMesh::NCList &list = pncmesh->GetNCList(entity);
 
          std::size_t lsize[3] =
          { list.conforming.size(), list.masters.size(), list.slaves.size() };
@@ -1834,13 +1831,16 @@ int ParFiniteElementSpace
                   (l == 1) ? (const MeshId&) list.masters[i]
                   /*    */ : (const MeshId&) list.slaves[i];
 
-               GetBareDofs(entity, id, dofs);
+               GroupId owner = pncmesh->GetEntityOwnerId(entity, id.index);
+               GroupId group = pncmesh->GetEntityGroupId(entity, id.index);
+
+               GetBareDofs(entity, id.index, dofs);
 
                for (int j = 0; j < dofs.Size(); j++)
                {
                   int dof = dofs[j];
-                  dof_owner[dof] = pncmesh->GetOwnerId(entity, id.index);
-                  dof_group[dof] = pncmesh->GetGroupId(entity, id.index);
+                  dof_owner[dof] = owner;
+                  dof_group[dof] = group;
                }
             }
          }

fem/pfespace.hpp

@@ -114,8 +114,7 @@ private:
 
    void GetGhostDofs(int entity, const MeshId &id, Array<int> &dofs) const;
    // Return the dofs associated with the interior of the given mesh entity.
-   // The MeshId may be the id of a regular or a ghost mesh entity.
-   void GetBareDofs(int entity, const MeshId &id, Array<int> &dofs) const;
+   void GetBareDofs(int entity, int index, Array<int> &dofs) const;
 
    int  PackDof(int entity, int index, int edof) const;
    void UnpackDof(int dof, int &entity, int &index, int &edof) const;
@@ -128,7 +127,7 @@ private:
                    std::map<int, class NeighborRowMessage> &send_msg) const;
 
 #ifdef MFEM_DEBUG_PMATRIX
-   void DebugDumpDOFs(std::ofstream &os,
+   void DebugDumpDOFs(std::ostream &os,
                       const SparseMatrix &deps,
                       const Array<GroupId> &dof_group,
                       const Array<GroupId> &dof_owner,

fem/sidredatacollection.cpp

@@ -45,11 +45,11 @@ SidreDataCollection::SidreDataCollection(const std::string& collection_name,
    sidre::Group * domain_grp =
       m_datastore_ptr->getRoot()->createGroup(collection_name);
 
-   bp_grp = domain_grp->createGroup("blueprint");
+   m_bp_grp = domain_grp->createGroup("blueprint");
    // Currently only rank 0 adds anything to bp_index.
-   bp_index_grp = global_grp->createGroup("blueprint_index/" + name);
+   m_bp_index_grp = global_grp->createGroup("blueprint_index/" + name);
 
-   named_bufs_grp = domain_grp->createGroup("named_buffers");
+   m_named_bufs_grp = domain_grp->createGroup("named_buffers");
 
    if (the_mesh)
    {
@@ -70,21 +70,19 @@ SidreDataCollection::SidreDataCollection(const std::string& collection_name,
 // in the future.  When this is available, all the blueprint index code can be
 // removed from the data collection class.
 SidreDataCollection::SidreDataCollection(const std::string& collection_name,
-                                         axom::sidre::Group* global_grp,
+                                         axom::sidre::Group* bp_index_grp,
                                          axom::sidre::Group* domain_grp,
                                          bool own_mesh_data)
    : mfem::DataCollection(collection_name),
      m_owns_datastore(false),
      m_owns_mesh_data(own_mesh_data),
      m_meshNodesGFName("mesh_nodes"),
-     m_datastore_ptr(NULL)
+     m_datastore_ptr(NULL),
+     m_bp_index_grp(bp_index_grp)
 {
-   bp_grp = domain_grp->createGroup("blueprint");
+   m_bp_grp = domain_grp->createGroup("blueprint");
 
-   // Currently only rank 0 adds anything to bp_index.
-   bp_index_grp = global_grp->createGroup("blueprint_index/" + name);
-
-   named_bufs_grp = domain_grp->createGroup("named_buffers");
+   m_named_bufs_grp = domain_grp->createGroup("named_buffers");
 
 #ifdef MFEM_USE_MPI
    m_comm = MPI_COMM_NULL;
@@ -113,10 +111,10 @@ void SidreDataCollection::SetComm(MPI_Comm comm)
 // protected method
 sidre::Group *SidreDataCollection::named_buffers_grp() const
 {
-   MFEM_ASSERT(named_bufs_grp != NULL,
+   MFEM_ASSERT(m_named_bufs_grp != NULL,
                "No group 'named_buffers' in data collection.  Verify that"
                " SetMesh was called to set the mesh in the data collection.");
-   return named_bufs_grp;
+   return m_named_bufs_grp;
 }
 
 // protected method
@@ -245,19 +243,19 @@ void SidreDataCollection::createMeshBlueprintStubs(bool hasBP)
 {
    if (!hasBP)
    {
-      bp_grp->createGroup("state");
-      bp_grp->createGroup("coordsets");
-      bp_grp->createGroup("topologies");
-      bp_grp->createGroup("fields");
+      m_bp_grp->createGroup("state");
+      m_bp_grp->createGroup("coordsets");
+      m_bp_grp->createGroup("topologies");
+      m_bp_grp->createGroup("fields");
    }
 
    // If rank is 0, set up blueprint index state group.
    if (myid == 0)
    {
-      bp_index_grp->createGroup("state");
-      bp_index_grp->createGroup("coordsets");
-      bp_index_grp->createGroup("topologies");
-      bp_index_grp->createGroup("fields");
+      m_bp_index_grp->createGroup("state");
+      m_bp_index_grp->createGroup("coordsets");
+      m_bp_index_grp->createGroup("topologies");
+      m_bp_index_grp->createGroup("fields");
    }
 }
 
@@ -267,18 +265,18 @@ void SidreDataCollection::createMeshBlueprintState(bool hasBP)
    if (!hasBP)
    {
       // Set up blueprint state group.
-      bp_grp->createViewScalar("state/cycle", 0);
-      bp_grp->createViewScalar("state/time", 0.);
-      bp_grp->createViewScalar("state/domain", myid);
-      bp_grp->createViewScalar("state/time_step", 0.);
+      m_bp_grp->createViewScalar("state/cycle", 0);
+      m_bp_grp->createViewScalar("state/time", 0.);
+      m_bp_grp->createViewScalar("state/domain", myid);
+      m_bp_grp->createViewScalar("state/time_step", 0.);
    }
 
    // If rank is 0, set up blueprint index state group.
    if (myid == 0)
    {
-      bp_index_grp->createViewScalar("state/cycle", 0);
-      bp_index_grp->createViewScalar("state/time", 0.);
-      bp_index_grp->createViewScalar("state/number_of_domains", num_procs);
+      m_bp_index_grp->createViewScalar("state/cycle", 0);
+      m_bp_index_grp->createViewScalar("state/time", 0.);
+      m_bp_index_grp->createViewScalar("state/number_of_domains", num_procs);
    }
 }
 
@@ -297,7 +295,7 @@ void SidreDataCollection::createMeshBlueprintCoordset(bool hasBP)
    // Add blueprint if not present
    if ( !hasBP )
    {
-      bp_grp->createViewString("coordsets/coords/type", "explicit");
+      m_bp_grp->createViewString("coordsets/coords/type", "explicit");
 
       sidre::DataType dtype =
          sidre::DataType::c_double(num_vertices);
@@ -306,17 +304,17 @@ void SidreDataCollection::createMeshBlueprintCoordset(bool hasBP)
 
       // Set up views for x, y, z values
       sidre::View *vx, *vy = NULL, *vz = NULL;
-      vx = bp_grp->createView("coordsets/coords/values/x", dtype);
+      vx = m_bp_grp->createView("coordsets/coords/values/x", dtype);
 
       if (dim >= 2)
       {
          dtype.set_offset(dtype.offset() + stride);
-         vy = bp_grp->createView("coordsets/coords/values/y", dtype);
+         vy = m_bp_grp->createView("coordsets/coords/values/y", dtype);
       }
       if (dim >= 3)
       {
          dtype.set_offset(dtype.offset() + stride);
-         vz = bp_grp->createView("coordsets/coords/values/z", dtype);
+         vz = m_bp_grp->createView("coordsets/coords/values/z", dtype);
       }
 
       if (m_owns_mesh_data)
@@ -343,27 +341,27 @@ void SidreDataCollection::createMeshBlueprintCoordset(bool hasBP)
    // If rank 0, set up blueprint index for coordinate set.
    if (myid == 0)
    {
-      bp_index_grp->createViewString(
-         "coordsets/coords/path", bp_grp->getPathName() + "/coordsets/coords");
+      m_bp_index_grp->createViewString(
+         "coordsets/coords/path", m_bp_grp->getPathName() + "/coordsets/coords");
 
-      bp_index_grp->getGroup("coordsets/coords")->copyView(
-         bp_grp->getView("coordsets/coords/type") );
+      m_bp_index_grp->getGroup("coordsets/coords")->copyView(
+         m_bp_grp->getView("coordsets/coords/type") );
 
-      bp_index_grp->createViewString(
+      m_bp_index_grp->createViewString(
          "coordsets/coords/coord_system/type", "cartesian");
 
       // These are empty views, their existence in the group tree is used to
       // define the number of dims
-      bp_index_grp->createView("coordsets/coords/coord_system/axes/x");
+      m_bp_index_grp->createView("coordsets/coords/coord_system/axes/x");
 
       if (dim >= 2)
       {
-         bp_index_grp->createView("coordsets/coords/coord_system/axes/y");
+         m_bp_index_grp->createView("coordsets/coords/coord_system/axes/y");
       }
 
       if (dim == 3)
       {
-         bp_index_grp->createView("coordsets/coords/coord_system/axes/z");
+         m_bp_index_grp->createView("coordsets/coords/coord_system/axes/z");
       }
    }
 
@@ -385,38 +383,41 @@ createMeshBlueprintTopologies(bool hasBP, const std::string& mesh_name)
                             ? mesh->GetNE()
                             : mesh->GetNBE();
 
-   MFEM_VERIFY(num_elements > 0,
-               "TODO: processors with 0 " << mesh_name << " elements");
-
-   const int element_size = !isBdry
-                            ? mesh->GetElement(0)->GetNVertices()
-                            : mesh->GetBdrElement(0)->GetNVertices();
-
-   const int num_indices = num_elements * element_size;
-
-   // Find the element shape
-   // Note: Assumes homogeneous elements, so only check the first element
-   const int geom =
-      isBdry ?
-      mesh->GetBdrElementBaseGeometry(0) :
-      mesh->GetElementBaseGeometry(0);
-   const std::string eltTypeStr =
-      !isBdry
-      ? getElementName( static_cast<Element::Type>(
-                           mesh->GetElement(0)->GetType() ) )
-      : getElementName( static_cast<Element::Type>(
-                           mesh->GetBdrElement(0)->GetType() ) );
-
    const std::string mesh_topo_str = "topologies/" + mesh_name;
    const std::string mesh_attr_str = mesh_name + "_material_attribute";
 
+   int element_size = 0;
+   int num_indices = 0;
+   int geom = 0;
+   std::string eltTypeStr = "point";
+
+   if (num_elements > 0)
+   {
+      element_size = !isBdry
+                     ? mesh->GetElement(0)->GetNVertices()
+                     : mesh->GetBdrElement(0)->GetNVertices();
+
+      num_indices = num_elements * element_size;
+
+      // Find the element shape
+      // Note: Assumes homogeneous elements, so only check the first element
+      geom = isBdry ?
+             mesh->GetBdrElementBaseGeometry(0) :
+             mesh->GetElementBaseGeometry(0);
+      eltTypeStr =
+         !isBdry
+         ? getElementName( static_cast<Element::Type>(
+                              mesh->GetElement(0)->GetType() ) )
+         : getElementName( static_cast<Element::Type>(
+                              mesh->GetBdrElement(0)->GetType() ) );
+   }
+
+   // Create the blueprint "topology" group, if not present
    if ( !hasBP )
    {
-      sidre::Group* topology_grp = bp_grp->createGroup(mesh_topo_str);
+      sidre::Group* topology_grp = m_bp_grp->createGroup(mesh_topo_str);
 
-      // Add mesh topology
       topology_grp->createViewString("type", "unstructured");
-      // Note: eltTypeStr comes form the mesh
       topology_grp->createViewString("elements/shape", eltTypeStr);
       topology_grp->createViewAndAllocate(
          "elements/connectivity", sidre::INT_ID, num_indices);
@@ -428,33 +429,52 @@ createMeshBlueprintTopologies(bool hasBP, const std::string& mesh_name)
       {
          topology_grp->createViewString("grid_function",m_meshNodesGFName);
       }
-
-      // Add the mesh's attributes as an attribute field
-      RegisterAttributeField(mesh_attr_str, isBdry);
    }
 
-   // If rank 0, set up blueprint index for topologies group and material
-   // attribute field.
+   // Add the mesh's attributes as an attribute field
+   RegisterAttributeField(mesh_attr_str, isBdry);
+
+   // Change ownership or copy the element arrays into Sidre
+   if (num_elements > 0)
+   {
+      sidre::View* conn_view =
+         m_bp_grp->getGroup(mesh_topo_str)->getView("elements/connectivity");
+
+      // The SidreDataCollection always owns these arrays:
+      Array<int> conn_array(conn_view->getData<int*>(), num_indices);
+      Array<int>* attr_array = attr_map.Get(mesh_attr_str);
+      if (!isBdry)
+      {
+         mesh->GetElementData(geom, conn_array, *attr_array);
+      }
+      else
+      {
+         mesh->GetBdrElementData(geom, conn_array, *attr_array);
+      }
+      MFEM_ASSERT(!conn_array.OwnsData(), "");
+      MFEM_ASSERT(!attr_array->OwnsData(), "");
+   }
+
+   // If rank 0, set up blueprint index for topologies group
    if (myid == 0)
    {
-      const std::string bp_grp_path = bp_grp->getPathName();
+      const std::string m_bp_grp_path = m_bp_grp->getPathName();
 
-      // Create blueprint index for topologies.
       if (isBdry)
       {
-         // "Shallow" copy the bp_grp view into the bp_index_grp sub-group.
+         // "Shallow" copy the m_bp_grp view into the m_bp_index_grp sub-group.
          // Note that the "topologies/mesh" sub-group has to exist, i.e. this
          // method should be called first with mesh_name = "mesh".
-         bp_index_grp->getGroup("topologies/mesh")
-         ->copyView( bp_grp->getView("topologies/mesh/boundary_topology") );
+         m_bp_index_grp->getGroup("topologies/mesh")
+         ->copyView( m_bp_grp->getView("topologies/mesh/boundary_topology") );
       }
 
       sidre::Group *bp_index_topo_grp =
-         bp_index_grp->createGroup(mesh_topo_str);
-      sidre::Group *topology_grp = bp_grp->getGroup(mesh_topo_str);
+         m_bp_index_grp->createGroup(mesh_topo_str);
+      sidre::Group *topology_grp = m_bp_grp->getGroup(mesh_topo_str);
 
       bp_index_topo_grp->createViewString(
-         "path", bp_grp_path + "/" + mesh_topo_str);
+         "path", m_bp_grp_path + "/" + mesh_topo_str);
       bp_index_topo_grp->copyView( topology_grp->getView("type") );
       bp_index_topo_grp->copyView( topology_grp->getView("coordset") );
 
@@ -465,24 +485,6 @@ createMeshBlueprintTopologies(bool hasBP, const std::string& mesh_name)
          bp_index_topo_grp->copyView(topology_grp->getView("grid_function"));
       }
    }
-
-   // Finally, change ownership or copy the element arrays into Sidre
-   sidre::View* conn_view =
-      bp_grp->getGroup(mesh_topo_str)->getView("elements/connectivity");
-
-   // The SidreDataCollection always owns these arrays:
-   Array<int> conn_array(conn_view->getData<int*>(), num_indices);
-   Array<int>* attr_array = attr_map.Get(mesh_attr_str);
-   if (!isBdry)
-   {
-      mesh->GetElementData(geom, conn_array, *attr_array);
-   }
-   else
-   {
-      mesh->GetBdrElementData(geom, conn_array, *attr_array);
-   }
-   MFEM_ASSERT(!conn_array.OwnsData(), "");
-   MFEM_ASSERT(!attr_array->OwnsData(), "");
 }
 
 // private method
@@ -501,11 +503,11 @@ void SidreDataCollection::createMeshBlueprintAdjacencies(bool hasBP)
    sidre::Group* adjset_grp = NULL;
    if (pmesh->GetNGroups() > 1)
    {
-      adjset_grp = bp_grp->createGroup("adjsets/mesh");
+      adjset_grp = m_bp_grp->createGroup("adjsets/mesh");
       adjset_grp->createViewString("association", "vertex");
       adjset_grp->createViewString("topology", "mesh");
 
-      if (myid == 0) { bp_index_grp->createGroup("adjsets"); }
+      if (myid == 0) { m_bp_index_grp->createGroup("adjsets"); }
    }
 
    for (int gi = 1; gi < pmesh->GetNGroups(); ++gi)
@@ -562,14 +564,36 @@ void SidreDataCollection::verifyMeshBlueprint()
    // Add call to that when it's available to check actual contents in sidre.
 }
 
+
+bool SidreDataCollection::HasBoundaryMesh() const
+{
+   // check if this rank has any boundary elements
+   int hasBndElts = mesh->GetNBE() > 0 ? 1 : 0;
+
+#ifdef MFEM_USE_MPI
+   // check if any rank has boundary elements
+   ParMesh *pmesh = dynamic_cast<ParMesh*>(mesh);
+   if (pmesh)
+   {
+      int hasBndElts_g;
+      MPI_Allreduce(&hasBndElts, &hasBndElts_g, 1,
+                    MPI_INT, MPI_MAX,pmesh->GetComm());
+
+      hasBndElts = hasBndElts_g;
+   }
+#endif
+
+   return hasBndElts > 0? true : false;
+}
+
 void SidreDataCollection::SetMesh(Mesh *new_mesh)
 {
    DataCollection::SetMesh(new_mesh);
 
    // hasBP is used to indicate if the data currently in the blueprint should be
    // used to replace the data in the mesh.
-   bool hasBP = bp_grp->getNumViews() > 0 || bp_grp->getNumGroups() > 0;
-   bool has_bnd_elts = (new_mesh->GetNBE() > 0);
+   bool hasBP = m_bp_grp->getNumViews() > 0 || m_bp_grp->getNumGroups() > 0;
+   bool has_bnd_elts = HasBoundaryMesh();
 
    createMeshBlueprintStubs(hasBP);
    createMeshBlueprintState(hasBP);
@@ -583,7 +607,7 @@ void SidreDataCollection::SetMesh(Mesh *new_mesh)
    if (has_bnd_elts)
    {
       // Set the "boundary_topology" of "mesh" to "boundary".
-      bp_grp->createViewString("topologies/mesh/boundary_topology", "boundary");
+      m_bp_grp->createViewString("topologies/mesh/boundary_topology", "boundary");
 
       // register the "boundary" topology in the blueprint.
       createMeshBlueprintTopologies(hasBP, "boundary");
@@ -605,7 +629,7 @@ void SidreDataCollection::SetMesh(Mesh *new_mesh)
       {
          // Get the bp mesh nodes name.
          sidre::View *v_bp_nodes_name =
-            bp_grp->getView("topologies/mesh/grid_function");
+            m_bp_grp->getView("topologies/mesh/grid_function");
          std::string bp_nodes_name(v_bp_nodes_name->getString());
 
          // Check that the names match, e.g. when loading the collection.
@@ -667,17 +691,15 @@ void SidreDataCollection::SetMesh(MPI_Comm comm, Mesh *new_mesh)
 #endif
 
 void SidreDataCollection::
-SetGroupPointers(axom::sidre::Group *global_grp,
+SetGroupPointers(axom::sidre::Group *bp_index_grp,
                  axom::sidre::Group *domain_grp)
 {
    MFEM_VERIFY(domain_grp->hasGroup("blueprint"),
                "Domain group does not contain a blueprint group.");
-   MFEM_VERIFY(global_grp->hasGroup("blueprint_index/" + name),
-               "Global group does not contain a blueprint index group.");
 
-   bp_grp = domain_grp->getGroup("blueprint");
-   bp_index_grp = global_grp->getGroup("blueprint_index/" + name);
-   named_bufs_grp = domain_grp->getGroup("named_buffers");
+   m_bp_grp = domain_grp->getGroup("blueprint");
+   m_bp_index_grp = bp_index_grp;
+   m_named_bufs_grp = domain_grp->getGroup("named_buffers");
 }
 
 void SidreDataCollection::Load(const std::string& path,
@@ -690,12 +712,12 @@ void SidreDataCollection::Load(const std::string& path,
    if (m_comm != MPI_COMM_NULL)
    {
       axom::sidre::IOManager reader(m_comm);
-      reader.read(bp_grp->getDataStore()->getRoot(), path);
+      reader.read(m_bp_grp->getDataStore()->getRoot(), path);
    }
    else
 #endif
    {
-      bp_grp->load(path, protocol);
+      m_bp_grp->load(path, protocol);
    }
 
    // If the data collection created the datastore, it knows the layout of where
@@ -719,32 +741,32 @@ void SidreDataCollection::LoadExternalData(const std::string& path)
    if (m_comm != MPI_COMM_NULL)
    {
       axom::sidre::IOManager reader(m_comm);
-      reader.loadExternalData(bp_grp->getDataStore()->getRoot(), path);
+      reader.loadExternalData(m_bp_grp->getDataStore()->getRoot(), path);
    }
    else
 #endif
    {
-      bp_grp->loadExternalData(path);
+      m_bp_grp->loadExternalData(path);
    }
 }
 
 void SidreDataCollection::UpdateStateFromDS()
 {
-   SetTime( bp_grp->getView("state/time")->getData<double>() );
-   SetCycle( bp_grp->getView("state/cycle")->getData<int>() );
-   SetTimeStep( bp_grp->getView("state/time_step")->getData<double>() );
+   SetTime( m_bp_grp->getView("state/time")->getData<double>() );
+   SetCycle( m_bp_grp->getView("state/cycle")->getData<int>() );
+   SetTimeStep( m_bp_grp->getView("state/time_step")->getData<double>() );
 }
 
 void SidreDataCollection::UpdateStateToDS()
 {
-   bp_grp->getView("state/cycle")->setScalar(GetCycle());
-   bp_grp->getView("state/time")->setScalar(GetTime());
-   bp_grp->getView("state/time_step")->setScalar(GetTimeStep());
+   m_bp_grp->getView("state/cycle")->setScalar(GetCycle());
+   m_bp_grp->getView("state/time")->setScalar(GetTime());
+   m_bp_grp->getView("state/time_step")->setScalar(GetTimeStep());
 
    if (myid == 0)
    {
-      bp_index_grp->getView("state/cycle")->setScalar(GetCycle());
-      bp_index_grp->getView("state/time")->setScalar(time);
+      m_bp_index_grp->getView("state/cycle")->setScalar(GetCycle());
+      m_bp_index_grp->getView("state/time")->setScalar(time);
    }
 }
 
@@ -771,12 +793,12 @@ void SidreDataCollection::Save(const std::string& filename,
 
    std::string file_path = get_file_path(filename);
 
-   sidre::Group * blueprint_indicies_grp = bp_index_grp->getParent();
+   sidre::Group * blueprint_indicies_grp = m_bp_index_grp->getParent();
 #ifdef MFEM_USE_MPI
    if (m_comm != MPI_COMM_NULL)
    {
       axom::sidre::IOManager writer(m_comm);
-      sidre::DataStore *datastore = bp_grp->getDataStore();
+      sidre::DataStore *datastore = m_bp_grp->getDataStore();
       writer.write(datastore->getRoot(), num_procs, file_path, protocol);
       if (myid == 0)
       {
@@ -797,7 +819,7 @@ void SidreDataCollection::Save(const std::string& filename,
 #endif
    {
       // If serial, use sidre group writer.
-      bp_grp->save(file_path, protocol);
+      m_bp_grp->save(file_path, protocol);
 
       blueprint_indicies_grp
       ->save(file_path + ".root", protocol);
@@ -810,7 +832,7 @@ addScalarBasedGridFunction(const std::string &field_name, GridFunction *gf,
                            const std::string &buffer_name,
                            axom::sidre::SidreLength offset)
 {
-   sidre::Group* grp = bp_grp->getGroup("fields/" + field_name);
+   sidre::Group* grp = m_bp_grp->getGroup("fields/" + field_name);
    MFEM_ASSERT(grp != NULL, "field " << field_name << " does not exist");
 
    const int numDofs = gf->FESpace()->GetVSize();
@@ -873,7 +895,7 @@ addVectorBasedGridFunction(const std::string& field_name, GridFunction *gf,
                            const std::string &buffer_name,
                            axom::sidre::SidreLength offset)
 {
-   sidre::Group* grp = bp_grp->getGroup("fields/" + field_name);
+   sidre::Group* grp = m_bp_grp->getGroup("fields/" + field_name);
    MFEM_ASSERT(grp != NULL, "field " << field_name << " does not exist");
 
    const int FLD_SZ = 20;
@@ -962,9 +984,9 @@ addVectorBasedGridFunction(const std::string& field_name, GridFunction *gf,
 void SidreDataCollection::
 RegisterFieldInBPIndex(const std::string& field_name, GridFunction *gf)
 {
-   sidre::Group *bp_field_grp = bp_grp->getGroup("fields/" + field_name);
+   sidre::Group *bp_field_grp = m_bp_grp->getGroup("fields/" + field_name);
    sidre::Group *bp_index_field_grp =
-      bp_index_grp->createGroup("fields/" + field_name);
+      m_bp_index_grp->createGroup("fields/" + field_name);
 
    bp_index_field_grp->createViewString( "path", bp_field_grp->getPathName() );
    bp_index_field_grp->copyView( bp_field_grp->getView("topology") );
@@ -983,7 +1005,7 @@ RegisterFieldInBPIndex(const std::string& field_name, GridFunction *gf)
 void SidreDataCollection::
 DeregisterFieldInBPIndex(const std::string& field_name)
 {
-   sidre::Group * fields_grp = bp_index_grp->getGroup("fields");
+   sidre::Group * fields_grp = m_bp_index_grp->getGroup("fields");
    MFEM_VERIFY(fields_grp->hasGroup(field_name),
                "No field exists in blueprint index with name " << name);
 
@@ -1005,7 +1027,7 @@ void SidreDataCollection::RegisterField(const std::string &field_name,
    }
 
    // Register field_name in the blueprint group.
-   sidre::Group* f = bp_grp->getGroup("fields");
+   sidre::Group* f = m_bp_grp->getGroup("fields");
 
    if (f->hasGroup( field_name ))
    {
@@ -1050,12 +1072,12 @@ void SidreDataCollection::RegisterField(const std::string &field_name,
    bool const isScalarValued = (gf->FESpace()->GetVDim() == 1);
    if (isScalarValued)
    {
-      // Set the View "<bp_grp>/fields/<field_name>/values"
+      // Set the View "<m_bp_grp>/fields/<field_name>/values"
       addScalarBasedGridFunction(field_name, gf, buffer_name, offset);
    }
    else // vector valued
    {
-      // Set the Group "<bp_grp>/fields/<field_name>/values"
+      // Set the Group "<m_bp_grp>/fields/<field_name>/values"
       addVectorBasedGridFunction(field_name, gf, buffer_name, offset);
    }
 
@@ -1077,7 +1099,7 @@ void SidreDataCollection::RegisterAttributeField(const std::string& attr_name,
       "Need to set mesh before registering attributes in SidreDataCollection.");
 
    // Register attr_name in the blueprint group.
-   sidre::Group* f = bp_grp->getGroup("fields");
+   sidre::Group* f = m_bp_grp->getGroup("fields");
    if (f->hasGroup( attr_name ))
    {
       bool isAttr = attr_map.Has(attr_name);
@@ -1107,7 +1129,7 @@ void SidreDataCollection::RegisterAttributeField(const std::string& attr_name,
 
    // Register new attribute array with attr_map
    sidre::View* a =
-      bp_grp->getGroup("fields")->getGroup(attr_name)->getView("values");
+      m_bp_grp->getGroup("fields")->getGroup(attr_name)->getView("values");
    Array<int>* attr = new Array<int>(a->getData<int*>(), a->getNumElements());
 
    attr_map.Register(attr_name, attr, own_data);
@@ -1116,20 +1138,20 @@ void SidreDataCollection::RegisterAttributeField(const std::string& attr_name,
 void SidreDataCollection::RegisterAttributeFieldInBPIndex(
    const std::string& attr_name)
 {
-   const std::string bp_grp_path = bp_grp->getPathName();
+   const std::string m_bp_grp_path = m_bp_grp->getPathName();
 
-   MFEM_ASSERT(bp_grp->getGroup("fields") != NULL,
+   MFEM_ASSERT(m_bp_grp->getGroup("fields") != NULL,
                "Mesh blueprint does not have 'fields' group");
-   MFEM_ASSERT(bp_index_grp->getGroup("fields") != NULL,
+   MFEM_ASSERT(m_bp_index_grp->getGroup("fields") != NULL,
                "Mesh blueprint index does not have 'fields' group");
 
    // get the BP attr group
    sidre::Group* attr_grp =
-      bp_grp->getGroup("fields")->getGroup(attr_name);
+      m_bp_grp->getGroup("fields")->getGroup(attr_name);
 
    // create blueprint index for this attribute
    sidre::Group *bp_index_attr_grp =
-      bp_index_grp->getGroup("fields")->createGroup(attr_name);
+      m_bp_index_grp->getGroup("fields")->createGroup(attr_name);
 
    bp_index_attr_grp->createViewString("path", attr_grp->getPathName() );
    bp_index_attr_grp->copyView( attr_grp->getView("association") );
@@ -1141,7 +1163,7 @@ void SidreDataCollection::DeregisterAttributeField(const std::string& attr_name)
 {
    attr_map.Deregister(name, own_data);
 
-   sidre::Group * attr_grp = bp_grp->getGroup("fields");
+   sidre::Group * attr_grp = m_bp_grp->getGroup("fields");
    MFEM_VERIFY(attr_grp->hasGroup(attr_name),
                "No field exists in blueprint with name " << attr_name);
 
@@ -1165,7 +1187,7 @@ void SidreDataCollection::DeregisterAttributeField(const std::string& attr_name)
 void SidreDataCollection::DeregisterAttributeFieldInBPIndex(
    const std::string& attr_name)
 {
-   sidre::Group * fields_grp = bp_index_grp->getGroup("fields");
+   sidre::Group * fields_grp = m_bp_index_grp->getGroup("fields");
    MFEM_VERIFY(fields_grp->hasGroup(attr_name),
                "No attribute exists in blueprint index with name " << attr_name);
 
@@ -1178,7 +1200,7 @@ void SidreDataCollection::DeregisterAttributeFieldInBPIndex(
 void SidreDataCollection::
 addIntegerAttributeField(const std::string& attr_name, bool is_bdry)
 {
-   sidre::Group* fld_grp = bp_grp->getGroup("fields");
+   sidre::Group* fld_grp = m_bp_grp->getGroup("fields");
    MFEM_ASSERT(fld_grp != NULL, "'fields' group does not exist");
 
    const int num_elem = is_bdry? mesh->GetNBE() : mesh->GetNE();
@@ -1195,7 +1217,7 @@ void SidreDataCollection::DeregisterField(const std::string& field_name)
    // Deregister field_name from field_map.
    DataCollection::DeregisterField(field_name);
 
-   sidre::Group * fields_grp = bp_grp->getGroup("fields");
+   sidre::Group * fields_grp = m_bp_grp->getGroup("fields");
    MFEM_VERIFY(fields_grp->hasGroup(field_name),
                "No field exists in blueprint with name " << field_name);
 

fem/sidredatacollection.hpp

@@ -184,8 +184,8 @@ public:
 
        @param[in] collection_name  Name of the collection used as a file name
                                    when saving
-       @param[in] global_grp       Pointer to the global group in the datastore,
-                                   see the above schematic
+       @param[in] bp_index_grp     Pointer to the blueprint index group in the
+                                   datastore, see the above schematic
        @param[in] domain_grp       Pointer to the domain group in the datastore,
                                    see the above schematic
        @param[in] owns_mesh_data   Does the SidreDC own the mesh vertices?
@@ -196,7 +196,7 @@ public:
        to be set with SetMesh() and fields registered with RegisterField().
     */
    SidreDataCollection(const std::string& collection_name,
-                       axom::sidre::Group * global_grp,
+                       axom::sidre::Group * bp_index_grp,
                        axom::sidre::Group * domain_grp,
                        bool owns_mesh_data = false);
 
@@ -255,6 +255,9 @@ public:
    bool HasAttributeField(const std::string& field_name) const
    { return attr_map.Has(field_name); }
 
+   /** Checks if any rank in the mesh has boundary elements */
+   bool HasBoundaryMesh() const;
+
    /// Set the name of the mesh nodes field.
    /** This name will be used by SetMesh() to register the mesh nodes, if not
        already registered. Also, this method should be called if the mesh nodes
@@ -295,8 +298,8 @@ public:
    void SetGroupPointers(axom::sidre::Group * global_grp,
                          axom::sidre::Group * domain_grp);
 
-   axom::sidre::Group * GetBPGroup() { return bp_grp; }
-   axom::sidre::Group * GetBPIndexGroup() { return bp_index_grp; }
+   axom::sidre::Group * GetBPGroup() { return m_bp_grp; }
+   axom::sidre::Group * GetBPIndexGroup() { return m_bp_index_grp; }
 
    /// Prepare the DataStore for writing
    virtual void PrepareToSave();
@@ -424,11 +427,11 @@ protected:
 private:
    // If the data collection does not own the datastore, it will need pointers
    // to the blueprint and blueprint index group to use.
-   axom::sidre::Group * bp_grp;
-   axom::sidre::Group * bp_index_grp;
+   axom::sidre::Group * m_bp_grp;
+   axom::sidre::Group * m_bp_index_grp;
 
    // This is stored for convenience.
-   axom::sidre::Group * named_bufs_grp;
+   axom::sidre::Group * m_named_bufs_grp;
 
    // Private helper functions
 

general/communication.cpp

@@ -324,6 +324,14 @@ void GroupTopology::Load(istream &in)
    Create(integer_sets, 823);
 }
 
+void GroupTopology::Copy(GroupTopology& copy) const
+{
+   copy.SetComm(MyComm);
+   group_lproc.Copy(copy.group_lproc);
+   groupmaster_lproc.Copy(copy.groupmaster_lproc);
+   lproc_proc.Copy(copy.lproc_proc);
+   group_mgroup.Copy(copy.group_mgroup);
+}
 
 // Initialize the static mpi_type for the specializations of MPITypeMap:
 const MPI_Datatype MPITypeMap<int>::mpi_type = MPI_INT;

general/communication.hpp

@@ -109,6 +109,9 @@ public:
    /// Load the data from a stream.
    void Load(std::istream &in);
 
+   /// Copy
+   void Copy(GroupTopology & copy) const;
+
    virtual ~GroupTopology() {}
 };
 

general/sort_pairs.hpp

@@ -58,6 +58,11 @@ public:
    A one;
    B two;
    C three;
+
+   Triple() { }
+
+   Triple(const A &one, const B &two, const C &three)
+      : one(one), two(two), three(three) { }
 };
 
 /// @brief Lexicographic comparison operator for class Triple.

general/stable3d.cpp

@@ -202,4 +202,22 @@ STable3D::~STable3D ()
    delete [] Rows;
 }
 
+void STable3D::Print(std::ostream & out) const
+{
+   out << NElem << endl;
+   for (int row = 0; row < Size; row++)
+   {
+      STable3DNode *node_p = Rows[row];
+      while (node_p != NULL)
+      {
+         out << row
+             << ' ' << node_p->Column
+             << ' ' << node_p->Floor
+             << ' ' << node_p->Number
+             << endl;
+         node_p = node_p->Prev;
+      }
+   }
+}
+
 }

general/stable3d.hpp

@@ -13,6 +13,7 @@
 #define MFEM_STABLE3D
 
 #include "mem_alloc.hpp"
+#include "../general/globals.hpp"
 
 namespace mfem
 {
@@ -50,6 +51,8 @@ public:
 
    int NumberOfElements() { return NElem; }
 
+   void Print(std::ostream &out = mfem::out) const;
+
    ~STable3D ();
 };
 

linalg/densemat.cpp

@@ -50,6 +50,9 @@ extern "C" void
 dsyev_(char *JOBZ, char *UPLO, int *N, double *A, int *LDA, double *W,
        double *WORK, int *LWORK, int *INFO);
 extern "C" void
+dsygv_ (int *ITYPE, char *JOBZ, char *UPLO, int * N, double *A, int *LDA,
+        double *B, int *LDB, double *W,  double *WORK, int *LWORK, int *INFO);
+extern "C" void
 dgesvd_(char *JOBU, char *JOBVT, int *M, int *N, double *A, int *LDA,
         double *S, double *U, int *LDU, double *VT, int *LDVT, double *WORK,
         int *LWORK, int *INFO);
@@ -858,9 +861,7 @@ void DenseMatrix::FNorm(double &scale_factor, double &scaled_fnorm2) const
 
 void dsyevr_Eigensystem(DenseMatrix &a, Vector &ev, DenseMatrix *evect)
 {
-
 #ifdef MFEM_USE_LAPACK
-
    ev.SetSize(a.Width());
 
    char      JOBZ     = 'N';
@@ -1020,9 +1021,7 @@ void dsyevr_Eigensystem(DenseMatrix &a, Vector &ev, DenseMatrix *evect)
 
 void dsyev_Eigensystem(DenseMatrix &a, Vector &ev, DenseMatrix *evect)
 {
-
 #ifdef MFEM_USE_LAPACK
-
    int   N      = a.Width();
    char  JOBZ   = 'N';
    char  UPLO   = 'U';
@@ -1070,7 +1069,6 @@ void dsyev_Eigensystem(DenseMatrix &a, Vector &ev, DenseMatrix *evect)
 
    delete [] WORK;
    if (evect == NULL) { delete [] A; }
-
 #endif
 }
 
@@ -1089,6 +1087,79 @@ void DenseMatrix::Eigensystem(Vector &ev, DenseMatrix *evect)
 #endif
 }
 
+void dsygv_Eigensystem(DenseMatrix &a, DenseMatrix &b, Vector &ev,
+                       DenseMatrix *evect)
+{
+#ifdef MFEM_USE_LAPACK
+   int   N      = a.Width();
+   int   ITYPE  = 1;
+   char  JOBZ   = 'N';
+   char  UPLO   = 'U';
+   int   LDA    = N;
+   int   LDB    = N;
+   int   LWORK  = -1; /* query optimal workspace size */
+   int   INFO;
+
+   ev.SetSize(N);
+
+   double *A    = NULL;
+   double *B    = new double[N*N];
+   double *W    = ev.GetData();
+   double *WORK = NULL;
+   double  QWORK;
+
+   if (evect)
+   {
+      JOBZ = 'V';
+      evect->SetSize(N);
+      A = evect->Data();
+   }
+   else
+   {
+      A = new double[N*N];
+   }
+
+   int hw = a.Height() * a.Width();
+   double *a_data = a.Data();
+   double *b_data = b.Data();
+   for (int i = 0; i < hw; i++)
+   {
+      A[i] = a_data[i];
+      B[i] = b_data[i];
+   }
+
+   dsygv_(&ITYPE, &JOBZ, &UPLO, &N, A, &LDA, B, &LDB, W, &QWORK, &LWORK, &INFO);
+
+   LWORK = (int) QWORK;
+   WORK = new double[LWORK];
+
+   dsygv_(&ITYPE, &JOBZ, &UPLO, &N, A, &LDA, B, &LDB, W, WORK, &LWORK, &INFO);
+
+   if (INFO != 0)
+   {
+      mfem::err << "dsygv_Eigensystem: DSYGV error code: " << INFO << endl;
+      mfem_error();
+   }
+
+   delete [] WORK;
+   delete [] B;
+   if (evect == NULL) { delete [] A; }
+#endif
+}
+
+void DenseMatrix::Eigensystem(DenseMatrix &b, Vector &ev,
+                              DenseMatrix *evect)
+{
+#ifdef MFEM_USE_LAPACK
+
+   dsygv_Eigensystem(*this, b, ev, evect);
+
+#else
+
+   mfem_error("DenseMatrix::Eigensystem for generalized eigenvalues");
+#endif
+}
+
 void DenseMatrix::SingularValues(Vector &sv) const
 {
 #ifdef MFEM_USE_LAPACK

linalg/densemat.hpp

@@ -31,6 +31,8 @@ private:
 
    void Eigensystem(Vector &ev, DenseMatrix *evect = NULL);
 
+   void Eigensystem(DenseMatrix &b, Vector &ev, DenseMatrix *evect = NULL);
+
    // Auxiliary method used in FNorm2() and FNorm()
    void FNorm(double &scale_factor, double &scaled_fnorm2) const;
 
@@ -216,15 +218,32 @@ public:
    /// Compute the square of the Frobenius norm of the matrix
    double FNorm2() const { double s, n2; FNorm(s, n2); return s*s*n2; }
 
+   /// Compute eigenvalues of A x = ev x where A = *this
    void Eigenvalues(Vector &ev)
    { Eigensystem(ev); }
 
+   /// Compute eigenvalues and eigenvectors of A x = ev x where A = *this
    void Eigenvalues(Vector &ev, DenseMatrix &evect)
    { Eigensystem(ev, &evect); }
 
+   /// Compute eigenvalues and eigenvectors of A x = ev x where A = *this
    void Eigensystem(Vector &ev, DenseMatrix &evect)
    { Eigensystem(ev, &evect); }
 
+   /** Compute generalized eigenvalues and eigenvectors of A x = ev B x,
+       where A = *this */
+   void Eigenvalues(DenseMatrix &b, Vector &ev)
+   { Eigensystem(b, ev); }
+
+   /// Compute generalized eigenvalues of A x = ev B x, where A = *this
+   void Eigenvalues(DenseMatrix &b, Vector &ev, DenseMatrix &evect)
+   { Eigensystem(b, ev, &evect); }
+
+   /** Compute generalized eigenvalues and eigenvectors of A x = ev B x,
+       where A = *this */
+   void Eigensystem(DenseMatrix &b, Vector &ev, DenseMatrix &evect)
+   { Eigensystem(b, ev, &evect); }
+
    void SingularValues(Vector &sv) const;
    int Rank(double tol) const;
 

linalg/petsc.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2016, Lawrence Livermore National Security, LLC. Produced at
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
 // the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
 // reserved. See file COPYRIGHT for details.
 //
@@ -834,7 +834,7 @@ void PetscParMatrix::ConvertOperator(MPI_Comm comm, const Operator &op, Mat* A,
                if (tid == PETSC_MATIS && needl2l)
                {
                   PetscContainer c;
-                  ierr = PetscObjectQuery((PetscObject)mats[i*nc+j],"__mfem_l2l",
+                  ierr = PetscObjectQuery((PetscObject)mats[i*nc+j],"_MatIS_PtAP_l2l",
                                           (PetscObject*)&c);
                   PCHKERRQ(mats[i*nc+j],ierr);
                   // special case for block operators: the local Vdofs should be
@@ -871,7 +871,7 @@ void PetscParMatrix::ConvertOperator(MPI_Comm comm, const Operator &op, Mat* A,
          ierr = PetscContainerSetPointer(c,vmatsl2l); PCHKERRQ(c,ierr);
          ierr = PetscContainerSetUserDestroy(c,__mfem_matarray_container_destroy);
          PCHKERRQ(c,ierr);
-         ierr = PetscObjectCompose((PetscObject)(*A),"__mfem_l2l",(PetscObject)c);
+         ierr = PetscObjectCompose((PetscObject)(*A),"_MatIS_PtAP_l2l",(PetscObject)c);
          PCHKERRQ((*A),ierr);
          ierr = PetscContainerDestroy(&c); CCHKERRQ(comm,ierr);
       }
@@ -1162,7 +1162,7 @@ PetscParMatrix * RAP(PetscParMatrix *Rt, PetscParMatrix *A, PetscParMatrix *P)
          ierr = PetscContainerSetPointer(c,vmatsl2l); PCHKERRQ(c,ierr);
          ierr = PetscContainerSetUserDestroy(c,__mfem_matarray_container_destroy);
          PCHKERRQ(c,ierr);
-         ierr = PetscObjectCompose((PetscObject)B,"__mfem_l2l",(PetscObject)c);
+         ierr = PetscObjectCompose((PetscObject)B,"_MatIS_PtAP_l2l",(PetscObject)c);
          PCHKERRQ(B,ierr);
          ierr = PetscContainerDestroy(&c); PCHKERRQ(B,ierr);
       }
@@ -2266,7 +2266,7 @@ void PetscBDDCSolver::BDDCSolverConstructor(const PetscBDDCSolverParams &opts)
    {
       PetscContainer c;
 
-      ierr = PetscObjectQuery((PetscObject)pA,"__mfem_l2l",(PetscObject*)&c);
+      ierr = PetscObjectQuery((PetscObject)pA,"_MatIS_PtAP_l2l",(PetscObject*)&c);
       MFEM_VERIFY(c,"Local-to-local PETSc container not present");
       ierr = PetscContainerGetPointer(c,(void**)&l2l); PCHKERRQ(c,ierr);
    }

linalg/petsc.hpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2016, Lawrence Livermore National Security, LLC. Produced at
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
 // the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
 // reserved. See file COPYRIGHT for details.
 //

linalg/sparsemat.cpp

@@ -184,6 +184,34 @@ SparseMatrix::SparseMatrix(const SparseMatrix &mat, bool copy_graph)
    isSorted = mat.isSorted;
 }
 
+SparseMatrix::SparseMatrix(const Vector &v)
+   : AbstractSparseMatrix(v.Size(), v.Size())
+   , Rows(NULL)
+   , ColPtrJ(NULL)
+   , ColPtrNode(NULL)
+   , ownGraph(true)
+   , ownData(true)
+   , isSorted(true)
+{
+#ifdef MFEM_USE_MEMALLOC
+   NodesMem = NULL;
+#endif
+   I = new int[height + 1];
+   J = new int[height];
+   A = new double[height];
+
+   for (int i = 0; i <= height; i++)
+   {
+      I[i] = i;
+   }
+
+   for (int r=0; r<height; r++)
+   {
+      J[r] = r;
+      A[r] = v[r];
+   }
+}
+
 SparseMatrix& SparseMatrix::operator=(const SparseMatrix &rhs)
 {
    Clear();
@@ -483,6 +511,37 @@ void SparseMatrix::GetDiag(Vector & d) const
    }
 }
 
+/// Produces a DenseMatrix from a SparseMatrix
+DenseMatrix *SparseMatrix::ToDenseMatrix() const
+{
+   int num_rows = this->Height();
+   int num_cols = this->Width();
+
+   DenseMatrix * B = new DenseMatrix(num_rows, num_cols);
+
+   this->ToDenseMatrix(*B);
+
+   return B;
+}
+
+/// Produces a DenseMatrix from a SparseMatrix
+void SparseMatrix::ToDenseMatrix(DenseMatrix & B) const
+{
+   B.SetSize(height, width);
+   B = 0.0;
+
+   for (int r=0; r<height; r++)
+   {
+      const int    * col = this->GetRowColumns(r);
+      const double * val = this->GetRowEntries(r);
+
+      for (int cj=0; cj<this->RowSize(r); cj++)
+      {
+         B(r, col[cj]) = val[cj];
+      }
+   }
+}
+
 void SparseMatrix::Mult(const Vector &x, Vector &y) const
 {
    y = 0.0;
@@ -1170,7 +1229,8 @@ void SparseMatrix::EliminateCol(int col, DiagonalPolicy dpolicy)
    }
 }
 
-void SparseMatrix::EliminateCols(const Array<int> &cols, Vector *x, Vector *b)
+void SparseMatrix::EliminateCols(const Array<int> &cols, const Vector *x,
+                                 Vector *b)
 {
    if (Rows == NULL)
    {
@@ -2986,6 +3046,14 @@ SparseMatrix *Mult (const SparseMatrix &A, const SparseMatrix &B,
    return C;
 }
 
+SparseMatrix * TransposeMult(const SparseMatrix &A, const SparseMatrix &B)
+{
+   SparseMatrix *At  = Transpose(A);
+   SparseMatrix *AtB = Mult(*At, B);
+   delete At;
+   return AtB;
+}
+
 SparseMatrix *MultAbstractSparseMatrix (const AbstractSparseMatrix &A,
                                         const AbstractSparseMatrix &B)
 {
@@ -3106,6 +3174,19 @@ DenseMatrix *RAP (const SparseMatrix &A, DenseMatrix &P)
    return _RAP;
 }
 
+DenseMatrix *RAP(DenseMatrix &A, const SparseMatrix &P)
+{
+   SparseMatrix *R  = Transpose(P);
+   DenseMatrix  *RA = Mult(*R, A);
+   DenseMatrix   AtP(*RA, 't');
+   delete RA;
+   DenseMatrix  *RAtP = Mult(*R, AtP);
+   delete R;
+   DenseMatrix * _RAP = new DenseMatrix(*RAtP, 't');
+   delete RAtP;
+   return _RAP;
+}
+
 SparseMatrix *RAP (const SparseMatrix &A, const SparseMatrix &R,
                    SparseMatrix *ORAP)
 {
@@ -3256,6 +3337,130 @@ SparseMatrix * Add(Array<SparseMatrix *> & Ai)
    return result;
 }
 
+/// B += alpha * A
+void Add(const SparseMatrix &A,
+         double alpha, DenseMatrix &B)
+{
+   for (int r = 0; r < B.Height(); r++)
+   {
+      const int    * colA = A.GetRowColumns(r);
+      const double * valA = A.GetRowEntries(r);
+      for (int i=0; i<A.RowSize(r); i++)
+      {
+         B(r, colA[i]) += alpha * valA[i];
+      }
+   }
+}
+
+/// Produces a block matrix with blocks A_{ij}*B
+DenseMatrix *OuterProduct(const DenseMatrix &A, const DenseMatrix &B)
+{
+   int mA = A.Height(), nA = A.Width();
+   int mB = B.Height(), nB = B.Width();
+
+   DenseMatrix *C = new DenseMatrix(mA * mB, nA * nB);
+   *C = 0.0;
+   for (int i=0; i<mA; i++)
+   {
+      for (int j=0; j<nA; j++)
+      {
+         C->AddMatrix(A(i,j), const_cast<DenseMatrix&>(B), i * mB, j * nB);
+      }
+   }
+   return C;
+}
+
+/// Produces a block matrix with blocks A_{ij}*B
+SparseMatrix *OuterProduct(const DenseMatrix &A, const SparseMatrix &B)
+{
+   int mA = A.Height(), nA = A.Width();
+   int mB = B.Height(), nB = B.Width();
+
+   SparseMatrix *C = new SparseMatrix(mA * mB, nA * nB);
+
+   for (int i=0; i<mA; i++)
+   {
+      for (int j=0; j<nA; j++)
+      {
+         for (int r=0; r<mB; r++)
+         {
+            const int    * colB = B.GetRowColumns(r);
+            const double * valB = B.GetRowEntries(r);
+
+            for (int cj=0; cj<B.RowSize(r); cj++)
+            {
+               C->Set(i * mB + r, j * nB + colB[cj], A(i,j) * valB[cj]);
+            }
+         }
+      }
+   }
+   C->Finalize();
+
+   return C;
+}
+
+/// Produces a block matrix with blocks A_{ij}*B
+SparseMatrix *OuterProduct(const SparseMatrix &A, const DenseMatrix &B)
+{
+   int mA = A.Height(), nA = A.Width();
+   int mB = B.Height(), nB = B.Width();
+
+   SparseMatrix *C = new SparseMatrix(mA * mB, nA * nB);
+
+   for (int r=0; r<mA; r++)
+   {
+      const int    * colA = A.GetRowColumns(r);
+      const double * valA = A.GetRowEntries(r);
+
+      for (int aj=0; aj<A.RowSize(r); aj++)
+      {
+         for (int i=0; i<mB; i++)
+         {
+            for (int j=0; j<nB; j++)
+            {
+               C->Set(r * mB + i, colA[aj] * nB + j, valA[aj] * B(i, j));
+            }
+         }
+      }
+   }
+   C->Finalize();
+
+   return C;
+}
+
+/// Produces a block matrix with blocks A_{ij}*B
+SparseMatrix *OuterProduct(const SparseMatrix &A, const SparseMatrix &B)
+{
+   int mA = A.Height(), nA = A.Width();
+   int mB = B.Height(), nB = B.Width();
+
+   SparseMatrix *C = new SparseMatrix(mA * mB, nA * nB);
+
+   for (int ar=0; ar<mA; ar++)
+   {
+      const int    * colA = A.GetRowColumns(ar);
+      const double * valA = A.GetRowEntries(ar);
+
+      for (int aj=0; aj<A.RowSize(ar); aj++)
+      {
+         for (int br=0; br<mB; br++)
+         {
+            const int    * colB = B.GetRowColumns(br);
+            const double * valB = B.GetRowEntries(br);
+
+            for (int bj=0; bj<B.RowSize(br); bj++)
+            {
+               C->Set(ar * mB + br, colA[aj] * nB + colB[bj],
+                      valA[aj] * valB[bj]);
+            }
+         }
+      }
+   }
+   C->Finalize();
+
+   return C;
+}
+
 void SparseMatrix::Swap(SparseMatrix &other)
 {
    mfem::Swap(width, other.width);