Include "mpi-ext.h" only if using OpenMPI.

pointers in MPI calls, regardless of the result of the
CUDA-aware MPI check.

.gitignore

@@ -112,6 +112,15 @@ examples/petsc/deformed.*
 examples/petsc/velocity.*
 examples/petsc/elastic_energy.*
 
+examples/pumi/ex1
+examples/pumi/ex[126]p
+
+examples/pumi/refined.mesh
+examples/pumi/sol.gf
+examples/pumi/mesh.*
+examples/pumi/sol.*
+examples/pumi/displaced.mesh
+
 miniapps/electromagnetics/volta
 miniapps/electromagnetics/tesla
 miniapps/electromagnetics/maxwell

CHANGELOG

@@ -8,11 +8,23 @@
                                http://mfem.org
 
 
-Version 3.3.3 (development)
+Version 3.4.1 (development)
 ===========================
+- Added support for reading linear and quadratic 2D quadrilateral and triangular
+  Cubit meshes.
+
+
+Version 3.4, released on May 29, 2018
+=====================================
+
+More general and efficient mesh adaptivity
+------------------------------------------
+- Added support for PUMI, the Parallel Unstructured Mesh Infrastructure from
+  https://scorec.rpi.edu/pumi. PUMI is an unstructured, distributed mesh data
+  management system that is capable of handling general non-manifold models and
+  effectively supports automated adaptive analysis. PUMI enables for the first
+  time support for parallel unstructured modifications of MFEM meshes.
 
-More efficient non-conforming adaptive mesh refinement
-------------------------------------------------------
 - Significantly reduced MPI communication in the construction of the parallel
   prolongation matrix in ParFiniteElementSpace, for much improved parallel
   scaling of non-conforming AMR on hundreds of thousands of MPI tasks. The
@@ -81,6 +93,11 @@ New and updated examples and miniapps
   NURBS meshes in the miniapps/nurbs directory. Currently the directory contains
   variable order NURBS versions of examples 1, 1p and 11p.
 
+- Added PUMI versions of examples ex1, ex1p, ex2 and ex6p in a new examples/pumi
+  directory. The new examples demonstrate the PUMI APIs for parallel and serial
+  mesh loading (ex1 and ex1p), applying BCs using classification (ex2), and
+  performing parallel mesh adaptation (ex6p).
+
 - Added two new miniapps related to DataCollection I/O in miniapps/tools:
   load-dc.cpp can be used to visualize fields saved via DataCollection classes;
   convert-dc.cpp demonstrates how to convert between MFEM's different concrete

CMakeLists.txt

@@ -45,7 +45,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 3.3.3)
+set(${PROJECT_NAME}_VERSION 3.4.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -139,7 +139,7 @@ if (MFEM_USE_MPI)
     set(PETSC_INCLUDE_DIRS ${PETSC_INCLUDES})
   endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU PETSC STRUMPACK)
+  set(PKGS_NEED_MPI SUPERLU PETSC STRUMPACK PUMI)
   foreach(PKG IN LISTS PKGS_NEED_MPI)
     if (MFEM_USE_${PKG})
       message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -246,6 +246,22 @@ if (MFEM_USE_SIDRE)
   find_package(Axom REQUIRED Sidre SLIC axom_utils)
 endif()
 
+# PUMI
+if (MFEM_USE_PUMI)
+  # If PUMI_DIR was specified, only link to that directory,
+  # i.e. don't link to another installation in /usr/lib by mistake
+  find_package(SCOREC 2.1.0 REQUIRED OPTIONAL_COMPONENTS gmi_sim
+    CONFIG PATHS ${PUMI_DIR} NO_DEFAULT_PATH)
+  if (SCOREC_FOUND)
+    # Define a header file with the MFEM_USE_SIMMETRIX preprocessor variable
+    set(MFEM_USE_SIMMETRIX ${SCOREC_gmi_sim_FOUND})
+    set(PUMI_FOUND ${SCOREC_FOUND})
+    get_target_property(PUMI_INCLUDE_DIRS
+      SCOREC::apf INTERFACE_INCLUDE_DIRECTORIES)
+    set(PUMI_LIBRARIES SCOREC::core)
+  endif()
+endif()
+
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -270,8 +286,8 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
-    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT GECKO GNUTLS NETCDF MPFR POSIXCLOCKS
-    MFEMBacktrace ZLIB)
+    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT GECKO GNUTLS NETCDF MPFR PUMI
+    POSIXCLOCKS MFEMBacktrace ZLIB)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")

CONTRIBUTING.md

@@ -1,3 +1,15 @@
+<p align="center">
+<a href="http://mfem.org/"><img alt="mfem" src="http://mfem.org/img/logo-300.png"></a>
+</p>
+
+<p align="center">
+<a href="https://github.com/mfem/mfem/blob/master/COPYRIGHT"><img alt="License" src="https://img.shields.io/badge/License-LGPL--2.1-brightgreen.svg"></a>
+<a href="https://travis-ci.org/mfem/mfem"><img alt="Build Status" src="https://travis-ci.org/mfem/mfem.svg?branch=master"></a>
+<a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
+<a href="http://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
+</p>
+
+
 # How to Contribute
 
 The MFEM team welcomes contributions at all levels: bugfixes; code
@@ -16,6 +28,7 @@ See the [Quick Summary](#quick-summary) section for the main highlights of our
 GitHub workflow. For more details, consult the following sections and refer
 back to them before issuing pull requests:
 
+- [Code Overview](#code-overview)
 - [GitHub Workflow](#github-workflow)
   - [MFEM Organization](#mfem-organization)
   - [New Feature Development](#new-feature-development)
@@ -53,6 +66,94 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
 - Don't hesitate to [contact us](#contact-information) if you have any questions.
 
 
+### Code Overview
+
+- The MFEM library uses object-orient design principles which reflect, in code,
+  the independent mathematical concepts of meshing, linear algebra and finite
+  element spaces and operators.
+
+- The MFEM source code has the following structure:
+  ```
+  .
+  ├── config
+  │   └── cmake
+  │       └── modules
+  ├── data
+  ├── doc
+  │   └── web
+  │       └── examples
+  ├── examples
+  │   ├── petsc
+  │   ├── pumi
+  │   └── sundials
+  ├── fem
+  ├── general
+  ├── linalg
+  ├── mesh
+  └── miniapps
+      ├── common
+      ├── electromagnetics
+      ├── meshing
+      ├── nurbs
+      ├── performance
+      └── tools
+  ```
+
+- The main directories are `fem/`, `mesh/` and `linalg/` containing the C++
+  classes implementing the finite element, mesh and linear algebra concepts
+  respectively.
+
+- The main mesh classes are:
+  + [`Mesh`](http://mfem.github.io/doxygen/html/classmfem_1_1Mesh.html)
+  + [`NCMesh`](http://mfem.github.io/doxygen/html/classmfem_1_1NCMesh.html)
+  + [`Element`](http://mfem.github.io/doxygen/html/classmfem_1_1Element.html)
+  + [`ElementTransformation`](http://mfem.github.io/doxygen/html/classmfem_1_1ElementTransformation.html)
+
+- The main finite element classes are:
+  + [`FiniteElement`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementCollection`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementSpace`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElementSpace.html)
+  + [`GridFunction`](http://mfem.github.io/doxygen/html/classmfem_1_1GridFunction.html)
+  + [`BilinearFormIntegrator`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html)
+  + [`LinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1MixedBilinearForm.html)
+
+- The main linear algebra classes and sources are
+  + [`Operator`](http://mfem.github.io/doxygen/html/classmfem_1_1Operator.html) and [`BilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html)
+  + [`Vector`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearForm.html)
+  + [`DenseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1SparseMatrix.html)
+  + Sparse [smoothers](http://mfem.github.io/doxygen/html/sparsesmoothers_8hpp.html) and linear [solvers](http://mfem.github.io/doxygen/html/solvers_8hpp.html)
+
+- Parallel MPI objects in MFEM inherit their serial counterparts, so a parallel
+  mesh for example is just a serial mesh on each task plus the information on
+  shared geometric entities between different tasks. The parallel source files
+  have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
+
+- The main parallel classes are
+  + [`ParMesh`](http://mfem.github.io/doxygen/html/solvers_8hpp.html)
+  + [`ParNCMesh`](http://mfem.github.io/doxygen/html/classmfem_1_1ParMesh.html)
+  + [`ParFiniteElementSpace`](http://mfem.github.io/doxygen/html/classmfem_1_1ParFiniteElementSpace.html)
+  + [`ParGridFunction`](http://mfem.github.io/doxygen/html/classmfem_1_1ParGridFunction.html)
+  + [`ParBilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1ParLinearForm.html)
+  + [`HypreParMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParVector.html)
+  + [`HypreSolver`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreSolver.html) and other [hypre classes](http://mfem.github.io/doxygen/html/hypre_8hpp.html)
+
+- The `general/` directory contains C++ classes that serve as utilities for
+  communication, error handling, arrays, (Boolean) tables, timing, etc.
+
+- The `config/` directory contains build-related files, both for the plain
+  Makefile and the CMake build options.
+
+- The `doc/` directory contains configuration for the Doxygen code documentation
+  that can either be build locally, or browsed online at
+  http://mfem.github.io/doxygen/html/index.html.
+
+- The `data/` directory contains a collection of small mesh files, that are used
+  in the simple example codes and more fully-featured mini applications in the
+  `examples/` and `miniapps/` directories.
+
+- See also the [code overview](http://mfem.org/code-overview/) section on the
+  MFEM website.
+
 ## GitHub Workflow
 
 The GitHub organization, https://github.com/mfem, is the main developer hub for
@@ -122,7 +223,7 @@ will allow us to reach you directly with project announcements.
   # Work on "feature-dev", add local commits
   # ...
 
-  # One time only) push the branch to github and setup your local
+  # (One time only) push the branch to github and setup your local
   # branch to track the github branch (for "git pull"):
   git push -u origin feature-dev
 
@@ -332,7 +433,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
     - [ ] `CHANGELOG`
     - [ ] `makefile`
     - [ ] `CMakeLists.txt`
-    - [ ] `doc/CodeDocumentation.conf`
+    - [ ] `doc/CodeDocumentation.conf.in`
 - [ ] (LLNL only) Make sure all `README.html` files in the source repo are up to date.
 - [ ] Tag the repository:
 

INSTALL

@@ -32,6 +32,9 @@ following package managers:
 - OpenHPC, http://openhpc.community
 - Homebrew/Science, https://github.com/Homebrew/homebrew-science
 
+We also recommend downloading and building the MFEM-based GLVis visualization
+tool which can be used to visualize the meshes and solution in MFEM's examples
+and miniapps. See http://glvis.org and http://mfem.org/building.
 
 Quick start with GNU make
 =========================
@@ -352,6 +355,13 @@ MFEM_USE_GZSTREAM = YES/NO
    before attempting to use it with MFEM.
    When enabled, this option uses the ZLIB_* library options, see below.
 
+MFEM_USE_PUMI = YES/NO
+   Enable the usage of PUMI (https://scorec.rpi.edu/pumi/) in MFEM. The Parallel
+   Unstructured Mesh Infrastructure (PUMI) is an unstructured, distributed mesh
+   data management system that is capable of handling general non-manifold
+   models and effectively supports automated adaptive analysis. PUMI enables
+   support for parallel unstructured mesh modifications in MFEM.
+
 MFEM_BUILD_TAG = (any value)
    An optional tag to characterize the build. Exported to config/config.mk.
    Can be used to identify the MFEM build from other makefiles.
@@ -461,6 +471,10 @@ The specific libraries and their options are:
        https://support.hdfgroup.org/HDF5 (HDF5)
   Options: CONDUIT_OPT, CONDUIT_LIB.
 
+- PUMI, used when MFEM_USE_PUMI = YES.
+  URL: https://scorec.rpi.edu/pumi
+  Options: PUMI_OPT, PUMI_LIB.
+
 - MPFR (optional), used when MFEM_USE_MPFR = YES.
   URL: http://mpfr.org, it depends on the GMP library: https://gmplib.org
   Options: MPFR_OPT, MPFR_LIB.
@@ -593,6 +607,7 @@ MFEM_USE_GNUTLS
 MFEM_USE_NETCDF
 MFEM_USE_MPFR
 MFEM_USE_GZSTREAM
+MFEM_USE_PUMI
 
 The following options are CMake specific:
 
@@ -638,6 +653,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - MPFR
  - LIBUNWIND
  - POSIXCLOCKS
+ - PUMI
 
 The following built-in CMake packages are also used:
 

README

@@ -12,11 +12,15 @@ to enable the research and development of scalable finite element discretization
 and solver algorithms through general finite element abstractions, accurate and
 flexible visualization, and tight integration with the hypre library.
 
-For building instructions, see the file INSTALL, or type "make help". Copyright
-information and licensing restrictions can be found in the file COPYRIGHT.
+* For building instructions, see the file INSTALL, or type "make help".
 
-The best starting point for new users interested in MFEM's features is the
-interactive documentation in examples/README.html.
+* Copyright and licensing information can be found in the file COPYRIGHT.
+
+* The best starting point for new users interested in MFEM's features is the
+  interactive documentation in examples/README.html.
+
+* Developers interested in contributing to the library, should read the
+  instructions and documentation in the CONTRIBUTING.md file.
 
 Conceptually, MFEM can be viewed as a finite element toolbox that provides the
 building blocks for developing finite element algorithms in a manner similar to
@@ -56,8 +60,8 @@ time integrators, etc.
 For examples of using MFEM, see the examples/ and miniapps/ directories, as well
 as the OpenGL visualization tool GLVis which is available at http://glvis.org.
 
-This project is released under the LGPL v2.1 license. See LICENSE file for full
-details.
+This project is released under the LGPL v2.1 license with static linking
+exception. See files COPYRIGHT and LICENSE file for full details.
 
 LLNL Release Number: LLNL-CODE-443211
 DOI: 10.11578/dc.20171025.1248

backends/base/array.hpp

@@ -109,10 +109,12 @@ public:
    PLayout &GetLayout() const { return *layout; }
 
    /// TODO
+   /// Note: we cannot use static_cast for class PArray.
    template <typename derived_t>
    derived_t &As() { return dynamic_cast<derived_t&>(*this); }
 
    /// TODO
+   /// Note: we cannot use static_cast for class PArray.
    template <typename derived_t>
    const derived_t &As() const { return dynamic_cast<const derived_t&>(*this); }
 

backends/base/engine.cpp

@@ -19,9 +19,29 @@
 namespace mfem
 {
 
-DFiniteElementSpace Engine::MakeFESpace(FiniteElementSpace &fes) const
+Engine::Engine(Backend *b, int n_mem, int n_workers)
+   : backend(b),
+#ifdef MFEM_USE_MPI
+     comm(MPI_COMM_NULL),
+#endif
+     num_mem_res(n_mem),
+     num_workers(n_workers),
+     memory_resources(new MemoryResource*[num_mem_res]()),
+     workers_weights(new double[num_workers]()),
+     workers_mem_res(new int[num_workers]())
 {
-   return DFiniteElementSpace(new PFiniteElementSpace(*this, fes));
+   // Note: all arrays are value-initialized with zeros.
+}
+
+Engine::~Engine()
+{
+   delete [] workers_mem_res;
+   delete [] workers_weights;
+   for (int i = 0; i < num_mem_res; i++)
+   {
+      delete memory_resources[i];
+   }
+   delete [] memory_resources;
 }
 
 } // namespace mfem

backends/base/engine.hpp

@@ -65,29 +65,10 @@ protected:
 
 public:
    /// TODO: doxygen
-   Engine(Backend *b, int n_mem, int n_workers)
-      : backend(b),
-#ifdef MFEM_USE_MPI
-        comm(MPI_COMM_NULL),
-#endif
-        num_mem_res(n_mem),
-        num_workers(n_workers),
-        memory_resources(new MemoryResource*[num_mem_res]()),
-        workers_weights(new double[num_workers]()),
-        workers_mem_res(new int[num_workers]())
-   { /* Note: all arrays are value-initialized with zeros. */ }
+   Engine(Backend *b, int n_mem, int n_workers);
 
    /// TODO: doxygen
-   virtual ~Engine()
-   {
-      delete [] workers_mem_res;
-      delete [] workers_weights;
-      for (int i = 0; i < num_mem_res; i++)
-      {
-         delete memory_resources[i];
-      }
-      delete [] memory_resources;
-   }
+   virtual ~Engine();
 
 
    /**
@@ -120,11 +101,11 @@ public:
 
    /// TODO
    template <typename derived_t>
-   derived_t &As() { *util::As<derived_t>(this); }
+   derived_t &As() { return *util::As<derived_t>(this); }
 
    /// TODO
    template <typename derived_t>
-   const derived_t &As() const { *util::As<const derived_t>(this); }
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
 
 
    // TODO: Error handling ... handle errors at the Engine level, at the class
@@ -139,10 +120,10 @@ public:
    // TODO: Asynchronous execution in this class ...
 
    /// Allocate and return a new layout for the given @a size.
-   /** The layout decomposition is determined automatically by the Engine using
-       a deterministic algorithm: calls to this method with the same @a size
-       will produce the same result, as long as the Engine remains unmodified
-       between the calls.
+   /** The layout decomposition (in the case of multiple workers) is determined
+       automatically by the Engine using a deterministic algorithm: calls to
+       this method with the same @a size will produce the same result, as long
+       as the Engine remains unmodified between the calls.
 
        The returned object is allocated with operator new and must be
        deallocated by the caller.
@@ -180,7 +161,7 @@ public:
                               int type_id = ScalarId<double>::value) const = 0;
 
    /// TODO: doxygen
-   virtual DFiniteElementSpace MakeFESpace(FiniteElementSpace &fes) const;
+   virtual DFiniteElementSpace MakeFESpace(FiniteElementSpace &fes) const = 0;
 
    /// TODO: doxygen
    virtual DBilinearForm MakeBilinearForm(BilinearForm &bf) const = 0;

backends/base/fespace.hpp

@@ -22,6 +22,7 @@ namespace mfem
 {
 
 class FiniteElementSpace;
+class QuadratureSpace;
 
 /// TODO: doxygen
 class PFiniteElementSpace : public RefCounted
@@ -30,11 +31,11 @@ protected:
    /// Engine with shared ownership
    SharedPtr<const Engine> engine;
    /// Not owned.
-   FiniteElementSpace *fes;
+   mfem::FiniteElementSpace *fes;
 
 public:
    /// TODO: doxygen
-   PFiniteElementSpace(const Engine &e, FiniteElementSpace &fespace)
+   PFiniteElementSpace(const Engine &e, mfem::FiniteElementSpace &fespace)
       : engine(&e), fes(&fespace) { }
 
    /// Virtual destructor
@@ -43,7 +44,8 @@ public:
    /// Get the associated engine
    const Engine &GetEngine() const { return *engine; }
 
-   mfem::FiniteElementSpace* GetFESpace() const { return fes; }
+   /// Return the associated mfem::FiniteElementSpace
+   mfem::FiniteElementSpace *GetFESpace() const { return fes; }
 
    /// TODO
    template <typename derived_t>
@@ -52,6 +54,41 @@ public:
    /// TODO
    template <typename derived_t>
    const derived_t &As() const { return *util::As<const derived_t>(this); }
+
+
+   /**
+       @name Virtual interface: finite element space functionality
+    */
+   ///@{
+
+   /// TODO
+   /** Return the operator mapping T-vectors to L-vectors. If a NULL pointer is
+       returned then the mapping is the idenity. */
+   virtual const mfem::Operator *GetProlongationOperator() const = 0;
+
+   /// TODO
+   /** Return the operator mapping L-vectors to T-vectors that extracts the
+       subset of all true dofs, i.e. no assembly is performed. If a NULL pointer
+       is returned then the mapping is the idenity. */
+   virtual const mfem::Operator *GetRestrictionOperator() const = 0;
+
+   /// TODO
+   /** Return the operator mapping L-vectors to Q-vectors that evaluates the
+       values of a GridFunction as a QuadratureFunction on the given
+       QuadratureSpace. If the returned pointer is NULL, then the mapping is the
+       identity. */
+   virtual const mfem::Operator *GetInterpolationOperator(
+      const mfem::QuadratureSpace &qspace) const = 0;
+
+   /// TODO
+   /** Return the operator mapping L-vectors to Q-vectors that evaluates the
+       _reference element_ gradients of a GridFunction as a QuadratureFunction
+       on the given QuadratureSpace. */
+   virtual const mfem::Operator *GetGradientOperator(
+      const mfem::QuadratureSpace &qspace) const = 0;
+
+   ///@}
+   // End: Virtual interface
 };
 
 } // namespace mfem

backends/base/layout.hpp

@@ -69,10 +69,16 @@ public:
    std::size_t Size() const { return size; }
 
    /// TODO
+   /** Useful in backends for down-casting to a backend-specific layout type.
+
+       When MFEM_DEBUG=YES, performs a type check using dynamic_cast. */
    template <typename derived_t>
    derived_t &As() { return *util::As<derived_t>(this); }
 
    /// TODO
+   /** Useful in backends for down-casting to a backend-specific layout type.
+
+       When MFEM_DEBUG=YES, performs a type check using dynamic_cast. */
    template <typename derived_t>
    const derived_t &As() const { return *util::As<const derived_t>(this); }
 

backends/base/smart_pointers.hpp

@@ -160,11 +160,12 @@ public:
 
    template <typename U>
    bool operator==(const SharedPtr<U> &other) const
-   { return ptr == other.Ptr(); }
+   { return ptr == other.Get(); }
    template <typename U>
    bool operator!=(const SharedPtr<U> &other) const
-   { return ptr != other.Ptr(); }
+   { return ptr != other.Get(); }
 
+   // Comparison to any type convertible to void *, e.g. the type of NULL.
    template <typename U>
    bool operator==(const U &p) const { return ptr == (void*) p; }
    template <typename U>

backends/occa/MassIntegrator.okl

@@ -0,0 +1,66 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  ELEMENT_BATCH   : How many elements are in each
+  .                   computation batch
+  NUM_DOFS_1D     : Dofs in the 1D segments
+  NUM_DOFS_2D     : Dofs in the 2D faces
+  NUM_DOFS_3D     : Dofs in the 3D domain
+  NUM_QUAD_1D     : Dofs in the 1D segments
+  NUM_QUAD_2D     : Dofs in the 2D faces
+  NUM_QUAD_3D     : Dofs in the 3D domain
+  NUM_MAX_1D      : max(NUM_QUAD_1D, NUM_DOFS_1D)
+  NUM_QUAD_DOFS_1D: NUM_QUAD_1D * NUM_DOFS_1D
+  COEFF_ARGS      : Code that passes required arguments to the kernel
+  COEFF           : Code that computes the coefficient
+  ================================================
+
+  [MISSING]
+  - Add support to auto-pick @dim and use @idxOrder on stack arrays
+  |   double a[2][2];
+  |   a[0][1]; <-- regular index
+  |   a(0,1);  <-- uses @idxOrder a[0][1] or a[1][0]
+  - Add support for @idxOrder to change indexing order after allocation
+  |   double a[2][2] @idxOrder(0,1);
+  |   a(0,1) -> a[1][0]
+  |   @set(a, idxOrder(1,0));
+  |   a(0,1) -> a[0][1]
+  - Add support to iterate over loop depending on mode
+  |   for(i; @inner) {
+  |     for(0 < j < N) {} <-- ++j or j += block?
+  |   }
+*/
+
+#include "mfem-occa://defines.okl"
+
+#if USING_TENSOR_OPS
+#  ifdef OCCA_USING_GPU
+#    if USING_LOW_ORDER
+#      include "mfem-occa://mass/tensor/gpuHighOrder.okl"
+#    else
+#      include "mfem-occa://mass/tensor/gpuHighOrder.okl"
+#    endif
+#  else
+#    include "mfem-occa://mass/tensor/cpu.okl"
+#  endif
+#else
+#  ifdef OCCA_USING_GPU
+#    if USING_LOW_ORDER
+#      include "mfem-occa://mass/simplex/gpuHighOrder.okl"
+#    else
+#      include "mfem-occa://mass/simplex/gpuHighOrder.okl"
+#    endif
+#  else
+#    include "mfem-occa://mass/simplex/cpu.okl"
+#  endif
+#endif

backends/occa/VectorMassIntegrator.okl

@@ -0,0 +1,38 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  ELEMENT_BATCH   : How many elements are in each
+  .                   computation batch
+  NUM_DOFS_1D     : Dofs in the 1D segments
+  NUM_DOFS_2D     : Dofs in the 2D faces
+  NUM_DOFS_3D     : Dofs in the 3D domain
+  NUM_QUAD_1D     : Dofs in the 1D segments
+  NUM_QUAD_2D     : Dofs in the 2D faces
+  NUM_QUAD_3D     : Dofs in the 3D domain
+  NUM_MAX_1D      : max(NUM_QUAD_1D, NUM_DOFS_1D)
+  NUM_QUAD_DOFS_1D: NUM_QUAD_1D * NUM_DOFS_1D
+  CONST_COEFF     : If the coefficient is constant, pass it
+  .                   as a define
+  ================================================
+
+  [MISSING]
+  See kernels/DiffusionIntegrator.okl
+*/
+
+#include "mfem-occa://defines.okl"
+
+#if USING_TENSOR_OPS
+#  ifndef OCCA_USING_GPU
+#    include "mfem-occa://vmass/tensor/cpu.okl"
+#  endif
+#endif

backends/occa/array.cpp

@@ -41,8 +41,7 @@ int Array::DoResize(PLayout &new_layout, void **buffer,
    MFEM_ASSERT(dynamic_cast<Layout *>(&new_layout) != NULL,
                "new_layout is not an OCCA Layout");
    Layout *lt = static_cast<Layout *>(&new_layout);
-   layout.Reset(lt); // Reset() checks if the pointer is the same
-   int err = ResizeData(lt, item_size);
+   int err = OccaResize(lt, item_size);
    if (!err && buffer)
    {
       *buffer = GetBuffer();
@@ -72,22 +71,22 @@ void Array::DoFill(const void *value_ptr, std::size_t item_size)
    switch (item_size)
    {
       case sizeof(int8_t):
-         OccaFill((const int8_t *)value_ptr);
+         OccaFill(*(const int8_t *)value_ptr);
          break;
       case sizeof(int16_t):
-         OccaFill((const int16_t *)value_ptr);
+         OccaFill(*(const int16_t *)value_ptr);
          break;
       case sizeof(int32_t):
-         OccaFill((const int32_t *)value_ptr);
+         OccaFill(*(const int32_t *)value_ptr);
          break;
       // case sizeof(int64_t):
-      //    OccaFill((const int64_t *)value_ptr);
+      //    OccaFill(*(const int64_t *)value_ptr);
       //    break;
       case sizeof(double):
-         OccaFill((const double *)value_ptr);
+         OccaFill(*(const double *)value_ptr);
          break;
       // case sizeof(::occa::double2):
-      //    OccaFill((const ::occa::double2 *)value_ptr);
+      //    OccaFill(*(const ::occa::double2 *)value_ptr);
       //    break;
       default:
          MFEM_ABORT("item_size = " << item_size << " is not supported");
@@ -112,8 +111,7 @@ void Array::DoAssign(const PArray &src, std::size_t item_size)
    //       class.
    const Array *source = dynamic_cast<const Array *>(&src);
    MFEM_ASSERT(source != NULL, "invalid source Array type");
-   MFEM_ASSERT(Size() == source->Size(), "");
-   slice.copyFrom(source->slice);
+   OccaAssign(*source);
 }
 
 } // namespace mfem::occa

backends/occa/array.hpp

@@ -60,16 +60,16 @@ protected:
 
    inline void *GetBuffer() const;
 
-   inline int ResizeData(const Layout *lt, std::size_t item_size);
-
-   template <typename T>
-   inline void OccaFill(const T *val_ptr)
-   { ::occa::linalg::operator_eq<T>(slice, *val_ptr); }
-
 public:
+   Array(const Engine &e)
+      : PArray(*(new Layout(e, 0))),
+        data(e.Alloc(0)),
+        slice(data)
+   { }
+
    Array(Layout &lt, std::size_t item_size)
       : PArray(lt),
-        data(lt.Alloc(lt.Size()*item_size)),
+        data(lt.OccaEngine().Alloc(lt.Size()*item_size)),
         slice(data)
    { }
 
@@ -77,11 +77,23 @@ public:
 
    inline void MakeRef(Array &master);
 
-   Layout &OccaLayout() const
-   { return *static_cast<Layout *>(layout.Get()); }
+   Layout &OccaLayout() const { return layout->As<Layout>(); }
+
+   const Engine &OccaEngine() const { return OccaLayout().OccaEngine(); }
 
    ::occa::memory &OccaMem() { return slice; }
    const ::occa::memory &OccaMem() const { return slice; }
+
+   inline int OccaResize(Layout *lt, std::size_t item_size);
+
+   inline int OccaResize(std::size_t new_size, std::size_t item_size);
+
+   template <typename T>
+   inline void OccaFill(const T val);
+
+   inline void OccaAssign(const Array &src);
+
+   inline void OccaPush(const void *src);
 };
 
 
@@ -98,13 +110,14 @@ inline void *Array::GetBuffer() const
    return NULL;
 }
 
-inline int Array::ResizeData(const Layout *lt, std::size_t item_size)
+inline int Array::OccaResize(Layout *lt, std::size_t item_size)
 {
+   layout.Reset(lt); // Reset() checks if the pointer is the same
    const std::size_t new_bytes = lt->Size()*item_size;
    if (data.size() < new_bytes ||
-       data.getDHandle() != lt->OccaEngine().GetDevice().getDHandle())
+       data.getDevice() != lt->OccaEngine().GetDevice())
    {
-      data = lt->Alloc(new_bytes);
+      data = lt->OccaEngine().Alloc(new_bytes);
       slice = data;
       // If memory allocation fails - an exception is thrown.
    }
@@ -122,6 +135,37 @@ inline void Array::MakeRef(Array &master)
    slice = master.slice;
 }
 
+inline int Array::OccaResize(std::size_t new_size, std::size_t item_size)
+{
+   Layout &ol = OccaLayout();
+   ol.OccaResize(new_size);
+   return OccaResize(&ol, item_size);
+}
+
+template <typename T>
+inline void Array::OccaFill(const T val)
+{
+   ::occa::linalg::operator_eq<T>(slice, val);
+}
+
+inline void Array::OccaAssign(const Array &src)
+{
+   if (slice != src.slice && slice.size() != 0)
+   {
+      MFEM_ASSERT(slice.size() == src.slice.size(), "");
+      slice.copyFrom(src.slice);
+   }
+}
+
+inline void Array::OccaPush(const void *src)
+{
+   if (slice.size() != 0)
+   {
+      slice.copyFrom(src);
+   }
+}
+
+
 } // namespace mfem::occa
 
 } // namespace mfem

backends/occa/bilinearform.cpp

@@ -24,8 +24,8 @@ namespace occa
 
 OccaBilinearForm::OccaBilinearForm(FiniteElementSpace *ofespace_) :
    Operator(ofespace_->OccaVLayout()),
-   localX((ofespace_->OccaEVLayout().DontDelete(), ofespace_->OccaEVLayout())),
-   localY((ofespace_->OccaEVLayout().DontDelete(), ofespace_->OccaEVLayout()))
+   localX(ofespace_->OccaEVLayout()),
+   localY(ofespace_->OccaEVLayout())
 {
    Init(ofespace_->OccaEngine(), ofespace_, ofespace_);
 }
@@ -34,8 +34,8 @@ OccaBilinearForm::OccaBilinearForm(FiniteElementSpace *otrialFESpace_,
                                    FiniteElementSpace *otestFESpace_) :
    Operator(otrialFESpace_->OccaVLayout(),
             otestFESpace_->OccaVLayout()),
-   localX((otrialFESpace_->OccaEVLayout().DontDelete(), otrialFESpace_->OccaEVLayout())),
-   localY((otestFESpace_->OccaEVLayout().DontDelete(), otestFESpace_->OccaEVLayout()))
+   localX(otrialFESpace_->OccaEVLayout()),
+   localY(otestFESpace_->OccaEVLayout())
 {
    Init(otrialFESpace_->OccaEngine(), otrialFESpace_, otestFESpace_);
 }
@@ -65,11 +65,9 @@ void OccaBilinearForm::Init(const Engine &e,
    if (GetDevice().mode() == "OpenMP")
    {
       const std::string &okl_path = OccaEngine().GetOklPath();
-      const std::string &okl_defines = OccaEngine().GetOklDefines();
       ::occa::kernel initLocalKernel =
          GetDevice().buildKernel(okl_path + "utils.okl",
-                                 "InitLocalVector",
-                                 okl_defines);
+                                 "InitLocalVector");
 
       const std::size_t sd = sizeof(double);
       const uint64_t trialEntries = sd * (elements * trialLocalDofs);
@@ -273,8 +271,6 @@ void OccaBilinearForm::InitRHS(const mfem::Array<int> &constraintList,
                                mfem::Vector &X, mfem::Vector &B,
                                int copy_interior)
 {
-   const std::string okl_defines = OccaEngine().GetOklDefines();
-
    // FIXME: move these kernels to the Backend?
    static ::occa::kernelBuilder get_subvector_builder =
       ::occa::linalg::customLinearMethod(
@@ -288,7 +284,7 @@ void OccaBilinearForm::InitRHS(const mfem::Array<int> &constraintList,
          "  VTYPE1: 'double',"
          "  VTYPE2: 'int',"
          "  TILESIZE: 128,"
-         "}" + okl_defines);
+         "}");
 
    static ::occa::kernelBuilder set_subvector_builder =
       ::occa::linalg::customLinearMethod(
@@ -302,7 +298,7 @@ void OccaBilinearForm::InitRHS(const mfem::Array<int> &constraintList,
          "  VTYPE1: 'double',"
          "  VTYPE2: 'int',"
          "  TILESIZE: 128,"
-         "}" + okl_defines);
+         "}");
 
    const mfem::Operator *P = GetTrialProlongation();
    const mfem::Operator *R = GetTrialRestriction();
@@ -345,6 +341,7 @@ void OccaBilinearForm::InitRHS(const mfem::Array<int> &constraintList,
                            constrList.OccaMem());
    }
 
+   // FIXME: add case for HypreParMatrix here
    OccaConstrainedOperator *cA = dynamic_cast<OccaConstrainedOperator*>(A);
    if (cA)
    {
@@ -361,7 +358,7 @@ void OccaBilinearForm::InitRHS(const mfem::Array<int> &constraintList,
 void OccaBilinearForm::Mult_(const Vector &x, Vector &y) const
 {
    otrialFESpace->GlobalToLocal(x, localX);
-   localY.Fill<double>(0.0);
+   localY.OccaFill<double>(0.0);
 
    const int integratorCount = (int) integrators.size();
    for (int i = 0; i < integratorCount; ++i)
@@ -376,7 +373,7 @@ void OccaBilinearForm::Mult_(const Vector &x, Vector &y) const
 void OccaBilinearForm::MultTranspose_(const Vector &x, Vector &y) const
 {
    otestFESpace->GlobalToLocal(x, localX);
-   localY.Fill<double>(0.0);
+   localY.OccaFill<double>(0.0);
 
    const int integratorCount = (int) integrators.size();
    for (int i = 0; i < integratorCount; ++i)
@@ -432,19 +429,42 @@ void BilinearForm::InitOccaBilinearForm()
       Coefficient *scal_coeff = dbfi[i]->GetScalarCoefficient();
       ConstantCoefficient *const_coeff =
          dynamic_cast<ConstantCoefficient*>(scal_coeff);
+      GridFunctionCoefficient *gridfunc_coeff =
+         dynamic_cast<GridFunctionCoefficient*>(scal_coeff);
       // TODO: other types of coefficients ...
-      double val = const_coeff ? const_coeff->constant : 1.0;
-      OccaCoefficient ocoeff(obform->OccaEngine(), val);
+
+      OccaCoefficient *ocoeff = NULL;
+      if (const_coeff)
+      {
+         ocoeff = new OccaCoefficient(obform->OccaEngine(),
+                                      const_coeff->constant);
+      }
+      else if (gridfunc_coeff)
+      {
+         ocoeff = new OccaCoefficient(obform->OccaEngine(),
+                                      *gridfunc_coeff->GetGridFunction(), true);
+      }
+      else if (!scal_coeff)
+      {
+         ocoeff = new OccaCoefficient(obform->OccaEngine(), 1.0);
+      }
+      else
+      {
+         MFEM_ABORT("Coefficient type not supported");
+      }
 
       OccaIntegrator *ointeg = NULL;
-
       if (integ_name == "(undefined)")
       {
          MFEM_ABORT("BilinearFormIntegrator does not define Name()");
       }
+      else if (integ_name == "mass")
+      {
+         ointeg = new OccaMassIntegrator(*ocoeff);
+      }
       else if (integ_name == "diffusion")
       {
-         ointeg = new OccaDiffusionIntegrator(ocoeff);
+         ointeg = new OccaDiffusionIntegrator(*ocoeff);
       }
       else
       {
@@ -452,6 +472,10 @@ void BilinearForm::InitOccaBilinearForm()
                     << "] is not supported");
       }
 
+      // NOTE: The integrators copy ocoeff, so it can be deleted here so there
+      //       is no memory leak.
+      delete ocoeff;
+
       const mfem::IntegrationRule *ir = dbfi[i]->GetIntRule();
       if (ir) { ointeg->SetIntegrationRule(*ir); }
 

backends/occa/bilininteg.cpp

@@ -107,11 +107,10 @@ OccaGeometry OccaGeometry::Get(::occa::device device,
    props["defines/STORE_JACOBIAN_DET"] = (flags & JacobianDet);
 
    const std::string &okl_path = ofespace.OccaEngine().GetOklPath();
-   const std::string &okl_defines = ofespace.OccaEngine().GetOklDefines();
    ::occa::kernel init = device.buildKernel(okl_path + "geometry.okl",
                                             stringWithDim("InitGeometryInfo",
                                                           fe.GetDim()),
-                                            props + okl_defines);
+                                            props);
    init(elements,
         maps.dofToQuadD,
         geom.meshNodes,
@@ -730,10 +729,9 @@ OccaGeometry OccaIntegrator::GetGeometry(const int flags)
 {
    const std::string filename = GetName() + ".okl";
    const std::string &okl_path = OccaEngine().GetOklPath();
-   const std::string &okl_defines = OccaEngine().GetOklDefines();
    return GetDevice().buildKernel(okl_path + filename,
                                   kernelName,
-                                  props + okl_defines);
+                                  props);
 }
 //====================================
 
@@ -786,8 +784,8 @@ void OccaDiffusionIntegrator::Assemble()
 
    OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
 
-   assembledOperator.Resize<double>(symmDims * quadraturePoints * elements,
-                                    NULL);
+   assembledOperator.OccaResize(symmDims * quadraturePoints * elements,
+                                sizeof(double));
 
    assembleKernel((int) mesh->GetNE(),
                   maps.quadWeights,

backends/occa/coefficient.cpp

@@ -119,17 +119,18 @@ void OccaVectorParameter::Setup(OccaIntegrator &integ,
 
 //---[ GridFunction Parameter ]-------
 OccaGridFunctionParameter::OccaGridFunctionParameter(const std::string &name_,
-                                                     OccaGridFunction &gf_,
+                                                     const Engine &e,
+                                                     mfem::GridFunction &gf_,
                                                      const bool useRestrict_)
    : name(name_),
      gf(gf_),
-     gfQuad(*(new Layout(gf_.OccaLayout().OccaEngine(), 0))),
+     gfQuad(e),
      useRestrict(useRestrict_) {}
 
 OccaParameter* OccaGridFunctionParameter::Clone()
 {
    OccaGridFunctionParameter *param =
-      new OccaGridFunctionParameter(name, gf, useRestrict);
+      new OccaGridFunctionParameter(name, gfQuad.OccaEngine(), gf, useRestrict);
    param->gfQuad.MakeRef(gfQuad);
    return param;
 }
@@ -141,15 +142,16 @@ void OccaGridFunctionParameter::Setup(OccaIntegrator &integ,
    std::string &args = (props["defines/COEFF_ARGS"]
                         .asString()
                         .string());
-   args += "const double *";
    if (useRestrict)
    {
-      args += " restrict ";
+      args += "@restrict ";
    }
+   args += "const double *";
    args += name;
    args += " @dim(NUM_QUAD, numElements),\n";
 
-   gf.ToQuad(integ.GetIntegrationRule(), gfQuad);
+   FiniteElementSpace &f = gf.FESpace()->Get_PFESpace()->As<FiniteElementSpace>();
+   ToQuad(integ.GetIntegrationRule(), f, gf.Get_PVector()->As<Vector>(), gfQuad);
 }
 
 ::occa::kernelArg OccaGridFunctionParameter::KernelArgs()
@@ -168,6 +170,16 @@ OccaCoefficient::OccaCoefficient(const Engine &e, const double value) :
    coeffValue = value;
 }
 
+OccaCoefficient::OccaCoefficient(const Engine &e, mfem::GridFunction &gf,
+                                 const bool useRestrict) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   coeffValue = "(u(q, e))";
+   AddGridFunction("u", gf, useRestrict);
+}
+
 OccaCoefficient::OccaCoefficient(const Engine &e, const std::string &source) :
    engine(&e),
    integ(NULL),
@@ -261,10 +273,13 @@ OccaCoefficient& OccaCoefficient::AddVector(const std::string &name_,
 }
 
 OccaCoefficient& OccaCoefficient::AddGridFunction(const std::string &name_,
-                                                  OccaGridFunction &gf,
+                                                  mfem::GridFunction &gf,
                                                   const bool useRestrict)
 {
-   return Add(new OccaGridFunctionParameter(name_, gf, useRestrict));
+   MFEM_ASSERT(engine->CheckVector(gf.Get_PVector()) &&
+               engine->CheckFESpace(gf.FESpace()->Get_PFESpace()),
+               "invalid device GridFunction");
+   return Add(new OccaGridFunctionParameter(name_, *engine, gf, useRestrict));
 }
 
 bool OccaCoefficient::IsConstant()
@@ -302,10 +317,9 @@ Vector OccaCoefficient::Eval()
 void OccaCoefficient::Eval(Vector &quadCoeff)
 {
    const std::string &okl_path = OccaEngine().GetOklPath();
-   const std::string &okl_defines = OccaEngine().GetOklDefines();
    static ::occa::kernelBuilder builder =
       ::occa::kernelBuilder::fromFile(okl_path + "coefficient.okl",
-                                      "CoefficientEval", okl_defines);
+                                      "CoefficientEval");
 
    if (integ == NULL)
    {
@@ -320,7 +334,6 @@ void OccaCoefficient::Eval(Vector &quadCoeff)
       kernelProps["defines/COEFF"]      = name;
       kernelProps["defines/COEFF_ARGS"] = name + "_ARGS";
    }
-   kernelProps += okl_defines;
 
    ::occa::kernel evalKernel = builder.build(GetDevice(), kernelProps);
    evalKernel(elements, *this, quadCoeff.OccaMem());

backends/occa/coefficient.hpp

@@ -178,13 +178,14 @@ class OccaGridFunctionParameter : public OccaParameter
 {
 private:
    const std::string name;
-   OccaGridFunction &gf;
+   mfem::GridFunction &gf;
    Vector gfQuad;
    bool useRestrict;
 
 public:
    OccaGridFunctionParameter(const std::string &name_,
-                             OccaGridFunction &gf_,
+                             const Engine &e,
+                             mfem::GridFunction &gf_,
                              const bool useRestrict_ = false);
 
    virtual OccaParameter* Clone();
@@ -219,6 +220,8 @@ private:
 
 public:
    OccaCoefficient(const Engine &e, const double value = 1.0);
+   OccaCoefficient(const Engine &e, mfem::GridFunction &gf,
+                   const bool useRestrict = false);
    OccaCoefficient(const Engine &e, const std::string &source);
    OccaCoefficient(const Engine &e, const char *source);
    ~OccaCoefficient();
@@ -263,7 +266,7 @@ public:
                               const bool useRestrict = false);
 
    OccaCoefficient& AddGridFunction(const std::string &name_,
-                                    OccaGridFunction &gf,
+                                    mfem::GridFunction &gf,
                                     const bool useRestrict = false);
 
    bool IsConstant();

backends/occa/diffusion/simplex/cpu.okl

@@ -13,10 +13,10 @@
 
 //---[ 2D ]-----------------------------
 @kernel void Assemble2D(const int numElements,
-                        const double * restrict quadWeights,
-                        const Jacobian2D_t restrict J,
+                        @restrict const double *quadWeights,
+                        @restrict const Jacobian2D_t J,
                         COEFF_ARGS
-                        SymmOperator2D_t restrict oper) {
+                        @restrict SymmOperator2D_t oper) {
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int q = 0; q < NUM_QUAD; ++q; @inner) {
       const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
@@ -32,13 +32,13 @@
 }
 
 @kernel void MultAdd2D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuadD2D_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDofD2D_t restrict quadToDofD,
-                       const SymmOperator2D_t restrict oper,
-                       const DLocal_t restrict solIn,
-                       DLocal_t restrict solOut) {
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD2D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD2D_t quadToDofD,
+                       @restrict const SymmOperator2D_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {
@@ -78,10 +78,10 @@
 
 //---[ 3D ]-----------------------------
 @kernel void Assemble3D(const int numElements,
-                        const double * restrict quadWeights,
-                        const Jacobian3D_t restrict J,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian3D_t J,
                         COEFF_ARGS
-                        SymmOperator3D_t restrict oper) {
+                        @restrict SymmOperator3D_t oper) {
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int q = 0; q < NUM_QUAD; ++q; @inner) {
       const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
@@ -118,13 +118,13 @@
 }
 
 @kernel void MultAdd3D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuadD3D_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDofD3D_t restrict quadToDofD,
-                       const SymmOperator3D_t restrict oper,
-                       const DLocal_t restrict solIn,
-                       DLocal_t restrict solOut) {
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD3D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD3D_t quadToDofD,
+                       @restrict const SymmOperator3D_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {

backends/occa/diffusion/simplex/gpuHighOrder.okl

@@ -38,13 +38,13 @@
 }
 
 @kernel void MultAdd2D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuadD2D_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDofD2D_t restrict quadToDofD,
-                       const SymmOperator2D_t restrict oper,
-                       const DLocal_t restrict solIn,
-                       DLocal_t restrict solOut) {
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD2D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD2D_t quadToDofD,
+                       @restrict const SymmOperator2D_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     @shared double s_gradX[NUM_QUAD];
@@ -131,13 +131,13 @@
 }
 
 @kernel void MultAdd3D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuadD3D_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDofD3D_t restrict quadToDofD,
-                       const SymmOperator3D_t restrict oper,
-                       const DLocal_t restrict solIn,
-                       DLocal_t restrict solOut) {
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD3D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD3D_t quadToDofD,
+                       @restrict const SymmOperator3D_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     @shared double s_gradX[NUM_QUAD];

backends/occa/diffusion/tensor/cpu.okl

@@ -13,10 +13,10 @@
 
 //---[ 1D ]-----------------------------
 @kernel void Assemble1D(const int numElements,
-                        const double * restrict quadWeights,
-                        const Jacobian1D_t restrict J,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian1D_t J,
                         COEFF_ARGS
-                        SymmOperator1D_t restrict oper) {
+                        @restrict SymmOperator1D_t oper) {
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
       oper(q, e) = quadWeights[q] * COEFF / J(q, e);
@@ -25,13 +25,13 @@
 }
 
 @kernel void MultAdd1D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuad_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDof_t restrict quadToDofD,
-                       const SymmOperator1D_t restrict oper,
-                       const DLocal1D_t restrict solIn,
-                       DLocal1D_t restrict solOut) {
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator1D_t oper,
+                       @restrict const DLocal1D_t solIn,
+                       @restrict DLocal1D_t solOut) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {
@@ -62,10 +62,10 @@
 
 //---[ 2D ]-----------------------------
 @kernel void Assemble2D(const int numElements,
-                        const double * restrict quadWeights,
-                        const Jacobian2D_t restrict J,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian2D_t J,
                         COEFF_ARGS
-                        SymmOperator2D_t restrict oper) {
+                        @restrict SymmOperator2D_t oper) {
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int q = 0; q < NUM_QUAD_2D; ++q; @inner) {
       const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
@@ -81,13 +81,13 @@
 }
 
 @kernel void MultAdd2D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuad_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDof_t restrict quadToDofD,
-                       const SymmOperator2D_t restrict oper,
-                       const DLocal2D_t restrict solIn,
-                       DLocal2D_t restrict solOut) {
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator2D_t oper,
+                       @restrict const DLocal2D_t solIn,
+                       @restrict DLocal2D_t solOut) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {
@@ -175,10 +175,10 @@
 
 //---[ 3D ]-----------------------------
 @kernel void Assemble3D(const int numElements,
-                        const double * restrict quadWeights,
-                        const Jacobian3D_t restrict J,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian3D_t J,
                         COEFF_ARGS
-                        SymmOperator3D_t restrict oper) {
+                        @restrict SymmOperator3D_t oper) {
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int q = 0; q < NUM_QUAD_3D; ++q; @inner) {
       const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
@@ -215,13 +215,13 @@
 }
 
 @kernel void MultAdd3D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuad_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDof_t restrict quadToDofD,
-                       const SymmOperator3D_t restrict oper,
-                       const DLocal3D_t restrict solIn,
-                       DLocal3D_t restrict solOut) {
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator3D_t oper,
+                       @restrict const DLocal3D_t solIn,
+                       @restrict DLocal3D_t solOut) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {

backends/occa/diffusion/tensor/gpuHighOrder.okl

@@ -29,13 +29,13 @@
 }
 
 @kernel void MultAdd1D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuad_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDof_t restrict quadToDofD,
-                       const SymmOperator1D_t restrict oper,
-                       const DLocal1D_t restrict solIn,
-                       DLocal1D_t restrict solOut) {
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator1D_t oper,
+                       @restrict const DLocal1D_t solIn,
+                       @restrict DLocal1D_t solOut) {
   // Iterate over elements
   for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
     @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
@@ -108,13 +108,13 @@
 }
 
 @kernel void MultAdd2D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuad_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDof_t restrict quadToDofD,
-                       const SymmOperator2D_t restrict oper,
-                       const DLocal2D_t restrict solIn,
-                       DLocal2D_t restrict solOut) {
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator2D_t oper,
+                       @restrict const DLocal2D_t solIn,
+                       @restrict DLocal2D_t solOut) {
   // Iterate over elements
   for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
     // Store dof <--> quad mappings
@@ -272,13 +272,13 @@
 }
 
 @kernel void MultAdd3D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuad_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDof_t restrict quadToDofD,
-                       const SymmOperator3D_t restrict oper,
-                       const DLocal3D_t restrict solIn,
-                       DLocal3D_t restrict solOut) {
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator3D_t oper,
+                       @restrict const DLocal3D_t solIn,
+                       @restrict DLocal3D_t solOut) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     // Store dof <--> quad mappings
@@ -391,6 +391,7 @@
           }
         }
       }
+      @barrier("s_z_s_Dz_sync_1");
     }
     // Iterate over xy planes to compute solution
     for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
@@ -427,6 +428,7 @@
           }
         }
       }
+      @barrier("s_z_s_Dz_s_xyDz_sync_1");
     }
   }
 }

backends/occa/engine.cpp

@@ -42,7 +42,6 @@ void Engine::Init(const std::string &engine_spec)
    device[0].setup(props);
 
    okl_path = "mfem-occa://";
-   // okl_defines = "...";
    if (!fileOpenerRegistered)
    {
       // The directories from "MFEM_OCCA_OKL_PATH", if any, have the highest
@@ -57,6 +56,9 @@ void Engine::Init(const std::string &engine_spec)
       ::occa::io::fileOpener::add(fo);
       fileOpenerRegistered = true;
    }
+   // std::cout << "OCCA device properties:\n" << device[0].properties();
+
+   force_cuda_aware_mpi = false;
 }
 
 Engine::Engine(const std::string &engine_spec)
@@ -74,6 +76,36 @@ Engine::Engine(MPI_Comm _comm, const std::string &engine_spec)
 }
 #endif
 
+bool Engine::CheckEngine(const mfem::Engine *engine) const
+{
+   return (engine != NULL && util::Is<const Engine>(engine) != NULL &&
+           *util::As<const Engine>(engine) == *this);
+}
+
+bool Engine::CheckLayout(const PLayout *layout) const
+{
+   return (layout != NULL && util::Is<const Layout>(layout) != NULL &&
+           layout->As<Layout>().OccaEngine() == *this);
+}
+
+bool Engine::CheckArray(const PArray *array) const
+{
+   return (array != NULL && util::Is<const Array>(array) != NULL &&
+           array->As<Array>().OccaEngine() == *this);
+}
+
+bool Engine::CheckVector(const PVector *vector) const
+{
+   return (vector != NULL && util::Is<const Vector>(vector) != NULL &&
+           vector->As<Vector>().OccaEngine() == *this);
+}
+
+bool Engine::CheckFESpace(const PFiniteElementSpace *fes) const
+{
+   return (fes != NULL && util::Is<const FiniteElementSpace>(fes) != NULL &&
+           fes->As<FiniteElementSpace>().OccaEngine() == *this);
+}
+
 DLayout Engine::MakeLayout(std::size_t size) const
 {
    return DLayout(new Layout(*this, size));
@@ -88,19 +120,14 @@ DLayout Engine::MakeLayout(const mfem::Array<std::size_t> &offsets) const
 
 DArray Engine::MakeArray(PLayout &layout, std::size_t item_size) const
 {
-   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
-               "invalid input layout");
-   Layout *lt = static_cast<Layout *>(&layout);
-   return DArray(new Array(*lt, item_size));
+   return DArray(new Array(layout.As<Layout>(), item_size));
 }
 
 DVector Engine::MakeVector(PLayout &layout, int type_id) const
 {
-   MFEM_ASSERT(type_id == ScalarId<double>::value, "invalid type_id");
-   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
-               "invalid input layout");
-   Layout *lt = static_cast<Layout *>(&layout);
-   return DVector(new Vector(*lt));
+   MFEM_ASSERT(type_id == ScalarId<double>::value, "type_id " << type_id
+               << " is not supported");
+   return DVector(new Vector(layout.As<Layout>()));
 }
 
 DFiniteElementSpace Engine::MakeFESpace(mfem::FiniteElementSpace &fespace) const

backends/occa/engine.hpp

@@ -41,18 +41,23 @@ protected:
    // int *workers_mem_res;
 
    static bool fileOpenerRegistered;
-   ::occa::device *device; // An array of OCCA devices
-   std::string okl_path, okl_defines;
+   /// An array of OCCA devices. Currently only a single device is supported.
+   ::occa::device *device;
+   std::string okl_path;
+   bool force_cuda_aware_mpi;
 
    void Init(const std::string &engine_spec);
 
 public:
+   /// TODO: doxygen
    Engine(const std::string &engine_spec);
 
 #ifdef MFEM_USE_MPI
+   /// TODO: doxygen
    Engine(MPI_Comm comm, const std::string &engine_spec);
 #endif
 
+   /// TODO: doxygen
    virtual ~Engine() { delete [] device; }
 
    /**
@@ -60,13 +65,41 @@ public:
     */
    ///@{
 
+   /// Get the associated OCCA device.
    ::occa::device GetDevice(int idx = 0) const { return device[idx]; }
 
    /// TODO: doxygen
    const std::string &GetOklPath() const { return okl_path; }
 
+   /// OCCA device memory allocation.
+   ::occa::memory Alloc(std::size_t bytes) const
+   { return GetDevice().malloc(bytes); }
+
+   /// Two mfem::occa::Engine%s are equal if they use the same OCCA device.
+   bool operator==(const Engine &other) const
+   { return GetDevice() == other.GetDevice(); }
+
    /// TODO: doxygen
-   const std::string &GetOklDefines() const { return okl_defines; }
+   bool CheckEngine(const mfem::Engine *e) const;
+
+   /// TODO: doxygen
+   bool CheckLayout(const PLayout *layout) const;
+
+   /// TODO: doxygen
+   bool CheckArray(const PArray *array) const;
+
+   /// TODO: doxygen
+   bool CheckVector(const PVector *vector) const;
+
+   /// TODO: doxygen
+   bool CheckFESpace(const PFiniteElementSpace *fes) const;
+
+#ifdef MFEM_USE_MPI
+   void SetForceCudaAwareMPI(bool force = true)
+   { force_cuda_aware_mpi = force; }
+
+   bool GetForceCudaAwareMPI() const { return force_cuda_aware_mpi; }
+#endif
 
    ///@}
    // End: OCCA specific interface

backends/occa/fespace.cpp

@@ -16,6 +16,12 @@
 #include "fespace.hpp"
 #include "interpolation.hpp"
 
+#ifdef MFEM_USE_MPI
+#ifdef OMPI_RELEASE_VERSION
+#include <mpi-ext.h> // Check for cuda support
+#endif
+#endif
+
 namespace mfem
 {
 
@@ -25,41 +31,42 @@ namespace occa
 FiniteElementSpace::FiniteElementSpace(const Engine &e,
                                        mfem::FiniteElementSpace &fespace)
    : PFiniteElementSpace(e, fespace),
-     e_layout(e, 0) // resized in SetupLocalGlobalMaps()
+     e_layout(new Layout(e, 0)) // resized in SetupLocalGlobalMaps()
 {
    vdim     = fespace.GetVDim();
    ordering = fespace.GetOrdering();
 
    SetupLocalGlobalMaps();
-   SetupOperators();
+   SetupOperators(); // calls virtual methods of 'fes'
    SetupKernels();
 }
 
 FiniteElementSpace::~FiniteElementSpace()
 {
-   delete [] elementDofMap;
-   delete [] elementDofMapInverse;
    delete restrictionOp;
    delete prolongationOp;
 }
 
 void FiniteElementSpace::SetupLocalGlobalMaps()
 {
-   const mfem::FiniteElement &fe = *(fes->GetFE(0));
+   const int elements = fes->GetNE();
+
+   if (elements == 0) { return; }
+
+   // Assuming of finite elements are the same.
+   const mfem::FiniteElement &fe = *fes->GetFE(0);
    const mfem::TensorBasisElement *el =
       dynamic_cast<const mfem::TensorBasisElement*>(&fe);
 
    const mfem::Table &e2dTable = fes->GetElementToDofTable();
    const int *elementMap = e2dTable.GetJ();
-   const int elements = fes->GetNE();
 
    globalDofs = fes->GetNDofs();
    localDofs  = fe.GetDof();
 
-   e_layout.Resize(localDofs * elements * fes->GetVDim());
+   e_layout->OccaResize(e2dTable.Size_of_connections());
 
-   elementDofMap        = new int[localDofs];
-   elementDofMapInverse = new int[localDofs];
+   int *elementDofMap = new int[localDofs];
    if (el)
    {
       ::memcpy(elementDofMap,
@@ -73,10 +80,6 @@ void FiniteElementSpace::SetupLocalGlobalMaps()
          elementDofMap[i] = i;
       }
    }
-   for (int i = 0; i < localDofs; ++i)
-   {
-      elementDofMapInverse[elementDofMap[i]] = i;
-   }
 
    // Allocate device offsets and indices
    globalToLocalOffsets.allocate(GetDevice(),
@@ -99,6 +102,7 @@ void FiniteElementSpace::SetupLocalGlobalMaps()
 
    for (int e = 0; e < elements; ++e)
    {
+      MFEM_ASSERT(e2dTable.RowSize(e) == localDofs, "");
       for (int d = 0; d < localDofs; ++d)
       {
          const int gid = elementMap[localDofs*e + d];
@@ -130,19 +134,64 @@ void FiniteElementSpace::SetupLocalGlobalMaps()
    }
    offsets[0] = 0;
 
+   delete [] elementDofMap;
+
    globalToLocalOffsets.keepInDevice();
    globalToLocalIndices.keepInDevice();
    localToGlobalMap.keepInDevice();
 }
 
-void FiniteElementSpace::SetupOperators()
+void FiniteElementSpace::SetupOperators() const
 {
+   // Construct 'restrictionOp' and 'prolongationOp'.
+
+   prolongationOp = restrictionOp = NULL;
+
    const mfem::SparseMatrix *R = fes->GetRestrictionMatrix();
    const mfem::Operator *P = fes->GetProlongationMatrix();
-   CreateRPOperators(OccaVLayout(), OccaTrueVLayout(),
-                     R, P,
-                     restrictionOp,
-                     prolongationOp);
+
+   if (!P) { return; }
+
+   Layout &v_layout = OccaVLayout();
+   Layout &t_layout = OccaTrueVLayout();
+
+   // Assuming R has one entry per row equal to 1.
+   MFEM_ASSERT(R->Finalized(), "");
+   const int tdofs = R->Height();
+   MFEM_ASSERT(tdofs == (int)t_layout.Size(), "");
+   MFEM_ASSERT(tdofs == R->GetI()[tdofs], "");
+   ::occa::array<int> ltdof_ldof(GetDevice(), tdofs, R->GetJ());
+   ltdof_ldof.keepInDevice();
+
+   restrictionOp = new RestrictionOperator(v_layout, t_layout, ltdof_ldof);
+
+   const mfem::SparseMatrix *pmat = dynamic_cast<const mfem::SparseMatrix*>(P);
+   if (pmat)
+   {
+      const mfem::SparseMatrix *pmatT = Transpose(*pmat);
+
+      OccaSparseMatrix *occaP  =
+         CreateMappedSparseMatrix(t_layout, v_layout, *pmat);
+      OccaSparseMatrix *occaPT =
+         CreateMappedSparseMatrix(v_layout, t_layout, *pmatT);
+
+      prolongationOp = new ProlongationOperator(*occaP, *occaPT);
+
+      delete occaPT;
+      delete occaP;
+   }
+#ifdef MFEM_USE_MPI
+   else if (fes->Conforming() && dynamic_cast<ParFiniteElementSpace*>(fes))
+   {
+      ParFiniteElementSpace *pfes = static_cast<ParFiniteElementSpace*>(fes);
+      prolongationOp = new OccaConformingProlongation(*this, *pfes,
+                                                      ltdof_ldof.memory());
+   }
+#endif
+   else
+   {
+      prolongationOp = new ProlongationOperator(t_layout, v_layout, P);
+   }
 }
 
 void FiniteElementSpace::SetupKernels()
@@ -158,15 +207,324 @@ void FiniteElementSpace::SetupKernels()
 
    ::occa::device device = GetDevice();
    const std::string &okl_path = OccaEngine().GetOklPath();
-   const std::string &okl_defines = OccaEngine().GetOklDefines();
    globalToLocalKernel = device.buildKernel(okl_path + "fespace.okl",
                                             "GlobalToLocal",
-                                            props + okl_defines);
+                                            props);
    localToGlobalKernel = device.buildKernel(okl_path + "fespace.okl",
                                             "LocalToGlobal",
-                                            props + okl_defines);
+                                            props);
 }
 
+
+#ifdef MFEM_USE_MPI
+
+OccaConformingProlongation::OccaConformingProlongation(
+   const FiniteElementSpace &ofes, const mfem::ParFiniteElementSpace &pfes,
+   ::occa::memory ltdof_ldof_)
+
+   : Operator(ofes.OccaTrueVLayout(), ofes.OccaVLayout()),
+     shr_ltdof(ofes.OccaEngine()),
+     ext_ldof(ofes.OccaEngine()),
+     shr_buf(shr_ltdof.OccaLayout(), sizeof(double)),
+     ext_buf(ext_ldof.OccaLayout(), sizeof(double)),
+     shr_buf_offsets(NULL), ext_buf_offsets(NULL),
+     ltdof_ldof(ltdof_ldof_),
+     gc(pfes.GroupComm())
+{
+   MFEM_ASSERT(pfes.Conforming(), "internal error");
+
+   const Engine &engine = ofes.OccaEngine();
+   const std::string &okl_path = engine.GetOklPath();
+   ::occa::device device = engine.GetDevice();
+
+   {
+      Table nbr_ltdof;
+      gc.GetNeighborLTDofTable(nbr_ltdof);
+      shr_ltdof.OccaResize(nbr_ltdof.Size_of_connections(), sizeof(int));
+      shr_ltdof.OccaPush(nbr_ltdof.GetJ());
+      shr_buf.OccaResize(&shr_ltdof.OccaLayout(), sizeof(double));
+      shr_buf_offsets = nbr_ltdof.GetI();
+      {
+         mfem::Array<int> shr_ltdof(nbr_ltdof.GetJ(),
+                                    nbr_ltdof.Size_of_connections());
+         mfem::Array<int> unique_ltdof(shr_ltdof);
+         unique_ltdof.Sort();
+         unique_ltdof.Unique();
+         // Note: the next loop modifies the J array of nbr_ltdof
+         for (int i = 0; i < shr_ltdof.Size(); i++)
+         {
+            shr_ltdof[i] = unique_ltdof.FindSorted(shr_ltdof[i]);
+            MFEM_ASSERT(shr_ltdof[i] != -1, "internal error");
+         }
+         Table unique_shr;
+         Transpose(shr_ltdof, unique_shr, unique_ltdof.Size());
+
+         unq_ltdof = device.malloc(unique_ltdof.Size()*sizeof(int),
+                                   unique_ltdof.GetData());
+         unq_shr_i = device.malloc((unique_shr.Size()+1)*sizeof(int),
+                                   unique_shr.GetI());
+         unq_shr_j = device.malloc(unique_shr.Size_of_connections()*sizeof(int),
+                                   unique_shr.GetJ());
+      }
+      delete [] nbr_ltdof.GetJ();
+      nbr_ltdof.LoseData();
+   }
+   {
+      Table nbr_ldof;
+      gc.GetNeighborLDofTable(nbr_ldof);
+      ext_ldof.OccaResize(nbr_ldof.Size_of_connections(), sizeof(int));
+      ext_ldof.OccaPush(nbr_ldof.GetJ());
+      ext_buf.OccaResize(&ext_ldof.OccaLayout(), sizeof(double));
+      ext_buf_offsets = nbr_ldof.GetI();
+      delete [] nbr_ldof.GetJ();
+      nbr_ldof.LoseData();
+   }
+   host_shr_buf = NULL;
+   host_ext_buf = NULL;
+   // If the device has a separate memory space (e.g. CUDA device) and the MPI
+   // library does not support buffers in that separate memory space, we
+   // allocate separate host buffers to use for MPI communication.
+   if (device.hasSeparateMemorySpace())
+   {
+      bool need_host_buf = true;
+      if (device.mode() == "CUDA")
+      {
+#ifdef MPIX_CUDA_AWARE_SUPPORT
+         need_host_buf = !MPIX_Query_cuda_support();
+#endif
+         if (engine.GetForceCudaAwareMPI()) { need_host_buf = false; }
+         if (gc.GetGroupTopology().MyRank() == 0)
+         {
+            mfem::out << "\nOccaConformingProlongation: CUDA-aware MPI: "
+                      << (need_host_buf ? "NO" : "YES") << "\n\n";
+         }
+      }
+      if (need_host_buf)
+      {
+         host_shr_buf = new char[shr_buf.OccaMem().size()];
+         host_ext_buf = new char[ext_buf.OccaMem().size()];
+      }
+   }
+
+   ExtractSubVector = device.buildKernel(okl_path + "mappings.okl",
+                                         "ExtractSubVector",
+                                         "defines: { TILESIZE: 256 }");
+   SetSubVector = device.buildKernel(okl_path + "mappings.okl",
+                                     "SetSubVector",
+                                     "defines: { TILESIZE: 256 }");
+   AddSubVector = device.buildKernel(okl_path + "mappings.okl",
+                                     "AddSubVector",
+                                     "defines: { TILESIZE: 256 }");
+
+   const GroupTopology &gtopo = gc.GetGroupTopology();
+   int req_counter = 0;
+   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
+   {
+      const int send_offset = shr_buf_offsets[nbr];
+      const int send_size = shr_buf_offsets[nbr+1] - send_offset;
+      if (send_size > 0) { req_counter++; }
+
+      const int recv_offset = ext_buf_offsets[nbr];
+      const int recv_size = ext_buf_offsets[nbr+1] - recv_offset;
+      if (recv_size > 0) { req_counter++; }
+   }
+   requests = new MPI_Request[req_counter];
+}
+
+OccaConformingProlongation::~OccaConformingProlongation()
+{
+   delete [] requests;
+   delete [] host_ext_buf;
+   delete [] host_shr_buf;
+   delete [] ext_buf_offsets;
+   delete [] shr_buf_offsets;
+}
+
+void OccaConformingProlongation::BcastBeginCopy(const ::occa::memory &src,
+                                                std::size_t item_size) const
+{
+   // shr_buf[i] = src[shr_ltdof[i]]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (shr_ltdof.Size() == 0) { return; }
+   ExtractSubVector((int)shr_ltdof.Size(), shr_ltdof.OccaMem(), src,
+                    shr_buf.OccaMem());
+   // If the above kernel is executed asynchronously, wait for it to complete:
+   shr_buf.OccaMem().getDevice().finish();
+   if (host_shr_buf)
+   {
+      shr_buf.OccaMem().copyTo(host_shr_buf);
+   }
+}
+
+void OccaConformingProlongation::BcastLocalCopy(const ::occa::memory &src,
+                                                ::occa::memory &dst,
+                                                std::size_t item_size) const
+{
+   // dst[ltdof_ldof[i]] = src[i]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (ltdof_ldof.size<int>() == 0) { return; }
+   SetSubVector((int)ltdof_ldof.size<int>(), ltdof_ldof, src, dst);
+}
+
+void OccaConformingProlongation::BcastEndCopy(::occa::memory &dst,
+                                              std::size_t item_size) const
+{
+   // dst[ext_ldof[i]] = ext_buf[i]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (ext_ldof.Size() == 0) { return; }
+   if (host_ext_buf)
+   {
+      ext_buf.OccaMem().copyFrom(host_ext_buf);
+   }
+   SetSubVector((int)ext_ldof.Size(), ext_ldof.OccaMem(),
+                ext_buf.OccaMem(), dst);
+}
+
+void OccaConformingProlongation::ReduceBeginCopy(const ::occa::memory &src,
+                                                 std::size_t item_size) const
+{
+   // ext_buf[i] = src[ext_ldof[i]]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (ext_ldof.Size() == 0) { return; }
+   ExtractSubVector((int)ext_ldof.Size(), ext_ldof.OccaMem(), src,
+                    ext_buf.OccaMem());
+   // If the above kernel is executed asynchronously, wait for it to complete:
+   ext_buf.OccaMem().getDevice().finish();
+   if (host_ext_buf)
+   {
+      ext_buf.OccaMem().copyTo(host_ext_buf);
+   }
+}
+
+void OccaConformingProlongation::ReduceLocalCopy(const ::occa::memory &src,
+                                                 ::occa::memory &dst,
+                                                 std::size_t item_size) const
+{
+   // dst[i] = src[ltdof_ldof[i]]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (ltdof_ldof.size<int>() == 0) { return; }
+   ExtractSubVector((int)ltdof_ldof.size<int>(), ltdof_ldof, src, dst);
+}
+
+void OccaConformingProlongation::ReduceEndAssemble(::occa::memory &dst,
+                                                   std::size_t item_size) const
+{
+   // dst[shr_ltdof[i]] += shr_buf[i]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (unq_ltdof.size<int>() == 0) { return; }
+   if (host_shr_buf)
+   {
+      shr_buf.OccaMem().copyFrom(host_shr_buf);
+   }
+   AddSubVector((int)unq_ltdof.size<int>(), unq_ltdof, unq_shr_i, unq_shr_j,
+                shr_buf.OccaMem(), dst);
+}
+
+void OccaConformingProlongation::Mult_(const Vector &x, Vector &y) const
+{
+   const GroupTopology &gtopo = gc.GetGroupTopology();
+
+   BcastBeginCopy(x.OccaMem(), sizeof(double)); // copy to 'shr_buf'
+
+   int req_counter = 0;
+   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
+   {
+      const int send_offset = shr_buf_offsets[nbr];
+      const int send_size = shr_buf_offsets[nbr+1] - send_offset;
+      if (send_size > 0)
+      {
+         void *send_buf;
+         if (host_shr_buf)
+         {
+            send_buf = host_shr_buf + send_offset*sizeof(double);
+         }
+         else
+         {
+            send_buf = (shr_buf.OccaMem() + send_offset*sizeof(double)).ptr();
+         }
+         MPI_Isend(send_buf, send_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
+                   41822, gtopo.GetComm(), &requests[req_counter++]);
+      }
+
+      const int recv_offset = ext_buf_offsets[nbr];
+      const int recv_size = ext_buf_offsets[nbr+1] - recv_offset;
+      if (recv_size > 0)
+      {
+         void *recv_buf;
+         if (host_ext_buf)
+         {
+            recv_buf = host_ext_buf + recv_offset*sizeof(double);
+         }
+         else
+         {
+            recv_buf = (ext_buf.OccaMem() + recv_offset*sizeof(double)).ptr();
+         }
+         MPI_Irecv(recv_buf, recv_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
+                   41822, gtopo.GetComm(), &requests[req_counter++]);
+      }
+   }
+
+   BcastLocalCopy(x.OccaMem(), y.OccaMem(), sizeof(double));
+
+   MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
+
+   BcastEndCopy(y.OccaMem(), sizeof(double)); // copy from 'ext_buf'
+}
+
+void OccaConformingProlongation::MultTranspose_(const Vector &x,
+                                                Vector &y) const
+{
+   const GroupTopology &gtopo = gc.GetGroupTopology();
+
+   ReduceBeginCopy(x.OccaMem(), sizeof(double)); // copy to 'ext_buf'
+
+   int req_counter = 0;
+   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
+   {
+      const int send_offset = ext_buf_offsets[nbr];
+      const int send_size = ext_buf_offsets[nbr+1] - send_offset;
+      if (send_size > 0)
+      {
+         void *send_buf;
+         if (host_ext_buf)
+         {
+            send_buf = host_ext_buf + send_offset*sizeof(double);
+         }
+         else
+         {
+            send_buf = (ext_buf.OccaMem() + send_offset*sizeof(double)).ptr();
+         }
+         MPI_Isend(send_buf, send_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
+                   41823, gtopo.GetComm(), &requests[req_counter++]);
+      }
+
+      const int recv_offset = shr_buf_offsets[nbr];
+      const int recv_size = shr_buf_offsets[nbr+1] - recv_offset;
+      if (recv_size > 0)
+      {
+         void *recv_buf;
+         if (host_shr_buf)
+         {
+            recv_buf = host_shr_buf + recv_offset*sizeof(double);
+         }
+         else
+         {
+            recv_buf = (shr_buf.OccaMem() + recv_offset*sizeof(double)).ptr();
+         }
+         MPI_Irecv(recv_buf, recv_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
+                   41823, gtopo.GetComm(), &requests[req_counter++]);
+      }
+   }
+
+   ReduceLocalCopy(x.OccaMem(), y.OccaMem(), sizeof(double));
+
+   MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
+
+   ReduceEndAssemble(y.OccaMem(), sizeof(double)); // assemble from 'shr_buf'
+}
+
+#endif // MFEM_USE_MPI
+
 } // namespace mfem::occa
 
 } // namespace mfem

backends/occa/fespace.hpp

@@ -35,10 +35,7 @@ protected:
    // SharedPtr<const mfem::Engine> engine;
    // mfem::FiniteElementSpace *fes;
 
-   Layout e_layout;
-
-   int *elementDofMap;
-   int *elementDofMapInverse;
+   SharedPtr<Layout> e_layout;
 
    ::occa::array<int> globalToLocalOffsets;
    ::occa::array<int> globalToLocalIndices;
@@ -50,10 +47,10 @@ protected:
    int globalDofs, localDofs;
    int vdim;
 
-   mfem::Operator *restrictionOp, *prolongationOp;
+   mutable Operator *prolongationOp, *restrictionOp;
 
    void SetupLocalGlobalMaps();
-   void SetupOperators();
+   void SetupOperators() const; // calls virtual methods of 'fes' !!!
    void SetupKernels();
 
 public:
@@ -64,8 +61,7 @@ public:
    virtual ~FiniteElementSpace();
 
    /// TODO: doxygen
-   const Engine &OccaEngine() const
-   { return *static_cast<const Engine *>(engine.Get()); }
+   const Engine &OccaEngine() const { return engine->As<Engine>(); }
 
    /// TODO: doxygen
    ::occa::device GetDevice(int idx = 0) const
@@ -79,13 +75,7 @@ public:
    Layout &OccaTrueVLayout() const
    { return *fes->GetTrueVLayout().As<Layout>(); }
 
-   Layout &OccaEVLayout() { return e_layout; }
-
-#ifdef MFEM_USE_MPI
-   bool isDistributed() const { return (OccaEngine().GetComm() != MPI_COMM_NULL); }
-#else
-   bool isDistributed() const { return false; }
-#endif
+   Layout &OccaEVLayout() { return *e_layout; }
 
    bool hasTensorBasis() const
    { return dynamic_cast<const mfem::TensorBasisElement*>(fes->GetFE(0)); }
@@ -105,20 +95,29 @@ public:
 
    int GetNE() const { return fes->GetNE(); }
 
-   const mfem::FiniteElementCollection* FEColl() const
+   const mfem::FiniteElementCollection *FEColl() const
    { return fes->FEColl(); }
-   const mfem::FiniteElement* GetFE(const int idx) const
+   const mfem::FiniteElement *GetFE(const int idx) const
    { return fes->GetFE(idx); }
 
-   const int* GetElementDofMap() const { return elementDofMap; }
-   const int* GetElementDofMapInverse() const { return elementDofMapInverse; }
+   virtual const mfem::Operator *GetProlongationOperator() const
+   { return prolongationOp; }
 
-   const mfem::Operator* GetRestrictionOperator() { return restrictionOp; }
-   const mfem::Operator* GetProlongationOperator() { return prolongationOp; }
+   virtual const mfem::Operator *GetRestrictionOperator() const
+   { return restrictionOp; }
+
+   virtual const mfem::Operator *GetInterpolationOperator(
+      const mfem::QuadratureSpace &qspace) const
+   { return NULL; /* FIXME */ }
+
+   virtual const mfem::Operator *GetGradientOperator(
+      const mfem::QuadratureSpace &qspace) const
+   { return NULL; /* FIXME */ }
 
    const ::occa::array<int> GetLocalToGlobalMap() const
    { return localToGlobalMap; }
 
+   /// L-vector to E-vector
    void GlobalToLocal(const Vector &globalVec, Vector &localVec) const
    {
       globalToLocalKernel(globalDofs,
@@ -127,6 +126,8 @@ public:
                           globalToLocalIndices,
                           globalVec.OccaMem(), localVec.OccaMem());
    }
+
+   /// E-vector to L-vector, transpose of GlobalToLocal
    void LocalToGlobal(const Vector &localVec, Vector &globalVec) const
    {
       localToGlobalKernel(globalDofs,
@@ -137,6 +138,69 @@ public:
    }
 };
 
+
+#ifdef MFEM_USE_MPI
+
+/// OCCA version of mfem::ConformingProlongationOperator
+class OccaConformingProlongation : public Operator
+{
+protected:
+   // size(shr_buf)=size(shr_ltdof)
+   // size(ext_buf)=size(ext_ldof)
+   Array shr_ltdof, ext_ldof;
+   mutable Array shr_buf, ext_buf;
+   mutable char *host_shr_buf, *host_ext_buf;
+   // Offsets into {shr,ext}_buf; size is num. neighbors, i.e.
+   // gc.GetGroupTopology().GetNumNeighbors():
+   int *shr_buf_offsets, *ext_buf_offsets;
+
+   ::occa::memory ltdof_ldof; // shared with the restriction operator
+
+   ::occa::memory unq_ltdof; // enumeration of the unique ltdofs in shr_ltdof
+   ::occa::memory unq_shr_i, unq_shr_j;
+
+   ::occa::kernel ExtractSubVector, SetSubVector, AddSubVector;
+
+   MPI_Request *requests;
+
+   const GroupCommunicator &gc;
+
+   // Kernel: copy ltdofs from 'src' to 'shr_buf' - prepare for send.
+   //         shr_buf[i] = src[shr_ltdof[i]]
+   void BcastBeginCopy(const ::occa::memory &src, std::size_t item_size) const;
+   // Kernel: copy ltdofs from 'src' to ldofs in 'dst'.
+   //         dst[ltdof_ldof[i]] = src[i]
+   void BcastLocalCopy(const ::occa::memory &src, ::occa::memory &dst,
+                       std::size_t item_size) const;
+   // Kernel: copy ext. dofs from 'ext_buf' to 'dst' - after recv.
+   //         dst[ext_ldof[i]] = ext_buf[i]
+   void BcastEndCopy(::occa::memory &dst, std::size_t item_size) const;
+
+   // Kernel: copy ext. dofs from 'src' to 'ext_buf' - prepare for send.
+   //         ext_buf[i] = src[ext_ldof[i]]
+   void ReduceBeginCopy(const ::occa::memory &src, std::size_t item_size) const;
+   // Kernel: copy owned ldofs from 'src' to ltdofs in 'dst'.
+   //         dst[i] = src[ltdof_ldof[i]]
+   void ReduceLocalCopy(const ::occa::memory &src, ::occa::memory &dst,
+                       std::size_t item_size) const;
+   // Kernel: assemble dofs from 'shr_buf' into to 'dst' - after recv.
+   //         dst[shr_ltdof[i]] += shr_buf[i]
+   void ReduceEndAssemble(::occa::memory &dst, std::size_t item_size) const;
+
+public:
+   OccaConformingProlongation(const FiniteElementSpace &ofes,
+                              const mfem::ParFiniteElementSpace &pfes,
+                              ::occa::memory ltdof_ldof_);
+
+   virtual ~OccaConformingProlongation();
+
+   // overrides
+   virtual void Mult_(const Vector &x, Vector &y) const;
+   virtual void MultTranspose_(const Vector &x, Vector &y) const;
+};
+
+#endif // MFEM_USE_MPI
+
 } // namespace mfem::occa
 
 } // namespace mfem

backends/occa/fespace.okl

@@ -25,10 +25,10 @@ typedef double *Local_t  @dim(NUM_VDIM, localEntries)  @dimOrder(1, 0);
 
 @kernel void GlobalToLocal(const int globalEntries,
                            const int localEntries,
-                           const int * restrict offsets,
-                           const int * restrict indices,
-                           const Global_t restrict globalX,
-                           Local_t restrict localX) {
+                           @restrict const int * offsets,
+                           @restrict const int * indices,
+                           @restrict const Global_t globalX,
+                           @restrict Local_t localX) {
 
   for (int i = 0; i < globalEntries; ++i; @tile(TILESIZE, @outer, @inner)) {
     if (i < globalEntries) {
@@ -46,10 +46,10 @@ typedef double *Local_t  @dim(NUM_VDIM, localEntries)  @dimOrder(1, 0);
 
 @kernel void LocalToGlobal(const int globalEntries,
                            const int localEntries,
-                           const int * restrict offsets,
-                           const int * restrict indices,
-                           const Local_t restrict localX,
-                           Global_t restrict globalX) {
+                           @restrict const int * offsets,
+                           @restrict const int * indices,
+                           @restrict const Local_t localX,
+                           @restrict Global_t globalX) {
 
   for (int i = 0; i < globalEntries; ++i; @tile(TILESIZE, @outer, @inner)) {
     if (i < globalEntries) {

backends/occa/geometry.okl

@@ -36,11 +36,11 @@ typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD, numElements);
 typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
 
 @kernel void InitGeometryInfo1D(const int numElements,
-                                const DofToQuadD1D_t restrict dofToQuadD,
-                                const Local1D_t restrict nodes,
-                                Jacobian1D_t restrict J,
-                                Jacobian1D_t restrict invJ,
-                                QLocal_t restrict detJ) {
+                                @restrict const DofToQuadD1D_t dofToQuadD,
+                                @restrict const Local1D_t nodes,
+                                @restrict Jacobian1D_t J,
+                                @restrict Jacobian1D_t invJ,
+                                @restrict QLocal_t detJ) {
 
   for (int e = 0; e < numElements; ++e; @outer) {
     @shared double s_nodes[NUM_DOFS];
@@ -70,11 +70,11 @@ typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
 }
 
 @kernel void InitGeometryInfo2D(const int numElements,
-                                const DofToQuadD2D_t restrict dofToQuadD,
-                                const Local2D_t restrict nodes,
-                                Jacobian2D_t restrict J,
-                                Jacobian2D_t restrict invJ,
-                                QLocal_t restrict detJ) {
+                                @restrict const DofToQuadD2D_t dofToQuadD,
+                                @restrict const Local2D_t nodes,
+                                @restrict Jacobian2D_t J,
+                                @restrict Jacobian2D_t invJ,
+                                @restrict QLocal_t detJ) {
 
   for (int e = 0; e < numElements; ++e; @outer) {
     @shared double s_nodes[2 * NUM_DOFS] @dim(2, NUM_DOFS);
@@ -119,11 +119,11 @@ typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
 }
 
 @kernel void InitGeometryInfo3D(const int numElements,
-                                const DofToQuadD3D_t restrict dofToQuadD,
-                                const Local3D_t restrict nodes,
-                                Jacobian3D_t restrict J,
-                                Jacobian3D_t restrict invJ,
-                                QLocal_t restrict detJ) {
+                                @restrict const DofToQuadD3D_t dofToQuadD,
+                                @restrict const Local3D_t nodes,
+                                @restrict Jacobian3D_t J,
+                                @restrict Jacobian3D_t invJ,
+                                @restrict QLocal_t detJ) {
 
   for (int e = 0; e < numElements; ++e; @outer) {
     @shared double s_nodes[3 * NUM_DOFS] @dim(3, NUM_DOFS);

backends/occa/gridfunc.cpp

@@ -59,135 +59,26 @@ std::map<std::string, ::occa::kernel> gridFunctionKernels;
    return kernel;
 }
 
-// OccaGridFunction::OccaGridFunction() :
-//    Vector(),
-//    ofespace(NULL),
-//    sequence(0) {}
-
-OccaGridFunction::OccaGridFunction(FiniteElementSpace *ofespace_)
-   : PArray(ofespace_->OccaVLayout()),
-     Array(ofespace_->OccaVLayout(), sizeof(double)),
-     Vector(ofespace_->OccaVLayout()),
-     ofespace(ofespace_),
-     sequence(0) {}
-
-// OccaGridFunction::OccaGridFunction(OccaFiniteElementSpace *ofespace_,
-//                                    OccaVectorRef ref) :
-//    OccaVector(ref),
-//    ofespace(ofespace_),
-//    sequence(0) {}
-
-OccaGridFunction::OccaGridFunction(const OccaGridFunction &v)
-   : PArray(v),
-     Array(v),
-     Vector(v),
-     ofespace(v.ofespace),
-     sequence(v.sequence) {}
-
-OccaGridFunction& OccaGridFunction::operator = (double value)
+void ToQuad(const IntegrationRule &ir, FiniteElementSpace &fespace, Vector &gf,
+            Vector &quadValues)
 {
-   Fill(value);
-   return *this;
-}
-
-OccaGridFunction& OccaGridFunction::operator = (const Vector &v)
-{
-   Assign<double>(v);
-   return *this;
-}
-
-// OccaGridFunction& OccaGridFunction::operator = (const OccaVectorRef &v)
-// {
-//    OccaVector::operator = (v);
-//    return *this;
-// }
-
-OccaGridFunction& OccaGridFunction::operator = (const OccaGridFunction &v)
-{
-   Assign<double>(v);
-   return *this;
-}
-
-// void OccaGridFunction::SetGridFunction(mfem::GridFunction &gf)
-// {
-//    Vector v = *this;
-//    gf.MakeRef(ofespace->GetFESpace(), v, 0);
-//    // Make gf the owner of the data
-//    v.Swap(gf);
-// }
-
-void OccaGridFunction::GetTrueDofs(Vector &v)
-{
-   const mfem::Operator *R = ofespace->GetRestrictionOperator();
-   if (!R)
-   {
-      v.MakeRef(*this);
-   }
-   else
-   {
-      v.Resize<double>(R->OutLayout(), NULL);
-      mfem::Vector mfem_v(v);
-      R->Mult(this->Wrap(), mfem_v);
-   }
-}
-
-void OccaGridFunction::SetFromTrueDofs(Vector &v)
-{
-   const mfem::Operator *P = ofespace->GetProlongationOperator();
-   if (!P)
-   {
-      MakeRef(v);
-   }
-   else
-   {
-      Resize<double>(P->OutLayout(), NULL);
-      mfem::Vector mfem_this(*this);
-      P->Mult(v.Wrap(), mfem_this);
-   }
-}
-
-mfem::FiniteElementSpace* OccaGridFunction::GetFESpace()
-{
-   return ofespace->GetFESpace();
-}
-
-const mfem::FiniteElementSpace* OccaGridFunction::GetFESpace() const
-{
-   return ofespace->GetFESpace();
-}
-
-void OccaGridFunction::ToQuad(const IntegrationRule &ir, Vector &quadValues)
-{
-   const Engine &engine = OccaLayout().OccaEngine();
+   const Engine &engine = fespace.OccaEngine();
    ::occa::device device = engine.GetDevice();
 
-   OccaDofQuadMaps &maps = OccaDofQuadMaps::Get(device, *ofespace, ir);
+   OccaDofQuadMaps &maps = OccaDofQuadMaps::Get(device, fespace, ir);
 
-   const int elements = ofespace->GetNE();
+   const int elements = fespace.GetNE();
    const int numQuad  = ir.GetNPoints();
-   quadValues.Resize<double>(*(new Layout(engine, numQuad * elements)), NULL);
+   quadValues.OccaResize(numQuad * elements, sizeof(double));
 
-   ::occa::kernel g2qKernel = GetGridFunctionKernel(device, *ofespace, ir);
+   ::occa::kernel g2qKernel = GetGridFunctionKernel(device, fespace, ir);
    g2qKernel(elements,
              maps.dofToQuad,
-             ofespace->GetLocalToGlobalMap(),
-             this->OccaMem(),
+             fespace.GetLocalToGlobalMap(),
+             gf.OccaMem(),
              quadValues.OccaMem());
 }
 
-void OccaGridFunction::Distribute(const Vector &v)
-{
-   if (ofespace->isDistributed())
-   {
-      mfem::Vector mfem_this(*this);
-      ofespace->GetProlongationOperator()->Mult(v.Wrap(), mfem_this);
-   }
-   else
-   {
-      *this = v;
-   }
-}
-
 } // namespace mfem::occa
 
 } // namespace mfem

backends/occa/gridfunc.hpp

@@ -22,14 +22,10 @@ namespace mfem
 {
 
 class IntegrationRule;
-class GridFunction;
 
 namespace occa
 {
 
-class OccaIntegrator;
-class OccaDofQuadMaps;
-
 // TODO: make this object part of the backend or the engine.
 extern std::map<std::string, ::occa::kernel> gridFunctionKernels;
 
@@ -38,41 +34,17 @@ extern std::map<std::string, ::occa::kernel> gridFunctionKernels;
                                      FiniteElementSpace &fespace,
                                      const mfem::IntegrationRule &ir);
 
-class OccaGridFunction : public Vector
-{
-protected:
-   FiniteElementSpace *ofespace;
-   long sequence;
-
-   ::occa::kernel gridFuncToQuad[3];
-
-public:
-   // OccaGridFunction();
-
-   OccaGridFunction(FiniteElementSpace *ofespace_);
-
-   // OccaGridFunction(FiniteElementSpace *ofespace_,
-   //                  OccaVectorRef ref);
-
-   OccaGridFunction(const OccaGridFunction &gf);
-
-   OccaGridFunction& operator = (double value);
-   OccaGridFunction& operator = (const Vector &v);
-   // OccaGridFunction& operator = (const OccaVectorRef &v);
-   OccaGridFunction& operator = (const OccaGridFunction &gf);
-
-   // void SetGridFunction(mfem::GridFunction &gf);
-
-   void GetTrueDofs(Vector &v);
-   void SetFromTrueDofs(Vector &v);
-
-   mfem::FiniteElementSpace* GetFESpace();
-   const mfem::FiniteElementSpace* GetFESpace() const;
-
-   void ToQuad(const mfem::IntegrationRule &ir, Vector &quadValues);
-
-   void Distribute(const Vector &v);
-};
+// ToQuad version without the deprecated class.
+//
+// FIXME: This is the action of a global B matrix, mapping L-vector to Q-vector,
+//        so it should be made into an operator that can be constructed by the
+//        FE space class. A batched version, where only a subset of the elements
+//        are processed should be defined as well.
+//
+//        The abstract operator construction method in the FE space class is:
+//           PFiniteElementSpace::GetInterpolationOperator(...)
+void ToQuad(const IntegrationRule &ir, FiniteElementSpace &ofespace, Vector &gf,
+            Vector &quadValues);
 
 } // namespace mfem::occa
 

backends/occa/gridfunc.okl

@@ -11,14 +11,14 @@
 
 #include "mfem-occa://defines.okl"
 
-#if USING_TENSOR_OPS
-#  if OCCA_USING_CPU
+#ifdef USING_TENSOR_OPS
+#  ifdef OCCA_USING_CPU
 #    include "mfem-occa://gridfunc/tensor/cpu.okl"
 #  else
 #    include "mfem-occa://gridfunc/tensor/gpuHighOrder.okl"
 #  endif
 #else
-#  if OCCA_USING_CPU
+#  ifdef OCCA_USING_CPU
 #    include "mfem-occa://gridfunc/simplex/cpu.okl"
 #  else
 #    include "mfem-occa://gridfunc/simplex/gpuHighOrder.okl"

backends/occa/gridfunc/simplex/cpu.okl

@@ -13,10 +13,10 @@
 
 //---[ 2D ]-----------------------------
 @kernel void GridFuncToQuad2D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap_t restrict l2gMap,
-                              const double * restrict gf,
-                              QVLocal_t restrict out) {
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal_t out) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {
@@ -39,10 +39,10 @@
 
 //---[ 3D ]-----------------------------
 @kernel void GridFuncToQuad3D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap_t restrict l2gMap,
-                              const double * restrict gf,
-                              QVLocal_t restrict out) {
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal_t out) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {

backends/occa/gridfunc/simplex/gpuHighOrder.okl

@@ -13,10 +13,10 @@
 
 //---[ 2D ]-----------------------------
 @kernel void GridFuncToQuad2D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap_t restrict l2gMap,
-                              const double * restrict gf,
-                              QVLocal_t restrict out) {
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal_t out) {
   for (int e = 0; e < numElements; ++e; @outer) {
     @shared double s_gf[NUM_VDIM][NUM_DOFS];
 
@@ -47,10 +47,10 @@
 
 //---[ 3D ]-----------------------------
 @kernel void GridFuncToQuad3D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap_t restrict l2gMap,
-                              const double * restrict gf,
-                              QVLocal_t restrict out) {
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal_t out) {
   for (int e = 0; e < numElements; ++e; @outer) {
     @shared double s_gf[NUM_VDIM][NUM_DOFS];
 

backends/occa/gridfunc/tensor/cpu.okl

@@ -13,10 +13,10 @@
 
 //---[ 1D ]-----------------------------
 @kernel void GridFuncToQuad1D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap1D_t restrict l2gMap,
-                              const double * restrict gf,
-                              QVLocal1D_t restrict out) {
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap1D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal1D_t out) {
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {
       double r_out[NUM_VDIM][NUM_QUAD_1D];
@@ -47,10 +47,10 @@
 
 //---[ 2D ]-----------------------------
 @kernel void GridFuncToQuad2D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap2D_t restrict l2gMap,
-                              const double * restrict gf,
-                              QVLocal2D_t restrict out) {
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap2D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal2D_t out) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {
@@ -106,10 +106,10 @@
 
 //---[ 3D ]-----------------------------
 @kernel void GridFuncToQuad3D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap3D_t restrict l2gMap,
-                              const double * restrict gf,
-                              QVLocal3D_t restrict out) {
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap3D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal3D_t out) {
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {
       double out_xyz[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D];

backends/occa/gridfunc/tensor/gpuHighOrder.okl

@@ -13,10 +13,10 @@
 
 //---[ 1D ]-----------------------------
 @kernel void GridFuncToQuad1D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap1D_t restrict l2gMap,
-                              const double * restrict gf,
-                              QLocal1D_t restrict out) {
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap1D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QLocal1D_t out) {
   // Iterate over elements
   for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
     @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
@@ -54,10 +54,10 @@
 
 //---[ 2D ]-----------------------------
 @kernel void GridFuncToQuad2D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap2D_t restrict l2gMap,
-                              const double * restrict gf,
-                              QLocal2D_t restrict out) {
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap2D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QLocal2D_t out) {
   // Iterate over elements
   for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
     // Store dof <--> quad mappings
@@ -109,10 +109,10 @@
 
 //---[ 3D ]-----------------------------
 @kernel void GridFuncToQuad3D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap3D_t restrict l2gMap,
-                              const double * restrict gf,
-                              QLocal3D_t restrict out) {
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap3D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QLocal3D_t out) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     // Store dof <--> quad mappings

backends/occa/interpolation.cpp

@@ -20,94 +20,51 @@ namespace mfem
 namespace occa
 {
 
-void CreateRPOperators(Layout &v_layout, Layout &t_layout,
-                       const mfem::SparseMatrix *R, const mfem::Operator *P,
-                       mfem::Operator *&OccaR, mfem::Operator *&OccaP)
-{
-   if (!P)
-   {
-      OccaR = new IdentityOperator(t_layout);
-      OccaP = new IdentityOperator(t_layout);
-      return;
-   }
-
-   const mfem::SparseMatrix *pmat = dynamic_cast<const mfem::SparseMatrix*>(P);
-   ::occa::device device = v_layout.OccaEngine().GetDevice();
-
-   if (R)
-   {
-      OccaSparseMatrix *occaR =
-         CreateMappedSparseMatrix(v_layout, t_layout, *R);
-      ::occa::array<int> reorderIndices = occaR->reorderIndices;
-      delete occaR;
-
-      OccaR = new RestrictionOperator(v_layout, t_layout, reorderIndices);
-   }
-
-   if (pmat)
-   {
-      const mfem::SparseMatrix *pmatT = Transpose(*pmat);
-
-      OccaSparseMatrix *occaP  =
-         CreateMappedSparseMatrix(t_layout, v_layout, *pmat);
-      OccaSparseMatrix *occaPT =
-         CreateMappedSparseMatrix(v_layout, t_layout, *pmatT);
-
-      OccaP = new ProlongationOperator(*occaP, *occaPT);
-   }
-   else
-   {
-      OccaP = new ProlongationOperator(t_layout, v_layout, P);
-   }
-}
-
 RestrictionOperator::RestrictionOperator(Layout &in_layout, Layout &out_layout,
-                                         ::occa::array<int> indices) :
-   Operator(in_layout, out_layout)
+                                         ::occa::array<int> indices)
+   : Operator(in_layout, out_layout)
 {
-
-   entries     = indices.size() / 2;
    trueIndices = indices;
 
-   // FIXME: paths ...
    ::occa::device device = in_layout.OccaEngine().GetDevice();
    const std::string &okl_path = in_layout.OccaEngine().GetOklPath();
-   const std::string &okl_defines = in_layout.OccaEngine().GetOklDefines();
    multOp = device.buildKernel(okl_path + "mappings.okl",
                                "ExtractSubVector",
-                               "defines: { TILESIZE: 256 }" + okl_defines);
+                               "defines: { TILESIZE: 256 }");
 
    multTransposeOp = device.buildKernel(okl_path + "mappings.okl",
                                         "SetSubVector",
-                                        "defines: { TILESIZE: 256 }" +
-                                        okl_defines);
+                                        "defines: { TILESIZE: 256 }");
 }
 
 void RestrictionOperator::Mult_(const Vector &x, Vector &y) const
 {
-   multOp(entries, trueIndices, x.OccaMem(), y.OccaMem());
+   // y[i] = x[trueIndices[i]]
+   multOp(height, trueIndices, x.OccaMem(), y.OccaMem());
 }
 
 void RestrictionOperator::MultTranspose_(const Vector &x, Vector &y) const
 {
-   y.Fill<double>(0.0);
-   multTransposeOp(entries, trueIndices, x.OccaMem(), y.OccaMem());
+   y.OccaFill<double>(0.0);
+   // y[trueIndices[i]] = x[i]
+   multTransposeOp(height, trueIndices, x.OccaMem(), y.OccaMem());
 }
 
 ProlongationOperator::ProlongationOperator(OccaSparseMatrix &multOp_,
-                                           OccaSparseMatrix &multTransposeOp_) :
-   Operator(multOp_),
-   pmat(NULL),
-   multOp(multOp_),
-   multTransposeOp(multTransposeOp_) {}
+                                           OccaSparseMatrix &multTransposeOp_)
+   : Operator(multOp_),
+     pmat(NULL),
+     multOp(multOp_),
+     multTransposeOp(multTransposeOp_)
+{ }
 
 ProlongationOperator::ProlongationOperator(Layout &in_layout,
                                            Layout &out_layout,
-                                           const mfem::Operator *pmat_) :
-   Operator(in_layout, out_layout),
-   pmat(pmat_),
-   multOp(*this),
-   multTransposeOp(*this)
+                                           const mfem::Operator *pmat_)
+   : Operator(in_layout, out_layout),
+     pmat(pmat_),
+     multOp(*this),
+     multTransposeOp(*this)
 { }
 
 void ProlongationOperator::Mult_(const Vector &x, Vector &y) const

backends/occa/interpolation.hpp

@@ -27,16 +27,10 @@ namespace mfem
 namespace occa
 {
 
-// [MISSING] Proper destructors
-void CreateRPOperators(Layout &v_layout, Layout &t_layout,
-                       const mfem::SparseMatrix *R, const mfem::Operator *P,
-                       mfem::Operator *&OccaR, mfem::Operator *&OccaP);
-
 class RestrictionOperator : public Operator
 {
 protected:
-   int entries;
-   ::occa::array<int> trueIndices;
+   ::occa::array<int> trueIndices; // ldof = trueIndices[ltdof]
    ::occa::kernel multOp, multTransposeOp;
 
 public:

backends/occa/layout.hpp

@@ -39,8 +39,7 @@ public:
    const Engine &OccaEngine() const
    { return *static_cast<const Engine *>(engine.Get()); }
 
-   ::occa::memory Alloc(std::size_t bytes) const
-   { return OccaEngine().GetDevice().malloc(bytes); }
+   void OccaResize(std::size_t new_size) { size = new_size; }
 
    virtual ~Layout() { }
 

backends/occa/mappings.okl

@@ -16,38 +16,60 @@
 */
 
 @kernel void ExtractSubVector(const int entries,
-                              const int * restrict indices,
-                              const double * restrict in,
-                              double * restrict out) {
+                              @restrict const int *indices,
+                              @restrict const double *in,
+                              @restrict double *out) {
 
   for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
     if (i < entries) {
-      out[i] = in[indices[i]];
+      out[i] = in[indices[i]]; // indices can be repeated
     }
   }
 }
 
 @kernel void SetSubVector(const int entries,
-                          const int * restrict indices,
-                          const double * restrict in,
-                          double * restrict out) {
+                          @restrict const int *indices,
+                          @restrict const double *in,
+                          @restrict double *out) {
 
   for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
     if (i < entries) {
-      out[indices[i]] = in[i];
+      out[indices[i]] = in[i]; // indices CANNOT be repeated
+    }
+  }
+}
+
+@kernel void AddSubVector(const int num_unique_dst_indices,
+                          @restrict const int *unique_dst_indices,
+                          @restrict const int *unique_to_src_offsets,
+                          @restrict const int *unique_to_src_indices,
+                          @restrict const double *src,
+                          @restrict double *dst) {
+
+  for (int i = 0; i < num_unique_dst_indices; ++i;
+       @tile(TILESIZE, @outer, @inner)) {
+
+    if (i < num_unique_dst_indices) {
+      const int dst_idx = unique_dst_indices[i];
+      double sum = dst[dst_idx];
+      const int end = unique_to_src_offsets[i+1];
+      for (int j = unique_to_src_offsets[i]; j != end; ++j) {
+        sum += src[unique_to_src_indices[j]];
+      }
+      dst[dst_idx] = sum;
     }
   }
 }
 
 @kernel void MapSubVector(const int entries,
-                          const int * restrict indices,
-                          const double * restrict in,
-                          double * restrict out) {
+                          @restrict const int *indices,
+                          @restrict const double *in,
+                          @restrict double *out) {
 
   for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
     if (i < entries) {
-      const int fromIdx = indices[2*i + 0];
-      const int toIdx   = indices[2*i + 1];
+      const int fromIdx = indices[2*i + 0]; // fromIdx indices can be repeated
+      const int toIdx   = indices[2*i + 1]; // toIdx indices CANNOT be repeated
       out[toIdx] = in[fromIdx];
     }
   }

backends/occa/mass/simplex/cpu.okl

@@ -0,0 +1,122 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian2D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+      oper(q, e) = quadWeights[q] * COEFF * ((J11 * J22) - (J21 * J12));
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD2D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD2D_t quadToDofD,
+                       @restrict const QLocal_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double r_sol[NUM_DOFS];
+
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        r_sol[d] = 0;
+      }
+
+      for (int q = 0; q < NUM_QUAD; ++q) {
+        double s = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          s += solIn(d, e) * quadToDof(d, q);
+        }
+        s *= oper(q, e);
+
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          r_sol[d] += (s * quadToDof(d, q));
+        }
+      }
+
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        solOut(d, e) += r_sol[d];
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian3D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+      oper(q, e) = quadWeights[q] * COEFF * detJ;
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD3D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD3D_t quadToDofD,
+                       @restrict const QLocal_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double r_sol[NUM_DOFS];
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        r_sol[d] = 0;
+      }
+
+      for (int q = 0; q < NUM_QUAD; ++q) {
+        double s = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          s += solIn(d, e) * quadToDof(d, q);
+        }
+        s *= oper(q, e);
+
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          r_sol[d] += (s * quadToDof(d, q));
+        }
+      }
+
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        solOut(d, e) += r_sol[d];
+      }
+    }
+  }
+}
+//======================================

backends/occa/mass/simplex/gpuHighOrder.okl

@@ -0,0 +1,129 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian2D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD; q += A2_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+            oper(q, e) = quadWeights[q] * COEFF * ((J11 * J22) - (J21 * J12));
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD2D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD2D_t quadToDofD,
+                       @restrict const QLocal_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_sol[NUM_QUAD];
+
+    for (int qOff = 0; qOff < M2_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M2_INNER_BATCH) {
+        double s = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          s += solIn(d, e) * dofToQuad(d, q);
+        }
+        s_sol[q] = s * oper(q, e);
+      }
+    }
+
+    for (int dOff = 0; dOff < M2_INNER_BATCH; ++dOff) {
+      for (int d = dOff; d < NUM_DOFS; d += M2_INNER_BATCH) {
+        double r_sol = 0;
+        for (int q = 0; q < NUM_QUAD; ++q) {
+          r_sol += (s_sol[q] * quadToDof(d, q));
+        }
+        solOut(d, e) += r_sol;
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian3D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD; q += A3_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+            oper(q, e) = quadWeights[q] * COEFF * detJ;
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD3D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD3D_t quadToDofD,
+                       @restrict const QLocal_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_sol[NUM_QUAD];
+
+    for (int qOff = 0; qOff < M3_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M3_INNER_BATCH) {
+        double s = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          s += solIn(d, e) * dofToQuad(d, q);
+        }
+        s_sol[q] = s * oper(q, e);
+      }
+    }
+
+    for (int dOff = 0; dOff < M3_INNER_BATCH; ++dOff) {
+      for (int d = dOff; d < NUM_DOFS; d += M3_INNER_BATCH) {
+        double r_sol = 0;
+        for (int q = 0; q < NUM_QUAD; ++q) {
+          r_sol += (s_sol[q] * quadToDof(d, q));
+        }
+        solOut(d, e) += r_sol;
+      }
+    }
+  }
+}
+//======================================

backends/occa/mass/tensor/cpu.okl

@@ -0,0 +1,281 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void Assemble1D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian1D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
+      const double detJ = J(q, e);
+      oper(q, e) = quadWeights[q] * COEFF * detJ;
+    }
+  }
+}
+
+@kernel void MultAdd1D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal1D_t oper,
+                       @restrict const DLocal1D_t solIn,
+                       @restrict DLocal1D_t solOut) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double sol_x[NUM_QUAD_1D];
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        sol_x[qx] = 0;
+      }
+      // sol_x{qx} = dofToQuad{qx,dx} * sol{dx}
+      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+        const double s = solIn(dx, e);
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          sol_x[qx] += s * dofToQuad(qx, dx);
+        }
+      }
+      // sol_x{q} *= oper{q}
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        sol_x[qx] *= oper(qx, e);
+      }
+      // sol{dx} = quadToDof{dx,qx} * sol_x{qx}
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          solOut(dx, e) += sol_x[qx] * quadToDof(dx, qx);
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian2D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_2D; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+      const double detJ = ((J11 * J22) - (J21 * J12));
+
+      oper(q, e) = quadWeights[q] * COEFF * detJ;
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal2D_t oper,
+                       @restrict const DLocal2D_t solIn,
+                       @restrict DLocal2D_t solOut) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double sol_xy[NUM_QUAD_1D][NUM_QUAD_1D];
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          sol_xy[qy][qx] = 0;
+        }
+      }
+
+      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+        double sol_x[NUM_QUAD_1D];
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          sol_x[qy] = 0;
+        }
+
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          const double s = solIn(dx, dy, e);
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_x[qx] += dofToQuad(qx, dx) * s;
+          }
+        }
+
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          const double d2q = dofToQuad(qy, dy);
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_xy[qy][qx] += d2q * sol_x[qx];
+          }
+        }
+      }
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          sol_xy[qy][qx] *= oper(qx, qy, e);
+        }
+      }
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        double sol_x[NUM_DOFS_1D];
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          sol_x[dx] = 0;
+        }
+
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          const double s = sol_xy[qy][qx];
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            sol_x[dx] += quadToDof(dx, qx) * s;
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          const double q2d = quadToDof(dy, qy);
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            solOut(dx, dy, e) += q2d * sol_x[dx];
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian3D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_3D; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+      oper(q, e) = quadWeights[q] * COEFF * detJ;
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal3D_t oper,
+                       @restrict const DLocal3D_t solIn,
+                       @restrict DLocal3D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double sol_xyz[NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D];
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_xyz[qz][qy][qx] = 0;
+          }
+        }
+      }
+
+      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+        double sol_xy[NUM_QUAD_1D][NUM_QUAD_1D];
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_xy[qy][qx] = 0;
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          double sol_x[NUM_QUAD_1D];
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_x[qx] = 0;
+          }
+
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double s = solIn(dx, dy, dz, e);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              sol_x[qx] += dofToQuad(qx, dx) * s;
+            }
+          }
+
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            const double wy = dofToQuad(qy, dy);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              sol_xy[qy][qx] += wy * sol_x[qx];
+            }
+          }
+        }
+
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          const double wz = dofToQuad(qz, dz);
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
+            }
+          }
+        }
+      }
+
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_xyz[qz][qy][qx] *= oper(qx, qy, qz, e);
+          }
+        }
+      }
+
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        double sol_xy[NUM_DOFS_1D][NUM_DOFS_1D];
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            sol_xy[dy][dx] = 0;
+          }
+        }
+
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          double sol_x[NUM_DOFS_1D];
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            sol_x[dx] = 0;
+          }
+
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            const double s = sol_xyz[qz][qy][qx];
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              sol_x[dx] += quadToDof(dx, qx) * s;
+            }
+          }
+
+          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+            const double wy = quadToDof(dy, qy);
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              sol_xy[dy][dx] += wy * sol_x[dx];
+            }
+          }
+        }
+
+        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+          const double wz = quadToDof(dz, qz);
+          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              solOut(dx, dy, dz, e) += wz * sol_xy[dy][dx];
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/mass/tensor/gpuHighOrder.okl

@@ -0,0 +1,341 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void Assemble1D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian1D_t J,
+                        COEFF_ARGS
+                        QLocal_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A1_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A1_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
+          oper(q, e) = quadWeights[q] * COEFF * J(q, e);
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd1D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal1D_t oper,
+                       @restrict const DLocal1D_t solIn,
+                       @restrict DLocal1D_t solOut) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    @exclusive double r_sol[NUM_QUAD_1D];
+
+    for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+      for (int i = el; i < NUM_QUAD_DOFS_1D; i += M1_INNER_ELEMENT_BATCH) {
+        s_dofToQuad[i] = dofToQuad[i];
+        s_quadToDof[i] = quadToDof[i];
+      }
+    }
+
+    for (int b = 0; b < M1_OUTER_ELEMENT_BATCH; ++b) {
+      for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+        const int e = eOff + b*M1_INNER_ELEMENT_BATCH + el;
+        if (e < numElements) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            r_sol[qx] = 0;
+          }
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double s = solIn(dx, e);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              r_sol[qx] += s * s_dofToQuad(qx, dx);
+            }
+          }
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            r_sol[qx] *= oper(qx, e);
+          }
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            double s = 0;
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              s += r_sol[qx] * s_quadToDof(dx, qx);
+            }
+            solOut(dx, e) += s;
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian2D_t J,
+                        COEFF_ARGS
+                        QLocal_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD_2D; q += A2_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+            oper(q, e) = quadWeights[q] * COEFF * ((J11 * J22) - (J21 * J12));
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal2D_t oper,
+                       @restrict const DLocal2D_t solIn,
+                       @restrict DLocal2D_t solOut) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    // Store xy planes in @shared memory
+    @shared double s_xy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_xy2[NUM_QUAD_2D] @dim(NUM_QUAD_1D, NUM_QUAD_1D);
+
+    @exclusive double r_x[NUM_MAX_1D];
+
+    for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+      for (int id = x; id < NUM_QUAD_DOFS_1D; id += NUM_MAX_1D) {
+        s_dofToQuad[id]  = dofToQuad[id];
+        s_quadToDof[id]  = quadToDof[id];
+      }
+    }
+
+    for (int e = eOff; e < (eOff + M2_ELEMENT_BATCH); ++e) {
+      if (e < numElements) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if (dx < NUM_DOFS_1D) {
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              s_xy(dx, qy) = 0;
+            }
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              r_x[dy] = solIn(dx, dy, e);
+            }
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              double xy = 0;
+              for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+                xy += r_x[dy] * s_dofToQuad(qy, dy);
+              }
+              s_xy(dx, qy) = xy;
+            }
+          }
+        }
+        for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+          if (qy < NUM_QUAD_1D) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              double s = 0;
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                s += s_xy(dx, qy) * s_dofToQuad(qx, dx);
+              }
+              s_xy2(qx, qy) = s * oper(qx, qy, e);
+            }
+          }
+        }
+
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if (qx < NUM_QUAD_1D) {
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              s_xy(dy, qx) = 0;
+            }
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              r_x[qy] = s_xy2(qx, qy);
+            }
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              double s = 0;
+              for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+                s += r_x[qy] * s_quadToDof(dy, qy);
+              }
+              s_xy(dy, qx) = s;
+            }
+          }
+        }
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if (dx < NUM_DOFS_1D) {
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              double s = 0;
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                s += (s_xy(dy, qx) * s_quadToDof(dx, qx));
+              }
+              solOut(dx, dy, e) += s;
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian3D_t J,
+                        COEFF_ARGS
+                        QLocal_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD_3D; q += A3_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+            oper(q, e) = quadWeights[q] * COEFF * detJ;
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal3D_t oper,
+                       @restrict const DLocal3D_t solIn,
+                       @restrict DLocal3D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    // Store xy planes in @shared memory
+    @shared double s_xy[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
+
+    // Store z axis as registers
+    @exclusive double r_z[NUM_QUAD_1D];
+    @exclusive double r_z2[NUM_DOFS_1D];
+
+    for (int y = 0; y < NUM_MAX_1D; ++y; @inner) {
+      for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+        const int id = (y * NUM_MAX_1D) + x;
+        // Fetch Q <--> D maps
+        if (id < NUM_QUAD_DOFS_1D) {
+          s_dofToQuad[id]  = dofToQuad[id];
+          s_quadToDof[id]  = quadToDof[id];
+        }
+        // Initialize our Z axis
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          r_z[qz] = 0;
+        }
+        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+          r_z2[dz] = 0;
+        }
+      }
+    }
+
+    for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+      for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+        if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+          for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+            const double s = solIn(dx, dy, dz, e);
+            // Calculate D -> Q in the Z axis
+            for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+              r_z[qz] += s * s_dofToQuad(qz, dz);
+            }
+          }
+        }
+      }
+    }
+    // For each xy plane
+    for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+      // Fill xy plane at given z position
+      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+            s_xy(dx, dy) = r_z[qz];
+          }
+        }
+      }
+      // Calculate Dxyz, xDyz, xyDz in plane
+      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
+            double s = 0;
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              const double wy = s_dofToQuad(qy, dy);
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                const double wx = s_dofToQuad(qx, dx);
+                s += wx * wy * s_xy(dx, dy);
+              }
+            }
+
+            s *= oper(qx, qy, qz, e);
+
+            for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+              const double wz  = s_quadToDof(dz, qz);
+              r_z2[dz] += wz * s;
+            }
+          }
+        }
+      }
+      @barrier("s_xy_sync_1");
+    }
+    // Iterate over xy planes to compute solution
+    for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+      // Place xy plane in @shared memory
+      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
+            s_xy(qx, qy) = r_z2[dz];
+          }
+        }
+      }
+      // Finalize solution in xy plane
+      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+            double solZ = 0;
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              const double wy = s_quadToDof(dy, qy);
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                const double wx = s_quadToDof(dx, qx);
+                solZ += wx * wy * s_xy(qx, qy);
+              }
+            }
+            solOut(dx, dy, dz, e) += solZ;
+          }
+        }
+      }
+      @barrier("s_xy_sync_2");
+    }
+  }
+}
+//======================================

backends/occa/operator.cpp

@@ -73,15 +73,21 @@ void OccaConstrainedOperator::Setup(::occa::device device_,
    own_A = own_A_;
 
    constraintIndices = constraintList_.Size();
-   constraintList = constraintList_.Get_PArray()->As<Array>().OccaMem();
+   if (constraintList_.Size() > 0)
+   {
+      constraintList = constraintList_.Get_PArray()->As<Array>().OccaMem();
+   }
+   else
+   {
+      // constraintList is not used
+   }
 }
 
 void OccaConstrainedOperator::EliminateRHS(const Vector &x, Vector &b) const
 {
-   const std::string &okl_defines = InLayout_().OccaEngine().GetOklDefines();
-   ::occa::kernel mapDofs = mapDofBuilder.build(device, okl_defines);
+   ::occa::kernel mapDofs = mapDofBuilder.build(device);
 
-   w.Fill<double>(0.0);
+   w.OccaFill(0.0);
 
    if (constraintIndices)
    {
@@ -107,11 +113,12 @@ void OccaConstrainedOperator::Mult_(const Vector &x, Vector &y) const
       return;
    }
 
-   const std::string &okl_defines = InLayout_().OccaEngine().GetOklDefines();
-   ::occa::kernel mapDofs   = mapDofBuilder.build(device, okl_defines);
-   ::occa::kernel clearDofs = clearDofBuilder.build(device, okl_defines);
+   ::occa::kernel mapDofs   = mapDofBuilder.build(device);
+   ::occa::kernel clearDofs = clearDofBuilder.build(device);
 
-   z.Assign<double>(x); // z = x
+   // z.OccaAssign(x); // z = x
+   // Is Axpy faster than DtoD copy on Volta?
+   z.Axpby(1.0, x, 0.0, x);
 
    clearDofs(constraintIndices, z.OccaMem(), constraintList);
 

backends/occa/operator.hpp

@@ -37,11 +37,9 @@ public:
    Operator(Layout &in_layout, Layout &out_layout)
       : mfem::Operator(in_layout, out_layout) { }
 
-   Layout &InLayout_() const
-   { return *static_cast<Layout*>(in_layout.Get()); }
+   Layout &InLayout_() const { return in_layout->As<Layout>(); }
 
-   Layout &OutLayout_() const
-   { return *static_cast<Layout*>(out_layout.Get()); }
+   Layout &OutLayout_() const { return out_layout->As<Layout>(); }
 
    virtual void Mult_(const Vector &x, Vector &y) const = 0;
 

backends/occa/sparse.okl

@@ -16,11 +16,11 @@
 */
 
 @kernel void Mult(const int entries,
-                  const int * restrict offsets,
-                  const int * restrict indices,
-                  const double * restrict weights,
-                  const double * restrict in,
-                  double * restrict out) {
+                  @restrict const int *offsets,
+                  @restrict const int *indices,
+                  @restrict const double *weights,
+                  @restrict const double *in,
+                  @restrict double *out) {
 
   for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
     if (i < entries) {
@@ -36,12 +36,12 @@
 }
 
 @kernel void MappedMult(const int entries,
-                        const int * restrict offsets,
-                        const int * restrict indices,
-                        const double * restrict weights,
-                        const int * restrict outIndices,
-                        const double * restrict in,
-                        double * restrict out) {
+                        @restrict const int *offsets,
+                        @restrict const int *indices,
+                        @restrict const double *weights,
+                        @restrict const int *outIndices,
+                        @restrict const double *in,
+                        @restrict double *out) {
 
   for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
     if (i < entries) {

backends/occa/sparsemat.cpp

@@ -22,54 +22,26 @@ namespace occa
 
 OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
                                    const mfem::SparseMatrix &m,
-                                   const ::occa::properties &props) :
-   Operator(in_layout, out_layout)
+                                   const ::occa::properties &props)
+   : Operator(in_layout, out_layout)
 {
-
    Setup(in_layout.OccaEngine().GetDevice(), m, props);
 }
 
-OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                                   const mfem::SparseMatrix &m,
-                                   ::occa::array<int> reorderIndices_,
-                                   ::occa::array<int> mappedIndices_,
-                                   const ::occa::properties &props) :
-   Operator(in_layout, out_layout)
-{
-
-   Setup(in_layout.OccaEngine().GetDevice(), m,
-         reorderIndices, mappedIndices_, props);
-}
-
-OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                                   ::occa::array<int> offsets_,
-                                   ::occa::array<int> indices_,
-                                   ::occa::array<double> weights_,
-                                   const ::occa::properties &props) :
-   Operator(in_layout, out_layout),
-   offsets(offsets_),
-   indices(indices_),
-   weights(weights_)
-{
-
-   SetupKernel(in_layout.OccaEngine().GetDevice(), props);
-}
-
 OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
                                    ::occa::array<int> offsets_,
                                    ::occa::array<int> indices_,
                                    ::occa::array<double> weights_,
                                    ::occa::array<int> reorderIndices_,
                                    ::occa::array<int> mappedIndices_,
-                                   const ::occa::properties &props) :
-   Operator(in_layout, out_layout),
-   offsets(offsets_),
-   indices(indices_),
-   weights(weights_),
-   reorderIndices(reorderIndices_),
-   mappedIndices(mappedIndices_)
+                                   const ::occa::properties &props)
+   : Operator(in_layout, out_layout),
+     offsets(offsets_),
+     indices(indices_),
+     weights(weights_),
+     reorderIndices(reorderIndices_),
+     mappedIndices(mappedIndices_)
 {
-
    SetupKernel(in_layout.OccaEngine().GetDevice(), props);
 }
 
@@ -84,6 +56,9 @@ void OccaSparseMatrix::Setup(::occa::device device, const SparseMatrix &m,
                              ::occa::array<int> mappedIndices_,
                              const ::occa::properties &props)
 {
+   MFEM_ASSERT(m.Finalized(), "");
+   MFEM_ASSERT(m.Height() == height, "");
+   MFEM_ASSERT(m.Width() == width, "");
 
    const int nnz = m.GetI()[height];
    offsets.allocate(device,
@@ -106,7 +81,6 @@ void OccaSparseMatrix::Setup(::occa::device device, const SparseMatrix &m,
 void OccaSparseMatrix::SetupKernel(::occa::device device,
                                    const ::occa::properties &props)
 {
-
    const bool hasOutIndices = mappedIndices.isInitialized();
 
    const ::occa::properties defaultProps("defines: {"
@@ -114,14 +88,13 @@ void OccaSparseMatrix::SetupKernel(::occa::device device,
                                          "}");
 
    const std::string &okl_path = InLayout_().OccaEngine().GetOklPath();
-   const std::string &okl_defines = InLayout_().OccaEngine().GetOklDefines();
    mapKernel = device.buildKernel(okl_path + "mappings.okl",
                                   "MapSubVector",
-                                  defaultProps + props + okl_defines);
+                                  defaultProps + props);
 
    multKernel = device.buildKernel(okl_path + "sparse.okl",
                                    hasOutIndices ? "MappedMult" : "Mult",
-                                   defaultProps + props + okl_defines);
+                                   defaultProps + props);
 }
 
 void OccaSparseMatrix::Mult_(const Vector &x, Vector &y) const

backends/occa/sparsemat.hpp

@@ -30,40 +30,12 @@ namespace occa
 /// TODO: doxygen
 class OccaSparseMatrix : public Operator
 {
-public:
+protected:
    ::occa::array<int> offsets, indices;
    ::occa::array<double> weights;
    ::occa::array<int> reorderIndices, mappedIndices;
    ::occa::kernel mapKernel, multKernel;
 
-   /// Construct an empty OccaSparseMatrix.
-   OccaSparseMatrix(const Operator &orig)
-      : Operator(orig) { }
-
-   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                    const mfem::SparseMatrix &m,
-                    const ::occa::properties &props = ::occa::properties());
-
-   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                    const mfem::SparseMatrix &m,
-                    ::occa::array<int> reorderIndices_,
-                    ::occa::array<int> mappedIndices_,
-                    const ::occa::properties &props = ::occa::properties());
-
-   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                    ::occa::array<int> offsets_,
-                    ::occa::array<int> indices_,
-                    ::occa::array<double> weights_,
-                    const ::occa::properties &props = ::occa::properties());
-
-   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                    ::occa::array<int> offsets_,
-                    ::occa::array<int> indices_,
-                    ::occa::array<double> weights_,
-                    ::occa::array<int> reorderIndices_,
-                    ::occa::array<int> mappedIndices_,
-                    const ::occa::properties &props = ::occa::properties());
-
    void Setup(::occa::device device, const mfem::SparseMatrix &m,
               const ::occa::properties &props);
 
@@ -75,6 +47,28 @@ public:
    void SetupKernel(::occa::device device,
                     const ::occa::properties &props);
 
+public:
+   /// Construct an empty OccaSparseMatrix.
+   OccaSparseMatrix(const Operator &orig)
+      : Operator(orig) { }
+
+   // Implicitly defined copy constructor.
+
+   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                    const mfem::SparseMatrix &m,
+                    const ::occa::properties &props = ::occa::properties());
+
+   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                    ::occa::array<int> offsets_,
+                    ::occa::array<int> indices_,
+                    ::occa::array<double> weights_,
+                    ::occa::array<int> reorderIndices_,
+                    ::occa::array<int> mappedIndices_,
+                    const ::occa::properties &props = ::occa::properties());
+
+   const ::occa::array<int> &GetReorderIndices() const
+   { return reorderIndices; }
+
    // override
    virtual void Mult_(const Vector &x, Vector &y) const;
 };

backends/occa/utils.okl

@@ -13,7 +13,7 @@ typedef double* Local_t @dim(numDofs, numElements);
 
 @kernel void InitLocalVector(const int numElements,
                              const int numDofs,
-                             Local_t restrict sol) {
+                             @restrict Local_t sol) {
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int d = 0; d < numDofs; ++d; @inner) {
       sol(d, e) = 0;

backends/occa/vector.cpp

@@ -45,17 +45,16 @@ void Vector::DoDotProduct(const PVector &x, void *result,
 
    MFEM_ASSERT(result_type_id == ScalarId<double>::value, "");
    double *res = (double *)result;
-   MFEM_ASSERT(dynamic_cast<const Vector *>(&x) != NULL, "invalid Vector type");
-   const Vector *xp = static_cast<const Vector *>(&x);
-   MFEM_ASSERT(this->Size() == xp->Size(), "");
-   *res = ::occa::linalg::dot<double, double, double>(this->slice, xp->slice);
+   const Vector &xp = x.As<Vector>();
+   MFEM_ASSERT(this->Size() == xp.Size(), "");
+   *res = ::occa::linalg::dot<double, double, double>(this->slice, xp.slice);
 
 #ifdef MFEM_USE_MPI
    double local_dot = *res;
    if (IsParallel())
    {
       MPI_Allreduce(&local_dot, res, 1, MPI_DOUBLE, MPI_SUM,
-                    OccaLayout().OccaEngine().GetComm());
+                    OccaEngine().GetComm());
    }
 #endif
 }
@@ -64,8 +63,6 @@ void Vector::DoAxpby(const void *a, const PVector &x,
                      const void *b, const PVector &y,
                      int ab_type_id)
 {
-   const std::string &okl_defines = OccaLayout().OccaEngine().GetOklDefines();
-
    //
    // TODO: move all kernel builders to class mfem::occa::Backend
    //
@@ -124,7 +121,7 @@ void Vector::DoAxpby(const void *a, const PVector &x,
    {
       if (db == 0.0)
       {
-         OccaFill(&da);
+         OccaFill(da);
       }
       else
       {
@@ -136,8 +133,7 @@ void Vector::DoAxpby(const void *a, const PVector &x,
          else
          {
             // *this = db * y
-            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice(),
-                                                         okl_defines);
+            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice());
             kernel((int)Size(), db, slice, yp->slice);
          }
       }
@@ -154,8 +150,7 @@ void Vector::DoAxpby(const void *a, const PVector &x,
          else
          {
             // *this = da * x
-            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice(),
-                                                         okl_defines);
+            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice());
             kernel((int)Size(), da, slice, xp->slice);
          }
       }
@@ -165,22 +160,19 @@ void Vector::DoAxpby(const void *a, const PVector &x,
          if (this->slice == xp->slice)
          {
             // *this = da * (*this) + db * y
-            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice(),
-                                                         okl_defines);
+            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice());
             kernel((int)Size(), da, db, slice, yp->slice);
          }
          else if (this->slice == yp->slice)
          {
             // *this = da * x + db * (*this)
-            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice(),
-                                                         okl_defines);
+            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice());
             kernel((int)Size(), db, da, slice, xp->slice);
          }
          else
          {
             // *this = da * x + db * y
-            ::occa::kernel kernel = axpby3_builder.build(slice.getDevice(),
-                                                         okl_defines);
+            ::occa::kernel kernel = axpby3_builder.build(slice.getDevice());
             kernel((int)Size(), da, db, slice, xp->slice, yp->slice);
          }
       }

backends/occa/vector.hpp

@@ -25,7 +25,12 @@ namespace mfem
 namespace occa
 {
 
+// FIXME: Once XL fixes this code quirk we can remove this #ifdef switch
+#ifdef __ibmxl__
+class Vector : public Array, public PVector
+#else
 class Vector : virtual public Array, public PVector
+#endif
 {
 protected:
    //
@@ -52,6 +57,10 @@ protected:
    // End: Virtual interface
 
 public:
+   Vector(const Engine &e)
+      : PArray(*(new Layout(e, 0))), Array(e), PVector(*layout)
+   { }
+
    Vector(Layout &lt)
       : PArray(lt), Array(lt, sizeof(double)), PVector(lt)
    { }
@@ -61,7 +70,7 @@ public:
    const mfem::Vector Wrap() const;
 
 #if defined(MFEM_USE_MPI)
-   bool IsParallel() const { return (OccaLayout().OccaEngine().GetComm() != MPI_COMM_NULL); }
+   bool IsParallel() const { return (OccaEngine().GetComm() != MPI_COMM_NULL); }
 #endif
 };
 

backends/occa/vmass/tensor/cpu.okl

@@ -0,0 +1,321 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void Assemble1D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian1D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
+      oper(q, e) = quadWeights[q] * COEFF * J(q, e);
+    }
+  }
+}
+
+@kernel void MultAdd1D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal_t oper,
+                       @restrict const DVLocal1D_t solIn,
+                       @restrict DVLocal1D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double sol_x[1][NUM_QUAD_1D];
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        sol_x[0][qx] = 0;
+      }
+      // sol_x{qx} = dofToQuad{qx,dx} * sol{dx}
+      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          sol_x[0][qx] += dofToQuad(qx, dx) * solIn(0, dx, e);
+        }
+      }
+      // sol_x{q} *= oper{q}
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        sol_x[0][qx] *= oper(qx, e);
+      }
+      // sol{dx} = quadToDof{dx,qx} * sol_x{qx}
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          solOut(0, dx, e) += sol_x[0][qx] * quadToDof(dx, qx);
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian2D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_2D; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+      oper(q, e) = quadWeights[q] * COEFF * ((J11 * J22) - (J21 * J12));
+    }
+  } // e
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal_t oper,
+                       @restrict const DVLocal2D_t solIn,
+                       @restrict DVLocal2D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy=0; dummy<1; ++dummy; @inner) {
+      double sol_xy[2][NUM_QUAD_1D][NUM_QUAD_1D];
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          sol_xy[0][qx][qy] = 0;
+          sol_xy[1][qx][qy] = 0;
+        }
+      }
+
+      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+        double sol_x[2][NUM_QUAD_1D];
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          sol_x[0][qy] = 0;
+          sol_x[1][qy] = 0;
+        }
+
+        // sol_x{vd, dx, qy} = dofToQuad{qy, dy} * sol{vd, dx, dy}
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            sol_x[0][qy] += dofToQuad(qy, dx) * solIn(0, dx, dy, e);
+            sol_x[1][qy] += dofToQuad(qy, dx) * solIn(1, dx, dy, e);
+          }
+        }
+
+        // sol_xy{qx, qy} = dofToQuad{qx, dx} * sol_x{dx, qy}
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          const double d2q = dofToQuad(qy, dy);
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_xy[0][qx][qy] += d2q * sol_x[0][qx];
+            sol_xy[1][qx][qy] += d2q * sol_x[1][qx];
+          }
+        }
+      } // dy
+
+      // sol_xy{qx, qy} = sol_xy{q} *= oper{q, e}
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          const int q = QUAD_2D_ID(qx, qy);
+          sol_xy[0][qx][qy] *= oper(q, e);
+          sol_xy[1][qx][qy] *= oper(q, e);
+        }
+      }
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        double sol_x[2][NUM_DOFS_1D];
+        for (int dx = 0; dx < NUM_QUAD_1D; ++dx) {
+          sol_x[0][dx] = 0;
+          sol_x[1][dx] = 0;
+        }
+
+        // sol_x{qx, dy} = quadToDof{dy, qy} * sol_xy{qx, qy}
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            sol_x[0][dx] += quadToDof(dx, qx) * sol_xy[0][qx][qy];
+            sol_x[1][dx] += quadToDof(dx, qx) * sol_xy[1][qx][qy];
+          }
+        }
+
+        // sol{dx, dy, e} = quadToDof{dx, qx} * sol_x{qx, dy}
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          const double q2d = quadToDof(dy, qy);
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            solOut(0, dx, dy, e) += q2d * sol_x[0][dx];
+            solOut(1, dx, dy, e) += q2d * sol_x[1][dx];
+          }
+        }
+      }
+    } // dummy
+  } // e
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian3D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_3D; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+      oper(q, e) = quadWeights[q] * COEFF * detJ;
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal_t oper,
+                       @restrict const DVLocal3D_t solIn,
+                       @restrict DVLocal3D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double sol_xyz[3][NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D];
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_xyz[0][qz][qy][qx] = 0;
+            sol_xyz[1][qz][qy][qx] = 0;
+            sol_xyz[2][qz][qy][qx] = 0;
+          }
+        }
+      }
+
+      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+        double sol_xy[3][NUM_QUAD_1D][NUM_QUAD_1D];
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_xy[0][qy][qx] = 0;
+            sol_xy[1][qy][qx] = 0;
+            sol_xy[2][qy][qx] = 0;
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          double sol_x[3][NUM_QUAD_1D];
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_x[0][qx] = 0;
+            sol_x[1][qx] = 0;
+            sol_x[2][qx] = 0;
+          }
+
+          // sol_x{qx} = dofToQuad{qx, dx} * sol{dx, dy, dz, e}
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              sol_x[0][qx] += dofToQuad(qx, dx) * solIn(0, dx, dy, dz, e);
+              sol_x[1][qx] += dofToQuad(qx, dx) * solIn(1, dx, dy, dz, e);
+              sol_x[2][qx] += dofToQuad(qx, dx) * solIn(2, dx, dy, dz, e);
+            }
+          }
+
+          // sol_xy{qx, qy} = dofToQuad{qy, dy} * sol_x{dx}
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            const double wy = dofToQuad(qy, dy);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              sol_xy[0][qy][qx] += wy * sol_x[0][qx];
+              sol_xy[1][qy][qx] += wy * sol_x[1][qx];
+              sol_xy[2][qy][qx] += wy * sol_x[2][qx];
+            }
+          }
+        } // dy
+
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          const double wz = dofToQuad(qz, dz);
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              sol_xyz[0][qz][qy][qx] += wz * sol_xy[0][qy][qx];
+              sol_xyz[1][qz][qy][qx] += wz * sol_xy[1][qy][qx];
+              sol_xyz[2][qz][qy][qx] += wz * sol_xy[2][qy][qx];
+            }
+          }
+        }
+      } // dz
+
+      // sol_xyz{qz, qy, qx} *= oper{q, e}
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            const int q = QUAD_3D_ID(qx, qy, qz);
+            sol_xyz[0][qz][qy][qx] *= oper(q, e);
+            sol_xyz[1][qz][qy][qx] *= oper(q, e);
+            sol_xyz[2][qz][qy][qx] *= oper(q, e);
+          }
+        }
+      }
+
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        double sol_xy[3][NUM_DOFS_1D][NUM_DOFS_1D];
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            sol_xy[0][dy][dx] = 0;
+            sol_xy[1][dy][dx] = 0;
+            sol_xy[2][dy][dx] = 0;
+          }
+        }
+
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          double sol_x[3][NUM_DOFS_1D];
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            sol_x[0][dx] = 0;
+            sol_x[1][dx] = 0;
+            sol_x[2][dx] = 0;
+          }
+
+          // sol_x{dx} = quadToDof{dx, qx} * sol_xyz{qz, qy, qx}
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              sol_x[0][dx] += quadToDof(dx, qx) * sol_xyz[0][qz][qy][qx];
+              sol_x[1][dx] += quadToDof(dx, qx) * sol_xyz[1][qz][qy][qx];
+              sol_x[2][dx] += quadToDof(dx, qx) * sol_xyz[2][qz][qy][qx];
+            }
+          }
+
+          // sol_xy{dy, dx} = quadToDof{dy, qy} * sol_x{dx}
+          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+            const double wy = quadToDof(dy, qy);
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              sol_xy[0][dy][dx] += wy * sol_x[0][dx];
+              sol_xy[1][dy][dx] += wy * sol_x[1][dx];
+              sol_xy[2][dy][dx] += wy * sol_x[2][dx];
+            }
+          }
+        } // qy
+
+        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+          const double wz = quadToDof(dz, qz);
+          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              solOut(0, dx, dy, dz, e) += wz * sol_xy[0][dy][dx];
+              solOut(1, dx, dy, dz, e) += wz * sol_xy[1][dy][dx];
+              solOut(2, dx, dy, dz, e) += wz * sol_xy[2][dy][dx];
+            }
+          }
+        }
+      } // qz
+    } // dummy
+  } // e
+
+}
+//======================================

config/cmake/MFEMConfig.cmake.in

@@ -39,6 +39,7 @@ set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
+set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
 
 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")

config/cmake/config.hpp.in

@@ -98,6 +98,9 @@
 // Enable MFEM functionality based on Conduit
 #cmakedefine MFEM_USE_CONDUIT
 
+// Enable MFEM functionality based on the PUMI library
+#cmakedefine MFEM_USE_PUMI
+
 // Which library functions to use in class StopWatch for measuring time.
 // For a list of the available options, see INSTALL.
 // If not defined, an option is selected automatically.
@@ -113,4 +116,8 @@
 // Version of HYPRE used for building MFEM.
 #cmakedefine MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
 
+// Macro defined when PUMI is built with support for the Simmetrix SimModSuite
+// library.
+#cmakedefine MFEM_USE_SIMMETRIX
+
 #endif // MFEM_CONFIG_HEADER

config/config.hpp

@@ -36,12 +36,15 @@
 #ifdef MFEM_USE_PETSC
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
+#ifdef MFEM_USE_PUMI
+#error Building with PUMI (MFEM_USE_PUMI=YES) requires MPI (MFEM_USE_MPI=YES)
+#endif
 #endif // MFEM_USE_MPI not defined
 
 // Macro that returns its first arg when MFEM_USE_BACKENDS is defined, and its
 // second arg if it is not defined.
 #ifdef MFEM_USE_BACKENDS
-#define MFEM_IF_BACKENDS(x,y) (x)
+#define MFEM_IF_BACKENDS(x,y) x
 #else
-#define MFEM_IF_BACKENDS(x,y) (y)
+#define MFEM_IF_BACKENDS(x,y) y
 #endif

config/config.hpp.in

@@ -115,6 +115,9 @@
 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR
 
+// Enable MFEM functionality based on the PUMI library
+// #define MFEM_USE_PUMI
+
 // Enable the use of MFEM backends.
 // #define MFEM_USE_BACKENDS
 
@@ -130,4 +133,8 @@
 // Version of HYPRE used for building MFEM.
 // #define MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
 
+// Macro defined when PUMI is built with support for the Simmetrix SimModSuite
+// library.
+// #define MFEM_USE_SIMMETRIX
+
 #endif // MFEM_CONFIG_HEADER

config/config.mk.in

@@ -39,6 +39,7 @@ MFEM_USE_PETSC       = @MFEM_USE_PETSC@
 MFEM_USE_MPFR        = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE       = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT     = @MFEM_USE_CONDUIT@
+MFEM_USE_PUMI        = @MFEM_USE_PUMI@
 MFEM_USE_BACKENDS    = @MFEM_USE_BACKENDS@
 MFEM_USE_OCCA        = @MFEM_USE_OCCA@
 

config/defaults.cmake

@@ -40,6 +40,7 @@ option(MFEM_USE_PETSC "Enable PETSc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
+option(MFEM_USE_PUMI "Enable PUMI" OFF)
 
 # Allow a user to disable testing, examples, and/or miniapps at CONFIGURE TIME
 # if they don't want/need them (e.g. if MFEM is "just a dependency" and all they
@@ -145,6 +146,9 @@ set(AXOM_DIR "${MFEM_DIR}/../axom" CACHE PATH "Path to the Axom library.")
 set(Axom_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
     "Additional packages required by Axom.")
 
+set(PUMI_DIR "${MFEM_DIR}/../pumi-2.1.0" CACHE STRING
+    "Directory where PUMI is installed")
+
 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")

config/defaults.mk

@@ -109,6 +109,7 @@ MFEM_USE_PETSC       = NO
 MFEM_USE_MPFR        = NO
 MFEM_USE_SIDRE       = NO
 MFEM_USE_CONDUIT     = NO
+MFEM_USE_PUMI        = NO
 
 # Compile and link options for zlib.
 ZLIB_DIR =
@@ -274,6 +275,13 @@ SIDRE_LIB = \
    -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
    -lsidre -lslic -laxom_utils -lconduit -lconduit_relay -lhdf5 $(ZLIB_LIB) -ldl
 
+# PUMI
+# Note that PUMI_DIR is needed -- it is used to check for gmi_sim.h
+PUMI_DIR = @MFEM_DIR@/../pumi-2.1.0
+PUMI_OPT = -I$(PUMI_DIR)/include
+PUMI_LIB = -L$(PUMI_DIR)/lib -lpumi -lcrv -lma -lmds -lapf -lpcu -lgmi -lparma\
+   -llion -lmth -lapf_zoltan -lspr
+
 OCCA_DIR = @MFEM_DIR@/../occa
 OCCA_OPT = -I$(OCCA_DIR)/include
 OCCA_LIB = -Wl,-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca

config/makefile

@@ -38,6 +38,9 @@ all: header config-mk
 MPI = $(MFEM_USE_MPI:NO=)
 GHV = get_hypre_version
 GHV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
+SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)
+SMX_PATH = $(PUMI_DIR)/include/gmi_sim.h
+SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))
 
 $(GHV): $(SRC)$(GHV).cpp
 	$(call mfem-info, Determining HYPRE version ...)
@@ -52,10 +55,16 @@ get-hypre-version: $(GHV).out
 	$(info HYPRE version: $(MFEM_HYPRE_VERSION)),\
 	$(error Unable to determine HYPRE version))
 
-header: $(if $(MPI),get-hypre-version,)
+check-smx:
+	$(call mfem-info, Checking for Simmetrix header [$(SMX_FILE)] ...)
+	$(eval MFEM_USE_SIMMETRIX:=$(if $(wildcard $(SMX_FILE)),YES,NO))
+	$(call mfem-info, MFEM_USE_SIMMETRIX = $(MFEM_USE_SIMMETRIX))
+	$(eval export MFEM_USE_SIMMETRIX)
+
+header: $(if $(MPI),get-hypre-version,) $(if $(SMX),check-smx)
 	$(call mfem-info, Writing $(CONFIG_HPP) ...)
 	@set -- && \
-	for def in $${MFEM_DEFINES} $(if $(MPI),MFEM_HYPRE_VERSION,); do \
+	for def in $${MFEM_DEFINES} $(if $(MPI),MFEM_HYPRE_VERSION) $(SMX); do \
 	   eval var=\$$$$def && \
 	   if [ "NO" != "$${var}" ]; then \
 	      set -- "$$@" -e "s|// \(#define $${def} \)|\1|" && \

config/test.mk

@@ -14,11 +14,13 @@
 # Colors used below:
 # green    '\033[0;32m'
 # red      '\033[0;31m'
+# yellow   '\033[0;33m'
 # no color '\033[0m'
 COLOR_PRINT = if [ -t 1 ]; then \
    printf $(1)$(2)'\033[0m'$(3); else printf $(2)$(3); fi
 PRINT_OK = $(call COLOR_PRINT,'\033[0;32m',OK,"  ($$1 $$2)\n")
 PRINT_FAILED = $(call COLOR_PRINT,'\033[0;31m',FAILED,"  ($$1 $$2)\n")
+PRINT_SKIP = $(call COLOR_PRINT,'\033[0;33m',SKIP,"\n")
 
 # Timing support
 define TIMECMD_detect

doc/CodeDocumentation.conf.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v3.3.3
+PROJECT_NUMBER         = v3.4.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -761,6 +761,7 @@ WARN_LOGFILE           =
 INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/mfem.hpp \
                          @MFEM_SOURCE_DIR@/backends/base \
+                         @MFEM_SOURCE_DIR@/backends/occa \
                          @MFEM_SOURCE_DIR@/config \
                          @MFEM_SOURCE_DIR@/general \
                          @MFEM_SOURCE_DIR@/linalg \
@@ -768,6 +769,7 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/fem \
                          @MFEM_SOURCE_DIR@/examples \
                          @MFEM_SOURCE_DIR@/examples/petsc \
+                         @MFEM_SOURCE_DIR@/examples/pumi \
                          @MFEM_SOURCE_DIR@/examples/sundials \
                          @MFEM_SOURCE_DIR@/miniapps/common \
                          @MFEM_SOURCE_DIR@/miniapps/meshing \

doc/CodeDocumentation.dox

@@ -36,8 +36,8 @@ namespace mfem {
  * - HypreSolver and other \link hypre.hpp hypre classes\endlink
  *
  * <H3>Example codes</H3>
- * - <a class="el" href="ex1_8cpp_source.html">Example 1</a>: nodal H1 FEM for the Laplace problem
- * - <a class="el" href="ex1p_8cpp_source.html">Example 1p</a>: parallel nodal H1 FEM for the Laplace problem
+ * - <a class="el" href="examples_2ex1_8cpp_source.html">Example 1</a>: nodal H1 FEM for the Laplace problem
+ * - <a class="el" href="examples_2ex1p_8cpp_source.html">Example 1p</a>: parallel nodal H1 FEM for the Laplace problem
  * - <a class="el" href="ex2_8cpp_source.html">Example 2</a>: vector FEM for linear elasticity
  * - <a class="el" href="ex2p_8cpp_source.html">Example 2p</a>: parallel vector FEM for linear elasticity
  * - <a class="el" href="ex3_8cpp_source.html">Example 3</a>: Nedelec H(curl) FEM for the definite Maxwell problem
@@ -56,7 +56,7 @@ namespace mfem {
  * - <a class="el" href="ex9p_8cpp_source.html">Example 9p</a>: parallel Discontinuous Galerkin (DG) time-dependent advection
  * - <a class="el" href="ex10_8cpp_source.html">Example 10</a>: time-dependent implicit nonlinear elasticity
  * - <a class="el" href="ex10p_8cpp_source.html">Example 10p</a>: parallel time-dependent implicit nonlinear elasticity
- * - <a class="el" href="ex11p_8cpp_source.html">Example 11p</a>: parallel Laplace eigensolver
+ * - <a class="el" href="examples_2ex11p_8cpp_source.html">Example 11p</a>: parallel Laplace eigensolver
  * - <a class="el" href="ex12p_8cpp_source.html">Example 12p</a>: parallel linear elasticity eigensolver
  * - <a class="el" href="ex13p_8cpp_source.html">Example 13p</a>: parallel Maxwell eigensolver
  * - <a class="el" href="ex14_8cpp_source.html">Example 14</a>: Discontinuous Galerkin (DG) for the Laplace problem
@@ -96,6 +96,15 @@ namespace mfem {
  *   <a class="el" href="petsc_2ex10p_8cpp_source.html">10p</a>
  *   demonstrating the use of MFEM's \link petsc.hpp PETSc classes\endlink
  *
+ * <H4>PUMI Examples</H4>
+ * - Variants of Examples
+ *   <a class="el" href="examples_2pumi_2ex1_8cpp_source.html">1</a>,
+ *   <a class="el" href="examples_2pumi_2ex1p_8cpp_source.html">1p</a>,
+ *   <a class="el" href="pumi_2ex2_8cpp_source.html">2</a>,
+ *   and
+ *   <a class="el" href="pumi_2ex6p_8cpp_source.html">6p</a>
+ *   demonstrating the use of MFEM's \link pumi.hpp PUMI classes\endlink
+ *
  * <H3>Miniapps</H3>
  * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
  * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code

examples/CMakeLists.txt

@@ -25,6 +25,7 @@ list(APPEND ALL_EXE_SRCS
   ex16.cpp
   ex17.cpp
   ex18.cpp
+  ex19.cpp
   )
 
 if (MFEM_USE_MPI)
@@ -47,6 +48,7 @@ if (MFEM_USE_MPI)
     ex16p.cpp
     ex17p.cpp
     ex18p.cpp
+    ex19p.cpp
     )
 endif()
 
@@ -61,8 +63,6 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
   get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
   string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
 
-  string(FIND ${TEST_NAME} "p" is_parallel_test)
-
   set(THIS_TEST_OPTIONS "-no-vis")
   if (${TEST_NAME} MATCHES "ex10p*")
     list(APPEND THIS_TEST_OPTIONS "-tf" "5")
@@ -70,7 +70,7 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
     list(APPEND THIS_TEST_OPTIONS "-e" "1")
   endif()
 
-  if (is_parallel_test EQUAL -1)
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
     add_test(NAME ${TEST_NAME}_ser
       COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
   else()
@@ -91,3 +91,8 @@ endif()
 if (MFEM_USE_PETSC)
   add_subdirectory(petsc)
 endif()
+
+# Include the examples/pumi directory if PUMI is enabled
+if (MFEM_USE_PUMI)
+  add_subdirectory(pumi)
+endif()

examples/ex16d.cpp

@@ -0,0 +1,441 @@
+//                                MFEM Example 16
+//
+// Compile with: make ex16
+//
+// Sample runs:  ex16
+//               ex16 -m ../data/inline-tri.mesh
+//               ex16 -m ../data/disc-nurbs.mesh -tf 2
+//               ex16 -s 1 -a 0.0 -k 1.0
+//               ex16 -s 2 -a 1.0 -k 0.0
+//               ex16 -s 3 -a 0.5 -k 0.5 -o 4
+//               ex16 -s 14 -dt 1.0e-4 -tf 4.0e-2 -vs 40
+//               ex16 -m ../data/fichera-q2.mesh
+//               ex16 -m ../data/escher.mesh
+//               ex16 -m ../data/beam-tet.mesh -tf 10 -dt 0.1
+//               ex16 -m ../data/amr-quad.mesh -o 4 -r 0
+//               ex16 -m ../data/amr-hex.mesh -o 2 -r 0
+//
+// Description:  This example solves a time dependent nonlinear heat equation
+//               problem of the form du/dt = C(u), with a non-linear diffusion
+//               operator C(u) = \nabla \cdot (\kappa + \alpha u) \nabla u.
+//
+//               The example demonstrates the use of nonlinear operators (the
+//               class ConductionOperator defining C(u)), as well as their
+//               implicit time integration. Note that implementing the method
+//               ConductionOperator::ImplicitSolve is the only requirement for
+//               high-order implicit (SDIRK) time integration.
+//
+//               We recommend viewing examples 2, 9 and 10 before viewing this
+//               example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+/** After spatial discretization, the conduction model can be written as:
+ *
+ *     du/dt = M^{-1}(-Ku)
+ *
+ *  where u is the vector representing the temperature, M is the mass matrix,
+ *  and K is the diffusion operator with diffusivity depending on u:
+ *  (\kappa + \alpha u).
+ *
+ *  Class ConductionOperator represents the right-hand side of the above ODE.
+ */
+class ConductionOperator : public TimeDependentOperator
+{
+protected:
+   FiniteElementSpace &fespace;
+   Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
+
+   BilinearForm M;
+   BilinearForm K;
+
+   GridFunction u_alpha_gf;
+   GridFunctionCoefficient u_coeff;
+
+   OperatorHandle Moper, Koper;
+   Operator *T; // T = M + dt K
+   double current_dt;
+
+   CGSolver M_solver; // Krylov solver for inverting the mass matrix M
+   // FIXME: add the preconditioner
+   // DSmoother M_prec;  // Preconditioner for the mass matrix M
+
+   CGSolver T_solver; // Implicit solver for T = M + dt K
+   // FIXME: add the preconditioner
+   // DSmoother T_prec;  // Preconditioner for the implicit solver
+
+   double alpha, kappa;
+
+   mutable Vector z; // auxiliary vector
+
+public:
+   ConductionOperator(FiniteElementSpace &f, const char *oper_spec,
+                      double alpha, double kappa, const Vector &u);
+
+   virtual void Mult(const Vector &u, Vector &du_dt) const;
+   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
+       This is the only requirement for high-order SDIRK implicit integration.*/
+   virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);
+
+   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
+   void SetParameters(const Vector &u);
+
+   virtual ~ConductionOperator();
+};
+
+
+class TimeDerivativeOperator : public Operator
+{
+   Operator *Moper;
+   Operator *Koper;
+   mutable Vector Kdu;
+   const double dt;
+
+public:
+   // FIXME: Sparse matrices should be changed to have PLayouts
+   // allocated so that this constructor works even when Moper and
+   // Koper were not created from engines.
+   TimeDerivativeOperator(Operator *_Moper, const double _dt, Operator *_Koper)
+      : Operator(*_Koper->InLayout(), *_Moper->OutLayout()),
+        Moper(_Moper),
+        Koper(_Koper),
+        Kdu(_Moper->OutLayout()),
+        dt(_dt) { }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      Moper->Mult(x, y);
+      Koper->Mult(x, Kdu);
+
+      y.Axpby(1.0, y, dt, Kdu);
+   }
+};
+
+double InitialTemperature(const Vector &x);
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int ref_levels = 2;
+   int order = 2;
+   int ode_solver_type = 3;
+   double t_final = 0.5;
+   double dt = 1.0e-2;
+   double alpha = 1.0e-2;
+   double kappa = 0.5;
+   bool visualization = true;
+   bool visit = false;
+   int vis_steps = 5;
+   const char *oper_spec = "representation: 'partial'";
+   const char *occa_spec = "mode: 'Serial'";
+
+   int precision = 8;
+   cout.precision(precision);
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ref_levels, "-r", "--refine",
+                  "Number of times to refine the mesh uniformly.");
+   args.AddOption(&order, "-o", "--order",
+                  "Order (degree) of the finite elements.");
+   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
+                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
+                  "\t   11 - Forward Euler, 12 - RK2, 13 - RK3 SSP, 14 - RK4.");
+   args.AddOption(&t_final, "-tf", "--t-final",
+                  "Final time; start time is 0.");
+   args.AddOption(&dt, "-dt", "--time-step",
+                  "Time step.");
+   args.AddOption(&alpha, "-a", "--alpha",
+                  "Alpha coefficient.");
+   args.AddOption(&kappa, "-k", "--kappa",
+                  "Kappa coefficient offset.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
+                  "--no-visit-datafiles",
+                  "Save data files for VisIt (visit.llnl.gov) visualization.");
+   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
+                  "Visualize every n-th timestep.");
+   args.AddOption(&oper_spec, "-s", "--oper-spec", "Operator specification");
+   args.AddOption(&occa_spec, "-os", "--occa-spec", "OCCA engine specification");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // Examples for OCCA specifications:
+   //   - CPU (serial): "mode: 'Serial'"
+   //   - CUDA GPU: "mode: 'CUDA', device_id: 0"
+   //   - OpenMP on CPUs: "mode: 'OpenMP', threads: 4"
+   //   - OpenCL on device 0: "mode: 'OpenCL', device_id: 0, platform_id: 0"
+
+   SharedPtr<Engine> engine(new mfem::occa::Engine(occa_spec));
+
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   mesh->SetEngine(*engine);
+   int dim = mesh->Dimension();
+
+   // 3. Define the ODE solver used for time integration. Several implicit
+   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
+   //    explicit Runge-Kutta methods are available.
+   ODESolver *ode_solver;
+   switch (ode_solver_type)
+   {
+      // Implicit L-stable methods
+      case 1:  ode_solver = new BackwardEulerSolver; break;
+      case 2:  ode_solver = new SDIRK23Solver(2); break;
+      case 3:  ode_solver = new SDIRK33Solver; break;
+      // Explicit methods
+      case 11: ode_solver = new ForwardEulerSolver; break;
+      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 13: ode_solver = new RK3SSPSolver; break;
+      case 14: ode_solver = new RK4Solver; break;
+      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }
+
+   // 4. Refine the mesh to increase the resolution. In this example we do
+   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
+   //    command-line parameter.
+   for (int lev = 0; lev < ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 5. Define the vector finite element space representing the current and the
+   //    initial temperature, u_ref.
+   H1_FECollection fe_coll(order, dim);
+   FiniteElementSpace fespace(mesh, &fe_coll);
+
+   int fe_size = fespace.GetTrueVSize();
+   cout << "Number of temperature unknowns: " << fe_size << endl;
+
+   GridFunction u_gf(&fespace);
+
+   // 6. Set the initial conditions for u. All boundaries are considered
+   //    natural. This computes this on the host, so pull/push is needed.
+   u_gf.Pull();
+   FunctionCoefficient u_0(InitialTemperature);
+   u_gf.ProjectCoefficient(u_0);
+   u_gf.Push();
+   Vector u;
+   u_gf.GetTrueDofs(u);
+
+   // 7. Initialize the conduction operator and the visualization.
+   ConductionOperator oper(fespace, oper_spec, alpha, kappa, u);
+
+   u_gf.SetFromTrueDofs(u);
+   {
+      ofstream omesh("ex16.mesh");
+      omesh.precision(precision);
+      mesh->Print(omesh);
+      ofstream osol("ex16-init.gf");
+      osol.precision(precision);
+      u_gf.Pull(); // pull back to host before saving
+      u_gf.Save(osol);
+   }
+
+   VisItDataCollection visit_dc("Example16", mesh);
+   visit_dc.RegisterField("temperature", &u_gf);
+   if (visit)
+   {
+      visit_dc.SetCycle(0);
+      visit_dc.SetTime(0.0);
+      visit_dc.Save();
+   }
+
+   socketstream sout;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      sout.open(vishost, visport);
+      if (!sout)
+      {
+         cout << "Unable to connect to GLVis server at "
+              << vishost << ':' << visport << endl;
+         visualization = false;
+         cout << "GLVis visualization disabled.\n";
+      }
+      else
+      {
+         sout.precision(precision);
+         sout << "solution\n" << *mesh << u_gf;
+         sout << "pause\n";
+         sout << flush;
+         cout << "GLVis visualization paused."
+              << " Press space (in the GLVis window) to resume it.\n";
+      }
+   }
+
+   // 8. Perform time-integration (looping over the time iterations, ti, with a
+   //    time-step dt).
+   ode_solver->Init(oper);
+   double t = 0.0;
+
+   bool last_step = false;
+   for (int ti = 1; !last_step; ti++)
+   {
+      if (t + dt >= t_final - dt/2)
+      {
+         last_step = true;
+      }
+
+      ode_solver->Step(u, t, dt);
+
+      if (last_step || (ti % vis_steps) == 0)
+      {
+         cout << "step " << ti << ", t = " << t << endl;
+
+         // u_gf and u are both on the device at this point.
+         u_gf.SetFromTrueDofs(u);
+         if (visualization)
+         {
+            u_gf.Pull(); // pull back to host before saving
+            sout << "solution\n" << *mesh << u_gf << flush;
+         }
+
+         if (visit)
+         {
+            visit_dc.SetCycle(ti);
+            visit_dc.SetTime(t);
+            visit_dc.Save();
+         }
+      }
+      oper.SetParameters(u);
+   }
+
+   // 9. Save the final solution. This output can be viewed later using GLVis:
+   //    "glvis -m ex16.mesh -g ex16-final.gf".
+   {
+      ofstream osol("ex16-final.gf");
+      osol.precision(precision);
+      u_gf.Pull(); // pull back to host before saving
+      u_gf.Save(osol);
+   }
+
+   // 10. Free the used memory.
+   delete ode_solver;
+   delete mesh;
+
+   return 0;
+}
+
+ConductionOperator::ConductionOperator(FiniteElementSpace &f,
+                                       const char *oper_spec, double al,
+                                       double kap, const Vector &u)
+   : TimeDependentOperator(*f.GetTrueVLayout()), fespace(f), M(&fespace),
+     K(&fespace), u_alpha_gf(&f), u_coeff(&u_alpha_gf), Moper(oper_spec),
+     Koper(oper_spec), T(NULL), current_dt(0.0), z(f.GetTrueVLayout())
+{
+   const double rel_tol = 1e-8;
+
+   M.AddDomainIntegrator(new MassIntegrator());
+   M.Assemble();
+   M.FormSystemMatrix(ess_tdof_list, Moper);
+
+   K.AddDomainIntegrator(new DiffusionIntegrator(u_coeff));
+
+   M_solver.iterative_mode = false;
+   M_solver.SetRelTol(rel_tol);
+   M_solver.SetAbsTol(0.0);
+   M_solver.SetMaxIter(200);
+   M_solver.SetPrintLevel(0);
+   // M_solver.SetPreconditioner(M_prec);
+   M_solver.SetOperator(*Moper.Ptr());
+
+   alpha = al;
+   kappa = kap;
+
+   T_solver.iterative_mode = false;
+   T_solver.SetRelTol(rel_tol);
+   T_solver.SetAbsTol(0.0);
+   T_solver.SetMaxIter(200);
+   T_solver.SetPrintLevel(0);
+   // T_solver.SetPreconditioner(T_prec);
+
+   SetParameters(u);
+}
+
+void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
+{
+   // Compute:
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
+   Koper.Ptr()->Mult(u, z);
+   z.Axpby(-1.0, z, 0.0, z);
+   M_solver.Mult(z, du_dt);
+}
+
+void ConductionOperator::ImplicitSolve(const double dt,
+                                       const Vector &u, Vector &du_dt)
+{
+   // Solve the equation:
+   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
+   // for du_dt
+   if (!T)
+   {
+      T = new TimeDerivativeOperator(Moper.Ptr(), dt, Koper.Ptr());
+      current_dt = dt;
+      T_solver.SetOperator(*T);
+   }
+   MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt
+   Koper.Ptr()->Mult(u, z);
+   z.Axpby(-1.0, z, 0.0, z);
+   T_solver.Mult(z, du_dt);
+}
+
+void ConductionOperator::SetParameters(const Vector &u)
+{
+   u_alpha_gf.SetFromTrueDofs(u);
+   u_alpha_gf.Pull();
+   for (int i = 0; i < u_alpha_gf.Size(); i++)
+   {
+      u_alpha_gf(i) = kappa + alpha*u_alpha_gf(i);
+   }
+   u_alpha_gf.Push();
+
+   // Reassemble after changing u_alpha_gf (and hence u_coeff)...
+   K.Assemble();
+   K.FormSystemMatrix(ess_tdof_list, Koper);
+
+   delete T;
+   T = NULL; // re-compute T on the next ImplicitSolve
+}
+
+ConductionOperator::~ConductionOperator()
+{
+   delete T;
+}
+
+double InitialTemperature(const Vector &x)
+{
+   if (x.Norml2() < 0.5)
+   {
+      return 2.0;
+   }
+   else
+   {
+      return 1.0;
+   }
+}

examples/ex19.cpp

@@ -50,7 +50,7 @@ using namespace mfem;
 //
 // and K^-1 is an approximation of the inverse of the displacement part of the
 // Jacobian and S^-1 is an approximation of the inverse of the Schur
-// complement S = B K^-1 B^T. The Schur complement is approximiated using
+// complement S = B K^-1 B^T. The Schur complement is approximated using
 // a mass matrix of the pressure variables.
 class JacobianPreconditioner : public Solver
 {

examples/ex19p.cpp

@@ -50,7 +50,7 @@ using namespace mfem;
 //
 // and K^-1 is an approximation of the inverse of the displacement part of the
 // Jacobian and S^-1 is an approximation of the inverse of the Schur
-// complement S = B K^-1 B^T. The Schur complement is approximiated using
+// complement S = B K^-1 B^T. The Schur complement is approximated using
 // a mass matrix of the pressure variables.
 class JacobianPreconditioner : public Solver
 {

examples/ex1d.cpp

@@ -12,17 +12,20 @@ int main(int argc, char *argv[])
    const char *spec = "cpu";
    const char *mesh_file = "../data/star.mesh";
    int order = 1;
+   int ref_levels = -1;
    bool static_cond = false;
    bool visualization = 1;
 
    OptionsParser args(argc, argv);
    args.AddOption(&spec, "-s", "--spec",
-                  "Compute resurce specification.");
+                  "Compute resource specification.");
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree) or -1 for"
                   " isoparametric space.");
+   args.AddOption(&ref_levels, "-r", "--refine-levels",
+                  "Number of uniform refinements to apply to the mesh.");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -36,20 +39,29 @@ int main(int argc, char *argv[])
    }
    args.PrintOptions(cout);
 
+#ifdef MFEM_USE_BACKENDS
    /// Engine *engine = EngineDepot.Select(spec);
 
-   string occa_spec("mode: 'Serial'");
-   // string occa_spec("mode: 'CUDA', deviceID: 0");
+   // string occa_spec("mode: 'Serial'");
+   string occa_spec("mode: 'CUDA', device_id: 0");
    // string occa_spec("mode: 'OpenMP', threads: 4");
-   // string occa_spec("mode: 'OpenCL', deviceID: 0, platformID: 0");
+   // string occa_spec("mode: 'OpenCL', device_id: 0, platform_id: 0");
+
+   // The following flag affects only 'Serial' and 'OpenMP' modes.
+   // In 'CUDA' mode, '-O3' affects only host code.
+   // In 'OpenCL' mode, adding '-O3' breaks compilation.
+   // occa_spec += ", kernel: { compiler_flags: '-O3' }";
 
    SharedPtr<Engine> engine(new mfem::occa::Engine(occa_spec));
+#endif
 
    // 2. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
    //    the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
+#ifdef MFEM_USE_BACKENDS
    mesh->SetEngine(*engine);
+#endif
    int dim = mesh->Dimension();
 
    // 3. Refine the mesh to increase the resolution. In this example we do
@@ -57,7 +69,7 @@ int main(int argc, char *argv[])
    //    largest number that gives a final mesh with no more than 50,000
    //    elements.
    {
-      int ref_levels =
+      ref_levels = ref_levels >= 0 ? ref_levels :
          (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
@@ -132,7 +144,12 @@ int main(int argc, char *argv[])
    cout << "Size of linear system: " << A.Ptr()->Height() << endl;
 
    // 10. Solve the system A X = B with CG.
-   CG(*A.Ptr(), B, X, 3, 1000, 1e-12, 0.0);
+   tic_toc.Clear();
+   tic_toc.Start();
+   const int print_level = 3;
+   CG(*A.Ptr(), B, X, print_level, 1000, 1e-12, 0.0);
+   tic_toc.Stop();
+   cout << "CG time: " << tic_toc.RealTime() << " sec." << endl;
 
    // 11. Recover the solution as a finite element grid function.
    a->RecoverFEMSolution(X, *b, x);

examples/ex1pd.cpp

@@ -16,6 +16,8 @@ int main(int argc, char *argv[])
 
    // 2. Parse command-line options.
    const char *mesh_file = "../data/star.mesh";
+   int ser_ref_levels = -1;
+   int par_ref_levels = -1;
    int order = 1;
    bool static_cond = false;
    bool visualization = 1;
@@ -23,6 +25,10 @@ int main(int argc, char *argv[])
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree) or -1 for"
                   " isoparametric space.");
@@ -46,20 +52,31 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
+#ifdef MFEM_USE_BACKENDS
    /// Engine *engine = EngineDepot.Select(spec);
 
-   string occa_spec("mode: 'Serial'");
-   // string occa_spec("mode: 'CUDA', deviceID: 0");
+   // string occa_spec("mode: 'Serial'");
+   string occa_spec;
+   {
+      stringstream occa_spec_ss;
+      occa_spec_ss << "mode: 'CUDA', device_id: 0";
+      // const int nGPUs = 4;
+      // occa_spec_ss << "mode: 'CUDA', device_id: " << (myid % nGPUs);
+      occa_spec = occa_spec_ss.str();
+   }
    // string occa_spec("mode: 'OpenMP', threads: 4");
-   // string occa_spec("mode: 'OpenCL', deviceID: 0, platformID: 0");
+   // string occa_spec("mode: 'OpenCL', device_id: 0, platform_id: 0");
 
    SharedPtr<Engine> engine(new mfem::occa::Engine(MPI_COMM_WORLD, occa_spec));
+#endif
 
    // 3. Read the (serial) mesh from the given mesh file on all processors.  We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
+#ifdef MFEM_USE_BACKENDS
    mesh->SetEngine(*engine);
+#endif
    int dim = mesh->Dimension();
 
    // 4. Refine the serial mesh on all processors to increase the resolution. In
@@ -67,8 +84,12 @@ int main(int argc, char *argv[])
    //    'ref_levels' to be the largest number that gives a final mesh with no
    //    more than 10,000 elements.
    {
-      int ref_levels =
-         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+      int ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+      ref_levels = ser_ref_levels >= 0 ? ser_ref_levels : ref_levels;
+      if (myid == 0)
+      {
+         cout << "Serial refinement levels: " << ref_levels << endl;
+      }
       for (int l = 0; l < ref_levels; l++)
       {
          mesh->UniformRefinement();
@@ -81,12 +102,18 @@ int main(int argc, char *argv[])
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
    {
-      int par_ref_levels = 2;
+      par_ref_levels = par_ref_levels >= 0 ? par_ref_levels : 2;
+      if (myid == 0)
+      {
+         cout << "Parallel refinement levels: " << par_ref_levels << endl;
+      }
       for (int l = 0; l < par_ref_levels; l++)
       {
          pmesh->UniformRefinement();
       }
    }
+   pmesh->PrintInfo(cout);
+   if (myid == 0) { cout << endl; }
 
    // 6. Define a parallel finite element space on the parallel mesh. Here we
    //    use continuous Lagrange finite elements of the specified order. If
@@ -164,7 +191,41 @@ int main(int argc, char *argv[])
    pcg->SetMaxIter(1000);
    pcg->SetPrintLevel(3);
    pcg->SetOperator(*A.Ptr());
+
+   // Run one CG iteration to make sure all kernels are loaded before measuring
+   // time.
+   if (myid == 0)
+   {
+      cout << "Running 1 CG iteration to load all kernels ..." << flush;
+   }
+   {
+      Vector X2(X);
+      pcg->SetMaxIter(1);
+      pcg->SetPrintLevel(-1);
+      pcg->Mult(B, X2);
+      pcg->SetMaxIter(1000);
+      pcg->SetPrintLevel(3);
+   }
+   if (myid == 0)
+   {
+      cout << " done." << endl;
+   }
+
+   double start_time = MPI_Wtime();
    pcg->Mult(B, X);
+   double end_time = MPI_Wtime();
+   double loc_time = end_time - start_time;
+   double max_time, min_time;
+   MPI_Allreduce(&loc_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
+   MPI_Allreduce(&loc_time, &min_time, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);
+   if (myid == 0)
+   {
+      cout << "CG time: " << max_time << " sec (min: " << min_time << " sec)\n"
+           << "DOFs/sec in CG: "
+           << 1e-6*size*pcg->GetNumIterations()/max_time << " ("
+           << 1e-6*size*pcg->GetNumIterations()/min_time << ") million.\n"
+           << endl;
+   }
 
    // 13. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.

examples/makefile

@@ -38,6 +38,9 @@ endif
 ifeq ($(MFEM_USE_PETSC),YES)
    SUBDIRS += petsc
 endif
+ifeq ($(MFEM_USE_PUMI),YES)
+   SUBDIRS += pumi
+endif
 SUBDIRS_ALL = $(addsuffix /all,$(SUBDIRS))
 SUBDIRS_TEST = $(addsuffix /test,$(SUBDIRS))
 SUBDIRS_CLEAN = $(addsuffix /clean,$(SUBDIRS))

examples/petsc/CMakeLists.txt

@@ -63,31 +63,29 @@ add_mfem_examples(PETSC_EXAMPLES_SRCS ${PFX} copy_petsc_rc_files test_petsc)
 #   ctest -R petsc
 
 # Command line options for the tests.
-set(EX1P_ARGS -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_ex1p)
-set(EX2P_ARGS -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p)
-set(EX3P_ARGS -m ../../data/klein-bottle.mesh
-    -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping)
-set(EX4P_ARGS -m ../../data/klein-bottle.mesh
-    -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping)
-set(EX5P_BDDC_ARGS -m ../../data/star.mesh
-    --usepetsc --petscopts rc_ex5p_bddc --nonoverlapping)
-set(EX5P_FSPL_ARGS -m ../../data/beam-tet.mesh
-    --usepetsc --petscopts rc_ex5p_fieldsplit)
-set(EX6P_ARGS -m ../../data/amr-quad.mesh --usepetsc)
-set(EX9P_E_ARGS -m ../../data/periodic-hexagon.mesh
-    --usepetsc --petscopts rc_ex9p_expl -dt 0.1)
-set(EX9P_ES_ARGS -m ../../data/periodic-hexagon.mesh
-    --usepetsc --petscopts rc_ex9p_expl --no-step)
-set(EX9P_IS_ARGS -m ../../data/periodic-hexagon.mesh
-    --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5)
-set(EX10P_ARGS -m ../../data/beam-quad.mesh
-    -tf 30 -s 3 -rs 2 -dt 3 --usepetsc --petscopts rc_ex10p)
+set(EX1_ARGS_W       -m ../../data/amr-quad.mesh --usepetsc)
+set(EX1_ARGS_P       -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_ex1p)
+set(EX2_ARGS         -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p)
+set(EX3_ARGS         -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping)
+set(EX4_ARGS         -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping)
+set(EX4_HYB_ARGS     -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping --hybridization)
+set(EX5_BDDC_LB_ARGS -m ../../data/star.mesh --usepetsc -o 0 --petscopts rc_ex5p_bddc --nonoverlapping --local-bdr)
+set(EX5_BDDC_GB_ARGS -m ../../data/star.mesh --usepetsc -o 0 --petscopts rc_ex5p_bddc --nonoverlapping)
+set(EX5_FSPL_ARGS    -m ../../data/beam-tet.mesh --usepetsc -o 0 --petscopts rc_ex5p_fieldsplit)
+set(EX6_ARGS         -m ../../data/amr-quad.mesh --usepetsc)
+set(EX6_NONOVL_ARGS  -m ../../data/amr-quad.mesh --usepetsc --nonoverlapping)
+set(EX9_E_ARGS       -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl -dt 0.1)
+set(EX9_ES_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step)
+set(EX9_IS_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5)
+set(EX10_ARGS        -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3)
 
 # Add the tests: one test per command-line-variable.
 foreach(TEST_OPTIONS_VAR
-        EX1P_ARGS EX2P_ARGS EX3P_ARGS EX4P_ARGS EX5P_BDDC_ARGS EX5P_FSPL_ARGS
-        EX6P_ARGS EX9P_E_ARGS EX9P_ES_ARGS EX9P_IS_ARGS EX10P_ARGS)
+        EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
+        EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
+        EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
   string(REGEX REPLACE "^(.+)_ARGS" "\\1" TEST_NAME_UC ${TEST_OPTIONS_VAR})
+  string(REGEX REPLACE "^([^_]+)" "\\1P" TEST_NAME_UC ${TEST_NAME_UC})
   string(TOLOWER ${TEST_NAME_UC} TEST_NAME_FULL)
   string(REGEX REPLACE "^([^_]+).*" "\\1" TEST_NAME ${TEST_NAME_FULL})
   set(TEST_NAME_FULL ${PFX}${TEST_NAME_FULL})

examples/petsc/ex5p.cpp

@@ -64,6 +64,7 @@ int main(int argc, char *argv[])
    bool visualization = 1;
    bool use_petsc = true;
    bool use_nonoverlapping = false;
+   bool local_bdr_spec = false;
    const char *petscrc_file = "";
 
    OptionsParser args(argc, argv);
@@ -86,6 +87,9 @@ int main(int argc, char *argv[])
                   "-no-nonoverlapping", "--no-nonoverlapping",
                   "Use or not the block diagonal PETSc's matrix format "
                   "for non-overlapping domain decomposition.");
+   args.AddOption(&local_bdr_spec, "-local-bdr", "--local-bdr", "-no-local-bdr",
+                  "--no-local-bdr",
+                  "Specify boundary dofs in local (Vdofs) ordering.");
    args.Parse();
    if (!args.Good())
    {
@@ -306,33 +310,38 @@ int main(int argc, char *argv[])
    {
       if (use_nonoverlapping)
       {
+         PetscBDDCSolverParams opts;
+
          // For saddle point problems, we need to provide BDDC the list of
          // boundary dofs either essential or natural.
          // Since R_space is the only space that may have boundary dofs and it
          // is ordered first then W_space, we don't need any local offset when
          // specifying the dofs.
          Array<int> bdr_tdof_list;
-         bool local = false;
          if (pmesh->bdr_attributes.Size())
          {
             Array<int> bdr(pmesh->bdr_attributes.Max());
             bdr = 1;
 
-            R_space->GetEssentialTrueDofs(bdr, bdr_tdof_list);
-            local = false;
-            // Alternatively, you can also provide the list of dofs in local
-            // ordering:
-            // R_space->GetEssentialVDofs(bdr, bdr_tdof_list);
-            // bdr_tdof_list.SetSize(R_space->GetVSize()+W_space->GetVSize(),0);
-            // local = true;
+            if (!local_bdr_spec)
+            {
+               // Essential dofs in global ordering
+               R_space->GetEssentialTrueDofs(bdr, bdr_tdof_list);
+            }
+            else
+            {
+               // Alternatively, you can also provide the list of dofs in local
+               // ordering
+               R_space->GetEssentialVDofs(bdr, bdr_tdof_list);
+               bdr_tdof_list.SetSize(R_space->GetVSize()+W_space->GetVSize(),0);
+            }
+            opts.SetNatBdrDofs(&bdr_tdof_list,local_bdr_spec);
          }
          else
          {
-            MFEM_ABORT("Need to know the boundary dofs");
+            MFEM_WARNING("Missing boundary dofs. This may cause solver failures.");
          }
 
-         PetscBDDCSolverParams opts;
-         opts.SetNatBdrDofs(&bdr_tdof_list,local);
          // See also command line options rc_ex5p_bddc
          pdarcyPr = new PetscBDDCSolver(MPI_COMM_WORLD,*darcyOp,opts,"prec_");
       }

examples/petsc/makefile

@@ -69,18 +69,21 @@ TESTNAME = Parallel PETSc example
 
 
 # Testing PETSc execution options.
-EX1_ARGS_W    := -m ../../data/amr-quad.mesh --usepetsc
-EX1_ARGS_P    := -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_ex1p
-EX2_ARGS      := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p
-EX3_ARGS      := -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping
-EX4_ARGS      := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping
-EX5_BDDC_ARGS := -m ../../data/star.mesh --usepetsc -o 0 --petscopts rc_ex5p_bddc --nonoverlapping
-EX5_FSPL_ARGS := -m ../../data/beam-tet.mesh --usepetsc -o 0 --petscopts rc_ex5p_fieldsplit
-EX6_ARGS      := -m ../../data/amr-quad.mesh --usepetsc
-EX9_E_ARGS    := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl -dt 0.1
-EX9_ES_ARGS   := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step
-EX9_IS_ARGS   := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5
-EX10_ARGS     := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3
+EX1_ARGS_W       := -m ../../data/amr-quad.mesh --usepetsc
+EX1_ARGS_P       := -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_ex1p
+EX2_ARGS         := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p
+EX3_ARGS         := -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping
+EX4_ARGS         := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping
+EX4_HYB_ARGS     := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping --hybridization
+EX5_BDDC_LB_ARGS := -m ../../data/star.mesh --usepetsc -o 0 --petscopts rc_ex5p_bddc --nonoverlapping --local-bdr
+EX5_BDDC_GB_ARGS := -m ../../data/star.mesh --usepetsc -o 0 --petscopts rc_ex5p_bddc --nonoverlapping
+EX5_FSPL_ARGS    := -m ../../data/beam-tet.mesh --usepetsc -o 0 --petscopts rc_ex5p_fieldsplit
+EX6_ARGS         := -m ../../data/amr-quad.mesh --usepetsc
+EX6_NONOVL_ARGS  := -m ../../data/amr-quad.mesh --usepetsc --nonoverlapping
+EX9_E_ARGS       := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl -dt 0.1
+EX9_ES_ARGS      := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step
+EX9_IS_ARGS      := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5
+EX10_ARGS        := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3
 ex1p-test-par: ex1p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_W))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_P))
@@ -90,11 +93,14 @@ ex3p-test-par: ex3p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX3_ARGS))
 ex4p-test-par: ex4p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX4_ARGS))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX4_HYB_ARGS))
 ex5p-test-par: ex5p
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX5_BDDC_ARGS))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX5_BDDC_LB_ARGS))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX5_BDDC_GB_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX5_FSPL_ARGS))
 ex6p-test-par: ex6p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX6_ARGS))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX6_NONOVL_ARGS))
 ex9p-test-par: ex9p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX9_E_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX9_ES_ARGS))

examples/petsc/rc_ex3p

@@ -6,4 +6,7 @@
 # it needs PETSc configured with MUMPS
 
 -solver_pc_type cholesky
+# Petsc 3.9
+-solver_pc_factor_mat_solver_type mumps
+# Older versions of PETSc
 -solver_pc_factor_mat_solver_package mumps

examples/petsc/rc_ex3p_bddc

@@ -16,7 +16,10 @@
 #-pc_bddc_adaptive_threshold 10
 
 # Customization of the local solvers
-#-pc_bddc_neumann_pc_factor_mat_solver_package mumps
-#-pc_bddc_dirichlet_pc_factor_mat_solver_package mumps
+# With PETSc versions older than 3.9
+# use "mat_solver_package" instead of "mat_solver_type"
+#
+#-pc_bddc_neumann_pc_factor_mat_solver_type mumps
+#-pc_bddc_dirichlet_pc_factor_mat_solver_type mumps
 #-pc_bddc_coarse_pc_type cholesky
-#-pc_bddc_coarse_pc_factor_mat_solver_package mumps
+#-pc_bddc_coarse_pc_factor_mat_solver_type mumps

examples/petsc/rc_ex4p

@@ -2,4 +2,7 @@
 # it needs PETSc configured with MUMPS
 
 -solver_pc_type cholesky
+# Petsc 3.9
+-solver_pc_factor_mat_solver_type mumps
+# Older versions of PETSc
 -solver_pc_factor_mat_solver_package mumps

examples/petsc/rc_ex4p_bddc

@@ -13,7 +13,10 @@
 #-pc_bddc_adaptive_threshold 10
 
 # Customization of the local solvers
-#-pc_bddc_neumann_pc_factor_mat_solver_package mumps
-#-pc_bddc_dirichlet_pc_factor_mat_solver_package mumps
+# With PETSc versions older than 3.9
+# use "mat_solver_package" instead of "mat_solver_type"
+#
+#-pc_bddc_neumann_pc_factor_mat_solver_type mumps
+#-pc_bddc_dirichlet_pc_factor_mat_solver_type mumps
 #-pc_bddc_coarse_pc_type cholesky
-#-pc_bddc_coarse_pc_factor_mat_solver_package mumps
+#-pc_bddc_coarse_pc_factor_mat_solver_type mumps

examples/petsc/rc_ex5p_bddc

@@ -25,15 +25,29 @@
 # verbose output
 #-prec_pc_bddc_check_level 1
 
-# local solvers (needs PETSc compiled with support for SuiteSparse)
-# default solvers will fail
+# local solvers (default "petsc" solvers will fail)
+# needs PETSc compiled with support for MUMPS or SuiteSparse
+# use "umfpack" in place of "mumps" if you want to use
+# SuiteSparse solvers
+#
+# With PETSc versions older than 3.9
+# use "mat_solver_package" instead of "mat_solver_type"
+#
 -prec_pc_bddc_neumann_pc_type lu
--prec_pc_bddc_neumann_pc_factor_mat_solver_package umfpack
+-prec_pc_bddc_neumann_pc_factor_mat_solver_type mumps
+-prec_pc_bddc_neumann_pc_factor_mat_solver_package mumps
 -prec_pc_bddc_dirichlet_pc_type lu
--prec_pc_bddc_dirichlet_pc_factor_mat_solver_package umfpack
+-prec_pc_bddc_dirichlet_pc_factor_mat_solver_type mumps
+-prec_pc_bddc_dirichlet_pc_factor_mat_solver_package mumps
+
+#  MUMPS sometimes fails with a very annoying error
+-mat_mumps_icntl_14 500
+-prec_pc_bddc_dirichlet_mat_mumps_icntl_14 500
+-prec_pc_bddc_neumann_mat_mumps_icntl_14 500
 
 # coarse solver (needs PETSc compiled with support for MUMPS)
 # default solver may fail
+-prec_pc_bddc_coarse_pc_factor_mat_solver_type mumps
 -prec_pc_bddc_coarse_pc_factor_mat_solver_package mumps
 -prec_pc_bddc_coarse_pc_type cholesky
 

examples/pumi/CMakeLists.txt

@@ -0,0 +1,72 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+set(PUMI_EXAMPLES_SRCS)
+# All PUMI examples require MPI
+if (MFEM_USE_MPI)
+  list(APPEND PUMI_EXAMPLES_SRCS
+    ex1.cpp
+    ex1p.cpp
+    ex2.cpp
+    ex6p.cpp
+    )
+endif()
+
+# Include the source directory where mfem.hpp and mfem-performance.hpp are.
+include_directories(BEFORE ${PROJECT_BINARY_DIR})
+
+# Add "test_pumi" target, see below.
+add_custom_target(test_pumi
+  ${CMAKE_CTEST_COMMAND} -R pumi USES_TERMINAL)
+
+# Add one executable per cpp file, adding "pumi_" as prefix. Sets
+# "test_pumi" as a target that depends on the given examples.
+set(PFX pumi_)
+add_mfem_examples(PUMI_EXAMPLES_SRCS ${PFX} "" test_pumi)
+
+# Testing.
+# The PUMI tests can be run separately using the target "test_pumi"
+# which builds the examples and runs:
+#   ctest -R pumi
+
+# Command line options for the tests.
+# TODO...
+
+# Set the number of processors for the parallel examples.
+set(EX1_TEST_NP 1)
+set(EX1P_TEST_NP 8)
+set(EX2_TEST_NP 1)
+set(EX6P_TEST_NP 8)
+
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${PUMI_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})
+
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+
+  # All PUMI examples require MPI
+  if (FALSE)
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    set(TEST_NP ${${UP_TEST_NAME}_TEST_NP})
+    add_test(NAME ${TEST_NAME}_np=${TEST_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${TEST_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()

examples/pumi/README

@@ -0,0 +1,18 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               http://mfem.org
+
+This directory contains modifications of the example codes that illustrate the
+use of MFEM features based on the Parallel Unstructured Mesh Infrastructure,
+PUMI, from https://scorec.rpi.edu/pumi.
+
+To build these examples, make sure that MFEM is configured with the option
+"MFEM_USE_PUMI = YES", see the top-level INSTALL file for details.
+
+We recommend comparing the original example codes with the corresponding files
+in the current directory.

examples/pumi/ex1.cpp

@@ -0,0 +1,262 @@
+//                                MFEM Example 1
+//                              PUMI Modification
+//
+// Compile with: make ex1
+//
+// Sample runs:
+//    ex1 -m ../../data/pumi/serial/Kova.smb -p ../../data/pumi/geom/Kova.dmg
+//
+// Note:         Example models + meshes for the PUMI examples can be downloaded
+//               from github.com/mfem/data/pumi. After downloading we recommend
+//               creating a symbolic link to the above directory in ../../data.
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Laplace problem
+//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
+//               Specifically, we discretize using a FE space of the specified
+//               order, or if order < 1 using an isoparametric/isogeometric
+//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of mesh refinement, finite
+//               element grid functions, as well as linear and bilinear forms
+//               corresponding to the left-hand side and right-hand side of the
+//               discrete linear system. We also cover the explicit elimination
+//               of essential boundary conditions, static condensation, and the
+//               optional connection to the GLVis tool for visualization.
+//
+//               This PUMI modification demonstrates how PUMI's API can be used
+//               to load a PUMI mesh classified on a geometric model and then
+//               convert it to the MFEM mesh format. The inputs are a Parasolid
+//               model, "*.xmt_txt" and a SCOREC mesh "*.smb". The option "-o"
+//               is used for the Finite Element order and "-go" is used for the
+//               geometry order. Note that they can be used independently, i.e.
+//               "-o 8 -go 3" solves for 8th order FE on a third order geometry.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#ifdef MFEM_USE_SIMMETRIX
+#include <SimUtil.h>
+#include <gmi_sim.h>
+#endif
+#include <apfMDS.h>
+#include <gmi_null.h>
+#include <PCU.h>
+#include <apfConvert.h>
+#include <gmi_mesh.h>
+#include <crv.h>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI (required by PUMI).
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/pumi/serial/Kova.smb";
+#ifdef MFEM_USE_SIMMETRIX
+   const char *model_file = "../../data/pumi/geom/Kova.x_t";
+#else
+   const char *model_file = "../../data/pumi/geom/Kova.dmg";
+#endif
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+   int geom_order = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&model_file, "-p", "--parasolid",
+                  "Parasolid model to use.");
+   args.AddOption(&geom_order, "-go", "--geometry_order",
+                  "Geometric order of the model");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 3. Read the SCOREC Mesh.
+   PCU_Comm_Init();
+#ifdef MFEM_USE_SIMMETRIX
+   Sim_readLicenseFile(0);
+   gmi_sim_start();
+   gmi_register_sim();
+#endif
+   gmi_register_mesh();
+
+   apf::Mesh2* pumi_mesh;
+   pumi_mesh = apf::loadMdsMesh(model_file, mesh_file);
+
+   // 4. Increase the geometry order if necessary.
+   if (geom_order > 1)
+   {
+      crv::BezierCurver bc(pumi_mesh, geom_order, 2);
+      bc.run();
+   }
+
+   pumi_mesh->verify();
+
+   // 5. Create the MFEM mesh object from the PUMI mesh. We can handle
+   //    triangular and tetrahedral meshes. Other inputs are the same as the
+   //    MFEM default constructor.
+   Mesh *mesh = new PumiMesh(pumi_mesh, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 6. Refine the mesh to increase the resolution. In this example we do
+   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
+   //    largest number that gives a final mesh with no more than 50,000
+   //    elements.
+   {
+      int ref_levels =
+         (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 7. Define a finite element space on the mesh. Here we use continuous
+   //    Lagrange finite elements of the specified order. If order < 1, we
+   //    instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (mesh->GetNodes())
+   {
+      fec = mesh->GetNodes()->OwnFEC();
+      cout << "Using isoparametric FEs: " << fec->Name() << endl;
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   cout << "Number of finite element unknowns: "
+        << fespace->GetTrueVSize() << endl;
+
+   // 8. Determine the list of true (i.e. conforming) essential boundary dofs.
+   //    In this example, the boundary conditions are defined by marking all
+   //    the boundary attributes from the mesh as essential (Dirichlet) and
+   //    converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 9. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
+   //    the basis functions in the finite element fespace.
+   LinearForm *b = new LinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   // 10. Define the solution vector x as a finite element grid function
+   //     corresponding to fespace. Initialize x with initial guess of zero,
+   //     which satisfies the boundary conditions.
+   GridFunction x(fespace);
+   x = 0.0;
+
+   // 11. Set up the bilinear form a(.,.) on the finite element space
+   //     corresponding to the Laplacian operator -Delta, by adding the
+   //     Diffusion domain integrator.
+   BilinearForm *a = new BilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 12. Assemble the bilinear form and the corresponding linear system,
+   //     applying any necessary transformations such as: eliminating boundary
+   //     conditions, applying conforming constraints for non-conforming AMR,
+   //     static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   SparseMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   cout << "Size of linear system: " << A.Height() << endl;
+
+#ifndef MFEM_USE_SUITESPARSE
+   // 13. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   //     solve the system A X = B with PCG.
+   GSSmoother M(A);
+   PCG(A, M, B, X, 1, 200, 1e-12, 0.0);
+#else
+   // 13. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+   UMFPackSolver umf_solver;
+   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   umf_solver.SetOperator(A);
+   umf_solver.Mult(B, X);
+#endif
+
+   // 14. Recover the solution as a finite element grid function.
+   a->RecoverFEMSolution(X, *b, x);
+
+   // 15. Save the refined mesh and the solution. This output can be viewed later
+   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
+   ofstream mesh_ofs("refined.mesh");
+   mesh_ofs.precision(8);
+   mesh->Print(mesh_ofs);
+   ofstream sol_ofs("sol.gf");
+   sol_ofs.precision(8);
+   x.Save(sol_ofs);
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *mesh << x << flush;
+   }
+
+   // 17. Free the used memory.
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete mesh;
+
+   pumi_mesh->destroyNative();
+   apf::destroyMesh(pumi_mesh);
+   PCU_Comm_Free();
+#ifdef MFEM_USE_SIMMETRIX
+   gmi_sim_stop();
+   Sim_unregisterAllKeys();
+#endif
+
+   MPI_Finalize();
+   return 0;
+}

examples/pumi/ex1p.cpp

@@ -0,0 +1,292 @@
+//                       MFEM Example 1 - Parallel Version
+//                              PUMI Modification
+//
+// Compile with: make ex1p
+//
+// Sample runs:
+//    mpirun -np 8 ex1p -m ../../data/pumi/parallel/Kova/Kova100k_8.smb
+//                      -p ../../data/pumi/geom/Kova.dmg -o 1 -go 2
+//
+// Note:         Example models + meshes for the PUMI examples can be downloaded
+//               from github.com/mfem/data/pumi. After downloading we recommend
+//               creating a symbolic link to the above directory in ../../data.
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Laplace problem
+//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
+//               Specifically, we discretize using a FE space of the specified
+//               order, or if order < 1 using an isoparametric/isogeometric
+//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of mesh refinement, finite
+//               element grid functions, as well as linear and bilinear forms
+//               corresponding to the left-hand side and right-hand side of the
+//               discrete linear system. We also cover the explicit elimination
+//               of essential boundary conditions, static condensation, and the
+//               optional connection to the GLVis tool for visualization.
+//
+//               This PUMI modification demonstrates how PUMI's API can be used
+//               to load a parallel PUMI mesh classified on a geometric model
+//               and then generate the corresponding parallel MFEM mesh. The
+//               example also performs a "uniform" refinement, similar to the
+//               MFEM examples, for coarse meshes. However, the refinement is
+//               performed using the PUMI API.  The inputs are a Parasolid
+//               model, "*.xmt_txt" and SCOREC parallel meshes "*.smb". The
+//               option "-o" is used for the Finite Element order and "-go" for
+//               the geometry order. Note that they can be used independently:
+//               "-o 8 -go 3" solves for 8th order FE on third order geometry.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#ifdef MFEM_USE_SIMMETRIX
+#include <SimUtil.h>
+#include <gmi_sim.h>
+#endif
+#include <apfMDS.h>
+#include <gmi_null.h>
+#include <PCU.h>
+#include <apfConvert.h>
+#include <gmi_mesh.h>
+#include <crv.h>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/pumi/parallel/Kova/Kova100k_8.smb";
+#ifdef MFEM_USE_SIMMETRIX
+   const char *model_file = "../../data/pumi/geom/Kova.x_t";
+#else
+   const char *model_file = "../../data/pumi/geom/Kova.dmg";
+#endif
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+   int geom_order = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&model_file, "-p", "--parasolid",
+                  "Parasolid model to use.");
+   args.AddOption(&geom_order, "-go", "--geometry_order",
+                  "Geometric order of the model");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 3. Read the SCOREC Mesh
+   PCU_Comm_Init();
+#ifdef MFEM_USE_SIMMETRIX
+   Sim_readLicenseFile(0);
+   gmi_sim_start();
+   gmi_register_sim();
+#endif
+   gmi_register_mesh();
+
+   apf::Mesh2* pumi_mesh;
+   pumi_mesh = apf::loadMdsMesh(model_file, mesh_file);
+
+   // 4. Increase the geometry order and refine the mesh if necessary. Parallel
+   //    uniform refinement is performed if the total number of elements is less
+   //    than 10,000.
+   int dim = pumi_mesh->getDimension();
+   int nEle = pumi_mesh->count(dim);
+   int ref_levels = (int)floor(log(10000./nEle)/log(2.)/dim);
+
+   if (geom_order > 1)
+   {
+      crv::BezierCurver bc(pumi_mesh, geom_order, 2);
+      bc.run();
+   }
+
+   // Perform Uniform refinement
+   if (ref_levels > 1)
+   {
+      ma::Input* uniInput = ma::configureUniformRefine(pumi_mesh, ref_levels);
+
+      if (geom_order > 1)
+      {
+         crv::adapt(uniInput);
+      }
+      else
+      {
+         ma::adapt(uniInput);
+      }
+   }
+
+   pumi_mesh->verify();
+
+   // 5. Create the parallel MFEM mesh object from the parallel PUMI mesh.
+   //    We can handle triangular and tetrahedral meshes. Note that the
+   //    mesh resolution is performed on the PUMI mesh.
+   ParMesh *pmesh = new ParPumiMesh(MPI_COMM_WORLD, pumi_mesh);
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+      if (myid == 0)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (1,phi_i) where phi_i are the basis functions in fespace.
+   ParLinearForm *b = new ParLinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   ParGridFunction x(fespace);
+   x = 0.0;
+
+   // 10. Set up the parallel bilinear form a(.,.) on the finite element space
+   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //     domain integrator.
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   HypreParMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+   }
+
+   // 12. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+   //     preconditioner from hypre.
+   HypreSolver *amg = new HypreBoomerAMG(A);
+   HyprePCG *pcg = new HyprePCG(A);
+   pcg->SetTol(1e-12);
+   pcg->SetMaxIter(200);
+   pcg->SetPrintLevel(2);
+   pcg->SetPreconditioner(*amg);
+   pcg->Mult(B, X);
+
+   // 13. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a->RecoverFEMSolution(X, *b, x);
+
+   // 14. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 15. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x << flush;
+   }
+
+   // 16. Free the used memory.
+   delete pcg;
+   delete amg;
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
+   pumi_mesh->destroyNative();
+   apf::destroyMesh(pumi_mesh);
+   PCU_Comm_Free();
+
+#ifdef MFEM_USE_SIMMETRIX
+   gmi_sim_stop();
+   Sim_unregisterAllKeys();
+#endif
+
+   MPI_Finalize();
+
+   return 0;
+}

examples/pumi/ex2.cpp

@@ -0,0 +1,413 @@
+//                                MFEM Example 2
+//                              PUMI Modification
+//
+// Compile with: make ex2
+//
+// Sample runs:
+//    ex2 -m ../../data/pumi/serial/pillbox.smb -p ../../data/pumi/geom/pillbox.dmg
+//        -bf ../../data/pumi/serial/boundary.mesh
+//
+// Note:         Example models + meshes for the PUMI examples can be downloaded
+//               from github.com/mfem/data/pumi. After downloading we recommend
+//               creating a symbolic link to the above directory in ../../data.
+//
+// Description:  This example code solves a simple linear elasticity problem
+//               describing a multi-material cantilever beam.
+//
+//               Specifically, we approximate the weak form of -div(sigma(u))=0
+//               where sigma(u)=lambda*div(u)*I+mu*(grad*u+u*grad) is the stress
+//               tensor corresponding to displacement field u, and lambda and mu
+//               are the material Lame constants. The boundary conditions are
+//               u=0 on the fixed part of the boundary with attribute 1, and
+//               sigma(u).n=f on the remainder with f being a constant pull down
+//               vector on boundary elements with attribute 2, and zero
+//               otherwise. The geometry of the domain is assumed to be as
+//               follows:
+//                                        boundary
+//                                       attribute 2
+//                                       (push down)
+//                                            ||
+//                                            \/
+//                                       +----------+
+//                                       |          |
+//                                       |          |
+//                             +---------| material |----------+
+//                boundary --->| material|    2     | material |<--- boundary
+//                attribute 1  |    1    |          |    3     |     attribute 1
+//                 (fixed)     +---------+----------+----------+     (fixed)
+//
+//               The example demonstrates the use of high-order and NURBS vector
+//               finite element spaces with the linear elasticity bilinear form,
+//               meshes with curved elements, and the definition of piece-wise
+//               constant and vector coefficient objects. Static condensation is
+//               also illustrated.
+//
+//               We recommend viewing Example 1 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#include "../../general/text.hpp"
+
+#ifdef MFEM_USE_SIMMETRIX
+#include <SimUtil.h>
+#include <gmi_sim.h>
+#endif
+#include <apfMDS.h>
+#include <gmi_null.h>
+#include <PCU.h>
+#include <apfConvert.h>
+#include <gmi_mesh.h>
+#include <crv.h>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI (required by PUMI).
+   int num_proc, myId;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_proc);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myId);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/pumi/serial/pillbox.smb";
+   const char *boundary_file = "../../data/pumi/serial/boundary.mesh";
+#ifdef MFEM_USE_SIMMETRIX
+   const char *model_file = "../../data/pumi/geom/pillbox.smd";
+#else
+   const char *model_file = "../../data/pumi/geom/pillbox.dmg";
+#endif
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+   int geom_order = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&model_file, "-p", "--parasolid",
+                  "Parasolid model to use.");
+   args.AddOption(&geom_order, "-go", "--geometry_order",
+                  "Geometric order of the model");
+   args.AddOption(&boundary_file, "-bf", "--txt",
+                  "txt file containing boundary tags");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 3. Read the SCOREC Mesh.
+   PCU_Comm_Init();
+#ifdef MFEM_USE_SIMMETRIX
+   Sim_readLicenseFile(0);
+   gmi_sim_start();
+   gmi_register_sim();
+#endif
+   gmi_register_mesh();
+
+   apf::Mesh2* pumi_mesh;
+   pumi_mesh = apf::loadMdsMesh(model_file, mesh_file);
+
+   // 4. Increase the geometry order if necessary.
+   if (geom_order > 1)
+   {
+      crv::BezierCurver bc(pumi_mesh, geom_order, 0);
+      bc.run();
+   }
+   pumi_mesh->verify();
+
+   // Read boundary
+   string bdr_tags;
+   named_ifgzstream input_bdr(boundary_file);
+   input_bdr >> ws;
+   getline(input_bdr, bdr_tags);
+   filter_dos(bdr_tags);
+   cout << " the boundary tag is : " << bdr_tags << endl;
+   Array<int> Dirichlet;
+   int numOfent;
+   if (bdr_tags == "Dirichlet")
+   {
+      input_bdr >> numOfent;
+      cout << " num of Dirichlet bdr conditions : " << numOfent << endl;
+      Dirichlet.SetSize(numOfent);
+      for (int kk = 0; kk < numOfent; kk++)
+      {
+         input_bdr >> Dirichlet[kk];
+      }
+   }
+   Dirichlet.Print();
+
+   Array<int> load_bdr;
+   skip_comment_lines(input_bdr, '#');
+   input_bdr >> bdr_tags;
+   filter_dos(bdr_tags);
+   cout << " the boundary tag is : " << bdr_tags << endl;
+   if (bdr_tags == "Load")
+   {
+      input_bdr >> numOfent;
+      load_bdr.SetSize(numOfent);
+      cout << " num of load bdr conditions : " << numOfent << endl;
+      for (int kk = 0; kk < numOfent; kk++)
+      {
+         input_bdr >> load_bdr[kk];
+      }
+   }
+   load_bdr.Print();
+
+   // 5. Create the MFEM mesh object from the PUMI mesh. We can handle triangular
+   //    and tetrahedral meshes. Other inputs are the same as MFEM default
+   //    constructor.
+   Mesh *mesh = new PumiMesh(pumi_mesh, 1, 1);
+   int dim = mesh->Dimension();
+
+   //  Boundary conditions hack.
+   apf::MeshIterator* itr = pumi_mesh->begin(dim-1);
+   apf::MeshEntity* ent ;
+   int bdr_cnt = 0;
+   while ((ent = pumi_mesh->iterate(itr)))
+   {
+      apf::ModelEntity *me = pumi_mesh->toModel(ent);
+      if (pumi_mesh->getModelType(me) == (dim-1))
+      {
+         // Everywhere 3 as initial
+         (mesh->GetBdrElement(bdr_cnt))->SetAttribute(3);
+         int tag = pumi_mesh->getModelTag(me);
+         if (Dirichlet.Find(tag) != -1)
+         {
+            // Dirichlet attr -> 1
+            (mesh->GetBdrElement(bdr_cnt))->SetAttribute(1);
+         }
+         else if (load_bdr.Find(tag) != -1)
+         {
+            // Load attr -> 2
+            (mesh->GetBdrElement(bdr_cnt))->SetAttribute(2);
+         }
+         bdr_cnt++;
+      }
+   }
+   pumi_mesh->end(itr);
+
+   // Assign attributes for elements.
+   double ppt[3];
+   Vector cent(ppt, dim);
+   for (int el = 0; el < mesh->GetNE(); el++)
+   {
+      (mesh->GetElementTransformation(el))->
+      Transform(Geometries.GetCenter(mesh->GetElementBaseGeometry(el)),cent);
+      if (cent(0) <= -0.05)
+      {
+         mesh->SetAttribute(el , 1);
+      }
+      else if (cent(0) >= 0.05)
+      {
+         mesh->SetAttribute(el , 2);
+      }
+      else
+      {
+         mesh->SetAttribute(el , 3);
+      }
+   }
+   mesh->SetAttributes();
+   if (mesh->attributes.Max() < 2 || mesh->bdr_attributes.Max() < 2)
+   {
+      cerr << "\nInput mesh should have at least two materials and "
+           << "two boundary attributes! (See schematic in ex2.cpp)\n"
+           << endl;
+      return 3;
+   }
+
+   // 6. Refine the mesh to increase the resolution. In this example we do
+   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
+   //    largest number that gives a final mesh with no more than 5,000
+   //    elements.
+   {
+      int ref_levels =
+         (int)floor(log(5000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 7. Define a finite element space on the mesh. Here we use vector finite
+   //    elements, i.e. dim copies of a scalar finite element space. The vector
+   //    dimension is specified by the last argument of the FiniteElementSpace
+   //    constructor. For NURBS meshes, we use the (degree elevated) NURBS space
+   //    associated with the mesh nodes.
+   FiniteElementCollection *fec;
+   FiniteElementSpace *fespace;
+   if (mesh->NURBSext)
+   {
+      fec = NULL;
+      fespace = mesh->GetNodes()->FESpace();
+   }
+   else
+   {
+      fec = new H1_FECollection(order, dim);
+      fespace = new FiniteElementSpace(mesh, fec, dim);
+   }
+   cout << "Number of finite element unknowns: " << fespace->GetTrueVSize()
+        << endl << "Assembling: " << flush;
+
+   // 8. Determine the list of true (i.e. conforming) essential boundary dofs.
+   //    In this example, the boundary conditions are defined by marking only
+   //    boundary attribute 1 from the mesh as essential and converting it to a
+   //    list of true dofs.
+   Array<int> ess_tdof_list, ess_bdr(mesh->bdr_attributes.Max());
+   ess_bdr = 0;
+   ess_bdr[0] = 1;
+   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+   // 9. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system. In this case, b_i equals the boundary integral
+   //    of f*phi_i where f represents a "pull down" force on the Neumann part
+   //    of the boundary and phi_i are the basis functions in the finite element
+   //    fespace. The force is defined by the VectorArrayCoefficient object f,
+   //    which is a vector of Coefficient objects. The fact that f is non-zero
+   //    on boundary attribute 2 is indicated by the use of piece-wise constants
+   //    coefficient for its last component.
+   VectorArrayCoefficient f(dim);
+   for (int i = 0; i < dim-1; i++)
+   {
+      f.Set(i, new ConstantCoefficient(0.0));
+   }
+   {
+      Vector pull_force(mesh->bdr_attributes.Max());
+      pull_force = 0.0;
+      pull_force(1) = -3.0e-2;
+      f.Set(dim-1, new PWConstCoefficient(pull_force));
+      f.Set(dim-2, new PWConstCoefficient(pull_force));
+   }
+
+   LinearForm *b = new LinearForm(fespace);
+   b->AddBoundaryIntegrator(new VectorBoundaryLFIntegrator(f));
+   cout << "r.h.s. ... " << flush;
+   b->Assemble();
+
+   // 10. Define the solution vector x as a finite element grid function
+   //     corresponding to fespace. Initialize x with initial guess of zero,
+   //     which satisfies the boundary conditions.
+   GridFunction x(fespace);
+   x = 0.0;
+
+   // 11. Set up the bilinear form a(.,.) on the finite element space
+   //     corresponding to the linear elasticity integrator with piece-wise
+   //     constants coefficient lambda and mu.
+   Vector lambda(mesh->attributes.Max());
+   lambda = 1.0;
+   lambda(0) = lambda(1)*10;
+   lambda(1) = lambda(1)*100;
+   PWConstCoefficient lambda_func(lambda);
+   Vector mu(mesh->attributes.Max());
+   mu = 1.0;
+   mu(0) = mu(1)*10;
+   mu(1) = mu(1)*100;
+   PWConstCoefficient mu_func(mu);
+
+   BilinearForm *a = new BilinearForm(fespace);
+   a->AddDomainIntegrator(new ElasticityIntegrator(lambda_func,mu_func));
+
+   // 12. Assemble the bilinear form and the corresponding linear system,
+   //     applying any necessary transformations such as: eliminating boundary
+   //     conditions, applying conforming constraints for non-conforming AMR,
+   //     static condensation, etc.
+   cout << "matrix ... " << flush;
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   SparseMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+   cout << "done." << endl;
+
+   cout << "Size of linear system: " << A.Height() << endl;
+
+#ifndef MFEM_USE_SUITESPARSE
+   // 13. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   //     solve the system Ax=b with PCG.
+   GSSmoother M(A);
+   PCG(A, M, B, X, 1, 500, 1e-8, 0.0);
+#else
+   // 13. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+   UMFPackSolver umf_solver;
+   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   umf_solver.SetOperator(A);
+   umf_solver.Mult(B, X);
+#endif
+
+   // 14. Recover the solution as a finite element grid function.
+   a->RecoverFEMSolution(X, *b, x);
+
+   // 15. For non-NURBS meshes, make the mesh curved based on the finite element
+   //     space. This means that we define the mesh elements through a fespace
+   //     based transformation of the reference element. This allows us to save
+   //     the displaced mesh as a curved mesh when using high-order finite
+   //     element displacement field. We assume that the initial mesh (read from
+   //     the file) is not higher order curved mesh compared to the chosen FE
+   //     space.
+   if (!mesh->NURBSext)
+   {
+      mesh->SetNodalFESpace(fespace);
+   }
+
+   // 16. Save the displaced mesh and the inverted solution (which gives the
+   //     backward displacements to the original grid). This output can be
+   //     viewed later using GLVis: "glvis -m displaced.mesh -g sol.gf".
+   {
+      GridFunction *nodes = mesh->GetNodes();
+      *nodes += x;
+      x *= -1;
+      ofstream mesh_ofs("displaced.mesh");
+      mesh_ofs.precision(8);
+      mesh->Print(mesh_ofs);
+      ofstream sol_ofs("sol.gf");
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 17. Send the above data by socket to a GLVis server. Use the "n" and "b"
+   //     keys in GLVis to visualize the displacements.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *mesh << x << flush;
+   }
+
+   // 18. Free the used memory.
+   delete a;
+   delete b;
+   if (fec)
+   {
+      delete fespace;
+      delete fec;
+   }
+   delete mesh;
+
+   pumi_mesh->destroyNative();
+   apf::destroyMesh(pumi_mesh);
+   PCU_Comm_Free();
+#ifdef MFEM_USE_SIMMETRIX
+   gmi_sim_stop();
+   Sim_unregisterAllKeys();
+#endif
+
+   MPI_Finalize();
+
+   return 0;
+}

examples/pumi/ex6p.cpp

@@ -0,0 +1,387 @@
+//                       MFEM Example 6 - Parallel Version
+//                              PUMI Modification
+//
+// Compile with: make ex1p
+//
+// Sample runs:  mpirun -np 8 ex6p
+//
+// Description:  This is a version of Example 1 with a simple adaptive mesh
+//               refinement loop. The problem being solved is again the Laplace
+//               equation -Delta u = 1 with homogeneous Dirichlet boundary
+//               conditions. The problem is solved on a sequence of meshes which
+//               are adapted in a conforming (tetrahedrons) manner according
+//               to a simple SPR ZZ error estimator.
+//
+//               This PUMI variation also performs a "uniform" refinement,
+//               similar to MFEM examples, for coarse meshes. However, the
+//               refinement is performed using the PUMI API. A new option "-ar"
+//               is added to modify the "adapt_ratio" which is the fraction of
+//               allowable error that scales the output size field of the error
+//               estimator.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#ifdef MFEM_USE_SIMMETRIX
+#include <SimUtil.h>
+#include <gmi_sim.h>
+#endif
+#include <apfMDS.h>
+#include <gmi_null.h>
+#include <PCU.h>
+#include <spr.h>
+#include <apfConvert.h>
+#include <gmi_mesh.h>
+#include <crv.h>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/pumi/parallel/Kova/Kova100k_8.smb";
+#ifdef MFEM_USE_SIMMETRIX
+   const char *model_file = "../../data/pumi/geom/Kova.x_t";
+   const char *smd_file = NULL;
+#else
+   const char *model_file = "../../data/pumi/geom/Kova.dmg";
+#endif
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+   int geom_order = 1;
+   double adapt_ratio = 0.05;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&model_file, "-p", "--model",
+                  "parasolid or .dmg model to use.");
+#ifdef MFEM_USE_SIMMETRIX
+   args.AddOption(&smd_file, "-sm", "--smd_model",
+                  "smd model file to use.");
+#endif
+   args.AddOption(&geom_order, "-go", "--geometry_order",
+                  "Geometric order of the model");
+   args.AddOption(&adapt_ratio, "-ar", "--adapt_ratio",
+                  "adaptation factor used in MeshAdapt");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 3. Read the SCOREC Mesh.
+   PCU_Comm_Init();
+#ifdef MFEM_USE_SIMMETRIX
+   Sim_readLicenseFile(0);
+   gmi_sim_start();
+   gmi_register_sim();
+#endif
+   gmi_register_mesh();
+
+   apf::Mesh2* pumi_mesh;
+#ifdef MFEM_USE_SIMMETRIX
+   if (smd_file)
+   {
+      gmi_model *mixed_model = gmi_sim_load(model_file, smd_file);
+      pumi_mesh = apf::loadMdsMesh(mixed_model, mesh_file);
+   }
+   else
+#endif
+   {
+      pumi_mesh = apf::loadMdsMesh(model_file, mesh_file);
+   }
+
+   // 4. Increase the geometry order and refine the mesh if necessary.  Parallel
+   //    uniform refinement is performed if the total number of elements is less
+   //    than 100,000.
+   int dim = pumi_mesh->getDimension();
+   int nEle = pumi_mesh->count(dim);
+   int ref_levels = (int)floor(log(100000./nEle)/log(2.)/dim);
+
+   if (geom_order > 1)
+   {
+      crv::BezierCurver bc(pumi_mesh, geom_order, 2);
+      bc.run();
+   }
+
+   // Perform Uniform refinement
+   if (myid == 1)
+   {
+      std::cout << " ref level : " <<     ref_levels << std::endl;
+   }
+
+   if (ref_levels > 1)
+   {
+      ma::Input* uniInput = ma::configureUniformRefine(pumi_mesh, ref_levels);
+
+      if ( geom_order > 1)
+      {
+         crv::adapt(uniInput);
+      }
+      else
+      {
+         ma::adapt(uniInput);
+      }
+   }
+
+   pumi_mesh->verify();
+
+   // 5. Create the parallel MFEM mesh object from the parallel PUMI mesh.  We
+   //    can handle triangular and tetrahedral meshes. Note that the mesh
+   //    resolution is performed on the PUMI mesh.
+   ParMesh *pmesh = new ParPumiMesh(MPI_COMM_WORLD, pumi_mesh);
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+      if (myid == 1)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 1)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (1,phi_i) where phi_i are the basis functions in fespace.
+   ParLinearForm *b = new ParLinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+
+   // 8. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   ParGridFunction x(fespace);
+   x = 0.0;
+
+   // 9. Connect to GLVis.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+
+   socketstream sout;
+   if (visualization)
+   {
+      sout.open(vishost, visport);
+      if (!sout)
+      {
+         if (myid == 0)
+         {
+            cout << "Unable to connect to GLVis server at "
+                 << vishost << ':' << visport << endl;
+            cout << "GLVis visualization disabled.\n";
+         }
+         visualization = false;
+      }
+
+      sout.precision(8);
+   }
+
+   // 10. Set up the parallel bilinear form a(.,.) on the finite element space
+   //     corresponding to the Laplacian operator -Delta, by adding the
+   //     Diffusion domain integrator.
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+
+   // 12. The main AMR loop. In each iteration we solve the problem on the
+   //     current mesh, visualize the solution, and adapt the mesh.
+   apf::Field* Tmag_field = 0;
+   apf::Field* temp_field = 0;
+   apf::Field* ipfield = 0;
+   apf::Field* sizefield = 0;
+   int max_iter = 3;
+
+   for (int Itr = 0; Itr < max_iter; Itr++)
+   {
+      HYPRE_Int global_dofs = fespace->GlobalTrueVSize();
+      if (myid == 1)
+      {
+         cout << "\nAMR iteration " << Itr << endl;
+         cout << "Number of unknowns: " << global_dofs << endl;
+      }
+
+      // Assemble.
+      a->Assemble();
+      b->Assemble();
+
+      // Essential boundary condition.
+      Array<int> ess_tdof_list;
+      if (pmesh->bdr_attributes.Size())
+      {
+         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+         ess_bdr = 1;
+         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      // Form linear system.
+      HypreParMatrix A;
+      Vector B, X;
+      const int copy_interior = 1;
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B, copy_interior);
+
+      // 13. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+      //     preconditioner from hypre.
+      HypreBoomerAMG amg;
+      amg.SetPrintLevel(0);
+      CGSolver pcg(A.GetComm());
+      pcg.SetPreconditioner(amg);
+      pcg.SetOperator(A);
+      pcg.SetRelTol(1e-6);
+      pcg.SetMaxIter(200);
+      pcg.SetPrintLevel(3); // print the first and the last iterations only
+      pcg.Mult(B, X);
+
+      // 14. Recover the parallel grid function corresponding to X. This is the
+      //     local finite element solution on each processor.
+      a->RecoverFEMSolution(X, *b, x);
+
+      // 15. Save in parallel the displaced mesh and the inverted solution (which
+      //     gives the backward displacements to the original grid). This output
+      //     can be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+      {
+         ostringstream mesh_name, sol_name;
+         mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+         sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+         ofstream mesh_ofs(mesh_name.str().c_str());
+         mesh_ofs.precision(8);
+         pmesh->Print(mesh_ofs);
+
+         ofstream sol_ofs(sol_name.str().c_str());
+         sol_ofs.precision(8);
+         x.Save(sol_ofs);
+      }
+
+      // 16. Send the above data by socket to a GLVis server.  Use the "n" and "b"
+      //     keys in GLVis to visualize the displacements.
+      if (visualization)
+      {
+         sout << "parallel " << num_procs << " " << myid << "\n";
+         sout << "solution\n" << *pmesh << x << flush;
+      }
+
+      // 17. Field transfer. Scalar solution field and magnitude field for error
+      //     estimation are created the PUMI mesh.
+      if (order > geom_order)
+      {
+         Tmag_field = apf::createField(pumi_mesh, "field_mag",
+                                       apf::SCALAR, apf::getLagrange(order));
+         temp_field = apf::createField(pumi_mesh, "T_field",
+                                       apf::SCALAR, apf::getLagrange(order));
+      }
+      else
+      {
+         Tmag_field = apf::createFieldOn(pumi_mesh, "field_mag",apf::SCALAR);
+         temp_field = apf::createFieldOn(pumi_mesh, "T_field", apf::SCALAR);
+      }
+
+      ParPumiMesh* pPPmesh = dynamic_cast<ParPumiMesh*>(pmesh);
+      pPPmesh->FieldMFEMtoPUMI(pumi_mesh, &x, temp_field, Tmag_field);
+
+      ipfield= spr::getGradIPField(Tmag_field, "MFEM_gradip", 2);
+      sizefield = spr::getSPRSizeField(ipfield, adapt_ratio);
+
+      apf::destroyField(Tmag_field);
+      apf::destroyField(ipfield);
+      apf::destroyNumbering(pumi_mesh->findNumbering("LocalVertexNumbering"));
+
+      // 18. Perform MesAdapt.
+      ma::Input* erinput = ma::configure(pumi_mesh, sizefield);
+      erinput->shouldFixShape = true;
+      erinput->maximumIterations = 2;
+      if ( geom_order > 1)
+      {
+         crv::adapt(erinput);
+      }
+      else
+      {
+         ma::adapt(erinput);
+      }
+
+      ParMesh* Adapmesh = new ParPumiMesh(MPI_COMM_WORLD, pumi_mesh);
+      pPPmesh->UpdateMesh(Adapmesh);
+      delete Adapmesh;
+
+      // 19. Update the FiniteElementSpace, GridFunction, and bilinear form.
+      fespace->Update();
+      x.Update();
+      x = 0.0;
+
+      pPPmesh->FieldPUMItoMFEM(pumi_mesh, temp_field, &x);
+      a->Update();
+      b->Update();
+
+      // Destroy fields.
+      apf::destroyField(temp_field);
+      apf::destroyField(sizefield);
+   }
+
+   // 20. Free the used memory.
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
+   pumi_mesh->destroyNative();
+   apf::destroyMesh(pumi_mesh);
+   PCU_Comm_Free();
+
+#ifdef MFEM_USE_SIMMETRIX
+   gmi_sim_stop();
+   Sim_unregisterAllKeys();
+#endif
+
+   MPI_Finalize();
+
+   return 0;
+}

examples/pumi/makefile

@@ -0,0 +1,94 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/pumi/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+# All PUMI examples require MPI
+SEQ_EXAMPLES =
+PAR_EXAMPLES = ex1 ex1p ex2 ex6p
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES)
+
+ifeq ($(MFEM_USE_PUMI),NO)
+$(EXAMPLES):
+	$(error MFEM is not configured with PUMI)
+endif
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+ifneq (,$(filter test%,$(MAKECMDGOALS)))
+   ifeq (,$(wildcard ../../data/pumi))
+      $(info PUMI data directory not found. The PUMI tests will be SKIPPED.)
+      mfem-test = printf "   $(3) [$(2) $(1) ... ]: "; $(PRINT_SKIP)
+   endif
+endif
+
+# Testing: Parallel vs. serial runs
+RUN_MPI_NP = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP)
+RUN_MPI = $(RUN_MPI_NP) $(MFEM_MPI_NP)
+SERIAL_NAME := Serial PUMI example
+PARALLEL_NAME := Parallel PUMI example
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
+%-test-seq: %
+	@$(call mfem-test,$<,, $(SERIAL_NAME))
+
+# Testing: Example-specific execution options:
+ex1-test-par: ex1
+	@$(call mfem-test,$<, $(RUN_MPI_NP) 1, $(PARALLEL_NAME))
+ex1p-test-par: ex1p
+	@$(call mfem-test,$<, $(RUN_MPI_NP) 8, $(PARALLEL_NAME))
+ex2-test-par: ex2
+	@$(call mfem-test,$<, $(RUN_MPI_NP) 1, $(PARALLEL_NAME))
+ex6p-test-par: ex6p
+	@$(call mfem-test,$<, $(RUN_MPI_NP) 8, $(PARALLEL_NAME))
+
+# Testing: "test" target and mfem-test* variables are defined in config/test.mk
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -f refined.mesh sol.gf mesh.* sol.* displaced.mesh

fem/bilinearform.cpp

@@ -688,7 +688,7 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
    else
 #endif
    {
-      if (A.Type() == Operator::MFEM_SPARSEMAT)
+      if (A.Type() == Operator::MFEM_SPARSEMAT || A.Type() == Operator::ANY_TYPE)
       {
          SparseMatrix A_sm;
          FormLinearSystem(ess_tdof_list, x, b, A_sm, X, B, copy_interior);
@@ -724,7 +724,7 @@ void BilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
    else
 #endif
    {
-      if (A.Type() == Operator::MFEM_SPARSEMAT)
+      if (A.Type() == Operator::MFEM_SPARSEMAT || A.Type() == Operator::ANY_TYPE)
       {
          SparseMatrix A_sm;
          FormSystemMatrix(ess_tdof_list, A_sm);
@@ -847,7 +847,7 @@ void BilinearForm::ComputeElementMatrices()
 }
 
 void BilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                                        Vector &sol, Vector &rhs,
+                                        const Vector &sol, Vector &rhs,
                                         DiagonalPolicy dpolicy)
 {
    Array<int> ess_dofs, conf_ess_dofs;
@@ -899,7 +899,7 @@ void BilinearForm::EliminateEssentialBCDiag (const Array<int> &bdr_attr_is_ess,
 }
 
 void BilinearForm::EliminateVDofs(const Array<int> &vdofs,
-                                  Vector &sol, Vector &rhs,
+                                  const Vector &sol, Vector &rhs,
                                   DiagonalPolicy dpolicy)
 {
    for (int i = 0; i < vdofs.Size(); i++)
@@ -939,7 +939,8 @@ void BilinearForm::EliminateVDofs(const Array<int> &vdofs,
 }
 
 void BilinearForm::EliminateEssentialBCFromDofs(
-   const Array<int> &ess_dofs, Vector &sol, Vector &rhs, DiagonalPolicy dpolicy)
+   const Array<int> &ess_dofs, const Vector &sol, Vector &rhs,
+   DiagonalPolicy dpolicy)
 {
    MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
    MFEM_ASSERT(sol.Size() == height, "incorrect sol Vector size");
@@ -1232,7 +1233,7 @@ void MixedBilinearForm::ConformingAssemble()
 }
 
 void MixedBilinearForm::EliminateTrialDofs (
-   Array<int> &bdr_attr_is_ess, Vector &sol, Vector &rhs )
+   Array<int> &bdr_attr_is_ess, const Vector &sol, Vector &rhs )
 {
    int i, j, k;
    Array<int> tr_vdofs, cols_marker (trial_fes -> GetVSize());
@@ -1255,7 +1256,7 @@ void MixedBilinearForm::EliminateTrialDofs (
 }
 
 void MixedBilinearForm::EliminateEssentialBCFromTrialDofs (
-   Array<int> &marked_vdofs, Vector &sol, Vector &rhs)
+   Array<int> &marked_vdofs, const Vector &sol, Vector &rhs)
 {
    mat -> EliminateCols (marked_vdofs, &sol, &rhs);
 }

fem/bilinearform.hpp

@@ -347,7 +347,7 @@ public:
        essential DOFs is set to 1.0. This behavior is controlled by the argument
        @a dpolicy. */
    void EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                             Vector &sol, Vector &rhs,
+                             const Vector &sol, Vector &rhs,
                              DiagonalPolicy dpolicy = DIAG_ONE);
 
    /// Eliminate essential boundary DOFs from the system matrix.
@@ -358,7 +358,7 @@ public:
                                  double value);
 
    /// Eliminate the given @a vdofs. NOTE: here, @a vdofs is a list of DOFs.
-   void EliminateVDofs(const Array<int> &vdofs, Vector &sol, Vector &rhs,
+   void EliminateVDofs(const Array<int> &vdofs, const Vector &sol, Vector &rhs,
                        DiagonalPolicy dpolicy = DIAG_ONE);
 
    /// Eliminate the given @a vdofs, storing the eliminated part internally.
@@ -369,10 +369,10 @@ public:
                        DiagonalPolicy dpolicy = DIAG_ONE);
 
    /** @brief Similar to
-       EliminateVDofs(const Array<int> &, Vector &, Vector &, DiagonalPolicy)
+       EliminateVDofs(const Array<int> &, const Vector &, Vector &, DiagonalPolicy)
        but here @a ess_dofs is a marker (boolean) array on all vector-dofs
        (@a ess_dofs[i] < 0 is true). */
-   void EliminateEssentialBCFromDofs(const Array<int> &ess_dofs, Vector &sol,
+   void EliminateEssentialBCFromDofs(const Array<int> &ess_dofs, const Vector &sol,
                                      Vector &rhs, DiagonalPolicy dpolicy = DIAG_ONE);
 
    /** @brief Similar to EliminateVDofs(const Array<int> &, DiagonalPolicy) but
@@ -496,10 +496,10 @@ public:
    void ConformingAssemble();
 
    void EliminateTrialDofs(Array<int> &bdr_attr_is_ess,
-                           Vector &sol, Vector &rhs);
+                           const Vector &sol, Vector &rhs);
 
    void EliminateEssentialBCFromTrialDofs(Array<int> &marked_vdofs,
-                                          Vector &sol, Vector &rhs);
+                                          const Vector &sol, Vector &rhs);
 
    virtual void EliminateTestDofs(Array<int> &bdr_attr_is_ess);
 

fem/bilininteg.cpp

@@ -1598,6 +1598,7 @@ void VectorCurlCurlIntegrator::AssembleElementMatrix(
       ir = &IntRules.Get(el.GetGeomType(), order);
    }
 
+   elmat.SetSize(dof*dim);
    elmat = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
    {

fem/coefficient.cpp

@@ -209,7 +209,7 @@ void VectorRestrictedCoefficient::Eval(
    }
    else
    {
-      M.SetSize(vdim);
+      M.SetSize(vdim, ir.GetNPoints());
       M = 0.0;
    }
 }

fem/conduitdatacollection.cpp

@@ -304,32 +304,43 @@ ConduitDataCollection::BlueprintMeshToMesh(const Node &n_mesh,
    if ( n_mesh_topo.has_child("boundary_topology") )
    {
       std::string bndry_topo_name = n_mesh_topo["boundary_topology"].as_string();
-      const Node &n_bndry_topo    = n_mesh["topologies"][bndry_topo_name];
-      std::string bndry_ele_shape = n_bndry_topo["elements/shape"].as_string();
 
-      bndry_geo = ShapeNameToGeomType(bndry_ele_shape);
-      int num_idxs_per_bndry_ele = Geometry::NumVerts[mesh_geo];
+      // In VisIt, we encountered a case were a mesh specified a boundary
+      // topology, but the boundary topology was omitted from the blueprint
+      // index, so it's data could not be obtained.
+      //
+      // This guard prevents an error in that case, allowing the mesh to be
+      // created without boundary info
 
-      const Node &n_bndry_conn = n_bndry_topo["elements/connectivity"];
-
-      // mfem requires ints, we could have int64s, etc convert if necessary
-      if ( n_bndry_conn.dtype().is_int() &&
-           n_bndry_conn.is_compact())
+      if (n_mesh["topologies"].has_child(bndry_topo_name))
       {
-         bndry_indices = n_bndry_conn.value();
-      }
-      else
-      {
-         Node &(n_bndry_conn_conv) =
-            n_conv["topologies"][bndry_topo_name]["elements/connectivity"];
-         n_bndry_conn.to_int_array(n_bndry_conn_conv);
-         bndry_indices = (n_bndry_conn_conv).value();
+         const Node &n_bndry_topo    = n_mesh["topologies"][bndry_topo_name];
+         std::string bndry_ele_shape = n_bndry_topo["elements/shape"].as_string();
 
-      }
+         bndry_geo = ShapeNameToGeomType(bndry_ele_shape);
+         int num_idxs_per_bndry_ele = Geometry::NumVerts[mesh_geo];
 
-      num_bndry_ele =
-         n_bndry_topo["elements/connectivity"].dtype().number_of_elements();
-      num_bndry_ele = num_bndry_ele / num_idxs_per_bndry_ele;
+         const Node &n_bndry_conn = n_bndry_topo["elements/connectivity"];
+
+         // mfem requires ints, we could have int64s, etc convert if necessary
+         if ( n_bndry_conn.dtype().is_int() &&
+              n_bndry_conn.is_compact())
+         {
+            bndry_indices = n_bndry_conn.value();
+         }
+         else
+         {
+            Node &(n_bndry_conn_conv) =
+               n_conv["topologies"][bndry_topo_name]["elements/connectivity"];
+            n_bndry_conn.to_int_array(n_bndry_conn_conv);
+            bndry_indices = (n_bndry_conn_conv).value();
+
+         }
+
+         num_bndry_ele =
+            n_bndry_topo["elements/connectivity"].dtype().number_of_elements();
+         num_bndry_ele = num_bndry_ele / num_idxs_per_bndry_ele;
+      }
    }
    else
    {

fem/datacollection.cpp

@@ -10,6 +10,7 @@
 // Software Foundation) version 2.1 dated February 1999.
 
 #include "fem.hpp"
+#include "../mesh/nurbs.hpp"
 #include "../general/text.hpp"
 #include "picojson.h"
 
@@ -331,6 +332,25 @@ DataCollection::~DataCollection()
 
 // class VisItDataCollection implementation
 
+void VisItDataCollection::UpdateMeshInfo()
+{
+   if (mesh)
+   {
+      spatial_dim = mesh->SpaceDimension();
+      topo_dim = mesh->Dimension();
+      if (mesh->NURBSext)
+      {
+         visit_levels_of_detail =
+            std::max(visit_levels_of_detail, mesh->NURBSext->GetOrder());
+      }
+   }
+   else
+   {
+      spatial_dim = 0;
+      topo_dim = 0;
+   }
+}
+
 VisItDataCollection::VisItDataCollection(const std::string& collection_name,
                                          Mesh *mesh)
    : DataCollection(collection_name, mesh)
@@ -338,17 +358,10 @@ VisItDataCollection::VisItDataCollection(const std::string& collection_name,
    appendRankToFileName = true; // always include rank in file names
    cycle = 0;                   // always include cycle in directory names
 
-   if (mesh)
-   {
-      spatial_dim = mesh->SpaceDimension();
-      topo_dim = mesh->Dimension();
-   }
-   else
-   {
-      spatial_dim = 0;
-      topo_dim = 0;
-   }
+   visit_levels_of_detail = 1;
    visit_max_levels_of_detail = 32;
+
+   UpdateMeshInfo();
 }
 
 #ifdef MFEM_USE_MPI
@@ -362,9 +375,11 @@ VisItDataCollection::VisItDataCollection(MPI_Comm comm,
    MPI_Comm_size(comm, &num_procs);
    appendRankToFileName = true; // always include rank in file names
    cycle = 0;                   // always include cycle in directory names
-   spatial_dim = 0;
-   topo_dim = 0;
+
+   visit_levels_of_detail = 1;
    visit_max_levels_of_detail = 32;
+
+   UpdateMeshInfo();
 }
 #endif
 
@@ -372,8 +387,7 @@ void VisItDataCollection::SetMesh(Mesh *new_mesh)
 {
    DataCollection::SetMesh(new_mesh);
    appendRankToFileName = true;
-   spatial_dim = mesh->SpaceDimension();
-   topo_dim = mesh->Dimension();
+   UpdateMeshInfo();
 }
 
 #ifdef MFEM_USE_MPI
@@ -392,6 +406,26 @@ void VisItDataCollection::RegisterField(const std::string& name,
 {
    DataCollection::RegisterField(name, gf);
    field_info_map[name] = VisItFieldInfo("nodes", gf->VectorDim());
+
+   int LOD = 1;
+   if (gf->FESpace()->GetNURBSext())
+   {
+      LOD = gf->FESpace()->GetNURBSext()->GetOrder();
+   }
+   else
+   {
+      for (int e=0; e<gf->FESpace()->GetNE() ; e++)
+      {
+         LOD = std::max(LOD,gf->FESpace()->GetFE(e)->GetOrder());
+      }
+   }
+
+   visit_levels_of_detail = std::max(visit_levels_of_detail, LOD);
+}
+
+void VisItDataCollection::SetLevelsOfDetail(int levels_of_detail)
+{
+   visit_levels_of_detail = levels_of_detail;
 }
 
 void VisItDataCollection::SetMaxLevelsOfDetail(int max_levels_of_detail)
@@ -595,6 +629,7 @@ std::string VisItDataCollection::GetVisItRootString()
    {
       ftags["assoc"] = picojson::value((it->second).association);
       ftags["comps"] = picojson::value(to_string((it->second).num_components));
+      ftags["lod"] = picojson::value(to_string(visit_levels_of_detail));
       field["path"] = picojson::value(path_str + it->first + file_ext_format);
       field["tags"] = picojson::value(ftags);
       fields[it->first] = picojson::value(field);

fem/datacollection.hpp

@@ -398,6 +398,7 @@ protected:
    // Additional data needed in the VisIt root file, which describes the mesh
    // and all the fields in the collection
    int spatial_dim, topo_dim;
+   int visit_levels_of_detail;
    int visit_max_levels_of_detail;
    std::map<std::string, VisItFieldInfo> field_info_map;
    typedef std::map<std::string, VisItFieldInfo>::iterator FieldInfoMapIterator;
@@ -407,6 +408,8 @@ protected:
    /// Read in a VisIt root file in JSON format
    void ParseVisItRootString(const std::string& json);
 
+   void UpdateMeshInfo();
+
    // Helper functions for Load()
    void LoadVisItRootFile(const std::string& root_name);
    void LoadMesh();
@@ -437,6 +440,9 @@ public:
    /// Add a grid function to the collection and update the root file
    virtual void RegisterField(const std::string& field_name, GridFunction *gf);
 
+   /// Set VisIt parameter: default levels of detail for the MultiresControl
+   void SetLevelsOfDetail(int levels_of_detail);
+
    /// Set VisIt parameter: maximum levels of detail for the MultiresControl
    void SetMaxLevelsOfDetail(int max_levels_of_detail);
 

fem/fe.cpp

@@ -6598,6 +6598,34 @@ void Poly_1D::CalcChebyshev(const int p, const double x, double *u, double *d)
    }
 }
 
+void Poly_1D::CalcChebyshev(const int p, const double x, double *u, double *d,
+                            double *dd)
+{
+   // recursive definition, z in [-1,1]
+   // T_0(z) = 1,  T_1(z) = z
+   // T_{n+1}(z) = 2*z*T_n(z) - T_{n-1}(z)
+   // T'_n(z) = n*U_{n-1}(z)
+   // U_0(z) = 1  U_1(z) = 2*z
+   // U_{n+1}(z) = 2*z*U_n(z) - U_{n-1}(z)
+   // U_n(z) = z*U_{n-1}(z) + T_n(z) = z*T'_n(z)/n + T_n(z)
+   // T'_{n+1}(z) = (n + 1)*(z*T'_n(z)/n + T_n(z))
+   // T''_{n+1}(z) = (n + 1)*(2*(n + 1)*T'_n(z) + z*T''_n(z)) / n
+   double z;
+   u[0] = 1.;
+   d[0] = 0.;
+   dd[0]= 0.;
+   if (p == 0) { return; }
+   u[1] = z = 2.*x - 1.;
+   d[1] = 2.;
+   dd[1] = 0;
+   for (int n = 1; n < p; n++)
+   {
+      u[n+1] = 2*z*u[n] - u[n-1];
+      d[n+1] = (n + 1)*(z*d[n]/n + 2*u[n]);
+      dd[n+1] = (n + 1)*(2.*(n + 1)*d[n] + z*dd[n])/n;
+   }
+}
+
 const double *Poly_1D::GetPoints(const int p, const int btype)
 {
    BasisType::Check(btype);
@@ -7463,8 +7491,12 @@ H1_TriangleElement::H1_TriangleElement(const int p, const int btype)
    dshape_x.SetSize(p + 1);
    dshape_y.SetSize(p + 1);
    dshape_l.SetSize(p + 1);
+   ddshape_x.SetSize(p + 1);
+   ddshape_y.SetSize(p + 1);
+   ddshape_l.SetSize(p + 1);
    u.SetSize(Dof);
    du.SetSize(Dof, Dim);
+   ddu.SetSize(Dof, (Dim * (Dim + 1)) / 2 );
 #else
    Vector shape_x(p + 1), shape_y(p + 1), shape_l(p + 1);
 #endif
@@ -7568,6 +7600,38 @@ void H1_TriangleElement::CalcDShape(const IntegrationPoint &ip,
    Ti.Mult(du, dshape);
 }
 
+void H1_TriangleElement::CalcHessian(const IntegrationPoint &ip,
+                                     DenseMatrix &ddshape) const
+{
+   const int p = Order;
+#ifdef MFEM_THREAD_SAFE
+   Vector   shape_x(p + 1),   shape_y(p + 1),   shape_l(p + 1);
+   Vector  dshape_x(p + 1),  dshape_y(p + 1),  dshape_l(p + 1);
+   Vector ddshape_x(p + 1), ddshape_y(p + 1), ddshape_l(p + 1);
+   DenseMatrix ddu(Dof, Dim);
+#endif
+
+   poly1d.CalcBasis(p, ip.x, shape_x, dshape_x, ddshape_x);
+   poly1d.CalcBasis(p, ip.y, shape_y, dshape_y, ddshape_y);
+   poly1d.CalcBasis(p, 1. - ip.x - ip.y, shape_l, dshape_l, ddshape_l);
+
+   for (int o = 0, j = 0; j <= p; j++)
+      for (int i = 0; i + j <= p; i++)
+      {
+         int k = p - i - j;
+         // u_xx, u_xy, u_yy
+         ddu(o,0) = ((ddshape_x(i) * shape_l(k)) - 2. * (dshape_x(i) * dshape_l(k)) +
+                     (shape_x(i) * ddshape_l(k))) * shape_y(j);
+         ddu(o,1) = (((shape_x(i) * ddshape_l(k)) - dshape_x(i) * dshape_l(k)) * shape_y(
+                        j)) + (((dshape_x(i) * shape_l(k)) - (shape_x(i) * dshape_l(k))) * dshape_y(j));
+         ddu(o,2) = ((ddshape_y(j) * shape_l(k)) - 2. * (dshape_y(j) * dshape_l(k)) +
+                     (shape_y(j) * ddshape_l(k))) * shape_x(i);
+         o++;
+      }
+
+   Ti.Mult(ddu, ddshape);
+}
+
 
 H1_TetrahedronElement::H1_TetrahedronElement(const int p, const int btype)
    : NodalFiniteElement(3, Geometry::TETRAHEDRON, ((p + 1)*(p + 2)*(p + 3))/6,
@@ -7584,8 +7648,13 @@ H1_TetrahedronElement::H1_TetrahedronElement(const int p, const int btype)
    dshape_y.SetSize(p + 1);
    dshape_z.SetSize(p + 1);
    dshape_l.SetSize(p + 1);
+   ddshape_x.SetSize(p + 1);
+   ddshape_y.SetSize(p + 1);
+   ddshape_z.SetSize(p + 1);
+   ddshape_l.SetSize(p + 1);
    u.SetSize(Dof);
    du.SetSize(Dof, Dim);
+   ddu.SetSize(Dof, (Dim * (Dim + 1)) / 2);
 #else
    Vector shape_x(p + 1), shape_y(p + 1), shape_z(p + 1), shape_l(p + 1);
 #endif
@@ -7738,6 +7807,51 @@ void H1_TetrahedronElement::CalcDShape(const IntegrationPoint &ip,
    Ti.Mult(du, dshape);
 }
 
+void H1_TetrahedronElement::CalcHessian(const IntegrationPoint &ip,
+                                        DenseMatrix &ddshape) const
+{
+   const int p = Order;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector   shape_x(p + 1),   shape_y(p + 1),   shape_z(p + 1),   shape_l(p + 1);
+   Vector  dshape_x(p + 1),  dshape_y(p + 1),  dshape_z(p + 1),  dshape_l(p + 1);
+   Vector ddshape_x(p + 1), ddshape_y(p + 1), ddshape_z(p + 1), ddshape_l(p + 1);
+   DenseMatrix ddu(Dof, ((Dim + 1) * Dim) / 2);
+#endif
+
+   poly1d.CalcBasis(p, ip.x, shape_x, dshape_x, ddshape_x);
+   poly1d.CalcBasis(p, ip.y, shape_y, dshape_y, ddshape_y);
+   poly1d.CalcBasis(p, ip.z, shape_z, dshape_z, ddshape_z);
+   poly1d.CalcBasis(p, 1. - ip.x - ip.y - ip.z, shape_l, dshape_l, ddshape_l);
+
+   for (int o = 0, k = 0; k <= p; k++)
+      for (int j = 0; j + k <= p; j++)
+         for (int i = 0; i + j + k <= p; i++)
+         {
+            // u_xx, u_xy, u_xz, u_yy, u_yz, u_zz
+            int l = p - i - j - k;
+            ddu(o,0) = ((ddshape_x(i) * shape_l(l)) - 2. * (dshape_x(i) * dshape_l(l)) +
+                        (shape_x(i) * ddshape_l(l))) * shape_y(j) * shape_z(k);
+            ddu(o,1) = ((dshape_y(j) * ((dshape_x(i) * shape_l(l)) -
+                                        (shape_x(i) * dshape_l(l)))) +
+                        (shape_y(j) * ((ddshape_l(l) * shape_x(i)) -
+                                       (dshape_x(i) * dshape_l(l)))))* shape_z(k);
+            ddu(o,2) = ((dshape_z(k) * ((dshape_x(i) * shape_l(l)) -
+                                        (shape_x(i) * dshape_l(l)))) +
+                        (shape_z(k) * ((ddshape_l(l) * shape_x(i)) -
+                                       (dshape_x(i) * dshape_l(l)))))* shape_y(j);
+            ddu(o,3) = ((ddshape_y(j) * shape_l(l)) - 2. * (dshape_y(j) * dshape_l(l)) +
+                        (shape_y(j) * ddshape_l(l))) * shape_x(i) * shape_z(k);
+            ddu(o,4) = ((dshape_z(k) * ((dshape_y(j) * shape_l(l)) -
+                                        (shape_y(j)*dshape_l(l))) ) +
+                        (shape_z(k)* ((ddshape_l(l)*shape_y(j)) -
+                                      (dshape_y(j) * dshape_l(l)) ) ) )* shape_x(i);
+            ddu(o,5) = ((ddshape_z(k) * shape_l(l)) - 2. * (dshape_z(k) * dshape_l(l)) +
+                        (shape_z(k) * ddshape_l(l))) * shape_y(j) * shape_x(i);
+            o++;
+         }
+   Ti.Mult(ddu, ddshape);
+}
 
 H1Pos_TriangleElement::H1Pos_TriangleElement(const int p)
    : PositiveFiniteElement(2, Geometry::TRIANGLE, ((p + 1)*(p + 2))/2, p,

fem/fe.hpp

@@ -1564,6 +1564,8 @@ private:
 
    static void CalcChebyshev(const int p, const double x, double *u);
    static void CalcChebyshev(const int p, const double x, double *u, double *d);
+   static void CalcChebyshev(const int p, const double x, double *u, double *d,
+                             double *dd);
 
    QuadratureFunctions1D quad_func;
 
@@ -1618,6 +1620,14 @@ public:
    // { CalcLegendre(p, x, u, d); }
    { CalcChebyshev(p, x, u, d); }
 
+   // Evaluate the values, derivatives and second derivatives of a hierarchical 1D basis at point x
+   static void CalcBasis(const int p, const double x, double *u, double *d,
+                         double *dd)
+   // { CalcMono(p, x, u, d); }
+   // { CalcBernstein(p, x, u, d); }
+   // { CalcLegendre(p, x, u, d); }
+   { CalcChebyshev(p, x, u, d, dd); }
+
    // Evaluate a representation of a Delta function at point x
    static double CalcDelta(const int p, const double x)
    { return pow(x, (double) p); }
@@ -1820,7 +1830,8 @@ class H1_TriangleElement : public NodalFiniteElement
 private:
 #ifndef MFEM_THREAD_SAFE
    mutable Vector shape_x, shape_y, shape_l, dshape_x, dshape_y, dshape_l, u;
-   mutable DenseMatrix du;
+   mutable Vector ddshape_x, ddshape_y, ddshape_l;
+   mutable DenseMatrix du, ddu;
 #endif
    DenseMatrixInverse Ti;
 
@@ -1829,6 +1840,8 @@ public:
    virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
    virtual void CalcDShape(const IntegrationPoint &ip,
                            DenseMatrix &dshape) const;
+   virtual void CalcHessian(const IntegrationPoint &ip,
+                            DenseMatrix &ddshape) const;
 };
 
 
@@ -1838,7 +1851,8 @@ private:
 #ifndef MFEM_THREAD_SAFE
    mutable Vector shape_x, shape_y, shape_z, shape_l;
    mutable Vector dshape_x, dshape_y, dshape_z, dshape_l, u;
-   mutable DenseMatrix du;
+   mutable Vector ddshape_x, ddshape_y, ddshape_z, ddshape_l;
+   mutable DenseMatrix du, ddu;
 #endif
    DenseMatrixInverse Ti;
 
@@ -1848,6 +1862,8 @@ public:
    virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
    virtual void CalcDShape(const IntegrationPoint &ip,
                            DenseMatrix &dshape) const;
+   virtual void CalcHessian(const IntegrationPoint &ip,
+                            DenseMatrix &ddshape) const;
 };