More attempts to make HYPRE sparse matrix work

.gitignore

@@ -112,15 +112,6 @@ examples/petsc/deformed.*
 examples/petsc/velocity.*
 examples/petsc/elastic_energy.*
 
-examples/pumi/ex1
-examples/pumi/ex[126]p
-
-examples/pumi/refined.mesh
-examples/pumi/sol.gf
-examples/pumi/mesh.*
-examples/pumi/sol.*
-examples/pumi/displaced.mesh
-
 miniapps/electromagnetics/volta
 miniapps/electromagnetics/tesla
 miniapps/electromagnetics/maxwell

CHANGELOG

@@ -8,23 +8,11 @@
                                http://mfem.org
 
 
-Version 3.4.1 (development)
+Version 3.3.3 (development)
 ===========================
-- Added support for reading linear and quadratic 2D quadrilateral and triangular
-  Cubit meshes.
-
-
-Version 3.4, released on May 29, 2018
-=====================================
-
-More general and efficient mesh adaptivity
-------------------------------------------
-- Added support for PUMI, the Parallel Unstructured Mesh Infrastructure from
-  https://scorec.rpi.edu/pumi. PUMI is an unstructured, distributed mesh data
-  management system that is capable of handling general non-manifold models and
-  effectively supports automated adaptive analysis. PUMI enables for the first
-  time support for parallel unstructured modifications of MFEM meshes.
 
+More efficient non-conforming adaptive mesh refinement
+------------------------------------------------------
 - Significantly reduced MPI communication in the construction of the parallel
   prolongation matrix in ParFiniteElementSpace, for much improved parallel
   scaling of non-conforming AMR on hundreds of thousands of MPI tasks. The
@@ -93,11 +81,6 @@ New and updated examples and miniapps
   NURBS meshes in the miniapps/nurbs directory. Currently the directory contains
   variable order NURBS versions of examples 1, 1p and 11p.
 
-- Added PUMI versions of examples ex1, ex1p, ex2 and ex6p in a new examples/pumi
-  directory. The new examples demonstrate the PUMI APIs for parallel and serial
-  mesh loading (ex1 and ex1p), applying BCs using classification (ex2), and
-  performing parallel mesh adaptation (ex6p).
-
 - Added two new miniapps related to DataCollection I/O in miniapps/tools:
   load-dc.cpp can be used to visualize fields saved via DataCollection classes;
   convert-dc.cpp demonstrates how to convert between MFEM's different concrete

CMakeLists.txt

@@ -45,7 +45,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 3.4.1)
+set(${PROJECT_NAME}_VERSION 3.3.3)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -139,7 +139,7 @@ if (MFEM_USE_MPI)
     set(PETSC_INCLUDE_DIRS ${PETSC_INCLUDES})
   endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU PETSC STRUMPACK PUMI)
+  set(PKGS_NEED_MPI SUPERLU PETSC STRUMPACK)
   foreach(PKG IN LISTS PKGS_NEED_MPI)
     if (MFEM_USE_${PKG})
       message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -246,22 +246,6 @@ if (MFEM_USE_SIDRE)
   find_package(Axom REQUIRED Sidre SLIC axom_utils)
 endif()
 
-# PUMI
-if (MFEM_USE_PUMI)
-  # If PUMI_DIR was specified, only link to that directory,
-  # i.e. don't link to another installation in /usr/lib by mistake
-  find_package(SCOREC 2.1.0 REQUIRED OPTIONAL_COMPONENTS gmi_sim
-    CONFIG PATHS ${PUMI_DIR} NO_DEFAULT_PATH)
-  if (SCOREC_FOUND)
-    # Define a header file with the MFEM_USE_SIMMETRIX preprocessor variable
-    set(MFEM_USE_SIMMETRIX ${SCOREC_gmi_sim_FOUND})
-    set(PUMI_FOUND ${SCOREC_FOUND})
-    get_target_property(PUMI_INCLUDE_DIRS
-      SCOREC::apf INTERFACE_INCLUDE_DIRECTORIES)
-    set(PUMI_LIBRARIES SCOREC::core)
-  endif()
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -286,8 +270,8 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
-    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT GECKO GNUTLS NETCDF MPFR PUMI
-    POSIXCLOCKS MFEMBacktrace ZLIB)
+    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT GECKO GNUTLS NETCDF MPFR POSIXCLOCKS
+    MFEMBacktrace ZLIB)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")

CONTRIBUTING.md

@@ -1,15 +1,3 @@
-<p align="center">
-<a href="http://mfem.org/"><img alt="mfem" src="http://mfem.org/img/logo-300.png"></a>
-</p>
-
-<p align="center">
-<a href="https://github.com/mfem/mfem/blob/master/COPYRIGHT"><img alt="License" src="https://img.shields.io/badge/License-LGPL--2.1-brightgreen.svg"></a>
-<a href="https://travis-ci.org/mfem/mfem"><img alt="Build Status" src="https://travis-ci.org/mfem/mfem.svg?branch=master"></a>
-<a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
-<a href="http://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
-</p>
-
-
 # How to Contribute
 
 The MFEM team welcomes contributions at all levels: bugfixes; code
@@ -28,7 +16,6 @@ See the [Quick Summary](#quick-summary) section for the main highlights of our
 GitHub workflow. For more details, consult the following sections and refer
 back to them before issuing pull requests:
 
-- [Code Overview](#code-overview)
 - [GitHub Workflow](#github-workflow)
   - [MFEM Organization](#mfem-organization)
   - [New Feature Development](#new-feature-development)
@@ -66,94 +53,6 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
 - Don't hesitate to [contact us](#contact-information) if you have any questions.
 
 
-### Code Overview
-
-- The MFEM library uses object-orient design principles which reflect, in code,
-  the independent mathematical concepts of meshing, linear algebra and finite
-  element spaces and operators.
-
-- The MFEM source code has the following structure:
-  ```
-  .
-  ├── config
-  │   └── cmake
-  │       └── modules
-  ├── data
-  ├── doc
-  │   └── web
-  │       └── examples
-  ├── examples
-  │   ├── petsc
-  │   ├── pumi
-  │   └── sundials
-  ├── fem
-  ├── general
-  ├── linalg
-  ├── mesh
-  └── miniapps
-      ├── common
-      ├── electromagnetics
-      ├── meshing
-      ├── nurbs
-      ├── performance
-      └── tools
-  ```
-
-- The main directories are `fem/`, `mesh/` and `linalg/` containing the C++
-  classes implementing the finite element, mesh and linear algebra concepts
-  respectively.
-
-- The main mesh classes are:
-  + [`Mesh`](http://mfem.github.io/doxygen/html/classmfem_1_1Mesh.html)
-  + [`NCMesh`](http://mfem.github.io/doxygen/html/classmfem_1_1NCMesh.html)
-  + [`Element`](http://mfem.github.io/doxygen/html/classmfem_1_1Element.html)
-  + [`ElementTransformation`](http://mfem.github.io/doxygen/html/classmfem_1_1ElementTransformation.html)
-
-- The main finite element classes are:
-  + [`FiniteElement`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
-  + [`FiniteElementCollection`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
-  + [`FiniteElementSpace`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElementSpace.html)
-  + [`GridFunction`](http://mfem.github.io/doxygen/html/classmfem_1_1GridFunction.html)
-  + [`BilinearFormIntegrator`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html)
-  + [`LinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1MixedBilinearForm.html)
-
-- The main linear algebra classes and sources are
-  + [`Operator`](http://mfem.github.io/doxygen/html/classmfem_1_1Operator.html) and [`BilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html)
-  + [`Vector`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearForm.html)
-  + [`DenseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1SparseMatrix.html)
-  + Sparse [smoothers](http://mfem.github.io/doxygen/html/sparsesmoothers_8hpp.html) and linear [solvers](http://mfem.github.io/doxygen/html/solvers_8hpp.html)
-
-- Parallel MPI objects in MFEM inherit their serial counterparts, so a parallel
-  mesh for example is just a serial mesh on each task plus the information on
-  shared geometric entities between different tasks. The parallel source files
-  have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
-
-- The main parallel classes are
-  + [`ParMesh`](http://mfem.github.io/doxygen/html/solvers_8hpp.html)
-  + [`ParNCMesh`](http://mfem.github.io/doxygen/html/classmfem_1_1ParMesh.html)
-  + [`ParFiniteElementSpace`](http://mfem.github.io/doxygen/html/classmfem_1_1ParFiniteElementSpace.html)
-  + [`ParGridFunction`](http://mfem.github.io/doxygen/html/classmfem_1_1ParGridFunction.html)
-  + [`ParBilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1ParLinearForm.html)
-  + [`HypreParMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParVector.html)
-  + [`HypreSolver`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreSolver.html) and other [hypre classes](http://mfem.github.io/doxygen/html/hypre_8hpp.html)
-
-- The `general/` directory contains C++ classes that serve as utilities for
-  communication, error handling, arrays, (Boolean) tables, timing, etc.
-
-- The `config/` directory contains build-related files, both for the plain
-  Makefile and the CMake build options.
-
-- The `doc/` directory contains configuration for the Doxygen code documentation
-  that can either be build locally, or browsed online at
-  http://mfem.github.io/doxygen/html/index.html.
-
-- The `data/` directory contains a collection of small mesh files, that are used
-  in the simple example codes and more fully-featured mini applications in the
-  `examples/` and `miniapps/` directories.
-
-- See also the [code overview](http://mfem.org/code-overview/) section on the
-  MFEM website.
-
 ## GitHub Workflow
 
 The GitHub organization, https://github.com/mfem, is the main developer hub for
@@ -223,7 +122,7 @@ will allow us to reach you directly with project announcements.
   # Work on "feature-dev", add local commits
   # ...
 
-  # (One time only) push the branch to github and setup your local
+  # One time only) push the branch to github and setup your local
   # branch to track the github branch (for "git pull"):
   git push -u origin feature-dev
 
@@ -433,7 +332,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
     - [ ] `CHANGELOG`
     - [ ] `makefile`
     - [ ] `CMakeLists.txt`
-    - [ ] `doc/CodeDocumentation.conf.in`
+    - [ ] `doc/CodeDocumentation.conf`
 - [ ] (LLNL only) Make sure all `README.html` files in the source repo are up to date.
 - [ ] Tag the repository:
 

INSTALL

@@ -32,9 +32,6 @@ following package managers:
 - OpenHPC, http://openhpc.community
 - Homebrew/Science, https://github.com/Homebrew/homebrew-science
 
-We also recommend downloading and building the MFEM-based GLVis visualization
-tool which can be used to visualize the meshes and solution in MFEM's examples
-and miniapps. See http://glvis.org and http://mfem.org/building.
 
 Quick start with GNU make
 =========================
@@ -355,13 +352,6 @@ MFEM_USE_GZSTREAM = YES/NO
    before attempting to use it with MFEM.
    When enabled, this option uses the ZLIB_* library options, see below.
 
-MFEM_USE_PUMI = YES/NO
-   Enable the usage of PUMI (https://scorec.rpi.edu/pumi/) in MFEM. The Parallel
-   Unstructured Mesh Infrastructure (PUMI) is an unstructured, distributed mesh
-   data management system that is capable of handling general non-manifold
-   models and effectively supports automated adaptive analysis. PUMI enables
-   support for parallel unstructured mesh modifications in MFEM.
-
 MFEM_BUILD_TAG = (any value)
    An optional tag to characterize the build. Exported to config/config.mk.
    Can be used to identify the MFEM build from other makefiles.
@@ -471,10 +461,6 @@ The specific libraries and their options are:
        https://support.hdfgroup.org/HDF5 (HDF5)
   Options: CONDUIT_OPT, CONDUIT_LIB.
 
-- PUMI, used when MFEM_USE_PUMI = YES.
-  URL: https://scorec.rpi.edu/pumi
-  Options: PUMI_OPT, PUMI_LIB.
-
 - MPFR (optional), used when MFEM_USE_MPFR = YES.
   URL: http://mpfr.org, it depends on the GMP library: https://gmplib.org
   Options: MPFR_OPT, MPFR_LIB.
@@ -607,7 +593,6 @@ MFEM_USE_GNUTLS
 MFEM_USE_NETCDF
 MFEM_USE_MPFR
 MFEM_USE_GZSTREAM
-MFEM_USE_PUMI
 
 The following options are CMake specific:
 
@@ -653,7 +638,6 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - MPFR
  - LIBUNWIND
  - POSIXCLOCKS
- - PUMI
 
 The following built-in CMake packages are also used:
 

README

@@ -12,15 +12,11 @@ to enable the research and development of scalable finite element discretization
 and solver algorithms through general finite element abstractions, accurate and
 flexible visualization, and tight integration with the hypre library.
 
-* For building instructions, see the file INSTALL, or type "make help".
+For building instructions, see the file INSTALL, or type "make help". Copyright
+information and licensing restrictions can be found in the file COPYRIGHT.
 
-* Copyright and licensing information can be found in the file COPYRIGHT.
-
-* The best starting point for new users interested in MFEM's features is the
-  interactive documentation in examples/README.html.
-
-* Developers interested in contributing to the library, should read the
-  instructions and documentation in the CONTRIBUTING.md file.
+The best starting point for new users interested in MFEM's features is the
+interactive documentation in examples/README.html.
 
 Conceptually, MFEM can be viewed as a finite element toolbox that provides the
 building blocks for developing finite element algorithms in a manner similar to
@@ -60,8 +56,8 @@ time integrators, etc.
 For examples of using MFEM, see the examples/ and miniapps/ directories, as well
 as the OpenGL visualization tool GLVis which is available at http://glvis.org.
 
-This project is released under the LGPL v2.1 license with static linking
-exception. See files COPYRIGHT and LICENSE file for full details.
+This project is released under the LGPL v2.1 license. See LICENSE file for full
+details.
 
 LLNL Release Number: LLNL-CODE-443211
 DOI: 10.11578/dc.20171025.1248

backends/all.hpp

@@ -22,6 +22,10 @@
 #include "occa/backend.hpp"
 #endif
 
+#ifdef MFEM_USE_OMP
+#include "omp/backend.hpp"
+#endif
+
 #endif // MFEM_USE_BACKENDS
 
 #endif // MFEM_BACKENDS_ALL_HPP

backends/base/array.hpp

@@ -34,6 +34,8 @@ protected:
     */
    ///@{
 
+   virtual void *DoGetData() const = 0;
+
    /** @brief Create and return a new array (in @a *clone) of the same dynamic
        type as this array using the same layout and ItemSize().
 
@@ -109,16 +111,13 @@ public:
    PLayout &GetLayout() const { return *layout; }
 
    /// TODO
-   /// Note: we cannot use static_cast for class PArray.
    template <typename derived_t>
    derived_t &As() { return dynamic_cast<derived_t&>(*this); }
 
    /// TODO
-   /// Note: we cannot use static_cast for class PArray.
    template <typename derived_t>
    const derived_t &As() const { return dynamic_cast<const derived_t&>(*this); }
 
-
    // TODO: Error handling ... handle errors at the Engine level, at the class
    //       level, or at the method level?
 
@@ -130,6 +129,9 @@ public:
     */
    ///@{
 
+   template <typename T=void>
+   T* GetData() const { return (T*) DoGetData(); }
+
    /** @brief Create and return a new array (in @a *clone) of the same dynamic
        type as this array using the same layout and ItemSize().
 

backends/base/engine.cpp

@@ -19,29 +19,9 @@
 namespace mfem
 {
 
-Engine::Engine(Backend *b, int n_mem, int n_workers)
-   : backend(b),
-#ifdef MFEM_USE_MPI
-     comm(MPI_COMM_NULL),
-#endif
-     num_mem_res(n_mem),
-     num_workers(n_workers),
-     memory_resources(new MemoryResource*[num_mem_res]()),
-     workers_weights(new double[num_workers]()),
-     workers_mem_res(new int[num_workers]())
+DFiniteElementSpace Engine::MakeFESpace(FiniteElementSpace &fes) const
 {
-   // Note: all arrays are value-initialized with zeros.
-}
-
-Engine::~Engine()
-{
-   delete [] workers_mem_res;
-   delete [] workers_weights;
-   for (int i = 0; i < num_mem_res; i++)
-   {
-      delete memory_resources[i];
-   }
-   delete [] memory_resources;
+   return DFiniteElementSpace(new PFiniteElementSpace(*this, fes));
 }
 
 } // namespace mfem

backends/base/engine.hpp

@@ -30,6 +30,7 @@ namespace mfem
 // Forward declarations.
 class Backend;
 template <typename T> class Array;
+class Vector;
 class Operator;
 class FiniteElementSpace;
 class LinearForm;
@@ -65,10 +66,29 @@ protected:
 
 public:
    /// TODO: doxygen
-   Engine(Backend *b, int n_mem, int n_workers);
+   Engine(Backend *b, int n_mem, int n_workers)
+      : backend(b),
+#ifdef MFEM_USE_MPI
+        comm(MPI_COMM_NULL),
+#endif
+        num_mem_res(n_mem),
+        num_workers(n_workers),
+        memory_resources(new MemoryResource*[num_mem_res]()),
+        workers_weights(new double[num_workers]()),
+        workers_mem_res(new int[num_workers]())
+   { /* Note: all arrays are value-initialized with zeros. */ }
 
    /// TODO: doxygen
-   virtual ~Engine();
+   virtual ~Engine()
+   {
+      delete [] workers_mem_res;
+      delete [] workers_weights;
+      for (int i = 0; i < num_mem_res; i++)
+      {
+         delete memory_resources[i];
+      }
+      delete [] memory_resources;
+   }
 
 
    /**
@@ -101,11 +121,11 @@ public:
 
    /// TODO
    template <typename derived_t>
-   derived_t &As() { return *util::As<derived_t>(this); }
+   derived_t &As() { *util::As<derived_t>(this); }
 
    /// TODO
    template <typename derived_t>
-   const derived_t &As() const { return *util::As<const derived_t>(this); }
+   const derived_t &As() const { *util::As<const derived_t>(this); }
 
 
    // TODO: Error handling ... handle errors at the Engine level, at the class
@@ -120,10 +140,10 @@ public:
    // TODO: Asynchronous execution in this class ...
 
    /// Allocate and return a new layout for the given @a size.
-   /** The layout decomposition (in the case of multiple workers) is determined
-       automatically by the Engine using a deterministic algorithm: calls to
-       this method with the same @a size will produce the same result, as long
-       as the Engine remains unmodified between the calls.
+   /** The layout decomposition is determined automatically by the Engine using
+       a deterministic algorithm: calls to this method with the same @a size
+       will produce the same result, as long as the Engine remains unmodified
+       between the calls.
 
        The returned object is allocated with operator new and must be
        deallocated by the caller.
@@ -161,7 +181,7 @@ public:
                               int type_id = ScalarId<double>::value) const = 0;
 
    /// TODO: doxygen
-   virtual DFiniteElementSpace MakeFESpace(FiniteElementSpace &fes) const = 0;
+   virtual DFiniteElementSpace MakeFESpace(FiniteElementSpace &fes) const;
 
    /// TODO: doxygen
    virtual DBilinearForm MakeBilinearForm(BilinearForm &bf) const = 0;

backends/base/fespace.hpp

@@ -22,7 +22,6 @@ namespace mfem
 {
 
 class FiniteElementSpace;
-class QuadratureSpace;
 
 /// TODO: doxygen
 class PFiniteElementSpace : public RefCounted
@@ -31,11 +30,11 @@ protected:
    /// Engine with shared ownership
    SharedPtr<const Engine> engine;
    /// Not owned.
-   mfem::FiniteElementSpace *fes;
+   FiniteElementSpace *fes;
 
 public:
    /// TODO: doxygen
-   PFiniteElementSpace(const Engine &e, mfem::FiniteElementSpace &fespace)
+   PFiniteElementSpace(const Engine &e, FiniteElementSpace &fespace)
       : engine(&e), fes(&fespace) { }
 
    /// Virtual destructor
@@ -44,8 +43,7 @@ public:
    /// Get the associated engine
    const Engine &GetEngine() const { return *engine; }
 
-   /// Return the associated mfem::FiniteElementSpace
-   mfem::FiniteElementSpace *GetFESpace() const { return fes; }
+   mfem::FiniteElementSpace* GetFESpace() const { return fes; }
 
    /// TODO
    template <typename derived_t>
@@ -54,41 +52,6 @@ public:
    /// TODO
    template <typename derived_t>
    const derived_t &As() const { return *util::As<const derived_t>(this); }
-
-
-   /**
-       @name Virtual interface: finite element space functionality
-    */
-   ///@{
-
-   /// TODO
-   /** Return the operator mapping T-vectors to L-vectors. If a NULL pointer is
-       returned then the mapping is the idenity. */
-   virtual const mfem::Operator *GetProlongationOperator() const = 0;
-
-   /// TODO
-   /** Return the operator mapping L-vectors to T-vectors that extracts the
-       subset of all true dofs, i.e. no assembly is performed. If a NULL pointer
-       is returned then the mapping is the idenity. */
-   virtual const mfem::Operator *GetRestrictionOperator() const = 0;
-
-   /// TODO
-   /** Return the operator mapping L-vectors to Q-vectors that evaluates the
-       values of a GridFunction as a QuadratureFunction on the given
-       QuadratureSpace. If the returned pointer is NULL, then the mapping is the
-       identity. */
-   virtual const mfem::Operator *GetInterpolationOperator(
-      const mfem::QuadratureSpace &qspace) const = 0;
-
-   /// TODO
-   /** Return the operator mapping L-vectors to Q-vectors that evaluates the
-       _reference element_ gradients of a GridFunction as a QuadratureFunction
-       on the given QuadratureSpace. */
-   virtual const mfem::Operator *GetGradientOperator(
-      const mfem::QuadratureSpace &qspace) const = 0;
-
-   ///@}
-   // End: Virtual interface
 };
 
 } // namespace mfem

backends/base/layout.hpp

@@ -69,16 +69,10 @@ public:
    std::size_t Size() const { return size; }
 
    /// TODO
-   /** Useful in backends for down-casting to a backend-specific layout type.
-
-       When MFEM_DEBUG=YES, performs a type check using dynamic_cast. */
    template <typename derived_t>
    derived_t &As() { return *util::As<derived_t>(this); }
 
    /// TODO
-   /** Useful in backends for down-casting to a backend-specific layout type.
-
-       When MFEM_DEBUG=YES, performs a type check using dynamic_cast. */
    template <typename derived_t>
    const derived_t &As() const { return *util::As<const derived_t>(this); }
 

backends/base/smart_pointers.hpp

@@ -160,12 +160,11 @@ public:
 
    template <typename U>
    bool operator==(const SharedPtr<U> &other) const
-   { return ptr == other.Get(); }
+   { return ptr == other.Ptr(); }
    template <typename U>
    bool operator!=(const SharedPtr<U> &other) const
-   { return ptr != other.Get(); }
+   { return ptr != other.Ptr(); }
 
-   // Comparison to any type convertible to void *, e.g. the type of NULL.
    template <typename U>
    bool operator==(const U &p) const { return ptr == (void*) p; }
    template <typename U>

backends/occa/MassIntegrator.okl

@@ -1,66 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-/*
-  ---[ Defines Known At Compile-Time ]------------
-  ELEMENT_BATCH   : How many elements are in each
-  .                   computation batch
-  NUM_DOFS_1D     : Dofs in the 1D segments
-  NUM_DOFS_2D     : Dofs in the 2D faces
-  NUM_DOFS_3D     : Dofs in the 3D domain
-  NUM_QUAD_1D     : Dofs in the 1D segments
-  NUM_QUAD_2D     : Dofs in the 2D faces
-  NUM_QUAD_3D     : Dofs in the 3D domain
-  NUM_MAX_1D      : max(NUM_QUAD_1D, NUM_DOFS_1D)
-  NUM_QUAD_DOFS_1D: NUM_QUAD_1D * NUM_DOFS_1D
-  COEFF_ARGS      : Code that passes required arguments to the kernel
-  COEFF           : Code that computes the coefficient
-  ================================================
-
-  [MISSING]
-  - Add support to auto-pick @dim and use @idxOrder on stack arrays
-  |   double a[2][2];
-  |   a[0][1]; <-- regular index
-  |   a(0,1);  <-- uses @idxOrder a[0][1] or a[1][0]
-  - Add support for @idxOrder to change indexing order after allocation
-  |   double a[2][2] @idxOrder(0,1);
-  |   a(0,1) -> a[1][0]
-  |   @set(a, idxOrder(1,0));
-  |   a(0,1) -> a[0][1]
-  - Add support to iterate over loop depending on mode
-  |   for(i; @inner) {
-  |     for(0 < j < N) {} <-- ++j or j += block?
-  |   }
-*/
-
-#include "mfem-occa://defines.okl"
-
-#if USING_TENSOR_OPS
-#  ifdef OCCA_USING_GPU
-#    if USING_LOW_ORDER
-#      include "mfem-occa://mass/tensor/gpuHighOrder.okl"
-#    else
-#      include "mfem-occa://mass/tensor/gpuHighOrder.okl"
-#    endif
-#  else
-#    include "mfem-occa://mass/tensor/cpu.okl"
-#  endif
-#else
-#  ifdef OCCA_USING_GPU
-#    if USING_LOW_ORDER
-#      include "mfem-occa://mass/simplex/gpuHighOrder.okl"
-#    else
-#      include "mfem-occa://mass/simplex/gpuHighOrder.okl"
-#    endif
-#  else
-#    include "mfem-occa://mass/simplex/cpu.okl"
-#  endif
-#endif

backends/occa/VectorMassIntegrator.okl

@@ -1,38 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-/*
-  ---[ Defines Known At Compile-Time ]------------
-  ELEMENT_BATCH   : How many elements are in each
-  .                   computation batch
-  NUM_DOFS_1D     : Dofs in the 1D segments
-  NUM_DOFS_2D     : Dofs in the 2D faces
-  NUM_DOFS_3D     : Dofs in the 3D domain
-  NUM_QUAD_1D     : Dofs in the 1D segments
-  NUM_QUAD_2D     : Dofs in the 2D faces
-  NUM_QUAD_3D     : Dofs in the 3D domain
-  NUM_MAX_1D      : max(NUM_QUAD_1D, NUM_DOFS_1D)
-  NUM_QUAD_DOFS_1D: NUM_QUAD_1D * NUM_DOFS_1D
-  CONST_COEFF     : If the coefficient is constant, pass it
-  .                   as a define
-  ================================================
-
-  [MISSING]
-  See kernels/DiffusionIntegrator.okl
-*/
-
-#include "mfem-occa://defines.okl"
-
-#if USING_TENSOR_OPS
-#  ifndef OCCA_USING_GPU
-#    include "mfem-occa://vmass/tensor/cpu.okl"
-#  endif
-#endif

backends/occa/array.cpp

@@ -41,7 +41,8 @@ int Array::DoResize(PLayout &new_layout, void **buffer,
    MFEM_ASSERT(dynamic_cast<Layout *>(&new_layout) != NULL,
                "new_layout is not an OCCA Layout");
    Layout *lt = static_cast<Layout *>(&new_layout);
-   int err = OccaResize(lt, item_size);
+   layout.Reset(lt); // Reset() checks if the pointer is the same
+   int err = ResizeData(lt, item_size);
    if (!err && buffer)
    {
       *buffer = GetBuffer();
@@ -71,22 +72,22 @@ void Array::DoFill(const void *value_ptr, std::size_t item_size)
    switch (item_size)
    {
       case sizeof(int8_t):
-         OccaFill(*(const int8_t *)value_ptr);
+         OccaFill((const int8_t *)value_ptr);
          break;
       case sizeof(int16_t):
-         OccaFill(*(const int16_t *)value_ptr);
+         OccaFill((const int16_t *)value_ptr);
          break;
       case sizeof(int32_t):
-         OccaFill(*(const int32_t *)value_ptr);
+         OccaFill((const int32_t *)value_ptr);
          break;
       // case sizeof(int64_t):
-      //    OccaFill(*(const int64_t *)value_ptr);
+      //    OccaFill((const int64_t *)value_ptr);
       //    break;
       case sizeof(double):
-         OccaFill(*(const double *)value_ptr);
+         OccaFill((const double *)value_ptr);
          break;
       // case sizeof(::occa::double2):
-      //    OccaFill(*(const ::occa::double2 *)value_ptr);
+      //    OccaFill((const ::occa::double2 *)value_ptr);
       //    break;
       default:
          MFEM_ABORT("item_size = " << item_size << " is not supported");
@@ -111,7 +112,8 @@ void Array::DoAssign(const PArray &src, std::size_t item_size)
    //       class.
    const Array *source = dynamic_cast<const Array *>(&src);
    MFEM_ASSERT(source != NULL, "invalid source Array type");
-   OccaAssign(*source);
+   MFEM_ASSERT(Size() == source->Size(), "");
+   slice.copyFrom(source->slice);
 }
 
 } // namespace mfem::occa

backends/occa/array.hpp

@@ -40,6 +40,8 @@ protected:
    // Virtual interface
    //
 
+   virtual void *DoGetData() const { return GetBuffer(); }
+
    virtual PArray *DoClone(bool copy_data, void **buffer,
                            std::size_t item_size) const;
 
@@ -60,16 +62,16 @@ protected:
 
    inline void *GetBuffer() const;
 
-public:
-   Array(const Engine &e)
-      : PArray(*(new Layout(e, 0))),
-        data(e.Alloc(0)),
-        slice(data)
-   { }
+   inline int ResizeData(const Layout *lt, std::size_t item_size);
 
+   template <typename T>
+   inline void OccaFill(const T *val_ptr)
+   { ::occa::linalg::operator_eq<T>(slice, *val_ptr); }
+
+public:
    Array(Layout &lt, std::size_t item_size)
       : PArray(lt),
-        data(lt.OccaEngine().Alloc(lt.Size()*item_size)),
+        data(lt.Alloc(lt.Size()*item_size)),
         slice(data)
    { }
 
@@ -77,23 +79,11 @@ public:
 
    inline void MakeRef(Array &master);
 
-   Layout &OccaLayout() const { return layout->As<Layout>(); }
-
-   const Engine &OccaEngine() const { return OccaLayout().OccaEngine(); }
+   Layout &OccaLayout() const
+   { return *static_cast<Layout *>(layout.Get()); }
 
    ::occa::memory &OccaMem() { return slice; }
    const ::occa::memory &OccaMem() const { return slice; }
-
-   inline int OccaResize(Layout *lt, std::size_t item_size);
-
-   inline int OccaResize(std::size_t new_size, std::size_t item_size);
-
-   template <typename T>
-   inline void OccaFill(const T val);
-
-   inline void OccaAssign(const Array &src);
-
-   inline void OccaPush(const void *src);
 };
 
 
@@ -110,14 +100,13 @@ inline void *Array::GetBuffer() const
    return NULL;
 }
 
-inline int Array::OccaResize(Layout *lt, std::size_t item_size)
+inline int Array::ResizeData(const Layout *lt, std::size_t item_size)
 {
-   layout.Reset(lt); // Reset() checks if the pointer is the same
    const std::size_t new_bytes = lt->Size()*item_size;
    if (data.size() < new_bytes ||
-       data.getDevice() != lt->OccaEngine().GetDevice())
+       data.getDHandle() != lt->OccaEngine().GetDevice().getDHandle())
    {
-      data = lt->OccaEngine().Alloc(new_bytes);
+      data = lt->Alloc(new_bytes);
       slice = data;
       // If memory allocation fails - an exception is thrown.
    }
@@ -135,37 +124,6 @@ inline void Array::MakeRef(Array &master)
    slice = master.slice;
 }
 
-inline int Array::OccaResize(std::size_t new_size, std::size_t item_size)
-{
-   Layout &ol = OccaLayout();
-   ol.OccaResize(new_size);
-   return OccaResize(&ol, item_size);
-}
-
-template <typename T>
-inline void Array::OccaFill(const T val)
-{
-   ::occa::linalg::operator_eq<T>(slice, val);
-}
-
-inline void Array::OccaAssign(const Array &src)
-{
-   if (slice != src.slice && slice.size() != 0)
-   {
-      MFEM_ASSERT(slice.size() == src.slice.size(), "");
-      slice.copyFrom(src.slice);
-   }
-}
-
-inline void Array::OccaPush(const void *src)
-{
-   if (slice.size() != 0)
-   {
-      slice.copyFrom(src);
-   }
-}
-
-
 } // namespace mfem::occa
 
 } // namespace mfem

backends/occa/bilinearform.cpp

@@ -24,8 +24,8 @@ namespace occa
 
 OccaBilinearForm::OccaBilinearForm(FiniteElementSpace *ofespace_) :
    Operator(ofespace_->OccaVLayout()),
-   localX(ofespace_->OccaEVLayout()),
-   localY(ofespace_->OccaEVLayout())
+   localX((ofespace_->OccaEVLayout().DontDelete(), ofespace_->OccaEVLayout())),
+   localY((ofespace_->OccaEVLayout().DontDelete(), ofespace_->OccaEVLayout()))
 {
    Init(ofespace_->OccaEngine(), ofespace_, ofespace_);
 }
@@ -34,8 +34,8 @@ OccaBilinearForm::OccaBilinearForm(FiniteElementSpace *otrialFESpace_,
                                    FiniteElementSpace *otestFESpace_) :
    Operator(otrialFESpace_->OccaVLayout(),
             otestFESpace_->OccaVLayout()),
-   localX(otrialFESpace_->OccaEVLayout()),
-   localY(otestFESpace_->OccaEVLayout())
+   localX((otrialFESpace_->OccaEVLayout().DontDelete(), otrialFESpace_->OccaEVLayout())),
+   localY((otestFESpace_->OccaEVLayout().DontDelete(), otestFESpace_->OccaEVLayout()))
 {
    Init(otrialFESpace_->OccaEngine(), otrialFESpace_, otestFESpace_);
 }
@@ -65,9 +65,11 @@ void OccaBilinearForm::Init(const Engine &e,
    if (GetDevice().mode() == "OpenMP")
    {
       const std::string &okl_path = OccaEngine().GetOklPath();
+      const std::string &okl_defines = OccaEngine().GetOklDefines();
       ::occa::kernel initLocalKernel =
          GetDevice().buildKernel(okl_path + "utils.okl",
-                                 "InitLocalVector");
+                                 "InitLocalVector",
+                                 okl_defines);
 
       const std::size_t sd = sizeof(double);
       const uint64_t trialEntries = sd * (elements * trialLocalDofs);
@@ -271,6 +273,8 @@ void OccaBilinearForm::InitRHS(const mfem::Array<int> &constraintList,
                                mfem::Vector &X, mfem::Vector &B,
                                int copy_interior)
 {
+   const std::string okl_defines = OccaEngine().GetOklDefines();
+
    // FIXME: move these kernels to the Backend?
    static ::occa::kernelBuilder get_subvector_builder =
       ::occa::linalg::customLinearMethod(
@@ -284,7 +288,7 @@ void OccaBilinearForm::InitRHS(const mfem::Array<int> &constraintList,
          "  VTYPE1: 'double',"
          "  VTYPE2: 'int',"
          "  TILESIZE: 128,"
-         "}");
+         "}" + okl_defines);
 
    static ::occa::kernelBuilder set_subvector_builder =
       ::occa::linalg::customLinearMethod(
@@ -298,7 +302,7 @@ void OccaBilinearForm::InitRHS(const mfem::Array<int> &constraintList,
          "  VTYPE1: 'double',"
          "  VTYPE2: 'int',"
          "  TILESIZE: 128,"
-         "}");
+         "}" + okl_defines);
 
    const mfem::Operator *P = GetTrialProlongation();
    const mfem::Operator *R = GetTrialRestriction();
@@ -341,7 +345,6 @@ void OccaBilinearForm::InitRHS(const mfem::Array<int> &constraintList,
                            constrList.OccaMem());
    }
 
-   // FIXME: add case for HypreParMatrix here
    OccaConstrainedOperator *cA = dynamic_cast<OccaConstrainedOperator*>(A);
    if (cA)
    {
@@ -358,7 +361,7 @@ void OccaBilinearForm::InitRHS(const mfem::Array<int> &constraintList,
 void OccaBilinearForm::Mult_(const Vector &x, Vector &y) const
 {
    otrialFESpace->GlobalToLocal(x, localX);
-   localY.OccaFill<double>(0.0);
+   localY.Fill<double>(0.0);
 
    const int integratorCount = (int) integrators.size();
    for (int i = 0; i < integratorCount; ++i)
@@ -373,7 +376,7 @@ void OccaBilinearForm::Mult_(const Vector &x, Vector &y) const
 void OccaBilinearForm::MultTranspose_(const Vector &x, Vector &y) const
 {
    otestFESpace->GlobalToLocal(x, localX);
-   localY.OccaFill<double>(0.0);
+   localY.Fill<double>(0.0);
 
    const int integratorCount = (int) integrators.size();
    for (int i = 0; i < integratorCount; ++i)
@@ -429,42 +432,19 @@ void BilinearForm::InitOccaBilinearForm()
       Coefficient *scal_coeff = dbfi[i]->GetScalarCoefficient();
       ConstantCoefficient *const_coeff =
          dynamic_cast<ConstantCoefficient*>(scal_coeff);
-      GridFunctionCoefficient *gridfunc_coeff =
-         dynamic_cast<GridFunctionCoefficient*>(scal_coeff);
       // TODO: other types of coefficients ...
-
-      OccaCoefficient *ocoeff = NULL;
-      if (const_coeff)
-      {
-         ocoeff = new OccaCoefficient(obform->OccaEngine(),
-                                      const_coeff->constant);
-      }
-      else if (gridfunc_coeff)
-      {
-         ocoeff = new OccaCoefficient(obform->OccaEngine(),
-                                      *gridfunc_coeff->GetGridFunction(), true);
-      }
-      else if (!scal_coeff)
-      {
-         ocoeff = new OccaCoefficient(obform->OccaEngine(), 1.0);
-      }
-      else
-      {
-         MFEM_ABORT("Coefficient type not supported");
-      }
+      double val = const_coeff ? const_coeff->constant : 1.0;
+      OccaCoefficient ocoeff(obform->OccaEngine(), val);
 
       OccaIntegrator *ointeg = NULL;
+
       if (integ_name == "(undefined)")
       {
          MFEM_ABORT("BilinearFormIntegrator does not define Name()");
       }
-      else if (integ_name == "mass")
-      {
-         ointeg = new OccaMassIntegrator(*ocoeff);
-      }
       else if (integ_name == "diffusion")
       {
-         ointeg = new OccaDiffusionIntegrator(*ocoeff);
+         ointeg = new OccaDiffusionIntegrator(ocoeff);
       }
       else
       {
@@ -472,10 +452,6 @@ void BilinearForm::InitOccaBilinearForm()
                     << "] is not supported");
       }
 
-      // NOTE: The integrators copy ocoeff, so it can be deleted here so there
-      //       is no memory leak.
-      delete ocoeff;
-
       const mfem::IntegrationRule *ir = dbfi[i]->GetIntRule();
       if (ir) { ointeg->SetIntegrationRule(*ir); }
 

backends/occa/bilininteg.cpp

@@ -107,10 +107,11 @@ OccaGeometry OccaGeometry::Get(::occa::device device,
    props["defines/STORE_JACOBIAN_DET"] = (flags & JacobianDet);
 
    const std::string &okl_path = ofespace.OccaEngine().GetOklPath();
+   const std::string &okl_defines = ofespace.OccaEngine().GetOklDefines();
    ::occa::kernel init = device.buildKernel(okl_path + "geometry.okl",
                                             stringWithDim("InitGeometryInfo",
                                                           fe.GetDim()),
-                                            props);
+                                            props + okl_defines);
    init(elements,
         maps.dofToQuadD,
         geom.meshNodes,
@@ -729,9 +730,10 @@ OccaGeometry OccaIntegrator::GetGeometry(const int flags)
 {
    const std::string filename = GetName() + ".okl";
    const std::string &okl_path = OccaEngine().GetOklPath();
+   const std::string &okl_defines = OccaEngine().GetOklDefines();
    return GetDevice().buildKernel(okl_path + filename,
                                   kernelName,
-                                  props);
+                                  props + okl_defines);
 }
 //====================================
 
@@ -784,8 +786,8 @@ void OccaDiffusionIntegrator::Assemble()
 
    OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
 
-   assembledOperator.OccaResize(symmDims * quadraturePoints * elements,
-                                sizeof(double));
+   assembledOperator.Resize<double>(symmDims * quadraturePoints * elements,
+                                    NULL);
 
    assembleKernel((int) mesh->GetNE(),
                   maps.quadWeights,

backends/occa/coefficient.cpp

@@ -119,18 +119,17 @@ void OccaVectorParameter::Setup(OccaIntegrator &integ,
 
 //---[ GridFunction Parameter ]-------
 OccaGridFunctionParameter::OccaGridFunctionParameter(const std::string &name_,
-                                                     const Engine &e,
-                                                     mfem::GridFunction &gf_,
+                                                     OccaGridFunction &gf_,
                                                      const bool useRestrict_)
    : name(name_),
      gf(gf_),
-     gfQuad(e),
+     gfQuad(*(new Layout(gf_.OccaLayout().OccaEngine(), 0))),
      useRestrict(useRestrict_) {}
 
 OccaParameter* OccaGridFunctionParameter::Clone()
 {
    OccaGridFunctionParameter *param =
-      new OccaGridFunctionParameter(name, gfQuad.OccaEngine(), gf, useRestrict);
+      new OccaGridFunctionParameter(name, gf, useRestrict);
    param->gfQuad.MakeRef(gfQuad);
    return param;
 }
@@ -142,16 +141,15 @@ void OccaGridFunctionParameter::Setup(OccaIntegrator &integ,
    std::string &args = (props["defines/COEFF_ARGS"]
                         .asString()
                         .string());
+   args += "const double *";
    if (useRestrict)
    {
-      args += "@restrict ";
+      args += " restrict ";
    }
-   args += "const double *";
    args += name;
    args += " @dim(NUM_QUAD, numElements),\n";
 
-   FiniteElementSpace &f = gf.FESpace()->Get_PFESpace()->As<FiniteElementSpace>();
-   ToQuad(integ.GetIntegrationRule(), f, gf.Get_PVector()->As<Vector>(), gfQuad);
+   gf.ToQuad(integ.GetIntegrationRule(), gfQuad);
 }
 
 ::occa::kernelArg OccaGridFunctionParameter::KernelArgs()
@@ -170,16 +168,6 @@ OccaCoefficient::OccaCoefficient(const Engine &e, const double value) :
    coeffValue = value;
 }
 
-OccaCoefficient::OccaCoefficient(const Engine &e, mfem::GridFunction &gf,
-                                 const bool useRestrict) :
-   engine(&e),
-   integ(NULL),
-   name("COEFF")
-{
-   coeffValue = "(u(q, e))";
-   AddGridFunction("u", gf, useRestrict);
-}
-
 OccaCoefficient::OccaCoefficient(const Engine &e, const std::string &source) :
    engine(&e),
    integ(NULL),
@@ -273,13 +261,10 @@ OccaCoefficient& OccaCoefficient::AddVector(const std::string &name_,
 }
 
 OccaCoefficient& OccaCoefficient::AddGridFunction(const std::string &name_,
-                                                  mfem::GridFunction &gf,
+                                                  OccaGridFunction &gf,
                                                   const bool useRestrict)
 {
-   MFEM_ASSERT(engine->CheckVector(gf.Get_PVector()) &&
-               engine->CheckFESpace(gf.FESpace()->Get_PFESpace()),
-               "invalid device GridFunction");
-   return Add(new OccaGridFunctionParameter(name_, *engine, gf, useRestrict));
+   return Add(new OccaGridFunctionParameter(name_, gf, useRestrict));
 }
 
 bool OccaCoefficient::IsConstant()
@@ -317,9 +302,10 @@ Vector OccaCoefficient::Eval()
 void OccaCoefficient::Eval(Vector &quadCoeff)
 {
    const std::string &okl_path = OccaEngine().GetOklPath();
+   const std::string &okl_defines = OccaEngine().GetOklDefines();
    static ::occa::kernelBuilder builder =
       ::occa::kernelBuilder::fromFile(okl_path + "coefficient.okl",
-                                      "CoefficientEval");
+                                      "CoefficientEval", okl_defines);
 
    if (integ == NULL)
    {
@@ -334,6 +320,7 @@ void OccaCoefficient::Eval(Vector &quadCoeff)
       kernelProps["defines/COEFF"]      = name;
       kernelProps["defines/COEFF_ARGS"] = name + "_ARGS";
    }
+   kernelProps += okl_defines;
 
    ::occa::kernel evalKernel = builder.build(GetDevice(), kernelProps);
    evalKernel(elements, *this, quadCoeff.OccaMem());

backends/occa/coefficient.hpp

@@ -178,14 +178,13 @@ class OccaGridFunctionParameter : public OccaParameter
 {
 private:
    const std::string name;
-   mfem::GridFunction &gf;
+   OccaGridFunction &gf;
    Vector gfQuad;
    bool useRestrict;
 
 public:
    OccaGridFunctionParameter(const std::string &name_,
-                             const Engine &e,
-                             mfem::GridFunction &gf_,
+                             OccaGridFunction &gf_,
                              const bool useRestrict_ = false);
 
    virtual OccaParameter* Clone();
@@ -220,8 +219,6 @@ private:
 
 public:
    OccaCoefficient(const Engine &e, const double value = 1.0);
-   OccaCoefficient(const Engine &e, mfem::GridFunction &gf,
-                   const bool useRestrict = false);
    OccaCoefficient(const Engine &e, const std::string &source);
    OccaCoefficient(const Engine &e, const char *source);
    ~OccaCoefficient();
@@ -266,7 +263,7 @@ public:
                               const bool useRestrict = false);
 
    OccaCoefficient& AddGridFunction(const std::string &name_,
-                                    mfem::GridFunction &gf,
+                                    OccaGridFunction &gf,
                                     const bool useRestrict = false);
 
    bool IsConstant();

backends/occa/diffusion/simplex/cpu.okl

@@ -13,10 +13,10 @@
 
 //---[ 2D ]-----------------------------
 @kernel void Assemble2D(const int numElements,
-                        @restrict const double *quadWeights,
-                        @restrict const Jacobian2D_t J,
+                        const double * restrict quadWeights,
+                        const Jacobian2D_t restrict J,
                         COEFF_ARGS
-                        @restrict SymmOperator2D_t oper) {
+                        SymmOperator2D_t restrict oper) {
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int q = 0; q < NUM_QUAD; ++q; @inner) {
       const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
@@ -32,13 +32,13 @@
 }
 
 @kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuadD2D_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDofD2D_t quadToDofD,
-                       @restrict const SymmOperator2D_t oper,
-                       @restrict const DLocal_t solIn,
-                       @restrict DLocal_t solOut) {
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuadD2D_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDofD2D_t restrict quadToDofD,
+                       const SymmOperator2D_t restrict oper,
+                       const DLocal_t restrict solIn,
+                       DLocal_t restrict solOut) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {
@@ -78,10 +78,10 @@
 
 //---[ 3D ]-----------------------------
 @kernel void Assemble3D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian3D_t J,
+                        const double * restrict quadWeights,
+                        const Jacobian3D_t restrict J,
                         COEFF_ARGS
-                        @restrict SymmOperator3D_t oper) {
+                        SymmOperator3D_t restrict oper) {
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int q = 0; q < NUM_QUAD; ++q; @inner) {
       const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
@@ -118,13 +118,13 @@
 }
 
 @kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuadD3D_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDofD3D_t quadToDofD,
-                       @restrict const SymmOperator3D_t oper,
-                       @restrict const DLocal_t solIn,
-                       @restrict DLocal_t solOut) {
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuadD3D_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDofD3D_t restrict quadToDofD,
+                       const SymmOperator3D_t restrict oper,
+                       const DLocal_t restrict solIn,
+                       DLocal_t restrict solOut) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {

backends/occa/diffusion/simplex/gpuHighOrder.okl

@@ -38,13 +38,13 @@
 }
 
 @kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuadD2D_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDofD2D_t quadToDofD,
-                       @restrict const SymmOperator2D_t oper,
-                       @restrict const DLocal_t solIn,
-                       @restrict DLocal_t solOut) {
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuadD2D_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDofD2D_t restrict quadToDofD,
+                       const SymmOperator2D_t restrict oper,
+                       const DLocal_t restrict solIn,
+                       DLocal_t restrict solOut) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     @shared double s_gradX[NUM_QUAD];
@@ -131,13 +131,13 @@
 }
 
 @kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuadD3D_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDofD3D_t quadToDofD,
-                       @restrict const SymmOperator3D_t oper,
-                       @restrict const DLocal_t solIn,
-                       @restrict DLocal_t solOut) {
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuadD3D_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDofD3D_t restrict quadToDofD,
+                       const SymmOperator3D_t restrict oper,
+                       const DLocal_t restrict solIn,
+                       DLocal_t restrict solOut) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     @shared double s_gradX[NUM_QUAD];

backends/occa/diffusion/tensor/cpu.okl

@@ -13,10 +13,10 @@
 
 //---[ 1D ]-----------------------------
 @kernel void Assemble1D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian1D_t J,
+                        const double * restrict quadWeights,
+                        const Jacobian1D_t restrict J,
                         COEFF_ARGS
-                        @restrict SymmOperator1D_t oper) {
+                        SymmOperator1D_t restrict oper) {
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
       oper(q, e) = quadWeights[q] * COEFF / J(q, e);
@@ -25,13 +25,13 @@
 }
 
 @kernel void MultAdd1D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const SymmOperator1D_t oper,
-                       @restrict const DLocal1D_t solIn,
-                       @restrict DLocal1D_t solOut) {
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator1D_t restrict oper,
+                       const DLocal1D_t restrict solIn,
+                       DLocal1D_t restrict solOut) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {
@@ -62,10 +62,10 @@
 
 //---[ 2D ]-----------------------------
 @kernel void Assemble2D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian2D_t J,
+                        const double * restrict quadWeights,
+                        const Jacobian2D_t restrict J,
                         COEFF_ARGS
-                        @restrict SymmOperator2D_t oper) {
+                        SymmOperator2D_t restrict oper) {
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int q = 0; q < NUM_QUAD_2D; ++q; @inner) {
       const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
@@ -81,13 +81,13 @@
 }
 
 @kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const SymmOperator2D_t oper,
-                       @restrict const DLocal2D_t solIn,
-                       @restrict DLocal2D_t solOut) {
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator2D_t restrict oper,
+                       const DLocal2D_t restrict solIn,
+                       DLocal2D_t restrict solOut) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {
@@ -175,10 +175,10 @@
 
 //---[ 3D ]-----------------------------
 @kernel void Assemble3D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian3D_t J,
+                        const double * restrict quadWeights,
+                        const Jacobian3D_t restrict J,
                         COEFF_ARGS
-                        @restrict SymmOperator3D_t oper) {
+                        SymmOperator3D_t restrict oper) {
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int q = 0; q < NUM_QUAD_3D; ++q; @inner) {
       const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
@@ -215,13 +215,13 @@
 }
 
 @kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const SymmOperator3D_t oper,
-                       @restrict const DLocal3D_t solIn,
-                       @restrict DLocal3D_t solOut) {
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator3D_t restrict oper,
+                       const DLocal3D_t restrict solIn,
+                       DLocal3D_t restrict solOut) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {

backends/occa/diffusion/tensor/gpuHighOrder.okl

@@ -29,13 +29,13 @@
 }
 
 @kernel void MultAdd1D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const SymmOperator1D_t oper,
-                       @restrict const DLocal1D_t solIn,
-                       @restrict DLocal1D_t solOut) {
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator1D_t restrict oper,
+                       const DLocal1D_t restrict solIn,
+                       DLocal1D_t restrict solOut) {
   // Iterate over elements
   for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
     @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
@@ -108,13 +108,13 @@
 }
 
 @kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const SymmOperator2D_t oper,
-                       @restrict const DLocal2D_t solIn,
-                       @restrict DLocal2D_t solOut) {
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator2D_t restrict oper,
+                       const DLocal2D_t restrict solIn,
+                       DLocal2D_t restrict solOut) {
   // Iterate over elements
   for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
     // Store dof <--> quad mappings
@@ -272,13 +272,13 @@
 }
 
 @kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const SymmOperator3D_t oper,
-                       @restrict const DLocal3D_t solIn,
-                       @restrict DLocal3D_t solOut) {
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator3D_t restrict oper,
+                       const DLocal3D_t restrict solIn,
+                       DLocal3D_t restrict solOut) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     // Store dof <--> quad mappings
@@ -391,7 +391,6 @@
           }
         }
       }
-      @barrier("s_z_s_Dz_sync_1");
     }
     // Iterate over xy planes to compute solution
     for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
@@ -428,7 +427,6 @@
           }
         }
       }
-      @barrier("s_z_s_Dz_s_xyDz_sync_1");
     }
   }
 }

backends/occa/engine.cpp

@@ -42,6 +42,7 @@ void Engine::Init(const std::string &engine_spec)
    device[0].setup(props);
 
    okl_path = "mfem-occa://";
+   // okl_defines = "...";
    if (!fileOpenerRegistered)
    {
       // The directories from "MFEM_OCCA_OKL_PATH", if any, have the highest
@@ -56,9 +57,6 @@ void Engine::Init(const std::string &engine_spec)
       ::occa::io::fileOpener::add(fo);
       fileOpenerRegistered = true;
    }
-   // std::cout << "OCCA device properties:\n" << device[0].properties();
-
-   force_cuda_aware_mpi = false;
 }
 
 Engine::Engine(const std::string &engine_spec)
@@ -76,36 +74,6 @@ Engine::Engine(MPI_Comm _comm, const std::string &engine_spec)
 }
 #endif
 
-bool Engine::CheckEngine(const mfem::Engine *engine) const
-{
-   return (engine != NULL && util::Is<const Engine>(engine) != NULL &&
-           *util::As<const Engine>(engine) == *this);
-}
-
-bool Engine::CheckLayout(const PLayout *layout) const
-{
-   return (layout != NULL && util::Is<const Layout>(layout) != NULL &&
-           layout->As<Layout>().OccaEngine() == *this);
-}
-
-bool Engine::CheckArray(const PArray *array) const
-{
-   return (array != NULL && util::Is<const Array>(array) != NULL &&
-           array->As<Array>().OccaEngine() == *this);
-}
-
-bool Engine::CheckVector(const PVector *vector) const
-{
-   return (vector != NULL && util::Is<const Vector>(vector) != NULL &&
-           vector->As<Vector>().OccaEngine() == *this);
-}
-
-bool Engine::CheckFESpace(const PFiniteElementSpace *fes) const
-{
-   return (fes != NULL && util::Is<const FiniteElementSpace>(fes) != NULL &&
-           fes->As<FiniteElementSpace>().OccaEngine() == *this);
-}
-
 DLayout Engine::MakeLayout(std::size_t size) const
 {
    return DLayout(new Layout(*this, size));
@@ -120,14 +88,19 @@ DLayout Engine::MakeLayout(const mfem::Array<std::size_t> &offsets) const
 
 DArray Engine::MakeArray(PLayout &layout, std::size_t item_size) const
 {
-   return DArray(new Array(layout.As<Layout>(), item_size));
+   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
+               "invalid input layout");
+   Layout *lt = static_cast<Layout *>(&layout);
+   return DArray(new Array(*lt, item_size));
 }
 
 DVector Engine::MakeVector(PLayout &layout, int type_id) const
 {
-   MFEM_ASSERT(type_id == ScalarId<double>::value, "type_id " << type_id
-               << " is not supported");
-   return DVector(new Vector(layout.As<Layout>()));
+   MFEM_ASSERT(type_id == ScalarId<double>::value, "invalid type_id");
+   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
+               "invalid input layout");
+   Layout *lt = static_cast<Layout *>(&layout);
+   return DVector(new Vector(*lt));
 }
 
 DFiniteElementSpace Engine::MakeFESpace(mfem::FiniteElementSpace &fespace) const

backends/occa/engine.hpp

@@ -41,23 +41,18 @@ protected:
    // int *workers_mem_res;
 
    static bool fileOpenerRegistered;
-   /// An array of OCCA devices. Currently only a single device is supported.
-   ::occa::device *device;
-   std::string okl_path;
-   bool force_cuda_aware_mpi;
+   ::occa::device *device; // An array of OCCA devices
+   std::string okl_path, okl_defines;
 
    void Init(const std::string &engine_spec);
 
 public:
-   /// TODO: doxygen
    Engine(const std::string &engine_spec);
 
 #ifdef MFEM_USE_MPI
-   /// TODO: doxygen
    Engine(MPI_Comm comm, const std::string &engine_spec);
 #endif
 
-   /// TODO: doxygen
    virtual ~Engine() { delete [] device; }
 
    /**
@@ -65,41 +60,13 @@ public:
     */
    ///@{
 
-   /// Get the associated OCCA device.
    ::occa::device GetDevice(int idx = 0) const { return device[idx]; }
 
    /// TODO: doxygen
    const std::string &GetOklPath() const { return okl_path; }
 
-   /// OCCA device memory allocation.
-   ::occa::memory Alloc(std::size_t bytes) const
-   { return GetDevice().malloc(bytes); }
-
-   /// Two mfem::occa::Engine%s are equal if they use the same OCCA device.
-   bool operator==(const Engine &other) const
-   { return GetDevice() == other.GetDevice(); }
-
    /// TODO: doxygen
-   bool CheckEngine(const mfem::Engine *e) const;
-
-   /// TODO: doxygen
-   bool CheckLayout(const PLayout *layout) const;
-
-   /// TODO: doxygen
-   bool CheckArray(const PArray *array) const;
-
-   /// TODO: doxygen
-   bool CheckVector(const PVector *vector) const;
-
-   /// TODO: doxygen
-   bool CheckFESpace(const PFiniteElementSpace *fes) const;
-
-#ifdef MFEM_USE_MPI
-   void SetForceCudaAwareMPI(bool force = true)
-   { force_cuda_aware_mpi = force; }
-
-   bool GetForceCudaAwareMPI() const { return force_cuda_aware_mpi; }
-#endif
+   const std::string &GetOklDefines() const { return okl_defines; }
 
    ///@}
    // End: OCCA specific interface

backends/occa/fespace.cpp

@@ -16,12 +16,6 @@
 #include "fespace.hpp"
 #include "interpolation.hpp"
 
-#ifdef MFEM_USE_MPI
-#ifdef OMPI_RELEASE_VERSION
-#include <mpi-ext.h> // Check for cuda support
-#endif
-#endif
-
 namespace mfem
 {
 
@@ -31,42 +25,41 @@ namespace occa
 FiniteElementSpace::FiniteElementSpace(const Engine &e,
                                        mfem::FiniteElementSpace &fespace)
    : PFiniteElementSpace(e, fespace),
-     e_layout(new Layout(e, 0)) // resized in SetupLocalGlobalMaps()
+     e_layout(e, 0) // resized in SetupLocalGlobalMaps()
 {
    vdim     = fespace.GetVDim();
    ordering = fespace.GetOrdering();
 
    SetupLocalGlobalMaps();
-   SetupOperators(); // calls virtual methods of 'fes'
+   SetupOperators();
    SetupKernels();
 }
 
 FiniteElementSpace::~FiniteElementSpace()
 {
+   delete [] elementDofMap;
+   delete [] elementDofMapInverse;
    delete restrictionOp;
    delete prolongationOp;
 }
 
 void FiniteElementSpace::SetupLocalGlobalMaps()
 {
-   const int elements = fes->GetNE();
-
-   if (elements == 0) { return; }
-
-   // Assuming of finite elements are the same.
-   const mfem::FiniteElement &fe = *fes->GetFE(0);
+   const mfem::FiniteElement &fe = *(fes->GetFE(0));
    const mfem::TensorBasisElement *el =
       dynamic_cast<const mfem::TensorBasisElement*>(&fe);
 
    const mfem::Table &e2dTable = fes->GetElementToDofTable();
    const int *elementMap = e2dTable.GetJ();
+   const int elements = fes->GetNE();
 
    globalDofs = fes->GetNDofs();
    localDofs  = fe.GetDof();
 
-   e_layout->OccaResize(e2dTable.Size_of_connections());
+   e_layout.Resize(localDofs * elements * fes->GetVDim());
 
-   int *elementDofMap = new int[localDofs];
+   elementDofMap        = new int[localDofs];
+   elementDofMapInverse = new int[localDofs];
    if (el)
    {
       ::memcpy(elementDofMap,
@@ -80,6 +73,10 @@ void FiniteElementSpace::SetupLocalGlobalMaps()
          elementDofMap[i] = i;
       }
    }
+   for (int i = 0; i < localDofs; ++i)
+   {
+      elementDofMapInverse[elementDofMap[i]] = i;
+   }
 
    // Allocate device offsets and indices
    globalToLocalOffsets.allocate(GetDevice(),
@@ -102,7 +99,6 @@ void FiniteElementSpace::SetupLocalGlobalMaps()
 
    for (int e = 0; e < elements; ++e)
    {
-      MFEM_ASSERT(e2dTable.RowSize(e) == localDofs, "");
       for (int d = 0; d < localDofs; ++d)
       {
          const int gid = elementMap[localDofs*e + d];
@@ -134,64 +130,19 @@ void FiniteElementSpace::SetupLocalGlobalMaps()
    }
    offsets[0] = 0;
 
-   delete [] elementDofMap;
-
    globalToLocalOffsets.keepInDevice();
    globalToLocalIndices.keepInDevice();
    localToGlobalMap.keepInDevice();
 }
 
-void FiniteElementSpace::SetupOperators() const
+void FiniteElementSpace::SetupOperators()
 {
-   // Construct 'restrictionOp' and 'prolongationOp'.
-
-   prolongationOp = restrictionOp = NULL;
-
    const mfem::SparseMatrix *R = fes->GetRestrictionMatrix();
    const mfem::Operator *P = fes->GetProlongationMatrix();
-
-   if (!P) { return; }
-
-   Layout &v_layout = OccaVLayout();
-   Layout &t_layout = OccaTrueVLayout();
-
-   // Assuming R has one entry per row equal to 1.
-   MFEM_ASSERT(R->Finalized(), "");
-   const int tdofs = R->Height();
-   MFEM_ASSERT(tdofs == (int)t_layout.Size(), "");
-   MFEM_ASSERT(tdofs == R->GetI()[tdofs], "");
-   ::occa::array<int> ltdof_ldof(GetDevice(), tdofs, R->GetJ());
-   ltdof_ldof.keepInDevice();
-
-   restrictionOp = new RestrictionOperator(v_layout, t_layout, ltdof_ldof);
-
-   const mfem::SparseMatrix *pmat = dynamic_cast<const mfem::SparseMatrix*>(P);
-   if (pmat)
-   {
-      const mfem::SparseMatrix *pmatT = Transpose(*pmat);
-
-      OccaSparseMatrix *occaP  =
-         CreateMappedSparseMatrix(t_layout, v_layout, *pmat);
-      OccaSparseMatrix *occaPT =
-         CreateMappedSparseMatrix(v_layout, t_layout, *pmatT);
-
-      prolongationOp = new ProlongationOperator(*occaP, *occaPT);
-
-      delete occaPT;
-      delete occaP;
-   }
-#ifdef MFEM_USE_MPI
-   else if (fes->Conforming() && dynamic_cast<ParFiniteElementSpace*>(fes))
-   {
-      ParFiniteElementSpace *pfes = static_cast<ParFiniteElementSpace*>(fes);
-      prolongationOp = new OccaConformingProlongation(*this, *pfes,
-                                                      ltdof_ldof.memory());
-   }
-#endif
-   else
-   {
-      prolongationOp = new ProlongationOperator(t_layout, v_layout, P);
-   }
+   CreateRPOperators(OccaVLayout(), OccaTrueVLayout(),
+                     R, P,
+                     restrictionOp,
+                     prolongationOp);
 }
 
 void FiniteElementSpace::SetupKernels()
@@ -207,324 +158,15 @@ void FiniteElementSpace::SetupKernels()
 
    ::occa::device device = GetDevice();
    const std::string &okl_path = OccaEngine().GetOklPath();
+   const std::string &okl_defines = OccaEngine().GetOklDefines();
    globalToLocalKernel = device.buildKernel(okl_path + "fespace.okl",
                                             "GlobalToLocal",
-                                            props);
+                                            props + okl_defines);
    localToGlobalKernel = device.buildKernel(okl_path + "fespace.okl",
                                             "LocalToGlobal",
-                                            props);
+                                            props + okl_defines);
 }
 
-
-#ifdef MFEM_USE_MPI
-
-OccaConformingProlongation::OccaConformingProlongation(
-   const FiniteElementSpace &ofes, const mfem::ParFiniteElementSpace &pfes,
-   ::occa::memory ltdof_ldof_)
-
-   : Operator(ofes.OccaTrueVLayout(), ofes.OccaVLayout()),
-     shr_ltdof(ofes.OccaEngine()),
-     ext_ldof(ofes.OccaEngine()),
-     shr_buf(shr_ltdof.OccaLayout(), sizeof(double)),
-     ext_buf(ext_ldof.OccaLayout(), sizeof(double)),
-     shr_buf_offsets(NULL), ext_buf_offsets(NULL),
-     ltdof_ldof(ltdof_ldof_),
-     gc(pfes.GroupComm())
-{
-   MFEM_ASSERT(pfes.Conforming(), "internal error");
-
-   const Engine &engine = ofes.OccaEngine();
-   const std::string &okl_path = engine.GetOklPath();
-   ::occa::device device = engine.GetDevice();
-
-   {
-      Table nbr_ltdof;
-      gc.GetNeighborLTDofTable(nbr_ltdof);
-      shr_ltdof.OccaResize(nbr_ltdof.Size_of_connections(), sizeof(int));
-      shr_ltdof.OccaPush(nbr_ltdof.GetJ());
-      shr_buf.OccaResize(&shr_ltdof.OccaLayout(), sizeof(double));
-      shr_buf_offsets = nbr_ltdof.GetI();
-      {
-         mfem::Array<int> shr_ltdof(nbr_ltdof.GetJ(),
-                                    nbr_ltdof.Size_of_connections());
-         mfem::Array<int> unique_ltdof(shr_ltdof);
-         unique_ltdof.Sort();
-         unique_ltdof.Unique();
-         // Note: the next loop modifies the J array of nbr_ltdof
-         for (int i = 0; i < shr_ltdof.Size(); i++)
-         {
-            shr_ltdof[i] = unique_ltdof.FindSorted(shr_ltdof[i]);
-            MFEM_ASSERT(shr_ltdof[i] != -1, "internal error");
-         }
-         Table unique_shr;
-         Transpose(shr_ltdof, unique_shr, unique_ltdof.Size());
-
-         unq_ltdof = device.malloc(unique_ltdof.Size()*sizeof(int),
-                                   unique_ltdof.GetData());
-         unq_shr_i = device.malloc((unique_shr.Size()+1)*sizeof(int),
-                                   unique_shr.GetI());
-         unq_shr_j = device.malloc(unique_shr.Size_of_connections()*sizeof(int),
-                                   unique_shr.GetJ());
-      }
-      delete [] nbr_ltdof.GetJ();
-      nbr_ltdof.LoseData();
-   }
-   {
-      Table nbr_ldof;
-      gc.GetNeighborLDofTable(nbr_ldof);
-      ext_ldof.OccaResize(nbr_ldof.Size_of_connections(), sizeof(int));
-      ext_ldof.OccaPush(nbr_ldof.GetJ());
-      ext_buf.OccaResize(&ext_ldof.OccaLayout(), sizeof(double));
-      ext_buf_offsets = nbr_ldof.GetI();
-      delete [] nbr_ldof.GetJ();
-      nbr_ldof.LoseData();
-   }
-   host_shr_buf = NULL;
-   host_ext_buf = NULL;
-   // If the device has a separate memory space (e.g. CUDA device) and the MPI
-   // library does not support buffers in that separate memory space, we
-   // allocate separate host buffers to use for MPI communication.
-   if (device.hasSeparateMemorySpace())
-   {
-      bool need_host_buf = true;
-      if (device.mode() == "CUDA")
-      {
-#ifdef MPIX_CUDA_AWARE_SUPPORT
-         need_host_buf = !MPIX_Query_cuda_support();
-#endif
-         if (engine.GetForceCudaAwareMPI()) { need_host_buf = false; }
-         if (gc.GetGroupTopology().MyRank() == 0)
-         {
-            mfem::out << "\nOccaConformingProlongation: CUDA-aware MPI: "
-                      << (need_host_buf ? "NO" : "YES") << "\n\n";
-         }
-      }
-      if (need_host_buf)
-      {
-         host_shr_buf = new char[shr_buf.OccaMem().size()];
-         host_ext_buf = new char[ext_buf.OccaMem().size()];
-      }
-   }
-
-   ExtractSubVector = device.buildKernel(okl_path + "mappings.okl",
-                                         "ExtractSubVector",
-                                         "defines: { TILESIZE: 256 }");
-   SetSubVector = device.buildKernel(okl_path + "mappings.okl",
-                                     "SetSubVector",
-                                     "defines: { TILESIZE: 256 }");
-   AddSubVector = device.buildKernel(okl_path + "mappings.okl",
-                                     "AddSubVector",
-                                     "defines: { TILESIZE: 256 }");
-
-   const GroupTopology &gtopo = gc.GetGroupTopology();
-   int req_counter = 0;
-   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
-   {
-      const int send_offset = shr_buf_offsets[nbr];
-      const int send_size = shr_buf_offsets[nbr+1] - send_offset;
-      if (send_size > 0) { req_counter++; }
-
-      const int recv_offset = ext_buf_offsets[nbr];
-      const int recv_size = ext_buf_offsets[nbr+1] - recv_offset;
-      if (recv_size > 0) { req_counter++; }
-   }
-   requests = new MPI_Request[req_counter];
-}
-
-OccaConformingProlongation::~OccaConformingProlongation()
-{
-   delete [] requests;
-   delete [] host_ext_buf;
-   delete [] host_shr_buf;
-   delete [] ext_buf_offsets;
-   delete [] shr_buf_offsets;
-}
-
-void OccaConformingProlongation::BcastBeginCopy(const ::occa::memory &src,
-                                                std::size_t item_size) const
-{
-   // shr_buf[i] = src[shr_ltdof[i]]
-   MFEM_ASSERT(item_size == sizeof(double), "");
-   if (shr_ltdof.Size() == 0) { return; }
-   ExtractSubVector((int)shr_ltdof.Size(), shr_ltdof.OccaMem(), src,
-                    shr_buf.OccaMem());
-   // If the above kernel is executed asynchronously, wait for it to complete:
-   shr_buf.OccaMem().getDevice().finish();
-   if (host_shr_buf)
-   {
-      shr_buf.OccaMem().copyTo(host_shr_buf);
-   }
-}
-
-void OccaConformingProlongation::BcastLocalCopy(const ::occa::memory &src,
-                                                ::occa::memory &dst,
-                                                std::size_t item_size) const
-{
-   // dst[ltdof_ldof[i]] = src[i]
-   MFEM_ASSERT(item_size == sizeof(double), "");
-   if (ltdof_ldof.size<int>() == 0) { return; }
-   SetSubVector((int)ltdof_ldof.size<int>(), ltdof_ldof, src, dst);
-}
-
-void OccaConformingProlongation::BcastEndCopy(::occa::memory &dst,
-                                              std::size_t item_size) const
-{
-   // dst[ext_ldof[i]] = ext_buf[i]
-   MFEM_ASSERT(item_size == sizeof(double), "");
-   if (ext_ldof.Size() == 0) { return; }
-   if (host_ext_buf)
-   {
-      ext_buf.OccaMem().copyFrom(host_ext_buf);
-   }
-   SetSubVector((int)ext_ldof.Size(), ext_ldof.OccaMem(),
-                ext_buf.OccaMem(), dst);
-}
-
-void OccaConformingProlongation::ReduceBeginCopy(const ::occa::memory &src,
-                                                 std::size_t item_size) const
-{
-   // ext_buf[i] = src[ext_ldof[i]]
-   MFEM_ASSERT(item_size == sizeof(double), "");
-   if (ext_ldof.Size() == 0) { return; }
-   ExtractSubVector((int)ext_ldof.Size(), ext_ldof.OccaMem(), src,
-                    ext_buf.OccaMem());
-   // If the above kernel is executed asynchronously, wait for it to complete:
-   ext_buf.OccaMem().getDevice().finish();
-   if (host_ext_buf)
-   {
-      ext_buf.OccaMem().copyTo(host_ext_buf);
-   }
-}
-
-void OccaConformingProlongation::ReduceLocalCopy(const ::occa::memory &src,
-                                                 ::occa::memory &dst,
-                                                 std::size_t item_size) const
-{
-   // dst[i] = src[ltdof_ldof[i]]
-   MFEM_ASSERT(item_size == sizeof(double), "");
-   if (ltdof_ldof.size<int>() == 0) { return; }
-   ExtractSubVector((int)ltdof_ldof.size<int>(), ltdof_ldof, src, dst);
-}
-
-void OccaConformingProlongation::ReduceEndAssemble(::occa::memory &dst,
-                                                   std::size_t item_size) const
-{
-   // dst[shr_ltdof[i]] += shr_buf[i]
-   MFEM_ASSERT(item_size == sizeof(double), "");
-   if (unq_ltdof.size<int>() == 0) { return; }
-   if (host_shr_buf)
-   {
-      shr_buf.OccaMem().copyFrom(host_shr_buf);
-   }
-   AddSubVector((int)unq_ltdof.size<int>(), unq_ltdof, unq_shr_i, unq_shr_j,
-                shr_buf.OccaMem(), dst);
-}
-
-void OccaConformingProlongation::Mult_(const Vector &x, Vector &y) const
-{
-   const GroupTopology &gtopo = gc.GetGroupTopology();
-
-   BcastBeginCopy(x.OccaMem(), sizeof(double)); // copy to 'shr_buf'
-
-   int req_counter = 0;
-   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
-   {
-      const int send_offset = shr_buf_offsets[nbr];
-      const int send_size = shr_buf_offsets[nbr+1] - send_offset;
-      if (send_size > 0)
-      {
-         void *send_buf;
-         if (host_shr_buf)
-         {
-            send_buf = host_shr_buf + send_offset*sizeof(double);
-         }
-         else
-         {
-            send_buf = (shr_buf.OccaMem() + send_offset*sizeof(double)).ptr();
-         }
-         MPI_Isend(send_buf, send_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
-                   41822, gtopo.GetComm(), &requests[req_counter++]);
-      }
-
-      const int recv_offset = ext_buf_offsets[nbr];
-      const int recv_size = ext_buf_offsets[nbr+1] - recv_offset;
-      if (recv_size > 0)
-      {
-         void *recv_buf;
-         if (host_ext_buf)
-         {
-            recv_buf = host_ext_buf + recv_offset*sizeof(double);
-         }
-         else
-         {
-            recv_buf = (ext_buf.OccaMem() + recv_offset*sizeof(double)).ptr();
-         }
-         MPI_Irecv(recv_buf, recv_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
-                   41822, gtopo.GetComm(), &requests[req_counter++]);
-      }
-   }
-
-   BcastLocalCopy(x.OccaMem(), y.OccaMem(), sizeof(double));
-
-   MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
-
-   BcastEndCopy(y.OccaMem(), sizeof(double)); // copy from 'ext_buf'
-}
-
-void OccaConformingProlongation::MultTranspose_(const Vector &x,
-                                                Vector &y) const
-{
-   const GroupTopology &gtopo = gc.GetGroupTopology();
-
-   ReduceBeginCopy(x.OccaMem(), sizeof(double)); // copy to 'ext_buf'
-
-   int req_counter = 0;
-   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
-   {
-      const int send_offset = ext_buf_offsets[nbr];
-      const int send_size = ext_buf_offsets[nbr+1] - send_offset;
-      if (send_size > 0)
-      {
-         void *send_buf;
-         if (host_ext_buf)
-         {
-            send_buf = host_ext_buf + send_offset*sizeof(double);
-         }
-         else
-         {
-            send_buf = (ext_buf.OccaMem() + send_offset*sizeof(double)).ptr();
-         }
-         MPI_Isend(send_buf, send_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
-                   41823, gtopo.GetComm(), &requests[req_counter++]);
-      }
-
-      const int recv_offset = shr_buf_offsets[nbr];
-      const int recv_size = shr_buf_offsets[nbr+1] - recv_offset;
-      if (recv_size > 0)
-      {
-         void *recv_buf;
-         if (host_shr_buf)
-         {
-            recv_buf = host_shr_buf + recv_offset*sizeof(double);
-         }
-         else
-         {
-            recv_buf = (shr_buf.OccaMem() + recv_offset*sizeof(double)).ptr();
-         }
-         MPI_Irecv(recv_buf, recv_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
-                   41823, gtopo.GetComm(), &requests[req_counter++]);
-      }
-   }
-
-   ReduceLocalCopy(x.OccaMem(), y.OccaMem(), sizeof(double));
-
-   MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
-
-   ReduceEndAssemble(y.OccaMem(), sizeof(double)); // assemble from 'shr_buf'
-}
-
-#endif // MFEM_USE_MPI
-
 } // namespace mfem::occa
 
 } // namespace mfem

backends/occa/fespace.hpp

@@ -35,7 +35,10 @@ protected:
    // SharedPtr<const mfem::Engine> engine;
    // mfem::FiniteElementSpace *fes;
 
-   SharedPtr<Layout> e_layout;
+   Layout e_layout;
+
+   int *elementDofMap;
+   int *elementDofMapInverse;
 
    ::occa::array<int> globalToLocalOffsets;
    ::occa::array<int> globalToLocalIndices;
@@ -47,10 +50,10 @@ protected:
    int globalDofs, localDofs;
    int vdim;
 
-   mutable Operator *prolongationOp, *restrictionOp;
+   mfem::Operator *restrictionOp, *prolongationOp;
 
    void SetupLocalGlobalMaps();
-   void SetupOperators() const; // calls virtual methods of 'fes' !!!
+   void SetupOperators();
    void SetupKernels();
 
 public:
@@ -61,7 +64,8 @@ public:
    virtual ~FiniteElementSpace();
 
    /// TODO: doxygen
-   const Engine &OccaEngine() const { return engine->As<Engine>(); }
+   const Engine &OccaEngine() const
+   { return *static_cast<const Engine *>(engine.Get()); }
 
    /// TODO: doxygen
    ::occa::device GetDevice(int idx = 0) const
@@ -75,7 +79,13 @@ public:
    Layout &OccaTrueVLayout() const
    { return *fes->GetTrueVLayout().As<Layout>(); }
 
-   Layout &OccaEVLayout() { return *e_layout; }
+   Layout &OccaEVLayout() { return e_layout; }
+
+#ifdef MFEM_USE_MPI
+   bool isDistributed() const { return (OccaEngine().GetComm() != MPI_COMM_NULL); }
+#else
+   bool isDistributed() const { return false; }
+#endif
 
    bool hasTensorBasis() const
    { return dynamic_cast<const mfem::TensorBasisElement*>(fes->GetFE(0)); }
@@ -95,29 +105,20 @@ public:
 
    int GetNE() const { return fes->GetNE(); }
 
-   const mfem::FiniteElementCollection *FEColl() const
+   const mfem::FiniteElementCollection* FEColl() const
    { return fes->FEColl(); }
-   const mfem::FiniteElement *GetFE(const int idx) const
+   const mfem::FiniteElement* GetFE(const int idx) const
    { return fes->GetFE(idx); }
 
-   virtual const mfem::Operator *GetProlongationOperator() const
-   { return prolongationOp; }
+   const int* GetElementDofMap() const { return elementDofMap; }
+   const int* GetElementDofMapInverse() const { return elementDofMapInverse; }
 
-   virtual const mfem::Operator *GetRestrictionOperator() const
-   { return restrictionOp; }
-
-   virtual const mfem::Operator *GetInterpolationOperator(
-      const mfem::QuadratureSpace &qspace) const
-   { return NULL; /* FIXME */ }
-
-   virtual const mfem::Operator *GetGradientOperator(
-      const mfem::QuadratureSpace &qspace) const
-   { return NULL; /* FIXME */ }
+   const mfem::Operator* GetRestrictionOperator() { return restrictionOp; }
+   const mfem::Operator* GetProlongationOperator() { return prolongationOp; }
 
    const ::occa::array<int> GetLocalToGlobalMap() const
    { return localToGlobalMap; }
 
-   /// L-vector to E-vector
    void GlobalToLocal(const Vector &globalVec, Vector &localVec) const
    {
       globalToLocalKernel(globalDofs,
@@ -126,8 +127,6 @@ public:
                           globalToLocalIndices,
                           globalVec.OccaMem(), localVec.OccaMem());
    }
-
-   /// E-vector to L-vector, transpose of GlobalToLocal
    void LocalToGlobal(const Vector &localVec, Vector &globalVec) const
    {
       localToGlobalKernel(globalDofs,
@@ -138,69 +137,6 @@ public:
    }
 };
 
-
-#ifdef MFEM_USE_MPI
-
-/// OCCA version of mfem::ConformingProlongationOperator
-class OccaConformingProlongation : public Operator
-{
-protected:
-   // size(shr_buf)=size(shr_ltdof)
-   // size(ext_buf)=size(ext_ldof)
-   Array shr_ltdof, ext_ldof;
-   mutable Array shr_buf, ext_buf;
-   mutable char *host_shr_buf, *host_ext_buf;
-   // Offsets into {shr,ext}_buf; size is num. neighbors, i.e.
-   // gc.GetGroupTopology().GetNumNeighbors():
-   int *shr_buf_offsets, *ext_buf_offsets;
-
-   ::occa::memory ltdof_ldof; // shared with the restriction operator
-
-   ::occa::memory unq_ltdof; // enumeration of the unique ltdofs in shr_ltdof
-   ::occa::memory unq_shr_i, unq_shr_j;
-
-   ::occa::kernel ExtractSubVector, SetSubVector, AddSubVector;
-
-   MPI_Request *requests;
-
-   const GroupCommunicator &gc;
-
-   // Kernel: copy ltdofs from 'src' to 'shr_buf' - prepare for send.
-   //         shr_buf[i] = src[shr_ltdof[i]]
-   void BcastBeginCopy(const ::occa::memory &src, std::size_t item_size) const;
-   // Kernel: copy ltdofs from 'src' to ldofs in 'dst'.
-   //         dst[ltdof_ldof[i]] = src[i]
-   void BcastLocalCopy(const ::occa::memory &src, ::occa::memory &dst,
-                       std::size_t item_size) const;
-   // Kernel: copy ext. dofs from 'ext_buf' to 'dst' - after recv.
-   //         dst[ext_ldof[i]] = ext_buf[i]
-   void BcastEndCopy(::occa::memory &dst, std::size_t item_size) const;
-
-   // Kernel: copy ext. dofs from 'src' to 'ext_buf' - prepare for send.
-   //         ext_buf[i] = src[ext_ldof[i]]
-   void ReduceBeginCopy(const ::occa::memory &src, std::size_t item_size) const;
-   // Kernel: copy owned ldofs from 'src' to ltdofs in 'dst'.
-   //         dst[i] = src[ltdof_ldof[i]]
-   void ReduceLocalCopy(const ::occa::memory &src, ::occa::memory &dst,
-                       std::size_t item_size) const;
-   // Kernel: assemble dofs from 'shr_buf' into to 'dst' - after recv.
-   //         dst[shr_ltdof[i]] += shr_buf[i]
-   void ReduceEndAssemble(::occa::memory &dst, std::size_t item_size) const;
-
-public:
-   OccaConformingProlongation(const FiniteElementSpace &ofes,
-                              const mfem::ParFiniteElementSpace &pfes,
-                              ::occa::memory ltdof_ldof_);
-
-   virtual ~OccaConformingProlongation();
-
-   // overrides
-   virtual void Mult_(const Vector &x, Vector &y) const;
-   virtual void MultTranspose_(const Vector &x, Vector &y) const;
-};
-
-#endif // MFEM_USE_MPI
-
 } // namespace mfem::occa
 
 } // namespace mfem

backends/occa/fespace.okl

@@ -25,10 +25,10 @@ typedef double *Local_t  @dim(NUM_VDIM, localEntries)  @dimOrder(1, 0);
 
 @kernel void GlobalToLocal(const int globalEntries,
                            const int localEntries,
-                           @restrict const int * offsets,
-                           @restrict const int * indices,
-                           @restrict const Global_t globalX,
-                           @restrict Local_t localX) {
+                           const int * restrict offsets,
+                           const int * restrict indices,
+                           const Global_t restrict globalX,
+                           Local_t restrict localX) {
 
   for (int i = 0; i < globalEntries; ++i; @tile(TILESIZE, @outer, @inner)) {
     if (i < globalEntries) {
@@ -46,10 +46,10 @@ typedef double *Local_t  @dim(NUM_VDIM, localEntries)  @dimOrder(1, 0);
 
 @kernel void LocalToGlobal(const int globalEntries,
                            const int localEntries,
-                           @restrict const int * offsets,
-                           @restrict const int * indices,
-                           @restrict const Local_t localX,
-                           @restrict Global_t globalX) {
+                           const int * restrict offsets,
+                           const int * restrict indices,
+                           const Local_t restrict localX,
+                           Global_t restrict globalX) {
 
   for (int i = 0; i < globalEntries; ++i; @tile(TILESIZE, @outer, @inner)) {
     if (i < globalEntries) {

backends/occa/geometry.okl

@@ -36,11 +36,11 @@ typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD, numElements);
 typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
 
 @kernel void InitGeometryInfo1D(const int numElements,
-                                @restrict const DofToQuadD1D_t dofToQuadD,
-                                @restrict const Local1D_t nodes,
-                                @restrict Jacobian1D_t J,
-                                @restrict Jacobian1D_t invJ,
-                                @restrict QLocal_t detJ) {
+                                const DofToQuadD1D_t restrict dofToQuadD,
+                                const Local1D_t restrict nodes,
+                                Jacobian1D_t restrict J,
+                                Jacobian1D_t restrict invJ,
+                                QLocal_t restrict detJ) {
 
   for (int e = 0; e < numElements; ++e; @outer) {
     @shared double s_nodes[NUM_DOFS];
@@ -70,11 +70,11 @@ typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
 }
 
 @kernel void InitGeometryInfo2D(const int numElements,
-                                @restrict const DofToQuadD2D_t dofToQuadD,
-                                @restrict const Local2D_t nodes,
-                                @restrict Jacobian2D_t J,
-                                @restrict Jacobian2D_t invJ,
-                                @restrict QLocal_t detJ) {
+                                const DofToQuadD2D_t restrict dofToQuadD,
+                                const Local2D_t restrict nodes,
+                                Jacobian2D_t restrict J,
+                                Jacobian2D_t restrict invJ,
+                                QLocal_t restrict detJ) {
 
   for (int e = 0; e < numElements; ++e; @outer) {
     @shared double s_nodes[2 * NUM_DOFS] @dim(2, NUM_DOFS);
@@ -119,11 +119,11 @@ typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
 }
 
 @kernel void InitGeometryInfo3D(const int numElements,
-                                @restrict const DofToQuadD3D_t dofToQuadD,
-                                @restrict const Local3D_t nodes,
-                                @restrict Jacobian3D_t J,
-                                @restrict Jacobian3D_t invJ,
-                                @restrict QLocal_t detJ) {
+                                const DofToQuadD3D_t restrict dofToQuadD,
+                                const Local3D_t restrict nodes,
+                                Jacobian3D_t restrict J,
+                                Jacobian3D_t restrict invJ,
+                                QLocal_t restrict detJ) {
 
   for (int e = 0; e < numElements; ++e; @outer) {
     @shared double s_nodes[3 * NUM_DOFS] @dim(3, NUM_DOFS);

backends/occa/gridfunc.cpp

@@ -59,26 +59,135 @@ std::map<std::string, ::occa::kernel> gridFunctionKernels;
    return kernel;
 }
 
-void ToQuad(const IntegrationRule &ir, FiniteElementSpace &fespace, Vector &gf,
-            Vector &quadValues)
+// OccaGridFunction::OccaGridFunction() :
+//    Vector(),
+//    ofespace(NULL),
+//    sequence(0) {}
+
+OccaGridFunction::OccaGridFunction(FiniteElementSpace *ofespace_)
+   : PArray(ofespace_->OccaVLayout()),
+     Array(ofespace_->OccaVLayout(), sizeof(double)),
+     Vector(ofespace_->OccaVLayout()),
+     ofespace(ofespace_),
+     sequence(0) {}
+
+// OccaGridFunction::OccaGridFunction(OccaFiniteElementSpace *ofespace_,
+//                                    OccaVectorRef ref) :
+//    OccaVector(ref),
+//    ofespace(ofespace_),
+//    sequence(0) {}
+
+OccaGridFunction::OccaGridFunction(const OccaGridFunction &v)
+   : PArray(v),
+     Array(v),
+     Vector(v),
+     ofespace(v.ofespace),
+     sequence(v.sequence) {}
+
+OccaGridFunction& OccaGridFunction::operator = (double value)
 {
-   const Engine &engine = fespace.OccaEngine();
+   Fill(value);
+   return *this;
+}
+
+OccaGridFunction& OccaGridFunction::operator = (const Vector &v)
+{
+   Assign<double>(v);
+   return *this;
+}
+
+// OccaGridFunction& OccaGridFunction::operator = (const OccaVectorRef &v)
+// {
+//    OccaVector::operator = (v);
+//    return *this;
+// }
+
+OccaGridFunction& OccaGridFunction::operator = (const OccaGridFunction &v)
+{
+   Assign<double>(v);
+   return *this;
+}
+
+// void OccaGridFunction::SetGridFunction(mfem::GridFunction &gf)
+// {
+//    Vector v = *this;
+//    gf.MakeRef(ofespace->GetFESpace(), v, 0);
+//    // Make gf the owner of the data
+//    v.Swap(gf);
+// }
+
+void OccaGridFunction::GetTrueDofs(Vector &v)
+{
+   const mfem::Operator *R = ofespace->GetRestrictionOperator();
+   if (!R)
+   {
+      v.MakeRef(*this);
+   }
+   else
+   {
+      v.Resize<double>(R->OutLayout(), NULL);
+      mfem::Vector mfem_v(v);
+      R->Mult(this->Wrap(), mfem_v);
+   }
+}
+
+void OccaGridFunction::SetFromTrueDofs(Vector &v)
+{
+   const mfem::Operator *P = ofespace->GetProlongationOperator();
+   if (!P)
+   {
+      MakeRef(v);
+   }
+   else
+   {
+      Resize<double>(P->OutLayout(), NULL);
+      mfem::Vector mfem_this(*this);
+      P->Mult(v.Wrap(), mfem_this);
+   }
+}
+
+mfem::FiniteElementSpace* OccaGridFunction::GetFESpace()
+{
+   return ofespace->GetFESpace();
+}
+
+const mfem::FiniteElementSpace* OccaGridFunction::GetFESpace() const
+{
+   return ofespace->GetFESpace();
+}
+
+void OccaGridFunction::ToQuad(const IntegrationRule &ir, Vector &quadValues)
+{
+   const Engine &engine = OccaLayout().OccaEngine();
    ::occa::device device = engine.GetDevice();
 
-   OccaDofQuadMaps &maps = OccaDofQuadMaps::Get(device, fespace, ir);
+   OccaDofQuadMaps &maps = OccaDofQuadMaps::Get(device, *ofespace, ir);
 
-   const int elements = fespace.GetNE();
+   const int elements = ofespace->GetNE();
    const int numQuad  = ir.GetNPoints();
-   quadValues.OccaResize(numQuad * elements, sizeof(double));
+   quadValues.Resize<double>(*(new Layout(engine, numQuad * elements)), NULL);
 
-   ::occa::kernel g2qKernel = GetGridFunctionKernel(device, fespace, ir);
+   ::occa::kernel g2qKernel = GetGridFunctionKernel(device, *ofespace, ir);
    g2qKernel(elements,
              maps.dofToQuad,
-             fespace.GetLocalToGlobalMap(),
-             gf.OccaMem(),
+             ofespace->GetLocalToGlobalMap(),
+             this->OccaMem(),
              quadValues.OccaMem());
 }
 
+void OccaGridFunction::Distribute(const Vector &v)
+{
+   if (ofespace->isDistributed())
+   {
+      mfem::Vector mfem_this(*this);
+      ofespace->GetProlongationOperator()->Mult(v.Wrap(), mfem_this);
+   }
+   else
+   {
+      *this = v;
+   }
+}
+
 } // namespace mfem::occa
 
 } // namespace mfem

backends/occa/gridfunc.hpp

@@ -22,10 +22,14 @@ namespace mfem
 {
 
 class IntegrationRule;
+class GridFunction;
 
 namespace occa
 {
 
+class OccaIntegrator;
+class OccaDofQuadMaps;
+
 // TODO: make this object part of the backend or the engine.
 extern std::map<std::string, ::occa::kernel> gridFunctionKernels;
 
@@ -34,17 +38,41 @@ extern std::map<std::string, ::occa::kernel> gridFunctionKernels;
                                      FiniteElementSpace &fespace,
                                      const mfem::IntegrationRule &ir);
 
-// ToQuad version without the deprecated class.
-//
-// FIXME: This is the action of a global B matrix, mapping L-vector to Q-vector,
-//        so it should be made into an operator that can be constructed by the
-//        FE space class. A batched version, where only a subset of the elements
-//        are processed should be defined as well.
-//
-//        The abstract operator construction method in the FE space class is:
-//           PFiniteElementSpace::GetInterpolationOperator(...)
-void ToQuad(const IntegrationRule &ir, FiniteElementSpace &ofespace, Vector &gf,
-            Vector &quadValues);
+class OccaGridFunction : public Vector
+{
+protected:
+   FiniteElementSpace *ofespace;
+   long sequence;
+
+   ::occa::kernel gridFuncToQuad[3];
+
+public:
+   // OccaGridFunction();
+
+   OccaGridFunction(FiniteElementSpace *ofespace_);
+
+   // OccaGridFunction(FiniteElementSpace *ofespace_,
+   //                  OccaVectorRef ref);
+
+   OccaGridFunction(const OccaGridFunction &gf);
+
+   OccaGridFunction& operator = (double value);
+   OccaGridFunction& operator = (const Vector &v);
+   // OccaGridFunction& operator = (const OccaVectorRef &v);
+   OccaGridFunction& operator = (const OccaGridFunction &gf);
+
+   // void SetGridFunction(mfem::GridFunction &gf);
+
+   void GetTrueDofs(Vector &v);
+   void SetFromTrueDofs(Vector &v);
+
+   mfem::FiniteElementSpace* GetFESpace();
+   const mfem::FiniteElementSpace* GetFESpace() const;
+
+   void ToQuad(const mfem::IntegrationRule &ir, Vector &quadValues);
+
+   void Distribute(const Vector &v);
+};
 
 } // namespace mfem::occa
 

backends/occa/gridfunc.okl

@@ -11,14 +11,14 @@
 
 #include "mfem-occa://defines.okl"
 
-#ifdef USING_TENSOR_OPS
-#  ifdef OCCA_USING_CPU
+#if USING_TENSOR_OPS
+#  if OCCA_USING_CPU
 #    include "mfem-occa://gridfunc/tensor/cpu.okl"
 #  else
 #    include "mfem-occa://gridfunc/tensor/gpuHighOrder.okl"
 #  endif
 #else
-#  ifdef OCCA_USING_CPU
+#  if OCCA_USING_CPU
 #    include "mfem-occa://gridfunc/simplex/cpu.okl"
 #  else
 #    include "mfem-occa://gridfunc/simplex/gpuHighOrder.okl"

backends/occa/gridfunc/simplex/cpu.okl

@@ -13,10 +13,10 @@
 
 //---[ 2D ]-----------------------------
 @kernel void GridFuncToQuad2D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QVLocal_t out) {
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal_t restrict out) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {
@@ -39,10 +39,10 @@
 
 //---[ 3D ]-----------------------------
 @kernel void GridFuncToQuad3D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QVLocal_t out) {
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal_t restrict out) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {

backends/occa/gridfunc/simplex/gpuHighOrder.okl

@@ -13,10 +13,10 @@
 
 //---[ 2D ]-----------------------------
 @kernel void GridFuncToQuad2D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QVLocal_t out) {
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal_t restrict out) {
   for (int e = 0; e < numElements; ++e; @outer) {
     @shared double s_gf[NUM_VDIM][NUM_DOFS];
 
@@ -47,10 +47,10 @@
 
 //---[ 3D ]-----------------------------
 @kernel void GridFuncToQuad3D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QVLocal_t out) {
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal_t restrict out) {
   for (int e = 0; e < numElements; ++e; @outer) {
     @shared double s_gf[NUM_VDIM][NUM_DOFS];
 

backends/occa/gridfunc/tensor/cpu.okl

@@ -13,10 +13,10 @@
 
 //---[ 1D ]-----------------------------
 @kernel void GridFuncToQuad1D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap1D_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QVLocal1D_t out) {
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap1D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal1D_t restrict out) {
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {
       double r_out[NUM_VDIM][NUM_QUAD_1D];
@@ -47,10 +47,10 @@
 
 //---[ 2D ]-----------------------------
 @kernel void GridFuncToQuad2D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap2D_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QVLocal2D_t out) {
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap2D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal2D_t restrict out) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {
@@ -106,10 +106,10 @@
 
 //---[ 3D ]-----------------------------
 @kernel void GridFuncToQuad3D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap3D_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QVLocal3D_t out) {
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap3D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal3D_t restrict out) {
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int dummy = 0; dummy < 1; ++dummy; @inner) {
       double out_xyz[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D];

backends/occa/gridfunc/tensor/gpuHighOrder.okl

@@ -13,10 +13,10 @@
 
 //---[ 1D ]-----------------------------
 @kernel void GridFuncToQuad1D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap1D_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QLocal1D_t out) {
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap1D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QLocal1D_t restrict out) {
   // Iterate over elements
   for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
     @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
@@ -54,10 +54,10 @@
 
 //---[ 2D ]-----------------------------
 @kernel void GridFuncToQuad2D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap2D_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QLocal2D_t out) {
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap2D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QLocal2D_t restrict out) {
   // Iterate over elements
   for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
     // Store dof <--> quad mappings
@@ -109,10 +109,10 @@
 
 //---[ 3D ]-----------------------------
 @kernel void GridFuncToQuad3D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap3D_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QLocal3D_t out) {
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap3D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QLocal3D_t restrict out) {
   // Iterate over elements
   for (int e = 0; e < numElements; ++e; @outer) {
     // Store dof <--> quad mappings

backends/occa/interpolation.cpp

@@ -20,51 +20,94 @@ namespace mfem
 namespace occa
 {
 
-RestrictionOperator::RestrictionOperator(Layout &in_layout, Layout &out_layout,
-                                         ::occa::array<int> indices)
-   : Operator(in_layout, out_layout)
+void CreateRPOperators(Layout &v_layout, Layout &t_layout,
+                       const mfem::SparseMatrix *R, const mfem::Operator *P,
+                       mfem::Operator *&OccaR, mfem::Operator *&OccaP)
 {
+   if (!P)
+   {
+      OccaR = new IdentityOperator(t_layout);
+      OccaP = new IdentityOperator(t_layout);
+      return;
+   }
+
+   const mfem::SparseMatrix *pmat = dynamic_cast<const mfem::SparseMatrix*>(P);
+   ::occa::device device = v_layout.OccaEngine().GetDevice();
+
+   if (R)
+   {
+      OccaSparseMatrix *occaR =
+         CreateMappedSparseMatrix(v_layout, t_layout, *R);
+      ::occa::array<int> reorderIndices = occaR->reorderIndices;
+      delete occaR;
+
+      OccaR = new RestrictionOperator(v_layout, t_layout, reorderIndices);
+   }
+
+   if (pmat)
+   {
+      const mfem::SparseMatrix *pmatT = Transpose(*pmat);
+
+      OccaSparseMatrix *occaP  =
+         CreateMappedSparseMatrix(t_layout, v_layout, *pmat);
+      OccaSparseMatrix *occaPT =
+         CreateMappedSparseMatrix(v_layout, t_layout, *pmatT);
+
+      OccaP = new ProlongationOperator(*occaP, *occaPT);
+   }
+   else
+   {
+      OccaP = new ProlongationOperator(t_layout, v_layout, P);
+   }
+}
+
+RestrictionOperator::RestrictionOperator(Layout &in_layout, Layout &out_layout,
+                                         ::occa::array<int> indices) :
+   Operator(in_layout, out_layout)
+{
+
+   entries     = indices.size() / 2;
    trueIndices = indices;
 
+   // FIXME: paths ...
    ::occa::device device = in_layout.OccaEngine().GetDevice();
    const std::string &okl_path = in_layout.OccaEngine().GetOklPath();
+   const std::string &okl_defines = in_layout.OccaEngine().GetOklDefines();
    multOp = device.buildKernel(okl_path + "mappings.okl",
                                "ExtractSubVector",
-                               "defines: { TILESIZE: 256 }");
+                               "defines: { TILESIZE: 256 }" + okl_defines);
 
    multTransposeOp = device.buildKernel(okl_path + "mappings.okl",
                                         "SetSubVector",
-                                        "defines: { TILESIZE: 256 }");
+                                        "defines: { TILESIZE: 256 }" +
+                                        okl_defines);
 }
 
 void RestrictionOperator::Mult_(const Vector &x, Vector &y) const
 {
-   // y[i] = x[trueIndices[i]]
-   multOp(height, trueIndices, x.OccaMem(), y.OccaMem());
+   multOp(entries, trueIndices, x.OccaMem(), y.OccaMem());
 }
 
 void RestrictionOperator::MultTranspose_(const Vector &x, Vector &y) const
 {
-   y.OccaFill<double>(0.0);
-   // y[trueIndices[i]] = x[i]
-   multTransposeOp(height, trueIndices, x.OccaMem(), y.OccaMem());
+   y.Fill<double>(0.0);
+   multTransposeOp(entries, trueIndices, x.OccaMem(), y.OccaMem());
 }
 
 ProlongationOperator::ProlongationOperator(OccaSparseMatrix &multOp_,
-                                           OccaSparseMatrix &multTransposeOp_)
-   : Operator(multOp_),
-     pmat(NULL),
-     multOp(multOp_),
-     multTransposeOp(multTransposeOp_)
-{ }
+                                           OccaSparseMatrix &multTransposeOp_) :
+   Operator(multOp_),
+   pmat(NULL),
+   multOp(multOp_),
+   multTransposeOp(multTransposeOp_) {}
 
 ProlongationOperator::ProlongationOperator(Layout &in_layout,
                                            Layout &out_layout,
-                                           const mfem::Operator *pmat_)
-   : Operator(in_layout, out_layout),
-     pmat(pmat_),
-     multOp(*this),
-     multTransposeOp(*this)
+                                           const mfem::Operator *pmat_) :
+   Operator(in_layout, out_layout),
+   pmat(pmat_),
+   multOp(*this),
+   multTransposeOp(*this)
 { }
 
 void ProlongationOperator::Mult_(const Vector &x, Vector &y) const

backends/occa/interpolation.hpp

@@ -27,10 +27,16 @@ namespace mfem
 namespace occa
 {
 
+// [MISSING] Proper destructors
+void CreateRPOperators(Layout &v_layout, Layout &t_layout,
+                       const mfem::SparseMatrix *R, const mfem::Operator *P,
+                       mfem::Operator *&OccaR, mfem::Operator *&OccaP);
+
 class RestrictionOperator : public Operator
 {
 protected:
-   ::occa::array<int> trueIndices; // ldof = trueIndices[ltdof]
+   int entries;
+   ::occa::array<int> trueIndices;
    ::occa::kernel multOp, multTransposeOp;
 
 public:

backends/occa/layout.hpp

@@ -39,7 +39,8 @@ public:
    const Engine &OccaEngine() const
    { return *static_cast<const Engine *>(engine.Get()); }
 
-   void OccaResize(std::size_t new_size) { size = new_size; }
+   ::occa::memory Alloc(std::size_t bytes) const
+   { return OccaEngine().GetDevice().malloc(bytes); }
 
    virtual ~Layout() { }
 

backends/occa/mappings.okl

@@ -16,60 +16,38 @@
 */
 
 @kernel void ExtractSubVector(const int entries,
-                              @restrict const int *indices,
-                              @restrict const double *in,
-                              @restrict double *out) {
+                              const int * restrict indices,
+                              const double * restrict in,
+                              double * restrict out) {
 
   for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
     if (i < entries) {
-      out[i] = in[indices[i]]; // indices can be repeated
+      out[i] = in[indices[i]];
     }
   }
 }
 
 @kernel void SetSubVector(const int entries,
-                          @restrict const int *indices,
-                          @restrict const double *in,
-                          @restrict double *out) {
+                          const int * restrict indices,
+                          const double * restrict in,
+                          double * restrict out) {
 
   for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
     if (i < entries) {
-      out[indices[i]] = in[i]; // indices CANNOT be repeated
-    }
-  }
-}
-
-@kernel void AddSubVector(const int num_unique_dst_indices,
-                          @restrict const int *unique_dst_indices,
-                          @restrict const int *unique_to_src_offsets,
-                          @restrict const int *unique_to_src_indices,
-                          @restrict const double *src,
-                          @restrict double *dst) {
-
-  for (int i = 0; i < num_unique_dst_indices; ++i;
-       @tile(TILESIZE, @outer, @inner)) {
-
-    if (i < num_unique_dst_indices) {
-      const int dst_idx = unique_dst_indices[i];
-      double sum = dst[dst_idx];
-      const int end = unique_to_src_offsets[i+1];
-      for (int j = unique_to_src_offsets[i]; j != end; ++j) {
-        sum += src[unique_to_src_indices[j]];
-      }
-      dst[dst_idx] = sum;
+      out[indices[i]] = in[i];
     }
   }
 }
 
 @kernel void MapSubVector(const int entries,
-                          @restrict const int *indices,
-                          @restrict const double *in,
-                          @restrict double *out) {
+                          const int * restrict indices,
+                          const double * restrict in,
+                          double * restrict out) {
 
   for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
     if (i < entries) {
-      const int fromIdx = indices[2*i + 0]; // fromIdx indices can be repeated
-      const int toIdx   = indices[2*i + 1]; // toIdx indices CANNOT be repeated
+      const int fromIdx = indices[2*i + 0];
+      const int toIdx   = indices[2*i + 1];
       out[toIdx] = in[fromIdx];
     }
   }

backends/occa/mass/simplex/cpu.okl

@@ -1,122 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian2D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-
-      oper(q, e) = quadWeights[q] * COEFF * ((J11 * J22) - (J21 * J12));
-    }
-  }
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuadD2D_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDofD2D_t quadToDofD,
-                       @restrict const QLocal_t oper,
-                       @restrict const DLocal_t solIn,
-                       @restrict DLocal_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double r_sol[NUM_DOFS];
-
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        r_sol[d] = 0;
-      }
-
-      for (int q = 0; q < NUM_QUAD; ++q) {
-        double s = 0;
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          s += solIn(d, e) * quadToDof(d, q);
-        }
-        s *= oper(q, e);
-
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          r_sol[d] += (s * quadToDof(d, q));
-        }
-      }
-
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        solOut(d, e) += r_sol[d];
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian3D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-      oper(q, e) = quadWeights[q] * COEFF * detJ;
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuadD3D_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDofD3D_t quadToDofD,
-                       @restrict const QLocal_t oper,
-                       @restrict const DLocal_t solIn,
-                       @restrict DLocal_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double r_sol[NUM_DOFS];
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        r_sol[d] = 0;
-      }
-
-      for (int q = 0; q < NUM_QUAD; ++q) {
-        double s = 0;
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          s += solIn(d, e) * quadToDof(d, q);
-        }
-        s *= oper(q, e);
-
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          r_sol[d] += (s * quadToDof(d, q));
-        }
-      }
-
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        solOut(d, e) += r_sol[d];
-      }
-    }
-  }
-}
-//======================================

backends/occa/mass/simplex/gpuHighOrder.okl

@@ -1,129 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian2D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
-          for (int q = qOff; q < NUM_QUAD; q += A2_QUAD_BATCH) {
-            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-
-            oper(q, e) = quadWeights[q] * COEFF * ((J11 * J22) - (J21 * J12));
-          }
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuadD2D_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDofD2D_t quadToDofD,
-                       @restrict const QLocal_t oper,
-                       @restrict const DLocal_t solIn,
-                       @restrict DLocal_t solOut) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_sol[NUM_QUAD];
-
-    for (int qOff = 0; qOff < M2_INNER_BATCH; ++qOff; @inner) {
-      for (int q = qOff; q < NUM_QUAD; q += M2_INNER_BATCH) {
-        double s = 0;
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          s += solIn(d, e) * dofToQuad(d, q);
-        }
-        s_sol[q] = s * oper(q, e);
-      }
-    }
-
-    for (int dOff = 0; dOff < M2_INNER_BATCH; ++dOff) {
-      for (int d = dOff; d < NUM_DOFS; d += M2_INNER_BATCH) {
-        double r_sol = 0;
-        for (int q = 0; q < NUM_QUAD; ++q) {
-          r_sol += (s_sol[q] * quadToDof(d, q));
-        }
-        solOut(d, e) += r_sol;
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian3D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
-          for (int q = qOff; q < NUM_QUAD; q += A3_QUAD_BATCH) {
-            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-            oper(q, e) = quadWeights[q] * COEFF * detJ;
-          }
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuadD3D_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDofD3D_t quadToDofD,
-                       @restrict const QLocal_t oper,
-                       @restrict const DLocal_t solIn,
-                       @restrict DLocal_t solOut) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_sol[NUM_QUAD];
-
-    for (int qOff = 0; qOff < M3_INNER_BATCH; ++qOff; @inner) {
-      for (int q = qOff; q < NUM_QUAD; q += M3_INNER_BATCH) {
-        double s = 0;
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          s += solIn(d, e) * dofToQuad(d, q);
-        }
-        s_sol[q] = s * oper(q, e);
-      }
-    }
-
-    for (int dOff = 0; dOff < M3_INNER_BATCH; ++dOff) {
-      for (int d = dOff; d < NUM_DOFS; d += M3_INNER_BATCH) {
-        double r_sol = 0;
-        for (int q = 0; q < NUM_QUAD; ++q) {
-          r_sol += (s_sol[q] * quadToDof(d, q));
-        }
-        solOut(d, e) += r_sol;
-      }
-    }
-  }
-}
-//======================================

backends/occa/mass/tensor/cpu.okl

@@ -1,281 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 1D ]-----------------------------
-@kernel void Assemble1D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian1D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
-      const double detJ = J(q, e);
-      oper(q, e) = quadWeights[q] * COEFF * detJ;
-    }
-  }
-}
-
-@kernel void MultAdd1D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal1D_t oper,
-                       @restrict const DLocal1D_t solIn,
-                       @restrict DLocal1D_t solOut) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double sol_x[NUM_QUAD_1D];
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        sol_x[qx] = 0;
-      }
-      // sol_x{qx} = dofToQuad{qx,dx} * sol{dx}
-      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-        const double s = solIn(dx, e);
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          sol_x[qx] += s * dofToQuad(qx, dx);
-        }
-      }
-      // sol_x{q} *= oper{q}
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        sol_x[qx] *= oper(qx, e);
-      }
-      // sol{dx} = quadToDof{dx,qx} * sol_x{qx}
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          solOut(dx, e) += sol_x[qx] * quadToDof(dx, qx);
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian2D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_2D; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-      const double detJ = ((J11 * J22) - (J21 * J12));
-
-      oper(q, e) = quadWeights[q] * COEFF * detJ;
-    }
-  }
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal2D_t oper,
-                       @restrict const DLocal2D_t solIn,
-                       @restrict DLocal2D_t solOut) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double sol_xy[NUM_QUAD_1D][NUM_QUAD_1D];
-
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          sol_xy[qy][qx] = 0;
-        }
-      }
-
-      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-        double sol_x[NUM_QUAD_1D];
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          sol_x[qy] = 0;
-        }
-
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          const double s = solIn(dx, dy, e);
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_x[qx] += dofToQuad(qx, dx) * s;
-          }
-        }
-
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          const double d2q = dofToQuad(qy, dy);
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_xy[qy][qx] += d2q * sol_x[qx];
-          }
-        }
-      }
-
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          sol_xy[qy][qx] *= oper(qx, qy, e);
-        }
-      }
-
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        double sol_x[NUM_DOFS_1D];
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          sol_x[dx] = 0;
-        }
-
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          const double s = sol_xy[qy][qx];
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            sol_x[dx] += quadToDof(dx, qx) * s;
-          }
-        }
-
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          const double q2d = quadToDof(dy, qy);
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            solOut(dx, dy, e) += q2d * sol_x[dx];
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian3D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_3D; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-      oper(q, e) = quadWeights[q] * COEFF * detJ;
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal3D_t oper,
-                       @restrict const DLocal3D_t solIn,
-                       @restrict DLocal3D_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double sol_xyz[NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D];
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_xyz[qz][qy][qx] = 0;
-          }
-        }
-      }
-
-      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-        double sol_xy[NUM_QUAD_1D][NUM_QUAD_1D];
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_xy[qy][qx] = 0;
-          }
-        }
-
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          double sol_x[NUM_QUAD_1D];
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_x[qx] = 0;
-          }
-
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            const double s = solIn(dx, dy, dz, e);
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              sol_x[qx] += dofToQuad(qx, dx) * s;
-            }
-          }
-
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            const double wy = dofToQuad(qy, dy);
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              sol_xy[qy][qx] += wy * sol_x[qx];
-            }
-          }
-        }
-
-        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-          const double wz = dofToQuad(qz, dz);
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
-            }
-          }
-        }
-      }
-
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_xyz[qz][qy][qx] *= oper(qx, qy, qz, e);
-          }
-        }
-      }
-
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        double sol_xy[NUM_DOFS_1D][NUM_DOFS_1D];
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            sol_xy[dy][dx] = 0;
-          }
-        }
-
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          double sol_x[NUM_DOFS_1D];
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            sol_x[dx] = 0;
-          }
-
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            const double s = sol_xyz[qz][qy][qx];
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              sol_x[dx] += quadToDof(dx, qx) * s;
-            }
-          }
-
-          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-            const double wy = quadToDof(dy, qy);
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              sol_xy[dy][dx] += wy * sol_x[dx];
-            }
-          }
-        }
-
-        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-          const double wz = quadToDof(dz, qz);
-          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              solOut(dx, dy, dz, e) += wz * sol_xy[dy][dx];
-            }
-          }
-        }
-      }
-    }
-  }
-}
-//======================================

backends/occa/mass/tensor/gpuHighOrder.okl

@@ -1,341 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 1D ]-----------------------------
-@kernel void Assemble1D(const int numElements,
-                        const double *quadWeights,
-                        const Jacobian1D_t J,
-                        COEFF_ARGS
-                        QLocal_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A1_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A1_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
-          oper(q, e) = quadWeights[q] * COEFF * J(q, e);
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd1D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal1D_t oper,
-                       @restrict const DLocal1D_t solIn,
-                       @restrict DLocal1D_t solOut) {
-  // Iterate over elements
-  for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
-    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-
-    @exclusive double r_sol[NUM_QUAD_1D];
-
-    for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
-      for (int i = el; i < NUM_QUAD_DOFS_1D; i += M1_INNER_ELEMENT_BATCH) {
-        s_dofToQuad[i] = dofToQuad[i];
-        s_quadToDof[i] = quadToDof[i];
-      }
-    }
-
-    for (int b = 0; b < M1_OUTER_ELEMENT_BATCH; ++b) {
-      for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
-        const int e = eOff + b*M1_INNER_ELEMENT_BATCH + el;
-        if (e < numElements) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            r_sol[qx] = 0;
-          }
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            const double s = solIn(dx, e);
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              r_sol[qx] += s * s_dofToQuad(qx, dx);
-            }
-          }
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            r_sol[qx] *= oper(qx, e);
-          }
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            double s = 0;
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              s += r_sol[qx] * s_quadToDof(dx, qx);
-            }
-            solOut(dx, e) += s;
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        const double *quadWeights,
-                        const Jacobian2D_t J,
-                        COEFF_ARGS
-                        QLocal_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
-          for (int q = qOff; q < NUM_QUAD_2D; q += A2_QUAD_BATCH) {
-            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-
-            oper(q, e) = quadWeights[q] * COEFF * ((J11 * J22) - (J21 * J12));
-          }
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal2D_t oper,
-                       @restrict const DLocal2D_t solIn,
-                       @restrict DLocal2D_t solOut) {
-  // Iterate over elements
-  for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
-    // Store dof <--> quad mappings
-    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-
-    // Store xy planes in @shared memory
-    @shared double s_xy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-    @shared double s_xy2[NUM_QUAD_2D] @dim(NUM_QUAD_1D, NUM_QUAD_1D);
-
-    @exclusive double r_x[NUM_MAX_1D];
-
-    for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
-      for (int id = x; id < NUM_QUAD_DOFS_1D; id += NUM_MAX_1D) {
-        s_dofToQuad[id]  = dofToQuad[id];
-        s_quadToDof[id]  = quadToDof[id];
-      }
-    }
-
-    for (int e = eOff; e < (eOff + M2_ELEMENT_BATCH); ++e) {
-      if (e < numElements) {
-        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-          if (dx < NUM_DOFS_1D) {
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              s_xy(dx, qy) = 0;
-            }
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              r_x[dy] = solIn(dx, dy, e);
-            }
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              double xy = 0;
-              for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-                xy += r_x[dy] * s_dofToQuad(qy, dy);
-              }
-              s_xy(dx, qy) = xy;
-            }
-          }
-        }
-        for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
-          if (qy < NUM_QUAD_1D) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              double s = 0;
-              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-                s += s_xy(dx, qy) * s_dofToQuad(qx, dx);
-              }
-              s_xy2(qx, qy) = s * oper(qx, qy, e);
-            }
-          }
-        }
-
-        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
-          if (qx < NUM_QUAD_1D) {
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              s_xy(dy, qx) = 0;
-            }
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              r_x[qy] = s_xy2(qx, qy);
-            }
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              double s = 0;
-              for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-                s += r_x[qy] * s_quadToDof(dy, qy);
-              }
-              s_xy(dy, qx) = s;
-            }
-          }
-        }
-        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-          if (dx < NUM_DOFS_1D) {
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              double s = 0;
-              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-                s += (s_xy(dy, qx) * s_quadToDof(dx, qx));
-              }
-              solOut(dx, dy, e) += s;
-            }
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        const double *quadWeights,
-                        const Jacobian3D_t J,
-                        COEFF_ARGS
-                        QLocal_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
-          for (int q = qOff; q < NUM_QUAD_3D; q += A3_QUAD_BATCH) {
-            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-            oper(q, e) = quadWeights[q] * COEFF * detJ;
-          }
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal3D_t oper,
-                       @restrict const DLocal3D_t solIn,
-                       @restrict DLocal3D_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    // Store dof <--> quad mappings
-    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-
-    // Store xy planes in @shared memory
-    @shared double s_xy[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
-
-    // Store z axis as registers
-    @exclusive double r_z[NUM_QUAD_1D];
-    @exclusive double r_z2[NUM_DOFS_1D];
-
-    for (int y = 0; y < NUM_MAX_1D; ++y; @inner) {
-      for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
-        const int id = (y * NUM_MAX_1D) + x;
-        // Fetch Q <--> D maps
-        if (id < NUM_QUAD_DOFS_1D) {
-          s_dofToQuad[id]  = dofToQuad[id];
-          s_quadToDof[id]  = quadToDof[id];
-        }
-        // Initialize our Z axis
-        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-          r_z[qz] = 0;
-        }
-        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-          r_z2[dz] = 0;
-        }
-      }
-    }
-
-    for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
-      for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-        if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
-          for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-            const double s = solIn(dx, dy, dz, e);
-            // Calculate D -> Q in the Z axis
-            for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-              r_z[qz] += s * s_dofToQuad(qz, dz);
-            }
-          }
-        }
-      }
-    }
-    // For each xy plane
-    for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-      // Fill xy plane at given z position
-      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
-        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
-            s_xy(dx, dy) = r_z[qz];
-          }
-        }
-      }
-      // Calculate Dxyz, xDyz, xyDz in plane
-      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
-        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
-          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
-            double s = 0;
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              const double wy = s_dofToQuad(qy, dy);
-              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-                const double wx = s_dofToQuad(qx, dx);
-                s += wx * wy * s_xy(dx, dy);
-              }
-            }
-
-            s *= oper(qx, qy, qz, e);
-
-            for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-              const double wz  = s_quadToDof(dz, qz);
-              r_z2[dz] += wz * s;
-            }
-          }
-        }
-      }
-      @barrier("s_xy_sync_1");
-    }
-    // Iterate over xy planes to compute solution
-    for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-      // Place xy plane in @shared memory
-      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
-        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
-          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
-            s_xy(qx, qy) = r_z2[dz];
-          }
-        }
-      }
-      // Finalize solution in xy plane
-      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
-        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
-            double solZ = 0;
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              const double wy = s_quadToDof(dy, qy);
-              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-                const double wx = s_quadToDof(dx, qx);
-                solZ += wx * wy * s_xy(qx, qy);
-              }
-            }
-            solOut(dx, dy, dz, e) += solZ;
-          }
-        }
-      }
-      @barrier("s_xy_sync_2");
-    }
-  }
-}
-//======================================

backends/occa/operator.cpp

@@ -73,21 +73,15 @@ void OccaConstrainedOperator::Setup(::occa::device device_,
    own_A = own_A_;
 
    constraintIndices = constraintList_.Size();
-   if (constraintList_.Size() > 0)
-   {
-      constraintList = constraintList_.Get_PArray()->As<Array>().OccaMem();
-   }
-   else
-   {
-      // constraintList is not used
-   }
+   constraintList = constraintList_.Get_PArray()->As<Array>().OccaMem();
 }
 
 void OccaConstrainedOperator::EliminateRHS(const Vector &x, Vector &b) const
 {
-   ::occa::kernel mapDofs = mapDofBuilder.build(device);
+   const std::string &okl_defines = InLayout_().OccaEngine().GetOklDefines();
+   ::occa::kernel mapDofs = mapDofBuilder.build(device, okl_defines);
 
-   w.OccaFill(0.0);
+   w.Fill<double>(0.0);
 
    if (constraintIndices)
    {
@@ -113,12 +107,11 @@ void OccaConstrainedOperator::Mult_(const Vector &x, Vector &y) const
       return;
    }
 
-   ::occa::kernel mapDofs   = mapDofBuilder.build(device);
-   ::occa::kernel clearDofs = clearDofBuilder.build(device);
+   const std::string &okl_defines = InLayout_().OccaEngine().GetOklDefines();
+   ::occa::kernel mapDofs   = mapDofBuilder.build(device, okl_defines);
+   ::occa::kernel clearDofs = clearDofBuilder.build(device, okl_defines);
 
-   // z.OccaAssign(x); // z = x
-   // Is Axpy faster than DtoD copy on Volta?
-   z.Axpby(1.0, x, 0.0, x);
+   z.Assign<double>(x); // z = x
 
    clearDofs(constraintIndices, z.OccaMem(), constraintList);
 

backends/occa/operator.hpp

@@ -37,9 +37,11 @@ public:
    Operator(Layout &in_layout, Layout &out_layout)
       : mfem::Operator(in_layout, out_layout) { }
 
-   Layout &InLayout_() const { return in_layout->As<Layout>(); }
+   Layout &InLayout_() const
+   { return *static_cast<Layout*>(in_layout.Get()); }
 
-   Layout &OutLayout_() const { return out_layout->As<Layout>(); }
+   Layout &OutLayout_() const
+   { return *static_cast<Layout*>(out_layout.Get()); }
 
    virtual void Mult_(const Vector &x, Vector &y) const = 0;
 

backends/occa/sparse.okl

@@ -16,11 +16,11 @@
 */
 
 @kernel void Mult(const int entries,
-                  @restrict const int *offsets,
-                  @restrict const int *indices,
-                  @restrict const double *weights,
-                  @restrict const double *in,
-                  @restrict double *out) {
+                  const int * restrict offsets,
+                  const int * restrict indices,
+                  const double * restrict weights,
+                  const double * restrict in,
+                  double * restrict out) {
 
   for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
     if (i < entries) {
@@ -36,12 +36,12 @@
 }
 
 @kernel void MappedMult(const int entries,
-                        @restrict const int *offsets,
-                        @restrict const int *indices,
-                        @restrict const double *weights,
-                        @restrict const int *outIndices,
-                        @restrict const double *in,
-                        @restrict double *out) {
+                        const int * restrict offsets,
+                        const int * restrict indices,
+                        const double * restrict weights,
+                        const int * restrict outIndices,
+                        const double * restrict in,
+                        double * restrict out) {
 
   for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
     if (i < entries) {

backends/occa/sparsemat.cpp

@@ -22,26 +22,54 @@ namespace occa
 
 OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
                                    const mfem::SparseMatrix &m,
-                                   const ::occa::properties &props)
-   : Operator(in_layout, out_layout)
+                                   const ::occa::properties &props) :
+   Operator(in_layout, out_layout)
 {
+
    Setup(in_layout.OccaEngine().GetDevice(), m, props);
 }
 
+OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                                   const mfem::SparseMatrix &m,
+                                   ::occa::array<int> reorderIndices_,
+                                   ::occa::array<int> mappedIndices_,
+                                   const ::occa::properties &props) :
+   Operator(in_layout, out_layout)
+{
+
+   Setup(in_layout.OccaEngine().GetDevice(), m,
+         reorderIndices, mappedIndices_, props);
+}
+
+OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                                   ::occa::array<int> offsets_,
+                                   ::occa::array<int> indices_,
+                                   ::occa::array<double> weights_,
+                                   const ::occa::properties &props) :
+   Operator(in_layout, out_layout),
+   offsets(offsets_),
+   indices(indices_),
+   weights(weights_)
+{
+
+   SetupKernel(in_layout.OccaEngine().GetDevice(), props);
+}
+
 OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
                                    ::occa::array<int> offsets_,
                                    ::occa::array<int> indices_,
                                    ::occa::array<double> weights_,
                                    ::occa::array<int> reorderIndices_,
                                    ::occa::array<int> mappedIndices_,
-                                   const ::occa::properties &props)
-   : Operator(in_layout, out_layout),
-     offsets(offsets_),
-     indices(indices_),
-     weights(weights_),
-     reorderIndices(reorderIndices_),
-     mappedIndices(mappedIndices_)
+                                   const ::occa::properties &props) :
+   Operator(in_layout, out_layout),
+   offsets(offsets_),
+   indices(indices_),
+   weights(weights_),
+   reorderIndices(reorderIndices_),
+   mappedIndices(mappedIndices_)
 {
+
    SetupKernel(in_layout.OccaEngine().GetDevice(), props);
 }
 
@@ -56,9 +84,6 @@ void OccaSparseMatrix::Setup(::occa::device device, const SparseMatrix &m,
                              ::occa::array<int> mappedIndices_,
                              const ::occa::properties &props)
 {
-   MFEM_ASSERT(m.Finalized(), "");
-   MFEM_ASSERT(m.Height() == height, "");
-   MFEM_ASSERT(m.Width() == width, "");
 
    const int nnz = m.GetI()[height];
    offsets.allocate(device,
@@ -81,6 +106,7 @@ void OccaSparseMatrix::Setup(::occa::device device, const SparseMatrix &m,
 void OccaSparseMatrix::SetupKernel(::occa::device device,
                                    const ::occa::properties &props)
 {
+
    const bool hasOutIndices = mappedIndices.isInitialized();
 
    const ::occa::properties defaultProps("defines: {"
@@ -88,13 +114,14 @@ void OccaSparseMatrix::SetupKernel(::occa::device device,
                                          "}");
 
    const std::string &okl_path = InLayout_().OccaEngine().GetOklPath();
+   const std::string &okl_defines = InLayout_().OccaEngine().GetOklDefines();
    mapKernel = device.buildKernel(okl_path + "mappings.okl",
                                   "MapSubVector",
-                                  defaultProps + props);
+                                  defaultProps + props + okl_defines);
 
    multKernel = device.buildKernel(okl_path + "sparse.okl",
                                    hasOutIndices ? "MappedMult" : "Mult",
-                                   defaultProps + props);
+                                   defaultProps + props + okl_defines);
 }
 
 void OccaSparseMatrix::Mult_(const Vector &x, Vector &y) const

backends/occa/sparsemat.hpp

@@ -30,12 +30,40 @@ namespace occa
 /// TODO: doxygen
 class OccaSparseMatrix : public Operator
 {
-protected:
+public:
    ::occa::array<int> offsets, indices;
    ::occa::array<double> weights;
    ::occa::array<int> reorderIndices, mappedIndices;
    ::occa::kernel mapKernel, multKernel;
 
+   /// Construct an empty OccaSparseMatrix.
+   OccaSparseMatrix(const Operator &orig)
+      : Operator(orig) { }
+
+   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                    const mfem::SparseMatrix &m,
+                    const ::occa::properties &props = ::occa::properties());
+
+   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                    const mfem::SparseMatrix &m,
+                    ::occa::array<int> reorderIndices_,
+                    ::occa::array<int> mappedIndices_,
+                    const ::occa::properties &props = ::occa::properties());
+
+   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                    ::occa::array<int> offsets_,
+                    ::occa::array<int> indices_,
+                    ::occa::array<double> weights_,
+                    const ::occa::properties &props = ::occa::properties());
+
+   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                    ::occa::array<int> offsets_,
+                    ::occa::array<int> indices_,
+                    ::occa::array<double> weights_,
+                    ::occa::array<int> reorderIndices_,
+                    ::occa::array<int> mappedIndices_,
+                    const ::occa::properties &props = ::occa::properties());
+
    void Setup(::occa::device device, const mfem::SparseMatrix &m,
               const ::occa::properties &props);
 
@@ -47,28 +75,6 @@ protected:
    void SetupKernel(::occa::device device,
                     const ::occa::properties &props);
 
-public:
-   /// Construct an empty OccaSparseMatrix.
-   OccaSparseMatrix(const Operator &orig)
-      : Operator(orig) { }
-
-   // Implicitly defined copy constructor.
-
-   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                    const mfem::SparseMatrix &m,
-                    const ::occa::properties &props = ::occa::properties());
-
-   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                    ::occa::array<int> offsets_,
-                    ::occa::array<int> indices_,
-                    ::occa::array<double> weights_,
-                    ::occa::array<int> reorderIndices_,
-                    ::occa::array<int> mappedIndices_,
-                    const ::occa::properties &props = ::occa::properties());
-
-   const ::occa::array<int> &GetReorderIndices() const
-   { return reorderIndices; }
-
    // override
    virtual void Mult_(const Vector &x, Vector &y) const;
 };

backends/occa/utils.okl

@@ -13,7 +13,7 @@ typedef double* Local_t @dim(numDofs, numElements);
 
 @kernel void InitLocalVector(const int numElements,
                              const int numDofs,
-                             @restrict Local_t sol) {
+                             Local_t restrict sol) {
   for (int e = 0; e < numElements; ++e; @outer) {
     for (int d = 0; d < numDofs; ++d; @inner) {
       sol(d, e) = 0;

backends/occa/vector.cpp

@@ -45,16 +45,17 @@ void Vector::DoDotProduct(const PVector &x, void *result,
 
    MFEM_ASSERT(result_type_id == ScalarId<double>::value, "");
    double *res = (double *)result;
-   const Vector &xp = x.As<Vector>();
-   MFEM_ASSERT(this->Size() == xp.Size(), "");
-   *res = ::occa::linalg::dot<double, double, double>(this->slice, xp.slice);
+   MFEM_ASSERT(dynamic_cast<const Vector *>(&x) != NULL, "invalid Vector type");
+   const Vector *xp = static_cast<const Vector *>(&x);
+   MFEM_ASSERT(this->Size() == xp->Size(), "");
+   *res = ::occa::linalg::dot<double, double, double>(this->slice, xp->slice);
 
 #ifdef MFEM_USE_MPI
    double local_dot = *res;
    if (IsParallel())
    {
       MPI_Allreduce(&local_dot, res, 1, MPI_DOUBLE, MPI_SUM,
-                    OccaEngine().GetComm());
+                    OccaLayout().OccaEngine().GetComm());
    }
 #endif
 }
@@ -63,6 +64,8 @@ void Vector::DoAxpby(const void *a, const PVector &x,
                      const void *b, const PVector &y,
                      int ab_type_id)
 {
+   const std::string &okl_defines = OccaLayout().OccaEngine().GetOklDefines();
+
    //
    // TODO: move all kernel builders to class mfem::occa::Backend
    //
@@ -121,7 +124,7 @@ void Vector::DoAxpby(const void *a, const PVector &x,
    {
       if (db == 0.0)
       {
-         OccaFill(da);
+         OccaFill(&da);
       }
       else
       {
@@ -133,7 +136,8 @@ void Vector::DoAxpby(const void *a, const PVector &x,
          else
          {
             // *this = db * y
-            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice());
+            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice(),
+                                                         okl_defines);
             kernel((int)Size(), db, slice, yp->slice);
          }
       }
@@ -150,7 +154,8 @@ void Vector::DoAxpby(const void *a, const PVector &x,
          else
          {
             // *this = da * x
-            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice());
+            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice(),
+                                                         okl_defines);
             kernel((int)Size(), da, slice, xp->slice);
          }
       }
@@ -160,19 +165,22 @@ void Vector::DoAxpby(const void *a, const PVector &x,
          if (this->slice == xp->slice)
          {
             // *this = da * (*this) + db * y
-            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice());
+            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice(),
+                                                         okl_defines);
             kernel((int)Size(), da, db, slice, yp->slice);
          }
          else if (this->slice == yp->slice)
          {
             // *this = da * x + db * (*this)
-            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice());
+            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice(),
+                                                         okl_defines);
             kernel((int)Size(), db, da, slice, xp->slice);
          }
          else
          {
             // *this = da * x + db * y
-            ::occa::kernel kernel = axpby3_builder.build(slice.getDevice());
+            ::occa::kernel kernel = axpby3_builder.build(slice.getDevice(),
+                                                         okl_defines);
             kernel((int)Size(), da, db, slice, xp->slice, yp->slice);
          }
       }

backends/occa/vector.hpp

@@ -25,12 +25,7 @@ namespace mfem
 namespace occa
 {
 
-// FIXME: Once XL fixes this code quirk we can remove this #ifdef switch
-#ifdef __ibmxl__
-class Vector : public Array, public PVector
-#else
 class Vector : virtual public Array, public PVector
-#endif
 {
 protected:
    //
@@ -57,10 +52,6 @@ protected:
    // End: Virtual interface
 
 public:
-   Vector(const Engine &e)
-      : PArray(*(new Layout(e, 0))), Array(e), PVector(*layout)
-   { }
-
    Vector(Layout &lt)
       : PArray(lt), Array(lt, sizeof(double)), PVector(lt)
    { }
@@ -70,7 +61,7 @@ public:
    const mfem::Vector Wrap() const;
 
 #if defined(MFEM_USE_MPI)
-   bool IsParallel() const { return (OccaEngine().GetComm() != MPI_COMM_NULL); }
+   bool IsParallel() const { return (OccaLayout().OccaEngine().GetComm() != MPI_COMM_NULL); }
 #endif
 };
 

backends/occa/vmass/tensor/cpu.okl

@@ -1,321 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 1D ]-----------------------------
-@kernel void Assemble1D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian1D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
-      oper(q, e) = quadWeights[q] * COEFF * J(q, e);
-    }
-  }
-}
-
-@kernel void MultAdd1D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal_t oper,
-                       @restrict const DVLocal1D_t solIn,
-                       @restrict DVLocal1D_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double sol_x[1][NUM_QUAD_1D];
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        sol_x[0][qx] = 0;
-      }
-      // sol_x{qx} = dofToQuad{qx,dx} * sol{dx}
-      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          sol_x[0][qx] += dofToQuad(qx, dx) * solIn(0, dx, e);
-        }
-      }
-      // sol_x{q} *= oper{q}
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        sol_x[0][qx] *= oper(qx, e);
-      }
-      // sol{dx} = quadToDof{dx,qx} * sol_x{qx}
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          solOut(0, dx, e) += sol_x[0][qx] * quadToDof(dx, qx);
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian2D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_2D; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-
-      oper(q, e) = quadWeights[q] * COEFF * ((J11 * J22) - (J21 * J12));
-    }
-  } // e
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal_t oper,
-                       @restrict const DVLocal2D_t solIn,
-                       @restrict DVLocal2D_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy=0; dummy<1; ++dummy; @inner) {
-      double sol_xy[2][NUM_QUAD_1D][NUM_QUAD_1D];
-
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          sol_xy[0][qx][qy] = 0;
-          sol_xy[1][qx][qy] = 0;
-        }
-      }
-
-      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-        double sol_x[2][NUM_QUAD_1D];
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          sol_x[0][qy] = 0;
-          sol_x[1][qy] = 0;
-        }
-
-        // sol_x{vd, dx, qy} = dofToQuad{qy, dy} * sol{vd, dx, dy}
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            sol_x[0][qy] += dofToQuad(qy, dx) * solIn(0, dx, dy, e);
-            sol_x[1][qy] += dofToQuad(qy, dx) * solIn(1, dx, dy, e);
-          }
-        }
-
-        // sol_xy{qx, qy} = dofToQuad{qx, dx} * sol_x{dx, qy}
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          const double d2q = dofToQuad(qy, dy);
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_xy[0][qx][qy] += d2q * sol_x[0][qx];
-            sol_xy[1][qx][qy] += d2q * sol_x[1][qx];
-          }
-        }
-      } // dy
-
-      // sol_xy{qx, qy} = sol_xy{q} *= oper{q, e}
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          const int q = QUAD_2D_ID(qx, qy);
-          sol_xy[0][qx][qy] *= oper(q, e);
-          sol_xy[1][qx][qy] *= oper(q, e);
-        }
-      }
-
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        double sol_x[2][NUM_DOFS_1D];
-        for (int dx = 0; dx < NUM_QUAD_1D; ++dx) {
-          sol_x[0][dx] = 0;
-          sol_x[1][dx] = 0;
-        }
-
-        // sol_x{qx, dy} = quadToDof{dy, qy} * sol_xy{qx, qy}
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            sol_x[0][dx] += quadToDof(dx, qx) * sol_xy[0][qx][qy];
-            sol_x[1][dx] += quadToDof(dx, qx) * sol_xy[1][qx][qy];
-          }
-        }
-
-        // sol{dx, dy, e} = quadToDof{dx, qx} * sol_x{qx, dy}
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          const double q2d = quadToDof(dy, qy);
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            solOut(0, dx, dy, e) += q2d * sol_x[0][dx];
-            solOut(1, dx, dy, e) += q2d * sol_x[1][dx];
-          }
-        }
-      }
-    } // dummy
-  } // e
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian3D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_3D; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-      oper(q, e) = quadWeights[q] * COEFF * detJ;
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal_t oper,
-                       @restrict const DVLocal3D_t solIn,
-                       @restrict DVLocal3D_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double sol_xyz[3][NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D];
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_xyz[0][qz][qy][qx] = 0;
-            sol_xyz[1][qz][qy][qx] = 0;
-            sol_xyz[2][qz][qy][qx] = 0;
-          }
-        }
-      }
-
-      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-        double sol_xy[3][NUM_QUAD_1D][NUM_QUAD_1D];
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_xy[0][qy][qx] = 0;
-            sol_xy[1][qy][qx] = 0;
-            sol_xy[2][qy][qx] = 0;
-          }
-        }
-
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          double sol_x[3][NUM_QUAD_1D];
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_x[0][qx] = 0;
-            sol_x[1][qx] = 0;
-            sol_x[2][qx] = 0;
-          }
-
-          // sol_x{qx} = dofToQuad{qx, dx} * sol{dx, dy, dz, e}
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              sol_x[0][qx] += dofToQuad(qx, dx) * solIn(0, dx, dy, dz, e);
-              sol_x[1][qx] += dofToQuad(qx, dx) * solIn(1, dx, dy, dz, e);
-              sol_x[2][qx] += dofToQuad(qx, dx) * solIn(2, dx, dy, dz, e);
-            }
-          }
-
-          // sol_xy{qx, qy} = dofToQuad{qy, dy} * sol_x{dx}
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            const double wy = dofToQuad(qy, dy);
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              sol_xy[0][qy][qx] += wy * sol_x[0][qx];
-              sol_xy[1][qy][qx] += wy * sol_x[1][qx];
-              sol_xy[2][qy][qx] += wy * sol_x[2][qx];
-            }
-          }
-        } // dy
-
-        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-          const double wz = dofToQuad(qz, dz);
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              sol_xyz[0][qz][qy][qx] += wz * sol_xy[0][qy][qx];
-              sol_xyz[1][qz][qy][qx] += wz * sol_xy[1][qy][qx];
-              sol_xyz[2][qz][qy][qx] += wz * sol_xy[2][qy][qx];
-            }
-          }
-        }
-      } // dz
-
-      // sol_xyz{qz, qy, qx} *= oper{q, e}
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            const int q = QUAD_3D_ID(qx, qy, qz);
-            sol_xyz[0][qz][qy][qx] *= oper(q, e);
-            sol_xyz[1][qz][qy][qx] *= oper(q, e);
-            sol_xyz[2][qz][qy][qx] *= oper(q, e);
-          }
-        }
-      }
-
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        double sol_xy[3][NUM_DOFS_1D][NUM_DOFS_1D];
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            sol_xy[0][dy][dx] = 0;
-            sol_xy[1][dy][dx] = 0;
-            sol_xy[2][dy][dx] = 0;
-          }
-        }
-
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          double sol_x[3][NUM_DOFS_1D];
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            sol_x[0][dx] = 0;
-            sol_x[1][dx] = 0;
-            sol_x[2][dx] = 0;
-          }
-
-          // sol_x{dx} = quadToDof{dx, qx} * sol_xyz{qz, qy, qx}
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              sol_x[0][dx] += quadToDof(dx, qx) * sol_xyz[0][qz][qy][qx];
-              sol_x[1][dx] += quadToDof(dx, qx) * sol_xyz[1][qz][qy][qx];
-              sol_x[2][dx] += quadToDof(dx, qx) * sol_xyz[2][qz][qy][qx];
-            }
-          }
-
-          // sol_xy{dy, dx} = quadToDof{dy, qy} * sol_x{dx}
-          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-            const double wy = quadToDof(dy, qy);
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              sol_xy[0][dy][dx] += wy * sol_x[0][dx];
-              sol_xy[1][dy][dx] += wy * sol_x[1][dx];
-              sol_xy[2][dy][dx] += wy * sol_x[2][dx];
-            }
-          }
-        } // qy
-
-        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-          const double wz = quadToDof(dz, qz);
-          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              solOut(0, dx, dy, dz, e) += wz * sol_xy[0][dy][dx];
-              solOut(1, dx, dy, dz, e) += wz * sol_xy[1][dy][dx];
-              solOut(2, dx, dy, dz, e) += wz * sol_xy[2][dy][dx];
-            }
-          }
-        }
-      } // qz
-    } // dummy
-  } // e
-
-}
-//======================================

backends/omp/adiffusioninteg.cpp

@@ -0,0 +1,675 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) &&               \
+   defined(MFEM_USE_OMP) &&                     \
+   defined(MFEM_USE_ACROTENSOR)
+
+#include "adiffusioninteg.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+PAIntegrator::PAIntegrator(Coefficient &q, FiniteElementSpace &f)
+{
+   Q = &q;
+   ofes = &f;
+   fes = ofes->GetFESpace();
+   onGPU = (ofes->OmpEngine().ExecTarget() == Device);
+   fe = fes->GetFE(0);
+   tfe = dynamic_cast<const TensorBasisElement*>(fe);
+   if (tfe)
+   {
+      tDofMap = tfe->GetDofMap();
+   }
+   else
+   {
+      tDofMap.SetSize(nDof);
+      for (int i = 0; i < nDof; ++i)
+      {
+         tDofMap[i] = i;
+      }
+   }
+
+   nElem  = fes->GetNE();
+   GeomType = fe->GetGeomType();
+   FEOrder = fe->GetOrder();
+   nDim    = fe->GetDim();
+   nDof   = fe->GetDof();
+
+   ElementTransformation *Trans = fes->GetElementTransformation(0);
+   int irorder = 2*fe->GetOrder() + Trans->OrderW();
+   ir = &IntRules.Get(GeomType, irorder);
+   nQuad = ir->GetNPoints();
+   hasTensorBasis = tfe ? true : false;
+
+   if (nDim > 3)
+   {
+      mfem_error("AcroIntegrator tensor computations don't support dim > 3.");
+   }
+}
+
+PAIntegrator::~PAIntegrator()
+{
+
+}
+
+AcroDiffusionIntegrator::AcroDiffusionIntegrator(Coefficient &q, FiniteElementSpace &f) :
+   PAIntegrator(q,f)
+{
+   if (onGPU)
+   {
+      //TE.SetExecutorType("OneOutPerThread");
+      TE.SetExecutorType("Cuda");
+      //TODO:  Set to an existing cuda context if one exists
+   }
+   else
+   {
+      TE.SetExecutorType("CPUInterpreted");
+   }
+
+   const IntegrationRule *ir1D = &IntRules.Get(Geometry::SEGMENT, ir->GetOrder());
+   nDof1D = FEOrder + 1;
+   nQuad1D = ir1D->GetNPoints();
+
+   if (hasTensorBasis)
+   {
+      H1_FECollection fec(FEOrder,1);
+      const FiniteElement *fe1D = fec.FiniteElementForGeometry(Geometry::SEGMENT);
+      mfem::Vector eval(nDof1D);
+      DenseMatrix deval(nDof1D,1);
+      B.Init(nQuad1D, nDof1D);
+      G.Init(nQuad1D, nDof1D);
+      std::vector<int> wdims(nDim, nQuad1D);
+      W.Init(wdims);
+
+      mfem::Vector w(nQuad1D);
+      for (int k = 0; k < nQuad1D; ++k)
+      {
+         const IntegrationPoint &ip = ir1D->IntPoint(k);
+         fe1D->CalcShape(ip, eval);
+         fe1D->CalcDShape(ip, deval);
+
+         B(k,0) = eval(0);
+         B(k,nDof1D-1) = eval(1);
+         G(k,0) = deval(0,0);
+         G(k,nDof1D-1) = deval(1,0);
+         for (int i = 1; i < nDof1D-1; ++i)
+         {
+            B(k,i) = eval(i+1);
+            G(k,i) = deval(i+1,0);
+         }
+         w(k) = ip.weight;
+      }
+
+      if (nDim == 1)
+      {
+         for (int k1 = 0; k1 < nQuad1D; ++k1)
+         {
+            W(k1) = w(k1);
+         }
+      }
+      else if (nDim == 2)
+      {
+         for (int k1 = 0; k1 < nQuad1D; ++k1)
+         {
+            for (int k2 = 0; k2 < nQuad1D; ++k2)
+            {
+               W(k1,k2) = w(k1)*w(k2);
+            }
+         }
+      }
+      else if (nDim == 3)
+      {
+         for (int k1 = 0; k1 < nQuad1D; ++k1)
+         {
+            for (int k2 = 0; k2 < nQuad1D; ++k2)
+            {
+               for (int k3 = 0; k3 < nQuad1D; ++k3)
+               {
+                  W(k1,k2,k3) = w(k1)*w(k2)*w(k3);
+               }
+            }
+         }
+      }
+   }
+   else
+   {
+      mfem::Vector eval(nDof);
+      DenseMatrix deval(nDof,nDim);
+      G.Init(nQuad, nDof,nDim);
+      W.Init(nQuad);
+      for (int k = 0; k < nQuad; ++k)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(k);
+         fe->CalcDShape(ip, deval);
+         for (int i = 0; i < nDof; ++i)
+         {
+            for (int d = 0; d < nDim; ++d)
+            {
+               G(k,i,d) = deval(i,d);
+            }
+         }
+         W(k) = ip.weight;
+      }
+   }
+
+   if (onGPU)
+   {
+      B.MapToGPU();
+      G.MapToGPU();
+      W.MapToGPU();
+   }
+
+   // Assemble in the constructor!
+   BatchedPartialAssemble();
+}
+
+
+AcroDiffusionIntegrator::~AcroDiffusionIntegrator()
+{
+   for (int i = 0; i < Btil.Size(); i++) delete Btil[i];
+}
+
+
+void AcroDiffusionIntegrator::ComputeBTilde()
+{
+   Btil.SetSize(nDim);
+   for (int d = 0; d < nDim; ++d)
+   {
+      Btil[d] = new acro::Tensor(nDim, nDim, nQuad1D, nDof1D, nDof1D);
+      for (int m = 0; m < nDim; ++m)
+      {
+         for (int n = 0; n < nDim; ++n)
+         {
+            acro::Tensor &BGM = (m == d) ? G : B;
+            acro::Tensor &BGN = (n == d) ? G : B;
+            for (int k = 0; k < nQuad1D; ++k)
+            {
+               for (int i = 0; i < nDof1D; ++i)
+               {
+                  for (int j = 0; j < nDof1D; ++j)
+                  {
+                     (*Btil[d])(m, n, k, i, j) = BGM(k,i)*BGN(k,j);
+                  }
+               }
+            }
+         }
+      }
+   }
+}
+
+
+void AcroDiffusionIntegrator::BatchedPartialAssemble()
+{
+   //Initilze the tensors
+   acro::Tensor J,Jinv,Jdet,C;
+   if (hasTensorBasis)
+   {
+      const IntegrationRule *ir1D = &IntRules.Get(Geometry::SEGMENT, ir->GetOrder());
+      IntegrationPoint ip;
+      if (nDim == 1)
+      {
+         D.Init(nElem, nDim, nDim, nQuad1D);
+         J.Init(nElem, nQuad1D, nDim, nDim);
+         Jinv.Init(nElem, nQuad1D, nDim, nDim);
+         Jdet.Init(nElem, nQuad1D);
+         C.Init(nElem, nQuad1D);
+
+         for (int e = 0; e < nElem; ++e)
+         {
+            ElementTransformation *Trans = fes->GetElementTransformation(e);
+            for (int k1 = 0; k1 < nQuad1D; ++k1)
+            {
+               ip.x = ir1D->IntPoint(k1).x;
+               ip.y = 0.0;
+               ip.z = 0.0;
+               Trans->SetIntPoint(&ip);
+               C(e,k1) = Q->Eval(*Trans, ip);
+               const DenseMatrix &JMat = Trans->Jacobian();
+               for (int m = 0; m < nDim; ++m)
+               {
+                  for (int n = 0; n < nDim; ++n)
+                  {
+                     J(e,k1,m,n) = JMat.Elem(m,n);
+                  }
+               }
+            }
+         }
+      }
+      else if (nDim == 2)
+      {
+         D.Init(nElem, nDim, nDim, nQuad1D, nQuad1D);
+         J.Init(nElem, nQuad1D, nQuad1D, nDim, nDim);
+         Jinv.Init(nElem, nQuad1D, nQuad1D, nDim, nDim);
+         Jdet.Init(nElem, nQuad1D, nQuad1D);
+         C.Init(nElem, nQuad1D, nQuad1D);
+
+         for (int e = 0; e < nElem; ++e)
+         {
+            ElementTransformation *Trans = fes->GetElementTransformation(e);
+            for (int k1 = 0; k1 < nQuad1D; ++k1)
+            {
+               for (int k2 = 0; k2 < nQuad1D; ++k2)
+               {
+                  ip.x = ir1D->IntPoint(k1).x;
+                  ip.y = ir1D->IntPoint(k2).y;
+                  ip.z = 0.0;
+                  Trans->SetIntPoint(&ip);
+                  C(e,k1,k2) = Q->Eval(*Trans, ip);
+                  const DenseMatrix &JMat = Trans->Jacobian();
+                  for (int m = 0; m < nDim; ++m)
+                  {
+                     for (int n = 0; n < nDim; ++n)
+                     {
+                        J(e,k1,k2,m,n) = JMat.Elem(m,n);
+                     }
+                  }
+               }
+            }
+         }
+      }
+      else if (nDim == 3)
+      {
+         D.Init(nElem, nDim, nDim, nQuad1D, nQuad1D, nQuad1D);
+         J.Init(nElem, nQuad1D, nQuad1D, nQuad1D, nDim, nDim);
+         Jinv.Init(nElem, nQuad1D, nQuad1D, nQuad1D, nDim, nDim);
+         Jdet.Init(nElem, nQuad1D, nQuad1D, nQuad1D);
+         C.Init(nElem, nQuad1D, nQuad1D, nQuad1D);
+
+         for (int e = 0; e < nElem; ++e)
+         {
+            ElementTransformation *Trans = fes->GetElementTransformation(e);
+            for (int k1 = 0; k1 < nQuad1D; ++k1)
+            {
+               for (int k2 = 0; k2 < nQuad1D; ++k2)
+               {
+                  for (int k3 = 0; k3 < nQuad1D; ++k3)
+                  {
+                     ip.x = ir1D->IntPoint(k1).x;
+                     ip.y = ir1D->IntPoint(k2).y;
+                     ip.z = ir1D->IntPoint(k3).z;
+                     Trans->SetIntPoint(&ip);
+                     C(e,k1,k2,k3) = Q->Eval(*Trans, ip);
+                     const DenseMatrix &JMat = Trans->Jacobian();
+                     for (int m = 0; m < nDim; ++m)
+                     {
+                        for (int n = 0; n < nDim; ++n)
+                        {
+                           J(e,k1,k2,k3,m,n) = JMat.Elem(m,n);
+                        }
+                     }
+                  }
+               }
+            }
+         }
+      }
+   }
+   else
+   {
+      D.Init(nElem, nDim, nDim, nQuad);
+      J.Init(nElem, nQuad, nDim, nDim);
+      Jinv.Init(nElem, nQuad, nDim, nDim);
+      Jdet.Init(nElem, nQuad);
+      C.Init(nElem, nQuad);
+
+      for (int e = 0; e < nElem; ++e)
+      {
+         ElementTransformation *Trans = fes->GetElementTransformation(e);
+         for (int k = 0; k < nQuad; ++k)
+         {
+            const IntegrationPoint &ip = ir->IntPoint(k);
+            Trans->SetIntPoint(&ip);
+            C(e,k) = Q->Eval(*Trans, ip);
+            const DenseMatrix &JMat = Trans->Jacobian();
+            for (int m = 0; m < nDim; ++m)
+            {
+               for (int n = 0; n < nDim; ++n)
+               {
+                  J(e,k,m,n) = JMat.Elem(m,n);
+               }
+            }
+         }
+      }
+   }
+
+   TE.BatchMatrixInvDet(Jinv, Jdet, J);
+
+   if (hasTensorBasis)
+   {
+      if (nDim == 1)
+      {
+         TE("D_e_m_n_k = W_k C_e_k Jdet_e_k Jinv_e_k_m_j Jinv_e_k_n_j",
+            D, W, C, Jdet, Jinv, Jinv);
+      }
+      else if (nDim == 2)
+      {
+         TE("D_e_m_n_k1_k2 = W_k1_k2 C_e_k1_k2 Jdet_e_k1_k2 Jinv_e_k1_k2_m_j Jinv_e_k1_k2_n_j",
+            D, W, C, Jdet, Jinv, Jinv);
+      }
+      else if (nDim == 3)
+      {
+         TE("D_e_m_n_k1_k2_k3 = W_k1_k2_k3 C_e_k1_k2_k3 Jdet_e_k1_k2_k3 Jinv_e_k1_k2_k3_n_j Jinv_e_k1_k2_k3_m_j",
+            D, W, C, Jdet, Jinv, Jinv);
+      }
+   }
+   else
+   {
+      TE("D_e_m_n_k = W_k C_e_k Jdet_e_k Jinv_e_k_m_j Jinv_e_k_n_j",
+         D, W, C, Jdet, Jinv, Jinv);
+   }
+}
+
+
+void AcroDiffusionIntegrator::BatchedAssembleElementMatrices(DenseTensor &elmats)
+{
+   if (hasTensorBasis && Btil.Size() == 0)
+   {
+      ComputeBTilde();
+   }
+
+   if (!D.IsInitialized())
+   {
+      BatchedPartialAssemble();
+   }
+
+   if (!S.IsInitialized())
+   {
+      if (hasTensorBasis)
+      {
+         if (nDim == 1)
+         {
+            S.Init(nElem, nDof1D, nDof1D);
+         }
+         else if (nDim == 2)
+         {
+            S.Init(nElem, nDof1D, nDof1D, nDof1D, nDof1D);
+         }
+         else if (nDim == 3)
+         {
+            S.Init(nElem, nDof1D, nDof1D, nDof1D, nDof1D, nDof1D, nDof1D);
+         }
+      }
+      else
+      {
+         S.Init(nElem, nDof, nDof);
+      }
+      if (onGPU) {S.SwitchToGPU();}
+   }
+
+
+   if (hasTensorBasis) {
+      if (nDim == 1) {
+         TE("S_e_i1_j1 = Btil_m_n_k1_i1_j1 D_e_m_n_k1",
+            S, *Btil[0], D);
+      }
+      else if (nDim == 2)
+      {
+         TE("S_e_i1_i2_j1_j2 = Btil1_m_n_k1_i1_j1 Btil2_m_n_k2_i2_j2 D_e_m_n_k1_k2",
+            S, *Btil[0], *Btil[1], D);
+      }
+      else if (nDim == 3)
+      {
+         TE("S_e_i1_i2_i3_j1_j2_j3 = Btil1_m_n_k1_i1_j1 Btil2_m_n_k2_i2_j2 Btil3_m_n_k3_i3_j3 D_e_m_n_k1_k2_k3",
+            S, *Btil[0], *Btil[1], *Btil[2], D);
+      }
+   }
+   else
+   {
+      TE("S_e_i_j = G_k_i_m G_k_i_n D_e_m_n_k",
+         S, G, G, D);
+   }
+
+   S.MoveFromGPU();
+   for (int e = 0; e < nElem; ++e)
+   {
+      for (int ei = 0; ei < nDof; ++ei)
+      {
+         for (int ej = 0; ej < nDof; ++ej)
+         {
+            elmats(tDofMap[ei], tDofMap[ej], e) = S[e*nDof*nDof + ei*nDof + ej];
+         }
+      }
+   }
+}
+
+
+void AcroDiffusionIntegrator::ComputeElementMatrices(Vector &elmats)
+{
+   if (hasTensorBasis && Btil.Size() == 0)
+   {
+      ComputeBTilde();
+   }
+
+   if (!D.IsInitialized())
+   {
+      BatchedPartialAssemble();
+   }
+
+   if (!S.IsInitialized())
+   {
+      if (hasTensorBasis)
+      {
+         if (nDim == 1)
+         {
+            S.Init(nElem, nDof1D, nDof1D);
+         }
+         else if (nDim == 2)
+         {
+            S.Init(nElem, nDof1D, nDof1D, nDof1D, nDof1D);
+         }
+         else if (nDim == 3)
+         {
+            S.Init(nElem, nDof1D, nDof1D, nDof1D, nDof1D, nDof1D, nDof1D);
+         }
+      }
+      else
+      {
+         S.Init(nElem, nDof, nDof);
+      }
+      if (onGPU) {S.SwitchToGPU();}
+   }
+
+
+   if (hasTensorBasis) {
+      if (nDim == 1) {
+         TE("S_e_i1_j1 += Btil_m_n_k1_i1_j1 D_e_m_n_k1",
+            S, *Btil[0], D);
+      }
+      else if (nDim == 2)
+      {
+         TE("S_e_i1_i2_j1_j2 += Btil1_m_n_k1_i1_j1 Btil2_m_n_k2_i2_j2 D_e_m_n_k1_k2",
+            S, *Btil[0], *Btil[1], D);
+      }
+      else if (nDim == 3)
+      {
+         TE("S_e_i1_i2_i3_j1_j2_j3 += Btil1_m_n_k1_i1_j1 Btil2_m_n_k2_i2_j2 Btil3_m_n_k3_i3_j3 D_e_m_n_k1_k2_k3",
+            S, *Btil[0], *Btil[1], *Btil[2], D);
+      }
+   }
+   else
+   {
+      TE("S_e_i_j += G_k_i_m G_k_i_n D_e_m_n_k",
+         S, G, G, D);
+   }
+
+   S.MoveFromGPU();
+
+   double *edata = elmats.GetData<double>();
+   for (int e = 0; e < nElem; ++e)
+   {
+      const int e_offset = e * nDof * nDof;
+      for (int ei = 0; ei < nDof; ++ei)
+      {
+         const int offset = e_offset + ei * tDofMap[ei] * nDof;
+         for (int ej = 0; ej < nDof; ++ej)
+         {
+            const int index = offset + tDofMap[ej];
+            edata[index] = S[e*nDof*nDof + ei*nDof + ej];
+         }
+      }
+   }
+}
+
+void AcroDiffusionIntegrator::ReassembleOperator()
+{
+   BatchedPartialAssemble();
+}
+
+void AcroDiffusionIntegrator::PAMult(const Vector &x, Vector &y)
+{
+   MFEM_ASSERT(hasTensorBasis,"AcroDiffusionIntegrator PAMult on simplices not supported");
+
+   if (!U.IsInitialized())
+   {
+      // NOTE: x and y are already sized for the fespace in the constructor
+      double *Xptr = const_cast<double*>(x.GetData<double>());
+      double *Yptr = y.GetData<double>();
+      if (nDim == 1) {
+         X.Init(nElem,nDof1D,Xptr,Xptr,onGPU);
+         Y.Init(nElem,nDof1D,Yptr,Yptr,onGPU);
+         U.Init(nDim, nElem, nQuad1D);
+         Z.Init(nDim, nElem, nQuad1D);
+         if (onGPU)
+         {
+            U.SwitchToGPU();
+            Z.SwitchToGPU();
+         }
+      }
+      else if (nDim == 2)
+      {
+         X.Init(nElem,nDof1D,nDof1D,Xptr,Xptr,onGPU);
+         Y.Init(nElem,nDof1D,nDof1D,Yptr,Yptr,onGPU);
+         U.Init(nDim, nElem, nQuad1D, nQuad1D);
+         Z.Init(nDim, nElem, nQuad1D, nQuad1D);
+         T1.Init(nElem,nDof1D,nQuad1D);
+         if (onGPU)
+         {
+            U.SwitchToGPU();
+            Z.SwitchToGPU();
+            T1.SwitchToGPU();
+         }
+      }
+      else if (nDim == 3)
+      {
+         X.Init(nElem,nDof1D,nDof1D,nDof1D,Xptr,Xptr,onGPU);
+         Y.Init(nElem,nDof1D,nDof1D,nDof1D,Yptr,Yptr,onGPU);
+         U.Init(nDim, nElem, nQuad1D, nQuad1D, nQuad1D);
+         Z.Init(nDim, nElem, nQuad1D, nQuad1D, nQuad1D);
+         T1.Init(nElem, nDof1D, nQuad1D, nQuad1D);
+         T2.Init(nElem, nDof1D, nDof1D, nQuad1D);
+         if (onGPU)
+         {
+            U.SwitchToGPU();
+            Z.SwitchToGPU();
+            T1.SwitchToGPU();
+            T2.SwitchToGPU();
+         }
+      }
+   }
+   else
+   {
+      // NOTE: x and y are already sized for the fespace in the constructor
+      double *Xptr = const_cast<double*>(x.GetData<double>());
+      double *Yptr = y.GetData<double>();
+      X.Retarget(Xptr,Xptr);
+      Y.Retarget(Yptr,Yptr);
+   }
+
+   acro::SliceTensor U1,U2,U3,Z1,Z2,Z3;
+   if (nDim == 1)
+   {
+      TE("U_n_e_k1 = G_k1_i1 X_e_i1", U, G, X);
+      TE("Z_m_e_k1 = D_e_m_n_k1 U_n_e_k1", Z, D, U);
+      TE("Y_e_i1 = G_k1_i1 Z_m_e_k1", Y, G, Z);
+   }
+   else if (nDim == 2)
+   {
+      U1.SliceInit(U, 0); U2.SliceInit(U, 1);
+      Z1.SliceInit(Z, 0); Z2.SliceInit(Z, 1);
+
+      //U1_e_k1_k2 = G_k1_i1 B_k2_i2 X_e_i1_i2
+      TE("BX_e_i1_k2 = B_k2_i2 X_e_i2_i1", T1, B, X);
+      TE("U1_e_k1_k2 = G_k1_i1 BX_e_i1_k2", U1, G, T1);
+
+      //U2_e_k1_k2 = B_k1_i1 G_k2_i2 X_e_i1_i2
+      TE("GX_e_i1_k2 = G_k2_i2 X_e_i2_i1", T1, G, X);
+      TE("U2_e_k1_k2 = B_k1_i1 GX_e_i1_k2", U2, B, T1);
+
+      TE("Z_m_e_k1_k2 = D_e_m_n_k1_k2 U_n_e_k1_k2", Z, D, U);
+
+      //Y_e_i1_i2 = G_k1_i1 B_k2_i2 Z1_e_k1_k2
+      TE("BZ1_e_i2_k1 = B_k2_i2 Z1_e_k1_k2", T1, B, Z1);
+      TE("Y_e_i2_i1 = G_k1_i1 BZ1_e_i2_k1", Y, G, T1);
+
+      //Y_e_i1_i2 += B_k1_i1 G_k2_i2 Z2_e_k1_k2
+      TE("GZ2_e_i2_k1 = G_k2_i2 Z2_e_k1_k2", T1, G, Z2);
+      TE("Y_e_i2_i1 += B_k1_i1 GZ2_e_i2_k1", Y, B, T1);
+   }
+   else if (nDim == 3)
+   {
+      U1.SliceInit(U, 0); U2.SliceInit(U, 1); U3.SliceInit(U, 2);
+      Z1.SliceInit(Z, 0); Z2.SliceInit(Z, 1); Z3.SliceInit(Z, 2);
+
+      TE.BeginMultiKernelLaunch();
+      //U1_e_k1_k2_k3 = G_k1_i1 B_k2_i2 B_k3_i3 X_e_i1_i2_i3
+      TE("T2_e_i1_i2_k3 = B_k3_i3 X_e_i1_i2_i3", T2, B, X);
+      TE("T1_e_i1_k2_k3 = B_k2_i2 T2_e_i1_i2_k3", T1, B, T2);
+      TE("U1_e_k1_k2_k3 = G_k1_i1 T1_e_i1_k2_k3", U1, G, T1);
+
+      //U2_e_k1_k2_k3 = B_k1_i1 G_k2_i2 B_k3_i3 X_e_i1_i2_i3
+      TE("T1_e_i1_k2_k3 = G_k2_i2 T2_e_i1_i2_k3", T1, G, T2);
+      TE("U2_e_k1_k2_k3 = B_k1_i1 T1_e_i1_k2_k3", U2, B, T1);
+
+      //U3_e_k1_k2_k3 = B_k1_i1 B_k2_i2 G_k3_i3 X_e_i1_i2_i3
+      TE("T2_e_i1_i2_k3 = G_k3_i3 X_e_i1_i2_i3", T2, G, X);
+      TE("T1_e_i1_k2_k3 = B_k2_i2 T2_e_i1_i2_k3", T1, B, T2);
+      TE("U3_e_k1_k2_k3 = B_k1_i1 T1_e_i1_k2_k3", U3, B, T1);
+
+      TE("Z_m_e_k1_k2_k3 = D_e_m_n_k1_k2_k3 U_n_e_k1_k2_k3", Z, D, U);
+
+      //Y_e_i1_i2_i3 =  G_k1_i1 B_k2_i2 B_k3_i3 Z1_e_k1_k2_k3
+      TE("T1_e_i3_k1_k2 = B_k3_i3 Z1_e_k1_k2_k3", T1, B, Z1);
+      TE("T2_e_i2_i3_k1 = B_k2_i2 T1_e_i3_k1_k2", T2, B, T1);
+      TE("Y_e_i1_i2_i3 = G_k1_i1 T2_e_i2_i3_k1", Y, G, T2);
+
+      //Y_e_i1_i2_i3 +=  B_k1_i1 G_k2_i2 B_k3_i3 Z2_e_k1_k2_k3
+      TE("T1_e_i3_k1_k2 = B_k3_i3 Z2_e_k1_k2_k3", T1, B, Z2);
+      TE("T2_e_i2_i3_k1 = G_k2_i2 T1_e_i3_k1_k2", T2, G, T1);
+      TE("Y_e_i1_i2_i3 += B_k1_i1 T2_e_i2_i3_k1", Y, B, T2);
+
+      //Y_e_i1_i2_i3 +=  B_k1_i1 B_k2_i2 G_k3_i3 Z3_e_k1_k2_k3
+      TE("T1_e_i3_k1_k2 = G_k3_i3 Z3_e_k1_k2_k3", T1, G, Z3);
+      TE("T2_e_i2_i3_k1 = B_k2_i2 T1_e_i3_k1_k2", T2, B, T1);
+      TE("Y_e_i1_i2_i3 += B_k1_i1 T2_e_i2_i3_k1", Y, B, T2);
+      TE.EndMultiKernelLaunch();
+   }
+}
+
+void AcroDiffusionIntegrator::MultAdd(const Vector &x, Vector &y) const
+{
+   const_cast<AcroDiffusionIntegrator*>(this)->PAMult(x, y);
+}
+
+void AcroDiffusionIntegrator::MultTransposeAdd(const Vector &x, Vector &y) const
+{
+   mfem_error("Not supported");
+}
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif

backends/omp/adiffusioninteg.hpp

@@ -0,0 +1,95 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_ADIFFUSIONINTEG_HPP
+#define MFEM_BACKENDS_OMP_ADIFFUSIONINTEG_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) &&               \
+   defined(MFEM_USE_OMP) &&                     \
+   defined(MFEM_USE_ACROTENSOR)
+
+#include "../../fem/bilininteg.hpp"
+#include "../../fem/fem.hpp"
+#include "vector.hpp"
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+#include "AcroTensor.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+class PAIntegrator : public TensorBilinearFormIntegrator
+{
+protected:
+   Coefficient *Q;
+   FiniteElementSpace *ofes;
+   mfem::FiniteElementSpace *fes;
+   const FiniteElement *fe;
+   const TensorBasisElement *tfe;
+   const IntegrationRule *ir;
+   mfem::Array<int> tDofMap;
+   int GeomType;
+   int FEOrder;
+   bool onGPU;
+   bool hasTensorBasis;
+   int nDim;
+   int nElem;
+   int nDof;
+   int nQuad;
+
+public:
+   PAIntegrator(Coefficient &q, FiniteElementSpace &f);
+   virtual ~PAIntegrator();
+};
+
+class AcroDiffusionIntegrator : public PAIntegrator
+{
+private:
+   acro::TensorEngine TE;
+   int nDof1D;
+   int nQuad1D;
+
+   acro::Tensor B, G;         //Basis and dbasis evaluated on the quad points
+   acro::Tensor W;            //Integration weights
+   mfem::Array<acro::Tensor*> Btil; //Btilde used to compute stiffness matrix
+   acro::Tensor D;            //Product of integration weight, physical consts, and element shape info
+   acro::Tensor S;            //The assembled local stiffness matrices
+   acro::Tensor U, Z, T1, T2; //Intermediate computations for tensor product partial assembly
+   acro::Tensor X, Y;
+
+   void ComputeBTilde();
+
+public:
+   AcroDiffusionIntegrator(BilinearFormIntegrator *integ);
+   AcroDiffusionIntegrator(Coefficient &q, FiniteElementSpace &f);
+   virtual ~AcroDiffusionIntegrator();
+
+   void BatchedPartialAssemble();
+   void BatchedAssembleElementMatrices(DenseTensor &elmats);
+   void ComputeElementMatrices(Vector &elmats);
+   void PAMult(const Vector &x, Vector &y);
+   virtual void MultTransposeAdd(const Vector &x, Vector &y) const;
+   virtual void MultAdd(const Vector &x, Vector &y) const;
+   virtual void ReassembleOperator();
+};
+
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif
+
+#endif

backends/omp/array.cpp

@@ -0,0 +1,128 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include <cstring>
+#include "array.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+PArray *Array::DoClone(bool copy_data, void **buffer,
+                       std::size_t item_size) const
+{
+   Array *new_array = new Array(OmpLayout(), item_size);
+   if (copy_data)
+   {
+      if (!ComputeOnDevice())
+         std::memcpy(new_array->GetData<void>(), data, bytes);
+      else
+      {
+         char *new_data = new_array->GetData<char>();
+         const bool use_target = ComputeOnDevice();
+         const bool use_parallel = Size() > 1000;
+#pragma omp target teams distribute parallel for       \
+   if (target: use_target) if (parallel: use_parallel) \
+   is_device_ptr(new_data)
+         for (std::size_t i = 0; i < bytes; i++) new_data[i] = data[i];
+      }
+   }
+   if (buffer)
+   {
+      *buffer = new_array->GetData<void>();
+   }
+   return new_array;
+}
+
+int Array::DoResize(PLayout &new_layout, void **buffer,
+                    std::size_t item_size)
+{
+   MFEM_ASSERT(dynamic_cast<Layout *>(&new_layout) != NULL,
+               "new_layout is not an OMP Layout");
+   Layout *lt = static_cast<Layout *>(&new_layout);
+   layout.Reset(lt); // Reset() checks if the pointer is the same
+   int err = ResizeData(lt, item_size);
+   if (!err && buffer)
+   {
+      *buffer = GetData<void>();
+   }
+   return err;
+}
+
+void *Array::DoPullData(void *buffer, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   if (!IsUnifiedMemory() && ComputeOnDevice() && (buffer != NULL))
+   {
+#pragma omp target update from(data)
+      std::memcpy(buffer, data, bytes);
+   }
+   else
+   {
+      buffer = data;
+   }
+
+   return buffer;
+}
+
+void Array::DoFill(const void *value_ptr, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   switch (item_size)
+   {
+   case sizeof(int):
+      OmpFill((const int *)value_ptr);
+      break;
+   case sizeof(double):
+      OmpFill((const double *)value_ptr);
+      break;
+   default:
+      MFEM_ABORT("item_size = " << item_size << " is not supported");
+   }
+}
+
+void Array::DoPushData(const void *src_buffer, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   std::memcpy(data, (char *) src_buffer, bytes);
+
+   if ((!IsUnifiedMemory() && ComputeOnDevice()) && (data != src_buffer))
+   {
+#pragma omp target update to(data)
+   }
+}
+
+void Array::DoAssign(const PArray &src, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   // Note: static_cast can not be used here since PArray is a virtual base
+   //       class.
+   const Array *source = dynamic_cast<const Array *>(&src);
+   MFEM_ASSERT(source != NULL, "invalid source Array type");
+   MFEM_ASSERT(Size() == source->Size(), "");
+   // All arrays from this engine are of the same type, so we can simply check *this and assume the same is used in src.
+   DoPushData(source->GetData<void>(), item_size);
+}
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/array.hpp

@@ -0,0 +1,143 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_ARRAY_HPP
+#define MFEM_BACKENDS_OMP_ARRAY_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "layout.hpp"
+#include "../base/array.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+class Array : public virtual mfem::PArray
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // DLayout layout;
+
+   bool own_data;
+   std::size_t bytes;
+   char *data;
+
+   //
+   // Virtual interface
+   //
+
+   virtual void *DoGetData() const { return (void *) data; }
+
+   virtual PArray *DoClone(bool copy_data, void **buffer,
+                           std::size_t item_size) const;
+
+   virtual int DoResize(PLayout &new_layout, void **buffer,
+                        std::size_t item_size);
+
+   virtual void *DoPullData(void *buffer, std::size_t item_size);
+
+   virtual void DoFill(const void *value_ptr, std::size_t item_size);
+
+   virtual void DoPushData(const void *src_buffer, std::size_t item_size);
+
+   virtual void DoAssign(const PArray &src, std::size_t item_size);
+
+   //
+   // Auxiliary methods
+   //
+
+   inline int ResizeData(const Layout *lt, std::size_t item_size);
+
+   inline bool IsUnifiedMemory() const { return OmpLayout().OmpEngine().UnifiedMemory(); }
+
+   template <typename T>
+   void OmpFill(const T *pval)
+   {
+      T *ptr = (T*) data;
+      T val = *pval;
+      const bool use_target = ComputeOnDevice();
+      const bool use_parallel = (use_target || layout->Size() > 1000);
+      const std::size_t size = layout->Size();
+#pragma omp target teams distribute parallel for        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel) map (to: ptr, val)
+      for (int i = 0; i < size; i++) ptr[i] = val;
+   }
+
+public:
+   Array(Layout &lt, std::size_t item_size)
+      : PArray(lt),
+        own_data(true),
+        bytes(lt.Size() * item_size),
+        data(static_cast<char *>(lt.Alloc(bytes)))
+   {
+#pragma omp target enter data map(alloc:data[:bytes]) if (!IsUnifiedMemory() && ComputeOnDevice())
+   }
+
+   Array(const Array &array)
+      : PArray(array.GetLayout()),
+        own_data(false),
+        bytes(array.bytes),
+        data(array.data) { }
+
+   inline bool ComputeOnDevice() const { return (OmpLayout().OmpEngine().ExecTarget() == Device); }
+
+   virtual ~Array()
+   {
+#pragma omp target exit data map(delete:data[:bytes]) if (!IsUnifiedMemory() && ComputeOnDevice())
+      if (own_data) layout->As<Layout>().Dealloc(data);
+   }
+
+   inline void MakeRef(Array &master);
+
+   Layout &OmpLayout() const
+   { return *static_cast<Layout *>(layout.Get()); }
+};
+
+
+//
+// Inline methods
+//
+
+inline int Array::ResizeData(const Layout *lt, std::size_t item_size)
+{
+   const std::size_t new_bytes = lt->Size() * item_size;
+   if (bytes < new_bytes)
+   {
+#pragma omp target exit data map(delete:data)
+      OmpLayout().Dealloc(data);
+      data = static_cast<char *>(OmpLayout().Alloc(new_bytes));
+      MFEM_VERIFY(data != NULL, "");
+      // If memory allocation fails - an exception is thrown.
+#pragma omp target enter data map(alloc:data[:new_bytes])
+   }
+   return 0;
+}
+
+inline void Array::MakeRef(Array &master)
+{
+   layout = master.layout;
+   data = master.data;
+}
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#endif // MFEM_BACKENDS_OMP_ARRAY_HPP

backends/omp/backend.cpp

@@ -0,0 +1,46 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "backend.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+bool Backend::Supports(const std::string &engine_spec) const
+{
+   return true;
+}
+
+mfem::Engine *Create(const std::string &engine_spec)
+{
+   return new Engine(engine_spec);
+}
+
+#ifdef MFEM_USE_MPI
+mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec)
+{
+   return new Engine(comm, engine_spec);
+}
+#endif
+
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/backend.hpp

@@ -0,0 +1,48 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_BACKEND_HPP
+#define MFEM_BACKENDS_OMP_BACKEND_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+// Only the Backend and Engine classes should be exposed through "backend.hpp"
+#include "../base/backend.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+class Backend : public mfem::Backend
+{
+public:
+   virtual ~Backend();
+
+   virtual bool Supports(const std::string &engine_spec) const;
+
+   virtual mfem::Engine *Create(const std::string &engine_spec);
+
+#ifdef MFEM_USE_MPI
+   virtual mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec);
+#endif
+};
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#endif // MFEM_BACKENDS_OMP_BACKEND_HPP

backends/omp/bilinearform.cpp

@@ -0,0 +1,399 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "backend.hpp"
+#include "bilinearform.hpp"
+#include "adiffusioninteg.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+BilinearForm::~BilinearForm()
+{
+   // Make sure all integrators free their data
+   for (int i = 0; i < tbfi.Size(); i++) delete tbfi[i];
+
+   delete element_matrices;
+}
+
+void BilinearForm::TransferIntegrators()
+{
+   mfem::Array<mfem::BilinearFormIntegrator*> &dbfi = *bform->GetDBFI();
+   for (int i = 0; i < dbfi.Size(); i++)
+   {
+      std::string integ_name(dbfi[i]->Name());
+      Coefficient *scal_coeff = dbfi[i]->GetScalarCoefficient();
+      // ConstantCoefficient *const_coeff =
+      //    dynamic_cast<ConstantCoefficient*>(scal_coeff);
+      // // TODO: other types of coefficients ...
+      // double val = const_coeff ? const_coeff->constant : 1.0;
+
+      if (integ_name == "(undefined)")
+      {
+         MFEM_ABORT("BilinearFormIntegrator does not define Name()");
+      }
+      else if (integ_name == "diffusion")
+      {
+         switch (OmpEngine().IntegType())
+         {
+         case Acrotensor:
+            tbfi.Append(new AcroDiffusionIntegrator(*scal_coeff, bform->FESpace()->Get_PFESpace()->As<FiniteElementSpace>()));
+            break;
+         default:
+            mfem_error("integrator is not supported for any MultType");
+            break;
+         }
+      }
+      else
+      {
+         MFEM_ABORT("BilinearFormIntegrator [Name() = " << integ_name
+                    << "] is not supported");
+      }
+   }
+}
+
+void BilinearForm::InitRHS(const mfem::Array<int> &ess_tdof_list,
+                           mfem::Vector &mfem_x, mfem::Vector &mfem_b,
+                           mfem::OperatorHandle &A,
+                           mfem::Vector &mfem_X, mfem::Vector &mfem_B,
+                           int copy_interior) const
+{
+   const mfem::Operator *P = GetProlongation();
+   const mfem::Operator *R = GetRestriction();
+
+   if (P)
+   {
+      // Variational restriction with P
+      mfem_B.Resize(P->InLayout());
+      P->MultTranspose(mfem_b, mfem_B);
+      mfem_X.Resize(R->OutLayout());
+      R->Mult(mfem_x, mfem_X);
+   }
+   else
+   {
+      // rap, X and B point to the same data as this, x and b
+      mfem_X.MakeRef(mfem_x);
+      mfem_B.MakeRef(mfem_b);
+   }
+
+   if (A.Type() != mfem::Operator::ANY_TYPE)
+   {
+      A.EliminateBC(mat_e, ess_tdof_list, mfem_X, mfem_B);
+   }
+
+   if (!copy_interior && ess_tdof_list.Size() > 0)
+   {
+      Vector &X = mfem_X.Get_PVector()->As<Vector>();
+      const Array &constraint_list = ess_tdof_list.Get_PArray()->As<Array>();
+
+      double *X_data = X.GetData<double>();
+      const int* constraint_data = constraint_list.GetData<int>();
+
+      Vector subvec(constraint_list.OmpLayout());
+      double *subvec_data = subvec.GetData<double>();
+
+      const std::size_t num_constraint = constraint_list.Size();
+      const bool use_target = constraint_list.ComputeOnDevice();
+      const bool use_parallel = (use_target || num_constraint > 1000);
+
+      // This operation is a general version of mfem::Vector::SetSubVectorComplement()
+      // {
+#pragma omp target teams distribute parallel for        \
+   map(to: subvec_data, constraint_data, X_data)        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel)
+      for (std::size_t i = 0; i < num_constraint; i++) subvec_data[i] = X_data[constraint_data[i]];
+
+      X.Fill(0.0);
+
+#pragma omp target teams distribute parallel for        \
+   map(to: X_data, constraint_data, subvec_data)        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel)
+      for (std::size_t i = 0; i < num_constraint; i++) X_data[constraint_data[i]] = subvec_data[i];
+      // }
+   }
+
+   if (A.Type() == mfem::Operator::ANY_TYPE)
+   {
+      ConstrainedOperator *A_constrained = static_cast<ConstrainedOperator*>(A.Ptr());
+      A_constrained->EliminateRHS(mfem_X, mfem_B);
+   }
+}
+
+
+bool BilinearForm::Assemble()
+{
+   if (!has_assembled)
+   {
+      TransferIntegrators();
+      has_assembled = true;
+   }
+
+   return true;
+}
+
+void BilinearForm::ComputeElementMatrices()
+{
+   // Only called if performing full assembly
+   const int nelements = trial_fes->GetFESpace()->GetNE();
+   const int trial_ndofs = trial_fes->GetFESpace()->GetFE(0)->GetDof() * trial_fes->GetFESpace()->GetVDim();
+   const int test_ndofs = test_fes->GetFESpace()->GetFE(0)->GetDof() * test_fes->GetFESpace()->GetVDim();
+   const std::size_t length = nelements * trial_ndofs * test_ndofs;
+
+   if (!element_matrices) element_matrices = new mfem::Vector(*(new Layout(OmpEngine(), length)));
+   else element_matrices->Push();
+
+   element_matrices->Fill(0.0);
+   Vector &elmats = element_matrices->Get_PVector()->As<Vector>();
+
+   tbfi[0]->ComputeElementMatrices(elmats);
+
+   if (tbfi.Size() > 1)
+   {
+      for (int k = 1; k < tbfi.Size(); k++)
+      {
+         tbfi[k]->ComputeElementMatrices(elmats);
+      }
+   }
+}
+
+void BilinearForm::FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
+                                    mfem::OperatorHandle &A)
+{
+   if (A.Type() == mfem::Operator::ANY_TYPE)
+   {
+      // FIXME: Support different test and trial spaces (MixedBilinearForm)
+      const mfem::Operator *P = GetProlongation();
+
+      mfem::Operator *rap = this;
+      if (P != NULL) rap = new mfem::RAPOperator(*P, *this, *P);
+
+      A.Reset(new ConstrainedOperator(rap, ess_tdof_list, (rap != this)));
+
+      return;
+   }
+   else
+   {
+      // ASSUMPTION: some sort of sparse matrix
+      // Compute the local matrices (stored in bform->element_matrices
+      ComputeElementMatrices();
+      bform->AllocateMatrix();
+      mfem::SparseMatrix &mat = bform->SpMat();
+
+      element_matrices->Pull();
+      double *data = element_matrices->GetData();
+
+      const bool skip_zeros = true;
+      mfem::Array<int> tr_vdofs, te_vdofs;
+      for (int i = 0; i < trial_fes->GetFESpace()->GetNE(); i++)
+      {
+         trial_fes->GetFESpace()->GetElementVDofs(i, tr_vdofs);
+         test_fes->GetFESpace()->GetElementVDofs(i, te_vdofs);
+         const mfem::DenseMatrix elmat(data, te_vdofs.Size(), tr_vdofs.Size());
+         mat.AddSubMatrix(te_vdofs, tr_vdofs, elmat, skip_zeros);
+         data += tr_vdofs.Size() * te_vdofs.Size();
+      }
+   }
+
+   if (A.Type() == mfem::Operator::MFEM_SPARSEMAT)
+   {
+      // This works because the FormSystemMatrix call with an explicit
+      // SparseMatrix doesnt call the backend version... This might
+      // change in the future.
+      bform->FormSystemMatrix(ess_tdof_list, static_cast<mfem::SparseMatrix&>(*A.Ptr()));
+   }
+#ifdef MFEM_USE_MPI
+   else if (A.Type() == mfem::Operator::Hypre_ParCSR)
+   {
+      mfem::SparseMatrix &mat = bform->SpMat();
+      mfem::ParBilinearForm *pbform = dynamic_cast<mfem::ParBilinearForm*>(bform);
+
+      const bool skip_zeros = false;
+      mat.Finalize(skip_zeros);
+
+      // -------- FOR SOME VERY AGGREVATING REASON THIS DOESN'T WORK ---------
+      // mfem::ParFiniteElementSpace *pfes = pbform->ParFESpace();
+      // OperatorHandle dA(Operator::Hypre_ParCSR);
+      // // construct a parallel block-diagonal matrix 'A' based on 'a'
+      // dA.MakeSquareBlockDiag(pfes->GetComm(), *engine->MakeLayout(pfes->GlobalTrueVSize()),
+      //                        pfes->GetDofOffsets(), &mat);
+      // OperatorHandle Ph(pfes->Dof_TrueDof_Matrix());
+      // A.MakePtAP(dA, Ph);
+      // A.SetOperatorOwner(false);
+      // -------- BUT THIS DOES ---------
+      pbform->ParallelAssemble(A, &mat);
+      A.SetOperatorOwner(false);
+      // ---------------------
+      mat.Clear();
+      mat_e.Clear();
+      std::cout << "operator size (FormSystemMatrix): " << A.Ptr()->InLayout()->Size() << " " << A.Ptr()->OutLayout()->Size() << std::endl;
+
+      mat_e.EliminateRowsCols(A, ess_tdof_list);
+   }
+#endif
+   else
+   {
+      MFEM_ABORT("Operator::Type is not supported, type = " << A.Type());
+   }
+}
+
+void BilinearForm::FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
+                                    mfem::Vector &x, mfem::Vector &b,
+                                    mfem::OperatorHandle &A, mfem::Vector &X, mfem::Vector &B,
+                                    int copy_interior)
+{
+   FormSystemMatrix(ess_tdof_list, A);
+   std::cout << "operator size (FormLinearSystem 1): " << A.Ptr()->InLayout()->Size() << " " << A.Ptr()->OutLayout()->Size() << std::endl;
+   InitRHS(ess_tdof_list, x, b, A, X, B, copy_interior);
+}
+
+void BilinearForm::RecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
+                                      mfem::Vector &x)
+{
+   const mfem::Operator *P = GetProlongation();
+   if (P)
+   {
+      // Apply conforming prolongation
+      x.Resize(P->OutLayout());
+      P->Mult(X, x);
+   }
+   // Otherwise X and x point to the same data
+}
+
+void BilinearForm::Mult(const mfem::Vector &x, mfem::Vector &y) const
+{
+   trial_fes->ToEVector(x.Get_PVector()->As<Vector>(), x_local);
+
+   y_local.Fill<double>(0.0);
+   for (int i = 0; i < tbfi.Size(); i++) tbfi[i]->MultAdd(x_local, y_local);
+
+   test_fes->ToLVector(y_local, y.Get_PVector()->As<Vector>());
+}
+
+void BilinearForm::MultTranspose(const mfem::Vector &x, mfem::Vector &y) const
+{ mfem_error("mfem::omp::BilinearForm::MultTranspose() is not supported!"); }
+
+
+ConstrainedOperator::ConstrainedOperator(mfem::Operator *A_,
+                                         const mfem::Array<int> &constraint_list_,
+                                         bool own_A_)
+   : Operator(A_->InLayout()->As<Layout>()),
+     A(A_),
+     own_A(own_A_),
+     // FIXME: @dudouit1 has a general fix for this
+     constraint_list(constraint_list_.Get_PArray()->As<Array>()),
+     z(OutLayout()->As<Layout>()),
+     w(OutLayout()->As<Layout>()),
+     mfem_z((z.DontDelete(), z)),
+     mfem_w((w.DontDelete(), w)) { }
+
+void ConstrainedOperator::EliminateRHS(const mfem::Vector &mfem_x, mfem::Vector &mfem_b) const
+{
+   w.Fill<double>(0.0);
+
+   const Vector &x = mfem_x.Get_PVector()->As<Vector>();
+   Vector &b = mfem_b.Get_PVector()->As<Vector>();
+
+   const double *x_data = x.GetData<double>();
+   double *b_data = b.GetData<double>();
+   double *w_data = w.GetData<double>();
+   const int* constraint_data = constraint_list.GetData<int>();
+
+   const std::size_t num_constraint = constraint_list.Size();
+   const bool use_target = constraint_list.ComputeOnDevice();
+   const bool use_parallel = (use_target || num_constraint > 1000);
+
+   if (num_constraint > 0)
+   {
+#pragma omp target teams distribute parallel for             \
+   map(to: w_data, constraint_data, x_data)                  \
+   if (target: use_target)                                   \
+   if (parallel: use_parallel)
+      for (std::size_t i = 0; i < num_constraint; i++)
+         w_data[constraint_data[i]] = x_data[constraint_data[i]];
+   }
+
+   A->Mult(mfem_w, mfem_z);
+
+   b.Axpby<double>(1.0, b, -1.0, z);
+
+   if (num_constraint > 0)
+   {
+#pragma omp target teams distribute parallel for        \
+   map(to: b_data, constraint_data, x_data)             \
+   if (target: use_target)                              \
+   if (parallel: use_parallel)
+      for (std::size_t i = 0; i < num_constraint; i++)
+         b_data[constraint_data[i]] = x_data[constraint_data[i]];
+   }
+}
+
+void ConstrainedOperator::Mult(const mfem::Vector &mfem_x, mfem::Vector &mfem_y) const
+{
+   if (constraint_list.Size() == 0)
+   {
+      A->Mult(mfem_x, mfem_y);
+      return;
+   }
+
+   const Vector &x = mfem_x.Get_PVector()->As<Vector>();
+   Vector &y = mfem_y.Get_PVector()->As<Vector>();
+
+   const double *x_data = x.GetData<double>();
+   double *y_data = y.GetData<double>();
+   double *z_data = z.GetData<double>();
+   const int* constraint_data = constraint_list.GetData<int>();
+
+   const std::size_t num_constraint = constraint_list.Size();
+   const bool use_target = constraint_list.ComputeOnDevice();
+   const bool use_parallel = (use_target || num_constraint > 1000);
+
+   z.Assign<double>(x); // z = x
+
+   // z[constraint_list] = 0.0
+#pragma omp target teams distribute parallel for             \
+   map(to: z_data, constraint_data)                          \
+   if (target: use_target)                                   \
+   if (parallel: use_parallel)
+   for (std::size_t i = 0; i < num_constraint; i++)
+      z_data[constraint_data[i]] = 0.0;
+
+   // y = A * z
+   A->Mult(mfem_z, mfem_y);
+
+   // y[constraint_list] = x[constraint_list]
+#pragma omp target teams distribute parallel for             \
+   map(to: y_data, constraint_data, x_data)                  \
+   if (target: use_target)                                   \
+   if (parallel: use_parallel)
+   for (std::size_t i = 0; i < num_constraint; i++)
+      y_data[constraint_data[i]] = x_data[constraint_data[i]];
+}
+
+// Destructor: destroys the unconstrained Operator @a A if @a own_A is true.
+ConstrainedOperator::~ConstrainedOperator()
+{
+   if (own_A) delete A;
+}
+
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/bilinearform.hpp

@@ -0,0 +1,176 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_BILINEARFORM_HPP
+#define MFEM_BACKENDS_OMP_BILINEARFORM_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "fespace.hpp"
+#include "array.hpp"
+#include "vector.hpp"
+#include "../../fem/bilininteg.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+class TensorBilinearFormIntegrator
+{
+public:
+   virtual ~TensorBilinearFormIntegrator() { }
+
+   virtual void ReassembleOperator() = 0;
+
+   virtual void ComputeElementMatrices(Vector &element_matrices)
+   { mfem_error("TensorBilinaerFormIntegrator::ComputeElementMatrices is not overloaded"); }
+
+   virtual void MultAdd(const Vector &x, Vector &y) const = 0;
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   { y.Fill<double>(0.0); MultAdd(x, y); }
+};
+
+/// TODO: doxygen
+class BilinearForm : public mfem::PBilinearForm, public mfem::Operator
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // mfem::BilinearForm *bform;
+
+   mfem::Array<TensorBilinearFormIntegrator*> tbfi;
+   bool has_assembled;
+
+   mutable FiniteElementSpace *trial_fes, *test_fes;
+
+   mutable Vector x_local, y_local;
+
+   mfem::Vector *element_matrices;
+   OperatorHandle mat_e;
+
+   void TransferIntegrators();
+
+   void ComputeElementMatrices();
+
+   void InitRHS(const mfem::Array<int> &constraint_list,
+                mfem::Vector &mfem_x, mfem::Vector &mfem_b,
+                mfem::OperatorHandle &A,
+                mfem::Vector &mfem_X, mfem::Vector &mfem_B,
+                int copy_interior = 0) const;
+
+public:
+   /// TODO: doxygen
+   BilinearForm(const Engine &e, mfem::BilinearForm &bf)
+      : mfem::PBilinearForm(e, bf),
+        // FIXME: for mixed bilinear forms
+        mfem::Operator(*bf.FESpace()->GetVLayout().As<Layout>()),
+        tbfi(),
+        has_assembled(false),
+        trial_fes(&bf.FESpace()->Get_PFESpace()->As<FiniteElementSpace>()),
+        test_fes(&bf.FESpace()->Get_PFESpace()->As<FiniteElementSpace>()),
+        x_local(trial_fes->GetELayout()),
+        y_local(test_fes->GetELayout()),
+        element_matrices(NULL),
+        mat_e() { }
+
+   /// Virtual destructor
+   virtual ~BilinearForm();
+
+   /// Return the engine as an OpenMP engine
+   const Engine &OmpEngine() { return static_cast<const Engine&>(*engine); }
+
+   /** @brief Prolongation operator from linear algebra (linear system) vectors,
+       to input vectors for the operator. `NULL` means identity. */
+   virtual const Operator *GetProlongation() const { return trial_fes->GetProlongation(); }
+
+   /** @brief Restriction operator from input vectors for the operator to linear
+       algebra (linear system) vectors. `NULL` means identity. */
+   virtual const Operator *GetRestriction() const  { return test_fes->GetRestriction(); }
+
+   /// Assemble the PBilinearForm.
+   /** This method is called from the method BilinearForm::Assemble() of the
+       associated BilinearForm #bform.
+       @returns True, if the host assembly should be skipped. */
+   virtual bool Assemble();
+
+   /// TODO: doxygen
+   virtual void FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
+                                 mfem::OperatorHandle &A);
+
+   /// TODO: doxygen
+   virtual void FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
+                                 mfem::Vector &x, mfem::Vector &b,
+                                 mfem::OperatorHandle &A, mfem::Vector &mfem_X, mfem::Vector &mfem_B,
+                                 int copy_interior);
+
+   /// TODO: doxygen
+   virtual void RecoverFEMSolution(const mfem::Vector &mfem_X, const mfem::Vector &mfem_b,
+                                   mfem::Vector &mfem_x);
+
+   /// Operator application: `y=A(x)`.
+   virtual void Mult(const mfem::Vector &mfem_x, mfem::Vector &mfem_y) const;
+
+   /** @brief Action of the transpose operator: `y=A^t(x)`. The default behavior
+       in class Operator is to generate an error. */
+   virtual void MultTranspose(const mfem::Vector &mfem_x, mfem::Vector &mfem_y) const;
+};
+
+class ConstrainedOperator : public mfem::Operator
+{
+   const mfem::Operator *A;
+   const bool own_A;
+   const Array constraint_list;
+   mutable Vector z, w;
+   mutable mfem::Vector mfem_z, mfem_w;
+
+public:
+   ConstrainedOperator(mfem::Operator *A_,
+                       const mfem::Array<int> &constraint_list_,
+                       bool own_A_ = false);
+
+   // Destructor: destroys the unconstrained Operator @a A if @a own_A is true.
+   virtual ~ConstrainedOperator();
+
+   /** @brief Eliminate "essential boundary condition" values specified in @a x
+       from the given right-hand side @a b.
+
+       Performs the following steps:
+
+       z = A((0,x_b));  b_i -= z_i;  b_b = x_b;
+
+       where the "_b" subscripts denote the essential (boundary) indices/dofs of
+       the vectors, and "_i" -- the rest of the entries. */
+   void EliminateRHS(const mfem::Vector &mfem_x, mfem::Vector &mfem_b) const;
+
+   /** @brief Constrained operator action.
+
+       Performs the following steps:
+
+       z = A((x_i,0));  y_i = z_i;  y_b = x_b;
+
+       where the "_b" subscripts denote the essential (boundary) indices/dofs of
+       the vectors, and "_i" -- the rest of the entries. */
+   virtual void Mult(const mfem::Vector &mfem_x, mfem::Vector &mfem_y) const;
+};
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#endif // MFEM_BACKENDS_OMP_BILINEAR_FORM_HPP

backends/omp/engine.cpp

@@ -0,0 +1,253 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "engine.hpp"
+#include "array.hpp"
+#include "layout.hpp"
+#include "vector.hpp"
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+#include "memory_resource.hpp"
+
+#include <map>
+
+namespace mfem
+{
+
+namespace omp
+{
+
+typedef std::map<std::string, std::string> keyval_pair_t;
+
+template<typename T, typename P>
+static T remove_if(T beg, T end, P pred)
+{
+   T dest = beg;
+   for (T itr = beg;itr != end; ++itr)
+      if (!pred(*itr))
+         *(dest++) = *itr;
+   return dest;
+}
+
+void parse_token(const std::string &token, std::string &key, std::string &val)
+{
+   std::size_t sep = token.find_first_of(':');
+   if (sep > token.size()) mfem_error("Parse error");
+
+   key = token.substr(0, sep);
+   key.erase(mfem::omp::remove_if(key.begin(), key.end(), isspace), key.end());
+   key.erase(std::remove(key.begin(), key.end(), '\''), key.end());
+
+   val = token.substr(sep+1);
+   val.erase(mfem::omp::remove_if(val.begin(), val.end(), isspace), val.end());
+   val.erase(std::remove(val.begin(), val.end(), '\''), val.end());
+}
+
+keyval_pair_t parse_engine_spec(const std::string &engine_spec)
+{
+   keyval_pair_t map;
+   std::size_t token_extent = 0;
+   std::string key, val;
+   while (token_extent < engine_spec.size())
+   {
+      const std::string remaining(engine_spec, token_extent);
+
+      std::size_t next_comma = remaining.find_first_of(',');
+      if (next_comma == std::string::npos) next_comma = engine_spec.size() - 1;
+
+      const std::string token(remaining, 0, next_comma);
+      parse_token(token, key, val);
+
+      map[key] = val;
+      token_extent += next_comma+1;
+   }
+   return map;
+}
+
+void Engine::Init(const std::string &engine_spec)
+{
+   keyval_pair_t tokens(parse_engine_spec(engine_spec));
+   keyval_pair_t::iterator it;
+
+   it = tokens.find("exec_target");
+   if (it != tokens.end())
+   {
+      if (!std::strncmp(it->second.data(), "device", 6))
+      {
+         exec_target = Device;
+         device_number = 0;
+      }
+      else if (!std::strncmp(it->second.data(), "host", 4))
+      {
+         exec_target = Host;
+         device_number = -1;
+      }
+      else
+      {
+         mfem_error("Parse error. Possible values for exec_target are: ['host', 'device']");
+      }
+   }
+   else
+   {
+      // Default to host if not specified
+      mfem::out << "Did not specify exec_target. Defaulting to host..." << std::endl;
+      exec_target = Host;
+      device_number = -1;
+   }
+
+   it = tokens.find("mem_type");
+   if (it != tokens.end())
+   {
+      if (!std::strncmp(it->second.data(), "unified", 7))
+      {
+#if defined(MFEM_USE_CUDAUM)
+         memory_resources[0] = new UnifiedMemoryResource();
+         unified_memory = true;
+#else
+         mfem_error("Have not compiled support for CUDA unified memory.");
+#endif
+      }
+      else if (!std::strncmp(it->second.data(), "separate", 4))
+      {
+         memory_resources[0] = new NewDeleteMemoryResource();
+         unified_memory = false;
+      }
+      else
+      {
+         mfem_error("Parse error. Possible values for mem_type are: ['separate', 'unified']");
+      }
+   }
+   else {
+      if (exec_target == Device)
+      {
+#if defined(MFEM_USE_CUDAUM)
+         mfem::out << "Did not specify mem_type in engine spec. Defaulting to unified memory..." << std::endl;
+         // Default to unified memory
+         memory_resources[0] = new UnifiedMemoryResource();
+         unified_memory = true;
+#else
+         mfem::out << "Did not specify mem_type in engine spec. Defaulting to standard host memory..." << std::endl;
+         memory_resources[0] = new NewDeleteMemoryResource();
+         unified_memory = false;
+#endif
+      }
+      else
+      {
+         mfem::out << "Did not specify mem_type in engine spec. Defaulting to standard host memory..." << std::endl;
+         memory_resources[0] = new NewDeleteMemoryResource();
+         unified_memory = false;
+      }
+   }
+
+   it = tokens.find("mult_engine");
+   if (it != tokens.end())
+   {
+      if (!std::strncmp(it->second.data(), "acrotensor", 10))
+      {
+         mult_type = Acrotensor;
+      }
+      else
+      {
+         mfem_error("Parse error. Possible values for mem_type are: ['acrotensor'].");
+      }
+   }
+   else
+   {
+      mfem::out << "Did not specify mult_engine in engine spec. Defaulting to Acrotensor..." << std::endl;
+#ifndef MFEM_USE_ACROTENSOR
+      mfem_error("Must compile with Acrotensor support");
+#endif
+      mult_type = Acrotensor;
+   }
+}
+
+Engine::Engine(const std::string &engine_spec)
+   : mfem::Engine(NULL, 1, 1)
+{
+   Init(engine_spec);
+}
+
+#ifdef MFEM_USE_MPI
+Engine::Engine(MPI_Comm _comm, const std::string &engine_spec)
+   : mfem::Engine(NULL, 1, 1)
+{
+   comm = _comm;
+   Init(engine_spec);
+}
+#endif
+
+DLayout Engine::MakeLayout(std::size_t size) const
+{
+   return DLayout(new Layout(*this, size));
+}
+
+DLayout Engine::MakeLayout(const mfem::Array<std::size_t> &offsets) const
+{
+   MFEM_ASSERT(offsets.Size() == 2,
+               "multiple workers are not supported yet");
+   return DLayout(new Layout(*this, offsets.Last()));
+}
+
+DArray Engine::MakeArray(PLayout &layout, std::size_t item_size) const
+{
+   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
+               "invalid input layout");
+   Layout *lt = static_cast<Layout *>(&layout);
+   return DArray(new Array(*lt, item_size));
+}
+
+DVector Engine::MakeVector(PLayout &layout, int type_id) const
+{
+   MFEM_ASSERT(type_id == ScalarId<double>::value, "invalid type_id");
+   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
+               "invalid input layout");
+   Layout *lt = static_cast<Layout *>(&layout);
+   return DVector(new Vector(*lt));
+}
+
+DFiniteElementSpace Engine::MakeFESpace(mfem::FiniteElementSpace &fespace) const
+{
+   return DFiniteElementSpace(new FiniteElementSpace(*this, fespace));
+}
+
+DBilinearForm Engine::MakeBilinearForm(mfem::BilinearForm &bf) const
+{
+   return DBilinearForm(new BilinearForm(*this, bf));
+}
+
+void Engine::AssembleLinearForm(LinearForm &l_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+}
+
+mfem::Operator *Engine::MakeOperator(const MixedBilinearForm &mbl_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+   return NULL;
+}
+
+mfem::Operator *Engine::MakeOperator(const NonlinearForm &nl_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+   return NULL;
+}
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/engine.hpp

@@ -0,0 +1,123 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_ENGINE_HPP
+#define MFEM_BACKENDS_OMP_ENGINE_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "../base/engine.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+enum ExecutionTarget { Host, Device };
+
+enum IntegratorType { Acrotensor };
+
+class Engine : public mfem::Engine
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // mfem::Backend *backend;
+#ifdef MFEM_USE_MPI
+   // MPI_Comm comm;
+#endif
+   // int num_mem_res;
+   // int num_workers;
+   // MemoryResource **memory_resources;
+   // double *workers_weights;
+   // int *workers_mem_res;
+
+   enum ExecutionTarget exec_target;
+   bool unified_memory;
+   int device_number;
+   IntegratorType mult_type;
+
+   void Init(const std::string &engine_spec);
+
+public:
+   Engine(const std::string &engine_spec);
+
+#ifdef MFEM_USE_MPI
+   Engine(MPI_Comm comm, const std::string &engine_spec);
+#endif
+
+   virtual ~Engine() { }
+
+   /**
+      @name OMP specific interface, used by other objects in the OMP backend
+   */
+   ///@{
+
+   IntegratorType IntegType() const { return mult_type; }
+
+   ExecutionTarget ExecTarget() const { return exec_target; }
+
+   inline bool UnifiedMemory() const { return unified_memory; }
+
+   void* Malloc(std::size_t bytes) const
+   {
+      return memory_resources[0]->Allocate(bytes, 16);
+   }
+
+   void Dealloc(void *ptr, std::size_t bytes = 0) const
+   {
+      memory_resources[0]->Deallocate(ptr, bytes);
+   }
+
+   ///@}
+   // End: OMP specific interface
+
+   /**
+      @name Virtual interface: finite element data structures and algorithms
+   */
+   ///@{
+
+   virtual DLayout MakeLayout(std::size_t size) const;
+   virtual DLayout MakeLayout(const mfem::Array<std::size_t> &offsets) const;
+
+   virtual DArray MakeArray(PLayout &layout, std::size_t item_size) const;
+
+   virtual DVector MakeVector(PLayout &layout,
+                              int type_id = ScalarId<double>::value) const;
+
+   virtual DFiniteElementSpace MakeFESpace(mfem::FiniteElementSpace &
+                                           fespace) const;
+
+   virtual DBilinearForm MakeBilinearForm(mfem::BilinearForm &bf) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual void AssembleLinearForm(LinearForm &l_form) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual mfem::Operator *MakeOperator(const MixedBilinearForm &mbl_form) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual mfem::Operator *MakeOperator(const NonlinearForm &nl_form) const;
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#endif // MFEM_BACKENDS_OMP_ENGINE_HPP

backends/omp/fespace.cpp

@@ -0,0 +1,237 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "fespace.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+FiniteElementSpace::FiniteElementSpace(const Engine &e,
+                                       mfem::FiniteElementSpace &fespace)
+   : PFiniteElementSpace(e, fespace),
+     e_layout(e, 0),
+     tensor_offsets(NULL),
+     tensor_indices(NULL),
+     prolongation(NULL),
+     restriction(NULL)
+{
+   std::size_t lsize = 0;
+   for (int e = 0; e < fespace.GetNE(); e++) { lsize += fespace.GetFE(e)->GetDof(); }
+   e_layout.Resize(lsize);
+   // The e_layout will be stored inside multiple shared DLayout objects
+   e_layout.DontDelete();
+}
+
+void FiniteElementSpace::BuildDofMaps()
+{
+   mfem::FiniteElementSpace *mfem_fes = GetFESpace();
+
+   const int local_size = GetELayout().Size();
+   const int global_size = mfem_fes->GetVLayout()->Size();
+   const int vdim = mfem_fes->GetVDim();
+
+   // Now we can allocate and fill the global map
+   tensor_offsets = new mfem::Array<int>(*(new Layout(OmpEngine(), global_size + 1)));
+   tensor_indices = new mfem::Array<int>(*(new Layout(OmpEngine(), local_size)));
+
+   mfem::Array<int> &offsets = *tensor_offsets;
+   mfem::Array<int> &indices = *tensor_indices;
+
+   mfem::Array<int> global_map(local_size);
+   mfem::Array<int> elem_vdof;
+
+   int offset = 0;
+   for (int e = 0; e < mfem_fes->GetNE(); e++)
+   {
+      const FiniteElement *fe = mfem_fes->GetFE(e);
+      const int dofs = fe->GetDof();
+      const TensorBasisElement *tfe = dynamic_cast<const TensorBasisElement *>(fe);
+      const mfem::Array<int> &dof_map = tfe->GetDofMap();
+
+      mfem_fes->GetElementVDofs(e, elem_vdof);
+
+      for (int vd = 0; vd < vdim; vd++)
+         for (int i = 0; i < dofs; i++)
+         {
+            global_map[offset + dofs*vd + i] = elem_vdof[dofs*vd + dof_map[i]];
+         }
+      offset += dofs * vdim;
+   }
+
+   // global_map[i] = index in global vector for local dof i
+   // NOTE: multiple i values will yield same global_map[i] for shared DOF.
+
+   // We want to now invert this map so we have indices[j] = (local dof for global dof j).
+
+   // Zero the offset vector
+   offsets = 0;
+
+   // Keep track of how many local dof point to its global dof
+   // Count how many times each dof gets hit
+   for (int i = 0; i < local_size; i++)
+   {
+      const int g = global_map[i];
+      ++offsets[g + 1];
+   }
+   // Aggregate the offsets
+   for (int i = 1; i <= global_size; i++)
+   {
+      offsets[i] += offsets[i - 1];
+   }
+
+   for (int i = 0; i < local_size; i++)
+   {
+      const int g = global_map[i];
+      indices[offsets[g]++] = i;
+   }
+
+   // Shift the offset vector back by one, since it was used as a
+   // counter above.
+   for (int i = global_size; i > 0; i--)
+   {
+      offsets[i] = offsets[i - 1];
+   }
+   offsets[0] = 0;
+
+   offsets.Push();
+   indices.Push();
+}
+
+/// Convert an E vector to L vector
+void FiniteElementSpace::ToLVector(const Vector &e_vector, Vector &l_vector)
+{
+   if (tensor_indices == NULL) BuildDofMaps();
+
+   if (l_vector.Size() != (std::size_t) GetFESpace()->GetVSize())
+   {
+      l_vector.Resize<double>(GetFESpace()->GetVLayout(), NULL);
+   }
+
+   const int lsize = l_vector.Size();
+   const int *offsets = tensor_offsets->Get_PArray()->As<Array>().GetData<int>();
+   const int *indices = tensor_indices->Get_PArray()->As<Array>().GetData<int>();
+
+   const double *e_data = e_vector.GetData<double>();
+   double *l_data = l_vector.GetData<double>();
+
+   const bool use_target = l_vector.ComputeOnDevice();
+   const bool use_parallel = (use_target || lsize > 1000);
+
+#pragma omp target teams distribute parallel for        \
+   map (to: offsets, indices, l_data, e_data)           \
+   if (target: use_target)                              \
+   if (parallel: use_parallel)
+   for (int i = 0; i < lsize; i++)
+   {
+      const int offset = offsets[i];
+      const int next_offset = offsets[i + 1];
+      double dof_value = 0;
+      for (int j = offset; j < next_offset; j++)
+      {
+         dof_value += e_data[indices[j]];
+      }
+      l_data[i] = dof_value;
+   }
+}
+
+/// Covert an L vector to E vector
+void FiniteElementSpace::ToEVector(const Vector &l_vector, Vector &e_vector)
+{
+   if (tensor_indices == NULL) BuildDofMaps();
+
+   if (e_vector.Size() != (std::size_t) e_layout.Size())
+   {
+      e_vector.Resize<double>(GetELayout(), NULL);
+   }
+
+   const int lsize = l_vector.Size();
+   const int *offsets = tensor_offsets->Get_PArray()->As<Array>().GetData<int>();
+   const int *indices = tensor_indices->Get_PArray()->As<Array>().GetData<int>();
+
+   const double *l_data = l_vector.GetData<double>();
+   double *e_data = e_vector.GetData<double>();
+
+   const bool use_target = l_vector.ComputeOnDevice();
+   const bool use_parallel = (use_target || lsize > 1000);
+
+#pragma omp target teams distribute parallel for         \
+   map (to: offsets, indices, l_data, e_data)            \
+   if (target: use_target)                               \
+   if (parallel: use_parallel)
+   for (int i = 0; i < lsize; i++)
+   {
+      const int offset = offsets[i];
+      const int next_offset = offsets[i + 1];
+      const double dof_value = l_data[i];
+      for (int j = offset; j < next_offset; j++)
+      {
+         e_data[indices[j]] = dof_value;
+      }
+   }
+}
+
+/// Get the finite element space prolongation matrix
+const Operator *FiniteElementSpace::GetProlongation() const
+{
+   // FIXME: This relies on unified memory if using a device other than the CPU
+   if (!prolongation)
+   {
+      Layout &v_layout = GetVLayout();
+      Layout &t_layout = GetTrueVLayout();
+
+      const mfem::Operator *op = GetFESpace()->GetProlongationMatrix();
+      if (!op)
+      {
+         prolongation = new mfem::IdentityOperator(t_layout);
+      }
+      else
+      {
+         prolongation = new BackendOperator(t_layout, v_layout, op);
+      }
+   }
+   return prolongation;
+}
+
+/// Get the finite element space restriction matrix
+const Operator *FiniteElementSpace::GetRestriction() const
+{
+   // FIXME: This relies on unified memory if using a device other than the CPU
+   if (!restriction)
+   {
+      Layout &v_layout = GetVLayout();
+      Layout &t_layout = GetTrueVLayout();
+
+      const mfem::Operator *op = GetFESpace()->GetRestrictionMatrix();
+      if (!op)
+      {
+         restriction = new mfem::IdentityOperator(t_layout);
+      }
+      else
+      {
+         restriction = new BackendOperator(v_layout, t_layout, op);
+      }
+   }
+   return restriction;
+}
+
+
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/fespace.hpp

@@ -0,0 +1,106 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_FESPACE_HPP
+#define MFEM_BACKENDS_OMP_FESPACE_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "engine.hpp"
+#include "array.hpp"
+#include "vector.hpp"
+#include "../../fem/fem.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+/*
+  Wraps an mfem::Operator that does not contain layout information.
+ */
+class BackendOperator : public mfem::Operator
+{
+   const mfem::Operator *op;
+
+public:
+   BackendOperator(Layout &in_layout, Layout &out_layout,
+                   const mfem::Operator *op_) : Operator(in_layout, out_layout), op(op_) { }
+
+   virtual void Mult(const mfem::Vector &x, mfem::Vector &y) const { op->Mult(x, y); }
+   virtual void MultTranspose(const mfem::Vector &x, mfem::Vector &y) const { op->MultTranspose(x, y); }
+};
+
+/// TODO: doxygen
+class FiniteElementSpace : public mfem::PFiniteElementSpace
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // mfem::FiniteElementSpace *fes;
+
+   Layout e_layout;
+
+   mfem::Array<int> *tensor_offsets, *tensor_indices;
+
+   mutable mfem::Operator *prolongation, *restriction;
+
+   void BuildDofMaps();
+
+public:
+   /// Nearly-empty class that stores a pointer to a mfem::FiniteElementSpace instance and the engine
+   FiniteElementSpace(const Engine &e, mfem::FiniteElementSpace &fespace);
+
+   /// Virtual destructor
+   virtual ~FiniteElementSpace()
+   {
+      delete tensor_offsets;
+      delete tensor_indices;
+      delete prolongation;
+      delete restriction;
+   }
+
+   Layout &GetELayout() { return e_layout; }
+
+   Layout &GetVLayout() const
+   { return *fes->GetVLayout().As<Layout>(); }
+
+   Layout &GetTrueVLayout() const
+   { return *fes->GetTrueVLayout().As<Layout>(); }
+
+   /// Return the engine as an OpenMP engine
+   const Engine &OmpEngine() { return static_cast<const Engine&>(*engine); }
+
+   /// Convert an E vector to L vector
+   void ToLVector(const Vector &e_vector, Vector &l_vector);
+
+   /// Covert an L vector to E vector
+   void ToEVector(const Vector &l_vector, Vector &e_vector);
+
+   /// Get the finite element space prolongation matrix
+   const Operator *GetProlongation() const;
+
+   /// Get the finite element space restriction matrix
+   const Operator *GetRestriction() const;
+
+};
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#endif // MFEM_BACKENDS_OMP_FESPACE_HPP

backends/omp/layout.cpp

@@ -0,0 +1,40 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "layout.hpp"
+#include "../../general/array.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+void Layout::Resize(std::size_t new_size)
+{
+   size = new_size;
+}
+
+void Layout::Resize(const Array<std::size_t> &offsets)
+{
+   MFEM_ASSERT(offsets.Size() == 2,
+               "multiple workers are not supported yet");
+   size = offsets.Last();
+}
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/layout.hpp

@@ -0,0 +1,71 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_LAYOUT_HPP
+#define MFEM_BACKENDS_OMP_LAYOUT_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "../base/layout.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+class Layout : public mfem::PLayout
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // std::size_t size;
+
+public:
+   Layout(const Engine &e, std::size_t s = 0) : PLayout(e, s) { }
+
+   const Engine &OmpEngine() const
+   { return *static_cast<const Engine *>(engine.Get()); }
+
+   void *Alloc(std::size_t bytes) const
+   { return OmpEngine().Malloc(bytes); }
+
+   void Dealloc(void *ptr) const
+   { return OmpEngine().Dealloc(ptr); }
+
+   virtual ~Layout() { }
+
+   /**
+      @name Virtual interface
+   */
+   ///@{
+
+   /// Resize the layout
+   virtual void Resize(std::size_t new_size);
+
+   /// Resize the layout based on the given worker offsets
+   virtual void Resize(const Array<std::size_t> &offsets);
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#endif // MFEM_BACKENDS_OMP_LAYOUT_HPP

backends/omp/memory_resource.cpp

@@ -0,0 +1,57 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "memory_resource.hpp"
+#include "../../general/error.hpp"
+
+#ifdef MFEM_USE_CUDAUM
+#include "cuda_runtime.h"
+#include "cuda.h"
+#endif
+
+namespace mfem
+{
+
+namespace omp
+{
+
+#ifdef MFEM_USE_CUDAUM
+void *UnifiedMemoryResource::DoAllocate(std::size_t bytes,
+                                        std::size_t alignment)
+{
+   void *p = NULL;
+   if (bytes > 0)
+   {
+      cudaError_t ret = cudaMallocManaged(&p, bytes);
+      MFEM_VERIFY(ret == cudaSuccess, "");
+   }
+   return p;
+}
+
+void UnifiedMemoryResource::DoDeallocate(void *p, std::size_t bytes,
+                                         std::size_t alignment)
+{
+   if (p != NULL)
+   {
+      cudaFree(p);
+   }
+}
+#endif
+
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/memory_resource.hpp

@@ -0,0 +1,44 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_MEMORY_RESOURCE_HPP
+#define MFEM_BACKENDS_OMP_MEMORY_RESOURCE_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "../../backends/base/memory_resource.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+/// Polymorphic memory resource. Similar to C++17's std::pmr::memory_resource.
+
+#ifdef MFEM_USE_CUDAUM
+/** @brief Memory resource using unified memory. */
+class UnifiedMemoryResource : public MemoryResource
+{
+protected:
+   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment);
+   virtual void DoDeallocate(void *p, std::size_t bytes, std::size_t alignment);
+};
+#endif
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#endif // MFEM_BACKENDS_OMP_MEMORY_RESOURCE_HPP

backends/omp/vector.cpp

@@ -0,0 +1,205 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "vector.hpp"
+#include "../../linalg/vector.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+PVector *Vector::DoVectorClone(bool copy_data, void **buffer,
+                               int buffer_type_id) const
+{
+   MFEM_ASSERT(buffer_type_id == ScalarId<double>::value, "");
+   Vector *new_vector = new Vector(OmpLayout());
+   if (copy_data)
+   {
+      const std::size_t total_size = sizeof(double) * OmpLayout().Size();
+      if (!ComputeOnDevice())
+         std::memcpy(new_vector->GetData<void>(), data, total_size);
+      else
+      {
+         char *new_data = new_vector->GetData<char>();
+#pragma omp target teams distribute parallel for is_device_ptr(new_data)
+         for (std::size_t i = 0; i < total_size; i++) new_data[i] = data[i];
+      }
+   }
+   if (buffer)
+   {
+      *buffer = new_vector->GetData<void>();
+   }
+   return new_vector;
+}
+
+void Vector::DoDotProduct(const PVector &x, void *result,
+                          int result_type_id) const
+{
+   // Can be called when Size() == 0, e.g. when an MPI-parallel vector has a
+   // local size of 0.
+
+   MFEM_ASSERT(result_type_id == ScalarId<double>::value, "");
+   double *res = (double *)result;
+   double local_dot = 0.;
+   MFEM_ASSERT(dynamic_cast<const Vector *>(&x) != NULL, "invalid Vector type");
+   const Vector *xp = static_cast<const Vector *>(&x);
+   MFEM_ASSERT(this->Size() == xp->Size(), "");
+
+   const double *ptr = GetData<double>();
+   const double *xptr = xp->GetData<double>();
+   const std::size_t size = Size();
+
+   if (!ComputeOnDevice())
+   {
+      for (std::size_t i = 0; i < size; i++) local_dot += ptr[i] * xptr[i];
+   }
+   else
+   {
+#pragma omp target teams distribute parallel for map(to: ptr, xptr) reduction(+:local_dot)
+      for (std::size_t i = 0; i < size; i++) local_dot += ptr[i] * xptr[i];
+   }
+
+   *res = local_dot;
+#ifdef MFEM_USE_MPI
+   MPI_Comm comm = OmpLayout().OmpEngine().GetComm();
+   if (comm != MPI_COMM_NULL)
+   {
+      MPI_Allreduce(&local_dot, res, 1, MPI_DOUBLE, MPI_SUM, comm);
+   }
+#endif
+}
+
+void Vector::DoAxpby(const void *a, const PVector &x,
+                     const void *b, const PVector &y,
+                     int ab_type_id)
+{
+   // called only when Size() != 0
+   MFEM_ASSERT(ab_type_id == ScalarId<double>::value, "");
+
+   const double da = *static_cast<const double *>(a);
+   const double db = *static_cast<const double *>(b);
+   MFEM_ASSERT(da == 0.0 || dynamic_cast<const Vector *>(&x) != NULL,
+               "invalid Vector x");
+   MFEM_ASSERT(db == 0.0 || dynamic_cast<const Vector *>(&y) != NULL,
+               "invalid Vector y");
+   const Vector *xp = static_cast<const Vector *>(&x);
+   const Vector *yp = static_cast<const Vector *>(&y);
+
+   MFEM_ASSERT(da == 0.0 || this->Size() == xp->Size(), "");
+   MFEM_ASSERT(db == 0.0 || this->Size() == yp->Size(), "");
+
+   const std::size_t size = Size();
+   const std::size_t critical_size = 1000;
+   const double *xd = xp->GetData<double>();
+   const double *yd = yp->GetData<double>();
+   double *td = GetData<double>();
+
+   const bool use_target = ComputeOnDevice();
+   const bool use_parallel = (use_target || size > critical_size);
+
+   if (da == 0.0)
+   {
+      if (db == 0.0)
+      {
+         OmpFill(&da);
+      }
+      else
+      {
+         if (td == yd)
+         {
+            // *this *= db
+#pragma omp target teams distribute parallel for        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel) map (to: db)
+            for (std::size_t i = 0; i < size; i++) td[i] *= db;
+         }
+         else
+         {
+            // *this = db * y
+#pragma omp target teams distribute parallel for        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel) map (to: yd, db)
+            for (std::size_t i = 0; i < size; i++) td[i] = yd[i] * db;
+         }
+      }
+   }
+   else
+   {
+      if (db == 0.0)
+      {
+         if (td == xd)
+         {
+            // *this *= da
+#pragma omp target teams distribute parallel for        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel) map (to: da)
+            for (std::size_t i = 0; i < size; i++) td[i] *= da;
+         }
+         else
+         {
+            // *this = da * x
+#pragma omp target teams distribute parallel for        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel) map (to: xd, da)
+            for (std::size_t i = 0; i < size; i++) td[i] = xd[i] * da;
+         }
+      }
+      else
+      {
+         MFEM_ASSERT(xd != yd, "invalid input");
+         if (td == xd)
+         {
+            // *this = da * (*this) + db * y
+#pragma omp target teams distribute parallel for        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel) map (to: da, td, db, yd)
+            for (std::size_t i = 0; i < size; i++) td[i] = da * td[i] + db * yd[i];
+         }
+         else if (td == yd)
+         {
+            // *this = da * x + db * (*this)
+#pragma omp target teams distribute parallel for        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel) map (to: da, xd, db, td)
+            for (std::size_t i = 0; i < size; i++) td[i] = da * xd[i] + db * td[i];
+         }
+         else
+         {
+            // *this = da * x + db * y
+#pragma omp target teams distribute parallel for        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel) map (to: da, xd, db, yd)
+            for (std::size_t i = 0; i < size; i++) td[i] = da * xd[i] + db * yd[i];
+         }
+      }
+   }
+}
+
+mfem::Vector Vector::Wrap()
+{
+   return mfem::Vector(*this);
+}
+
+const mfem::Vector Vector::Wrap() const
+{
+   return mfem::Vector(*const_cast<Vector*>(this));
+}
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/vector.hpp

@@ -0,0 +1,71 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_VECTOR_HPP
+#define MFEM_BACKENDS_OMP_VECTOR_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "../base/vector.hpp"
+#include "array.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+class Vector : virtual public Array, public mfem::PVector
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // DLayout layout;
+   // char *data;
+   // std::size_t size;
+
+   /**
+      @name Virtual interface
+   */
+   ///@{
+
+   virtual PVector *DoVectorClone(bool copy_data, void **buffer,
+                                  int buffer_type_id) const;
+
+   virtual void DoDotProduct(const PVector &x, void *result,
+                             int result_type_id) const;
+
+   virtual void DoAxpby(const void *a, const PVector &x,
+                        const void *b, const PVector &y,
+                        int ab_type_id);
+
+   ///@}
+   // End: Virtual interface
+
+public:
+   Vector(Layout &lt)
+      : PArray(lt), Array(lt, sizeof(double)), PVector(lt)
+   { }
+
+   mfem::Vector Wrap();
+
+   const mfem::Vector Wrap() const;
+};
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#endif // MFEM_BACKENDS_OMP_VECTOR_HPP

config/cmake/MFEMConfig.cmake.in

@@ -39,7 +39,6 @@ set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
-set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
 
 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")

config/cmake/config.hpp.in

@@ -98,9 +98,6 @@
 // Enable MFEM functionality based on Conduit
 #cmakedefine MFEM_USE_CONDUIT
 
-// Enable MFEM functionality based on the PUMI library
-#cmakedefine MFEM_USE_PUMI
-
 // Which library functions to use in class StopWatch for measuring time.
 // For a list of the available options, see INSTALL.
 // If not defined, an option is selected automatically.
@@ -116,8 +113,4 @@
 // Version of HYPRE used for building MFEM.
 #cmakedefine MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
 
-// Macro defined when PUMI is built with support for the Simmetrix SimModSuite
-// library.
-#cmakedefine MFEM_USE_SIMMETRIX
-
 #endif // MFEM_CONFIG_HEADER

config/config.hpp

@@ -36,15 +36,12 @@
 #ifdef MFEM_USE_PETSC
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
-#ifdef MFEM_USE_PUMI
-#error Building with PUMI (MFEM_USE_PUMI=YES) requires MPI (MFEM_USE_MPI=YES)
-#endif
 #endif // MFEM_USE_MPI not defined
 
 // Macro that returns its first arg when MFEM_USE_BACKENDS is defined, and its
 // second arg if it is not defined.
 #ifdef MFEM_USE_BACKENDS
-#define MFEM_IF_BACKENDS(x,y) x
+#define MFEM_IF_BACKENDS(x,y) (x)
 #else
-#define MFEM_IF_BACKENDS(x,y) y
+#define MFEM_IF_BACKENDS(x,y) (y)
 #endif

config/config.hpp.in

@@ -115,15 +115,21 @@
 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR
 
-// Enable MFEM functionality based on the PUMI library
-// #define MFEM_USE_PUMI
-
 // Enable the use of MFEM backends.
 // #define MFEM_USE_BACKENDS
 
 // Enable the OCCA backend.
 // #define MFEM_USE_OCCA
 
+// Enable the OMP backend.
+// #define MFEM_USE_OMP
+
+// Enable use of acrotensor in backends.
+// #define MFEM_USE_ACROTENSOR
+
+// Enable use of unified memory.
+// #define MFEM_USE_CUDAUM
+
 // Windows specific options
 #ifdef _WIN32
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)
@@ -133,8 +139,4 @@
 // Version of HYPRE used for building MFEM.
 // #define MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
 
-// Macro defined when PUMI is built with support for the Simmetrix SimModSuite
-// library.
-// #define MFEM_USE_SIMMETRIX
-
 #endif // MFEM_CONFIG_HEADER

config/config.mk.in

@@ -39,9 +39,11 @@ MFEM_USE_PETSC       = @MFEM_USE_PETSC@
 MFEM_USE_MPFR        = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE       = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT     = @MFEM_USE_CONDUIT@
-MFEM_USE_PUMI        = @MFEM_USE_PUMI@
 MFEM_USE_BACKENDS    = @MFEM_USE_BACKENDS@
 MFEM_USE_OCCA        = @MFEM_USE_OCCA@
+MFEM_USE_OMP         = @MFEM_USE_OMP@
+MFEM_USE_ACROTENSOR  = @MFEM_USE_ACROTENSOR@
+MFEM_USE_CUDAUM      = @MFEM_USE_CUDAUM@
 
 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@

config/defaults.cmake

@@ -40,7 +40,6 @@ option(MFEM_USE_PETSC "Enable PETSc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
-option(MFEM_USE_PUMI "Enable PUMI" OFF)
 
 # Allow a user to disable testing, examples, and/or miniapps at CONFIGURE TIME
 # if they don't want/need them (e.g. if MFEM is "just a dependency" and all they
@@ -146,9 +145,6 @@ set(AXOM_DIR "${MFEM_DIR}/../axom" CACHE PATH "Path to the Axom library.")
 set(Axom_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
     "Additional packages required by Axom.")
 
-set(PUMI_DIR "${MFEM_DIR}/../pumi-2.1.0" CACHE STRING
-    "Directory where PUMI is installed")
-
 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")

config/defaults.mk

@@ -109,7 +109,10 @@ MFEM_USE_PETSC       = NO
 MFEM_USE_MPFR        = NO
 MFEM_USE_SIDRE       = NO
 MFEM_USE_CONDUIT     = NO
-MFEM_USE_PUMI        = NO
+# FIXME: add MFEM_USE_OMP and MFEM_USE_ACROTENSOR to the CMake build system
+MFEM_USE_OMP         = NO
+MFEM_USE_ACROTENSOR  = NO
+MFEM_USE_CUDAUM      = NO
 
 # Compile and link options for zlib.
 ZLIB_DIR =
@@ -275,17 +278,31 @@ SIDRE_LIB = \
    -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
    -lsidre -lslic -laxom_utils -lconduit -lconduit_relay -lhdf5 $(ZLIB_LIB) -ldl
 
-# PUMI
-# Note that PUMI_DIR is needed -- it is used to check for gmi_sim.h
-PUMI_DIR = @MFEM_DIR@/../pumi-2.1.0
-PUMI_OPT = -I$(PUMI_DIR)/include
-PUMI_LIB = -L$(PUMI_DIR)/lib -lpumi -lcrv -lma -lmds -lapf -lpcu -lgmi -lparma\
-   -llion -lmth -lapf_zoltan -lspr
-
 OCCA_DIR = @MFEM_DIR@/../occa
 OCCA_OPT = -I$(OCCA_DIR)/include
 OCCA_LIB = -Wl,-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca
 
+CUDA_DIR = /usr/local/cuda
+CUDAUM_LIB = -L$(CUDA_DIR)/lib64 -lcudart
+CUDAUM_OPT = -I$(CUDA_DIR)/include
+
+OMP_OPT = -qsmp=omp -qoffload
+
+ACROTENSOR_DIR = @MFEM_DIR@/../acrotensor
+ACROTENSOR_OPT = -std=c++11 -I$(ACROTENSOR_DIR)/inc
+ACROTENSOR_LIB = -Wl,-rpath,$(ACROTENSOR_DIR)/lib/shared -L$(ACROTENSOR_DIR)/lib/shared -lacrotensor
+# If Acrotensor was compile with CUDA support, but MFEM_USE_CUDAUM==NO, then uncomment the lines below
+# ACROTENSOR_OPT += -I$(CUDA_DIR)/include
+# ACROTENSOR_LIB += -L$(CUDA_DIR)/lib64 -lcuda -lcudart -lnvrtc
+ifeq ($(MFEM_USE_CUDAUM),YES)
+ifeq ($(MFEM_USE_MPI),YES)
+# HYPRE needs some extra libraries in parallel on the GPU
+# FIXME: We need another solution for compilers other than XL for the
+#        dlink CUDA step, but fixes need to happen elsewhere as well.
+HYPRE_LIB += -qcuda -lcublas -lcusparse -lnvToolsExt
+endif
+endif
+
 # If YES, enable some informational messages
 VERBOSE = NO
 

config/makefile

@@ -38,9 +38,6 @@ all: header config-mk
 MPI = $(MFEM_USE_MPI:NO=)
 GHV = get_hypre_version
 GHV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
-SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)
-SMX_PATH = $(PUMI_DIR)/include/gmi_sim.h
-SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))
 
 $(GHV): $(SRC)$(GHV).cpp
 	$(call mfem-info, Determining HYPRE version ...)
@@ -55,16 +52,10 @@ get-hypre-version: $(GHV).out
 	$(info HYPRE version: $(MFEM_HYPRE_VERSION)),\
 	$(error Unable to determine HYPRE version))
 
-check-smx:
-	$(call mfem-info, Checking for Simmetrix header [$(SMX_FILE)] ...)
-	$(eval MFEM_USE_SIMMETRIX:=$(if $(wildcard $(SMX_FILE)),YES,NO))
-	$(call mfem-info, MFEM_USE_SIMMETRIX = $(MFEM_USE_SIMMETRIX))
-	$(eval export MFEM_USE_SIMMETRIX)
-
-header: $(if $(MPI),get-hypre-version,) $(if $(SMX),check-smx)
+header: $(if $(MPI),get-hypre-version,)
 	$(call mfem-info, Writing $(CONFIG_HPP) ...)
 	@set -- && \
-	for def in $${MFEM_DEFINES} $(if $(MPI),MFEM_HYPRE_VERSION) $(SMX); do \
+	for def in $${MFEM_DEFINES} $(if $(MPI),MFEM_HYPRE_VERSION,); do \
 	   eval var=\$$$$def && \
 	   if [ "NO" != "$${var}" ]; then \
 	      set -- "$$@" -e "s|// \(#define $${def} \)|\1|" && \

config/test.mk

@@ -14,13 +14,11 @@
 # Colors used below:
 # green    '\033[0;32m'
 # red      '\033[0;31m'
-# yellow   '\033[0;33m'
 # no color '\033[0m'
 COLOR_PRINT = if [ -t 1 ]; then \
    printf $(1)$(2)'\033[0m'$(3); else printf $(2)$(3); fi
 PRINT_OK = $(call COLOR_PRINT,'\033[0;32m',OK,"  ($$1 $$2)\n")
 PRINT_FAILED = $(call COLOR_PRINT,'\033[0;31m',FAILED,"  ($$1 $$2)\n")
-PRINT_SKIP = $(call COLOR_PRINT,'\033[0;33m',SKIP,"\n")
 
 # Timing support
 define TIMECMD_detect

doc/CodeDocumentation.conf.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v3.4.1
+PROJECT_NUMBER         = v3.3.3
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -761,7 +761,6 @@ WARN_LOGFILE           =
 INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/mfem.hpp \
                          @MFEM_SOURCE_DIR@/backends/base \
-                         @MFEM_SOURCE_DIR@/backends/occa \
                          @MFEM_SOURCE_DIR@/config \
                          @MFEM_SOURCE_DIR@/general \
                          @MFEM_SOURCE_DIR@/linalg \
@@ -769,7 +768,6 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/fem \
                          @MFEM_SOURCE_DIR@/examples \
                          @MFEM_SOURCE_DIR@/examples/petsc \
-                         @MFEM_SOURCE_DIR@/examples/pumi \
                          @MFEM_SOURCE_DIR@/examples/sundials \
                          @MFEM_SOURCE_DIR@/miniapps/common \
                          @MFEM_SOURCE_DIR@/miniapps/meshing \

doc/CodeDocumentation.dox

@@ -36,8 +36,8 @@ namespace mfem {
  * - HypreSolver and other \link hypre.hpp hypre classes\endlink
  *
  * <H3>Example codes</H3>
- * - <a class="el" href="examples_2ex1_8cpp_source.html">Example 1</a>: nodal H1 FEM for the Laplace problem
- * - <a class="el" href="examples_2ex1p_8cpp_source.html">Example 1p</a>: parallel nodal H1 FEM for the Laplace problem
+ * - <a class="el" href="ex1_8cpp_source.html">Example 1</a>: nodal H1 FEM for the Laplace problem
+ * - <a class="el" href="ex1p_8cpp_source.html">Example 1p</a>: parallel nodal H1 FEM for the Laplace problem
  * - <a class="el" href="ex2_8cpp_source.html">Example 2</a>: vector FEM for linear elasticity
  * - <a class="el" href="ex2p_8cpp_source.html">Example 2p</a>: parallel vector FEM for linear elasticity
  * - <a class="el" href="ex3_8cpp_source.html">Example 3</a>: Nedelec H(curl) FEM for the definite Maxwell problem
@@ -56,7 +56,7 @@ namespace mfem {
  * - <a class="el" href="ex9p_8cpp_source.html">Example 9p</a>: parallel Discontinuous Galerkin (DG) time-dependent advection
  * - <a class="el" href="ex10_8cpp_source.html">Example 10</a>: time-dependent implicit nonlinear elasticity
  * - <a class="el" href="ex10p_8cpp_source.html">Example 10p</a>: parallel time-dependent implicit nonlinear elasticity
- * - <a class="el" href="examples_2ex11p_8cpp_source.html">Example 11p</a>: parallel Laplace eigensolver
+ * - <a class="el" href="ex11p_8cpp_source.html">Example 11p</a>: parallel Laplace eigensolver
  * - <a class="el" href="ex12p_8cpp_source.html">Example 12p</a>: parallel linear elasticity eigensolver
  * - <a class="el" href="ex13p_8cpp_source.html">Example 13p</a>: parallel Maxwell eigensolver
  * - <a class="el" href="ex14_8cpp_source.html">Example 14</a>: Discontinuous Galerkin (DG) for the Laplace problem
@@ -96,15 +96,6 @@ namespace mfem {
  *   <a class="el" href="petsc_2ex10p_8cpp_source.html">10p</a>
  *   demonstrating the use of MFEM's \link petsc.hpp PETSc classes\endlink
  *
- * <H4>PUMI Examples</H4>
- * - Variants of Examples
- *   <a class="el" href="examples_2pumi_2ex1_8cpp_source.html">1</a>,
- *   <a class="el" href="examples_2pumi_2ex1p_8cpp_source.html">1p</a>,
- *   <a class="el" href="pumi_2ex2_8cpp_source.html">2</a>,
- *   and
- *   <a class="el" href="pumi_2ex6p_8cpp_source.html">6p</a>
- *   demonstrating the use of MFEM's \link pumi.hpp PUMI classes\endlink
- *
  * <H3>Miniapps</H3>
  * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
  * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code

examples/CMakeLists.txt

@@ -25,7 +25,6 @@ list(APPEND ALL_EXE_SRCS
   ex16.cpp
   ex17.cpp
   ex18.cpp
-  ex19.cpp
   )
 
 if (MFEM_USE_MPI)
@@ -48,7 +47,6 @@ if (MFEM_USE_MPI)
     ex16p.cpp
     ex17p.cpp
     ex18p.cpp
-    ex19p.cpp
     )
 endif()
 
@@ -63,6 +61,8 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
   get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
   string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
 
+  string(FIND ${TEST_NAME} "p" is_parallel_test)
+
   set(THIS_TEST_OPTIONS "-no-vis")
   if (${TEST_NAME} MATCHES "ex10p*")
     list(APPEND THIS_TEST_OPTIONS "-tf" "5")
@@ -70,7 +70,7 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
     list(APPEND THIS_TEST_OPTIONS "-e" "1")
   endif()
 
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+  if (is_parallel_test EQUAL -1)
     add_test(NAME ${TEST_NAME}_ser
       COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
   else()
@@ -91,8 +91,3 @@ endif()
 if (MFEM_USE_PETSC)
   add_subdirectory(petsc)
 endif()
-
-# Include the examples/pumi directory if PUMI is enabled
-if (MFEM_USE_PUMI)
-  add_subdirectory(pumi)
-endif()

examples/ex16d.cpp

@@ -1,441 +0,0 @@
-//                                MFEM Example 16
-//
-// Compile with: make ex16
-//
-// Sample runs:  ex16
-//               ex16 -m ../data/inline-tri.mesh
-//               ex16 -m ../data/disc-nurbs.mesh -tf 2
-//               ex16 -s 1 -a 0.0 -k 1.0
-//               ex16 -s 2 -a 1.0 -k 0.0
-//               ex16 -s 3 -a 0.5 -k 0.5 -o 4
-//               ex16 -s 14 -dt 1.0e-4 -tf 4.0e-2 -vs 40
-//               ex16 -m ../data/fichera-q2.mesh
-//               ex16 -m ../data/escher.mesh
-//               ex16 -m ../data/beam-tet.mesh -tf 10 -dt 0.1
-//               ex16 -m ../data/amr-quad.mesh -o 4 -r 0
-//               ex16 -m ../data/amr-hex.mesh -o 2 -r 0
-//
-// Description:  This example solves a time dependent nonlinear heat equation
-//               problem of the form du/dt = C(u), with a non-linear diffusion
-//               operator C(u) = \nabla \cdot (\kappa + \alpha u) \nabla u.
-//
-//               The example demonstrates the use of nonlinear operators (the
-//               class ConductionOperator defining C(u)), as well as their
-//               implicit time integration. Note that implementing the method
-//               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration.
-//
-//               We recommend viewing examples 2, 9 and 10 before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-/** After spatial discretization, the conduction model can be written as:
- *
- *     du/dt = M^{-1}(-Ku)
- *
- *  where u is the vector representing the temperature, M is the mass matrix,
- *  and K is the diffusion operator with diffusivity depending on u:
- *  (\kappa + \alpha u).
- *
- *  Class ConductionOperator represents the right-hand side of the above ODE.
- */
-class ConductionOperator : public TimeDependentOperator
-{
-protected:
-   FiniteElementSpace &fespace;
-   Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
-
-   BilinearForm M;
-   BilinearForm K;
-
-   GridFunction u_alpha_gf;
-   GridFunctionCoefficient u_coeff;
-
-   OperatorHandle Moper, Koper;
-   Operator *T; // T = M + dt K
-   double current_dt;
-
-   CGSolver M_solver; // Krylov solver for inverting the mass matrix M
-   // FIXME: add the preconditioner
-   // DSmoother M_prec;  // Preconditioner for the mass matrix M
-
-   CGSolver T_solver; // Implicit solver for T = M + dt K
-   // FIXME: add the preconditioner
-   // DSmoother T_prec;  // Preconditioner for the implicit solver
-
-   double alpha, kappa;
-
-   mutable Vector z; // auxiliary vector
-
-public:
-   ConductionOperator(FiniteElementSpace &f, const char *oper_spec,
-                      double alpha, double kappa, const Vector &u);
-
-   virtual void Mult(const Vector &u, Vector &du_dt) const;
-   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
-       This is the only requirement for high-order SDIRK implicit integration.*/
-   virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);
-
-   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
-   void SetParameters(const Vector &u);
-
-   virtual ~ConductionOperator();
-};
-
-
-class TimeDerivativeOperator : public Operator
-{
-   Operator *Moper;
-   Operator *Koper;
-   mutable Vector Kdu;
-   const double dt;
-
-public:
-   // FIXME: Sparse matrices should be changed to have PLayouts
-   // allocated so that this constructor works even when Moper and
-   // Koper were not created from engines.
-   TimeDerivativeOperator(Operator *_Moper, const double _dt, Operator *_Koper)
-      : Operator(*_Koper->InLayout(), *_Moper->OutLayout()),
-        Moper(_Moper),
-        Koper(_Koper),
-        Kdu(_Moper->OutLayout()),
-        dt(_dt) { }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      Moper->Mult(x, y);
-      Koper->Mult(x, Kdu);
-
-      y.Axpby(1.0, y, dt, Kdu);
-   }
-};
-
-double InitialTemperature(const Vector &x);
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int ref_levels = 2;
-   int order = 2;
-   int ode_solver_type = 3;
-   double t_final = 0.5;
-   double dt = 1.0e-2;
-   double alpha = 1.0e-2;
-   double kappa = 0.5;
-   bool visualization = true;
-   bool visit = false;
-   int vis_steps = 5;
-   const char *oper_spec = "representation: 'partial'";
-   const char *occa_spec = "mode: 'Serial'";
-
-   int precision = 8;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
-                  "\t   11 - Forward Euler, 12 - RK2, 13 - RK3 SSP, 14 - RK4.");
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time step.");
-   args.AddOption(&alpha, "-a", "--alpha",
-                  "Alpha coefficient.");
-   args.AddOption(&kappa, "-k", "--kappa",
-                  "Kappa coefficient offset.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.AddOption(&oper_spec, "-s", "--oper-spec", "Operator specification");
-   args.AddOption(&occa_spec, "-os", "--occa-spec", "OCCA engine specification");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // Examples for OCCA specifications:
-   //   - CPU (serial): "mode: 'Serial'"
-   //   - CUDA GPU: "mode: 'CUDA', device_id: 0"
-   //   - OpenMP on CPUs: "mode: 'OpenMP', threads: 4"
-   //   - OpenCL on device 0: "mode: 'OpenCL', device_id: 0, platform_id: 0"
-
-   SharedPtr<Engine> engine(new mfem::occa::Engine(occa_spec));
-
-   // 2. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   mesh->SetEngine(*engine);
-   int dim = mesh->Dimension();
-
-   // 3. Define the ODE solver used for time integration. Several implicit
-   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
-   //    explicit Runge-Kutta methods are available.
-   ODESolver *ode_solver;
-   switch (ode_solver_type)
-   {
-      // Implicit L-stable methods
-      case 1:  ode_solver = new BackwardEulerSolver; break;
-      case 2:  ode_solver = new SDIRK23Solver(2); break;
-      case 3:  ode_solver = new SDIRK33Solver; break;
-      // Explicit methods
-      case 11: ode_solver = new ForwardEulerSolver; break;
-      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 13: ode_solver = new RK3SSPSolver; break;
-      case 14: ode_solver = new RK4Solver; break;
-      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
-      // Implicit A-stable methods (not L-stable)
-      case 22: ode_solver = new ImplicitMidpointSolver; break;
-      case 23: ode_solver = new SDIRK23Solver; break;
-      case 24: ode_solver = new SDIRK34Solver; break;
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         delete mesh;
-         return 3;
-   }
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
-   //    command-line parameter.
-   for (int lev = 0; lev < ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 5. Define the vector finite element space representing the current and the
-   //    initial temperature, u_ref.
-   H1_FECollection fe_coll(order, dim);
-   FiniteElementSpace fespace(mesh, &fe_coll);
-
-   int fe_size = fespace.GetTrueVSize();
-   cout << "Number of temperature unknowns: " << fe_size << endl;
-
-   GridFunction u_gf(&fespace);
-
-   // 6. Set the initial conditions for u. All boundaries are considered
-   //    natural. This computes this on the host, so pull/push is needed.
-   u_gf.Pull();
-   FunctionCoefficient u_0(InitialTemperature);
-   u_gf.ProjectCoefficient(u_0);
-   u_gf.Push();
-   Vector u;
-   u_gf.GetTrueDofs(u);
-
-   // 7. Initialize the conduction operator and the visualization.
-   ConductionOperator oper(fespace, oper_spec, alpha, kappa, u);
-
-   u_gf.SetFromTrueDofs(u);
-   {
-      ofstream omesh("ex16.mesh");
-      omesh.precision(precision);
-      mesh->Print(omesh);
-      ofstream osol("ex16-init.gf");
-      osol.precision(precision);
-      u_gf.Pull(); // pull back to host before saving
-      u_gf.Save(osol);
-   }
-
-   VisItDataCollection visit_dc("Example16", mesh);
-   visit_dc.RegisterField("temperature", &u_gf);
-   if (visit)
-   {
-      visit_dc.SetCycle(0);
-      visit_dc.SetTime(0.0);
-      visit_dc.Save();
-   }
-
-   socketstream sout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      if (!sout)
-      {
-         cout << "Unable to connect to GLVis server at "
-              << vishost << ':' << visport << endl;
-         visualization = false;
-         cout << "GLVis visualization disabled.\n";
-      }
-      else
-      {
-         sout.precision(precision);
-         sout << "solution\n" << *mesh << u_gf;
-         sout << "pause\n";
-         sout << flush;
-         cout << "GLVis visualization paused."
-              << " Press space (in the GLVis window) to resume it.\n";
-      }
-   }
-
-   // 8. Perform time-integration (looping over the time iterations, ti, with a
-   //    time-step dt).
-   ode_solver->Init(oper);
-   double t = 0.0;
-
-   bool last_step = false;
-   for (int ti = 1; !last_step; ti++)
-   {
-      if (t + dt >= t_final - dt/2)
-      {
-         last_step = true;
-      }
-
-      ode_solver->Step(u, t, dt);
-
-      if (last_step || (ti % vis_steps) == 0)
-      {
-         cout << "step " << ti << ", t = " << t << endl;
-
-         // u_gf and u are both on the device at this point.
-         u_gf.SetFromTrueDofs(u);
-         if (visualization)
-         {
-            u_gf.Pull(); // pull back to host before saving
-            sout << "solution\n" << *mesh << u_gf << flush;
-         }
-
-         if (visit)
-         {
-            visit_dc.SetCycle(ti);
-            visit_dc.SetTime(t);
-            visit_dc.Save();
-         }
-      }
-      oper.SetParameters(u);
-   }
-
-   // 9. Save the final solution. This output can be viewed later using GLVis:
-   //    "glvis -m ex16.mesh -g ex16-final.gf".
-   {
-      ofstream osol("ex16-final.gf");
-      osol.precision(precision);
-      u_gf.Pull(); // pull back to host before saving
-      u_gf.Save(osol);
-   }
-
-   // 10. Free the used memory.
-   delete ode_solver;
-   delete mesh;
-
-   return 0;
-}
-
-ConductionOperator::ConductionOperator(FiniteElementSpace &f,
-                                       const char *oper_spec, double al,
-                                       double kap, const Vector &u)
-   : TimeDependentOperator(*f.GetTrueVLayout()), fespace(f), M(&fespace),
-     K(&fespace), u_alpha_gf(&f), u_coeff(&u_alpha_gf), Moper(oper_spec),
-     Koper(oper_spec), T(NULL), current_dt(0.0), z(f.GetTrueVLayout())
-{
-   const double rel_tol = 1e-8;
-
-   M.AddDomainIntegrator(new MassIntegrator());
-   M.Assemble();
-   M.FormSystemMatrix(ess_tdof_list, Moper);
-
-   K.AddDomainIntegrator(new DiffusionIntegrator(u_coeff));
-
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(rel_tol);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(200);
-   M_solver.SetPrintLevel(0);
-   // M_solver.SetPreconditioner(M_prec);
-   M_solver.SetOperator(*Moper.Ptr());
-
-   alpha = al;
-   kappa = kap;
-
-   T_solver.iterative_mode = false;
-   T_solver.SetRelTol(rel_tol);
-   T_solver.SetAbsTol(0.0);
-   T_solver.SetMaxIter(200);
-   T_solver.SetPrintLevel(0);
-   // T_solver.SetPreconditioner(T_prec);
-
-   SetParameters(u);
-}
-
-void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
-{
-   // Compute:
-   //    du_dt = M^{-1}*-K(u)
-   // for du_dt
-   Koper.Ptr()->Mult(u, z);
-   z.Axpby(-1.0, z, 0.0, z);
-   M_solver.Mult(z, du_dt);
-}
-
-void ConductionOperator::ImplicitSolve(const double dt,
-                                       const Vector &u, Vector &du_dt)
-{
-   // Solve the equation:
-   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt
-   if (!T)
-   {
-      T = new TimeDerivativeOperator(Moper.Ptr(), dt, Koper.Ptr());
-      current_dt = dt;
-      T_solver.SetOperator(*T);
-   }
-   MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt
-   Koper.Ptr()->Mult(u, z);
-   z.Axpby(-1.0, z, 0.0, z);
-   T_solver.Mult(z, du_dt);
-}
-
-void ConductionOperator::SetParameters(const Vector &u)
-{
-   u_alpha_gf.SetFromTrueDofs(u);
-   u_alpha_gf.Pull();
-   for (int i = 0; i < u_alpha_gf.Size(); i++)
-   {
-      u_alpha_gf(i) = kappa + alpha*u_alpha_gf(i);
-   }
-   u_alpha_gf.Push();
-
-   // Reassemble after changing u_alpha_gf (and hence u_coeff)...
-   K.Assemble();
-   K.FormSystemMatrix(ess_tdof_list, Koper);
-
-   delete T;
-   T = NULL; // re-compute T on the next ImplicitSolve
-}
-
-ConductionOperator::~ConductionOperator()
-{
-   delete T;
-}
-
-double InitialTemperature(const Vector &x)
-{
-   if (x.Norml2() < 0.5)
-   {
-      return 2.0;
-   }
-   else
-   {
-      return 1.0;
-   }
-}

examples/ex19.cpp

@@ -50,7 +50,7 @@ using namespace mfem;
 //
 // and K^-1 is an approximation of the inverse of the displacement part of the
 // Jacobian and S^-1 is an approximation of the inverse of the Schur
-// complement S = B K^-1 B^T. The Schur complement is approximated using
+// complement S = B K^-1 B^T. The Schur complement is approximiated using
 // a mass matrix of the pressure variables.
 class JacobianPreconditioner : public Solver
 {

examples/ex19p.cpp

@@ -50,7 +50,7 @@ using namespace mfem;
 //
 // and K^-1 is an approximation of the inverse of the displacement part of the
 // Jacobian and S^-1 is an approximation of the inverse of the Schur
-// complement S = B K^-1 B^T. The Schur complement is approximated using
+// complement S = B K^-1 B^T. The Schur complement is approximiated using
 // a mass matrix of the pressure variables.
 class JacobianPreconditioner : public Solver
 {

examples/ex1d.cpp

@@ -12,25 +12,28 @@ int main(int argc, char *argv[])
    const char *spec = "cpu";
    const char *mesh_file = "../data/star.mesh";
    int order = 1;
-   int ref_levels = -1;
    bool static_cond = false;
    bool visualization = 1;
+   const char *engine_type = "omp";
+   const char *engine_spec = "mult_engine:'acrotensor', exec_target:'device', mem_type:'unified'";
+   int ref_levels = -1;
 
    OptionsParser args(argc, argv);
    args.AddOption(&spec, "-s", "--spec",
-                  "Compute resource specification.");
+                  "Compute resurce specification.");
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree) or -1 for"
                   " isoparametric space.");
-   args.AddOption(&ref_levels, "-r", "--refine-levels",
-                  "Number of uniform refinements to apply to the mesh.");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
+   args.AddOption(&engine_type, "-et", "--engine-type", "Engine type");
+   args.AddOption(&engine_spec, "-es", "--engine-spec", "Engine specification");
+   args.AddOption(&ref_levels, "-r", "--refs", "Number of uniform refinements (negative implies dof ~ 50000)");
    args.Parse();
    if (!args.Good())
    {
@@ -39,29 +42,29 @@ int main(int argc, char *argv[])
    }
    args.PrintOptions(cout);
 
-#ifdef MFEM_USE_BACKENDS
    /// Engine *engine = EngineDepot.Select(spec);
 
-   // string occa_spec("mode: 'Serial'");
-   string occa_spec("mode: 'CUDA', device_id: 0");
-   // string occa_spec("mode: 'OpenMP', threads: 4");
-   // string occa_spec("mode: 'OpenCL', device_id: 0, platform_id: 0");
+   // string spec("mode: 'Serial'");
+   // string spec("mode: 'CUDA', deviceID: 0");
+   // string spec("mode: 'OpenMP', threads: 4");
+   // string spec("mode: 'OpenCL', deviceID: 0, platformID: 0");
+   // SharedPtr<Engine> engine(new mfem::occa::Engine(spec));
 
-   // The following flag affects only 'Serial' and 'OpenMP' modes.
-   // In 'CUDA' mode, '-O3' affects only host code.
-   // In 'OpenCL' mode, adding '-O3' breaks compilation.
-   // occa_spec += ", kernel: { compiler_flags: '-O3' }";
-
-   SharedPtr<Engine> engine(new mfem::occa::Engine(occa_spec));
-#endif
+   SharedPtr<Engine> engine;
+   if (!strncmp(engine_type, "omp", 3))
+   {
+      engine.Reset(new mfem::omp::Engine(engine_spec));
+   }
+   else if (!strncmp(engine_type, "occa", 4))
+   {
+      engine.Reset(new mfem::occa::Engine(engine_spec));
+   }
 
    // 2. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
    //    the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
-#ifdef MFEM_USE_BACKENDS
    mesh->SetEngine(*engine);
-#endif
    int dim = mesh->Dimension();
 
    // 3. Refine the mesh to increase the resolution. In this example we do
@@ -69,8 +72,7 @@ int main(int argc, char *argv[])
    //    largest number that gives a final mesh with no more than 50,000
    //    elements.
    {
-      ref_levels = ref_levels >= 0 ? ref_levels :
-         (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
+      if (ref_levels < 0) ref_levels = (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
          mesh->UniformRefinement();
@@ -144,12 +146,7 @@ int main(int argc, char *argv[])
    cout << "Size of linear system: " << A.Ptr()->Height() << endl;
 
    // 10. Solve the system A X = B with CG.
-   tic_toc.Clear();
-   tic_toc.Start();
-   const int print_level = 3;
-   CG(*A.Ptr(), B, X, print_level, 1000, 1e-12, 0.0);
-   tic_toc.Stop();
-   cout << "CG time: " << tic_toc.RealTime() << " sec." << endl;
+   CG(*A.Ptr(), B, X, 3, 500, 1e-12, 0.0);
 
    // 11. Recover the solution as a finite element grid function.
    a->RecoverFEMSolution(X, *b, x);

examples/ex1pd.cpp

@@ -16,19 +16,18 @@ int main(int argc, char *argv[])
 
    // 2. Parse command-line options.
    const char *mesh_file = "../data/star.mesh";
-   int ser_ref_levels = -1;
-   int par_ref_levels = -1;
    int order = 1;
    bool static_cond = false;
    bool visualization = 1;
+   const char *engine_type = "omp";
+   const char *engine_spec = "mult_engine:'acrotensor', exec_target:'device', mem_type:'unified'";
+   int serial_ref_levels = -1;
+   int parallel_ref_levels = 2;
+   bool use_preconditioner = false;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree) or -1 for"
                   " isoparametric space.");
@@ -37,6 +36,11 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
+   args.AddOption(&engine_type, "-et", "--engine-type", "Engine type");
+   args.AddOption(&engine_spec, "-es", "--engine-spec", "Engine specification");
+   args.AddOption(&serial_ref_levels, "-sr", "--serial-refs", "Number of serial uniform refinements (negative implies dof ~ 10000)");
+   args.AddOption(&parallel_ref_levels, "-pr", "--parallel-refs", "Number of parallel uniform refinements");
+   args.AddOption(&use_preconditioner, "-prec", "--use-preconditioner", "-no-prec", "--no-preconditioner", "Enable HYPRE AMG preconditioner");
    args.Parse();
    if (!args.Good())
    {
@@ -52,31 +56,29 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-#ifdef MFEM_USE_BACKENDS
    /// Engine *engine = EngineDepot.Select(spec);
 
    // string occa_spec("mode: 'Serial'");
-   string occa_spec;
-   {
-      stringstream occa_spec_ss;
-      occa_spec_ss << "mode: 'CUDA', device_id: 0";
-      // const int nGPUs = 4;
-      // occa_spec_ss << "mode: 'CUDA', device_id: " << (myid % nGPUs);
-      occa_spec = occa_spec_ss.str();
-   }
+   // string occa_spec("mode: 'CUDA', deviceID: 0");
    // string occa_spec("mode: 'OpenMP', threads: 4");
-   // string occa_spec("mode: 'OpenCL', device_id: 0, platform_id: 0");
+   // string occa_spec("mode: 'OpenCL', deviceID: 0, platformID: 0");
+   // SharedPtr<Engine> engine(new mfem::occa::Engine(MPI_COMM_WORLD, occa_spec));
 
-   SharedPtr<Engine> engine(new mfem::occa::Engine(MPI_COMM_WORLD, occa_spec));
-#endif
+   SharedPtr<Engine> engine;
+   if (!strncmp(engine_type, "omp", 3))
+   {
+      engine.Reset(new mfem::omp::Engine(MPI_COMM_WORLD, engine_spec));
+   }
+   else if (!strncmp(engine_type, "occa", 4))
+   {
+      engine.Reset(new mfem::occa::Engine(MPI_COMM_WORLD, engine_spec));
+   }
 
    // 3. Read the (serial) mesh from the given mesh file on all processors.  We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
-#ifdef MFEM_USE_BACKENDS
    mesh->SetEngine(*engine);
-#endif
    int dim = mesh->Dimension();
 
    // 4. Refine the serial mesh on all processors to increase the resolution. In
@@ -84,13 +86,9 @@ int main(int argc, char *argv[])
    //    'ref_levels' to be the largest number that gives a final mesh with no
    //    more than 10,000 elements.
    {
-      int ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
-      ref_levels = ser_ref_levels >= 0 ? ser_ref_levels : ref_levels;
-      if (myid == 0)
-      {
-         cout << "Serial refinement levels: " << ref_levels << endl;
-      }
-      for (int l = 0; l < ref_levels; l++)
+      if (serial_ref_levels < 0)
+         serial_ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < serial_ref_levels; l++)
       {
          mesh->UniformRefinement();
       }
@@ -102,18 +100,11 @@ int main(int argc, char *argv[])
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
    {
-      par_ref_levels = par_ref_levels >= 0 ? par_ref_levels : 2;
-      if (myid == 0)
-      {
-         cout << "Parallel refinement levels: " << par_ref_levels << endl;
-      }
-      for (int l = 0; l < par_ref_levels; l++)
+      for (int l = 0; l < parallel_ref_levels; l++)
       {
          pmesh->UniformRefinement();
       }
    }
-   pmesh->PrintInfo(cout);
-   if (myid == 0) { cout << endl; }
 
    // 6. Define a parallel finite element space on the parallel mesh. Here we
    //    use continuous Lagrange finite elements of the specified order. If
@@ -181,51 +172,36 @@ int main(int argc, char *argv[])
    if (static_cond) { a->EnableStaticCondensation(); }
    a->Assemble();
 
-   OperatorHandle A(Operator::ANY_TYPE);
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
    CGSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+   HypreSolver *amg = NULL;
    pcg->SetRelTol(1e-6);
    pcg->SetAbsTol(0.0);
-   pcg->SetMaxIter(1000);
-   pcg->SetPrintLevel(3);
-   pcg->SetOperator(*A.Ptr());
+   pcg->SetMaxIter(500);
+   pcg->SetPrintLevel(1);
 
-   // Run one CG iteration to make sure all kernels are loaded before measuring
-   // time.
-   if (myid == 0)
+   Vector B, X;
+   if (!use_preconditioner)
    {
-      cout << "Running 1 CG iteration to load all kernels ..." << flush;
-   }
-   {
-      Vector X2(X);
-      pcg->SetMaxIter(1);
-      pcg->SetPrintLevel(-1);
-      pcg->Mult(B, X2);
-      pcg->SetMaxIter(1000);
-      pcg->SetPrintLevel(3);
-   }
-   if (myid == 0)
-   {
-      cout << " done." << endl;
-   }
+      OperatorHandle A(Operator::ANY_TYPE);
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
 
-   double start_time = MPI_Wtime();
-   pcg->Mult(B, X);
-   double end_time = MPI_Wtime();
-   double loc_time = end_time - start_time;
-   double max_time, min_time;
-   MPI_Allreduce(&loc_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
-   MPI_Allreduce(&loc_time, &min_time, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);
-   if (myid == 0)
-   {
-      cout << "CG time: " << max_time << " sec (min: " << min_time << " sec)\n"
-           << "DOFs/sec in CG: "
-           << 1e-6*size*pcg->GetNumIterations()/max_time << " ("
-           << 1e-6*size*pcg->GetNumIterations()/min_time << ") million.\n"
-           << endl;
+      pcg->SetOperator(*A.Ptr());
+      pcg->Mult(B, X);
    }
+   else
+   {
+      HypreParMatrix A;
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+      amg = new HypreBoomerAMG(A);
+      std::cout << "operator size: " << A.InLayout()->Size() << " " << A.OutLayout()->Size() << std::endl;
+      pcg->SetOperator(A);
+      pcg->SetPreconditioner(*amg);
+      Vector X_backend(*X.Get_PVector()), B_backend(*B.Get_PVector());
+      pcg->Mult(B_backend, X_backend);
+      delete amg;
+   }
+   delete pcg;
 
    // 13. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.
@@ -260,7 +236,6 @@ int main(int argc, char *argv[])
    }
 
    // 16. Free the used memory.
-   delete pcg;
    delete a;
    delete b;
    delete fespace;

examples/makefile

@@ -38,9 +38,6 @@ endif
 ifeq ($(MFEM_USE_PETSC),YES)
    SUBDIRS += petsc
 endif
-ifeq ($(MFEM_USE_PUMI),YES)
-   SUBDIRS += pumi
-endif
 SUBDIRS_ALL = $(addsuffix /all,$(SUBDIRS))
 SUBDIRS_TEST = $(addsuffix /test,$(SUBDIRS))
 SUBDIRS_CLEAN = $(addsuffix /clean,$(SUBDIRS))

examples/petsc/CMakeLists.txt

@@ -63,29 +63,31 @@ add_mfem_examples(PETSC_EXAMPLES_SRCS ${PFX} copy_petsc_rc_files test_petsc)
 #   ctest -R petsc
 
 # Command line options for the tests.
-set(EX1_ARGS_W       -m ../../data/amr-quad.mesh --usepetsc)
-set(EX1_ARGS_P       -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_ex1p)
-set(EX2_ARGS         -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p)
-set(EX3_ARGS         -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping)
-set(EX4_ARGS         -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping)
-set(EX4_HYB_ARGS     -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping --hybridization)
-set(EX5_BDDC_LB_ARGS -m ../../data/star.mesh --usepetsc -o 0 --petscopts rc_ex5p_bddc --nonoverlapping --local-bdr)
-set(EX5_BDDC_GB_ARGS -m ../../data/star.mesh --usepetsc -o 0 --petscopts rc_ex5p_bddc --nonoverlapping)
-set(EX5_FSPL_ARGS    -m ../../data/beam-tet.mesh --usepetsc -o 0 --petscopts rc_ex5p_fieldsplit)
-set(EX6_ARGS         -m ../../data/amr-quad.mesh --usepetsc)
-set(EX6_NONOVL_ARGS  -m ../../data/amr-quad.mesh --usepetsc --nonoverlapping)
-set(EX9_E_ARGS       -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl -dt 0.1)
-set(EX9_ES_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step)
-set(EX9_IS_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5)
-set(EX10_ARGS        -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3)
+set(EX1P_ARGS -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_ex1p)
+set(EX2P_ARGS -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p)
+set(EX3P_ARGS -m ../../data/klein-bottle.mesh
+    -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping)
+set(EX4P_ARGS -m ../../data/klein-bottle.mesh
+    -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping)
+set(EX5P_BDDC_ARGS -m ../../data/star.mesh
+    --usepetsc --petscopts rc_ex5p_bddc --nonoverlapping)
+set(EX5P_FSPL_ARGS -m ../../data/beam-tet.mesh
+    --usepetsc --petscopts rc_ex5p_fieldsplit)
+set(EX6P_ARGS -m ../../data/amr-quad.mesh --usepetsc)
+set(EX9P_E_ARGS -m ../../data/periodic-hexagon.mesh
+    --usepetsc --petscopts rc_ex9p_expl -dt 0.1)
+set(EX9P_ES_ARGS -m ../../data/periodic-hexagon.mesh
+    --usepetsc --petscopts rc_ex9p_expl --no-step)
+set(EX9P_IS_ARGS -m ../../data/periodic-hexagon.mesh
+    --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5)
+set(EX10P_ARGS -m ../../data/beam-quad.mesh
+    -tf 30 -s 3 -rs 2 -dt 3 --usepetsc --petscopts rc_ex10p)
 
 # Add the tests: one test per command-line-variable.
 foreach(TEST_OPTIONS_VAR
-        EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
-        EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
-        EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
+        EX1P_ARGS EX2P_ARGS EX3P_ARGS EX4P_ARGS EX5P_BDDC_ARGS EX5P_FSPL_ARGS
+        EX6P_ARGS EX9P_E_ARGS EX9P_ES_ARGS EX9P_IS_ARGS EX10P_ARGS)
   string(REGEX REPLACE "^(.+)_ARGS" "\\1" TEST_NAME_UC ${TEST_OPTIONS_VAR})
-  string(REGEX REPLACE "^([^_]+)" "\\1P" TEST_NAME_UC ${TEST_NAME_UC})
   string(TOLOWER ${TEST_NAME_UC} TEST_NAME_FULL)
   string(REGEX REPLACE "^([^_]+).*" "\\1" TEST_NAME ${TEST_NAME_FULL})
   set(TEST_NAME_FULL ${PFX}${TEST_NAME_FULL})

examples/petsc/ex5p.cpp

@@ -64,7 +64,6 @@ int main(int argc, char *argv[])
    bool visualization = 1;
    bool use_petsc = true;
    bool use_nonoverlapping = false;
-   bool local_bdr_spec = false;
    const char *petscrc_file = "";
 
    OptionsParser args(argc, argv);
@@ -87,9 +86,6 @@ int main(int argc, char *argv[])
                   "-no-nonoverlapping", "--no-nonoverlapping",
                   "Use or not the block diagonal PETSc's matrix format "
                   "for non-overlapping domain decomposition.");
-   args.AddOption(&local_bdr_spec, "-local-bdr", "--local-bdr", "-no-local-bdr",
-                  "--no-local-bdr",
-                  "Specify boundary dofs in local (Vdofs) ordering.");
    args.Parse();
    if (!args.Good())
    {
@@ -310,38 +306,33 @@ int main(int argc, char *argv[])
    {
       if (use_nonoverlapping)
       {
-         PetscBDDCSolverParams opts;
-
          // For saddle point problems, we need to provide BDDC the list of
          // boundary dofs either essential or natural.
          // Since R_space is the only space that may have boundary dofs and it
          // is ordered first then W_space, we don't need any local offset when
          // specifying the dofs.
          Array<int> bdr_tdof_list;
+         bool local = false;
          if (pmesh->bdr_attributes.Size())
          {
             Array<int> bdr(pmesh->bdr_attributes.Max());
             bdr = 1;
 
-            if (!local_bdr_spec)
-            {
-               // Essential dofs in global ordering
-               R_space->GetEssentialTrueDofs(bdr, bdr_tdof_list);
-            }
-            else
-            {
-               // Alternatively, you can also provide the list of dofs in local
-               // ordering
-               R_space->GetEssentialVDofs(bdr, bdr_tdof_list);
-               bdr_tdof_list.SetSize(R_space->GetVSize()+W_space->GetVSize(),0);
-            }
-            opts.SetNatBdrDofs(&bdr_tdof_list,local_bdr_spec);
+            R_space->GetEssentialTrueDofs(bdr, bdr_tdof_list);
+            local = false;
+            // Alternatively, you can also provide the list of dofs in local
+            // ordering:
+            // R_space->GetEssentialVDofs(bdr, bdr_tdof_list);
+            // bdr_tdof_list.SetSize(R_space->GetVSize()+W_space->GetVSize(),0);
+            // local = true;
          }
          else
          {
-            MFEM_WARNING("Missing boundary dofs. This may cause solver failures.");
+            MFEM_ABORT("Need to know the boundary dofs");
          }
 
+         PetscBDDCSolverParams opts;
+         opts.SetNatBdrDofs(&bdr_tdof_list,local);
          // See also command line options rc_ex5p_bddc
          pdarcyPr = new PetscBDDCSolver(MPI_COMM_WORLD,*darcyOp,opts,"prec_");
       }

examples/petsc/makefile

@@ -69,21 +69,18 @@ TESTNAME = Parallel PETSc example
 
 
 # Testing PETSc execution options.
-EX1_ARGS_W       := -m ../../data/amr-quad.mesh --usepetsc
-EX1_ARGS_P       := -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_ex1p
-EX2_ARGS         := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p
-EX3_ARGS         := -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping
-EX4_ARGS         := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping
-EX4_HYB_ARGS     := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping --hybridization
-EX5_BDDC_LB_ARGS := -m ../../data/star.mesh --usepetsc -o 0 --petscopts rc_ex5p_bddc --nonoverlapping --local-bdr
-EX5_BDDC_GB_ARGS := -m ../../data/star.mesh --usepetsc -o 0 --petscopts rc_ex5p_bddc --nonoverlapping
-EX5_FSPL_ARGS    := -m ../../data/beam-tet.mesh --usepetsc -o 0 --petscopts rc_ex5p_fieldsplit
-EX6_ARGS         := -m ../../data/amr-quad.mesh --usepetsc
-EX6_NONOVL_ARGS  := -m ../../data/amr-quad.mesh --usepetsc --nonoverlapping
-EX9_E_ARGS       := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl -dt 0.1
-EX9_ES_ARGS      := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step
-EX9_IS_ARGS      := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5
-EX10_ARGS        := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3
+EX1_ARGS_W    := -m ../../data/amr-quad.mesh --usepetsc
+EX1_ARGS_P    := -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_ex1p
+EX2_ARGS      := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p
+EX3_ARGS      := -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping
+EX4_ARGS      := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping
+EX5_BDDC_ARGS := -m ../../data/star.mesh --usepetsc -o 0 --petscopts rc_ex5p_bddc --nonoverlapping
+EX5_FSPL_ARGS := -m ../../data/beam-tet.mesh --usepetsc -o 0 --petscopts rc_ex5p_fieldsplit
+EX6_ARGS      := -m ../../data/amr-quad.mesh --usepetsc
+EX9_E_ARGS    := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl -dt 0.1
+EX9_ES_ARGS   := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step
+EX9_IS_ARGS   := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5
+EX10_ARGS     := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3
 ex1p-test-par: ex1p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_W))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_P))
@@ -93,14 +90,11 @@ ex3p-test-par: ex3p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX3_ARGS))
 ex4p-test-par: ex4p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX4_ARGS))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX4_HYB_ARGS))
 ex5p-test-par: ex5p
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX5_BDDC_LB_ARGS))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX5_BDDC_GB_ARGS))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX5_BDDC_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX5_FSPL_ARGS))
 ex6p-test-par: ex6p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX6_ARGS))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX6_NONOVL_ARGS))
 ex9p-test-par: ex9p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX9_E_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX9_ES_ARGS))

examples/petsc/rc_ex3p

@@ -6,7 +6,4 @@
 # it needs PETSc configured with MUMPS
 
 -solver_pc_type cholesky
-# Petsc 3.9
--solver_pc_factor_mat_solver_type mumps
-# Older versions of PETSc
 -solver_pc_factor_mat_solver_package mumps

examples/petsc/rc_ex3p_bddc

@@ -16,10 +16,7 @@
 #-pc_bddc_adaptive_threshold 10
 
 # Customization of the local solvers
-# With PETSc versions older than 3.9
-# use "mat_solver_package" instead of "mat_solver_type"
-#
-#-pc_bddc_neumann_pc_factor_mat_solver_type mumps
-#-pc_bddc_dirichlet_pc_factor_mat_solver_type mumps
+#-pc_bddc_neumann_pc_factor_mat_solver_package mumps
+#-pc_bddc_dirichlet_pc_factor_mat_solver_package mumps
 #-pc_bddc_coarse_pc_type cholesky
-#-pc_bddc_coarse_pc_factor_mat_solver_type mumps
+#-pc_bddc_coarse_pc_factor_mat_solver_package mumps