More attempts to make HYPRE sparse matrix work

to call Pull() for the input mfem::Vector inside Operator::Mult()
implementation.

Simplify the implementation of mfem::occa::ProlongationOperator
which still needs OCCA operator(s) for the MPI case.

INSTALL

@@ -28,7 +28,7 @@ without GNU make or CMake can be found at the end of this file.
 In addition to the native build systems, MFEM packages are also available in the
 following package managers:
 
-- Spack, https://github.com/LLNL/spack
+- Spack, https://github.com/spack/spack
 - OpenHPC, http://openhpc.community
 - Homebrew/Science, https://github.com/Homebrew/homebrew-science
 

backends/all.hpp

@@ -0,0 +1,31 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_ALL_HPP
+#define MFEM_BACKENDS_ALL_HPP
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_BACKENDS
+
+#include "base/backend.hpp"
+
+#ifdef MFEM_USE_OCCA
+#include "occa/backend.hpp"
+#endif
+
+#ifdef MFEM_USE_OMP
+#include "omp/backend.hpp"
+#endif
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_ALL_HPP

backends/base/array.hpp

@@ -0,0 +1,215 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_ARRAY_HPP
+#define MFEM_BACKENDS_BASE_ARRAY_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "layout.hpp"
+#include "utils.hpp"
+
+namespace mfem
+{
+
+/// Extension to the template class Array<T>
+class PArray : public RefCounted
+{
+protected:
+   /// Layout with shared ownership (smart pointer)
+   DLayout layout;
+
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   virtual void *DoGetData() const = 0;
+
+   /** @brief Create and return a new array (in @a *clone) of the same dynamic
+       type as this array using the same layout and ItemSize().
+
+       Set @a *clone to NULL if allocation fails.
+
+       If @a copy_data is true, the contents of this array is copied to the new
+       array; otherwise, the new array remains uninitialized.
+
+       If @a buffer is not NULL, return the array data of the newly created
+       object (in @a *buffer) , if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   virtual PArray *DoClone(bool copy_data, void **buffer,
+                           std::size_t item_size) const = 0;
+
+   /// Resize the array, reallocating its data if necessary.
+   /** If @a buffer is not NULL, return the array data (in @a *buffer), if it
+       is stored as a contiguous array on the host; otherwise, set @a *buffer to
+       NULL. Returns 0 on success and non-zero otherwise, e.g. if memory
+       allocation fails.
+
+       If the @a new_layout is not supported, a non-zero error code will be
+       returned.
+
+       The @a new_layout has to be valid, i.e. new_layout != NULL and
+       new_layout->HasEngine() == true.
+
+       @note If reallocation is performed, the previous content of the array is
+       NOT copied to the new location. */
+   virtual int DoResize(PLayout &new_layout, void **buffer,
+                        std::size_t item_size) = 0;
+
+   /** @brief Get access to the contents of the array in host memory, as a
+       contiguous array. */
+   /** If the array data is stored as a contiguous array in host memory, return
+       a pointer to it. Otherwise, copy the data to @a buffer (if @a buffer is
+       not NULL) and return @a buffer.
+       @note If not NULL, @a buffer is assumed to be of size greater than or
+       equal to Size(). */
+   virtual void *DoPullData(void *buffer, std::size_t item_size) = 0;
+
+   /** @brief Set all entries of the array to the (single) value pointed to by
+       @a value_ptr. */
+   virtual void DoFill(const void *value_ptr, std::size_t item_size) = 0;
+
+   /** @brief Set all Size() entries of the array from the given contiguous
+       array, @a src_buffer, on the host. */
+   virtual void DoPushData(const void *src_buffer, std::size_t item_size) = 0;
+
+   /// Copy the data from @a src to @a *this.
+   /** Both arrays must have the same dynamic type, layout, and item_size. */
+   virtual void DoAssign(const PArray &src, std::size_t item_size) = 0;
+
+   ///@}
+   // End: Virtual interface
+
+public:
+   /** @brief The @a layout parameter will be reference counted and therefore it
+       should be dynamically allocated. */
+   /** The @a layout must be valid in the sense that layout != NULL and
+       layout->HasEngine() == true. */
+   PArray(PLayout &p_layout)
+      : layout(&p_layout)
+   {
+      MFEM_ASSERT(layout && layout->HasEngine(), "invalid layout");
+   }
+
+   virtual ~PArray() { }
+
+   /// Get the current size of the array.
+   std::size_t Size() const { return layout->Size(); }
+
+   /// Get the current layout of the array.
+   PLayout &GetLayout() const { return *layout; }
+
+   /// TODO
+   template <typename derived_t>
+   derived_t &As() { return dynamic_cast<derived_t&>(*this); }
+
+   /// TODO
+   template <typename derived_t>
+   const derived_t &As() const { return dynamic_cast<const derived_t&>(*this); }
+
+   // TODO: Error handling ... handle errors at the Engine level, at the class
+   //       level, or at the method level?
+
+   // TODO: Asynchronous execution interface ...
+
+
+   /**
+       @name Public virtual interface
+    */
+   ///@{
+
+   template <typename T=void>
+   T* GetData() const { return (T*) DoGetData(); }
+
+   /** @brief Create and return a new array (in @a *clone) of the same dynamic
+       type as this array using the same layout and ItemSize().
+
+       Set @a *clone to NULL if allocation fails.
+
+       If @a copy_data is true, the contents of this array is copied to the new
+       array; otherwise, the new array remains uninitialized.
+
+       If @a buffer is not NULL, return the array data of the newly created
+       object (in @a *buffer) , if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   template <typename T>
+   DArray Clone(bool copy_data, T **buffer) const
+   { return DArray(DoClone(copy_data, (void**)buffer, sizeof(T))); }
+
+   /// Resize the array, reallocating its data if necessary.
+   /** If @a buffer is not NULL, return the array data (in @a *buffer), if it
+       is stored as a contiguous array on the host; otherwise, set @a *buffer to
+       NULL. Returns 0 on success and non-zero otherwise, e.g. if memory
+       allocation fails.
+
+       If the @a new_layout is not supported, a non-zero error code will be
+       returned.
+
+       The @a new_layout has to be valid, i.e. new_layout != NULL and
+       new_layout->HasEngine() == true.
+
+       @note If reallocation is performed, the previous content of the array is
+       NOT copied to the new location. */
+   template <typename T>
+   int Resize(PLayout &new_layout, T **buffer)
+   { return DoResize(new_layout, (void**)buffer, sizeof(T)); }
+
+   /// Shortcut for Resize(*layout, buffer).
+   /** This method is useful for updating the array after its layout is changed
+       externally. */
+   template <typename T>
+   int Update(T **buffer)
+   { return DoResize(*layout, (void**)buffer, sizeof(T)); }
+
+   /// Shortcut for layout->Resize(new_size) followed by Update()
+   template <typename T>
+   int Resize(std::size_t new_size, T **buffer)
+   { layout->Resize(new_size); return Update(buffer); }
+
+   /** @brief Get access to the contents of the array in host memory, as a
+       contiguous array. */
+   /** If the array data is stored as a contiguous array in host memory, return
+       a pointer to it. Otherwise, copy the data to @a buffer (if @a buffer is
+       not NULL) and return @a buffer.
+       @note If not NULL, @a buffer is assumed to be of size greater than or
+       equal to Size(). */
+   template <typename T>
+   T *PullData(T *buffer)
+   { return Size() ? (T*)DoPullData((void*)buffer, sizeof(T)) : NULL; }
+
+   /** @brief Set all entries of the array to the (single) value pointed to by
+       @a value_ptr. */
+   template <typename T>
+   void Fill(const T &value) { if (Size()) { DoFill(&value, sizeof(T)); } }
+
+   /** @brief Set all Size() entries of the array from the given contiguous
+       array, @a src_buffer, on the host. */
+   template <typename T>
+   void PushData(const T *src_buffer)
+   { if (Size()) { DoPushData(src_buffer, sizeof(T)); } }
+
+   /// Copy the data from @a src to @a *this.
+   /** Both arrays must have the same dynamic type, layout, and entry type. */
+   template <typename T>
+   void Assign(const PArray &src) { if (Size()) { DoAssign(src, sizeof(T)); } }
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_ARRAY_HPP

backends/base/backend.hpp

@@ -0,0 +1,57 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_BACKEND_HPP
+#define MFEM_BACKENDS_BASE_BACKEND_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "memory_resource.hpp"
+#include "engine.hpp"
+#include "array.hpp"
+#include "vector.hpp"
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+
+#include <string>
+
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#endif
+
+namespace mfem
+{
+
+/// TODO
+class Backend
+{
+public:
+   /// TODO
+   virtual ~Backend() { }
+
+   /// TODO
+   virtual bool Supports(const std::string &engine_spec) const = 0;
+
+   /// TODO
+   virtual Engine *Create(const std::string &engine_spec) = 0;
+
+#ifdef MFEM_USE_MPI
+   /// TODO
+   virtual Engine *Create(MPI_Comm comm, const std::string &engine_spec) = 0;
+#endif
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_BACKEND_HPP

backends/base/bilinearform.hpp

@@ -0,0 +1,72 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_BILINEARFORM_HPP
+#define MFEM_BACKENDS_BASE_BILINEARFORM_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "engine.hpp"
+
+namespace mfem
+{
+
+class Vector;
+class OperatorHandle;
+class BilinearForm;
+
+/// TODO: doxygen
+class PBilinearForm : public RefCounted
+{
+protected:
+   /// Engine with shared ownership
+   SharedPtr<const Engine> engine;
+   /// Not owned.
+   BilinearForm *bform;
+
+public:
+   /// TODO: doxygen
+   PBilinearForm(const Engine &e, BilinearForm &bf)
+      : engine(&e), bform(&bf) { }
+
+   /// Virtual destructor
+   virtual ~PBilinearForm() { }
+
+   /// Get the associated Engine
+   const Engine &GetEngine() const { return *engine; }
+
+   /// Assemble the PBilinearForm.
+   /** This method is called from the method BilinearForm::Assemble() of the
+       associated BilinearForm #bform.
+       @returns True, if the host assembly should be skipped. */
+   virtual bool Assemble() = 0;
+
+   /// TODO: doxygen
+   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                 OperatorHandle &A) = 0;
+
+   /// TODO: doxygen
+   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
+                                 Vector &x, Vector &b,
+                                 OperatorHandle &A, Vector &X, Vector &B,
+                                 int copy_interior) = 0;
+
+   /// TODO: doxygen
+   virtual void RecoverFEMSolution(const Vector &X, const Vector &b,
+                                   Vector &x) = 0;
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_BILINEARFORM_HPP

backends/base/engine.cpp

@@ -0,0 +1,29 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "engine.hpp"
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+
+namespace mfem
+{
+
+DFiniteElementSpace Engine::MakeFESpace(FiniteElementSpace &fes) const
+{
+   return DFiniteElementSpace(new PFiniteElementSpace(*this, fes));
+}
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS

backends/base/engine.hpp

@@ -0,0 +1,210 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_ENGINE_HPP
+#define MFEM_BACKENDS_BASE_ENGINE_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "../../general/scalars.hpp"
+#include "memory_resource.hpp"
+#include "smart_pointers.hpp"
+#include "utils.hpp"
+
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#endif
+
+namespace mfem
+{
+
+// Forward declarations.
+class Backend;
+template <typename T> class Array;
+class Vector;
+class Operator;
+class FiniteElementSpace;
+class LinearForm;
+class BilinearForm;
+class MixedBilinearForm;
+class NonlinearForm;
+
+
+/// In parallel, each MPI rank will usually create a single engine.
+class Engine : public RefCounted
+{
+protected:
+   Backend *backend; ///< Backend that created the engine. Not owned.
+
+#ifdef MFEM_USE_MPI
+   MPI_Comm comm; ///< Associated MPI communicator (may be MPI_COMM_NULL).
+#endif
+
+   /// Number of memory resources used by the Engine.
+   int num_mem_res;
+   /// Number of workers used by the Engine.
+   int num_workers;
+
+   /// Memory resources used by the engine - array of pointers.
+   /** Both the array and the entries are owned. */
+   MemoryResource **memory_resources;
+
+   /// Relative computational speed of the workers. Owned.
+   double *workers_weights;
+
+   /// For each worker, which memory resource it uses.
+   int *workers_mem_res;
+
+public:
+   /// TODO: doxygen
+   Engine(Backend *b, int n_mem, int n_workers)
+      : backend(b),
+#ifdef MFEM_USE_MPI
+        comm(MPI_COMM_NULL),
+#endif
+        num_mem_res(n_mem),
+        num_workers(n_workers),
+        memory_resources(new MemoryResource*[num_mem_res]()),
+        workers_weights(new double[num_workers]()),
+        workers_mem_res(new int[num_workers]())
+   { /* Note: all arrays are value-initialized with zeros. */ }
+
+   /// TODO: doxygen
+   virtual ~Engine()
+   {
+      delete [] workers_mem_res;
+      delete [] workers_weights;
+      for (int i = 0; i < num_mem_res; i++)
+      {
+         delete memory_resources[i];
+      }
+      delete [] memory_resources;
+   }
+
+
+   /**
+       @name Machine resources interface
+    */
+   ///@{
+
+#ifdef MFEM_USE_MPI
+   /// Get the associated MPI_Comm
+   MPI_Comm GetComm() const { return comm; }
+#endif
+
+   /// TODO
+   int GetNumMemRes() const { return num_mem_res; }
+
+   /// TODO
+   MemoryResource &GetMemRes(int idx) const { return *memory_resources[idx]; }
+
+   /// TODO
+   int GetNumWorkers() const { return num_workers; }
+
+   /// TODO
+   const double *GetWorkersWeights() const { return workers_weights; }
+
+   /// TODO
+   const int *GetWorkersMemRes() const { return workers_mem_res; }
+
+   ///@}
+   // End: Machine resources interface
+
+   /// TODO
+   template <typename derived_t>
+   derived_t &As() { *util::As<derived_t>(this); }
+
+   /// TODO
+   template <typename derived_t>
+   const derived_t &As() const { *util::As<const derived_t>(this); }
+
+
+   // TODO: Error handling ... handle errors at the Engine level, at the class
+   //       level, or at the method level?
+
+
+   /**
+       @name Virtual interface: finite element data structures and algorithms
+    */
+   ///@{
+
+   // TODO: Asynchronous execution in this class ...
+
+   /// Allocate and return a new layout for the given @a size.
+   /** The layout decomposition is determined automatically by the Engine using
+       a deterministic algorithm: calls to this method with the same @a size
+       will produce the same result, as long as the Engine remains unmodified
+       between the calls.
+
+       The returned object is allocated with operator new and must be
+       deallocated by the caller.
+
+       TODO: Returns NULL if memory allocation fails?
+   */
+   virtual DLayout MakeLayout(std::size_t size) const = 0;
+
+   /// Allocate and return a new layout for the given worker decomposition.
+   /** The returned object is allocated with operator new and must be
+       deallocated by the caller.
+
+       TODO: Returns NULL if memory allocation fails?
+
+       The @a offsets should satisfy: offsets.Size() == number of workers + 1,
+       offsets[0] == 0, and offsets[i] <= offsets[i+1], for i: 0 <= i < number
+       of workers. */
+   virtual DLayout MakeLayout(const Array<std::size_t> &offsets) const = 0;
+
+   // Note: There may be other ways to construct layouts in the future, e.g.
+   //       block-vector layouts, or multi-vector layouts.
+
+   /// TODO
+   virtual DArray MakeArray(PLayout &layout, std::size_t item_size) const = 0;
+
+   /// Allocate and return a new vector using the given @a layout.
+   /** The returned object is a smart pointer that will automatically deallocate
+       the vector.
+
+       TODO: Produce an error if memory allocation fails?
+
+       Only layouts returned by this Engine are guaranteed to be supported.
+       Using a type that is not supported will produce an error. */
+   virtual DVector MakeVector(PLayout &layout,
+                              int type_id = ScalarId<double>::value) const = 0;
+
+   /// TODO: doxygen
+   virtual DFiniteElementSpace MakeFESpace(FiniteElementSpace &fes) const;
+
+   /// TODO: doxygen
+   virtual DBilinearForm MakeBilinearForm(BilinearForm &bf) const = 0;
+
+
+   // Question: How do we construct coefficients?
+
+
+   /// FIXME - What will the actual parameters be?
+   virtual void AssembleLinearForm(LinearForm &l_form) const = 0;
+
+   /// FIXME - What will the actual parameters be?
+   virtual Operator *MakeOperator(const MixedBilinearForm &mbl_form) const = 0;
+
+   /// FIXME - What will the actual parameters be?
+   virtual Operator *MakeOperator(const NonlinearForm &nl_form) const = 0;
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_ENGINE_HPP

backends/base/fespace.hpp

@@ -0,0 +1,61 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_FE_SPACE_HPP
+#define MFEM_BACKENDS_BASE_FE_SPACE_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "engine.hpp"
+#include "utils.hpp"
+
+namespace mfem
+{
+
+class FiniteElementSpace;
+
+/// TODO: doxygen
+class PFiniteElementSpace : public RefCounted
+{
+protected:
+   /// Engine with shared ownership
+   SharedPtr<const Engine> engine;
+   /// Not owned.
+   FiniteElementSpace *fes;
+
+public:
+   /// TODO: doxygen
+   PFiniteElementSpace(const Engine &e, FiniteElementSpace &fespace)
+      : engine(&e), fes(&fespace) { }
+
+   /// Virtual destructor
+   virtual ~PFiniteElementSpace() { }
+
+   /// Get the associated engine
+   const Engine &GetEngine() const { return *engine; }
+
+   mfem::FiniteElementSpace* GetFESpace() const { return fes; }
+
+   /// TODO
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   /// TODO
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_FE_SPACE_HPP

backends/base/layout.hpp

@@ -0,0 +1,104 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_LAYOUT_HPP
+#define MFEM_BACKENDS_BASE_LAYOUT_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "smart_pointers.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+/// Polymorphic layout (array/vector layout descriptor)
+class PLayout : public RefCounted
+{
+protected:
+   /// Engine with shared ownership
+   SharedPtr<const Engine> engine;
+   std::size_t size;
+
+   template <typename DObject>
+   struct Maker
+   {
+      template <typename entry_t>
+      static DObject MakeNew(PLayout &layout);
+   };
+
+public:
+   explicit PLayout(std::size_t s = 0) : engine(NULL), size(s) { }
+
+   explicit PLayout(const Engine &e, std::size_t s = 0)
+      : engine(&e), size(s) { }
+
+   virtual ~PLayout() { }
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   /// Resize the layout
+   virtual void Resize(std::size_t new_size) { size = new_size; }
+
+   /// Resize the layout based on the given worker offsets
+   virtual void Resize(const Array<std::size_t> &offsets)
+   { MFEM_ABORT("method not supported"); }
+
+   ///@}
+   // End: Virtual interface
+
+   /// Layouts without engine cannot create DArray, DVector, etc.
+   bool HasEngine() const { return engine != NULL; }
+
+   /// TODO: doxygen
+   const Engine &GetEngine() const { return *engine; }
+
+   /// TODO: doxygen
+   std::size_t Size() const { return size; }
+
+   /// TODO
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   /// TODO
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+
+   /// TODO: doxygen
+   template <typename DObject, typename entry_t>
+   DObject Make()
+   {
+      MFEM_ASSERT(HasEngine(), "this method requires an Engine");
+      return Maker<DObject>::template MakeNew<entry_t>(*this);
+   }
+};
+
+template <> struct PLayout::Maker<DArray>
+{
+   template <typename entry_t> static DArray MakeNew(PLayout &layout)
+   { return layout.GetEngine().MakeArray(layout, sizeof(entry_t)); }
+};
+
+template <> struct PLayout::Maker<DVector>
+{
+   template <typename entry_t> static DVector MakeNew(PLayout &layout)
+   { return layout.GetEngine().MakeVector(layout, ScalarId<entry_t>::value); }
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_LAYOUT_HPP

backends/base/memory_resource.cpp

@@ -0,0 +1,59 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "memory_resource.hpp"
+#include "../../general/error.hpp"
+
+#include <cstdlib>
+#include <cstring>
+#include <cerrno>
+
+namespace mfem
+{
+
+void *NewDeleteMemoryResource::DoAllocate(std::size_t bytes,
+                                          std::size_t alignment)
+{
+   void *p = ::operator new[](bytes);
+   MFEM_VERIFY(!alignment || (std::size_t)(p) % alignment == 0,
+               "invalid alignment");
+   return p;
+}
+
+void NewDeleteMemoryResource::DoDeallocate(void *p, std::size_t bytes,
+                                           std::size_t alignment)
+{
+   ::operator delete[](p);
+}
+
+
+void *AlignedMemoryResource::DoAllocate(std::size_t bytes,
+                                        std::size_t alignment)
+{
+   void *p;
+   if (!alignment) { alignment = sizeof(long double); }
+   MFEM_VERIFY(posix_memalign(&p, alignment, bytes) == 0,
+               "error in posix_memalign(): " << strerror(errno));
+   return p;
+}
+
+void AlignedMemoryResource::DoDeallocate(void *p, std::size_t bytes,
+                                         std::size_t alignment)
+{
+   free(p);
+}
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS

backends/base/memory_resource.hpp

@@ -0,0 +1,70 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP
+#define MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include <cstddef>
+
+namespace mfem
+{
+
+/// Polymorphic memory resource. Similar to C++17's std::pmr::memory_resource.
+class MemoryResource
+{
+protected:
+   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment) = 0;
+   virtual void DoDeallocate(void* p, std::size_t bytes,
+                             std::size_t alignment) = 0;
+
+public:
+   // Implicitly defined default & copy constructors
+
+   /// Virtual destructor.
+   virtual ~MemoryResource() { }
+
+   /// If alignment == 0, use default alignment.
+   void *Allocate(std::size_t bytes, std::size_t alignment = 0)
+   { return DoAllocate(bytes, alignment); }
+
+   /// If alignment == 0, use default alignment.
+   void Deallocate(void *p, std::size_t bytes, std::size_t alignment = 0)
+   { DoDeallocate(p, bytes, alignment); }
+};
+
+
+/** @brief Dynamic host memory resource using operator new[](std::size_t) for
+    allocation and operator delete[](void*) for deallocation. */
+class NewDeleteMemoryResource : public MemoryResource
+{
+protected:
+   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment);
+   virtual void DoDeallocate(void *p, std::size_t bytes, std::size_t alignment);
+};
+
+
+/** @brief Dynamic host memory resource using posix_memalign() for aligned
+    allocation and free() for deallocation. */
+class AlignedMemoryResource : public MemoryResource
+{
+protected:
+   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment);
+   virtual void DoDeallocate(void *p, std::size_t bytes, std::size_t alignment);
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP

backends/base/smart_pointers.hpp

@@ -0,0 +1,233 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_SMART_POINTERS_HPP
+#define MFEM_BACKENDS_BASE_SMART_POINTERS_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "utils.hpp"
+#include "../../general/error.hpp"
+#include <cstddef>
+
+// #define MFEM_TRACE_SHARED_PTR
+#ifdef MFEM_TRACE_SHARED_PTR
+#include "../../general/globals.hpp"
+#endif
+
+namespace mfem
+{
+
+/// Base class for classes with simple reference counting.
+/** Reference counting is performed by the class SharedPtr. */
+class RefCounted
+{
+private:
+   mutable unsigned ref_count;
+
+   /// Only class SharedPtr can access ref_count.
+   template <typename T> friend class SharedPtr;
+
+public:
+   RefCounted() : ref_count(0) { }
+
+   /** @brief Prevent SharedPtr objects from deleting this object by
+       incrementing the reference counter by one. */
+   void DontDelete() const { ++ref_count; }
+};
+
+
+/** @brief Smart pointer class that manages objects of type T derived from class
+    RefCounted. */
+/** This class is generally meant to work with dynamically allocated object,
+    specifically objects allocated with operator new(). It will invoke operator
+    delete() to destroy the managed object when its reference counter reaches
+    zero. This behavior can be overriden by calling RefCounted::DontDelete() to
+    ensure that an object will not be deleted by a SharedPtr that holds a
+    pointer to it.
+    @note This class is NOT thread-safe and does not support circular ownership.
+ */
+template <typename T>
+class SharedPtr
+{
+public:
+   typedef T stored_type;
+
+private:
+   T *ptr;
+
+   void Init(T *new_ptr)
+   {
+      ptr = new_ptr;
+      if (ptr) { ++ptr->RefCounted::ref_count; }
+#ifdef MFEM_TRACE_SHARED_PTR
+#elif 0
+      mfem::out << "  [" << _MFEM_FUNC_NAME << "]: ptr = " << ptr;
+      if (ptr)
+      {
+         mfem::out << ", new ref_count = " << ptr->RefCounted::ref_count;
+      }
+      mfem::out << '\n';
+#endif
+   }
+   void Destroy()
+   {
+      MFEM_ASSERT(!ptr || ptr->RefCounted::ref_count >= 1, "invalid use");
+      if (ptr && --ptr->RefCounted::ref_count == 0) { delete ptr; }
+#ifdef MFEM_TRACE_SHARED_PTR
+#elif 0
+      mfem::out << "  [" << _MFEM_FUNC_NAME << "]: ptr = " << ptr;
+      if (ptr)
+      {
+         mfem::out << ", new ref_count = " << ptr->RefCounted::ref_count;
+      }
+      mfem::out << '\n';
+#endif
+   }
+
+public:
+   SharedPtr() : ptr(NULL)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]: ptr = " << ptr << '\n';
+#endif
+   }
+
+   SharedPtr(const SharedPtr &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Init(other.ptr);
+   }
+
+   template <typename U>
+   SharedPtr(const SharedPtr<U> &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Init(other.Get());
+   }
+
+   explicit SharedPtr(T *p)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Init(p);
+   }
+
+   ~SharedPtr()
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Destroy();
+   }
+
+   SharedPtr &operator=(const SharedPtr &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Reset(other.ptr); return *this;
+   }
+
+   template <typename U>
+   SharedPtr &operator=(const SharedPtr<U> &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Reset(other.Get()); return *this;
+   }
+
+   T &operator*() const { return *ptr; }
+   T *operator->() const { return ptr; }
+
+   operator bool() const { return ptr; }
+   bool operator!() const { return !ptr; }
+
+   template <typename U>
+   bool operator==(const SharedPtr<U> &other) const
+   { return ptr == other.Ptr(); }
+   template <typename U>
+   bool operator!=(const SharedPtr<U> &other) const
+   { return ptr != other.Ptr(); }
+
+   template <typename U>
+   bool operator==(const U &p) const { return ptr == (void*) p; }
+   template <typename U>
+   bool operator!=(const U &p) const { return ptr != (void*) p; }
+
+   T *Get() const { return ptr; }
+
+   /// TODO
+   template <typename derived_t>
+   derived_t *As() const { return util::As<derived_t>(ptr); }
+
+   unsigned UseCount() const { return ptr ? ptr->RefCounted::ref_count : 0; }
+
+   void Reset()
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Destroy();
+      ptr = NULL;
+   }
+
+   /// The type U* needs to be implicitly convertible to T*
+   template <typename U>
+   void Reset(U *new_ptr)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      if (ptr != new_ptr) { Destroy(); Init(new_ptr); }
+   }
+
+   void Swap(SharedPtr &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      std::swap(ptr, other.ptr);
+   }
+};
+
+
+template <class T>
+inline void Swap(SharedPtr<T> &a, SharedPtr<T> &b) { a.Swap(b); }
+
+
+class PLayout;
+typedef SharedPtr<PLayout> DLayout;
+
+class PArray;
+typedef SharedPtr<PArray> DArray;
+
+class PVector;
+typedef SharedPtr<PVector> DVector;
+
+class PFiniteElementSpace;
+typedef SharedPtr<PFiniteElementSpace> DFiniteElementSpace;
+
+class PBilinearForm;
+typedef SharedPtr<PBilinearForm> DBilinearForm;
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_SMART_POINTERS_HPP

backends/base/utils.hpp

@@ -0,0 +1,52 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_UTILS_HPP
+#define MFEM_BACKENDS_BASE_UTILS_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "../../general/error.hpp"
+
+namespace mfem
+{
+
+namespace util
+{
+
+//
+// Inline methods
+//
+
+/// TODO: doxygen
+template <typename derived_t, typename base_t>
+inline derived_t *As(base_t *base_obj)
+{
+   MFEM_ASSERT(dynamic_cast<derived_t*>(base_obj) != NULL,
+               "invalid object type");
+   return static_cast<derived_t*>(base_obj);
+}
+
+/// TODO: doxygen
+template <typename derived_t, typename base_t>
+inline derived_t *Is(base_t *base_obj)
+{
+   return dynamic_cast<derived_t*>(base_obj);
+}
+
+} // namespace mfem::util
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_UTILS_HPP

backends/base/vector.hpp

@@ -0,0 +1,153 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_VECTOR_HPP
+#define MFEM_BACKENDS_BASE_VECTOR_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "../../general/scalars.hpp"
+#include "array.hpp"
+
+namespace mfem
+{
+
+/// Polymorphic vector - array of scalars.
+class PVector : virtual public PArray
+{
+protected:
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new vector of the same dynamic type as this
+       vector using the same layout with entries specified by @a buffer_type_id
+       which should be a constant defined by the `value` field in a
+       specialization of the template class mfem::ScalarId.
+
+       Returns NULL if allocation fails.
+
+       If @a copy_data is true, the contents of this vector is copied to the new
+       vector; otherwise, the new vector remains uninitialized.
+
+       If @a buffer is not NULL, return the vector data of the newly created
+       object (in @a *buffer), if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   virtual PVector *DoVectorClone(bool copy_data, void **buffer,
+                                  int buffer_type_id) const = 0;
+
+   /** @brief Compute and return the dot product of @a *this and @a x. In the
+       case of an MPI-parallel vector, the result must be the MPI-global dot
+       product. */
+   /** Both vectors must have the same dynamic type and layout. */
+   virtual void DoDotProduct(const PVector &x, void *result,
+                             int result_type_id) const = 0;
+
+   // TODO: add reduction operations: min, max, sum
+
+   /// Perform the operation @a *this = @a a @a x + @a b @a y.
+   /** Rules:
+       - the dynamic type of both @a x and @a y is the same as that of @a *this
+       - if @a a == 0, neither @a x nor its data are accessed
+       - if @a b == 0, neither @a y nor its data are accessed
+       - @a x's data is never the same as @a y's data, unless @a a == 0, or
+         @a b == 0
+       - @a x's data or @a y's data may be the same as the data of @a *this
+       - all accessed vectors, @a x, @a y, and @a *this have the same layout. */
+   virtual void DoAxpby(const void *a, const PVector &x,
+                        const void *b, const PVector &y,
+                        int ab_type_id) = 0;
+
+   ///@}
+   // End: Virtual interface
+
+public:
+   /** @brief Create a PVector. */
+   /** The @a layout must be valid in the sense that layout != NULL and
+       layout->HasEngine() == true. */
+   PVector(PLayout &p_layout)
+      : PArray(p_layout) { }
+
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+
+
+   // TODO: Error handling ... handle errors at the Engine level, at the class
+   //       level, or at the method level?
+
+   // TODO: Asynchronous execution interface ...
+
+   // TODO: Multi-vector interface ...
+
+
+   /**
+       @name Public virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new vector of the same dynamic type as this
+       vector using the same layout with entries of type @a scalar_t.
+
+       If @a copy_data is true, the contents of this vector is copied to the new
+       vector; otherwise, the new vector remains uninitialized.
+
+       If @a buffer is not NULL, return the vector data of the newly created
+       object (in @a *buffer) , if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   template <typename scalar_t>
+   DVector Clone(bool copy_data, scalar_t **buffer) const
+   {
+      return DVector(DoVectorClone(copy_data, (void**)buffer,
+                                   ScalarId<scalar_t>::value));
+   }
+
+   /** @brief Compute and return the dot product of @a *this and @a x. In the
+       case of an MPI-parallel vector, the result must be the MPI-global dot
+       product. */
+   /** Both vectors must have the same dynamic type and layout. */
+   template <typename scalar_t>
+   scalar_t DotProduct(const PVector &x) const
+   {
+      scalar_t result;
+      DoDotProduct(x, &result, ScalarId<scalar_t>::value);
+      return result;
+   }
+
+   // TODO: add reduction operations: min, max, sum
+
+   /// Perform the operation @a *this = @a a @a x + @a b @a y.
+   /** Rules:
+       - the dynamic type of both @a x and @a y is the same as that of @a *this
+       - if @a a == 0, neither @a x nor its data are accessed
+       - if @a b == 0, neither @a y nor its data are accessed
+       - @a x's data is never the same as @a y's data, unless @a a == 0, or
+         @a b == 0
+       - @a x's data or @a y's data may be the same as the data of @a *this
+       - all accessed vectors, @a x, @a y, and @a *this have the same layout. */
+   template <typename scalar_t>
+   void Axpby(const scalar_t &a, const PVector &x,
+              const scalar_t &b, const PVector &y)
+   { if (Size()) { DoAxpby(&a, x, &b, y, ScalarId<scalar_t>::value); } }
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_VECTOR_HPP

backends/occa/DiffusionIntegrator.okl

@@ -0,0 +1,66 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  ELEMENT_BATCH   : How many elements are in each
+  .                   computation batch
+  NUM_DOFS_1D     : Dofs in the 1D segments
+  NUM_DOFS_2D     : Dofs in the 2D faces
+  NUM_DOFS_3D     : Dofs in the 3D domain
+  NUM_QUAD_1D     : Dofs in the 1D segments
+  NUM_QUAD_2D     : Dofs in the 2D faces
+  NUM_QUAD_3D     : Dofs in the 3D domain
+  NUM_MAX_1D      : max(NUM_QUAD_1D, NUM_DOFS_1D)
+  NUM_QUAD_DOFS_1D: NUM_QUAD_1D * NUM_DOFS_1D
+  COEFF_ARGS      : Code that passes required arguments to the kernel
+  COEFF           : Code that computes the coefficient
+  ================================================
+
+  [MISSING]
+  - Add support to auto-pick @dim and use @idxOrder on stack arrays
+  |   double a[2][2];
+  |   a[0][1]; <-- regular index
+  |   a(0,1);  <-- uses @idxOrder a[0][1] or a[1][0]
+  - Add support for @idxOrder to change indexing order after allocation
+  |   double a[2][2] @idxOrder(0,1);
+  |   a(0,1) -> a[1][0]
+  |   @set(a, idxOrder(1,0));
+  |   a(0,1) -> a[0][1]
+  - Add support to iterate over loop depending on mode
+  |   for(i; @inner) {
+  |     for(0 < j < N) {} <-- ++j or j += block?
+  |   }
+*/
+
+#include "mfem-occa://defines.okl"
+
+#if USING_TENSOR_OPS
+#  ifdef OCCA_USING_GPU
+#    if USING_LOW_ORDER
+#      include "mfem-occa://diffusion/tensor/gpuHighOrder.okl"
+#    else
+#      include "mfem-occa://diffusion/tensor/gpuHighOrder.okl"
+#    endif
+#  else
+#    include "mfem-occa://diffusion/tensor/cpu.okl"
+#  endif
+#else
+#  ifdef OCCA_USING_GPU
+#    if USING_LOW_ORDER
+#      include "mfem-occa://diffusion/simplex/gpuHighOrder.okl"
+#    else
+#      include "mfem-occa://diffusion/simplex/gpuHighOrder.okl"
+#    endif
+#  else
+#    include "mfem-occa://diffusion/simplex/cpu.okl"
+#  endif
+#endif

backends/occa/array.cpp

@@ -0,0 +1,123 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+PArray *Array::DoClone(bool copy_data, void **buffer,
+                       std::size_t item_size) const
+{
+   Array *new_array = new Array(OccaLayout(), item_size);
+   if (copy_data)
+   {
+      new_array->slice.copyFrom(slice);
+   }
+   if (buffer)
+   {
+      *buffer = new_array->GetBuffer();
+   }
+   return new_array;
+}
+
+int Array::DoResize(PLayout &new_layout, void **buffer,
+                    std::size_t item_size)
+{
+   MFEM_ASSERT(dynamic_cast<Layout *>(&new_layout) != NULL,
+               "new_layout is not an OCCA Layout");
+   Layout *lt = static_cast<Layout *>(&new_layout);
+   layout.Reset(lt); // Reset() checks if the pointer is the same
+   int err = ResizeData(lt, item_size);
+   if (!err && buffer)
+   {
+      *buffer = GetBuffer();
+   }
+   return err;
+}
+
+void *Array::DoPullData(void *buffer, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   if (!slice.getDevice().hasSeparateMemorySpace())
+   {
+      return slice.ptr();
+   }
+   if (buffer)
+   {
+      slice.copyTo(buffer);
+   }
+   return buffer;
+}
+
+void Array::DoFill(const void *value_ptr, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   switch (item_size)
+   {
+      case sizeof(int8_t):
+         OccaFill((const int8_t *)value_ptr);
+         break;
+      case sizeof(int16_t):
+         OccaFill((const int16_t *)value_ptr);
+         break;
+      case sizeof(int32_t):
+         OccaFill((const int32_t *)value_ptr);
+         break;
+      // case sizeof(int64_t):
+      //    OccaFill((const int64_t *)value_ptr);
+      //    break;
+      case sizeof(double):
+         OccaFill((const double *)value_ptr);
+         break;
+      // case sizeof(::occa::double2):
+      //    OccaFill((const ::occa::double2 *)value_ptr);
+      //    break;
+      default:
+         MFEM_ABORT("item_size = " << item_size << " is not supported");
+   }
+}
+
+void Array::DoPushData(const void *src_buffer, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   if (slice.getDevice().hasSeparateMemorySpace() || slice.ptr() != src_buffer)
+   {
+      slice.copyFrom(src_buffer);
+   }
+}
+
+void Array::DoAssign(const PArray &src, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   // Note: static_cast can not be used here since PArray is a virtual base
+   //       class.
+   const Array *source = dynamic_cast<const Array *>(&src);
+   MFEM_ASSERT(source != NULL, "invalid source Array type");
+   MFEM_ASSERT(Size() == source->Size(), "");
+   slice.copyFrom(source->slice);
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/array.hpp

@@ -0,0 +1,133 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_ARRAY_HPP
+#define MFEM_BACKENDS_OCCA_ARRAY_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include <occa.hpp>
+#include "layout.hpp"
+#include "../base/array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Array : public virtual PArray
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // DLayout layout;
+
+   // Always true: Size()*item_size == slice.size() <= data.size()
+   mutable ::occa::memory data, slice;
+
+   //
+   // Virtual interface
+   //
+
+   virtual void *DoGetData() const { return GetBuffer(); }
+
+   virtual PArray *DoClone(bool copy_data, void **buffer,
+                           std::size_t item_size) const;
+
+   virtual int DoResize(PLayout &new_layout, void **buffer,
+                        std::size_t item_size);
+
+   virtual void *DoPullData(void *buffer, std::size_t item_size);
+
+   virtual void DoFill(const void *value_ptr, std::size_t item_size);
+
+   virtual void DoPushData(const void *src_buffer, std::size_t item_size);
+
+   virtual void DoAssign(const PArray &src, std::size_t item_size);
+
+   //
+   // Auxiliary methods
+   //
+
+   inline void *GetBuffer() const;
+
+   inline int ResizeData(const Layout *lt, std::size_t item_size);
+
+   template <typename T>
+   inline void OccaFill(const T *val_ptr)
+   { ::occa::linalg::operator_eq<T>(slice, *val_ptr); }
+
+public:
+   Array(Layout &lt, std::size_t item_size)
+      : PArray(lt),
+        data(lt.Alloc(lt.Size()*item_size)),
+        slice(data)
+   { }
+
+   virtual ~Array() { }
+
+   inline void MakeRef(Array &master);
+
+   Layout &OccaLayout() const
+   { return *static_cast<Layout *>(layout.Get()); }
+
+   ::occa::memory &OccaMem() { return slice; }
+   const ::occa::memory &OccaMem() const { return slice; }
+};
+
+
+//
+// Inline methods
+//
+
+inline void *Array::GetBuffer() const
+{
+   if (!slice.getDevice().hasSeparateMemorySpace())
+   {
+      return slice.ptr();
+   }
+   return NULL;
+}
+
+inline int Array::ResizeData(const Layout *lt, std::size_t item_size)
+{
+   const std::size_t new_bytes = lt->Size()*item_size;
+   if (data.size() < new_bytes ||
+       data.getDHandle() != lt->OccaEngine().GetDevice().getDHandle())
+   {
+      data = lt->Alloc(new_bytes);
+      slice = data;
+      // If memory allocation fails - an exception is thrown.
+   }
+   else if (slice.size() != new_bytes)
+   {
+      slice = data.slice(0, new_bytes);
+   }
+   return 0;
+}
+
+inline void Array::MakeRef(Array &master)
+{
+   layout = master.layout;
+   data = master.data;
+   slice = master.slice;
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_ARRAY_HPP

backends/occa/backend.cpp

@@ -0,0 +1,47 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "backend.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+bool Backend::Supports(const std::string &engine_spec) const
+{
+   // TODO: check if 'engine_spec' is valid OCCA string.
+   return true;
+}
+
+mfem::Engine *Create(const std::string &engine_spec)
+{
+   return new Engine(engine_spec);
+}
+
+#ifdef MFEM_USE_MPI
+mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec)
+{
+   return new Engine(comm, engine_spec);
+}
+#endif
+
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/backend.hpp

@@ -0,0 +1,49 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_BACKEND_HPP
+#define MFEM_BACKENDS_OCCA_BACKEND_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+// Only the Backend and Engine classes should be exposed through "backend.hpp"
+#include "../base/backend.hpp"
+#include "engine.hpp"
+#include <occa.hpp>
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Backend : public mfem::Backend
+{
+public:
+   virtual ~Backend();
+
+   virtual bool Supports(const std::string &engine_spec) const;
+
+   virtual mfem::Engine *Create(const std::string &engine_spec);
+
+#ifdef MFEM_USE_MPI
+   virtual mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec);
+#endif
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_BACKEND_HPP

backends/occa/bilinearform.cpp

@@ -0,0 +1,514 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "backend.hpp"
+#include "bilininteg.hpp"
+#include "../../fem/bilinearform.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+OccaBilinearForm::OccaBilinearForm(FiniteElementSpace *ofespace_) :
+   Operator(ofespace_->OccaVLayout()),
+   localX((ofespace_->OccaEVLayout().DontDelete(), ofespace_->OccaEVLayout())),
+   localY((ofespace_->OccaEVLayout().DontDelete(), ofespace_->OccaEVLayout()))
+{
+   Init(ofespace_->OccaEngine(), ofespace_, ofespace_);
+}
+
+OccaBilinearForm::OccaBilinearForm(FiniteElementSpace *otrialFESpace_,
+                                   FiniteElementSpace *otestFESpace_) :
+   Operator(otrialFESpace_->OccaVLayout(),
+            otestFESpace_->OccaVLayout()),
+   localX((otrialFESpace_->OccaEVLayout().DontDelete(), otrialFESpace_->OccaEVLayout())),
+   localY((otestFESpace_->OccaEVLayout().DontDelete(), otestFESpace_->OccaEVLayout()))
+{
+   Init(otrialFESpace_->OccaEngine(), otrialFESpace_, otestFESpace_);
+}
+
+void OccaBilinearForm::Init(const Engine &e,
+                            FiniteElementSpace *otrialFESpace_,
+                            FiniteElementSpace *otestFESpace_)
+{
+   engine.Reset(&e);
+
+   otrialFESpace = otrialFESpace_;
+   trialFESpace  = otrialFESpace_->GetFESpace();
+
+   otestFESpace = otestFESpace_;
+   testFESpace  = otestFESpace_->GetFESpace();
+
+   mesh = trialFESpace->GetMesh();
+
+   const int elements = GetNE();
+
+   const int trialVDim = trialFESpace->GetVDim();
+
+   const int trialLocalDofs = otrialFESpace->GetLocalDofs();
+   const int testLocalDofs  = otestFESpace->GetLocalDofs();
+
+   // First-touch policy when running with OpenMP
+   if (GetDevice().mode() == "OpenMP")
+   {
+      const std::string &okl_path = OccaEngine().GetOklPath();
+      const std::string &okl_defines = OccaEngine().GetOklDefines();
+      ::occa::kernel initLocalKernel =
+         GetDevice().buildKernel(okl_path + "utils.okl",
+                                 "InitLocalVector",
+                                 okl_defines);
+
+      const std::size_t sd = sizeof(double);
+      const uint64_t trialEntries = sd * (elements * trialLocalDofs);
+      const uint64_t testEntries  = sd * (elements * testLocalDofs);
+      for (int v = 0; v < trialVDim; ++v)
+      {
+         const uint64_t trialOffset = v * trialEntries;
+         const uint64_t testOffset  = v * testEntries;
+
+         initLocalKernel(elements, trialLocalDofs,
+                         localX.OccaMem().slice(trialOffset, trialEntries));
+         initLocalKernel(elements, testLocalDofs,
+                         localY.OccaMem().slice(testOffset, testEntries));
+      }
+   }
+}
+
+int OccaBilinearForm::BaseGeom() const
+{
+   return mesh->GetElementBaseGeometry();
+}
+
+int OccaBilinearForm::GetDim() const
+{
+   return mesh->Dimension();
+}
+
+int64_t OccaBilinearForm::GetNE() const
+{
+   return mesh->GetNE();
+}
+
+Mesh& OccaBilinearForm::GetMesh() const
+{
+   return *mesh;
+}
+
+FiniteElementSpace& OccaBilinearForm::GetTrialOccaFESpace() const
+{
+   return *otrialFESpace;
+}
+
+FiniteElementSpace& OccaBilinearForm::GetTestOccaFESpace() const
+{
+   return *otestFESpace;
+}
+
+mfem::FiniteElementSpace& OccaBilinearForm::GetTrialFESpace() const
+{
+   return *trialFESpace;
+}
+
+mfem::FiniteElementSpace& OccaBilinearForm::GetTestFESpace() const
+{
+   return *testFESpace;
+}
+
+int64_t OccaBilinearForm::GetTrialNDofs() const
+{
+   return trialFESpace->GetNDofs();
+}
+
+int64_t OccaBilinearForm::GetTestNDofs() const
+{
+   return testFESpace->GetNDofs();
+}
+
+int64_t OccaBilinearForm::GetTrialVDim() const
+{
+   return trialFESpace->GetVDim();
+}
+
+int64_t OccaBilinearForm::GetTestVDim() const
+{
+   return testFESpace->GetVDim();
+}
+
+const FiniteElement& OccaBilinearForm::GetTrialFE(const int i) const
+{
+   return *(trialFESpace->GetFE(i));
+}
+
+const FiniteElement& OccaBilinearForm::GetTestFE(const int i) const
+{
+   return *(testFESpace->GetFE(i));
+}
+
+// Adds new Domain Integrator.
+void OccaBilinearForm::AddDomainIntegrator(OccaIntegrator *integrator,
+                                           const ::occa::properties &props)
+{
+   AddIntegrator(integrator, props, DomainIntegrator);
+}
+
+// Adds new Boundary Integrator.
+void OccaBilinearForm::AddBoundaryIntegrator(OccaIntegrator *integrator,
+                                             const ::occa::properties &props)
+{
+   AddIntegrator(integrator, props, BoundaryIntegrator);
+}
+
+// Adds new interior Face Integrator.
+void OccaBilinearForm::AddInteriorFaceIntegrator(OccaIntegrator *integrator,
+                                                 const ::occa::properties &props)
+{
+   AddIntegrator(integrator, props, InteriorFaceIntegrator);
+}
+
+// Adds new boundary Face Integrator.
+void OccaBilinearForm::AddBoundaryFaceIntegrator(OccaIntegrator *integrator,
+                                                 const ::occa::properties &props)
+{
+   AddIntegrator(integrator, props, BoundaryFaceIntegrator);
+}
+
+// Adds Integrator based on OccaIntegratorType
+void OccaBilinearForm::AddIntegrator(OccaIntegrator *integrator,
+                                     const ::occa::properties &props,
+                                     const OccaIntegratorType itype)
+{
+   if (integrator == NULL)
+   {
+      std::stringstream error_ss;
+      error_ss << "OccaBilinearForm::";
+      switch (itype)
+      {
+         case DomainIntegrator      : error_ss << "AddDomainIntegrator";       break;
+         case BoundaryIntegrator    : error_ss << "AddBoundaryIntegrator";     break;
+         case InteriorFaceIntegrator: error_ss << "AddInteriorFaceIntegrator"; break;
+         case BoundaryFaceIntegrator: error_ss << "AddBoundaryFaceIntegrator"; break;
+      }
+      error_ss << " (...):\n"
+               << "  Integrator is NULL";
+      const std::string error = error_ss.str();
+      mfem_error(error.c_str());
+   }
+   integrator->SetupIntegrator(*this, baseKernelProps + props, itype);
+   integrators.push_back(integrator);
+}
+
+const mfem::Operator* OccaBilinearForm::GetTrialProlongation() const
+{
+   return otrialFESpace->GetProlongationOperator();
+}
+
+const mfem::Operator* OccaBilinearForm::GetTestProlongation() const
+{
+   return otestFESpace->GetProlongationOperator();
+}
+
+const mfem::Operator* OccaBilinearForm::GetTrialRestriction() const
+{
+   return otrialFESpace->GetRestrictionOperator();
+}
+
+const mfem::Operator* OccaBilinearForm::GetTestRestriction() const
+{
+   return otestFESpace->GetRestrictionOperator();
+}
+
+void OccaBilinearForm::Assemble()
+{
+   // [MISSING] Find geometric information that is needed by intergrators
+   //             to share between integrators.
+   const int integratorCount = (int) integrators.size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->Assemble();
+   }
+}
+
+void OccaBilinearForm::FormLinearSystem(const mfem::Array<int> &constraintList,
+                                        mfem::Vector &x, mfem::Vector &b,
+                                        mfem::Operator *&Aout,
+                                        mfem::Vector &X, mfem::Vector &B,
+                                        int copy_interior)
+{
+   FormOperator(constraintList, Aout);
+   InitRHS(constraintList, x, b, Aout, X, B, copy_interior);
+}
+
+void OccaBilinearForm::FormOperator(const mfem::Array<int> &constraintList,
+                                    mfem::Operator *&Aout)
+{
+   const mfem::Operator *trialP = GetTrialProlongation();
+   const mfem::Operator *testP  = GetTestProlongation();
+   mfem::Operator *rap = this;
+
+   if (trialP)
+   {
+      rap = new RAPOperator(*testP, *this, *trialP);
+   }
+
+   Aout = new OccaConstrainedOperator(rap, constraintList,
+                                      rap != this);
+}
+
+void OccaBilinearForm::InitRHS(const mfem::Array<int> &constraintList,
+                               mfem::Vector &x, mfem::Vector &b,
+                               mfem::Operator *A,
+                               mfem::Vector &X, mfem::Vector &B,
+                               int copy_interior)
+{
+   const std::string okl_defines = OccaEngine().GetOklDefines();
+
+   // FIXME: move these kernels to the Backend?
+   static ::occa::kernelBuilder get_subvector_builder =
+      ::occa::linalg::customLinearMethod(
+         "vector_get_subvector",
+
+         "const int dof_i = v2[i];"
+         "v0[i] = dof_i >= 0 ? v1[dof_i] : -v1[-dof_i - 1];",
+
+         "defines: {"
+         "  VTYPE0: 'double',"
+         "  VTYPE1: 'double',"
+         "  VTYPE2: 'int',"
+         "  TILESIZE: 128,"
+         "}" + okl_defines);
+
+   static ::occa::kernelBuilder set_subvector_builder =
+      ::occa::linalg::customLinearMethod(
+         "vector_set_subvector",
+         "const int dof_i = v2[i];"
+         "if (dof_i >= 0) { v0[dof_i]      = v1[i]; }"
+         "else            { v0[-dof_i - 1] = -v1[i]; }",
+
+         "defines: {"
+         "  VTYPE0: 'double',"
+         "  VTYPE1: 'double',"
+         "  VTYPE2: 'int',"
+         "  TILESIZE: 128,"
+         "}" + okl_defines);
+
+   const mfem::Operator *P = GetTrialProlongation();
+   const mfem::Operator *R = GetTrialRestriction();
+
+   if (P)
+   {
+      // Variational restriction with P
+      B.Resize(P->InLayout());
+      P->MultTranspose(b, B);
+      X.Resize(R->OutLayout());
+      R->Mult(x, X);
+   }
+   else
+   {
+      // rap, X and B point to the same data as this, x and b
+      X.MakeRef(x);
+      B.MakeRef(b);
+   }
+
+   if (!copy_interior && constraintList.Size() > 0)
+   {
+      ::occa::kernel get_subvector_kernel =
+         get_subvector_builder.build(GetDevice());
+      ::occa::kernel set_subvector_kernel =
+         set_subvector_builder.build(GetDevice());
+
+      const Array &constrList = constraintList.Get_PArray()->As<Array>();
+      Vector subvec(constrList.OccaLayout());
+
+      get_subvector_kernel(constraintList.Size(),
+                           subvec.OccaMem(),
+                           X.Get_PVector()->As<Vector>().OccaMem(),
+                           constrList.OccaMem());
+
+      X.Fill(0.0);
+
+      set_subvector_kernel(constraintList.Size(),
+                           X.Get_PVector()->As<Vector>().OccaMem(),
+                           subvec.OccaMem(),
+                           constrList.OccaMem());
+   }
+
+   OccaConstrainedOperator *cA = dynamic_cast<OccaConstrainedOperator*>(A);
+   if (cA)
+   {
+      cA->EliminateRHS(X.Get_PVector()->As<Vector>(),
+                       B.Get_PVector()->As<Vector>());
+   }
+   else
+   {
+      mfem_error("OccaBilinearForm::InitRHS expects an OccaConstrainedOperator");
+   }
+}
+
+// Matrix vector multiplication.
+void OccaBilinearForm::Mult_(const Vector &x, Vector &y) const
+{
+   otrialFESpace->GlobalToLocal(x, localX);
+   localY.Fill<double>(0.0);
+
+   const int integratorCount = (int) integrators.size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->MultAdd(localX, localY);
+   }
+
+   otestFESpace->LocalToGlobal(localY, y);
+}
+
+// Matrix transpose vector multiplication.
+void OccaBilinearForm::MultTranspose_(const Vector &x, Vector &y) const
+{
+   otestFESpace->GlobalToLocal(x, localX);
+   localY.Fill<double>(0.0);
+
+   const int integratorCount = (int) integrators.size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->MultTransposeAdd(localX, localY);
+   }
+
+   otrialFESpace->LocalToGlobal(localY, y);
+}
+
+void OccaBilinearForm::OccaRecoverFEMSolution(const mfem::Vector &X,
+                                              const mfem::Vector &b,
+                                              mfem::Vector &x)
+{
+   const mfem::Operator *P = this->GetTrialProlongation();
+   if (P)
+   {
+      // Apply conforming prolongation
+      x.Resize(P->OutLayout());
+      P->Mult(X, x);
+   }
+   // Otherwise X and x point to the same data
+}
+
+// Frees memory bilinear form.
+OccaBilinearForm::~OccaBilinearForm()
+{
+   // Make sure all integrators free their data
+   IntegratorVector::iterator it = integrators.begin();
+   while (it != integrators.end())
+   {
+      delete *it;
+      ++it;
+   }
+}
+
+
+void BilinearForm::InitOccaBilinearForm()
+{
+   // Init 'obform' using 'bform'
+   MFEM_ASSERT(bform != NULL, "");
+   MFEM_ASSERT(obform == NULL, "");
+
+   FiniteElementSpace &ofes =
+      bform->FESpace()->Get_PFESpace()->As<FiniteElementSpace>();
+   obform = new OccaBilinearForm(&ofes);
+
+   // Transfer domain integrators
+   mfem::Array<mfem::BilinearFormIntegrator*> &dbfi = *bform->GetDBFI();
+   for (int i = 0; i < dbfi.Size(); i++)
+   {
+      std::string integ_name(dbfi[i]->Name());
+      Coefficient *scal_coeff = dbfi[i]->GetScalarCoefficient();
+      ConstantCoefficient *const_coeff =
+         dynamic_cast<ConstantCoefficient*>(scal_coeff);
+      // TODO: other types of coefficients ...
+      double val = const_coeff ? const_coeff->constant : 1.0;
+      OccaCoefficient ocoeff(obform->OccaEngine(), val);
+
+      OccaIntegrator *ointeg = NULL;
+
+      if (integ_name == "(undefined)")
+      {
+         MFEM_ABORT("BilinearFormIntegrator does not define Name()");
+      }
+      else if (integ_name == "diffusion")
+      {
+         ointeg = new OccaDiffusionIntegrator(ocoeff);
+      }
+      else
+      {
+         MFEM_ABORT("BilinearFormIntegrator [Name() = " << integ_name
+                    << "] is not supported");
+      }
+
+      const mfem::IntegrationRule *ir = dbfi[i]->GetIntRule();
+      if (ir) { ointeg->SetIntegrationRule(*ir); }
+
+      obform->AddDomainIntegrator(ointeg);
+   }
+
+   // TODO: other types of integrators ...
+}
+
+bool BilinearForm::Assemble()
+{
+   if (obform == NULL) { InitOccaBilinearForm(); }
+
+   obform->Assemble();
+
+   return true; // --> host assembly is not needed
+}
+
+void BilinearForm::FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
+                                    mfem::OperatorHandle &A)
+{
+   if (A.Type() == mfem::Operator::ANY_TYPE)
+   {
+      mfem::Operator *Aout = NULL;
+      obform->FormOperator(ess_tdof_list, Aout);
+      A.Reset(Aout);
+   }
+   else
+   {
+      MFEM_ABORT("Operator::Type is not supported, type = " << A.Type());
+   }
+}
+
+void BilinearForm::FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
+                                    mfem::Vector &x, mfem::Vector &b,
+                                    mfem::OperatorHandle &A,
+                                    mfem::Vector &X, mfem::Vector &B,
+                                    int copy_interior)
+{
+   FormSystemMatrix(ess_tdof_list, A);
+   obform->InitRHS(ess_tdof_list, x, b, A.Ptr(), X, B, copy_interior);
+}
+
+void BilinearForm::RecoverFEMSolution(const mfem::Vector &X,
+                                      const mfem::Vector &b,
+                                      mfem::Vector &x)
+{
+   obform->OccaRecoverFEMSolution(X, b, x);
+}
+
+BilinearForm::~BilinearForm()
+{
+   delete obform;
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/bilinearform.hpp

@@ -0,0 +1,213 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP
+#define MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "fespace.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+enum OccaIntegratorType
+{
+   DomainIntegrator       = 0,
+   BoundaryIntegrator     = 1,
+   InteriorFaceIntegrator = 2,
+   BoundaryFaceIntegrator = 3
+};
+
+class OccaIntegrator;
+
+
+/** Class for bilinear form - "Matrix" with associated FE space and
+    BLFIntegrators. */
+class OccaBilinearForm : public Operator
+{
+   friend class OccaIntegrator;
+
+protected:
+   typedef std::vector<OccaIntegrator*> IntegratorVector;
+
+   SharedPtr<const Engine> engine;
+
+   // State information
+   mutable mfem::Mesh *mesh;
+
+   mutable FiniteElementSpace *otrialFESpace;
+   mutable mfem::FiniteElementSpace *trialFESpace;
+
+   mutable FiniteElementSpace *otestFESpace;
+   mutable mfem::FiniteElementSpace *testFESpace;
+
+   IntegratorVector integrators;
+
+   // Device data
+   ::occa::properties baseKernelProps;
+
+   // The input and output vectors are mapped to local nodes for efficient
+   // operations. In other words, they are E-vectors.
+   // The size is: (number of elements) * (nodes in element) * (vector dim)
+   mutable Vector localX, localY;
+
+public:
+   OccaBilinearForm(FiniteElementSpace *ofespace_);
+
+   OccaBilinearForm(FiniteElementSpace *otrialFESpace_,
+                    FiniteElementSpace *otestFESpace_);
+
+   void Init(const Engine &e,
+             FiniteElementSpace *otrialFESpace_,
+             FiniteElementSpace *otestFESpace_);
+
+   const Engine &OccaEngine() const { return *engine; }
+
+   ::occa::device GetDevice(int idx = 0) const
+   { return engine->GetDevice(idx); }
+
+   // Useful mesh Information
+   int BaseGeom() const;
+   int GetDim() const;
+   int64_t GetNE() const;
+
+   mfem::Mesh& GetMesh() const;
+
+   FiniteElementSpace& GetTrialOccaFESpace() const;
+   FiniteElementSpace& GetTestOccaFESpace() const;
+
+   mfem::FiniteElementSpace& GetTrialFESpace() const;
+   mfem::FiniteElementSpace& GetTestFESpace() const;
+
+   // Useful FE information
+   int64_t GetTrialNDofs() const;
+   int64_t GetTestNDofs() const;
+
+   int64_t GetTrialVDim() const;
+   int64_t GetTestVDim() const;
+
+   const mfem::FiniteElement& GetTrialFE(const int i) const;
+   const mfem::FiniteElement& GetTestFE(const int i) const;
+
+   // Adds new Domain Integrator.
+   void AddDomainIntegrator(OccaIntegrator *integrator,
+                            const ::occa::properties &props =
+                               ::occa::properties());
+
+   // Adds new Boundary Integrator.
+   void AddBoundaryIntegrator(OccaIntegrator *integrator,
+                              const ::occa::properties &props =
+                                 ::occa::properties());
+
+   // Adds new interior Face Integrator.
+   void AddInteriorFaceIntegrator(OccaIntegrator *integrator,
+                                  const ::occa::properties &props =
+                                     ::occa::properties());
+
+   // Adds new boundary Face Integrator.
+   void AddBoundaryFaceIntegrator(OccaIntegrator *integrator,
+                                  const ::occa::properties &props =
+                                     ::occa::properties());
+
+   // Adds Integrator based on OccaIntegratorType
+   void AddIntegrator(OccaIntegrator *integrator,
+                      const ::occa::properties &props,
+                      const OccaIntegratorType itype);
+
+   virtual const mfem::Operator *GetTrialProlongation() const;
+   virtual const mfem::Operator *GetTestProlongation() const;
+
+   virtual const mfem::Operator *GetTrialRestriction() const;
+   virtual const mfem::Operator *GetTestRestriction() const;
+
+   // Assembles the form i.e. sums over all domain/bdr integrators.
+   virtual void Assemble();
+
+   void FormLinearSystem(const mfem::Array<int> &constraintList,
+                         mfem::Vector &x, mfem::Vector &b,
+                         mfem::Operator *&Aout,
+                         mfem::Vector &X, mfem::Vector &B,
+                         int copy_interior = 0);
+
+   void FormOperator(const mfem::Array<int> &constraintList,
+                     mfem::Operator *&Aout);
+
+   void InitRHS(const mfem::Array<int> &constraintList,
+                mfem::Vector &x, mfem::Vector &b,
+                mfem::Operator *Aout,
+                mfem::Vector &X, mfem::Vector &B,
+                int copy_interior = 0);
+
+   // overrides
+   virtual void Mult_(const Vector &x, Vector &y) const;
+   virtual void MultTranspose_(const Vector &x, Vector &y) const;
+
+   void OccaRecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
+                               mfem::Vector &x);
+
+   // Destroys bilinear form.
+   ~OccaBilinearForm();
+};
+
+
+/// TODO: doxygen
+class BilinearForm : public mfem::PBilinearForm
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // mfem::BilinearForm *bform;
+   OccaBilinearForm *obform;
+
+   // Called from Assemble() if obform is NULL to initialize obform.
+   void InitOccaBilinearForm();
+
+public:
+   /// TODO: doxygen
+   BilinearForm(const Engine &e, mfem::BilinearForm &bf)
+      : mfem::PBilinearForm(e, bf), obform(NULL) { }
+
+   /// Virtual destructor
+   virtual ~BilinearForm();
+
+   /// Assemble the PBilinearForm.
+   /** This method is called from the method mfem::BilinearForm::Assemble() of
+       the associated mfem::BilinearForm, #bform.
+       @returns True, if the host assembly should NOT be performed. */
+   virtual bool Assemble();
+
+   virtual void FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
+                                 mfem::OperatorHandle &A);
+
+   virtual void FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
+                                 mfem::Vector &x, mfem::Vector &b,
+                                 mfem::OperatorHandle &A,
+                                 mfem::Vector &X, mfem::Vector &B,
+                                 int copy_interior);
+
+   virtual void RecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
+                                   mfem::Vector &x);
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP

backends/occa/bilininteg.cpp

@@ -0,0 +1,956 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "bilininteg.hpp"
+#include "../../fem/fem.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+std::map<std::string, OccaDofQuadMaps> OccaDofQuadMaps::AllDofQuadMaps;
+
+OccaGeometry OccaGeometry::Get(::occa::device device,
+                               FiniteElementSpace &ofespace,
+                               const mfem::IntegrationRule &ir,
+                               const int flags)
+{
+   OccaGeometry geom;
+
+   mfem::Mesh &mesh = *(ofespace.GetMesh());
+   if (!mesh.GetNodes())
+   {
+      mesh.SetCurvature(1, false, -1, mfem::Ordering::byVDIM);
+   }
+   mfem::GridFunction &nodes = *(mesh.GetNodes());
+   const mfem::FiniteElementSpace &fespace = *(nodes.FESpace());
+   const mfem::FiniteElement &fe = *(fespace.GetFE(0));
+
+   const int dims     = fe.GetDim();
+   const int elements = fespace.GetNE();
+   const int numDofs  = fe.GetDof();
+   const int numQuad  = ir.GetNPoints();
+
+   MFEM_ASSERT(dims == mesh.SpaceDimension(), "");
+
+   geom.meshNodes.allocate(device,
+                           dims, numDofs, elements);
+
+   const mfem::Table &e2dTable = fespace.GetElementToDofTable();
+   const int *elementMap = e2dTable.GetJ();
+   nodes.Pull();
+   for (int e = 0; e < elements; ++e)
+   {
+      for (int dof = 0; dof < numDofs; ++dof)
+      {
+         const int gid = elementMap[dof + numDofs*e];
+         for (int dim = 0; dim < dims; ++dim)
+         {
+            geom.meshNodes(dim, dof, e) = nodes[fespace.DofToVDof(gid,dim)];
+         }
+      }
+   }
+   geom.meshNodes.keepInDevice();
+
+   if (flags & Jacobian)
+   {
+      geom.J.allocate(device,
+                      dims, dims, numQuad, elements);
+   }
+   else
+   {
+      geom.J.allocate(device, 1);
+   }
+   if (flags & JacobianInv)
+   {
+      geom.invJ.allocate(device,
+                         dims, dims, numQuad, elements);
+   }
+   else
+   {
+      geom.invJ.allocate(device, 1);
+   }
+   if (flags & JacobianDet)
+   {
+      geom.detJ.allocate(device,
+                         numQuad, elements);
+   }
+   else
+   {
+      geom.detJ.allocate(device, 1);
+   }
+
+   geom.J.stopManaging();
+   geom.invJ.stopManaging();
+   geom.detJ.stopManaging();
+
+   OccaDofQuadMaps &maps = OccaDofQuadMaps::GetSimplexMaps(device, fe, ir);
+
+   ::occa::properties props;
+   props["defines/NUM_DOFS"] = numDofs;
+   props["defines/NUM_QUAD"] = numQuad;
+   props["defines/STORE_JACOBIAN"]     = (flags & Jacobian);
+   props["defines/STORE_JACOBIAN_INV"] = (flags & JacobianInv);
+   props["defines/STORE_JACOBIAN_DET"] = (flags & JacobianDet);
+
+   const std::string &okl_path = ofespace.OccaEngine().GetOklPath();
+   const std::string &okl_defines = ofespace.OccaEngine().GetOklDefines();
+   ::occa::kernel init = device.buildKernel(okl_path + "geometry.okl",
+                                            stringWithDim("InitGeometryInfo",
+                                                          fe.GetDim()),
+                                            props + okl_defines);
+   init(elements,
+        maps.dofToQuadD,
+        geom.meshNodes,
+        geom.J, geom.invJ, geom.detJ);
+
+   return geom;
+}
+
+OccaDofQuadMaps::OccaDofQuadMaps() :
+   hash() {}
+
+OccaDofQuadMaps::OccaDofQuadMaps(const OccaDofQuadMaps &maps)
+{
+   *this = maps;
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::operator = (const OccaDofQuadMaps &maps)
+{
+   hash = maps.hash;
+   dofToQuad   = maps.dofToQuad;
+   dofToQuadD  = maps.dofToQuadD;
+   quadToDof   = maps.quadToDof;
+   quadToDofD  = maps.quadToDofD;
+   quadWeights = maps.quadWeights;
+   return *this;
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
+                                      const FiniteElementSpace &fespace,
+                                      const mfem::IntegrationRule &ir,
+                                      const bool transpose)
+{
+   return Get(device,
+              *fespace.GetFE(0),
+              *fespace.GetFE(0),
+              ir,
+              transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
+                                      const mfem::FiniteElement &fe,
+                                      const mfem::IntegrationRule &ir,
+                                      const bool transpose)
+{
+   return Get(device, fe, fe, ir, transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
+                                      const FiniteElementSpace &trialFESpace,
+                                      const FiniteElementSpace &testFESpace,
+                                      const mfem::IntegrationRule &ir,
+                                      const bool transpose)
+{
+   return Get(device,
+              *trialFESpace.GetFE(0),
+              *testFESpace.GetFE(0),
+              ir,
+              transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
+                                      const mfem::FiniteElement &trialFE,
+                                      const mfem::FiniteElement &testFE,
+                                      const mfem::IntegrationRule &ir,
+                                      const bool transpose)
+{
+   return (dynamic_cast<const mfem::TensorBasisElement*>(&trialFE)
+           ? GetTensorMaps(device, trialFE, testFE, ir, transpose)
+           : GetSimplexMaps(device, trialFE, testFE, ir, transpose));
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::GetTensorMaps(::occa::device device,
+                                                const mfem::FiniteElement &fe,
+                                                const mfem::IntegrationRule &ir,
+                                                const bool transpose)
+{
+   return GetTensorMaps(device,
+                        fe, fe,
+                        ir, transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::GetTensorMaps(::occa::device device,
+                                                const mfem::FiniteElement &trialFE,
+                                                const mfem::FiniteElement &testFE,
+                                                const mfem::IntegrationRule &ir,
+                                                const bool transpose)
+{
+   const mfem::TensorBasisElement &trialTFE =
+      dynamic_cast<const mfem::TensorBasisElement&>(trialFE);
+   const mfem::TensorBasisElement &testTFE =
+      dynamic_cast<const mfem::TensorBasisElement&>(testFE);
+
+   std::stringstream ss;
+   ss << ::occa::hash(device)
+      << "Tensor"
+      << "O1:"  << trialFE.GetOrder()
+      << "O2:"  << testFE.GetOrder()
+      << "BT1:" << trialTFE.GetBasisType()
+      << "BT2:" << testTFE.GetBasisType()
+      << "Q:"   << ir.GetNPoints();
+   std::string hash = ss.str();
+
+   // If we've already made the dof-quad maps, reuse them
+   OccaDofQuadMaps &maps = AllDofQuadMaps[hash];
+   if (!maps.hash.size())
+   {
+      // Create the dof-quad maps
+      maps.hash = hash;
+
+      OccaDofQuadMaps trialMaps = GetD2QTensorMaps(device, trialFE, ir);
+      OccaDofQuadMaps testMaps  = GetD2QTensorMaps(device, testFE , ir, true);
+
+      maps.dofToQuad   = trialMaps.dofToQuad;
+      maps.dofToQuadD  = trialMaps.dofToQuadD;
+      maps.quadToDof   = testMaps.dofToQuad;
+      maps.quadToDofD  = testMaps.dofToQuadD;
+      maps.quadWeights = testMaps.quadWeights;
+   }
+   return maps;
+}
+
+OccaDofQuadMaps OccaDofQuadMaps::GetD2QTensorMaps(::occa::device device,
+                                                  const mfem::FiniteElement &fe,
+                                                  const mfem::IntegrationRule &ir,
+                                                  const bool transpose)
+{
+   const mfem::TensorBasisElement &tfe =
+      dynamic_cast<const mfem::TensorBasisElement&>(fe);
+
+   const mfem::Poly_1D::Basis &basis = tfe.GetBasis1D();
+   const int order = fe.GetOrder();
+   // [MISSING] Get 1D dofs
+   const int dofs = order + 1;
+   const int dims = fe.GetDim();
+
+   // Create the dof -> quadrature point map
+   const mfem::IntegrationRule &ir1D =
+      mfem::IntRules.Get(mfem::Geometry::SEGMENT, ir.GetOrder());
+   const int quadPoints = ir1D.GetNPoints();
+   const int quadPoints2D = quadPoints*quadPoints;
+   const int quadPoints3D = quadPoints2D*quadPoints;
+   const int quadPointsND = ((dims == 1) ? quadPoints :
+                             ((dims == 2) ? quadPoints2D : quadPoints3D));
+
+   OccaDofQuadMaps maps;
+   // Initialize the dof -> quad mapping
+   maps.dofToQuad.allocate(device,
+                           quadPoints, dofs);
+   maps.dofToQuadD.allocate(device,
+                            quadPoints, dofs);
+
+   double *quadWeights1DData = NULL;
+
+   if (transpose)
+   {
+      maps.dofToQuad.reindex(1,0);
+      maps.dofToQuadD.reindex(1,0);
+      // Initialize quad weights only for transpose
+      maps.quadWeights.allocate(device,
+                                quadPointsND);
+      quadWeights1DData = new double[quadPoints];
+   }
+
+   mfem::Vector d2q(dofs);
+   mfem::Vector d2qD(dofs);
+   for (int q = 0; q < quadPoints; ++q)
+   {
+      const mfem::IntegrationPoint &ip = ir1D.IntPoint(q);
+      basis.Eval(ip.x, d2q, d2qD);
+      if (transpose)
+      {
+         quadWeights1DData[q] = ip.weight;
+      }
+      for (int d = 0; d < dofs; ++d)
+      {
+         maps.dofToQuad(q, d)  = d2q[d];
+         maps.dofToQuadD(q, d) = d2qD[d];
+      }
+   }
+
+   maps.dofToQuad.keepInDevice();
+   maps.dofToQuadD.keepInDevice();
+
+   if (transpose)
+   {
+      for (int q = 0; q < quadPointsND; ++q)
+      {
+         const int qx = q % quadPoints;
+         const int qz = q / quadPoints2D;
+         const int qy = (q - qz*quadPoints2D) / quadPoints;
+         double w = quadWeights1DData[qx];
+         if (dims > 1)
+         {
+            w *= quadWeights1DData[qy];
+         }
+         if (dims > 2)
+         {
+            w *= quadWeights1DData[qz];
+         }
+         maps.quadWeights[q] = w;
+      }
+      maps.quadWeights.keepInDevice();
+      delete [] quadWeights1DData;
+   }
+
+   return maps;
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::GetSimplexMaps(::occa::device device,
+                                                 const mfem::FiniteElement &fe,
+                                                 const mfem::IntegrationRule &ir,
+                                                 const bool transpose)
+{
+   return GetSimplexMaps(device,
+                         fe, fe,
+                         ir, transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::GetSimplexMaps(::occa::device device,
+                                                 const mfem::FiniteElement &trialFE,
+                                                 const mfem::FiniteElement &testFE,
+                                                 const mfem::IntegrationRule &ir,
+                                                 const bool transpose)
+{
+   std::stringstream ss;
+   ss << ::occa::hash(device)
+      << "Simplex"
+      << "O1:" << trialFE.GetOrder()
+      << "O2:" << testFE.GetOrder()
+      << "Q:"  << ir.GetNPoints();
+   std::string hash = ss.str();
+
+   // If we've already made the dof-quad maps, reuse them
+   OccaDofQuadMaps &maps = AllDofQuadMaps[hash];
+   if (!maps.hash.size())
+   {
+      // Create the dof-quad maps
+      maps.hash = hash;
+
+      OccaDofQuadMaps trialMaps = GetD2QSimplexMaps(device, trialFE, ir);
+      OccaDofQuadMaps testMaps  = GetD2QSimplexMaps(device, testFE , ir, true);
+
+      maps.dofToQuad   = trialMaps.dofToQuad;
+      maps.dofToQuadD  = trialMaps.dofToQuadD;
+      maps.quadToDof   = testMaps.dofToQuad;
+      maps.quadToDofD  = testMaps.dofToQuadD;
+      maps.quadWeights = testMaps.quadWeights;
+   }
+   return maps;
+}
+
+OccaDofQuadMaps OccaDofQuadMaps::GetD2QSimplexMaps(::occa::device device,
+                                                   const mfem::FiniteElement &fe,
+                                                   const mfem::IntegrationRule &ir,
+                                                   const bool transpose)
+{
+   const int dims = fe.GetDim();
+   const int numDofs = fe.GetDof();
+   const int numQuad = ir.GetNPoints();
+
+   OccaDofQuadMaps maps;
+   // Initialize the dof -> quad mapping
+   maps.dofToQuad.allocate(device,
+                           numQuad, numDofs);
+   maps.dofToQuadD.allocate(device,
+                            dims, numQuad, numDofs);
+
+   if (transpose)
+   {
+      maps.dofToQuad.reindex(1,0);
+      maps.dofToQuadD.reindex(1,0);
+      // Initialize quad weights only for transpose
+      maps.quadWeights.allocate(device,
+                                numQuad);
+   }
+
+   mfem::Vector d2q(numDofs);
+   mfem::DenseMatrix d2qD(numDofs, dims);
+   for (int q = 0; q < numQuad; ++q)
+   {
+      const mfem::IntegrationPoint &ip = ir.IntPoint(q);
+      if (transpose)
+      {
+         maps.quadWeights[q] = ip.weight;
+      }
+      fe.CalcShape(ip, d2q);
+      fe.CalcDShape(ip, d2qD);
+      for (int d = 0; d < numDofs; ++d)
+      {
+         const double w = d2q[d];
+         maps.dofToQuad(q, d) = w;
+         for (int dim = 0; dim < dims; ++dim)
+         {
+            const double wD = d2qD(d, dim);
+            maps.dofToQuadD(dim, q, d) = wD;
+         }
+      }
+   }
+
+   maps.dofToQuad.keepInDevice();
+   maps.dofToQuadD.keepInDevice();
+   if (transpose)
+   {
+      maps.quadWeights.keepInDevice();
+   }
+
+   return maps;
+}
+
+//---[ Integrator Defines ]-----------
+std::string stringWithDim(const std::string &s, const int dim)
+{
+   std::string ret = s;
+   ret += ('0' + (char) dim);
+   ret += 'D';
+   return ret;
+}
+
+int closestWarpBatchTo(const int value)
+{
+   return ((value + 31) / 32) * 32;
+}
+
+int closestMultipleWarpBatch(const int multiple, const int maxSize)
+{
+   if (multiple > maxSize)
+   {
+      return maxSize;
+   }
+   int batch = (32 / multiple);
+   int minDiff = 32 - (multiple * batch);
+   for (int i = 64; i <= maxSize; i += 32)
+   {
+      const int newDiff = i - (multiple * (i / multiple));
+      if (newDiff < minDiff)
+      {
+         batch = (i / multiple);
+         minDiff = newDiff;
+      }
+   }
+   return batch;
+}
+
+void SetProperties(FiniteElementSpace &fespace,
+                   const mfem::IntegrationRule &ir,
+                   ::occa::properties &props)
+{
+   SetProperties(fespace, fespace, ir, props);
+}
+
+void SetProperties(FiniteElementSpace &trialFESpace,
+                   FiniteElementSpace &testFESpace,
+                   const mfem::IntegrationRule &ir,
+                   ::occa::properties &props)
+{
+   props["defines/TRIAL_VDIM"] = trialFESpace.GetVDim();
+   props["defines/TEST_VDIM"]  = testFESpace.GetVDim();
+   props["defines/NUM_DIM"]    = trialFESpace.GetDim();
+
+   if (trialFESpace.hasTensorBasis())
+   {
+      SetTensorProperties(trialFESpace, testFESpace, ir, props);
+   }
+   else
+   {
+      SetSimplexProperties(trialFESpace, testFESpace, ir, props);
+   }
+}
+
+void SetTensorProperties(FiniteElementSpace &fespace,
+                         const mfem::IntegrationRule &ir,
+                         ::occa::properties &props)
+{
+   SetTensorProperties(fespace, fespace, ir, props);
+}
+
+void SetTensorProperties(FiniteElementSpace &trialFESpace,
+                         FiniteElementSpace &testFESpace,
+                         const mfem::IntegrationRule &ir,
+                         ::occa::properties &props)
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace.GetFE(0));
+   const mfem::FiniteElement &testFE = *(testFESpace.GetFE(0));
+
+   const mfem::IntegrationRule &ir1D =
+      mfem::IntRules.Get(mfem::Geometry::SEGMENT, ir.GetOrder());
+
+   const int trialDofs = trialFE.GetDof();
+   const int testDofs  = testFE.GetDof();
+   const int numQuad  = ir.GetNPoints();
+
+   const int trialDofs1D = trialFE.GetOrder() + 1;
+   const int testDofs1D  = testFE.GetOrder() + 1;
+   const int quad1D  = ir1D.GetNPoints();
+   int trialDofsND = trialDofs1D;
+   int testDofsND  = testDofs1D;
+   int quadND  = quad1D;
+
+   const bool trialByVDIM = (trialFESpace.GetOrdering() == mfem::Ordering::byVDIM);
+   const bool testByVDIM  = (testFESpace.GetOrdering()  == mfem::Ordering::byVDIM);
+
+   props["defines/ORDERING_BY_NODES"] = 0;
+   props["defines/ORDERING_BY_VDIM"]  = 1;
+   props["defines/VDIM_ORDERING"]  = (int) trialByVDIM;
+   props["defines/TRIAL_ORDERING"] = (int) trialByVDIM;
+   props["defines/TEST_ORDERING"]  = (int) testByVDIM;
+
+   props["defines/USING_TENSOR_OPS"] = 1;
+   props["defines/NUM_DOFS"]  = trialDofs;
+   props["defines/NUM_QUAD"]  = numQuad;
+
+   props["defines/TRIAL_DOFS"] = trialDofs;
+   props["defines/TEST_DOFS"]  = testDofs;
+
+   for (int d = 1; d <= 3; ++d)
+   {
+      if (d > 1)
+      {
+         trialDofsND *= trialDofs1D;
+         testDofsND  *= testDofs1D;
+         quadND *= quad1D;
+      }
+      props["defines"][stringWithDim("NUM_DOFS_", d)] = trialDofsND;
+      props["defines"][stringWithDim("NUM_QUAD_", d)] = quadND;
+
+      props["defines"][stringWithDim("TRIAL_DOFS_", d)] = trialDofsND;
+      props["defines"][stringWithDim("TEST_DOFS_" , d)] = testDofsND;
+   }
+
+   // 1D Defines
+   const int m1InnerBatch = 32 * ((quad1D + 31) / 32);
+   props["defines/A1_ELEMENT_BATCH"]       = closestMultipleWarpBatch(quad1D, 512);
+   props["defines/M1_OUTER_ELEMENT_BATCH"] = closestMultipleWarpBatch(m1InnerBatch,
+                                                                      512);
+   props["defines/M1_INNER_ELEMENT_BATCH"] = m1InnerBatch;
+
+   // 2D Defines
+   props["defines/A2_ELEMENT_BATCH"] = 1;
+   props["defines/A2_QUAD_BATCH"]    = 1;
+   props["defines/M2_ELEMENT_BATCH"] = 32;
+
+   // 3D Defines
+   const int a3QuadBatch = closestMultipleWarpBatch(quadND, 512);
+   props["defines/A3_ELEMENT_BATCH"] = closestMultipleWarpBatch(a3QuadBatch, 512);
+   props["defines/A3_QUAD_BATCH"]    = a3QuadBatch;
+}
+
+void SetSimplexProperties(FiniteElementSpace &fespace,
+                          const mfem::IntegrationRule &ir,
+                          ::occa::properties &props)
+{
+   SetSimplexProperties(fespace, fespace, ir, props);
+}
+
+void SetSimplexProperties(FiniteElementSpace &trialFESpace,
+                          FiniteElementSpace &testFESpace,
+                          const mfem::IntegrationRule &ir,
+                          ::occa::properties &props)
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace.GetFE(0));
+   const mfem::FiniteElement &testFE = *(testFESpace.GetFE(0));
+
+   const int trialDofs = trialFE.GetDof();
+   const int testDofs  = testFE.GetDof();
+   const int numQuad = ir.GetNPoints();
+   const int maxDQ   = std::max(std::max(trialDofs, testDofs), numQuad);
+
+   const bool trialByVDIM = (trialFESpace.GetOrdering() == mfem::Ordering::byVDIM);
+   const bool testByVDIM  = (testFESpace.GetOrdering()  == mfem::Ordering::byVDIM);
+
+   props["defines/ORDERING_BY_NODES"] = 0;
+   props["defines/ORDERING_BY_VDIM"]  = 1;
+   props["defines/VDIM_ORDERING"]  = (int) trialByVDIM;
+   props["defines/TRIAL_ORDERING"] = (int) trialByVDIM;
+   props["defines/TEST_ORDERING"]  = (int) testByVDIM;
+
+   props["defines/USING_TENSOR_OPS"] = 0;
+   props["defines/NUM_DOFS"]  = trialDofs;
+   props["defines/NUM_QUAD"]  = numQuad;
+
+   props["defines/TRIAL_DOFS"] = trialDofs;
+   props["defines/TEST_DOFS"]  = testDofs;
+
+   // 2D Defines
+   const int quadBatch = closestWarpBatchTo(numQuad);
+   props["defines/A2_ELEMENT_BATCH"] = closestMultipleWarpBatch(quadBatch, 2048);
+   props["defines/A2_QUAD_BATCH"]    = quadBatch;
+   props["defines/M2_INNER_BATCH"]   = closestWarpBatchTo(maxDQ);
+
+   // 3D Defines
+   props["defines/A3_ELEMENT_BATCH"] = closestMultipleWarpBatch(quadBatch, 2048);
+   props["defines/A3_QUAD_BATCH"]    = quadBatch;
+   props["defines/M3_INNER_BATCH"]   = closestWarpBatchTo(maxDQ);
+}
+
+
+//---[ Base Integrator ]--------------
+OccaIntegrator::OccaIntegrator(const Engine &e)
+   : engine(&e),
+     bform(),
+     mesh(),
+     otrialFESpace(),
+     otestFESpace(),
+     trialFESpace(),
+     testFESpace(),
+     itype(DomainIntegrator),
+     ir(NULL),
+     hasTensorBasis(false) { }
+
+OccaIntegrator::~OccaIntegrator() {}
+
+void OccaIntegrator::SetupMaps()
+{
+   maps = OccaDofQuadMaps::Get(GetDevice(),
+                               *otrialFESpace,
+                               *otestFESpace,
+                               *ir);
+
+   mapsTranspose = OccaDofQuadMaps::Get(GetDevice(),
+                                        *otestFESpace,
+                                        *otrialFESpace,
+                                        *ir);
+}
+
+FiniteElementSpace& OccaIntegrator::GetTrialOccaFESpace() const
+{
+   return *otrialFESpace;
+}
+
+FiniteElementSpace& OccaIntegrator::GetTestOccaFESpace() const
+{
+   return *otestFESpace;
+}
+
+mfem::FiniteElementSpace& OccaIntegrator::GetTrialFESpace() const
+{
+   return *trialFESpace;
+}
+
+mfem::FiniteElementSpace& OccaIntegrator::GetTestFESpace() const
+{
+   return *testFESpace;
+}
+
+void OccaIntegrator::SetIntegrationRule(const mfem::IntegrationRule &ir_)
+{
+   ir = &ir_;
+}
+
+const mfem::IntegrationRule& OccaIntegrator::GetIntegrationRule() const
+{
+   return *ir;
+}
+
+OccaDofQuadMaps& OccaIntegrator::GetDofQuadMaps()
+{
+   return maps;
+}
+
+void OccaIntegrator::SetupIntegrator(OccaBilinearForm &bform_,
+                                     const ::occa::properties &props_,
+                                     const OccaIntegratorType itype_)
+{
+   MFEM_ASSERT(engine == &bform_.OccaEngine(), "");
+   bform     = &bform_;
+   mesh      = &(bform_.GetMesh());
+
+   otrialFESpace = &(bform_.GetTrialOccaFESpace());
+   otestFESpace  = &(bform_.GetTestOccaFESpace());
+
+   trialFESpace = &(bform_.GetTrialFESpace());
+   testFESpace  = &(bform_.GetTestFESpace());
+
+   hasTensorBasis = otrialFESpace->hasTensorBasis();
+
+   props = props_;
+   itype = itype_;
+
+   if (ir == NULL)
+   {
+      SetupIntegrationRule();
+   }
+
+   SetupMaps();
+
+   SetProperties(*otrialFESpace,
+                 *otestFESpace,
+                 *ir,
+                 props);
+
+   Setup();
+}
+
+OccaGeometry OccaIntegrator::GetGeometry(const int flags)
+{
+   return OccaGeometry::Get(GetDevice(), *otrialFESpace, *ir, flags);
+}
+
+::occa::kernel OccaIntegrator::GetAssembleKernel(const ::occa::properties
+                                                 &props)
+{
+   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
+   return GetKernel(stringWithDim("Assemble", fe.GetDim()),
+                    props);
+}
+
+::occa::kernel OccaIntegrator::GetMultAddKernel(const ::occa::properties &props)
+{
+   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
+   return GetKernel(stringWithDim("MultAdd", fe.GetDim()),
+                    props);
+}
+
+::occa::kernel OccaIntegrator::GetKernel(const std::string &kernelName,
+                                         const ::occa::properties &props)
+{
+   const std::string filename = GetName() + ".okl";
+   const std::string &okl_path = OccaEngine().GetOklPath();
+   const std::string &okl_defines = OccaEngine().GetOklDefines();
+   return GetDevice().buildKernel(okl_path + filename,
+                                  kernelName,
+                                  props + okl_defines);
+}
+//====================================
+
+
+//---[ Diffusion Integrator ]---------
+OccaDiffusionIntegrator::OccaDiffusionIntegrator(const OccaCoefficient &coeff_)
+   :
+   OccaIntegrator(coeff_.OccaEngine()),
+   coeff(coeff_),
+   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
+{
+   coeff.SetName("COEFF");
+}
+
+OccaDiffusionIntegrator::~OccaDiffusionIntegrator() {}
+
+
+std::string OccaDiffusionIntegrator::GetName()
+{
+   return "DiffusionIntegrator";
+}
+
+void OccaDiffusionIntegrator::SetupIntegrationRule()
+{
+   const FiniteElement &trialFE = *(trialFESpace->GetFE(0));
+   const FiniteElement &testFE  = *(testFESpace->GetFE(0));
+   ir = &mfem::DiffusionIntegrator::GetRule(trialFE, testFE);
+}
+
+void OccaDiffusionIntegrator::Setup()
+{
+   ::occa::properties kernelProps = props;
+
+   coeff.Setup(*this, kernelProps);
+
+   // Setup assemble and mult kernels
+   assembleKernel = GetAssembleKernel(kernelProps);
+   multKernel     = GetMultAddKernel(kernelProps);
+}
+
+void OccaDiffusionIntegrator::Assemble()
+{
+   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
+
+   const int dims = fe.GetDim();
+   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+
+   const int elements = trialFESpace->GetNE();
+   const int quadraturePoints = ir->GetNPoints();
+
+   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
+
+   assembledOperator.Resize<double>(symmDims * quadraturePoints * elements,
+                                    NULL);
+
+   assembleKernel((int) mesh->GetNE(),
+                  maps.quadWeights,
+                  geom.J,
+                  coeff,
+                  assembledOperator.OccaMem());
+}
+
+void OccaDiffusionIntegrator::MultAdd(Vector &x, Vector &y)
+{
+   // Note: x and y are E-vectors
+
+   multKernel((int) mesh->GetNE(),
+              maps.dofToQuad,
+              maps.dofToQuadD,
+              maps.quadToDof,
+              maps.quadToDofD,
+              assembledOperator.OccaMem(),
+              x.OccaMem(), y.OccaMem());
+}
+//====================================
+
+
+//---[ Mass Integrator ]--------------
+OccaMassIntegrator::OccaMassIntegrator(const OccaCoefficient &coeff_) :
+   OccaIntegrator(coeff_.OccaEngine()),
+   coeff(coeff_),
+   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
+{
+   coeff.SetName("COEFF");
+}
+
+OccaMassIntegrator::~OccaMassIntegrator() {}
+
+std::string OccaMassIntegrator::GetName()
+{
+   return "MassIntegrator";
+}
+
+void OccaMassIntegrator::SetupIntegrationRule()
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace->GetFE(0));
+   const mfem::FiniteElement &testFE  = *(testFESpace->GetFE(0));
+   mfem::ElementTransformation &T = *trialFESpace->GetElementTransformation(0);
+   ir = &mfem::MassIntegrator::GetRule(trialFE, testFE, T);
+}
+
+void OccaMassIntegrator::Setup()
+{
+   ::occa::properties kernelProps = props;
+
+   coeff.Setup(*this, kernelProps);
+
+   // Setup assemble and mult kernels
+   assembleKernel = GetAssembleKernel(kernelProps);
+   multKernel     = GetMultAddKernel(kernelProps);
+}
+
+void OccaMassIntegrator::Assemble()
+{
+   if (assembledOperator.Size())
+   {
+      return;
+   }
+
+   const int elements = trialFESpace->GetNE();
+   const int quadraturePoints = ir->GetNPoints();
+
+   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
+
+   assembledOperator.Resize<double>(quadraturePoints * elements, NULL);
+
+   assembleKernel((int) mesh->GetNE(),
+                  maps.quadWeights,
+                  geom.J,
+                  coeff,
+                  assembledOperator.OccaMem());
+}
+
+void OccaMassIntegrator::SetOperator(Vector &v)
+{
+   assembledOperator = v;
+}
+
+void OccaMassIntegrator::MultAdd(Vector &x, Vector &y)
+{
+   multKernel((int) mesh->GetNE(),
+              maps.dofToQuad,
+              maps.dofToQuadD,
+              maps.quadToDof,
+              maps.quadToDofD,
+              assembledOperator.OccaMem(),
+              x.OccaMem(), y.OccaMem());
+}
+//====================================
+
+
+//---[ Vector Mass Integrator ]--------------
+OccaVectorMassIntegrator::OccaVectorMassIntegrator(const OccaCoefficient &
+                                                   coeff_)
+   :
+   OccaIntegrator(coeff_.OccaEngine()),
+   coeff(coeff_),
+   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
+{
+   coeff.SetName("COEFF");
+}
+
+OccaVectorMassIntegrator::~OccaVectorMassIntegrator() {}
+
+std::string OccaVectorMassIntegrator::GetName()
+{
+   return "VectorMassIntegrator";
+}
+
+void OccaVectorMassIntegrator::SetupIntegrationRule()
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace->GetFE(0));
+   const mfem::FiniteElement &testFE  = *(testFESpace->GetFE(0));
+   mfem::ElementTransformation &T = *trialFESpace->GetElementTransformation(0);
+   ir = &mfem::MassIntegrator::GetRule(trialFE, testFE, T);
+}
+
+void OccaVectorMassIntegrator::Setup()
+{
+   ::occa::properties kernelProps = props;
+
+   coeff.Setup(*this, kernelProps);
+
+   // Setup assemble and mult kernels
+   assembleKernel = GetAssembleKernel(kernelProps);
+   multKernel     = GetMultAddKernel(kernelProps);
+}
+
+void OccaVectorMassIntegrator::Assemble()
+{
+   const int elements = trialFESpace->GetNE();
+   const int quadraturePoints = ir->GetNPoints();
+
+   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
+
+   assembledOperator.Resize<double>(quadraturePoints * elements, NULL);
+
+   assembleKernel((int) mesh->GetNE(),
+                  maps.quadWeights,
+                  geom.J,
+                  coeff,
+                  assembledOperator.OccaMem());
+}
+
+void OccaVectorMassIntegrator::MultAdd(Vector &x, Vector &y)
+{
+   multKernel((int) mesh->GetNE(),
+              maps.dofToQuad,
+              maps.dofToQuadD,
+              maps.quadToDof,
+              maps.quadToDofD,
+              assembledOperator.OccaMem(),
+              x.OccaMem(), y.OccaMem());
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/bilininteg.hpp

@@ -0,0 +1,323 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP
+#define MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+#include "coefficient.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class OccaGeometry
+{
+public:
+   ::occa::array<double> meshNodes;
+   ::occa::array<double> J, invJ, detJ;
+
+   // byVDIM  -> [x y z x y z x y z]
+   // byNodes -> [x x x y y y z z z]
+   static const int Jacobian    = (1 << 0);
+   static const int JacobianInv = (1 << 1);
+   static const int JacobianDet = (1 << 2);
+
+   static OccaGeometry Get(::occa::device device,
+                           FiniteElementSpace &ofespace,
+                           const IntegrationRule &ir,
+                           const int flags = (Jacobian    |
+                                              JacobianInv |
+                                              JacobianDet));
+};
+
+class OccaDofQuadMaps
+{
+private:
+   // Reuse dof-quad maps
+   static std::map<std::string, OccaDofQuadMaps> AllDofQuadMaps;
+   std::string hash;
+
+public:
+   // Local stiffness matrices (B and B^T operators)
+   ::occa::array<double, ::occa::dynamic> dofToQuad, dofToQuadD; // B
+   ::occa::array<double, ::occa::dynamic> quadToDof, quadToDofD; // B^T
+   ::occa::array<double> quadWeights;
+
+   OccaDofQuadMaps();
+   OccaDofQuadMaps(const OccaDofQuadMaps &maps);
+   OccaDofQuadMaps& operator = (const OccaDofQuadMaps &maps);
+
+   // [[x y] [x y] [x y]]
+   // [[x y z] [x y z] [x y z]]
+   // mfem::GridFunction* mfem::Mesh::GetNodes() { return Nodes; }
+
+   // mfem::FiniteElementSpace *Nodes->FESpace()
+   // 25
+   // 1D [x x x x x x]
+   // 2D [x y x y x y]
+   // GetVdim()
+   // 3D ordering == byVDIM  -> [x y z x y z x y z x y z x y z x y z]
+   //    ordering == byNODES -> [x x x x x x y y y y y y z z z z z z]
+   static OccaDofQuadMaps& Get(::occa::device device,
+                               const FiniteElementSpace &fespace,
+                               const mfem::IntegrationRule &ir,
+                               const bool transpose = false);
+
+   static OccaDofQuadMaps& Get(::occa::device device,
+                               const mfem::FiniteElement &fe,
+                               const mfem::IntegrationRule &ir,
+                               const bool transpose = false);
+
+   static OccaDofQuadMaps& Get(::occa::device device,
+                               const FiniteElementSpace &trialFESpace,
+                               const FiniteElementSpace &testFESpace,
+                               const mfem::IntegrationRule &ir,
+                               const bool transpose = false);
+
+   static OccaDofQuadMaps& Get(::occa::device device,
+                               const mfem::FiniteElement &trialFE,
+                               const mfem::FiniteElement &testFE,
+                               const mfem::IntegrationRule &ir,
+                               const bool transpose = false);
+
+   static OccaDofQuadMaps& GetTensorMaps(::occa::device device,
+                                         const mfem::FiniteElement &fe,
+                                         const mfem::IntegrationRule &ir,
+                                         const bool transpose = false);
+
+   static OccaDofQuadMaps& GetTensorMaps(::occa::device device,
+                                         const mfem::FiniteElement &trialFE,
+                                         const mfem::FiniteElement &testFE,
+                                         const mfem::IntegrationRule &ir,
+                                         const bool transpose = false);
+
+   static OccaDofQuadMaps GetD2QTensorMaps(::occa::device device,
+                                           const mfem::FiniteElement &fe,
+                                           const mfem::IntegrationRule &ir,
+                                           const bool transpose = false);
+
+   static OccaDofQuadMaps& GetSimplexMaps(::occa::device device,
+                                          const mfem::FiniteElement &fe,
+                                          const mfem::IntegrationRule &ir,
+                                          const bool transpose = false);
+
+   static OccaDofQuadMaps& GetSimplexMaps(::occa::device device,
+                                          const mfem::FiniteElement &trialFE,
+                                          const mfem::FiniteElement &testFE,
+                                          const mfem::IntegrationRule &ir,
+                                          const bool transpose = false);
+
+   static OccaDofQuadMaps GetD2QSimplexMaps(::occa::device device,
+                                            const mfem::FiniteElement &fe,
+                                            const mfem::IntegrationRule &ir,
+                                            const bool transpose = false);
+};
+
+//---[ Define Methods ]---------------
+std::string stringWithDim(const std::string &s, const int dim);
+int closestWarpBatch(const int multiple, const int maxSize);
+
+void SetProperties(FiniteElementSpace &fespace,
+                   const mfem::IntegrationRule &ir,
+                   ::occa::properties &props);
+
+void SetProperties(FiniteElementSpace &trialFESpace,
+                   FiniteElementSpace &testFESpace,
+                   const mfem::IntegrationRule &ir,
+                   ::occa::properties &props);
+
+void SetTensorProperties(FiniteElementSpace &fespace,
+                         const mfem::IntegrationRule &ir,
+                         ::occa::properties &props);
+
+void SetTensorProperties(FiniteElementSpace &trialFESpace,
+                         FiniteElementSpace &testFESpace,
+                         const IntegrationRule &ir,
+                         ::occa::properties &props);
+
+void SetSimplexProperties(FiniteElementSpace &fespace,
+                          const IntegrationRule &ir,
+                          ::occa::properties &props);
+
+void SetSimplexProperties(FiniteElementSpace &trialFESpace,
+                          FiniteElementSpace &testFESpace,
+                          const IntegrationRule &ir,
+                          ::occa::properties &props);
+
+//---[ Base Integrator ]--------------
+class OccaIntegrator
+{
+protected:
+   SharedPtr<const Engine> engine;
+
+   OccaBilinearForm *bform;
+   mfem::Mesh *mesh;
+
+   FiniteElementSpace *otrialFESpace;
+   FiniteElementSpace *otestFESpace;
+
+   mfem::FiniteElementSpace *trialFESpace;
+   mfem::FiniteElementSpace *testFESpace;
+
+   ::occa::properties props;
+   OccaIntegratorType itype;
+
+   const IntegrationRule *ir;
+   bool hasTensorBasis;
+   OccaDofQuadMaps maps;
+   OccaDofQuadMaps mapsTranspose;
+
+public:
+   OccaIntegrator(const Engine &e);
+   virtual ~OccaIntegrator();
+
+   const Engine &OccaEngine() const { return *engine; }
+
+   ::occa::device GetDevice(int idx = 0) const
+   { return engine->GetDevice(idx); }
+
+   virtual std::string GetName() = 0;
+
+   FiniteElementSpace& GetTrialOccaFESpace() const;
+   FiniteElementSpace& GetTestOccaFESpace() const;
+
+   mfem::FiniteElementSpace& GetTrialFESpace() const;
+   mfem::FiniteElementSpace& GetTestFESpace() const;
+
+   void SetIntegrationRule(const mfem::IntegrationRule &ir_);
+   const mfem::IntegrationRule& GetIntegrationRule() const;
+
+   OccaDofQuadMaps& GetDofQuadMaps();
+
+   void SetupMaps();
+
+   virtual void SetupIntegrationRule() = 0;
+
+   virtual void SetupIntegrator(OccaBilinearForm &bform_,
+                                const ::occa::properties &props_,
+                                const OccaIntegratorType itype_);
+
+   virtual void Setup() = 0;
+
+   virtual void Assemble() = 0;
+   /// This method works on E-vectors!
+   virtual void MultAdd(Vector &x, Vector &y) = 0;
+
+   virtual void MultTransposeAdd(Vector &x, Vector &y)
+   {
+      mfem_error("OccaIntegrator::MultTransposeAdd() is not overloaded!");
+   }
+
+   OccaGeometry GetGeometry(const int flags = (OccaGeometry::Jacobian    |
+                                               OccaGeometry::JacobianInv |
+                                               OccaGeometry::JacobianDet));
+
+   ::occa::kernel GetAssembleKernel(const ::occa::properties &props);
+   ::occa::kernel GetMultAddKernel(const ::occa::properties &props);
+
+   ::occa::kernel GetKernel(const std::string &kernelName,
+                            const ::occa::properties &props);
+};
+//====================================
+
+
+//---[ Diffusion Integrator ]---------
+class OccaDiffusionIntegrator : public OccaIntegrator
+{
+private:
+   OccaCoefficient coeff;
+
+   ::occa::kernel assembleKernel, multKernel;
+
+   Vector assembledOperator;
+
+public:
+   OccaDiffusionIntegrator(const OccaCoefficient &coeff_);
+   virtual ~OccaDiffusionIntegrator();
+
+   virtual std::string GetName();
+
+   virtual void SetupIntegrationRule();
+
+   virtual void Setup();
+
+   virtual void Assemble();
+   virtual void MultAdd(Vector &x, Vector &y);
+};
+//====================================
+
+
+//---[ Mass Integrator ]--------------
+class OccaMassIntegrator : public OccaIntegrator
+{
+private:
+   OccaCoefficient coeff;
+
+   ::occa::kernel assembleKernel, multKernel;
+
+   Vector assembledOperator;
+
+public:
+   OccaMassIntegrator(const OccaCoefficient &coeff_);
+   virtual ~OccaMassIntegrator();
+
+   virtual std::string GetName();
+
+   virtual void SetupIntegrationRule();
+
+   virtual void Setup();
+
+   virtual void Assemble();
+   void SetOperator(Vector &v);
+
+   virtual void MultAdd(Vector &x, Vector &y);
+};
+//====================================
+
+//---[ Vector Mass Integrator ]--------------
+class OccaVectorMassIntegrator : public OccaIntegrator
+{
+private:
+   OccaCoefficient coeff;
+
+   ::occa::kernel assembleKernel, multKernel;
+
+   Vector assembledOperator;
+
+public:
+   OccaVectorMassIntegrator(const OccaCoefficient &coeff_);
+   virtual ~OccaVectorMassIntegrator();
+
+   virtual std::string GetName();
+
+   virtual void SetupIntegrationRule();
+
+   virtual void Setup();
+
+   virtual void Assemble();
+
+   virtual void MultAdd(Vector &x, Vector &y);
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP

backends/occa/coefficient.cpp

@@ -0,0 +1,344 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "coefficient.hpp"
+#include "bilininteg.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+//---[ Parameter ]------------
+OccaParameter::~OccaParameter() {}
+
+void OccaParameter::Setup(OccaIntegrator &integ,
+                          ::occa::properties &props) {}
+
+::occa::kernelArg OccaParameter::KernelArgs()
+{
+   return ::occa::kernelArg();
+}
+//====================================
+
+
+//---[ Include Parameter ]------------
+OccaIncludeParameter::OccaIncludeParameter(const std::string &filename_) :
+   filename(filename_) {}
+
+OccaParameter* OccaIncludeParameter::Clone()
+{
+   return new OccaIncludeParameter(filename);
+}
+
+void OccaIncludeParameter::Setup(OccaIntegrator &integ,
+                                 ::occa::properties &props)
+{
+   props["headers"].asArray() += "#include " + filename;
+}
+//====================================
+
+
+//---[ Source Parameter ]------------
+OccaSourceParameter::OccaSourceParameter(const std::string &source_) :
+   source(source_) {}
+
+OccaParameter* OccaSourceParameter::Clone()
+{
+   return new OccaSourceParameter(source);
+}
+
+void OccaSourceParameter::Setup(OccaIntegrator &integ,
+                                ::occa::properties &props)
+{
+   props["headers"].asArray() += source;
+}
+//====================================
+
+//---[ Vector Parameter ]-------
+OccaVectorParameter::OccaVectorParameter(const std::string &name_,
+                                         Vector &v_,
+                                         const bool useRestrict_) :
+   name(name_),
+   v(v_),
+   useRestrict(useRestrict_),
+   attr("") {}
+
+OccaVectorParameter::OccaVectorParameter(const std::string &name_,
+                                         Vector &v_,
+                                         const std::string &attr_,
+                                         const bool useRestrict_) :
+   name(name_),
+   v(v_),
+   useRestrict(useRestrict_),
+   attr(attr_) {}
+
+OccaParameter* OccaVectorParameter::Clone()
+{
+   return new OccaVectorParameter(name, v, attr, useRestrict);
+}
+
+void OccaVectorParameter::Setup(OccaIntegrator &integ,
+                                ::occa::properties &props)
+{
+   std::string &args = (props["defines/COEFF_ARGS"]
+                        .asString()
+                        .string());
+   args += "const double *";
+   if (useRestrict)
+   {
+      args += " restrict ";
+   }
+   args += name;
+   if (attr.size())
+   {
+      args += ' ';
+      args += attr;
+   }
+   args += ",\n";
+}
+
+::occa::kernelArg OccaVectorParameter::KernelArgs()
+{
+   return ::occa::kernelArg(v.OccaMem());
+}
+//====================================
+
+
+//---[ GridFunction Parameter ]-------
+OccaGridFunctionParameter::OccaGridFunctionParameter(const std::string &name_,
+                                                     OccaGridFunction &gf_,
+                                                     const bool useRestrict_)
+   : name(name_),
+     gf(gf_),
+     gfQuad(*(new Layout(gf_.OccaLayout().OccaEngine(), 0))),
+     useRestrict(useRestrict_) {}
+
+OccaParameter* OccaGridFunctionParameter::Clone()
+{
+   OccaGridFunctionParameter *param =
+      new OccaGridFunctionParameter(name, gf, useRestrict);
+   param->gfQuad.MakeRef(gfQuad);
+   return param;
+}
+
+void OccaGridFunctionParameter::Setup(OccaIntegrator &integ,
+                                      ::occa::properties &props)
+{
+
+   std::string &args = (props["defines/COEFF_ARGS"]
+                        .asString()
+                        .string());
+   args += "const double *";
+   if (useRestrict)
+   {
+      args += " restrict ";
+   }
+   args += name;
+   args += " @dim(NUM_QUAD, numElements),\n";
+
+   gf.ToQuad(integ.GetIntegrationRule(), gfQuad);
+}
+
+::occa::kernelArg OccaGridFunctionParameter::KernelArgs()
+{
+   return gfQuad.OccaMem();
+}
+//====================================
+
+
+//---[ Coefficient ]------------------
+OccaCoefficient::OccaCoefficient(const Engine &e, const double value) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   coeffValue = value;
+}
+
+OccaCoefficient::OccaCoefficient(const Engine &e, const std::string &source) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   coeffValue = source;
+}
+
+OccaCoefficient::OccaCoefficient(const Engine &e, const char *source) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   coeffValue = source;
+}
+
+OccaCoefficient::OccaCoefficient(const OccaCoefficient &coeff) :
+   engine(coeff.engine),
+   integ(NULL),
+   name(coeff.name),
+   coeffValue(coeff.coeffValue)
+{
+
+   const int paramCount = (int) coeff.params.size();
+   for (int i = 0; i < paramCount; ++i)
+   {
+      params.push_back(coeff.params[i]->Clone());
+   }
+}
+
+OccaCoefficient::~OccaCoefficient()
+{
+   const int paramCount = (int) params.size();
+   for (int i = 0; i < paramCount; ++i)
+   {
+      delete params[i];
+   }
+}
+
+OccaCoefficient& OccaCoefficient::SetName(const std::string &name_)
+{
+   name = name_;
+   return *this;
+}
+
+void OccaCoefficient::Setup(OccaIntegrator &integ_,
+                            ::occa::properties &props_)
+{
+   integ = &integ_;
+
+   const int paramCount = (int) params.size();
+   props_["defines"][name + "_ARGS"] = "";
+   for (int i = 0; i < paramCount; ++i)
+   {
+      params[i]->Setup(integ_, props_);
+   }
+   props_["defines"][name] = coeffValue;
+
+   props = props_;
+}
+
+OccaCoefficient& OccaCoefficient::Add(OccaParameter *param)
+{
+   params.push_back(param);
+   return *this;
+}
+
+OccaCoefficient& OccaCoefficient::IncludeHeader(const std::string &filename)
+{
+   return Add(new OccaIncludeParameter(filename));
+}
+
+OccaCoefficient& OccaCoefficient::IncludeSource(const std::string &source)
+{
+   return Add(new OccaSourceParameter(source));
+}
+
+OccaCoefficient& OccaCoefficient::AddVector(const std::string &name_,
+                                            Vector &v,
+                                            const bool useRestrict)
+{
+   return Add(new OccaVectorParameter(name_, v, useRestrict));
+}
+
+OccaCoefficient& OccaCoefficient::AddVector(const std::string &name_,
+                                            Vector &v,
+                                            const std::string &attr,
+                                            const bool useRestrict)
+{
+   return Add(new OccaVectorParameter(name_, v, attr, useRestrict));
+}
+
+OccaCoefficient& OccaCoefficient::AddGridFunction(const std::string &name_,
+                                                  OccaGridFunction &gf,
+                                                  const bool useRestrict)
+{
+   return Add(new OccaGridFunctionParameter(name_, gf, useRestrict));
+}
+
+bool OccaCoefficient::IsConstant()
+{
+   return coeffValue.isNumber();
+}
+
+double OccaCoefficient::GetConstantValue()
+{
+   if (!IsConstant())
+   {
+      mfem_error("OccaCoefficient is not constant");
+   }
+   return coeffValue.number();
+}
+
+Vector OccaCoefficient::Eval()
+{
+   if (integ == NULL)
+   {
+      mfem_error("OccaCoefficient requires a Setup() call before Eval()");
+   }
+
+   mfem::FiniteElementSpace &fespace = integ->GetTrialFESpace();
+   const mfem::IntegrationRule &ir   = integ->GetIntegrationRule();
+
+   const int elements = fespace.GetNE();
+   const int numQuad  = ir.GetNPoints();
+
+   Vector quadCoeff(*(new Layout(OccaEngine(), numQuad * elements)));
+   Eval(quadCoeff);
+   return quadCoeff;
+}
+
+void OccaCoefficient::Eval(Vector &quadCoeff)
+{
+   const std::string &okl_path = OccaEngine().GetOklPath();
+   const std::string &okl_defines = OccaEngine().GetOklDefines();
+   static ::occa::kernelBuilder builder =
+      ::occa::kernelBuilder::fromFile(okl_path + "coefficient.okl",
+                                      "CoefficientEval", okl_defines);
+
+   if (integ == NULL)
+   {
+      mfem_error("OccaCoefficient requires a Setup() call before Eval()");
+   }
+
+   const int elements = integ->GetTrialFESpace().GetNE();
+
+   ::occa::properties kernelProps = props;
+   if (name != "COEFF")
+   {
+      kernelProps["defines/COEFF"]      = name;
+      kernelProps["defines/COEFF_ARGS"] = name + "_ARGS";
+   }
+   kernelProps += okl_defines;
+
+   ::occa::kernel evalKernel = builder.build(GetDevice(), kernelProps);
+   evalKernel(elements, *this, quadCoeff.OccaMem());
+}
+
+OccaCoefficient::operator ::occa::kernelArg ()
+{
+   ::occa::kernelArg kArg;
+   const int paramCount = (int) params.size();
+   for (int i = 0; i < paramCount; ++i)
+   {
+      kArg.add(params[i]->KernelArgs());
+   }
+   return kArg;
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/coefficient.hpp

@@ -0,0 +1,284 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_COEFFICIENT_HPP
+#define MFEM_BACKENDS_OCCA_COEFFICIENT_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "vector.hpp"
+#include "gridfunc.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class OccaIntegrator;
+
+
+class OccaParameter
+{
+public:
+   virtual ~OccaParameter();
+
+   virtual OccaParameter* Clone() = 0;
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+
+   virtual ::occa::kernelArg KernelArgs();
+};
+
+
+//---[ Include Parameter ]------------
+class OccaIncludeParameter : public OccaParameter
+{
+private:
+   std::string filename;
+
+public:
+   OccaIncludeParameter(const std::string &filename_);
+
+   virtual OccaParameter* Clone();
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+};
+//====================================
+
+
+//---[ Source Parameter ]------------
+class OccaSourceParameter : public OccaParameter
+{
+private:
+   std::string source;
+
+public:
+   OccaSourceParameter(const std::string &filename_);
+
+   virtual OccaParameter* Clone();
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+};
+//====================================
+
+
+//---[ Define Parameter ]------------
+template <class TM>
+class OccaDefineParameter : public OccaParameter
+{
+private:
+   const std::string name;
+   TM value;
+
+public:
+   OccaDefineParameter(const std::string &name_,
+                       const TM &value_) :
+      name(name_),
+      value(value_) {}
+
+   virtual OccaParameter* Clone()
+   {
+      return new OccaDefineParameter(name, value);
+   }
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props)
+   {
+      props["defines"][name] = value;
+   }
+};
+//====================================
+
+
+//---[ Variable Parameter ]-----------
+template <class TM>
+class OccaVariableParameter : public OccaParameter
+{
+private:
+   const std::string name;
+   const TM &value;
+
+public:
+   OccaVariableParameter(const std::string &name_,
+                         const TM &value_) :
+      name(name_),
+      value(value_) {}
+
+   virtual OccaParameter* Clone()
+   {
+      return new OccaVariableParameter(name, value);
+   }
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props)
+   {
+      std::string &args = (props["defines/COEFF_ARGS"]
+                           .asString()
+                           .string());
+      // const TM name,\n"
+      args += "const ";
+      args += ::occa::primitiveinfo<TM>::name;
+      args += ' ';
+      args += name;
+      args += ",\n";
+   }
+
+   virtual ::occa::kernelArg KernelArgs()
+   {
+      return ::occa::kernelArg(value);
+   }
+};
+//====================================
+
+
+//---[ Vector Parameter ]-------
+class OccaVectorParameter : public OccaParameter
+{
+private:
+   const std::string name;
+   Vector v;
+   bool useRestrict;
+   std::string attr;
+
+public:
+   OccaVectorParameter(const std::string &name_,
+                       Vector &v_,
+                       const bool useRestrict_ = false);
+
+   OccaVectorParameter(const std::string &name_,
+                       Vector &v_,
+                       const std::string &attr_,
+                       const bool useRestrict_ = false);
+
+   virtual OccaParameter* Clone();
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+
+   virtual ::occa::kernelArg KernelArgs();
+};
+//====================================
+
+
+//---[ GridFunction Parameter ]-------
+class OccaGridFunctionParameter : public OccaParameter
+{
+private:
+   const std::string name;
+   OccaGridFunction &gf;
+   Vector gfQuad;
+   bool useRestrict;
+
+public:
+   OccaGridFunctionParameter(const std::string &name_,
+                             OccaGridFunction &gf_,
+                             const bool useRestrict_ = false);
+
+   virtual OccaParameter* Clone();
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+
+   virtual ::occa::kernelArg KernelArgs();
+};
+//====================================
+
+
+//---[ Coefficient ]------------------
+// [MISSING]
+// Needs to know about the integrator's
+//   - fespace
+//   - ir
+// Step where parameters that need the ir get called for setup
+// For example, GridFunction (d, e) -> (q, e)
+class OccaCoefficient
+{
+private:
+   SharedPtr<const Engine> engine;
+
+   OccaIntegrator *integ;
+
+   std::string name;
+   ::occa::json coeffValue;
+
+   ::occa::properties props;
+   std::vector<OccaParameter*> params;
+
+public:
+   OccaCoefficient(const Engine &e, const double value = 1.0);
+   OccaCoefficient(const Engine &e, const std::string &source);
+   OccaCoefficient(const Engine &e, const char *source);
+   ~OccaCoefficient();
+
+   OccaCoefficient(const OccaCoefficient &coeff);
+
+   const Engine &OccaEngine() const { return *engine; }
+
+   ::occa::device GetDevice(int idx = 0) const
+   { return engine->GetDevice(idx); }
+
+   OccaCoefficient& SetName(const std::string &name_);
+
+   void Setup(OccaIntegrator &integ_,
+              ::occa::properties &props_);
+
+   OccaCoefficient& Add(OccaParameter *param);
+
+   OccaCoefficient& IncludeHeader(const std::string &filename);
+   OccaCoefficient& IncludeSource(const std::string &source);
+
+   template <class TM>
+   OccaCoefficient& AddDefine(const std::string &name_, const TM &value)
+   {
+      return Add(new OccaDefineParameter<TM>(name_, value));
+   }
+
+   template <class TM>
+   OccaCoefficient& AddVariable(const std::string &name_, const TM &value)
+   {
+      return Add(new OccaVariableParameter<TM>(name_, value));
+   }
+
+   OccaCoefficient& AddVector(const std::string &name_,
+                              Vector &v,
+                              const bool useRestrict = false);
+
+
+   OccaCoefficient& AddVector(const std::string &name_,
+                              Vector &v,
+                              const std::string &attr,
+                              const bool useRestrict = false);
+
+   OccaCoefficient& AddGridFunction(const std::string &name_,
+                                    OccaGridFunction &gf,
+                                    const bool useRestrict = false);
+
+   bool IsConstant();
+   double GetConstantValue();
+
+   Vector Eval();
+   void Eval(Vector &quadCoeff);
+
+   operator ::occa::kernelArg ();
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_COEFFICIENT_HPP

backends/occa/defines.okl

@@ -0,0 +1,40 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_OCCA_DEFINES
+#define MFEM_OCCA_DEFINES
+
+#ifndef USING_TENSOR_OPS
+#  define USING_TENSOR_OPS 0
+#endif
+
+#ifdef OCCA_USING_GPU
+#  define GPU_ORDER_2(I0, I1)         @dimOrder(I0, I1)
+#  define GPU_ORDER_3(I0, I1, I2)     @dimOrder(I0, I1, I2)
+#  define GPU_ORDER_4(I0, I1, I2, I3) @dimOrder(I0, I1, I2, I3)
+#else
+#  define GPU_ORDER_2(I0, I1)         @dimOrder(0, 1)
+#  define GPU_ORDER_3(I0, I1, I2)     @dimOrder(0, 1, 2)
+#  define GPU_ORDER_4(I0, I1, I2, I3) @dimOrder(0, 1, 2, 3)
+#endif
+
+#ifndef COEFF
+#  define COEFF 1.0
+#  define COEFF_ARGS
+#endif
+
+#if USING_TENSOR_OPS
+#  include "mfem-occa://defines/tensor.okl"
+#else
+#  include "mfem-occa://defines/simplex.okl"
+#endif
+
+#endif

backends/occa/defines/simplex.okl

@@ -0,0 +1,40 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#define USING_LOW_ORDER 1
+#define USING_HI_ORDER  0
+
+typedef double* DofToQuad_t @dim(NUM_QUAD, NUM_DOFS);
+typedef double* DofToQuadD2D_t @dim(2, NUM_QUAD, NUM_DOFS);
+typedef double* DofToQuadD3D_t @dim(3, NUM_QUAD, NUM_DOFS);
+
+typedef double* QuadToDof_t @dim(NUM_DOFS, NUM_QUAD);
+typedef double* QuadToDofD2D_t @dim(2, NUM_DOFS, NUM_QUAD);
+typedef double* QuadToDofD3D_t @dim(3, NUM_DOFS, NUM_QUAD);
+
+typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD, numElements);
+typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
+
+typedef double* SymmOperator2D_t @dim(3, NUM_QUAD, numElements);
+typedef double* SymmOperator3D_t @dim(6, NUM_QUAD, numElements);
+
+typedef double* DLocal_t @dim(NUM_DOFS, numElements);
+typedef double* QLocal_t @dim(NUM_QUAD, numElements);
+
+#if VDIM_ORDERING == ORDERING_BY_VDIM
+typedef double* DVLocal_t @dim(NUM_VDIM, NUM_DOFS, numElements);
+typedef double* QVLocal_t @dim(NUM_VDIM, NUM_QUAD, numElements);
+#else
+typedef double* DVLocal_t @dim(NUM_VDIM, NUM_DOFS, numElements) @dimOrder(2,0,1);
+typedef double* QVLocal_t @dim(NUM_VDIM, NUM_QUAD, numElements) @dimOrder(2,0,1);
+#endif
+
+typedef int* DLocalMap_t @dim(NUM_DOFS, numElements);

backends/occa/defines/tensor.okl

@@ -0,0 +1,85 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#if NUM_QUAD_1D < NUM_DOFS_1D
+#  define NUM_MAX_1D NUM_DOFS_1D
+#else
+#  define NUM_MAX_1D NUM_QUAD_1D
+#endif
+
+#define NUM_MAX_2D (NUM_MAX_1D * NUM_MAX_1D)
+
+#define NUM_QUAD_DOFS_1D (NUM_QUAD_1D * NUM_DOFS_1D)
+
+#define QUAD_2D_ID(X, Y) (X + ((Y) * NUM_QUAD_1D))
+#define DOFS_2D_ID(X, Y) (X + ((Y) * NUM_DOFS_1D))
+
+#define QUAD_3D_ID(X, Y, Z) (X + ((Y) * NUM_QUAD_1D) + ((Z) * NUM_QUAD_2D))
+#define DOFS_3D_ID(X, Y, Z) (X + ((Y) * NUM_DOFS_1D) + ((Z) * NUM_DOFS_2D))
+
+#if NUM_MAX_1D < 8
+#  define USING_LOW_ORDER 1
+#  define USING_HI_ORDER  0
+#else
+#  define USING_LOW_ORDER 0
+#  define USING_HI_ORDER  1
+#endif
+
+#define M1_ELEMENT_BATCHES (M1_OUTER_ELEMENT_BATCH * M1_INNER_ELEMENT_BATCH)
+
+typedef double* DofToQuad_t @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+typedef double* QuadToDof_t @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+typedef double* Jacobian_t   @dim(NUM_DIM, NUM_DIM, numElements);
+typedef double* Jacobian1D_t @dim(NUM_QUAD_1D, numElements);
+typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD_2D, numElements);
+typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD_3D, numElements);
+
+typedef double* SymmOperator1D_t @dim(NUM_QUAD_1D, numElements);
+typedef double* SymmOperator2D_t @dim(3, NUM_QUAD_2D, numElements);
+typedef double* SymmOperator3D_t @dim(6, NUM_QUAD_3D, numElements);
+
+typedef double* DLocal_t   @dim(NUM_DOFS, numElements);
+typedef double* DLocal1D_t @dim(NUM_DOFS_1D, numElements);
+typedef double* DLocal2D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+typedef double* DLocal3D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+
+typedef double* QLocal_t   @dim(NUM_QUAD, numElements);
+typedef double* QLocal1D_t @dim(NUM_QUAD_1D, numElements);
+typedef double* QLocal2D_t @dim(NUM_QUAD_1D, NUM_QUAD_1D, numElements);
+typedef double* QLocal3D_t @dim(NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
+
+#if VDIM_ORDERING == ORDERING_BY_VDIM
+typedef double* DVLocal_t   @dim(NUM_VDIM, NUM_DOFS, numElements);
+typedef double* DVLocal1D_t @dim(NUM_VDIM, NUM_DOFS_1D, numElements);
+typedef double* DVLocal2D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+typedef double* DVLocal3D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+
+typedef double* QVLocal_t   @dim(NUM_VDIM, NUM_QUAD, numElements);
+typedef double* QVLocal1D_t @dim(NUM_VDIM, NUM_QUAD_1D, numElements);
+typedef double* QVLocal2D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
+typedef double* QVLocal3D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
+#else
+typedef double* DVLocal_t   @dim(NUM_VDIM, NUM_DOFS, numElements) @dimOrder(2,0,1);
+typedef double* DVLocal1D_t @dim(NUM_VDIM, NUM_DOFS_1D, numElements) @dimOrder(2,0,1);
+typedef double* DVLocal2D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, numElements) @dimOrder(3,0,1,2);
+typedef double* DVLocal3D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements) @dimOrder(4,0,1,2,3);
+
+typedef double* QVLocal_t   @dim(NUM_VDIM, NUM_QUAD, numElements) @dimOrder(2,0,1);
+typedef double* QVLocal1D_t @dim(NUM_VDIM, NUM_QUAD_1D, numElements) @dimOrder(2,0,1);
+typedef double* QVLocal2D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, numElements) @dimOrder(3,0,1,2);
+typedef double* QVLocal3D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements) @dimOrder(4,0,1,2,3);
+#endif
+
+typedef int* DLocalMap_t   @dim(NUM_DOFS, numElements);
+typedef int* DLocalMap1D_t @dim(NUM_DOFS_1D, numElements);
+typedef int* DLocalMap2D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+typedef int* DLocalMap3D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);

backends/occa/diffusion/simplex/cpu.okl

@@ -0,0 +1,168 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        const double * restrict quadWeights,
+                        const Jacobian2D_t restrict J,
+                        COEFF_ARGS
+                        SymmOperator2D_t restrict oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+      const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
+
+      oper(0, q, e) =  c_detJ * (J12*J12 + J22*J22); // (1,1)
+      oper(1, q, e) = -c_detJ * (J12*J11 + J22*J21); // (1,2) + (2,1)
+      oper(2, q, e) =  c_detJ * (J11*J11 + J21*J21); // (2,2)
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuadD2D_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDofD2D_t restrict quadToDofD,
+                       const SymmOperator2D_t restrict oper,
+                       const DLocal_t restrict solIn,
+                       DLocal_t restrict solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double r_sol[NUM_DOFS];
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        r_sol[d] = 0;
+      }
+
+      for (int q = 0; q < NUM_QUAD; ++q) {
+        double gradX = 0, gradY = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          const double s = solIn(d, e);
+          gradX += s * quadToDofD(0, d, q);
+          gradY += s * quadToDofD(1, d, q);
+        }
+        const double O11 = oper(0, q, e);
+        const double O12 = oper(1, q, e);
+        const double O22 = oper(2, q, e);
+
+        const double gradX2 = (O11 * gradX) + (O12 * gradY);
+        const double gradY2 = (O12 * gradX) + (O22 * gradY);
+
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          r_sol[d] += ((gradX2 * quadToDofD(0, d, q)) +
+                       (gradY2 * quadToDofD(1, d, q)));
+        }
+      }
+
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        solOut(d, e) += r_sol[d];
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        const double * restrict quadWeights,
+                        const Jacobian3D_t restrict J,
+                        COEFF_ARGS
+                        SymmOperator3D_t restrict oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+      const double c_detJ = quadWeights[q] * COEFF / detJ;
+
+      // adj(J)
+      const double A11 = (J22 * J33) - (J23 * J32);
+      const double A12 = (J23 * J31) - (J21 * J33);
+      const double A13 = (J21 * J32) - (J22 * J31);
+
+      const double A21 = (J13 * J32) - (J12 * J33);
+      const double A22 = (J11 * J33) - (J13 * J31);
+      const double A23 = (J12 * J31) - (J11 * J32);
+
+      const double A31 = (J12 * J23) - (J13 * J22);
+      const double A32 = (J13 * J21) - (J11 * J23);
+      const double A33 = (J11 * J22) - (J12 * J21);
+
+      // adj(J)^Tadj(J)
+      oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
+      oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2) + (2,1)
+      oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3) + (3,1)
+      oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
+      oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3) + (3,2)
+      oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuadD3D_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDofD3D_t restrict quadToDofD,
+                       const SymmOperator3D_t restrict oper,
+                       const DLocal_t restrict solIn,
+                       DLocal_t restrict solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double r_sol[NUM_DOFS];
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        r_sol[d] = 0;
+      }
+
+      for (int q = 0; q < NUM_QUAD; ++q) {
+        double gradX = 0, gradY = 0, gradZ = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          const double s = solIn(d, e);
+          gradX += s * quadToDofD(0, d, q);
+          gradY += s * quadToDofD(1, d, q);
+          gradZ += s * quadToDofD(2, d, q);
+        }
+        const double O11 = oper(0, q, e);
+        const double O12 = oper(1, q, e);
+        const double O13 = oper(2, q, e);
+        const double O22 = oper(3, q, e);
+        const double O23 = oper(4, q, e);
+        const double O33 = oper(5, q, e);
+
+        const double gradX2 = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
+        const double gradY2 = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
+        const double gradZ2 = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
+
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          r_sol[d] += ((gradX2 * quadToDofD(0, d, q)) +
+                       (gradY2 * quadToDofD(1, d, q)) +
+                       (gradZ2 * quadToDofD(2, d, q)));
+        }
+      }
+
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        solOut(d, e) += r_sol[d];
+      }
+    }
+  }
+}
+//======================================

backends/occa/diffusion/simplex/gpuHighOrder.okl

@@ -0,0 +1,182 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian2D_t J,
+                        COEFF_ARGS
+                        SymmOperator2D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD; q += A2_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+            const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
+
+            oper(0, q, e) =  c_detJ * (J12*J12 + J22*J22); // (1,1)
+            oper(1, q, e) = -c_detJ * (J12*J11 + J22*J21); // (1,2) + (2,1)
+            oper(2, q, e) =  c_detJ * (J11*J11 + J21*J21); // (2,2)
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuadD2D_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDofD2D_t restrict quadToDofD,
+                       const SymmOperator2D_t restrict oper,
+                       const DLocal_t restrict solIn,
+                       DLocal_t restrict solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_gradX[NUM_QUAD];
+    @shared double s_gradY[NUM_QUAD];
+
+    for (int qOff = 0; qOff < M2_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M2_INNER_BATCH) {
+        double gradX = 0, gradY = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          const double s = solIn(d, e);
+          gradX += s * quadToDofD(0, d, q);
+          gradY += s * quadToDofD(1, d, q);
+        }
+        const double O11 = oper(0, q, e);
+        const double O12 = oper(1, q, e);
+        const double O22 = oper(2, q, e);
+
+        s_gradX[q] = (O11 * gradX) + (O12 * gradY);
+        s_gradY[q] = (O12 * gradX) + (O22 * gradY);
+      }
+    }
+
+    for (int dOff = 0; dOff < M2_INNER_BATCH; ++dOff) {
+      for (int d = dOff; d < NUM_DOFS; d += M2_INNER_BATCH) {
+        double r_sol = 0;
+        for (int q = 0; q < NUM_QUAD; ++q) {
+          // FIXME: s_gradX and s_gradY are @shared used outside of @inner
+          r_sol += ((s_gradX[q] * quadToDofD(0, d, q)) +
+                    (s_gradY[q] * quadToDofD(1, d, q)));
+        }
+        solOut(d, e) += r_sol;
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian3D_t J,
+                        COEFF_ARGS
+                        SymmOperator3D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD; q += A3_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+            const double c_detJ = quadWeights[q] * COEFF / detJ;
+
+            // adj(J)
+            const double A11 = (J22 * J33) - (J23 * J32);
+            const double A12 = (J23 * J31) - (J21 * J33);
+            const double A13 = (J21 * J32) - (J22 * J31);
+
+            const double A21 = (J13 * J32) - (J12 * J33);
+            const double A22 = (J11 * J33) - (J13 * J31);
+            const double A23 = (J12 * J31) - (J11 * J32);
+
+            const double A31 = (J12 * J23) - (J13 * J22);
+            const double A32 = (J13 * J21) - (J11 * J23);
+            const double A33 = (J11 * J22) - (J12 * J21);
+
+            // adj(J)^Tadj(J)
+            oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
+            oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2) + (2,1)
+            oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3) + (3,1)
+            oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
+            oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3) + (3,2)
+            oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuadD3D_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDofD3D_t restrict quadToDofD,
+                       const SymmOperator3D_t restrict oper,
+                       const DLocal_t restrict solIn,
+                       DLocal_t restrict solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_gradX[NUM_QUAD];
+    @shared double s_gradY[NUM_QUAD];
+    @shared double s_gradZ[NUM_QUAD];
+
+    for (int qOff = 0; qOff < M3_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M3_INNER_BATCH) {
+        double gradX = 0, gradY = 0, gradZ = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          const double s = solIn(d, e);
+          gradX += s * quadToDofD(0, d, q);
+          gradY += s * quadToDofD(1, d, q);
+          gradZ += s * quadToDofD(2, d, q);
+        }
+        const double O11 = oper(0, q, e);
+        const double O12 = oper(1, q, e);
+        const double O13 = oper(2, q, e);
+        const double O22 = oper(3, q, e);
+        const double O23 = oper(4, q, e);
+        const double O33 = oper(5, q, e);
+
+        s_gradX[q] = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
+        s_gradY[q] = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
+        s_gradZ[q] = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
+      }
+    }
+
+    for (int dOff = 0; dOff < M3_INNER_BATCH; ++dOff) {
+      for (int d = dOff; d < NUM_DOFS; d += M3_INNER_BATCH) {
+        double r_sol = 0;
+        for (int q = 0; q < NUM_QUAD; ++q) {
+          r_sol += ((s_gradX[q] * quadToDofD(0, d, q)) +
+                    (s_gradY[q] * quadToDofD(1, d, q)) +
+                    (s_gradZ[q] * quadToDofD(2, d, q)));
+        }
+        solOut(d, e) += r_sol;
+      }
+    }
+  }
+}
+//======================================

backends/occa/diffusion/tensor/cpu.okl

@@ -0,0 +1,370 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void Assemble1D(const int numElements,
+                        const double * restrict quadWeights,
+                        const Jacobian1D_t restrict J,
+                        COEFF_ARGS
+                        SymmOperator1D_t restrict oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
+      oper(q, e) = quadWeights[q] * COEFF / J(q, e);
+    }
+  }
+}
+
+@kernel void MultAdd1D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator1D_t restrict oper,
+                       const DLocal1D_t restrict solIn,
+                       DLocal1D_t restrict solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double grad[NUM_QUAD_1D];
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        grad[qx] = 0;
+      }
+      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+        const double s = solIn(dx, e);
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          grad[qx] += s * dofToQuadD(qx, dx);
+        }
+      }
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        grad[qx] *= oper(qx, e);
+      }
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        const double gradX = grad[qx];
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          solOut(dx, e) += gradX * quadToDofD(dx, qx);
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        const double * restrict quadWeights,
+                        const Jacobian2D_t restrict J,
+                        COEFF_ARGS
+                        SymmOperator2D_t restrict oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_2D; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+      const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
+
+      oper(0, q, e) =  c_detJ * (J21*J21 + J22*J22); // (1,1)
+      oper(1, q, e) = -c_detJ * (J21*J11 + J22*J12); // (1,2), (2,1)
+      oper(2, q, e) =  c_detJ * (J11*J11 + J12*J12); // (2,2)
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator2D_t restrict oper,
+                       const DLocal2D_t restrict solIn,
+                       DLocal2D_t restrict solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double grad[NUM_QUAD_1D][NUM_QUAD_1D][2];
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          grad[qy][qx][0] = 0;
+          grad[qy][qx][1] = 0;
+        }
+      }
+
+      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+        double gradX[NUM_QUAD_1D][2];
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          gradX[qx][0] = 0;
+          gradX[qx][1] = 0;
+        }
+
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          const double s = solIn(dx, dy, e);
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            gradX[qx][0] += s * dofToQuad(qx, dx);
+            gradX[qx][1] += s * dofToQuadD(qx, dx);
+          }
+        }
+
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          const double wy  = dofToQuad(qy, dy);
+          const double wDy = dofToQuadD(qy, dy);
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            grad[qy][qx][0] += gradX[qx][1] * wy;
+            grad[qy][qx][1] += gradX[qx][0] * wDy;
+          }
+        }
+      }
+
+      // Calculate Dxy, xDy in plane
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          const int q = QUAD_2D_ID(qx, qy);
+          const double O11 = oper(0, q, e);
+          const double O12 = oper(1, q, e);
+          const double O22 = oper(2, q, e);
+
+          const double gradX = grad[qy][qx][0];
+          const double gradY = grad[qy][qx][1];
+
+          grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
+          grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
+        }
+      }
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        double gradX[NUM_DOFS_1D][2];
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          gradX[dx][0] = 0;
+          gradX[dx][1] = 0;
+        }
+
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          const double gX = grad[qy][qx][0];
+          const double gY = grad[qy][qx][1];
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double wx  = quadToDof(dx, qx);
+            const double wDx = quadToDofD(dx, qx);
+            gradX[dx][0] += gX * wDx;
+            gradX[dx][1] += gY * wx;
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          const double wy  = quadToDof(dy, qy);
+          const double wDy = quadToDofD(dy, qy);
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            solOut(dx, dy, e) += ((gradX[dx][0] * wy) +
+                                  (gradX[dx][1] * wDy));
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        const double * restrict quadWeights,
+                        const Jacobian3D_t restrict J,
+                        COEFF_ARGS
+                        SymmOperator3D_t restrict oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_3D; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+      const double c_detJ = quadWeights[q] * COEFF / detJ;
+
+      // adj(J)
+      const double A11 = (J22 * J33) - (J23 * J32);
+      const double A12 = (J23 * J31) - (J21 * J33);
+      const double A13 = (J21 * J32) - (J22 * J31);
+
+      const double A21 = (J13 * J32) - (J12 * J33);
+      const double A22 = (J11 * J33) - (J13 * J31);
+      const double A23 = (J12 * J31) - (J11 * J32);
+
+      const double A31 = (J12 * J23) - (J13 * J22);
+      const double A32 = (J13 * J21) - (J11 * J23);
+      const double A33 = (J11 * J22) - (J12 * J21);
+
+      // adj(J)^Tadj(J)
+      oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
+      oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2), (2,1)
+      oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3), (3,1)
+      oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
+      oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3), (3,2)
+      oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator3D_t restrict oper,
+                       const DLocal3D_t restrict solIn,
+                       DLocal3D_t restrict solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double grad[NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D][4];
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            grad[qz][qy][qx][0] = 0;
+            grad[qz][qy][qx][1] = 0;
+            grad[qz][qy][qx][2] = 0;
+          }
+        }
+      }
+
+      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+        double gradXY[NUM_QUAD_1D][NUM_QUAD_1D][4];
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            gradXY[qy][qx][0] = 0;
+            gradXY[qy][qx][1] = 0;
+            gradXY[qy][qx][2] = 0;
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          double gradX[NUM_QUAD_1D][2];
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            gradX[qx][0] = 0;
+            gradX[qx][1] = 0;
+          }
+
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double s = solIn(dx, dy, dz, e);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              gradX[qx][0] += s * dofToQuad(qx, dx);
+              gradX[qx][1] += s * dofToQuadD(qx, dx);
+            }
+          }
+
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            const double wy  = dofToQuad(qy, dy);
+            const double wDy = dofToQuadD(qy, dy);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              const double wx  = gradX[qx][0];
+              const double wDx = gradX[qx][1];
+              gradXY[qy][qx][0] += wDx * wy;
+              gradXY[qy][qx][1] += wx  * wDy;
+              gradXY[qy][qx][2] += wx  * wy;
+            }
+          }
+        }
+
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          const double wz  = dofToQuad(qz, dz);
+          const double wDz = dofToQuadD(qz, dz);
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              grad[qz][qy][qx][0] += gradXY[qy][qx][0] * wz;
+              grad[qz][qy][qx][1] += gradXY[qy][qx][1] * wz;
+              grad[qz][qy][qx][2] += gradXY[qy][qx][2] * wDz;
+            }
+          }
+        }
+      }
+
+      // Calculate Dxyz, xDyz, xyDz in plane
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            const int q = QUAD_3D_ID(qx, qy, qz);
+            const double O11 = oper(0, q, e);
+            const double O12 = oper(1, q, e);
+            const double O13 = oper(2, q, e);
+            const double O22 = oper(3, q, e);
+            const double O23 = oper(4, q, e);
+            const double O33 = oper(5, q, e);
+
+            const double gradX = grad[qz][qy][qx][0];
+            const double gradY = grad[qz][qy][qx][1];
+            const double gradZ = grad[qz][qy][qx][2];
+
+            grad[qz][qy][qx][0] = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
+            grad[qz][qy][qx][1] = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
+            grad[qz][qy][qx][2] = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
+          }
+        }
+      }
+
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        double gradXY[NUM_DOFS_1D][NUM_DOFS_1D][4];
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            gradXY[dy][dx][0] = 0;
+            gradXY[dy][dx][1] = 0;
+            gradXY[dy][dx][2] = 0;
+          }
+        }
+
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          double gradX[NUM_DOFS_1D][4];
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            gradX[dx][0] = 0;
+            gradX[dx][1] = 0;
+            gradX[dx][2] = 0;
+          }
+
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            const double gX = grad[qz][qy][qx][0];
+            const double gY = grad[qz][qy][qx][1];
+            const double gZ = grad[qz][qy][qx][2];
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              const double wx  = quadToDof(dx, qx);
+              const double wDx = quadToDofD(dx, qx);
+              gradX[dx][0] += gX * wDx;
+              gradX[dx][1] += gY * wx;
+              gradX[dx][2] += gZ * wx;
+            }
+          }
+
+          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+            const double wy  = quadToDof(dy, qy);
+            const double wDy = quadToDofD(dy, qy);
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              gradXY[dy][dx][0] += gradX[dx][0] * wy;
+              gradXY[dy][dx][1] += gradX[dx][1] * wDy;
+              gradXY[dy][dx][2] += gradX[dx][2] * wy;
+            }
+          }
+        }
+
+        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+          const double wz  = quadToDof(dz, qz);
+          const double wDz = quadToDofD(dz, qz);
+          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              solOut(dx, dy, dz, e) += ((gradXY[dy][dx][0] * wz) +
+                                        (gradXY[dy][dx][1] * wz) +
+                                        (gradXY[dy][dx][2] * wDz));
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/diffusion/tensor/gpuHighOrder.okl

@@ -0,0 +1,433 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void Assemble1D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian1D_t J,
+                        COEFF_ARGS
+                        SymmOperator1D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A1_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A1_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
+          oper(q, e) = quadWeights[q] * COEFF / J(q, e);
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd1D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator1D_t restrict oper,
+                       const DLocal1D_t restrict solIn,
+                       DLocal1D_t restrict solOut) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
+    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    @exclusive double grad[NUM_QUAD_1D];
+
+    for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+      for (int i = el; i < NUM_QUAD_DOFS_1D; i += M1_INNER_ELEMENT_BATCH) {
+        s_dofToQuadD[i] = dofToQuadD[i];
+        s_quadToDofD[i] = quadToDofD[i];
+      }
+    }
+
+    for (int b = 0; b < M1_OUTER_ELEMENT_BATCH; ++b) {
+      for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+        const int e = eOff + b*M1_INNER_ELEMENT_BATCH + el;
+        if (e < numElements) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            grad[qx] = 0;
+          }
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double s = solIn(dx, e);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              grad[qx] += s * s_dofToQuadD(qx, dx);
+            }
+          }
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            grad[qx] *= oper(qx, e);
+          }
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            double s = 0;
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              s += grad[qx] * s_quadToDofD(dx, qx);
+            }
+            solOut(dx, e) += s;
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian2D_t J,
+                        COEFF_ARGS
+                        SymmOperator2D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD_2D; q += A2_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+            const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
+
+            oper(0, q, e) =  c_detJ * (J21*J21 + J22*J22); // (1,1)
+            oper(1, q, e) = -c_detJ * (J21*J11 + J22*J12); // (1,2), (2,1)
+            oper(2, q, e) =  c_detJ * (J11*J11 + J12*J12); // (2,2)
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator2D_t restrict oper,
+                       const DLocal2D_t restrict solIn,
+                       DLocal2D_t restrict solOut) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    // Store xy planes in shared memory
+    @shared double s_xy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_xDy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_grad[2 * NUM_QUAD_2D] @dim(2, NUM_QUAD_1D, NUM_QUAD_1D);
+
+    @exclusive double r_x[NUM_MAX_1D];
+    @exclusive double r_y[NUM_QUAD_1D];
+
+    for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+      for (int id = x; id < NUM_QUAD_DOFS_1D; id += NUM_MAX_1D) {
+        s_dofToQuad[id]  = dofToQuad[id];
+        s_dofToQuadD[id] = dofToQuadD[id];
+        s_quadToDof[id]  = quadToDof[id];
+        s_quadToDofD[id] = quadToDofD[id];
+      }
+    }
+
+    for (int e = eOff; e < (eOff + M2_ELEMENT_BATCH); ++e) {
+      if (e < numElements) {
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx; @inner) {
+          if (dx < NUM_DOFS_1D) {
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              s_xy(dx, qy) = 0;
+              s_xDy(dx, qy) = 0;
+            }
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              r_x[dy] = solIn(dx, dy, e);
+            }
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              double xy = 0;
+              double xDy = 0;
+              for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+                xy  += r_x[dy] * s_dofToQuad(qy, dy);
+                xDy += r_x[dy] * s_dofToQuadD(qy, dy);
+              }
+              s_xy(dx, qy)  = xy;
+              s_xDy(dx, qy) = xDy;
+            }
+          }
+        }
+        for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+          if (qy < NUM_QUAD_1D) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              double gradX = 0, gradY = 0;
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                gradX += s_xy(dx, qy)  * s_dofToQuadD(qx, dx);
+                gradY += s_xDy(dx, qy) * s_dofToQuad(qx, dx);
+              }
+
+              const int q = QUAD_2D_ID(qx, qy);
+              const double O11 = oper(0, q, e);
+              const double O12 = oper(1, q, e);
+              const double O22 = oper(2, q, e);
+
+              s_grad(0, qx, qy) = (O11 * gradX) + (O12 * gradY);
+              s_grad(1, qx, qy) = (O12 * gradX) + (O22 * gradY);
+            }
+          }
+        }
+
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx; @inner) {
+          if (qx < NUM_QUAD_1D) {
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              s_xy(dy, qx) = 0;
+              s_xDy(dy, qx) = 0;
+            }
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              r_x[qy] = s_grad(0, qx, qy);
+              r_y[qy] = s_grad(1, qx, qy);
+            }
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              double xy  = 0;
+              double xDy = 0;
+              for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+                xy  += r_x[qy] * s_quadToDof(dy, qy);
+                xDy += r_y[qy] * s_quadToDofD(dy, qy);
+              }
+              s_xy(dy, qx) = xy;
+              s_xDy(dy, qx) = xDy;
+            }
+          }
+        }
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx; @inner) {
+          if (dx < NUM_DOFS_1D) {
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              double s = 0;
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                s += ((s_xy(dy, qx) * s_quadToDofD(dx, qx)) +
+                      (s_xDy(dy, qx) * s_quadToDof(dx, qx)));
+              }
+              solOut(dx, dy, e) += s;
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian3D_t J,
+                        COEFF_ARGS
+                        SymmOperator3D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD_3D; q += A3_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+            const double c_detJ = quadWeights[q] * COEFF / detJ;
+
+            // adj(J)
+            const double A11 = (J22 * J33) - (J23 * J32);
+            const double A12 = (J23 * J31) - (J21 * J33);
+            const double A13 = (J21 * J32) - (J22 * J31);
+
+            const double A21 = (J13 * J32) - (J12 * J33);
+            const double A22 = (J11 * J33) - (J13 * J31);
+            const double A23 = (J12 * J31) - (J11 * J32);
+
+            const double A31 = (J12 * J23) - (J13 * J22);
+            const double A32 = (J13 * J21) - (J11 * J23);
+            const double A33 = (J11 * J22) - (J12 * J21);
+
+            // adj(J)^Tadj(J)
+            oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
+            oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2), (2,1)
+            oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3), (3,1)
+            oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
+            oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3), (3,2)
+            oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator3D_t restrict oper,
+                       const DLocal3D_t restrict solIn,
+                       DLocal3D_t restrict solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    // Store xy planes in shared memory
+    @shared double s_z[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
+    @shared double s_Dz[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
+    @shared double s_xyDz[NUM_QUAD_2D] @dim(NUM_QUAD_1D, NUM_QUAD_1D);
+
+    // Store z axis as registers
+    @exclusive double r_qz[NUM_QUAD_1D];
+    @exclusive double r_qDz[NUM_QUAD_1D];
+    @exclusive double r_dDxyz[NUM_DOFS_1D];
+    @exclusive double r_dxDyz[NUM_DOFS_1D];
+    @exclusive double r_dxyDz[NUM_DOFS_1D];
+
+    for (int y = 0; y < NUM_MAX_1D; ++y; @inner) {
+      for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+        const int id = (y * NUM_MAX_1D) + x;
+        // Fetch Q <--> D maps
+        if (id < NUM_QUAD_DOFS_1D) {
+          s_dofToQuad[id]  = dofToQuad[id];
+          s_dofToQuadD[id] = dofToQuadD[id];
+          s_quadToDof[id]  = quadToDof[id];
+          s_quadToDofD[id] = quadToDofD[id];
+        }
+        // Initialize our Z axis
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          r_qz[qz] = 0;
+          r_qDz[qz] = 0;
+        }
+        // Initialize our solution updates in the Z axis
+        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+          r_dDxyz[dz] = 0;
+          r_dxDyz[dz] = 0;
+          r_dxyDz[dz] = 0;
+        }
+      }
+    }
+
+    for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+      for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+        if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+          for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+            const double s = solIn(dx, dy, dz, e);
+            // Calculate D -> Q in the Z axis
+            for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+              r_qz[qz]  += s * s_dofToQuad(qz, dz);
+              r_qDz[qz] += s * s_dofToQuadD(qz, dz);
+            }
+          }
+        }
+      }
+    }
+    // For each xy plane
+    for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+      // Fill xy plane at given z position
+      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+            s_z(dx, dy)  = r_qz[qz];
+            s_Dz(dx, dy) = r_qDz[qz];
+          }
+        }
+      }
+      // Calculate Dxyz, xDyz, xyDz in plane
+      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
+            double Dxyz = 0;
+            double xDyz = 0;
+            double xyDz = 0;
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              const double wy  = s_dofToQuad(qy, dy);
+              const double wDy = s_dofToQuadD(qy, dy);
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                const double wx  = s_dofToQuad(qx, dx);
+                const double wDx = s_dofToQuadD(qx, dx);
+                const double z  = s_z(dx, dy);
+                const double Dz = s_Dz(dx, dy);
+                Dxyz += wDx * wy  * z;
+                xDyz += wx  * wDy * z;
+                xyDz += wx  * wy  * Dz;
+              }
+            }
+
+            const int q = QUAD_3D_ID(qx, qy, qz);
+            const double O11 = oper(0, q, e);
+            const double O12 = oper(1, q, e);
+            const double O13 = oper(2, q, e);
+            const double O22 = oper(3, q, e);
+            const double O23 = oper(4, q, e);
+            const double O33 = oper(5, q, e);
+
+            const double qDxyz = (O11 * Dxyz) + (O12 * xDyz) + (O13 * xyDz);
+            const double qxDyz = (O12 * Dxyz) + (O22 * xDyz) + (O23 * xyDz);
+            const double qxyDz = (O13 * Dxyz) + (O23 * xDyz) + (O33 * xyDz);
+
+            for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+              const double wz  = s_quadToDof(dz, qz);
+              const double wDz = s_quadToDofD(dz, qz);
+              r_dDxyz[dz] += wz  * qDxyz;
+              r_dxDyz[dz] += wz  * qxDyz;
+              r_dxyDz[dz] += wDz * qxyDz;
+            }
+          }
+        }
+      }
+    }
+    // Iterate over xy planes to compute solution
+    for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+      // Place xy plane in shared memory
+      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
+            s_z(qx, qy)    = r_dDxyz[dz];
+            s_Dz(qx, qy)   = r_dxDyz[dz];
+            s_xyDz(qx, qy) = r_dxyDz[dz];
+          }
+        }
+      }
+      // Finalize solution in xy plane
+      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+            double solZ = 0;
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              const double wy  = s_quadToDof(dy, qy);
+              const double wDy = s_quadToDofD(dy, qy);
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                const double wx  = s_quadToDof(dx, qx);
+                const double wDx = s_quadToDofD(dx, qx);
+                const double Dxyz = s_z(qx, qy);
+                const double xDyz = s_Dz(qx, qy);
+                const double xyDz = s_xyDz(qx, qy);
+                solZ += ((wDx * wy  * Dxyz) +
+                         (wx  * wDy * xDyz) +
+                         (wx  * wy  * xyDz));
+              }
+            }
+            solOut(dx, dy, dz, e) += solZ;
+          }
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/engine.cpp

@@ -0,0 +1,140 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "backend.hpp"
+#include "url_handler.hpp"
+#include "bilinearform.hpp"
+#include "../../general/array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+bool Engine::fileOpenerRegistered = false;
+
+void Engine::Init(const std::string &engine_spec)
+{
+   //
+   // Initialize inherited fields
+   //
+   memory_resources[0] = NULL;
+   workers_weights[0]= 1.0;
+   workers_mem_res[0] = 0;
+
+   //
+   // Initialize the OCCA engine
+   //
+   ::occa::properties props(engine_spec);
+   device = new ::occa::device[1];
+   device[0].setup(props);
+
+   okl_path = "mfem-occa://";
+   // okl_defines = "...";
+   if (!fileOpenerRegistered)
+   {
+      // The directories from "MFEM_OCCA_OKL_PATH", if any, have the highest
+      // priority.
+      FileOpener *fo = new FileOpener("mfem-occa://", "MFEM_OCCA_OKL_PATH");
+      // Next in priority is the source path, if it exists.
+      std::string mfem_src_prefix = mfem::GetSourcePath();
+      fo->AddDir(mfem_src_prefix + "/backends/occa");
+      // And last in priority is the install path, if it exists.
+      std::string mfem_install_prefix = mfem::GetInstallPath();
+      fo->AddDir(mfem_install_prefix + "/lib/mfem/occa");
+      ::occa::io::fileOpener::add(fo);
+      fileOpenerRegistered = true;
+   }
+}
+
+Engine::Engine(const std::string &engine_spec)
+   : mfem::Engine(NULL, 1, 1)
+{
+   Init(engine_spec);
+}
+
+#ifdef MFEM_USE_MPI
+Engine::Engine(MPI_Comm _comm, const std::string &engine_spec)
+   : mfem::Engine(NULL, 1, 1)
+{
+   comm = _comm;
+   Init(engine_spec);
+}
+#endif
+
+DLayout Engine::MakeLayout(std::size_t size) const
+{
+   return DLayout(new Layout(*this, size));
+}
+
+DLayout Engine::MakeLayout(const mfem::Array<std::size_t> &offsets) const
+{
+   MFEM_ASSERT(offsets.Size() == 2,
+               "multiple workers are not supported yet");
+   return DLayout(new Layout(*this, offsets.Last()));
+}
+
+DArray Engine::MakeArray(PLayout &layout, std::size_t item_size) const
+{
+   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
+               "invalid input layout");
+   Layout *lt = static_cast<Layout *>(&layout);
+   return DArray(new Array(*lt, item_size));
+}
+
+DVector Engine::MakeVector(PLayout &layout, int type_id) const
+{
+   MFEM_ASSERT(type_id == ScalarId<double>::value, "invalid type_id");
+   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
+               "invalid input layout");
+   Layout *lt = static_cast<Layout *>(&layout);
+   return DVector(new Vector(*lt));
+}
+
+DFiniteElementSpace Engine::MakeFESpace(mfem::FiniteElementSpace &fespace) const
+{
+   return DFiniteElementSpace(new FiniteElementSpace(*this, fespace));
+}
+
+DBilinearForm Engine::MakeBilinearForm(mfem::BilinearForm &bf) const
+{
+   return DBilinearForm(new BilinearForm(*this, bf));
+}
+
+void Engine::AssembleLinearForm(LinearForm &l_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+}
+
+mfem::Operator *Engine::MakeOperator(const MixedBilinearForm &mbl_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+   return NULL;
+}
+
+mfem::Operator *Engine::MakeOperator(const NonlinearForm &nl_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+   return NULL;
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/engine.hpp

@@ -0,0 +1,111 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_ENGINE_HPP
+#define MFEM_BACKENDS_OCCA_ENGINE_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "../base/backend.hpp"
+#include <occa.hpp>
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Engine : public mfem::Engine
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // mfem::Backend *backend;
+#ifdef MFEM_USE_MPI
+   // MPI_Comm comm;
+#endif
+   // int num_mem_res;
+   // int num_workers;
+   // MemoryResource **memory_resources;
+   // double *workers_weights;
+   // int *workers_mem_res;
+
+   static bool fileOpenerRegistered;
+   ::occa::device *device; // An array of OCCA devices
+   std::string okl_path, okl_defines;
+
+   void Init(const std::string &engine_spec);
+
+public:
+   Engine(const std::string &engine_spec);
+
+#ifdef MFEM_USE_MPI
+   Engine(MPI_Comm comm, const std::string &engine_spec);
+#endif
+
+   virtual ~Engine() { delete [] device; }
+
+   /**
+       @name OCCA specific interface, used by other objects in the OCCA backend
+    */
+   ///@{
+
+   ::occa::device GetDevice(int idx = 0) const { return device[idx]; }
+
+   /// TODO: doxygen
+   const std::string &GetOklPath() const { return okl_path; }
+
+   /// TODO: doxygen
+   const std::string &GetOklDefines() const { return okl_defines; }
+
+   ///@}
+   // End: OCCA specific interface
+
+   /**
+       @name Virtual interface: finite element data structures and algorithms
+    */
+   ///@{
+
+   virtual DLayout MakeLayout(std::size_t size) const;
+   virtual DLayout MakeLayout(const mfem::Array<std::size_t> &offsets) const;
+
+   virtual DArray MakeArray(PLayout &layout, std::size_t item_size) const;
+
+   virtual DVector MakeVector(PLayout &layout,
+                              int type_id = ScalarId<double>::value) const;
+
+   virtual DFiniteElementSpace MakeFESpace(mfem::FiniteElementSpace &
+                                           fespace) const;
+
+   virtual DBilinearForm MakeBilinearForm(mfem::BilinearForm &bf) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual void AssembleLinearForm(LinearForm &l_form) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual mfem::Operator *MakeOperator(const MixedBilinearForm &mbl_form) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual mfem::Operator *MakeOperator(const NonlinearForm &nl_form) const;
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_ENGINE_HPP

backends/occa/fespace.cpp

@@ -0,0 +1,174 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "backend.hpp"
+#include "fespace.hpp"
+#include "interpolation.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+FiniteElementSpace::FiniteElementSpace(const Engine &e,
+                                       mfem::FiniteElementSpace &fespace)
+   : PFiniteElementSpace(e, fespace),
+     e_layout(e, 0) // resized in SetupLocalGlobalMaps()
+{
+   vdim     = fespace.GetVDim();
+   ordering = fespace.GetOrdering();
+
+   SetupLocalGlobalMaps();
+   SetupOperators();
+   SetupKernels();
+}
+
+FiniteElementSpace::~FiniteElementSpace()
+{
+   delete [] elementDofMap;
+   delete [] elementDofMapInverse;
+   delete restrictionOp;
+   delete prolongationOp;
+}
+
+void FiniteElementSpace::SetupLocalGlobalMaps()
+{
+   const mfem::FiniteElement &fe = *(fes->GetFE(0));
+   const mfem::TensorBasisElement *el =
+      dynamic_cast<const mfem::TensorBasisElement*>(&fe);
+
+   const mfem::Table &e2dTable = fes->GetElementToDofTable();
+   const int *elementMap = e2dTable.GetJ();
+   const int elements = fes->GetNE();
+
+   globalDofs = fes->GetNDofs();
+   localDofs  = fe.GetDof();
+
+   e_layout.Resize(localDofs * elements * fes->GetVDim());
+
+   elementDofMap        = new int[localDofs];
+   elementDofMapInverse = new int[localDofs];
+   if (el)
+   {
+      ::memcpy(elementDofMap,
+               el->GetDofMap().GetData(),
+               localDofs * sizeof(int));
+   }
+   else
+   {
+      for (int i = 0; i < localDofs; ++i)
+      {
+         elementDofMap[i] = i;
+      }
+   }
+   for (int i = 0; i < localDofs; ++i)
+   {
+      elementDofMapInverse[elementDofMap[i]] = i;
+   }
+
+   // Allocate device offsets and indices
+   globalToLocalOffsets.allocate(GetDevice(),
+                                 globalDofs + 1);
+   globalToLocalIndices.allocate(GetDevice(),
+                                 localDofs, elements);
+   localToGlobalMap.allocate(GetDevice(),
+                             localDofs, elements);
+
+   int *offsets = globalToLocalOffsets.ptr();
+   int *indices = globalToLocalIndices.ptr();
+   int *l2gMap  = localToGlobalMap.ptr();
+
+   // We'll be keeping a count of how many local nodes point
+   //   to its global dof
+   for (int i = 0; i <= globalDofs; ++i)
+   {
+      offsets[i] = 0;
+   }
+
+   for (int e = 0; e < elements; ++e)
+   {
+      for (int d = 0; d < localDofs; ++d)
+      {
+         const int gid = elementMap[localDofs*e + d];
+         ++offsets[gid + 1];
+      }
+   }
+   // Aggregate to find offsets for each global dof
+   for (int i = 1; i <= globalDofs; ++i)
+   {
+      offsets[i] += offsets[i - 1];
+   }
+   // For each global dof, fill in all local nodes that point
+   //   to it
+   for (int e = 0; e < elements; ++e)
+   {
+      for (int d = 0; d < localDofs; ++d)
+      {
+         const int gid = elementMap[localDofs*e + elementDofMap[d]];
+         const int lid = localDofs*e + d;
+         indices[offsets[gid]++] = lid;
+         l2gMap[lid] = gid;
+      }
+   }
+   // We shifted the offsets vector by 1 by using it
+   //   as a counter. Now we shift it back.
+   for (int i = globalDofs; i > 0; --i)
+   {
+      offsets[i] = offsets[i - 1];
+   }
+   offsets[0] = 0;
+
+   globalToLocalOffsets.keepInDevice();
+   globalToLocalIndices.keepInDevice();
+   localToGlobalMap.keepInDevice();
+}
+
+void FiniteElementSpace::SetupOperators()
+{
+   const mfem::SparseMatrix *R = fes->GetRestrictionMatrix();
+   const mfem::Operator *P = fes->GetProlongationMatrix();
+   CreateRPOperators(OccaVLayout(), OccaTrueVLayout(),
+                     R, P,
+                     restrictionOp,
+                     prolongationOp);
+}
+
+void FiniteElementSpace::SetupKernels()
+{
+   ::occa::properties props("defines: {"
+                            "  TILESIZE: 256,"
+                            "}");
+   props["defines/NUM_VDIM"] = vdim;
+
+   props["defines/ORDERING_BY_NODES"] = 0;
+   props["defines/ORDERING_BY_VDIM"]  = 1;
+   props["defines/VDIM_ORDERING"] = (int) (ordering == Ordering::byVDIM);
+
+   ::occa::device device = GetDevice();
+   const std::string &okl_path = OccaEngine().GetOklPath();
+   const std::string &okl_defines = OccaEngine().GetOklDefines();
+   globalToLocalKernel = device.buildKernel(okl_path + "fespace.okl",
+                                            "GlobalToLocal",
+                                            props + okl_defines);
+   localToGlobalKernel = device.buildKernel(okl_path + "fespace.okl",
+                                            "LocalToGlobal",
+                                            props + okl_defines);
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/fespace.hpp

@@ -0,0 +1,146 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_FE_SPACE_HPP
+#define MFEM_BACKENDS_OCCA_FE_SPACE_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "engine.hpp"
+#include "operator.hpp"
+#include "../../fem/fem.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+/// TODO: doxygen
+class FiniteElementSpace : public mfem::PFiniteElementSpace
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // mfem::FiniteElementSpace *fes;
+
+   Layout e_layout;
+
+   int *elementDofMap;
+   int *elementDofMapInverse;
+
+   ::occa::array<int> globalToLocalOffsets;
+   ::occa::array<int> globalToLocalIndices;
+   ::occa::array<int> localToGlobalMap;
+   ::occa::kernel globalToLocalKernel, localToGlobalKernel;
+
+   mfem::Ordering::Type ordering;
+
+   int globalDofs, localDofs;
+   int vdim;
+
+   mfem::Operator *restrictionOp, *prolongationOp;
+
+   void SetupLocalGlobalMaps();
+   void SetupOperators();
+   void SetupKernels();
+
+public:
+   /// TODO: doxygen
+   FiniteElementSpace(const Engine &e, mfem::FiniteElementSpace &fespace);
+
+   /// Virtual destructor
+   virtual ~FiniteElementSpace();
+
+   /// TODO: doxygen
+   const Engine &OccaEngine() const
+   { return *static_cast<const Engine *>(engine.Get()); }
+
+   /// TODO: doxygen
+   ::occa::device GetDevice(int idx = 0) const
+   { return OccaEngine().GetDevice(idx); }
+
+   mfem::Mesh* GetMesh() const { return fes->GetMesh(); }
+
+   Layout &OccaVLayout() const
+   { return *fes->GetVLayout().As<Layout>(); }
+
+   Layout &OccaTrueVLayout() const
+   { return *fes->GetTrueVLayout().As<Layout>(); }
+
+   Layout &OccaEVLayout() { return e_layout; }
+
+#ifdef MFEM_USE_MPI
+   bool isDistributed() const { return (OccaEngine().GetComm() != MPI_COMM_NULL); }
+#else
+   bool isDistributed() const { return false; }
+#endif
+
+   bool hasTensorBasis() const
+   { return dynamic_cast<const mfem::TensorBasisElement*>(fes->GetFE(0)); }
+
+   mfem::Ordering::Type GetOrdering() const { return ordering; }
+
+   int GetGlobalDofs() const { return globalDofs; }
+   int GetLocalDofs() const { return localDofs; }
+
+   int GetDim() const { return fes->GetMesh()->Dimension(); }
+   int GetVDim() const { return vdim; }
+
+   int GetVSize() const { return globalDofs * vdim; }
+   int GetTrueVSize() const { return fes->GetTrueVSize(); }
+   int GetGlobalVSize() const { return globalDofs*vdim; /* FIXME: MPI */ }
+   int GetGlobalTrueVSize() const { return fes->GetTrueVSize(); }
+
+   int GetNE() const { return fes->GetNE(); }
+
+   const mfem::FiniteElementCollection* FEColl() const
+   { return fes->FEColl(); }
+   const mfem::FiniteElement* GetFE(const int idx) const
+   { return fes->GetFE(idx); }
+
+   const int* GetElementDofMap() const { return elementDofMap; }
+   const int* GetElementDofMapInverse() const { return elementDofMapInverse; }
+
+   const mfem::Operator* GetRestrictionOperator() { return restrictionOp; }
+   const mfem::Operator* GetProlongationOperator() { return prolongationOp; }
+
+   const ::occa::array<int> GetLocalToGlobalMap() const
+   { return localToGlobalMap; }
+
+   void GlobalToLocal(const Vector &globalVec, Vector &localVec) const
+   {
+      globalToLocalKernel(globalDofs,
+                          localDofs * fes->GetNE(),
+                          globalToLocalOffsets,
+                          globalToLocalIndices,
+                          globalVec.OccaMem(), localVec.OccaMem());
+   }
+   void LocalToGlobal(const Vector &localVec, Vector &globalVec) const
+   {
+      localToGlobalKernel(globalDofs,
+                          localDofs * fes->GetNE(),
+                          globalToLocalOffsets,
+                          globalToLocalIndices,
+                          localVec.OccaMem(), globalVec.OccaMem());
+   }
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_FE_SPACE_HPP

backends/occa/fespace.okl

@@ -0,0 +1,67 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  TILESIZE : Tilesize for iterating over entries
+  ================================================
+*/
+
+#if VDIM_ORDERING == ORDERING_BY_VDIM
+typedef double *Global_t @dim(NUM_VDIM, globalEntries);
+typedef double *Local_t  @dim(NUM_VDIM, localEntries);
+#else
+typedef double *Global_t @dim(NUM_VDIM, globalEntries) @dimOrder(1, 0);
+typedef double *Local_t  @dim(NUM_VDIM, localEntries)  @dimOrder(1, 0);
+#endif
+
+@kernel void GlobalToLocal(const int globalEntries,
+                           const int localEntries,
+                           const int * restrict offsets,
+                           const int * restrict indices,
+                           const Global_t restrict globalX,
+                           Local_t restrict localX) {
+
+  for (int i = 0; i < globalEntries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < globalEntries) {
+      const int offset = offsets[i];
+      const int nextOffset = offsets[i + 1];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        const double dofValue = globalX(v, i);
+        for (int j = offset; j < nextOffset; ++j) {
+          localX(v, indices[j]) = dofValue;
+        }
+      }
+    }
+  }
+}
+
+@kernel void LocalToGlobal(const int globalEntries,
+                           const int localEntries,
+                           const int * restrict offsets,
+                           const int * restrict indices,
+                           const Local_t restrict localX,
+                           Global_t restrict globalX) {
+
+  for (int i = 0; i < globalEntries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < globalEntries) {
+      const int offset = offsets[i];
+      const int nextOffset = offsets[i + 1];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        double dofValue = 0;
+        for (int j = offset; j < nextOffset; ++j) {
+          dofValue += localX(v, indices[j]);
+        }
+        globalX(v, i) = dofValue;
+      }
+    }
+  }
+}

backends/occa/geometry.okl

@@ -0,0 +1,181 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef STORE_JACOBIAN
+#  define STORE_JACOBIAN 1
+#endif
+
+#ifndef STORE_JACOBIAN_INV
+#  define STORE_JACOBIAN_INV 1
+#endif
+
+#ifndef STORE_JACOBIAN_DET
+#  define STORE_JACOBIAN_DET 1
+#endif
+
+typedef double* Local1D_t @dim(1, NUM_DOFS, numElements);
+typedef double* Local2D_t @dim(2, NUM_DOFS, numElements);
+typedef double* Local3D_t @dim(3, NUM_DOFS, numElements);
+
+typedef double* QLocal_t @dim(NUM_QUAD, numElements);
+
+typedef double* DofToQuadD1D_t @dim(NUM_QUAD, NUM_DOFS);
+typedef double* DofToQuadD2D_t @dim(2, NUM_QUAD, NUM_DOFS);
+typedef double* DofToQuadD3D_t @dim(3, NUM_QUAD, NUM_DOFS);
+
+typedef double* Jacobian1D_t @dim(NUM_QUAD, numElements);
+typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD, numElements);
+typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
+
+@kernel void InitGeometryInfo1D(const int numElements,
+                                const DofToQuadD1D_t restrict dofToQuadD,
+                                const Local1D_t restrict nodes,
+                                Jacobian1D_t restrict J,
+                                Jacobian1D_t restrict invJ,
+                                QLocal_t restrict detJ) {
+
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_nodes[NUM_DOFS];
+
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
+        s_nodes[d] = nodes(0, d, e);
+      }
+    }
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      double J11 = 0;
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const double wx = dofToQuadD(q, d);
+        J11 += wx * s_nodes[d];
+      }
+#if STORE_JACOBIAN
+      J(q, e) = J11;
+#endif
+#if STORE_JACOBIAN_INV
+      invJ(q, e) = 1.0 / J11;
+#endif
+#if STORE_JACOBIAN_DET
+      detJ(q, e) = J11;
+#endif
+    }
+  }
+}
+
+@kernel void InitGeometryInfo2D(const int numElements,
+                                const DofToQuadD2D_t restrict dofToQuadD,
+                                const Local2D_t restrict nodes,
+                                Jacobian2D_t restrict J,
+                                Jacobian2D_t restrict invJ,
+                                QLocal_t restrict detJ) {
+
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_nodes[2 * NUM_DOFS] @dim(2, NUM_DOFS);
+
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
+        s_nodes(0, d) = nodes(0, d, e);
+        s_nodes(1, d) = nodes(1, d, e);
+      }
+    }
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      double J11 = 0, J12 = 0;
+      double J21 = 0, J22 = 0;
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const double wx = dofToQuadD(0, q, d);
+        const double wy = dofToQuadD(1, q, d);
+        const double x = s_nodes(0, d);
+        const double y = s_nodes(1, d);
+        J11 += (wx * x); J12 += (wx * y);
+        J21 += (wy * x); J22 += (wy * y);
+      }
+#if STORE_JACOBIAN_INV || STORE_JACOBIAN_DET
+      const double r_detJ = (J11 * J22) - (J12 * J21);
+#endif
+#if STORE_JACOBIAN
+      J(0, 0, q, e) = J11; J(1, 0, q, e) = J12;
+      J(0, 1, q, e) = J21; J(1, 1, q, e) = J22;
+#endif
+#if STORE_JACOBIAN_INV
+      const double r_idetJ = 1.0 / r_detJ;
+      invJ(0, 0, q, e) =  J22 * r_idetJ;
+      invJ(1, 0, q, e) = -J12 * r_idetJ;
+
+      invJ(0, 1, q, e) = -J21 * r_idetJ;
+      invJ(1, 1, q, e) =  J11 * r_idetJ;
+#endif
+#if STORE_JACOBIAN_DET
+      detJ(q, e) = r_detJ;
+#endif
+    }
+  }
+}
+
+@kernel void InitGeometryInfo3D(const int numElements,
+                                const DofToQuadD3D_t restrict dofToQuadD,
+                                const Local3D_t restrict nodes,
+                                Jacobian3D_t restrict J,
+                                Jacobian3D_t restrict invJ,
+                                QLocal_t restrict detJ) {
+
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_nodes[3 * NUM_DOFS] @dim(3, NUM_DOFS);
+
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
+        s_nodes(0, d) = nodes(0, d, e);
+        s_nodes(1, d) = nodes(1, d, e);
+        s_nodes(2, d) = nodes(2, d, e);
+      }
+    }
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      double J11 = 0, J12 = 0, J13 = 0;
+      double J21 = 0, J22 = 0, J23 = 0;
+      double J31 = 0, J32 = 0, J33 = 0;
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const double wx = dofToQuadD(0, q, d);
+        const double wy = dofToQuadD(1, q, d);
+        const double wz = dofToQuadD(2, q, d);
+        const double x = s_nodes(0, d);
+        const double y = s_nodes(1, d);
+        const double z = s_nodes(2, d);
+        J11 += (wx * x); J12 += (wx * y); J13 += (wx * z);
+        J21 += (wy * x); J22 += (wy * y); J23 += (wy * z);
+        J31 += (wz * x); J32 += (wz * y); J33 += (wz * z);
+      }
+#if STORE_JACOBIAN_INV || STORE_JACOBIAN_DET
+      const double r_detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                             (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+#endif
+#if STORE_JACOBIAN
+      J(0, 0, q, e) = J11; J(1, 0, q, e) = J12; J(2, 0, q, e) = J13;
+      J(0, 1, q, e) = J21; J(1, 1, q, e) = J22; J(2, 1, q, e) = J23;
+      J(0, 2, q, e) = J31; J(1, 2, q, e) = J32; J(2, 2, q, e) = J33;
+#endif
+#if STORE_JACOBIAN_INV
+      const double r_idetJ = 1.0 / r_detJ;
+      invJ(0, 0, q, e) = r_idetJ * ((J22 * J33) - (J23 * J32));
+      invJ(1, 0, q, e) = r_idetJ * ((J32 * J13) - (J33 * J12));
+      invJ(2, 0, q, e) = r_idetJ * ((J12 * J23) - (J13 * J22));
+
+      invJ(0, 1, q, e) = r_idetJ * ((J23 * J31) - (J21 * J33));
+      invJ(1, 1, q, e) = r_idetJ * ((J33 * J11) - (J31 * J13));
+      invJ(2, 1, q, e) = r_idetJ * ((J13 * J21) - (J11 * J23));
+
+      invJ(0, 2, q, e) = r_idetJ * ((J21 * J32) - (J22 * J31));
+      invJ(1, 2, q, e) = r_idetJ * ((J31 * J12) - (J32 * J11));
+      invJ(2, 2, q, e) = r_idetJ * ((J11 * J22) - (J12 * J21));
+#endif
+#if STORE_JACOBIAN_DET
+      detJ(q, e) = r_detJ;
+#endif
+    }
+  }
+}

backends/occa/gridfunc.cpp

@@ -0,0 +1,195 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "gridfunc.hpp"
+#include "bilininteg.hpp"
+#include "../../fem/gridfunc.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+std::map<std::string, ::occa::kernel> gridFunctionKernels;
+
+::occa::kernel GetGridFunctionKernel(::occa::device device,
+                                     FiniteElementSpace &fespace,
+                                     const mfem::IntegrationRule &ir)
+{
+   const int numQuad = ir.GetNPoints();
+
+   const FiniteElement &fe = *(fespace.GetFE(0));
+   const int dim  = fe.GetDim();
+   const int vdim = fespace.GetVDim();
+
+   std::stringstream ss;
+   ss << ::occa::hash(device)
+      << "FEColl : " << fespace.FEColl()->Name()
+      << "Quad: "    << numQuad
+      << "Dim: "     << dim
+      << "VDim: "    << vdim;
+   std::string hash = ss.str();
+
+   // Kernel defines
+   ::occa::properties props;
+   props["defines/NUM_VDIM"] = vdim;
+
+   SetProperties(fespace, ir, props);
+
+   ::occa::kernel kernel = gridFunctionKernels[hash];
+   if (!kernel.isInitialized())
+   {
+      const std::string &okl_path = fespace.OccaEngine().GetOklPath();
+      kernel = device.buildKernel(okl_path + "gridfunc.okl",
+                                  stringWithDim("GridFuncToQuad", dim),
+                                  props);
+   }
+   return kernel;
+}
+
+// OccaGridFunction::OccaGridFunction() :
+//    Vector(),
+//    ofespace(NULL),
+//    sequence(0) {}
+
+OccaGridFunction::OccaGridFunction(FiniteElementSpace *ofespace_)
+   : PArray(ofespace_->OccaVLayout()),
+     Array(ofespace_->OccaVLayout(), sizeof(double)),
+     Vector(ofespace_->OccaVLayout()),
+     ofespace(ofespace_),
+     sequence(0) {}
+
+// OccaGridFunction::OccaGridFunction(OccaFiniteElementSpace *ofespace_,
+//                                    OccaVectorRef ref) :
+//    OccaVector(ref),
+//    ofespace(ofespace_),
+//    sequence(0) {}
+
+OccaGridFunction::OccaGridFunction(const OccaGridFunction &v)
+   : PArray(v),
+     Array(v),
+     Vector(v),
+     ofespace(v.ofespace),
+     sequence(v.sequence) {}
+
+OccaGridFunction& OccaGridFunction::operator = (double value)
+{
+   Fill(value);
+   return *this;
+}
+
+OccaGridFunction& OccaGridFunction::operator = (const Vector &v)
+{
+   Assign<double>(v);
+   return *this;
+}
+
+// OccaGridFunction& OccaGridFunction::operator = (const OccaVectorRef &v)
+// {
+//    OccaVector::operator = (v);
+//    return *this;
+// }
+
+OccaGridFunction& OccaGridFunction::operator = (const OccaGridFunction &v)
+{
+   Assign<double>(v);
+   return *this;
+}
+
+// void OccaGridFunction::SetGridFunction(mfem::GridFunction &gf)
+// {
+//    Vector v = *this;
+//    gf.MakeRef(ofespace->GetFESpace(), v, 0);
+//    // Make gf the owner of the data
+//    v.Swap(gf);
+// }
+
+void OccaGridFunction::GetTrueDofs(Vector &v)
+{
+   const mfem::Operator *R = ofespace->GetRestrictionOperator();
+   if (!R)
+   {
+      v.MakeRef(*this);
+   }
+   else
+   {
+      v.Resize<double>(R->OutLayout(), NULL);
+      mfem::Vector mfem_v(v);
+      R->Mult(this->Wrap(), mfem_v);
+   }
+}
+
+void OccaGridFunction::SetFromTrueDofs(Vector &v)
+{
+   const mfem::Operator *P = ofespace->GetProlongationOperator();
+   if (!P)
+   {
+      MakeRef(v);
+   }
+   else
+   {
+      Resize<double>(P->OutLayout(), NULL);
+      mfem::Vector mfem_this(*this);
+      P->Mult(v.Wrap(), mfem_this);
+   }
+}
+
+mfem::FiniteElementSpace* OccaGridFunction::GetFESpace()
+{
+   return ofespace->GetFESpace();
+}
+
+const mfem::FiniteElementSpace* OccaGridFunction::GetFESpace() const
+{
+   return ofespace->GetFESpace();
+}
+
+void OccaGridFunction::ToQuad(const IntegrationRule &ir, Vector &quadValues)
+{
+   const Engine &engine = OccaLayout().OccaEngine();
+   ::occa::device device = engine.GetDevice();
+
+   OccaDofQuadMaps &maps = OccaDofQuadMaps::Get(device, *ofespace, ir);
+
+   const int elements = ofespace->GetNE();
+   const int numQuad  = ir.GetNPoints();
+   quadValues.Resize<double>(*(new Layout(engine, numQuad * elements)), NULL);
+
+   ::occa::kernel g2qKernel = GetGridFunctionKernel(device, *ofespace, ir);
+   g2qKernel(elements,
+             maps.dofToQuad,
+             ofespace->GetLocalToGlobalMap(),
+             this->OccaMem(),
+             quadValues.OccaMem());
+}
+
+void OccaGridFunction::Distribute(const Vector &v)
+{
+   if (ofespace->isDistributed())
+   {
+      mfem::Vector mfem_this(*this);
+      ofespace->GetProlongationOperator()->Mult(v.Wrap(), mfem_this);
+   }
+   else
+   {
+      *this = v;
+   }
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/gridfunc.hpp

@@ -0,0 +1,83 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_GRID_FUNC_HPP
+#define MFEM_BACKENDS_OCCA_GRID_FUNC_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "vector.hpp"
+#include "fespace.hpp"
+
+namespace mfem
+{
+
+class IntegrationRule;
+class GridFunction;
+
+namespace occa
+{
+
+class OccaIntegrator;
+class OccaDofQuadMaps;
+
+// TODO: make this object part of the backend or the engine.
+extern std::map<std::string, ::occa::kernel> gridFunctionKernels;
+
+// TODO: make this a method of the backend or the engine.
+::occa::kernel GetGridFunctionKernel(::occa::device device,
+                                     FiniteElementSpace &fespace,
+                                     const mfem::IntegrationRule &ir);
+
+class OccaGridFunction : public Vector
+{
+protected:
+   FiniteElementSpace *ofespace;
+   long sequence;
+
+   ::occa::kernel gridFuncToQuad[3];
+
+public:
+   // OccaGridFunction();
+
+   OccaGridFunction(FiniteElementSpace *ofespace_);
+
+   // OccaGridFunction(FiniteElementSpace *ofespace_,
+   //                  OccaVectorRef ref);
+
+   OccaGridFunction(const OccaGridFunction &gf);
+
+   OccaGridFunction& operator = (double value);
+   OccaGridFunction& operator = (const Vector &v);
+   // OccaGridFunction& operator = (const OccaVectorRef &v);
+   OccaGridFunction& operator = (const OccaGridFunction &gf);
+
+   // void SetGridFunction(mfem::GridFunction &gf);
+
+   void GetTrueDofs(Vector &v);
+   void SetFromTrueDofs(Vector &v);
+
+   mfem::FiniteElementSpace* GetFESpace();
+   const mfem::FiniteElementSpace* GetFESpace() const;
+
+   void ToQuad(const mfem::IntegrationRule &ir, Vector &quadValues);
+
+   void Distribute(const Vector &v);
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_GRID_FUNC_HPP

backends/occa/gridfunc.okl

@@ -0,0 +1,26 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+#if USING_TENSOR_OPS
+#  if OCCA_USING_CPU
+#    include "mfem-occa://gridfunc/tensor/cpu.okl"
+#  else
+#    include "mfem-occa://gridfunc/tensor/gpuHighOrder.okl"
+#  endif
+#else
+#  if OCCA_USING_CPU
+#    include "mfem-occa://gridfunc/simplex/cpu.okl"
+#  else
+#    include "mfem-occa://gridfunc/simplex/gpuHighOrder.okl"
+#  endif
+#endif

backends/occa/gridfunc/simplex/cpu.okl

@@ -0,0 +1,63 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void GridFuncToQuad2D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal_t restrict out) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const int gid = l2gMap(d, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          const double r_gf = gf[v + gid*NUM_VDIM];
+          double r_out = 0;
+          for (int q = 0; q < NUM_QUAD; ++q) {
+            r_out += r_gf * dofToQuad(d, q);
+          }
+          out(v, d, e) = r_out;
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void GridFuncToQuad3D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal_t restrict out) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const int gid = l2gMap(d, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          const double r_gf = gf[v + gid*NUM_VDIM];
+          double r_out = 0;
+          for (int q = 0; q < NUM_QUAD; ++q) {
+            r_out += r_gf * dofToQuad(d, q);
+          }
+          out(v, d, e) = r_out;
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/gridfunc/simplex/gpuHighOrder.okl

@@ -0,0 +1,79 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void GridFuncToQuad2D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal_t restrict out) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_gf[NUM_VDIM][NUM_DOFS];
+
+    for (int dOff = 0; dOff < M2_INNER_BATCH; ++dOff; @inner) {
+      for (int d = dOff; d < NUM_DOFS; d += M2_INNER_BATCH) {
+        const int gid = l2gMap(d, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          s_gf[v][d] = gf[v + gid*NUM_VDIM]];
+        }
+      }
+    }
+
+    for (int qOff = 0; qOff < M2_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M2_INNER_BATCH) {
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          double r_out = 0;
+          for (int d = 0; d < NUM_DOFS; ++d) {
+            r_out += s_gf[v][d] * dofToQuad(d, q);
+          }
+          out(v, q, e) = r_out;
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void GridFuncToQuad3D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal_t restrict out) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_gf[NUM_VDIM][NUM_DOFS];
+
+    for (int dOff = 0; dOff < M3_INNER_BATCH; ++dOff; @inner) {
+      for (int d = dOff; d < NUM_DOFS; d += M3_INNER_BATCH) {
+        const int gid = l2gMap(d, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          s_gf[v][d] = gf[v + gid*NUM_VDIM]];
+        }
+      }
+    }
+
+    for (int qOff = 0; qOff < M3_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M3_INNER_BATCH) {
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          double r_out = 0;
+          for (int d = 0; d < NUM_DOFS; ++d) {
+            r_out += s_gf[v][d] * dofToQuad(d, q);
+          }
+          out(v, q, e) = r_out;
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/gridfunc/tensor/cpu.okl

@@ -0,0 +1,188 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void GridFuncToQuad1D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap1D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal1D_t restrict out) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double r_out[NUM_VDIM][NUM_QUAD_1D];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          r_out[v][qx] = 0;
+        }
+      }
+      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+        const int gid = l2gMap(dx, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          const double r_gf = gf[v + gid*NUM_VDIM];
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            r_out[v][qx] += r_gf * dofToQuad(qx, dx);
+          }
+        }
+      }
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          out(v, qx, e) = r_out[v][qx];
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void GridFuncToQuad2D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap2D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal2D_t restrict out) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double out_xy[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            out_xy[v][qy][qx] = 0;
+          }
+        }
+      }
+
+      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+        double out_x[NUM_VDIM][NUM_QUAD_1D];
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            out_x[v][qy] = 0;
+          }
+        }
+
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          const int gid = l2gMap(dx, dy, e);
+          for (int v = 0; v < NUM_VDIM; ++v) {
+            const double r_gf = gf[v + gid*NUM_VDIM];
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              out_x[v][qy] += r_gf * dofToQuad(qy, dx);
+            }
+          }
+        }
+
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            const double d2q = dofToQuad(qy, dy);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              out_xy[v][qy][qx] += d2q * out_x[v][qx];
+            }
+          }
+        }
+      }
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          for (int v = 0; v < NUM_VDIM; ++v) {
+            out(v, qx, qy, e) = out_xy[v][qy][qx];
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void GridFuncToQuad3D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap3D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal3D_t restrict out) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double out_xyz[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              out_xyz[v][qz][qy][qx] = 0;
+            }
+          }
+        }
+      }
+
+      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+        double out_xy[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D];
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              out_xy[v][qy][qx] = 0;
+            }
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          double out_x[NUM_VDIM][NUM_QUAD_1D];
+          for (int v = 0; v < NUM_VDIM; ++v) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              out_x[v][qx] = 0;
+            }
+          }
+
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const int gid = l2gMap(dx, dy, dz, e);
+            for (int v = 0; v < NUM_VDIM; ++v) {
+              const double r_gf = gf[v + gid*NUM_VDIM];
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                out_x[v][qx] += r_gf * dofToQuad(qx, dx);
+              }
+            }
+          }
+
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            const double wy = dofToQuad(qy, dy);
+            for (int v = 0; v < NUM_VDIM; ++v) {
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                out_xy[v][qy][qx] += wy * out_x[v][qx];
+              }
+            }
+          }
+        }
+
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          const double wz = dofToQuad(qz, dz);
+          for (int v = 0; v < NUM_VDIM; ++v) {
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                out_xyz[v][qz][qy][qx] += wz * out_xy[v][qy][qx];
+              }
+            }
+          }
+        }
+      }
+
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            for (int v = 0; v < NUM_VDIM; ++v) {
+              out(v, qx, qy, qz, e) = out_xyz[v][qz][qy][qx];
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/gridfunc/tensor/gpuHighOrder.okl

@@ -0,0 +1,183 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void GridFuncToQuad1D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap1D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QLocal1D_t restrict out) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @exclusive double r_out[NUM_QUAD_1D];
+
+    for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+      for (int i = el; i < NUM_QUAD_DOFS_1D; i += M1_INNER_ELEMENT_BATCH) {
+        s_dofToQuad[i] = dofToQuad[i];
+      }
+    }
+
+    for (int b = 0; b < M1_OUTER_ELEMENT_BATCH; ++b) {
+      for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+        const int e = eOff + b*M1_INNER_ELEMENT_BATCH + el;
+        if (e < numElements) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            r_out[qx] = 0;
+          }
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double r_gf = gf[l2gMap(dx, e)];
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              r_out[qx] += r_gf * s_dofToQuad(qx, dx);
+            }
+          }
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            out(qx, e) = r_out[qx];
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void GridFuncToQuad2D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap2D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QLocal2D_t restrict out) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+
+    // Store xy planes in shared memory
+    @shared double s_xy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+      for (int id = x; id < NUM_QUAD_DOFS_1D; id += NUM_MAX_1D) {
+        s_dofToQuad[id] = dofToQuad[id];
+      }
+    }
+
+    for (int e = eOff; e < (eOff + M2_ELEMENT_BATCH); ++e) {
+      if (e < numElements) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if (dx < NUM_DOFS_1D) {
+            double r_x[NUM_DOFS_1D];
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              r_x[dy] = gf[l2gMap(dx, dy, e)];
+            }
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              double xy = 0;
+              for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+                xy += r_x[dy] * s_dofToQuad(qy, dy);
+              }
+              s_xy(dx, qy) = xy;
+            }
+          }
+        }
+        for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+          if (qy < NUM_QUAD_1D) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              double val = 0;
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                val += s_xy(dx, qy) * s_dofToQuad(qx, dx);
+              }
+              out(qx, qy, e) = val;
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void GridFuncToQuad3D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap3D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QLocal3D_t restrict out) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+
+    // Store xy planes in shared memory
+    @shared double s_z[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
+
+    // Store z axis as registers
+    @exclusive double r_qz[NUM_QUAD_1D];
+
+    for (int y = 0; y < NUM_MAX_1D; ++y; @inner) {
+      for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+        const int id = (y * NUM_MAX_1D) + x;
+        // Fetch Q <--> D maps
+        if (id < NUM_QUAD_DOFS_1D) {
+          s_dofToQuad[id] = dofToQuad[id];
+        }
+        // Initialize our Z axis
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          r_qz[qz] = 0;
+        }
+      }
+    }
+
+    for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+      for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+        if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+          for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+            const double val = gf[l2gMap(dx, dy, dz, e)];
+            // Calculate D -> Q in the Z axis
+            for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+              r_qz[qz] += val * s_dofToQuad(qz, dz);
+            }
+          }
+        }
+      }
+    }
+    // For each xy plane
+    for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+      // Fill xy plane at given z position
+      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+            s_z(dx, dy) = r_qz[qz];
+          }
+        }
+      }
+      // Calculate Dxyz, xDyz, xyDz in plane
+      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
+            double val = 0;
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              const double wy = s_dofToQuad(qy, dy);
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                const double wx = s_dofToQuad(qx, dx);
+                val += wx * wy * s_z(dx, dy);
+              }
+            }
+            out(qx, qy, qz, e) = val;
+          }
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/interpolation.cpp

@@ -0,0 +1,162 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "interpolation.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+void CreateRPOperators(Layout &v_layout, Layout &t_layout,
+                       const mfem::SparseMatrix *R, const mfem::Operator *P,
+                       mfem::Operator *&OccaR, mfem::Operator *&OccaP)
+{
+   if (!P)
+   {
+      OccaR = new IdentityOperator(t_layout);
+      OccaP = new IdentityOperator(t_layout);
+      return;
+   }
+
+   const mfem::SparseMatrix *pmat = dynamic_cast<const mfem::SparseMatrix*>(P);
+   ::occa::device device = v_layout.OccaEngine().GetDevice();
+
+   if (R)
+   {
+      OccaSparseMatrix *occaR =
+         CreateMappedSparseMatrix(v_layout, t_layout, *R);
+      ::occa::array<int> reorderIndices = occaR->reorderIndices;
+      delete occaR;
+
+      OccaR = new RestrictionOperator(v_layout, t_layout, reorderIndices);
+   }
+
+   if (pmat)
+   {
+      const mfem::SparseMatrix *pmatT = Transpose(*pmat);
+
+      OccaSparseMatrix *occaP  =
+         CreateMappedSparseMatrix(t_layout, v_layout, *pmat);
+      OccaSparseMatrix *occaPT =
+         CreateMappedSparseMatrix(v_layout, t_layout, *pmatT);
+
+      OccaP = new ProlongationOperator(*occaP, *occaPT);
+   }
+   else
+   {
+      OccaP = new ProlongationOperator(t_layout, v_layout, P);
+   }
+}
+
+RestrictionOperator::RestrictionOperator(Layout &in_layout, Layout &out_layout,
+                                         ::occa::array<int> indices) :
+   Operator(in_layout, out_layout)
+{
+
+   entries     = indices.size() / 2;
+   trueIndices = indices;
+
+   // FIXME: paths ...
+   ::occa::device device = in_layout.OccaEngine().GetDevice();
+   const std::string &okl_path = in_layout.OccaEngine().GetOklPath();
+   const std::string &okl_defines = in_layout.OccaEngine().GetOklDefines();
+   multOp = device.buildKernel(okl_path + "mappings.okl",
+                               "ExtractSubVector",
+                               "defines: { TILESIZE: 256 }" + okl_defines);
+
+   multTransposeOp = device.buildKernel(okl_path + "mappings.okl",
+                                        "SetSubVector",
+                                        "defines: { TILESIZE: 256 }" +
+                                        okl_defines);
+}
+
+void RestrictionOperator::Mult_(const Vector &x, Vector &y) const
+{
+   multOp(entries, trueIndices, x.OccaMem(), y.OccaMem());
+}
+
+void RestrictionOperator::MultTranspose_(const Vector &x, Vector &y) const
+{
+   y.Fill<double>(0.0);
+   multTransposeOp(entries, trueIndices, x.OccaMem(), y.OccaMem());
+}
+
+ProlongationOperator::ProlongationOperator(OccaSparseMatrix &multOp_,
+                                           OccaSparseMatrix &multTransposeOp_) :
+   Operator(multOp_),
+   pmat(NULL),
+   multOp(multOp_),
+   multTransposeOp(multTransposeOp_) {}
+
+ProlongationOperator::ProlongationOperator(Layout &in_layout,
+                                           Layout &out_layout,
+                                           const mfem::Operator *pmat_) :
+   Operator(in_layout, out_layout),
+   pmat(pmat_),
+   multOp(*this),
+   multTransposeOp(*this)
+{ }
+
+void ProlongationOperator::Mult_(const Vector &x, Vector &y) const
+{
+   MFEM_VERIFY(pmat == NULL, "");
+   multOp.Mult_(x, y);
+}
+
+void ProlongationOperator::MultTranspose_(const Vector &x, Vector &y) const
+{
+   MFEM_VERIFY(pmat == NULL, "");
+   multTransposeOp.Mult_(x, y);
+}
+
+void ProlongationOperator::Mult(const mfem::Vector &x, mfem::Vector &y) const
+{
+   if (pmat)
+   {
+      // FIXME: create an OCCA version of 'pmat'
+      x.Pull();
+      y.Pull(false);
+      pmat->Mult(x, y);
+      y.Push();
+   }
+   else
+   {
+      multOp.Mult(x, y);
+   }
+}
+
+void ProlongationOperator::MultTranspose(const mfem::Vector &x,
+                                         mfem::Vector &y) const
+{
+   if (pmat)
+   {
+      // FIXME: create an OCCA version of 'pmat'
+      x.Pull();
+      y.Pull(false);
+      pmat->MultTranspose(x, y);
+      y.Push();
+   }
+   else
+   {
+      multTransposeOp.Mult(x, y);
+   }
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/interpolation.hpp

@@ -0,0 +1,79 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_INTERPOLATION_HPP
+#define MFEM_BACKENDS_OCCA_INTERPOLATION_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include <occa.hpp>
+#include "vector.hpp"
+#include "engine.hpp"
+#include "sparsemat.hpp"
+#include "../../fem/fem.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+// [MISSING] Proper destructors
+void CreateRPOperators(Layout &v_layout, Layout &t_layout,
+                       const mfem::SparseMatrix *R, const mfem::Operator *P,
+                       mfem::Operator *&OccaR, mfem::Operator *&OccaP);
+
+class RestrictionOperator : public Operator
+{
+protected:
+   int entries;
+   ::occa::array<int> trueIndices;
+   ::occa::kernel multOp, multTransposeOp;
+
+public:
+   RestrictionOperator(Layout &in_layout, Layout &out_layout,
+                       ::occa::array<int> indices);
+
+   // overrides
+   virtual void Mult_(const Vector &x, Vector &y) const;
+   virtual void MultTranspose_(const Vector &x, Vector &y) const;
+};
+
+class ProlongationOperator : public Operator
+{
+protected:
+   const mfem::Operator *pmat;
+   OccaSparseMatrix multOp, multTransposeOp;
+
+public:
+   ProlongationOperator(OccaSparseMatrix &multOp_,
+                        OccaSparseMatrix &multTransposeOp_);
+
+   ProlongationOperator(Layout &in_layout, Layout &out_layout,
+                        const mfem::Operator *pmat_);
+
+   // overrides
+   virtual void Mult_(const Vector &x, Vector &y) const;
+   virtual void MultTranspose_(const Vector &x, Vector &y) const;
+
+   // overrides
+   virtual void Mult(const mfem::Vector &x, mfem::Vector &y) const;
+   virtual void MultTranspose(const mfem::Vector &x, mfem::Vector &y) const;
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_INTERPOLATION_HPP

backends/occa/layout.cpp

@@ -0,0 +1,40 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "layout.hpp"
+#include "../../general/array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+void Layout::Resize(std::size_t new_size)
+{
+   size = new_size;
+}
+
+void Layout::Resize(const Array<std::size_t> &offsets)
+{
+   MFEM_ASSERT(offsets.Size() == 2,
+               "multiple workers are not supported yet");
+   size = offsets.Last();
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/layout.hpp

@@ -0,0 +1,68 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_LAYOUT_HPP
+#define MFEM_BACKENDS_OCCA_LAYOUT_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "../base/layout.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Layout : public PLayout
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // std::size_t size;
+
+public:
+   Layout(const Engine &e, std::size_t s = 0) : PLayout(e, s) { }
+
+   const Engine &OccaEngine() const
+   { return *static_cast<const Engine *>(engine.Get()); }
+
+   ::occa::memory Alloc(std::size_t bytes) const
+   { return OccaEngine().GetDevice().malloc(bytes); }
+
+   virtual ~Layout() { }
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   /// Resize the layout
+   virtual void Resize(std::size_t new_size);
+
+   /// Resize the layout based on the given worker offsets
+   virtual void Resize(const Array<std::size_t> &offsets);
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_LAYOUT_HPP

backends/occa/mappings.okl

@@ -0,0 +1,54 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  TILESIZE : Tilesize for iterating over entries
+  ================================================
+*/
+
+@kernel void ExtractSubVector(const int entries,
+                              const int * restrict indices,
+                              const double * restrict in,
+                              double * restrict out) {
+
+  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < entries) {
+      out[i] = in[indices[i]];
+    }
+  }
+}
+
+@kernel void SetSubVector(const int entries,
+                          const int * restrict indices,
+                          const double * restrict in,
+                          double * restrict out) {
+
+  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < entries) {
+      out[indices[i]] = in[i];
+    }
+  }
+}
+
+@kernel void MapSubVector(const int entries,
+                          const int * restrict indices,
+                          const double * restrict in,
+                          double * restrict out) {
+
+  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < entries) {
+      const int fromIdx = indices[2*i + 0];
+      const int toIdx   = indices[2*i + 1];
+      out[toIdx] = in[fromIdx];
+    }
+  }
+}

backends/occa/operator.cpp

@@ -0,0 +1,135 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "operator.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+// FIXME: move this object to the Backend?
+::occa::kernelBuilder OccaConstrainedOperator::mapDofBuilder =
+   ::occa::linalg::customLinearMethod(
+      "vector_map_dofs",
+
+      "const int idx = v2[i];"
+      "v0[idx] = v1[idx];",
+
+      "defines: {"
+      "  VTYPE0: 'double',"
+      "  VTYPE1: 'double',"
+      "  VTYPE2: 'int',"
+      "  TILESIZE: 128,"
+      "}");
+
+// FIXME: move this object to the Backend?
+::occa::kernelBuilder OccaConstrainedOperator::clearDofBuilder =
+   ::occa::linalg::customLinearMethod(
+      "vector_clear_dofs",
+
+      "v0[v1[i]] = 0.0;",
+
+      "defines: {"
+      "  VTYPE0: 'double',"
+      "  VTYPE1: 'int',"
+      "  TILESIZE: 128,"
+      "}");
+
+OccaConstrainedOperator::OccaConstrainedOperator(
+   mfem::Operator *A_,
+   const mfem::Array<int> &constraintList_,
+   bool own_A_)
+
+   : Operator(A_->InLayout()->As<Layout>()),
+     z(OutLayout_()),
+     w(OutLayout_()),
+     mfem_z((z.DontDelete(), z)),
+     mfem_w((w.DontDelete(), w))
+{
+   Setup(OutLayout_().OccaEngine().GetDevice(), A_, constraintList_, own_A_);
+}
+
+void OccaConstrainedOperator::Setup(::occa::device device_,
+                                    mfem::Operator *A_,
+                                    const mfem::Array<int> &constraintList_,
+                                    bool own_A_)
+{
+   device = device_;
+
+   A = A_;
+   own_A = own_A_;
+
+   constraintIndices = constraintList_.Size();
+   constraintList = constraintList_.Get_PArray()->As<Array>().OccaMem();
+}
+
+void OccaConstrainedOperator::EliminateRHS(const Vector &x, Vector &b) const
+{
+   const std::string &okl_defines = InLayout_().OccaEngine().GetOklDefines();
+   ::occa::kernel mapDofs = mapDofBuilder.build(device, okl_defines);
+
+   w.Fill<double>(0.0);
+
+   if (constraintIndices)
+   {
+      mapDofs(constraintIndices, w.OccaMem(), x.OccaMem(), constraintList);
+   }
+
+   A->Mult(mfem_w, mfem_z);
+
+   b.Axpby<double>(1.0, b, -1.0, z);
+
+   if (constraintIndices)
+   {
+      mapDofs(constraintIndices, b.OccaMem(), x.OccaMem(), constraintList);
+   }
+}
+
+void OccaConstrainedOperator::Mult_(const Vector &x, Vector &y) const
+{
+   mfem::Vector mfem_y(y);
+   if (constraintIndices == 0)
+   {
+      A->Mult(x.Wrap(), mfem_y);
+      return;
+   }
+
+   const std::string &okl_defines = InLayout_().OccaEngine().GetOklDefines();
+   ::occa::kernel mapDofs   = mapDofBuilder.build(device, okl_defines);
+   ::occa::kernel clearDofs = clearDofBuilder.build(device, okl_defines);
+
+   z.Assign<double>(x); // z = x
+
+   clearDofs(constraintIndices, z.OccaMem(), constraintList);
+
+   A->Mult(mfem_z, mfem_y);
+
+   mapDofs(constraintIndices, y.OccaMem(), x.OccaMem(), constraintList);
+}
+
+OccaConstrainedOperator::~OccaConstrainedOperator()
+{
+   if (own_A)
+   {
+      delete A;
+   }
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/operator.hpp

@@ -0,0 +1,129 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_OPERATOR_HPP
+#define MFEM_BACKENDS_OCCA_OPERATOR_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "vector.hpp"
+#include "../../linalg/operator.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Operator : public mfem::Operator
+{
+public:
+   /// Creare an operator with the same dimensions as @a orig.
+   Operator(const Operator &orig)
+      : mfem::Operator(orig) { }
+
+   Operator(Layout &layout)
+      : mfem::Operator(layout) { }
+
+   Operator(Layout &in_layout, Layout &out_layout)
+      : mfem::Operator(in_layout, out_layout) { }
+
+   Layout &InLayout_() const
+   { return *static_cast<Layout*>(in_layout.Get()); }
+
+   Layout &OutLayout_() const
+   { return *static_cast<Layout*>(out_layout.Get()); }
+
+   virtual void Mult_(const Vector &x, Vector &y) const = 0;
+
+   virtual void MultTranspose_(const Vector &x, Vector &y) const
+   { MFEM_ABORT("method is not supported"); }
+
+   // override
+   virtual void Mult(const mfem::Vector &x, mfem::Vector &y) const
+   {
+      Mult_(x.Get_PVector()->As<Vector>(),
+            y.Get_PVector()->As<Vector>());
+   }
+
+   // override
+   virtual void MultTranspose(const mfem::Vector &x, mfem::Vector &y) const
+   {
+      MultTranspose_(x.Get_PVector()->As<Vector>(),
+                     y.Get_PVector()->As<Vector>());
+   }
+};
+
+
+class OccaConstrainedOperator : public Operator
+{
+protected:
+   ::occa::device device;
+
+   mfem::Operator *A;              //< The unconstrained Operator.
+   bool own_A;                     //< Ownership flag for A.
+   ::occa::memory constraintList;  //< List of constrained indices/dofs.
+   int constraintIndices;
+   mutable Vector z, w;       //< Auxiliary vectors.
+   mutable mfem::Vector mfem_z, mfem_w; // Wrap z, w
+
+   static ::occa::kernelBuilder mapDofBuilder, clearDofBuilder;
+
+public:
+   /** @brief Constructor from a general Operator and a list of essential
+       indices/dofs.
+
+       Specify the unconstrained operator @a *A and a @a list of indices to
+       constrain, i.e. each entry @a list[i] represents an essential-dof. If the
+       ownership flag @a own_A is true, the operator @a *A will be destroyed
+       when this object is destroyed. */
+   OccaConstrainedOperator(mfem::Operator *A_,
+                           const mfem::Array<int> &constraintList_,
+                           bool own_A_ = false);
+
+   void Setup(::occa::device device_,
+              mfem::Operator *A_,
+              const mfem::Array<int> &constraintList_,
+              bool own_A_ = false);
+
+   /** @brief Eliminate "essential boundary condition" values specified in @a x
+       from the given right-hand side @a b.
+
+       Performs the following steps:
+
+       z = A((0,x_b));  b_i -= z_i;  b_b = x_b;
+
+       where the "_b" subscripts denote the essential (boundary) indices/dofs of
+       the vectors, and "_i" -- the rest of the entries. */
+   void EliminateRHS(const Vector &x, Vector &b) const;
+
+   /** @brief Constrained operator action.
+
+       Performs the following steps:
+
+       z = A((x_i,0));  y_i = z_i;  y_b = x_b;
+
+       where the "_b" subscripts denote the essential (boundary) indices/dofs of
+       the vectors, and "_i" -- the rest of the entries. */
+   virtual void Mult_(const Vector &x, Vector &y) const;
+
+   // Destructor: destroys the unconstrained Operator @a A if @a own_A is true.
+   virtual ~OccaConstrainedOperator();
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_OPERATOR_HPP

backends/occa/sparse.okl

@@ -0,0 +1,57 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  TILESIZE : Tilesize for iterating over dofs
+  ================================================
+*/
+
+@kernel void Mult(const int entries,
+                  const int * restrict offsets,
+                  const int * restrict indices,
+                  const double * restrict weights,
+                  const double * restrict in,
+                  double * restrict out) {
+
+  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < entries) {
+      const int offset = offsets[i];
+      const int nextOffset = offsets[i + 1];
+      double value = 0;
+      for (int j = offset; j < nextOffset; ++j) {
+        value += weights[j] * in[indices[j]];
+      }
+      out[i] = value;
+    }
+  }
+}
+
+@kernel void MappedMult(const int entries,
+                        const int * restrict offsets,
+                        const int * restrict indices,
+                        const double * restrict weights,
+                        const int * restrict outIndices,
+                        const double * restrict in,
+                        double * restrict out) {
+
+  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < entries) {
+      const int offset = offsets[i];
+      const int nextOffset = offsets[i + 1];
+      double value = 0;
+      for (int j = offset; j < nextOffset; ++j) {
+        value += weights[j] * in[indices[j]];
+      }
+      out[outIndices[i]] = value;
+    }
+  }
+}

backends/occa/sparsemat.cpp

@@ -0,0 +1,248 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "sparsemat.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                                   const mfem::SparseMatrix &m,
+                                   const ::occa::properties &props) :
+   Operator(in_layout, out_layout)
+{
+
+   Setup(in_layout.OccaEngine().GetDevice(), m, props);
+}
+
+OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                                   const mfem::SparseMatrix &m,
+                                   ::occa::array<int> reorderIndices_,
+                                   ::occa::array<int> mappedIndices_,
+                                   const ::occa::properties &props) :
+   Operator(in_layout, out_layout)
+{
+
+   Setup(in_layout.OccaEngine().GetDevice(), m,
+         reorderIndices, mappedIndices_, props);
+}
+
+OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                                   ::occa::array<int> offsets_,
+                                   ::occa::array<int> indices_,
+                                   ::occa::array<double> weights_,
+                                   const ::occa::properties &props) :
+   Operator(in_layout, out_layout),
+   offsets(offsets_),
+   indices(indices_),
+   weights(weights_)
+{
+
+   SetupKernel(in_layout.OccaEngine().GetDevice(), props);
+}
+
+OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                                   ::occa::array<int> offsets_,
+                                   ::occa::array<int> indices_,
+                                   ::occa::array<double> weights_,
+                                   ::occa::array<int> reorderIndices_,
+                                   ::occa::array<int> mappedIndices_,
+                                   const ::occa::properties &props) :
+   Operator(in_layout, out_layout),
+   offsets(offsets_),
+   indices(indices_),
+   weights(weights_),
+   reorderIndices(reorderIndices_),
+   mappedIndices(mappedIndices_)
+{
+
+   SetupKernel(in_layout.OccaEngine().GetDevice(), props);
+}
+
+void OccaSparseMatrix::Setup(::occa::device device, const mfem::SparseMatrix &m,
+                             const ::occa::properties &props)
+{
+   Setup(device, m, ::occa::array<int>(), ::occa::array<int>(), props);
+}
+
+void OccaSparseMatrix::Setup(::occa::device device, const SparseMatrix &m,
+                             ::occa::array<int> reorderIndices_,
+                             ::occa::array<int> mappedIndices_,
+                             const ::occa::properties &props)
+{
+
+   const int nnz = m.GetI()[height];
+   offsets.allocate(device,
+                    height + 1, m.GetI());
+   indices.allocate(device,
+                    nnz, m.GetJ());
+   weights.allocate(device,
+                    nnz, m.GetData());
+
+   offsets.keepInDevice();
+   indices.keepInDevice();
+   weights.keepInDevice();
+
+   reorderIndices = reorderIndices_;
+   mappedIndices  = mappedIndices_;
+
+   SetupKernel(device, props);
+}
+
+void OccaSparseMatrix::SetupKernel(::occa::device device,
+                                   const ::occa::properties &props)
+{
+
+   const bool hasOutIndices = mappedIndices.isInitialized();
+
+   const ::occa::properties defaultProps("defines: {"
+                                         "  TILESIZE: 256,"
+                                         "}");
+
+   const std::string &okl_path = InLayout_().OccaEngine().GetOklPath();
+   const std::string &okl_defines = InLayout_().OccaEngine().GetOklDefines();
+   mapKernel = device.buildKernel(okl_path + "mappings.okl",
+                                  "MapSubVector",
+                                  defaultProps + props + okl_defines);
+
+   multKernel = device.buildKernel(okl_path + "sparse.okl",
+                                   hasOutIndices ? "MappedMult" : "Mult",
+                                   defaultProps + props + okl_defines);
+}
+
+void OccaSparseMatrix::Mult_(const Vector &x, Vector &y) const
+{
+   if (reorderIndices.isInitialized() ||
+       mappedIndices.isInitialized())
+   {
+      if (reorderIndices.isInitialized())
+      {
+         mapKernel((int) (reorderIndices.size() / 2),
+                   reorderIndices,
+                   x.OccaMem(), y.OccaMem());
+      }
+      if (mappedIndices.isInitialized())
+      {
+         multKernel((int) (mappedIndices.size()),
+                    offsets, indices, weights,
+                    mappedIndices,
+                    x.OccaMem(), y.OccaMem());
+      }
+   }
+   else
+   {
+      multKernel((int) height,
+                 offsets, indices, weights,
+                 x.OccaMem(), y.OccaMem());
+   }
+}
+
+
+OccaSparseMatrix* CreateMappedSparseMatrix(Layout &in_layout,
+                                           Layout &out_layout,
+                                           const mfem::SparseMatrix &m,
+                                           const ::occa::properties &props)
+{
+   const int mHeight = m.Height();
+   // const int mWidth  = m.Width();
+
+   // Count indices that are only reordered (true dofs)
+   const int *I = m.GetI();
+   const int *J = m.GetJ();
+   const double *D = m.GetData();
+
+   int trueCount = 0;
+   for (int i = 0; i < mHeight; ++i)
+   {
+      trueCount += ((I[i + 1] - I[i]) == 1);
+   }
+   const int dupCount = (mHeight - trueCount);
+
+   // Create the reordering map for entries that aren't modified (true dofs)
+   ::occa::device device(in_layout.OccaEngine().GetDevice());
+   ::occa::array<int> reorderIndices(device,
+                                     2 * trueCount);
+   ::occa::array<int> mappedIndices, offsets, indices;
+   ::occa::array<double> weights;
+
+   if (dupCount)
+   {
+      mappedIndices.allocate(device,
+                             dupCount);
+   }
+   int trueIdx = 0, dupIdx = 0;
+   for (int i = 0; i < mHeight; ++i)
+   {
+      const int i1 = I[i];
+      if ((I[i + 1] - i1) == 1)
+      {
+         reorderIndices[trueIdx++] = J[i1];
+         reorderIndices[trueIdx++] = i;
+      }
+      else
+      {
+         mappedIndices[dupIdx++] = i;
+      }
+   }
+   reorderIndices.keepInDevice();
+
+   if (dupCount)
+   {
+      mappedIndices.keepInDevice();
+
+      // Extract sparse matrix without reordered identity
+      const int dupNnz = I[mHeight] - trueCount;
+
+      offsets.allocate(device,
+                       dupCount + 1);
+      indices.allocate(device,
+                       dupNnz);
+      weights.allocate(device,
+                       dupNnz);
+
+      int nnz = 0;
+      offsets[0] = 0;
+      for (int i = 0; i < dupCount; ++i)
+      {
+         const int idx = mappedIndices[i];
+         const int offStart = I[idx];
+         const int offEnd   = I[idx + 1];
+         offsets[i + 1] = offsets[i] + (offEnd - offStart);
+         for (int j = offStart; j < offEnd; ++j)
+         {
+            indices[nnz] = J[j];
+            weights[nnz] = D[j];
+            ++nnz;
+         }
+      }
+
+      offsets.keepInDevice();
+      indices.keepInDevice();
+      weights.keepInDevice();
+   }
+
+   return new OccaSparseMatrix(in_layout, out_layout,
+                               offsets, indices, weights,
+                               reorderIndices, mappedIndices,
+                               props);
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/sparsemat.hpp

@@ -0,0 +1,95 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_SPARSE_MAT_HPP
+#define MFEM_BACKENDS_OCCA_SPARSE_MAT_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include <occa.hpp>
+#include "vector.hpp"
+#include "engine.hpp"
+#include "operator.hpp"
+#include "../../linalg/sparsemat.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+/// TODO: doxygen
+class OccaSparseMatrix : public Operator
+{
+public:
+   ::occa::array<int> offsets, indices;
+   ::occa::array<double> weights;
+   ::occa::array<int> reorderIndices, mappedIndices;
+   ::occa::kernel mapKernel, multKernel;
+
+   /// Construct an empty OccaSparseMatrix.
+   OccaSparseMatrix(const Operator &orig)
+      : Operator(orig) { }
+
+   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                    const mfem::SparseMatrix &m,
+                    const ::occa::properties &props = ::occa::properties());
+
+   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                    const mfem::SparseMatrix &m,
+                    ::occa::array<int> reorderIndices_,
+                    ::occa::array<int> mappedIndices_,
+                    const ::occa::properties &props = ::occa::properties());
+
+   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                    ::occa::array<int> offsets_,
+                    ::occa::array<int> indices_,
+                    ::occa::array<double> weights_,
+                    const ::occa::properties &props = ::occa::properties());
+
+   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                    ::occa::array<int> offsets_,
+                    ::occa::array<int> indices_,
+                    ::occa::array<double> weights_,
+                    ::occa::array<int> reorderIndices_,
+                    ::occa::array<int> mappedIndices_,
+                    const ::occa::properties &props = ::occa::properties());
+
+   void Setup(::occa::device device, const mfem::SparseMatrix &m,
+              const ::occa::properties &props);
+
+   void Setup(::occa::device device, const mfem::SparseMatrix &m,
+              ::occa::array<int> reorderIndices_,
+              ::occa::array<int> mappedIndices_,
+              const ::occa::properties &props);
+
+   void SetupKernel(::occa::device device,
+                    const ::occa::properties &props);
+
+   // override
+   virtual void Mult_(const Vector &x, Vector &y) const;
+};
+
+
+/// TODO: doxygen
+OccaSparseMatrix* CreateMappedSparseMatrix(
+   Layout &in_layout, Layout &out_layout,
+   const mfem::SparseMatrix &m,
+   const ::occa::properties &props = ::occa::properties());
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_SPARSE_MAT_HPP

backends/occa/url_handler.cpp

@@ -0,0 +1,81 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "url_handler.hpp"
+#include "../../general/error.hpp"
+#include <cstdlib>
+#include <sys/stat.h>
+
+namespace mfem
+{
+
+namespace occa
+{
+
+FileOpener::FileOpener(const std::string &prefix,
+                       const std::string &env_variable)
+   : pfx(prefix)
+{
+   const char *env_path = getenv(env_variable.c_str());
+   if (!env_path) { return; }
+   std::string path(env_path);
+   for (std::size_t start = 0, end; start < path.size(); start = end + 1)
+   {
+      end = path.find(':', start);
+      if (end == std::string::npos)
+      {
+         AddDir(path.substr(start, end));
+         break;
+      }
+      AddDir(path.substr(start, end - start));
+   }
+}
+
+bool FileOpener::AddDir(const std::string &dir)
+{
+   if (dir.size() == 0 || dir[0] != '/') { return false; }
+   struct stat dir_stat;
+   if (stat(dir.c_str(), &dir_stat)) { return false; }
+   if (!S_ISDIR(dir_stat.st_mode)) { return false; }
+   paths.push_back(dir + (*dir.rbegin() == '/' ? "" : "/"));
+   return true;
+}
+
+bool FileOpener::handles(const std::string &filename)
+{
+   return filename.size() >= pfx.size() &&
+          filename.compare(0, pfx.size(), pfx) == 0;
+}
+
+std::string FileOpener::expand(const std::string &filename)
+{
+   std::string sfx(filename.substr(pfx.size()));
+   for (std::size_t i = 0; i < paths.size(); i++)
+   {
+      std::string file = paths[i] + sfx;
+      struct stat file_stat;
+      if (stat(file.c_str(), &file_stat) == 0 && S_ISREG(file_stat.st_mode))
+      {
+         return file;
+      }
+   }
+   MFEM_ABORT("invalid url: " << filename);
+   return sfx;
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/url_handler.hpp

@@ -0,0 +1,47 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_URL_HANDLER_HPP
+#define MFEM_BACKENDS_OCCA_URL_HANDLER_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include <occa.hpp>
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class FileOpener : public ::occa::io::fileOpener
+{
+protected:
+   std::string pfx; // prefix, e.g. "mfem://"
+   std::vector<std::string> paths; // paths to search for prefix replacement
+
+public:
+   FileOpener(const std::string &prefix, const std::string &env_variable);
+
+   bool AddDir(const std::string &dir);
+
+   virtual bool handles(const std::string &filename);
+   virtual std::string expand(const std::string &filename);
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_URL_HANDLER_HPP

backends/occa/utils.okl

@@ -0,0 +1,22 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+typedef double* Local_t @dim(numDofs, numElements);
+
+@kernel void InitLocalVector(const int numElements,
+                             const int numDofs,
+                             Local_t restrict sol) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int d = 0; d < numDofs; ++d; @inner) {
+      sol(d, e) = 0;
+    }
+  }
+}

backends/occa/vector.cpp

@@ -0,0 +1,204 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "vector.hpp"
+#include "../../linalg/vector.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+PVector *Vector::DoVectorClone(bool copy_data, void **buffer,
+                               int buffer_type_id) const
+{
+   MFEM_ASSERT(buffer_type_id == ScalarId<double>::value, "");
+   Vector *new_vector = new Vector(OccaLayout());
+   if (copy_data)
+   {
+      new_vector->slice.copyFrom(slice);
+   }
+   if (buffer)
+   {
+      *buffer = new_vector->GetBuffer();
+   }
+   return new_vector;
+}
+
+void Vector::DoDotProduct(const PVector &x, void *result,
+                          int result_type_id) const
+{
+   // Can be called when Size() == 0, e.g. when an MPI-parallel vector has a
+   // local size of 0.
+
+   MFEM_ASSERT(result_type_id == ScalarId<double>::value, "");
+   double *res = (double *)result;
+   MFEM_ASSERT(dynamic_cast<const Vector *>(&x) != NULL, "invalid Vector type");
+   const Vector *xp = static_cast<const Vector *>(&x);
+   MFEM_ASSERT(this->Size() == xp->Size(), "");
+   *res = ::occa::linalg::dot<double, double, double>(this->slice, xp->slice);
+
+#ifdef MFEM_USE_MPI
+   double local_dot = *res;
+   if (IsParallel())
+   {
+      MPI_Allreduce(&local_dot, res, 1, MPI_DOUBLE, MPI_SUM,
+                    OccaLayout().OccaEngine().GetComm());
+   }
+#endif
+}
+
+void Vector::DoAxpby(const void *a, const PVector &x,
+                     const void *b, const PVector &y,
+                     int ab_type_id)
+{
+   const std::string &okl_defines = OccaLayout().OccaEngine().GetOklDefines();
+
+   //
+   // TODO: move all kernel builders to class mfem::occa::Backend
+   //
+   static ::occa::kernelBuilder axpby1_builder =
+      ::occa::linalg::customLinearMethod(
+         "mfem_occa_axpby1",
+         "v0[i] = c0 * v1[i];",
+         "defines: {"
+         "  CTYPE0: 'double',"
+         "  VTYPE0: 'double',"
+         "  VTYPE1: 'double',"
+         "  TILESIZE: '128',"
+         "}");
+
+   static ::occa::kernelBuilder axpby2_builder =
+      ::occa::linalg::customLinearMethod(
+         "mfem_occa_axpby2",
+         "v0[i] = c0 * v0[i] + c1 * v1[i];",
+         "defines: {"
+         "  CTYPE0: 'double',"
+         "  CTYPE1: 'double',"
+         "  VTYPE0: 'double',"
+         "  VTYPE1: 'double',"
+         "  TILESIZE: '128',"
+         "}");
+
+   static ::occa::kernelBuilder axpby3_builder =
+      ::occa::linalg::customLinearMethod(
+         "mfem_occa_axpby3",
+         "v0[i] = c0 * v1[i] + c1 * v2[i];",
+         "defines: {"
+         "  CTYPE0: 'double',"
+         "  CTYPE1: 'double',"
+         "  VTYPE0: 'double',"
+         "  VTYPE1: 'double',"
+         "  VTYPE2: 'double',"
+         "  TILESIZE: '128',"
+         "}");
+
+   // called only when Size() != 0
+
+   MFEM_ASSERT(ab_type_id == ScalarId<double>::value, "");
+   const double da = *static_cast<const double *>(a);
+   const double db = *static_cast<const double *>(b);
+   MFEM_ASSERT(da == 0.0 || dynamic_cast<const Vector *>(&x) != NULL,
+               "invalid Vector x");
+   MFEM_ASSERT(db == 0.0 || dynamic_cast<const Vector *>(&y) != NULL,
+               "invalid Vector y");
+   const Vector *xp = static_cast<const Vector *>(&x);
+   const Vector *yp = static_cast<const Vector *>(&y);
+
+   MFEM_ASSERT(da == 0.0 || this->Size() == xp->Size(), "");
+   MFEM_ASSERT(db == 0.0 || this->Size() == yp->Size(), "");
+
+   if (da == 0.0)
+   {
+      if (db == 0.0)
+      {
+         OccaFill(&da);
+      }
+      else
+      {
+         if (this->slice == yp->slice)
+         {
+            // *this *= db
+            ::occa::linalg::operator_mult_eq(slice, db);
+         }
+         else
+         {
+            // *this = db * y
+            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice(),
+                                                         okl_defines);
+            kernel((int)Size(), db, slice, yp->slice);
+         }
+      }
+   }
+   else
+   {
+      if (db == 0.0)
+      {
+         if (this->slice == xp->slice)
+         {
+            // *this *= da
+            ::occa::linalg::operator_mult_eq(slice, da);
+         }
+         else
+         {
+            // *this = da * x
+            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice(),
+                                                         okl_defines);
+            kernel((int)Size(), da, slice, xp->slice);
+         }
+      }
+      else
+      {
+         MFEM_ASSERT(xp->slice != yp->slice, "invalid input");
+         if (this->slice == xp->slice)
+         {
+            // *this = da * (*this) + db * y
+            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice(),
+                                                         okl_defines);
+            kernel((int)Size(), da, db, slice, yp->slice);
+         }
+         else if (this->slice == yp->slice)
+         {
+            // *this = da * x + db * (*this)
+            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice(),
+                                                         okl_defines);
+            kernel((int)Size(), db, da, slice, xp->slice);
+         }
+         else
+         {
+            // *this = da * x + db * y
+            ::occa::kernel kernel = axpby3_builder.build(slice.getDevice(),
+                                                         okl_defines);
+            kernel((int)Size(), da, db, slice, xp->slice, yp->slice);
+         }
+      }
+   }
+}
+
+mfem::Vector Vector::Wrap()
+{
+   return mfem::Vector(*this);
+}
+
+const mfem::Vector Vector::Wrap() const
+{
+   return mfem::Vector(*const_cast<Vector*>(this));
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/vector.hpp

@@ -0,0 +1,74 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_VECTOR_HPP
+#define MFEM_BACKENDS_OCCA_VECTOR_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include <occa.hpp>
+#include "../base/vector.hpp"
+#include "array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Vector : virtual public Array, public PVector
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // DLayout layout;
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   virtual PVector *DoVectorClone(bool copy_data, void **buffer,
+                                  int buffer_type_id) const;
+
+   virtual void DoDotProduct(const PVector &x, void *result,
+                             int result_type_id) const;
+
+   virtual void DoAxpby(const void *a, const PVector &x,
+                        const void *b, const PVector &y,
+                        int ab_type_id);
+
+   ///@}
+   // End: Virtual interface
+
+public:
+   Vector(Layout &lt)
+      : PArray(lt), Array(lt, sizeof(double)), PVector(lt)
+   { }
+
+   mfem::Vector Wrap();
+
+   const mfem::Vector Wrap() const;
+
+#if defined(MFEM_USE_MPI)
+   bool IsParallel() const { return (OccaLayout().OccaEngine().GetComm() != MPI_COMM_NULL); }
+#endif
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_VECTOR_HPP

backends/omp/adiffusioninteg.cpp

@@ -0,0 +1,675 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) &&               \
+   defined(MFEM_USE_OMP) &&                     \
+   defined(MFEM_USE_ACROTENSOR)
+
+#include "adiffusioninteg.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+PAIntegrator::PAIntegrator(Coefficient &q, FiniteElementSpace &f)
+{
+   Q = &q;
+   ofes = &f;
+   fes = ofes->GetFESpace();
+   onGPU = (ofes->OmpEngine().ExecTarget() == Device);
+   fe = fes->GetFE(0);
+   tfe = dynamic_cast<const TensorBasisElement*>(fe);
+   if (tfe)
+   {
+      tDofMap = tfe->GetDofMap();
+   }
+   else
+   {
+      tDofMap.SetSize(nDof);
+      for (int i = 0; i < nDof; ++i)
+      {
+         tDofMap[i] = i;
+      }
+   }
+
+   nElem  = fes->GetNE();
+   GeomType = fe->GetGeomType();
+   FEOrder = fe->GetOrder();
+   nDim    = fe->GetDim();
+   nDof   = fe->GetDof();
+
+   ElementTransformation *Trans = fes->GetElementTransformation(0);
+   int irorder = 2*fe->GetOrder() + Trans->OrderW();
+   ir = &IntRules.Get(GeomType, irorder);
+   nQuad = ir->GetNPoints();
+   hasTensorBasis = tfe ? true : false;
+
+   if (nDim > 3)
+   {
+      mfem_error("AcroIntegrator tensor computations don't support dim > 3.");
+   }
+}
+
+PAIntegrator::~PAIntegrator()
+{
+
+}
+
+AcroDiffusionIntegrator::AcroDiffusionIntegrator(Coefficient &q, FiniteElementSpace &f) :
+   PAIntegrator(q,f)
+{
+   if (onGPU)
+   {
+      //TE.SetExecutorType("OneOutPerThread");
+      TE.SetExecutorType("Cuda");
+      //TODO:  Set to an existing cuda context if one exists
+   }
+   else
+   {
+      TE.SetExecutorType("CPUInterpreted");
+   }
+
+   const IntegrationRule *ir1D = &IntRules.Get(Geometry::SEGMENT, ir->GetOrder());
+   nDof1D = FEOrder + 1;
+   nQuad1D = ir1D->GetNPoints();
+
+   if (hasTensorBasis)
+   {
+      H1_FECollection fec(FEOrder,1);
+      const FiniteElement *fe1D = fec.FiniteElementForGeometry(Geometry::SEGMENT);
+      mfem::Vector eval(nDof1D);
+      DenseMatrix deval(nDof1D,1);
+      B.Init(nQuad1D, nDof1D);
+      G.Init(nQuad1D, nDof1D);
+      std::vector<int> wdims(nDim, nQuad1D);
+      W.Init(wdims);
+
+      mfem::Vector w(nQuad1D);
+      for (int k = 0; k < nQuad1D; ++k)
+      {
+         const IntegrationPoint &ip = ir1D->IntPoint(k);
+         fe1D->CalcShape(ip, eval);
+         fe1D->CalcDShape(ip, deval);
+
+         B(k,0) = eval(0);
+         B(k,nDof1D-1) = eval(1);
+         G(k,0) = deval(0,0);
+         G(k,nDof1D-1) = deval(1,0);
+         for (int i = 1; i < nDof1D-1; ++i)
+         {
+            B(k,i) = eval(i+1);
+            G(k,i) = deval(i+1,0);
+         }
+         w(k) = ip.weight;
+      }
+
+      if (nDim == 1)
+      {
+         for (int k1 = 0; k1 < nQuad1D; ++k1)
+         {
+            W(k1) = w(k1);
+         }
+      }
+      else if (nDim == 2)
+      {
+         for (int k1 = 0; k1 < nQuad1D; ++k1)
+         {
+            for (int k2 = 0; k2 < nQuad1D; ++k2)
+            {
+               W(k1,k2) = w(k1)*w(k2);
+            }
+         }
+      }
+      else if (nDim == 3)
+      {
+         for (int k1 = 0; k1 < nQuad1D; ++k1)
+         {
+            for (int k2 = 0; k2 < nQuad1D; ++k2)
+            {
+               for (int k3 = 0; k3 < nQuad1D; ++k3)
+               {
+                  W(k1,k2,k3) = w(k1)*w(k2)*w(k3);
+               }
+            }
+         }
+      }
+   }
+   else
+   {
+      mfem::Vector eval(nDof);
+      DenseMatrix deval(nDof,nDim);
+      G.Init(nQuad, nDof,nDim);
+      W.Init(nQuad);
+      for (int k = 0; k < nQuad; ++k)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(k);
+         fe->CalcDShape(ip, deval);
+         for (int i = 0; i < nDof; ++i)
+         {
+            for (int d = 0; d < nDim; ++d)
+            {
+               G(k,i,d) = deval(i,d);
+            }
+         }
+         W(k) = ip.weight;
+      }
+   }
+
+   if (onGPU)
+   {
+      B.MapToGPU();
+      G.MapToGPU();
+      W.MapToGPU();
+   }
+
+   // Assemble in the constructor!
+   BatchedPartialAssemble();
+}
+
+
+AcroDiffusionIntegrator::~AcroDiffusionIntegrator()
+{
+   for (int i = 0; i < Btil.Size(); i++) delete Btil[i];
+}
+
+
+void AcroDiffusionIntegrator::ComputeBTilde()
+{
+   Btil.SetSize(nDim);
+   for (int d = 0; d < nDim; ++d)
+   {
+      Btil[d] = new acro::Tensor(nDim, nDim, nQuad1D, nDof1D, nDof1D);
+      for (int m = 0; m < nDim; ++m)
+      {
+         for (int n = 0; n < nDim; ++n)
+         {
+            acro::Tensor &BGM = (m == d) ? G : B;
+            acro::Tensor &BGN = (n == d) ? G : B;
+            for (int k = 0; k < nQuad1D; ++k)
+            {
+               for (int i = 0; i < nDof1D; ++i)
+               {
+                  for (int j = 0; j < nDof1D; ++j)
+                  {
+                     (*Btil[d])(m, n, k, i, j) = BGM(k,i)*BGN(k,j);
+                  }
+               }
+            }
+         }
+      }
+   }
+}
+
+
+void AcroDiffusionIntegrator::BatchedPartialAssemble()
+{
+   //Initilze the tensors
+   acro::Tensor J,Jinv,Jdet,C;
+   if (hasTensorBasis)
+   {
+      const IntegrationRule *ir1D = &IntRules.Get(Geometry::SEGMENT, ir->GetOrder());
+      IntegrationPoint ip;
+      if (nDim == 1)
+      {
+         D.Init(nElem, nDim, nDim, nQuad1D);
+         J.Init(nElem, nQuad1D, nDim, nDim);
+         Jinv.Init(nElem, nQuad1D, nDim, nDim);
+         Jdet.Init(nElem, nQuad1D);
+         C.Init(nElem, nQuad1D);
+
+         for (int e = 0; e < nElem; ++e)
+         {
+            ElementTransformation *Trans = fes->GetElementTransformation(e);
+            for (int k1 = 0; k1 < nQuad1D; ++k1)
+            {
+               ip.x = ir1D->IntPoint(k1).x;
+               ip.y = 0.0;
+               ip.z = 0.0;
+               Trans->SetIntPoint(&ip);
+               C(e,k1) = Q->Eval(*Trans, ip);
+               const DenseMatrix &JMat = Trans->Jacobian();
+               for (int m = 0; m < nDim; ++m)
+               {
+                  for (int n = 0; n < nDim; ++n)
+                  {
+                     J(e,k1,m,n) = JMat.Elem(m,n);
+                  }
+               }
+            }
+         }
+      }
+      else if (nDim == 2)
+      {
+         D.Init(nElem, nDim, nDim, nQuad1D, nQuad1D);
+         J.Init(nElem, nQuad1D, nQuad1D, nDim, nDim);
+         Jinv.Init(nElem, nQuad1D, nQuad1D, nDim, nDim);
+         Jdet.Init(nElem, nQuad1D, nQuad1D);
+         C.Init(nElem, nQuad1D, nQuad1D);
+
+         for (int e = 0; e < nElem; ++e)
+         {
+            ElementTransformation *Trans = fes->GetElementTransformation(e);
+            for (int k1 = 0; k1 < nQuad1D; ++k1)
+            {
+               for (int k2 = 0; k2 < nQuad1D; ++k2)
+               {
+                  ip.x = ir1D->IntPoint(k1).x;
+                  ip.y = ir1D->IntPoint(k2).y;
+                  ip.z = 0.0;
+                  Trans->SetIntPoint(&ip);
+                  C(e,k1,k2) = Q->Eval(*Trans, ip);
+                  const DenseMatrix &JMat = Trans->Jacobian();
+                  for (int m = 0; m < nDim; ++m)
+                  {
+                     for (int n = 0; n < nDim; ++n)
+                     {
+                        J(e,k1,k2,m,n) = JMat.Elem(m,n);
+                     }
+                  }
+               }
+            }
+         }
+      }
+      else if (nDim == 3)
+      {
+         D.Init(nElem, nDim, nDim, nQuad1D, nQuad1D, nQuad1D);
+         J.Init(nElem, nQuad1D, nQuad1D, nQuad1D, nDim, nDim);
+         Jinv.Init(nElem, nQuad1D, nQuad1D, nQuad1D, nDim, nDim);
+         Jdet.Init(nElem, nQuad1D, nQuad1D, nQuad1D);
+         C.Init(nElem, nQuad1D, nQuad1D, nQuad1D);
+
+         for (int e = 0; e < nElem; ++e)
+         {
+            ElementTransformation *Trans = fes->GetElementTransformation(e);
+            for (int k1 = 0; k1 < nQuad1D; ++k1)
+            {
+               for (int k2 = 0; k2 < nQuad1D; ++k2)
+               {
+                  for (int k3 = 0; k3 < nQuad1D; ++k3)
+                  {
+                     ip.x = ir1D->IntPoint(k1).x;
+                     ip.y = ir1D->IntPoint(k2).y;
+                     ip.z = ir1D->IntPoint(k3).z;
+                     Trans->SetIntPoint(&ip);
+                     C(e,k1,k2,k3) = Q->Eval(*Trans, ip);
+                     const DenseMatrix &JMat = Trans->Jacobian();
+                     for (int m = 0; m < nDim; ++m)
+                     {
+                        for (int n = 0; n < nDim; ++n)
+                        {
+                           J(e,k1,k2,k3,m,n) = JMat.Elem(m,n);
+                        }
+                     }
+                  }
+               }
+            }
+         }
+      }
+   }
+   else
+   {
+      D.Init(nElem, nDim, nDim, nQuad);
+      J.Init(nElem, nQuad, nDim, nDim);
+      Jinv.Init(nElem, nQuad, nDim, nDim);
+      Jdet.Init(nElem, nQuad);
+      C.Init(nElem, nQuad);
+
+      for (int e = 0; e < nElem; ++e)
+      {
+         ElementTransformation *Trans = fes->GetElementTransformation(e);
+         for (int k = 0; k < nQuad; ++k)
+         {
+            const IntegrationPoint &ip = ir->IntPoint(k);
+            Trans->SetIntPoint(&ip);
+            C(e,k) = Q->Eval(*Trans, ip);
+            const DenseMatrix &JMat = Trans->Jacobian();
+            for (int m = 0; m < nDim; ++m)
+            {
+               for (int n = 0; n < nDim; ++n)
+               {
+                  J(e,k,m,n) = JMat.Elem(m,n);
+               }
+            }
+         }
+      }
+   }
+
+   TE.BatchMatrixInvDet(Jinv, Jdet, J);
+
+   if (hasTensorBasis)
+   {
+      if (nDim == 1)
+      {
+         TE("D_e_m_n_k = W_k C_e_k Jdet_e_k Jinv_e_k_m_j Jinv_e_k_n_j",
+            D, W, C, Jdet, Jinv, Jinv);
+      }
+      else if (nDim == 2)
+      {
+         TE("D_e_m_n_k1_k2 = W_k1_k2 C_e_k1_k2 Jdet_e_k1_k2 Jinv_e_k1_k2_m_j Jinv_e_k1_k2_n_j",
+            D, W, C, Jdet, Jinv, Jinv);
+      }
+      else if (nDim == 3)
+      {
+         TE("D_e_m_n_k1_k2_k3 = W_k1_k2_k3 C_e_k1_k2_k3 Jdet_e_k1_k2_k3 Jinv_e_k1_k2_k3_n_j Jinv_e_k1_k2_k3_m_j",
+            D, W, C, Jdet, Jinv, Jinv);
+      }
+   }
+   else
+   {
+      TE("D_e_m_n_k = W_k C_e_k Jdet_e_k Jinv_e_k_m_j Jinv_e_k_n_j",
+         D, W, C, Jdet, Jinv, Jinv);
+   }
+}
+
+
+void AcroDiffusionIntegrator::BatchedAssembleElementMatrices(DenseTensor &elmats)
+{
+   if (hasTensorBasis && Btil.Size() == 0)
+   {
+      ComputeBTilde();
+   }
+
+   if (!D.IsInitialized())
+   {
+      BatchedPartialAssemble();
+   }
+
+   if (!S.IsInitialized())
+   {
+      if (hasTensorBasis)
+      {
+         if (nDim == 1)
+         {
+            S.Init(nElem, nDof1D, nDof1D);
+         }
+         else if (nDim == 2)
+         {
+            S.Init(nElem, nDof1D, nDof1D, nDof1D, nDof1D);
+         }
+         else if (nDim == 3)
+         {
+            S.Init(nElem, nDof1D, nDof1D, nDof1D, nDof1D, nDof1D, nDof1D);
+         }
+      }
+      else
+      {
+         S.Init(nElem, nDof, nDof);
+      }
+      if (onGPU) {S.SwitchToGPU();}
+   }
+
+
+   if (hasTensorBasis) {
+      if (nDim == 1) {
+         TE("S_e_i1_j1 = Btil_m_n_k1_i1_j1 D_e_m_n_k1",
+            S, *Btil[0], D);
+      }
+      else if (nDim == 2)
+      {
+         TE("S_e_i1_i2_j1_j2 = Btil1_m_n_k1_i1_j1 Btil2_m_n_k2_i2_j2 D_e_m_n_k1_k2",
+            S, *Btil[0], *Btil[1], D);
+      }
+      else if (nDim == 3)
+      {
+         TE("S_e_i1_i2_i3_j1_j2_j3 = Btil1_m_n_k1_i1_j1 Btil2_m_n_k2_i2_j2 Btil3_m_n_k3_i3_j3 D_e_m_n_k1_k2_k3",
+            S, *Btil[0], *Btil[1], *Btil[2], D);
+      }
+   }
+   else
+   {
+      TE("S_e_i_j = G_k_i_m G_k_i_n D_e_m_n_k",
+         S, G, G, D);
+   }
+
+   S.MoveFromGPU();
+   for (int e = 0; e < nElem; ++e)
+   {
+      for (int ei = 0; ei < nDof; ++ei)
+      {
+         for (int ej = 0; ej < nDof; ++ej)
+         {
+            elmats(tDofMap[ei], tDofMap[ej], e) = S[e*nDof*nDof + ei*nDof + ej];
+         }
+      }
+   }
+}
+
+
+void AcroDiffusionIntegrator::ComputeElementMatrices(Vector &elmats)
+{
+   if (hasTensorBasis && Btil.Size() == 0)
+   {
+      ComputeBTilde();
+   }
+
+   if (!D.IsInitialized())
+   {
+      BatchedPartialAssemble();
+   }
+
+   if (!S.IsInitialized())
+   {
+      if (hasTensorBasis)
+      {
+         if (nDim == 1)
+         {
+            S.Init(nElem, nDof1D, nDof1D);
+         }
+         else if (nDim == 2)
+         {
+            S.Init(nElem, nDof1D, nDof1D, nDof1D, nDof1D);
+         }
+         else if (nDim == 3)
+         {
+            S.Init(nElem, nDof1D, nDof1D, nDof1D, nDof1D, nDof1D, nDof1D);
+         }
+      }
+      else
+      {
+         S.Init(nElem, nDof, nDof);
+      }
+      if (onGPU) {S.SwitchToGPU();}
+   }
+
+
+   if (hasTensorBasis) {
+      if (nDim == 1) {
+         TE("S_e_i1_j1 += Btil_m_n_k1_i1_j1 D_e_m_n_k1",
+            S, *Btil[0], D);
+      }
+      else if (nDim == 2)
+      {
+         TE("S_e_i1_i2_j1_j2 += Btil1_m_n_k1_i1_j1 Btil2_m_n_k2_i2_j2 D_e_m_n_k1_k2",
+            S, *Btil[0], *Btil[1], D);
+      }
+      else if (nDim == 3)
+      {
+         TE("S_e_i1_i2_i3_j1_j2_j3 += Btil1_m_n_k1_i1_j1 Btil2_m_n_k2_i2_j2 Btil3_m_n_k3_i3_j3 D_e_m_n_k1_k2_k3",
+            S, *Btil[0], *Btil[1], *Btil[2], D);
+      }
+   }
+   else
+   {
+      TE("S_e_i_j += G_k_i_m G_k_i_n D_e_m_n_k",
+         S, G, G, D);
+   }
+
+   S.MoveFromGPU();
+
+   double *edata = elmats.GetData<double>();
+   for (int e = 0; e < nElem; ++e)
+   {
+      const int e_offset = e * nDof * nDof;
+      for (int ei = 0; ei < nDof; ++ei)
+      {
+         const int offset = e_offset + ei * tDofMap[ei] * nDof;
+         for (int ej = 0; ej < nDof; ++ej)
+         {
+            const int index = offset + tDofMap[ej];
+            edata[index] = S[e*nDof*nDof + ei*nDof + ej];
+         }
+      }
+   }
+}
+
+void AcroDiffusionIntegrator::ReassembleOperator()
+{
+   BatchedPartialAssemble();
+}
+
+void AcroDiffusionIntegrator::PAMult(const Vector &x, Vector &y)
+{
+   MFEM_ASSERT(hasTensorBasis,"AcroDiffusionIntegrator PAMult on simplices not supported");
+
+   if (!U.IsInitialized())
+   {
+      // NOTE: x and y are already sized for the fespace in the constructor
+      double *Xptr = const_cast<double*>(x.GetData<double>());
+      double *Yptr = y.GetData<double>();
+      if (nDim == 1) {
+         X.Init(nElem,nDof1D,Xptr,Xptr,onGPU);
+         Y.Init(nElem,nDof1D,Yptr,Yptr,onGPU);
+         U.Init(nDim, nElem, nQuad1D);
+         Z.Init(nDim, nElem, nQuad1D);
+         if (onGPU)
+         {
+            U.SwitchToGPU();
+            Z.SwitchToGPU();
+         }
+      }
+      else if (nDim == 2)
+      {
+         X.Init(nElem,nDof1D,nDof1D,Xptr,Xptr,onGPU);
+         Y.Init(nElem,nDof1D,nDof1D,Yptr,Yptr,onGPU);
+         U.Init(nDim, nElem, nQuad1D, nQuad1D);
+         Z.Init(nDim, nElem, nQuad1D, nQuad1D);
+         T1.Init(nElem,nDof1D,nQuad1D);
+         if (onGPU)
+         {
+            U.SwitchToGPU();
+            Z.SwitchToGPU();
+            T1.SwitchToGPU();
+         }
+      }
+      else if (nDim == 3)
+      {
+         X.Init(nElem,nDof1D,nDof1D,nDof1D,Xptr,Xptr,onGPU);
+         Y.Init(nElem,nDof1D,nDof1D,nDof1D,Yptr,Yptr,onGPU);
+         U.Init(nDim, nElem, nQuad1D, nQuad1D, nQuad1D);
+         Z.Init(nDim, nElem, nQuad1D, nQuad1D, nQuad1D);
+         T1.Init(nElem, nDof1D, nQuad1D, nQuad1D);
+         T2.Init(nElem, nDof1D, nDof1D, nQuad1D);
+         if (onGPU)
+         {
+            U.SwitchToGPU();
+            Z.SwitchToGPU();
+            T1.SwitchToGPU();
+            T2.SwitchToGPU();
+         }
+      }
+   }
+   else
+   {
+      // NOTE: x and y are already sized for the fespace in the constructor
+      double *Xptr = const_cast<double*>(x.GetData<double>());
+      double *Yptr = y.GetData<double>();
+      X.Retarget(Xptr,Xptr);
+      Y.Retarget(Yptr,Yptr);
+   }
+
+   acro::SliceTensor U1,U2,U3,Z1,Z2,Z3;
+   if (nDim == 1)
+   {
+      TE("U_n_e_k1 = G_k1_i1 X_e_i1", U, G, X);
+      TE("Z_m_e_k1 = D_e_m_n_k1 U_n_e_k1", Z, D, U);
+      TE("Y_e_i1 = G_k1_i1 Z_m_e_k1", Y, G, Z);
+   }
+   else if (nDim == 2)
+   {
+      U1.SliceInit(U, 0); U2.SliceInit(U, 1);
+      Z1.SliceInit(Z, 0); Z2.SliceInit(Z, 1);
+
+      //U1_e_k1_k2 = G_k1_i1 B_k2_i2 X_e_i1_i2
+      TE("BX_e_i1_k2 = B_k2_i2 X_e_i2_i1", T1, B, X);
+      TE("U1_e_k1_k2 = G_k1_i1 BX_e_i1_k2", U1, G, T1);
+
+      //U2_e_k1_k2 = B_k1_i1 G_k2_i2 X_e_i1_i2
+      TE("GX_e_i1_k2 = G_k2_i2 X_e_i2_i1", T1, G, X);
+      TE("U2_e_k1_k2 = B_k1_i1 GX_e_i1_k2", U2, B, T1);
+
+      TE("Z_m_e_k1_k2 = D_e_m_n_k1_k2 U_n_e_k1_k2", Z, D, U);
+
+      //Y_e_i1_i2 = G_k1_i1 B_k2_i2 Z1_e_k1_k2
+      TE("BZ1_e_i2_k1 = B_k2_i2 Z1_e_k1_k2", T1, B, Z1);
+      TE("Y_e_i2_i1 = G_k1_i1 BZ1_e_i2_k1", Y, G, T1);
+
+      //Y_e_i1_i2 += B_k1_i1 G_k2_i2 Z2_e_k1_k2
+      TE("GZ2_e_i2_k1 = G_k2_i2 Z2_e_k1_k2", T1, G, Z2);
+      TE("Y_e_i2_i1 += B_k1_i1 GZ2_e_i2_k1", Y, B, T1);
+   }
+   else if (nDim == 3)
+   {
+      U1.SliceInit(U, 0); U2.SliceInit(U, 1); U3.SliceInit(U, 2);
+      Z1.SliceInit(Z, 0); Z2.SliceInit(Z, 1); Z3.SliceInit(Z, 2);
+
+      TE.BeginMultiKernelLaunch();
+      //U1_e_k1_k2_k3 = G_k1_i1 B_k2_i2 B_k3_i3 X_e_i1_i2_i3
+      TE("T2_e_i1_i2_k3 = B_k3_i3 X_e_i1_i2_i3", T2, B, X);
+      TE("T1_e_i1_k2_k3 = B_k2_i2 T2_e_i1_i2_k3", T1, B, T2);
+      TE("U1_e_k1_k2_k3 = G_k1_i1 T1_e_i1_k2_k3", U1, G, T1);
+
+      //U2_e_k1_k2_k3 = B_k1_i1 G_k2_i2 B_k3_i3 X_e_i1_i2_i3
+      TE("T1_e_i1_k2_k3 = G_k2_i2 T2_e_i1_i2_k3", T1, G, T2);
+      TE("U2_e_k1_k2_k3 = B_k1_i1 T1_e_i1_k2_k3", U2, B, T1);
+
+      //U3_e_k1_k2_k3 = B_k1_i1 B_k2_i2 G_k3_i3 X_e_i1_i2_i3
+      TE("T2_e_i1_i2_k3 = G_k3_i3 X_e_i1_i2_i3", T2, G, X);
+      TE("T1_e_i1_k2_k3 = B_k2_i2 T2_e_i1_i2_k3", T1, B, T2);
+      TE("U3_e_k1_k2_k3 = B_k1_i1 T1_e_i1_k2_k3", U3, B, T1);
+
+      TE("Z_m_e_k1_k2_k3 = D_e_m_n_k1_k2_k3 U_n_e_k1_k2_k3", Z, D, U);
+
+      //Y_e_i1_i2_i3 =  G_k1_i1 B_k2_i2 B_k3_i3 Z1_e_k1_k2_k3
+      TE("T1_e_i3_k1_k2 = B_k3_i3 Z1_e_k1_k2_k3", T1, B, Z1);
+      TE("T2_e_i2_i3_k1 = B_k2_i2 T1_e_i3_k1_k2", T2, B, T1);
+      TE("Y_e_i1_i2_i3 = G_k1_i1 T2_e_i2_i3_k1", Y, G, T2);
+
+      //Y_e_i1_i2_i3 +=  B_k1_i1 G_k2_i2 B_k3_i3 Z2_e_k1_k2_k3
+      TE("T1_e_i3_k1_k2 = B_k3_i3 Z2_e_k1_k2_k3", T1, B, Z2);
+      TE("T2_e_i2_i3_k1 = G_k2_i2 T1_e_i3_k1_k2", T2, G, T1);
+      TE("Y_e_i1_i2_i3 += B_k1_i1 T2_e_i2_i3_k1", Y, B, T2);
+
+      //Y_e_i1_i2_i3 +=  B_k1_i1 B_k2_i2 G_k3_i3 Z3_e_k1_k2_k3
+      TE("T1_e_i3_k1_k2 = G_k3_i3 Z3_e_k1_k2_k3", T1, G, Z3);
+      TE("T2_e_i2_i3_k1 = B_k2_i2 T1_e_i3_k1_k2", T2, B, T1);
+      TE("Y_e_i1_i2_i3 += B_k1_i1 T2_e_i2_i3_k1", Y, B, T2);
+      TE.EndMultiKernelLaunch();
+   }
+}
+
+void AcroDiffusionIntegrator::MultAdd(const Vector &x, Vector &y) const
+{
+   const_cast<AcroDiffusionIntegrator*>(this)->PAMult(x, y);
+}
+
+void AcroDiffusionIntegrator::MultTransposeAdd(const Vector &x, Vector &y) const
+{
+   mfem_error("Not supported");
+}
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif

backends/omp/adiffusioninteg.hpp

@@ -0,0 +1,95 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_ADIFFUSIONINTEG_HPP
+#define MFEM_BACKENDS_OMP_ADIFFUSIONINTEG_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) &&               \
+   defined(MFEM_USE_OMP) &&                     \
+   defined(MFEM_USE_ACROTENSOR)
+
+#include "../../fem/bilininteg.hpp"
+#include "../../fem/fem.hpp"
+#include "vector.hpp"
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+#include "AcroTensor.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+class PAIntegrator : public TensorBilinearFormIntegrator
+{
+protected:
+   Coefficient *Q;
+   FiniteElementSpace *ofes;
+   mfem::FiniteElementSpace *fes;
+   const FiniteElement *fe;
+   const TensorBasisElement *tfe;
+   const IntegrationRule *ir;
+   mfem::Array<int> tDofMap;
+   int GeomType;
+   int FEOrder;
+   bool onGPU;
+   bool hasTensorBasis;
+   int nDim;
+   int nElem;
+   int nDof;
+   int nQuad;
+
+public:
+   PAIntegrator(Coefficient &q, FiniteElementSpace &f);
+   virtual ~PAIntegrator();
+};
+
+class AcroDiffusionIntegrator : public PAIntegrator
+{
+private:
+   acro::TensorEngine TE;
+   int nDof1D;
+   int nQuad1D;
+
+   acro::Tensor B, G;         //Basis and dbasis evaluated on the quad points
+   acro::Tensor W;            //Integration weights
+   mfem::Array<acro::Tensor*> Btil; //Btilde used to compute stiffness matrix
+   acro::Tensor D;            //Product of integration weight, physical consts, and element shape info
+   acro::Tensor S;            //The assembled local stiffness matrices
+   acro::Tensor U, Z, T1, T2; //Intermediate computations for tensor product partial assembly
+   acro::Tensor X, Y;
+
+   void ComputeBTilde();
+
+public:
+   AcroDiffusionIntegrator(BilinearFormIntegrator *integ);
+   AcroDiffusionIntegrator(Coefficient &q, FiniteElementSpace &f);
+   virtual ~AcroDiffusionIntegrator();
+
+   void BatchedPartialAssemble();
+   void BatchedAssembleElementMatrices(DenseTensor &elmats);
+   void ComputeElementMatrices(Vector &elmats);
+   void PAMult(const Vector &x, Vector &y);
+   virtual void MultTransposeAdd(const Vector &x, Vector &y) const;
+   virtual void MultAdd(const Vector &x, Vector &y) const;
+   virtual void ReassembleOperator();
+};
+
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif
+
+#endif

backends/omp/array.cpp

@@ -0,0 +1,128 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include <cstring>
+#include "array.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+PArray *Array::DoClone(bool copy_data, void **buffer,
+                       std::size_t item_size) const
+{
+   Array *new_array = new Array(OmpLayout(), item_size);
+   if (copy_data)
+   {
+      if (!ComputeOnDevice())
+         std::memcpy(new_array->GetData<void>(), data, bytes);
+      else
+      {
+         char *new_data = new_array->GetData<char>();
+         const bool use_target = ComputeOnDevice();
+         const bool use_parallel = Size() > 1000;
+#pragma omp target teams distribute parallel for       \
+   if (target: use_target) if (parallel: use_parallel) \
+   is_device_ptr(new_data)
+         for (std::size_t i = 0; i < bytes; i++) new_data[i] = data[i];
+      }
+   }
+   if (buffer)
+   {
+      *buffer = new_array->GetData<void>();
+   }
+   return new_array;
+}
+
+int Array::DoResize(PLayout &new_layout, void **buffer,
+                    std::size_t item_size)
+{
+   MFEM_ASSERT(dynamic_cast<Layout *>(&new_layout) != NULL,
+               "new_layout is not an OMP Layout");
+   Layout *lt = static_cast<Layout *>(&new_layout);
+   layout.Reset(lt); // Reset() checks if the pointer is the same
+   int err = ResizeData(lt, item_size);
+   if (!err && buffer)
+   {
+      *buffer = GetData<void>();
+   }
+   return err;
+}
+
+void *Array::DoPullData(void *buffer, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   if (!IsUnifiedMemory() && ComputeOnDevice() && (buffer != NULL))
+   {
+#pragma omp target update from(data)
+      std::memcpy(buffer, data, bytes);
+   }
+   else
+   {
+      buffer = data;
+   }
+
+   return buffer;
+}
+
+void Array::DoFill(const void *value_ptr, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   switch (item_size)
+   {
+   case sizeof(int):
+      OmpFill((const int *)value_ptr);
+      break;
+   case sizeof(double):
+      OmpFill((const double *)value_ptr);
+      break;
+   default:
+      MFEM_ABORT("item_size = " << item_size << " is not supported");
+   }
+}
+
+void Array::DoPushData(const void *src_buffer, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   std::memcpy(data, (char *) src_buffer, bytes);
+
+   if ((!IsUnifiedMemory() && ComputeOnDevice()) && (data != src_buffer))
+   {
+#pragma omp target update to(data)
+   }
+}
+
+void Array::DoAssign(const PArray &src, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   // Note: static_cast can not be used here since PArray is a virtual base
+   //       class.
+   const Array *source = dynamic_cast<const Array *>(&src);
+   MFEM_ASSERT(source != NULL, "invalid source Array type");
+   MFEM_ASSERT(Size() == source->Size(), "");
+   // All arrays from this engine are of the same type, so we can simply check *this and assume the same is used in src.
+   DoPushData(source->GetData<void>(), item_size);
+}
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/array.hpp

@@ -0,0 +1,143 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_ARRAY_HPP
+#define MFEM_BACKENDS_OMP_ARRAY_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "layout.hpp"
+#include "../base/array.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+class Array : public virtual mfem::PArray
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // DLayout layout;
+
+   bool own_data;
+   std::size_t bytes;
+   char *data;
+
+   //
+   // Virtual interface
+   //
+
+   virtual void *DoGetData() const { return (void *) data; }
+
+   virtual PArray *DoClone(bool copy_data, void **buffer,
+                           std::size_t item_size) const;
+
+   virtual int DoResize(PLayout &new_layout, void **buffer,
+                        std::size_t item_size);
+
+   virtual void *DoPullData(void *buffer, std::size_t item_size);
+
+   virtual void DoFill(const void *value_ptr, std::size_t item_size);
+
+   virtual void DoPushData(const void *src_buffer, std::size_t item_size);
+
+   virtual void DoAssign(const PArray &src, std::size_t item_size);
+
+   //
+   // Auxiliary methods
+   //
+
+   inline int ResizeData(const Layout *lt, std::size_t item_size);
+
+   inline bool IsUnifiedMemory() const { return OmpLayout().OmpEngine().UnifiedMemory(); }
+
+   template <typename T>
+   void OmpFill(const T *pval)
+   {
+      T *ptr = (T*) data;
+      T val = *pval;
+      const bool use_target = ComputeOnDevice();
+      const bool use_parallel = (use_target || layout->Size() > 1000);
+      const std::size_t size = layout->Size();
+#pragma omp target teams distribute parallel for        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel) map (to: ptr, val)
+      for (int i = 0; i < size; i++) ptr[i] = val;
+   }
+
+public:
+   Array(Layout &lt, std::size_t item_size)
+      : PArray(lt),
+        own_data(true),
+        bytes(lt.Size() * item_size),
+        data(static_cast<char *>(lt.Alloc(bytes)))
+   {
+#pragma omp target enter data map(alloc:data[:bytes]) if (!IsUnifiedMemory() && ComputeOnDevice())
+   }
+
+   Array(const Array &array)
+      : PArray(array.GetLayout()),
+        own_data(false),
+        bytes(array.bytes),
+        data(array.data) { }
+
+   inline bool ComputeOnDevice() const { return (OmpLayout().OmpEngine().ExecTarget() == Device); }
+
+   virtual ~Array()
+   {
+#pragma omp target exit data map(delete:data[:bytes]) if (!IsUnifiedMemory() && ComputeOnDevice())
+      if (own_data) layout->As<Layout>().Dealloc(data);
+   }
+
+   inline void MakeRef(Array &master);
+
+   Layout &OmpLayout() const
+   { return *static_cast<Layout *>(layout.Get()); }
+};
+
+
+//
+// Inline methods
+//
+
+inline int Array::ResizeData(const Layout *lt, std::size_t item_size)
+{
+   const std::size_t new_bytes = lt->Size() * item_size;
+   if (bytes < new_bytes)
+   {
+#pragma omp target exit data map(delete:data)
+      OmpLayout().Dealloc(data);
+      data = static_cast<char *>(OmpLayout().Alloc(new_bytes));
+      MFEM_VERIFY(data != NULL, "");
+      // If memory allocation fails - an exception is thrown.
+#pragma omp target enter data map(alloc:data[:new_bytes])
+   }
+   return 0;
+}
+
+inline void Array::MakeRef(Array &master)
+{
+   layout = master.layout;
+   data = master.data;
+}
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#endif // MFEM_BACKENDS_OMP_ARRAY_HPP

backends/omp/backend.cpp

@@ -0,0 +1,46 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "backend.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+bool Backend::Supports(const std::string &engine_spec) const
+{
+   return true;
+}
+
+mfem::Engine *Create(const std::string &engine_spec)
+{
+   return new Engine(engine_spec);
+}
+
+#ifdef MFEM_USE_MPI
+mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec)
+{
+   return new Engine(comm, engine_spec);
+}
+#endif
+
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/backend.hpp

@@ -0,0 +1,48 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_BACKEND_HPP
+#define MFEM_BACKENDS_OMP_BACKEND_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+// Only the Backend and Engine classes should be exposed through "backend.hpp"
+#include "../base/backend.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+class Backend : public mfem::Backend
+{
+public:
+   virtual ~Backend();
+
+   virtual bool Supports(const std::string &engine_spec) const;
+
+   virtual mfem::Engine *Create(const std::string &engine_spec);
+
+#ifdef MFEM_USE_MPI
+   virtual mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec);
+#endif
+};
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#endif // MFEM_BACKENDS_OMP_BACKEND_HPP

backends/omp/bilinearform.cpp

@@ -0,0 +1,399 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "backend.hpp"
+#include "bilinearform.hpp"
+#include "adiffusioninteg.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+BilinearForm::~BilinearForm()
+{
+   // Make sure all integrators free their data
+   for (int i = 0; i < tbfi.Size(); i++) delete tbfi[i];
+
+   delete element_matrices;
+}
+
+void BilinearForm::TransferIntegrators()
+{
+   mfem::Array<mfem::BilinearFormIntegrator*> &dbfi = *bform->GetDBFI();
+   for (int i = 0; i < dbfi.Size(); i++)
+   {
+      std::string integ_name(dbfi[i]->Name());
+      Coefficient *scal_coeff = dbfi[i]->GetScalarCoefficient();
+      // ConstantCoefficient *const_coeff =
+      //    dynamic_cast<ConstantCoefficient*>(scal_coeff);
+      // // TODO: other types of coefficients ...
+      // double val = const_coeff ? const_coeff->constant : 1.0;
+
+      if (integ_name == "(undefined)")
+      {
+         MFEM_ABORT("BilinearFormIntegrator does not define Name()");
+      }
+      else if (integ_name == "diffusion")
+      {
+         switch (OmpEngine().IntegType())
+         {
+         case Acrotensor:
+            tbfi.Append(new AcroDiffusionIntegrator(*scal_coeff, bform->FESpace()->Get_PFESpace()->As<FiniteElementSpace>()));
+            break;
+         default:
+            mfem_error("integrator is not supported for any MultType");
+            break;
+         }
+      }
+      else
+      {
+         MFEM_ABORT("BilinearFormIntegrator [Name() = " << integ_name
+                    << "] is not supported");
+      }
+   }
+}
+
+void BilinearForm::InitRHS(const mfem::Array<int> &ess_tdof_list,
+                           mfem::Vector &mfem_x, mfem::Vector &mfem_b,
+                           mfem::OperatorHandle &A,
+                           mfem::Vector &mfem_X, mfem::Vector &mfem_B,
+                           int copy_interior) const
+{
+   const mfem::Operator *P = GetProlongation();
+   const mfem::Operator *R = GetRestriction();
+
+   if (P)
+   {
+      // Variational restriction with P
+      mfem_B.Resize(P->InLayout());
+      P->MultTranspose(mfem_b, mfem_B);
+      mfem_X.Resize(R->OutLayout());
+      R->Mult(mfem_x, mfem_X);
+   }
+   else
+   {
+      // rap, X and B point to the same data as this, x and b
+      mfem_X.MakeRef(mfem_x);
+      mfem_B.MakeRef(mfem_b);
+   }
+
+   if (A.Type() != mfem::Operator::ANY_TYPE)
+   {
+      A.EliminateBC(mat_e, ess_tdof_list, mfem_X, mfem_B);
+   }
+
+   if (!copy_interior && ess_tdof_list.Size() > 0)
+   {
+      Vector &X = mfem_X.Get_PVector()->As<Vector>();
+      const Array &constraint_list = ess_tdof_list.Get_PArray()->As<Array>();
+
+      double *X_data = X.GetData<double>();
+      const int* constraint_data = constraint_list.GetData<int>();
+
+      Vector subvec(constraint_list.OmpLayout());
+      double *subvec_data = subvec.GetData<double>();
+
+      const std::size_t num_constraint = constraint_list.Size();
+      const bool use_target = constraint_list.ComputeOnDevice();
+      const bool use_parallel = (use_target || num_constraint > 1000);
+
+      // This operation is a general version of mfem::Vector::SetSubVectorComplement()
+      // {
+#pragma omp target teams distribute parallel for        \
+   map(to: subvec_data, constraint_data, X_data)        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel)
+      for (std::size_t i = 0; i < num_constraint; i++) subvec_data[i] = X_data[constraint_data[i]];
+
+      X.Fill(0.0);
+
+#pragma omp target teams distribute parallel for        \
+   map(to: X_data, constraint_data, subvec_data)        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel)
+      for (std::size_t i = 0; i < num_constraint; i++) X_data[constraint_data[i]] = subvec_data[i];
+      // }
+   }
+
+   if (A.Type() == mfem::Operator::ANY_TYPE)
+   {
+      ConstrainedOperator *A_constrained = static_cast<ConstrainedOperator*>(A.Ptr());
+      A_constrained->EliminateRHS(mfem_X, mfem_B);
+   }
+}
+
+
+bool BilinearForm::Assemble()
+{
+   if (!has_assembled)
+   {
+      TransferIntegrators();
+      has_assembled = true;
+   }
+
+   return true;
+}
+
+void BilinearForm::ComputeElementMatrices()
+{
+   // Only called if performing full assembly
+   const int nelements = trial_fes->GetFESpace()->GetNE();
+   const int trial_ndofs = trial_fes->GetFESpace()->GetFE(0)->GetDof() * trial_fes->GetFESpace()->GetVDim();
+   const int test_ndofs = test_fes->GetFESpace()->GetFE(0)->GetDof() * test_fes->GetFESpace()->GetVDim();
+   const std::size_t length = nelements * trial_ndofs * test_ndofs;
+
+   if (!element_matrices) element_matrices = new mfem::Vector(*(new Layout(OmpEngine(), length)));
+   else element_matrices->Push();
+
+   element_matrices->Fill(0.0);
+   Vector &elmats = element_matrices->Get_PVector()->As<Vector>();
+
+   tbfi[0]->ComputeElementMatrices(elmats);
+
+   if (tbfi.Size() > 1)
+   {
+      for (int k = 1; k < tbfi.Size(); k++)
+      {
+         tbfi[k]->ComputeElementMatrices(elmats);
+      }
+   }
+}
+
+void BilinearForm::FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
+                                    mfem::OperatorHandle &A)
+{
+   if (A.Type() == mfem::Operator::ANY_TYPE)
+   {
+      // FIXME: Support different test and trial spaces (MixedBilinearForm)
+      const mfem::Operator *P = GetProlongation();
+
+      mfem::Operator *rap = this;
+      if (P != NULL) rap = new mfem::RAPOperator(*P, *this, *P);
+
+      A.Reset(new ConstrainedOperator(rap, ess_tdof_list, (rap != this)));
+
+      return;
+   }
+   else
+   {
+      // ASSUMPTION: some sort of sparse matrix
+      // Compute the local matrices (stored in bform->element_matrices
+      ComputeElementMatrices();
+      bform->AllocateMatrix();
+      mfem::SparseMatrix &mat = bform->SpMat();
+
+      element_matrices->Pull();
+      double *data = element_matrices->GetData();
+
+      const bool skip_zeros = true;
+      mfem::Array<int> tr_vdofs, te_vdofs;
+      for (int i = 0; i < trial_fes->GetFESpace()->GetNE(); i++)
+      {
+         trial_fes->GetFESpace()->GetElementVDofs(i, tr_vdofs);
+         test_fes->GetFESpace()->GetElementVDofs(i, te_vdofs);
+         const mfem::DenseMatrix elmat(data, te_vdofs.Size(), tr_vdofs.Size());
+         mat.AddSubMatrix(te_vdofs, tr_vdofs, elmat, skip_zeros);
+         data += tr_vdofs.Size() * te_vdofs.Size();
+      }
+   }
+
+   if (A.Type() == mfem::Operator::MFEM_SPARSEMAT)
+   {
+      // This works because the FormSystemMatrix call with an explicit
+      // SparseMatrix doesnt call the backend version... This might
+      // change in the future.
+      bform->FormSystemMatrix(ess_tdof_list, static_cast<mfem::SparseMatrix&>(*A.Ptr()));
+   }
+#ifdef MFEM_USE_MPI
+   else if (A.Type() == mfem::Operator::Hypre_ParCSR)
+   {
+      mfem::SparseMatrix &mat = bform->SpMat();
+      mfem::ParBilinearForm *pbform = dynamic_cast<mfem::ParBilinearForm*>(bform);
+
+      const bool skip_zeros = false;
+      mat.Finalize(skip_zeros);
+
+      // -------- FOR SOME VERY AGGREVATING REASON THIS DOESN'T WORK ---------
+      // mfem::ParFiniteElementSpace *pfes = pbform->ParFESpace();
+      // OperatorHandle dA(Operator::Hypre_ParCSR);
+      // // construct a parallel block-diagonal matrix 'A' based on 'a'
+      // dA.MakeSquareBlockDiag(pfes->GetComm(), *engine->MakeLayout(pfes->GlobalTrueVSize()),
+      //                        pfes->GetDofOffsets(), &mat);
+      // OperatorHandle Ph(pfes->Dof_TrueDof_Matrix());
+      // A.MakePtAP(dA, Ph);
+      // A.SetOperatorOwner(false);
+      // -------- BUT THIS DOES ---------
+      pbform->ParallelAssemble(A, &mat);
+      A.SetOperatorOwner(false);
+      // ---------------------
+      mat.Clear();
+      mat_e.Clear();
+      std::cout << "operator size (FormSystemMatrix): " << A.Ptr()->InLayout()->Size() << " " << A.Ptr()->OutLayout()->Size() << std::endl;
+
+      mat_e.EliminateRowsCols(A, ess_tdof_list);
+   }
+#endif
+   else
+   {
+      MFEM_ABORT("Operator::Type is not supported, type = " << A.Type());
+   }
+}
+
+void BilinearForm::FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
+                                    mfem::Vector &x, mfem::Vector &b,
+                                    mfem::OperatorHandle &A, mfem::Vector &X, mfem::Vector &B,
+                                    int copy_interior)
+{
+   FormSystemMatrix(ess_tdof_list, A);
+   std::cout << "operator size (FormLinearSystem 1): " << A.Ptr()->InLayout()->Size() << " " << A.Ptr()->OutLayout()->Size() << std::endl;
+   InitRHS(ess_tdof_list, x, b, A, X, B, copy_interior);
+}
+
+void BilinearForm::RecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
+                                      mfem::Vector &x)
+{
+   const mfem::Operator *P = GetProlongation();
+   if (P)
+   {
+      // Apply conforming prolongation
+      x.Resize(P->OutLayout());
+      P->Mult(X, x);
+   }
+   // Otherwise X and x point to the same data
+}
+
+void BilinearForm::Mult(const mfem::Vector &x, mfem::Vector &y) const
+{
+   trial_fes->ToEVector(x.Get_PVector()->As<Vector>(), x_local);
+
+   y_local.Fill<double>(0.0);
+   for (int i = 0; i < tbfi.Size(); i++) tbfi[i]->MultAdd(x_local, y_local);
+
+   test_fes->ToLVector(y_local, y.Get_PVector()->As<Vector>());
+}
+
+void BilinearForm::MultTranspose(const mfem::Vector &x, mfem::Vector &y) const
+{ mfem_error("mfem::omp::BilinearForm::MultTranspose() is not supported!"); }
+
+
+ConstrainedOperator::ConstrainedOperator(mfem::Operator *A_,
+                                         const mfem::Array<int> &constraint_list_,
+                                         bool own_A_)
+   : Operator(A_->InLayout()->As<Layout>()),
+     A(A_),
+     own_A(own_A_),
+     // FIXME: @dudouit1 has a general fix for this
+     constraint_list(constraint_list_.Get_PArray()->As<Array>()),
+     z(OutLayout()->As<Layout>()),
+     w(OutLayout()->As<Layout>()),
+     mfem_z((z.DontDelete(), z)),
+     mfem_w((w.DontDelete(), w)) { }
+
+void ConstrainedOperator::EliminateRHS(const mfem::Vector &mfem_x, mfem::Vector &mfem_b) const
+{
+   w.Fill<double>(0.0);
+
+   const Vector &x = mfem_x.Get_PVector()->As<Vector>();
+   Vector &b = mfem_b.Get_PVector()->As<Vector>();
+
+   const double *x_data = x.GetData<double>();
+   double *b_data = b.GetData<double>();
+   double *w_data = w.GetData<double>();
+   const int* constraint_data = constraint_list.GetData<int>();
+
+   const std::size_t num_constraint = constraint_list.Size();
+   const bool use_target = constraint_list.ComputeOnDevice();
+   const bool use_parallel = (use_target || num_constraint > 1000);
+
+   if (num_constraint > 0)
+   {
+#pragma omp target teams distribute parallel for             \
+   map(to: w_data, constraint_data, x_data)                  \
+   if (target: use_target)                                   \
+   if (parallel: use_parallel)
+      for (std::size_t i = 0; i < num_constraint; i++)
+         w_data[constraint_data[i]] = x_data[constraint_data[i]];
+   }
+
+   A->Mult(mfem_w, mfem_z);
+
+   b.Axpby<double>(1.0, b, -1.0, z);
+
+   if (num_constraint > 0)
+   {
+#pragma omp target teams distribute parallel for        \
+   map(to: b_data, constraint_data, x_data)             \
+   if (target: use_target)                              \
+   if (parallel: use_parallel)
+      for (std::size_t i = 0; i < num_constraint; i++)
+         b_data[constraint_data[i]] = x_data[constraint_data[i]];
+   }
+}
+
+void ConstrainedOperator::Mult(const mfem::Vector &mfem_x, mfem::Vector &mfem_y) const
+{
+   if (constraint_list.Size() == 0)
+   {
+      A->Mult(mfem_x, mfem_y);
+      return;
+   }
+
+   const Vector &x = mfem_x.Get_PVector()->As<Vector>();
+   Vector &y = mfem_y.Get_PVector()->As<Vector>();
+
+   const double *x_data = x.GetData<double>();
+   double *y_data = y.GetData<double>();
+   double *z_data = z.GetData<double>();
+   const int* constraint_data = constraint_list.GetData<int>();
+
+   const std::size_t num_constraint = constraint_list.Size();
+   const bool use_target = constraint_list.ComputeOnDevice();
+   const bool use_parallel = (use_target || num_constraint > 1000);
+
+   z.Assign<double>(x); // z = x
+
+   // z[constraint_list] = 0.0
+#pragma omp target teams distribute parallel for             \
+   map(to: z_data, constraint_data)                          \
+   if (target: use_target)                                   \
+   if (parallel: use_parallel)
+   for (std::size_t i = 0; i < num_constraint; i++)
+      z_data[constraint_data[i]] = 0.0;
+
+   // y = A * z
+   A->Mult(mfem_z, mfem_y);
+
+   // y[constraint_list] = x[constraint_list]
+#pragma omp target teams distribute parallel for             \
+   map(to: y_data, constraint_data, x_data)                  \
+   if (target: use_target)                                   \
+   if (parallel: use_parallel)
+   for (std::size_t i = 0; i < num_constraint; i++)
+      y_data[constraint_data[i]] = x_data[constraint_data[i]];
+}
+
+// Destructor: destroys the unconstrained Operator @a A if @a own_A is true.
+ConstrainedOperator::~ConstrainedOperator()
+{
+   if (own_A) delete A;
+}
+
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/bilinearform.hpp

@@ -0,0 +1,176 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_BILINEARFORM_HPP
+#define MFEM_BACKENDS_OMP_BILINEARFORM_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "fespace.hpp"
+#include "array.hpp"
+#include "vector.hpp"
+#include "../../fem/bilininteg.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+class TensorBilinearFormIntegrator
+{
+public:
+   virtual ~TensorBilinearFormIntegrator() { }
+
+   virtual void ReassembleOperator() = 0;
+
+   virtual void ComputeElementMatrices(Vector &element_matrices)
+   { mfem_error("TensorBilinaerFormIntegrator::ComputeElementMatrices is not overloaded"); }
+
+   virtual void MultAdd(const Vector &x, Vector &y) const = 0;
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   { y.Fill<double>(0.0); MultAdd(x, y); }
+};
+
+/// TODO: doxygen
+class BilinearForm : public mfem::PBilinearForm, public mfem::Operator
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // mfem::BilinearForm *bform;
+
+   mfem::Array<TensorBilinearFormIntegrator*> tbfi;
+   bool has_assembled;
+
+   mutable FiniteElementSpace *trial_fes, *test_fes;
+
+   mutable Vector x_local, y_local;
+
+   mfem::Vector *element_matrices;
+   OperatorHandle mat_e;
+
+   void TransferIntegrators();
+
+   void ComputeElementMatrices();
+
+   void InitRHS(const mfem::Array<int> &constraint_list,
+                mfem::Vector &mfem_x, mfem::Vector &mfem_b,
+                mfem::OperatorHandle &A,
+                mfem::Vector &mfem_X, mfem::Vector &mfem_B,
+                int copy_interior = 0) const;
+
+public:
+   /// TODO: doxygen
+   BilinearForm(const Engine &e, mfem::BilinearForm &bf)
+      : mfem::PBilinearForm(e, bf),
+        // FIXME: for mixed bilinear forms
+        mfem::Operator(*bf.FESpace()->GetVLayout().As<Layout>()),
+        tbfi(),
+        has_assembled(false),
+        trial_fes(&bf.FESpace()->Get_PFESpace()->As<FiniteElementSpace>()),
+        test_fes(&bf.FESpace()->Get_PFESpace()->As<FiniteElementSpace>()),
+        x_local(trial_fes->GetELayout()),
+        y_local(test_fes->GetELayout()),
+        element_matrices(NULL),
+        mat_e() { }
+
+   /// Virtual destructor
+   virtual ~BilinearForm();
+
+   /// Return the engine as an OpenMP engine
+   const Engine &OmpEngine() { return static_cast<const Engine&>(*engine); }
+
+   /** @brief Prolongation operator from linear algebra (linear system) vectors,
+       to input vectors for the operator. `NULL` means identity. */
+   virtual const Operator *GetProlongation() const { return trial_fes->GetProlongation(); }
+
+   /** @brief Restriction operator from input vectors for the operator to linear
+       algebra (linear system) vectors. `NULL` means identity. */
+   virtual const Operator *GetRestriction() const  { return test_fes->GetRestriction(); }
+
+   /// Assemble the PBilinearForm.
+   /** This method is called from the method BilinearForm::Assemble() of the
+       associated BilinearForm #bform.
+       @returns True, if the host assembly should be skipped. */
+   virtual bool Assemble();
+
+   /// TODO: doxygen
+   virtual void FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
+                                 mfem::OperatorHandle &A);
+
+   /// TODO: doxygen
+   virtual void FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
+                                 mfem::Vector &x, mfem::Vector &b,
+                                 mfem::OperatorHandle &A, mfem::Vector &mfem_X, mfem::Vector &mfem_B,
+                                 int copy_interior);
+
+   /// TODO: doxygen
+   virtual void RecoverFEMSolution(const mfem::Vector &mfem_X, const mfem::Vector &mfem_b,
+                                   mfem::Vector &mfem_x);
+
+   /// Operator application: `y=A(x)`.
+   virtual void Mult(const mfem::Vector &mfem_x, mfem::Vector &mfem_y) const;
+
+   /** @brief Action of the transpose operator: `y=A^t(x)`. The default behavior
+       in class Operator is to generate an error. */
+   virtual void MultTranspose(const mfem::Vector &mfem_x, mfem::Vector &mfem_y) const;
+};
+
+class ConstrainedOperator : public mfem::Operator
+{
+   const mfem::Operator *A;
+   const bool own_A;
+   const Array constraint_list;
+   mutable Vector z, w;
+   mutable mfem::Vector mfem_z, mfem_w;
+
+public:
+   ConstrainedOperator(mfem::Operator *A_,
+                       const mfem::Array<int> &constraint_list_,
+                       bool own_A_ = false);
+
+   // Destructor: destroys the unconstrained Operator @a A if @a own_A is true.
+   virtual ~ConstrainedOperator();
+
+   /** @brief Eliminate "essential boundary condition" values specified in @a x
+       from the given right-hand side @a b.
+
+       Performs the following steps:
+
+       z = A((0,x_b));  b_i -= z_i;  b_b = x_b;
+
+       where the "_b" subscripts denote the essential (boundary) indices/dofs of
+       the vectors, and "_i" -- the rest of the entries. */
+   void EliminateRHS(const mfem::Vector &mfem_x, mfem::Vector &mfem_b) const;
+
+   /** @brief Constrained operator action.
+
+       Performs the following steps:
+
+       z = A((x_i,0));  y_i = z_i;  y_b = x_b;
+
+       where the "_b" subscripts denote the essential (boundary) indices/dofs of
+       the vectors, and "_i" -- the rest of the entries. */
+   virtual void Mult(const mfem::Vector &mfem_x, mfem::Vector &mfem_y) const;
+};
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#endif // MFEM_BACKENDS_OMP_BILINEAR_FORM_HPP

backends/omp/engine.cpp

@@ -0,0 +1,253 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "engine.hpp"
+#include "array.hpp"
+#include "layout.hpp"
+#include "vector.hpp"
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+#include "memory_resource.hpp"
+
+#include <map>
+
+namespace mfem
+{
+
+namespace omp
+{
+
+typedef std::map<std::string, std::string> keyval_pair_t;
+
+template<typename T, typename P>
+static T remove_if(T beg, T end, P pred)
+{
+   T dest = beg;
+   for (T itr = beg;itr != end; ++itr)
+      if (!pred(*itr))
+         *(dest++) = *itr;
+   return dest;
+}
+
+void parse_token(const std::string &token, std::string &key, std::string &val)
+{
+   std::size_t sep = token.find_first_of(':');
+   if (sep > token.size()) mfem_error("Parse error");
+
+   key = token.substr(0, sep);
+   key.erase(mfem::omp::remove_if(key.begin(), key.end(), isspace), key.end());
+   key.erase(std::remove(key.begin(), key.end(), '\''), key.end());
+
+   val = token.substr(sep+1);
+   val.erase(mfem::omp::remove_if(val.begin(), val.end(), isspace), val.end());
+   val.erase(std::remove(val.begin(), val.end(), '\''), val.end());
+}
+
+keyval_pair_t parse_engine_spec(const std::string &engine_spec)
+{
+   keyval_pair_t map;
+   std::size_t token_extent = 0;
+   std::string key, val;
+   while (token_extent < engine_spec.size())
+   {
+      const std::string remaining(engine_spec, token_extent);
+
+      std::size_t next_comma = remaining.find_first_of(',');
+      if (next_comma == std::string::npos) next_comma = engine_spec.size() - 1;
+
+      const std::string token(remaining, 0, next_comma);
+      parse_token(token, key, val);
+
+      map[key] = val;
+      token_extent += next_comma+1;
+   }
+   return map;
+}
+
+void Engine::Init(const std::string &engine_spec)
+{
+   keyval_pair_t tokens(parse_engine_spec(engine_spec));
+   keyval_pair_t::iterator it;
+
+   it = tokens.find("exec_target");
+   if (it != tokens.end())
+   {
+      if (!std::strncmp(it->second.data(), "device", 6))
+      {
+         exec_target = Device;
+         device_number = 0;
+      }
+      else if (!std::strncmp(it->second.data(), "host", 4))
+      {
+         exec_target = Host;
+         device_number = -1;
+      }
+      else
+      {
+         mfem_error("Parse error. Possible values for exec_target are: ['host', 'device']");
+      }
+   }
+   else
+   {
+      // Default to host if not specified
+      mfem::out << "Did not specify exec_target. Defaulting to host..." << std::endl;
+      exec_target = Host;
+      device_number = -1;
+   }
+
+   it = tokens.find("mem_type");
+   if (it != tokens.end())
+   {
+      if (!std::strncmp(it->second.data(), "unified", 7))
+      {
+#if defined(MFEM_USE_CUDAUM)
+         memory_resources[0] = new UnifiedMemoryResource();
+         unified_memory = true;
+#else
+         mfem_error("Have not compiled support for CUDA unified memory.");
+#endif
+      }
+      else if (!std::strncmp(it->second.data(), "separate", 4))
+      {
+         memory_resources[0] = new NewDeleteMemoryResource();
+         unified_memory = false;
+      }
+      else
+      {
+         mfem_error("Parse error. Possible values for mem_type are: ['separate', 'unified']");
+      }
+   }
+   else {
+      if (exec_target == Device)
+      {
+#if defined(MFEM_USE_CUDAUM)
+         mfem::out << "Did not specify mem_type in engine spec. Defaulting to unified memory..." << std::endl;
+         // Default to unified memory
+         memory_resources[0] = new UnifiedMemoryResource();
+         unified_memory = true;
+#else
+         mfem::out << "Did not specify mem_type in engine spec. Defaulting to standard host memory..." << std::endl;
+         memory_resources[0] = new NewDeleteMemoryResource();
+         unified_memory = false;
+#endif
+      }
+      else
+      {
+         mfem::out << "Did not specify mem_type in engine spec. Defaulting to standard host memory..." << std::endl;
+         memory_resources[0] = new NewDeleteMemoryResource();
+         unified_memory = false;
+      }
+   }
+
+   it = tokens.find("mult_engine");
+   if (it != tokens.end())
+   {
+      if (!std::strncmp(it->second.data(), "acrotensor", 10))
+      {
+         mult_type = Acrotensor;
+      }
+      else
+      {
+         mfem_error("Parse error. Possible values for mem_type are: ['acrotensor'].");
+      }
+   }
+   else
+   {
+      mfem::out << "Did not specify mult_engine in engine spec. Defaulting to Acrotensor..." << std::endl;
+#ifndef MFEM_USE_ACROTENSOR
+      mfem_error("Must compile with Acrotensor support");
+#endif
+      mult_type = Acrotensor;
+   }
+}
+
+Engine::Engine(const std::string &engine_spec)
+   : mfem::Engine(NULL, 1, 1)
+{
+   Init(engine_spec);
+}
+
+#ifdef MFEM_USE_MPI
+Engine::Engine(MPI_Comm _comm, const std::string &engine_spec)
+   : mfem::Engine(NULL, 1, 1)
+{
+   comm = _comm;
+   Init(engine_spec);
+}
+#endif
+
+DLayout Engine::MakeLayout(std::size_t size) const
+{
+   return DLayout(new Layout(*this, size));
+}
+
+DLayout Engine::MakeLayout(const mfem::Array<std::size_t> &offsets) const
+{
+   MFEM_ASSERT(offsets.Size() == 2,
+               "multiple workers are not supported yet");
+   return DLayout(new Layout(*this, offsets.Last()));
+}
+
+DArray Engine::MakeArray(PLayout &layout, std::size_t item_size) const
+{
+   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
+               "invalid input layout");
+   Layout *lt = static_cast<Layout *>(&layout);
+   return DArray(new Array(*lt, item_size));
+}
+
+DVector Engine::MakeVector(PLayout &layout, int type_id) const
+{
+   MFEM_ASSERT(type_id == ScalarId<double>::value, "invalid type_id");
+   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
+               "invalid input layout");
+   Layout *lt = static_cast<Layout *>(&layout);
+   return DVector(new Vector(*lt));
+}
+
+DFiniteElementSpace Engine::MakeFESpace(mfem::FiniteElementSpace &fespace) const
+{
+   return DFiniteElementSpace(new FiniteElementSpace(*this, fespace));
+}
+
+DBilinearForm Engine::MakeBilinearForm(mfem::BilinearForm &bf) const
+{
+   return DBilinearForm(new BilinearForm(*this, bf));
+}
+
+void Engine::AssembleLinearForm(LinearForm &l_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+}
+
+mfem::Operator *Engine::MakeOperator(const MixedBilinearForm &mbl_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+   return NULL;
+}
+
+mfem::Operator *Engine::MakeOperator(const NonlinearForm &nl_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+   return NULL;
+}
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/engine.hpp

@@ -0,0 +1,123 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_ENGINE_HPP
+#define MFEM_BACKENDS_OMP_ENGINE_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "../base/engine.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+enum ExecutionTarget { Host, Device };
+
+enum IntegratorType { Acrotensor };
+
+class Engine : public mfem::Engine
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // mfem::Backend *backend;
+#ifdef MFEM_USE_MPI
+   // MPI_Comm comm;
+#endif
+   // int num_mem_res;
+   // int num_workers;
+   // MemoryResource **memory_resources;
+   // double *workers_weights;
+   // int *workers_mem_res;
+
+   enum ExecutionTarget exec_target;
+   bool unified_memory;
+   int device_number;
+   IntegratorType mult_type;
+
+   void Init(const std::string &engine_spec);
+
+public:
+   Engine(const std::string &engine_spec);
+
+#ifdef MFEM_USE_MPI
+   Engine(MPI_Comm comm, const std::string &engine_spec);
+#endif
+
+   virtual ~Engine() { }
+
+   /**
+      @name OMP specific interface, used by other objects in the OMP backend
+   */
+   ///@{
+
+   IntegratorType IntegType() const { return mult_type; }
+
+   ExecutionTarget ExecTarget() const { return exec_target; }
+
+   inline bool UnifiedMemory() const { return unified_memory; }
+
+   void* Malloc(std::size_t bytes) const
+   {
+      return memory_resources[0]->Allocate(bytes, 16);
+   }
+
+   void Dealloc(void *ptr, std::size_t bytes = 0) const
+   {
+      memory_resources[0]->Deallocate(ptr, bytes);
+   }
+
+   ///@}
+   // End: OMP specific interface
+
+   /**
+      @name Virtual interface: finite element data structures and algorithms
+   */
+   ///@{
+
+   virtual DLayout MakeLayout(std::size_t size) const;
+   virtual DLayout MakeLayout(const mfem::Array<std::size_t> &offsets) const;
+
+   virtual DArray MakeArray(PLayout &layout, std::size_t item_size) const;
+
+   virtual DVector MakeVector(PLayout &layout,
+                              int type_id = ScalarId<double>::value) const;
+
+   virtual DFiniteElementSpace MakeFESpace(mfem::FiniteElementSpace &
+                                           fespace) const;
+
+   virtual DBilinearForm MakeBilinearForm(mfem::BilinearForm &bf) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual void AssembleLinearForm(LinearForm &l_form) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual mfem::Operator *MakeOperator(const MixedBilinearForm &mbl_form) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual mfem::Operator *MakeOperator(const NonlinearForm &nl_form) const;
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#endif // MFEM_BACKENDS_OMP_ENGINE_HPP

backends/omp/fespace.cpp

@@ -0,0 +1,237 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "fespace.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+FiniteElementSpace::FiniteElementSpace(const Engine &e,
+                                       mfem::FiniteElementSpace &fespace)
+   : PFiniteElementSpace(e, fespace),
+     e_layout(e, 0),
+     tensor_offsets(NULL),
+     tensor_indices(NULL),
+     prolongation(NULL),
+     restriction(NULL)
+{
+   std::size_t lsize = 0;
+   for (int e = 0; e < fespace.GetNE(); e++) { lsize += fespace.GetFE(e)->GetDof(); }
+   e_layout.Resize(lsize);
+   // The e_layout will be stored inside multiple shared DLayout objects
+   e_layout.DontDelete();
+}
+
+void FiniteElementSpace::BuildDofMaps()
+{
+   mfem::FiniteElementSpace *mfem_fes = GetFESpace();
+
+   const int local_size = GetELayout().Size();
+   const int global_size = mfem_fes->GetVLayout()->Size();
+   const int vdim = mfem_fes->GetVDim();
+
+   // Now we can allocate and fill the global map
+   tensor_offsets = new mfem::Array<int>(*(new Layout(OmpEngine(), global_size + 1)));
+   tensor_indices = new mfem::Array<int>(*(new Layout(OmpEngine(), local_size)));
+
+   mfem::Array<int> &offsets = *tensor_offsets;
+   mfem::Array<int> &indices = *tensor_indices;
+
+   mfem::Array<int> global_map(local_size);
+   mfem::Array<int> elem_vdof;
+
+   int offset = 0;
+   for (int e = 0; e < mfem_fes->GetNE(); e++)
+   {
+      const FiniteElement *fe = mfem_fes->GetFE(e);
+      const int dofs = fe->GetDof();
+      const TensorBasisElement *tfe = dynamic_cast<const TensorBasisElement *>(fe);
+      const mfem::Array<int> &dof_map = tfe->GetDofMap();
+
+      mfem_fes->GetElementVDofs(e, elem_vdof);
+
+      for (int vd = 0; vd < vdim; vd++)
+         for (int i = 0; i < dofs; i++)
+         {
+            global_map[offset + dofs*vd + i] = elem_vdof[dofs*vd + dof_map[i]];
+         }
+      offset += dofs * vdim;
+   }
+
+   // global_map[i] = index in global vector for local dof i
+   // NOTE: multiple i values will yield same global_map[i] for shared DOF.
+
+   // We want to now invert this map so we have indices[j] = (local dof for global dof j).
+
+   // Zero the offset vector
+   offsets = 0;
+
+   // Keep track of how many local dof point to its global dof
+   // Count how many times each dof gets hit
+   for (int i = 0; i < local_size; i++)
+   {
+      const int g = global_map[i];
+      ++offsets[g + 1];
+   }
+   // Aggregate the offsets
+   for (int i = 1; i <= global_size; i++)
+   {
+      offsets[i] += offsets[i - 1];
+   }
+
+   for (int i = 0; i < local_size; i++)
+   {
+      const int g = global_map[i];
+      indices[offsets[g]++] = i;
+   }
+
+   // Shift the offset vector back by one, since it was used as a
+   // counter above.
+   for (int i = global_size; i > 0; i--)
+   {
+      offsets[i] = offsets[i - 1];
+   }
+   offsets[0] = 0;
+
+   offsets.Push();
+   indices.Push();
+}
+
+/// Convert an E vector to L vector
+void FiniteElementSpace::ToLVector(const Vector &e_vector, Vector &l_vector)
+{
+   if (tensor_indices == NULL) BuildDofMaps();
+
+   if (l_vector.Size() != (std::size_t) GetFESpace()->GetVSize())
+   {
+      l_vector.Resize<double>(GetFESpace()->GetVLayout(), NULL);
+   }
+
+   const int lsize = l_vector.Size();
+   const int *offsets = tensor_offsets->Get_PArray()->As<Array>().GetData<int>();
+   const int *indices = tensor_indices->Get_PArray()->As<Array>().GetData<int>();
+
+   const double *e_data = e_vector.GetData<double>();
+   double *l_data = l_vector.GetData<double>();
+
+   const bool use_target = l_vector.ComputeOnDevice();
+   const bool use_parallel = (use_target || lsize > 1000);
+
+#pragma omp target teams distribute parallel for        \
+   map (to: offsets, indices, l_data, e_data)           \
+   if (target: use_target)                              \
+   if (parallel: use_parallel)
+   for (int i = 0; i < lsize; i++)
+   {
+      const int offset = offsets[i];
+      const int next_offset = offsets[i + 1];
+      double dof_value = 0;
+      for (int j = offset; j < next_offset; j++)
+      {
+         dof_value += e_data[indices[j]];
+      }
+      l_data[i] = dof_value;
+   }
+}
+
+/// Covert an L vector to E vector
+void FiniteElementSpace::ToEVector(const Vector &l_vector, Vector &e_vector)
+{
+   if (tensor_indices == NULL) BuildDofMaps();
+
+   if (e_vector.Size() != (std::size_t) e_layout.Size())
+   {
+      e_vector.Resize<double>(GetELayout(), NULL);
+   }
+
+   const int lsize = l_vector.Size();
+   const int *offsets = tensor_offsets->Get_PArray()->As<Array>().GetData<int>();
+   const int *indices = tensor_indices->Get_PArray()->As<Array>().GetData<int>();
+
+   const double *l_data = l_vector.GetData<double>();
+   double *e_data = e_vector.GetData<double>();
+
+   const bool use_target = l_vector.ComputeOnDevice();
+   const bool use_parallel = (use_target || lsize > 1000);
+
+#pragma omp target teams distribute parallel for         \
+   map (to: offsets, indices, l_data, e_data)            \
+   if (target: use_target)                               \
+   if (parallel: use_parallel)
+   for (int i = 0; i < lsize; i++)
+   {
+      const int offset = offsets[i];
+      const int next_offset = offsets[i + 1];
+      const double dof_value = l_data[i];
+      for (int j = offset; j < next_offset; j++)
+      {
+         e_data[indices[j]] = dof_value;
+      }
+   }
+}
+
+/// Get the finite element space prolongation matrix
+const Operator *FiniteElementSpace::GetProlongation() const
+{
+   // FIXME: This relies on unified memory if using a device other than the CPU
+   if (!prolongation)
+   {
+      Layout &v_layout = GetVLayout();
+      Layout &t_layout = GetTrueVLayout();
+
+      const mfem::Operator *op = GetFESpace()->GetProlongationMatrix();
+      if (!op)
+      {
+         prolongation = new mfem::IdentityOperator(t_layout);
+      }
+      else
+      {
+         prolongation = new BackendOperator(t_layout, v_layout, op);
+      }
+   }
+   return prolongation;
+}
+
+/// Get the finite element space restriction matrix
+const Operator *FiniteElementSpace::GetRestriction() const
+{
+   // FIXME: This relies on unified memory if using a device other than the CPU
+   if (!restriction)
+   {
+      Layout &v_layout = GetVLayout();
+      Layout &t_layout = GetTrueVLayout();
+
+      const mfem::Operator *op = GetFESpace()->GetRestrictionMatrix();
+      if (!op)
+      {
+         restriction = new mfem::IdentityOperator(t_layout);
+      }
+      else
+      {
+         restriction = new BackendOperator(v_layout, t_layout, op);
+      }
+   }
+   return restriction;
+}
+
+
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/fespace.hpp

@@ -0,0 +1,106 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_FESPACE_HPP
+#define MFEM_BACKENDS_OMP_FESPACE_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "engine.hpp"
+#include "array.hpp"
+#include "vector.hpp"
+#include "../../fem/fem.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+/*
+  Wraps an mfem::Operator that does not contain layout information.
+ */
+class BackendOperator : public mfem::Operator
+{
+   const mfem::Operator *op;
+
+public:
+   BackendOperator(Layout &in_layout, Layout &out_layout,
+                   const mfem::Operator *op_) : Operator(in_layout, out_layout), op(op_) { }
+
+   virtual void Mult(const mfem::Vector &x, mfem::Vector &y) const { op->Mult(x, y); }
+   virtual void MultTranspose(const mfem::Vector &x, mfem::Vector &y) const { op->MultTranspose(x, y); }
+};
+
+/// TODO: doxygen
+class FiniteElementSpace : public mfem::PFiniteElementSpace
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // mfem::FiniteElementSpace *fes;
+
+   Layout e_layout;
+
+   mfem::Array<int> *tensor_offsets, *tensor_indices;
+
+   mutable mfem::Operator *prolongation, *restriction;
+
+   void BuildDofMaps();
+
+public:
+   /// Nearly-empty class that stores a pointer to a mfem::FiniteElementSpace instance and the engine
+   FiniteElementSpace(const Engine &e, mfem::FiniteElementSpace &fespace);
+
+   /// Virtual destructor
+   virtual ~FiniteElementSpace()
+   {
+      delete tensor_offsets;
+      delete tensor_indices;
+      delete prolongation;
+      delete restriction;
+   }
+
+   Layout &GetELayout() { return e_layout; }
+
+   Layout &GetVLayout() const
+   { return *fes->GetVLayout().As<Layout>(); }
+
+   Layout &GetTrueVLayout() const
+   { return *fes->GetTrueVLayout().As<Layout>(); }
+
+   /// Return the engine as an OpenMP engine
+   const Engine &OmpEngine() { return static_cast<const Engine&>(*engine); }
+
+   /// Convert an E vector to L vector
+   void ToLVector(const Vector &e_vector, Vector &l_vector);
+
+   /// Covert an L vector to E vector
+   void ToEVector(const Vector &l_vector, Vector &e_vector);
+
+   /// Get the finite element space prolongation matrix
+   const Operator *GetProlongation() const;
+
+   /// Get the finite element space restriction matrix
+   const Operator *GetRestriction() const;
+
+};
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#endif // MFEM_BACKENDS_OMP_FESPACE_HPP

backends/omp/layout.cpp

@@ -0,0 +1,40 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "layout.hpp"
+#include "../../general/array.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+void Layout::Resize(std::size_t new_size)
+{
+   size = new_size;
+}
+
+void Layout::Resize(const Array<std::size_t> &offsets)
+{
+   MFEM_ASSERT(offsets.Size() == 2,
+               "multiple workers are not supported yet");
+   size = offsets.Last();
+}
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/layout.hpp

@@ -0,0 +1,71 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_LAYOUT_HPP
+#define MFEM_BACKENDS_OMP_LAYOUT_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "../base/layout.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+class Layout : public mfem::PLayout
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // std::size_t size;
+
+public:
+   Layout(const Engine &e, std::size_t s = 0) : PLayout(e, s) { }
+
+   const Engine &OmpEngine() const
+   { return *static_cast<const Engine *>(engine.Get()); }
+
+   void *Alloc(std::size_t bytes) const
+   { return OmpEngine().Malloc(bytes); }
+
+   void Dealloc(void *ptr) const
+   { return OmpEngine().Dealloc(ptr); }
+
+   virtual ~Layout() { }
+
+   /**
+      @name Virtual interface
+   */
+   ///@{
+
+   /// Resize the layout
+   virtual void Resize(std::size_t new_size);
+
+   /// Resize the layout based on the given worker offsets
+   virtual void Resize(const Array<std::size_t> &offsets);
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#endif // MFEM_BACKENDS_OMP_LAYOUT_HPP

backends/omp/memory_resource.cpp

@@ -0,0 +1,57 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "memory_resource.hpp"
+#include "../../general/error.hpp"
+
+#ifdef MFEM_USE_CUDAUM
+#include "cuda_runtime.h"
+#include "cuda.h"
+#endif
+
+namespace mfem
+{
+
+namespace omp
+{
+
+#ifdef MFEM_USE_CUDAUM
+void *UnifiedMemoryResource::DoAllocate(std::size_t bytes,
+                                        std::size_t alignment)
+{
+   void *p = NULL;
+   if (bytes > 0)
+   {
+      cudaError_t ret = cudaMallocManaged(&p, bytes);
+      MFEM_VERIFY(ret == cudaSuccess, "");
+   }
+   return p;
+}
+
+void UnifiedMemoryResource::DoDeallocate(void *p, std::size_t bytes,
+                                         std::size_t alignment)
+{
+   if (p != NULL)
+   {
+      cudaFree(p);
+   }
+}
+#endif
+
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/memory_resource.hpp

@@ -0,0 +1,44 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_MEMORY_RESOURCE_HPP
+#define MFEM_BACKENDS_OMP_MEMORY_RESOURCE_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "../../backends/base/memory_resource.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+/// Polymorphic memory resource. Similar to C++17's std::pmr::memory_resource.
+
+#ifdef MFEM_USE_CUDAUM
+/** @brief Memory resource using unified memory. */
+class UnifiedMemoryResource : public MemoryResource
+{
+protected:
+   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment);
+   virtual void DoDeallocate(void *p, std::size_t bytes, std::size_t alignment);
+};
+#endif
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#endif // MFEM_BACKENDS_OMP_MEMORY_RESOURCE_HPP

backends/omp/vector.cpp

@@ -0,0 +1,205 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "vector.hpp"
+#include "../../linalg/vector.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+PVector *Vector::DoVectorClone(bool copy_data, void **buffer,
+                               int buffer_type_id) const
+{
+   MFEM_ASSERT(buffer_type_id == ScalarId<double>::value, "");
+   Vector *new_vector = new Vector(OmpLayout());
+   if (copy_data)
+   {
+      const std::size_t total_size = sizeof(double) * OmpLayout().Size();
+      if (!ComputeOnDevice())
+         std::memcpy(new_vector->GetData<void>(), data, total_size);
+      else
+      {
+         char *new_data = new_vector->GetData<char>();
+#pragma omp target teams distribute parallel for is_device_ptr(new_data)
+         for (std::size_t i = 0; i < total_size; i++) new_data[i] = data[i];
+      }
+   }
+   if (buffer)
+   {
+      *buffer = new_vector->GetData<void>();
+   }
+   return new_vector;
+}
+
+void Vector::DoDotProduct(const PVector &x, void *result,
+                          int result_type_id) const
+{
+   // Can be called when Size() == 0, e.g. when an MPI-parallel vector has a
+   // local size of 0.
+
+   MFEM_ASSERT(result_type_id == ScalarId<double>::value, "");
+   double *res = (double *)result;
+   double local_dot = 0.;
+   MFEM_ASSERT(dynamic_cast<const Vector *>(&x) != NULL, "invalid Vector type");
+   const Vector *xp = static_cast<const Vector *>(&x);
+   MFEM_ASSERT(this->Size() == xp->Size(), "");
+
+   const double *ptr = GetData<double>();
+   const double *xptr = xp->GetData<double>();
+   const std::size_t size = Size();
+
+   if (!ComputeOnDevice())
+   {
+      for (std::size_t i = 0; i < size; i++) local_dot += ptr[i] * xptr[i];
+   }
+   else
+   {
+#pragma omp target teams distribute parallel for map(to: ptr, xptr) reduction(+:local_dot)
+      for (std::size_t i = 0; i < size; i++) local_dot += ptr[i] * xptr[i];
+   }
+
+   *res = local_dot;
+#ifdef MFEM_USE_MPI
+   MPI_Comm comm = OmpLayout().OmpEngine().GetComm();
+   if (comm != MPI_COMM_NULL)
+   {
+      MPI_Allreduce(&local_dot, res, 1, MPI_DOUBLE, MPI_SUM, comm);
+   }
+#endif
+}
+
+void Vector::DoAxpby(const void *a, const PVector &x,
+                     const void *b, const PVector &y,
+                     int ab_type_id)
+{
+   // called only when Size() != 0
+   MFEM_ASSERT(ab_type_id == ScalarId<double>::value, "");
+
+   const double da = *static_cast<const double *>(a);
+   const double db = *static_cast<const double *>(b);
+   MFEM_ASSERT(da == 0.0 || dynamic_cast<const Vector *>(&x) != NULL,
+               "invalid Vector x");
+   MFEM_ASSERT(db == 0.0 || dynamic_cast<const Vector *>(&y) != NULL,
+               "invalid Vector y");
+   const Vector *xp = static_cast<const Vector *>(&x);
+   const Vector *yp = static_cast<const Vector *>(&y);
+
+   MFEM_ASSERT(da == 0.0 || this->Size() == xp->Size(), "");
+   MFEM_ASSERT(db == 0.0 || this->Size() == yp->Size(), "");
+
+   const std::size_t size = Size();
+   const std::size_t critical_size = 1000;
+   const double *xd = xp->GetData<double>();
+   const double *yd = yp->GetData<double>();
+   double *td = GetData<double>();
+
+   const bool use_target = ComputeOnDevice();
+   const bool use_parallel = (use_target || size > critical_size);
+
+   if (da == 0.0)
+   {
+      if (db == 0.0)
+      {
+         OmpFill(&da);
+      }
+      else
+      {
+         if (td == yd)
+         {
+            // *this *= db
+#pragma omp target teams distribute parallel for        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel) map (to: db)
+            for (std::size_t i = 0; i < size; i++) td[i] *= db;
+         }
+         else
+         {
+            // *this = db * y
+#pragma omp target teams distribute parallel for        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel) map (to: yd, db)
+            for (std::size_t i = 0; i < size; i++) td[i] = yd[i] * db;
+         }
+      }
+   }
+   else
+   {
+      if (db == 0.0)
+      {
+         if (td == xd)
+         {
+            // *this *= da
+#pragma omp target teams distribute parallel for        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel) map (to: da)
+            for (std::size_t i = 0; i < size; i++) td[i] *= da;
+         }
+         else
+         {
+            // *this = da * x
+#pragma omp target teams distribute parallel for        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel) map (to: xd, da)
+            for (std::size_t i = 0; i < size; i++) td[i] = xd[i] * da;
+         }
+      }
+      else
+      {
+         MFEM_ASSERT(xd != yd, "invalid input");
+         if (td == xd)
+         {
+            // *this = da * (*this) + db * y
+#pragma omp target teams distribute parallel for        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel) map (to: da, td, db, yd)
+            for (std::size_t i = 0; i < size; i++) td[i] = da * td[i] + db * yd[i];
+         }
+         else if (td == yd)
+         {
+            // *this = da * x + db * (*this)
+#pragma omp target teams distribute parallel for        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel) map (to: da, xd, db, td)
+            for (std::size_t i = 0; i < size; i++) td[i] = da * xd[i] + db * td[i];
+         }
+         else
+         {
+            // *this = da * x + db * y
+#pragma omp target teams distribute parallel for        \
+   if (target: use_target)                              \
+   if (parallel: use_parallel) map (to: da, xd, db, yd)
+            for (std::size_t i = 0; i < size; i++) td[i] = da * xd[i] + db * yd[i];
+         }
+      }
+   }
+}
+
+mfem::Vector Vector::Wrap()
+{
+   return mfem::Vector(*this);
+}
+
+const mfem::Vector Vector::Wrap() const
+{
+   return mfem::Vector(*const_cast<Vector*>(this));
+}
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/vector.hpp

@@ -0,0 +1,71 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OMP_VECTOR_HPP
+#define MFEM_BACKENDS_OMP_VECTOR_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#include "../base/vector.hpp"
+#include "array.hpp"
+
+namespace mfem
+{
+
+namespace omp
+{
+
+class Vector : virtual public Array, public mfem::PVector
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // DLayout layout;
+   // char *data;
+   // std::size_t size;
+
+   /**
+      @name Virtual interface
+   */
+   ///@{
+
+   virtual PVector *DoVectorClone(bool copy_data, void **buffer,
+                                  int buffer_type_id) const;
+
+   virtual void DoDotProduct(const PVector &x, void *result,
+                             int result_type_id) const;
+
+   virtual void DoAxpby(const void *a, const PVector &x,
+                        const void *b, const PVector &y,
+                        int ab_type_id);
+
+   ///@}
+   // End: Virtual interface
+
+public:
+   Vector(Layout &lt)
+      : PArray(lt), Array(lt, sizeof(double)), PVector(lt)
+   { }
+
+   mfem::Vector Wrap();
+
+   const mfem::Vector Wrap() const;
+};
+
+} // namespace mfem::omp
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
+
+#endif // MFEM_BACKENDS_OMP_VECTOR_HPP

config/config.hpp

@@ -37,3 +37,11 @@
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
 #endif // MFEM_USE_MPI not defined
+
+// Macro that returns its first arg when MFEM_USE_BACKENDS is defined, and its
+// second arg if it is not defined.
+#ifdef MFEM_USE_BACKENDS
+#define MFEM_IF_BACKENDS(x,y) (x)
+#else
+#define MFEM_IF_BACKENDS(x,y) (y)
+#endif

config/config.hpp.in

@@ -33,6 +33,12 @@
 // Description of the git commit used to build MFEM.
 // #define MFEM_GIT_STRING "@MFEM_GIT_STRING@"
 
+// The absolute path of the MFEM source prefix
+// #define MFEM_SOURCE_DIR "@MFEM_SOURCE_DIR@"
+
+// The absolute path of the MFEM installation prefix
+// #define MFEM_INSTALL_DIR "@MFEM_INSTALL_DIR@"
+
 // Build the parallel MFEM library.
 // Requires an MPI compiler, and the libraries HYPRE and METIS.
 // #define MFEM_USE_MPI
@@ -109,6 +115,21 @@
 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR
 
+// Enable the use of MFEM backends.
+// #define MFEM_USE_BACKENDS
+
+// Enable the OCCA backend.
+// #define MFEM_USE_OCCA
+
+// Enable the OMP backend.
+// #define MFEM_USE_OMP
+
+// Enable use of acrotensor in backends.
+// #define MFEM_USE_ACROTENSOR
+
+// Enable use of unified memory.
+// #define MFEM_USE_CUDAUM
+
 // Windows specific options
 #ifdef _WIN32
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)

config/config.mk.in

@@ -13,6 +13,8 @@
 MFEM_VERSION         = @MFEM_VERSION@
 MFEM_VERSION_STRING  = @MFEM_VERSION_STRING@
 MFEM_GIT_STRING      = @MFEM_GIT_STRING@
+MFEM_SOURCE_DIR      = @MFEM_SOURCE_DIR@
+MFEM_INSTALL_DIR     = @MFEM_INSTALL_DIR@
 MFEM_USE_MPI         = @MFEM_USE_MPI@
 MFEM_USE_METIS       = @MFEM_USE_METIS@
 MFEM_USE_METIS_5     = @MFEM_USE_METIS_5@
@@ -37,6 +39,11 @@ MFEM_USE_PETSC       = @MFEM_USE_PETSC@
 MFEM_USE_MPFR        = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE       = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT     = @MFEM_USE_CONDUIT@
+MFEM_USE_BACKENDS    = @MFEM_USE_BACKENDS@
+MFEM_USE_OCCA        = @MFEM_USE_OCCA@
+MFEM_USE_OMP         = @MFEM_USE_OMP@
+MFEM_USE_ACROTENSOR  = @MFEM_USE_ACROTENSOR@
+MFEM_USE_CUDAUM      = @MFEM_USE_CUDAUM@
 
 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@

config/defaults.mk

@@ -83,6 +83,9 @@ MFEM_MPI_NP = 4
 # in config.mk and config.hpp.
 
 MFEM_USE_MPI         = NO
+# FIXME: add MFEM_USE_BACKENDS, MFEM_USE_OCCA to the CMake build system
+MFEM_USE_BACKENDS    = YES
+MFEM_USE_OCCA        = YES
 MFEM_USE_METIS       = $(MFEM_USE_MPI)
 MFEM_USE_METIS_5     = NO
 MFEM_DEBUG           = NO
@@ -106,6 +109,10 @@ MFEM_USE_PETSC       = NO
 MFEM_USE_MPFR        = NO
 MFEM_USE_SIDRE       = NO
 MFEM_USE_CONDUIT     = NO
+# FIXME: add MFEM_USE_OMP and MFEM_USE_ACROTENSOR to the CMake build system
+MFEM_USE_OMP         = NO
+MFEM_USE_ACROTENSOR  = NO
+MFEM_USE_CUDAUM      = NO
 
 # Compile and link options for zlib.
 ZLIB_DIR =
@@ -271,6 +278,31 @@ SIDRE_LIB = \
    -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
    -lsidre -lslic -laxom_utils -lconduit -lconduit_relay -lhdf5 $(ZLIB_LIB) -ldl
 
+OCCA_DIR = @MFEM_DIR@/../occa
+OCCA_OPT = -I$(OCCA_DIR)/include
+OCCA_LIB = -Wl,-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca
+
+CUDA_DIR = /usr/local/cuda
+CUDAUM_LIB = -L$(CUDA_DIR)/lib64 -lcudart
+CUDAUM_OPT = -I$(CUDA_DIR)/include
+
+OMP_OPT = -qsmp=omp -qoffload
+
+ACROTENSOR_DIR = @MFEM_DIR@/../acrotensor
+ACROTENSOR_OPT = -std=c++11 -I$(ACROTENSOR_DIR)/inc
+ACROTENSOR_LIB = -Wl,-rpath,$(ACROTENSOR_DIR)/lib/shared -L$(ACROTENSOR_DIR)/lib/shared -lacrotensor
+# If Acrotensor was compile with CUDA support, but MFEM_USE_CUDAUM==NO, then uncomment the lines below
+# ACROTENSOR_OPT += -I$(CUDA_DIR)/include
+# ACROTENSOR_LIB += -L$(CUDA_DIR)/lib64 -lcuda -lcudart -lnvrtc
+ifeq ($(MFEM_USE_CUDAUM),YES)
+ifeq ($(MFEM_USE_MPI),YES)
+# HYPRE needs some extra libraries in parallel on the GPU
+# FIXME: We need another solution for compilers other than XL for the
+#        dlink CUDA step, but fixes need to happen elsewhere as well.
+HYPRE_LIB += -qcuda -lcublas -lcusparse -lnvToolsExt
+endif
+endif
+
 # If YES, enable some informational messages
 VERBOSE = NO
 

doc/CodeDocumentation.conf.in

@@ -760,6 +760,7 @@ WARN_LOGFILE           =
 
 INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/mfem.hpp \
+                         @MFEM_SOURCE_DIR@/backends/base \
                          @MFEM_SOURCE_DIR@/config \
                          @MFEM_SOURCE_DIR@/general \
                          @MFEM_SOURCE_DIR@/linalg \

examples/ex1d.cpp

@@ -0,0 +1,182 @@
+
+#include <mfem.hpp>
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *spec = "cpu";
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+   const char *engine_type = "omp";
+   const char *engine_spec = "mult_engine:'acrotensor', exec_target:'device', mem_type:'unified'";
+   int ref_levels = -1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&spec, "-s", "--spec",
+                  "Compute resurce specification.");
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&engine_type, "-et", "--engine-type", "Engine type");
+   args.AddOption(&engine_spec, "-es", "--engine-spec", "Engine specification");
+   args.AddOption(&ref_levels, "-r", "--refs", "Number of uniform refinements (negative implies dof ~ 50000)");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   /// Engine *engine = EngineDepot.Select(spec);
+
+   // string spec("mode: 'Serial'");
+   // string spec("mode: 'CUDA', deviceID: 0");
+   // string spec("mode: 'OpenMP', threads: 4");
+   // string spec("mode: 'OpenCL', deviceID: 0, platformID: 0");
+   // SharedPtr<Engine> engine(new mfem::occa::Engine(spec));
+
+   SharedPtr<Engine> engine;
+   if (!strncmp(engine_type, "omp", 3))
+   {
+      engine.Reset(new mfem::omp::Engine(engine_spec));
+   }
+   else if (!strncmp(engine_type, "occa", 4))
+   {
+      engine.Reset(new mfem::occa::Engine(engine_spec));
+   }
+
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
+   //    the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   mesh->SetEngine(*engine);
+   int dim = mesh->Dimension();
+
+   // 3. Refine the mesh to increase the resolution. In this example we do
+   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
+   //    largest number that gives a final mesh with no more than 50,000
+   //    elements.
+   {
+      if (ref_levels < 0) ref_levels = (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 4. Define a finite element space on the mesh. Here we use continuous
+   //    Lagrange finite elements of the specified order. If order < 1, we
+   //    instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (mesh->GetNodes())
+   {
+      fec = mesh->GetNodes()->OwnFEC();
+      cout << "Using isoparametric FEs: " << fec->Name() << endl;
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   cout << "Number of finite element unknowns: "
+        << fespace->GetTrueVSize() << endl;
+
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
+   //    In this example, the boundary conditions are defined by marking all
+   //    the boundary attributes from the mesh as essential (Dirichlet) and
+   //    converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
+   //    the basis functions in the finite element fespace.
+   LinearForm *b = new LinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   // 7. Define the solution vector x as a finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   GridFunction x(fespace);
+   x.Fill(0.0);
+
+   // 8. Set up the bilinear form a(.,.) on the finite element space
+   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //    domain integrator.
+   BilinearForm *a = new BilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 9. Assemble the bilinear form and the corresponding linear system,
+   //    applying any necessary transformations such as: eliminating boundary
+   //    conditions, applying conforming constraints for non-conforming AMR,
+   //    static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   OperatorHandle A(Operator::ANY_TYPE);
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   cout << "Size of linear system: " << A.Ptr()->Height() << endl;
+
+   // 10. Solve the system A X = B with CG.
+   CG(*A.Ptr(), B, X, 3, 500, 1e-12, 0.0);
+
+   // 11. Recover the solution as a finite element grid function.
+   a->RecoverFEMSolution(X, *b, x);
+   x.Pull();
+
+   // 12. Save the refined mesh and the solution. This output can be viewed
+   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
+   ofstream mesh_ofs("refined.mesh");
+   mesh_ofs.precision(8);
+   mesh->Print(mesh_ofs);
+   ofstream sol_ofs("sol.gf");
+   sol_ofs.precision(8);
+   x.Save(sol_ofs);
+
+   // 13. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *mesh << x << flush;
+   }
+
+   // 14. Free the used memory.
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete mesh;
+
+   return 0;
+}

examples/ex1pd.cpp

@@ -0,0 +1,248 @@
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+   const char *engine_type = "omp";
+   const char *engine_spec = "mult_engine:'acrotensor', exec_target:'device', mem_type:'unified'";
+   int serial_ref_levels = -1;
+   int parallel_ref_levels = 2;
+   bool use_preconditioner = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&engine_type, "-et", "--engine-type", "Engine type");
+   args.AddOption(&engine_spec, "-es", "--engine-spec", "Engine specification");
+   args.AddOption(&serial_ref_levels, "-sr", "--serial-refs", "Number of serial uniform refinements (negative implies dof ~ 10000)");
+   args.AddOption(&parallel_ref_levels, "-pr", "--parallel-refs", "Number of parallel uniform refinements");
+   args.AddOption(&use_preconditioner, "-prec", "--use-preconditioner", "-no-prec", "--no-preconditioner", "Enable HYPRE AMG preconditioner");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   /// Engine *engine = EngineDepot.Select(spec);
+
+   // string occa_spec("mode: 'Serial'");
+   // string occa_spec("mode: 'CUDA', deviceID: 0");
+   // string occa_spec("mode: 'OpenMP', threads: 4");
+   // string occa_spec("mode: 'OpenCL', deviceID: 0, platformID: 0");
+   // SharedPtr<Engine> engine(new mfem::occa::Engine(MPI_COMM_WORLD, occa_spec));
+
+   SharedPtr<Engine> engine;
+   if (!strncmp(engine_type, "omp", 3))
+   {
+      engine.Reset(new mfem::omp::Engine(MPI_COMM_WORLD, engine_spec));
+   }
+   else if (!strncmp(engine_type, "occa", 4))
+   {
+      engine.Reset(new mfem::occa::Engine(MPI_COMM_WORLD, engine_spec));
+   }
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   mesh->SetEngine(*engine);
+   int dim = mesh->Dimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 10,000 elements.
+   {
+      if (serial_ref_levels < 0)
+         serial_ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < serial_ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   {
+      for (int l = 0; l < parallel_ref_levels; l++)
+      {
+         pmesh->UniformRefinement();
+      }
+   }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+      if (myid == 0)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (1,phi_i) where phi_i are the basis functions in fespace.
+   ParLinearForm *b = new ParLinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   ParGridFunction x(fespace);
+   x.Fill(0.0);
+
+   // 10. Set up the parallel bilinear form a(.,.) on the finite element space
+   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //     domain integrator.
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   CGSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+   HypreSolver *amg = NULL;
+   pcg->SetRelTol(1e-6);
+   pcg->SetAbsTol(0.0);
+   pcg->SetMaxIter(500);
+   pcg->SetPrintLevel(1);
+
+   Vector B, X;
+   if (!use_preconditioner)
+   {
+      OperatorHandle A(Operator::ANY_TYPE);
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+      pcg->SetOperator(*A.Ptr());
+      pcg->Mult(B, X);
+   }
+   else
+   {
+      HypreParMatrix A;
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+      amg = new HypreBoomerAMG(A);
+      std::cout << "operator size: " << A.InLayout()->Size() << " " << A.OutLayout()->Size() << std::endl;
+      pcg->SetOperator(A);
+      pcg->SetPreconditioner(*amg);
+      Vector X_backend(*X.Get_PVector()), B_backend(*B.Get_PVector());
+      pcg->Mult(B_backend, X_backend);
+      delete amg;
+   }
+   delete pcg;
+
+   // 13. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a->RecoverFEMSolution(X, *b, x);
+   x.Pull();
+
+   // 14. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 15. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x << flush;
+   }
+
+   // 16. Free the used memory.
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}

fem/bilinearform.cpp

@@ -74,6 +74,12 @@ BilinearForm::BilinearForm (FiniteElementSpace * f)
    hybridization = NULL;
    precompute_sparsity = 0;
    diag_policy = DIAG_KEEP;
+#ifdef MFEM_USE_BACKENDS
+   if (fes->GetVLayout()->HasEngine())
+   {
+      dev_ext = fes->GetVLayout()->GetEngine().MakeBilinearForm(*this);
+   }
+#endif
 }
 
 BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
@@ -126,6 +132,14 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
 void BilinearForm::EnableStaticCondensation()
 {
    delete static_cond;
+#ifdef MFEM_USE_BACKENDS
+   if (fes->GetVLayout()->HasEngine())
+   {
+      static_cond = NULL;
+      MFEM_WARNING("Engine interface does not support static condensation yet");
+      return;
+   }
+#endif
    static_cond = new StaticCondensation(fes);
    if (static_cond->ReducesTrueVSize())
    {
@@ -145,6 +159,15 @@ void BilinearForm::EnableHybridization(FiniteElementSpace *constr_space,
                                        const Array<int> &ess_tdof_list)
 {
    delete hybridization;
+#ifdef MFEM_USE_BACKENDS
+   if (fes->GetVLayout()->HasEngine())
+   {
+      delete constr_integ;
+      hybridization = NULL;
+      MFEM_WARNING("Engine interface does not support hybridization yet");
+      return;
+   }
+#endif
    hybridization = new Hybridization(fes, constr_space);
    hybridization->SetConstraintIntegrator(constr_integ);
    hybridization->Init(ess_tdof_list);
@@ -313,7 +336,15 @@ void BilinearForm::Assemble (int skip_zeros)
    Mesh *mesh = fes -> GetMesh();
    DenseMatrix elmat, *elmat_p;
 
-   int i;
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      // TODO: push the 'skip_zeros' as a parameter to 'dev_ext'
+
+      const bool assembly_done = dev_ext->Assemble();
+      if (assembly_done) { return; }
+   }
+#endif
 
    if (mat == NULL)
    {
@@ -331,7 +362,7 @@ void BilinearForm::Assemble (int skip_zeros)
 
    if (dbfi.Size())
    {
-      for (i = 0; i < fes -> GetNE(); i++)
+      for (int i = 0; i < fes -> GetNE(); i++)
       {
          fes->GetElementVDofs(i, vdofs);
          if (element_matrices)
@@ -388,7 +419,7 @@ void BilinearForm::Assemble (int skip_zeros)
          }
       }
 
-      for (i = 0; i < fes -> GetNBE(); i++)
+      for (int i = 0; i < fes -> GetNBE(); i++)
       {
          const int bdr_attr = mesh->GetBdrAttribute(i);
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
@@ -426,7 +457,7 @@ void BilinearForm::Assemble (int skip_zeros)
       Array<int> vdofs2;
 
       int nfaces = mesh->GetNumFaces();
-      for (i = 0; i < nfaces; i++)
+      for (int i = 0; i < nfaces; i++)
       {
          tr = mesh -> GetInteriorFaceTransformations (i);
          if (tr != NULL)
@@ -471,7 +502,7 @@ void BilinearForm::Assemble (int skip_zeros)
          }
       }
 
-      for (i = 0; i < fes -> GetNBE(); i++)
+      for (int i = 0; i < fes -> GetNBE(); i++)
       {
          const int bdr_attr = mesh->GetBdrAttribute(i);
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
@@ -643,9 +674,91 @@ void BilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
    }
 }
 
+void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
+                                    Vector &x, Vector &b,
+                                    OperatorHandle &A, Vector &X, Vector &B,
+                                    int copy_interior)
+{
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      MFEM_VERIFY(!static_cond && !hybridization, "");
+      dev_ext->FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
+   }
+   else
+#endif
+   {
+      if (A.Type() == Operator::MFEM_SPARSEMAT)
+      {
+         SparseMatrix A_sm;
+         FormLinearSystem(ess_tdof_list, x, b, A_sm, X, B, copy_interior);
+         if (static_cond)
+         {
+            A.Reset(&static_cond->GetMatrix(), false);
+         }
+         else if (hybridization)
+         {
+            A.Reset(&hybridization->GetMatrix(), false);
+         }
+         else
+         {
+            A.Reset(mat, false);
+         }
+      }
+      else
+      {
+         MFEM_ABORT("Operator::Type is not supported: type_id = " << A.Type());
+      }
+   }
+}
+
+void BilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                    OperatorHandle &A)
+{
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      MFEM_VERIFY(!static_cond && !hybridization, "");
+      dev_ext->FormSystemMatrix(ess_tdof_list, A);
+   }
+   else
+#endif
+   {
+      if (A.Type() == Operator::MFEM_SPARSEMAT)
+      {
+         SparseMatrix A_sm;
+         FormSystemMatrix(ess_tdof_list, A_sm);
+         if (static_cond)
+         {
+            A.Reset(&static_cond->GetMatrix(), false);
+         }
+         else if (hybridization)
+         {
+            A.Reset(&hybridization->GetMatrix(), false);
+         }
+         else
+         {
+            A.Reset(mat, false);
+         }
+      }
+      else
+      {
+         MFEM_ABORT("Operator::Type is not supported: type_id = " << A.Type());
+      }
+   }
+}
+
 void BilinearForm::RecoverFEMSolution(const Vector &X,
                                       const Vector &b, Vector &x)
 {
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      dev_ext->RecoverFEMSolution(X, b, x);
+      return;
+   }
+#endif
+
    const SparseMatrix *P = fes->GetConformingProlongation();
    if (!P) // conforming space
    {
@@ -734,7 +847,8 @@ void BilinearForm::ComputeElementMatrices()
 }
 
 void BilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                                        Vector &sol, Vector &rhs, DiagonalPolicy dpolicy)
+                                        Vector &sol, Vector &rhs,
+                                        DiagonalPolicy dpolicy)
 {
    Array<int> ess_dofs, conf_ess_dofs;
    fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);

fem/bilinearform.hpp

@@ -38,6 +38,11 @@ protected:
    /// FE space on which the form lives.
    FiniteElementSpace *fes;
 
+#ifdef MFEM_USE_BACKENDS
+   /// Device/Engine extension (smart shared pointer)
+   DBilinearForm dev_ext;
+#endif
+
    /// Indicates the Mesh::sequence corresponding to the current state of the
    /// BilinearForm.
    long sequence;
@@ -285,6 +290,37 @@ public:
    /// Form the linear system matrix A, see FormLinearSystem() for details.
    void FormSystemMatrix(const Array<int> &ess_tdof_list, SparseMatrix &A);
 
+   /** Form the linear system @a A @a X = @a B, corresponding to the bilinear
+       form and the r.h.s. linear form (vector) @a b, by applying any necessary
+       transformations such as: eliminating boundary conditions; applying
+       conforming constraints for non-conforming AMR; parallel assembly; static
+       condensation; hybridization.
+
+       The GridFunction-size vector @a x must contain the essential b.c. The
+       BilinearForm and the LinearForm-size vector @a b must be assembled.
+
+       The vector @a X is initialized with a suitable initial guess: when using
+       hybridization, the vector @a X is set to zero; otherwise, the essential
+       entries of @a X are set to the corresponding b.c. and all other entries
+       are set to zero (if @a copy_interior == 0) or copied from @a x (if
+       @a copy_interior != 0).
+
+       This method can be called multiple times (with the same @a ess_tdof_list
+       array) to initialize different right-hand sides and boundary condition
+       values.
+
+       After solving the linear system, the finite element solution @a x can be
+       recovered by calling RecoverFEMSolution() (with the same vectors @a X,
+       @a b, and @a x). */
+   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
+                                 Vector &x, Vector &b,
+                                 OperatorHandle &A, Vector &X, Vector &B,
+                                 int copy_interior = 0);
+
+   /// Form the linear system matrix @a A, see FormLinearSystem() for details.
+   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                 OperatorHandle &A);
+
    /// Recover the solution of a linear system formed with FormLinearSystem().
    /** Call this method after solving a linear system constructed using the
        FormLinearSystem() method to recover the solution as a GridFunction-size

fem/bilininteg.cpp

@@ -361,6 +361,28 @@ void MixedScalarVectorIntegrator::AssembleElementMatrix2(
    }
 }
 
+
+const IntegrationRule &DiffusionIntegrator::GetRule(
+   const FiniteElement &trial_fe, const FiniteElement &test_fe)
+{
+   int order;
+   if (trial_fe.Space() == FunctionSpace::Pk)
+   {
+      order = trial_fe.GetOrder() + test_fe.GetOrder() - 2;
+   }
+   else
+   {
+      // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
+      order = trial_fe.GetOrder() + test_fe.GetOrder() + trial_fe.GetDim() - 1;
+   }
+
+   if (trial_fe.Space() == FunctionSpace::rQk)
+   {
+      return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
+   }
+   return IntRules.Get(trial_fe.GetGeomType(), order);
+}
+
 void DiffusionIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
   DenseMatrix &elmat )
@@ -383,25 +405,7 @@ void DiffusionIntegrator::AssembleElementMatrix
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int order;
-      if (el.Space() == FunctionSpace::Pk)
-      {
-         order = 2*el.GetOrder() - 2;
-      }
-      else
-         // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
-      {
-         order = 2*el.GetOrder() + dim - 1;
-      }
-
-      if (el.Space() == FunctionSpace::rQk)
-      {
-         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
-      }
-      else
-      {
-         ir = &IntRules.Get(el.GetGeomType(), order);
-      }
+      ir = &GetRule(el, el);
    }
 
    elmat = 0.0;
@@ -461,24 +465,7 @@ void DiffusionIntegrator::AssembleElementMatrix2(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int order;
-      if (trial_fe.Space() == FunctionSpace::Pk)
-      {
-         order = trial_fe.GetOrder() + test_fe.GetOrder() - 2;
-      }
-      else
-      {
-         order = trial_fe.GetOrder() + test_fe.GetOrder() + dim - 1;
-      }
-
-      if (trial_fe.Space() == FunctionSpace::rQk)
-      {
-         ir = &RefinedIntRules.Get(trial_fe.GetGeomType(), order);
-      }
-      else
-      {
-         ir = &IntRules.Get(trial_fe.GetGeomType(), order);
-      }
+      ir = &GetRule(trial_fe, test_fe);
    }
 
    elmat = 0.0;
@@ -717,6 +704,22 @@ double DiffusionIntegrator::ComputeFluxEnergy
 }
 
 
+const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
+                                               const FiniteElement &test_fe,
+                                               ElementTransformation &Trans,
+                                               int coeff_order)
+{
+   // int order = trial_fe.GetOrder() + test_fe.GetOrder();
+   int order = trial_fe.GetOrder() + test_fe.GetOrder() +
+               Trans.OrderW() + coeff_order;
+
+   if (trial_fe.Space() == FunctionSpace::rQk)
+   {
+      return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
+   }
+   return IntRules.Get(trial_fe.GetGeomType(), order);
+}
+
 void MassIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
   DenseMatrix &elmat )
@@ -734,17 +737,7 @@ void MassIntegrator::AssembleElementMatrix
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      // int order = 2 * el.GetOrder();
-      int order = 2 * el.GetOrder() + Trans.OrderW();
-
-      if (el.Space() == FunctionSpace::rQk)
-      {
-         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
-      }
-      else
-      {
-         ir = &IntRules.Get(el.GetGeomType(), order);
-      }
+      ir = &GetRule(el, el, Trans);
    }
 
    elmat = 0.0;
@@ -783,9 +776,7 @@ void MassIntegrator::AssembleElementMatrix2(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
-
-      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+      ir = &GetRule(trial_fe, test_fe, Trans);
    }
 
    elmat = 0.0;
@@ -980,16 +971,7 @@ void VectorMassIntegrator::AssembleElementMatrix
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int order = 2 * el.GetOrder() + Trans.OrderW() + Q_order;
-
-      if (el.Space() == FunctionSpace::rQk)
-      {
-         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
-      }
-      else
-      {
-         ir = &IntRules.Get(el.GetGeomType(), order);
-      }
+      ir = &MassIntegrator::GetRule(el, el, Trans, Q_order);
    }
 
    elmat = 0.0;
@@ -1065,17 +1047,7 @@ void VectorMassIntegrator::AssembleElementMatrix2(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int order = (trial_fe.GetOrder() + test_fe.GetOrder() +
-                   Trans.OrderW() + Q_order);
-
-      if (trial_fe.Space() == FunctionSpace::rQk)
-      {
-         ir = &RefinedIntRules.Get(trial_fe.GetGeomType(), order);
-      }
-      else
-      {
-         ir = &IntRules.Get(trial_fe.GetGeomType(), order);
-      }
+      ir = &MassIntegrator::GetRule(trial_fe, test_fe, Trans, Q_order);
    }
 
    elmat = 0.0;

fem/bilininteg.hpp

@@ -1622,6 +1622,13 @@ public:
    virtual double ComputeFluxEnergy(const FiniteElement &fluxelem,
                                     ElementTransformation &Trans,
                                     Vector &flux, Vector *d_energy = NULL);
+
+   virtual const char *Name() const { return "diffusion"; }
+
+   virtual Coefficient *GetScalarCoefficient() const { return Q; }
+
+   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
+                                         const FiniteElement &test_fe);
 };
 
 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
@@ -1649,6 +1656,15 @@ public:
                                        const FiniteElement &test_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
+
+   virtual const char *Name() const { return "mass"; }
+
+   virtual Coefficient *GetScalarCoefficient() const { return Q; }
+
+   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
+                                         const FiniteElement &test_fe,
+                                         ElementTransformation &Trans,
+                                         int coeff_order = 0);
 };
 
 class BoundaryMassIntegrator : public MassIntegrator

fem/fe_coll.cpp

@@ -165,8 +165,10 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
                                 BasisType::GetType(name[3]));
    }
    else if (!strncmp(name, "L2_T", 4))
+   {
       fec = new L2_FECollection(atoi(name + 10), atoi(name + 6),
                                 atoi(name + 4));
+   }
    else if (!strncmp(name, "L2_", 3))
    {
       fec = new L2_FECollection(atoi(name + 7), atoi(name + 3));

fem/fespace.cpp

@@ -377,6 +377,12 @@ void FiniteElementSpace::GetEssentialTrueDofs(const Array<int> &bdr_attr_is_ess,
    {
       R->BooleanMult(ess_vdofs, ess_tdofs);
    }
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      ess_tdof_list.SetEngine(dev_ext->GetEngine());
+   }
+#endif
    MarkerToList(ess_tdofs, ess_tdof_list);
 }
 
@@ -389,12 +395,14 @@ void FiniteElementSpace::MarkerToList(const Array<int> &marker,
    {
       if (marker[i]) { num_marked++; }
    }
+   list.Resize(num_marked);
+   list.Pull(false);
    list.SetSize(0);
-   list.Reserve(num_marked);
    for (int i = 0; i < marker.Size(); i++)
    {
       if (marker[i]) { list.Append(i); }
    }
+   list.Push();
 }
 
 // static method
@@ -642,9 +650,23 @@ void FiniteElementSpace::BuildConformingInterpolation() const
    if (n_true_dofs == ndofs)
    {
       cP = cR = NULL; // will be treated as identities
+#ifdef MFEM_USE_BACKENDS
+      t_layout = v_layout;
+#endif
       return;
    }
 
+#ifdef MFEM_USE_BACKENDS
+   if (mesh->HasEngine())
+   {
+      t_layout = mesh->GetEngine().MakeLayout(n_true_dofs*vdim);
+   }
+   else
+   {
+      t_layout.Reset(new PLayout(n_true_dofs*vdim));
+   }
+#endif
+
    // create the conforming restriction matrix cR
    int *cR_J;
    {
@@ -757,27 +779,6 @@ void FiniteElementSpace::MakeVDimMatrix(SparseMatrix &mat) const
    delete vmat;
 }
 
-
-const SparseMatrix* FiniteElementSpace::GetConformingProlongation() const
-{
-   if (Conforming()) { return NULL; }
-   if (!cP_is_set) { BuildConformingInterpolation(); }
-   return cP;
-}
-
-const SparseMatrix* FiniteElementSpace::GetConformingRestriction() const
-{
-   if (Conforming()) { return NULL; }
-   if (!cP_is_set) { BuildConformingInterpolation(); }
-   return cR;
-}
-
-int FiniteElementSpace::GetNConformingDofs() const
-{
-   const SparseMatrix* P = GetConformingProlongation();
-   return P ? (P->Width() / vdim) : ndofs;
-}
-
 SparseMatrix *FiniteElementSpace::RefinementMatrix_main(
    const int coarse_ndofs, const Table &coarse_elem_dof,
    const DenseTensor &localP) const
@@ -1108,7 +1109,7 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
    {
       if (!mesh->NURBSext)
       {
-         mfem_error("FiniteElementSpace::FiniteElementSpace :\n"
+         mfem_error("FiniteElementSpace::Constructor :\n"
                     "   NURBS FE space requires NURBS mesh.");
       }
 
@@ -1124,7 +1125,7 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
       }
       UpdateNURBS();
       cP = cR = NULL;
-      cP_is_set = false;
+      cP_is_set = true;
    }
    else
    {
@@ -1133,6 +1134,22 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
       Construct();
    }
    BuildElementToDofTable();
+
+#ifdef MFEM_USE_BACKENDS
+   if (mesh->HasEngine())
+   {
+      v_layout = mesh->GetEngine().MakeLayout(GetVSize());
+      if (cP_is_set) { t_layout = v_layout; }
+      // Ensure GetVLayout() and GetTrueVLayout() will work correctly before
+      // calling MakeFESpace().
+      dev_ext = mesh->GetEngine().MakeFESpace(*this);
+   }
+   else
+   {
+      v_layout.Reset(new PLayout(GetVSize()));
+      if (cP_is_set) { t_layout = v_layout; }
+   }
+#endif
 }
 
 NURBSExtension *FiniteElementSpace::StealNURBSext()
@@ -1160,10 +1177,14 @@ void FiniteElementSpace::UpdateNURBS()
    ndofs = NURBSext->GetNDof();
    elem_dof = NURBSext->GetElementDofTable();
    bdrElem_dof = NURBSext->GetBdrElementDofTable();
+
+   // TODO: update v_layout, t_layout
 }
 
 void FiniteElementSpace::Construct()
 {
+   // called in parallel by ParFiniteElementSpace::Update()
+
    // This method should be used only for non-NURBS spaces.
    MFEM_ASSERT(!NURBSext, "internal error");
 
@@ -1188,7 +1209,7 @@ void FiniteElementSpace::Construct()
    fdofs = NULL;
    cP = NULL;
    cR = NULL;
-   cP_is_set = false;
+   cP_is_set = Conforming();
    // Th is initialized/destroyed before this method is called.
 
    if (mesh->Dimension() == 3 && mesh->GetNE())
@@ -1635,6 +1656,10 @@ FiniteElementSpace::~FiniteElementSpace()
 
 void FiniteElementSpace::Destroy()
 {
+   // called in parallel by ParFiniteElementSpace::Update()
+
+   // For now, do not reset dev_ext and/or v_layout, t_layout
+
    delete cR;
    delete cP;
    Th.Clear();

fem/fespace.hpp

@@ -16,6 +16,11 @@
 #include "../linalg/sparsemat.hpp"
 #include "../mesh/mesh.hpp"
 #include "fe_coll.hpp"
+
+#ifdef MFEM_USE_BACKENDS
+#include "../backends/base/backend.hpp"
+#endif
+
 #include <iostream>
 
 namespace mfem
@@ -72,6 +77,12 @@ protected:
    /// Associated FE collection (not owned).
    const FiniteElementCollection *fec;
 
+#ifdef MFEM_USE_BACKENDS
+   DFiniteElementSpace dev_ext;
+   DLayout v_layout;
+   mutable DLayout t_layout;
+#endif
+
    /// %Vector dimension (number of unknowns per degree of freedom).
    int vdim;
 
@@ -124,6 +135,9 @@ protected:
    /// Calculate the cP and cR matrices for a nonconforming mesh.
    void BuildConformingInterpolation() const;
 
+   // Shortcut
+   void SetCP() const { if (!cP_is_set) { BuildConformingInterpolation(); } }
+
    static void AddDependencies(SparseMatrix& deps, Array<int>& master_dofs,
                                Array<int>& slave_dofs, DenseMatrix& I);
 
@@ -230,8 +244,8 @@ public:
    bool Conforming() const { return mesh->Conforming(); }
    bool Nonconforming() const { return mesh->Nonconforming(); }
 
-   const SparseMatrix *GetConformingProlongation() const;
-   const SparseMatrix *GetConformingRestriction() const;
+   const SparseMatrix *GetConformingProlongation() const { SetCP(); return cP; }
+   const SparseMatrix *GetConformingRestriction() const { SetCP(); return cR; }
 
    virtual const Operator *GetProlongationMatrix() const
    { return GetConformingProlongation(); }
@@ -255,11 +269,24 @@ public:
    /// Return the number of vector true (conforming) dofs.
    virtual int GetTrueVSize() const { return GetConformingVSize(); }
 
+#ifdef MFEM_USE_BACKENDS
+   /// TODO: doxygen
+   DLayout &GetVLayout() { return v_layout; }
+
+   /// TODO: doxygen
+   DLayout &GetTrueVLayout() { SetCP(); return t_layout; }
+
+   /// TODO: doxygen
+   const DFiniteElementSpace &Get_PFESpace() const { return dev_ext; }
+#endif
+
    /// Returns the number of conforming ("true") degrees of freedom
    /// (if the space is on a nonconforming mesh with hanging nodes).
-   int GetNConformingDofs() const;
+   int GetNConformingDofs() const
+   { SetCP(); return cP ? (cP->Width() / vdim) : ndofs; }
 
-   int GetConformingVSize() const { return vdim * GetNConformingDofs(); }
+   int GetConformingVSize() const
+   { SetCP(); return cP ? cP->Width() : GetVSize(); }
 
    /// Return the ordering method.
    inline Ordering::Type GetOrdering() const { return ordering; }
@@ -411,12 +438,16 @@ public:
    /** Get a list of essential true dofs, ess_tdof_list, corresponding to the
        boundary attributes marked in the array bdr_attr_is_ess.
        For spaces with 'vdim' > 1, the 'component' parameter can be used
-       to restricts the marked tDOFs to the specified component. */
+       to restricts the marked tDOFs to the specified component.
+
+       If the FE space is linked to an Engine (as defined by its Mesh), the out
+       @a ess_tdof_list will be transferred to the engine as well. */
    virtual void GetEssentialTrueDofs(const Array<int> &bdr_attr_is_ess,
                                      Array<int> &ess_tdof_list,
                                      int component = -1);
 
    /// Convert a Boolean marker array to a list containing all marked indices.
+   /** This method supports device arrays for the output parameter, @a list. */
    static void MarkerToList(const Array<int> &marker, Array<int> &list);
 
    /** Convert an array of indices (list) to a Boolean marker array where all

fem/gridfunc.cpp

@@ -33,6 +33,9 @@ GridFunction::GridFunction(Mesh *m, std::istream &input)
    fes = new FiniteElementSpace;
    fec = fes->Load(m, input);
 
+   Resize(MFEM_IF_BACKENDS(fes->GetVLayout(), fes->GetVSize()));
+   Pull(false);
+
    skip_comment_lines(input, '#');
    istream::int_type next_char = input.peek();
    if (next_char == 'N') // First letter of "NURBS_patches"
@@ -56,6 +59,7 @@ GridFunction::GridFunction(Mesh *m, std::istream &input)
       Vector::Load(input, fes->GetVSize());
    }
    sequence = fes->GetSequence();
+   Push();
 }
 
 GridFunction::GridFunction(Mesh *m, GridFunction *gf_array[], int num_pieces)
@@ -1431,6 +1435,7 @@ void GridFunction::ProjectCoefficient(VectorCoefficient &vcoeff)
    Array<int> vdofs;
    Vector vals;
 
+   Pull(false);
    for (i = 0; i < fes->GetNE(); i++)
    {
       fes->GetElementVDofs(i, vdofs);
@@ -1438,6 +1443,7 @@ void GridFunction::ProjectCoefficient(VectorCoefficient &vcoeff)
       fes->GetFE(i)->Project(vcoeff, *fes->GetElementTransformation(i), vals);
       SetSubVector(vdofs, vals);
    }
+   Push();
 }
 
 void GridFunction::ProjectCoefficient(

fem/gridfunc.hpp

@@ -61,6 +61,8 @@ protected:
        degree of freedom. */
    void ProjectDiscCoefficient(VectorCoefficient &coeff, Array<int> &dof_attr);
 
+   // There is a method with the same name in the base class Vector.
+   // This version of Destroy does not call Vector::Destroy.
    void Destroy();
 
 public:
@@ -72,7 +74,8 @@ public:
       : Vector(orig), fes(orig.fes), fec(NULL), sequence(orig.sequence) { }
 
    /// Construct a GridFunction associated with the FiniteElementSpace @a *f.
-   GridFunction(FiniteElementSpace *f) : Vector(f->GetVSize())
+   GridFunction(FiniteElementSpace *f)
+      : Vector(MFEM_IF_BACKENDS(f->GetVLayout(), f->GetVSize()))
    { fes = f; fec = NULL; sequence = f->GetSequence(); }
 
    /// Construct a GridFunction using previously allocated array @a data.
@@ -81,8 +84,7 @@ public:
        for externally allocated array, the pointer @a data can be NULL. The data
        array can be replaced later using the method SetData().
     */
-   GridFunction(FiniteElementSpace *f, double *data) : Vector(data, f->GetVSize())
-   { fes = f; fec = NULL; sequence = f->GetSequence(); }
+   inline GridFunction(FiniteElementSpace *f, double *data);
 
    /// Construct a GridFunction on the given Mesh, using the data from @a input.
    /** The content of @a input should be in the format created by the method
@@ -533,7 +535,21 @@ GridFunction *Extrude1DGridFunction(Mesh *mesh, Mesh *mesh2d,
                                     GridFunction *sol, const int ny);
 
 
-// Inline methods
+// Inline methods: class GridFunction
+
+inline GridFunction::GridFunction(FiniteElementSpace *f, double *data)
+   : Vector(data, f->GetVSize())
+{
+#ifdef MFEM_USE_BACKENDS
+   MFEM_ASSERT(f->GetVLayout()->HasEngine() == false, "not supported");
+#endif
+   fes = f;
+   fec = NULL;
+   sequence = f->GetSequence();
+}
+
+
+// Inline methods: class QuadratureFunction
 
 inline void QuadratureFunction::SetSpace(QuadratureSpace *qspace_, int vdim_)
 {

fem/linearform.cpp

@@ -54,12 +54,14 @@ void LinearForm::Assemble()
    ElementTransformation *eltrans;
    Vector elemvect;
 
-   int i;
+   // FIXME: Use the Engine when set
 
+   // Assemble on host
+   Pull(false); // copy_data = false
    Vector::operator=(0.0);
 
    if (dlfi.Size())
-      for (i = 0; i < fes -> GetNE(); i++)
+      for (int i = 0; i < fes -> GetNE(); i++)
       {
          fes -> GetElementVDofs (i, vdofs);
          eltrans = fes -> GetElementTransformation (i);
@@ -73,7 +75,7 @@ void LinearForm::Assemble()
    AssembleDelta();
 
    if (blfi.Size())
-      for (i = 0; i < fes -> GetNBE(); i++)
+      for (int i = 0; i < fes -> GetNBE(); i++)
       {
          fes -> GetBdrElementVDofs (i, vdofs);
          eltrans = fes -> GetBdrElementTransformation (i);
@@ -110,7 +112,7 @@ void LinearForm::Assemble()
          }
       }
 
-      for (i = 0; i < mesh->GetNBE(); i++)
+      for (int i = 0; i < mesh->GetNBE(); i++)
       {
          const int bdr_attr = mesh->GetBdrAttribute(i);
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
@@ -131,11 +133,16 @@ void LinearForm::Assemble()
          }
       }
    }
+
+   Push();
 }
 
 void LinearForm::Update(FiniteElementSpace *f, Vector &v, int v_offset)
 {
    fes = f;
+#ifdef MFEM_USE_BACKENDS
+   MFEM_VERIFY(f->GetMesh()->HasEngine() == false, "not supported yet");
+#endif
    NewDataAndSize((double *)v + v_offset, fes->GetVSize());
    ResetDeltaLocations();
 }

fem/linearform.hpp

@@ -53,8 +53,8 @@ private:
 
 public:
    /// Creates linear form associated with FE space *f.
-   LinearForm (FiniteElementSpace * f) : Vector (f -> GetVSize())
-   { fes = f; }
+   LinearForm(FiniteElementSpace *f)
+      : Vector(MFEM_IF_BACKENDS(f->GetVLayout(), f->GetVSize())), fes(f) { }
 
    LinearForm() { fes = NULL; }
 
@@ -87,10 +87,13 @@ public:
    /// Assembles delta functions of the linear form
    void AssembleDelta();
 
-   void Update() { SetSize(fes->GetVSize()); ResetDeltaLocations(); }
+   void Update()
+   {
+      Resize(MFEM_IF_BACKENDS(fes->GetVLayout(), fes->GetVSize()));
+      ResetDeltaLocations();
+   }
 
-   void Update(FiniteElementSpace *f)
-   { fes = f; SetSize(f->GetVSize()); ResetDeltaLocations(); }
+   void Update(FiniteElementSpace *f) { fes = f; Update(); }
 
    void Update(FiniteElementSpace *f, Vector &v, int v_offset);
 

fem/nonlininteg.hpp

@@ -39,6 +39,10 @@ public:
    /// Prescribe a fixed IntegrationRule to use.
    void SetIntegrationRule(const IntegrationRule &irule) { IntRule = &irule; }
 
+   /** @brief Get the prescribed IntegrationRule, if set by SetIntRule() or
+       SetIntegrationRule(). */
+   const IntegrationRule *GetIntRule() const { return IntRule; }
+
    /// Perform the local action of the NonlinearFormIntegrator
    virtual void AssembleElementVector(const FiniteElement &el,
                                       ElementTransformation &Tr,
@@ -68,6 +72,13 @@ public:
                                    ElementTransformation &Tr,
                                    const Vector &elfun);
 
+   /// Return a string that identifies the integrator.
+   virtual const char *Name() const { return "(undefined)"; }
+
+   /** If the integrator uses a scalar Coefficient, return a pointer to it.
+       Otherwise return NULL. */
+   virtual Coefficient *GetScalarCoefficient() const { return NULL; }
+
    virtual ~NonlinearFormIntegrator() { }
 };
 

fem/pbilinearform.cpp

@@ -133,8 +133,18 @@ void ParBilinearForm::ParallelAssemble(OperatorHandle &A, SparseMatrix *A_local)
    if (fbfi.Size() == 0)
    {
       // construct a parallel block-diagonal matrix 'A' based on 'a'
-      dA.MakeSquareBlockDiag(pfes->GetComm(), pfes->GlobalVSize(),
-                             pfes->GetDofOffsets(), A_local);
+#ifdef MFEM_USE_BACKENDS
+      if (pfes->GetMesh()->HasEngine())
+      {
+         dA.MakeSquareBlockDiag(pfes->GetComm(), *pfes->GetMesh()->GetEngine().MakeLayout(pfes->GlobalVSize()),
+                                pfes->GetDofOffsets(), A_local);
+      }
+      else
+#endif
+      {
+         dA.MakeSquareBlockDiag(pfes->GetComm(), pfes->GlobalVSize(),
+                                pfes->GetDofOffsets(), A_local);
+      }
    }
    else
    {
@@ -284,49 +294,67 @@ void ParBilinearForm::FormLinearSystem(
    const Array<int> &ess_tdof_list, Vector &x, Vector &b,
    OperatorHandle &A, Vector &X, Vector &B, int copy_interior)
 {
-   // Finish the matrix assembly and perform BC elimination, storing the
-   // eliminated part of the matrix.
-   FormSystemMatrix(ess_tdof_list, A);
-
-   const Operator &P = *pfes->GetProlongationMatrix();
-   const SparseMatrix &R = *pfes->GetRestrictionMatrix();
-
-   // Transform the system and perform the elimination in B, based on the
-   // essential BC values from x. Restrict the BC part of x in X, and set the
-   // non-BC part to zero. Since there is no good initial guess for the Lagrange
-   // multipliers, set X = 0.0 for hybridization.
-   if (static_cond)
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
    {
-      // Schur complement reduction to the exposed dofs
-      static_cond->ReduceSystem(x, b, X, B, copy_interior);
-   }
-   else if (hybridization)
-   {
-      // Reduction to the Lagrange multipliers system
-      HypreParVector true_X(pfes), true_B(pfes);
-      P.MultTranspose(b, true_B);
-      R.Mult(x, true_X);
-      p_mat.EliminateBC(p_mat_e, ess_tdof_list, true_X, true_B);
-      R.MultTranspose(true_B, b);
-      hybridization->ReduceRHS(true_B, B);
-      X.SetSize(B.Size());
-      X = 0.0;
+      MFEM_VERIFY(!static_cond && !hybridization, "");
+      dev_ext->FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
    }
    else
+#endif
    {
-      // Variational restriction with P
-      X.SetSize(pfes->TrueVSize());
-      B.SetSize(X.Size());
-      P.MultTranspose(b, B);
-      R.Mult(x, X);
-      p_mat.EliminateBC(p_mat_e, ess_tdof_list, X, B);
-      if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
+      // Finish the matrix assembly and perform BC elimination, storing the
+      // eliminated part of the matrix.
+      FormSystemMatrix(ess_tdof_list, A);
+
+      const Operator &P = *pfes->GetProlongationMatrix();
+      const SparseMatrix &R = *pfes->GetRestrictionMatrix();
+
+      // Transform the system and perform the elimination in B, based on the
+      // essential BC values from x. Restrict the BC part of x in X, and set the
+      // non-BC part to zero. Since there is no good initial guess for the Lagrange
+      // multipliers, set X = 0.0 for hybridization.
+      if (static_cond)
+      {
+         // Schur complement reduction to the exposed dofs
+         static_cond->ReduceSystem(x, b, X, B, copy_interior);
+      }
+      else if (hybridization)
+      {
+         // Reduction to the Lagrange multipliers system
+         HypreParVector true_X(pfes), true_B(pfes);
+         P.MultTranspose(b, true_B);
+         R.Mult(x, true_X);
+         p_mat.EliminateBC(p_mat_e, ess_tdof_list, true_X, true_B);
+         R.MultTranspose(true_B, b);
+         hybridization->ReduceRHS(true_B, B);
+         X.SetSize(B.Size());
+         X = 0.0;
+      }
+      else
+      {
+         // Variational restriction with P
+         X.SetSize(pfes->TrueVSize());
+         B.SetSize(X.Size());
+         P.MultTranspose(b, B);
+         R.Mult(x, X);
+         p_mat.EliminateBC(p_mat_e, ess_tdof_list, X, B);
+         if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
+      }
    }
 }
 
 void ParBilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
                                        OperatorHandle &A)
 {
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      dev_ext->FormSystemMatrix(ess_tdof_list, A);
+      return;
+   }
+#endif
+
    // Finish the matrix assembly and perform BC elimination, storing the
    // eliminated part of the matrix.
    if (static_cond)
@@ -369,6 +397,14 @@ void ParBilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
 void ParBilinearForm::RecoverFEMSolution(
    const Vector &X, const Vector &b, Vector &x)
 {
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      dev_ext->RecoverFEMSolution(X, b, x);
+      return;
+   }
+#endif
+
    const Operator &P = *pfes->GetProlongationMatrix();
 
    if (static_cond)

fem/pbilinearform.hpp

@@ -145,30 +145,10 @@ public:
    virtual const Operator *GetRestriction() const
    { return pfes->GetRestrictionMatrix(); }
 
-   /** Form the linear system A X = B, corresponding to the current bilinear
-       form and b(.), by applying any necessary transformations such as:
-       eliminating boundary conditions; applying conforming constraints for
-       non-conforming AMR; parallel assembly; static condensation;
-       hybridization.
-
-       The ParGridFunction-size vector x must contain the essential b.c. The
-       ParBilinearForm and the ParLinearForm-size vector b must be assembled.
-
-       The vector X is initialized with a suitable initial guess: when using
-       hybridization, the vector X is set to zero; otherwise, the essential
-       entries of X are set to the corresponding b.c. and all other entries are
-       set to zero (copy_interior == 0) or copied from x (copy_interior != 0).
-
-       This method can be called multiple times (with the same ess_tdof_list
-       array) to initialize different right-hand sides and boundary condition
-       values.
-
-       After solving the linear system, the finite element solution x can be
-       recovered by calling RecoverFEMSolution (with the same vectors X, b, and
-       x). */
-   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
-                         OperatorHandle &A, Vector &X, Vector &B,
-                         int copy_interior = 0);
+   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
+                                 Vector &x, Vector &b,
+                                 OperatorHandle &A, Vector &X, Vector &B,
+                                 int copy_interior = 0);
 
    /** Version of the method FormLinearSystem() where the system matrix is
        returned in the variable @a A, of type OpType, holding a *reference* to
@@ -187,8 +167,20 @@ public:
       A.MakeRef(*A_ptr);
    }
 
-   /// Form the linear system matrix @a A, see FormLinearSystem() for details.
-   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
+   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
+                         HypreParMatrix &A, Vector &X, Vector &B,
+                         int copy_interior = 0)
+   {
+      OperatorHandle Ah(Operator::Hypre_ParCSR);
+      FormLinearSystem(ess_tdof_list, x, b, Ah, X, B, copy_interior);
+      HypreParMatrix *A_ptr = Ah.As<HypreParMatrix>();
+      Ah.SetOperatorOwner(false);
+      MFEM_VERIFY(A_ptr, "invalid OpType used");
+      A.MakeRef(*A_ptr);
+   }
+
+   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                 OperatorHandle &A);
 
    /** Version of the method FormSystemMatrix() where the system matrix is
        returned in the variable @a A, of type OpType, holding a *reference* to

fem/pfespace.cpp

@@ -124,6 +124,16 @@ void ParFiniteElementSpace::ParInit(ParMesh *pm)
    {
       ApplyLDofSigns(*elem_dof);
    }
+
+#ifdef MFEM_USE_BACKENDS
+   if (pmesh->HasEngine())
+   {
+      // Ensure GetVLayout() and GetTrueVLayout() will work correctly before
+      // calling MakeFESpace().
+      // Overwrite dev_ext with one that uses parallel finite element space
+      dev_ext = pmesh->GetEngine().MakeFESpace(*this);
+   }
+#endif
 }
 
 void ParFiniteElementSpace::Construct()
@@ -169,6 +179,19 @@ void ParFiniteElementSpace::Construct()
       // to overlap its communication with processing between this constructor
       // and the point where the P matrix is actually needed.
    }
+
+#ifdef MFEM_USE_BACKENDS
+   // Now that we have the local true dof size (ltdof_size), we need to initialize t_layout
+   MFEM_ASSERT(t_layout != NULL, "Internal error");
+   if (pmesh->HasEngine())
+   {
+      t_layout = pmesh->GetEngine().MakeLayout(ltdof_size);
+   }
+   else
+   {
+      t_layout.Reset(new PLayout(ltdof_size));
+   }
+#endif
 }
 
 void ParFiniteElementSpace::GetGroupComm(
@@ -482,6 +505,14 @@ void ParFiniteElementSpace::GenerateGlobalOffsets() const
       delete [] statuses;
       delete [] requests;
    }
+
+#ifdef MFEM_USE_BACKENDS
+   if (pmesh->HasEngine())
+   {
+      dof_offsets.Push();
+      tdof_offsets.Push();
+   }
+#endif
 }
 
 void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
@@ -493,15 +524,15 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
    int ldof  = GetVSize();
    int ltdof = TrueVSize();
 
-   HYPRE_Int *i_diag = new HYPRE_Int[ldof+1];
-   HYPRE_Int *j_diag = new HYPRE_Int[ltdof];
+   HYPRE_Int *i_diag = mfem_hypre_CTAlloc(HYPRE_Int, ldof+1);
+   HYPRE_Int *j_diag = mfem_hypre_CTAlloc(HYPRE_Int, ltdof);
    int diag_counter;
 
-   HYPRE_Int *i_offd = new HYPRE_Int[ldof+1];
-   HYPRE_Int *j_offd = new HYPRE_Int[ldof-ltdof];
+   HYPRE_Int *i_offd = mfem_hypre_CTAlloc(HYPRE_Int, ldof+1);
+   HYPRE_Int *j_offd = mfem_hypre_CTAlloc(HYPRE_Int, ldof-ltdof);
    int offd_counter;
 
-   HYPRE_Int *cmap   = new HYPRE_Int[ldof-ltdof];
+   HYPRE_Int *cmap   = mfem_hypre_CTAlloc(HYPRE_Int, ldof-ltdof);
 
    HYPRE_Int *col_starts = GetTrueDofOffsets();
    HYPRE_Int *row_starts = GetDofOffsets();
@@ -535,7 +566,12 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
       j_offd[cmap_j_offd[i].two] = i;
    }
 
-   P = new HypreParMatrix(MyComm, MyRank, NRanks, row_starts, col_starts,
+   ParFiniteElementSpace &this_pfes = const_cast<ParFiniteElementSpace&>(*this);
+
+   // FIXME: This needs ifdef guards
+   P = new HypreParMatrix(MyComm, MyRank, NRanks,
+                          *this_pfes.GetVLayout(), *this_pfes.GetTrueVLayout(),
+                          row_starts, col_starts,
                           i_diag, j_diag, i_offd, j_offd, cmap, offd_counter);
 
    SparseMatrix Pdiag;
@@ -644,6 +680,12 @@ void ParFiniteElementSpace::GetEssentialTrueDofs(const Array<int>
       if (bool(true_ess_dofs[i]) != bool(true_ess_dofs2[i])) { counter++; }
    }
    MFEM_VERIFY(counter == 0, "internal MFEM error: counter = " << counter);
+#endif
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      ess_tdof_list.SetEngine(dev_ext->GetEngine());
+   }
 #endif
    MarkerToList(true_ess_dofs, ess_tdof_list);
 }
@@ -2087,7 +2129,7 @@ HypreParMatrix* ParFiniteElementSpace
    }
 
    // create offd column mapping
-   HYPRE_Int *cmap = new HYPRE_Int[col_map.size()];
+   HYPRE_Int *cmap = mfem_hypre_CTAlloc(HYPRE_Int, col_map.size());
    int offd_col = 0;
    for (std::map<HYPRE_Int, int>::iterator
         it = col_map.begin(); it != col_map.end(); ++it)
@@ -2096,14 +2138,14 @@ HypreParMatrix* ParFiniteElementSpace
       it->second = offd_col++;
    }
 
-   HYPRE_Int *I_diag = new HYPRE_Int[vdim*local_rows + 1];
-   HYPRE_Int *I_offd = new HYPRE_Int[vdim*local_rows + 1];
+   HYPRE_Int *I_diag = mfem_hypre_CTAlloc(HYPRE_Int, vdim*local_rows + 1);
+   HYPRE_Int *I_offd = mfem_hypre_CTAlloc(HYPRE_Int, vdim*local_rows + 1);
 
-   HYPRE_Int *J_diag = new HYPRE_Int[nnz_diag];
-   HYPRE_Int *J_offd = new HYPRE_Int[nnz_offd];
+   HYPRE_Int *J_diag = mfem_hypre_CTAlloc(HYPRE_Int, nnz_diag);
+   HYPRE_Int *J_offd = mfem_hypre_CTAlloc(HYPRE_Int, nnz_offd);
 
-   double *A_diag = new double[nnz_diag];
-   double *A_offd = new double[nnz_offd];
+   double *A_diag = mfem_hypre_CTAlloc(HYPRE_Real, nnz_diag);
+   double *A_offd = mfem_hypre_CTAlloc(HYPRE_Real, nnz_offd);
 
    int vdim1 = bynodes ? vdim : 1;
    int vdim2 = bynodes ? 1 : vdim;
@@ -2154,7 +2196,7 @@ HypreParMatrix* ParFiniteElementSpace
 
 static HYPRE_Int* make_i_array(int nrows)
 {
-   HYPRE_Int *I = new HYPRE_Int[nrows+1];
+   HYPRE_Int *I = mfem_hypre_CTAlloc(HYPRE_Int, nrows+1);
    for (int i = 0; i <= nrows; i++) { I[i] = -1; }
    return I;
 }
@@ -2166,7 +2208,7 @@ static HYPRE_Int* make_j_array(HYPRE_Int* I, int nrows)
    {
       if (I[i] >= 0) { nnz++; }
    }
-   HYPRE_Int *J = new HYPRE_Int[nnz];
+   HYPRE_Int *J = mfem_hypre_CTAlloc(HYPRE_Int, nnz);
 
    I[nrows] = -1;
    for (int i = 0, k = 0; i <= nrows; i++)
@@ -2265,7 +2307,7 @@ ParFiniteElementSpace::RebalanceMatrix(int old_ndofs,
    }
    SortPairs<HYPRE_Int, int>(cmap_offd, offd_cols);
 
-   HYPRE_Int* cmap = new HYPRE_Int[offd_cols];
+   HYPRE_Int* cmap = mfem_hypre_CTAlloc(HYPRE_Int, offd_cols);
    for (int i = 0; i < offd_cols; i++)
    {
       cmap[i] = cmap_offd[i].one;
@@ -2461,7 +2503,7 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
    offd->SetWidth(col_map.size());
 
    // create offd column mapping for use by hypre
-   HYPRE_Int *cmap = new HYPRE_Int[offd->Width()];
+   HYPRE_Int *cmap = mfem_hypre_CTAlloc(HYPRE_Int, offd->Width());
    for (std::map<HYPRE_Int, int>::iterator
         it = col_map.begin(); it != col_map.end(); ++it)
    {