First commit defining an FCT Equation using the partial assembly backend.

- Gives a hierarchical way of building Integrator. From Matrix Free, to Partial Assembly, to Local Matrices, to Sparse Matrices.
- Attempt to make the backend generic so that CUDA kernels can be integrated soon.

INSTALL

@@ -28,7 +28,7 @@ without GNU make or CMake can be found at the end of this file.
 In addition to the native build systems, MFEM packages are also available in the
 following package managers:
 
-- Spack, https://github.com/LLNL/spack
+- Spack, https://github.com/spack/spack
 - OpenHPC, http://openhpc.community
 - Homebrew/Science, https://github.com/Homebrew/homebrew-science
 

backends/all.hpp

@@ -0,0 +1,31 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_ALL_HPP
+#define MFEM_BACKENDS_ALL_HPP
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_BACKENDS
+
+#include "base/backend.hpp"
+
+#ifdef MFEM_USE_OCCA
+#include "occa/backend.hpp"
+#endif
+
+#ifdef MFEM_USE_PA
+#include "partialassembly/backend.hpp"
+#endif
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_ALL_HPP

backends/base/array.hpp

@@ -0,0 +1,211 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_ARRAY_HPP
+#define MFEM_BACKENDS_BASE_ARRAY_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "layout.hpp"
+#include "utils.hpp"
+
+namespace mfem
+{
+
+/// Extension to the template class Array<T>
+class PArray : public RefCounted
+{
+protected:
+   /// Layout with shared ownership (smart pointer)
+   DLayout layout;
+
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new array (in @a *clone) of the same dynamic
+       type as this array using the same layout and ItemSize().
+
+       Set @a *clone to NULL if allocation fails.
+
+       If @a copy_data is true, the contents of this array is copied to the new
+       array; otherwise, the new array remains uninitialized.
+
+       If @a buffer is not NULL, return the array data of the newly created
+       object (in @a *buffer) , if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   virtual PArray *DoClone(bool copy_data, void **buffer,
+                           std::size_t item_size) const = 0;
+
+   /// Resize the array, reallocating its data if necessary.
+   /** If @a buffer is not NULL, return the array data (in @a *buffer), if it
+       is stored as a contiguous array on the host; otherwise, set @a *buffer to
+       NULL. Returns 0 on success and non-zero otherwise, e.g. if memory
+       allocation fails.
+
+       If the @a new_layout is not supported, a non-zero error code will be
+       returned.
+
+       The @a new_layout has to be valid, i.e. new_layout != NULL and
+       new_layout->HasEngine() == true.
+
+       @note If reallocation is performed, the previous content of the array is
+       NOT copied to the new location. */
+   virtual int DoResize(PLayout &new_layout, void **buffer,
+                        std::size_t item_size) = 0;
+
+   /** @brief Get access to the contents of the array in host memory, as a
+       contiguous array. */
+   /** If the array data is stored as a contiguous array in host memory, return
+       a pointer to it. Otherwise, copy the data to @a buffer (if @a buffer is
+       not NULL) and return @a buffer.
+       @note If not NULL, @a buffer is assumed to be of size greater than or
+       equal to Size(). */
+   virtual void *DoPullData(void *buffer, std::size_t item_size) = 0;
+
+   /** @brief Set all entries of the array to the (single) value pointed to by
+       @a value_ptr. */
+   virtual void DoFill(const void *value_ptr, std::size_t item_size) = 0;
+
+   /** @brief Set all Size() entries of the array from the given contiguous
+       array, @a src_buffer, on the host. */
+   virtual void DoPushData(const void *src_buffer, std::size_t item_size) = 0;
+
+   /// Copy the data from @a src to @a *this.
+   /** Both arrays must have the same dynamic type, layout, and item_size. */
+   virtual void DoAssign(const PArray &src, std::size_t item_size) = 0;
+
+   ///@}
+   // End: Virtual interface
+
+public:
+   /** @brief The @a layout parameter will be reference counted and therefore it
+       should be dynamically allocated. */
+   /** The @a layout must be valid in the sense that layout != NULL and
+       layout->HasEngine() == true. */
+   PArray(PLayout &p_layout)
+      : layout(&p_layout)
+   {
+      MFEM_ASSERT(layout && layout->HasEngine(), "invalid layout");
+   }
+
+   virtual ~PArray() { }
+
+   /// Get the current size of the array.
+   std::size_t Size() const { return layout->Size(); }
+
+   /// Get the current layout of the array.
+   PLayout &GetLayout() const { return *layout; }
+
+   /// TODO
+   template <typename derived_t>
+   derived_t &As() { return dynamic_cast<derived_t&>(*this); }
+
+   /// TODO
+   template <typename derived_t>
+   const derived_t &As() const { return dynamic_cast<const derived_t&>(*this); }
+
+
+   // TODO: Error handling ... handle errors at the Engine level, at the class
+   //       level, or at the method level?
+
+   // TODO: Asynchronous execution interface ...
+
+
+   /**
+       @name Public virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new array (in @a *clone) of the same dynamic
+       type as this array using the same layout and ItemSize().
+
+       Set @a *clone to NULL if allocation fails.
+
+       If @a copy_data is true, the contents of this array is copied to the new
+       array; otherwise, the new array remains uninitialized.
+
+       If @a buffer is not NULL, return the array data of the newly created
+       object (in @a *buffer) , if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   template <typename T>
+   DArray Clone(bool copy_data, T **buffer) const
+   { return DArray(DoClone(copy_data, (void**)buffer, sizeof(T))); }
+
+   /// Resize the array, reallocating its data if necessary.
+   /** If @a buffer is not NULL, return the array data (in @a *buffer), if it
+       is stored as a contiguous array on the host; otherwise, set @a *buffer to
+       NULL. Returns 0 on success and non-zero otherwise, e.g. if memory
+       allocation fails.
+
+       If the @a new_layout is not supported, a non-zero error code will be
+       returned.
+
+       The @a new_layout has to be valid, i.e. new_layout != NULL and
+       new_layout->HasEngine() == true.
+
+       @note If reallocation is performed, the previous content of the array is
+       NOT copied to the new location. */
+   template <typename T>
+   int Resize(PLayout &new_layout, T **buffer)
+   { return DoResize(new_layout, (void**)buffer, sizeof(T)); }
+
+   /// Shortcut for Resize(*layout, buffer).
+   /** This method is useful for updating the array after its layout is changed
+       externally. */
+   template <typename T>
+   int Update(T **buffer)
+   { return DoResize(*layout, (void**)buffer, sizeof(T)); }
+
+   /// Shortcut for layout->Resize(new_size) followed by Update()
+   template <typename T>
+   int Resize(std::size_t new_size, T **buffer)
+   { layout->Resize(new_size); return Update(buffer); }
+
+   /** @brief Get access to the contents of the array in host memory, as a
+       contiguous array. */
+   /** If the array data is stored as a contiguous array in host memory, return
+       a pointer to it. Otherwise, copy the data to @a buffer (if @a buffer is
+       not NULL) and return @a buffer.
+       @note If not NULL, @a buffer is assumed to be of size greater than or
+       equal to Size(). */
+   template <typename T>
+   T *PullData(T *buffer)
+   { return Size() ? (T*)DoPullData((void*)buffer, sizeof(T)) : NULL; }
+
+   /** @brief Set all entries of the array to the (single) value pointed to by
+       @a value_ptr. */
+   template <typename T>
+   void Fill(const T &value) { if (Size()) { DoFill(&value, sizeof(T)); } }
+
+   /** @brief Set all Size() entries of the array from the given contiguous
+       array, @a src_buffer, on the host. */
+   template <typename T>
+   void PushData(const T *src_buffer)
+   { if (Size()) { DoPushData(src_buffer, sizeof(T)); } }
+
+   /// Copy the data from @a src to @a *this.
+   /** Both arrays must have the same dynamic type, layout, and entry type. */
+   template <typename T>
+   void Assign(const PArray &src) { if (Size()) { DoAssign(src, sizeof(T)); } }
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_ARRAY_HPP

backends/base/backend.hpp

@@ -0,0 +1,57 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_BACKEND_HPP
+#define MFEM_BACKENDS_BASE_BACKEND_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "memory_resource.hpp"
+#include "engine.hpp"
+#include "array.hpp"
+#include "vector.hpp"
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+
+#include <string>
+
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#endif
+
+namespace mfem
+{
+
+/// TODO
+class Backend
+{
+public:
+   /// TODO
+   virtual ~Backend() { }
+
+   /// TODO
+   virtual bool Supports(const std::string &engine_spec) const = 0;
+
+   /// TODO
+   virtual Engine *Create(const std::string &engine_spec) = 0;
+
+#ifdef MFEM_USE_MPI
+   /// TODO
+   virtual Engine *Create(MPI_Comm comm, const std::string &engine_spec) = 0;
+#endif
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_BACKEND_HPP

backends/base/bilinearform.hpp

@@ -0,0 +1,72 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_BILINEARFORM_HPP
+#define MFEM_BACKENDS_BASE_BILINEARFORM_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "engine.hpp"
+
+namespace mfem
+{
+
+class Vector;
+class OperatorHandle;
+class BilinearForm;
+
+/// TODO: doxygen
+class PBilinearForm : public RefCounted
+{
+protected:
+   /// Engine with shared ownership
+   SharedPtr<const Engine> engine;
+   /// Not owned.
+   BilinearForm *bform;
+
+public:
+   /// TODO: doxygen
+   PBilinearForm(const Engine &e, BilinearForm &bf)
+      : engine(&e), bform(&bf) { }
+
+   /// Virtual destructor
+   virtual ~PBilinearForm() { }
+
+   /// Get the associated Engine
+   const Engine &GetEngine() const { return *engine; }
+
+   /// Assemble the PBilinearForm.
+   /** This method is called from the method BilinearForm::Assemble() of the
+       associated BilinearForm #bform.
+       @returns True, if the host assembly should be skipped. */
+   virtual bool Assemble() = 0;
+
+   /// TODO: doxygen
+   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                 OperatorHandle &A) = 0;
+
+   /// TODO: doxygen
+   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
+                                 Vector &x, Vector &b,
+                                 OperatorHandle &A, Vector &X, Vector &B,
+                                 int copy_interior) = 0;
+
+   /// TODO: doxygen
+   virtual void RecoverFEMSolution(const Vector &X, const Vector &b,
+                                   Vector &x) = 0;
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_BILINEARFORM_HPP

backends/base/engine.cpp

@@ -0,0 +1,29 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "engine.hpp"
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+
+namespace mfem
+{
+
+DFiniteElementSpace Engine::MakeFESpace(FiniteElementSpace &fes) const
+{
+   return DFiniteElementSpace(new PFiniteElementSpace(*this, fes));
+}
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS

backends/base/engine.hpp

@@ -0,0 +1,209 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_ENGINE_HPP
+#define MFEM_BACKENDS_BASE_ENGINE_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "../../general/scalars.hpp"
+#include "memory_resource.hpp"
+#include "smart_pointers.hpp"
+#include "utils.hpp"
+
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#endif
+
+namespace mfem
+{
+
+// Forward declarations.
+class Backend;
+template <typename T> class Array;
+class Operator;
+class FiniteElementSpace;
+class LinearForm;
+class BilinearForm;
+class MixedBilinearForm;
+class NonlinearForm;
+
+
+/// In parallel, each MPI rank will usually create a single engine.
+class Engine : public RefCounted
+{
+protected:
+   Backend *backend; ///< Backend that created the engine. Not owned.
+
+#ifdef MFEM_USE_MPI
+   MPI_Comm comm; ///< Associated MPI communicator (may be MPI_COMM_NULL).
+#endif
+
+   /// Number of memory resources used by the Engine.
+   int num_mem_res;
+   /// Number of workers used by the Engine.
+   int num_workers;
+
+   /// Memory resources used by the engine - array of pointers.
+   /** Both the array and the entries are owned. */
+   MemoryResource **memory_resources;
+
+   /// Relative computational speed of the workers. Owned.
+   double *workers_weights;
+
+   /// For each worker, which memory resource it uses.
+   int *workers_mem_res;
+
+public:
+   /// TODO: doxygen
+   Engine(Backend *b, int n_mem, int n_workers)
+      : backend(b),
+#ifdef MFEM_USE_MPI
+        comm(MPI_COMM_NULL),
+#endif
+        num_mem_res(n_mem),
+        num_workers(n_workers),
+        memory_resources(new MemoryResource*[num_mem_res]()),
+        workers_weights(new double[num_workers]()),
+        workers_mem_res(new int[num_workers]())
+   { /* Note: all arrays are value-initialized with zeros. */ }
+
+   /// TODO: doxygen
+   virtual ~Engine()
+   {
+      delete [] workers_mem_res;
+      delete [] workers_weights;
+      for (int i = 0; i < num_mem_res; i++)
+      {
+         delete memory_resources[i];
+      }
+      delete [] memory_resources;
+   }
+
+
+   /**
+       @name Machine resources interface
+    */
+   ///@{
+
+#ifdef MFEM_USE_MPI
+   /// Get the associated MPI_Comm
+   MPI_Comm GetComm() const { return comm; }
+#endif
+
+   /// TODO
+   int GetNumMemRes() const { return num_mem_res; }
+
+   /// TODO
+   MemoryResource &GetMemRes(int idx) const { return *memory_resources[idx]; }
+
+   /// TODO
+   int GetNumWorkers() const { return num_workers; }
+
+   /// TODO
+   const double *GetWorkersWeights() const { return workers_weights; }
+
+   /// TODO
+   const int *GetWorkersMemRes() const { return workers_mem_res; }
+
+   ///@}
+   // End: Machine resources interface
+
+   /// TODO
+   template <typename derived_t>
+   derived_t &As() { *util::As<derived_t>(this); }
+
+   /// TODO
+   template <typename derived_t>
+   const derived_t &As() const { *util::As<const derived_t>(this); }
+
+
+   // TODO: Error handling ... handle errors at the Engine level, at the class
+   //       level, or at the method level?
+
+
+   /**
+       @name Virtual interface: finite element data structures and algorithms
+    */
+   ///@{
+
+   // TODO: Asynchronous execution in this class ...
+
+   /// Allocate and return a new layout for the given @a size.
+   /** The layout decomposition is determined automatically by the Engine using
+       a deterministic algorithm: calls to this method with the same @a size
+       will produce the same result, as long as the Engine remains unmodified
+       between the calls.
+
+       The returned object is allocated with operator new and must be
+       deallocated by the caller.
+
+       TODO: Returns NULL if memory allocation fails?
+   */
+   virtual DLayout MakeLayout(std::size_t size) const = 0;
+
+   /// Allocate and return a new layout for the given worker decomposition.
+   /** The returned object is allocated with operator new and must be
+       deallocated by the caller.
+
+       TODO: Returns NULL if memory allocation fails?
+
+       The @a offsets should satisfy: offsets.Size() == number of workers + 1,
+       offsets[0] == 0, and offsets[i] <= offsets[i+1], for i: 0 <= i < number
+       of workers. */
+   virtual DLayout MakeLayout(const Array<std::size_t> &offsets) const = 0;
+
+   // Note: There may be other ways to construct layouts in the future, e.g.
+   //       block-vector layouts, or multi-vector layouts.
+
+   /// TODO
+   virtual DArray MakeArray(PLayout &layout, std::size_t item_size) const = 0;
+
+   /// Allocate and return a new vector using the given @a layout.
+   /** The returned object is a smart pointer that will automatically deallocate
+       the vector.
+
+       TODO: Produce an error if memory allocation fails?
+
+       Only layouts returned by this Engine are guaranteed to be supported.
+       Using a type that is not supported will produce an error. */
+   virtual DVector MakeVector(PLayout &layout,
+                              int type_id = ScalarId<double>::value) const = 0;
+
+   /// TODO: doxygen
+   virtual DFiniteElementSpace MakeFESpace(FiniteElementSpace &fes) const;
+
+   /// TODO: doxygen
+   virtual DBilinearForm MakeBilinearForm(BilinearForm &bf) const = 0;
+
+
+   // Question: How do we construct coefficients?
+
+
+   /// FIXME - What will the actual parameters be?
+   virtual void AssembleLinearForm(LinearForm &l_form) const = 0;
+
+   /// FIXME - What will the actual parameters be?
+   virtual Operator *MakeOperator(const MixedBilinearForm &mbl_form) const = 0;
+
+   /// FIXME - What will the actual parameters be?
+   virtual Operator *MakeOperator(const NonlinearForm &nl_form) const = 0;
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_ENGINE_HPP

backends/base/fespace.hpp

@@ -0,0 +1,61 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_FE_SPACE_HPP
+#define MFEM_BACKENDS_BASE_FE_SPACE_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "engine.hpp"
+#include "utils.hpp"
+
+namespace mfem
+{
+
+class FiniteElementSpace;
+
+/// TODO: doxygen
+class PFiniteElementSpace : public RefCounted
+{
+protected:
+   /// Engine with shared ownership
+   SharedPtr<const Engine> engine;
+   /// Not owned.
+   FiniteElementSpace *fes;
+
+public:
+   /// TODO: doxygen
+   PFiniteElementSpace(const Engine &e, FiniteElementSpace &fespace)
+      : engine(&e), fes(&fespace) { }
+
+   /// Virtual destructor
+   virtual ~PFiniteElementSpace() { }
+
+   /// Get the associated engine
+   const Engine &GetEngine() const { return *engine; }
+
+   mfem::FiniteElementSpace* GetFESpace() const { return fes; }
+
+   /// TODO
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   /// TODO
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_FE_SPACE_HPP

backends/base/layout.hpp

@@ -0,0 +1,104 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_LAYOUT_HPP
+#define MFEM_BACKENDS_BASE_LAYOUT_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "smart_pointers.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+/// Polymorphic layout (array/vector layout descriptor)
+class PLayout : public RefCounted
+{
+protected:
+   /// Engine with shared ownership
+   SharedPtr<const Engine> engine;
+   std::size_t size;
+
+   template <typename DObject>
+   struct Maker
+   {
+      template <typename entry_t>
+      static DObject MakeNew(PLayout &layout);
+   };
+
+public:
+   explicit PLayout(std::size_t s = 0) : engine(NULL), size(s) { }
+
+   explicit PLayout(const Engine &e, std::size_t s = 0)
+      : engine(&e), size(s) { }
+
+   virtual ~PLayout() { }
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   /// Resize the layout
+   virtual void Resize(std::size_t new_size) { size = new_size; }
+
+   /// Resize the layout based on the given worker offsets
+   virtual void Resize(const Array<std::size_t> &offsets)
+   { MFEM_ABORT("method not supported"); }
+
+   ///@}
+   // End: Virtual interface
+
+   /// Layouts without engine cannot create DArray, DVector, etc.
+   bool HasEngine() const { return engine != NULL; }
+
+   /// TODO: doxygen
+   const Engine &GetEngine() const { return *engine; }
+
+   /// TODO: doxygen
+   std::size_t Size() const { return size; }
+
+   /// TODO
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   /// TODO
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+
+   /// TODO: doxygen
+   template <typename DObject, typename entry_t>
+   DObject Make()
+   {
+      MFEM_ASSERT(HasEngine(), "this method requires an Engine");
+      return Maker<DObject>::template MakeNew<entry_t>(*this);
+   }
+};
+
+template <> struct PLayout::Maker<DArray>
+{
+   template <typename entry_t> static DArray MakeNew(PLayout &layout)
+   { return layout.GetEngine().MakeArray(layout, sizeof(entry_t)); }
+};
+
+template <> struct PLayout::Maker<DVector>
+{
+   template <typename entry_t> static DVector MakeNew(PLayout &layout)
+   { return layout.GetEngine().MakeVector(layout, ScalarId<entry_t>::value); }
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_LAYOUT_HPP

backends/base/memory_resource.cpp

@@ -0,0 +1,59 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "memory_resource.hpp"
+#include "../../general/error.hpp"
+
+#include <cstdlib>
+#include <cstring>
+#include <cerrno>
+
+namespace mfem
+{
+
+void *NewDeleteMemoryResource::DoAllocate(std::size_t bytes,
+                                          std::size_t alignment)
+{
+   void *p = ::operator new[](bytes);
+   MFEM_VERIFY(!alignment || (std::size_t)(p) % alignment == 0,
+               "invalid alignment");
+   return p;
+}
+
+void NewDeleteMemoryResource::DoDeallocate(void *p, std::size_t bytes,
+                                           std::size_t alignment)
+{
+   ::operator delete[](p);
+}
+
+
+void *AlignedMemoryResource::DoAllocate(std::size_t bytes,
+                                        std::size_t alignment)
+{
+   void *p;
+   if (!alignment) { alignment = sizeof(long double); }
+   MFEM_VERIFY(posix_memalign(&p, alignment, bytes) == 0,
+               "error in posix_memalign(): " << strerror(errno));
+   return p;
+}
+
+void AlignedMemoryResource::DoDeallocate(void *p, std::size_t bytes,
+                                         std::size_t alignment)
+{
+   free(p);
+}
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS

backends/base/memory_resource.hpp

@@ -0,0 +1,70 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP
+#define MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include <cstddef>
+
+namespace mfem
+{
+
+/// Polymorphic memory resource. Similar to C++17's std::pmr::memory_resource.
+class MemoryResource
+{
+protected:
+   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment) = 0;
+   virtual void DoDeallocate(void* p, std::size_t bytes,
+                             std::size_t alignment) = 0;
+
+public:
+   // Implicitly defined default & copy constructors
+
+   /// Virtual destructor.
+   virtual ~MemoryResource() { }
+
+   /// If alignment == 0, use default alignment.
+   void *Allocate(std::size_t bytes, std::size_t alignment = 0)
+   { return DoAllocate(bytes, alignment); }
+
+   /// If alignment == 0, use default alignment.
+   void Deallocate(void *p, std::size_t bytes, std::size_t alignment = 0)
+   { DoDeallocate(p, bytes, alignment); }
+};
+
+
+/** @brief Dynamic host memory resource using operator new[](std::size_t) for
+    allocation and operator delete[](void*) for deallocation. */
+class NewDeleteMemoryResource : public MemoryResource
+{
+protected:
+   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment);
+   virtual void DoDeallocate(void *p, std::size_t bytes, std::size_t alignment);
+};
+
+
+/** @brief Dynamic host memory resource using posix_memalign() for aligned
+    allocation and free() for deallocation. */
+class AlignedMemoryResource : public MemoryResource
+{
+protected:
+   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment);
+   virtual void DoDeallocate(void *p, std::size_t bytes, std::size_t alignment);
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP

backends/base/smart_pointers.hpp

@@ -0,0 +1,233 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_SMART_POINTERS_HPP
+#define MFEM_BACKENDS_BASE_SMART_POINTERS_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "utils.hpp"
+#include "../../general/error.hpp"
+#include <cstddef>
+
+// #define MFEM_TRACE_SHARED_PTR
+#ifdef MFEM_TRACE_SHARED_PTR
+#include "../../general/globals.hpp"
+#endif
+
+namespace mfem
+{
+
+/// Base class for classes with simple reference counting.
+/** Reference counting is performed by the class SharedPtr. */
+class RefCounted
+{
+private:
+   mutable unsigned ref_count;
+
+   /// Only class SharedPtr can access ref_count.
+   template <typename T> friend class SharedPtr;
+
+public:
+   RefCounted() : ref_count(0) { }
+
+   /** @brief Prevent SharedPtr objects from deleting this object by
+       incrementing the reference counter by one. */
+   void DontDelete() const { ++ref_count; }
+};
+
+
+/** @brief Smart pointer class that manages objects of type T derived from class
+    RefCounted. */
+/** This class is generally meant to work with dynamically allocated object,
+    specifically objects allocated with operator new(). It will invoke operator
+    delete() to destroy the managed object when its reference counter reaches
+    zero. This behavior can be overriden by calling RefCounted::DontDelete() to
+    ensure that an object will not be deleted by a SharedPtr that holds a
+    pointer to it.
+    @note This class is NOT thread-safe and does not support circular ownership.
+ */
+template <typename T>
+class SharedPtr
+{
+public:
+   typedef T stored_type;
+
+private:
+   T *ptr;
+
+   void Init(T *new_ptr)
+   {
+      ptr = new_ptr;
+      if (ptr) { ++ptr->RefCounted::ref_count; }
+#ifdef MFEM_TRACE_SHARED_PTR
+#elif 0
+      mfem::out << "  [" << _MFEM_FUNC_NAME << "]: ptr = " << ptr;
+      if (ptr)
+      {
+         mfem::out << ", new ref_count = " << ptr->RefCounted::ref_count;
+      }
+      mfem::out << '\n';
+#endif
+   }
+   void Destroy()
+   {
+      MFEM_ASSERT(!ptr || ptr->RefCounted::ref_count >= 1, "invalid use");
+      if (ptr && --ptr->RefCounted::ref_count == 0) { delete ptr; }
+#ifdef MFEM_TRACE_SHARED_PTR
+#elif 0
+      mfem::out << "  [" << _MFEM_FUNC_NAME << "]: ptr = " << ptr;
+      if (ptr)
+      {
+         mfem::out << ", new ref_count = " << ptr->RefCounted::ref_count;
+      }
+      mfem::out << '\n';
+#endif
+   }
+
+public:
+   SharedPtr() : ptr(NULL)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]: ptr = " << ptr << '\n';
+#endif
+   }
+
+   SharedPtr(const SharedPtr &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Init(other.ptr);
+   }
+
+   template <typename U>
+   SharedPtr(const SharedPtr<U> &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Init(other.Get());
+   }
+
+   explicit SharedPtr(T *p)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Init(p);
+   }
+
+   ~SharedPtr()
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Destroy();
+   }
+
+   SharedPtr &operator=(const SharedPtr &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Reset(other.ptr); return *this;
+   }
+
+   template <typename U>
+   SharedPtr &operator=(const SharedPtr<U> &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Reset(other.Get()); return *this;
+   }
+
+   T &operator*() const { return *ptr; }
+   T *operator->() const { return ptr; }
+
+   operator bool() const { return ptr; }
+   bool operator!() const { return !ptr; }
+
+   template <typename U>
+   bool operator==(const SharedPtr<U> &other) const
+   { return ptr == other.Ptr(); }
+   template <typename U>
+   bool operator!=(const SharedPtr<U> &other) const
+   { return ptr != other.Ptr(); }
+
+   template <typename U>
+   bool operator==(const U &p) const { return ptr == (void*) p; }
+   template <typename U>
+   bool operator!=(const U &p) const { return ptr != (void*) p; }
+
+   T *Get() const { return ptr; }
+
+   /// TODO
+   template <typename derived_t>
+   derived_t *As() const { return util::As<derived_t>(ptr); }
+
+   unsigned UseCount() const { return ptr ? ptr->RefCounted::ref_count : 0; }
+
+   void Reset()
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Destroy();
+      ptr = NULL;
+   }
+
+   /// The type U* needs to be implicitly convertible to T*
+   template <typename U>
+   void Reset(U *new_ptr)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      if (ptr != new_ptr) { Destroy(); Init(new_ptr); }
+   }
+
+   void Swap(SharedPtr &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      std::swap(ptr, other.ptr);
+   }
+};
+
+
+template <class T>
+inline void Swap(SharedPtr<T> &a, SharedPtr<T> &b) { a.Swap(b); }
+
+
+class PLayout;
+typedef SharedPtr<PLayout> DLayout;
+
+class PArray;
+typedef SharedPtr<PArray> DArray;
+
+class PVector;
+typedef SharedPtr<PVector> DVector;
+
+class PFiniteElementSpace;
+typedef SharedPtr<PFiniteElementSpace> DFiniteElementSpace;
+
+class PBilinearForm;
+typedef SharedPtr<PBilinearForm> DBilinearForm;
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_SMART_POINTERS_HPP

backends/base/utils.hpp

@@ -0,0 +1,52 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_UTILS_HPP
+#define MFEM_BACKENDS_BASE_UTILS_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "../../general/error.hpp"
+
+namespace mfem
+{
+
+namespace util
+{
+
+//
+// Inline methods
+//
+
+/// TODO: doxygen
+template <typename derived_t, typename base_t>
+inline derived_t *As(base_t *base_obj)
+{
+   MFEM_ASSERT(dynamic_cast<derived_t*>(base_obj) != NULL,
+               "invalid object type");
+   return static_cast<derived_t*>(base_obj);
+}
+
+/// TODO: doxygen
+template <typename derived_t, typename base_t>
+inline derived_t *Is(base_t *base_obj)
+{
+   return dynamic_cast<derived_t*>(base_obj);
+}
+
+} // namespace mfem::util
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_UTILS_HPP

backends/base/vector.hpp

@@ -0,0 +1,153 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_VECTOR_HPP
+#define MFEM_BACKENDS_BASE_VECTOR_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "../../general/scalars.hpp"
+#include "array.hpp"
+
+namespace mfem
+{
+
+/// Polymorphic vector - array of scalars.
+class PVector : virtual public PArray
+{
+protected:
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new vector of the same dynamic type as this
+       vector using the same layout with entries specified by @a buffer_type_id
+       which should be a constant defined by the `value` field in a
+       specialization of the template class mfem::ScalarId.
+
+       Returns NULL if allocation fails.
+
+       If @a copy_data is true, the contents of this vector is copied to the new
+       vector; otherwise, the new vector remains uninitialized.
+
+       If @a buffer is not NULL, return the vector data of the newly created
+       object (in @a *buffer), if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   virtual PVector *DoVectorClone(bool copy_data, void **buffer,
+                                  int buffer_type_id) const = 0;
+
+   /** @brief Compute and return the dot product of @a *this and @a x. In the
+       case of an MPI-parallel vector, the result must be the MPI-global dot
+       product. */
+   /** Both vectors must have the same dynamic type and layout. */
+   virtual void DoDotProduct(const PVector &x, void *result,
+                             int result_type_id) const = 0;
+
+   // TODO: add reduction operations: min, max, sum
+
+   /// Perform the operation @a *this = @a a @a x + @a b @a y.
+   /** Rules:
+       - the dynamic type of both @a x and @a y is the same as that of @a *this
+       - if @a a == 0, neither @a x nor its data are accessed
+       - if @a b == 0, neither @a y nor its data are accessed
+       - @a x's data is never the same as @a y's data, unless @a a == 0, or
+         @a b == 0
+       - @a x's data or @a y's data may be the same as the data of @a *this
+       - all accessed vectors, @a x, @a y, and @a *this have the same layout. */
+   virtual void DoAxpby(const void *a, const PVector &x,
+                        const void *b, const PVector &y,
+                        int ab_type_id) = 0;
+
+   ///@}
+   // End: Virtual interface
+
+public:
+   /** @brief Create a PVector. */
+   /** The @a layout must be valid in the sense that layout != NULL and
+       layout->HasEngine() == true. */
+   PVector(PLayout &p_layout)
+      : PArray(p_layout) { }
+
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+
+
+   // TODO: Error handling ... handle errors at the Engine level, at the class
+   //       level, or at the method level?
+
+   // TODO: Asynchronous execution interface ...
+
+   // TODO: Multi-vector interface ...
+
+
+   /**
+       @name Public virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new vector of the same dynamic type as this
+       vector using the same layout with entries of type @a scalar_t.
+
+       If @a copy_data is true, the contents of this vector is copied to the new
+       vector; otherwise, the new vector remains uninitialized.
+
+       If @a buffer is not NULL, return the vector data of the newly created
+       object (in @a *buffer) , if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   template <typename scalar_t>
+   DVector Clone(bool copy_data, scalar_t **buffer) const
+   {
+      return DVector(DoVectorClone(copy_data, (void**)buffer,
+                                   ScalarId<scalar_t>::value));
+   }
+
+   /** @brief Compute and return the dot product of @a *this and @a x. In the
+       case of an MPI-parallel vector, the result must be the MPI-global dot
+       product. */
+   /** Both vectors must have the same dynamic type and layout. */
+   template <typename scalar_t>
+   scalar_t DotProduct(const PVector &x) const
+   {
+      scalar_t result;
+      DoDotProduct(x, &result, ScalarId<scalar_t>::value);
+      return result;
+   }
+
+   // TODO: add reduction operations: min, max, sum
+
+   /// Perform the operation @a *this = @a a @a x + @a b @a y.
+   /** Rules:
+       - the dynamic type of both @a x and @a y is the same as that of @a *this
+       - if @a a == 0, neither @a x nor its data are accessed
+       - if @a b == 0, neither @a y nor its data are accessed
+       - @a x's data is never the same as @a y's data, unless @a a == 0, or
+         @a b == 0
+       - @a x's data or @a y's data may be the same as the data of @a *this
+       - all accessed vectors, @a x, @a y, and @a *this have the same layout. */
+   template <typename scalar_t>
+   void Axpby(const scalar_t &a, const PVector &x,
+              const scalar_t &b, const PVector &y)
+   { if (Size()) { DoAxpby(&a, x, &b, y, ScalarId<scalar_t>::value); } }
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_VECTOR_HPP

backends/occa/DiffusionIntegrator.okl

@@ -0,0 +1,66 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  ELEMENT_BATCH   : How many elements are in each
+  .                   computation batch
+  NUM_DOFS_1D     : Dofs in the 1D segments
+  NUM_DOFS_2D     : Dofs in the 2D faces
+  NUM_DOFS_3D     : Dofs in the 3D domain
+  NUM_QUAD_1D     : Dofs in the 1D segments
+  NUM_QUAD_2D     : Dofs in the 2D faces
+  NUM_QUAD_3D     : Dofs in the 3D domain
+  NUM_MAX_1D      : max(NUM_QUAD_1D, NUM_DOFS_1D)
+  NUM_QUAD_DOFS_1D: NUM_QUAD_1D * NUM_DOFS_1D
+  COEFF_ARGS      : Code that passes required arguments to the kernel
+  COEFF           : Code that computes the coefficient
+  ================================================
+
+  [MISSING]
+  - Add support to auto-pick @dim and use @idxOrder on stack arrays
+  |   double a[2][2];
+  |   a[0][1]; <-- regular index
+  |   a(0,1);  <-- uses @idxOrder a[0][1] or a[1][0]
+  - Add support for @idxOrder to change indexing order after allocation
+  |   double a[2][2] @idxOrder(0,1);
+  |   a(0,1) -> a[1][0]
+  |   @set(a, idxOrder(1,0));
+  |   a(0,1) -> a[0][1]
+  - Add support to iterate over loop depending on mode
+  |   for(i; @inner) {
+  |     for(0 < j < N) {} <-- ++j or j += block?
+  |   }
+*/
+
+#include "mfem-occa://defines.okl"
+
+#if USING_TENSOR_OPS
+#  ifdef OCCA_USING_GPU
+#    if USING_LOW_ORDER
+#      include "mfem-occa://diffusion/tensor/gpuHighOrder.okl"
+#    else
+#      include "mfem-occa://diffusion/tensor/gpuHighOrder.okl"
+#    endif
+#  else
+#    include "mfem-occa://diffusion/tensor/cpu.okl"
+#  endif
+#else
+#  ifdef OCCA_USING_GPU
+#    if USING_LOW_ORDER
+#      include "mfem-occa://diffusion/simplex/gpuHighOrder.okl"
+#    else
+#      include "mfem-occa://diffusion/simplex/gpuHighOrder.okl"
+#    endif
+#  else
+#    include "mfem-occa://diffusion/simplex/cpu.okl"
+#  endif
+#endif

backends/occa/array.cpp

@@ -0,0 +1,123 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+PArray *Array::DoClone(bool copy_data, void **buffer,
+                       std::size_t item_size) const
+{
+   Array *new_array = new Array(OccaLayout(), item_size);
+   if (copy_data)
+   {
+      new_array->slice.copyFrom(slice);
+   }
+   if (buffer)
+   {
+      *buffer = new_array->GetBuffer();
+   }
+   return new_array;
+}
+
+int Array::DoResize(PLayout &new_layout, void **buffer,
+                    std::size_t item_size)
+{
+   MFEM_ASSERT(dynamic_cast<Layout *>(&new_layout) != NULL,
+               "new_layout is not an OCCA Layout");
+   Layout *lt = static_cast<Layout *>(&new_layout);
+   layout.Reset(lt); // Reset() checks if the pointer is the same
+   int err = ResizeData(lt, item_size);
+   if (!err && buffer)
+   {
+      *buffer = GetBuffer();
+   }
+   return err;
+}
+
+void *Array::DoPullData(void *buffer, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   if (!slice.getDevice().hasSeparateMemorySpace())
+   {
+      return slice.ptr();
+   }
+   if (buffer)
+   {
+      slice.copyTo(buffer);
+   }
+   return buffer;
+}
+
+void Array::DoFill(const void *value_ptr, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   switch (item_size)
+   {
+      case sizeof(int8_t):
+         OccaFill((const int8_t *)value_ptr);
+         break;
+      case sizeof(int16_t):
+         OccaFill((const int16_t *)value_ptr);
+         break;
+      case sizeof(int32_t):
+         OccaFill((const int32_t *)value_ptr);
+         break;
+      // case sizeof(int64_t):
+      //    OccaFill((const int64_t *)value_ptr);
+      //    break;
+      case sizeof(double):
+         OccaFill((const double *)value_ptr);
+         break;
+      // case sizeof(::occa::double2):
+      //    OccaFill((const ::occa::double2 *)value_ptr);
+      //    break;
+      default:
+         MFEM_ABORT("item_size = " << item_size << " is not supported");
+   }
+}
+
+void Array::DoPushData(const void *src_buffer, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   if (slice.getDevice().hasSeparateMemorySpace() || slice.ptr() != src_buffer)
+   {
+      slice.copyFrom(src_buffer);
+   }
+}
+
+void Array::DoAssign(const PArray &src, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   // Note: static_cast can not be used here since PArray is a virtual base
+   //       class.
+   const Array *source = dynamic_cast<const Array *>(&src);
+   MFEM_ASSERT(source != NULL, "invalid source Array type");
+   MFEM_ASSERT(Size() == source->Size(), "");
+   slice.copyFrom(source->slice);
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/array.hpp

@@ -0,0 +1,131 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_ARRAY_HPP
+#define MFEM_BACKENDS_OCCA_ARRAY_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include <occa.hpp>
+#include "layout.hpp"
+#include "../base/array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Array : public virtual PArray
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // DLayout layout;
+
+   // Always true: Size()*item_size == slice.size() <= data.size()
+   mutable ::occa::memory data, slice;
+
+   //
+   // Virtual interface
+   //
+
+   virtual PArray *DoClone(bool copy_data, void **buffer,
+                           std::size_t item_size) const;
+
+   virtual int DoResize(PLayout &new_layout, void **buffer,
+                        std::size_t item_size);
+
+   virtual void *DoPullData(void *buffer, std::size_t item_size);
+
+   virtual void DoFill(const void *value_ptr, std::size_t item_size);
+
+   virtual void DoPushData(const void *src_buffer, std::size_t item_size);
+
+   virtual void DoAssign(const PArray &src, std::size_t item_size);
+
+   //
+   // Auxiliary methods
+   //
+
+   inline void *GetBuffer() const;
+
+   inline int ResizeData(const Layout *lt, std::size_t item_size);
+
+   template <typename T>
+   inline void OccaFill(const T *val_ptr)
+   { ::occa::linalg::operator_eq<T>(slice, *val_ptr); }
+
+public:
+   Array(Layout &lt, std::size_t item_size)
+      : PArray(lt),
+        data(lt.Alloc(lt.Size()*item_size)),
+        slice(data)
+   { }
+
+   virtual ~Array() { }
+
+   inline void MakeRef(Array &master);
+
+   Layout &OccaLayout() const
+   { return *static_cast<Layout *>(layout.Get()); }
+
+   ::occa::memory &OccaMem() { return slice; }
+   const ::occa::memory &OccaMem() const { return slice; }
+};
+
+
+//
+// Inline methods
+//
+
+inline void *Array::GetBuffer() const
+{
+   if (!slice.getDevice().hasSeparateMemorySpace())
+   {
+      return slice.ptr();
+   }
+   return NULL;
+}
+
+inline int Array::ResizeData(const Layout *lt, std::size_t item_size)
+{
+   const std::size_t new_bytes = lt->Size()*item_size;
+   if (data.size() < new_bytes ||
+       data.getDHandle() != lt->OccaEngine().GetDevice().getDHandle())
+   {
+      data = lt->Alloc(new_bytes);
+      slice = data;
+      // If memory allocation fails - an exception is thrown.
+   }
+   else if (slice.size() != new_bytes)
+   {
+      slice = data.slice(0, new_bytes);
+   }
+   return 0;
+}
+
+inline void Array::MakeRef(Array &master)
+{
+   layout = master.layout;
+   data = master.data;
+   slice = master.slice;
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_ARRAY_HPP

backends/occa/backend.cpp

@@ -0,0 +1,47 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "backend.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+bool Backend::Supports(const std::string &engine_spec) const
+{
+   // TODO: check if 'engine_spec' is valid OCCA string.
+   return true;
+}
+
+mfem::Engine *Create(const std::string &engine_spec)
+{
+   return new Engine(engine_spec);
+}
+
+#ifdef MFEM_USE_MPI
+mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec)
+{
+   return new Engine(comm, engine_spec);
+}
+#endif
+
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/backend.hpp

@@ -0,0 +1,49 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_BACKEND_HPP
+#define MFEM_BACKENDS_OCCA_BACKEND_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+// Only the Backend and Engine classes should be exposed through "backend.hpp"
+#include "../base/backend.hpp"
+#include "engine.hpp"
+#include <occa.hpp>
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Backend : public mfem::Backend
+{
+public:
+   virtual ~Backend();
+
+   virtual bool Supports(const std::string &engine_spec) const;
+
+   virtual mfem::Engine *Create(const std::string &engine_spec);
+
+#ifdef MFEM_USE_MPI
+   virtual mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec);
+#endif
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_BACKEND_HPP

backends/occa/bilinearform.cpp

@@ -0,0 +1,514 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "backend.hpp"
+#include "bilininteg.hpp"
+#include "../../fem/bilinearform.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+OccaBilinearForm::OccaBilinearForm(FiniteElementSpace *ofespace_) :
+   Operator(ofespace_->OccaVLayout()),
+   localX(ofespace_->OccaEVLayout()),
+   localY(ofespace_->OccaEVLayout())
+{
+   Init(ofespace_->OccaEngine(), ofespace_, ofespace_);
+}
+
+OccaBilinearForm::OccaBilinearForm(FiniteElementSpace *otrialFESpace_,
+                                   FiniteElementSpace *otestFESpace_) :
+   Operator(otrialFESpace_->OccaVLayout(),
+            otestFESpace_->OccaVLayout()),
+   localX(otrialFESpace_->OccaEVLayout()),
+   localY(otestFESpace_->OccaEVLayout())
+{
+   Init(otrialFESpace_->OccaEngine(), otrialFESpace_, otestFESpace_);
+}
+
+void OccaBilinearForm::Init(const Engine &e,
+                            FiniteElementSpace *otrialFESpace_,
+                            FiniteElementSpace *otestFESpace_)
+{
+   engine.Reset(&e);
+
+   otrialFESpace = otrialFESpace_;
+   trialFESpace  = otrialFESpace_->GetFESpace();
+
+   otestFESpace = otestFESpace_;
+   testFESpace  = otestFESpace_->GetFESpace();
+
+   mesh = trialFESpace->GetMesh();
+
+   const int elements = GetNE();
+
+   const int trialVDim = trialFESpace->GetVDim();
+
+   const int trialLocalDofs = otrialFESpace->GetLocalDofs();
+   const int testLocalDofs  = otestFESpace->GetLocalDofs();
+
+   // First-touch policy when running with OpenMP
+   if (GetDevice().mode() == "OpenMP")
+   {
+      const std::string &okl_path = OccaEngine().GetOklPath();
+      const std::string &okl_defines = OccaEngine().GetOklDefines();
+      ::occa::kernel initLocalKernel =
+         GetDevice().buildKernel(okl_path + "utils.okl",
+                                 "InitLocalVector",
+                                 okl_defines);
+
+      const std::size_t sd = sizeof(double);
+      const uint64_t trialEntries = sd * (elements * trialLocalDofs);
+      const uint64_t testEntries  = sd * (elements * testLocalDofs);
+      for (int v = 0; v < trialVDim; ++v)
+      {
+         const uint64_t trialOffset = v * trialEntries;
+         const uint64_t testOffset  = v * testEntries;
+
+         initLocalKernel(elements, trialLocalDofs,
+                         localX.OccaMem().slice(trialOffset, trialEntries));
+         initLocalKernel(elements, testLocalDofs,
+                         localY.OccaMem().slice(testOffset, testEntries));
+      }
+   }
+}
+
+int OccaBilinearForm::BaseGeom() const
+{
+   return mesh->GetElementBaseGeometry();
+}
+
+int OccaBilinearForm::GetDim() const
+{
+   return mesh->Dimension();
+}
+
+int64_t OccaBilinearForm::GetNE() const
+{
+   return mesh->GetNE();
+}
+
+Mesh& OccaBilinearForm::GetMesh() const
+{
+   return *mesh;
+}
+
+FiniteElementSpace& OccaBilinearForm::GetTrialOccaFESpace() const
+{
+   return *otrialFESpace;
+}
+
+FiniteElementSpace& OccaBilinearForm::GetTestOccaFESpace() const
+{
+   return *otestFESpace;
+}
+
+mfem::FiniteElementSpace& OccaBilinearForm::GetTrialFESpace() const
+{
+   return *trialFESpace;
+}
+
+mfem::FiniteElementSpace& OccaBilinearForm::GetTestFESpace() const
+{
+   return *testFESpace;
+}
+
+int64_t OccaBilinearForm::GetTrialNDofs() const
+{
+   return trialFESpace->GetNDofs();
+}
+
+int64_t OccaBilinearForm::GetTestNDofs() const
+{
+   return testFESpace->GetNDofs();
+}
+
+int64_t OccaBilinearForm::GetTrialVDim() const
+{
+   return trialFESpace->GetVDim();
+}
+
+int64_t OccaBilinearForm::GetTestVDim() const
+{
+   return testFESpace->GetVDim();
+}
+
+const FiniteElement& OccaBilinearForm::GetTrialFE(const int i) const
+{
+   return *(trialFESpace->GetFE(i));
+}
+
+const FiniteElement& OccaBilinearForm::GetTestFE(const int i) const
+{
+   return *(testFESpace->GetFE(i));
+}
+
+// Adds new Domain Integrator.
+void OccaBilinearForm::AddDomainIntegrator(OccaIntegrator *integrator,
+                                           const ::occa::properties &props)
+{
+   AddIntegrator(integrator, props, DomainIntegrator);
+}
+
+// Adds new Boundary Integrator.
+void OccaBilinearForm::AddBoundaryIntegrator(OccaIntegrator *integrator,
+                                             const ::occa::properties &props)
+{
+   AddIntegrator(integrator, props, BoundaryIntegrator);
+}
+
+// Adds new interior Face Integrator.
+void OccaBilinearForm::AddInteriorFaceIntegrator(OccaIntegrator *integrator,
+                                                 const ::occa::properties &props)
+{
+   AddIntegrator(integrator, props, InteriorFaceIntegrator);
+}
+
+// Adds new boundary Face Integrator.
+void OccaBilinearForm::AddBoundaryFaceIntegrator(OccaIntegrator *integrator,
+                                                 const ::occa::properties &props)
+{
+   AddIntegrator(integrator, props, BoundaryFaceIntegrator);
+}
+
+// Adds Integrator based on OccaIntegratorType
+void OccaBilinearForm::AddIntegrator(OccaIntegrator *integrator,
+                                     const ::occa::properties &props,
+                                     const OccaIntegratorType itype)
+{
+   if (integrator == NULL)
+   {
+      std::stringstream error_ss;
+      error_ss << "OccaBilinearForm::";
+      switch (itype)
+      {
+         case DomainIntegrator      : error_ss << "AddDomainIntegrator";       break;
+         case BoundaryIntegrator    : error_ss << "AddBoundaryIntegrator";     break;
+         case InteriorFaceIntegrator: error_ss << "AddInteriorFaceIntegrator"; break;
+         case BoundaryFaceIntegrator: error_ss << "AddBoundaryFaceIntegrator"; break;
+      }
+      error_ss << " (...):\n"
+               << "  Integrator is NULL";
+      const std::string error = error_ss.str();
+      mfem_error(error.c_str());
+   }
+   integrator->SetupIntegrator(*this, baseKernelProps + props, itype);
+   integrators.push_back(integrator);
+}
+
+const mfem::Operator* OccaBilinearForm::GetTrialProlongation() const
+{
+   return otrialFESpace->GetProlongationOperator();
+}
+
+const mfem::Operator* OccaBilinearForm::GetTestProlongation() const
+{
+   return otestFESpace->GetProlongationOperator();
+}
+
+const mfem::Operator* OccaBilinearForm::GetTrialRestriction() const
+{
+   return otrialFESpace->GetRestrictionOperator();
+}
+
+const mfem::Operator* OccaBilinearForm::GetTestRestriction() const
+{
+   return otestFESpace->GetRestrictionOperator();
+}
+
+void OccaBilinearForm::Assemble()
+{
+   // [MISSING] Find geometric information that is needed by intergrators
+   //             to share between integrators.
+   const int integratorCount = (int) integrators.size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->Assemble();
+   }
+}
+
+void OccaBilinearForm::FormLinearSystem(const mfem::Array<int> &constraintList,
+                                        mfem::Vector &x, mfem::Vector &b,
+                                        mfem::Operator *&Aout,
+                                        mfem::Vector &X, mfem::Vector &B,
+                                        int copy_interior)
+{
+   FormOperator(constraintList, Aout);
+   InitRHS(constraintList, x, b, Aout, X, B, copy_interior);
+}
+
+void OccaBilinearForm::FormOperator(const mfem::Array<int> &constraintList,
+                                    mfem::Operator *&Aout)
+{
+   const mfem::Operator *trialP = GetTrialProlongation();
+   const mfem::Operator *testP  = GetTestProlongation();
+   mfem::Operator *rap = this;
+
+   if (trialP)
+   {
+      rap = new RAPOperator(*testP, *this, *trialP);
+   }
+
+   Aout = new OccaConstrainedOperator(rap, constraintList,
+                                      rap != this);
+}
+
+void OccaBilinearForm::InitRHS(const mfem::Array<int> &constraintList,
+                               mfem::Vector &x, mfem::Vector &b,
+                               mfem::Operator *A,
+                               mfem::Vector &X, mfem::Vector &B,
+                               int copy_interior)
+{
+   const std::string okl_defines = OccaEngine().GetOklDefines();
+
+   // FIXME: move these kernels to the Backend?
+   static ::occa::kernelBuilder get_subvector_builder =
+      ::occa::linalg::customLinearMethod(
+         "vector_get_subvector",
+
+         "const int dof_i = v2[i];"
+         "v0[i] = dof_i >= 0 ? v1[dof_i] : -v1[-dof_i - 1];",
+
+         "defines: {"
+         "  VTYPE0: 'double',"
+         "  VTYPE1: 'double',"
+         "  VTYPE2: 'int',"
+         "  TILESIZE: 128,"
+         "}" + okl_defines);
+
+   static ::occa::kernelBuilder set_subvector_builder =
+      ::occa::linalg::customLinearMethod(
+         "vector_set_subvector",
+         "const int dof_i = v2[i];"
+         "if (dof_i >= 0) { v0[dof_i]      = v1[i]; }"
+         "else            { v0[-dof_i - 1] = -v1[i]; }",
+
+         "defines: {"
+         "  VTYPE0: 'double',"
+         "  VTYPE1: 'double',"
+         "  VTYPE2: 'int',"
+         "  TILESIZE: 128,"
+         "}" + okl_defines);
+
+   const mfem::Operator *P = GetTrialProlongation();
+   const mfem::Operator *R = GetTrialRestriction();
+
+   if (P)
+   {
+      // Variational restriction with P
+      B.Resize(P->InLayout());
+      P->MultTranspose(b, B);
+      X.Resize(R->OutLayout());
+      R->Mult(x, X);
+   }
+   else
+   {
+      // rap, X and B point to the same data as this, x and b
+      X.MakeRef(x);
+      B.MakeRef(b);
+   }
+
+   if (!copy_interior && constraintList.Size() > 0)
+   {
+      ::occa::kernel get_subvector_kernel =
+         get_subvector_builder.build(GetDevice());
+      ::occa::kernel set_subvector_kernel =
+         set_subvector_builder.build(GetDevice());
+
+      const Array &constrList = constraintList.Get_PArray()->As<Array>();
+      Vector subvec(constrList.OccaLayout());
+
+      get_subvector_kernel(constraintList.Size(),
+                           subvec.OccaMem(),
+                           X.Get_PVector()->As<Vector>().OccaMem(),
+                           constrList.OccaMem());
+
+      X.Fill(0.0);
+
+      set_subvector_kernel(constraintList.Size(),
+                           X.Get_PVector()->As<Vector>().OccaMem(),
+                           subvec.OccaMem(),
+                           constrList.OccaMem());
+   }
+
+   OccaConstrainedOperator *cA = dynamic_cast<OccaConstrainedOperator*>(A);
+   if (cA)
+   {
+      cA->EliminateRHS(X.Get_PVector()->As<Vector>(),
+                       B.Get_PVector()->As<Vector>());
+   }
+   else
+   {
+      mfem_error("OccaBilinearForm::InitRHS expects an OccaConstrainedOperator");
+   }
+}
+
+// Matrix vector multiplication.
+void OccaBilinearForm::Mult_(const Vector &x, Vector &y) const
+{
+   otrialFESpace->GlobalToLocal(x, localX);
+   localY.Fill<double>(0.0);
+
+   const int integratorCount = (int) integrators.size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->MultAdd(localX, localY);
+   }
+
+   otestFESpace->LocalToGlobal(localY, y);
+}
+
+// Matrix transpose vector multiplication.
+void OccaBilinearForm::MultTranspose_(const Vector &x, Vector &y) const
+{
+   otestFESpace->GlobalToLocal(x, localX);
+   localY.Fill<double>(0.0);
+
+   const int integratorCount = (int) integrators.size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->MultTransposeAdd(localX, localY);
+   }
+
+   otrialFESpace->LocalToGlobal(localY, y);
+}
+
+void OccaBilinearForm::OccaRecoverFEMSolution(const mfem::Vector &X,
+                                              const mfem::Vector &b,
+                                              mfem::Vector &x)
+{
+   const mfem::Operator *P = this->GetTrialProlongation();
+   if (P)
+   {
+      // Apply conforming prolongation
+      x.Resize(P->OutLayout());
+      P->Mult(X, x);
+   }
+   // Otherwise X and x point to the same data
+}
+
+// Frees memory bilinear form.
+OccaBilinearForm::~OccaBilinearForm()
+{
+   // Make sure all integrators free their data
+   IntegratorVector::iterator it = integrators.begin();
+   while (it != integrators.end())
+   {
+      delete *it;
+      ++it;
+   }
+}
+
+
+void BilinearForm::InitOccaBilinearForm()
+{
+   // Init 'obform' using 'bform'
+   MFEM_ASSERT(bform != NULL, "");
+   MFEM_ASSERT(obform == NULL, "");
+
+   FiniteElementSpace &ofes =
+      bform->FESpace()->Get_PFESpace()->As<FiniteElementSpace>();
+   obform = new OccaBilinearForm(&ofes);
+
+   // Transfer domain integrators
+   mfem::Array<mfem::BilinearFormIntegrator*> &dbfi = *bform->GetDBFI();
+   for (int i = 0; i < dbfi.Size(); i++)
+   {
+      std::string integ_name(dbfi[i]->Name());
+      Coefficient *scal_coeff = dbfi[i]->GetScalarCoefficient();
+      ConstantCoefficient *const_coeff =
+         dynamic_cast<ConstantCoefficient*>(scal_coeff);
+      // TODO: other types of coefficients ...
+      double val = const_coeff ? const_coeff->constant : 1.0;
+      OccaCoefficient ocoeff(obform->OccaEngine(), val);
+
+      OccaIntegrator *ointeg = NULL;
+
+      if (integ_name == "(undefined)")
+      {
+         MFEM_ABORT("BilinearFormIntegrator does not define Name()");
+      }
+      else if (integ_name == "diffusion")
+      {
+         ointeg = new OccaDiffusionIntegrator(ocoeff);
+      }
+      else
+      {
+         MFEM_ABORT("BilinearFormIntegrator [Name() = " << integ_name
+                    << "] is not supported");
+      }
+
+      const mfem::IntegrationRule *ir = dbfi[i]->GetIntRule();
+      if (ir) { ointeg->SetIntegrationRule(*ir); }
+
+      obform->AddDomainIntegrator(ointeg);
+   }
+
+   // TODO: other types of integrators ...
+}
+
+bool BilinearForm::Assemble()
+{
+   if (obform == NULL) { InitOccaBilinearForm(); }
+
+   obform->Assemble();
+
+   return true; // --> host assembly is not needed
+}
+
+void BilinearForm::FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
+                                    mfem::OperatorHandle &A)
+{
+   if (A.Type() == mfem::Operator::ANY_TYPE)
+   {
+      mfem::Operator *Aout = NULL;
+      obform->FormOperator(ess_tdof_list, Aout);
+      A.Reset(Aout);
+   }
+   else
+   {
+      MFEM_ABORT("Operator::Type is not supported, type = " << A.Type());
+   }
+}
+
+void BilinearForm::FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
+                                    mfem::Vector &x, mfem::Vector &b,
+                                    mfem::OperatorHandle &A,
+                                    mfem::Vector &X, mfem::Vector &B,
+                                    int copy_interior)
+{
+   FormSystemMatrix(ess_tdof_list, A);
+   obform->InitRHS(ess_tdof_list, x, b, A.Ptr(), X, B, copy_interior);
+}
+
+void BilinearForm::RecoverFEMSolution(const mfem::Vector &X,
+                                      const mfem::Vector &b,
+                                      mfem::Vector &x)
+{
+   obform->OccaRecoverFEMSolution(X, b, x);
+}
+
+BilinearForm::~BilinearForm()
+{
+   delete obform;
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/bilinearform.hpp

@@ -0,0 +1,213 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP
+#define MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "fespace.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+enum OccaIntegratorType
+{
+   DomainIntegrator       = 0,
+   BoundaryIntegrator     = 1,
+   InteriorFaceIntegrator = 2,
+   BoundaryFaceIntegrator = 3
+};
+
+class OccaIntegrator;
+
+
+/** Class for bilinear form - "Matrix" with associated FE space and
+    BLFIntegrators. */
+class OccaBilinearForm : public Operator
+{
+   friend class OccaIntegrator;
+
+protected:
+   typedef std::vector<OccaIntegrator*> IntegratorVector;
+
+   SharedPtr<const Engine> engine;
+
+   // State information
+   mutable mfem::Mesh *mesh;
+
+   mutable FiniteElementSpace *otrialFESpace;
+   mutable mfem::FiniteElementSpace *trialFESpace;
+
+   mutable FiniteElementSpace *otestFESpace;
+   mutable mfem::FiniteElementSpace *testFESpace;
+
+   IntegratorVector integrators;
+
+   // Device data
+   ::occa::properties baseKernelProps;
+
+   // The input and output vectors are mapped to local nodes for efficient
+   // operations. In other words, they are E-vectors.
+   // The size is: (number of elements) * (nodes in element) * (vector dim)
+   mutable Vector localX, localY;
+
+public:
+   OccaBilinearForm(FiniteElementSpace *ofespace_);
+
+   OccaBilinearForm(FiniteElementSpace *otrialFESpace_,
+                    FiniteElementSpace *otestFESpace_);
+
+   void Init(const Engine &e,
+             FiniteElementSpace *otrialFESpace_,
+             FiniteElementSpace *otestFESpace_);
+
+   const Engine &OccaEngine() const { return *engine; }
+
+   ::occa::device GetDevice(int idx = 0) const
+   { return engine->GetDevice(idx); }
+
+   // Useful mesh Information
+   int BaseGeom() const;
+   int GetDim() const;
+   int64_t GetNE() const;
+
+   mfem::Mesh& GetMesh() const;
+
+   FiniteElementSpace& GetTrialOccaFESpace() const;
+   FiniteElementSpace& GetTestOccaFESpace() const;
+
+   mfem::FiniteElementSpace& GetTrialFESpace() const;
+   mfem::FiniteElementSpace& GetTestFESpace() const;
+
+   // Useful FE information
+   int64_t GetTrialNDofs() const;
+   int64_t GetTestNDofs() const;
+
+   int64_t GetTrialVDim() const;
+   int64_t GetTestVDim() const;
+
+   const mfem::FiniteElement& GetTrialFE(const int i) const;
+   const mfem::FiniteElement& GetTestFE(const int i) const;
+
+   // Adds new Domain Integrator.
+   void AddDomainIntegrator(OccaIntegrator *integrator,
+                            const ::occa::properties &props =
+                               ::occa::properties());
+
+   // Adds new Boundary Integrator.
+   void AddBoundaryIntegrator(OccaIntegrator *integrator,
+                              const ::occa::properties &props =
+                                 ::occa::properties());
+
+   // Adds new interior Face Integrator.
+   void AddInteriorFaceIntegrator(OccaIntegrator *integrator,
+                                  const ::occa::properties &props =
+                                     ::occa::properties());
+
+   // Adds new boundary Face Integrator.
+   void AddBoundaryFaceIntegrator(OccaIntegrator *integrator,
+                                  const ::occa::properties &props =
+                                     ::occa::properties());
+
+   // Adds Integrator based on OccaIntegratorType
+   void AddIntegrator(OccaIntegrator *integrator,
+                      const ::occa::properties &props,
+                      const OccaIntegratorType itype);
+
+   virtual const mfem::Operator *GetTrialProlongation() const;
+   virtual const mfem::Operator *GetTestProlongation() const;
+
+   virtual const mfem::Operator *GetTrialRestriction() const;
+   virtual const mfem::Operator *GetTestRestriction() const;
+
+   // Assembles the form i.e. sums over all domain/bdr integrators.
+   virtual void Assemble();
+
+   void FormLinearSystem(const mfem::Array<int> &constraintList,
+                         mfem::Vector &x, mfem::Vector &b,
+                         mfem::Operator *&Aout,
+                         mfem::Vector &X, mfem::Vector &B,
+                         int copy_interior = 0);
+
+   void FormOperator(const mfem::Array<int> &constraintList,
+                     mfem::Operator *&Aout);
+
+   void InitRHS(const mfem::Array<int> &constraintList,
+                mfem::Vector &x, mfem::Vector &b,
+                mfem::Operator *Aout,
+                mfem::Vector &X, mfem::Vector &B,
+                int copy_interior = 0);
+
+   // overrides
+   virtual void Mult_(const Vector &x, Vector &y) const;
+   virtual void MultTranspose_(const Vector &x, Vector &y) const;
+
+   void OccaRecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
+                               mfem::Vector &x);
+
+   // Destroys bilinear form.
+   ~OccaBilinearForm();
+};
+
+
+/// TODO: doxygen
+class BilinearForm : public mfem::PBilinearForm
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // mfem::BilinearForm *bform;
+   OccaBilinearForm *obform;
+
+   // Called from Assemble() if obform is NULL to initialize obform.
+   void InitOccaBilinearForm();
+
+public:
+   /// TODO: doxygen
+   BilinearForm(const Engine &e, mfem::BilinearForm &bf)
+      : mfem::PBilinearForm(e, bf), obform(NULL) { }
+
+   /// Virtual destructor
+   virtual ~BilinearForm();
+
+   /// Assemble the PBilinearForm.
+   /** This method is called from the method mfem::BilinearForm::Assemble() of
+       the associated mfem::BilinearForm, #bform.
+       @returns True, if the host assembly should NOT be performed. */
+   virtual bool Assemble();
+
+   virtual void FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
+                                 mfem::OperatorHandle &A);
+
+   virtual void FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
+                                 mfem::Vector &x, mfem::Vector &b,
+                                 mfem::OperatorHandle &A,
+                                 mfem::Vector &X, mfem::Vector &B,
+                                 int copy_interior);
+
+   virtual void RecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
+                                   mfem::Vector &x);
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP

backends/occa/bilininteg.cpp

@@ -0,0 +1,956 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "bilininteg.hpp"
+#include "../../fem/fem.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+std::map<std::string, OccaDofQuadMaps> OccaDofQuadMaps::AllDofQuadMaps;
+
+OccaGeometry OccaGeometry::Get(::occa::device device,
+                               FiniteElementSpace &ofespace,
+                               const mfem::IntegrationRule &ir,
+                               const int flags)
+{
+   OccaGeometry geom;
+
+   mfem::Mesh &mesh = *(ofespace.GetMesh());
+   if (!mesh.GetNodes())
+   {
+      mesh.SetCurvature(1, false, -1, mfem::Ordering::byVDIM);
+   }
+   mfem::GridFunction &nodes = *(mesh.GetNodes());
+   const mfem::FiniteElementSpace &fespace = *(nodes.FESpace());
+   const mfem::FiniteElement &fe = *(fespace.GetFE(0));
+
+   const int dims     = fe.GetDim();
+   const int elements = fespace.GetNE();
+   const int numDofs  = fe.GetDof();
+   const int numQuad  = ir.GetNPoints();
+
+   MFEM_ASSERT(dims == mesh.SpaceDimension(), "");
+
+   geom.meshNodes.allocate(device,
+                           dims, numDofs, elements);
+
+   const mfem::Table &e2dTable = fespace.GetElementToDofTable();
+   const int *elementMap = e2dTable.GetJ();
+   nodes.Pull();
+   for (int e = 0; e < elements; ++e)
+   {
+      for (int dof = 0; dof < numDofs; ++dof)
+      {
+         const int gid = elementMap[dof + numDofs*e];
+         for (int dim = 0; dim < dims; ++dim)
+         {
+            geom.meshNodes(dim, dof, e) = nodes[fespace.DofToVDof(gid,dim)];
+         }
+      }
+   }
+   geom.meshNodes.keepInDevice();
+
+   if (flags & Jacobian)
+   {
+      geom.J.allocate(device,
+                      dims, dims, numQuad, elements);
+   }
+   else
+   {
+      geom.J.allocate(device, 1);
+   }
+   if (flags & JacobianInv)
+   {
+      geom.invJ.allocate(device,
+                         dims, dims, numQuad, elements);
+   }
+   else
+   {
+      geom.invJ.allocate(device, 1);
+   }
+   if (flags & JacobianDet)
+   {
+      geom.detJ.allocate(device,
+                         numQuad, elements);
+   }
+   else
+   {
+      geom.detJ.allocate(device, 1);
+   }
+
+   geom.J.stopManaging();
+   geom.invJ.stopManaging();
+   geom.detJ.stopManaging();
+
+   OccaDofQuadMaps &maps = OccaDofQuadMaps::GetSimplexMaps(device, fe, ir);
+
+   ::occa::properties props;
+   props["defines/NUM_DOFS"] = numDofs;
+   props["defines/NUM_QUAD"] = numQuad;
+   props["defines/STORE_JACOBIAN"]     = (flags & Jacobian);
+   props["defines/STORE_JACOBIAN_INV"] = (flags & JacobianInv);
+   props["defines/STORE_JACOBIAN_DET"] = (flags & JacobianDet);
+
+   const std::string &okl_path = ofespace.OccaEngine().GetOklPath();
+   const std::string &okl_defines = ofespace.OccaEngine().GetOklDefines();
+   ::occa::kernel init = device.buildKernel(okl_path + "geometry.okl",
+                                            stringWithDim("InitGeometryInfo",
+                                                          fe.GetDim()),
+                                            props + okl_defines);
+   init(elements,
+        maps.dofToQuadD,
+        geom.meshNodes,
+        geom.J, geom.invJ, geom.detJ);
+
+   return geom;
+}
+
+OccaDofQuadMaps::OccaDofQuadMaps() :
+   hash() {}
+
+OccaDofQuadMaps::OccaDofQuadMaps(const OccaDofQuadMaps &maps)
+{
+   *this = maps;
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::operator = (const OccaDofQuadMaps &maps)
+{
+   hash = maps.hash;
+   dofToQuad   = maps.dofToQuad;
+   dofToQuadD  = maps.dofToQuadD;
+   quadToDof   = maps.quadToDof;
+   quadToDofD  = maps.quadToDofD;
+   quadWeights = maps.quadWeights;
+   return *this;
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
+                                      const FiniteElementSpace &fespace,
+                                      const mfem::IntegrationRule &ir,
+                                      const bool transpose)
+{
+   return Get(device,
+              *fespace.GetFE(0),
+              *fespace.GetFE(0),
+              ir,
+              transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
+                                      const mfem::FiniteElement &fe,
+                                      const mfem::IntegrationRule &ir,
+                                      const bool transpose)
+{
+   return Get(device, fe, fe, ir, transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
+                                      const FiniteElementSpace &trialFESpace,
+                                      const FiniteElementSpace &testFESpace,
+                                      const mfem::IntegrationRule &ir,
+                                      const bool transpose)
+{
+   return Get(device,
+              *trialFESpace.GetFE(0),
+              *testFESpace.GetFE(0),
+              ir,
+              transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
+                                      const mfem::FiniteElement &trialFE,
+                                      const mfem::FiniteElement &testFE,
+                                      const mfem::IntegrationRule &ir,
+                                      const bool transpose)
+{
+   return (dynamic_cast<const mfem::TensorBasisElement*>(&trialFE)
+           ? GetTensorMaps(device, trialFE, testFE, ir, transpose)
+           : GetSimplexMaps(device, trialFE, testFE, ir, transpose));
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::GetTensorMaps(::occa::device device,
+                                                const mfem::FiniteElement &fe,
+                                                const mfem::IntegrationRule &ir,
+                                                const bool transpose)
+{
+   return GetTensorMaps(device,
+                        fe, fe,
+                        ir, transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::GetTensorMaps(::occa::device device,
+                                                const mfem::FiniteElement &trialFE,
+                                                const mfem::FiniteElement &testFE,
+                                                const mfem::IntegrationRule &ir,
+                                                const bool transpose)
+{
+   const mfem::TensorBasisElement &trialTFE =
+      dynamic_cast<const mfem::TensorBasisElement&>(trialFE);
+   const mfem::TensorBasisElement &testTFE =
+      dynamic_cast<const mfem::TensorBasisElement&>(testFE);
+
+   std::stringstream ss;
+   ss << ::occa::hash(device)
+      << "Tensor"
+      << "O1:"  << trialFE.GetOrder()
+      << "O2:"  << testFE.GetOrder()
+      << "BT1:" << trialTFE.GetBasisType()
+      << "BT2:" << testTFE.GetBasisType()
+      << "Q:"   << ir.GetNPoints();
+   std::string hash = ss.str();
+
+   // If we've already made the dof-quad maps, reuse them
+   OccaDofQuadMaps &maps = AllDofQuadMaps[hash];
+   if (!maps.hash.size())
+   {
+      // Create the dof-quad maps
+      maps.hash = hash;
+
+      OccaDofQuadMaps trialMaps = GetD2QTensorMaps(device, trialFE, ir);
+      OccaDofQuadMaps testMaps  = GetD2QTensorMaps(device, testFE , ir, true);
+
+      maps.dofToQuad   = trialMaps.dofToQuad;
+      maps.dofToQuadD  = trialMaps.dofToQuadD;
+      maps.quadToDof   = testMaps.dofToQuad;
+      maps.quadToDofD  = testMaps.dofToQuadD;
+      maps.quadWeights = testMaps.quadWeights;
+   }
+   return maps;
+}
+
+OccaDofQuadMaps OccaDofQuadMaps::GetD2QTensorMaps(::occa::device device,
+                                                  const mfem::FiniteElement &fe,
+                                                  const mfem::IntegrationRule &ir,
+                                                  const bool transpose)
+{
+   const mfem::TensorBasisElement &tfe =
+      dynamic_cast<const mfem::TensorBasisElement&>(fe);
+
+   const mfem::Poly_1D::Basis &basis = tfe.GetBasis1D();
+   const int order = fe.GetOrder();
+   // [MISSING] Get 1D dofs
+   const int dofs = order + 1;
+   const int dims = fe.GetDim();
+
+   // Create the dof -> quadrature point map
+   const mfem::IntegrationRule &ir1D =
+      mfem::IntRules.Get(mfem::Geometry::SEGMENT, ir.GetOrder());
+   const int quadPoints = ir1D.GetNPoints();
+   const int quadPoints2D = quadPoints*quadPoints;
+   const int quadPoints3D = quadPoints2D*quadPoints;
+   const int quadPointsND = ((dims == 1) ? quadPoints :
+                             ((dims == 2) ? quadPoints2D : quadPoints3D));
+
+   OccaDofQuadMaps maps;
+   // Initialize the dof -> quad mapping
+   maps.dofToQuad.allocate(device,
+                           quadPoints, dofs);
+   maps.dofToQuadD.allocate(device,
+                            quadPoints, dofs);
+
+   double *quadWeights1DData = NULL;
+
+   if (transpose)
+   {
+      maps.dofToQuad.reindex(1,0);
+      maps.dofToQuadD.reindex(1,0);
+      // Initialize quad weights only for transpose
+      maps.quadWeights.allocate(device,
+                                quadPointsND);
+      quadWeights1DData = new double[quadPoints];
+   }
+
+   mfem::Vector d2q(dofs);
+   mfem::Vector d2qD(dofs);
+   for (int q = 0; q < quadPoints; ++q)
+   {
+      const mfem::IntegrationPoint &ip = ir1D.IntPoint(q);
+      basis.Eval(ip.x, d2q, d2qD);
+      if (transpose)
+      {
+         quadWeights1DData[q] = ip.weight;
+      }
+      for (int d = 0; d < dofs; ++d)
+      {
+         maps.dofToQuad(q, d)  = d2q[d];
+         maps.dofToQuadD(q, d) = d2qD[d];
+      }
+   }
+
+   maps.dofToQuad.keepInDevice();
+   maps.dofToQuadD.keepInDevice();
+
+   if (transpose)
+   {
+      for (int q = 0; q < quadPointsND; ++q)
+      {
+         const int qx = q % quadPoints;
+         const int qz = q / quadPoints2D;
+         const int qy = (q - qz*quadPoints2D) / quadPoints;
+         double w = quadWeights1DData[qx];
+         if (dims > 1)
+         {
+            w *= quadWeights1DData[qy];
+         }
+         if (dims > 2)
+         {
+            w *= quadWeights1DData[qz];
+         }
+         maps.quadWeights[q] = w;
+      }
+      maps.quadWeights.keepInDevice();
+      delete [] quadWeights1DData;
+   }
+
+   return maps;
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::GetSimplexMaps(::occa::device device,
+                                                 const mfem::FiniteElement &fe,
+                                                 const mfem::IntegrationRule &ir,
+                                                 const bool transpose)
+{
+   return GetSimplexMaps(device,
+                         fe, fe,
+                         ir, transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::GetSimplexMaps(::occa::device device,
+                                                 const mfem::FiniteElement &trialFE,
+                                                 const mfem::FiniteElement &testFE,
+                                                 const mfem::IntegrationRule &ir,
+                                                 const bool transpose)
+{
+   std::stringstream ss;
+   ss << ::occa::hash(device)
+      << "Simplex"
+      << "O1:" << trialFE.GetOrder()
+      << "O2:" << testFE.GetOrder()
+      << "Q:"  << ir.GetNPoints();
+   std::string hash = ss.str();
+
+   // If we've already made the dof-quad maps, reuse them
+   OccaDofQuadMaps &maps = AllDofQuadMaps[hash];
+   if (!maps.hash.size())
+   {
+      // Create the dof-quad maps
+      maps.hash = hash;
+
+      OccaDofQuadMaps trialMaps = GetD2QSimplexMaps(device, trialFE, ir);
+      OccaDofQuadMaps testMaps  = GetD2QSimplexMaps(device, testFE , ir, true);
+
+      maps.dofToQuad   = trialMaps.dofToQuad;
+      maps.dofToQuadD  = trialMaps.dofToQuadD;
+      maps.quadToDof   = testMaps.dofToQuad;
+      maps.quadToDofD  = testMaps.dofToQuadD;
+      maps.quadWeights = testMaps.quadWeights;
+   }
+   return maps;
+}
+
+OccaDofQuadMaps OccaDofQuadMaps::GetD2QSimplexMaps(::occa::device device,
+                                                   const mfem::FiniteElement &fe,
+                                                   const mfem::IntegrationRule &ir,
+                                                   const bool transpose)
+{
+   const int dims = fe.GetDim();
+   const int numDofs = fe.GetDof();
+   const int numQuad = ir.GetNPoints();
+
+   OccaDofQuadMaps maps;
+   // Initialize the dof -> quad mapping
+   maps.dofToQuad.allocate(device,
+                           numQuad, numDofs);
+   maps.dofToQuadD.allocate(device,
+                            dims, numQuad, numDofs);
+
+   if (transpose)
+   {
+      maps.dofToQuad.reindex(1,0);
+      maps.dofToQuadD.reindex(1,0);
+      // Initialize quad weights only for transpose
+      maps.quadWeights.allocate(device,
+                                numQuad);
+   }
+
+   mfem::Vector d2q(numDofs);
+   mfem::DenseMatrix d2qD(numDofs, dims);
+   for (int q = 0; q < numQuad; ++q)
+   {
+      const mfem::IntegrationPoint &ip = ir.IntPoint(q);
+      if (transpose)
+      {
+         maps.quadWeights[q] = ip.weight;
+      }
+      fe.CalcShape(ip, d2q);
+      fe.CalcDShape(ip, d2qD);
+      for (int d = 0; d < numDofs; ++d)
+      {
+         const double w = d2q[d];
+         maps.dofToQuad(q, d) = w;
+         for (int dim = 0; dim < dims; ++dim)
+         {
+            const double wD = d2qD(d, dim);
+            maps.dofToQuadD(dim, q, d) = wD;
+         }
+      }
+   }
+
+   maps.dofToQuad.keepInDevice();
+   maps.dofToQuadD.keepInDevice();
+   if (transpose)
+   {
+      maps.quadWeights.keepInDevice();
+   }
+
+   return maps;
+}
+
+//---[ Integrator Defines ]-----------
+std::string stringWithDim(const std::string &s, const int dim)
+{
+   std::string ret = s;
+   ret += ('0' + (char) dim);
+   ret += 'D';
+   return ret;
+}
+
+int closestWarpBatchTo(const int value)
+{
+   return ((value + 31) / 32) * 32;
+}
+
+int closestMultipleWarpBatch(const int multiple, const int maxSize)
+{
+   if (multiple > maxSize)
+   {
+      return maxSize;
+   }
+   int batch = (32 / multiple);
+   int minDiff = 32 - (multiple * batch);
+   for (int i = 64; i <= maxSize; i += 32)
+   {
+      const int newDiff = i - (multiple * (i / multiple));
+      if (newDiff < minDiff)
+      {
+         batch = (i / multiple);
+         minDiff = newDiff;
+      }
+   }
+   return batch;
+}
+
+void SetProperties(FiniteElementSpace &fespace,
+                   const mfem::IntegrationRule &ir,
+                   ::occa::properties &props)
+{
+   SetProperties(fespace, fespace, ir, props);
+}
+
+void SetProperties(FiniteElementSpace &trialFESpace,
+                   FiniteElementSpace &testFESpace,
+                   const mfem::IntegrationRule &ir,
+                   ::occa::properties &props)
+{
+   props["defines/TRIAL_VDIM"] = trialFESpace.GetVDim();
+   props["defines/TEST_VDIM"]  = testFESpace.GetVDim();
+   props["defines/NUM_DIM"]    = trialFESpace.GetDim();
+
+   if (trialFESpace.hasTensorBasis())
+   {
+      SetTensorProperties(trialFESpace, testFESpace, ir, props);
+   }
+   else
+   {
+      SetSimplexProperties(trialFESpace, testFESpace, ir, props);
+   }
+}
+
+void SetTensorProperties(FiniteElementSpace &fespace,
+                         const mfem::IntegrationRule &ir,
+                         ::occa::properties &props)
+{
+   SetTensorProperties(fespace, fespace, ir, props);
+}
+
+void SetTensorProperties(FiniteElementSpace &trialFESpace,
+                         FiniteElementSpace &testFESpace,
+                         const mfem::IntegrationRule &ir,
+                         ::occa::properties &props)
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace.GetFE(0));
+   const mfem::FiniteElement &testFE = *(testFESpace.GetFE(0));
+
+   const mfem::IntegrationRule &ir1D =
+      mfem::IntRules.Get(mfem::Geometry::SEGMENT, ir.GetOrder());
+
+   const int trialDofs = trialFE.GetDof();
+   const int testDofs  = testFE.GetDof();
+   const int numQuad  = ir.GetNPoints();
+
+   const int trialDofs1D = trialFE.GetOrder() + 1;
+   const int testDofs1D  = testFE.GetOrder() + 1;
+   const int quad1D  = ir1D.GetNPoints();
+   int trialDofsND = trialDofs1D;
+   int testDofsND  = testDofs1D;
+   int quadND  = quad1D;
+
+   const bool trialByVDIM = (trialFESpace.GetOrdering() == mfem::Ordering::byVDIM);
+   const bool testByVDIM  = (testFESpace.GetOrdering()  == mfem::Ordering::byVDIM);
+
+   props["defines/ORDERING_BY_NODES"] = 0;
+   props["defines/ORDERING_BY_VDIM"]  = 1;
+   props["defines/VDIM_ORDERING"]  = (int) trialByVDIM;
+   props["defines/TRIAL_ORDERING"] = (int) trialByVDIM;
+   props["defines/TEST_ORDERING"]  = (int) testByVDIM;
+
+   props["defines/USING_TENSOR_OPS"] = 1;
+   props["defines/NUM_DOFS"]  = trialDofs;
+   props["defines/NUM_QUAD"]  = numQuad;
+
+   props["defines/TRIAL_DOFS"] = trialDofs;
+   props["defines/TEST_DOFS"]  = testDofs;
+
+   for (int d = 1; d <= 3; ++d)
+   {
+      if (d > 1)
+      {
+         trialDofsND *= trialDofs1D;
+         testDofsND  *= testDofs1D;
+         quadND *= quad1D;
+      }
+      props["defines"][stringWithDim("NUM_DOFS_", d)] = trialDofsND;
+      props["defines"][stringWithDim("NUM_QUAD_", d)] = quadND;
+
+      props["defines"][stringWithDim("TRIAL_DOFS_", d)] = trialDofsND;
+      props["defines"][stringWithDim("TEST_DOFS_" , d)] = testDofsND;
+   }
+
+   // 1D Defines
+   const int m1InnerBatch = 32 * ((quad1D + 31) / 32);
+   props["defines/A1_ELEMENT_BATCH"]       = closestMultipleWarpBatch(quad1D, 512);
+   props["defines/M1_OUTER_ELEMENT_BATCH"] = closestMultipleWarpBatch(m1InnerBatch,
+                                                                      512);
+   props["defines/M1_INNER_ELEMENT_BATCH"] = m1InnerBatch;
+
+   // 2D Defines
+   props["defines/A2_ELEMENT_BATCH"] = 1;
+   props["defines/A2_QUAD_BATCH"]    = 1;
+   props["defines/M2_ELEMENT_BATCH"] = 32;
+
+   // 3D Defines
+   const int a3QuadBatch = closestMultipleWarpBatch(quadND, 512);
+   props["defines/A3_ELEMENT_BATCH"] = closestMultipleWarpBatch(a3QuadBatch, 512);
+   props["defines/A3_QUAD_BATCH"]    = a3QuadBatch;
+}
+
+void SetSimplexProperties(FiniteElementSpace &fespace,
+                          const mfem::IntegrationRule &ir,
+                          ::occa::properties &props)
+{
+   SetSimplexProperties(fespace, fespace, ir, props);
+}
+
+void SetSimplexProperties(FiniteElementSpace &trialFESpace,
+                          FiniteElementSpace &testFESpace,
+                          const mfem::IntegrationRule &ir,
+                          ::occa::properties &props)
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace.GetFE(0));
+   const mfem::FiniteElement &testFE = *(testFESpace.GetFE(0));
+
+   const int trialDofs = trialFE.GetDof();
+   const int testDofs  = testFE.GetDof();
+   const int numQuad = ir.GetNPoints();
+   const int maxDQ   = std::max(std::max(trialDofs, testDofs), numQuad);
+
+   const bool trialByVDIM = (trialFESpace.GetOrdering() == mfem::Ordering::byVDIM);
+   const bool testByVDIM  = (testFESpace.GetOrdering()  == mfem::Ordering::byVDIM);
+
+   props["defines/ORDERING_BY_NODES"] = 0;
+   props["defines/ORDERING_BY_VDIM"]  = 1;
+   props["defines/VDIM_ORDERING"]  = (int) trialByVDIM;
+   props["defines/TRIAL_ORDERING"] = (int) trialByVDIM;
+   props["defines/TEST_ORDERING"]  = (int) testByVDIM;
+
+   props["defines/USING_TENSOR_OPS"] = 0;
+   props["defines/NUM_DOFS"]  = trialDofs;
+   props["defines/NUM_QUAD"]  = numQuad;
+
+   props["defines/TRIAL_DOFS"] = trialDofs;
+   props["defines/TEST_DOFS"]  = testDofs;
+
+   // 2D Defines
+   const int quadBatch = closestWarpBatchTo(numQuad);
+   props["defines/A2_ELEMENT_BATCH"] = closestMultipleWarpBatch(quadBatch, 2048);
+   props["defines/A2_QUAD_BATCH"]    = quadBatch;
+   props["defines/M2_INNER_BATCH"]   = closestWarpBatchTo(maxDQ);
+
+   // 3D Defines
+   props["defines/A3_ELEMENT_BATCH"] = closestMultipleWarpBatch(quadBatch, 2048);
+   props["defines/A3_QUAD_BATCH"]    = quadBatch;
+   props["defines/M3_INNER_BATCH"]   = closestWarpBatchTo(maxDQ);
+}
+
+
+//---[ Base Integrator ]--------------
+OccaIntegrator::OccaIntegrator(const Engine &e)
+   : engine(&e),
+     bform(),
+     mesh(),
+     otrialFESpace(),
+     otestFESpace(),
+     trialFESpace(),
+     testFESpace(),
+     itype(DomainIntegrator),
+     ir(NULL),
+     hasTensorBasis(false) { }
+
+OccaIntegrator::~OccaIntegrator() {}
+
+void OccaIntegrator::SetupMaps()
+{
+   maps = OccaDofQuadMaps::Get(GetDevice(),
+                               *otrialFESpace,
+                               *otestFESpace,
+                               *ir);
+
+   mapsTranspose = OccaDofQuadMaps::Get(GetDevice(),
+                                        *otestFESpace,
+                                        *otrialFESpace,
+                                        *ir);
+}
+
+FiniteElementSpace& OccaIntegrator::GetTrialOccaFESpace() const
+{
+   return *otrialFESpace;
+}
+
+FiniteElementSpace& OccaIntegrator::GetTestOccaFESpace() const
+{
+   return *otestFESpace;
+}
+
+mfem::FiniteElementSpace& OccaIntegrator::GetTrialFESpace() const
+{
+   return *trialFESpace;
+}
+
+mfem::FiniteElementSpace& OccaIntegrator::GetTestFESpace() const
+{
+   return *testFESpace;
+}
+
+void OccaIntegrator::SetIntegrationRule(const mfem::IntegrationRule &ir_)
+{
+   ir = &ir_;
+}
+
+const mfem::IntegrationRule& OccaIntegrator::GetIntegrationRule() const
+{
+   return *ir;
+}
+
+OccaDofQuadMaps& OccaIntegrator::GetDofQuadMaps()
+{
+   return maps;
+}
+
+void OccaIntegrator::SetupIntegrator(OccaBilinearForm &bform_,
+                                     const ::occa::properties &props_,
+                                     const OccaIntegratorType itype_)
+{
+   MFEM_ASSERT(engine == &bform_.OccaEngine(), "");
+   bform     = &bform_;
+   mesh      = &(bform_.GetMesh());
+
+   otrialFESpace = &(bform_.GetTrialOccaFESpace());
+   otestFESpace  = &(bform_.GetTestOccaFESpace());
+
+   trialFESpace = &(bform_.GetTrialFESpace());
+   testFESpace  = &(bform_.GetTestFESpace());
+
+   hasTensorBasis = otrialFESpace->hasTensorBasis();
+
+   props = props_;
+   itype = itype_;
+
+   if (ir == NULL)
+   {
+      SetupIntegrationRule();
+   }
+
+   SetupMaps();
+
+   SetProperties(*otrialFESpace,
+                 *otestFESpace,
+                 *ir,
+                 props);
+
+   Setup();
+}
+
+OccaGeometry OccaIntegrator::GetGeometry(const int flags)
+{
+   return OccaGeometry::Get(GetDevice(), *otrialFESpace, *ir, flags);
+}
+
+::occa::kernel OccaIntegrator::GetAssembleKernel(const ::occa::properties
+                                                 &props)
+{
+   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
+   return GetKernel(stringWithDim("Assemble", fe.GetDim()),
+                    props);
+}
+
+::occa::kernel OccaIntegrator::GetMultAddKernel(const ::occa::properties &props)
+{
+   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
+   return GetKernel(stringWithDim("MultAdd", fe.GetDim()),
+                    props);
+}
+
+::occa::kernel OccaIntegrator::GetKernel(const std::string &kernelName,
+                                         const ::occa::properties &props)
+{
+   const std::string filename = GetName() + ".okl";
+   const std::string &okl_path = OccaEngine().GetOklPath();
+   const std::string &okl_defines = OccaEngine().GetOklDefines();
+   return GetDevice().buildKernel(okl_path + filename,
+                                  kernelName,
+                                  props + okl_defines);
+}
+//====================================
+
+
+//---[ Diffusion Integrator ]---------
+OccaDiffusionIntegrator::OccaDiffusionIntegrator(const OccaCoefficient &coeff_)
+   :
+   OccaIntegrator(coeff_.OccaEngine()),
+   coeff(coeff_),
+   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
+{
+   coeff.SetName("COEFF");
+}
+
+OccaDiffusionIntegrator::~OccaDiffusionIntegrator() {}
+
+
+std::string OccaDiffusionIntegrator::GetName()
+{
+   return "DiffusionIntegrator";
+}
+
+void OccaDiffusionIntegrator::SetupIntegrationRule()
+{
+   const FiniteElement &trialFE = *(trialFESpace->GetFE(0));
+   const FiniteElement &testFE  = *(testFESpace->GetFE(0));
+   ir = &mfem::DiffusionIntegrator::GetRule(trialFE, testFE);
+}
+
+void OccaDiffusionIntegrator::Setup()
+{
+   ::occa::properties kernelProps = props;
+
+   coeff.Setup(*this, kernelProps);
+
+   // Setup assemble and mult kernels
+   assembleKernel = GetAssembleKernel(kernelProps);
+   multKernel     = GetMultAddKernel(kernelProps);
+}
+
+void OccaDiffusionIntegrator::Assemble()
+{
+   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
+
+   const int dims = fe.GetDim();
+   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+
+   const int elements = trialFESpace->GetNE();
+   const int quadraturePoints = ir->GetNPoints();
+
+   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
+
+   assembledOperator.Resize<double>(symmDims * quadraturePoints * elements,
+                                    NULL);
+
+   assembleKernel((int) mesh->GetNE(),
+                  maps.quadWeights,
+                  geom.J,
+                  coeff,
+                  assembledOperator.OccaMem());
+}
+
+void OccaDiffusionIntegrator::MultAdd(Vector &x, Vector &y)
+{
+   // Note: x and y are E-vectors
+
+   multKernel((int) mesh->GetNE(),
+              maps.dofToQuad,
+              maps.dofToQuadD,
+              maps.quadToDof,
+              maps.quadToDofD,
+              assembledOperator.OccaMem(),
+              x.OccaMem(), y.OccaMem());
+}
+//====================================
+
+
+//---[ Mass Integrator ]--------------
+OccaMassIntegrator::OccaMassIntegrator(const OccaCoefficient &coeff_) :
+   OccaIntegrator(coeff_.OccaEngine()),
+   coeff(coeff_),
+   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
+{
+   coeff.SetName("COEFF");
+}
+
+OccaMassIntegrator::~OccaMassIntegrator() {}
+
+std::string OccaMassIntegrator::GetName()
+{
+   return "MassIntegrator";
+}
+
+void OccaMassIntegrator::SetupIntegrationRule()
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace->GetFE(0));
+   const mfem::FiniteElement &testFE  = *(testFESpace->GetFE(0));
+   mfem::ElementTransformation &T = *trialFESpace->GetElementTransformation(0);
+   ir = &mfem::MassIntegrator::GetRule(trialFE, testFE, T);
+}
+
+void OccaMassIntegrator::Setup()
+{
+   ::occa::properties kernelProps = props;
+
+   coeff.Setup(*this, kernelProps);
+
+   // Setup assemble and mult kernels
+   assembleKernel = GetAssembleKernel(kernelProps);
+   multKernel     = GetMultAddKernel(kernelProps);
+}
+
+void OccaMassIntegrator::Assemble()
+{
+   if (assembledOperator.Size())
+   {
+      return;
+   }
+
+   const int elements = trialFESpace->GetNE();
+   const int quadraturePoints = ir->GetNPoints();
+
+   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
+
+   assembledOperator.Resize<double>(quadraturePoints * elements, NULL);
+
+   assembleKernel((int) mesh->GetNE(),
+                  maps.quadWeights,
+                  geom.J,
+                  coeff,
+                  assembledOperator.OccaMem());
+}
+
+void OccaMassIntegrator::SetOperator(Vector &v)
+{
+   assembledOperator = v;
+}
+
+void OccaMassIntegrator::MultAdd(Vector &x, Vector &y)
+{
+   multKernel((int) mesh->GetNE(),
+              maps.dofToQuad,
+              maps.dofToQuadD,
+              maps.quadToDof,
+              maps.quadToDofD,
+              assembledOperator.OccaMem(),
+              x.OccaMem(), y.OccaMem());
+}
+//====================================
+
+
+//---[ Vector Mass Integrator ]--------------
+OccaVectorMassIntegrator::OccaVectorMassIntegrator(const OccaCoefficient &
+                                                   coeff_)
+   :
+   OccaIntegrator(coeff_.OccaEngine()),
+   coeff(coeff_),
+   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
+{
+   coeff.SetName("COEFF");
+}
+
+OccaVectorMassIntegrator::~OccaVectorMassIntegrator() {}
+
+std::string OccaVectorMassIntegrator::GetName()
+{
+   return "VectorMassIntegrator";
+}
+
+void OccaVectorMassIntegrator::SetupIntegrationRule()
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace->GetFE(0));
+   const mfem::FiniteElement &testFE  = *(testFESpace->GetFE(0));
+   mfem::ElementTransformation &T = *trialFESpace->GetElementTransformation(0);
+   ir = &mfem::MassIntegrator::GetRule(trialFE, testFE, T);
+}
+
+void OccaVectorMassIntegrator::Setup()
+{
+   ::occa::properties kernelProps = props;
+
+   coeff.Setup(*this, kernelProps);
+
+   // Setup assemble and mult kernels
+   assembleKernel = GetAssembleKernel(kernelProps);
+   multKernel     = GetMultAddKernel(kernelProps);
+}
+
+void OccaVectorMassIntegrator::Assemble()
+{
+   const int elements = trialFESpace->GetNE();
+   const int quadraturePoints = ir->GetNPoints();
+
+   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
+
+   assembledOperator.Resize<double>(quadraturePoints * elements, NULL);
+
+   assembleKernel((int) mesh->GetNE(),
+                  maps.quadWeights,
+                  geom.J,
+                  coeff,
+                  assembledOperator.OccaMem());
+}
+
+void OccaVectorMassIntegrator::MultAdd(Vector &x, Vector &y)
+{
+   multKernel((int) mesh->GetNE(),
+              maps.dofToQuad,
+              maps.dofToQuadD,
+              maps.quadToDof,
+              maps.quadToDofD,
+              assembledOperator.OccaMem(),
+              x.OccaMem(), y.OccaMem());
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/bilininteg.hpp

@@ -0,0 +1,323 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP
+#define MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+#include "coefficient.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class OccaGeometry
+{
+public:
+   ::occa::array<double> meshNodes;
+   ::occa::array<double> J, invJ, detJ;
+
+   // byVDIM  -> [x y z x y z x y z]
+   // byNodes -> [x x x y y y z z z]
+   static const int Jacobian    = (1 << 0);
+   static const int JacobianInv = (1 << 1);
+   static const int JacobianDet = (1 << 2);
+
+   static OccaGeometry Get(::occa::device device,
+                           FiniteElementSpace &ofespace,
+                           const IntegrationRule &ir,
+                           const int flags = (Jacobian    |
+                                              JacobianInv |
+                                              JacobianDet));
+};
+
+class OccaDofQuadMaps
+{
+private:
+   // Reuse dof-quad maps
+   static std::map<std::string, OccaDofQuadMaps> AllDofQuadMaps;
+   std::string hash;
+
+public:
+   // Local stiffness matrices (B and B^T operators)
+   ::occa::array<double, ::occa::dynamic> dofToQuad, dofToQuadD; // B
+   ::occa::array<double, ::occa::dynamic> quadToDof, quadToDofD; // B^T
+   ::occa::array<double> quadWeights;
+
+   OccaDofQuadMaps();
+   OccaDofQuadMaps(const OccaDofQuadMaps &maps);
+   OccaDofQuadMaps& operator = (const OccaDofQuadMaps &maps);
+
+   // [[x y] [x y] [x y]]
+   // [[x y z] [x y z] [x y z]]
+   // mfem::GridFunction* mfem::Mesh::GetNodes() { return Nodes; }
+
+   // mfem::FiniteElementSpace *Nodes->FESpace()
+   // 25
+   // 1D [x x x x x x]
+   // 2D [x y x y x y]
+   // GetVdim()
+   // 3D ordering == byVDIM  -> [x y z x y z x y z x y z x y z x y z]
+   //    ordering == byNODES -> [x x x x x x y y y y y y z z z z z z]
+   static OccaDofQuadMaps& Get(::occa::device device,
+                               const FiniteElementSpace &fespace,
+                               const mfem::IntegrationRule &ir,
+                               const bool transpose = false);
+
+   static OccaDofQuadMaps& Get(::occa::device device,
+                               const mfem::FiniteElement &fe,
+                               const mfem::IntegrationRule &ir,
+                               const bool transpose = false);
+
+   static OccaDofQuadMaps& Get(::occa::device device,
+                               const FiniteElementSpace &trialFESpace,
+                               const FiniteElementSpace &testFESpace,
+                               const mfem::IntegrationRule &ir,
+                               const bool transpose = false);
+
+   static OccaDofQuadMaps& Get(::occa::device device,
+                               const mfem::FiniteElement &trialFE,
+                               const mfem::FiniteElement &testFE,
+                               const mfem::IntegrationRule &ir,
+                               const bool transpose = false);
+
+   static OccaDofQuadMaps& GetTensorMaps(::occa::device device,
+                                         const mfem::FiniteElement &fe,
+                                         const mfem::IntegrationRule &ir,
+                                         const bool transpose = false);
+
+   static OccaDofQuadMaps& GetTensorMaps(::occa::device device,
+                                         const mfem::FiniteElement &trialFE,
+                                         const mfem::FiniteElement &testFE,
+                                         const mfem::IntegrationRule &ir,
+                                         const bool transpose = false);
+
+   static OccaDofQuadMaps GetD2QTensorMaps(::occa::device device,
+                                           const mfem::FiniteElement &fe,
+                                           const mfem::IntegrationRule &ir,
+                                           const bool transpose = false);
+
+   static OccaDofQuadMaps& GetSimplexMaps(::occa::device device,
+                                          const mfem::FiniteElement &fe,
+                                          const mfem::IntegrationRule &ir,
+                                          const bool transpose = false);
+
+   static OccaDofQuadMaps& GetSimplexMaps(::occa::device device,
+                                          const mfem::FiniteElement &trialFE,
+                                          const mfem::FiniteElement &testFE,
+                                          const mfem::IntegrationRule &ir,
+                                          const bool transpose = false);
+
+   static OccaDofQuadMaps GetD2QSimplexMaps(::occa::device device,
+                                            const mfem::FiniteElement &fe,
+                                            const mfem::IntegrationRule &ir,
+                                            const bool transpose = false);
+};
+
+//---[ Define Methods ]---------------
+std::string stringWithDim(const std::string &s, const int dim);
+int closestWarpBatch(const int multiple, const int maxSize);
+
+void SetProperties(FiniteElementSpace &fespace,
+                   const mfem::IntegrationRule &ir,
+                   ::occa::properties &props);
+
+void SetProperties(FiniteElementSpace &trialFESpace,
+                   FiniteElementSpace &testFESpace,
+                   const mfem::IntegrationRule &ir,
+                   ::occa::properties &props);
+
+void SetTensorProperties(FiniteElementSpace &fespace,
+                         const mfem::IntegrationRule &ir,
+                         ::occa::properties &props);
+
+void SetTensorProperties(FiniteElementSpace &trialFESpace,
+                         FiniteElementSpace &testFESpace,
+                         const IntegrationRule &ir,
+                         ::occa::properties &props);
+
+void SetSimplexProperties(FiniteElementSpace &fespace,
+                          const IntegrationRule &ir,
+                          ::occa::properties &props);
+
+void SetSimplexProperties(FiniteElementSpace &trialFESpace,
+                          FiniteElementSpace &testFESpace,
+                          const IntegrationRule &ir,
+                          ::occa::properties &props);
+
+//---[ Base Integrator ]--------------
+class OccaIntegrator
+{
+protected:
+   SharedPtr<const Engine> engine;
+
+   OccaBilinearForm *bform;
+   mfem::Mesh *mesh;
+
+   FiniteElementSpace *otrialFESpace;
+   FiniteElementSpace *otestFESpace;
+
+   mfem::FiniteElementSpace *trialFESpace;
+   mfem::FiniteElementSpace *testFESpace;
+
+   ::occa::properties props;
+   OccaIntegratorType itype;
+
+   const IntegrationRule *ir;
+   bool hasTensorBasis;
+   OccaDofQuadMaps maps;
+   OccaDofQuadMaps mapsTranspose;
+
+public:
+   OccaIntegrator(const Engine &e);
+   virtual ~OccaIntegrator();
+
+   const Engine &OccaEngine() const { return *engine; }
+
+   ::occa::device GetDevice(int idx = 0) const
+   { return engine->GetDevice(idx); }
+
+   virtual std::string GetName() = 0;
+
+   FiniteElementSpace& GetTrialOccaFESpace() const;
+   FiniteElementSpace& GetTestOccaFESpace() const;
+
+   mfem::FiniteElementSpace& GetTrialFESpace() const;
+   mfem::FiniteElementSpace& GetTestFESpace() const;
+
+   void SetIntegrationRule(const mfem::IntegrationRule &ir_);
+   const mfem::IntegrationRule& GetIntegrationRule() const;
+
+   OccaDofQuadMaps& GetDofQuadMaps();
+
+   void SetupMaps();
+
+   virtual void SetupIntegrationRule() = 0;
+
+   virtual void SetupIntegrator(OccaBilinearForm &bform_,
+                                const ::occa::properties &props_,
+                                const OccaIntegratorType itype_);
+
+   virtual void Setup() = 0;
+
+   virtual void Assemble() = 0;
+   /// This method works on E-vectors!
+   virtual void MultAdd(Vector &x, Vector &y) = 0;
+
+   virtual void MultTransposeAdd(Vector &x, Vector &y)
+   {
+      mfem_error("OccaIntegrator::MultTransposeAdd() is not overloaded!");
+   }
+
+   OccaGeometry GetGeometry(const int flags = (OccaGeometry::Jacobian    |
+                                               OccaGeometry::JacobianInv |
+                                               OccaGeometry::JacobianDet));
+
+   ::occa::kernel GetAssembleKernel(const ::occa::properties &props);
+   ::occa::kernel GetMultAddKernel(const ::occa::properties &props);
+
+   ::occa::kernel GetKernel(const std::string &kernelName,
+                            const ::occa::properties &props);
+};
+//====================================
+
+
+//---[ Diffusion Integrator ]---------
+class OccaDiffusionIntegrator : public OccaIntegrator
+{
+private:
+   OccaCoefficient coeff;
+
+   ::occa::kernel assembleKernel, multKernel;
+
+   Vector assembledOperator;
+
+public:
+   OccaDiffusionIntegrator(const OccaCoefficient &coeff_);
+   virtual ~OccaDiffusionIntegrator();
+
+   virtual std::string GetName();
+
+   virtual void SetupIntegrationRule();
+
+   virtual void Setup();
+
+   virtual void Assemble();
+   virtual void MultAdd(Vector &x, Vector &y);
+};
+//====================================
+
+
+//---[ Mass Integrator ]--------------
+class OccaMassIntegrator : public OccaIntegrator
+{
+private:
+   OccaCoefficient coeff;
+
+   ::occa::kernel assembleKernel, multKernel;
+
+   Vector assembledOperator;
+
+public:
+   OccaMassIntegrator(const OccaCoefficient &coeff_);
+   virtual ~OccaMassIntegrator();
+
+   virtual std::string GetName();
+
+   virtual void SetupIntegrationRule();
+
+   virtual void Setup();
+
+   virtual void Assemble();
+   void SetOperator(Vector &v);
+
+   virtual void MultAdd(Vector &x, Vector &y);
+};
+//====================================
+
+//---[ Vector Mass Integrator ]--------------
+class OccaVectorMassIntegrator : public OccaIntegrator
+{
+private:
+   OccaCoefficient coeff;
+
+   ::occa::kernel assembleKernel, multKernel;
+
+   Vector assembledOperator;
+
+public:
+   OccaVectorMassIntegrator(const OccaCoefficient &coeff_);
+   virtual ~OccaVectorMassIntegrator();
+
+   virtual std::string GetName();
+
+   virtual void SetupIntegrationRule();
+
+   virtual void Setup();
+
+   virtual void Assemble();
+
+   virtual void MultAdd(Vector &x, Vector &y);
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP

backends/occa/coefficient.cpp

@@ -0,0 +1,344 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "coefficient.hpp"
+#include "bilininteg.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+//---[ Parameter ]------------
+OccaParameter::~OccaParameter() {}
+
+void OccaParameter::Setup(OccaIntegrator &integ,
+                          ::occa::properties &props) {}
+
+::occa::kernelArg OccaParameter::KernelArgs()
+{
+   return ::occa::kernelArg();
+}
+//====================================
+
+
+//---[ Include Parameter ]------------
+OccaIncludeParameter::OccaIncludeParameter(const std::string &filename_) :
+   filename(filename_) {}
+
+OccaParameter* OccaIncludeParameter::Clone()
+{
+   return new OccaIncludeParameter(filename);
+}
+
+void OccaIncludeParameter::Setup(OccaIntegrator &integ,
+                                 ::occa::properties &props)
+{
+   props["headers"].asArray() += "#include " + filename;
+}
+//====================================
+
+
+//---[ Source Parameter ]------------
+OccaSourceParameter::OccaSourceParameter(const std::string &source_) :
+   source(source_) {}
+
+OccaParameter* OccaSourceParameter::Clone()
+{
+   return new OccaSourceParameter(source);
+}
+
+void OccaSourceParameter::Setup(OccaIntegrator &integ,
+                                ::occa::properties &props)
+{
+   props["headers"].asArray() += source;
+}
+//====================================
+
+//---[ Vector Parameter ]-------
+OccaVectorParameter::OccaVectorParameter(const std::string &name_,
+                                         Vector &v_,
+                                         const bool useRestrict_) :
+   name(name_),
+   v(v_),
+   useRestrict(useRestrict_),
+   attr("") {}
+
+OccaVectorParameter::OccaVectorParameter(const std::string &name_,
+                                         Vector &v_,
+                                         const std::string &attr_,
+                                         const bool useRestrict_) :
+   name(name_),
+   v(v_),
+   useRestrict(useRestrict_),
+   attr(attr_) {}
+
+OccaParameter* OccaVectorParameter::Clone()
+{
+   return new OccaVectorParameter(name, v, attr, useRestrict);
+}
+
+void OccaVectorParameter::Setup(OccaIntegrator &integ,
+                                ::occa::properties &props)
+{
+   std::string &args = (props["defines/COEFF_ARGS"]
+                        .asString()
+                        .string());
+   args += "const double *";
+   if (useRestrict)
+   {
+      args += " restrict ";
+   }
+   args += name;
+   if (attr.size())
+   {
+      args += ' ';
+      args += attr;
+   }
+   args += ",\n";
+}
+
+::occa::kernelArg OccaVectorParameter::KernelArgs()
+{
+   return ::occa::kernelArg(v.OccaMem());
+}
+//====================================
+
+
+//---[ GridFunction Parameter ]-------
+OccaGridFunctionParameter::OccaGridFunctionParameter(const std::string &name_,
+                                                     OccaGridFunction &gf_,
+                                                     const bool useRestrict_)
+   : name(name_),
+     gf(gf_),
+     gfQuad(*(new Layout(gf_.OccaLayout().OccaEngine(), 0))),
+     useRestrict(useRestrict_) {}
+
+OccaParameter* OccaGridFunctionParameter::Clone()
+{
+   OccaGridFunctionParameter *param =
+      new OccaGridFunctionParameter(name, gf, useRestrict);
+   param->gfQuad.MakeRef(gfQuad);
+   return param;
+}
+
+void OccaGridFunctionParameter::Setup(OccaIntegrator &integ,
+                                      ::occa::properties &props)
+{
+
+   std::string &args = (props["defines/COEFF_ARGS"]
+                        .asString()
+                        .string());
+   args += "const double *";
+   if (useRestrict)
+   {
+      args += " restrict ";
+   }
+   args += name;
+   args += " @dim(NUM_QUAD, numElements),\n";
+
+   gf.ToQuad(integ.GetIntegrationRule(), gfQuad);
+}
+
+::occa::kernelArg OccaGridFunctionParameter::KernelArgs()
+{
+   return gfQuad.OccaMem();
+}
+//====================================
+
+
+//---[ Coefficient ]------------------
+OccaCoefficient::OccaCoefficient(const Engine &e, const double value) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   coeffValue = value;
+}
+
+OccaCoefficient::OccaCoefficient(const Engine &e, const std::string &source) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   coeffValue = source;
+}
+
+OccaCoefficient::OccaCoefficient(const Engine &e, const char *source) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   coeffValue = source;
+}
+
+OccaCoefficient::OccaCoefficient(const OccaCoefficient &coeff) :
+   engine(coeff.engine),
+   integ(NULL),
+   name(coeff.name),
+   coeffValue(coeff.coeffValue)
+{
+
+   const int paramCount = (int) coeff.params.size();
+   for (int i = 0; i < paramCount; ++i)
+   {
+      params.push_back(coeff.params[i]->Clone());
+   }
+}
+
+OccaCoefficient::~OccaCoefficient()
+{
+   const int paramCount = (int) params.size();
+   for (int i = 0; i < paramCount; ++i)
+   {
+      delete params[i];
+   }
+}
+
+OccaCoefficient& OccaCoefficient::SetName(const std::string &name_)
+{
+   name = name_;
+   return *this;
+}
+
+void OccaCoefficient::Setup(OccaIntegrator &integ_,
+                            ::occa::properties &props_)
+{
+   integ = &integ_;
+
+   const int paramCount = (int) params.size();
+   props_["defines"][name + "_ARGS"] = "";
+   for (int i = 0; i < paramCount; ++i)
+   {
+      params[i]->Setup(integ_, props_);
+   }
+   props_["defines"][name] = coeffValue;
+
+   props = props_;
+}
+
+OccaCoefficient& OccaCoefficient::Add(OccaParameter *param)
+{
+   params.push_back(param);
+   return *this;
+}
+
+OccaCoefficient& OccaCoefficient::IncludeHeader(const std::string &filename)
+{
+   return Add(new OccaIncludeParameter(filename));
+}
+
+OccaCoefficient& OccaCoefficient::IncludeSource(const std::string &source)
+{
+   return Add(new OccaSourceParameter(source));
+}
+
+OccaCoefficient& OccaCoefficient::AddVector(const std::string &name_,
+                                            Vector &v,
+                                            const bool useRestrict)
+{
+   return Add(new OccaVectorParameter(name_, v, useRestrict));
+}
+
+OccaCoefficient& OccaCoefficient::AddVector(const std::string &name_,
+                                            Vector &v,
+                                            const std::string &attr,
+                                            const bool useRestrict)
+{
+   return Add(new OccaVectorParameter(name_, v, attr, useRestrict));
+}
+
+OccaCoefficient& OccaCoefficient::AddGridFunction(const std::string &name_,
+                                                  OccaGridFunction &gf,
+                                                  const bool useRestrict)
+{
+   return Add(new OccaGridFunctionParameter(name_, gf, useRestrict));
+}
+
+bool OccaCoefficient::IsConstant()
+{
+   return coeffValue.isNumber();
+}
+
+double OccaCoefficient::GetConstantValue()
+{
+   if (!IsConstant())
+   {
+      mfem_error("OccaCoefficient is not constant");
+   }
+   return coeffValue.number();
+}
+
+Vector OccaCoefficient::Eval()
+{
+   if (integ == NULL)
+   {
+      mfem_error("OccaCoefficient requires a Setup() call before Eval()");
+   }
+
+   mfem::FiniteElementSpace &fespace = integ->GetTrialFESpace();
+   const mfem::IntegrationRule &ir   = integ->GetIntegrationRule();
+
+   const int elements = fespace.GetNE();
+   const int numQuad  = ir.GetNPoints();
+
+   Vector quadCoeff(*(new Layout(OccaEngine(), numQuad * elements)));
+   Eval(quadCoeff);
+   return quadCoeff;
+}
+
+void OccaCoefficient::Eval(Vector &quadCoeff)
+{
+   const std::string &okl_path = OccaEngine().GetOklPath();
+   const std::string &okl_defines = OccaEngine().GetOklDefines();
+   static ::occa::kernelBuilder builder =
+      ::occa::kernelBuilder::fromFile(okl_path + "coefficient.okl",
+                                      "CoefficientEval", okl_defines);
+
+   if (integ == NULL)
+   {
+      mfem_error("OccaCoefficient requires a Setup() call before Eval()");
+   }
+
+   const int elements = integ->GetTrialFESpace().GetNE();
+
+   ::occa::properties kernelProps = props;
+   if (name != "COEFF")
+   {
+      kernelProps["defines/COEFF"]      = name;
+      kernelProps["defines/COEFF_ARGS"] = name + "_ARGS";
+   }
+   kernelProps += okl_defines;
+
+   ::occa::kernel evalKernel = builder.build(GetDevice(), kernelProps);
+   evalKernel(elements, *this, quadCoeff.OccaMem());
+}
+
+OccaCoefficient::operator ::occa::kernelArg ()
+{
+   ::occa::kernelArg kArg;
+   const int paramCount = (int) params.size();
+   for (int i = 0; i < paramCount; ++i)
+   {
+      kArg.add(params[i]->KernelArgs());
+   }
+   return kArg;
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/coefficient.hpp

@@ -0,0 +1,284 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_COEFFICIENT_HPP
+#define MFEM_BACKENDS_OCCA_COEFFICIENT_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "vector.hpp"
+#include "gridfunc.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class OccaIntegrator;
+
+
+class OccaParameter
+{
+public:
+   virtual ~OccaParameter();
+
+   virtual OccaParameter* Clone() = 0;
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+
+   virtual ::occa::kernelArg KernelArgs();
+};
+
+
+//---[ Include Parameter ]------------
+class OccaIncludeParameter : public OccaParameter
+{
+private:
+   std::string filename;
+
+public:
+   OccaIncludeParameter(const std::string &filename_);
+
+   virtual OccaParameter* Clone();
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+};
+//====================================
+
+
+//---[ Source Parameter ]------------
+class OccaSourceParameter : public OccaParameter
+{
+private:
+   std::string source;
+
+public:
+   OccaSourceParameter(const std::string &filename_);
+
+   virtual OccaParameter* Clone();
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+};
+//====================================
+
+
+//---[ Define Parameter ]------------
+template <class TM>
+class OccaDefineParameter : public OccaParameter
+{
+private:
+   const std::string name;
+   TM value;
+
+public:
+   OccaDefineParameter(const std::string &name_,
+                       const TM &value_) :
+      name(name_),
+      value(value_) {}
+
+   virtual OccaParameter* Clone()
+   {
+      return new OccaDefineParameter(name, value);
+   }
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props)
+   {
+      props["defines"][name] = value;
+   }
+};
+//====================================
+
+
+//---[ Variable Parameter ]-----------
+template <class TM>
+class OccaVariableParameter : public OccaParameter
+{
+private:
+   const std::string name;
+   const TM &value;
+
+public:
+   OccaVariableParameter(const std::string &name_,
+                         const TM &value_) :
+      name(name_),
+      value(value_) {}
+
+   virtual OccaParameter* Clone()
+   {
+      return new OccaVariableParameter(name, value);
+   }
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props)
+   {
+      std::string &args = (props["defines/COEFF_ARGS"]
+                           .asString()
+                           .string());
+      // const TM name,\n"
+      args += "const ";
+      args += ::occa::primitiveinfo<TM>::name;
+      args += ' ';
+      args += name;
+      args += ",\n";
+   }
+
+   virtual ::occa::kernelArg KernelArgs()
+   {
+      return ::occa::kernelArg(value);
+   }
+};
+//====================================
+
+
+//---[ Vector Parameter ]-------
+class OccaVectorParameter : public OccaParameter
+{
+private:
+   const std::string name;
+   Vector v;
+   bool useRestrict;
+   std::string attr;
+
+public:
+   OccaVectorParameter(const std::string &name_,
+                       Vector &v_,
+                       const bool useRestrict_ = false);
+
+   OccaVectorParameter(const std::string &name_,
+                       Vector &v_,
+                       const std::string &attr_,
+                       const bool useRestrict_ = false);
+
+   virtual OccaParameter* Clone();
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+
+   virtual ::occa::kernelArg KernelArgs();
+};
+//====================================
+
+
+//---[ GridFunction Parameter ]-------
+class OccaGridFunctionParameter : public OccaParameter
+{
+private:
+   const std::string name;
+   OccaGridFunction &gf;
+   Vector gfQuad;
+   bool useRestrict;
+
+public:
+   OccaGridFunctionParameter(const std::string &name_,
+                             OccaGridFunction &gf_,
+                             const bool useRestrict_ = false);
+
+   virtual OccaParameter* Clone();
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+
+   virtual ::occa::kernelArg KernelArgs();
+};
+//====================================
+
+
+//---[ Coefficient ]------------------
+// [MISSING]
+// Needs to know about the integrator's
+//   - fespace
+//   - ir
+// Step where parameters that need the ir get called for setup
+// For example, GridFunction (d, e) -> (q, e)
+class OccaCoefficient
+{
+private:
+   SharedPtr<const Engine> engine;
+
+   OccaIntegrator *integ;
+
+   std::string name;
+   ::occa::json coeffValue;
+
+   ::occa::properties props;
+   std::vector<OccaParameter*> params;
+
+public:
+   OccaCoefficient(const Engine &e, const double value = 1.0);
+   OccaCoefficient(const Engine &e, const std::string &source);
+   OccaCoefficient(const Engine &e, const char *source);
+   ~OccaCoefficient();
+
+   OccaCoefficient(const OccaCoefficient &coeff);
+
+   const Engine &OccaEngine() const { return *engine; }
+
+   ::occa::device GetDevice(int idx = 0) const
+   { return engine->GetDevice(idx); }
+
+   OccaCoefficient& SetName(const std::string &name_);
+
+   void Setup(OccaIntegrator &integ_,
+              ::occa::properties &props_);
+
+   OccaCoefficient& Add(OccaParameter *param);
+
+   OccaCoefficient& IncludeHeader(const std::string &filename);
+   OccaCoefficient& IncludeSource(const std::string &source);
+
+   template <class TM>
+   OccaCoefficient& AddDefine(const std::string &name_, const TM &value)
+   {
+      return Add(new OccaDefineParameter<TM>(name_, value));
+   }
+
+   template <class TM>
+   OccaCoefficient& AddVariable(const std::string &name_, const TM &value)
+   {
+      return Add(new OccaVariableParameter<TM>(name_, value));
+   }
+
+   OccaCoefficient& AddVector(const std::string &name_,
+                              Vector &v,
+                              const bool useRestrict = false);
+
+
+   OccaCoefficient& AddVector(const std::string &name_,
+                              Vector &v,
+                              const std::string &attr,
+                              const bool useRestrict = false);
+
+   OccaCoefficient& AddGridFunction(const std::string &name_,
+                                    OccaGridFunction &gf,
+                                    const bool useRestrict = false);
+
+   bool IsConstant();
+   double GetConstantValue();
+
+   Vector Eval();
+   void Eval(Vector &quadCoeff);
+
+   operator ::occa::kernelArg ();
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_COEFFICIENT_HPP

backends/occa/defines.okl

@@ -0,0 +1,40 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_OCCA_DEFINES
+#define MFEM_OCCA_DEFINES
+
+#ifndef USING_TENSOR_OPS
+#  define USING_TENSOR_OPS 0
+#endif
+
+#ifdef OCCA_USING_GPU
+#  define GPU_ORDER_2(I0, I1)         @dimOrder(I0, I1)
+#  define GPU_ORDER_3(I0, I1, I2)     @dimOrder(I0, I1, I2)
+#  define GPU_ORDER_4(I0, I1, I2, I3) @dimOrder(I0, I1, I2, I3)
+#else
+#  define GPU_ORDER_2(I0, I1)         @dimOrder(0, 1)
+#  define GPU_ORDER_3(I0, I1, I2)     @dimOrder(0, 1, 2)
+#  define GPU_ORDER_4(I0, I1, I2, I3) @dimOrder(0, 1, 2, 3)
+#endif
+
+#ifndef COEFF
+#  define COEFF 1.0
+#  define COEFF_ARGS
+#endif
+
+#if USING_TENSOR_OPS
+#  include "mfem-occa://defines/tensor.okl"
+#else
+#  include "mfem-occa://defines/simplex.okl"
+#endif
+
+#endif

backends/occa/defines/simplex.okl

@@ -0,0 +1,40 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#define USING_LOW_ORDER 1
+#define USING_HI_ORDER  0
+
+typedef double* DofToQuad_t @dim(NUM_QUAD, NUM_DOFS);
+typedef double* DofToQuadD2D_t @dim(2, NUM_QUAD, NUM_DOFS);
+typedef double* DofToQuadD3D_t @dim(3, NUM_QUAD, NUM_DOFS);
+
+typedef double* QuadToDof_t @dim(NUM_DOFS, NUM_QUAD);
+typedef double* QuadToDofD2D_t @dim(2, NUM_DOFS, NUM_QUAD);
+typedef double* QuadToDofD3D_t @dim(3, NUM_DOFS, NUM_QUAD);
+
+typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD, numElements);
+typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
+
+typedef double* SymmOperator2D_t @dim(3, NUM_QUAD, numElements);
+typedef double* SymmOperator3D_t @dim(6, NUM_QUAD, numElements);
+
+typedef double* DLocal_t @dim(NUM_DOFS, numElements);
+typedef double* QLocal_t @dim(NUM_QUAD, numElements);
+
+#if VDIM_ORDERING == ORDERING_BY_VDIM
+typedef double* DVLocal_t @dim(NUM_VDIM, NUM_DOFS, numElements);
+typedef double* QVLocal_t @dim(NUM_VDIM, NUM_QUAD, numElements);
+#else
+typedef double* DVLocal_t @dim(NUM_VDIM, NUM_DOFS, numElements) @dimOrder(2,0,1);
+typedef double* QVLocal_t @dim(NUM_VDIM, NUM_QUAD, numElements) @dimOrder(2,0,1);
+#endif
+
+typedef int* DLocalMap_t @dim(NUM_DOFS, numElements);

backends/occa/defines/tensor.okl

@@ -0,0 +1,85 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#if NUM_QUAD_1D < NUM_DOFS_1D
+#  define NUM_MAX_1D NUM_DOFS_1D
+#else
+#  define NUM_MAX_1D NUM_QUAD_1D
+#endif
+
+#define NUM_MAX_2D (NUM_MAX_1D * NUM_MAX_1D)
+
+#define NUM_QUAD_DOFS_1D (NUM_QUAD_1D * NUM_DOFS_1D)
+
+#define QUAD_2D_ID(X, Y) (X + ((Y) * NUM_QUAD_1D))
+#define DOFS_2D_ID(X, Y) (X + ((Y) * NUM_DOFS_1D))
+
+#define QUAD_3D_ID(X, Y, Z) (X + ((Y) * NUM_QUAD_1D) + ((Z) * NUM_QUAD_2D))
+#define DOFS_3D_ID(X, Y, Z) (X + ((Y) * NUM_DOFS_1D) + ((Z) * NUM_DOFS_2D))
+
+#if NUM_MAX_1D < 8
+#  define USING_LOW_ORDER 1
+#  define USING_HI_ORDER  0
+#else
+#  define USING_LOW_ORDER 0
+#  define USING_HI_ORDER  1
+#endif
+
+#define M1_ELEMENT_BATCHES (M1_OUTER_ELEMENT_BATCH * M1_INNER_ELEMENT_BATCH)
+
+typedef double* DofToQuad_t @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+typedef double* QuadToDof_t @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+typedef double* Jacobian_t   @dim(NUM_DIM, NUM_DIM, numElements);
+typedef double* Jacobian1D_t @dim(NUM_QUAD_1D, numElements);
+typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD_2D, numElements);
+typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD_3D, numElements);
+
+typedef double* SymmOperator1D_t @dim(NUM_QUAD_1D, numElements);
+typedef double* SymmOperator2D_t @dim(3, NUM_QUAD_2D, numElements);
+typedef double* SymmOperator3D_t @dim(6, NUM_QUAD_3D, numElements);
+
+typedef double* DLocal_t   @dim(NUM_DOFS, numElements);
+typedef double* DLocal1D_t @dim(NUM_DOFS_1D, numElements);
+typedef double* DLocal2D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+typedef double* DLocal3D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+
+typedef double* QLocal_t   @dim(NUM_QUAD, numElements);
+typedef double* QLocal1D_t @dim(NUM_QUAD_1D, numElements);
+typedef double* QLocal2D_t @dim(NUM_QUAD_1D, NUM_QUAD_1D, numElements);
+typedef double* QLocal3D_t @dim(NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
+
+#if VDIM_ORDERING == ORDERING_BY_VDIM
+typedef double* DVLocal_t   @dim(NUM_VDIM, NUM_DOFS, numElements);
+typedef double* DVLocal1D_t @dim(NUM_VDIM, NUM_DOFS_1D, numElements);
+typedef double* DVLocal2D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+typedef double* DVLocal3D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+
+typedef double* QVLocal_t   @dim(NUM_VDIM, NUM_QUAD, numElements);
+typedef double* QVLocal1D_t @dim(NUM_VDIM, NUM_QUAD_1D, numElements);
+typedef double* QVLocal2D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
+typedef double* QVLocal3D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
+#else
+typedef double* DVLocal_t   @dim(NUM_VDIM, NUM_DOFS, numElements) @dimOrder(2,0,1);
+typedef double* DVLocal1D_t @dim(NUM_VDIM, NUM_DOFS_1D, numElements) @dimOrder(2,0,1);
+typedef double* DVLocal2D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, numElements) @dimOrder(3,0,1,2);
+typedef double* DVLocal3D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements) @dimOrder(4,0,1,2,3);
+
+typedef double* QVLocal_t   @dim(NUM_VDIM, NUM_QUAD, numElements) @dimOrder(2,0,1);
+typedef double* QVLocal1D_t @dim(NUM_VDIM, NUM_QUAD_1D, numElements) @dimOrder(2,0,1);
+typedef double* QVLocal2D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, numElements) @dimOrder(3,0,1,2);
+typedef double* QVLocal3D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements) @dimOrder(4,0,1,2,3);
+#endif
+
+typedef int* DLocalMap_t   @dim(NUM_DOFS, numElements);
+typedef int* DLocalMap1D_t @dim(NUM_DOFS_1D, numElements);
+typedef int* DLocalMap2D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+typedef int* DLocalMap3D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);

backends/occa/diffusion/simplex/cpu.okl

@@ -0,0 +1,168 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        const double * restrict quadWeights,
+                        const Jacobian2D_t restrict J,
+                        COEFF_ARGS
+                        SymmOperator2D_t restrict oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+      const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
+
+      oper(0, q, e) =  c_detJ * (J12*J12 + J22*J22); // (1,1)
+      oper(1, q, e) = -c_detJ * (J12*J11 + J22*J21); // (1,2) + (2,1)
+      oper(2, q, e) =  c_detJ * (J11*J11 + J21*J21); // (2,2)
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuadD2D_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDofD2D_t restrict quadToDofD,
+                       const SymmOperator2D_t restrict oper,
+                       const DLocal_t restrict solIn,
+                       DLocal_t restrict solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double r_sol[NUM_DOFS];
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        r_sol[d] = 0;
+      }
+
+      for (int q = 0; q < NUM_QUAD; ++q) {
+        double gradX = 0, gradY = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          const double s = solIn(d, e);
+          gradX += s * quadToDofD(0, d, q);
+          gradY += s * quadToDofD(1, d, q);
+        }
+        const double O11 = oper(0, q, e);
+        const double O12 = oper(1, q, e);
+        const double O22 = oper(2, q, e);
+
+        const double gradX2 = (O11 * gradX) + (O12 * gradY);
+        const double gradY2 = (O12 * gradX) + (O22 * gradY);
+
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          r_sol[d] += ((gradX2 * quadToDofD(0, d, q)) +
+                       (gradY2 * quadToDofD(1, d, q)));
+        }
+      }
+
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        solOut(d, e) += r_sol[d];
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        const double * restrict quadWeights,
+                        const Jacobian3D_t restrict J,
+                        COEFF_ARGS
+                        SymmOperator3D_t restrict oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+      const double c_detJ = quadWeights[q] * COEFF / detJ;
+
+      // adj(J)
+      const double A11 = (J22 * J33) - (J23 * J32);
+      const double A12 = (J23 * J31) - (J21 * J33);
+      const double A13 = (J21 * J32) - (J22 * J31);
+
+      const double A21 = (J13 * J32) - (J12 * J33);
+      const double A22 = (J11 * J33) - (J13 * J31);
+      const double A23 = (J12 * J31) - (J11 * J32);
+
+      const double A31 = (J12 * J23) - (J13 * J22);
+      const double A32 = (J13 * J21) - (J11 * J23);
+      const double A33 = (J11 * J22) - (J12 * J21);
+
+      // adj(J)^Tadj(J)
+      oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
+      oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2) + (2,1)
+      oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3) + (3,1)
+      oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
+      oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3) + (3,2)
+      oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuadD3D_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDofD3D_t restrict quadToDofD,
+                       const SymmOperator3D_t restrict oper,
+                       const DLocal_t restrict solIn,
+                       DLocal_t restrict solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double r_sol[NUM_DOFS];
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        r_sol[d] = 0;
+      }
+
+      for (int q = 0; q < NUM_QUAD; ++q) {
+        double gradX = 0, gradY = 0, gradZ = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          const double s = solIn(d, e);
+          gradX += s * quadToDofD(0, d, q);
+          gradY += s * quadToDofD(1, d, q);
+          gradZ += s * quadToDofD(2, d, q);
+        }
+        const double O11 = oper(0, q, e);
+        const double O12 = oper(1, q, e);
+        const double O13 = oper(2, q, e);
+        const double O22 = oper(3, q, e);
+        const double O23 = oper(4, q, e);
+        const double O33 = oper(5, q, e);
+
+        const double gradX2 = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
+        const double gradY2 = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
+        const double gradZ2 = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
+
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          r_sol[d] += ((gradX2 * quadToDofD(0, d, q)) +
+                       (gradY2 * quadToDofD(1, d, q)) +
+                       (gradZ2 * quadToDofD(2, d, q)));
+        }
+      }
+
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        solOut(d, e) += r_sol[d];
+      }
+    }
+  }
+}
+//======================================

backends/occa/diffusion/simplex/gpuHighOrder.okl

@@ -0,0 +1,182 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian2D_t J,
+                        COEFF_ARGS
+                        SymmOperator2D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD; q += A2_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+            const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
+
+            oper(0, q, e) =  c_detJ * (J12*J12 + J22*J22); // (1,1)
+            oper(1, q, e) = -c_detJ * (J12*J11 + J22*J21); // (1,2) + (2,1)
+            oper(2, q, e) =  c_detJ * (J11*J11 + J21*J21); // (2,2)
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuadD2D_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDofD2D_t restrict quadToDofD,
+                       const SymmOperator2D_t restrict oper,
+                       const DLocal_t restrict solIn,
+                       DLocal_t restrict solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_gradX[NUM_QUAD];
+    @shared double s_gradY[NUM_QUAD];
+
+    for (int qOff = 0; qOff < M2_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M2_INNER_BATCH) {
+        double gradX = 0, gradY = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          const double s = solIn(d, e);
+          gradX += s * quadToDofD(0, d, q);
+          gradY += s * quadToDofD(1, d, q);
+        }
+        const double O11 = oper(0, q, e);
+        const double O12 = oper(1, q, e);
+        const double O22 = oper(2, q, e);
+
+        s_gradX[q] = (O11 * gradX) + (O12 * gradY);
+        s_gradY[q] = (O12 * gradX) + (O22 * gradY);
+      }
+    }
+
+    for (int dOff = 0; dOff < M2_INNER_BATCH; ++dOff) {
+      for (int d = dOff; d < NUM_DOFS; d += M2_INNER_BATCH) {
+        double r_sol = 0;
+        for (int q = 0; q < NUM_QUAD; ++q) {
+          // FIXME: s_gradX and s_gradY are @shared used outside of @inner
+          r_sol += ((s_gradX[q] * quadToDofD(0, d, q)) +
+                    (s_gradY[q] * quadToDofD(1, d, q)));
+        }
+        solOut(d, e) += r_sol;
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian3D_t J,
+                        COEFF_ARGS
+                        SymmOperator3D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD; q += A3_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+            const double c_detJ = quadWeights[q] * COEFF / detJ;
+
+            // adj(J)
+            const double A11 = (J22 * J33) - (J23 * J32);
+            const double A12 = (J23 * J31) - (J21 * J33);
+            const double A13 = (J21 * J32) - (J22 * J31);
+
+            const double A21 = (J13 * J32) - (J12 * J33);
+            const double A22 = (J11 * J33) - (J13 * J31);
+            const double A23 = (J12 * J31) - (J11 * J32);
+
+            const double A31 = (J12 * J23) - (J13 * J22);
+            const double A32 = (J13 * J21) - (J11 * J23);
+            const double A33 = (J11 * J22) - (J12 * J21);
+
+            // adj(J)^Tadj(J)
+            oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
+            oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2) + (2,1)
+            oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3) + (3,1)
+            oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
+            oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3) + (3,2)
+            oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuadD3D_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDofD3D_t restrict quadToDofD,
+                       const SymmOperator3D_t restrict oper,
+                       const DLocal_t restrict solIn,
+                       DLocal_t restrict solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_gradX[NUM_QUAD];
+    @shared double s_gradY[NUM_QUAD];
+    @shared double s_gradZ[NUM_QUAD];
+
+    for (int qOff = 0; qOff < M3_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M3_INNER_BATCH) {
+        double gradX = 0, gradY = 0, gradZ = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          const double s = solIn(d, e);
+          gradX += s * quadToDofD(0, d, q);
+          gradY += s * quadToDofD(1, d, q);
+          gradZ += s * quadToDofD(2, d, q);
+        }
+        const double O11 = oper(0, q, e);
+        const double O12 = oper(1, q, e);
+        const double O13 = oper(2, q, e);
+        const double O22 = oper(3, q, e);
+        const double O23 = oper(4, q, e);
+        const double O33 = oper(5, q, e);
+
+        s_gradX[q] = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
+        s_gradY[q] = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
+        s_gradZ[q] = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
+      }
+    }
+
+    for (int dOff = 0; dOff < M3_INNER_BATCH; ++dOff) {
+      for (int d = dOff; d < NUM_DOFS; d += M3_INNER_BATCH) {
+        double r_sol = 0;
+        for (int q = 0; q < NUM_QUAD; ++q) {
+          r_sol += ((s_gradX[q] * quadToDofD(0, d, q)) +
+                    (s_gradY[q] * quadToDofD(1, d, q)) +
+                    (s_gradZ[q] * quadToDofD(2, d, q)));
+        }
+        solOut(d, e) += r_sol;
+      }
+    }
+  }
+}
+//======================================

backends/occa/diffusion/tensor/cpu.okl

@@ -0,0 +1,370 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void Assemble1D(const int numElements,
+                        const double * restrict quadWeights,
+                        const Jacobian1D_t restrict J,
+                        COEFF_ARGS
+                        SymmOperator1D_t restrict oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
+      oper(q, e) = quadWeights[q] * COEFF / J(q, e);
+    }
+  }
+}
+
+@kernel void MultAdd1D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator1D_t restrict oper,
+                       const DLocal1D_t restrict solIn,
+                       DLocal1D_t restrict solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double grad[NUM_QUAD_1D];
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        grad[qx] = 0;
+      }
+      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+        const double s = solIn(dx, e);
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          grad[qx] += s * dofToQuadD(qx, dx);
+        }
+      }
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        grad[qx] *= oper(qx, e);
+      }
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        const double gradX = grad[qx];
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          solOut(dx, e) += gradX * quadToDofD(dx, qx);
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        const double * restrict quadWeights,
+                        const Jacobian2D_t restrict J,
+                        COEFF_ARGS
+                        SymmOperator2D_t restrict oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_2D; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+      const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
+
+      oper(0, q, e) =  c_detJ * (J21*J21 + J22*J22); // (1,1)
+      oper(1, q, e) = -c_detJ * (J21*J11 + J22*J12); // (1,2), (2,1)
+      oper(2, q, e) =  c_detJ * (J11*J11 + J12*J12); // (2,2)
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator2D_t restrict oper,
+                       const DLocal2D_t restrict solIn,
+                       DLocal2D_t restrict solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double grad[NUM_QUAD_1D][NUM_QUAD_1D][2];
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          grad[qy][qx][0] = 0;
+          grad[qy][qx][1] = 0;
+        }
+      }
+
+      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+        double gradX[NUM_QUAD_1D][2];
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          gradX[qx][0] = 0;
+          gradX[qx][1] = 0;
+        }
+
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          const double s = solIn(dx, dy, e);
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            gradX[qx][0] += s * dofToQuad(qx, dx);
+            gradX[qx][1] += s * dofToQuadD(qx, dx);
+          }
+        }
+
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          const double wy  = dofToQuad(qy, dy);
+          const double wDy = dofToQuadD(qy, dy);
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            grad[qy][qx][0] += gradX[qx][1] * wy;
+            grad[qy][qx][1] += gradX[qx][0] * wDy;
+          }
+        }
+      }
+
+      // Calculate Dxy, xDy in plane
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          const int q = QUAD_2D_ID(qx, qy);
+          const double O11 = oper(0, q, e);
+          const double O12 = oper(1, q, e);
+          const double O22 = oper(2, q, e);
+
+          const double gradX = grad[qy][qx][0];
+          const double gradY = grad[qy][qx][1];
+
+          grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
+          grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
+        }
+      }
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        double gradX[NUM_DOFS_1D][2];
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          gradX[dx][0] = 0;
+          gradX[dx][1] = 0;
+        }
+
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          const double gX = grad[qy][qx][0];
+          const double gY = grad[qy][qx][1];
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double wx  = quadToDof(dx, qx);
+            const double wDx = quadToDofD(dx, qx);
+            gradX[dx][0] += gX * wDx;
+            gradX[dx][1] += gY * wx;
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          const double wy  = quadToDof(dy, qy);
+          const double wDy = quadToDofD(dy, qy);
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            solOut(dx, dy, e) += ((gradX[dx][0] * wy) +
+                                  (gradX[dx][1] * wDy));
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        const double * restrict quadWeights,
+                        const Jacobian3D_t restrict J,
+                        COEFF_ARGS
+                        SymmOperator3D_t restrict oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_3D; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+      const double c_detJ = quadWeights[q] * COEFF / detJ;
+
+      // adj(J)
+      const double A11 = (J22 * J33) - (J23 * J32);
+      const double A12 = (J23 * J31) - (J21 * J33);
+      const double A13 = (J21 * J32) - (J22 * J31);
+
+      const double A21 = (J13 * J32) - (J12 * J33);
+      const double A22 = (J11 * J33) - (J13 * J31);
+      const double A23 = (J12 * J31) - (J11 * J32);
+
+      const double A31 = (J12 * J23) - (J13 * J22);
+      const double A32 = (J13 * J21) - (J11 * J23);
+      const double A33 = (J11 * J22) - (J12 * J21);
+
+      // adj(J)^Tadj(J)
+      oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
+      oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2), (2,1)
+      oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3), (3,1)
+      oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
+      oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3), (3,2)
+      oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator3D_t restrict oper,
+                       const DLocal3D_t restrict solIn,
+                       DLocal3D_t restrict solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double grad[NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D][4];
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            grad[qz][qy][qx][0] = 0;
+            grad[qz][qy][qx][1] = 0;
+            grad[qz][qy][qx][2] = 0;
+          }
+        }
+      }
+
+      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+        double gradXY[NUM_QUAD_1D][NUM_QUAD_1D][4];
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            gradXY[qy][qx][0] = 0;
+            gradXY[qy][qx][1] = 0;
+            gradXY[qy][qx][2] = 0;
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          double gradX[NUM_QUAD_1D][2];
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            gradX[qx][0] = 0;
+            gradX[qx][1] = 0;
+          }
+
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double s = solIn(dx, dy, dz, e);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              gradX[qx][0] += s * dofToQuad(qx, dx);
+              gradX[qx][1] += s * dofToQuadD(qx, dx);
+            }
+          }
+
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            const double wy  = dofToQuad(qy, dy);
+            const double wDy = dofToQuadD(qy, dy);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              const double wx  = gradX[qx][0];
+              const double wDx = gradX[qx][1];
+              gradXY[qy][qx][0] += wDx * wy;
+              gradXY[qy][qx][1] += wx  * wDy;
+              gradXY[qy][qx][2] += wx  * wy;
+            }
+          }
+        }
+
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          const double wz  = dofToQuad(qz, dz);
+          const double wDz = dofToQuadD(qz, dz);
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              grad[qz][qy][qx][0] += gradXY[qy][qx][0] * wz;
+              grad[qz][qy][qx][1] += gradXY[qy][qx][1] * wz;
+              grad[qz][qy][qx][2] += gradXY[qy][qx][2] * wDz;
+            }
+          }
+        }
+      }
+
+      // Calculate Dxyz, xDyz, xyDz in plane
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            const int q = QUAD_3D_ID(qx, qy, qz);
+            const double O11 = oper(0, q, e);
+            const double O12 = oper(1, q, e);
+            const double O13 = oper(2, q, e);
+            const double O22 = oper(3, q, e);
+            const double O23 = oper(4, q, e);
+            const double O33 = oper(5, q, e);
+
+            const double gradX = grad[qz][qy][qx][0];
+            const double gradY = grad[qz][qy][qx][1];
+            const double gradZ = grad[qz][qy][qx][2];
+
+            grad[qz][qy][qx][0] = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
+            grad[qz][qy][qx][1] = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
+            grad[qz][qy][qx][2] = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
+          }
+        }
+      }
+
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        double gradXY[NUM_DOFS_1D][NUM_DOFS_1D][4];
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            gradXY[dy][dx][0] = 0;
+            gradXY[dy][dx][1] = 0;
+            gradXY[dy][dx][2] = 0;
+          }
+        }
+
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          double gradX[NUM_DOFS_1D][4];
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            gradX[dx][0] = 0;
+            gradX[dx][1] = 0;
+            gradX[dx][2] = 0;
+          }
+
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            const double gX = grad[qz][qy][qx][0];
+            const double gY = grad[qz][qy][qx][1];
+            const double gZ = grad[qz][qy][qx][2];
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              const double wx  = quadToDof(dx, qx);
+              const double wDx = quadToDofD(dx, qx);
+              gradX[dx][0] += gX * wDx;
+              gradX[dx][1] += gY * wx;
+              gradX[dx][2] += gZ * wx;
+            }
+          }
+
+          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+            const double wy  = quadToDof(dy, qy);
+            const double wDy = quadToDofD(dy, qy);
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              gradXY[dy][dx][0] += gradX[dx][0] * wy;
+              gradXY[dy][dx][1] += gradX[dx][1] * wDy;
+              gradXY[dy][dx][2] += gradX[dx][2] * wy;
+            }
+          }
+        }
+
+        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+          const double wz  = quadToDof(dz, qz);
+          const double wDz = quadToDofD(dz, qz);
+          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              solOut(dx, dy, dz, e) += ((gradXY[dy][dx][0] * wz) +
+                                        (gradXY[dy][dx][1] * wz) +
+                                        (gradXY[dy][dx][2] * wDz));
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/diffusion/tensor/gpuHighOrder.okl

@@ -0,0 +1,433 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void Assemble1D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian1D_t J,
+                        COEFF_ARGS
+                        SymmOperator1D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A1_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A1_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
+          oper(q, e) = quadWeights[q] * COEFF / J(q, e);
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd1D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator1D_t restrict oper,
+                       const DLocal1D_t restrict solIn,
+                       DLocal1D_t restrict solOut) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
+    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    @exclusive double grad[NUM_QUAD_1D];
+
+    for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+      for (int i = el; i < NUM_QUAD_DOFS_1D; i += M1_INNER_ELEMENT_BATCH) {
+        s_dofToQuadD[i] = dofToQuadD[i];
+        s_quadToDofD[i] = quadToDofD[i];
+      }
+    }
+
+    for (int b = 0; b < M1_OUTER_ELEMENT_BATCH; ++b) {
+      for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+        const int e = eOff + b*M1_INNER_ELEMENT_BATCH + el;
+        if (e < numElements) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            grad[qx] = 0;
+          }
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double s = solIn(dx, e);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              grad[qx] += s * s_dofToQuadD(qx, dx);
+            }
+          }
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            grad[qx] *= oper(qx, e);
+          }
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            double s = 0;
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              s += grad[qx] * s_quadToDofD(dx, qx);
+            }
+            solOut(dx, e) += s;
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian2D_t J,
+                        COEFF_ARGS
+                        SymmOperator2D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD_2D; q += A2_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+            const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
+
+            oper(0, q, e) =  c_detJ * (J21*J21 + J22*J22); // (1,1)
+            oper(1, q, e) = -c_detJ * (J21*J11 + J22*J12); // (1,2), (2,1)
+            oper(2, q, e) =  c_detJ * (J11*J11 + J12*J12); // (2,2)
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator2D_t restrict oper,
+                       const DLocal2D_t restrict solIn,
+                       DLocal2D_t restrict solOut) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    // Store xy planes in shared memory
+    @shared double s_xy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_xDy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_grad[2 * NUM_QUAD_2D] @dim(2, NUM_QUAD_1D, NUM_QUAD_1D);
+
+    @exclusive double r_x[NUM_MAX_1D];
+    @exclusive double r_y[NUM_QUAD_1D];
+
+    for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+      for (int id = x; id < NUM_QUAD_DOFS_1D; id += NUM_MAX_1D) {
+        s_dofToQuad[id]  = dofToQuad[id];
+        s_dofToQuadD[id] = dofToQuadD[id];
+        s_quadToDof[id]  = quadToDof[id];
+        s_quadToDofD[id] = quadToDofD[id];
+      }
+    }
+
+    for (int e = eOff; e < (eOff + M2_ELEMENT_BATCH); ++e) {
+      if (e < numElements) {
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx; @inner) {
+          if (dx < NUM_DOFS_1D) {
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              s_xy(dx, qy) = 0;
+              s_xDy(dx, qy) = 0;
+            }
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              r_x[dy] = solIn(dx, dy, e);
+            }
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              double xy = 0;
+              double xDy = 0;
+              for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+                xy  += r_x[dy] * s_dofToQuad(qy, dy);
+                xDy += r_x[dy] * s_dofToQuadD(qy, dy);
+              }
+              s_xy(dx, qy)  = xy;
+              s_xDy(dx, qy) = xDy;
+            }
+          }
+        }
+        for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+          if (qy < NUM_QUAD_1D) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              double gradX = 0, gradY = 0;
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                gradX += s_xy(dx, qy)  * s_dofToQuadD(qx, dx);
+                gradY += s_xDy(dx, qy) * s_dofToQuad(qx, dx);
+              }
+
+              const int q = QUAD_2D_ID(qx, qy);
+              const double O11 = oper(0, q, e);
+              const double O12 = oper(1, q, e);
+              const double O22 = oper(2, q, e);
+
+              s_grad(0, qx, qy) = (O11 * gradX) + (O12 * gradY);
+              s_grad(1, qx, qy) = (O12 * gradX) + (O22 * gradY);
+            }
+          }
+        }
+
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx; @inner) {
+          if (qx < NUM_QUAD_1D) {
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              s_xy(dy, qx) = 0;
+              s_xDy(dy, qx) = 0;
+            }
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              r_x[qy] = s_grad(0, qx, qy);
+              r_y[qy] = s_grad(1, qx, qy);
+            }
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              double xy  = 0;
+              double xDy = 0;
+              for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+                xy  += r_x[qy] * s_quadToDof(dy, qy);
+                xDy += r_y[qy] * s_quadToDofD(dy, qy);
+              }
+              s_xy(dy, qx) = xy;
+              s_xDy(dy, qx) = xDy;
+            }
+          }
+        }
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx; @inner) {
+          if (dx < NUM_DOFS_1D) {
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              double s = 0;
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                s += ((s_xy(dy, qx) * s_quadToDofD(dx, qx)) +
+                      (s_xDy(dy, qx) * s_quadToDof(dx, qx)));
+              }
+              solOut(dx, dy, e) += s;
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian3D_t J,
+                        COEFF_ARGS
+                        SymmOperator3D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD_3D; q += A3_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+            const double c_detJ = quadWeights[q] * COEFF / detJ;
+
+            // adj(J)
+            const double A11 = (J22 * J33) - (J23 * J32);
+            const double A12 = (J23 * J31) - (J21 * J33);
+            const double A13 = (J21 * J32) - (J22 * J31);
+
+            const double A21 = (J13 * J32) - (J12 * J33);
+            const double A22 = (J11 * J33) - (J13 * J31);
+            const double A23 = (J12 * J31) - (J11 * J32);
+
+            const double A31 = (J12 * J23) - (J13 * J22);
+            const double A32 = (J13 * J21) - (J11 * J23);
+            const double A33 = (J11 * J22) - (J12 * J21);
+
+            // adj(J)^Tadj(J)
+            oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
+            oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2), (2,1)
+            oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3), (3,1)
+            oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
+            oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3), (3,2)
+            oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       const DofToQuad_t restrict dofToQuad,
+                       const DofToQuad_t restrict dofToQuadD,
+                       const QuadToDof_t restrict quadToDof,
+                       const QuadToDof_t restrict quadToDofD,
+                       const SymmOperator3D_t restrict oper,
+                       const DLocal3D_t restrict solIn,
+                       DLocal3D_t restrict solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    // Store xy planes in shared memory
+    @shared double s_z[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
+    @shared double s_Dz[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
+    @shared double s_xyDz[NUM_QUAD_2D] @dim(NUM_QUAD_1D, NUM_QUAD_1D);
+
+    // Store z axis as registers
+    @exclusive double r_qz[NUM_QUAD_1D];
+    @exclusive double r_qDz[NUM_QUAD_1D];
+    @exclusive double r_dDxyz[NUM_DOFS_1D];
+    @exclusive double r_dxDyz[NUM_DOFS_1D];
+    @exclusive double r_dxyDz[NUM_DOFS_1D];
+
+    for (int y = 0; y < NUM_MAX_1D; ++y; @inner) {
+      for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+        const int id = (y * NUM_MAX_1D) + x;
+        // Fetch Q <--> D maps
+        if (id < NUM_QUAD_DOFS_1D) {
+          s_dofToQuad[id]  = dofToQuad[id];
+          s_dofToQuadD[id] = dofToQuadD[id];
+          s_quadToDof[id]  = quadToDof[id];
+          s_quadToDofD[id] = quadToDofD[id];
+        }
+        // Initialize our Z axis
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          r_qz[qz] = 0;
+          r_qDz[qz] = 0;
+        }
+        // Initialize our solution updates in the Z axis
+        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+          r_dDxyz[dz] = 0;
+          r_dxDyz[dz] = 0;
+          r_dxyDz[dz] = 0;
+        }
+      }
+    }
+
+    for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+      for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+        if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+          for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+            const double s = solIn(dx, dy, dz, e);
+            // Calculate D -> Q in the Z axis
+            for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+              r_qz[qz]  += s * s_dofToQuad(qz, dz);
+              r_qDz[qz] += s * s_dofToQuadD(qz, dz);
+            }
+          }
+        }
+      }
+    }
+    // For each xy plane
+    for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+      // Fill xy plane at given z position
+      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+            s_z(dx, dy)  = r_qz[qz];
+            s_Dz(dx, dy) = r_qDz[qz];
+          }
+        }
+      }
+      // Calculate Dxyz, xDyz, xyDz in plane
+      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
+            double Dxyz = 0;
+            double xDyz = 0;
+            double xyDz = 0;
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              const double wy  = s_dofToQuad(qy, dy);
+              const double wDy = s_dofToQuadD(qy, dy);
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                const double wx  = s_dofToQuad(qx, dx);
+                const double wDx = s_dofToQuadD(qx, dx);
+                const double z  = s_z(dx, dy);
+                const double Dz = s_Dz(dx, dy);
+                Dxyz += wDx * wy  * z;
+                xDyz += wx  * wDy * z;
+                xyDz += wx  * wy  * Dz;
+              }
+            }
+
+            const int q = QUAD_3D_ID(qx, qy, qz);
+            const double O11 = oper(0, q, e);
+            const double O12 = oper(1, q, e);
+            const double O13 = oper(2, q, e);
+            const double O22 = oper(3, q, e);
+            const double O23 = oper(4, q, e);
+            const double O33 = oper(5, q, e);
+
+            const double qDxyz = (O11 * Dxyz) + (O12 * xDyz) + (O13 * xyDz);
+            const double qxDyz = (O12 * Dxyz) + (O22 * xDyz) + (O23 * xyDz);
+            const double qxyDz = (O13 * Dxyz) + (O23 * xDyz) + (O33 * xyDz);
+
+            for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+              const double wz  = s_quadToDof(dz, qz);
+              const double wDz = s_quadToDofD(dz, qz);
+              r_dDxyz[dz] += wz  * qDxyz;
+              r_dxDyz[dz] += wz  * qxDyz;
+              r_dxyDz[dz] += wDz * qxyDz;
+            }
+          }
+        }
+      }
+    }
+    // Iterate over xy planes to compute solution
+    for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+      // Place xy plane in shared memory
+      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
+            s_z(qx, qy)    = r_dDxyz[dz];
+            s_Dz(qx, qy)   = r_dxDyz[dz];
+            s_xyDz(qx, qy) = r_dxyDz[dz];
+          }
+        }
+      }
+      // Finalize solution in xy plane
+      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+            double solZ = 0;
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              const double wy  = s_quadToDof(dy, qy);
+              const double wDy = s_quadToDofD(dy, qy);
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                const double wx  = s_quadToDof(dx, qx);
+                const double wDx = s_quadToDofD(dx, qx);
+                const double Dxyz = s_z(qx, qy);
+                const double xDyz = s_Dz(qx, qy);
+                const double xyDz = s_xyDz(qx, qy);
+                solZ += ((wDx * wy  * Dxyz) +
+                         (wx  * wDy * xDyz) +
+                         (wx  * wy  * xyDz));
+              }
+            }
+            solOut(dx, dy, dz, e) += solZ;
+          }
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/engine.cpp

@@ -0,0 +1,140 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "backend.hpp"
+#include "url_handler.hpp"
+#include "bilinearform.hpp"
+#include "../../general/array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+bool Engine::fileOpenerRegistered = false;
+
+void Engine::Init(const std::string &engine_spec)
+{
+   //
+   // Initialize inherited fields
+   //
+   memory_resources[0] = NULL;
+   workers_weights[0]= 1.0;
+   workers_mem_res[0] = 0;
+
+   //
+   // Initialize the OCCA engine
+   //
+   ::occa::properties props(engine_spec);
+   device = new ::occa::device[1];
+   device[0].setup(props);
+
+   okl_path = "mfem-occa://";
+   // okl_defines = "...";
+   if (!fileOpenerRegistered)
+   {
+      // The directories from "MFEM_OCCA_OKL_PATH", if any, have the highest
+      // priority.
+      FileOpener *fo = new FileOpener("mfem-occa://", "MFEM_OCCA_OKL_PATH");
+      // Next in priority is the source path, if it exists.
+      std::string mfem_src_prefix = mfem::GetSourcePath();
+      fo->AddDir(mfem_src_prefix + "/backends/occa");
+      // And last in priority is the install path, if it exists.
+      std::string mfem_install_prefix = mfem::GetInstallPath();
+      fo->AddDir(mfem_install_prefix + "/lib/mfem/occa");
+      ::occa::io::fileOpener::add(fo);
+      fileOpenerRegistered = true;
+   }
+}
+
+Engine::Engine(const std::string &engine_spec)
+   : mfem::Engine(NULL, 1, 1)
+{
+   Init(engine_spec);
+}
+
+#ifdef MFEM_USE_MPI
+Engine::Engine(MPI_Comm _comm, const std::string &engine_spec)
+   : mfem::Engine(NULL, 1, 1)
+{
+   comm = _comm;
+   Init(engine_spec);
+}
+#endif
+
+DLayout Engine::MakeLayout(std::size_t size) const
+{
+   return DLayout(new Layout(*this, size));
+}
+
+DLayout Engine::MakeLayout(const mfem::Array<std::size_t> &offsets) const
+{
+   MFEM_ASSERT(offsets.Size() == 2,
+               "multiple workers are not supported yet");
+   return DLayout(new Layout(*this, offsets.Last()));
+}
+
+DArray Engine::MakeArray(PLayout &layout, std::size_t item_size) const
+{
+   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
+               "invalid input layout");
+   Layout *lt = static_cast<Layout *>(&layout);
+   return DArray(new Array(*lt, item_size));
+}
+
+DVector Engine::MakeVector(PLayout &layout, int type_id) const
+{
+   MFEM_ASSERT(type_id == ScalarId<double>::value, "invalid type_id");
+   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
+               "invalid input layout");
+   Layout *lt = static_cast<Layout *>(&layout);
+   return DVector(new Vector(*lt));
+}
+
+DFiniteElementSpace Engine::MakeFESpace(mfem::FiniteElementSpace &fespace) const
+{
+   return DFiniteElementSpace(new FiniteElementSpace(*this, fespace));
+}
+
+DBilinearForm Engine::MakeBilinearForm(mfem::BilinearForm &bf) const
+{
+   return DBilinearForm(new BilinearForm(*this, bf));
+}
+
+void Engine::AssembleLinearForm(LinearForm &l_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+}
+
+mfem::Operator *Engine::MakeOperator(const MixedBilinearForm &mbl_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+   return NULL;
+}
+
+mfem::Operator *Engine::MakeOperator(const NonlinearForm &nl_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+   return NULL;
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/engine.hpp

@@ -0,0 +1,111 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_ENGINE_HPP
+#define MFEM_BACKENDS_OCCA_ENGINE_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "../base/backend.hpp"
+#include <occa.hpp>
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Engine : public mfem::Engine
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // mfem::Backend *backend;
+#ifdef MFEM_USE_MPI
+   // MPI_Comm comm;
+#endif
+   // int num_mem_res;
+   // int num_workers;
+   // MemoryResource **memory_resources;
+   // double *workers_weights;
+   // int *workers_mem_res;
+
+   static bool fileOpenerRegistered;
+   ::occa::device *device; // An array of OCCA devices
+   std::string okl_path, okl_defines;
+
+   void Init(const std::string &engine_spec);
+
+public:
+   Engine(const std::string &engine_spec);
+
+#ifdef MFEM_USE_MPI
+   Engine(MPI_Comm comm, const std::string &engine_spec);
+#endif
+
+   virtual ~Engine() { delete [] device; }
+
+   /**
+       @name OCCA specific interface, used by other objects in the OCCA backend
+    */
+   ///@{
+
+   ::occa::device GetDevice(int idx = 0) const { return device[idx]; }
+
+   /// TODO: doxygen
+   const std::string &GetOklPath() const { return okl_path; }
+
+   /// TODO: doxygen
+   const std::string &GetOklDefines() const { return okl_defines; }
+
+   ///@}
+   // End: OCCA specific interface
+
+   /**
+       @name Virtual interface: finite element data structures and algorithms
+    */
+   ///@{
+
+   virtual DLayout MakeLayout(std::size_t size) const;
+   virtual DLayout MakeLayout(const mfem::Array<std::size_t> &offsets) const;
+
+   virtual DArray MakeArray(PLayout &layout, std::size_t item_size) const;
+
+   virtual DVector MakeVector(PLayout &layout,
+                              int type_id = ScalarId<double>::value) const;
+
+   virtual DFiniteElementSpace MakeFESpace(mfem::FiniteElementSpace &
+                                           fespace) const;
+
+   virtual DBilinearForm MakeBilinearForm(mfem::BilinearForm &bf) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual void AssembleLinearForm(LinearForm &l_form) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual mfem::Operator *MakeOperator(const MixedBilinearForm &mbl_form) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual mfem::Operator *MakeOperator(const NonlinearForm &nl_form) const;
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_ENGINE_HPP

backends/occa/fespace.cpp

@@ -0,0 +1,176 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "backend.hpp"
+#include "fespace.hpp"
+#include "interpolation.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+FiniteElementSpace::FiniteElementSpace(const Engine &e,
+                                       mfem::FiniteElementSpace &fespace)
+   : PFiniteElementSpace(e, fespace),
+     e_layout(e, 0) // resized in SetupLocalGlobalMaps()
+{
+   vdim     = fespace.GetVDim();
+   ordering = fespace.GetOrdering();
+
+   SetupLocalGlobalMaps();
+   SetupOperators();
+   SetupKernels();
+
+   e_layout.DontDelete();
+}
+
+FiniteElementSpace::~FiniteElementSpace()
+{
+   delete [] elementDofMap;
+   delete [] elementDofMapInverse;
+   delete restrictionOp;
+   delete prolongationOp;
+}
+
+void FiniteElementSpace::SetupLocalGlobalMaps()
+{
+   const mfem::FiniteElement &fe = *(fes->GetFE(0));
+   const mfem::TensorBasisElement *el =
+      dynamic_cast<const mfem::TensorBasisElement*>(&fe);
+
+   const mfem::Table &e2dTable = fes->GetElementToDofTable();
+   const int *elementMap = e2dTable.GetJ();
+   const int elements = fes->GetNE();
+
+   globalDofs = fes->GetNDofs();
+   localDofs  = fe.GetDof();
+
+   e_layout.Resize(localDofs * elements * fes->GetVDim());
+
+   elementDofMap        = new int[localDofs];
+   elementDofMapInverse = new int[localDofs];
+   if (el)
+   {
+      ::memcpy(elementDofMap,
+               el->GetDofMap().GetData(),
+               localDofs * sizeof(int));
+   }
+   else
+   {
+      for (int i = 0; i < localDofs; ++i)
+      {
+         elementDofMap[i] = i;
+      }
+   }
+   for (int i = 0; i < localDofs; ++i)
+   {
+      elementDofMapInverse[elementDofMap[i]] = i;
+   }
+
+   // Allocate device offsets and indices
+   globalToLocalOffsets.allocate(GetDevice(),
+                                 globalDofs + 1);
+   globalToLocalIndices.allocate(GetDevice(),
+                                 localDofs, elements);
+   localToGlobalMap.allocate(GetDevice(),
+                             localDofs, elements);
+
+   int *offsets = globalToLocalOffsets.ptr();
+   int *indices = globalToLocalIndices.ptr();
+   int *l2gMap  = localToGlobalMap.ptr();
+
+   // We'll be keeping a count of how many local nodes point
+   //   to its global dof
+   for (int i = 0; i <= globalDofs; ++i)
+   {
+      offsets[i] = 0;
+   }
+
+   for (int e = 0; e < elements; ++e)
+   {
+      for (int d = 0; d < localDofs; ++d)
+      {
+         const int gid = elementMap[localDofs*e + d];
+         ++offsets[gid + 1];
+      }
+   }
+   // Aggregate to find offsets for each global dof
+   for (int i = 1; i <= globalDofs; ++i)
+   {
+      offsets[i] += offsets[i - 1];
+   }
+   // For each global dof, fill in all local nodes that point
+   //   to it
+   for (int e = 0; e < elements; ++e)
+   {
+      for (int d = 0; d < localDofs; ++d)
+      {
+         const int gid = elementMap[localDofs*e + elementDofMap[d]];
+         const int lid = localDofs*e + d;
+         indices[offsets[gid]++] = lid;
+         l2gMap[lid] = gid;
+      }
+   }
+   // We shifted the offsets vector by 1 by using it
+   //   as a counter. Now we shift it back.
+   for (int i = globalDofs; i > 0; --i)
+   {
+      offsets[i] = offsets[i - 1];
+   }
+   offsets[0] = 0;
+
+   globalToLocalOffsets.keepInDevice();
+   globalToLocalIndices.keepInDevice();
+   localToGlobalMap.keepInDevice();
+}
+
+void FiniteElementSpace::SetupOperators()
+{
+   const mfem::SparseMatrix *R = fes->GetRestrictionMatrix();
+   const mfem::Operator *P = fes->GetProlongationMatrix();
+   CreateRPOperators(OccaVLayout(), OccaTrueVLayout(),
+                     R, P,
+                     restrictionOp,
+                     prolongationOp);
+}
+
+void FiniteElementSpace::SetupKernels()
+{
+   ::occa::properties props("defines: {"
+                            "  TILESIZE: 256,"
+                            "}");
+   props["defines/NUM_VDIM"] = vdim;
+
+   props["defines/ORDERING_BY_NODES"] = 0;
+   props["defines/ORDERING_BY_VDIM"]  = 1;
+   props["defines/VDIM_ORDERING"] = (int) (ordering == Ordering::byVDIM);
+
+   ::occa::device device = GetDevice();
+   const std::string &okl_path = OccaEngine().GetOklPath();
+   const std::string &okl_defines = OccaEngine().GetOklDefines();
+   globalToLocalKernel = device.buildKernel(okl_path + "fespace.okl",
+                                            "GlobalToLocal",
+                                            props + okl_defines);
+   localToGlobalKernel = device.buildKernel(okl_path + "fespace.okl",
+                                            "LocalToGlobal",
+                                            props + okl_defines);
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/fespace.hpp

@@ -0,0 +1,146 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_FE_SPACE_HPP
+#define MFEM_BACKENDS_OCCA_FE_SPACE_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "engine.hpp"
+#include "operator.hpp"
+#include "../../fem/fem.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+/// TODO: doxygen
+class FiniteElementSpace : public mfem::PFiniteElementSpace
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // mfem::FiniteElementSpace *fes;
+
+   Layout e_layout;
+
+   int *elementDofMap;
+   int *elementDofMapInverse;
+
+   ::occa::array<int> globalToLocalOffsets;
+   ::occa::array<int> globalToLocalIndices;
+   ::occa::array<int> localToGlobalMap;
+   ::occa::kernel globalToLocalKernel, localToGlobalKernel;
+
+   mfem::Ordering::Type ordering;
+
+   int globalDofs, localDofs;
+   int vdim;
+
+   mfem::Operator *restrictionOp, *prolongationOp;
+
+   void SetupLocalGlobalMaps();
+   void SetupOperators();
+   void SetupKernels();
+
+public:
+   /// TODO: doxygen
+   FiniteElementSpace(const Engine &e, mfem::FiniteElementSpace &fespace);
+
+   /// Virtual destructor
+   virtual ~FiniteElementSpace();
+
+   /// TODO: doxygen
+   const Engine &OccaEngine() const
+   { return *static_cast<const Engine *>(engine.Get()); }
+
+   /// TODO: doxygen
+   ::occa::device GetDevice(int idx = 0) const
+   { return OccaEngine().GetDevice(idx); }
+
+   mfem::Mesh* GetMesh() const { return fes->GetMesh(); }
+
+   Layout &OccaVLayout() const
+   { return *fes->GetVLayout().As<Layout>(); }
+
+   Layout &OccaTrueVLayout() const
+   { return *fes->GetTrueVLayout().As<Layout>(); }
+
+   Layout &OccaEVLayout() { return e_layout; }
+
+#ifdef MFEM_USE_MPI
+   bool isDistributed() const { return (OccaEngine().GetComm() != MPI_COMM_NULL); }
+#else
+   bool isDistributed() const { return false; }
+#endif
+
+   bool hasTensorBasis() const
+   { return dynamic_cast<const mfem::TensorBasisElement*>(fes->GetFE(0)); }
+
+   mfem::Ordering::Type GetOrdering() const { return ordering; }
+
+   int GetGlobalDofs() const { return globalDofs; }
+   int GetLocalDofs() const { return localDofs; }
+
+   int GetDim() const { return fes->GetMesh()->Dimension(); }
+   int GetVDim() const { return vdim; }
+
+   int GetVSize() const { return globalDofs * vdim; }
+   int GetTrueVSize() const { return fes->GetTrueVSize(); }
+   int GetGlobalVSize() const { return globalDofs*vdim; /* FIXME: MPI */ }
+   int GetGlobalTrueVSize() const { return fes->GetTrueVSize(); }
+
+   int GetNE() const { return fes->GetNE(); }
+
+   const mfem::FiniteElementCollection* FEColl() const
+   { return fes->FEColl(); }
+   const mfem::FiniteElement* GetFE(const int idx) const
+   { return fes->GetFE(idx); }
+
+   const int* GetElementDofMap() const { return elementDofMap; }
+   const int* GetElementDofMapInverse() const { return elementDofMapInverse; }
+
+   const mfem::Operator* GetRestrictionOperator() { return restrictionOp; }
+   const mfem::Operator* GetProlongationOperator() { return prolongationOp; }
+
+   const ::occa::array<int> GetLocalToGlobalMap() const
+   { return localToGlobalMap; }
+
+   void GlobalToLocal(const Vector &globalVec, Vector &localVec) const
+   {
+      globalToLocalKernel(globalDofs,
+                          localDofs * fes->GetNE(),
+                          globalToLocalOffsets,
+                          globalToLocalIndices,
+                          globalVec.OccaMem(), localVec.OccaMem());
+   }
+   void LocalToGlobal(const Vector &localVec, Vector &globalVec) const
+   {
+      localToGlobalKernel(globalDofs,
+                          localDofs * fes->GetNE(),
+                          globalToLocalOffsets,
+                          globalToLocalIndices,
+                          localVec.OccaMem(), globalVec.OccaMem());
+   }
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_FE_SPACE_HPP

backends/occa/fespace.okl

@@ -0,0 +1,67 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  TILESIZE : Tilesize for iterating over entries
+  ================================================
+*/
+
+#if VDIM_ORDERING == ORDERING_BY_VDIM
+typedef double *Global_t @dim(NUM_VDIM, globalEntries);
+typedef double *Local_t  @dim(NUM_VDIM, localEntries);
+#else
+typedef double *Global_t @dim(NUM_VDIM, globalEntries) @dimOrder(1, 0);
+typedef double *Local_t  @dim(NUM_VDIM, localEntries)  @dimOrder(1, 0);
+#endif
+
+@kernel void GlobalToLocal(const int globalEntries,
+                           const int localEntries,
+                           const int * restrict offsets,
+                           const int * restrict indices,
+                           const Global_t restrict globalX,
+                           Local_t restrict localX) {
+
+  for (int i = 0; i < globalEntries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < globalEntries) {
+      const int offset = offsets[i];
+      const int nextOffset = offsets[i + 1];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        const double dofValue = globalX(v, i);
+        for (int j = offset; j < nextOffset; ++j) {
+          localX(v, indices[j]) = dofValue;
+        }
+      }
+    }
+  }
+}
+
+@kernel void LocalToGlobal(const int globalEntries,
+                           const int localEntries,
+                           const int * restrict offsets,
+                           const int * restrict indices,
+                           const Local_t restrict localX,
+                           Global_t restrict globalX) {
+
+  for (int i = 0; i < globalEntries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < globalEntries) {
+      const int offset = offsets[i];
+      const int nextOffset = offsets[i + 1];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        double dofValue = 0;
+        for (int j = offset; j < nextOffset; ++j) {
+          dofValue += localX(v, indices[j]);
+        }
+        globalX(v, i) = dofValue;
+      }
+    }
+  }
+}

backends/occa/geometry.okl

@@ -0,0 +1,181 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef STORE_JACOBIAN
+#  define STORE_JACOBIAN 1
+#endif
+
+#ifndef STORE_JACOBIAN_INV
+#  define STORE_JACOBIAN_INV 1
+#endif
+
+#ifndef STORE_JACOBIAN_DET
+#  define STORE_JACOBIAN_DET 1
+#endif
+
+typedef double* Local1D_t @dim(1, NUM_DOFS, numElements);
+typedef double* Local2D_t @dim(2, NUM_DOFS, numElements);
+typedef double* Local3D_t @dim(3, NUM_DOFS, numElements);
+
+typedef double* QLocal_t @dim(NUM_QUAD, numElements);
+
+typedef double* DofToQuadD1D_t @dim(NUM_QUAD, NUM_DOFS);
+typedef double* DofToQuadD2D_t @dim(2, NUM_QUAD, NUM_DOFS);
+typedef double* DofToQuadD3D_t @dim(3, NUM_QUAD, NUM_DOFS);
+
+typedef double* Jacobian1D_t @dim(NUM_QUAD, numElements);
+typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD, numElements);
+typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
+
+@kernel void InitGeometryInfo1D(const int numElements,
+                                const DofToQuadD1D_t restrict dofToQuadD,
+                                const Local1D_t restrict nodes,
+                                Jacobian1D_t restrict J,
+                                Jacobian1D_t restrict invJ,
+                                QLocal_t restrict detJ) {
+
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_nodes[NUM_DOFS];
+
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
+        s_nodes[d] = nodes(0, d, e);
+      }
+    }
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      double J11 = 0;
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const double wx = dofToQuadD(q, d);
+        J11 += wx * s_nodes[d];
+      }
+#if STORE_JACOBIAN
+      J(q, e) = J11;
+#endif
+#if STORE_JACOBIAN_INV
+      invJ(q, e) = 1.0 / J11;
+#endif
+#if STORE_JACOBIAN_DET
+      detJ(q, e) = J11;
+#endif
+    }
+  }
+}
+
+@kernel void InitGeometryInfo2D(const int numElements,
+                                const DofToQuadD2D_t restrict dofToQuadD,
+                                const Local2D_t restrict nodes,
+                                Jacobian2D_t restrict J,
+                                Jacobian2D_t restrict invJ,
+                                QLocal_t restrict detJ) {
+
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_nodes[2 * NUM_DOFS] @dim(2, NUM_DOFS);
+
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
+        s_nodes(0, d) = nodes(0, d, e);
+        s_nodes(1, d) = nodes(1, d, e);
+      }
+    }
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      double J11 = 0, J12 = 0;
+      double J21 = 0, J22 = 0;
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const double wx = dofToQuadD(0, q, d);
+        const double wy = dofToQuadD(1, q, d);
+        const double x = s_nodes(0, d);
+        const double y = s_nodes(1, d);
+        J11 += (wx * x); J12 += (wx * y);
+        J21 += (wy * x); J22 += (wy * y);
+      }
+#if STORE_JACOBIAN_INV || STORE_JACOBIAN_DET
+      const double r_detJ = (J11 * J22) - (J12 * J21);
+#endif
+#if STORE_JACOBIAN
+      J(0, 0, q, e) = J11; J(1, 0, q, e) = J12;
+      J(0, 1, q, e) = J21; J(1, 1, q, e) = J22;
+#endif
+#if STORE_JACOBIAN_INV
+      const double r_idetJ = 1.0 / r_detJ;
+      invJ(0, 0, q, e) =  J22 * r_idetJ;
+      invJ(1, 0, q, e) = -J12 * r_idetJ;
+
+      invJ(0, 1, q, e) = -J21 * r_idetJ;
+      invJ(1, 1, q, e) =  J11 * r_idetJ;
+#endif
+#if STORE_JACOBIAN_DET
+      detJ(q, e) = r_detJ;
+#endif
+    }
+  }
+}
+
+@kernel void InitGeometryInfo3D(const int numElements,
+                                const DofToQuadD3D_t restrict dofToQuadD,
+                                const Local3D_t restrict nodes,
+                                Jacobian3D_t restrict J,
+                                Jacobian3D_t restrict invJ,
+                                QLocal_t restrict detJ) {
+
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_nodes[3 * NUM_DOFS] @dim(3, NUM_DOFS);
+
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
+        s_nodes(0, d) = nodes(0, d, e);
+        s_nodes(1, d) = nodes(1, d, e);
+        s_nodes(2, d) = nodes(2, d, e);
+      }
+    }
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      double J11 = 0, J12 = 0, J13 = 0;
+      double J21 = 0, J22 = 0, J23 = 0;
+      double J31 = 0, J32 = 0, J33 = 0;
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const double wx = dofToQuadD(0, q, d);
+        const double wy = dofToQuadD(1, q, d);
+        const double wz = dofToQuadD(2, q, d);
+        const double x = s_nodes(0, d);
+        const double y = s_nodes(1, d);
+        const double z = s_nodes(2, d);
+        J11 += (wx * x); J12 += (wx * y); J13 += (wx * z);
+        J21 += (wy * x); J22 += (wy * y); J23 += (wy * z);
+        J31 += (wz * x); J32 += (wz * y); J33 += (wz * z);
+      }
+#if STORE_JACOBIAN_INV || STORE_JACOBIAN_DET
+      const double r_detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                             (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+#endif
+#if STORE_JACOBIAN
+      J(0, 0, q, e) = J11; J(1, 0, q, e) = J12; J(2, 0, q, e) = J13;
+      J(0, 1, q, e) = J21; J(1, 1, q, e) = J22; J(2, 1, q, e) = J23;
+      J(0, 2, q, e) = J31; J(1, 2, q, e) = J32; J(2, 2, q, e) = J33;
+#endif
+#if STORE_JACOBIAN_INV
+      const double r_idetJ = 1.0 / r_detJ;
+      invJ(0, 0, q, e) = r_idetJ * ((J22 * J33) - (J23 * J32));
+      invJ(1, 0, q, e) = r_idetJ * ((J32 * J13) - (J33 * J12));
+      invJ(2, 0, q, e) = r_idetJ * ((J12 * J23) - (J13 * J22));
+
+      invJ(0, 1, q, e) = r_idetJ * ((J23 * J31) - (J21 * J33));
+      invJ(1, 1, q, e) = r_idetJ * ((J33 * J11) - (J31 * J13));
+      invJ(2, 1, q, e) = r_idetJ * ((J13 * J21) - (J11 * J23));
+
+      invJ(0, 2, q, e) = r_idetJ * ((J21 * J32) - (J22 * J31));
+      invJ(1, 2, q, e) = r_idetJ * ((J31 * J12) - (J32 * J11));
+      invJ(2, 2, q, e) = r_idetJ * ((J11 * J22) - (J12 * J21));
+#endif
+#if STORE_JACOBIAN_DET
+      detJ(q, e) = r_detJ;
+#endif
+    }
+  }
+}

backends/occa/gridfunc.cpp

@@ -0,0 +1,195 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "gridfunc.hpp"
+#include "bilininteg.hpp"
+#include "../../fem/gridfunc.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+std::map<std::string, ::occa::kernel> gridFunctionKernels;
+
+::occa::kernel GetGridFunctionKernel(::occa::device device,
+                                     FiniteElementSpace &fespace,
+                                     const mfem::IntegrationRule &ir)
+{
+   const int numQuad = ir.GetNPoints();
+
+   const FiniteElement &fe = *(fespace.GetFE(0));
+   const int dim  = fe.GetDim();
+   const int vdim = fespace.GetVDim();
+
+   std::stringstream ss;
+   ss << ::occa::hash(device)
+      << "FEColl : " << fespace.FEColl()->Name()
+      << "Quad: "    << numQuad
+      << "Dim: "     << dim
+      << "VDim: "    << vdim;
+   std::string hash = ss.str();
+
+   // Kernel defines
+   ::occa::properties props;
+   props["defines/NUM_VDIM"] = vdim;
+
+   SetProperties(fespace, ir, props);
+
+   ::occa::kernel kernel = gridFunctionKernels[hash];
+   if (!kernel.isInitialized())
+   {
+      const std::string &okl_path = fespace.OccaEngine().GetOklPath();
+      kernel = device.buildKernel(okl_path + "gridfunc.okl",
+                                  stringWithDim("GridFuncToQuad", dim),
+                                  props);
+   }
+   return kernel;
+}
+
+// OccaGridFunction::OccaGridFunction() :
+//    Vector(),
+//    ofespace(NULL),
+//    sequence(0) {}
+
+OccaGridFunction::OccaGridFunction(FiniteElementSpace *ofespace_)
+   : PArray(ofespace_->OccaVLayout()),
+     Array(ofespace_->OccaVLayout(), sizeof(double)),
+     Vector(ofespace_->OccaVLayout()),
+     ofespace(ofespace_),
+     sequence(0) {}
+
+// OccaGridFunction::OccaGridFunction(OccaFiniteElementSpace *ofespace_,
+//                                    OccaVectorRef ref) :
+//    OccaVector(ref),
+//    ofespace(ofespace_),
+//    sequence(0) {}
+
+OccaGridFunction::OccaGridFunction(const OccaGridFunction &v)
+   : PArray(v),
+     Array(v),
+     Vector(v),
+     ofespace(v.ofespace),
+     sequence(v.sequence) {}
+
+OccaGridFunction& OccaGridFunction::operator = (double value)
+{
+   Fill(value);
+   return *this;
+}
+
+OccaGridFunction& OccaGridFunction::operator = (const Vector &v)
+{
+   Assign<double>(v);
+   return *this;
+}
+
+// OccaGridFunction& OccaGridFunction::operator = (const OccaVectorRef &v)
+// {
+//    OccaVector::operator = (v);
+//    return *this;
+// }
+
+OccaGridFunction& OccaGridFunction::operator = (const OccaGridFunction &v)
+{
+   Assign<double>(v);
+   return *this;
+}
+
+// void OccaGridFunction::SetGridFunction(mfem::GridFunction &gf)
+// {
+//    Vector v = *this;
+//    gf.MakeRef(ofespace->GetFESpace(), v, 0);
+//    // Make gf the owner of the data
+//    v.Swap(gf);
+// }
+
+void OccaGridFunction::GetTrueDofs(Vector &v)
+{
+   const mfem::Operator *R = ofespace->GetRestrictionOperator();
+   if (!R)
+   {
+      v.MakeRef(*this);
+   }
+   else
+   {
+      v.Resize<double>(R->OutLayout(), NULL);
+      mfem::Vector mfem_v(v);
+      R->Mult(this->Wrap(), mfem_v);
+   }
+}
+
+void OccaGridFunction::SetFromTrueDofs(Vector &v)
+{
+   const mfem::Operator *P = ofespace->GetProlongationOperator();
+   if (!P)
+   {
+      MakeRef(v);
+   }
+   else
+   {
+      Resize<double>(P->OutLayout(), NULL);
+      mfem::Vector mfem_this(*this);
+      P->Mult(v.Wrap(), mfem_this);
+   }
+}
+
+mfem::FiniteElementSpace* OccaGridFunction::GetFESpace()
+{
+   return ofespace->GetFESpace();
+}
+
+const mfem::FiniteElementSpace* OccaGridFunction::GetFESpace() const
+{
+   return ofespace->GetFESpace();
+}
+
+void OccaGridFunction::ToQuad(const IntegrationRule &ir, Vector &quadValues)
+{
+   const Engine &engine = OccaLayout().OccaEngine();
+   ::occa::device device = engine.GetDevice();
+
+   OccaDofQuadMaps &maps = OccaDofQuadMaps::Get(device, *ofespace, ir);
+
+   const int elements = ofespace->GetNE();
+   const int numQuad  = ir.GetNPoints();
+   quadValues.Resize<double>(*(new Layout(engine, numQuad * elements)), NULL);
+
+   ::occa::kernel g2qKernel = GetGridFunctionKernel(device, *ofespace, ir);
+   g2qKernel(elements,
+             maps.dofToQuad,
+             ofespace->GetLocalToGlobalMap(),
+             this->OccaMem(),
+             quadValues.OccaMem());
+}
+
+void OccaGridFunction::Distribute(const Vector &v)
+{
+   if (ofespace->isDistributed())
+   {
+      mfem::Vector mfem_this(*this);
+      ofespace->GetProlongationOperator()->Mult(v.Wrap(), mfem_this);
+   }
+   else
+   {
+      *this = v;
+   }
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/gridfunc.hpp

@@ -0,0 +1,83 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_GRID_FUNC_HPP
+#define MFEM_BACKENDS_OCCA_GRID_FUNC_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "vector.hpp"
+#include "fespace.hpp"
+
+namespace mfem
+{
+
+class IntegrationRule;
+class GridFunction;
+
+namespace occa
+{
+
+class OccaIntegrator;
+class OccaDofQuadMaps;
+
+// TODO: make this object part of the backend or the engine.
+extern std::map<std::string, ::occa::kernel> gridFunctionKernels;
+
+// TODO: make this a method of the backend or the engine.
+::occa::kernel GetGridFunctionKernel(::occa::device device,
+                                     FiniteElementSpace &fespace,
+                                     const mfem::IntegrationRule &ir);
+
+class OccaGridFunction : public Vector
+{
+protected:
+   FiniteElementSpace *ofespace;
+   long sequence;
+
+   ::occa::kernel gridFuncToQuad[3];
+
+public:
+   // OccaGridFunction();
+
+   OccaGridFunction(FiniteElementSpace *ofespace_);
+
+   // OccaGridFunction(FiniteElementSpace *ofespace_,
+   //                  OccaVectorRef ref);
+
+   OccaGridFunction(const OccaGridFunction &gf);
+
+   OccaGridFunction& operator = (double value);
+   OccaGridFunction& operator = (const Vector &v);
+   // OccaGridFunction& operator = (const OccaVectorRef &v);
+   OccaGridFunction& operator = (const OccaGridFunction &gf);
+
+   // void SetGridFunction(mfem::GridFunction &gf);
+
+   void GetTrueDofs(Vector &v);
+   void SetFromTrueDofs(Vector &v);
+
+   mfem::FiniteElementSpace* GetFESpace();
+   const mfem::FiniteElementSpace* GetFESpace() const;
+
+   void ToQuad(const mfem::IntegrationRule &ir, Vector &quadValues);
+
+   void Distribute(const Vector &v);
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_GRID_FUNC_HPP

backends/occa/gridfunc.okl

@@ -0,0 +1,26 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+#if USING_TENSOR_OPS
+#  if OCCA_USING_CPU
+#    include "mfem-occa://gridfunc/tensor/cpu.okl"
+#  else
+#    include "mfem-occa://gridfunc/tensor/gpuHighOrder.okl"
+#  endif
+#else
+#  if OCCA_USING_CPU
+#    include "mfem-occa://gridfunc/simplex/cpu.okl"
+#  else
+#    include "mfem-occa://gridfunc/simplex/gpuHighOrder.okl"
+#  endif
+#endif

backends/occa/gridfunc/simplex/cpu.okl

@@ -0,0 +1,63 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void GridFuncToQuad2D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal_t restrict out) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const int gid = l2gMap(d, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          const double r_gf = gf[v + gid*NUM_VDIM];
+          double r_out = 0;
+          for (int q = 0; q < NUM_QUAD; ++q) {
+            r_out += r_gf * dofToQuad(d, q);
+          }
+          out(v, d, e) = r_out;
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void GridFuncToQuad3D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal_t restrict out) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const int gid = l2gMap(d, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          const double r_gf = gf[v + gid*NUM_VDIM];
+          double r_out = 0;
+          for (int q = 0; q < NUM_QUAD; ++q) {
+            r_out += r_gf * dofToQuad(d, q);
+          }
+          out(v, d, e) = r_out;
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/gridfunc/simplex/gpuHighOrder.okl

@@ -0,0 +1,79 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void GridFuncToQuad2D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal_t restrict out) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_gf[NUM_VDIM][NUM_DOFS];
+
+    for (int dOff = 0; dOff < M2_INNER_BATCH; ++dOff; @inner) {
+      for (int d = dOff; d < NUM_DOFS; d += M2_INNER_BATCH) {
+        const int gid = l2gMap(d, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          s_gf[v][d] = gf[v + gid*NUM_VDIM]];
+        }
+      }
+    }
+
+    for (int qOff = 0; qOff < M2_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M2_INNER_BATCH) {
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          double r_out = 0;
+          for (int d = 0; d < NUM_DOFS; ++d) {
+            r_out += s_gf[v][d] * dofToQuad(d, q);
+          }
+          out(v, q, e) = r_out;
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void GridFuncToQuad3D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal_t restrict out) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_gf[NUM_VDIM][NUM_DOFS];
+
+    for (int dOff = 0; dOff < M3_INNER_BATCH; ++dOff; @inner) {
+      for (int d = dOff; d < NUM_DOFS; d += M3_INNER_BATCH) {
+        const int gid = l2gMap(d, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          s_gf[v][d] = gf[v + gid*NUM_VDIM]];
+        }
+      }
+    }
+
+    for (int qOff = 0; qOff < M3_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M3_INNER_BATCH) {
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          double r_out = 0;
+          for (int d = 0; d < NUM_DOFS; ++d) {
+            r_out += s_gf[v][d] * dofToQuad(d, q);
+          }
+          out(v, q, e) = r_out;
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/gridfunc/tensor/cpu.okl

@@ -0,0 +1,188 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void GridFuncToQuad1D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap1D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal1D_t restrict out) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double r_out[NUM_VDIM][NUM_QUAD_1D];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          r_out[v][qx] = 0;
+        }
+      }
+      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+        const int gid = l2gMap(dx, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          const double r_gf = gf[v + gid*NUM_VDIM];
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            r_out[v][qx] += r_gf * dofToQuad(qx, dx);
+          }
+        }
+      }
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          out(v, qx, e) = r_out[v][qx];
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void GridFuncToQuad2D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap2D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal2D_t restrict out) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double out_xy[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            out_xy[v][qy][qx] = 0;
+          }
+        }
+      }
+
+      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+        double out_x[NUM_VDIM][NUM_QUAD_1D];
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            out_x[v][qy] = 0;
+          }
+        }
+
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          const int gid = l2gMap(dx, dy, e);
+          for (int v = 0; v < NUM_VDIM; ++v) {
+            const double r_gf = gf[v + gid*NUM_VDIM];
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              out_x[v][qy] += r_gf * dofToQuad(qy, dx);
+            }
+          }
+        }
+
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            const double d2q = dofToQuad(qy, dy);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              out_xy[v][qy][qx] += d2q * out_x[v][qx];
+            }
+          }
+        }
+      }
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          for (int v = 0; v < NUM_VDIM; ++v) {
+            out(v, qx, qy, e) = out_xy[v][qy][qx];
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void GridFuncToQuad3D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap3D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QVLocal3D_t restrict out) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double out_xyz[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              out_xyz[v][qz][qy][qx] = 0;
+            }
+          }
+        }
+      }
+
+      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+        double out_xy[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D];
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              out_xy[v][qy][qx] = 0;
+            }
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          double out_x[NUM_VDIM][NUM_QUAD_1D];
+          for (int v = 0; v < NUM_VDIM; ++v) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              out_x[v][qx] = 0;
+            }
+          }
+
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const int gid = l2gMap(dx, dy, dz, e);
+            for (int v = 0; v < NUM_VDIM; ++v) {
+              const double r_gf = gf[v + gid*NUM_VDIM];
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                out_x[v][qx] += r_gf * dofToQuad(qx, dx);
+              }
+            }
+          }
+
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            const double wy = dofToQuad(qy, dy);
+            for (int v = 0; v < NUM_VDIM; ++v) {
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                out_xy[v][qy][qx] += wy * out_x[v][qx];
+              }
+            }
+          }
+        }
+
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          const double wz = dofToQuad(qz, dz);
+          for (int v = 0; v < NUM_VDIM; ++v) {
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                out_xyz[v][qz][qy][qx] += wz * out_xy[v][qy][qx];
+              }
+            }
+          }
+        }
+      }
+
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            for (int v = 0; v < NUM_VDIM; ++v) {
+              out(v, qx, qy, qz, e) = out_xyz[v][qz][qy][qx];
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/gridfunc/tensor/gpuHighOrder.okl

@@ -0,0 +1,183 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void GridFuncToQuad1D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap1D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QLocal1D_t restrict out) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @exclusive double r_out[NUM_QUAD_1D];
+
+    for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+      for (int i = el; i < NUM_QUAD_DOFS_1D; i += M1_INNER_ELEMENT_BATCH) {
+        s_dofToQuad[i] = dofToQuad[i];
+      }
+    }
+
+    for (int b = 0; b < M1_OUTER_ELEMENT_BATCH; ++b) {
+      for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+        const int e = eOff + b*M1_INNER_ELEMENT_BATCH + el;
+        if (e < numElements) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            r_out[qx] = 0;
+          }
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double r_gf = gf[l2gMap(dx, e)];
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              r_out[qx] += r_gf * s_dofToQuad(qx, dx);
+            }
+          }
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            out(qx, e) = r_out[qx];
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void GridFuncToQuad2D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap2D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QLocal2D_t restrict out) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+
+    // Store xy planes in shared memory
+    @shared double s_xy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+      for (int id = x; id < NUM_QUAD_DOFS_1D; id += NUM_MAX_1D) {
+        s_dofToQuad[id] = dofToQuad[id];
+      }
+    }
+
+    for (int e = eOff; e < (eOff + M2_ELEMENT_BATCH); ++e) {
+      if (e < numElements) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if (dx < NUM_DOFS_1D) {
+            double r_x[NUM_DOFS_1D];
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              r_x[dy] = gf[l2gMap(dx, dy, e)];
+            }
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              double xy = 0;
+              for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+                xy += r_x[dy] * s_dofToQuad(qy, dy);
+              }
+              s_xy(dx, qy) = xy;
+            }
+          }
+        }
+        for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+          if (qy < NUM_QUAD_1D) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              double val = 0;
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                val += s_xy(dx, qy) * s_dofToQuad(qx, dx);
+              }
+              out(qx, qy, e) = val;
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void GridFuncToQuad3D(const int numElements,
+                              const DofToQuad_t restrict dofToQuad,
+                              const DLocalMap3D_t restrict l2gMap,
+                              const double * restrict gf,
+                              QLocal3D_t restrict out) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+
+    // Store xy planes in shared memory
+    @shared double s_z[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
+
+    // Store z axis as registers
+    @exclusive double r_qz[NUM_QUAD_1D];
+
+    for (int y = 0; y < NUM_MAX_1D; ++y; @inner) {
+      for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+        const int id = (y * NUM_MAX_1D) + x;
+        // Fetch Q <--> D maps
+        if (id < NUM_QUAD_DOFS_1D) {
+          s_dofToQuad[id] = dofToQuad[id];
+        }
+        // Initialize our Z axis
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          r_qz[qz] = 0;
+        }
+      }
+    }
+
+    for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+      for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+        if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+          for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+            const double val = gf[l2gMap(dx, dy, dz, e)];
+            // Calculate D -> Q in the Z axis
+            for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+              r_qz[qz] += val * s_dofToQuad(qz, dz);
+            }
+          }
+        }
+      }
+    }
+    // For each xy plane
+    for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+      // Fill xy plane at given z position
+      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+            s_z(dx, dy) = r_qz[qz];
+          }
+        }
+      }
+      // Calculate Dxyz, xDyz, xyDz in plane
+      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
+            double val = 0;
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              const double wy = s_dofToQuad(qy, dy);
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                const double wx = s_dofToQuad(qx, dx);
+                val += wx * wy * s_z(dx, dy);
+              }
+            }
+            out(qx, qy, qz, e) = val;
+          }
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/interpolation.cpp

@@ -0,0 +1,162 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "interpolation.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+void CreateRPOperators(Layout &v_layout, Layout &t_layout,
+                       const mfem::SparseMatrix *R, const mfem::Operator *P,
+                       mfem::Operator *&OccaR, mfem::Operator *&OccaP)
+{
+   if (!P)
+   {
+      OccaR = new IdentityOperator(t_layout);
+      OccaP = new IdentityOperator(t_layout);
+      return;
+   }
+
+   const mfem::SparseMatrix *pmat = dynamic_cast<const mfem::SparseMatrix*>(P);
+   ::occa::device device = v_layout.OccaEngine().GetDevice();
+
+   if (R)
+   {
+      OccaSparseMatrix *occaR =
+         CreateMappedSparseMatrix(v_layout, t_layout, *R);
+      ::occa::array<int> reorderIndices = occaR->reorderIndices;
+      delete occaR;
+
+      OccaR = new RestrictionOperator(v_layout, t_layout, reorderIndices);
+   }
+
+   if (pmat)
+   {
+      const mfem::SparseMatrix *pmatT = Transpose(*pmat);
+
+      OccaSparseMatrix *occaP  =
+         CreateMappedSparseMatrix(t_layout, v_layout, *pmat);
+      OccaSparseMatrix *occaPT =
+         CreateMappedSparseMatrix(v_layout, t_layout, *pmatT);
+
+      OccaP = new ProlongationOperator(*occaP, *occaPT);
+   }
+   else
+   {
+      OccaP = new ProlongationOperator(t_layout, v_layout, P);
+   }
+}
+
+RestrictionOperator::RestrictionOperator(Layout &in_layout, Layout &out_layout,
+                                         ::occa::array<int> indices) :
+   Operator(in_layout, out_layout)
+{
+
+   entries     = indices.size() / 2;
+   trueIndices = indices;
+
+   // FIXME: paths ...
+   ::occa::device device = in_layout.OccaEngine().GetDevice();
+   const std::string &okl_path = in_layout.OccaEngine().GetOklPath();
+   const std::string &okl_defines = in_layout.OccaEngine().GetOklDefines();
+   multOp = device.buildKernel(okl_path + "mappings.okl",
+                               "ExtractSubVector",
+                               "defines: { TILESIZE: 256 }" + okl_defines);
+
+   multTransposeOp = device.buildKernel(okl_path + "mappings.okl",
+                                        "SetSubVector",
+                                        "defines: { TILESIZE: 256 }" +
+                                        okl_defines);
+}
+
+void RestrictionOperator::Mult_(const Vector &x, Vector &y) const
+{
+   multOp(entries, trueIndices, x.OccaMem(), y.OccaMem());
+}
+
+void RestrictionOperator::MultTranspose_(const Vector &x, Vector &y) const
+{
+   y.Fill<double>(0.0);
+   multTransposeOp(entries, trueIndices, x.OccaMem(), y.OccaMem());
+}
+
+ProlongationOperator::ProlongationOperator(OccaSparseMatrix &multOp_,
+                                           OccaSparseMatrix &multTransposeOp_) :
+   Operator(multOp_),
+   pmat(NULL),
+   multOp(multOp_),
+   multTransposeOp(multTransposeOp_) {}
+
+ProlongationOperator::ProlongationOperator(Layout &in_layout,
+                                           Layout &out_layout,
+                                           const mfem::Operator *pmat_) :
+   Operator(in_layout, out_layout),
+   pmat(pmat_),
+   multOp(*this),
+   multTransposeOp(*this)
+{ }
+
+void ProlongationOperator::Mult_(const Vector &x, Vector &y) const
+{
+   MFEM_VERIFY(pmat == NULL, "");
+   multOp.Mult_(x, y);
+}
+
+void ProlongationOperator::MultTranspose_(const Vector &x, Vector &y) const
+{
+   MFEM_VERIFY(pmat == NULL, "");
+   multTransposeOp.Mult_(x, y);
+}
+
+void ProlongationOperator::Mult(const mfem::Vector &x, mfem::Vector &y) const
+{
+   if (pmat)
+   {
+      // FIXME: create an OCCA version of 'pmat'
+      x.Pull();
+      y.Pull(false);
+      pmat->Mult(x, y);
+      y.Push();
+   }
+   else
+   {
+      multOp.Mult(x, y);
+   }
+}
+
+void ProlongationOperator::MultTranspose(const mfem::Vector &x,
+                                         mfem::Vector &y) const
+{
+   if (pmat)
+   {
+      // FIXME: create an OCCA version of 'pmat'
+      x.Pull();
+      y.Pull(false);
+      pmat->MultTranspose(x, y);
+      y.Push();
+   }
+   else
+   {
+      multTransposeOp.Mult(x, y);
+   }
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/interpolation.hpp

@@ -0,0 +1,79 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_INTERPOLATION_HPP
+#define MFEM_BACKENDS_OCCA_INTERPOLATION_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include <occa.hpp>
+#include "vector.hpp"
+#include "engine.hpp"
+#include "sparsemat.hpp"
+#include "../../fem/fem.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+// [MISSING] Proper destructors
+void CreateRPOperators(Layout &v_layout, Layout &t_layout,
+                       const mfem::SparseMatrix *R, const mfem::Operator *P,
+                       mfem::Operator *&OccaR, mfem::Operator *&OccaP);
+
+class RestrictionOperator : public Operator
+{
+protected:
+   int entries;
+   ::occa::array<int> trueIndices;
+   ::occa::kernel multOp, multTransposeOp;
+
+public:
+   RestrictionOperator(Layout &in_layout, Layout &out_layout,
+                       ::occa::array<int> indices);
+
+   // overrides
+   virtual void Mult_(const Vector &x, Vector &y) const;
+   virtual void MultTranspose_(const Vector &x, Vector &y) const;
+};
+
+class ProlongationOperator : public Operator
+{
+protected:
+   const mfem::Operator *pmat;
+   OccaSparseMatrix multOp, multTransposeOp;
+
+public:
+   ProlongationOperator(OccaSparseMatrix &multOp_,
+                        OccaSparseMatrix &multTransposeOp_);
+
+   ProlongationOperator(Layout &in_layout, Layout &out_layout,
+                        const mfem::Operator *pmat_);
+
+   // overrides
+   virtual void Mult_(const Vector &x, Vector &y) const;
+   virtual void MultTranspose_(const Vector &x, Vector &y) const;
+
+   // overrides
+   virtual void Mult(const mfem::Vector &x, mfem::Vector &y) const;
+   virtual void MultTranspose(const mfem::Vector &x, mfem::Vector &y) const;
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_INTERPOLATION_HPP

backends/occa/layout.cpp

@@ -0,0 +1,40 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "layout.hpp"
+#include "../../general/array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+void Layout::Resize(std::size_t new_size)
+{
+   size = new_size;
+}
+
+void Layout::Resize(const Array<std::size_t> &offsets)
+{
+   MFEM_ASSERT(offsets.Size() == 2,
+               "multiple workers are not supported yet");
+   size = offsets.Last();
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/layout.hpp

@@ -0,0 +1,68 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_LAYOUT_HPP
+#define MFEM_BACKENDS_OCCA_LAYOUT_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "../base/layout.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Layout : public PLayout
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // std::size_t size;
+
+public:
+   Layout(const Engine &e, std::size_t s = 0) : PLayout(e, s) { }
+
+   const Engine &OccaEngine() const
+   { return *static_cast<const Engine *>(engine.Get()); }
+
+   ::occa::memory Alloc(std::size_t bytes) const
+   { return OccaEngine().GetDevice().malloc(bytes); }
+
+   virtual ~Layout() { }
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   /// Resize the layout
+   virtual void Resize(std::size_t new_size);
+
+   /// Resize the layout based on the given worker offsets
+   virtual void Resize(const Array<std::size_t> &offsets);
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_LAYOUT_HPP

backends/occa/mappings.okl

@@ -0,0 +1,54 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  TILESIZE : Tilesize for iterating over entries
+  ================================================
+*/
+
+@kernel void ExtractSubVector(const int entries,
+                              const int * restrict indices,
+                              const double * restrict in,
+                              double * restrict out) {
+
+  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < entries) {
+      out[i] = in[indices[i]];
+    }
+  }
+}
+
+@kernel void SetSubVector(const int entries,
+                          const int * restrict indices,
+                          const double * restrict in,
+                          double * restrict out) {
+
+  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < entries) {
+      out[indices[i]] = in[i];
+    }
+  }
+}
+
+@kernel void MapSubVector(const int entries,
+                          const int * restrict indices,
+                          const double * restrict in,
+                          double * restrict out) {
+
+  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < entries) {
+      const int fromIdx = indices[2*i + 0];
+      const int toIdx   = indices[2*i + 1];
+      out[toIdx] = in[fromIdx];
+    }
+  }
+}

backends/occa/operator.cpp

@@ -0,0 +1,135 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "operator.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+// FIXME: move this object to the Backend?
+::occa::kernelBuilder OccaConstrainedOperator::mapDofBuilder =
+   ::occa::linalg::customLinearMethod(
+      "vector_map_dofs",
+
+      "const int idx = v2[i];"
+      "v0[idx] = v1[idx];",
+
+      "defines: {"
+      "  VTYPE0: 'double',"
+      "  VTYPE1: 'double',"
+      "  VTYPE2: 'int',"
+      "  TILESIZE: 128,"
+      "}");
+
+// FIXME: move this object to the Backend?
+::occa::kernelBuilder OccaConstrainedOperator::clearDofBuilder =
+   ::occa::linalg::customLinearMethod(
+      "vector_clear_dofs",
+
+      "v0[v1[i]] = 0.0;",
+
+      "defines: {"
+      "  VTYPE0: 'double',"
+      "  VTYPE1: 'int',"
+      "  TILESIZE: 128,"
+      "}");
+
+OccaConstrainedOperator::OccaConstrainedOperator(
+   mfem::Operator *A_,
+   const mfem::Array<int> &constraintList_,
+   bool own_A_)
+
+   : Operator(A_->InLayout()->As<Layout>()),
+     z(OutLayout_()),
+     w(OutLayout_()),
+     mfem_z((z.DontDelete(), z)),
+     mfem_w((w.DontDelete(), w))
+{
+   Setup(OutLayout_().OccaEngine().GetDevice(), A_, constraintList_, own_A_);
+}
+
+void OccaConstrainedOperator::Setup(::occa::device device_,
+                                    mfem::Operator *A_,
+                                    const mfem::Array<int> &constraintList_,
+                                    bool own_A_)
+{
+   device = device_;
+
+   A = A_;
+   own_A = own_A_;
+
+   constraintIndices = constraintList_.Size();
+   constraintList = constraintList_.Get_PArray()->As<Array>().OccaMem();
+}
+
+void OccaConstrainedOperator::EliminateRHS(const Vector &x, Vector &b) const
+{
+   const std::string &okl_defines = InLayout_().OccaEngine().GetOklDefines();
+   ::occa::kernel mapDofs = mapDofBuilder.build(device, okl_defines);
+
+   w.Fill<double>(0.0);
+
+   if (constraintIndices)
+   {
+      mapDofs(constraintIndices, w.OccaMem(), x.OccaMem(), constraintList);
+   }
+
+   A->Mult(mfem_w, mfem_z);
+
+   b.Axpby<double>(1.0, b, -1.0, z);
+
+   if (constraintIndices)
+   {
+      mapDofs(constraintIndices, b.OccaMem(), x.OccaMem(), constraintList);
+   }
+}
+
+void OccaConstrainedOperator::Mult_(const Vector &x, Vector &y) const
+{
+   mfem::Vector mfem_y(y);
+   if (constraintIndices == 0)
+   {
+      A->Mult(x.Wrap(), mfem_y);
+      return;
+   }
+
+   const std::string &okl_defines = InLayout_().OccaEngine().GetOklDefines();
+   ::occa::kernel mapDofs   = mapDofBuilder.build(device, okl_defines);
+   ::occa::kernel clearDofs = clearDofBuilder.build(device, okl_defines);
+
+   z.Assign<double>(x); // z = x
+
+   clearDofs(constraintIndices, z.OccaMem(), constraintList);
+
+   A->Mult(mfem_z, mfem_y);
+
+   mapDofs(constraintIndices, y.OccaMem(), x.OccaMem(), constraintList);
+}
+
+OccaConstrainedOperator::~OccaConstrainedOperator()
+{
+   if (own_A)
+   {
+      delete A;
+   }
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/operator.hpp

@@ -0,0 +1,129 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_OPERATOR_HPP
+#define MFEM_BACKENDS_OCCA_OPERATOR_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "vector.hpp"
+#include "../../linalg/operator.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Operator : public mfem::Operator
+{
+public:
+   /// Creare an operator with the same dimensions as @a orig.
+   Operator(const Operator &orig)
+      : mfem::Operator(orig) { }
+
+   Operator(Layout &layout)
+      : mfem::Operator(layout) { }
+
+   Operator(Layout &in_layout, Layout &out_layout)
+      : mfem::Operator(in_layout, out_layout) { }
+
+   Layout &InLayout_() const
+   { return *static_cast<Layout*>(in_layout.Get()); }
+
+   Layout &OutLayout_() const
+   { return *static_cast<Layout*>(out_layout.Get()); }
+
+   virtual void Mult_(const Vector &x, Vector &y) const = 0;
+
+   virtual void MultTranspose_(const Vector &x, Vector &y) const
+   { MFEM_ABORT("method is not supported"); }
+
+   // override
+   virtual void Mult(const mfem::Vector &x, mfem::Vector &y) const
+   {
+      Mult_(x.Get_PVector()->As<Vector>(),
+            y.Get_PVector()->As<Vector>());
+   }
+
+   // override
+   virtual void MultTranspose(const mfem::Vector &x, mfem::Vector &y) const
+   {
+      MultTranspose_(x.Get_PVector()->As<Vector>(),
+                     y.Get_PVector()->As<Vector>());
+   }
+};
+
+
+class OccaConstrainedOperator : public Operator
+{
+protected:
+   ::occa::device device;
+
+   mfem::Operator *A;              //< The unconstrained Operator.
+   bool own_A;                     //< Ownership flag for A.
+   ::occa::memory constraintList;  //< List of constrained indices/dofs.
+   int constraintIndices;
+   mutable Vector z, w;       //< Auxiliary vectors.
+   mutable mfem::Vector mfem_z, mfem_w; // Wrap z, w
+
+   static ::occa::kernelBuilder mapDofBuilder, clearDofBuilder;
+
+public:
+   /** @brief Constructor from a general Operator and a list of essential
+       indices/dofs.
+
+       Specify the unconstrained operator @a *A and a @a list of indices to
+       constrain, i.e. each entry @a list[i] represents an essential-dof. If the
+       ownership flag @a own_A is true, the operator @a *A will be destroyed
+       when this object is destroyed. */
+   OccaConstrainedOperator(mfem::Operator *A_,
+                           const mfem::Array<int> &constraintList_,
+                           bool own_A_ = false);
+
+   void Setup(::occa::device device_,
+              mfem::Operator *A_,
+              const mfem::Array<int> &constraintList_,
+              bool own_A_ = false);
+
+   /** @brief Eliminate "essential boundary condition" values specified in @a x
+       from the given right-hand side @a b.
+
+       Performs the following steps:
+
+       z = A((0,x_b));  b_i -= z_i;  b_b = x_b;
+
+       where the "_b" subscripts denote the essential (boundary) indices/dofs of
+       the vectors, and "_i" -- the rest of the entries. */
+   void EliminateRHS(const Vector &x, Vector &b) const;
+
+   /** @brief Constrained operator action.
+
+       Performs the following steps:
+
+       z = A((x_i,0));  y_i = z_i;  y_b = x_b;
+
+       where the "_b" subscripts denote the essential (boundary) indices/dofs of
+       the vectors, and "_i" -- the rest of the entries. */
+   virtual void Mult_(const Vector &x, Vector &y) const;
+
+   // Destructor: destroys the unconstrained Operator @a A if @a own_A is true.
+   virtual ~OccaConstrainedOperator();
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_OPERATOR_HPP

backends/occa/sparse.okl

@@ -0,0 +1,57 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  TILESIZE : Tilesize for iterating over dofs
+  ================================================
+*/
+
+@kernel void Mult(const int entries,
+                  const int * restrict offsets,
+                  const int * restrict indices,
+                  const double * restrict weights,
+                  const double * restrict in,
+                  double * restrict out) {
+
+  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < entries) {
+      const int offset = offsets[i];
+      const int nextOffset = offsets[i + 1];
+      double value = 0;
+      for (int j = offset; j < nextOffset; ++j) {
+        value += weights[j] * in[indices[j]];
+      }
+      out[i] = value;
+    }
+  }
+}
+
+@kernel void MappedMult(const int entries,
+                        const int * restrict offsets,
+                        const int * restrict indices,
+                        const double * restrict weights,
+                        const int * restrict outIndices,
+                        const double * restrict in,
+                        double * restrict out) {
+
+  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < entries) {
+      const int offset = offsets[i];
+      const int nextOffset = offsets[i + 1];
+      double value = 0;
+      for (int j = offset; j < nextOffset; ++j) {
+        value += weights[j] * in[indices[j]];
+      }
+      out[outIndices[i]] = value;
+    }
+  }
+}

backends/occa/sparsemat.cpp

@@ -0,0 +1,248 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "sparsemat.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                                   const mfem::SparseMatrix &m,
+                                   const ::occa::properties &props) :
+   Operator(in_layout, out_layout)
+{
+
+   Setup(in_layout.OccaEngine().GetDevice(), m, props);
+}
+
+OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                                   const mfem::SparseMatrix &m,
+                                   ::occa::array<int> reorderIndices_,
+                                   ::occa::array<int> mappedIndices_,
+                                   const ::occa::properties &props) :
+   Operator(in_layout, out_layout)
+{
+
+   Setup(in_layout.OccaEngine().GetDevice(), m,
+         reorderIndices, mappedIndices_, props);
+}
+
+OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                                   ::occa::array<int> offsets_,
+                                   ::occa::array<int> indices_,
+                                   ::occa::array<double> weights_,
+                                   const ::occa::properties &props) :
+   Operator(in_layout, out_layout),
+   offsets(offsets_),
+   indices(indices_),
+   weights(weights_)
+{
+
+   SetupKernel(in_layout.OccaEngine().GetDevice(), props);
+}
+
+OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                                   ::occa::array<int> offsets_,
+                                   ::occa::array<int> indices_,
+                                   ::occa::array<double> weights_,
+                                   ::occa::array<int> reorderIndices_,
+                                   ::occa::array<int> mappedIndices_,
+                                   const ::occa::properties &props) :
+   Operator(in_layout, out_layout),
+   offsets(offsets_),
+   indices(indices_),
+   weights(weights_),
+   reorderIndices(reorderIndices_),
+   mappedIndices(mappedIndices_)
+{
+
+   SetupKernel(in_layout.OccaEngine().GetDevice(), props);
+}
+
+void OccaSparseMatrix::Setup(::occa::device device, const mfem::SparseMatrix &m,
+                             const ::occa::properties &props)
+{
+   Setup(device, m, ::occa::array<int>(), ::occa::array<int>(), props);
+}
+
+void OccaSparseMatrix::Setup(::occa::device device, const SparseMatrix &m,
+                             ::occa::array<int> reorderIndices_,
+                             ::occa::array<int> mappedIndices_,
+                             const ::occa::properties &props)
+{
+
+   const int nnz = m.GetI()[height];
+   offsets.allocate(device,
+                    height + 1, m.GetI());
+   indices.allocate(device,
+                    nnz, m.GetJ());
+   weights.allocate(device,
+                    nnz, m.GetData());
+
+   offsets.keepInDevice();
+   indices.keepInDevice();
+   weights.keepInDevice();
+
+   reorderIndices = reorderIndices_;
+   mappedIndices  = mappedIndices_;
+
+   SetupKernel(device, props);
+}
+
+void OccaSparseMatrix::SetupKernel(::occa::device device,
+                                   const ::occa::properties &props)
+{
+
+   const bool hasOutIndices = mappedIndices.isInitialized();
+
+   const ::occa::properties defaultProps("defines: {"
+                                         "  TILESIZE: 256,"
+                                         "}");
+
+   const std::string &okl_path = InLayout_().OccaEngine().GetOklPath();
+   const std::string &okl_defines = InLayout_().OccaEngine().GetOklDefines();
+   mapKernel = device.buildKernel(okl_path + "mappings.okl",
+                                  "MapSubVector",
+                                  defaultProps + props + okl_defines);
+
+   multKernel = device.buildKernel(okl_path + "sparse.okl",
+                                   hasOutIndices ? "MappedMult" : "Mult",
+                                   defaultProps + props + okl_defines);
+}
+
+void OccaSparseMatrix::Mult_(const Vector &x, Vector &y) const
+{
+   if (reorderIndices.isInitialized() ||
+       mappedIndices.isInitialized())
+   {
+      if (reorderIndices.isInitialized())
+      {
+         mapKernel((int) (reorderIndices.size() / 2),
+                   reorderIndices,
+                   x.OccaMem(), y.OccaMem());
+      }
+      if (mappedIndices.isInitialized())
+      {
+         multKernel((int) (mappedIndices.size()),
+                    offsets, indices, weights,
+                    mappedIndices,
+                    x.OccaMem(), y.OccaMem());
+      }
+   }
+   else
+   {
+      multKernel((int) height,
+                 offsets, indices, weights,
+                 x.OccaMem(), y.OccaMem());
+   }
+}
+
+
+OccaSparseMatrix* CreateMappedSparseMatrix(Layout &in_layout,
+                                           Layout &out_layout,
+                                           const mfem::SparseMatrix &m,
+                                           const ::occa::properties &props)
+{
+   const int mHeight = m.Height();
+   // const int mWidth  = m.Width();
+
+   // Count indices that are only reordered (true dofs)
+   const int *I = m.GetI();
+   const int *J = m.GetJ();
+   const double *D = m.GetData();
+
+   int trueCount = 0;
+   for (int i = 0; i < mHeight; ++i)
+   {
+      trueCount += ((I[i + 1] - I[i]) == 1);
+   }
+   const int dupCount = (mHeight - trueCount);
+
+   // Create the reordering map for entries that aren't modified (true dofs)
+   ::occa::device device(in_layout.OccaEngine().GetDevice());
+   ::occa::array<int> reorderIndices(device,
+                                     2 * trueCount);
+   ::occa::array<int> mappedIndices, offsets, indices;
+   ::occa::array<double> weights;
+
+   if (dupCount)
+   {
+      mappedIndices.allocate(device,
+                             dupCount);
+   }
+   int trueIdx = 0, dupIdx = 0;
+   for (int i = 0; i < mHeight; ++i)
+   {
+      const int i1 = I[i];
+      if ((I[i + 1] - i1) == 1)
+      {
+         reorderIndices[trueIdx++] = J[i1];
+         reorderIndices[trueIdx++] = i;
+      }
+      else
+      {
+         mappedIndices[dupIdx++] = i;
+      }
+   }
+   reorderIndices.keepInDevice();
+
+   if (dupCount)
+   {
+      mappedIndices.keepInDevice();
+
+      // Extract sparse matrix without reordered identity
+      const int dupNnz = I[mHeight] - trueCount;
+
+      offsets.allocate(device,
+                       dupCount + 1);
+      indices.allocate(device,
+                       dupNnz);
+      weights.allocate(device,
+                       dupNnz);
+
+      int nnz = 0;
+      offsets[0] = 0;
+      for (int i = 0; i < dupCount; ++i)
+      {
+         const int idx = mappedIndices[i];
+         const int offStart = I[idx];
+         const int offEnd   = I[idx + 1];
+         offsets[i + 1] = offsets[i] + (offEnd - offStart);
+         for (int j = offStart; j < offEnd; ++j)
+         {
+            indices[nnz] = J[j];
+            weights[nnz] = D[j];
+            ++nnz;
+         }
+      }
+
+      offsets.keepInDevice();
+      indices.keepInDevice();
+      weights.keepInDevice();
+   }
+
+   return new OccaSparseMatrix(in_layout, out_layout,
+                               offsets, indices, weights,
+                               reorderIndices, mappedIndices,
+                               props);
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/sparsemat.hpp

@@ -0,0 +1,95 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_SPARSE_MAT_HPP
+#define MFEM_BACKENDS_OCCA_SPARSE_MAT_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include <occa.hpp>
+#include "vector.hpp"
+#include "engine.hpp"
+#include "operator.hpp"
+#include "../../linalg/sparsemat.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+/// TODO: doxygen
+class OccaSparseMatrix : public Operator
+{
+public:
+   ::occa::array<int> offsets, indices;
+   ::occa::array<double> weights;
+   ::occa::array<int> reorderIndices, mappedIndices;
+   ::occa::kernel mapKernel, multKernel;
+
+   /// Construct an empty OccaSparseMatrix.
+   OccaSparseMatrix(const Operator &orig)
+      : Operator(orig) { }
+
+   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                    const mfem::SparseMatrix &m,
+                    const ::occa::properties &props = ::occa::properties());
+
+   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                    const mfem::SparseMatrix &m,
+                    ::occa::array<int> reorderIndices_,
+                    ::occa::array<int> mappedIndices_,
+                    const ::occa::properties &props = ::occa::properties());
+
+   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                    ::occa::array<int> offsets_,
+                    ::occa::array<int> indices_,
+                    ::occa::array<double> weights_,
+                    const ::occa::properties &props = ::occa::properties());
+
+   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                    ::occa::array<int> offsets_,
+                    ::occa::array<int> indices_,
+                    ::occa::array<double> weights_,
+                    ::occa::array<int> reorderIndices_,
+                    ::occa::array<int> mappedIndices_,
+                    const ::occa::properties &props = ::occa::properties());
+
+   void Setup(::occa::device device, const mfem::SparseMatrix &m,
+              const ::occa::properties &props);
+
+   void Setup(::occa::device device, const mfem::SparseMatrix &m,
+              ::occa::array<int> reorderIndices_,
+              ::occa::array<int> mappedIndices_,
+              const ::occa::properties &props);
+
+   void SetupKernel(::occa::device device,
+                    const ::occa::properties &props);
+
+   // override
+   virtual void Mult_(const Vector &x, Vector &y) const;
+};
+
+
+/// TODO: doxygen
+OccaSparseMatrix* CreateMappedSparseMatrix(
+   Layout &in_layout, Layout &out_layout,
+   const mfem::SparseMatrix &m,
+   const ::occa::properties &props = ::occa::properties());
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_SPARSE_MAT_HPP

backends/occa/url_handler.cpp

@@ -0,0 +1,81 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "url_handler.hpp"
+#include "../../general/error.hpp"
+#include <cstdlib>
+#include <sys/stat.h>
+
+namespace mfem
+{
+
+namespace occa
+{
+
+FileOpener::FileOpener(const std::string &prefix,
+                       const std::string &env_variable)
+   : pfx(prefix)
+{
+   const char *env_path = getenv(env_variable.c_str());
+   if (!env_path) { return; }
+   std::string path(env_path);
+   for (std::size_t start = 0, end; start < path.size(); start = end + 1)
+   {
+      end = path.find(':', start);
+      if (end == std::string::npos)
+      {
+         AddDir(path.substr(start, end));
+         break;
+      }
+      AddDir(path.substr(start, end - start));
+   }
+}
+
+bool FileOpener::AddDir(const std::string &dir)
+{
+   if (dir.size() == 0 || dir[0] != '/') { return false; }
+   struct stat dir_stat;
+   if (stat(dir.c_str(), &dir_stat)) { return false; }
+   if (!S_ISDIR(dir_stat.st_mode)) { return false; }
+   paths.push_back(dir + (*dir.rbegin() == '/' ? "" : "/"));
+   return true;
+}
+
+bool FileOpener::handles(const std::string &filename)
+{
+   return filename.size() >= pfx.size() &&
+          filename.compare(0, pfx.size(), pfx) == 0;
+}
+
+std::string FileOpener::expand(const std::string &filename)
+{
+   std::string sfx(filename.substr(pfx.size()));
+   for (std::size_t i = 0; i < paths.size(); i++)
+   {
+      std::string file = paths[i] + sfx;
+      struct stat file_stat;
+      if (stat(file.c_str(), &file_stat) == 0 && S_ISREG(file_stat.st_mode))
+      {
+         return file;
+      }
+   }
+   MFEM_ABORT("invalid url: " << filename);
+   return sfx;
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/url_handler.hpp

@@ -0,0 +1,47 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_URL_HANDLER_HPP
+#define MFEM_BACKENDS_OCCA_URL_HANDLER_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include <occa.hpp>
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class FileOpener : public ::occa::io::fileOpener
+{
+protected:
+   std::string pfx; // prefix, e.g. "mfem://"
+   std::vector<std::string> paths; // paths to search for prefix replacement
+
+public:
+   FileOpener(const std::string &prefix, const std::string &env_variable);
+
+   bool AddDir(const std::string &dir);
+
+   virtual bool handles(const std::string &filename);
+   virtual std::string expand(const std::string &filename);
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_URL_HANDLER_HPP

backends/occa/utils.okl

@@ -0,0 +1,22 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+typedef double* Local_t @dim(numDofs, numElements);
+
+@kernel void InitLocalVector(const int numElements,
+                             const int numDofs,
+                             Local_t restrict sol) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int d = 0; d < numDofs; ++d; @inner) {
+      sol(d, e) = 0;
+    }
+  }
+}

backends/occa/vector.cpp

@@ -0,0 +1,204 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "vector.hpp"
+#include "../../linalg/vector.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+PVector *Vector::DoVectorClone(bool copy_data, void **buffer,
+                               int buffer_type_id) const
+{
+   MFEM_ASSERT(buffer_type_id == ScalarId<double>::value, "");
+   Vector *new_vector = new Vector(OccaLayout());
+   if (copy_data)
+   {
+      new_vector->slice.copyFrom(slice);
+   }
+   if (buffer)
+   {
+      *buffer = new_vector->GetBuffer();
+   }
+   return new_vector;
+}
+
+void Vector::DoDotProduct(const PVector &x, void *result,
+                          int result_type_id) const
+{
+   // Can be called when Size() == 0, e.g. when an MPI-parallel vector has a
+   // local size of 0.
+
+   MFEM_ASSERT(result_type_id == ScalarId<double>::value, "");
+   double *res = (double *)result;
+   MFEM_ASSERT(dynamic_cast<const Vector *>(&x) != NULL, "invalid Vector type");
+   const Vector *xp = static_cast<const Vector *>(&x);
+   MFEM_ASSERT(this->Size() == xp->Size(), "");
+   *res = ::occa::linalg::dot<double, double, double>(this->slice, xp->slice);
+
+#ifdef MFEM_USE_MPI
+   double local_dot = *res;
+   if (IsParallel())
+   {
+      MPI_Allreduce(&local_dot, res, 1, MPI_DOUBLE, MPI_SUM,
+                    OccaLayout().OccaEngine().GetComm());
+   }
+#endif
+}
+
+void Vector::DoAxpby(const void *a, const PVector &x,
+                     const void *b, const PVector &y,
+                     int ab_type_id)
+{
+   const std::string &okl_defines = OccaLayout().OccaEngine().GetOklDefines();
+
+   //
+   // TODO: move all kernel builders to class mfem::occa::Backend
+   //
+   static ::occa::kernelBuilder axpby1_builder =
+      ::occa::linalg::customLinearMethod(
+         "mfem_occa_axpby1",
+         "v0[i] = c0 * v1[i];",
+         "defines: {"
+         "  CTYPE0: 'double',"
+         "  VTYPE0: 'double',"
+         "  VTYPE1: 'double',"
+         "  TILESIZE: '128',"
+         "}");
+
+   static ::occa::kernelBuilder axpby2_builder =
+      ::occa::linalg::customLinearMethod(
+         "mfem_occa_axpby2",
+         "v0[i] = c0 * v0[i] + c1 * v1[i];",
+         "defines: {"
+         "  CTYPE0: 'double',"
+         "  CTYPE1: 'double',"
+         "  VTYPE0: 'double',"
+         "  VTYPE1: 'double',"
+         "  TILESIZE: '128',"
+         "}");
+
+   static ::occa::kernelBuilder axpby3_builder =
+      ::occa::linalg::customLinearMethod(
+         "mfem_occa_axpby3",
+         "v0[i] = c0 * v1[i] + c1 * v2[i];",
+         "defines: {"
+         "  CTYPE0: 'double',"
+         "  CTYPE1: 'double',"
+         "  VTYPE0: 'double',"
+         "  VTYPE1: 'double',"
+         "  VTYPE2: 'double',"
+         "  TILESIZE: '128',"
+         "}");
+
+   // called only when Size() != 0
+
+   MFEM_ASSERT(ab_type_id == ScalarId<double>::value, "");
+   const double da = *static_cast<const double *>(a);
+   const double db = *static_cast<const double *>(b);
+   MFEM_ASSERT(da == 0.0 || dynamic_cast<const Vector *>(&x) != NULL,
+               "invalid Vector x");
+   MFEM_ASSERT(db == 0.0 || dynamic_cast<const Vector *>(&y) != NULL,
+               "invalid Vector y");
+   const Vector *xp = static_cast<const Vector *>(&x);
+   const Vector *yp = static_cast<const Vector *>(&y);
+
+   MFEM_ASSERT(da == 0.0 || this->Size() == xp->Size(), "");
+   MFEM_ASSERT(db == 0.0 || this->Size() == yp->Size(), "");
+
+   if (da == 0.0)
+   {
+      if (db == 0.0)
+      {
+         OccaFill(&da);
+      }
+      else
+      {
+         if (this->slice == yp->slice)
+         {
+            // *this *= db
+            ::occa::linalg::operator_mult_eq(slice, db);
+         }
+         else
+         {
+            // *this = db * y
+            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice(),
+                                                         okl_defines);
+            kernel((int)Size(), db, slice, yp->slice);
+         }
+      }
+   }
+   else
+   {
+      if (db == 0.0)
+      {
+         if (this->slice == xp->slice)
+         {
+            // *this *= da
+            ::occa::linalg::operator_mult_eq(slice, da);
+         }
+         else
+         {
+            // *this = da * x
+            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice(),
+                                                         okl_defines);
+            kernel((int)Size(), da, slice, xp->slice);
+         }
+      }
+      else
+      {
+         MFEM_ASSERT(xp->slice != yp->slice, "invalid input");
+         if (this->slice == xp->slice)
+         {
+            // *this = da * (*this) + db * y
+            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice(),
+                                                         okl_defines);
+            kernel((int)Size(), da, db, slice, yp->slice);
+         }
+         else if (this->slice == yp->slice)
+         {
+            // *this = da * x + db * (*this)
+            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice(),
+                                                         okl_defines);
+            kernel((int)Size(), db, da, slice, xp->slice);
+         }
+         else
+         {
+            // *this = da * x + db * y
+            ::occa::kernel kernel = axpby3_builder.build(slice.getDevice(),
+                                                         okl_defines);
+            kernel((int)Size(), da, db, slice, xp->slice, yp->slice);
+         }
+      }
+   }
+}
+
+mfem::Vector Vector::Wrap()
+{
+   return mfem::Vector(*this);
+}
+
+const mfem::Vector Vector::Wrap() const
+{
+   return mfem::Vector(*const_cast<Vector*>(this));
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/vector.hpp

@@ -0,0 +1,74 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_VECTOR_HPP
+#define MFEM_BACKENDS_OCCA_VECTOR_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include <occa.hpp>
+#include "../base/vector.hpp"
+#include "array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Vector : virtual public Array, public PVector
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // DLayout layout;
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   virtual PVector *DoVectorClone(bool copy_data, void **buffer,
+                                  int buffer_type_id) const;
+
+   virtual void DoDotProduct(const PVector &x, void *result,
+                             int result_type_id) const;
+
+   virtual void DoAxpby(const void *a, const PVector &x,
+                        const void *b, const PVector &y,
+                        int ab_type_id);
+
+   ///@}
+   // End: Virtual interface
+
+public:
+   Vector(Layout &lt)
+      : PArray(lt), Array(lt, sizeof(double)), PVector(lt)
+   { }
+
+   mfem::Vector Wrap();
+
+   const mfem::Vector Wrap() const;
+
+#if defined(MFEM_USE_MPI)
+   bool IsParallel() const { return (OccaLayout().OccaEngine().GetComm() != MPI_COMM_NULL); }
+#endif
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_VECTOR_HPP

backends/partialassembly/array.cpp

@@ -0,0 +1,87 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#include "array.hpp"
+#include <cstring>
+
+namespace mfem
+{
+
+namespace pa
+{
+
+////////////
+// CpuArray
+
+Array *Array::DoClone(bool copy_data, void **buffer,
+                      std::size_t item_size) const
+{
+   Layout& lay = dynamic_cast<Layout&>(*layout);
+   Array *new_array = new Array(lay, item_size);
+   if (copy_data)
+   {
+      memcpy(new_array->data, data, size);
+   }
+   if (buffer)
+   {
+      *buffer = new_array->data;
+   }
+   return new_array;
+}
+
+int Array::DoResize(PLayout &new_layout, void **buffer,
+                    std::size_t item_size)
+{
+   Layout *lt = static_cast<Layout *>(&new_layout);
+   layout.Reset(lt); // Reset() checks if the pointer is the same
+   int err = ResizeData(lt, item_size);
+   if (!err && buffer)
+   {
+      *buffer = this->data;
+   }
+   return err;
+}
+
+void* Array::DoPullData(void *buffer, std::size_t item_size)
+{
+   //Always on host
+   return data;
+}
+
+void Array::DoFill(const void *value_ptr, std::size_t item_size)
+{
+   char* this_data = GetTypedData<char>();
+   for (std::size_t i = 0; i < size; i += item_size)
+   {
+      memcpy(this_data + i, value_ptr, item_size);
+   }
+}
+
+void Array::DoPushData(const void *src_buffer, std::size_t item_size)
+{
+   if (src_buffer) memcpy(data, src_buffer, size);
+}
+
+void Array::DoAssign(const PArray &src, std::size_t item_size)
+{
+   // called only when Size() != 0
+   const Array* src_array = dynamic_cast<const Array*>(&src);
+   memcpy(data, src_array->data, size);
+}
+
+} // namespace mfem::pa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)

backends/partialassembly/array.hpp

@@ -0,0 +1,112 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_PA_ARRAY_HPP
+#define MFEM_BACKENDS_PA_ARRAY_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#include "layout.hpp"
+#include "../base/array.hpp"
+
+namespace mfem
+{
+
+namespace pa
+{
+
+/**
+*  Simple cpu backend array
+*/
+class Array : public virtual PArray
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // DLayout layout;
+
+   void* data;
+   std::size_t size;
+
+   //
+   // Virtual interface
+   //
+
+   virtual Array* DoClone(bool copy_data, void **buffer,
+                          std::size_t item_size) const;
+
+   virtual int DoResize(PLayout &new_layout, void **buffer,
+                        std::size_t item_size);
+
+   virtual void* DoPullData(void *buffer, std::size_t item_size);
+
+   virtual void DoFill(const void *value_ptr, std::size_t item_size);
+
+   virtual void DoPushData(const void *src_buffer, std::size_t item_size);
+
+   virtual void DoAssign(const PArray &src, std::size_t item_size);
+
+   //
+   // Auxiliary methods
+   //
+
+   inline int ResizeData(const Layout *lt, std::size_t item_size);
+
+public:
+   Array(Layout &lt, std::size_t item_size)
+      : PArray(lt),
+        data(lt.Alloc(lt.Size() * item_size)),
+        size(lt.Size() * item_size)
+   { }
+
+   virtual ~Array() { delete [] static_cast<char*>(data); }
+
+   /**
+   *  An unsafe way to access the data, tries to provide a semblance of type safety.
+   */
+   template <typename T>
+   T* GetTypedData() { return static_cast<T*>(data); }
+
+   template <typename T>
+   const T* GetTypedData() const { return static_cast<const T*>(data); }
+
+   /**
+   *  Overload the GetLayout of PArray to return Layout instead of PLayout to avoid to have to cast in the backend
+   *  This is compliant with PArray's definition since Layout is a Covariant type of PLayout.
+   */
+   Layout &GetLayout() const
+   { return *static_cast<Layout *>(layout.Get()); }
+};
+
+//
+// Inline methods
+//
+
+inline int Array::ResizeData(const Layout *lt, std::size_t item_size)
+{
+   const std::size_t new_bytes = lt->Size() * item_size;
+   if (size < new_bytes)
+   {
+      data = lt->Alloc(new_bytes);
+      size = new_bytes;
+   }
+   return 0;
+}
+
+} // namespace mfem::pa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#endif // MFEM_BACKENDS_PA_ARRAY_HPP

backends/partialassembly/backend.cpp

@@ -0,0 +1,46 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#include "backend.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+namespace pa
+{
+
+bool Backend::Supports(const std::string &engine_spec) const
+{
+	return true;
+}
+
+mfem::Engine *Create(const std::string &engine_spec)
+{
+	return new Engine(engine_spec);
+}
+
+#ifdef MFEM_USE_MPI
+mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec)
+{
+	return new Engine(comm, engine_spec);
+}
+#endif
+
+
+} // namespace mfem::pa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)

backends/partialassembly/backend.hpp

@@ -0,0 +1,48 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_PA_BACKEND_HPP
+#define MFEM_BACKENDS_PA_BACKEND_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+// Only the Backend and Engine classes should be exposed through "backend.hpp"
+#include "../base/backend.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+namespace pa
+{
+
+class Backend : public mfem::Backend
+{
+public:
+	virtual ~Backend();
+
+	virtual bool Supports(const std::string &engine_spec) const;
+
+	virtual mfem::Engine *Create(const std::string &engine_spec);
+
+#ifdef MFEM_USE_MPI
+	virtual mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec);
+#endif
+};
+
+} // namespace mfem::pa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#endif // MFEM_BACKENDS_PA_BACKEND_HPP

backends/partialassembly/bilinearform.cpp

@@ -0,0 +1,337 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#include "backend.hpp"
+#include "bilinearform.hpp"
+
+namespace mfem
+{
+
+namespace pa
+{
+
+BilinearForm::~BilinearForm()
+{
+   // Make sure all integrators free their data
+   for (int i = 0; i < tbfi.Size(); i++) delete tbfi[i];
+}
+
+void BilinearForm::TransferIntegrators(mfem::Array<mfem::BilinearFormIntegrator*>& bfi) {
+   for (int i = 0; i < bfi.Size(); i++)
+   {
+      mfem::FiniteElementSpace* fes = bform->FESpace();
+      const int order = fes->GetFE(0)->GetOrder();
+      const int ir_order = 2 * order + 1;
+      std::string integ_name(bfi[i]->Name());
+      // A better approach to this would be to use a map containing function pointers to create the desired Integrator
+      // using the integ_name as a key.
+      if (integ_name == "(undefined)")
+      {
+         MFEM_ABORT("BilinearFormIntegrator does not define Name()");
+      }
+      else if (integ_name == "mass")
+      {
+         std::cout << "=> " << integ_name << " Integrator transfered" << std::endl;
+         MassIntegrator* integ = dynamic_cast<MassIntegrator*>(bfi[i]);
+         Coefficient* coef;
+         integ->GetParameters(coef);
+         if (coef) {
+            std::cout << "==> with Coefficient" << std::endl;
+            typename MassEquation::ArgsCoeff args(*coef);
+            AddIntegrator( new PADomainInt<MassEquation, Vector<double>>(fes, ir_order, args) );
+         } else {
+            std::cout << "==> without Coefficient" << std::endl;
+            typename MassEquation::ArgsEmpty args;
+            AddIntegrator( new PADomainInt<MassEquation, Vector<double>>(fes, ir_order, args) );
+         }
+      }
+      else if (integ_name == "diffusion")
+      {
+         std::cout << "=> " << integ_name << " Integrator transfered" << std::endl;
+         DiffusionIntegrator* integ = dynamic_cast<DiffusionIntegrator*>(bfi[i]);
+         Coefficient* coef;
+         integ->GetParameters(coef);
+         typename DiffusionEquation::Args args(*coef);
+         AddIntegrator( new PADomainInt<DiffusionEquation, Vector<double>, TensorDomainMult>(fes, ir_order, args) );
+      }
+      else if (integ_name == "convection")
+      {
+         std::cout << "=> " << integ_name << " Integrator transfered" << std::endl;
+         ConvectionIntegrator* integ = dynamic_cast<ConvectionIntegrator*>(bfi[i]);
+         VectorCoefficient* u;
+         double* alpha;
+         integ->GetParameters(u, alpha);
+         typename DGConvectionEquation::Args args(*u, *alpha);
+         AddIntegrator( new PADomainInt<DGConvectionEquation, Vector<double>>(fes, ir_order, args) );
+      }
+      else if (integ_name == "transpose")
+      {
+         std::cout << "=> " << integ_name << " Integrator transfered" << std::endl;
+         TransposeIntegrator* transInteg = dynamic_cast<TransposeIntegrator*>(bfi[i]);
+         BilinearFormIntegrator* bf;
+         transInteg->GetParameters(bf);
+         integ_name = bf->Name();
+         if (integ_name == "dgtrace")
+         {
+            std::cout << "==> " << integ_name << " Integrator transfered" << std::endl;
+            DGTraceIntegrator* integ = dynamic_cast<DGTraceIntegrator*>(bf);
+            Coefficient* rho;
+            VectorCoefficient* u;
+            double* alpha;
+            double* beta;
+            integ->GetParameters(rho, u, alpha, beta);
+            typename DGConvectionEquation::Args args(*u, -(*alpha), *beta);
+            AddIntegrator( new PAFaceInt<DGConvectionEquation, Vector<double>>(fes, ir_order, args) );
+         }
+         else
+         {
+            MFEM_ABORT("Transpose BilinearFormIntegrator [Name() = " << integ_name
+                       << "] is not supported");
+         }
+      }
+      else if (integ_name == "fct")
+      {
+         std::cout << "=> " << integ_name << " Integrator transfered" << std::endl;
+         FCTIntegrator* integ = dynamic_cast<FCTIntegrator*>(bfi[i]);
+         VectorCoefficient* q;
+         mfem::Vector* d_e;
+         double* a;
+         double* b;
+         integ->GetParameters(q,d_e,a,b);
+         typename FCTEquation::Args args = {*q,*d_e,*a,*b};
+         AddIntegrator( new PAFaceInt<FCTEquation, Vector<double>>(fes, ir_order, args) );
+      }
+      else
+      {
+         MFEM_ABORT("BilinearFormIntegrator [Name() = " << integ_name
+                    << "] is not supported");
+      }
+   }
+}
+
+void BilinearForm::InitRHS(const mfem::Array<int> &ess_tdof_list,
+                           mfem::Vector &mfem_x, mfem::Vector &mfem_b,
+                           mfem::OperatorHandle& A,
+                           mfem::Vector &mfem_X, mfem::Vector &mfem_B,
+                           int copy_interior) const
+{
+   const mfem::Operator *P = GetProlongation();
+   const mfem::Operator *R = GetRestriction();
+
+   if (P)
+   {
+      // Variational restriction with P
+      mfem_B.Resize(P->InLayout());
+      P->MultTranspose(mfem_b, mfem_B);
+      mfem_X.Resize(R->OutLayout());
+      R->Mult(mfem_x, mfem_X);
+   }
+   else
+   {
+      // rap, X and B point to the same data as this, x and b
+      mfem_X.MakeRef(mfem_x);
+      mfem_B.MakeRef(mfem_b);
+   }
+
+   if (A.Type() != mfem::Operator::ANY_TYPE)
+   {
+      OperatorHandle mat_e;
+      A.EliminateBC(mat_e, ess_tdof_list, mfem_X, mfem_B);
+   }
+
+   if (!copy_interior && ess_tdof_list.Size() > 0)
+   {
+      Vector<double> &X = mfem_X.Get_PVector()->As<Vector<double>>();
+      const Array &constraint_list = ess_tdof_list.Get_PArray()->As<Array>();
+
+      double *X_data = X.GetData();
+      const int* constraint_data = constraint_list.GetTypedData<int>();
+
+      Vector<double> subvec(constraint_list.GetLayout());
+      double *subvec_data = subvec.GetData();
+
+      const std::size_t num_constraint = constraint_list.Size();
+
+      for (std::size_t i = 0; i < num_constraint; i++) subvec_data[i] = X_data[constraint_data[i]];
+
+      X.Fill(0.0);
+
+      for (std::size_t i = 0; i < num_constraint; i++) X_data[constraint_data[i]] = subvec_data[i];
+   }
+
+   if (A.Type() == mfem::Operator::ANY_TYPE)
+   {
+      ConstrainedOperator *A_constrained = static_cast<ConstrainedOperator*>(A.Ptr());
+      A_constrained->EliminateRHS(mfem_X, mfem_B);
+   }
+}
+
+
+bool BilinearForm::Assemble()
+{
+   if (!has_assembled)
+   {
+      TransferIntegrators(*bform->GetDBFI());
+      TransferIntegrators(*bform->GetFBFI());
+      has_assembled = true;
+   }
+
+   return true;
+}
+
+void BilinearForm::FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
+                                    mfem::OperatorHandle &A)
+{
+   if (A.Type() == mfem::Operator::ANY_TYPE)
+   {
+      // FIXME: Support different test and trial spaces (MixedBilinearForm)
+      const mfem::Operator *P = GetProlongation();
+
+      mfem::Operator *rap = this;
+      if (P != NULL) rap = new mfem::RAPOperator(*P, *this, *P);
+
+      A.Reset(new ConstrainedOperator(rap, ess_tdof_list, (rap != this)));
+   }
+   else
+   {
+      MFEM_ABORT("Operator::Type is not supported, type = " << A.Type());
+   }
+}
+
+void BilinearForm::FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
+                                    mfem::Vector &x, mfem::Vector &b,
+                                    mfem::OperatorHandle &A, mfem::Vector &X, mfem::Vector &B,
+                                    int copy_interior)
+{
+   FormSystemMatrix(ess_tdof_list, A);
+   InitRHS(ess_tdof_list, x, b, A, X, B, copy_interior);
+}
+
+void BilinearForm::RecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
+                                      mfem::Vector &x)
+{
+   const mfem::Operator *P = GetProlongation();
+   if (P)
+   {
+      // Apply conforming prolongation
+      x.Resize(P->OutLayout());
+      P->Mult(X, x);
+   }
+   // Otherwise X and x point to the same data
+}
+
+void BilinearForm::Mult(const mfem::Vector &x, mfem::Vector &y) const
+{
+   trial_fes->ToEVector(x.Get_PVector()->As<Vector<double>>(), x_local);
+
+   y_local.Fill<double>(0.0);
+   for (int i = 0; i < tbfi.Size(); i++) tbfi[i]->MultAdd(x_local, y_local);
+
+   test_fes->ToLVector(y_local, y.Get_PVector()->As<Vector<double>>());
+}
+
+void BilinearForm::MultTranspose(const mfem::Vector &x, mfem::Vector &y) const
+{ mfem_error("mfem::pa::BilinearForm::MultTranspose() is not supported!"); }
+
+
+ConstrainedOperator::ConstrainedOperator(mfem::Operator *A_,
+      const mfem::Array<int> &constraint_list_,
+      bool own_A_)
+   : Operator(A_->InLayout()->As<Layout>()),
+     A(A_),
+     own_A(own_A_),
+     constraint_list(*InLayout()->GetEngine().MakeLayout(constraint_list_.Size()).As<Layout>(), sizeof(int)),
+     z(OutLayout()->As<Layout>()),
+     w(OutLayout()->As<Layout>()),
+     mfem_z((z.DontDelete(), z)),
+     mfem_w((w.DontDelete(), w))
+{
+   constraint_list.PushData(constraint_list_.GetData());
+}
+
+void ConstrainedOperator::EliminateRHS(const mfem::Vector &mfem_x, mfem::Vector &mfem_b) const
+{
+   w.Fill<double>(0.0);
+
+   const Vector<double> &x = mfem_x.Get_PVector()->As<Vector<double>>();
+   Vector<double> &b = mfem_b.Get_PVector()->As<Vector<double>>();
+
+   const double *x_data = x.GetData();
+   double *b_data = b.GetData();
+   double *w_data = w.GetData();
+   const int* constraint_data = constraint_list.GetTypedData<int>();
+
+   const std::size_t num_constraint = constraint_list.Size();
+
+   if (num_constraint > 0)
+   {
+      for (std::size_t i = 0; i < num_constraint; i++)
+         w_data[constraint_data[i]] = x_data[constraint_data[i]];
+   }
+
+   A->Mult(mfem_w, mfem_z);
+
+   b.Axpby<double>(1.0, b, -1.0, z);
+
+   if (num_constraint > 0)
+   {
+      for (std::size_t i = 0; i < num_constraint; i++)
+         b_data[constraint_data[i]] = x_data[constraint_data[i]];
+   }
+}
+
+void ConstrainedOperator::Mult(const mfem::Vector &mfem_x, mfem::Vector &mfem_y) const
+{
+   if (constraint_list.Size() == 0)
+   {
+      A->Mult(mfem_x, mfem_y);
+      return;
+   }
+
+   const Vector<double> &x = mfem_x.Get_PVector()->As<Vector<double>>();
+   Vector<double> &y = mfem_y.Get_PVector()->As<Vector<double>>();
+
+   const double *x_data = x.GetData();
+   double *y_data = y.GetData();
+   double *z_data = z.GetData();
+   const int* constraint_data = constraint_list.GetTypedData<int>();
+
+   const std::size_t num_constraint = constraint_list.Size();
+
+   z.Assign<double>(x); // z = x
+
+   // z[constraint_list] = 0.0
+   for (std::size_t i = 0; i < num_constraint; i++)
+      z_data[constraint_data[i]] = 0.0;
+
+   // y = A * z
+   A->Mult(mfem_z, mfem_y);
+
+   // y[constraint_list] = x[constraint_list]
+   for (std::size_t i = 0; i < num_constraint; i++)
+      y_data[constraint_data[i]] = x_data[constraint_data[i]];
+}
+
+// Destructor: destroys the unconstrained Operator @a A if @a own_A is true.
+ConstrainedOperator::~ConstrainedOperator()
+{
+   if (own_A) delete A;
+}
+
+
+} // namespace mfem::pa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)

backends/partialassembly/bilinearform.hpp

@@ -0,0 +1,155 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_PA_BILINEARFORM_HPP
+#define MFEM_BACKENDS_PA_BILINEARFORM_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#include "fespace.hpp"
+#include "array.hpp"
+#include "vector.hpp"
+#include "../../fem/bilininteg.hpp"
+#include "partialassemblykernel.hpp"
+#include "integrator.hpp"
+
+namespace mfem
+{
+
+namespace pa
+{
+
+/**
+*  A backend BilinearForm for cpu that mostly duplicate the mfem::BilinearForm code...
+*/
+class BilinearForm : public mfem::PBilinearForm, public mfem::Operator
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // mfem::BilinearForm *bform;
+
+   mfem::Array<TensorBilinearFormIntegrator*> tbfi;
+   bool has_assembled;
+
+   mutable FiniteElementSpace *trial_fes, *test_fes;
+
+   mutable Vector<double> x_local, y_local;
+
+   /**
+   *  This function transfers the mfem::BilinearFormIntegrator to backend integrators.
+   */
+   void TransferIntegrators(mfem::Array<mfem::BilinearFormIntegrator*>& bfi);
+
+   void InitRHS(const mfem::Array<int> &constraint_list,
+                mfem::Vector &mfem_x, mfem::Vector &mfem_b,
+                mfem::OperatorHandle& A,
+                mfem::Vector &mfem_X, mfem::Vector &mfem_B,
+                int copy_interior = 0) const;
+
+   void AddIntegrator(TensorBilinearFormIntegrator* integrator){ tbfi.Append(integrator); }
+
+public:
+   BilinearForm(const Engine &e, mfem::BilinearForm &bf)
+      : mfem::PBilinearForm(e, bf),
+        // FIXME: for mixed bilinear forms
+        mfem::Operator(*bf.FESpace()->GetVLayout().As<Layout>()),
+        tbfi(),
+        has_assembled(false),
+        trial_fes(&bf.FESpace()->Get_PFESpace()->As<FiniteElementSpace>()),
+        test_fes(&bf.FESpace()->Get_PFESpace()->As<FiniteElementSpace>()),
+        x_local(trial_fes->GetELayout()),
+        y_local(test_fes->GetELayout()) { }
+
+   /// Virtual destructor
+   virtual ~BilinearForm();
+
+   /// Return the engine as a backend engine
+   const Engine &GetEngine() { return static_cast<const Engine&>(*engine); }
+
+   /** @brief Prolongation operator from linear algebra (linear system) vectors,
+       to input vectors for the operator. `NULL` means identity. */
+   virtual const Operator *GetProlongation() const { return bform->GetProlongation(); }
+
+   /** @brief Restriction operator from input vectors for the operator to linear
+       algebra (linear system) vectors. `NULL` means identity. */
+   virtual const Operator *GetRestriction() const  { return bform->GetRestriction(); }
+
+   /// Assemble the PBilinearForm.
+   /** This method is called from the method BilinearForm::Assemble() of the
+       associated BilinearForm #bform.
+       @returns True, if the host assembly should be skipped. */
+   virtual bool Assemble();
+
+   virtual void FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
+                                 mfem::OperatorHandle &A);
+
+   virtual void FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
+                                 mfem::Vector &x, mfem::Vector &b,
+                                 mfem::OperatorHandle &A, mfem::Vector &mfem_X, mfem::Vector &mfem_B,
+                                 int copy_interior);
+
+   virtual void RecoverFEMSolution(const mfem::Vector &mfem_X, const mfem::Vector &mfem_b,
+                                   mfem::Vector &mfem_x);
+
+   /// Operator application: `y=A(x)`.
+   virtual void Mult(const mfem::Vector &mfem_x, mfem::Vector &mfem_y) const;
+
+   /** @brief Action of the transpose operator: `y=A^t(x)`. The default behavior
+       in class Operator is to generate an error. */
+   virtual void MultTranspose(const mfem::Vector &mfem_x, mfem::Vector &mfem_y) const;
+};
+
+/**
+*  Duplicates the mfem::ConstrainedOperator in the backend
+*/
+class ConstrainedOperator : public mfem::Operator
+{
+   const mfem::Operator *A;
+   const bool own_A;
+   Array constraint_list;
+   mutable Vector<double> z, w;
+   mutable mfem::Vector mfem_z, mfem_w;
+
+public:
+   ConstrainedOperator(mfem::Operator *A_,
+                       const mfem::Array<int> &constraint_list_,
+                       bool own_A_ = false);
+
+   // Destructor: destroys the unconstrained Operator @a A if @a own_A is true.
+   virtual ~ConstrainedOperator();
+
+   /** @brief Eliminate "essential boundary condition" values specified in @a x
+       from the given right-hand side @a b.
+       Performs the following steps:
+       z = A((0,x_b));  b_i -= z_i;  b_b = x_b;
+       where the "_b" subscripts denote the essential (boundary) indices/dofs of
+       the vectors, and "_i" -- the rest of the entries. */
+   void EliminateRHS(const mfem::Vector &mfem_x, mfem::Vector &mfem_b) const;
+
+   /** @brief Constrained operator action.
+       Performs the following steps:
+       z = A((x_i,0));  y_i = z_i;  y_b = x_b;
+       where the "_b" subscripts denote the essential (boundary) indices/dofs of
+       the vectors, and "_i" -- the rest of the entries. */
+   virtual void Mult(const mfem::Vector &mfem_x, mfem::Vector &mfem_y) const;
+};
+
+} // namespace mfem::pa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#endif // MFEM_BACKENDS_PA_BILINEAR_FORM_HPP

backends/partialassembly/dgfacefunctions.cpp

@@ -0,0 +1,692 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+//This file contains useful functions to compute fluxes for DG methods.
+
+#include <vector>
+// #include "fem.hpp"
+#include "tensor.hpp"
+#include "../../linalg/vector.hpp"
+#include "../../mesh/mesh.hpp"
+
+using std::vector;
+using std::pair;
+
+namespace mfem
+{
+
+namespace pa
+{
+
+/**
+*	Returns the canonical coordinate vectors e_1 and e_2.
+*/
+void getBaseVector2D(mfem::Vector& e1, mfem::Vector& e2)
+{
+	e1.SetSize(2);
+	e1(0) = 1;
+	e1(1) = 0;
+	e2.SetSize(2);
+	e2(0) = 0;
+	e2(1) = 1;
+}
+
+/**
+*	Returns the canonical coordinate vectors e_1, e_2 and e_3.
+*/
+void getBaseVector3D(mfem::Vector& e1, mfem::Vector& e2, mfem::Vector& e3)
+{
+	e1.SetSize(3);
+	e1(0) = 1;
+	e1(1) = 0;
+	e1(2) = 0;
+	e2.SetSize(3);
+	e2(0) = 0;
+	e2(1) = 1;
+	e2(2) = 0;
+	e3.SetSize(3);
+	e3(0) = 0;
+	e3(1) = 0;
+	e3(2) = 1;
+}
+
+/** 
+*   A function that initialize the local coordinate base for a face with
+*   indice face_ind.
+*   This returns the local face coordinate base expressed in reference
+*   element coordinate.
+*/
+// Highly dependent of the node ordering from geom.cpp
+void InitFaceCoord2D(const int face_id, IntMatrix& base)
+{
+	//Vector e1,e2;
+	//getBaseVector2D(e1,e2);
+	base.zero();
+	switch(face_id)
+	{
+		case 0://SOUTH
+			base(0,0)= 1;//base.SetCol(0, e1);
+			base(1,1)=-1;//base.SetCol(1,-e2);
+			break;
+		case 1://EAST
+			base(1,0)= 1;//base.SetCol(0, e2);
+			base(0,1)= 1;//base.SetCol(1, e1);
+			break;
+		case 2://NORTH
+			base(0,0)=-1;//base.SetCol(0,-e1);
+			base(1,1)= 1;//base.SetCol(1, e2);
+			break;
+		case 3://WEST
+			base(1,0)=-1;//base.SetCol(0,-e2);
+			base(0,1)= 1;//base.SetCol(1, e1);
+			break;
+		default:
+			mfem_error("The face_ind exceeds the number of faces in this dimension.");
+			break;
+	}
+}
+
+// Highly dependent of the node ordering from geom.cpp
+void InitFaceCoord3D(const int face_id, IntMatrix& base)
+{
+	//Vector e1,e2,e3;
+	//getBaseVector3D(e1,e2,e3);
+	base.zero();
+	switch(face_id)
+	{
+		case 0://BOTTOM
+			base(0,0)= 1;//base.SetCol(0, e1);
+			base(1,1)=-1;//base.SetCol(1,-e2);
+			base(2,2)=-1;//base.SetCol(2,-e3);
+			break;		
+		case 1://SOUTH
+			base(0,0)= 1;//base.SetCol(0, e1);
+			base(2,1)= 1;//base.SetCol(1, e3);
+			base(1,2)=-1;//base.SetCol(2,-e2);
+			break;
+		case 2://EAST
+			base(1,0)= 1;//base.SetCol(0, e2);
+			base(2,1)= 1;//base.SetCol(1, e3);
+			base(0,2)= 1;//base.SetCol(2, e1);
+			break;
+		case 3://NORTH
+			base(0,0)=-1;//base.SetCol(0,-e1);
+			base(2,1)= 1;//base.SetCol(1, e3);
+			base(1,2)= 1;//base.SetCol(2, e2);
+			break;
+		case 4://WEST
+			base(1,0)=-1;//base.SetCol(0,-e2);
+			base(2,1)= 1;//base.SetCol(1, e3);
+			base(0,2)=-1;//base.SetCol(2,-e1);
+			break;
+		case 5://TOP
+			base(0,0)= 1;//base.SetCol(0, e1);
+			base(1,1)= 1;//base.SetCol(1, e2);
+			base(2,2)= 1;//base.SetCol(2, e3);
+			break;
+		default:
+			mfem_error("The face_ind exceeds the number of faces in this dimension.");
+			break;
+	}
+}
+
+/** Maps the coordinate vectors of the first face to the coordinate vectors of the second face.
+*   nb_rot is the number of rotation to opperate so that the first node of each face match.
+*   The result map contains pairs of int, where the first int is the cofficient, and the 
+*   second int is the indice of the second face vector.
+*/
+// There shouldn't be any rotation in 2D.
+void GetLocalCoordMap2D(vector<pair<int,int> >& map, const int nb_rot)
+{
+	map.resize(2);
+	//First and second coordinate vectors should always be of opposite direction in 2D.
+	//TODO Maybe not
+	map[0] = pair<int,int>(-1,0);
+	map[1] = pair<int,int>(-1,1);
+}
+
+void GetLocalCoordMap3D(vector< pair<int,int> >& map, const int orientation)
+{
+	map.resize(3);
+	// orientation determines how local coordinates are oriented from one face to the other.
+	// See case 2 for an example.
+	switch(orientation)
+	{
+		case 0://{0, 1, 2, 3}
+			map[0] = pair<int,int>( 1,0);
+			map[1] = pair<int,int>( 1,1);
+			map[2] = pair<int,int>( 1,2);
+			break;
+		case 1://{0, 3, 2, 1}
+			map[0] = pair<int,int>( 1,1);
+			map[1] = pair<int,int>( 1,0);
+			map[2] = pair<int,int>(-1,2);
+			break;
+		case 2://{1, 2, 3, 0}
+			//first vector equals -1 times the second vector of the other face coordinates
+			map[0] = pair<int,int>(-1,1);
+			//second vector equals -1 times the first vector of the other face coordinates
+			map[1] = pair<int,int>( 1,0);
+			//third vector equals -1 times the third vector of the other face coordinates
+			map[2] = pair<int,int>( 1,2);
+			break;
+		case 3://{1, 0, 3, 2}
+			map[0] = pair<int,int>(-1,0);
+			map[1] = pair<int,int>( 1,1);
+			map[2] = pair<int,int>(-1,2);
+			break;
+		case 4://{2, 3, 0, 1}
+			map[0] = pair<int,int>(-1,0);
+			map[1] = pair<int,int>(-1,1);
+			map[2] = pair<int,int>( 1,2);
+			break;
+		case 5://{2, 1, 0, 3}
+			map[0] = pair<int,int>(-1,1);
+			map[1] = pair<int,int>(-1,0);
+			map[2] = pair<int,int>(-1,2);
+			break;
+		case 6://{3, 0, 1, 2}
+			map[0] = pair<int,int>( 1,1);
+			map[1] = pair<int,int>(-1,0);
+			map[2] = pair<int,int>( 1,2);
+			break;
+		case 7://{3, 2, 1, 0}
+			map[0] = pair<int,int>( 1,0);
+			map[1] = pair<int,int>(-1,1);
+			map[2] = pair<int,int>(-1,2);
+			break;
+		default:
+			mfem_error("There shouldn't be that many orientations.");
+			break;
+	}
+}
+
+/**
+*	Returns the change of matrix P from base_K2 to base_K1 according to the mapping map.
+*/
+void GetChangeOfBasis(const IntMatrix& base_K1, IntMatrix& base_K2,
+								const vector<pair<int,int> >& map, IntMatrix& P)
+{
+	int dim = base_K1.Height();
+	int i,j,ind;
+	double coeff;
+	for (int n = 0; n < dim; ++n)
+	{
+		i = 0;
+		while( base_K1(i,n)   == 0 ) ++i;
+		j = 0;
+		ind = map[n].second;
+		while( base_K2(j,ind) == 0 ) ++j;
+		coeff = map[n].first;
+		P(i,j) = coeff * base_K1(i,n) * base_K2(j,ind);
+	}
+}
+
+void GetChangeOfBasis2D(const int face_id1, const int face_id2, IntMatrix& P)
+{
+	// We add 8 because of C++ stupid definition of modulo
+	int nb_rot = (8 + face_id2 - face_id1 - 2)%4;
+	P.zero();
+	switch(nb_rot)
+	{
+	case 0://Id=R^4
+		P(0,0) = 1;
+		P(1,1) = 1;
+		break;
+	case 1://R
+		P(1,0) = 1;
+		P(0,1) =-1;
+		break;
+	case 2://R²
+		P(0,0) =-1;
+		P(1,1) =-1;
+		break;
+	case 3://R³
+		P(1,0) =-1;
+		P(0,1) = 1;
+		break;
+	default:mfem_error("C++ modulo error in GetChangeOfBasis2D");
+	}
+}
+
+void GetChangeOfBasis(const int permutation, IntMatrix& P)
+{
+	int code1 = permutation/100;
+	int ind1  = code1/2;
+	int val1  = code1%2==0?-1:1;
+	int code2 = (permutation%100)/10;
+	int ind2  = code2/2;
+	int val2  = code2%2==0?-1:1;
+	int code3 = permutation%10;
+	int ind3  = code3/2;
+	int val3  = code3%2==0?-1:1;
+	P.zero();
+	P(ind1,0) = val1;
+	P(ind2,1) = val2;
+	P(ind3,2) = val3;
+}
+
+/**
+*	Returns the face_id that identifies the face on the reference element, and nb_rot the
+*  "rotations" the face did between reference to physical spaces.
+*/
+void GetIdRotInfo(const int face_info, int& face_id, int& nb_rot){
+	int orientation = face_info % 64;
+	face_id = face_info / 64;
+	nb_rot = orientation;
+}
+
+void GetFaceInfo(const Mesh* mesh, const int face, int& ind_elt1, int& ind_elt2, int& face_id1, int& face_id2, int& nb_rot1, int& nb_rot2)
+{
+	// We collect the indices of the two elements on the face, element1 is the master element,
+	// the one that defines the normal to the face.
+	mesh->GetFaceElements(face,&ind_elt1,&ind_elt2);
+	int info_elt1, info_elt2;
+	// We collect the informations on the face for the two elements.
+	mesh->GetFaceInfos(face,&info_elt1,&info_elt2);
+	GetIdRotInfo(info_elt1,face_id1,nb_rot1);//nb_rot1 is always 0 by convention
+	GetIdRotInfo(info_elt2,face_id2,nb_rot2);
+}
+
+/**
+*	Returns the permutation id, so that we can permute dofs to be in a structured case.
+*/
+int Permutation2D(const int face_id_trial, const int face_id_test)
+{
+	int perm = face_id_trial - face_id_test - 2;
+	perm = perm < 0 ? perm+4 : perm;
+	return perm;
+}
+
+/**
+*  Returns an integer that encrypts P.
+*/
+void Permutation3D(const int face_id1, const int face_id2, const int orientation, int& perm1, int& perm2)
+{
+	IntMatrix K1(3,3);
+	K1.zero();
+	InitFaceCoord3D(face_id1, K1);
+	IntMatrix K2(3,3);
+	K2.zero();
+	InitFaceCoord3D(face_id2, K2);
+	vector< pair<int,int> > map;
+	GetLocalCoordMap3D(map, orientation);
+	IntMatrix P(3,3);
+	P.zero();
+	GetChangeOfBasis(K1, K2, map, P);
+	perm1  = 0;
+	// Encrypts first column
+	perm1 += 100*(0*(P(0,0)==-1) + 1*(P(0,0)==1) + 2*(P(1,0)==-1) + 3*(P(1,0)==1) + 4*(P(2,0)==-1) + 5*(P(2,0)==1));
+	// Encrypts second column
+	perm1 += 10 *(0*(P(0,1)==-1) + 1*(P(0,1)==1) + 2*(P(1,1)==-1) + 3*(P(1,1)==1) + 4*(P(2,1)==-1) + 5*(P(2,1)==1));
+	// Encrypts third column
+	perm1 +=     (0*(P(0,2)==-1) + 1*(P(0,2)==1) + 2*(P(1,2)==-1) + 3*(P(1,2)==1) + 4*(P(2,2)==-1) + 5*(P(2,2)==1));
+	// Encrypts the transposed permutation matrix in a second integer.
+	perm2  = 0;
+	perm2 += 100*(0*(P(0,0)==-1) + 1*(P(0,0)==1) + 2*(P(0,1)==-1) + 3*(P(0,1)==1) + 4*(P(0,2)==-1) + 5*(P(0,2)==1));
+	perm2 += 10 *(0*(P(1,0)==-1) + 1*(P(1,0)==1) + 2*(P(1,1)==-1) + 3*(P(1,1)==1) + 4*(P(1,2)==-1) + 5*(P(1,2)==1));
+	perm2 +=     (0*(P(2,0)==-1) + 1*(P(2,0)==1) + 2*(P(2,1)==-1) + 3*(P(2,1)==1) + 4*(P(2,2)==-1) + 5*(P(2,2)==1));
+}
+
+void GetPermutation(const int dim, const int face_id1, const int face_id2, const int orientation, int& perm1, int& perm2)
+{
+	switch(dim){
+		case 1:
+			mfem_error("Not yet implemented");
+			break;
+		case 2:
+			perm1 = Permutation2D(face_id1, face_id2);
+			perm2 = Permutation2D(face_id2, face_id1);
+			break;
+		case 3:
+			Permutation3D(face_id1, face_id2, orientation, perm1, perm2);
+			break;
+		default:
+			mfem_error("Dimension of the problem too high.");
+			break;
+	}
+}
+
+/**
+*	Hardcoded permutation due to arbitrary hardcoded orientation in geom.cpp.
+*   Will break if geom.cpp changes.
+*   This function could be improved by returning the 'permutation' parameters once,
+*   instead of recomputing them for every quadrature point...
+*/
+int GetFaceQuadIndex3D(const int face_id, const int orientation, const int qind, const int quads, Tensor<1,int>& ind_f)
+{
+	int& k1 = ind_f(0);
+	int& k2 = ind_f(1);
+	int kf1,kf2;
+	kf1 = qind%quads;
+	kf2 = qind/quads;
+	switch(face_id)
+	{
+		case 0://BOTTOM
+			switch(orientation)
+			{
+				case 0://{0, 1, 2, 3}
+					k1 = kf1;
+					k2 = quads-1-kf2;
+					break;
+				case 1://{0, 3, 2, 1}
+					k1 = quads-1-kf2;
+					k2 = kf1;
+					break;
+				case 2://{1, 2, 3, 0}
+					k1 = quads-1-kf2;
+					k2 = quads-1-kf1;
+					break;
+				case 3://{1, 0, 3, 2}
+					k1 = quads-1-kf1;
+					k2 = quads-1-kf2;
+					break;
+				case 4://{2, 3, 0, 1}
+					k1 = quads-1-kf1;
+					k2 = kf2;
+					break;
+				case 5://{2, 1, 0, 3}
+					k1 = kf2;
+					k2 = quads-1-kf1;
+					break;
+				case 6://{3, 0, 1, 2}
+					k1 = kf2;
+					k2 = kf1;
+					break;
+				case 7://{3, 2, 1, 0}
+					k1 = kf1;
+					k2 = kf2;
+					break;
+				default:
+					mfem_error("This orientation does not exist in 3D");
+					break;
+			}
+			break;
+		case 1://SOUTH
+			switch(orientation)
+			{
+				case 0://{0, 1, 2, 3}
+					k1 = kf1;
+					k2 = kf2;
+					break;
+				case 1://{0, 3, 2, 1}
+					k1 = kf2;
+					k2 = kf1;
+					break;
+				case 2://{1, 2, 3, 0}
+					k1 = kf2;
+					k2 = quads-1-kf1;
+					break;
+				case 3://{1, 0, 3, 2}
+					k1 = quads-1-kf1;
+					k2 = kf2;
+					break;
+				case 4://{2, 3, 0, 1}
+					k1 = quads-1-kf1;
+					k2 = quads-1-kf2;
+					break;
+				case 5://{2, 1, 0, 3}
+					k1 = quads-1-kf2;
+					k2 = quads-1-kf1;
+					break;
+				case 6://{3, 0, 1, 2}
+					k1 = quads-1-kf2;
+					k2 = kf1;
+					break;
+				case 7://{3, 2, 1, 0}
+					k1 = kf1;
+					k2 = quads-1-kf2;
+					break;
+				default:
+					mfem_error("This orientation does not exist in 3D");
+					break;
+			}
+			break;
+		case 2://EAST
+			switch(orientation)
+			{
+				case 0://{0, 1, 2, 3}
+					k1 = kf1;
+					k2 = kf2;
+					break;
+				case 1://{0, 3, 2, 1}
+					k1 = kf2;
+					k2 = kf1;
+					break;
+				case 2://{1, 2, 3, 0}
+					k1 = kf2;
+					k2 = quads-1-kf1;
+					break;
+				case 3://{1, 0, 3, 2}
+					k1 = quads-1-kf1;
+					k2 = kf2;
+					break;
+				case 4://{2, 3, 0, 1}
+					k1 = quads-1-kf1;
+					k2 = quads-1-kf2;
+					break;
+				case 5://{2, 1, 0, 3}
+					k1 = quads-1-kf2;
+					k2 = quads-1-kf1;
+					break;
+				case 6://{3, 0, 1, 2}
+					k1 = quads-1-kf2;
+					k2 = kf1;
+					break;
+				case 7://{3, 2, 1, 0}
+					k1 = kf1;
+					k2 = quads-1-kf2;
+					break;
+				default:
+					mfem_error("This orientation does not exist in 3D");
+					break;
+			}
+			break;
+		case 3://NORTH
+			switch(orientation)
+			{
+				case 0://{0, 1, 2, 3}
+					k1 = quads-1-kf1;
+					k2 = kf2;
+					break;
+				case 1://{0, 3, 2, 1}
+					k1 = kf2;
+					k2 = quads-1-kf1;
+					break;
+				case 2://{1, 2, 3, 0}
+					k1 = kf2;
+					k2 = kf1;
+					break;
+				case 3://{1, 0, 3, 2}
+					k1 = kf1;
+					k2 = kf2;
+					break;
+				case 4://{2, 3, 0, 1}
+					k1 = kf1;
+					k2 = quads-1-kf2;
+					break;
+				case 5://{2, 1, 0, 3}
+					k1 = quads-1-kf2;
+					k2 = kf1;
+					break;
+				case 6://{3, 0, 1, 2}
+					k1 = quads-1-kf2;
+					k2 = quads-1-kf1;
+					break;
+				case 7://{3, 2, 1, 0}
+					k1 = quads-1-kf1;
+					k2 = quads-1-kf2;
+					break;
+				default:
+					mfem_error("This orientation does not exist in 3D");
+					break;
+			}
+			break;
+		case 4://WEST
+			switch(orientation)
+			{
+				case 0://{0, 1, 2, 3}
+					k1 = quads-1-kf1;
+					k2 = kf2;
+					break;
+				case 1://{0, 3, 2, 1}
+					k1 = kf2;
+					k2 = quads-1-kf1;
+					break;
+				case 2://{1, 2, 3, 0}
+					k1 = kf2;
+					k2 = kf1;
+					break;
+				case 3://{1, 0, 3, 2}
+					k1 = kf1;
+					k2 = kf2;
+					break;
+				case 4://{2, 3, 0, 1}
+					k1 = kf1;
+					k2 = quads-1-kf2;
+					break;
+				case 5://{2, 1, 0, 3}
+					k1 = quads-1-kf2;
+					k2 = kf1;
+					break;
+				case 6://{3, 0, 1, 2}
+					k1 = quads-1-kf2;
+					k2 = quads-1-kf1;
+					break;
+				case 7://{3, 2, 1, 0}
+					k1 = quads-1-kf1;
+					k2 = quads-1-kf2;
+					break;
+				default:
+					mfem_error("This orientation does not exist in 3D");
+					break;
+			}
+			break;
+		case 5://TOP
+			switch(orientation)
+			{
+				case 0://{0, 1, 2, 3}
+					k1 = kf1;
+					k2 = kf2;
+					break;
+				case 1://{0, 3, 2, 1}
+					k1 = kf2;
+					k2 = kf1;
+					break;
+				case 2://{1, 2, 3, 0}
+					k1 = kf2;
+					k2 = quads-1-kf1;
+					break;
+				case 3://{1, 0, 3, 2}
+					k1 = quads-1-kf1;
+					k2 = kf2;
+					break;
+				case 4://{2, 3, 0, 1}
+					k1 = quads-1-kf1;
+					k2 = quads-1-kf2;
+					break;
+				case 5://{2, 1, 0, 3}
+					k1 = quads-1-kf2;
+					k2 = quads-1-kf1;
+					break;
+				case 6://{3, 0, 1, 2}
+					k1 = quads-1-kf2;
+					k2 = kf1;
+					break;
+				case 7://{3, 2, 1, 0}
+					k1 = kf1;
+					k2 = quads-1-kf2;
+					break;
+				default:
+					mfem_error("This orientation does not exist in 3D");
+					break;
+			}
+			break;
+		default:
+			mfem_error("This face_id does not exist in 3D");
+			break;	
+	}
+	return k1 + quads*k2;
+}
+
+int GetFaceQuadIndex(const int dim, const int face_id, const int orientation, const int qind, const int quads, Tensor<1,int>& ind_f)
+{
+	int res = 0;
+	switch(dim)
+	{
+		case 1:
+			break;
+		case 2:
+            if(face_id<=1){//SOUTH or EAST (canonical ordering)
+               res = ind_f(0) = qind;
+            }else{//NORTH or WEST (counter-canonical ordering)
+               res = ind_f(0) = quads-1-qind;
+            }
+            break;
+        case 3:
+			res = GetFaceQuadIndex3D(face_id, orientation, qind, quads, ind_f);
+			break;
+		default:
+			mfem_error("Dimension too high.");
+			break;
+	}
+	return res;
+}
+
+const int GetGlobalQuadIndex(const int dim, const int face_id, const int quads, Tensor<1,int>& ind_f)
+{
+   switch(dim)
+   {
+      case 1:
+         if (face_id==0)//WEST
+         {
+            return 0;
+         }else{//EAST
+            return quads-1;
+         }
+      case 2:
+            switch(face_id)
+            {
+               case 0://SOUTH
+                  return ind_f(0);
+               case 1://EAST
+                  return quads-1 + ind_f(0)*quads;
+               case 2://NORTH
+                  return ind_f(0) + (quads-1)*quads;
+               case 3://WEST
+                  return ind_f(0)*quads;
+            }
+        case 3:
+         switch(face_id)
+         {
+            case 0://BOTTOM
+               return ind_f(0) + ind_f(1)*quads;
+            case 1://SOUTH
+               return ind_f(0) + ind_f(1)*quads*quads;
+            case 2://EAST
+               return (quads-1) + ind_f(0)*quads + ind_f(1)*quads*quads;
+            case 3://NORTH
+               return ind_f(0) + (quads-1)*quads + ind_f(1)*quads*quads;
+            case 4://WEST
+               return ind_f(0)*quads + ind_f(1)*quads*quads;
+            case 5://TOP
+               return ind_f(0) + ind_f(1)*quads + (quads-1)*quads*quads;
+         }
+      default:
+         mfem_error("Dimension too high.");
+         break;
+   }
+   return -1;
+}
+
+}
+
+}

backends/partialassembly/dgfacefunctions.hpp

@@ -0,0 +1,129 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+//This file contains useful functions to compute fluxes for DG methods.
+
+
+#ifndef MFEM_DGFACEFUNC
+#define MFEM_DGFACEFUNC
+#include "tensor.hpp"
+#include <vector>
+#include <utility>
+#include "../../linalg/vector.hpp"
+#include "../../mesh/mesh.hpp"
+#include "tensorialfunctions.hpp"
+
+using std::vector;
+using std::pair;
+
+namespace mfem
+{
+
+namespace pa
+{
+
+/**
+*	Returns the canonical coordinate vectors e_1 and e_2.
+*/
+void getBaseVector2D(mfem::Vector& e1, mfem::Vector& e2);
+
+/**
+*	Returns the canonical coordinate vectors e_1, e_2 and e_3.
+*/
+void getBaseVector3D(mfem::Vector& e1, mfem::Vector& e2, mfem::Vector& e3);
+
+/** A function that initialize the local coordinate base for a face with
+*   indice face_ind.
+*   This returns the local face coordinate base expressed in reference
+*   element coordinate.
+*/
+// Highly dependent of the node ordering from geom.cpp
+void InitFaceCoord2D(const int face_id, IntMatrix& base);
+
+// Highly dependent of the node ordering from geom.cpp
+void InitFaceCoord3D(const int face_id, IntMatrix& base);
+
+/** Maps the coordinate vectors of the first face to the coordinate vectors of the second face.
+*   nb_rot is the number of rotation to opperate so that the first node of each face match.
+*   The result map contains pairs of int, where the first int is a direction cofficient,
+*	 and the second int is the indice of the second face vector.
+*/
+// There shouldn't be any rotation in 2D.
+void GetLocalCoordMap2D(vector<pair<int,int> >& map, const int nb_rot = 0);
+
+// Rotations follow the ordering of the nodes.
+void GetLocalCoordMap3D(vector<pair<int,int> >& map, const int nb_rot);
+
+/**
+*	Returns the change of matrix P from base_K2 to base_K1 according to the mapping map.
+*/
+void GetChangeOfBasis(const IntMatrix& base_K1, IntMatrix& base_K2,
+								const vector<pair<int,int> >& map, IntMatrix& P);
+
+void GetChangeOfBasis(const int permutation, IntMatrix& P);
+
+/**
+*	Returns the change of coordinate from second element to first element on a 2D face.
+*/
+void GetChangeOfBasis2D(const int face_id1, const int face_id2, IntMatrix& P);
+
+/**
+*	Returns the indices, face ID, and number of rotations, of the two element sharing a face.
+*  The number of rotations is relative to the element 1, so nb_rot1 is always 0.
+*/
+void GetFaceInfo(const Mesh* mesh, const int face,
+						int& ind_elt1, int& ind_elt2,
+						int& face_id1, int& face_id2,
+						int& nb_rot1, int& nb_rot2);
+
+/**
+*	Returns the face_id that identifies the face on the reference element, and nb_rot the
+*  "rotations" the face did between reference to physical spaces.
+*/
+void GetIdRotInfo(const int face_info, int& face_id, int& nb_rot);
+
+/**
+*	Returns an integer identifying the permutation to apply to be in structured-
+*  like configuration for 2D hex meshes.
+*/
+int Permutation2D(const int face_id_trial, const int face_id_test);
+
+/**
+*	Returns an integer identifying the permutation to apply to be in structured-
+*  like configuration for 3D hex meshes.
+*/
+void Permutation3D(const int face_id1, const int face_id2, const int orientation, int& perm1, int& perm2);
+
+/**
+*	Returns an integer identifying the permutation to apply to be in structured-
+*  like configuration.
+*/
+void GetPermutation(const int dim, const int face_id1, const int face_id2, const int orientation, int& perm1, int& perm2);
+
+int GetFaceQuadIndex3D(const int face_id, const int orientation, const int qind, const int quads, Tensor<1,int>& ind_f);
+
+/**
+*	Returns the indices of a quadrature point on the face of an hex element relative to the index of the quadrature
+*  point on the reference face.
+*/
+int GetFaceQuadIndex(const int dim, const int face_id, const int orientation, const int qind, const int quads, Tensor<1,int>& ind_f);
+
+/**
+*	Returns the indices of a quadrature point on the element relative to the index of the quadrature
+*  point on the reference face.
+*/
+const int GetGlobalQuadIndex(const int dim, const int face_id, const int quads, Tensor<1,int>& ind_f);
+
+}
+
+}
+
+#endif // MFEM_DGFACEFUNC

backends/partialassembly/dgpabilininteg.hpp

@@ -0,0 +1,216 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+
+// This file contains a prototype version for Discontinuous Galerkin Partial assembly
+
+#ifndef MFEM_DGPABILININTEG
+#define MFEM_DGPABILININTEG
+
+// #include "bilininteg.hpp"
+#include "tensor.hpp"
+
+// #include "fem.hpp"
+// #include <cmath>
+// #include <algorithm>
+// #include "../linalg/vector.hpp"
+
+namespace mfem
+{
+
+namespace pa
+{
+
+/**
+*  The different operators available for the Kernels
+*/
+enum PAOp { BtDB, BtDG, GtDB, GtDG };
+
+
+class DiffusionEquation
+{
+public:
+   static const PAOp OpName = GtDG;
+
+   struct Args {
+      Args(Coefficient& q) : q(q) {}
+      Coefficient& q;
+   };
+
+   void evalD(Tensor<2>& res, ElementTransformation *Tr, const IntegrationPoint& ip,
+                  const Tensor<2>& Jac, const Args& args)
+   {
+      const int dim = res.size(0);
+      Tensor<2> Adj(dim,dim);
+      adjugate(Jac,Adj);
+      double val = 0.0;
+      double qval = 1.0;
+      double detJ = det(Jac);
+      qval = 1.0;//args.q.Eval(*Tr, ip);//FIXME
+      for (int i = 0; i < dim; ++i)
+      {
+         for (int j = 0; j < dim; ++j)
+         {
+            val = 0.0;
+            for (int k = 0; k < dim; ++k)
+            {
+               val += Adj(i,k)*Adj(j,k); //Adj*Adj^T
+            }
+            res(i,j) = ip.weight * qval / detJ * val;
+         }
+      }
+   }
+};
+
+/**
+*	A class that describes the Convection Equation using DG for Partial Assembly.
+*/
+class DGConvectionEquation
+{
+public:
+   /**
+   *  Defines the Kernel to apply to the Domain
+   */
+   static const PAOp OpName = BtDG;
+
+   /**
+   *  Defines the variables needed to build D for the Domain kernel
+   */
+   struct Args {
+      Args(VectorCoefficient& _q, double _a, double _b = 0.0) : q(_q), a(_a), b(_b) {}
+      VectorCoefficient& q;
+      double a;
+      double b;
+   };
+
+   /**
+   *  Returns the values of the D tensor at a given integration Point.
+   */
+   void evalD(Tensor<1>& res, ElementTransformation *Tr, const IntegrationPoint& ip,
+                  const Tensor<2>& Jac, const Args& args)
+   {
+      const int dim = res.size(0);
+      mfem::Vector qvec(dim);
+      args.q.Eval(qvec, *Tr, ip);
+      Tensor<2> Adj(dim,dim);
+      adjugate(Jac,Adj);
+      for (int i = 0; i < dim; ++i)
+      {
+         double val = 0.0;
+         for (int j = 0; j < dim; ++j)
+         {
+            val += Adj(i,j) * qvec(j);
+         }
+         res(i) = ip.weight * args.a * val;
+      }
+   }
+
+   /**
+   *  Defines the Kernel to apply to the Faces
+   */
+   static const PAOp FaceOpName = BtDB;
+
+   /**
+   *  Returns the values of the Dint and Dext tensors at a given integration Point for
+   *  each element over a face.
+   */
+   void evalFaceD(double& res11, double& res21, double& res22, double& res12,
+      const FaceElementTransformations* face_tr, const mfem::Vector& normal,
+      const IntegrationPoint& ip1, const IntegrationPoint& ip2,
+      const Tensor<2>& Jac1, const Tensor<2>& Jac2,
+      const Args& args)
+   {
+      const int dim = normal.Size();
+      mfem::Vector qvec(dim);
+      // FIXME: qvec might be discontinuous if not constant with a periodic mesh
+      // We should then use the evaluation on Elem2 and eip2
+      args.q.Eval( qvec, *(face_tr->Elem1), ip1 );
+      const double res = qvec * normal;
+      const double a = -args.a, b = args.b;
+      res11 = ip1.weight * (   a/2 * res + b * abs(res) );
+      res21 = ip1.weight * (   a/2 * res - b * abs(res) );
+      res22 = ip1.weight * ( - a/2 * res + b * abs(res) );
+      res12 = ip1.weight * ( - a/2 * res - b * abs(res) );
+   }  
+};
+
+class FCTEquation
+{
+public:
+   struct Args {
+      VectorCoefficient& q;
+      mfem::Vector& d_e;
+      double a;
+      double b;
+   };
+
+   static const PAOp FaceOpName = BtDB;
+
+   void evalFaceD(double& res11, double& res21, double& res22, double& res12,
+      const FaceElementTransformations* face_tr, const mfem::Vector& normal,
+      const IntegrationPoint& ip1, const IntegrationPoint& ip2,
+      const Tensor<2>& Jac1, const Tensor<2>& Jac2,
+      const Args& args)
+   {
+      const int dim = normal.Size();
+      mfem::Vector qvec(dim);
+      // FIXME: qvec might be discontinuous if not constant with a periodic mesh
+      // We should then use the evaluation on Elem2 and eip2
+      args.q.Eval( qvec, *(face_tr->Elem1), ip1 );
+      const double res = qvec * normal;
+      const double a = -args.a, b = args.b;
+      res11 = 0.0; //ip1.weight * (   a/2 * res + b * abs(res) );
+      res21 = 0.0; //ip1.weight * (   a/2 * res - b * abs(res) );
+      res22 = 0.0; //ip1.weight * ( - a/2 * res + b * abs(res) );
+      res12 = 0.0; //ip1.weight * ( - a/2 * res - b * abs(res) );      
+   }
+};
+
+/**
+*  A class that describes a Mass Equation using Partial Assembly
+*/
+class MassEquation
+{
+public:
+   static const PAOp OpName = BtDB;
+
+   struct ArgsEmpty{};
+
+   void evalD(double& res, ElementTransformation* Tr, const IntegrationPoint& ip,
+               const Tensor<2>& Jac, const ArgsEmpty& args)
+   {
+      res = ip.weight * det(Jac);
+   }
+
+   struct ArgsCoeff
+   {
+      ArgsCoeff(Coefficient& coeff): coeff(coeff) {}
+      Coefficient& coeff;
+   };
+
+   void evalD(double& res, ElementTransformation* Tr, const IntegrationPoint& ip,
+               const Tensor<2>& Jac, const ArgsCoeff& args)
+   {
+      res = args.coeff.Eval(*Tr, ip) * ip.weight * det(Jac);
+   }
+
+   void evalD(double& res, ElementTransformation* Tr, const IntegrationPoint& ip,
+               const Tensor<2>& Jac, Coefficient& coeff)
+   {
+      res = coeff.Eval(*Tr, ip) * ip.weight * det(Jac);
+   }
+};
+
+}
+
+}
+
+#endif //MFEM_DGPABILININTEG

backends/partialassembly/engine.cpp

@@ -0,0 +1,128 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#include "backend.hpp"
+#include "bilinearform.hpp"
+#include "../../general/array.hpp"
+
+namespace mfem
+{
+
+namespace pa
+{
+
+void Engine::Init(const std::string &engine_spec)
+{
+   //
+   // Initialize inherited fields
+   //
+   memory_resources[0] = NULL;
+   workers_weights[0] = 1.0;
+   workers_mem_res[0] = 0;
+}
+
+Engine::Engine()
+   : mfem::Engine(NULL, 1, 1)
+{
+   Init("");
+}
+
+Engine::Engine(const std::string &engine_spec)
+   : mfem::Engine(NULL, 1, 1)
+{
+   Init(engine_spec);
+}
+
+#ifdef MFEM_USE_MPI
+Engine::Engine(MPI_Comm _comm, const std::string &engine_spec)
+   : mfem::Engine(NULL, 1, 1)
+{
+   comm = _comm;
+   Init(engine_spec);
+}
+#endif
+
+DLayout Engine::MakeLayout(std::size_t size) const
+{
+   return DLayout(new Layout(*this, size));
+}
+
+DLayout Engine::MakeLayout(const mfem::Array<std::size_t> &offsets) const
+{
+   MFEM_ASSERT(offsets.Size() == 2,
+               "multiple workers are not supported yet");
+   return DLayout(new Layout(*this, offsets.Last()));
+}
+
+DArray Engine::MakeArray(PLayout &layout, std::size_t item_size) const
+{
+   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
+               "invalid input layout");
+   Layout *lt = static_cast<Layout *>(&layout);
+   return DArray(new Array(*lt, item_size));
+}
+
+DVector Engine::MakeVector(PLayout &layout, int type_id) const
+{
+   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
+               "invalid input layout");
+   Layout *lt = static_cast<Layout *>(&layout);
+   switch (type_id)
+   {
+   case ScalarId<double>::value:
+      return DVector(new Vector<double>(*lt));
+   case ScalarId<std::complex<double>>::value:
+      return DVector(new Vector<std::complex<double>>(*lt));
+   // case ScalarId<int>::value:
+   //    return DVector(new Vector<int>(*lt));
+   default:
+      mfem_error("Invalid type_id");
+   }
+}
+
+DFiniteElementSpace Engine::MakeFESpace(mfem::FiniteElementSpace &fespace) const
+{
+   return DFiniteElementSpace(new FiniteElementSpace(*this, fespace));
+}
+
+DBilinearForm Engine::MakeBilinearForm(mfem::BilinearForm &bf) const
+{
+   return DBilinearForm(new BilinearForm(*this, bf));
+}
+
+void Engine::AssembleLinearForm(LinearForm &l_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+}
+
+mfem::Operator *Engine::MakeOperator(const MixedBilinearForm &mbl_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+   return NULL;
+}
+
+mfem::Operator *Engine::MakeOperator(const NonlinearForm &nl_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+   return NULL;
+}
+
+} // namespace mfem::pa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)

backends/partialassembly/engine.hpp

@@ -0,0 +1,91 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_PA_ENGINE_HPP
+#define MFEM_BACKENDS_PA_ENGINE_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#include "../base/backend.hpp"
+
+namespace mfem
+{
+
+namespace pa
+{
+
+class Engine : public mfem::Engine
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // mfem::Backend *backend;
+#ifdef MFEM_USE_MPI
+   // MPI_Comm comm;
+#endif
+   // int num_mem_res;
+   // int num_workers;
+   // MemoryResource **memory_resources;
+   // double *workers_weights;
+   // int *workers_mem_res;
+
+   void Init(const std::string &engine_spec);
+
+public:
+   Engine();
+   Engine(const std::string &engine_spec);
+
+#ifdef MFEM_USE_MPI
+   Engine(MPI_Comm comm, const std::string &engine_spec);
+#endif
+
+   virtual ~Engine() { }
+
+   /**
+       @name Virtual interface: finite element data structures and algorithms
+    */
+   ///@{
+
+   virtual DLayout MakeLayout(std::size_t size) const;
+   virtual DLayout MakeLayout(const mfem::Array<std::size_t> &offsets) const;
+
+   virtual DArray MakeArray(PLayout &layout, std::size_t item_size) const;
+
+   virtual DVector MakeVector(PLayout &layout,
+                              int type_id = ScalarId<double>::value) const;
+
+   virtual DFiniteElementSpace MakeFESpace(mfem::FiniteElementSpace &
+                                           fespace) const;
+
+   virtual DBilinearForm MakeBilinearForm(mfem::BilinearForm &bf) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual void AssembleLinearForm(LinearForm &l_form) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual mfem::Operator *MakeOperator(const MixedBilinearForm &mbl_form) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual mfem::Operator *MakeOperator(const NonlinearForm &nl_form) const;
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem::pa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#endif // MFEM_BACKENDS_PA_ENGINE_HPP

backends/partialassembly/facekernels.cpp

@@ -0,0 +1,137 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "facekernels.hpp"
+
+namespace mfem{
+
+namespace pa{
+
+void Permutation::Permutation2d(int face_id, int nbe, int dofs1d, const Tensor3d& T0, Tensor3d& T0p) const
+{
+   for (int e = 0; e < nbe; ++e)
+   {
+      const int trial = kernel_data(e,face_id).indirection;
+      const int permutation = kernel_data(e,face_id).permutation;
+      if(trial!=-1)
+      {
+         switch(permutation)
+         {
+         case 0:
+            for (int i2 = 0; i2 < dofs1d; ++i2)
+            {
+               for (int i1 = 0; i1 < dofs1d; ++i1)
+               {
+                  T0p(i1,i2,e) = T0(i1,i2,trial);
+               }
+            }
+            break;
+         case 1:
+            for (int i2 = 0, j1 = dofs1d-1; i2 < dofs1d; ++i2, --j1)
+            {
+               for (int i1 = 0, j2 = 0; i1 < dofs1d; ++i1, ++j2)
+               {
+                  T0p(i1,i2,e) = T0(j1,j2,trial);
+               }
+            }
+            break;
+         case 2:
+            for (int i2 = 0, j2 = dofs1d-1; i2 < dofs1d; ++i2, --j2)
+            {
+               for (int i1 = 0, j1 = dofs1d-1; i1 < dofs1d; ++i1, --j1)
+               {
+                  T0p(i1,i2,e) = T0(j1,j2,trial);
+               }
+            }
+            break;
+         case 3:
+            for (int i2 = 0, j1 = 0; i2 < dofs1d; ++i2, ++j1)
+            {
+               for (int i1 = 0, j2 = dofs1d-1; i1 < dofs1d; ++i1, --j2)
+               {
+                  T0p(i1,i2,e) = T0(j1,j2,trial);
+               }
+            }
+            break;
+         default:
+            mfem_error("This permutation id does not exist in 2D");
+         }
+      }else{
+         for (int i2 = 0; i2 < dofs1d; ++i2)
+         {
+            for (int i1 = 0; i1 < dofs1d; ++i1)
+            {
+               T0p(i1,i2,e) = 0.0;
+            }
+         }
+      }
+   }
+}
+
+/**
+*  The Permutation3d works differently than the 2d version, here we receive a change of basis matrix encrypted in one integer.
+*/
+void Permutation::Permutation3d(int face_id, int nbe, int dofs1d, const Tensor4d& T0, Tensor4d& T0p) const
+{
+   const double* U = T0.getData();
+   int elt, ii, jj, kk;
+   const int step_elt = dofs1d*dofs1d*dofs1d;
+   for (int e = 0; e < nbe; ++e)
+   {
+      const int trial = kernel_data(e,face_id).indirection;
+      const int permutation = kernel_data(e,face_id).permutation;
+      if (trial!=-1)
+      {
+         elt = trial*step_elt;
+         IntMatrix P(3,3);
+         GetChangeOfBasis(permutation, P);
+         int begin_ii = (P(0,0)==-1)*(dofs1d-1) + (P(1,0)==-1)*(dofs1d*dofs1d-1) + (P(2,0)==-1)*(dofs1d*dofs1d*dofs1d-1);
+         int begin_jj = (P(0,1)==-1)*(dofs1d-1) + (P(1,1)==-1)*(dofs1d*dofs1d-1) + (P(2,1)==-1)*(dofs1d*dofs1d*dofs1d-1);
+         int begin_kk = (P(0,2)==-1)*(dofs1d-1) + (P(1,2)==-1)*(dofs1d*dofs1d-1) + (P(2,2)==-1)*(dofs1d*dofs1d*dofs1d-1);
+         int step_ii  = P(0,0) + P(1,0)*dofs1d + P(2,0)*dofs1d*dofs1d;
+         int step_jj  = P(0,1) + P(1,1)*dofs1d + P(2,1)*dofs1d*dofs1d;
+         int step_kk  = P(0,2) + P(1,2)*dofs1d + P(2,2)*dofs1d*dofs1d;
+         kk = begin_kk;
+         for (int k = 0; k < dofs1d; ++k)
+         {
+            jj = begin_jj;
+            for (int j = 0; j < dofs1d; ++j)
+            {
+               ii = begin_ii;
+               for (int i = 0; i < dofs1d; ++i)
+               {
+                  T0p(i,j,k,e) = U[ elt + ii + jj + kk ];
+                  ii += step_ii;
+               }
+               jj += step_jj;
+            }
+            kk += step_kk;
+         }
+      }
+      else
+      {
+         for (int k = 0; k < dofs1d; ++k)
+         {
+            for (int j = 0; j < dofs1d; ++j)
+            {
+               for (int i = 0; i < dofs1d; ++i)
+               {
+                  T0p(i,j,k,e) = 0.0;
+               }
+            }
+         }
+      }
+   }
+}
+
+}
+
+}

backends/partialassembly/fespace.cpp

@@ -0,0 +1,185 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#include "fespace.hpp"
+
+namespace mfem
+{
+
+namespace pa
+{
+
+FiniteElementSpace::FiniteElementSpace(const Engine &e,
+                                       mfem::FiniteElementSpace &fespace)
+   : PFiniteElementSpace(e, fespace),
+     e_layout(e, 0),
+     tensor_offsets(NULL),
+     tensor_indices(NULL)
+{
+   std::size_t lsize = 0;
+   for (int e = 0; e < fespace.GetNE(); e++) { lsize += fespace.GetFE(e)->GetDof(); }
+   e_layout.Resize(lsize);
+   e_layout.DontDelete();
+}
+
+void FiniteElementSpace::BuildDofMaps()
+{
+   mfem::FiniteElementSpace *mfem_fes = fes;
+
+   const int local_size = GetELayout().Size();
+   const int global_size = mfem_fes->GetVLayout()->Size();
+   const int vdim = mfem_fes->GetVDim();
+
+   // Now we can allocate and fill the global map
+   tensor_offsets = new mfem::Array<int>(*(new Layout(GetEngine(), global_size + 1)));
+   tensor_indices = new mfem::Array<int>(*(new Layout(GetEngine(), local_size)));
+
+   mfem::Array<int> &offsets = *tensor_offsets;
+   mfem::Array<int> &indices = *tensor_indices;
+
+   mfem::Array<int> global_map(local_size);
+   mfem::Array<int> elem_vdof;
+
+   int offset = 0;
+   for (int e = 0; e < mfem_fes->GetNE(); e++)
+   {
+      const FiniteElement *fe = mfem_fes->GetFE(e);
+      const int dofs = fe->GetDof();
+      const int vdofs = dofs * vdim;
+      const TensorBasisElement *tfe = dynamic_cast<const TensorBasisElement *>(fe);
+      const mfem::Array<int> &dof_map = tfe->GetDofMap();
+
+      mfem_fes->GetElementVDofs(e, elem_vdof);
+
+      if (dof_map.Size()==0)
+      {
+         for (int vd = 0; vd < vdim; vd++)
+            for (int i = 0; i < vdofs; i++)
+            {
+               global_map[offset + dofs*vd + i] = elem_vdof[dofs*vd + i];
+            }
+      }else{
+         for (int vd = 0; vd < vdim; vd++)
+            for (int i = 0; i < vdofs; i++)
+            {
+               global_map[offset + dofs*vd + i] = elem_vdof[dofs*vd + dof_map[i]];
+            }
+      }
+      offset += vdofs;
+   }
+
+   // global_map[i] = index in global vector for local dof i
+   // NOTE: multiple i values will yield same global_map[i] for shared DOF.
+
+   // We want to now invert this map so we have indices[j] = (local dof for global dof j).
+
+   // Zero the offset vector
+   offsets = 0;
+
+   // Keep track of how many local dof point to its global dof
+   // Count how many times each dof gets hit
+   for (int i = 0; i < local_size; i++)
+   {
+      const int g = global_map[i];
+      ++offsets[g + 1];
+   }
+   // Aggregate the offsets
+   for (int i = 1; i <= global_size; i++)
+   {
+      offsets[i] += offsets[i - 1];
+   }
+
+   for (int i = 0; i < local_size; i++)
+   {
+      const int g = global_map[i];
+      indices[offsets[g]++] = i;
+   }
+
+   // Shift the offset vector back by one, since it was used as a
+   // counter above.
+   for (int i = global_size; i > 0; i--)
+   {
+      offsets[i] = offsets[i - 1];
+   }
+   offsets[0] = 0;
+
+   offsets.Push();
+   indices.Push();
+}
+
+/// Convert an E vector to L vector
+void FiniteElementSpace::ToLVector(const Vector<double>& e_vector, Vector<double>& l_vector)
+{
+   if (tensor_indices == NULL) BuildDofMaps();
+
+   if (l_vector.Size() != (std::size_t) GetFESpace()->GetVSize())
+   {
+      l_vector.Resize<double>(GetFESpace()->GetVLayout(), NULL);
+   }
+
+   const int lsize = l_vector.Size();
+   const int *offsets = tensor_offsets->Get_PArray()->As<Array>().GetTypedData<int>();
+   const int *indices = tensor_indices->Get_PArray()->As<Array>().GetTypedData<int>();
+
+   const double *e_data = e_vector.GetData();
+   double *l_data = l_vector.GetData();
+
+   for (int i = 0; i < lsize; i++)
+   {
+      const int offset = offsets[i];
+      const int next_offset = offsets[i + 1];
+      double dof_value = 0;
+      for (int j = offset; j < next_offset; j++)
+      {
+         dof_value += e_data[indices[j]];
+      }
+      l_data[i] = dof_value;
+   }
+}
+
+/// Covert an L vector to E vector
+void FiniteElementSpace::ToEVector(const Vector<double>& l_vector, Vector<double>& e_vector)
+{
+   if (tensor_indices == NULL) BuildDofMaps();
+
+   if (e_vector.Size() != (std::size_t) e_layout.Size())
+   {
+      e_vector.Resize<double>(GetELayout(), NULL);
+   }
+
+   const int lsize = l_vector.Size();
+   const int *offsets = tensor_offsets->Get_PArray()->As<Array>().GetTypedData<int>();
+   const int *indices = tensor_indices->Get_PArray()->As<Array>().GetTypedData<int>();
+
+   const double *l_data = l_vector.GetData();
+   double *e_data = e_vector.GetData();
+
+   for (int i = 0; i < lsize; i++)
+   {
+      const int offset = offsets[i];
+      const int next_offset = offsets[i + 1];
+      const double dof_value = l_data[i];
+      for (int j = offset; j < next_offset; j++)
+      {
+         e_data[indices[j]] = dof_value;
+      }
+   }
+}
+
+
+} // namespace mfem::pa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)

backends/partialassembly/fespace.hpp

@@ -0,0 +1,88 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_PA_FESPACE_HPP
+#define MFEM_BACKENDS_PA_FESPACE_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#include "engine.hpp"
+#include "array.hpp"
+#include "vector.hpp"
+#include "../../fem/fem.hpp"
+
+namespace mfem
+{
+
+namespace pa
+{
+
+/// TODO: doxygen
+class FiniteElementSpace : public mfem::PFiniteElementSpace
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // mfem::FiniteElementSpace *fes;
+
+   Layout e_layout;
+
+   mfem::Array<int> *tensor_offsets, *tensor_indices;
+
+   void BuildDofMaps();
+
+public:
+   FiniteElementSpace() = delete;
+
+   /// Nearly-empty class that stores a pointer to a mfem::FiniteElementSpace instance and the engine
+   FiniteElementSpace(const Engine &e, mfem::FiniteElementSpace &fespace);
+
+   /// Virtual destructor
+   virtual ~FiniteElementSpace()
+   {
+      delete tensor_offsets;
+      delete tensor_indices;
+   }
+
+   Layout &GetELayout() { return e_layout; }
+
+   /// Return the engine as an OpenMP engine
+   const Engine &GetEngine() { return static_cast<const Engine&>(*engine); }
+
+   /// Convert an E vector to L vector
+   void ToLVector(const Vector<double>& e_vector, Vector<double>& l_vector);
+
+   /// Covert an L vector to E vector
+   void ToEVector(const Vector<double>& l_vector, Vector<double>& e_vector);
+
+   const FiniteElement *GetFE(int i) const { return fes->GetFE(i); }
+
+   /// Returns number of degrees of freedom in each direction.
+   inline const int GetNDofs1d() const { return GetFE(0)->GetOrder() + 1; }
+
+   /// Returns number of quadrature points in each direction.
+   inline const int GetNQuads1d(const int order) const
+   {
+      const IntegrationRule &ir1d = IntRules.Get(Geometry::SEGMENT, order);
+      return ir1d.GetNPoints();
+   }
+};
+
+} // namespace mfem::pa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#endif // MFEM_BACKENDS_PA_FESPACE_HPP

backends/partialassembly/integrator.hpp

@@ -0,0 +1,51 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_PA_INTEGRATOR_HPP
+#define MFEM_BACKENDS_PA_INTEGRATOR_HPP
+
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#include "vector.hpp"
+
+namespace mfem
+{
+
+namespace pa
+{
+
+/**
+* A simple class to represent a backend partial Assembly Integrator.
+*/
+class TensorBilinearFormIntegrator
+{
+public:
+   virtual ~TensorBilinearFormIntegrator() { }
+
+   virtual void ReassembleOperator() = 0;
+
+   virtual void ComputeElementMatrices(DenseTensor &element_matrices)
+   { mfem_error("TensorBilinaerFormIntegrator::ComputeElementMatrices is not overloaded"); }
+
+   virtual void MultAdd(const Vector<double>& x, Vector<double>& y) const = 0;
+
+   virtual void Mult(const Vector<double>& x, Vector<double>& y) const
+   { y.Fill<double>(0.0); MultAdd(x, y); }
+};
+
+
+} // namespace mfem::pa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#endif // MFEM_BACKENDS_PA_INTEGRATOR_HPP

backends/partialassembly/layout.cpp

@@ -0,0 +1,40 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#include "layout.hpp"
+#include "../../general/array.hpp"
+
+namespace mfem
+{
+
+namespace pa
+{
+
+void Layout::Resize(std::size_t new_size)
+{
+	size = new_size;
+}
+
+void Layout::Resize(const Array<std::size_t> &offsets)
+{
+	MFEM_ASSERT(offsets.Size() == 2,
+	            "multiple workers are not supported yet");
+	size = offsets.Last();
+}
+
+} // namespace mfem::pa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)

backends/partialassembly/layout.hpp

@@ -0,0 +1,73 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_PA_LAYOUT_HPP
+#define MFEM_BACKENDS_PA_LAYOUT_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#include "../base/layout.hpp"
+#include "engine.hpp"
+#include <stdlib.h>
+
+namespace mfem
+{
+
+namespace pa
+{
+
+class Layout : public PLayout
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // std::size_t size;
+
+public:
+   Layout(const Engine &e, std::size_t s = 0) : PLayout(e, s) { }
+
+   virtual ~Layout() { }
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+   template <typename T>
+   T* Alloc(std::size_t size) const
+   {
+      return new T[size];
+   }
+
+   void* Alloc(std::size_t size) const
+   {
+      return new char[size];
+   }
+
+   /// Resize the layout
+   virtual void Resize(std::size_t new_size);
+
+   /// Resize the layout based on the given worker offsets
+   virtual void Resize(const Array<std::size_t> &offsets);
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem::pa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#endif // MFEM_BACKENDS_PA_LAYOUT_HPP

backends/partialassembly/partialassemblykernel.hpp

@@ -0,0 +1,261 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+// This file contains operator-based bilinear form integrators used
+// with BilinearFormOperator.
+
+#ifndef MFEM_PAK
+#define MFEM_PAK
+
+// #include "fem.hpp"
+// #include "../config/config.hpp"
+// #include "bilininteg.hpp"
+// #include "dalg.hpp"
+// #include "dgfacefunctions.hpp"
+#include "domainkernels.hpp"
+#include "facekernels.hpp"
+// #include "solverkernels.hpp"
+// #include <iostream>
+#include "integrator.hpp"
+#include "dgpabilininteg.hpp"
+#include "tensor.hpp"
+
+namespace mfem
+{
+
+namespace pa
+{
+
+/////////////////////////////////////////////////
+//                                             //
+//                                             //
+//        PARTIAL ASSEMBLY INTEGRATORS         //
+//                                             //
+//                                             //
+/////////////////////////////////////////////////
+
+
+/////////////////////////////
+// Domain Kernel Interface //
+/////////////////////////////
+
+/**
+*  A partial assembly Integrator class for domain integrals.
+*  Takes an 'Equation' template parameter, that must contain 'OpName' of
+*  type 'PAOp' and a function named 'evalD', that receives a 'res' vector,
+*  the element transformation and the integration point, and then whatever
+*  is needed to compute at the point (Coefficient, VectorCoeffcient, etc...).
+*  The 'IMPL' template parameter allows to switch between different implementations
+*  of the tensor contraction kernels.
+*/
+template < typename Equation, typename Vector = mfem::Vector,
+           template<typename, PAOp, typename> class IMPL = DomainMult>
+class PADomainInt
+   : public TensorBilinearFormIntegrator, public IMPL<Equation, Equation::OpName, Vector>
+{
+private:
+   typedef IMPL<Equation, Equation::OpName, Vector> Op;
+
+public:
+   /**
+   *  The constructor is templated so that the argument needed for 'evalD' can be
+   *  packed arbitrarily ('evalD' with the corresponding signature must exist).
+   */
+   template <typename Args>
+   PADomainInt(mfem::FiniteElementSpace *fes, const int order, Args& args)
+      : Op(fes, order, args)
+   {
+      const int nb_elts = fes->GetNE();
+      const IntegrationRule& ir = IntRules.Get(fes->GetFE(0)->GetGeomType(), order);
+      const int quads  = ir.GetNPoints();
+      const FiniteElement* fe = fes->GetFE(0);
+      const int dim = fe->GetDim();
+      this->InitD(dim, quads, nb_elts);
+      Tensor<1> Jac1D(dim * dim * quads * nb_elts);
+      EvalJacobians(dim, fes, order, Jac1D);
+      Tensor<4> Jac(Jac1D.getData(), dim, dim, quads, nb_elts);
+      for (int e = 0; e < nb_elts; ++e)
+      {
+         ElementTransformation *Tr = fes->GetElementTransformation(e);
+         for (int k = 0; k < quads; ++k)
+         {
+            Tensor<2> J_ek(&Jac(0, 0, k, e), dim, dim);
+            const IntegrationPoint &ip = ir.IntPoint(k);
+            Tr->SetIntPoint(&ip);
+            this->evalEq(dim, k, e, Tr, ip, J_ek, args);
+         }
+      }
+   }
+
+   const typename Op::DTensor& getD() const
+   {
+      return Op::getD();
+   }
+
+   /**
+   *  Applies the partial assembly operator.
+   */
+   virtual void MultAdd(const Vector& u, Vector& v) const
+   {
+      int dim = this->fes->GetFE(0)->GetDim();
+      switch (dim)
+      {
+      case 1: this->Mult1d(u, v); break;
+      case 2: this->Mult2d(u, v); break;
+      case 3: this->Mult3d(u, v); break;
+      default: mfem_error("More than # dimension not yet supported"); break;
+      }
+   }
+
+   virtual void ReassembleOperator() { }
+
+};
+
+///////////////////////////
+// Face Kernel Interface //
+///////////////////////////
+
+/**
+*  A partial assembly Integrator interface class for face integrals.
+*  The template parameters have the same role as for 'PADomainInt'.
+*/
+template <typename Equation, typename Vector = mfem::Vector,
+          template<typename, PAOp, typename> class IMPL = FaceMult>
+class PAFaceInt
+   : public TensorBilinearFormIntegrator, public IMPL<Equation, Equation::FaceOpName, Vector>
+{
+private:
+   typedef IMPL<Equation, Equation::FaceOpName, Vector> Op;
+
+public:
+   template <typename Args>
+   PAFaceInt(mfem::FiniteElementSpace* fes, const int order, Args& args)
+      : Op(fes, order, args)
+   {
+      const int dim = fes->GetFE(0)->GetDim();
+      // const int quads1d = GetNQuads1d(order);
+      // Mesh* mesh = fes->GetMesh();
+      const int nb_elts = fes->GetNE();
+      const int nb_faces_elt = 2 * dim;
+      int geom;
+      switch (dim) {
+      case 1: geom = Geometry::POINT; break;
+      case 2: geom = Geometry::SEGMENT; break;
+      case 3: geom = Geometry::SQUARE; break;
+      }
+      const IntegrationRule& ir = IntRules.Get(geom, order);
+      const int quads  = ir.GetNPoints();
+      this->init(dim, quads, nb_elts, nb_faces_elt);
+      Assemble(fes, order, args);
+   }
+
+   // Perform the action of the BilinearFormIntegrator
+   virtual void MultAdd(const Vector& u, Vector& v) const
+   {
+      int dim = this->fes->GetFE(0)->GetDim();
+      switch (dim)
+      {
+      case 1:
+         mfem_error("Not yet implemented");
+         break;
+      case 2:
+         this->EvalInt2D(u, v);
+         this->EvalExt2D(u, v);
+         break;
+      case 3:
+         this->EvalInt3D(u, v);
+         this->EvalExt3D(u, v);
+         break;
+      default:
+         mfem_error("Face Kernel does not exist for this dimension.");
+         break;
+      }
+   }
+
+   virtual void ReassembleOperator() { }
+
+private:
+   template <typename Args>
+   void Assemble(mfem::FiniteElementSpace* fes, const int order, Args& args)
+   {
+      const int dim = fes->GetFE(0)->GetDim();
+      const int quads1d = GetNQuads1d(order);
+      Mesh* mesh = fes->GetMesh();
+      const int nb_elts = fes->GetNE();
+      // const int nb_faces_elt = 2 * dim;
+      const int nb_faces = mesh->GetNumFaces();
+      int geom;
+      switch (dim) {
+      case 1: geom = Geometry::POINT; break;
+      case 2: geom = Geometry::SEGMENT; break;
+      case 3: geom = Geometry::SQUARE; break;
+      }
+      const IntegrationRule& ir = IntRules.Get(geom, order);
+      const int quads  = ir.GetNPoints();
+      mfem::Vector qvec(dim);
+      Tensor<1> normal(dim);
+      mfem::Vector n(normal.getData(), dim);
+      // Vector n(dim);
+      // !!! Should not be recomputed... !!!
+      Tensor<1> Jac1D(dim * dim * quads * quads1d * nb_elts);
+      EvalJacobians(dim, fes, order, Jac1D);
+      Tensor<4> Jac(Jac1D.getData(), dim, dim, quads * quads1d, nb_elts); // Creating a view
+      // !!!                             !!!
+      // We have a per face approach for the fluxes
+      for (int face = 0; face < nb_faces; ++face)
+      {
+         int ind_elt1, ind_elt2;
+         int face_id1, face_id2;
+         int nb_rot1, nb_rot2;
+         GetFaceInfo(mesh, face, ind_elt1, ind_elt2, face_id1, face_id2, nb_rot1, nb_rot2);
+         FaceElementTransformations* face_tr = mesh->GetFaceElementTransformations(face);
+         int perm1, perm2;
+         // cout << "ind_elt1=" << ind_elt1 << ", face_id1=" << face_id1 << ", nb_rot1=" << nb_rot1 << ", ind_elt2=" << ind_elt2 << ", face_id2=" << face_id2 << ", nb_rot2=" << nb_rot2 << endl;
+         for (int kf = 0; kf < quads; ++kf)
+         {
+            const IntegrationPoint& ip = ir.IntPoint(kf);
+            if (ind_elt2 != -1) { //Not a boundary face
+               Tensor<1, int> ind_f1(dim - 1), ind_f2(dim - 1);
+               // We compute the lexicographical index on each face
+               int k1 = GetFaceQuadIndex(dim, face_id1, nb_rot1, kf, quads1d, ind_f1);
+               int k2 = GetFaceQuadIndex(dim, face_id2, nb_rot2, kf, quads1d, ind_f2);
+               this->initFaceData(dim, ind_elt1, face_id1, nb_rot1, perm1, ind_elt2, face_id2, nb_rot2, perm2);
+               face_tr->Face->SetIntPoint( &ip );
+               IntegrationPoint eip1;
+               face_tr->Loc1.Transform(ip, eip1);
+               eip1.weight = ip.weight;//Sets the weight since Transform doesn't do it...
+               IntegrationPoint eip2;
+               face_tr->Loc2.Transform(ip, eip2);
+               eip2.weight = ip.weight;//Sets the weight since Transform doesn't do it...
+               int kg1 = GetGlobalQuadIndex(dim, face_id1, quads1d, ind_f1);
+               int kg2 = GetGlobalQuadIndex(dim, face_id2, quads1d, ind_f2);
+               Tensor<2> J_e1(&Jac(0, 0, kg1, ind_elt1), dim, dim);
+               Tensor<2> J_e2(&Jac(0, 0, kg2, ind_elt2), dim, dim);
+               Tensor<2> Adj(dim, dim);
+               adjugate(J_e1, Adj);
+               calcOrtho( Adj, face_id1, normal); // normal*determinant (risky, bug prone)
+               this->evalEq(dim, k1, k2, n, ind_elt1, face_id1, ind_elt2, face_id2, face_tr, eip1, eip2, J_e1, J_e2, args);
+            } else { //Boundary face
+               this->initBoundaryFaceData(ind_elt1, face_id1);
+               // TODO: Something should be done here when there is boundary conditions!
+               // D11(ind) = 0;
+            }
+         }
+      }
+   }
+
+};
+
+}
+
+}
+
+#endif //MFEM_PAK

backends/partialassembly/tensor.hpp

@@ -0,0 +1,985 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+// This file contains operator-based bilinear form integrators used
+// with BilinearFormOperator.
+
+#ifndef MFEM_TENSOR
+#define MFEM_TENSOR
+
+// #include "bilininteg.hpp"
+// #include <vector>
+#include <iostream>
+#include "../../general/error.hpp"
+
+namespace mfem
+{
+
+namespace pa
+{
+
+/**
+* A Class to compute the real indice from the multi-indice of a tensor
+*/
+template <int N, int Dim, typename T, typename... Args>
+class TensorInd
+{
+public:
+   static int result(const int* sizes, T first, Args... args)
+   {
+      MFEM_ASSERT(first<sizes[N-1],"Trying to access out of boundary.");
+      return first + sizes[N - 1] * TensorInd < N + 1, Dim, Args... >::result(sizes, args...);
+   }
+};
+//Terminal case
+template <int Dim, typename T, typename... Args>
+class TensorInd<Dim, Dim, T, Args...>
+{
+public:
+   static int result(const int* sizes, T first, Args... args)
+   {
+      MFEM_ASSERT(first<sizes[Dim-1],"Trying to access out of boundary.");
+      return first;
+   }
+};
+
+/**
+* A class to initialize the size of a Tensor
+*/
+template <int N, int Dim, typename T, typename... Args>
+class Init
+{
+public:
+   static int result(int* sizes, T first, Args... args) {
+      sizes[N - 1] = first;
+      return first * Init < N + 1, Dim, Args... >::result(sizes, args...);
+   }
+};
+//Terminal case
+template <int Dim, typename T, typename... Args>
+class Init<Dim, Dim, T, Args...>
+{
+public:
+   static int result(int* sizes, T first, Args... args) {
+      sizes[Dim - 1] = first;
+      return first;
+   }
+};
+
+/**
+*  A basic generic Tensor class
+*/
+template<int Dim, typename Scalar = double>
+class Tensor
+{
+protected:
+   int capacity;
+   Scalar* data;
+   bool own_data;
+   int sizes[Dim];
+
+public:
+   /**
+   *  A default constructor
+   */
+   explicit Tensor()
+      : capacity(0), data(NULL), own_data(false)
+   {
+   }
+
+   /**
+   *  The destructor
+   */
+   ~Tensor()
+   {
+      if (own_data)
+      {
+         delete [] data;
+      }
+   }
+
+   /**
+   *  A constructor to initialize the sizes of a tensor with an array of integers
+   */
+   Tensor(int* _sizes)
+      : own_data(true)
+   {
+      int nb = 1;
+      for (int i = 0; i < Dim; ++i)
+      {
+         sizes[i] = _sizes[i];
+         nb *= sizes[i];
+      }
+      capacity = nb;
+      data = new Scalar[nb];
+   }
+
+   /**
+   *  A constructor to initialize a tensor from a different size Tensor
+   */
+   template <int Dim1, typename... Args>
+   Tensor(Tensor<Dim1, Scalar>& t, Args... args)
+      : own_data(false)
+   {
+      static_assert(sizeof...(args) == Dim, "Wrong number of arguments");
+      // Initialize sizes, and compute the number of values
+      long int nb = Init<1, Dim, Args...>::result(sizes, args...);
+      capacity = nb;
+      data = t.getData();
+   }
+
+   /**
+   *  A constructor to initialize the sizes of a tensor with a variadic function
+   */
+   template <typename... Args>
+   Tensor(Args... args)
+      : own_data(true)
+   {
+      static_assert(sizeof...(args) == Dim, "Wrong number of arguments");
+      // Initialize sizes, and compute the number of values
+      long int nb = Init<1, Dim, Args...>::result(sizes, args...);
+      capacity = nb;
+      data = new Scalar[nb];
+   }
+
+   /**
+   *  A constructor to initialize a tensor from the Scalar array _data
+   */
+   template <typename... Args>
+   Tensor(Scalar* _data, Args... args)
+      : own_data(false)
+   {
+      static_assert(sizeof...(args) == Dim, "Wrong number of arguments");
+      // Initialize sizes, and compute the number of values
+      long int nb = Init<1, Dim, Args...>::result(sizes, args...);
+      capacity = nb;
+      data = _data;
+   }
+
+   // Let's write some uggly code
+   template <typename... Args>
+   Tensor(const Scalar* _data, Args... args)
+      : own_data(false)
+   {
+      static_assert(sizeof...(args) == Dim, "Wrong number of arguments");
+      // Initialize sizes, and compute the number of values
+      long int nb = Init<1, Dim, Args...>::result(sizes, args...);
+      capacity = nb;
+      data = const_cast<Scalar*>(_data);
+   }
+
+   /**
+   *  A copy constructor
+   */
+   Tensor(const Tensor& t)
+      : capacity(t.length()), data(new Scalar[capacity]), own_data(true)
+   {
+      for (int i = 0; i < Dim; ++i)
+      {
+         sizes[i] = t.size(i);
+      }
+      const Scalar* data_t = t.getData();
+      for (int i = 0; i < capacity; ++i)
+      {
+         data[i] = data_t[i];
+      }
+   }
+
+   /**
+   *  A copy assignment operator (do not resize Tensors)
+   */
+   Tensor& operator=(const Tensor& t)
+   {
+      if (this == &t)
+      {
+         return *this;
+      }
+      const int nb = t.length();
+      for (int i = 0; i < Dim; ++i)
+      {
+         if (sizes[i] != t.size(i))
+         {
+            // std::cout << sizes[i] << " | " << t.size(i) << std::endl;
+            mfem_error("The Tensors have different sizes.");
+         }
+         // sizes[i] = t.size(i);
+      }
+      for (int i = 0; i < nb; ++i)
+      {
+         data[i] = t[i];
+      }
+      return *this;
+   }
+
+   /**
+   *  Operator += for Tensors of the same size.
+   */
+   Tensor& operator+=(const Tensor& t)
+   {
+      const int nb = t.length();
+      for (int i = 0; i < Dim; ++i)
+      {
+         if (sizes[i] != t.size(i))
+         {
+            // std::cout << sizes[i] << " | " << t.size(i) << std::endl;
+            mfem_error("The Tensors have different sizes.");
+         }
+         // sizes[i] = t.size(i);
+      }
+      for (int i = 0; i < nb; ++i)
+      {
+         data[i] += t[i];
+      }
+      return *this;
+   }
+
+   /**
+   *  Sets the size of the tensor, and allocate memory if necessary
+   */
+   template <typename... Args>
+   void setSize(Args... args)
+   {
+      static_assert(sizeof...(args) == Dim, "Wrong number of arguments");
+      // Initialize sizes, and compute the number of values
+      long int nb = Init<1, Dim, Args...>::result(sizes, args...);
+      if (nb > capacity)
+      {
+         Scalar* _data = new Scalar[nb];
+         for (int i = 0; i < capacity; ++i)
+         {
+            _data[i] = data[i];
+         }
+         for (int i = capacity; i < nb; ++i)
+         {
+            _data[i] = Scalar();
+         }
+         if (own_data)
+         {
+            delete [] data;
+         }
+         data = _data;
+         own_data = true;
+         capacity = nb;
+      }
+   }
+
+   Tensor& setView(Scalar* ptr){
+      MFEM_ASSERT(own_data,"you should get rid of your data first.");
+      data = ptr;
+      return *this;
+   }
+
+   /**
+   *  A const accessor for the data
+   */
+   template <typename... Args>
+   const Scalar& operator()(Args... args) const
+   {
+      static_assert(sizeof...(args) == Dim, "Wrong number of arguments");
+      return data[ TensorInd<1, Dim, Args...>::result(sizes, args...) ];
+   }
+
+   /**
+   *  A reference accessor to the data
+   */
+   template <typename... Args>
+   Scalar& operator()(Args... args)
+   {
+      static_assert(sizeof...(args) == Dim, "Wrong number of arguments");
+      return data[ TensorInd<1, Dim, Args...>::result(sizes, args...) ];
+   }
+
+   const Scalar& operator[](int i) const
+   {
+      return data[i];
+   }
+
+   Scalar& operator[](int i)
+   {
+      return data[i];
+   }
+
+   void zero()
+   {
+      for (int i = 0; i < capacity; ++i)
+      {
+         data[i] = Scalar();
+      }
+   }
+
+   /**
+   *  Returns the size of the i-th dimension #UNSAFE#
+   */
+   const int size(int i) const
+   {
+      return sizes[i];
+   }
+
+   const int Height() const
+   {
+      static_assert(Dim == 2, "Height() should only be used for second order tensors");
+      return sizes[0];
+   }
+
+   const int Width() const
+   {
+      static_assert(Dim == 2, "Width() should only be used for second order tensors");
+      return sizes[1];
+   }
+
+   /**
+   *  Returns the length of the Tensor (number of values, may be different from capacity)
+   */
+   const int length() const
+   {
+      int res = 1;
+      for (int i = 0; i < Dim; ++i)
+      {
+         res *= sizes[i];
+      }
+      return res;
+   }
+
+   /**
+   *  Returns the dimension of the tensor.
+   */
+   const int dimension() const
+   {
+      return Dim;
+   }
+
+   /**
+   *  Returns the Scalar array data (Really unsafe and ugly method)
+   *  Mostly exists to remap Tensors, so could be avoided by using more constructors...
+   */
+   Scalar* getData()
+   {
+      return data;
+   }
+
+   const Scalar* getData() const
+   {
+      return data;
+   }
+
+   /**
+   *  Returns a sub-tensor contained in the last dimension at @index in the Tensor @T.
+   */
+   void slice(Tensor<Dim+1, Scalar>& T, const int index) {
+      if (own_data)
+      {
+         mfem_error("I didn't expect you to do that");
+      }
+      int offset = 1;
+      for (int i = 0; i < Dim; ++i)
+      {
+         const int size = T.size(i);
+         offset *= size;
+         this->sizes[i] = size;
+      }
+      this->capacity = offset;
+      const int ind = offset * index;
+      this->data = T.getData() + ind;
+      this->own_data = false;
+   }
+
+   /**
+   *  Basic printing method.
+   */
+   friend std::ostream& operator<<(std::ostream& os, const Tensor& T)
+   {
+      int nb_elts = 1;
+      for (int i = 0; i < T.dimension(); ++i)
+      {
+         nb_elts *= T.size(i);
+      }
+      for (int i = 0; i < nb_elts; ++i)
+      {
+         os << T.data[i] << " ";
+         if ((i + 1) % T.sizes[0] == 0)
+         {
+            os << "\n";
+         }
+      }
+      os << "\n";
+      return os;
+   }
+};
+
+template <typename Scalar>
+class Tensor<0,Scalar>{
+private:
+   Scalar* data;
+   bool own_data;
+public:
+   Tensor(): data(new Scalar), own_data(true) {}
+
+   Tensor& operator=(Scalar& val) {
+      *data = val;
+      return *this;
+   }
+
+   Tensor& operator=(Scalar val) {
+      *data = val;
+      return *this;
+   }
+
+   void slice(Tensor<1,Scalar>& T, const int index) {
+      data = &T(index);
+   }
+
+   friend std::ostream& operator<<(std::ostream& os, const Tensor& T)
+   {
+      os << *T.data;
+      return os;
+   }
+};
+
+template <typename Scalar>
+inline void adjugate(const Tensor<2, Scalar>& A, Tensor<2, Scalar>& Adj)
+{
+   const int dim = A.Height();
+   switch (dim) {
+   case 1:
+      Adj(0, 0) = A(0, 0);
+      break;
+   case 2:
+      Adj(0, 0) =  A(1, 1); Adj(0, 1) = -A(0, 1);
+      Adj(1, 0) = -A(1, 0); Adj(1, 1) =  A(0, 0);
+      break;
+   case 3:
+      Adj(0, 0) =  A(1, 1) * A(2, 2) - A(1, 2) * A(2, 1);
+      Adj(0, 1) = -A(0, 1) * A(2, 2) + A(0, 2) * A(2, 1);
+      Adj(0, 2) =  A(0, 1) * A(1, 2) - A(0, 2) * A(1, 1);
+      //
+      Adj(1, 0) = -A(1, 0) * A(2, 2) + A(1, 2) * A(2, 0);
+      Adj(1, 1) =  A(0, 0) * A(2, 2) - A(0, 2) * A(2, 0);
+      Adj(1, 2) = -A(0, 0) * A(1, 2) + A(0, 2) * A(1, 0);
+      //
+      Adj(2, 0) =  A(1, 0) * A(2, 1) - A(1, 1) * A(2, 0);
+      Adj(2, 1) = -A(0, 0) * A(2, 1) + A(0, 1) * A(2, 0);
+      Adj(2, 2) =  A(0, 0) * A(1, 1) - A(0, 1) * A(1, 0);
+      break;
+   default:
+      mfem_error("adjugate not defined for this size");
+      break;
+   }
+}
+
+template <typename Scalar>
+inline Scalar det(const Tensor<2, Scalar>& A)
+{
+   MFEM_ASSERT(A.Height() == A.Width(), "You're attempting to compute the determinant of a non square matrix.");
+   const int dim = A.Height();
+   switch (dim) {
+   case 1:
+      return A(0, 0);
+   case 2:
+      return A(0, 0) * A(1, 1) - A(0, 1) * A(1, 0);
+   case 3:
+      return A(0, 0) * (A(1, 1) * A(2, 2) - A(1, 2) * A(2, 1))
+             - A(1, 0) * (A(0, 1) * A(2, 2) - A(0, 2) * A(2, 1))
+             + A(2, 0) * (A(0, 1) * A(1, 2) - A(0, 2) * A(1, 1));
+   default:
+      mfem_error("determinant not defined for this size");
+      break;
+   }
+   return Scalar();
+}
+
+template <typename Scalar>
+inline Scalar norm2sq(const Tensor<1, Scalar>& t)
+{
+   Scalar res = 0.0;
+   for (int i = 0; i < t.size(0); ++i)
+   {
+      res += t(i) * t(i);
+   }
+   return res;
+}
+
+template <typename Scalar>
+inline Scalar dot(const Tensor<1, Scalar>& t1, const Tensor<1, Scalar>& t2)
+{
+   Scalar res = 0.0;
+   MFEM_ASSERT(t1.size(0) == t2.size(0), "Tensor<1> t1 and t2 are of different size");
+   for (int i = 0; i < t1.size(0); ++i)
+   {
+      res += t1(i) * t2(i);
+   }
+   return res;
+}
+
+template <typename Scalar>
+inline void calcOrtho(const Tensor<2, Scalar>& J, const int& face_id, Tensor<1>& n)
+{
+   const int dim = n.length();
+   switch (dim)
+   {
+   case 1:
+      n(0) = face_id == 0 ? -J(0, 0) : J(0, 0);
+      break;
+   case 2:
+      //FIXME: looks strange we access 2D Jacobians in a different way than 3D
+      switch (face_id)
+      {
+      case 0://SOUTH ( 0,-1)
+         // n(0) = -J(0,1); n(1) = -J(1,1);
+         n(0) = -J(1, 0); n(1) = -J(1, 1);
+         break;
+      case 1://EAST  ( 1, 0)
+         // n(0) =  J(0,0); n(1) =  J(1,0);
+         n(0) =  J(0, 0); n(1) =  J(0, 1);
+         break;
+      case 2://NORTH ( 0, 1)
+         // n(0) =  J(0,1); n(1) =  J(1,1);
+         n(0) =  J(1, 0); n(1) =  J(1, 1);
+         break;
+      case 3://WEST  (-1, 0)
+         // n(0) = -J(0,0); n(1) = -J(1,0);
+         n(0) = -J(0, 0); n(1) = -J(0, 1);
+         break;
+      }
+      break;
+   case 3:
+      switch (face_id)
+      {
+      case 0://BOTTOM ( 0, 0,-1)
+         n(0) = -J(0, 2); n(1) = -J(1, 2); n(2) = -J(2, 2);
+         break;
+      case 1://SOUTH  ( 0,-1, 0)
+         n(0) = -J(0, 1); n(1) = -J(1, 1); n(2) = -J(2, 1);
+         break;
+      case 2://EAST   ( 1, 0, 0)
+         n(0) =  J(0, 0); n(1) =  J(1, 0); n(2) =  J(2, 0);
+         break;
+      case 3://NORTH  ( 0, 1, 0)
+         n(0) =  J(0, 1); n(1) =  J(1, 1); n(2) =  J(2, 1);
+         break;
+      case 4://WEST   (-1, 0, 0)
+         n(0) = -J(0, 0); n(1) = -J(1, 0); n(2) = -J(2, 0);
+         break;
+      case 5://TOP    ( 0, 0, 1)
+         n(0) =  J(0, 2); n(1) =  J(1, 2); n(2) =  J(2, 2);
+         break;
+      }
+      break;
+   }
+}
+
+/**
+*  A type trait to obtain the the Scalar type underloying a type.
+*/
+template <typename T>
+struct value_type;
+
+template <int N, typename Scalar>
+struct value_type<Tensor<N, Scalar>>
+{
+   typedef Scalar type;
+};
+
+template <typename T>
+using value_type_t = typename value_type<T>::type;
+
+///////////////////////////
+// "Volume" contractions //
+///////////////////////////
+
+
+// Would defining those contractions for abstract templated types be better?
+///////
+// 1d
+template <typename Scalar>
+inline void contract(const Tensor<2, Scalar>& B, const Tensor<1, Scalar>& U, Tensor<1, Scalar>& V)
+{
+   MFEM_ASSERT(B.size(0) == U.size(0), "Size mismatch for contraction.");
+   for (int j = 0; j < B.size(1); ++j)
+   {
+      V(j) = Scalar();
+      for (int i = 0; i < B.size(0); ++i)
+      {
+         V(j) += B(i, j) * U(i);
+      }
+   }
+}
+
+template <typename Scalar>
+inline void contractT(const Tensor<2, Scalar>& B, const Tensor<1, Scalar>& U, Tensor<1, Scalar>& V)
+{
+   MFEM_ASSERT(B.size(1) == U.size(0), "Size mismatch for contraction.");
+   for (int j = 0; j < B.size(0); ++j)
+   {
+      V(j) = Scalar();
+      for (int i = 0; i < B.size(1); ++i)
+      {
+         V(j) += B(j, i) * U(i);
+      }
+   }
+}
+
+///////
+// 2d
+template <typename Scalar>
+inline void contract(const Tensor<2, Scalar>& B, const Tensor<2, Scalar>& U, Tensor<2, Scalar>& V)
+{
+   MFEM_ASSERT(B.size(0) == U.size(0), "Size mismatch for contraction.");
+   for (int j1 = 0; j1 < B.size(1); ++j1)
+   {
+      for (int i2 = 0; i2 < U.size(1); ++i2)
+      {
+         V(i2, j1) = Scalar();
+         for (int i1 = 0; i1 < B.size(0); ++i1)
+         {
+            V(i2, j1) += B(i1, j1) * U(i1, i2);
+         }
+      }
+   }
+}
+
+template <typename Scalar>
+inline void contractT(const Tensor<2, Scalar>& B, const Tensor<2, Scalar>& U, Tensor<2, Scalar>& V)
+{
+   MFEM_ASSERT(B.size(1) == U.size(0), "Size mismatch for contraction.");
+   for (int j1 = 0; j1 < B.size(0); ++j1)
+   {
+      for (int i2 = 0; i2 < U.size(1); ++i2)
+      {
+         V(i2, j1) = Scalar();
+         for (int i1 = 0; i1 < B.size(1); ++i1)
+         {
+            V(i2, j1) += B(j1, i1) * U(i1, i2);
+         }
+      }
+   }
+}
+
+///////
+// 3d
+template <typename Scalar>
+inline void contract(const Tensor<2, Scalar>& B, const Tensor<3, Scalar>& U, Tensor<3, Scalar>& V)
+{
+   MFEM_ASSERT(B.size(0) == U.size(0), "Size mismatch for contraction.");
+   for (int j1 = 0; j1 < B.size(1); ++j1)
+   {
+      for (int i3 = 0; i3 < U.size(2); ++i3)
+      {
+         for (int i2 = 0; i2 < U.size(1); ++i2)
+         {
+            V(i2, i3, j1) = Scalar();
+            for (int i1 = 0; i1 < B.size(0); ++i1)
+            {
+               V(i2, i3, j1) += B(i1, j1) * U(i1, i2, i3);
+            }
+         }
+      }
+   }
+}
+
+template <typename Scalar>
+inline void contractT(const Tensor<2, Scalar>& B, const Tensor<3, Scalar>& U, Tensor<3, Scalar>& V)
+{
+   MFEM_ASSERT(B.size(1) == U.size(0), "Size mismatch for contraction.");
+   for (int j1 = 0; j1 < B.size(0); ++j1)
+   {
+      for (int i3 = 0; i3 < U.size(2); ++i3)
+      {
+         for (int i2 = 0; i2 < U.size(1); ++i2)
+         {
+            V(i2, i3, j1) = Scalar();
+            for (int i1 = 0; i1 < B.size(1); ++i1)
+            {
+               V(i2, i3, j1) += B(j1, i1) * U(i1, i2, i3);
+            }
+         }
+      }
+   }
+}
+
+/////////////////////////
+// "Face" contractions //
+/////////////////////////
+
+///////
+// 1d
+template <typename Scalar>
+inline void contractX(const Tensor<1, Scalar>& B, const Tensor<1, Scalar>& U, Scalar& V)
+{
+   MFEM_ASSERT(B.size(0) == U.size(0), "Size mismatch for contraction.");
+   V = Scalar();
+   for (int i = 0; i < B.size(0); ++i)
+   {
+      V += B(i) * U(i);
+   }
+}
+
+template <typename Scalar>
+inline void contractTX(const Tensor<1, Scalar>& B, const Scalar& U, Tensor<1, Scalar>& V)
+{
+   for (int i = 0; i < B.size(0); ++i)
+   {
+      V(i) = B(i) * U;
+   }
+}
+
+///////
+// 2d
+template <typename Scalar>
+inline void contractX(const Tensor<2, Scalar>& B, const Tensor<2, Scalar>& U, Tensor<1, Scalar>& V)
+{
+   MFEM_ASSERT(B.size(0) == U.size(0), "Size mismatch for contraction.");
+   for (int i2 = 0; i2 < U.size(1); ++i2)
+   {
+      V(i2) = Scalar();
+      for (int i1 = 0; i1 < B.size(0); ++i1)
+      {
+         V(i2) += B(i1, 0) * U(i1, i2);
+      }
+   }
+}
+
+template <typename Scalar>
+inline void contractTX(const Tensor<2, Scalar>& B, const Tensor<1, Scalar>& U, Tensor<2, Scalar>& V)
+{
+   for (int i2 = 0; i2 < U.size(0); ++i2)
+   {
+      for (int i1 = 0; i1 < B.size(0); ++i1)
+      {
+         V(i1, i2) += B(i1, 0) * U(i2);
+      }
+   }
+}
+
+template <typename Scalar>
+inline void contractY(const Tensor<2, Scalar>& B, const Tensor<2, Scalar>& U, Tensor<1, Scalar>& V)
+{
+   MFEM_ASSERT(B.size(0) == U.size(1), "Size mismatch for contraction.");
+
+   for (int i1 = 0; i1 < B.size(0); ++i1)
+   {
+      V(i1) = Scalar();
+      for (int i2 = 0; i2 < U.size(1); ++i2)
+      {
+         // V(i1) += B(i2,0) * U(i1,i2);
+         V(i1) += B[i2] * U(i1, i2);
+      }
+   }
+}
+
+template <typename Scalar>
+inline void contractTY(const Tensor<2, Scalar>& B, const Tensor<1, Scalar>& U, Tensor<2, Scalar>& V)
+{
+   for (int i2 = 0; i2 < B.size(0); ++i2)
+   {
+      for (int i1 = 0; i1 < U.size(0); ++i1)
+      {
+         // V(i1,i2) += B(i2,0) * U(i1);
+         V(i1, i2) += B[i2] * U(i1);
+      }
+   }
+}
+
+///////
+// 3d
+template <typename Scalar>
+inline void contractX(const Tensor<1, Scalar>& B, const Tensor<3, Scalar>& U, Tensor<2, Scalar>& V)
+{
+   MFEM_ASSERT(B.size(0) == U.size(0), "Size mismatch for contraction.");
+   for (int i3 = 0; i3 < U.size(2); ++i3)
+   {
+      for (int i2 = 0; i2 < U.size(1); ++i2)
+      {
+         V(i2, i3) = Scalar();
+         for (int i1 = 0; i1 < B.size(0); ++i1)
+         {
+            V(i2, i3) += B(i1) * U(i1, i2, i3);
+         }
+      }
+   }
+}
+
+template <typename Scalar>
+inline void contractTX(const Tensor<1, Scalar>& B, const Tensor<2, Scalar>& U, Tensor<3, Scalar>& V)
+{
+   for (int i3 = 0; i3 < U.size(1); ++i3)
+   {
+      for (int i2 = 0; i2 < U.size(0); ++i2)
+      {
+         for (int i1 = 0; i1 < B.size(0); ++i1)
+         {
+            V(i1, i2, i3) = B(i1) * U(i2, i3);
+         }
+      }
+   }
+}
+
+template <typename Scalar>
+inline void contractY(const Tensor<1, Scalar>& B, const Tensor<3, Scalar>& U, Tensor<2, Scalar>& V)
+{
+   MFEM_ASSERT(B.size(0) == U.size(1), "Size mismatch for contraction.");
+   V.zero();
+   for (int i3 = 0; i3 < U.size(2); ++i3)
+   {
+      for (int i2 = 0; i2 < U.size(1); ++i2)
+      {
+         for (int i1 = 0; i1 < B.size(0); ++i1)
+         {
+            V(i1, i3) += B(i2) * U(i1, i2, i3);
+         }
+      }
+   }
+}
+
+template <typename Scalar>
+inline void contractTY(const Tensor<1, Scalar>& B, const Tensor<2, Scalar>& U, Tensor<3, Scalar>& V)
+{
+   for (int i3 = 0; i3 < U.size(1); ++i3)
+   {
+      for (int i2 = 0; i2 < B.size(0); ++i2)
+      {
+         for (int i1 = 0; i1 < U.size(0); ++i1)
+         {
+            V(i1, i2, i3) = B(i2) * U(i1, i3);
+         }
+      }
+   }
+}
+
+template <typename Scalar>
+inline void contractZ(const Tensor<1, Scalar>& B, const Tensor<3, Scalar>& U, Tensor<2, Scalar>& V)
+{
+   MFEM_ASSERT(B.size(0) == U.size(2), "Size mismatch for contraction.");
+   V.zero();
+   for (int i3 = 0; i3 < U.size(2); ++i3)
+   {
+      for (int i2 = 0; i2 < U.size(1); ++i2)
+      {
+         for (int i1 = 0; i1 < B.size(0); ++i1)
+         {
+            V(i1, i2) += B(i3) * U(i1, i2, i3);
+         }
+      }
+   }
+}
+
+template <typename Scalar>
+inline void contractTZ(const Tensor<1, Scalar>& B, const Tensor<2, Scalar>& U, Tensor<3, Scalar>& V)
+{
+   for (int i3 = 0; i3 < B.size(0); ++i3)
+   {
+      for (int i2 = 0; i2 < U.size(1); ++i2)
+      {
+         for (int i1 = 0; i1 < U.size(0); ++i1)
+         {
+            V(i1, i2, i3) = B(i3) * U(i1, i2);
+         }
+      }
+   }
+}
+///////////////////////////////////////
+//  Coefficient-wise multiplication  //
+///////////////////////////////////////
+
+template <int N, typename Scalar>
+inline void cWiseMult(const Tensor<N, Scalar>& D, const Tensor<N, Scalar>& U, Tensor<N, Scalar>& V)
+{
+   MFEM_ASSERT(D.length() == U.length() && U.length() == V.length(), "The Tensors do not contain the same number of elements.")
+   for (int i = 0; i < U.length(); ++i)
+   {
+      V[i] = D[i] * U[i];
+   }
+}
+
+template <typename Scalar>
+inline void cWiseMult(const Tensor<3, Scalar>& D,
+                      const Tensor<2, Scalar>& BGT, const Tensor<2, Scalar>& GBT,
+                      Tensor<2, Scalar>& DGT)
+{
+   for (int i2 = 0; i2 < D.size(2); ++i2)
+   {
+      for (int i1 = 0; i1 < D.size(1); ++i1)
+      {
+         DGT(i1, i2) = D(0, i1, i2) * BGT(i1, i2)
+                       + D(1, i1, i2) * GBT(i1, i2);
+      }
+   }
+}
+
+template <typename Scalar>
+inline void cWiseMult(const Tensor<4, Scalar>& D,
+                      const Tensor<2, Scalar>& BGT, const Tensor<2, Scalar>& GBT,
+                      Tensor<2, Scalar>& D0GT, Tensor<2, Scalar>& D1GT)
+{
+   for (int i2 = 0; i2 < D.size(3); ++i2)
+   {
+      for (int i1 = 0; i1 < D.size(2); ++i1)
+      {
+         D0GT(i1, i2) = D(0, 0, i1, i2) * BGT(i1, i2)
+                        + D(0, 1, i1, i2) * GBT(i1, i2);
+
+         D1GT(i1, i2) = D(1, 0, i1, i2) * BGT(i1, i2)
+                        + D(1, 1, i1, i2) * GBT(i1, i2);
+      }
+   }
+}
+
+template <typename Scalar>
+inline void cWiseMult(const Tensor<4, Scalar>& D,
+                      const Tensor<3, Scalar>& BBGT, const Tensor<3, Scalar>& BGBT, const Tensor<3, Scalar>& GBBT,
+                      Tensor<3, Scalar>& DGT)
+{
+   for (int i3 = 0; i3 < D.size(3); ++i3)
+   {
+      for (int i2 = 0; i2 < D.size(2); ++i2)
+      {
+         for (int i1 = 0; i1 < D.size(1); ++i1)
+         {
+            DGT(i1, i2, i3) = D(0, i1, i2, i3) * BBGT(i1, i2, i3)
+                              + D(1, i1, i2, i3) * BGBT(i1, i2, i3)
+                              + D(2, i1, i2, i3) * GBBT(i1, i2, i3);
+         }
+      }
+   }
+}
+
+template <typename Scalar>
+inline void cWiseMult(const Tensor<5, Scalar>& D,
+                      const Tensor<3, Scalar>& BBGT, const Tensor<3, Scalar>& BGBT, const Tensor<3, Scalar>& GBBT,
+                      Tensor<3, Scalar>& D0GT, Tensor<3, Scalar>& D1GT, Tensor<3, Scalar>& D2GT)
+{
+   for (int i3 = 0; i3 < D.size(4); ++i3)
+   {
+      for (int i2 = 0; i2 < D.size(3); ++i2)
+      {
+         for (int i1 = 0; i1 < D.size(2); ++i1)
+         {
+            const Scalar v1 = BBGT(i1, i2, i3), v2 = BGBT(i1, i2, i3), v3 = GBBT(i1, i2, i3);
+            D0GT(i1, i2, i3) = D(0, 0, i1, i2, i3) * v1
+                               + D(0, 1, i1, i2, i3) * v2
+                               + D(0, 2, i1, i2, i3) * v3;
+            D1GT(i1, i2, i3) = D(1, 0, i1, i2, i3) * v1
+                               + D(1, 1, i1, i2, i3) * v2
+                               + D(1, 2, i1, i2, i3) * v3;
+            D2GT(i1, i2, i3) = D(2, 0, i1, i2, i3) * v1
+                               + D(2, 1, i1, i2, i3) * v2
+                               + D(2, 2, i1, i2, i3) * v3;
+         }
+      }
+   }
+}
+
+typedef Tensor<2, int> IntMatrix;
+
+}
+
+}
+
+#endif //MFEM_DUMMYALGEBRA

backends/partialassembly/tensorialfunctions.cpp

@@ -0,0 +1,289 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+// This file contains operator-based bilinear form integrators used
+// with BilinearFormOperator.
+
+#ifndef MFEM_TENSORFUNC_CPP
+#define MFEM_TENSORFUNC_CPP
+
+#include "tensor.hpp"
+#include "../../fem/fespace.hpp"
+#include "tensorialfunctions.hpp"
+#include "../../fem/gridfunc.hpp"
+// #include "fem.hpp"
+// #include "dgpabilininteg.hpp"
+
+namespace mfem
+{
+
+namespace pa
+{
+
+void ScatterDofs(const mfem::FiniteElementSpace* mfes, const Table& eldof, const mfem::Array<int>& dof_map,
+                  const GridFunction* nodes, const int dofs, const int dim, const int e,
+                  Tensor<2>& LexPointMat)
+{
+   if (dof_map.Size()==0)
+   {
+      if(mfes->GetOrdering()==Ordering::byVDIM)
+      {
+         for (int i = 0; i < dofs; ++i)
+         {
+            for (int j = 0; j < dim; ++j)
+            {
+               LexPointMat(j,i) = (*nodes)( ( e*dofs + i )*dim + j );
+            }
+         }
+      }else{
+         for (int i = 0; i < dofs; ++i)
+         {
+            for (int j = 0; j < dim; ++j)
+            {
+               LexPointMat(j,i) = (*nodes)( ( e*dofs + i ) + j*mfes->GetNDofs() );
+            }
+         }
+      }
+   }else{
+      if(mfes->GetOrdering()==Ordering::byVDIM)
+      {
+         for (int i = 0; i < dofs; ++i)
+         {
+            const int pivot = dof_map[i];
+            for (int j = 0; j < dim; ++j)
+            {
+               LexPointMat(j,i) = (*nodes)(eldof.GetJ()[ e*dofs + pivot ]*dim + j);
+            }
+         }
+      }else{
+         for (int i = 0; i < dofs; ++i)
+         {
+            const int pivot = dof_map[i];
+            for (int j = 0; j < dim; ++j)
+            {
+               LexPointMat(j,i) = (*nodes)(eldof.GetJ()[ e*dofs + pivot ] + j*mfes->GetNDofs());
+            }
+         }
+      }
+   }   
+}
+
+static void EvalJacobians1D(const mfem::FiniteElementSpace* fes, const int order, Tensor<1>& J)
+{
+   const int dim = 1;
+
+   const Mesh* mesh = fes->GetMesh();
+   const mfem::FiniteElementSpace* mfes = mesh->GetNodalFESpace();
+   const Table& eldof = mfes->GetElementToDofTable();
+   const FiniteElement* fe = mfes->GetFE(0);
+   const TensorBasisElement* tfe = dynamic_cast<const TensorBasisElement*>(fe);
+   const mfem::Array<int>& dof_map = tfe->GetDofMap();
+   const GridFunction* nodes = mesh->GetNodes();
+   Tensor<2> shape1d(GetNDofs1d(mfes),GetNQuads1d(order)), dshape1d(GetNDofs1d(mfes),GetNQuads1d(order));
+   ComputeBasis1d( fe, order, shape1d, dshape1d );
+
+   const int NE = fes->GetNE();
+
+   const int quads1d = shape1d.Width();
+   const int dofs1d = shape1d.Height();
+   const int dofs = dofs1d;
+
+   Tensor<2> Jac(J.getData(),quads1d,NE);
+   Jac.zero();
+   Tensor<2> LexPointMat(dim,dofs);
+
+   Tensor<1> T0(LexPointMat.getData(),dofs1d);
+   for (int e = 0; e < NE; ++e)
+   {
+      ScatterDofs(mfes, eldof, dof_map, nodes, dofs, dim, e, LexPointMat);
+      // Computing the Jacobian with the tensor product structure
+      for (int j1 = 0; j1 < quads1d; ++j1)
+      {
+         for (int i1 = 0; i1 < dofs1d; ++i1)
+         {
+            for (int d = 0; d < dim; ++d)
+            {
+               Jac(j1,e) += T0(i1) * dshape1d(i1,j1);
+            }
+         }
+      }
+   }
+}
+
+static void EvalJacobians2D(const mfem::FiniteElementSpace* fes, const int order, Tensor<1>& J)
+{
+   const int dim = 2;
+
+   const Mesh* mesh = fes->GetMesh();
+   const mfem::FiniteElementSpace* mfes = mesh->GetNodalFESpace();
+   const Table& eldof = mfes->GetElementToDofTable();
+   const FiniteElement* fe = mfes->GetFE(0);
+   const TensorBasisElement* tfe = dynamic_cast<const TensorBasisElement*>(fe);
+   const mfem::Array<int>& dof_map = tfe->GetDofMap();
+   const GridFunction* nodes = mesh->GetNodes();
+   Tensor<2> shape1d(GetNDofs1d(mfes),GetNQuads1d(order)), dshape1d(GetNDofs1d(mfes),GetNQuads1d(order));
+   ComputeBasis1d( fe, order, shape1d, dshape1d );
+
+   const int NE = mfes->GetNE();
+
+   const int quads1d = shape1d.Width();
+   const int dofs1d  = shape1d.Height();
+   const int dofs = dofs1d * dofs1d;
+
+
+   Tensor<5> Jac(J.getData(),dim,dim,quads1d,quads1d,NE);
+   Jac.zero();
+   Tensor<2> LexPointMat(dim,dofs);
+
+   Tensor<3> T0(LexPointMat.getData(),dim,dofs1d,dofs1d);
+   Tensor<2> T1b(dim,quads1d), T1d(dim,quads1d);
+   for (int e = 0; e < NE; ++e)
+   {
+      ScatterDofs(mfes, eldof, dof_map, nodes, dofs, dim, e, LexPointMat);
+      // Computing the Jacobian with the tensor product structure
+      for (int i2 = 0; i2 < dofs1d; ++i2)
+      {
+         T1b.zero();
+         T1d.zero();
+         for (int j1 = 0; j1 < quads1d; ++j1)
+         {
+            for (int i1 = 0; i1 < dofs1d; ++i1)
+            {
+               for (int d = 0; d < dim; ++d)
+               {
+                  T1b(d,j1) += T0(d,i1,i2) *  shape1d(i1,j1);
+                  T1d(d,j1) += T0(d,i1,i2) * dshape1d(i1,j1);
+               }
+            }
+         }
+         for (int j2 = 0; j2 < quads1d; ++j2)
+         {
+            for (int j1 = 0; j1 < quads1d; ++j1)
+            {
+               for (int d = 0; d < dim; ++d)
+               {
+                  Jac(d,0,j1,j2,e) += T1d(d,j1) *  shape1d(i2,j2);
+                  Jac(d,1,j1,j2,e) += T1b(d,j1) * dshape1d(i2,j2);
+               }
+            }
+         }
+      }
+   }
+}
+
+static void EvalJacobians3D(const mfem::FiniteElementSpace* fes, const int order, Tensor<1>& J)
+{
+   const int dim = 3;
+
+   const Mesh* mesh = fes->GetMesh();
+   const mfem::FiniteElementSpace* mfes = mesh->GetNodalFESpace();
+   const Table& eldof = mfes->GetElementToDofTable();
+   const FiniteElement* fe = mfes->GetFE(0);
+   const TensorBasisElement* tfe = dynamic_cast<const TensorBasisElement*>(fe);
+   const mfem::Array<int>& dof_map = tfe->GetDofMap();
+   const GridFunction* nodes = mesh->GetNodes();
+   Tensor<2> shape1d(GetNDofs1d(mfes),GetNQuads1d(order)), dshape1d(GetNDofs1d(mfes),GetNQuads1d(order));
+   ComputeBasis1d( fe, order, shape1d, dshape1d );
+
+   const int NE = fes->GetNE();
+
+   const int quads1d = shape1d.Width();
+   const int dofs1d = shape1d.Height();
+   const int dofs = dofs1d * dofs1d * dofs1d;
+
+   Tensor<6> Jac(J.getData(),dim,dim,quads1d,quads1d,quads1d,NE);
+   Jac.zero();
+   Tensor<2> LexPointMat(dim,dofs);
+
+   Tensor<4> T0(LexPointMat.getData(),dim,dofs1d,dofs1d,dofs1d);
+   Tensor<2> T1b(dim,quads1d), T1d(dim,quads1d);
+   Tensor<3> T2bb(dim,quads1d,quads1d), T2db(dim,quads1d,quads1d), T2bd(dim,quads1d,quads1d);
+   for (int e = 0; e < NE; ++e)
+   {
+      // Modifies T0 in the same time...
+      ScatterDofs(mfes, eldof, dof_map, nodes, dofs, dim, e, LexPointMat);
+      // Computing the Jacobian with the tensor product structure
+      for (int i3 = 0; i3 < dofs1d; ++i3)
+      {
+         T2bb.zero();
+         T2db.zero();
+         T2bd.zero();
+         for (int i2 = 0; i2 < dofs1d; ++i2)
+         {
+            T1b.zero();
+            T1d.zero();
+            for (int j1 = 0; j1 < quads1d; ++j1)
+            {
+               for (int i1 = 0; i1 < dofs1d; ++i1)
+               {
+                  for (int d = 0; d < dim; ++d)
+                  {
+                     T1b(d,j1) += T0(d,i1,i2,i3) *  shape1d(i1,j1);
+                     T1d(d,j1) += T0(d,i1,i2,i3) * dshape1d(i1,j1);
+                  }
+               }
+            }
+            for (int j2 = 0; j2 < quads1d; ++j2)
+            {
+               for (int j1 = 0; j1 < quads1d; ++j1)
+               {
+                  for (int d = 0; d < dim; ++d)
+                  {
+                     T2bb(d,j1,j2) += T1b(d,j1) *  shape1d(i2,j2);
+                     T2bd(d,j1,j2) += T1b(d,j1) * dshape1d(i2,j2);
+                     T2db(d,j1,j2) += T1d(d,j1) *  shape1d(i2,j2);
+                  }
+               }
+            }
+         }
+         for (int j3 = 0; j3 < quads1d; ++j3)
+         {
+            for (int j2 = 0; j2 < quads1d; ++j2)
+            {
+               for (int j1 = 0; j1 < quads1d; ++j1)
+               {
+                  for (int d = 0; d < dim; ++d)
+                  {
+                     Jac(d,0,j1,j2,j3,e) += T2db(d,j1,j2) *  shape1d(i3,j3);
+                     Jac(d,1,j1,j2,j3,e) += T2bd(d,j1,j2) *  shape1d(i3,j3);
+                     Jac(d,2,j1,j2,j3,e) += T2bb(d,j1,j2) * dshape1d(i3,j3);
+                  }
+               }
+            }
+         }
+      }
+   }
+}
+
+void EvalJacobians( const int dim, const mfem::FiniteElementSpace* fes, const int order,
+                     Tensor<1>& J )
+{
+   switch(dim)
+   {
+      case 1:
+         EvalJacobians1D( fes, order, J );
+         break;
+      case 2:
+         EvalJacobians2D( fes, order, J );
+         break;
+      case 3:
+         EvalJacobians3D( fes, order, J );
+         break;
+      default:
+         mfem_error("This orientation does not exist in 3D");
+   }
+}
+
+}
+
+}
+
+#endif

backends/partialassembly/tensorialfunctions.hpp

@@ -0,0 +1,801 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+// This file contains operator-based bilinear form integrators used
+// with BilinearFormOperator.
+
+#ifndef MFEM_TENSORFUNC
+#define MFEM_TENSORFUNC
+
+#include "tensor.hpp"
+// #include "fem.hpp"
+#include "dgpabilininteg.hpp"
+#include "../../fem/fespace.hpp"
+
+namespace mfem
+{
+
+namespace pa
+{
+
+/// Returns number of degrees of freedom in each direction.
+inline const int GetNDofs1d(const mfem::FiniteElementSpace& fes){ return fes.GetFE(0)->GetOrder() + 1; }
+
+inline const int GetNDofs1d(const mfem::FiniteElementSpace* fes){ return GetNDofs1d(*fes); }
+
+/// Returns number of quadrature points in each direction.
+inline const int GetNQuads1d(const int order)
+{
+   const IntegrationRule &ir1d = IntRules.Get(Geometry::SEGMENT, order);
+   return ir1d.GetNPoints();
+}
+
+/**
+* Gives the evaluation of the 1d basis functions and their derivative at one point @param x
+*/
+template <typename Tensor>
+void ComputeBasis0d(const FiniteElement *fe, double x,
+                     Tensor& shape0d, Tensor& dshape0d)
+{
+   const TensorBasisElement* tfe(dynamic_cast<const TensorBasisElement*>(fe));
+   const Poly_1D::Basis &basis0d = tfe->GetBasis1D();
+
+   const int quads0d = 1;
+   const int dofs = fe->GetOrder() + 1;
+
+   mfem::Vector u(dofs);
+   mfem::Vector d(dofs);
+   basis0d.Eval(x, u, d);
+   for (int i = 0; i < dofs; i++)
+   {
+      shape0d(i, 0) = u(i);
+      dshape0d(i, 0) = d(i);
+   }
+}
+
+/**
+* Gives the evaluation of the 1d basis functions and their derivative at all quadrature points
+*/
+template <typename Tensor>
+void ComputeBasis1d(const FiniteElement *fe, int order, Tensor& shape1d,
+                           Tensor& dshape1d, bool backward=false)
+{
+   const TensorBasisElement* tfe(dynamic_cast<const TensorBasisElement*>(fe));
+   const Poly_1D::Basis &basis1d = tfe->GetBasis1D();
+   const IntegrationRule &ir1d = IntRules.Get(Geometry::SEGMENT, order);
+
+   const int quads1d = ir1d.GetNPoints();
+   const int dofs = fe->GetOrder() + 1;
+
+   mfem::Vector u(dofs);
+   mfem::Vector d(dofs);
+   for (int k = 0; k < quads1d; k++)
+   {
+      int ind = backward ? quads1d -1 - k : k;
+      const IntegrationPoint &ip = ir1d.IntPoint(k);
+      basis1d.Eval(ip.x, u, d);
+      for (int i = 0; i < dofs; i++)
+      {
+         shape1d(i, ind) = u(i);
+         dshape1d(i, ind) = d(i);
+      }
+   }
+}
+
+/**
+* Gives the evaluation of the 1d basis functions at one point @param x
+*/
+template <typename Tensor>
+void ComputeBasis0d(const FiniteElement *fe, double x, Tensor& shape0d)
+{
+   const TensorBasisElement* tfe(dynamic_cast<const TensorBasisElement*>(fe));
+   const Poly_1D::Basis &basis0d = tfe->GetBasis1D();
+
+   const int dofs = fe->GetOrder() + 1;
+
+   mfem::Vector u(dofs);
+   mfem::Vector d(dofs);
+   basis0d.Eval(x, u, d);
+   for (int i = 0; i < dofs; i++)
+   {
+      shape0d(i, 0) = u(i);
+   }
+}
+
+/**
+* Gives the evaluation of the 1d basis functions at all quadrature points
+*/
+template <typename Tensor>
+void ComputeBasis1d(const FiniteElement *fe, int order, Tensor& shape1d, bool backward=false)
+{
+   const TensorBasisElement* tfe(dynamic_cast<const TensorBasisElement*>(fe));
+   const Poly_1D::Basis &basis1d = tfe->GetBasis1D();
+   const IntegrationRule &ir1d = IntRules.Get(Geometry::SEGMENT, order);
+
+   const int quads1d = ir1d.GetNPoints();
+   const int dofs = fe->GetOrder() + 1;
+
+   mfem::Vector u(dofs);
+   mfem::Vector d(dofs);
+   for (int k = 0; k < quads1d; k++)
+   {
+      int ind = backward ? quads1d -1 - k : k;
+      const IntegrationPoint &ip = ir1d.IntPoint(k);
+      basis1d.Eval(ip.x, u, d);
+      for (int i = 0; i < dofs; i++)
+      {
+         shape1d(i, ind) = u(i);
+      }
+   }
+}
+
+template <PAOp Op>
+class TensorBasis
+{
+protected:
+   const int dim;
+
+private:
+   typedef Tensor<2> Tensor2d;
+   Tensor2d shape1d, dshape1d;
+
+public:
+   TensorBasis(mfem::FiniteElementSpace* fes, const int order)
+   : dim(fes->GetFE(0)->GetDim()),
+     shape1d(GetNDofs1d(fes),GetNQuads1d(order)),
+     dshape1d(GetNDofs1d(fes),GetNQuads1d(order))
+   {
+      // Store the 1d shape functions and gradients
+      ComputeBasis1d(fes->GetFE(0), order, shape1d, dshape1d);
+   }
+
+   const Tensor2d& getB() const
+   {
+      return shape1d;
+   }
+
+   const Tensor2d& getG() const
+   {
+      return dshape1d;
+   }
+
+};
+
+template <>
+class TensorBasis<BtDB>
+{
+protected:
+   const int dim;
+
+private:
+   typedef Tensor<2> Tensor2d;
+   Tensor2d shape1d;
+
+public:
+   TensorBasis(mfem::FiniteElementSpace* fes, const int order)
+   : dim(fes->GetFE(0)->GetDim()),
+     shape1d(GetNDofs1d(fes),GetNQuads1d(order))
+   {
+      // Store the 1d shape functions and gradients
+      ComputeBasis1d(fes->GetFE(0), order, shape1d);
+   }
+
+   const Tensor2d& getB() const
+   {
+      return shape1d;
+   }
+
+};
+
+
+template <PAOp Op>
+class FaceTensorBasis: public TensorBasis<Op>
+{
+private:
+   typedef Tensor<2> Tensor2d;
+   Tensor2d shape0d0, shape0d1;
+   Tensor2d dshape0d0, dshape0d1;
+
+public:
+   FaceTensorBasis(mfem::FiniteElementSpace* fes, const int order)
+   : TensorBasis<Op>(fes,order),
+     shape0d0(GetNDofs1d(fes),GetNQuads1d(order)),
+     shape0d1(GetNDofs1d(fes),GetNQuads1d(order))
+   {
+      // Store the two 0d shape functions and gradients
+      // in x = 0.0
+      ComputeBasis0d(fes->GetFE(0), 0.0  , shape0d0, dshape0d0);
+      // in x = 1.0
+      ComputeBasis0d(fes->GetFE(0), 1.0  , shape0d1, dshape0d1);
+   }
+
+   const Tensor2d& getB0d(const int face_id) const
+   {
+      switch(this->dim)
+      {
+         case 1:
+         switch(face_id)
+         {
+            case 0://WEST
+               return shape0d0;
+            case 1://EAST
+               return shape0d1;
+         }
+         case 2:
+         switch(face_id)
+         {
+            case 0://SOUTH
+               return shape0d0;
+            case 1://EAST
+               return shape0d1;
+            case 2://NORTH
+               return shape0d1;
+            case 3://WEST
+               return shape0d0;
+         }
+         case 3:
+         switch(face_id)
+         {
+            case 0://BOTTOM
+               return shape0d0;
+            case 1://SOUTH
+               return shape0d0;
+            case 2://EAST
+               return shape0d1;
+            case 3://NORTH
+               return shape0d1;
+            case 4://WEST
+               return shape0d0;
+            case 5://TOP
+               return shape0d1;
+         }
+      }
+   }
+
+   const Tensor2d& getB0dTrial(const int face_id) const
+   {
+      switch(this->dim)
+      {
+         case 1:
+         switch(face_id)
+         {
+            case 0://WEST
+               return shape0d1;
+            case 1://EAST
+               return shape0d0;
+         }
+         case 2:
+         switch(face_id)
+         {
+            case 0://SOUTH
+               return shape0d1;
+            case 1://EAST
+               return shape0d0;
+            case 2://NORTH
+               return shape0d0;
+            case 3://WEST
+               return shape0d1;
+         }
+         case 3:
+         switch(face_id)
+         {
+            case 0://BOTTOM
+               return shape0d1;
+            case 1://SOUTH
+               return shape0d1;
+            case 2://EAST
+               return shape0d0;
+            case 3://NORTH
+               return shape0d0;
+            case 4://WEST
+               return shape0d1;
+            case 5://TOP
+               return shape0d0;
+         }
+      }
+   }
+
+   const Tensor2d& getB0dTest(const int face_id) const
+   {
+      switch(this->dim)
+      {
+         case 1:
+         switch(face_id)
+         {
+            case 0://WEST
+               return shape0d0;
+            case 1://EAST
+               return shape0d1;
+         }
+         case 2:
+         switch(face_id)
+         {
+            case 0://SOUTH
+               return shape0d0;
+            case 1://EAST
+               return shape0d1;
+            case 2://NORTH
+               return shape0d1;
+            case 3://WEST
+               return shape0d0;
+         }
+         case 3:
+         switch(face_id)
+         {
+            case 0://BOTTOM
+               return shape0d0;
+            case 1://SOUTH
+               return shape0d0;
+            case 2://EAST
+               return shape0d1;
+            case 3://NORTH
+               return shape0d1;
+            case 4://WEST
+               return shape0d0;
+            case 5://TOP
+               return shape0d1;
+         }
+      }
+   }
+
+   const Tensor2d& getG0d(const int face_id) const
+   {
+      switch(this->dim)
+      {
+         case 1:
+         switch(face_id)
+         {
+            case 0://WEST
+               return dshape0d0;
+            case 1://EAST
+               return dshape0d1;
+         }
+         case 2:
+         switch(face_id)
+         {
+            case 0://SOUTH
+               return dshape0d0;
+            case 1://EAST
+               return dshape0d1;
+            case 2://NORTH
+               return dshape0d1;
+            case 3://WEST
+               return dshape0d0;
+         }
+         case 3:
+         switch(face_id)
+         {
+            case 0://BOTTOM
+               return dshape0d0;
+            case 1://SOUTH
+               return dshape0d0;
+            case 2://EAST
+               return dshape0d1;
+            case 3://NORTH
+               return dshape0d1;
+            case 4://WEST
+               return dshape0d0;
+            case 5://TOP
+               return dshape0d1;
+         }
+      }
+   }
+
+   const Tensor2d& getG0dTrial(const int face_id) const
+   {
+      switch(this->dim)
+      {
+         case 1:
+         switch(face_id)
+         {
+            case 0://WEST
+               return dshape0d1;
+            case 1://EAST
+               return dshape0d0;
+         }
+         case 2:
+         switch(face_id)
+         {
+            case 0://SOUTH
+               return dshape0d1;
+            case 1://EAST
+               return dshape0d0;
+            case 2://NORTH
+               return dshape0d0;
+            case 3://WEST
+               return dshape0d1;
+         }
+         case 3:
+         switch(face_id)
+         {
+            case 0://BOTTOM
+               return dshape0d1;
+            case 1://SOUTH
+               return dshape0d1;
+            case 2://EAST
+               return dshape0d0;
+            case 3://NORTH
+               return dshape0d0;
+            case 4://WEST
+               return dshape0d1;
+            case 5://TOP
+               return dshape0d0;
+         }
+      }
+   }
+
+   const Tensor2d& getG0dTest(const int face_id) const
+   {
+      switch(this->dim)
+      {
+         case 1:
+         switch(face_id)
+         {
+            case 0://WEST
+               return dshape0d0;
+            case 1://EAST
+               return dshape0d1;
+         }
+         case 2:
+         switch(face_id)
+         {
+            case 0://SOUTH
+               return dshape0d0;
+            case 1://EAST
+               return dshape0d1;
+            case 2://NORTH
+               return dshape0d1;
+            case 3://WEST
+               return dshape0d0;
+         }
+         case 3:
+         switch(face_id)
+         {
+            case 0://BOTTOM
+               return dshape0d0;
+            case 1://SOUTH
+               return dshape0d0;
+            case 2://EAST
+               return dshape0d1;
+            case 3://NORTH
+               return dshape0d1;
+            case 4://WEST
+               return dshape0d0;
+            case 5://TOP
+               return dshape0d1;
+         }
+      }
+   }
+
+};
+
+template <>
+class FaceTensorBasis<BtDB>: public TensorBasis<BtDB>
+{
+private:
+   typedef Tensor<2> Tensor2d;
+   Tensor2d shape0d0, shape0d1;
+
+public:
+   FaceTensorBasis(mfem::FiniteElementSpace* fes, const int order)
+   : TensorBasis<BtDB>(fes,order),
+     shape0d0(GetNDofs1d(fes),GetNQuads1d(order)),
+     shape0d1(GetNDofs1d(fes),GetNQuads1d(order))
+   {
+      // Store the two 0d shape functions and gradients
+      // in x = 0.0
+      ComputeBasis0d(fes->GetFE(0), 0.0  , shape0d0);
+      // in x = 1.0
+      ComputeBasis0d(fes->GetFE(0), 1.0  , shape0d1);
+   }
+
+   const Tensor2d& getB0d(const int face_id) const
+   {
+      switch(this->dim)
+      {
+         case 1:
+         switch(face_id)
+         {
+            case 0://WEST
+               return shape0d0;
+            case 1://EAST
+               return shape0d1;
+         }
+         case 2:
+         switch(face_id)
+         {
+            case 0://SOUTH
+               return shape0d0;
+            case 1://EAST
+               return shape0d1;
+            case 2://NORTH
+               return shape0d1;
+            case 3://WEST
+               return shape0d0;
+         }
+         case 3:
+         switch(face_id)
+         {
+            case 0://BOTTOM
+               return shape0d0;
+            case 1://SOUTH
+               return shape0d0;
+            case 2://EAST
+               return shape0d1;
+            case 3://NORTH
+               return shape0d1;
+            case 4://WEST
+               return shape0d0;
+            case 5://TOP
+               return shape0d1;
+         }
+      }
+   }
+
+   const Tensor2d& getB0dTrial(const int face_id) const
+   {
+      switch(this->dim)
+      {
+         case 1:
+         switch(face_id)
+         {
+            case 0://WEST
+               return shape0d1;
+            case 1://EAST
+               return shape0d0;
+         }
+         case 2:
+         switch(face_id)
+         {
+            case 0://SOUTH
+               return shape0d1;
+            case 1://EAST
+               return shape0d0;
+            case 2://NORTH
+               return shape0d0;
+            case 3://WEST
+               return shape0d1;
+         }
+         case 3:
+         switch(face_id)
+         {
+            case 0://BOTTOM
+               return shape0d1;
+            case 1://SOUTH
+               return shape0d1;
+            case 2://EAST
+               return shape0d0;
+            case 3://NORTH
+               return shape0d0;
+            case 4://WEST
+               return shape0d1;
+            case 5://TOP
+               return shape0d0;
+         }
+      }
+   }
+
+   const Tensor2d& getB0dTest(const int face_id) const
+   {
+      switch(this->dim)
+      {
+         case 1:
+         switch(face_id)
+         {
+            case 0://WEST
+               return shape0d0;
+            case 1://EAST
+               return shape0d1;
+         }
+         case 2:
+         switch(face_id)
+         {
+            case 0://SOUTH
+               return shape0d0;
+            case 1://EAST
+               return shape0d1;
+            case 2://NORTH
+               return shape0d1;
+            case 3://WEST
+               return shape0d0;
+         }
+         case 3:
+         switch(face_id)
+         {
+            case 0://BOTTOM
+               return shape0d0;
+            case 1://SOUTH
+               return shape0d0;
+            case 2://EAST
+               return shape0d1;
+            case 3://NORTH
+               return shape0d1;
+            case 4://WEST
+               return shape0d0;
+            case 5://TOP
+               return shape0d1;
+         }
+      }
+   }
+};
+
+
+void ScatterDofs(const mfem::FiniteElementSpace* mfes, const Table& eldof, const mfem::Array<int>& dof_map,
+                  const GridFunction* nodes, const int dofs, const int dim, const int e,
+                  Tensor<2>& LexPointMat);
+
+/**
+*  Eval the Jacobian of all elements in a Finite Element Space.
+*/
+void EvalJacobians( const int dim, const mfem::FiniteElementSpace* fes, const int order,
+                     Tensor<1>& J );
+
+/**
+*  Return the diagonal of a 1d Partial Assembly Operator
+*/
+template <int Dim, typename Op>
+void GetDiag1d(const mfem::FiniteElementSpace& fes, const int order, const Op& op, Tensor<Dim>& diag)
+{
+   const int dofs1d = GetNDofs1d(fes);
+   const int quads1d = GetNQuads1d(order);
+   const int nb_elts = fes.GetNE();
+   Tensor<2> diagT(diag, dofs1d, nb_elts);
+   auto Dlin = op.getD();
+   Tensor<2> D(Dlin, quads1d, nb_elts);
+   Tensor<2> shape1d(dofs1d,quads1d);
+   ComputeBasis1d( fes.GetFE(0), order, shape1d);
+
+   diagT.zero();
+   for (int e = 0; e < nb_elts; ++e)
+   {
+      for (int j1 = 0; j1 < quads1d; ++j1)
+      {
+         for (int i1 = 0; i1 < dofs1d; ++i1)
+         {
+               diagT(i1,e) += D(j1,e) * shape1d(i1,j1) * shape1d(i1,j1);
+         }
+      }
+   }
+}
+
+/**
+*  Return the diagonal of a 2d Partial Assembly Operator
+*/
+template <int Dim, typename Op>
+void GetDiag2d(const mfem::FiniteElementSpace& fes, const int order, const Op& op, Tensor<Dim>& diag)
+{
+   const int dofs1d = GetNDofs1d(fes);
+   const int quads1d = GetNQuads1d(order);
+   const int nb_elts = fes.GetNE();
+   Tensor<3> diagT(diag, dofs1d, dofs1d, nb_elts);
+   auto Dlin = op.getD();
+   Tensor<3> D(Dlin, quads1d, quads1d, nb_elts);
+   Tensor<2> shape1d(dofs1d,quads1d);
+   ComputeBasis1d( fes.GetFE(0), order, shape1d);
+
+   Tensor<2> T1(dofs1d,quads1d);
+
+   diagT.zero();
+   for (int e = 0; e < nb_elts; ++e)
+   {
+      T1.zero();
+      for (int j2 = 0; j2 < quads1d; ++j2)
+      {
+         for (int j1 = 0; j1 < quads1d; ++j1)
+         {
+            for (int i1 = 0; i1 < dofs1d; ++i1)
+            {
+                  T1(i1,j2) += D(j1,j2,e) * shape1d(i1,j1) * shape1d(i1,j1);
+            }
+         }
+      }
+      for (int j2 = 0; j2 < quads1d; ++j2)
+      {
+         for (int i2 = 0; i2 < dofs1d; ++i2)
+         {
+            for (int i1 = 0; i1 < dofs1d; ++i1)
+            {
+                  diagT(i1,i2,e) += T1(i1,j2) * shape1d(i2,j2) * shape1d(i2,j2);
+            }
+         }
+      }
+   }
+}
+
+/**
+*  Return the diagonal of a 3d Partial Assembly Operator
+*/
+template <int Dim, typename Op>
+void GetDiag3d(const mfem::FiniteElementSpace& fes, const int order, const Op& op, Tensor<Dim>& diag)
+{
+   const int dofs1d = GetNDofs1d(fes);
+   const int quads1d = GetNQuads1d(order);
+   const int nb_elts = fes.GetNE();
+   Tensor<4> diagT(diag, dofs1d, dofs1d, dofs1d, nb_elts);
+   auto Dlin = op.getD();
+   Tensor<4> D(Dlin, quads1d, quads1d, quads1d, nb_elts);
+   Tensor<2> shape1d(dofs1d,quads1d);
+   ComputeBasis1d( fes.GetFE(0), order, shape1d);
+
+   Tensor<3> T1(dofs1d,quads1d,quads1d),T2(dofs1d,dofs1d,quads1d);
+
+   diagT.zero();
+   for (int e = 0; e < nb_elts; ++e)
+   {
+      T1.zero();
+      for (int j3 = 0; j3 < quads1d; ++j3)
+      {
+         for (int j2 = 0; j2 < quads1d; ++j2)
+         {
+            for (int j1 = 0; j1 < quads1d; ++j1)
+            {
+               for (int i1 = 0; i1 < dofs1d; ++i1)
+               {
+                     T1(i1,j2,j3) += D(j1,j2,j3,e) * shape1d(i1,j1) * shape1d(i1,j1);
+               }
+            }
+         }
+      }
+      T2.zero();
+      for (int j3 = 0; j3 < quads1d; ++j3)
+      {
+         for (int j2 = 0; j2 < quads1d; ++j2)
+         {
+            for (int i2 = 0; i2 < dofs1d; ++i2)
+            {
+               for (int i1 = 0; i1 < dofs1d; ++i1)
+               {
+                     T2(i1,i2,j3) += T1(i1,j2,j3) * shape1d(i2,j2) * shape1d(i2,j2);
+               }
+            }
+         }
+      }
+      for (int j3 = 0; j3 < quads1d; ++j3)
+      {
+         for (int i3 = 0; i3 < dofs1d; ++i3)
+         {
+            for (int i2 = 0; i2 < dofs1d; ++i2)
+            {
+               for (int i1 = 0; i1 < dofs1d; ++i1)
+               {
+                     diagT(i1,i2,i3,e) += T2(i1,i2,j3) * shape1d(i3,j3) * shape1d(i3,j3);
+               }
+            }
+         }
+      }
+   }
+}
+
+/**
+*  Return the diagonal of a Partial Assembly Operator
+*/
+template <int Dim, typename Op>
+void GetDiag(const FiniteElementSpace& fes, const int order, const Op& op, Tensor<Dim>& diag)
+{
+   switch(fes.GetFE(0)->GetDim())
+   {
+      case 1:
+         GetDiag1d(fes, order, op, diag);
+         break;
+      case 2:
+         GetDiag2d(fes, order, op, diag);
+         break;
+      case 3:
+         GetDiag3d(fes, order, op, diag);
+         break;
+   }
+}
+
+}
+
+}
+
+#endif

backends/partialassembly/vector.cpp

@@ -0,0 +1,28 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "vector.hpp"
+#include "../../linalg/vector.hpp"
+
+namespace mfem
+{
+
+namespace pa
+{
+
+} // namespace mfem::pa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)

backends/partialassembly/vector.hpp

@@ -0,0 +1,171 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_PA_VECTOR_HPP
+#define MFEM_BACKENDS_PA_VECTOR_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#include "../base/vector.hpp"
+#include "array.hpp"
+#include "../../linalg/vector.hpp"
+#include "../../linalg/densemat.hpp"
+#include <cstring>
+
+namespace mfem
+{
+
+namespace pa
+{
+
+/**
+*  A simple Vector class with type information.
+*/
+template <typename T>
+class Vector : public Array, public PVector
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // DLayout layout;
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   virtual PVector *DoVectorClone(bool copy_data, void **buffer,
+                                  int buffer_type_id) const;
+
+   virtual void DoDotProduct(const PVector &x, void *result,
+                             int result_type_id) const;
+
+   virtual void DoAxpby(const void *a, const PVector &x,
+                        const void *b, const PVector &y,
+                        int ab_type_id);
+
+   ///@}
+   // End: Virtual interface
+
+public:
+   Vector(Layout &lt)
+      : PArray(lt), Array(lt, sizeof(T)), PVector(lt)
+   { }
+
+   T* GetData() { return Array::GetTypedData<T>(); }
+   const T* GetData() const { return Array::GetTypedData<T>(); }
+   const mfem::Vector GetVectorView(const int offset, const int size) const
+   {
+      return mfem::Vector(static_cast<T*>(data) + offset, size);
+   }
+   const mfem::DenseMatrix GetMatrixView(const int offset, const int height, const int width) const
+   {
+      return mfem::DenseMatrix(static_cast<T*>(data) + offset, height, width);
+   }
+   const mfem::DenseTensor GetTensorView(const int offset, const int i, const int j, const int k) const
+   {
+      return mfem::DenseTensor(static_cast<T*>(data) + offset, i, j, k);
+      // return mfem::DenseTensor().UseExternalData(data+offset,i,j,k);
+   }
+
+   mfem::Vector Wrap();
+
+   const mfem::Vector Wrap() const;
+
+};
+
+template<typename T>
+PVector* Vector<T>::DoVectorClone(bool copy_data, void **buffer,
+                                  int buffer_type_id) const
+{
+   MFEM_ASSERT(buffer_type_id == ScalarId<T>::value, "The buffer has a different type.");
+   Layout& lt = static_cast<Layout&>(*layout);
+   Vector<T> *new_vector = new Vector<T>(lt);
+   if (copy_data)
+   {
+      memcpy(new_vector->GetData(), data, layout->Size()*sizeof(T));
+   }
+   if (buffer)
+   {
+      *buffer = new_vector->GetData();
+   }
+   return new_vector;
+}
+
+template<typename T>
+void Vector<T>::DoDotProduct(const PVector &x, void *result,
+                             int result_type_id) const
+{
+   MFEM_ASSERT(result_type_id == ScalarId<T>::value, "The buffer has a different type.");
+   const T* data_v1 = this->GetData();
+   const T* data_v2 = x.As<Vector<T>>().GetData();
+   T& result_d = *static_cast<T*>(result);
+   result_d = 0;
+   for (std::size_t i = 0; i < layout->Size(); ++i)
+   {
+      result_d += data_v1[i] * data_v2[i];
+   }
+}
+
+template<typename T>
+void Vector<T>::DoAxpby(const void *a, const PVector &x,
+                        const void *b, const PVector &y,
+                        int ab_type_id)
+{
+   MFEM_ASSERT(ab_type_id == ScalarId<T>::value, "The buffer has a different type.");
+   const T& va = *static_cast<const T*>(a);
+   const T& vb = *static_cast<const T*>(b);
+   if (va != 0.0 && vb != 0.0) {
+      const T* vx = x.As<Vector<T>>().GetData();
+      const T* vy = y.As<Vector<T>>().GetData();
+      T* typed_data = GetData();
+      for (std::size_t i = 0; i < layout->Size(); ++i)
+      {
+         typed_data[i] = va * vx[i] + vb * vy[i];
+      }
+   } else if (va == 0.0) {
+      const T* vy = y.As<Vector<T>>().GetData();
+      T* typed_data = GetData();
+      for (std::size_t i = 0; i < layout->Size(); ++i)
+      {
+         typed_data[i] = vb * vy[i];
+      }
+   } else if (vb == 0.0) {
+      const T* vx = x.As<Vector<T>>().GetData();
+      T* typed_data = GetData();
+      for (std::size_t i = 0; i < layout->Size(); ++i)
+      {
+         typed_data[i] = va * vx[i];
+      }
+   }
+}
+
+template<typename T>
+mfem::Vector Vector<T>::Wrap()
+{
+   return mfem::Vector(*this);
+}
+
+template<typename T>
+const mfem::Vector Vector<T>::Wrap() const
+{
+   return mfem::Vector(*const_cast<Vector<T>*>(this));
+}
+
+} // namespace mfem::pa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_PA)
+
+#endif // MFEM_BACKENDS_PA_VECTOR_HPP

config/config.hpp

@@ -37,3 +37,11 @@
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
 #endif // MFEM_USE_MPI not defined
+
+// Macro that returns its first arg when MFEM_USE_BACKENDS is defined, and its
+// second arg if it is not defined.
+#ifdef MFEM_USE_BACKENDS
+#define MFEM_IF_BACKENDS(x,y) (x)
+#else
+#define MFEM_IF_BACKENDS(x,y) (y)
+#endif

config/config.hpp.in

@@ -33,6 +33,12 @@
 // Description of the git commit used to build MFEM.
 // #define MFEM_GIT_STRING "@MFEM_GIT_STRING@"
 
+// The absolute path of the MFEM source prefix
+// #define MFEM_SOURCE_DIR "@MFEM_SOURCE_DIR@"
+
+// The absolute path of the MFEM installation prefix
+// #define MFEM_INSTALL_DIR "@MFEM_INSTALL_DIR@"
+
 // Build the parallel MFEM library.
 // Requires an MPI compiler, and the libraries HYPRE and METIS.
 // #define MFEM_USE_MPI
@@ -109,6 +115,15 @@
 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR
 
+// Enable the use of MFEM backends.
+// #define MFEM_USE_BACKENDS
+
+// Enable the OCCA backend.
+// #define MFEM_USE_OCCA
+
+// Enable the PA backend.
+// #define MFEM_USE_PA
+
 // Windows specific options
 #ifdef _WIN32
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)

config/config.mk.in

@@ -13,6 +13,8 @@
 MFEM_VERSION         = @MFEM_VERSION@
 MFEM_VERSION_STRING  = @MFEM_VERSION_STRING@
 MFEM_GIT_STRING      = @MFEM_GIT_STRING@
+MFEM_SOURCE_DIR      = @MFEM_SOURCE_DIR@
+MFEM_INSTALL_DIR     = @MFEM_INSTALL_DIR@
 MFEM_USE_MPI         = @MFEM_USE_MPI@
 MFEM_USE_METIS       = @MFEM_USE_METIS@
 MFEM_USE_METIS_5     = @MFEM_USE_METIS_5@
@@ -37,6 +39,9 @@ MFEM_USE_PETSC       = @MFEM_USE_PETSC@
 MFEM_USE_MPFR        = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE       = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT     = @MFEM_USE_CONDUIT@
+MFEM_USE_BACKENDS    = @MFEM_USE_BACKENDS@
+MFEM_USE_OCCA        = @MFEM_USE_OCCA@
+MFEM_USE_PA          = @MFEM_USE_PA@
 
 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@

config/defaults.mk

@@ -83,6 +83,9 @@ MFEM_MPI_NP = 4
 # in config.mk and config.hpp.
 
 MFEM_USE_MPI         = NO
+# FIXME: add MFEM_USE_BACKENDS, MFEM_USE_OCCA to the CMake build system
+MFEM_USE_BACKENDS    = YES
+MFEM_USE_OCCA        = YES
 MFEM_USE_METIS       = $(MFEM_USE_MPI)
 MFEM_USE_METIS_5     = NO
 MFEM_DEBUG           = NO
@@ -271,6 +274,10 @@ SIDRE_LIB = \
    -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
    -lsidre -lslic -laxom_utils -lconduit -lconduit_relay -lhdf5 $(ZLIB_LIB) -ldl
 
+OCCA_DIR = @MFEM_DIR@/../occa
+OCCA_OPT = -I$(OCCA_DIR)/include
+OCCA_LIB = -Wl,-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca
+
 # If YES, enable some informational messages
 VERBOSE = NO
 

doc/CodeDocumentation.conf.in

@@ -760,6 +760,7 @@ WARN_LOGFILE           =
 
 INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/mfem.hpp \
+                         @MFEM_SOURCE_DIR@/backends/base \
                          @MFEM_SOURCE_DIR@/config \
                          @MFEM_SOURCE_DIR@/general \
                          @MFEM_SOURCE_DIR@/linalg \

examples/ex1.cpp

@@ -44,6 +44,8 @@ using namespace mfem;
 
 int main(int argc, char *argv[])
 {
+   tic_toc.Clear();
+   tic_toc.Start();
    // 1. Parse command-line options.
    const char *mesh_file = "../data/star.mesh";
    int order = 1;
@@ -154,18 +156,25 @@ int main(int argc, char *argv[])
 
    cout << "Size of linear system: " << A.Height() << endl;
 
-#ifndef MFEM_USE_SUITESPARSE
-   // 10. Define a simple symmetric Gauss-Seidel preconditioner and use it to
-   //     solve the system A X = B with PCG.
-   GSSmoother M(A);
-   PCG(A, M, B, X, 1, 200, 1e-12, 0.0);
-#else
-   // 10. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-   UMFPackSolver umf_solver;
-   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-   umf_solver.SetOperator(A);
-   umf_solver.Mult(B, X);
-#endif
+// #ifndef MFEM_USE_SUITESPARSE
+//    // 10. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+//    //     solve the system A X = B with PCG.
+//    GSSmoother M(A);
+//    PCG(A, M, B, X, 1, 200, 1e-12, 0.0);
+// #else
+//    // 10. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+//    UMFPackSolver umf_solver;
+//    umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+//    umf_solver.SetOperator(A);
+//    umf_solver.Mult(B, X);
+// #endif
+   tic_toc.Stop();
+   cout << " Initialization time: " << tic_toc.RealTime() << "s." << endl;
+   tic_toc.Clear();
+   tic_toc.Start();
+   CG(A, B, X, 3, 1000, 1e-12, 0.0);
+   tic_toc.Stop();
+   cout << " Computation time: " << tic_toc.RealTime() << "s." << endl;
 
    // 11. Recover the solution as a finite element grid function.
    a->RecoverFEMSolution(X, *b, x);

examples/ex1d.cpp

@@ -0,0 +1,180 @@
+
+#include <mfem.hpp>
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   tic_toc.Clear();
+   tic_toc.Start();
+   // 1. Parse command-line options.
+   const char *spec = "cpu";
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&spec, "-s", "--spec",
+                  "Compute resurce specification.");
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   /// Engine *engine = EngineDepot.Select(spec);
+   // string occa_spec("mode: 'Serial'");
+   // string occa_spec("mode: 'CUDA', deviceID: 0");
+   // string occa_spec("mode: 'OpenMP', threads: 4");
+   // string occa_spec("mode: 'OpenCL', deviceID: 0, platformID: 0");
+   string pa_spec("Hello world");
+
+   // SharedPtr<Engine> engine(new mfem::occa::Engine(occa_spec));
+   // SharedPtr<Engine> engine(new mfem::pa::Engine("hello world"));
+   SharedPtr<Engine> engine(new mfem::pa::Engine());
+
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
+   //    the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   mesh->SetEngine(*engine);
+   mesh->SetCurvature(1);
+   
+   int dim = mesh->Dimension();
+
+   // 3. Refine the mesh to increase the resolution. In this example we do
+   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
+   //    largest number that gives a final mesh with no more than 50,000
+   //    elements.
+   {
+      int ref_levels =
+         (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 4. Define a finite element space on the mesh. Here we use continuous
+   //    Lagrange finite elements of the specified order. If order < 1, we
+   //    instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (mesh->GetNodes())
+   {
+      fec = mesh->GetNodes()->OwnFEC();
+      cout << "Using isoparametric FEs: " << fec->Name() << endl;
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   cout << "Number of finite element unknowns: "
+        << fespace->GetTrueVSize() << endl;
+
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
+   //    In this example, the boundary conditions are defined by marking all
+   //    the boundary attributes from the mesh as essential (Dirichlet) and
+   //    converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
+   //    the basis functions in the finite element fespace.
+   LinearForm *b = new LinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   // 7. Define the solution vector x as a finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   GridFunction x(fespace);
+   x.Fill(0.0);
+
+   // 8. Set up the bilinear form a(.,.) on the finite element space
+   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //    domain integrator.
+   BilinearForm *a = new BilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 9. Assemble the bilinear form and the corresponding linear system,
+   //    applying any necessary transformations such as: eliminating boundary
+   //    conditions, applying conforming constraints for non-conforming AMR,
+   //    static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   OperatorHandle A(Operator::ANY_TYPE);
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   cout << "Size of linear system: " << A.Ptr()->Height() << endl;
+
+   // 10. Solve the system A X = B with CG.
+   tic_toc.Stop();
+   cout << " Initialization time: " << tic_toc.RealTime() << "s." << endl;
+   tic_toc.Clear();
+   tic_toc.Start();
+   CG(*A.Ptr(), B, X, 3, 1000, 1e-12, 0.0);
+   tic_toc.Stop();
+   cout << " Computation time: " << tic_toc.RealTime() << "s." << endl;
+
+   // 11. Recover the solution as a finite element grid function.
+   a->RecoverFEMSolution(X, *b, x);
+   x.Pull();
+
+   // 12. Save the refined mesh and the solution. This output can be viewed
+   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
+   ofstream mesh_ofs("refined.mesh");
+   mesh_ofs.precision(8);
+   mesh->Print(mesh_ofs);
+   ofstream sol_ofs("sol.gf");
+   sol_ofs.precision(8);
+   x.Save(sol_ofs);
+
+   // 13. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *mesh << x << flush;
+   }
+
+   // 14. Free the used memory.
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete mesh;
+
+   return 0;
+}

examples/ex1pd.cpp

@@ -0,0 +1,212 @@
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   /// Engine *engine = EngineDepot.Select(spec);
+
+   string occa_spec("mode: 'Serial'");
+   // string occa_spec("mode: 'CUDA', deviceID: 0");
+   // string occa_spec("mode: 'OpenMP', threads: 4");
+   // string occa_spec("mode: 'OpenCL', deviceID: 0, platformID: 0");
+
+   SharedPtr<Engine> engine(new mfem::occa::Engine(MPI_COMM_WORLD, occa_spec));
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   mesh->SetEngine(*engine);
+   int dim = mesh->Dimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 10,000 elements.
+   {
+      int ref_levels =
+         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   {
+      int par_ref_levels = 2;
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh->UniformRefinement();
+      }
+   }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+      if (myid == 0)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (1,phi_i) where phi_i are the basis functions in fespace.
+   ParLinearForm *b = new ParLinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   ParGridFunction x(fespace);
+   x.Fill(0.0);
+
+   // 10. Set up the parallel bilinear form a(.,.) on the finite element space
+   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //     domain integrator.
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   OperatorHandle A(Operator::ANY_TYPE);
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   CGSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+   pcg->SetRelTol(1e-6);
+   pcg->SetAbsTol(0.0);
+   pcg->SetMaxIter(1000);
+   pcg->SetPrintLevel(3);
+   pcg->SetOperator(*A.Ptr());
+   pcg->Mult(B, X);
+
+   // 13. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a->RecoverFEMSolution(X, *b, x);
+   x.Pull();
+
+   // 14. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 15. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x << flush;
+   }
+
+   // 16. Free the used memory.
+   delete pcg;
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}

examples/ex9d.cpp

@@ -0,0 +1,511 @@
+//                                MFEM Example 9
+//
+// Compile with: make ex9
+//
+// Sample runs:
+//    ex9 -m ../data/periodic-segment.mesh -p 0 -r 2 -dt 0.005
+//    ex9 -m ../data/periodic-square.mesh -p 0 -r 2 -dt 0.01 -tf 10
+//    ex9 -m ../data/periodic-hexagon.mesh -p 0 -r 2 -dt 0.01 -tf 10
+//    ex9 -m ../data/periodic-square.mesh -p 1 -r 2 -dt 0.005 -tf 9
+//    ex9 -m ../data/periodic-hexagon.mesh -p 1 -r 2 -dt 0.005 -tf 9
+//    ex9 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.002 -tf 9
+//    ex9 -m ../data/star-q3.mesh -p 1 -r 2 -dt 0.005 -tf 9
+//    ex9 -m ../data/disc-nurbs.mesh -p 1 -r 3 -dt 0.005 -tf 9
+//    ex9 -m ../data/disc-nurbs.mesh -p 2 -r 3 -dt 0.005 -tf 9
+//    ex9 -m ../data/periodic-square.mesh -p 3 -r 4 -dt 0.0025 -tf 9 -vs 20
+//    ex9 -m ../data/periodic-cube.mesh -p 0 -r 2 -o 2 -dt 0.02 -tf 8
+//
+// Description:  This example code solves the time-dependent advection equation
+//               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
+//               u0(x)=u(0,x) is a given initial condition.
+//
+//               The example demonstrates the use of Discontinuous Galerkin (DG)
+//               bilinear forms in MFEM (face integrators), the use of explicit
+//               ODE time integrators, the definition of periodic boundary
+//               conditions through periodic meshes, as well as the use of GLVis
+//               for persistent visualization of a time-evolving solution. The
+//               saving of time-dependent data files for external visualization
+//               with VisIt (visit.llnl.gov) is also illustrated.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include <algorithm>
+
+using namespace std;
+using namespace mfem;
+
+// Choice for the problem setup. The fluid velocity, initial condition and
+// inflow boundary condition are chosen based on this parameter.
+int problem;
+
+// Velocity coefficient
+void velocity_function(const Vector &x, Vector &v);
+
+// Initial condition
+double u0_function(const Vector &x);
+
+// Inflow boundary condition
+double inflow_function(const Vector &x);
+
+// Mesh bounding box
+Vector bb_min, bb_max;
+
+
+/** A time-dependent operator for the right-hand side of the ODE. The DG weak
+    form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
+    and advection matrices, and b describes the flow on the boundary. This can
+    be written as a general ODE, du/dt = M^{-1} (K u + b), and this class is
+    used to evaluate the right-hand side. */
+// class FE_Evolution : public TimeDependentOperator
+// {
+// private:
+//    SparseMatrix &M, &K;
+//    const Vector &b;
+//    DSmoother M_prec;
+//    CGSolver M_solver;
+
+//    mutable Vector z;
+
+// public:
+//    FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b);
+
+//    virtual void Mult(const Vector &x, Vector &y) const;
+
+//    virtual ~FE_Evolution() { }
+// };
+
+class FE_Evolution : public TimeDependentOperator
+{
+private:
+   OperatorHandle &M, &K;
+   const Vector &b;
+
+   mutable Vector z;
+
+public:
+   FE_Evolution(OperatorHandle &_M, OperatorHandle &_K, const Vector &_b);
+
+   virtual void Mult(const Vector &x, Vector &y) const;
+
+   virtual ~FE_Evolution() { }
+};
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   problem = 0;
+   const char *mesh_file = "../data/periodic-hexagon.mesh";
+   int ref_levels = 2;
+   int order = 3;
+   int ode_solver_type = 4;
+   double t_final = 10.0;
+   double dt = 0.01;
+   bool visualization = true;
+   bool visit = false;
+   bool binary = false;
+   int vis_steps = 5;
+
+   int precision = 8;
+   cout.precision(precision);
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&problem, "-p", "--problem",
+                  "Problem setup to use. See options in velocity_function().");
+   args.AddOption(&ref_levels, "-r", "--refine",
+                  "Number of times to refine the mesh uniformly.");
+   args.AddOption(&order, "-o", "--order",
+                  "Order (degree) of the finite elements.");
+   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
+                  "ODE solver: 1 - Forward Euler,\n\t"
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
+   args.AddOption(&t_final, "-tf", "--t-final",
+                  "Final time; start time is 0.");
+   args.AddOption(&dt, "-dt", "--time-step",
+                  "Time step.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
+                  "--no-visit-datafiles",
+                  "Save data files for VisIt (visit.llnl.gov) visualization.");
+   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
+                  "--ascii-datafiles",
+                  "Use binary (Sidre) or ascii format for VisIt data files.");
+   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
+                  "Visualize every n-th timestep.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   //Engine stuff
+   string pa_spec("Hello world");
+   SharedPtr<Engine> engine(new mfem::pa::Engine());
+
+
+   // 2. Read the mesh from the given mesh file. We can handle geometrically
+   //    periodic meshes in this code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   mesh->SetEngine(*engine);
+   int dim = mesh->Dimension();
+
+   // 3. Define the ODE solver used for time integration. Several explicit
+   //    Runge-Kutta methods are available.
+   ODESolver *ode_solver = NULL;
+   switch (ode_solver_type)
+   {
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(1.0); break;
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      case 6: ode_solver = new RK6Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }
+
+   // 4. Refine the mesh to increase the resolution. In this example we do
+   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
+   //    command-line parameter. If the mesh is of NURBS type, we convert it to
+   //    a (piecewise-polynomial) high-order mesh.
+   for (int lev = 0; lev < ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+   if (mesh->NURBSext)
+   {
+      mesh->SetCurvature(max(order, 1));
+   }
+   mesh->GetBoundingBox(bb_min, bb_max, max(order, 1));
+
+   // 5. Define the discontinuous DG finite element space of the given
+   //    polynomial order on the refined mesh.
+   DG_FECollection fec(order, dim);
+   FiniteElementSpace fes(mesh, &fec);
+
+   cout << "Number of unknowns: " << fes.GetVSize() << endl;
+
+   tic_toc.Clear();
+   tic_toc.Start();
+
+   // 6. Set up and assemble the bilinear and linear forms corresponding to the
+   //    DG discretization. The DGTraceIntegrator involves integrals over mesh
+   //    interior faces.
+   VectorFunctionCoefficient velocity(dim, velocity_function);
+   FunctionCoefficient inflow(inflow_function);
+   FunctionCoefficient u0(u0_function);
+
+   BilinearForm m(&fes);
+   m.AddDomainIntegrator(new MassIntegrator(NULL));
+   BilinearForm k(&fes);
+   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
+   k.AddInteriorFaceIntegrator(
+      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+   Vector d_e;
+   k.AddInteriorFaceIntegrator(new FCTIntegrator(velocity,d_e,1.0,-0.5));
+   k.AddBdrFaceIntegrator(
+      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+
+   LinearForm b(&fes);
+   b.AddBdrFaceIntegrator(
+      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));
+
+   m.Assemble();
+   // m.Finalize();
+   k.Assemble();
+   // int skip_zeros = 0;
+   // k.Assemble(skip_zeros);
+   // k.Finalize(skip_zeros);
+   b.Assemble();
+   
+   // 7. Define the initial conditions, save the corresponding grid function to
+   //    a file and (optionally) save data in the VisIt format and initialize
+   //    GLVis visualization.
+   GridFunction u(&fes);
+   u.ProjectCoefficient(u0);
+   // u.Fill(0.0);
+
+   // 8.
+   Array<int> ess_tdof_list;
+   OperatorHandle K(Operator::ANY_TYPE);
+   Vector B, X;
+   k.FormLinearSystem(ess_tdof_list, u, b, K, X, B);
+   OperatorHandle M(Operator::ANY_TYPE);
+   m.FormSystemMatrix(ess_tdof_list, M);
+
+   tic_toc.Stop();
+   cout << " Initialization time: " << tic_toc.RealTime() << "s." << endl;
+
+   {
+      ofstream omesh("ex9.mesh");
+      omesh.precision(precision);
+      mesh->Print(omesh);
+      ofstream osol("ex9-init.gf");
+      osol.precision(precision);
+      u.Save(osol);
+   }
+
+   // Create data collection for solution output: either VisItDataCollection for
+   // ascii data files, or SidreDataCollection for binary data files.
+   DataCollection *dc = NULL;
+   if (visit)
+   {
+      if (binary)
+      {
+#ifdef MFEM_USE_SIDRE
+         dc = new SidreDataCollection("Example9", mesh);
+#else
+         MFEM_ABORT("Must build with MFEM_USE_SIDRE=YES for binary output.");
+#endif
+      }
+      else
+      {
+         dc = new VisItDataCollection("Example9", mesh);
+         dc->SetPrecision(precision);
+      }
+      dc->RegisterField("solution", &u);
+      dc->SetCycle(0);
+      dc->SetTime(0.0);
+      dc->Save();
+   }
+
+   socketstream sout;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      sout.open(vishost, visport);
+      if (!sout)
+      {
+         cout << "Unable to connect to GLVis server at "
+              << vishost << ':' << visport << endl;
+         visualization = false;
+         cout << "GLVis visualization disabled.\n";
+      }
+      else
+      {
+         sout.precision(precision);
+         sout << "solution\n" << *mesh << u;
+         sout << "pause\n";
+         sout << flush;
+         cout << "GLVis visualization paused."
+              << " Press space (in the GLVis window) to resume it.\n";
+      }
+   }
+
+   // 8. Define the time-dependent evolution operator describing the ODE
+   //    right-hand side, and perform time-integration (looping over the time
+   //    iterations, ti, with a time-step dt).
+   FE_Evolution adv(M, K, B);
+
+   double t = 0.0;
+   adv.SetTime(t);
+   ode_solver->Init(adv);
+
+   tic_toc.Clear();
+   tic_toc.Start();
+
+   bool done = false;
+   for (int ti = 0; !done; )
+   {
+      double dt_real = min(dt, t_final - t);
+      ode_solver->Step(u, t, dt_real);
+      ti++;
+
+      done = (t >= t_final - 1e-8*dt);
+
+      if (done || ti % vis_steps == 0)
+      {
+         cout << "time step: " << ti << ", time: " << t << endl;
+
+         if (visualization)
+         {
+            sout << "solution\n" << *mesh << u << flush;
+         }
+
+         if (visit)
+         {
+            dc->SetCycle(ti);
+            dc->SetTime(t);
+            dc->Save();
+         }
+      }
+   }
+
+   tic_toc.Stop();
+   cout << " done, " << tic_toc.RealTime() << "s." << endl;
+   
+   // 9. Save the final solution. This output can be viewed later using GLVis:
+   //    "glvis -m ex9.mesh -g ex9-final.gf".
+   {
+      ofstream osol("ex9-final.gf");
+      osol.precision(precision);
+      u.Save(osol);
+   }
+
+   // 10. Free the used memory.
+   delete ode_solver;
+   delete dc;
+
+   return 0;
+}
+
+
+// Implementation of class FE_Evolution
+// FE_Evolution::FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b)
+//    : TimeDependentOperator(_M.Size()), M(_M), K(_K), b(_b), z(_M.Size())
+// {
+//    M_solver.SetPreconditioner(M_prec);
+//    M_solver.SetOperator(M);
+
+//    M_solver.iterative_mode = false;
+//    M_solver.SetRelTol(1e-9);
+//    M_solver.SetAbsTol(0.0);
+//    M_solver.SetMaxIter(100);
+//    M_solver.SetPrintLevel(0);
+// }
+FE_Evolution::FE_Evolution(OperatorHandle &_M, OperatorHandle &_K, const Vector &_b)
+   : TimeDependentOperator(*_K.Ptr()), M(_M), K(_K), b(_b), z(_b,false) 
+{
+}
+
+void FE_Evolution::Mult(const Vector &x, Vector &y) const
+{
+   // y = M^{-1} (K x + b)
+   K.Ptr()->Mult(x, z);
+   // z += b;
+   z.Axpby(1.0, z, 1.0, b);
+   // M.Mult(z, y);
+   CG(*M.Ptr(), z, y, 0, 1000, 1e-12, 0.0);
+}
+
+
+// Velocity coefficient
+void velocity_function(const Vector &x, Vector &v)
+{
+   int dim = x.Size();
+
+   // map to the reference [-1,1] domain
+   Vector X(dim);
+   for (int i = 0; i < dim; i++)
+   {
+      double center = (bb_min[i] + bb_max[i]) * 0.5;
+      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
+   }
+
+   switch (problem)
+   {
+      case 0:
+      {
+         // Translations in 1D, 2D, and 3D
+         switch (dim)
+         {
+            case 1: v(0) = 1.0; break;
+            case 2: v(0) = sqrt(2./3.); v(1) = sqrt(1./3.); break;
+            case 3: v(0) = sqrt(3./6.); v(1) = sqrt(2./6.); v(2) = sqrt(1./6.);
+               break;
+         }
+         break;
+      }
+      case 1:
+      case 2:
+      {
+         // Clockwise rotation in 2D around the origin
+         const double w = M_PI/2;
+         switch (dim)
+         {
+            case 1: v(0) = 1.0; break;
+            case 2: v(0) = w*X(1); v(1) = -w*X(0); break;
+            case 3: v(0) = w*X(1); v(1) = -w*X(0); v(2) = 0.0; break;
+         }
+         break;
+      }
+      case 3:
+      {
+         // Clockwise twisting rotation in 2D around the origin
+         const double w = M_PI/2;
+         double d = max((X(0)+1.)*(1.-X(0)),0.) * max((X(1)+1.)*(1.-X(1)),0.);
+         d = d*d;
+         switch (dim)
+         {
+            case 1: v(0) = 1.0; break;
+            case 2: v(0) = d*w*X(1); v(1) = -d*w*X(0); break;
+            case 3: v(0) = d*w*X(1); v(1) = -d*w*X(0); v(2) = 0.0; break;
+         }
+         break;
+      }
+   }
+}
+
+// Initial condition
+double u0_function(const Vector &x)
+{
+   int dim = x.Size();
+
+   // map to the reference [-1,1] domain
+   Vector X(dim);
+   for (int i = 0; i < dim; i++)
+   {
+      double center = (bb_min[i] + bb_max[i]) * 0.5;
+      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
+   }
+
+   switch (problem)
+   {
+      case 0:
+      case 1:
+      {
+         switch (dim)
+         {
+            case 1:
+               return exp(-40.*pow(X(0)-0.5,2));
+            case 2:
+            case 3:
+            {
+               double rx = 0.45, ry = 0.25, cx = 0., cy = -0.2, w = 10.;
+               if (dim == 3)
+               {
+                  const double s = (1. + 0.25*cos(2*M_PI*X(2)));
+                  rx *= s;
+                  ry *= s;
+               }
+               return ( erfc(w*(X(0)-cx-rx))*erfc(-w*(X(0)-cx+rx)) *
+                        erfc(w*(X(1)-cy-ry))*erfc(-w*(X(1)-cy+ry)) )/16;
+            }
+         }
+      }
+      case 2:
+      {
+         double x_ = X(0), y_ = X(1), rho, phi;
+         rho = hypot(x_, y_);
+         phi = atan2(y_, x_);
+         return pow(sin(M_PI*rho),2)*sin(3*phi);
+      }
+      case 3:
+      {
+         const double f = M_PI;
+         return sin(f*X(0))*sin(f*X(1));
+      }
+   }
+   return 0.0;
+}
+
+// Inflow boundary condition (zero for the problems considered in this example)
+double inflow_function(const Vector &x)
+{
+   switch (problem)
+   {
+      case 0:
+      case 1:
+      case 2:
+      case 3: return 0.0;
+   }
+   return 0.0;
+}

fem/bilinearform.cpp

@@ -74,6 +74,12 @@ BilinearForm::BilinearForm (FiniteElementSpace * f)
    hybridization = NULL;
    precompute_sparsity = 0;
    diag_policy = DIAG_KEEP;
+#ifdef MFEM_USE_BACKENDS
+   if (fes->GetVLayout()->HasEngine())
+   {
+      dev_ext = fes->GetVLayout()->GetEngine().MakeBilinearForm(*this);
+   }
+#endif
 }
 
 BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
@@ -126,6 +132,14 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
 void BilinearForm::EnableStaticCondensation()
 {
    delete static_cond;
+#ifdef MFEM_USE_BACKENDS
+   if (fes->GetVLayout()->HasEngine())
+   {
+      static_cond = NULL;
+      MFEM_WARNING("Engine interface does not support static condensation yet");
+      return;
+   }
+#endif
    static_cond = new StaticCondensation(fes);
    if (static_cond->ReducesTrueVSize())
    {
@@ -145,6 +159,15 @@ void BilinearForm::EnableHybridization(FiniteElementSpace *constr_space,
                                        const Array<int> &ess_tdof_list)
 {
    delete hybridization;
+#ifdef MFEM_USE_BACKENDS
+   if (fes->GetVLayout()->HasEngine())
+   {
+      delete constr_integ;
+      hybridization = NULL;
+      MFEM_WARNING("Engine interface does not support hybridization yet");
+      return;
+   }
+#endif
    hybridization = new Hybridization(fes, constr_space);
    hybridization->SetConstraintIntegrator(constr_integ);
    hybridization->Init(ess_tdof_list);
@@ -313,7 +336,15 @@ void BilinearForm::Assemble (int skip_zeros)
    Mesh *mesh = fes -> GetMesh();
    DenseMatrix elmat, *elmat_p;
 
-   int i;
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      // TODO: push the 'skip_zeros' as a parameter to 'dev_ext'
+
+      const bool assembly_done = dev_ext->Assemble();
+      if (assembly_done) { return; }
+   }
+#endif
 
    if (mat == NULL)
    {
@@ -331,7 +362,7 @@ void BilinearForm::Assemble (int skip_zeros)
 
    if (dbfi.Size())
    {
-      for (i = 0; i < fes -> GetNE(); i++)
+      for (int i = 0; i < fes -> GetNE(); i++)
       {
          fes->GetElementVDofs(i, vdofs);
          if (element_matrices)
@@ -388,7 +419,7 @@ void BilinearForm::Assemble (int skip_zeros)
          }
       }
 
-      for (i = 0; i < fes -> GetNBE(); i++)
+      for (int i = 0; i < fes -> GetNBE(); i++)
       {
          const int bdr_attr = mesh->GetBdrAttribute(i);
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
@@ -426,7 +457,7 @@ void BilinearForm::Assemble (int skip_zeros)
       Array<int> vdofs2;
 
       int nfaces = mesh->GetNumFaces();
-      for (i = 0; i < nfaces; i++)
+      for (int i = 0; i < nfaces; i++)
       {
          tr = mesh -> GetInteriorFaceTransformations (i);
          if (tr != NULL)
@@ -471,7 +502,7 @@ void BilinearForm::Assemble (int skip_zeros)
          }
       }
 
-      for (i = 0; i < fes -> GetNBE(); i++)
+      for (int i = 0; i < fes -> GetNBE(); i++)
       {
          const int bdr_attr = mesh->GetBdrAttribute(i);
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
@@ -643,9 +674,91 @@ void BilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
    }
 }
 
+void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
+                                    Vector &x, Vector &b,
+                                    OperatorHandle &A, Vector &X, Vector &B,
+                                    int copy_interior)
+{
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      MFEM_VERIFY(!static_cond && !hybridization, "");
+      dev_ext->FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
+   }
+   else
+#endif
+   {
+      if (A.Type() == Operator::MFEM_SPARSEMAT)
+      {
+         SparseMatrix A_sm;
+         FormLinearSystem(ess_tdof_list, x, b, A_sm, X, B, copy_interior);
+         if (static_cond)
+         {
+            A.Reset(&static_cond->GetMatrix(), false);
+         }
+         else if (hybridization)
+         {
+            A.Reset(&hybridization->GetMatrix(), false);
+         }
+         else
+         {
+            A.Reset(mat, false);
+         }
+      }
+      else
+      {
+         MFEM_ABORT("Operator::Type is not supported: type_id = " << A.Type());
+      }
+   }
+}
+
+void BilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                    OperatorHandle &A)
+{
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      MFEM_VERIFY(!static_cond && !hybridization, "");
+      dev_ext->FormSystemMatrix(ess_tdof_list, A);
+   }
+   else
+#endif
+   {
+      if (A.Type() == Operator::MFEM_SPARSEMAT)
+      {
+         SparseMatrix A_sm;
+         FormSystemMatrix(ess_tdof_list, A_sm);
+         if (static_cond)
+         {
+            A.Reset(&static_cond->GetMatrix(), false);
+         }
+         else if (hybridization)
+         {
+            A.Reset(&hybridization->GetMatrix(), false);
+         }
+         else
+         {
+            A.Reset(mat, false);
+         }
+      }
+      else
+      {
+         MFEM_ABORT("Operator::Type is not supported: type_id = " << A.Type());
+      }
+   }
+}
+
 void BilinearForm::RecoverFEMSolution(const Vector &X,
                                       const Vector &b, Vector &x)
 {
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      dev_ext->RecoverFEMSolution(X, b, x);
+      return;
+   }
+#endif
+
    const SparseMatrix *P = fes->GetConformingProlongation();
    if (!P) // conforming space
    {
@@ -734,7 +847,8 @@ void BilinearForm::ComputeElementMatrices()
 }
 
 void BilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                                        Vector &sol, Vector &rhs, DiagonalPolicy dpolicy)
+                                        Vector &sol, Vector &rhs,
+                                        DiagonalPolicy dpolicy)
 {
    Array<int> ess_dofs, conf_ess_dofs;
    fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);

fem/bilinearform.hpp

@@ -38,6 +38,11 @@ protected:
    /// FE space on which the form lives.
    FiniteElementSpace *fes;
 
+#ifdef MFEM_USE_BACKENDS
+   /// Device/Engine extension (smart shared pointer)
+   DBilinearForm dev_ext;
+#endif
+
    /// Indicates the Mesh::sequence corresponding to the current state of the
    /// BilinearForm.
    long sequence;
@@ -285,6 +290,37 @@ public:
    /// Form the linear system matrix A, see FormLinearSystem() for details.
    void FormSystemMatrix(const Array<int> &ess_tdof_list, SparseMatrix &A);
 
+   /** Form the linear system @a A @a X = @a B, corresponding to the bilinear
+       form and the r.h.s. linear form (vector) @a b, by applying any necessary
+       transformations such as: eliminating boundary conditions; applying
+       conforming constraints for non-conforming AMR; parallel assembly; static
+       condensation; hybridization.
+
+       The GridFunction-size vector @a x must contain the essential b.c. The
+       BilinearForm and the LinearForm-size vector @a b must be assembled.
+
+       The vector @a X is initialized with a suitable initial guess: when using
+       hybridization, the vector @a X is set to zero; otherwise, the essential
+       entries of @a X are set to the corresponding b.c. and all other entries
+       are set to zero (if @a copy_interior == 0) or copied from @a x (if
+       @a copy_interior != 0).
+
+       This method can be called multiple times (with the same @a ess_tdof_list
+       array) to initialize different right-hand sides and boundary condition
+       values.
+
+       After solving the linear system, the finite element solution @a x can be
+       recovered by calling RecoverFEMSolution() (with the same vectors @a X,
+       @a b, and @a x). */
+   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
+                                 Vector &x, Vector &b,
+                                 OperatorHandle &A, Vector &X, Vector &B,
+                                 int copy_interior = 0);
+
+   /// Form the linear system matrix @a A, see FormLinearSystem() for details.
+   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                 OperatorHandle &A);
+
    /// Recover the solution of a linear system formed with FormLinearSystem().
    /** Call this method after solving a linear system constructed using the
        FormLinearSystem() method to recover the solution as a GridFunction-size

fem/bilininteg.cpp

@@ -361,6 +361,28 @@ void MixedScalarVectorIntegrator::AssembleElementMatrix2(
    }
 }
 
+
+const IntegrationRule &DiffusionIntegrator::GetRule(
+   const FiniteElement &trial_fe, const FiniteElement &test_fe)
+{
+   int order;
+   if (trial_fe.Space() == FunctionSpace::Pk)
+   {
+      order = trial_fe.GetOrder() + test_fe.GetOrder() - 2;
+   }
+   else
+   {
+      // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
+      order = trial_fe.GetOrder() + test_fe.GetOrder() + trial_fe.GetDim() - 1;
+   }
+
+   if (trial_fe.Space() == FunctionSpace::rQk)
+   {
+      return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
+   }
+   return IntRules.Get(trial_fe.GetGeomType(), order);
+}
+
 void DiffusionIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
   DenseMatrix &elmat )
@@ -383,25 +405,7 @@ void DiffusionIntegrator::AssembleElementMatrix
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int order;
-      if (el.Space() == FunctionSpace::Pk)
-      {
-         order = 2*el.GetOrder() - 2;
-      }
-      else
-         // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
-      {
-         order = 2*el.GetOrder() + dim - 1;
-      }
-
-      if (el.Space() == FunctionSpace::rQk)
-      {
-         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
-      }
-      else
-      {
-         ir = &IntRules.Get(el.GetGeomType(), order);
-      }
+      ir = &GetRule(el, el);
    }
 
    elmat = 0.0;
@@ -461,24 +465,7 @@ void DiffusionIntegrator::AssembleElementMatrix2(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int order;
-      if (trial_fe.Space() == FunctionSpace::Pk)
-      {
-         order = trial_fe.GetOrder() + test_fe.GetOrder() - 2;
-      }
-      else
-      {
-         order = trial_fe.GetOrder() + test_fe.GetOrder() + dim - 1;
-      }
-
-      if (trial_fe.Space() == FunctionSpace::rQk)
-      {
-         ir = &RefinedIntRules.Get(trial_fe.GetGeomType(), order);
-      }
-      else
-      {
-         ir = &IntRules.Get(trial_fe.GetGeomType(), order);
-      }
+      ir = &GetRule(trial_fe, test_fe);
    }
 
    elmat = 0.0;
@@ -717,6 +704,22 @@ double DiffusionIntegrator::ComputeFluxEnergy
 }
 
 
+const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
+                                               const FiniteElement &test_fe,
+                                               ElementTransformation &Trans,
+                                               int coeff_order)
+{
+   // int order = trial_fe.GetOrder() + test_fe.GetOrder();
+   int order = trial_fe.GetOrder() + test_fe.GetOrder() +
+               Trans.OrderW() + coeff_order;
+
+   if (trial_fe.Space() == FunctionSpace::rQk)
+   {
+      return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
+   }
+   return IntRules.Get(trial_fe.GetGeomType(), order);
+}
+
 void MassIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
   DenseMatrix &elmat )
@@ -734,17 +737,7 @@ void MassIntegrator::AssembleElementMatrix
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      // int order = 2 * el.GetOrder();
-      int order = 2 * el.GetOrder() + Trans.OrderW();
-
-      if (el.Space() == FunctionSpace::rQk)
-      {
-         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
-      }
-      else
-      {
-         ir = &IntRules.Get(el.GetGeomType(), order);
-      }
+      ir = &GetRule(el, el, Trans);
    }
 
    elmat = 0.0;
@@ -783,9 +776,7 @@ void MassIntegrator::AssembleElementMatrix2(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
-
-      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+      ir = &GetRule(trial_fe, test_fe, Trans);
    }
 
    elmat = 0.0;
@@ -980,16 +971,7 @@ void VectorMassIntegrator::AssembleElementMatrix
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int order = 2 * el.GetOrder() + Trans.OrderW() + Q_order;
-
-      if (el.Space() == FunctionSpace::rQk)
-      {
-         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
-      }
-      else
-      {
-         ir = &IntRules.Get(el.GetGeomType(), order);
-      }
+      ir = &MassIntegrator::GetRule(el, el, Trans, Q_order);
    }
 
    elmat = 0.0;
@@ -1065,17 +1047,7 @@ void VectorMassIntegrator::AssembleElementMatrix2(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int order = (trial_fe.GetOrder() + test_fe.GetOrder() +
-                   Trans.OrderW() + Q_order);
-
-      if (trial_fe.Space() == FunctionSpace::rQk)
-      {
-         ir = &RefinedIntRules.Get(trial_fe.GetGeomType(), order);
-      }
-      else
-      {
-         ir = &IntRules.Get(trial_fe.GetGeomType(), order);
-      }
+      ir = &MassIntegrator::GetRule(trial_fe, test_fe, Trans, Q_order);
    }
 
    elmat = 0.0;

fem/bilininteg.hpp

@@ -18,6 +18,7 @@
 namespace mfem
 {
 
+
 /// Abstract base class BilinearFormIntegrator
 class BilinearFormIntegrator : public NonlinearFormIntegrator
 {
@@ -83,6 +84,7 @@ public:
    virtual ~BilinearFormIntegrator() { }
 };
 
+
 class TransposeIntegrator : public BilinearFormIntegrator
 {
 private:
@@ -110,6 +112,10 @@ public:
                                    FaceElementTransformations &Trans,
                                    DenseMatrix &elmat);
 
+   void GetParameters(BilinearFormIntegrator*& _bfi){_bfi=bfi;}
+
+   virtual const char *Name() const { return "transpose"; }
+
    virtual ~TransposeIntegrator() { if (own_bfi) { delete bfi; } }
 };
 
@@ -1622,6 +1628,15 @@ public:
    virtual double ComputeFluxEnergy(const FiniteElement &fluxelem,
                                     ElementTransformation &Trans,
                                     Vector &flux, Vector *d_energy = NULL);
+
+   virtual const char *Name() const { return "diffusion"; }
+
+   virtual Coefficient *GetScalarCoefficient() const { return Q; }
+
+   void GetParameters(Coefficient*& coef) const { coef = Q; }
+
+   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
+                                         const FiniteElement &test_fe);
 };
 
 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
@@ -1649,6 +1664,39 @@ public:
                                        const FiniteElement &test_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
+
+   virtual const char *Name() const { return "mass"; }
+
+   virtual Coefficient *GetScalarCoefficient() const { return Q; }
+
+   void GetParameters(Coefficient*& coef) const { coef = Q; }
+
+   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
+                                         const FiniteElement &test_fe,
+                                         ElementTransformation &Trans,
+                                         int coeff_order = 0);
+};
+
+class FCTIntegrator : public BilinearFormIntegrator
+{
+private:
+   VectorCoefficient* q;
+   Vector* d_e;
+   double a;
+   double b;
+public:
+   FCTIntegrator(VectorCoefficient& q, Vector& d_e, const double a, const double b, const IntegrationRule* ir = NULL)
+   : BilinearFormIntegrator(ir),
+     q(&q), d_e(&d_e), a(a), b(b) { }
+
+   virtual const char *Name() const { return "fct"; }
+
+   void GetParameters(VectorCoefficient*& q_ptr, Vector*& d_e_ptr, double*& a_ptr, double*& b_ptr){
+     q_ptr = q;
+     d_e_ptr = d_e;
+     a_ptr = &a;
+     b_ptr = &b;
+   }
 };
 
 class BoundaryMassIntegrator : public MassIntegrator
@@ -1681,6 +1729,14 @@ public:
    virtual void AssembleElementMatrix(const FiniteElement &,
                                       ElementTransformation &,
                                       DenseMatrix &);
+
+   virtual const char *Name() const { return "convection"; }
+
+   void GetParameters(VectorCoefficient*& vcoef, double*& a)
+   {
+    vcoef = &Q;
+    a = α
+   }
 };
 
 /// alpha (q . grad u, v) using the "group" FE discretization
@@ -2066,6 +2122,16 @@ public:
                                    const FiniteElement &el2,
                                    FaceElementTransformations &Trans,
                                    DenseMatrix &elmat);
+
+   virtual const char *Name() const { return "dgtrace"; }
+
+   void GetParameters(Coefficient*& _rho, VectorCoefficient*& _u, double*& _alpha, double*& _beta)
+   {
+    _rho   = rho;
+    _u     = u;
+    _alpha = α
+    _beta  = β
+   }
 };
 
 /** Integrator for the DG form:

fem/fe_coll.cpp

@@ -165,8 +165,10 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
                                 BasisType::GetType(name[3]));
    }
    else if (!strncmp(name, "L2_T", 4))
+   {
       fec = new L2_FECollection(atoi(name + 10), atoi(name + 6),
                                 atoi(name + 4));
+   }
    else if (!strncmp(name, "L2_", 3))
    {
       fec = new L2_FECollection(atoi(name + 7), atoi(name + 3));

fem/fespace.cpp

@@ -377,6 +377,12 @@ void FiniteElementSpace::GetEssentialTrueDofs(const Array<int> &bdr_attr_is_ess,
    {
       R->BooleanMult(ess_vdofs, ess_tdofs);
    }
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      ess_tdof_list.SetEngine(dev_ext->GetEngine());
+   }
+#endif
    MarkerToList(ess_tdofs, ess_tdof_list);
 }
 
@@ -389,12 +395,14 @@ void FiniteElementSpace::MarkerToList(const Array<int> &marker,
    {
       if (marker[i]) { num_marked++; }
    }
+   list.Resize(num_marked);
+   list.Pull(false);
    list.SetSize(0);
-   list.Reserve(num_marked);
    for (int i = 0; i < marker.Size(); i++)
    {
       if (marker[i]) { list.Append(i); }
    }
+   list.Push();
 }
 
 // static method
@@ -642,9 +650,23 @@ void FiniteElementSpace::BuildConformingInterpolation() const
    if (n_true_dofs == ndofs)
    {
       cP = cR = NULL; // will be treated as identities
+#ifdef MFEM_USE_BACKENDS
+      t_layout = v_layout;
+#endif
       return;
    }
 
+#ifdef MFEM_USE_BACKENDS
+   if (mesh->HasEngine())
+   {
+      t_layout = mesh->GetEngine().MakeLayout(n_true_dofs*vdim);
+   }
+   else
+   {
+      t_layout.Reset(new PLayout(n_true_dofs*vdim));
+   }
+#endif
+
    // create the conforming restriction matrix cR
    int *cR_J;
    {
@@ -757,27 +779,6 @@ void FiniteElementSpace::MakeVDimMatrix(SparseMatrix &mat) const
    delete vmat;
 }
 
-
-const SparseMatrix* FiniteElementSpace::GetConformingProlongation() const
-{
-   if (Conforming()) { return NULL; }
-   if (!cP_is_set) { BuildConformingInterpolation(); }
-   return cP;
-}
-
-const SparseMatrix* FiniteElementSpace::GetConformingRestriction() const
-{
-   if (Conforming()) { return NULL; }
-   if (!cP_is_set) { BuildConformingInterpolation(); }
-   return cR;
-}
-
-int FiniteElementSpace::GetNConformingDofs() const
-{
-   const SparseMatrix* P = GetConformingProlongation();
-   return P ? (P->Width() / vdim) : ndofs;
-}
-
 SparseMatrix *FiniteElementSpace::RefinementMatrix_main(
    const int coarse_ndofs, const Table &coarse_elem_dof,
    const DenseTensor &localP) const
@@ -1108,7 +1109,7 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
    {
       if (!mesh->NURBSext)
       {
-         mfem_error("FiniteElementSpace::FiniteElementSpace :\n"
+         mfem_error("FiniteElementSpace::Constructor :\n"
                     "   NURBS FE space requires NURBS mesh.");
       }
 
@@ -1124,7 +1125,7 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
       }
       UpdateNURBS();
       cP = cR = NULL;
-      cP_is_set = false;
+      cP_is_set = true;
    }
    else
    {
@@ -1133,6 +1134,22 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
       Construct();
    }
    BuildElementToDofTable();
+
+#ifdef MFEM_USE_BACKENDS
+   if (mesh->HasEngine())
+   {
+      v_layout = mesh->GetEngine().MakeLayout(GetVSize());
+      if (cP_is_set) { t_layout = v_layout; }
+      // Ensure GetVLayout() and GetTrueVLayout() will work correctly before
+      // calling MakeFESpace().
+      dev_ext = mesh->GetEngine().MakeFESpace(*this);
+   }
+   else
+   {
+      v_layout.Reset(new PLayout(GetVSize()));
+      if (cP_is_set) { t_layout = v_layout; }
+   }
+#endif
 }
 
 NURBSExtension *FiniteElementSpace::StealNURBSext()
@@ -1160,10 +1177,14 @@ void FiniteElementSpace::UpdateNURBS()
    ndofs = NURBSext->GetNDof();
    elem_dof = NURBSext->GetElementDofTable();
    bdrElem_dof = NURBSext->GetBdrElementDofTable();
+
+   // TODO: update v_layout, t_layout
 }
 
 void FiniteElementSpace::Construct()
 {
+   // called in parallel by ParFiniteElementSpace::Update()
+
    // This method should be used only for non-NURBS spaces.
    MFEM_ASSERT(!NURBSext, "internal error");
 
@@ -1188,7 +1209,7 @@ void FiniteElementSpace::Construct()
    fdofs = NULL;
    cP = NULL;
    cR = NULL;
-   cP_is_set = false;
+   cP_is_set = Conforming();
    // Th is initialized/destroyed before this method is called.
 
    if (mesh->Dimension() == 3 && mesh->GetNE())
@@ -1635,6 +1656,10 @@ FiniteElementSpace::~FiniteElementSpace()
 
 void FiniteElementSpace::Destroy()
 {
+   // called in parallel by ParFiniteElementSpace::Update()
+
+   // For now, do not reset dev_ext and/or v_layout, t_layout
+
    delete cR;
    delete cP;
    Th.Clear();