Include "mpi-ext.h" only if using OpenMPI.

pointers in MPI calls, regardless of the result of the
CUDA-aware MPI check.

backends/all.hpp

@@ -0,0 +1,27 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_ALL_HPP
+#define MFEM_BACKENDS_ALL_HPP
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_BACKENDS
+
+#include "base/backend.hpp"
+
+#ifdef MFEM_USE_OCCA
+#include "occa/backend.hpp"
+#endif
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_ALL_HPP

backends/base/array.hpp

@@ -0,0 +1,213 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_ARRAY_HPP
+#define MFEM_BACKENDS_BASE_ARRAY_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "layout.hpp"
+#include "utils.hpp"
+
+namespace mfem
+{
+
+/// Extension to the template class Array<T>
+class PArray : public RefCounted
+{
+protected:
+   /// Layout with shared ownership (smart pointer)
+   DLayout layout;
+
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new array (in @a *clone) of the same dynamic
+       type as this array using the same layout and ItemSize().
+
+       Set @a *clone to NULL if allocation fails.
+
+       If @a copy_data is true, the contents of this array is copied to the new
+       array; otherwise, the new array remains uninitialized.
+
+       If @a buffer is not NULL, return the array data of the newly created
+       object (in @a *buffer) , if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   virtual PArray *DoClone(bool copy_data, void **buffer,
+                           std::size_t item_size) const = 0;
+
+   /// Resize the array, reallocating its data if necessary.
+   /** If @a buffer is not NULL, return the array data (in @a *buffer), if it
+       is stored as a contiguous array on the host; otherwise, set @a *buffer to
+       NULL. Returns 0 on success and non-zero otherwise, e.g. if memory
+       allocation fails.
+
+       If the @a new_layout is not supported, a non-zero error code will be
+       returned.
+
+       The @a new_layout has to be valid, i.e. new_layout != NULL and
+       new_layout->HasEngine() == true.
+
+       @note If reallocation is performed, the previous content of the array is
+       NOT copied to the new location. */
+   virtual int DoResize(PLayout &new_layout, void **buffer,
+                        std::size_t item_size) = 0;
+
+   /** @brief Get access to the contents of the array in host memory, as a
+       contiguous array. */
+   /** If the array data is stored as a contiguous array in host memory, return
+       a pointer to it. Otherwise, copy the data to @a buffer (if @a buffer is
+       not NULL) and return @a buffer.
+       @note If not NULL, @a buffer is assumed to be of size greater than or
+       equal to Size(). */
+   virtual void *DoPullData(void *buffer, std::size_t item_size) = 0;
+
+   /** @brief Set all entries of the array to the (single) value pointed to by
+       @a value_ptr. */
+   virtual void DoFill(const void *value_ptr, std::size_t item_size) = 0;
+
+   /** @brief Set all Size() entries of the array from the given contiguous
+       array, @a src_buffer, on the host. */
+   virtual void DoPushData(const void *src_buffer, std::size_t item_size) = 0;
+
+   /// Copy the data from @a src to @a *this.
+   /** Both arrays must have the same dynamic type, layout, and item_size. */
+   virtual void DoAssign(const PArray &src, std::size_t item_size) = 0;
+
+   ///@}
+   // End: Virtual interface
+
+public:
+   /** @brief The @a layout parameter will be reference counted and therefore it
+       should be dynamically allocated. */
+   /** The @a layout must be valid in the sense that layout != NULL and
+       layout->HasEngine() == true. */
+   PArray(PLayout &p_layout)
+      : layout(&p_layout)
+   {
+      MFEM_ASSERT(layout && layout->HasEngine(), "invalid layout");
+   }
+
+   virtual ~PArray() { }
+
+   /// Get the current size of the array.
+   std::size_t Size() const { return layout->Size(); }
+
+   /// Get the current layout of the array.
+   PLayout &GetLayout() const { return *layout; }
+
+   /// TODO
+   /// Note: we cannot use static_cast for class PArray.
+   template <typename derived_t>
+   derived_t &As() { return dynamic_cast<derived_t&>(*this); }
+
+   /// TODO
+   /// Note: we cannot use static_cast for class PArray.
+   template <typename derived_t>
+   const derived_t &As() const { return dynamic_cast<const derived_t&>(*this); }
+
+
+   // TODO: Error handling ... handle errors at the Engine level, at the class
+   //       level, or at the method level?
+
+   // TODO: Asynchronous execution interface ...
+
+
+   /**
+       @name Public virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new array (in @a *clone) of the same dynamic
+       type as this array using the same layout and ItemSize().
+
+       Set @a *clone to NULL if allocation fails.
+
+       If @a copy_data is true, the contents of this array is copied to the new
+       array; otherwise, the new array remains uninitialized.
+
+       If @a buffer is not NULL, return the array data of the newly created
+       object (in @a *buffer) , if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   template <typename T>
+   DArray Clone(bool copy_data, T **buffer) const
+   { return DArray(DoClone(copy_data, (void**)buffer, sizeof(T))); }
+
+   /// Resize the array, reallocating its data if necessary.
+   /** If @a buffer is not NULL, return the array data (in @a *buffer), if it
+       is stored as a contiguous array on the host; otherwise, set @a *buffer to
+       NULL. Returns 0 on success and non-zero otherwise, e.g. if memory
+       allocation fails.
+
+       If the @a new_layout is not supported, a non-zero error code will be
+       returned.
+
+       The @a new_layout has to be valid, i.e. new_layout != NULL and
+       new_layout->HasEngine() == true.
+
+       @note If reallocation is performed, the previous content of the array is
+       NOT copied to the new location. */
+   template <typename T>
+   int Resize(PLayout &new_layout, T **buffer)
+   { return DoResize(new_layout, (void**)buffer, sizeof(T)); }
+
+   /// Shortcut for Resize(*layout, buffer).
+   /** This method is useful for updating the array after its layout is changed
+       externally. */
+   template <typename T>
+   int Update(T **buffer)
+   { return DoResize(*layout, (void**)buffer, sizeof(T)); }
+
+   /// Shortcut for layout->Resize(new_size) followed by Update()
+   template <typename T>
+   int Resize(std::size_t new_size, T **buffer)
+   { layout->Resize(new_size); return Update(buffer); }
+
+   /** @brief Get access to the contents of the array in host memory, as a
+       contiguous array. */
+   /** If the array data is stored as a contiguous array in host memory, return
+       a pointer to it. Otherwise, copy the data to @a buffer (if @a buffer is
+       not NULL) and return @a buffer.
+       @note If not NULL, @a buffer is assumed to be of size greater than or
+       equal to Size(). */
+   template <typename T>
+   T *PullData(T *buffer)
+   { return Size() ? (T*)DoPullData((void*)buffer, sizeof(T)) : NULL; }
+
+   /** @brief Set all entries of the array to the (single) value pointed to by
+       @a value_ptr. */
+   template <typename T>
+   void Fill(const T &value) { if (Size()) { DoFill(&value, sizeof(T)); } }
+
+   /** @brief Set all Size() entries of the array from the given contiguous
+       array, @a src_buffer, on the host. */
+   template <typename T>
+   void PushData(const T *src_buffer)
+   { if (Size()) { DoPushData(src_buffer, sizeof(T)); } }
+
+   /// Copy the data from @a src to @a *this.
+   /** Both arrays must have the same dynamic type, layout, and entry type. */
+   template <typename T>
+   void Assign(const PArray &src) { if (Size()) { DoAssign(src, sizeof(T)); } }
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_ARRAY_HPP

backends/base/backend.hpp

@@ -0,0 +1,57 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_BACKEND_HPP
+#define MFEM_BACKENDS_BASE_BACKEND_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "memory_resource.hpp"
+#include "engine.hpp"
+#include "array.hpp"
+#include "vector.hpp"
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+
+#include <string>
+
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#endif
+
+namespace mfem
+{
+
+/// TODO
+class Backend
+{
+public:
+   /// TODO
+   virtual ~Backend() { }
+
+   /// TODO
+   virtual bool Supports(const std::string &engine_spec) const = 0;
+
+   /// TODO
+   virtual Engine *Create(const std::string &engine_spec) = 0;
+
+#ifdef MFEM_USE_MPI
+   /// TODO
+   virtual Engine *Create(MPI_Comm comm, const std::string &engine_spec) = 0;
+#endif
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_BACKEND_HPP

backends/base/bilinearform.hpp

@@ -0,0 +1,72 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_BILINEARFORM_HPP
+#define MFEM_BACKENDS_BASE_BILINEARFORM_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "engine.hpp"
+
+namespace mfem
+{
+
+class Vector;
+class OperatorHandle;
+class BilinearForm;
+
+/// TODO: doxygen
+class PBilinearForm : public RefCounted
+{
+protected:
+   /// Engine with shared ownership
+   SharedPtr<const Engine> engine;
+   /// Not owned.
+   BilinearForm *bform;
+
+public:
+   /// TODO: doxygen
+   PBilinearForm(const Engine &e, BilinearForm &bf)
+      : engine(&e), bform(&bf) { }
+
+   /// Virtual destructor
+   virtual ~PBilinearForm() { }
+
+   /// Get the associated Engine
+   const Engine &GetEngine() const { return *engine; }
+
+   /// Assemble the PBilinearForm.
+   /** This method is called from the method BilinearForm::Assemble() of the
+       associated BilinearForm #bform.
+       @returns True, if the host assembly should be skipped. */
+   virtual bool Assemble() = 0;
+
+   /// TODO: doxygen
+   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                 OperatorHandle &A) = 0;
+
+   /// TODO: doxygen
+   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
+                                 Vector &x, Vector &b,
+                                 OperatorHandle &A, Vector &X, Vector &B,
+                                 int copy_interior) = 0;
+
+   /// TODO: doxygen
+   virtual void RecoverFEMSolution(const Vector &X, const Vector &b,
+                                   Vector &x) = 0;
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_BILINEARFORM_HPP

backends/base/engine.cpp

@@ -0,0 +1,49 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "engine.hpp"
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+
+namespace mfem
+{
+
+Engine::Engine(Backend *b, int n_mem, int n_workers)
+   : backend(b),
+#ifdef MFEM_USE_MPI
+     comm(MPI_COMM_NULL),
+#endif
+     num_mem_res(n_mem),
+     num_workers(n_workers),
+     memory_resources(new MemoryResource*[num_mem_res]()),
+     workers_weights(new double[num_workers]()),
+     workers_mem_res(new int[num_workers]())
+{
+   // Note: all arrays are value-initialized with zeros.
+}
+
+Engine::~Engine()
+{
+   delete [] workers_mem_res;
+   delete [] workers_weights;
+   for (int i = 0; i < num_mem_res; i++)
+   {
+      delete memory_resources[i];
+   }
+   delete [] memory_resources;
+}
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS

backends/base/engine.hpp

@@ -0,0 +1,190 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_ENGINE_HPP
+#define MFEM_BACKENDS_BASE_ENGINE_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "../../general/scalars.hpp"
+#include "memory_resource.hpp"
+#include "smart_pointers.hpp"
+#include "utils.hpp"
+
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#endif
+
+namespace mfem
+{
+
+// Forward declarations.
+class Backend;
+template <typename T> class Array;
+class Operator;
+class FiniteElementSpace;
+class LinearForm;
+class BilinearForm;
+class MixedBilinearForm;
+class NonlinearForm;
+
+
+/// In parallel, each MPI rank will usually create a single engine.
+class Engine : public RefCounted
+{
+protected:
+   Backend *backend; ///< Backend that created the engine. Not owned.
+
+#ifdef MFEM_USE_MPI
+   MPI_Comm comm; ///< Associated MPI communicator (may be MPI_COMM_NULL).
+#endif
+
+   /// Number of memory resources used by the Engine.
+   int num_mem_res;
+   /// Number of workers used by the Engine.
+   int num_workers;
+
+   /// Memory resources used by the engine - array of pointers.
+   /** Both the array and the entries are owned. */
+   MemoryResource **memory_resources;
+
+   /// Relative computational speed of the workers. Owned.
+   double *workers_weights;
+
+   /// For each worker, which memory resource it uses.
+   int *workers_mem_res;
+
+public:
+   /// TODO: doxygen
+   Engine(Backend *b, int n_mem, int n_workers);
+
+   /// TODO: doxygen
+   virtual ~Engine();
+
+
+   /**
+       @name Machine resources interface
+    */
+   ///@{
+
+#ifdef MFEM_USE_MPI
+   /// Get the associated MPI_Comm
+   MPI_Comm GetComm() const { return comm; }
+#endif
+
+   /// TODO
+   int GetNumMemRes() const { return num_mem_res; }
+
+   /// TODO
+   MemoryResource &GetMemRes(int idx) const { return *memory_resources[idx]; }
+
+   /// TODO
+   int GetNumWorkers() const { return num_workers; }
+
+   /// TODO
+   const double *GetWorkersWeights() const { return workers_weights; }
+
+   /// TODO
+   const int *GetWorkersMemRes() const { return workers_mem_res; }
+
+   ///@}
+   // End: Machine resources interface
+
+   /// TODO
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   /// TODO
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+
+
+   // TODO: Error handling ... handle errors at the Engine level, at the class
+   //       level, or at the method level?
+
+
+   /**
+       @name Virtual interface: finite element data structures and algorithms
+    */
+   ///@{
+
+   // TODO: Asynchronous execution in this class ...
+
+   /// Allocate and return a new layout for the given @a size.
+   /** The layout decomposition (in the case of multiple workers) is determined
+       automatically by the Engine using a deterministic algorithm: calls to
+       this method with the same @a size will produce the same result, as long
+       as the Engine remains unmodified between the calls.
+
+       The returned object is allocated with operator new and must be
+       deallocated by the caller.
+
+       TODO: Returns NULL if memory allocation fails?
+   */
+   virtual DLayout MakeLayout(std::size_t size) const = 0;
+
+   /// Allocate and return a new layout for the given worker decomposition.
+   /** The returned object is allocated with operator new and must be
+       deallocated by the caller.
+
+       TODO: Returns NULL if memory allocation fails?
+
+       The @a offsets should satisfy: offsets.Size() == number of workers + 1,
+       offsets[0] == 0, and offsets[i] <= offsets[i+1], for i: 0 <= i < number
+       of workers. */
+   virtual DLayout MakeLayout(const Array<std::size_t> &offsets) const = 0;
+
+   // Note: There may be other ways to construct layouts in the future, e.g.
+   //       block-vector layouts, or multi-vector layouts.
+
+   /// TODO
+   virtual DArray MakeArray(PLayout &layout, std::size_t item_size) const = 0;
+
+   /// Allocate and return a new vector using the given @a layout.
+   /** The returned object is a smart pointer that will automatically deallocate
+       the vector.
+
+       TODO: Produce an error if memory allocation fails?
+
+       Only layouts returned by this Engine are guaranteed to be supported.
+       Using a type that is not supported will produce an error. */
+   virtual DVector MakeVector(PLayout &layout,
+                              int type_id = ScalarId<double>::value) const = 0;
+
+   /// TODO: doxygen
+   virtual DFiniteElementSpace MakeFESpace(FiniteElementSpace &fes) const = 0;
+
+   /// TODO: doxygen
+   virtual DBilinearForm MakeBilinearForm(BilinearForm &bf) const = 0;
+
+
+   // Question: How do we construct coefficients?
+
+
+   /// FIXME - What will the actual parameters be?
+   virtual void AssembleLinearForm(LinearForm &l_form) const = 0;
+
+   /// FIXME - What will the actual parameters be?
+   virtual Operator *MakeOperator(const MixedBilinearForm &mbl_form) const = 0;
+
+   /// FIXME - What will the actual parameters be?
+   virtual Operator *MakeOperator(const NonlinearForm &nl_form) const = 0;
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_ENGINE_HPP

backends/base/fespace.hpp

@@ -0,0 +1,98 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_FE_SPACE_HPP
+#define MFEM_BACKENDS_BASE_FE_SPACE_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "engine.hpp"
+#include "utils.hpp"
+
+namespace mfem
+{
+
+class FiniteElementSpace;
+class QuadratureSpace;
+
+/// TODO: doxygen
+class PFiniteElementSpace : public RefCounted
+{
+protected:
+   /// Engine with shared ownership
+   SharedPtr<const Engine> engine;
+   /// Not owned.
+   mfem::FiniteElementSpace *fes;
+
+public:
+   /// TODO: doxygen
+   PFiniteElementSpace(const Engine &e, mfem::FiniteElementSpace &fespace)
+      : engine(&e), fes(&fespace) { }
+
+   /// Virtual destructor
+   virtual ~PFiniteElementSpace() { }
+
+   /// Get the associated engine
+   const Engine &GetEngine() const { return *engine; }
+
+   /// Return the associated mfem::FiniteElementSpace
+   mfem::FiniteElementSpace *GetFESpace() const { return fes; }
+
+   /// TODO
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   /// TODO
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+
+
+   /**
+       @name Virtual interface: finite element space functionality
+    */
+   ///@{
+
+   /// TODO
+   /** Return the operator mapping T-vectors to L-vectors. If a NULL pointer is
+       returned then the mapping is the idenity. */
+   virtual const mfem::Operator *GetProlongationOperator() const = 0;
+
+   /// TODO
+   /** Return the operator mapping L-vectors to T-vectors that extracts the
+       subset of all true dofs, i.e. no assembly is performed. If a NULL pointer
+       is returned then the mapping is the idenity. */
+   virtual const mfem::Operator *GetRestrictionOperator() const = 0;
+
+   /// TODO
+   /** Return the operator mapping L-vectors to Q-vectors that evaluates the
+       values of a GridFunction as a QuadratureFunction on the given
+       QuadratureSpace. If the returned pointer is NULL, then the mapping is the
+       identity. */
+   virtual const mfem::Operator *GetInterpolationOperator(
+      const mfem::QuadratureSpace &qspace) const = 0;
+
+   /// TODO
+   /** Return the operator mapping L-vectors to Q-vectors that evaluates the
+       _reference element_ gradients of a GridFunction as a QuadratureFunction
+       on the given QuadratureSpace. */
+   virtual const mfem::Operator *GetGradientOperator(
+      const mfem::QuadratureSpace &qspace) const = 0;
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_FE_SPACE_HPP

backends/base/layout.hpp

@@ -0,0 +1,110 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_LAYOUT_HPP
+#define MFEM_BACKENDS_BASE_LAYOUT_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "smart_pointers.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+/// Polymorphic layout (array/vector layout descriptor)
+class PLayout : public RefCounted
+{
+protected:
+   /// Engine with shared ownership
+   SharedPtr<const Engine> engine;
+   std::size_t size;
+
+   template <typename DObject>
+   struct Maker
+   {
+      template <typename entry_t>
+      static DObject MakeNew(PLayout &layout);
+   };
+
+public:
+   explicit PLayout(std::size_t s = 0) : engine(NULL), size(s) { }
+
+   explicit PLayout(const Engine &e, std::size_t s = 0)
+      : engine(&e), size(s) { }
+
+   virtual ~PLayout() { }
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   /// Resize the layout
+   virtual void Resize(std::size_t new_size) { size = new_size; }
+
+   /// Resize the layout based on the given worker offsets
+   virtual void Resize(const Array<std::size_t> &offsets)
+   { MFEM_ABORT("method not supported"); }
+
+   ///@}
+   // End: Virtual interface
+
+   /// Layouts without engine cannot create DArray, DVector, etc.
+   bool HasEngine() const { return engine != NULL; }
+
+   /// TODO: doxygen
+   const Engine &GetEngine() const { return *engine; }
+
+   /// TODO: doxygen
+   std::size_t Size() const { return size; }
+
+   /// TODO
+   /** Useful in backends for down-casting to a backend-specific layout type.
+
+       When MFEM_DEBUG=YES, performs a type check using dynamic_cast. */
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   /// TODO
+   /** Useful in backends for down-casting to a backend-specific layout type.
+
+       When MFEM_DEBUG=YES, performs a type check using dynamic_cast. */
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+
+   /// TODO: doxygen
+   template <typename DObject, typename entry_t>
+   DObject Make()
+   {
+      MFEM_ASSERT(HasEngine(), "this method requires an Engine");
+      return Maker<DObject>::template MakeNew<entry_t>(*this);
+   }
+};
+
+template <> struct PLayout::Maker<DArray>
+{
+   template <typename entry_t> static DArray MakeNew(PLayout &layout)
+   { return layout.GetEngine().MakeArray(layout, sizeof(entry_t)); }
+};
+
+template <> struct PLayout::Maker<DVector>
+{
+   template <typename entry_t> static DVector MakeNew(PLayout &layout)
+   { return layout.GetEngine().MakeVector(layout, ScalarId<entry_t>::value); }
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_LAYOUT_HPP

backends/base/memory_resource.cpp

@@ -0,0 +1,59 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "memory_resource.hpp"
+#include "../../general/error.hpp"
+
+#include <cstdlib>
+#include <cstring>
+#include <cerrno>
+
+namespace mfem
+{
+
+void *NewDeleteMemoryResource::DoAllocate(std::size_t bytes,
+                                          std::size_t alignment)
+{
+   void *p = ::operator new[](bytes);
+   MFEM_VERIFY(!alignment || (std::size_t)(p) % alignment == 0,
+               "invalid alignment");
+   return p;
+}
+
+void NewDeleteMemoryResource::DoDeallocate(void *p, std::size_t bytes,
+                                           std::size_t alignment)
+{
+   ::operator delete[](p);
+}
+
+
+void *AlignedMemoryResource::DoAllocate(std::size_t bytes,
+                                        std::size_t alignment)
+{
+   void *p;
+   if (!alignment) { alignment = sizeof(long double); }
+   MFEM_VERIFY(posix_memalign(&p, alignment, bytes) == 0,
+               "error in posix_memalign(): " << strerror(errno));
+   return p;
+}
+
+void AlignedMemoryResource::DoDeallocate(void *p, std::size_t bytes,
+                                         std::size_t alignment)
+{
+   free(p);
+}
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS

backends/base/memory_resource.hpp

@@ -0,0 +1,70 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP
+#define MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include <cstddef>
+
+namespace mfem
+{
+
+/// Polymorphic memory resource. Similar to C++17's std::pmr::memory_resource.
+class MemoryResource
+{
+protected:
+   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment) = 0;
+   virtual void DoDeallocate(void* p, std::size_t bytes,
+                             std::size_t alignment) = 0;
+
+public:
+   // Implicitly defined default & copy constructors
+
+   /// Virtual destructor.
+   virtual ~MemoryResource() { }
+
+   /// If alignment == 0, use default alignment.
+   void *Allocate(std::size_t bytes, std::size_t alignment = 0)
+   { return DoAllocate(bytes, alignment); }
+
+   /// If alignment == 0, use default alignment.
+   void Deallocate(void *p, std::size_t bytes, std::size_t alignment = 0)
+   { DoDeallocate(p, bytes, alignment); }
+};
+
+
+/** @brief Dynamic host memory resource using operator new[](std::size_t) for
+    allocation and operator delete[](void*) for deallocation. */
+class NewDeleteMemoryResource : public MemoryResource
+{
+protected:
+   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment);
+   virtual void DoDeallocate(void *p, std::size_t bytes, std::size_t alignment);
+};
+
+
+/** @brief Dynamic host memory resource using posix_memalign() for aligned
+    allocation and free() for deallocation. */
+class AlignedMemoryResource : public MemoryResource
+{
+protected:
+   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment);
+   virtual void DoDeallocate(void *p, std::size_t bytes, std::size_t alignment);
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP

backends/base/smart_pointers.hpp

@@ -0,0 +1,234 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_SMART_POINTERS_HPP
+#define MFEM_BACKENDS_BASE_SMART_POINTERS_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "utils.hpp"
+#include "../../general/error.hpp"
+#include <cstddef>
+
+// #define MFEM_TRACE_SHARED_PTR
+#ifdef MFEM_TRACE_SHARED_PTR
+#include "../../general/globals.hpp"
+#endif
+
+namespace mfem
+{
+
+/// Base class for classes with simple reference counting.
+/** Reference counting is performed by the class SharedPtr. */
+class RefCounted
+{
+private:
+   mutable unsigned ref_count;
+
+   /// Only class SharedPtr can access ref_count.
+   template <typename T> friend class SharedPtr;
+
+public:
+   RefCounted() : ref_count(0) { }
+
+   /** @brief Prevent SharedPtr objects from deleting this object by
+       incrementing the reference counter by one. */
+   void DontDelete() const { ++ref_count; }
+};
+
+
+/** @brief Smart pointer class that manages objects of type T derived from class
+    RefCounted. */
+/** This class is generally meant to work with dynamically allocated object,
+    specifically objects allocated with operator new(). It will invoke operator
+    delete() to destroy the managed object when its reference counter reaches
+    zero. This behavior can be overriden by calling RefCounted::DontDelete() to
+    ensure that an object will not be deleted by a SharedPtr that holds a
+    pointer to it.
+    @note This class is NOT thread-safe and does not support circular ownership.
+ */
+template <typename T>
+class SharedPtr
+{
+public:
+   typedef T stored_type;
+
+private:
+   T *ptr;
+
+   void Init(T *new_ptr)
+   {
+      ptr = new_ptr;
+      if (ptr) { ++ptr->RefCounted::ref_count; }
+#ifdef MFEM_TRACE_SHARED_PTR
+#elif 0
+      mfem::out << "  [" << _MFEM_FUNC_NAME << "]: ptr = " << ptr;
+      if (ptr)
+      {
+         mfem::out << ", new ref_count = " << ptr->RefCounted::ref_count;
+      }
+      mfem::out << '\n';
+#endif
+   }
+   void Destroy()
+   {
+      MFEM_ASSERT(!ptr || ptr->RefCounted::ref_count >= 1, "invalid use");
+      if (ptr && --ptr->RefCounted::ref_count == 0) { delete ptr; }
+#ifdef MFEM_TRACE_SHARED_PTR
+#elif 0
+      mfem::out << "  [" << _MFEM_FUNC_NAME << "]: ptr = " << ptr;
+      if (ptr)
+      {
+         mfem::out << ", new ref_count = " << ptr->RefCounted::ref_count;
+      }
+      mfem::out << '\n';
+#endif
+   }
+
+public:
+   SharedPtr() : ptr(NULL)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]: ptr = " << ptr << '\n';
+#endif
+   }
+
+   SharedPtr(const SharedPtr &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Init(other.ptr);
+   }
+
+   template <typename U>
+   SharedPtr(const SharedPtr<U> &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Init(other.Get());
+   }
+
+   explicit SharedPtr(T *p)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Init(p);
+   }
+
+   ~SharedPtr()
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Destroy();
+   }
+
+   SharedPtr &operator=(const SharedPtr &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Reset(other.ptr); return *this;
+   }
+
+   template <typename U>
+   SharedPtr &operator=(const SharedPtr<U> &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Reset(other.Get()); return *this;
+   }
+
+   T &operator*() const { return *ptr; }
+   T *operator->() const { return ptr; }
+
+   operator bool() const { return ptr; }
+   bool operator!() const { return !ptr; }
+
+   template <typename U>
+   bool operator==(const SharedPtr<U> &other) const
+   { return ptr == other.Get(); }
+   template <typename U>
+   bool operator!=(const SharedPtr<U> &other) const
+   { return ptr != other.Get(); }
+
+   // Comparison to any type convertible to void *, e.g. the type of NULL.
+   template <typename U>
+   bool operator==(const U &p) const { return ptr == (void*) p; }
+   template <typename U>
+   bool operator!=(const U &p) const { return ptr != (void*) p; }
+
+   T *Get() const { return ptr; }
+
+   /// TODO
+   template <typename derived_t>
+   derived_t *As() const { return util::As<derived_t>(ptr); }
+
+   unsigned UseCount() const { return ptr ? ptr->RefCounted::ref_count : 0; }
+
+   void Reset()
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Destroy();
+      ptr = NULL;
+   }
+
+   /// The type U* needs to be implicitly convertible to T*
+   template <typename U>
+   void Reset(U *new_ptr)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      if (ptr != new_ptr) { Destroy(); Init(new_ptr); }
+   }
+
+   void Swap(SharedPtr &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      std::swap(ptr, other.ptr);
+   }
+};
+
+
+template <class T>
+inline void Swap(SharedPtr<T> &a, SharedPtr<T> &b) { a.Swap(b); }
+
+
+class PLayout;
+typedef SharedPtr<PLayout> DLayout;
+
+class PArray;
+typedef SharedPtr<PArray> DArray;
+
+class PVector;
+typedef SharedPtr<PVector> DVector;
+
+class PFiniteElementSpace;
+typedef SharedPtr<PFiniteElementSpace> DFiniteElementSpace;
+
+class PBilinearForm;
+typedef SharedPtr<PBilinearForm> DBilinearForm;
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_SMART_POINTERS_HPP

backends/base/utils.hpp

@@ -0,0 +1,52 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_UTILS_HPP
+#define MFEM_BACKENDS_BASE_UTILS_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "../../general/error.hpp"
+
+namespace mfem
+{
+
+namespace util
+{
+
+//
+// Inline methods
+//
+
+/// TODO: doxygen
+template <typename derived_t, typename base_t>
+inline derived_t *As(base_t *base_obj)
+{
+   MFEM_ASSERT(dynamic_cast<derived_t*>(base_obj) != NULL,
+               "invalid object type");
+   return static_cast<derived_t*>(base_obj);
+}
+
+/// TODO: doxygen
+template <typename derived_t, typename base_t>
+inline derived_t *Is(base_t *base_obj)
+{
+   return dynamic_cast<derived_t*>(base_obj);
+}
+
+} // namespace mfem::util
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_UTILS_HPP

backends/base/vector.hpp

@@ -0,0 +1,153 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_VECTOR_HPP
+#define MFEM_BACKENDS_BASE_VECTOR_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "../../general/scalars.hpp"
+#include "array.hpp"
+
+namespace mfem
+{
+
+/// Polymorphic vector - array of scalars.
+class PVector : virtual public PArray
+{
+protected:
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new vector of the same dynamic type as this
+       vector using the same layout with entries specified by @a buffer_type_id
+       which should be a constant defined by the `value` field in a
+       specialization of the template class mfem::ScalarId.
+
+       Returns NULL if allocation fails.
+
+       If @a copy_data is true, the contents of this vector is copied to the new
+       vector; otherwise, the new vector remains uninitialized.
+
+       If @a buffer is not NULL, return the vector data of the newly created
+       object (in @a *buffer), if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   virtual PVector *DoVectorClone(bool copy_data, void **buffer,
+                                  int buffer_type_id) const = 0;
+
+   /** @brief Compute and return the dot product of @a *this and @a x. In the
+       case of an MPI-parallel vector, the result must be the MPI-global dot
+       product. */
+   /** Both vectors must have the same dynamic type and layout. */
+   virtual void DoDotProduct(const PVector &x, void *result,
+                             int result_type_id) const = 0;
+
+   // TODO: add reduction operations: min, max, sum
+
+   /// Perform the operation @a *this = @a a @a x + @a b @a y.
+   /** Rules:
+       - the dynamic type of both @a x and @a y is the same as that of @a *this
+       - if @a a == 0, neither @a x nor its data are accessed
+       - if @a b == 0, neither @a y nor its data are accessed
+       - @a x's data is never the same as @a y's data, unless @a a == 0, or
+         @a b == 0
+       - @a x's data or @a y's data may be the same as the data of @a *this
+       - all accessed vectors, @a x, @a y, and @a *this have the same layout. */
+   virtual void DoAxpby(const void *a, const PVector &x,
+                        const void *b, const PVector &y,
+                        int ab_type_id) = 0;
+
+   ///@}
+   // End: Virtual interface
+
+public:
+   /** @brief Create a PVector. */
+   /** The @a layout must be valid in the sense that layout != NULL and
+       layout->HasEngine() == true. */
+   PVector(PLayout &p_layout)
+      : PArray(p_layout) { }
+
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+
+
+   // TODO: Error handling ... handle errors at the Engine level, at the class
+   //       level, or at the method level?
+
+   // TODO: Asynchronous execution interface ...
+
+   // TODO: Multi-vector interface ...
+
+
+   /**
+       @name Public virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new vector of the same dynamic type as this
+       vector using the same layout with entries of type @a scalar_t.
+
+       If @a copy_data is true, the contents of this vector is copied to the new
+       vector; otherwise, the new vector remains uninitialized.
+
+       If @a buffer is not NULL, return the vector data of the newly created
+       object (in @a *buffer) , if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   template <typename scalar_t>
+   DVector Clone(bool copy_data, scalar_t **buffer) const
+   {
+      return DVector(DoVectorClone(copy_data, (void**)buffer,
+                                   ScalarId<scalar_t>::value));
+   }
+
+   /** @brief Compute and return the dot product of @a *this and @a x. In the
+       case of an MPI-parallel vector, the result must be the MPI-global dot
+       product. */
+   /** Both vectors must have the same dynamic type and layout. */
+   template <typename scalar_t>
+   scalar_t DotProduct(const PVector &x) const
+   {
+      scalar_t result;
+      DoDotProduct(x, &result, ScalarId<scalar_t>::value);
+      return result;
+   }
+
+   // TODO: add reduction operations: min, max, sum
+
+   /// Perform the operation @a *this = @a a @a x + @a b @a y.
+   /** Rules:
+       - the dynamic type of both @a x and @a y is the same as that of @a *this
+       - if @a a == 0, neither @a x nor its data are accessed
+       - if @a b == 0, neither @a y nor its data are accessed
+       - @a x's data is never the same as @a y's data, unless @a a == 0, or
+         @a b == 0
+       - @a x's data or @a y's data may be the same as the data of @a *this
+       - all accessed vectors, @a x, @a y, and @a *this have the same layout. */
+   template <typename scalar_t>
+   void Axpby(const scalar_t &a, const PVector &x,
+              const scalar_t &b, const PVector &y)
+   { if (Size()) { DoAxpby(&a, x, &b, y, ScalarId<scalar_t>::value); } }
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_VECTOR_HPP

backends/occa/DiffusionIntegrator.okl

@@ -0,0 +1,66 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  ELEMENT_BATCH   : How many elements are in each
+  .                   computation batch
+  NUM_DOFS_1D     : Dofs in the 1D segments
+  NUM_DOFS_2D     : Dofs in the 2D faces
+  NUM_DOFS_3D     : Dofs in the 3D domain
+  NUM_QUAD_1D     : Dofs in the 1D segments
+  NUM_QUAD_2D     : Dofs in the 2D faces
+  NUM_QUAD_3D     : Dofs in the 3D domain
+  NUM_MAX_1D      : max(NUM_QUAD_1D, NUM_DOFS_1D)
+  NUM_QUAD_DOFS_1D: NUM_QUAD_1D * NUM_DOFS_1D
+  COEFF_ARGS      : Code that passes required arguments to the kernel
+  COEFF           : Code that computes the coefficient
+  ================================================
+
+  [MISSING]
+  - Add support to auto-pick @dim and use @idxOrder on stack arrays
+  |   double a[2][2];
+  |   a[0][1]; <-- regular index
+  |   a(0,1);  <-- uses @idxOrder a[0][1] or a[1][0]
+  - Add support for @idxOrder to change indexing order after allocation
+  |   double a[2][2] @idxOrder(0,1);
+  |   a(0,1) -> a[1][0]
+  |   @set(a, idxOrder(1,0));
+  |   a(0,1) -> a[0][1]
+  - Add support to iterate over loop depending on mode
+  |   for(i; @inner) {
+  |     for(0 < j < N) {} <-- ++j or j += block?
+  |   }
+*/
+
+#include "mfem-occa://defines.okl"
+
+#if USING_TENSOR_OPS
+#  ifdef OCCA_USING_GPU
+#    if USING_LOW_ORDER
+#      include "mfem-occa://diffusion/tensor/gpuHighOrder.okl"
+#    else
+#      include "mfem-occa://diffusion/tensor/gpuHighOrder.okl"
+#    endif
+#  else
+#    include "mfem-occa://diffusion/tensor/cpu.okl"
+#  endif
+#else
+#  ifdef OCCA_USING_GPU
+#    if USING_LOW_ORDER
+#      include "mfem-occa://diffusion/simplex/gpuHighOrder.okl"
+#    else
+#      include "mfem-occa://diffusion/simplex/gpuHighOrder.okl"
+#    endif
+#  else
+#    include "mfem-occa://diffusion/simplex/cpu.okl"
+#  endif
+#endif

backends/occa/MassIntegrator.okl

@@ -0,0 +1,66 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  ELEMENT_BATCH   : How many elements are in each
+  .                   computation batch
+  NUM_DOFS_1D     : Dofs in the 1D segments
+  NUM_DOFS_2D     : Dofs in the 2D faces
+  NUM_DOFS_3D     : Dofs in the 3D domain
+  NUM_QUAD_1D     : Dofs in the 1D segments
+  NUM_QUAD_2D     : Dofs in the 2D faces
+  NUM_QUAD_3D     : Dofs in the 3D domain
+  NUM_MAX_1D      : max(NUM_QUAD_1D, NUM_DOFS_1D)
+  NUM_QUAD_DOFS_1D: NUM_QUAD_1D * NUM_DOFS_1D
+  COEFF_ARGS      : Code that passes required arguments to the kernel
+  COEFF           : Code that computes the coefficient
+  ================================================
+
+  [MISSING]
+  - Add support to auto-pick @dim and use @idxOrder on stack arrays
+  |   double a[2][2];
+  |   a[0][1]; <-- regular index
+  |   a(0,1);  <-- uses @idxOrder a[0][1] or a[1][0]
+  - Add support for @idxOrder to change indexing order after allocation
+  |   double a[2][2] @idxOrder(0,1);
+  |   a(0,1) -> a[1][0]
+  |   @set(a, idxOrder(1,0));
+  |   a(0,1) -> a[0][1]
+  - Add support to iterate over loop depending on mode
+  |   for(i; @inner) {
+  |     for(0 < j < N) {} <-- ++j or j += block?
+  |   }
+*/
+
+#include "mfem-occa://defines.okl"
+
+#if USING_TENSOR_OPS
+#  ifdef OCCA_USING_GPU
+#    if USING_LOW_ORDER
+#      include "mfem-occa://mass/tensor/gpuHighOrder.okl"
+#    else
+#      include "mfem-occa://mass/tensor/gpuHighOrder.okl"
+#    endif
+#  else
+#    include "mfem-occa://mass/tensor/cpu.okl"
+#  endif
+#else
+#  ifdef OCCA_USING_GPU
+#    if USING_LOW_ORDER
+#      include "mfem-occa://mass/simplex/gpuHighOrder.okl"
+#    else
+#      include "mfem-occa://mass/simplex/gpuHighOrder.okl"
+#    endif
+#  else
+#    include "mfem-occa://mass/simplex/cpu.okl"
+#  endif
+#endif

backends/occa/VectorMassIntegrator.okl

@@ -0,0 +1,38 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  ELEMENT_BATCH   : How many elements are in each
+  .                   computation batch
+  NUM_DOFS_1D     : Dofs in the 1D segments
+  NUM_DOFS_2D     : Dofs in the 2D faces
+  NUM_DOFS_3D     : Dofs in the 3D domain
+  NUM_QUAD_1D     : Dofs in the 1D segments
+  NUM_QUAD_2D     : Dofs in the 2D faces
+  NUM_QUAD_3D     : Dofs in the 3D domain
+  NUM_MAX_1D      : max(NUM_QUAD_1D, NUM_DOFS_1D)
+  NUM_QUAD_DOFS_1D: NUM_QUAD_1D * NUM_DOFS_1D
+  CONST_COEFF     : If the coefficient is constant, pass it
+  .                   as a define
+  ================================================
+
+  [MISSING]
+  See kernels/DiffusionIntegrator.okl
+*/
+
+#include "mfem-occa://defines.okl"
+
+#if USING_TENSOR_OPS
+#  ifndef OCCA_USING_GPU
+#    include "mfem-occa://vmass/tensor/cpu.okl"
+#  endif
+#endif

backends/occa/array.cpp

@@ -0,0 +1,121 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+PArray *Array::DoClone(bool copy_data, void **buffer,
+                       std::size_t item_size) const
+{
+   Array *new_array = new Array(OccaLayout(), item_size);
+   if (copy_data)
+   {
+      new_array->slice.copyFrom(slice);
+   }
+   if (buffer)
+   {
+      *buffer = new_array->GetBuffer();
+   }
+   return new_array;
+}
+
+int Array::DoResize(PLayout &new_layout, void **buffer,
+                    std::size_t item_size)
+{
+   MFEM_ASSERT(dynamic_cast<Layout *>(&new_layout) != NULL,
+               "new_layout is not an OCCA Layout");
+   Layout *lt = static_cast<Layout *>(&new_layout);
+   int err = OccaResize(lt, item_size);
+   if (!err && buffer)
+   {
+      *buffer = GetBuffer();
+   }
+   return err;
+}
+
+void *Array::DoPullData(void *buffer, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   if (!slice.getDevice().hasSeparateMemorySpace())
+   {
+      return slice.ptr();
+   }
+   if (buffer)
+   {
+      slice.copyTo(buffer);
+   }
+   return buffer;
+}
+
+void Array::DoFill(const void *value_ptr, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   switch (item_size)
+   {
+      case sizeof(int8_t):
+         OccaFill(*(const int8_t *)value_ptr);
+         break;
+      case sizeof(int16_t):
+         OccaFill(*(const int16_t *)value_ptr);
+         break;
+      case sizeof(int32_t):
+         OccaFill(*(const int32_t *)value_ptr);
+         break;
+      // case sizeof(int64_t):
+      //    OccaFill(*(const int64_t *)value_ptr);
+      //    break;
+      case sizeof(double):
+         OccaFill(*(const double *)value_ptr);
+         break;
+      // case sizeof(::occa::double2):
+      //    OccaFill(*(const ::occa::double2 *)value_ptr);
+      //    break;
+      default:
+         MFEM_ABORT("item_size = " << item_size << " is not supported");
+   }
+}
+
+void Array::DoPushData(const void *src_buffer, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   if (slice.getDevice().hasSeparateMemorySpace() || slice.ptr() != src_buffer)
+   {
+      slice.copyFrom(src_buffer);
+   }
+}
+
+void Array::DoAssign(const PArray &src, std::size_t item_size)
+{
+   // called only when Size() != 0
+
+   // Note: static_cast can not be used here since PArray is a virtual base
+   //       class.
+   const Array *source = dynamic_cast<const Array *>(&src);
+   MFEM_ASSERT(source != NULL, "invalid source Array type");
+   OccaAssign(*source);
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/array.hpp

@@ -0,0 +1,175 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_ARRAY_HPP
+#define MFEM_BACKENDS_OCCA_ARRAY_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include <occa.hpp>
+#include "layout.hpp"
+#include "../base/array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Array : public virtual PArray
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // DLayout layout;
+
+   // Always true: Size()*item_size == slice.size() <= data.size()
+   mutable ::occa::memory data, slice;
+
+   //
+   // Virtual interface
+   //
+
+   virtual PArray *DoClone(bool copy_data, void **buffer,
+                           std::size_t item_size) const;
+
+   virtual int DoResize(PLayout &new_layout, void **buffer,
+                        std::size_t item_size);
+
+   virtual void *DoPullData(void *buffer, std::size_t item_size);
+
+   virtual void DoFill(const void *value_ptr, std::size_t item_size);
+
+   virtual void DoPushData(const void *src_buffer, std::size_t item_size);
+
+   virtual void DoAssign(const PArray &src, std::size_t item_size);
+
+   //
+   // Auxiliary methods
+   //
+
+   inline void *GetBuffer() const;
+
+public:
+   Array(const Engine &e)
+      : PArray(*(new Layout(e, 0))),
+        data(e.Alloc(0)),
+        slice(data)
+   { }
+
+   Array(Layout &lt, std::size_t item_size)
+      : PArray(lt),
+        data(lt.OccaEngine().Alloc(lt.Size()*item_size)),
+        slice(data)
+   { }
+
+   virtual ~Array() { }
+
+   inline void MakeRef(Array &master);
+
+   Layout &OccaLayout() const { return layout->As<Layout>(); }
+
+   const Engine &OccaEngine() const { return OccaLayout().OccaEngine(); }
+
+   ::occa::memory &OccaMem() { return slice; }
+   const ::occa::memory &OccaMem() const { return slice; }
+
+   inline int OccaResize(Layout *lt, std::size_t item_size);
+
+   inline int OccaResize(std::size_t new_size, std::size_t item_size);
+
+   template <typename T>
+   inline void OccaFill(const T val);
+
+   inline void OccaAssign(const Array &src);
+
+   inline void OccaPush(const void *src);
+};
+
+
+//
+// Inline methods
+//
+
+inline void *Array::GetBuffer() const
+{
+   if (!slice.getDevice().hasSeparateMemorySpace())
+   {
+      return slice.ptr();
+   }
+   return NULL;
+}
+
+inline int Array::OccaResize(Layout *lt, std::size_t item_size)
+{
+   layout.Reset(lt); // Reset() checks if the pointer is the same
+   const std::size_t new_bytes = lt->Size()*item_size;
+   if (data.size() < new_bytes ||
+       data.getDevice() != lt->OccaEngine().GetDevice())
+   {
+      data = lt->OccaEngine().Alloc(new_bytes);
+      slice = data;
+      // If memory allocation fails - an exception is thrown.
+   }
+   else if (slice.size() != new_bytes)
+   {
+      slice = data.slice(0, new_bytes);
+   }
+   return 0;
+}
+
+inline void Array::MakeRef(Array &master)
+{
+   layout = master.layout;
+   data = master.data;
+   slice = master.slice;
+}
+
+inline int Array::OccaResize(std::size_t new_size, std::size_t item_size)
+{
+   Layout &ol = OccaLayout();
+   ol.OccaResize(new_size);
+   return OccaResize(&ol, item_size);
+}
+
+template <typename T>
+inline void Array::OccaFill(const T val)
+{
+   ::occa::linalg::operator_eq<T>(slice, val);
+}
+
+inline void Array::OccaAssign(const Array &src)
+{
+   if (slice != src.slice && slice.size() != 0)
+   {
+      MFEM_ASSERT(slice.size() == src.slice.size(), "");
+      slice.copyFrom(src.slice);
+   }
+}
+
+inline void Array::OccaPush(const void *src)
+{
+   if (slice.size() != 0)
+   {
+      slice.copyFrom(src);
+   }
+}
+
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_ARRAY_HPP

backends/occa/backend.cpp

@@ -0,0 +1,47 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "backend.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+bool Backend::Supports(const std::string &engine_spec) const
+{
+   // TODO: check if 'engine_spec' is valid OCCA string.
+   return true;
+}
+
+mfem::Engine *Create(const std::string &engine_spec)
+{
+   return new Engine(engine_spec);
+}
+
+#ifdef MFEM_USE_MPI
+mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec)
+{
+   return new Engine(comm, engine_spec);
+}
+#endif
+
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/backend.hpp

@@ -0,0 +1,49 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_BACKEND_HPP
+#define MFEM_BACKENDS_OCCA_BACKEND_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+// Only the Backend and Engine classes should be exposed through "backend.hpp"
+#include "../base/backend.hpp"
+#include "engine.hpp"
+#include <occa.hpp>
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Backend : public mfem::Backend
+{
+public:
+   virtual ~Backend();
+
+   virtual bool Supports(const std::string &engine_spec) const;
+
+   virtual mfem::Engine *Create(const std::string &engine_spec);
+
+#ifdef MFEM_USE_MPI
+   virtual mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec);
+#endif
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_BACKEND_HPP

backends/occa/bilinearform.cpp

@@ -0,0 +1,538 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "backend.hpp"
+#include "bilininteg.hpp"
+#include "../../fem/bilinearform.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+OccaBilinearForm::OccaBilinearForm(FiniteElementSpace *ofespace_) :
+   Operator(ofespace_->OccaVLayout()),
+   localX(ofespace_->OccaEVLayout()),
+   localY(ofespace_->OccaEVLayout())
+{
+   Init(ofespace_->OccaEngine(), ofespace_, ofespace_);
+}
+
+OccaBilinearForm::OccaBilinearForm(FiniteElementSpace *otrialFESpace_,
+                                   FiniteElementSpace *otestFESpace_) :
+   Operator(otrialFESpace_->OccaVLayout(),
+            otestFESpace_->OccaVLayout()),
+   localX(otrialFESpace_->OccaEVLayout()),
+   localY(otestFESpace_->OccaEVLayout())
+{
+   Init(otrialFESpace_->OccaEngine(), otrialFESpace_, otestFESpace_);
+}
+
+void OccaBilinearForm::Init(const Engine &e,
+                            FiniteElementSpace *otrialFESpace_,
+                            FiniteElementSpace *otestFESpace_)
+{
+   engine.Reset(&e);
+
+   otrialFESpace = otrialFESpace_;
+   trialFESpace  = otrialFESpace_->GetFESpace();
+
+   otestFESpace = otestFESpace_;
+   testFESpace  = otestFESpace_->GetFESpace();
+
+   mesh = trialFESpace->GetMesh();
+
+   const int elements = GetNE();
+
+   const int trialVDim = trialFESpace->GetVDim();
+
+   const int trialLocalDofs = otrialFESpace->GetLocalDofs();
+   const int testLocalDofs  = otestFESpace->GetLocalDofs();
+
+   // First-touch policy when running with OpenMP
+   if (GetDevice().mode() == "OpenMP")
+   {
+      const std::string &okl_path = OccaEngine().GetOklPath();
+      ::occa::kernel initLocalKernel =
+         GetDevice().buildKernel(okl_path + "utils.okl",
+                                 "InitLocalVector");
+
+      const std::size_t sd = sizeof(double);
+      const uint64_t trialEntries = sd * (elements * trialLocalDofs);
+      const uint64_t testEntries  = sd * (elements * testLocalDofs);
+      for (int v = 0; v < trialVDim; ++v)
+      {
+         const uint64_t trialOffset = v * trialEntries;
+         const uint64_t testOffset  = v * testEntries;
+
+         initLocalKernel(elements, trialLocalDofs,
+                         localX.OccaMem().slice(trialOffset, trialEntries));
+         initLocalKernel(elements, testLocalDofs,
+                         localY.OccaMem().slice(testOffset, testEntries));
+      }
+   }
+}
+
+int OccaBilinearForm::BaseGeom() const
+{
+   return mesh->GetElementBaseGeometry();
+}
+
+int OccaBilinearForm::GetDim() const
+{
+   return mesh->Dimension();
+}
+
+int64_t OccaBilinearForm::GetNE() const
+{
+   return mesh->GetNE();
+}
+
+Mesh& OccaBilinearForm::GetMesh() const
+{
+   return *mesh;
+}
+
+FiniteElementSpace& OccaBilinearForm::GetTrialOccaFESpace() const
+{
+   return *otrialFESpace;
+}
+
+FiniteElementSpace& OccaBilinearForm::GetTestOccaFESpace() const
+{
+   return *otestFESpace;
+}
+
+mfem::FiniteElementSpace& OccaBilinearForm::GetTrialFESpace() const
+{
+   return *trialFESpace;
+}
+
+mfem::FiniteElementSpace& OccaBilinearForm::GetTestFESpace() const
+{
+   return *testFESpace;
+}
+
+int64_t OccaBilinearForm::GetTrialNDofs() const
+{
+   return trialFESpace->GetNDofs();
+}
+
+int64_t OccaBilinearForm::GetTestNDofs() const
+{
+   return testFESpace->GetNDofs();
+}
+
+int64_t OccaBilinearForm::GetTrialVDim() const
+{
+   return trialFESpace->GetVDim();
+}
+
+int64_t OccaBilinearForm::GetTestVDim() const
+{
+   return testFESpace->GetVDim();
+}
+
+const FiniteElement& OccaBilinearForm::GetTrialFE(const int i) const
+{
+   return *(trialFESpace->GetFE(i));
+}
+
+const FiniteElement& OccaBilinearForm::GetTestFE(const int i) const
+{
+   return *(testFESpace->GetFE(i));
+}
+
+// Adds new Domain Integrator.
+void OccaBilinearForm::AddDomainIntegrator(OccaIntegrator *integrator,
+                                           const ::occa::properties &props)
+{
+   AddIntegrator(integrator, props, DomainIntegrator);
+}
+
+// Adds new Boundary Integrator.
+void OccaBilinearForm::AddBoundaryIntegrator(OccaIntegrator *integrator,
+                                             const ::occa::properties &props)
+{
+   AddIntegrator(integrator, props, BoundaryIntegrator);
+}
+
+// Adds new interior Face Integrator.
+void OccaBilinearForm::AddInteriorFaceIntegrator(OccaIntegrator *integrator,
+                                                 const ::occa::properties &props)
+{
+   AddIntegrator(integrator, props, InteriorFaceIntegrator);
+}
+
+// Adds new boundary Face Integrator.
+void OccaBilinearForm::AddBoundaryFaceIntegrator(OccaIntegrator *integrator,
+                                                 const ::occa::properties &props)
+{
+   AddIntegrator(integrator, props, BoundaryFaceIntegrator);
+}
+
+// Adds Integrator based on OccaIntegratorType
+void OccaBilinearForm::AddIntegrator(OccaIntegrator *integrator,
+                                     const ::occa::properties &props,
+                                     const OccaIntegratorType itype)
+{
+   if (integrator == NULL)
+   {
+      std::stringstream error_ss;
+      error_ss << "OccaBilinearForm::";
+      switch (itype)
+      {
+         case DomainIntegrator      : error_ss << "AddDomainIntegrator";       break;
+         case BoundaryIntegrator    : error_ss << "AddBoundaryIntegrator";     break;
+         case InteriorFaceIntegrator: error_ss << "AddInteriorFaceIntegrator"; break;
+         case BoundaryFaceIntegrator: error_ss << "AddBoundaryFaceIntegrator"; break;
+      }
+      error_ss << " (...):\n"
+               << "  Integrator is NULL";
+      const std::string error = error_ss.str();
+      mfem_error(error.c_str());
+   }
+   integrator->SetupIntegrator(*this, baseKernelProps + props, itype);
+   integrators.push_back(integrator);
+}
+
+const mfem::Operator* OccaBilinearForm::GetTrialProlongation() const
+{
+   return otrialFESpace->GetProlongationOperator();
+}
+
+const mfem::Operator* OccaBilinearForm::GetTestProlongation() const
+{
+   return otestFESpace->GetProlongationOperator();
+}
+
+const mfem::Operator* OccaBilinearForm::GetTrialRestriction() const
+{
+   return otrialFESpace->GetRestrictionOperator();
+}
+
+const mfem::Operator* OccaBilinearForm::GetTestRestriction() const
+{
+   return otestFESpace->GetRestrictionOperator();
+}
+
+void OccaBilinearForm::Assemble()
+{
+   // [MISSING] Find geometric information that is needed by intergrators
+   //             to share between integrators.
+   const int integratorCount = (int) integrators.size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->Assemble();
+   }
+}
+
+void OccaBilinearForm::FormLinearSystem(const mfem::Array<int> &constraintList,
+                                        mfem::Vector &x, mfem::Vector &b,
+                                        mfem::Operator *&Aout,
+                                        mfem::Vector &X, mfem::Vector &B,
+                                        int copy_interior)
+{
+   FormOperator(constraintList, Aout);
+   InitRHS(constraintList, x, b, Aout, X, B, copy_interior);
+}
+
+void OccaBilinearForm::FormOperator(const mfem::Array<int> &constraintList,
+                                    mfem::Operator *&Aout)
+{
+   const mfem::Operator *trialP = GetTrialProlongation();
+   const mfem::Operator *testP  = GetTestProlongation();
+   mfem::Operator *rap = this;
+
+   if (trialP)
+   {
+      rap = new RAPOperator(*testP, *this, *trialP);
+   }
+
+   Aout = new OccaConstrainedOperator(rap, constraintList,
+                                      rap != this);
+}
+
+void OccaBilinearForm::InitRHS(const mfem::Array<int> &constraintList,
+                               mfem::Vector &x, mfem::Vector &b,
+                               mfem::Operator *A,
+                               mfem::Vector &X, mfem::Vector &B,
+                               int copy_interior)
+{
+   // FIXME: move these kernels to the Backend?
+   static ::occa::kernelBuilder get_subvector_builder =
+      ::occa::linalg::customLinearMethod(
+         "vector_get_subvector",
+
+         "const int dof_i = v2[i];"
+         "v0[i] = dof_i >= 0 ? v1[dof_i] : -v1[-dof_i - 1];",
+
+         "defines: {"
+         "  VTYPE0: 'double',"
+         "  VTYPE1: 'double',"
+         "  VTYPE2: 'int',"
+         "  TILESIZE: 128,"
+         "}");
+
+   static ::occa::kernelBuilder set_subvector_builder =
+      ::occa::linalg::customLinearMethod(
+         "vector_set_subvector",
+         "const int dof_i = v2[i];"
+         "if (dof_i >= 0) { v0[dof_i]      = v1[i]; }"
+         "else            { v0[-dof_i - 1] = -v1[i]; }",
+
+         "defines: {"
+         "  VTYPE0: 'double',"
+         "  VTYPE1: 'double',"
+         "  VTYPE2: 'int',"
+         "  TILESIZE: 128,"
+         "}");
+
+   const mfem::Operator *P = GetTrialProlongation();
+   const mfem::Operator *R = GetTrialRestriction();
+
+   if (P)
+   {
+      // Variational restriction with P
+      B.Resize(P->InLayout());
+      P->MultTranspose(b, B);
+      X.Resize(R->OutLayout());
+      R->Mult(x, X);
+   }
+   else
+   {
+      // rap, X and B point to the same data as this, x and b
+      X.MakeRef(x);
+      B.MakeRef(b);
+   }
+
+   if (!copy_interior && constraintList.Size() > 0)
+   {
+      ::occa::kernel get_subvector_kernel =
+         get_subvector_builder.build(GetDevice());
+      ::occa::kernel set_subvector_kernel =
+         set_subvector_builder.build(GetDevice());
+
+      const Array &constrList = constraintList.Get_PArray()->As<Array>();
+      Vector subvec(constrList.OccaLayout());
+
+      get_subvector_kernel(constraintList.Size(),
+                           subvec.OccaMem(),
+                           X.Get_PVector()->As<Vector>().OccaMem(),
+                           constrList.OccaMem());
+
+      X.Fill(0.0);
+
+      set_subvector_kernel(constraintList.Size(),
+                           X.Get_PVector()->As<Vector>().OccaMem(),
+                           subvec.OccaMem(),
+                           constrList.OccaMem());
+   }
+
+   // FIXME: add case for HypreParMatrix here
+   OccaConstrainedOperator *cA = dynamic_cast<OccaConstrainedOperator*>(A);
+   if (cA)
+   {
+      cA->EliminateRHS(X.Get_PVector()->As<Vector>(),
+                       B.Get_PVector()->As<Vector>());
+   }
+   else
+   {
+      mfem_error("OccaBilinearForm::InitRHS expects an OccaConstrainedOperator");
+   }
+}
+
+// Matrix vector multiplication.
+void OccaBilinearForm::Mult_(const Vector &x, Vector &y) const
+{
+   otrialFESpace->GlobalToLocal(x, localX);
+   localY.OccaFill<double>(0.0);
+
+   const int integratorCount = (int) integrators.size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->MultAdd(localX, localY);
+   }
+
+   otestFESpace->LocalToGlobal(localY, y);
+}
+
+// Matrix transpose vector multiplication.
+void OccaBilinearForm::MultTranspose_(const Vector &x, Vector &y) const
+{
+   otestFESpace->GlobalToLocal(x, localX);
+   localY.OccaFill<double>(0.0);
+
+   const int integratorCount = (int) integrators.size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->MultTransposeAdd(localX, localY);
+   }
+
+   otrialFESpace->LocalToGlobal(localY, y);
+}
+
+void OccaBilinearForm::OccaRecoverFEMSolution(const mfem::Vector &X,
+                                              const mfem::Vector &b,
+                                              mfem::Vector &x)
+{
+   const mfem::Operator *P = this->GetTrialProlongation();
+   if (P)
+   {
+      // Apply conforming prolongation
+      x.Resize(P->OutLayout());
+      P->Mult(X, x);
+   }
+   // Otherwise X and x point to the same data
+}
+
+// Frees memory bilinear form.
+OccaBilinearForm::~OccaBilinearForm()
+{
+   // Make sure all integrators free their data
+   IntegratorVector::iterator it = integrators.begin();
+   while (it != integrators.end())
+   {
+      delete *it;
+      ++it;
+   }
+}
+
+
+void BilinearForm::InitOccaBilinearForm()
+{
+   // Init 'obform' using 'bform'
+   MFEM_ASSERT(bform != NULL, "");
+   MFEM_ASSERT(obform == NULL, "");
+
+   FiniteElementSpace &ofes =
+      bform->FESpace()->Get_PFESpace()->As<FiniteElementSpace>();
+   obform = new OccaBilinearForm(&ofes);
+
+   // Transfer domain integrators
+   mfem::Array<mfem::BilinearFormIntegrator*> &dbfi = *bform->GetDBFI();
+   for (int i = 0; i < dbfi.Size(); i++)
+   {
+      std::string integ_name(dbfi[i]->Name());
+      Coefficient *scal_coeff = dbfi[i]->GetScalarCoefficient();
+      ConstantCoefficient *const_coeff =
+         dynamic_cast<ConstantCoefficient*>(scal_coeff);
+      GridFunctionCoefficient *gridfunc_coeff =
+         dynamic_cast<GridFunctionCoefficient*>(scal_coeff);
+      // TODO: other types of coefficients ...
+
+      OccaCoefficient *ocoeff = NULL;
+      if (const_coeff)
+      {
+         ocoeff = new OccaCoefficient(obform->OccaEngine(),
+                                      const_coeff->constant);
+      }
+      else if (gridfunc_coeff)
+      {
+         ocoeff = new OccaCoefficient(obform->OccaEngine(),
+                                      *gridfunc_coeff->GetGridFunction(), true);
+      }
+      else if (!scal_coeff)
+      {
+         ocoeff = new OccaCoefficient(obform->OccaEngine(), 1.0);
+      }
+      else
+      {
+         MFEM_ABORT("Coefficient type not supported");
+      }
+
+      OccaIntegrator *ointeg = NULL;
+      if (integ_name == "(undefined)")
+      {
+         MFEM_ABORT("BilinearFormIntegrator does not define Name()");
+      }
+      else if (integ_name == "mass")
+      {
+         ointeg = new OccaMassIntegrator(*ocoeff);
+      }
+      else if (integ_name == "diffusion")
+      {
+         ointeg = new OccaDiffusionIntegrator(*ocoeff);
+      }
+      else
+      {
+         MFEM_ABORT("BilinearFormIntegrator [Name() = " << integ_name
+                    << "] is not supported");
+      }
+
+      // NOTE: The integrators copy ocoeff, so it can be deleted here so there
+      //       is no memory leak.
+      delete ocoeff;
+
+      const mfem::IntegrationRule *ir = dbfi[i]->GetIntRule();
+      if (ir) { ointeg->SetIntegrationRule(*ir); }
+
+      obform->AddDomainIntegrator(ointeg);
+   }
+
+   // TODO: other types of integrators ...
+}
+
+bool BilinearForm::Assemble()
+{
+   if (obform == NULL) { InitOccaBilinearForm(); }
+
+   obform->Assemble();
+
+   return true; // --> host assembly is not needed
+}
+
+void BilinearForm::FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
+                                    mfem::OperatorHandle &A)
+{
+   if (A.Type() == mfem::Operator::ANY_TYPE)
+   {
+      mfem::Operator *Aout = NULL;
+      obform->FormOperator(ess_tdof_list, Aout);
+      A.Reset(Aout);
+   }
+   else
+   {
+      MFEM_ABORT("Operator::Type is not supported, type = " << A.Type());
+   }
+}
+
+void BilinearForm::FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
+                                    mfem::Vector &x, mfem::Vector &b,
+                                    mfem::OperatorHandle &A,
+                                    mfem::Vector &X, mfem::Vector &B,
+                                    int copy_interior)
+{
+   FormSystemMatrix(ess_tdof_list, A);
+   obform->InitRHS(ess_tdof_list, x, b, A.Ptr(), X, B, copy_interior);
+}
+
+void BilinearForm::RecoverFEMSolution(const mfem::Vector &X,
+                                      const mfem::Vector &b,
+                                      mfem::Vector &x)
+{
+   obform->OccaRecoverFEMSolution(X, b, x);
+}
+
+BilinearForm::~BilinearForm()
+{
+   delete obform;
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/bilinearform.hpp

@@ -0,0 +1,213 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP
+#define MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "fespace.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+enum OccaIntegratorType
+{
+   DomainIntegrator       = 0,
+   BoundaryIntegrator     = 1,
+   InteriorFaceIntegrator = 2,
+   BoundaryFaceIntegrator = 3
+};
+
+class OccaIntegrator;
+
+
+/** Class for bilinear form - "Matrix" with associated FE space and
+    BLFIntegrators. */
+class OccaBilinearForm : public Operator
+{
+   friend class OccaIntegrator;
+
+protected:
+   typedef std::vector<OccaIntegrator*> IntegratorVector;
+
+   SharedPtr<const Engine> engine;
+
+   // State information
+   mutable mfem::Mesh *mesh;
+
+   mutable FiniteElementSpace *otrialFESpace;
+   mutable mfem::FiniteElementSpace *trialFESpace;
+
+   mutable FiniteElementSpace *otestFESpace;
+   mutable mfem::FiniteElementSpace *testFESpace;
+
+   IntegratorVector integrators;
+
+   // Device data
+   ::occa::properties baseKernelProps;
+
+   // The input and output vectors are mapped to local nodes for efficient
+   // operations. In other words, they are E-vectors.
+   // The size is: (number of elements) * (nodes in element) * (vector dim)
+   mutable Vector localX, localY;
+
+public:
+   OccaBilinearForm(FiniteElementSpace *ofespace_);
+
+   OccaBilinearForm(FiniteElementSpace *otrialFESpace_,
+                    FiniteElementSpace *otestFESpace_);
+
+   void Init(const Engine &e,
+             FiniteElementSpace *otrialFESpace_,
+             FiniteElementSpace *otestFESpace_);
+
+   const Engine &OccaEngine() const { return *engine; }
+
+   ::occa::device GetDevice(int idx = 0) const
+   { return engine->GetDevice(idx); }
+
+   // Useful mesh Information
+   int BaseGeom() const;
+   int GetDim() const;
+   int64_t GetNE() const;
+
+   mfem::Mesh& GetMesh() const;
+
+   FiniteElementSpace& GetTrialOccaFESpace() const;
+   FiniteElementSpace& GetTestOccaFESpace() const;
+
+   mfem::FiniteElementSpace& GetTrialFESpace() const;
+   mfem::FiniteElementSpace& GetTestFESpace() const;
+
+   // Useful FE information
+   int64_t GetTrialNDofs() const;
+   int64_t GetTestNDofs() const;
+
+   int64_t GetTrialVDim() const;
+   int64_t GetTestVDim() const;
+
+   const mfem::FiniteElement& GetTrialFE(const int i) const;
+   const mfem::FiniteElement& GetTestFE(const int i) const;
+
+   // Adds new Domain Integrator.
+   void AddDomainIntegrator(OccaIntegrator *integrator,
+                            const ::occa::properties &props =
+                               ::occa::properties());
+
+   // Adds new Boundary Integrator.
+   void AddBoundaryIntegrator(OccaIntegrator *integrator,
+                              const ::occa::properties &props =
+                                 ::occa::properties());
+
+   // Adds new interior Face Integrator.
+   void AddInteriorFaceIntegrator(OccaIntegrator *integrator,
+                                  const ::occa::properties &props =
+                                     ::occa::properties());
+
+   // Adds new boundary Face Integrator.
+   void AddBoundaryFaceIntegrator(OccaIntegrator *integrator,
+                                  const ::occa::properties &props =
+                                     ::occa::properties());
+
+   // Adds Integrator based on OccaIntegratorType
+   void AddIntegrator(OccaIntegrator *integrator,
+                      const ::occa::properties &props,
+                      const OccaIntegratorType itype);
+
+   virtual const mfem::Operator *GetTrialProlongation() const;
+   virtual const mfem::Operator *GetTestProlongation() const;
+
+   virtual const mfem::Operator *GetTrialRestriction() const;
+   virtual const mfem::Operator *GetTestRestriction() const;
+
+   // Assembles the form i.e. sums over all domain/bdr integrators.
+   virtual void Assemble();
+
+   void FormLinearSystem(const mfem::Array<int> &constraintList,
+                         mfem::Vector &x, mfem::Vector &b,
+                         mfem::Operator *&Aout,
+                         mfem::Vector &X, mfem::Vector &B,
+                         int copy_interior = 0);
+
+   void FormOperator(const mfem::Array<int> &constraintList,
+                     mfem::Operator *&Aout);
+
+   void InitRHS(const mfem::Array<int> &constraintList,
+                mfem::Vector &x, mfem::Vector &b,
+                mfem::Operator *Aout,
+                mfem::Vector &X, mfem::Vector &B,
+                int copy_interior = 0);
+
+   // overrides
+   virtual void Mult_(const Vector &x, Vector &y) const;
+   virtual void MultTranspose_(const Vector &x, Vector &y) const;
+
+   void OccaRecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
+                               mfem::Vector &x);
+
+   // Destroys bilinear form.
+   ~OccaBilinearForm();
+};
+
+
+/// TODO: doxygen
+class BilinearForm : public mfem::PBilinearForm
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // mfem::BilinearForm *bform;
+   OccaBilinearForm *obform;
+
+   // Called from Assemble() if obform is NULL to initialize obform.
+   void InitOccaBilinearForm();
+
+public:
+   /// TODO: doxygen
+   BilinearForm(const Engine &e, mfem::BilinearForm &bf)
+      : mfem::PBilinearForm(e, bf), obform(NULL) { }
+
+   /// Virtual destructor
+   virtual ~BilinearForm();
+
+   /// Assemble the PBilinearForm.
+   /** This method is called from the method mfem::BilinearForm::Assemble() of
+       the associated mfem::BilinearForm, #bform.
+       @returns True, if the host assembly should NOT be performed. */
+   virtual bool Assemble();
+
+   virtual void FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
+                                 mfem::OperatorHandle &A);
+
+   virtual void FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
+                                 mfem::Vector &x, mfem::Vector &b,
+                                 mfem::OperatorHandle &A,
+                                 mfem::Vector &X, mfem::Vector &B,
+                                 int copy_interior);
+
+   virtual void RecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
+                                   mfem::Vector &x);
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP

backends/occa/bilininteg.cpp

@@ -0,0 +1,954 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "bilininteg.hpp"
+#include "../../fem/fem.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+std::map<std::string, OccaDofQuadMaps> OccaDofQuadMaps::AllDofQuadMaps;
+
+OccaGeometry OccaGeometry::Get(::occa::device device,
+                               FiniteElementSpace &ofespace,
+                               const mfem::IntegrationRule &ir,
+                               const int flags)
+{
+   OccaGeometry geom;
+
+   mfem::Mesh &mesh = *(ofespace.GetMesh());
+   if (!mesh.GetNodes())
+   {
+      mesh.SetCurvature(1, false, -1, mfem::Ordering::byVDIM);
+   }
+   mfem::GridFunction &nodes = *(mesh.GetNodes());
+   const mfem::FiniteElementSpace &fespace = *(nodes.FESpace());
+   const mfem::FiniteElement &fe = *(fespace.GetFE(0));
+
+   const int dims     = fe.GetDim();
+   const int elements = fespace.GetNE();
+   const int numDofs  = fe.GetDof();
+   const int numQuad  = ir.GetNPoints();
+
+   MFEM_ASSERT(dims == mesh.SpaceDimension(), "");
+
+   geom.meshNodes.allocate(device,
+                           dims, numDofs, elements);
+
+   const mfem::Table &e2dTable = fespace.GetElementToDofTable();
+   const int *elementMap = e2dTable.GetJ();
+   nodes.Pull();
+   for (int e = 0; e < elements; ++e)
+   {
+      for (int dof = 0; dof < numDofs; ++dof)
+      {
+         const int gid = elementMap[dof + numDofs*e];
+         for (int dim = 0; dim < dims; ++dim)
+         {
+            geom.meshNodes(dim, dof, e) = nodes[fespace.DofToVDof(gid,dim)];
+         }
+      }
+   }
+   geom.meshNodes.keepInDevice();
+
+   if (flags & Jacobian)
+   {
+      geom.J.allocate(device,
+                      dims, dims, numQuad, elements);
+   }
+   else
+   {
+      geom.J.allocate(device, 1);
+   }
+   if (flags & JacobianInv)
+   {
+      geom.invJ.allocate(device,
+                         dims, dims, numQuad, elements);
+   }
+   else
+   {
+      geom.invJ.allocate(device, 1);
+   }
+   if (flags & JacobianDet)
+   {
+      geom.detJ.allocate(device,
+                         numQuad, elements);
+   }
+   else
+   {
+      geom.detJ.allocate(device, 1);
+   }
+
+   geom.J.stopManaging();
+   geom.invJ.stopManaging();
+   geom.detJ.stopManaging();
+
+   OccaDofQuadMaps &maps = OccaDofQuadMaps::GetSimplexMaps(device, fe, ir);
+
+   ::occa::properties props;
+   props["defines/NUM_DOFS"] = numDofs;
+   props["defines/NUM_QUAD"] = numQuad;
+   props["defines/STORE_JACOBIAN"]     = (flags & Jacobian);
+   props["defines/STORE_JACOBIAN_INV"] = (flags & JacobianInv);
+   props["defines/STORE_JACOBIAN_DET"] = (flags & JacobianDet);
+
+   const std::string &okl_path = ofespace.OccaEngine().GetOklPath();
+   ::occa::kernel init = device.buildKernel(okl_path + "geometry.okl",
+                                            stringWithDim("InitGeometryInfo",
+                                                          fe.GetDim()),
+                                            props);
+   init(elements,
+        maps.dofToQuadD,
+        geom.meshNodes,
+        geom.J, geom.invJ, geom.detJ);
+
+   return geom;
+}
+
+OccaDofQuadMaps::OccaDofQuadMaps() :
+   hash() {}
+
+OccaDofQuadMaps::OccaDofQuadMaps(const OccaDofQuadMaps &maps)
+{
+   *this = maps;
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::operator = (const OccaDofQuadMaps &maps)
+{
+   hash = maps.hash;
+   dofToQuad   = maps.dofToQuad;
+   dofToQuadD  = maps.dofToQuadD;
+   quadToDof   = maps.quadToDof;
+   quadToDofD  = maps.quadToDofD;
+   quadWeights = maps.quadWeights;
+   return *this;
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
+                                      const FiniteElementSpace &fespace,
+                                      const mfem::IntegrationRule &ir,
+                                      const bool transpose)
+{
+   return Get(device,
+              *fespace.GetFE(0),
+              *fespace.GetFE(0),
+              ir,
+              transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
+                                      const mfem::FiniteElement &fe,
+                                      const mfem::IntegrationRule &ir,
+                                      const bool transpose)
+{
+   return Get(device, fe, fe, ir, transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
+                                      const FiniteElementSpace &trialFESpace,
+                                      const FiniteElementSpace &testFESpace,
+                                      const mfem::IntegrationRule &ir,
+                                      const bool transpose)
+{
+   return Get(device,
+              *trialFESpace.GetFE(0),
+              *testFESpace.GetFE(0),
+              ir,
+              transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
+                                      const mfem::FiniteElement &trialFE,
+                                      const mfem::FiniteElement &testFE,
+                                      const mfem::IntegrationRule &ir,
+                                      const bool transpose)
+{
+   return (dynamic_cast<const mfem::TensorBasisElement*>(&trialFE)
+           ? GetTensorMaps(device, trialFE, testFE, ir, transpose)
+           : GetSimplexMaps(device, trialFE, testFE, ir, transpose));
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::GetTensorMaps(::occa::device device,
+                                                const mfem::FiniteElement &fe,
+                                                const mfem::IntegrationRule &ir,
+                                                const bool transpose)
+{
+   return GetTensorMaps(device,
+                        fe, fe,
+                        ir, transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::GetTensorMaps(::occa::device device,
+                                                const mfem::FiniteElement &trialFE,
+                                                const mfem::FiniteElement &testFE,
+                                                const mfem::IntegrationRule &ir,
+                                                const bool transpose)
+{
+   const mfem::TensorBasisElement &trialTFE =
+      dynamic_cast<const mfem::TensorBasisElement&>(trialFE);
+   const mfem::TensorBasisElement &testTFE =
+      dynamic_cast<const mfem::TensorBasisElement&>(testFE);
+
+   std::stringstream ss;
+   ss << ::occa::hash(device)
+      << "Tensor"
+      << "O1:"  << trialFE.GetOrder()
+      << "O2:"  << testFE.GetOrder()
+      << "BT1:" << trialTFE.GetBasisType()
+      << "BT2:" << testTFE.GetBasisType()
+      << "Q:"   << ir.GetNPoints();
+   std::string hash = ss.str();
+
+   // If we've already made the dof-quad maps, reuse them
+   OccaDofQuadMaps &maps = AllDofQuadMaps[hash];
+   if (!maps.hash.size())
+   {
+      // Create the dof-quad maps
+      maps.hash = hash;
+
+      OccaDofQuadMaps trialMaps = GetD2QTensorMaps(device, trialFE, ir);
+      OccaDofQuadMaps testMaps  = GetD2QTensorMaps(device, testFE , ir, true);
+
+      maps.dofToQuad   = trialMaps.dofToQuad;
+      maps.dofToQuadD  = trialMaps.dofToQuadD;
+      maps.quadToDof   = testMaps.dofToQuad;
+      maps.quadToDofD  = testMaps.dofToQuadD;
+      maps.quadWeights = testMaps.quadWeights;
+   }
+   return maps;
+}
+
+OccaDofQuadMaps OccaDofQuadMaps::GetD2QTensorMaps(::occa::device device,
+                                                  const mfem::FiniteElement &fe,
+                                                  const mfem::IntegrationRule &ir,
+                                                  const bool transpose)
+{
+   const mfem::TensorBasisElement &tfe =
+      dynamic_cast<const mfem::TensorBasisElement&>(fe);
+
+   const mfem::Poly_1D::Basis &basis = tfe.GetBasis1D();
+   const int order = fe.GetOrder();
+   // [MISSING] Get 1D dofs
+   const int dofs = order + 1;
+   const int dims = fe.GetDim();
+
+   // Create the dof -> quadrature point map
+   const mfem::IntegrationRule &ir1D =
+      mfem::IntRules.Get(mfem::Geometry::SEGMENT, ir.GetOrder());
+   const int quadPoints = ir1D.GetNPoints();
+   const int quadPoints2D = quadPoints*quadPoints;
+   const int quadPoints3D = quadPoints2D*quadPoints;
+   const int quadPointsND = ((dims == 1) ? quadPoints :
+                             ((dims == 2) ? quadPoints2D : quadPoints3D));
+
+   OccaDofQuadMaps maps;
+   // Initialize the dof -> quad mapping
+   maps.dofToQuad.allocate(device,
+                           quadPoints, dofs);
+   maps.dofToQuadD.allocate(device,
+                            quadPoints, dofs);
+
+   double *quadWeights1DData = NULL;
+
+   if (transpose)
+   {
+      maps.dofToQuad.reindex(1,0);
+      maps.dofToQuadD.reindex(1,0);
+      // Initialize quad weights only for transpose
+      maps.quadWeights.allocate(device,
+                                quadPointsND);
+      quadWeights1DData = new double[quadPoints];
+   }
+
+   mfem::Vector d2q(dofs);
+   mfem::Vector d2qD(dofs);
+   for (int q = 0; q < quadPoints; ++q)
+   {
+      const mfem::IntegrationPoint &ip = ir1D.IntPoint(q);
+      basis.Eval(ip.x, d2q, d2qD);
+      if (transpose)
+      {
+         quadWeights1DData[q] = ip.weight;
+      }
+      for (int d = 0; d < dofs; ++d)
+      {
+         maps.dofToQuad(q, d)  = d2q[d];
+         maps.dofToQuadD(q, d) = d2qD[d];
+      }
+   }
+
+   maps.dofToQuad.keepInDevice();
+   maps.dofToQuadD.keepInDevice();
+
+   if (transpose)
+   {
+      for (int q = 0; q < quadPointsND; ++q)
+      {
+         const int qx = q % quadPoints;
+         const int qz = q / quadPoints2D;
+         const int qy = (q - qz*quadPoints2D) / quadPoints;
+         double w = quadWeights1DData[qx];
+         if (dims > 1)
+         {
+            w *= quadWeights1DData[qy];
+         }
+         if (dims > 2)
+         {
+            w *= quadWeights1DData[qz];
+         }
+         maps.quadWeights[q] = w;
+      }
+      maps.quadWeights.keepInDevice();
+      delete [] quadWeights1DData;
+   }
+
+   return maps;
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::GetSimplexMaps(::occa::device device,
+                                                 const mfem::FiniteElement &fe,
+                                                 const mfem::IntegrationRule &ir,
+                                                 const bool transpose)
+{
+   return GetSimplexMaps(device,
+                         fe, fe,
+                         ir, transpose);
+}
+
+OccaDofQuadMaps& OccaDofQuadMaps::GetSimplexMaps(::occa::device device,
+                                                 const mfem::FiniteElement &trialFE,
+                                                 const mfem::FiniteElement &testFE,
+                                                 const mfem::IntegrationRule &ir,
+                                                 const bool transpose)
+{
+   std::stringstream ss;
+   ss << ::occa::hash(device)
+      << "Simplex"
+      << "O1:" << trialFE.GetOrder()
+      << "O2:" << testFE.GetOrder()
+      << "Q:"  << ir.GetNPoints();
+   std::string hash = ss.str();
+
+   // If we've already made the dof-quad maps, reuse them
+   OccaDofQuadMaps &maps = AllDofQuadMaps[hash];
+   if (!maps.hash.size())
+   {
+      // Create the dof-quad maps
+      maps.hash = hash;
+
+      OccaDofQuadMaps trialMaps = GetD2QSimplexMaps(device, trialFE, ir);
+      OccaDofQuadMaps testMaps  = GetD2QSimplexMaps(device, testFE , ir, true);
+
+      maps.dofToQuad   = trialMaps.dofToQuad;
+      maps.dofToQuadD  = trialMaps.dofToQuadD;
+      maps.quadToDof   = testMaps.dofToQuad;
+      maps.quadToDofD  = testMaps.dofToQuadD;
+      maps.quadWeights = testMaps.quadWeights;
+   }
+   return maps;
+}
+
+OccaDofQuadMaps OccaDofQuadMaps::GetD2QSimplexMaps(::occa::device device,
+                                                   const mfem::FiniteElement &fe,
+                                                   const mfem::IntegrationRule &ir,
+                                                   const bool transpose)
+{
+   const int dims = fe.GetDim();
+   const int numDofs = fe.GetDof();
+   const int numQuad = ir.GetNPoints();
+
+   OccaDofQuadMaps maps;
+   // Initialize the dof -> quad mapping
+   maps.dofToQuad.allocate(device,
+                           numQuad, numDofs);
+   maps.dofToQuadD.allocate(device,
+                            dims, numQuad, numDofs);
+
+   if (transpose)
+   {
+      maps.dofToQuad.reindex(1,0);
+      maps.dofToQuadD.reindex(1,0);
+      // Initialize quad weights only for transpose
+      maps.quadWeights.allocate(device,
+                                numQuad);
+   }
+
+   mfem::Vector d2q(numDofs);
+   mfem::DenseMatrix d2qD(numDofs, dims);
+   for (int q = 0; q < numQuad; ++q)
+   {
+      const mfem::IntegrationPoint &ip = ir.IntPoint(q);
+      if (transpose)
+      {
+         maps.quadWeights[q] = ip.weight;
+      }
+      fe.CalcShape(ip, d2q);
+      fe.CalcDShape(ip, d2qD);
+      for (int d = 0; d < numDofs; ++d)
+      {
+         const double w = d2q[d];
+         maps.dofToQuad(q, d) = w;
+         for (int dim = 0; dim < dims; ++dim)
+         {
+            const double wD = d2qD(d, dim);
+            maps.dofToQuadD(dim, q, d) = wD;
+         }
+      }
+   }
+
+   maps.dofToQuad.keepInDevice();
+   maps.dofToQuadD.keepInDevice();
+   if (transpose)
+   {
+      maps.quadWeights.keepInDevice();
+   }
+
+   return maps;
+}
+
+//---[ Integrator Defines ]-----------
+std::string stringWithDim(const std::string &s, const int dim)
+{
+   std::string ret = s;
+   ret += ('0' + (char) dim);
+   ret += 'D';
+   return ret;
+}
+
+int closestWarpBatchTo(const int value)
+{
+   return ((value + 31) / 32) * 32;
+}
+
+int closestMultipleWarpBatch(const int multiple, const int maxSize)
+{
+   if (multiple > maxSize)
+   {
+      return maxSize;
+   }
+   int batch = (32 / multiple);
+   int minDiff = 32 - (multiple * batch);
+   for (int i = 64; i <= maxSize; i += 32)
+   {
+      const int newDiff = i - (multiple * (i / multiple));
+      if (newDiff < minDiff)
+      {
+         batch = (i / multiple);
+         minDiff = newDiff;
+      }
+   }
+   return batch;
+}
+
+void SetProperties(FiniteElementSpace &fespace,
+                   const mfem::IntegrationRule &ir,
+                   ::occa::properties &props)
+{
+   SetProperties(fespace, fespace, ir, props);
+}
+
+void SetProperties(FiniteElementSpace &trialFESpace,
+                   FiniteElementSpace &testFESpace,
+                   const mfem::IntegrationRule &ir,
+                   ::occa::properties &props)
+{
+   props["defines/TRIAL_VDIM"] = trialFESpace.GetVDim();
+   props["defines/TEST_VDIM"]  = testFESpace.GetVDim();
+   props["defines/NUM_DIM"]    = trialFESpace.GetDim();
+
+   if (trialFESpace.hasTensorBasis())
+   {
+      SetTensorProperties(trialFESpace, testFESpace, ir, props);
+   }
+   else
+   {
+      SetSimplexProperties(trialFESpace, testFESpace, ir, props);
+   }
+}
+
+void SetTensorProperties(FiniteElementSpace &fespace,
+                         const mfem::IntegrationRule &ir,
+                         ::occa::properties &props)
+{
+   SetTensorProperties(fespace, fespace, ir, props);
+}
+
+void SetTensorProperties(FiniteElementSpace &trialFESpace,
+                         FiniteElementSpace &testFESpace,
+                         const mfem::IntegrationRule &ir,
+                         ::occa::properties &props)
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace.GetFE(0));
+   const mfem::FiniteElement &testFE = *(testFESpace.GetFE(0));
+
+   const mfem::IntegrationRule &ir1D =
+      mfem::IntRules.Get(mfem::Geometry::SEGMENT, ir.GetOrder());
+
+   const int trialDofs = trialFE.GetDof();
+   const int testDofs  = testFE.GetDof();
+   const int numQuad  = ir.GetNPoints();
+
+   const int trialDofs1D = trialFE.GetOrder() + 1;
+   const int testDofs1D  = testFE.GetOrder() + 1;
+   const int quad1D  = ir1D.GetNPoints();
+   int trialDofsND = trialDofs1D;
+   int testDofsND  = testDofs1D;
+   int quadND  = quad1D;
+
+   const bool trialByVDIM = (trialFESpace.GetOrdering() == mfem::Ordering::byVDIM);
+   const bool testByVDIM  = (testFESpace.GetOrdering()  == mfem::Ordering::byVDIM);
+
+   props["defines/ORDERING_BY_NODES"] = 0;
+   props["defines/ORDERING_BY_VDIM"]  = 1;
+   props["defines/VDIM_ORDERING"]  = (int) trialByVDIM;
+   props["defines/TRIAL_ORDERING"] = (int) trialByVDIM;
+   props["defines/TEST_ORDERING"]  = (int) testByVDIM;
+
+   props["defines/USING_TENSOR_OPS"] = 1;
+   props["defines/NUM_DOFS"]  = trialDofs;
+   props["defines/NUM_QUAD"]  = numQuad;
+
+   props["defines/TRIAL_DOFS"] = trialDofs;
+   props["defines/TEST_DOFS"]  = testDofs;
+
+   for (int d = 1; d <= 3; ++d)
+   {
+      if (d > 1)
+      {
+         trialDofsND *= trialDofs1D;
+         testDofsND  *= testDofs1D;
+         quadND *= quad1D;
+      }
+      props["defines"][stringWithDim("NUM_DOFS_", d)] = trialDofsND;
+      props["defines"][stringWithDim("NUM_QUAD_", d)] = quadND;
+
+      props["defines"][stringWithDim("TRIAL_DOFS_", d)] = trialDofsND;
+      props["defines"][stringWithDim("TEST_DOFS_" , d)] = testDofsND;
+   }
+
+   // 1D Defines
+   const int m1InnerBatch = 32 * ((quad1D + 31) / 32);
+   props["defines/A1_ELEMENT_BATCH"]       = closestMultipleWarpBatch(quad1D, 512);
+   props["defines/M1_OUTER_ELEMENT_BATCH"] = closestMultipleWarpBatch(m1InnerBatch,
+                                                                      512);
+   props["defines/M1_INNER_ELEMENT_BATCH"] = m1InnerBatch;
+
+   // 2D Defines
+   props["defines/A2_ELEMENT_BATCH"] = 1;
+   props["defines/A2_QUAD_BATCH"]    = 1;
+   props["defines/M2_ELEMENT_BATCH"] = 32;
+
+   // 3D Defines
+   const int a3QuadBatch = closestMultipleWarpBatch(quadND, 512);
+   props["defines/A3_ELEMENT_BATCH"] = closestMultipleWarpBatch(a3QuadBatch, 512);
+   props["defines/A3_QUAD_BATCH"]    = a3QuadBatch;
+}
+
+void SetSimplexProperties(FiniteElementSpace &fespace,
+                          const mfem::IntegrationRule &ir,
+                          ::occa::properties &props)
+{
+   SetSimplexProperties(fespace, fespace, ir, props);
+}
+
+void SetSimplexProperties(FiniteElementSpace &trialFESpace,
+                          FiniteElementSpace &testFESpace,
+                          const mfem::IntegrationRule &ir,
+                          ::occa::properties &props)
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace.GetFE(0));
+   const mfem::FiniteElement &testFE = *(testFESpace.GetFE(0));
+
+   const int trialDofs = trialFE.GetDof();
+   const int testDofs  = testFE.GetDof();
+   const int numQuad = ir.GetNPoints();
+   const int maxDQ   = std::max(std::max(trialDofs, testDofs), numQuad);
+
+   const bool trialByVDIM = (trialFESpace.GetOrdering() == mfem::Ordering::byVDIM);
+   const bool testByVDIM  = (testFESpace.GetOrdering()  == mfem::Ordering::byVDIM);
+
+   props["defines/ORDERING_BY_NODES"] = 0;
+   props["defines/ORDERING_BY_VDIM"]  = 1;
+   props["defines/VDIM_ORDERING"]  = (int) trialByVDIM;
+   props["defines/TRIAL_ORDERING"] = (int) trialByVDIM;
+   props["defines/TEST_ORDERING"]  = (int) testByVDIM;
+
+   props["defines/USING_TENSOR_OPS"] = 0;
+   props["defines/NUM_DOFS"]  = trialDofs;
+   props["defines/NUM_QUAD"]  = numQuad;
+
+   props["defines/TRIAL_DOFS"] = trialDofs;
+   props["defines/TEST_DOFS"]  = testDofs;
+
+   // 2D Defines
+   const int quadBatch = closestWarpBatchTo(numQuad);
+   props["defines/A2_ELEMENT_BATCH"] = closestMultipleWarpBatch(quadBatch, 2048);
+   props["defines/A2_QUAD_BATCH"]    = quadBatch;
+   props["defines/M2_INNER_BATCH"]   = closestWarpBatchTo(maxDQ);
+
+   // 3D Defines
+   props["defines/A3_ELEMENT_BATCH"] = closestMultipleWarpBatch(quadBatch, 2048);
+   props["defines/A3_QUAD_BATCH"]    = quadBatch;
+   props["defines/M3_INNER_BATCH"]   = closestWarpBatchTo(maxDQ);
+}
+
+
+//---[ Base Integrator ]--------------
+OccaIntegrator::OccaIntegrator(const Engine &e)
+   : engine(&e),
+     bform(),
+     mesh(),
+     otrialFESpace(),
+     otestFESpace(),
+     trialFESpace(),
+     testFESpace(),
+     itype(DomainIntegrator),
+     ir(NULL),
+     hasTensorBasis(false) { }
+
+OccaIntegrator::~OccaIntegrator() {}
+
+void OccaIntegrator::SetupMaps()
+{
+   maps = OccaDofQuadMaps::Get(GetDevice(),
+                               *otrialFESpace,
+                               *otestFESpace,
+                               *ir);
+
+   mapsTranspose = OccaDofQuadMaps::Get(GetDevice(),
+                                        *otestFESpace,
+                                        *otrialFESpace,
+                                        *ir);
+}
+
+FiniteElementSpace& OccaIntegrator::GetTrialOccaFESpace() const
+{
+   return *otrialFESpace;
+}
+
+FiniteElementSpace& OccaIntegrator::GetTestOccaFESpace() const
+{
+   return *otestFESpace;
+}
+
+mfem::FiniteElementSpace& OccaIntegrator::GetTrialFESpace() const
+{
+   return *trialFESpace;
+}
+
+mfem::FiniteElementSpace& OccaIntegrator::GetTestFESpace() const
+{
+   return *testFESpace;
+}
+
+void OccaIntegrator::SetIntegrationRule(const mfem::IntegrationRule &ir_)
+{
+   ir = &ir_;
+}
+
+const mfem::IntegrationRule& OccaIntegrator::GetIntegrationRule() const
+{
+   return *ir;
+}
+
+OccaDofQuadMaps& OccaIntegrator::GetDofQuadMaps()
+{
+   return maps;
+}
+
+void OccaIntegrator::SetupIntegrator(OccaBilinearForm &bform_,
+                                     const ::occa::properties &props_,
+                                     const OccaIntegratorType itype_)
+{
+   MFEM_ASSERT(engine == &bform_.OccaEngine(), "");
+   bform     = &bform_;
+   mesh      = &(bform_.GetMesh());
+
+   otrialFESpace = &(bform_.GetTrialOccaFESpace());
+   otestFESpace  = &(bform_.GetTestOccaFESpace());
+
+   trialFESpace = &(bform_.GetTrialFESpace());
+   testFESpace  = &(bform_.GetTestFESpace());
+
+   hasTensorBasis = otrialFESpace->hasTensorBasis();
+
+   props = props_;
+   itype = itype_;
+
+   if (ir == NULL)
+   {
+      SetupIntegrationRule();
+   }
+
+   SetupMaps();
+
+   SetProperties(*otrialFESpace,
+                 *otestFESpace,
+                 *ir,
+                 props);
+
+   Setup();
+}
+
+OccaGeometry OccaIntegrator::GetGeometry(const int flags)
+{
+   return OccaGeometry::Get(GetDevice(), *otrialFESpace, *ir, flags);
+}
+
+::occa::kernel OccaIntegrator::GetAssembleKernel(const ::occa::properties
+                                                 &props)
+{
+   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
+   return GetKernel(stringWithDim("Assemble", fe.GetDim()),
+                    props);
+}
+
+::occa::kernel OccaIntegrator::GetMultAddKernel(const ::occa::properties &props)
+{
+   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
+   return GetKernel(stringWithDim("MultAdd", fe.GetDim()),
+                    props);
+}
+
+::occa::kernel OccaIntegrator::GetKernel(const std::string &kernelName,
+                                         const ::occa::properties &props)
+{
+   const std::string filename = GetName() + ".okl";
+   const std::string &okl_path = OccaEngine().GetOklPath();
+   return GetDevice().buildKernel(okl_path + filename,
+                                  kernelName,
+                                  props);
+}
+//====================================
+
+
+//---[ Diffusion Integrator ]---------
+OccaDiffusionIntegrator::OccaDiffusionIntegrator(const OccaCoefficient &coeff_)
+   :
+   OccaIntegrator(coeff_.OccaEngine()),
+   coeff(coeff_),
+   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
+{
+   coeff.SetName("COEFF");
+}
+
+OccaDiffusionIntegrator::~OccaDiffusionIntegrator() {}
+
+
+std::string OccaDiffusionIntegrator::GetName()
+{
+   return "DiffusionIntegrator";
+}
+
+void OccaDiffusionIntegrator::SetupIntegrationRule()
+{
+   const FiniteElement &trialFE = *(trialFESpace->GetFE(0));
+   const FiniteElement &testFE  = *(testFESpace->GetFE(0));
+   ir = &mfem::DiffusionIntegrator::GetRule(trialFE, testFE);
+}
+
+void OccaDiffusionIntegrator::Setup()
+{
+   ::occa::properties kernelProps = props;
+
+   coeff.Setup(*this, kernelProps);
+
+   // Setup assemble and mult kernels
+   assembleKernel = GetAssembleKernel(kernelProps);
+   multKernel     = GetMultAddKernel(kernelProps);
+}
+
+void OccaDiffusionIntegrator::Assemble()
+{
+   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
+
+   const int dims = fe.GetDim();
+   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+
+   const int elements = trialFESpace->GetNE();
+   const int quadraturePoints = ir->GetNPoints();
+
+   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
+
+   assembledOperator.OccaResize(symmDims * quadraturePoints * elements,
+                                sizeof(double));
+
+   assembleKernel((int) mesh->GetNE(),
+                  maps.quadWeights,
+                  geom.J,
+                  coeff,
+                  assembledOperator.OccaMem());
+}
+
+void OccaDiffusionIntegrator::MultAdd(Vector &x, Vector &y)
+{
+   // Note: x and y are E-vectors
+
+   multKernel((int) mesh->GetNE(),
+              maps.dofToQuad,
+              maps.dofToQuadD,
+              maps.quadToDof,
+              maps.quadToDofD,
+              assembledOperator.OccaMem(),
+              x.OccaMem(), y.OccaMem());
+}
+//====================================
+
+
+//---[ Mass Integrator ]--------------
+OccaMassIntegrator::OccaMassIntegrator(const OccaCoefficient &coeff_) :
+   OccaIntegrator(coeff_.OccaEngine()),
+   coeff(coeff_),
+   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
+{
+   coeff.SetName("COEFF");
+}
+
+OccaMassIntegrator::~OccaMassIntegrator() {}
+
+std::string OccaMassIntegrator::GetName()
+{
+   return "MassIntegrator";
+}
+
+void OccaMassIntegrator::SetupIntegrationRule()
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace->GetFE(0));
+   const mfem::FiniteElement &testFE  = *(testFESpace->GetFE(0));
+   mfem::ElementTransformation &T = *trialFESpace->GetElementTransformation(0);
+   ir = &mfem::MassIntegrator::GetRule(trialFE, testFE, T);
+}
+
+void OccaMassIntegrator::Setup()
+{
+   ::occa::properties kernelProps = props;
+
+   coeff.Setup(*this, kernelProps);
+
+   // Setup assemble and mult kernels
+   assembleKernel = GetAssembleKernel(kernelProps);
+   multKernel     = GetMultAddKernel(kernelProps);
+}
+
+void OccaMassIntegrator::Assemble()
+{
+   if (assembledOperator.Size())
+   {
+      return;
+   }
+
+   const int elements = trialFESpace->GetNE();
+   const int quadraturePoints = ir->GetNPoints();
+
+   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
+
+   assembledOperator.Resize<double>(quadraturePoints * elements, NULL);
+
+   assembleKernel((int) mesh->GetNE(),
+                  maps.quadWeights,
+                  geom.J,
+                  coeff,
+                  assembledOperator.OccaMem());
+}
+
+void OccaMassIntegrator::SetOperator(Vector &v)
+{
+   assembledOperator = v;
+}
+
+void OccaMassIntegrator::MultAdd(Vector &x, Vector &y)
+{
+   multKernel((int) mesh->GetNE(),
+              maps.dofToQuad,
+              maps.dofToQuadD,
+              maps.quadToDof,
+              maps.quadToDofD,
+              assembledOperator.OccaMem(),
+              x.OccaMem(), y.OccaMem());
+}
+//====================================
+
+
+//---[ Vector Mass Integrator ]--------------
+OccaVectorMassIntegrator::OccaVectorMassIntegrator(const OccaCoefficient &
+                                                   coeff_)
+   :
+   OccaIntegrator(coeff_.OccaEngine()),
+   coeff(coeff_),
+   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
+{
+   coeff.SetName("COEFF");
+}
+
+OccaVectorMassIntegrator::~OccaVectorMassIntegrator() {}
+
+std::string OccaVectorMassIntegrator::GetName()
+{
+   return "VectorMassIntegrator";
+}
+
+void OccaVectorMassIntegrator::SetupIntegrationRule()
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace->GetFE(0));
+   const mfem::FiniteElement &testFE  = *(testFESpace->GetFE(0));
+   mfem::ElementTransformation &T = *trialFESpace->GetElementTransformation(0);
+   ir = &mfem::MassIntegrator::GetRule(trialFE, testFE, T);
+}
+
+void OccaVectorMassIntegrator::Setup()
+{
+   ::occa::properties kernelProps = props;
+
+   coeff.Setup(*this, kernelProps);
+
+   // Setup assemble and mult kernels
+   assembleKernel = GetAssembleKernel(kernelProps);
+   multKernel     = GetMultAddKernel(kernelProps);
+}
+
+void OccaVectorMassIntegrator::Assemble()
+{
+   const int elements = trialFESpace->GetNE();
+   const int quadraturePoints = ir->GetNPoints();
+
+   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
+
+   assembledOperator.Resize<double>(quadraturePoints * elements, NULL);
+
+   assembleKernel((int) mesh->GetNE(),
+                  maps.quadWeights,
+                  geom.J,
+                  coeff,
+                  assembledOperator.OccaMem());
+}
+
+void OccaVectorMassIntegrator::MultAdd(Vector &x, Vector &y)
+{
+   multKernel((int) mesh->GetNE(),
+              maps.dofToQuad,
+              maps.dofToQuadD,
+              maps.quadToDof,
+              maps.quadToDofD,
+              assembledOperator.OccaMem(),
+              x.OccaMem(), y.OccaMem());
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/bilininteg.hpp

@@ -0,0 +1,323 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP
+#define MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+#include "coefficient.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class OccaGeometry
+{
+public:
+   ::occa::array<double> meshNodes;
+   ::occa::array<double> J, invJ, detJ;
+
+   // byVDIM  -> [x y z x y z x y z]
+   // byNodes -> [x x x y y y z z z]
+   static const int Jacobian    = (1 << 0);
+   static const int JacobianInv = (1 << 1);
+   static const int JacobianDet = (1 << 2);
+
+   static OccaGeometry Get(::occa::device device,
+                           FiniteElementSpace &ofespace,
+                           const IntegrationRule &ir,
+                           const int flags = (Jacobian    |
+                                              JacobianInv |
+                                              JacobianDet));
+};
+
+class OccaDofQuadMaps
+{
+private:
+   // Reuse dof-quad maps
+   static std::map<std::string, OccaDofQuadMaps> AllDofQuadMaps;
+   std::string hash;
+
+public:
+   // Local stiffness matrices (B and B^T operators)
+   ::occa::array<double, ::occa::dynamic> dofToQuad, dofToQuadD; // B
+   ::occa::array<double, ::occa::dynamic> quadToDof, quadToDofD; // B^T
+   ::occa::array<double> quadWeights;
+
+   OccaDofQuadMaps();
+   OccaDofQuadMaps(const OccaDofQuadMaps &maps);
+   OccaDofQuadMaps& operator = (const OccaDofQuadMaps &maps);
+
+   // [[x y] [x y] [x y]]
+   // [[x y z] [x y z] [x y z]]
+   // mfem::GridFunction* mfem::Mesh::GetNodes() { return Nodes; }
+
+   // mfem::FiniteElementSpace *Nodes->FESpace()
+   // 25
+   // 1D [x x x x x x]
+   // 2D [x y x y x y]
+   // GetVdim()
+   // 3D ordering == byVDIM  -> [x y z x y z x y z x y z x y z x y z]
+   //    ordering == byNODES -> [x x x x x x y y y y y y z z z z z z]
+   static OccaDofQuadMaps& Get(::occa::device device,
+                               const FiniteElementSpace &fespace,
+                               const mfem::IntegrationRule &ir,
+                               const bool transpose = false);
+
+   static OccaDofQuadMaps& Get(::occa::device device,
+                               const mfem::FiniteElement &fe,
+                               const mfem::IntegrationRule &ir,
+                               const bool transpose = false);
+
+   static OccaDofQuadMaps& Get(::occa::device device,
+                               const FiniteElementSpace &trialFESpace,
+                               const FiniteElementSpace &testFESpace,
+                               const mfem::IntegrationRule &ir,
+                               const bool transpose = false);
+
+   static OccaDofQuadMaps& Get(::occa::device device,
+                               const mfem::FiniteElement &trialFE,
+                               const mfem::FiniteElement &testFE,
+                               const mfem::IntegrationRule &ir,
+                               const bool transpose = false);
+
+   static OccaDofQuadMaps& GetTensorMaps(::occa::device device,
+                                         const mfem::FiniteElement &fe,
+                                         const mfem::IntegrationRule &ir,
+                                         const bool transpose = false);
+
+   static OccaDofQuadMaps& GetTensorMaps(::occa::device device,
+                                         const mfem::FiniteElement &trialFE,
+                                         const mfem::FiniteElement &testFE,
+                                         const mfem::IntegrationRule &ir,
+                                         const bool transpose = false);
+
+   static OccaDofQuadMaps GetD2QTensorMaps(::occa::device device,
+                                           const mfem::FiniteElement &fe,
+                                           const mfem::IntegrationRule &ir,
+                                           const bool transpose = false);
+
+   static OccaDofQuadMaps& GetSimplexMaps(::occa::device device,
+                                          const mfem::FiniteElement &fe,
+                                          const mfem::IntegrationRule &ir,
+                                          const bool transpose = false);
+
+   static OccaDofQuadMaps& GetSimplexMaps(::occa::device device,
+                                          const mfem::FiniteElement &trialFE,
+                                          const mfem::FiniteElement &testFE,
+                                          const mfem::IntegrationRule &ir,
+                                          const bool transpose = false);
+
+   static OccaDofQuadMaps GetD2QSimplexMaps(::occa::device device,
+                                            const mfem::FiniteElement &fe,
+                                            const mfem::IntegrationRule &ir,
+                                            const bool transpose = false);
+};
+
+//---[ Define Methods ]---------------
+std::string stringWithDim(const std::string &s, const int dim);
+int closestWarpBatch(const int multiple, const int maxSize);
+
+void SetProperties(FiniteElementSpace &fespace,
+                   const mfem::IntegrationRule &ir,
+                   ::occa::properties &props);
+
+void SetProperties(FiniteElementSpace &trialFESpace,
+                   FiniteElementSpace &testFESpace,
+                   const mfem::IntegrationRule &ir,
+                   ::occa::properties &props);
+
+void SetTensorProperties(FiniteElementSpace &fespace,
+                         const mfem::IntegrationRule &ir,
+                         ::occa::properties &props);
+
+void SetTensorProperties(FiniteElementSpace &trialFESpace,
+                         FiniteElementSpace &testFESpace,
+                         const IntegrationRule &ir,
+                         ::occa::properties &props);
+
+void SetSimplexProperties(FiniteElementSpace &fespace,
+                          const IntegrationRule &ir,
+                          ::occa::properties &props);
+
+void SetSimplexProperties(FiniteElementSpace &trialFESpace,
+                          FiniteElementSpace &testFESpace,
+                          const IntegrationRule &ir,
+                          ::occa::properties &props);
+
+//---[ Base Integrator ]--------------
+class OccaIntegrator
+{
+protected:
+   SharedPtr<const Engine> engine;
+
+   OccaBilinearForm *bform;
+   mfem::Mesh *mesh;
+
+   FiniteElementSpace *otrialFESpace;
+   FiniteElementSpace *otestFESpace;
+
+   mfem::FiniteElementSpace *trialFESpace;
+   mfem::FiniteElementSpace *testFESpace;
+
+   ::occa::properties props;
+   OccaIntegratorType itype;
+
+   const IntegrationRule *ir;
+   bool hasTensorBasis;
+   OccaDofQuadMaps maps;
+   OccaDofQuadMaps mapsTranspose;
+
+public:
+   OccaIntegrator(const Engine &e);
+   virtual ~OccaIntegrator();
+
+   const Engine &OccaEngine() const { return *engine; }
+
+   ::occa::device GetDevice(int idx = 0) const
+   { return engine->GetDevice(idx); }
+
+   virtual std::string GetName() = 0;
+
+   FiniteElementSpace& GetTrialOccaFESpace() const;
+   FiniteElementSpace& GetTestOccaFESpace() const;
+
+   mfem::FiniteElementSpace& GetTrialFESpace() const;
+   mfem::FiniteElementSpace& GetTestFESpace() const;
+
+   void SetIntegrationRule(const mfem::IntegrationRule &ir_);
+   const mfem::IntegrationRule& GetIntegrationRule() const;
+
+   OccaDofQuadMaps& GetDofQuadMaps();
+
+   void SetupMaps();
+
+   virtual void SetupIntegrationRule() = 0;
+
+   virtual void SetupIntegrator(OccaBilinearForm &bform_,
+                                const ::occa::properties &props_,
+                                const OccaIntegratorType itype_);
+
+   virtual void Setup() = 0;
+
+   virtual void Assemble() = 0;
+   /// This method works on E-vectors!
+   virtual void MultAdd(Vector &x, Vector &y) = 0;
+
+   virtual void MultTransposeAdd(Vector &x, Vector &y)
+   {
+      mfem_error("OccaIntegrator::MultTransposeAdd() is not overloaded!");
+   }
+
+   OccaGeometry GetGeometry(const int flags = (OccaGeometry::Jacobian    |
+                                               OccaGeometry::JacobianInv |
+                                               OccaGeometry::JacobianDet));
+
+   ::occa::kernel GetAssembleKernel(const ::occa::properties &props);
+   ::occa::kernel GetMultAddKernel(const ::occa::properties &props);
+
+   ::occa::kernel GetKernel(const std::string &kernelName,
+                            const ::occa::properties &props);
+};
+//====================================
+
+
+//---[ Diffusion Integrator ]---------
+class OccaDiffusionIntegrator : public OccaIntegrator
+{
+private:
+   OccaCoefficient coeff;
+
+   ::occa::kernel assembleKernel, multKernel;
+
+   Vector assembledOperator;
+
+public:
+   OccaDiffusionIntegrator(const OccaCoefficient &coeff_);
+   virtual ~OccaDiffusionIntegrator();
+
+   virtual std::string GetName();
+
+   virtual void SetupIntegrationRule();
+
+   virtual void Setup();
+
+   virtual void Assemble();
+   virtual void MultAdd(Vector &x, Vector &y);
+};
+//====================================
+
+
+//---[ Mass Integrator ]--------------
+class OccaMassIntegrator : public OccaIntegrator
+{
+private:
+   OccaCoefficient coeff;
+
+   ::occa::kernel assembleKernel, multKernel;
+
+   Vector assembledOperator;
+
+public:
+   OccaMassIntegrator(const OccaCoefficient &coeff_);
+   virtual ~OccaMassIntegrator();
+
+   virtual std::string GetName();
+
+   virtual void SetupIntegrationRule();
+
+   virtual void Setup();
+
+   virtual void Assemble();
+   void SetOperator(Vector &v);
+
+   virtual void MultAdd(Vector &x, Vector &y);
+};
+//====================================
+
+//---[ Vector Mass Integrator ]--------------
+class OccaVectorMassIntegrator : public OccaIntegrator
+{
+private:
+   OccaCoefficient coeff;
+
+   ::occa::kernel assembleKernel, multKernel;
+
+   Vector assembledOperator;
+
+public:
+   OccaVectorMassIntegrator(const OccaCoefficient &coeff_);
+   virtual ~OccaVectorMassIntegrator();
+
+   virtual std::string GetName();
+
+   virtual void SetupIntegrationRule();
+
+   virtual void Setup();
+
+   virtual void Assemble();
+
+   virtual void MultAdd(Vector &x, Vector &y);
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP

backends/occa/coefficient.cpp

@@ -0,0 +1,357 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "coefficient.hpp"
+#include "bilininteg.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+//---[ Parameter ]------------
+OccaParameter::~OccaParameter() {}
+
+void OccaParameter::Setup(OccaIntegrator &integ,
+                          ::occa::properties &props) {}
+
+::occa::kernelArg OccaParameter::KernelArgs()
+{
+   return ::occa::kernelArg();
+}
+//====================================
+
+
+//---[ Include Parameter ]------------
+OccaIncludeParameter::OccaIncludeParameter(const std::string &filename_) :
+   filename(filename_) {}
+
+OccaParameter* OccaIncludeParameter::Clone()
+{
+   return new OccaIncludeParameter(filename);
+}
+
+void OccaIncludeParameter::Setup(OccaIntegrator &integ,
+                                 ::occa::properties &props)
+{
+   props["headers"].asArray() += "#include " + filename;
+}
+//====================================
+
+
+//---[ Source Parameter ]------------
+OccaSourceParameter::OccaSourceParameter(const std::string &source_) :
+   source(source_) {}
+
+OccaParameter* OccaSourceParameter::Clone()
+{
+   return new OccaSourceParameter(source);
+}
+
+void OccaSourceParameter::Setup(OccaIntegrator &integ,
+                                ::occa::properties &props)
+{
+   props["headers"].asArray() += source;
+}
+//====================================
+
+//---[ Vector Parameter ]-------
+OccaVectorParameter::OccaVectorParameter(const std::string &name_,
+                                         Vector &v_,
+                                         const bool useRestrict_) :
+   name(name_),
+   v(v_),
+   useRestrict(useRestrict_),
+   attr("") {}
+
+OccaVectorParameter::OccaVectorParameter(const std::string &name_,
+                                         Vector &v_,
+                                         const std::string &attr_,
+                                         const bool useRestrict_) :
+   name(name_),
+   v(v_),
+   useRestrict(useRestrict_),
+   attr(attr_) {}
+
+OccaParameter* OccaVectorParameter::Clone()
+{
+   return new OccaVectorParameter(name, v, attr, useRestrict);
+}
+
+void OccaVectorParameter::Setup(OccaIntegrator &integ,
+                                ::occa::properties &props)
+{
+   std::string &args = (props["defines/COEFF_ARGS"]
+                        .asString()
+                        .string());
+   args += "const double *";
+   if (useRestrict)
+   {
+      args += " restrict ";
+   }
+   args += name;
+   if (attr.size())
+   {
+      args += ' ';
+      args += attr;
+   }
+   args += ",\n";
+}
+
+::occa::kernelArg OccaVectorParameter::KernelArgs()
+{
+   return ::occa::kernelArg(v.OccaMem());
+}
+//====================================
+
+
+//---[ GridFunction Parameter ]-------
+OccaGridFunctionParameter::OccaGridFunctionParameter(const std::string &name_,
+                                                     const Engine &e,
+                                                     mfem::GridFunction &gf_,
+                                                     const bool useRestrict_)
+   : name(name_),
+     gf(gf_),
+     gfQuad(e),
+     useRestrict(useRestrict_) {}
+
+OccaParameter* OccaGridFunctionParameter::Clone()
+{
+   OccaGridFunctionParameter *param =
+      new OccaGridFunctionParameter(name, gfQuad.OccaEngine(), gf, useRestrict);
+   param->gfQuad.MakeRef(gfQuad);
+   return param;
+}
+
+void OccaGridFunctionParameter::Setup(OccaIntegrator &integ,
+                                      ::occa::properties &props)
+{
+
+   std::string &args = (props["defines/COEFF_ARGS"]
+                        .asString()
+                        .string());
+   if (useRestrict)
+   {
+      args += "@restrict ";
+   }
+   args += "const double *";
+   args += name;
+   args += " @dim(NUM_QUAD, numElements),\n";
+
+   FiniteElementSpace &f = gf.FESpace()->Get_PFESpace()->As<FiniteElementSpace>();
+   ToQuad(integ.GetIntegrationRule(), f, gf.Get_PVector()->As<Vector>(), gfQuad);
+}
+
+::occa::kernelArg OccaGridFunctionParameter::KernelArgs()
+{
+   return gfQuad.OccaMem();
+}
+//====================================
+
+
+//---[ Coefficient ]------------------
+OccaCoefficient::OccaCoefficient(const Engine &e, const double value) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   coeffValue = value;
+}
+
+OccaCoefficient::OccaCoefficient(const Engine &e, mfem::GridFunction &gf,
+                                 const bool useRestrict) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   coeffValue = "(u(q, e))";
+   AddGridFunction("u", gf, useRestrict);
+}
+
+OccaCoefficient::OccaCoefficient(const Engine &e, const std::string &source) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   coeffValue = source;
+}
+
+OccaCoefficient::OccaCoefficient(const Engine &e, const char *source) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   coeffValue = source;
+}
+
+OccaCoefficient::OccaCoefficient(const OccaCoefficient &coeff) :
+   engine(coeff.engine),
+   integ(NULL),
+   name(coeff.name),
+   coeffValue(coeff.coeffValue)
+{
+
+   const int paramCount = (int) coeff.params.size();
+   for (int i = 0; i < paramCount; ++i)
+   {
+      params.push_back(coeff.params[i]->Clone());
+   }
+}
+
+OccaCoefficient::~OccaCoefficient()
+{
+   const int paramCount = (int) params.size();
+   for (int i = 0; i < paramCount; ++i)
+   {
+      delete params[i];
+   }
+}
+
+OccaCoefficient& OccaCoefficient::SetName(const std::string &name_)
+{
+   name = name_;
+   return *this;
+}
+
+void OccaCoefficient::Setup(OccaIntegrator &integ_,
+                            ::occa::properties &props_)
+{
+   integ = &integ_;
+
+   const int paramCount = (int) params.size();
+   props_["defines"][name + "_ARGS"] = "";
+   for (int i = 0; i < paramCount; ++i)
+   {
+      params[i]->Setup(integ_, props_);
+   }
+   props_["defines"][name] = coeffValue;
+
+   props = props_;
+}
+
+OccaCoefficient& OccaCoefficient::Add(OccaParameter *param)
+{
+   params.push_back(param);
+   return *this;
+}
+
+OccaCoefficient& OccaCoefficient::IncludeHeader(const std::string &filename)
+{
+   return Add(new OccaIncludeParameter(filename));
+}
+
+OccaCoefficient& OccaCoefficient::IncludeSource(const std::string &source)
+{
+   return Add(new OccaSourceParameter(source));
+}
+
+OccaCoefficient& OccaCoefficient::AddVector(const std::string &name_,
+                                            Vector &v,
+                                            const bool useRestrict)
+{
+   return Add(new OccaVectorParameter(name_, v, useRestrict));
+}
+
+OccaCoefficient& OccaCoefficient::AddVector(const std::string &name_,
+                                            Vector &v,
+                                            const std::string &attr,
+                                            const bool useRestrict)
+{
+   return Add(new OccaVectorParameter(name_, v, attr, useRestrict));
+}
+
+OccaCoefficient& OccaCoefficient::AddGridFunction(const std::string &name_,
+                                                  mfem::GridFunction &gf,
+                                                  const bool useRestrict)
+{
+   MFEM_ASSERT(engine->CheckVector(gf.Get_PVector()) &&
+               engine->CheckFESpace(gf.FESpace()->Get_PFESpace()),
+               "invalid device GridFunction");
+   return Add(new OccaGridFunctionParameter(name_, *engine, gf, useRestrict));
+}
+
+bool OccaCoefficient::IsConstant()
+{
+   return coeffValue.isNumber();
+}
+
+double OccaCoefficient::GetConstantValue()
+{
+   if (!IsConstant())
+   {
+      mfem_error("OccaCoefficient is not constant");
+   }
+   return coeffValue.number();
+}
+
+Vector OccaCoefficient::Eval()
+{
+   if (integ == NULL)
+   {
+      mfem_error("OccaCoefficient requires a Setup() call before Eval()");
+   }
+
+   mfem::FiniteElementSpace &fespace = integ->GetTrialFESpace();
+   const mfem::IntegrationRule &ir   = integ->GetIntegrationRule();
+
+   const int elements = fespace.GetNE();
+   const int numQuad  = ir.GetNPoints();
+
+   Vector quadCoeff(*(new Layout(OccaEngine(), numQuad * elements)));
+   Eval(quadCoeff);
+   return quadCoeff;
+}
+
+void OccaCoefficient::Eval(Vector &quadCoeff)
+{
+   const std::string &okl_path = OccaEngine().GetOklPath();
+   static ::occa::kernelBuilder builder =
+      ::occa::kernelBuilder::fromFile(okl_path + "coefficient.okl",
+                                      "CoefficientEval");
+
+   if (integ == NULL)
+   {
+      mfem_error("OccaCoefficient requires a Setup() call before Eval()");
+   }
+
+   const int elements = integ->GetTrialFESpace().GetNE();
+
+   ::occa::properties kernelProps = props;
+   if (name != "COEFF")
+   {
+      kernelProps["defines/COEFF"]      = name;
+      kernelProps["defines/COEFF_ARGS"] = name + "_ARGS";
+   }
+
+   ::occa::kernel evalKernel = builder.build(GetDevice(), kernelProps);
+   evalKernel(elements, *this, quadCoeff.OccaMem());
+}
+
+OccaCoefficient::operator ::occa::kernelArg ()
+{
+   ::occa::kernelArg kArg;
+   const int paramCount = (int) params.size();
+   for (int i = 0; i < paramCount; ++i)
+   {
+      kArg.add(params[i]->KernelArgs());
+   }
+   return kArg;
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/coefficient.hpp

@@ -0,0 +1,287 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_COEFFICIENT_HPP
+#define MFEM_BACKENDS_OCCA_COEFFICIENT_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "vector.hpp"
+#include "gridfunc.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class OccaIntegrator;
+
+
+class OccaParameter
+{
+public:
+   virtual ~OccaParameter();
+
+   virtual OccaParameter* Clone() = 0;
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+
+   virtual ::occa::kernelArg KernelArgs();
+};
+
+
+//---[ Include Parameter ]------------
+class OccaIncludeParameter : public OccaParameter
+{
+private:
+   std::string filename;
+
+public:
+   OccaIncludeParameter(const std::string &filename_);
+
+   virtual OccaParameter* Clone();
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+};
+//====================================
+
+
+//---[ Source Parameter ]------------
+class OccaSourceParameter : public OccaParameter
+{
+private:
+   std::string source;
+
+public:
+   OccaSourceParameter(const std::string &filename_);
+
+   virtual OccaParameter* Clone();
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+};
+//====================================
+
+
+//---[ Define Parameter ]------------
+template <class TM>
+class OccaDefineParameter : public OccaParameter
+{
+private:
+   const std::string name;
+   TM value;
+
+public:
+   OccaDefineParameter(const std::string &name_,
+                       const TM &value_) :
+      name(name_),
+      value(value_) {}
+
+   virtual OccaParameter* Clone()
+   {
+      return new OccaDefineParameter(name, value);
+   }
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props)
+   {
+      props["defines"][name] = value;
+   }
+};
+//====================================
+
+
+//---[ Variable Parameter ]-----------
+template <class TM>
+class OccaVariableParameter : public OccaParameter
+{
+private:
+   const std::string name;
+   const TM &value;
+
+public:
+   OccaVariableParameter(const std::string &name_,
+                         const TM &value_) :
+      name(name_),
+      value(value_) {}
+
+   virtual OccaParameter* Clone()
+   {
+      return new OccaVariableParameter(name, value);
+   }
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props)
+   {
+      std::string &args = (props["defines/COEFF_ARGS"]
+                           .asString()
+                           .string());
+      // const TM name,\n"
+      args += "const ";
+      args += ::occa::primitiveinfo<TM>::name;
+      args += ' ';
+      args += name;
+      args += ",\n";
+   }
+
+   virtual ::occa::kernelArg KernelArgs()
+   {
+      return ::occa::kernelArg(value);
+   }
+};
+//====================================
+
+
+//---[ Vector Parameter ]-------
+class OccaVectorParameter : public OccaParameter
+{
+private:
+   const std::string name;
+   Vector v;
+   bool useRestrict;
+   std::string attr;
+
+public:
+   OccaVectorParameter(const std::string &name_,
+                       Vector &v_,
+                       const bool useRestrict_ = false);
+
+   OccaVectorParameter(const std::string &name_,
+                       Vector &v_,
+                       const std::string &attr_,
+                       const bool useRestrict_ = false);
+
+   virtual OccaParameter* Clone();
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+
+   virtual ::occa::kernelArg KernelArgs();
+};
+//====================================
+
+
+//---[ GridFunction Parameter ]-------
+class OccaGridFunctionParameter : public OccaParameter
+{
+private:
+   const std::string name;
+   mfem::GridFunction &gf;
+   Vector gfQuad;
+   bool useRestrict;
+
+public:
+   OccaGridFunctionParameter(const std::string &name_,
+                             const Engine &e,
+                             mfem::GridFunction &gf_,
+                             const bool useRestrict_ = false);
+
+   virtual OccaParameter* Clone();
+
+   virtual void Setup(OccaIntegrator &integ,
+                      ::occa::properties &props);
+
+   virtual ::occa::kernelArg KernelArgs();
+};
+//====================================
+
+
+//---[ Coefficient ]------------------
+// [MISSING]
+// Needs to know about the integrator's
+//   - fespace
+//   - ir
+// Step where parameters that need the ir get called for setup
+// For example, GridFunction (d, e) -> (q, e)
+class OccaCoefficient
+{
+private:
+   SharedPtr<const Engine> engine;
+
+   OccaIntegrator *integ;
+
+   std::string name;
+   ::occa::json coeffValue;
+
+   ::occa::properties props;
+   std::vector<OccaParameter*> params;
+
+public:
+   OccaCoefficient(const Engine &e, const double value = 1.0);
+   OccaCoefficient(const Engine &e, mfem::GridFunction &gf,
+                   const bool useRestrict = false);
+   OccaCoefficient(const Engine &e, const std::string &source);
+   OccaCoefficient(const Engine &e, const char *source);
+   ~OccaCoefficient();
+
+   OccaCoefficient(const OccaCoefficient &coeff);
+
+   const Engine &OccaEngine() const { return *engine; }
+
+   ::occa::device GetDevice(int idx = 0) const
+   { return engine->GetDevice(idx); }
+
+   OccaCoefficient& SetName(const std::string &name_);
+
+   void Setup(OccaIntegrator &integ_,
+              ::occa::properties &props_);
+
+   OccaCoefficient& Add(OccaParameter *param);
+
+   OccaCoefficient& IncludeHeader(const std::string &filename);
+   OccaCoefficient& IncludeSource(const std::string &source);
+
+   template <class TM>
+   OccaCoefficient& AddDefine(const std::string &name_, const TM &value)
+   {
+      return Add(new OccaDefineParameter<TM>(name_, value));
+   }
+
+   template <class TM>
+   OccaCoefficient& AddVariable(const std::string &name_, const TM &value)
+   {
+      return Add(new OccaVariableParameter<TM>(name_, value));
+   }
+
+   OccaCoefficient& AddVector(const std::string &name_,
+                              Vector &v,
+                              const bool useRestrict = false);
+
+
+   OccaCoefficient& AddVector(const std::string &name_,
+                              Vector &v,
+                              const std::string &attr,
+                              const bool useRestrict = false);
+
+   OccaCoefficient& AddGridFunction(const std::string &name_,
+                                    mfem::GridFunction &gf,
+                                    const bool useRestrict = false);
+
+   bool IsConstant();
+   double GetConstantValue();
+
+   Vector Eval();
+   void Eval(Vector &quadCoeff);
+
+   operator ::occa::kernelArg ();
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_COEFFICIENT_HPP

backends/occa/defines.okl

@@ -0,0 +1,40 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_OCCA_DEFINES
+#define MFEM_OCCA_DEFINES
+
+#ifndef USING_TENSOR_OPS
+#  define USING_TENSOR_OPS 0
+#endif
+
+#ifdef OCCA_USING_GPU
+#  define GPU_ORDER_2(I0, I1)         @dimOrder(I0, I1)
+#  define GPU_ORDER_3(I0, I1, I2)     @dimOrder(I0, I1, I2)
+#  define GPU_ORDER_4(I0, I1, I2, I3) @dimOrder(I0, I1, I2, I3)
+#else
+#  define GPU_ORDER_2(I0, I1)         @dimOrder(0, 1)
+#  define GPU_ORDER_3(I0, I1, I2)     @dimOrder(0, 1, 2)
+#  define GPU_ORDER_4(I0, I1, I2, I3) @dimOrder(0, 1, 2, 3)
+#endif
+
+#ifndef COEFF
+#  define COEFF 1.0
+#  define COEFF_ARGS
+#endif
+
+#if USING_TENSOR_OPS
+#  include "mfem-occa://defines/tensor.okl"
+#else
+#  include "mfem-occa://defines/simplex.okl"
+#endif
+
+#endif

backends/occa/defines/simplex.okl

@@ -0,0 +1,40 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#define USING_LOW_ORDER 1
+#define USING_HI_ORDER  0
+
+typedef double* DofToQuad_t @dim(NUM_QUAD, NUM_DOFS);
+typedef double* DofToQuadD2D_t @dim(2, NUM_QUAD, NUM_DOFS);
+typedef double* DofToQuadD3D_t @dim(3, NUM_QUAD, NUM_DOFS);
+
+typedef double* QuadToDof_t @dim(NUM_DOFS, NUM_QUAD);
+typedef double* QuadToDofD2D_t @dim(2, NUM_DOFS, NUM_QUAD);
+typedef double* QuadToDofD3D_t @dim(3, NUM_DOFS, NUM_QUAD);
+
+typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD, numElements);
+typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
+
+typedef double* SymmOperator2D_t @dim(3, NUM_QUAD, numElements);
+typedef double* SymmOperator3D_t @dim(6, NUM_QUAD, numElements);
+
+typedef double* DLocal_t @dim(NUM_DOFS, numElements);
+typedef double* QLocal_t @dim(NUM_QUAD, numElements);
+
+#if VDIM_ORDERING == ORDERING_BY_VDIM
+typedef double* DVLocal_t @dim(NUM_VDIM, NUM_DOFS, numElements);
+typedef double* QVLocal_t @dim(NUM_VDIM, NUM_QUAD, numElements);
+#else
+typedef double* DVLocal_t @dim(NUM_VDIM, NUM_DOFS, numElements) @dimOrder(2,0,1);
+typedef double* QVLocal_t @dim(NUM_VDIM, NUM_QUAD, numElements) @dimOrder(2,0,1);
+#endif
+
+typedef int* DLocalMap_t @dim(NUM_DOFS, numElements);

backends/occa/defines/tensor.okl

@@ -0,0 +1,85 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#if NUM_QUAD_1D < NUM_DOFS_1D
+#  define NUM_MAX_1D NUM_DOFS_1D
+#else
+#  define NUM_MAX_1D NUM_QUAD_1D
+#endif
+
+#define NUM_MAX_2D (NUM_MAX_1D * NUM_MAX_1D)
+
+#define NUM_QUAD_DOFS_1D (NUM_QUAD_1D * NUM_DOFS_1D)
+
+#define QUAD_2D_ID(X, Y) (X + ((Y) * NUM_QUAD_1D))
+#define DOFS_2D_ID(X, Y) (X + ((Y) * NUM_DOFS_1D))
+
+#define QUAD_3D_ID(X, Y, Z) (X + ((Y) * NUM_QUAD_1D) + ((Z) * NUM_QUAD_2D))
+#define DOFS_3D_ID(X, Y, Z) (X + ((Y) * NUM_DOFS_1D) + ((Z) * NUM_DOFS_2D))
+
+#if NUM_MAX_1D < 8
+#  define USING_LOW_ORDER 1
+#  define USING_HI_ORDER  0
+#else
+#  define USING_LOW_ORDER 0
+#  define USING_HI_ORDER  1
+#endif
+
+#define M1_ELEMENT_BATCHES (M1_OUTER_ELEMENT_BATCH * M1_INNER_ELEMENT_BATCH)
+
+typedef double* DofToQuad_t @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+typedef double* QuadToDof_t @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+typedef double* Jacobian_t   @dim(NUM_DIM, NUM_DIM, numElements);
+typedef double* Jacobian1D_t @dim(NUM_QUAD_1D, numElements);
+typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD_2D, numElements);
+typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD_3D, numElements);
+
+typedef double* SymmOperator1D_t @dim(NUM_QUAD_1D, numElements);
+typedef double* SymmOperator2D_t @dim(3, NUM_QUAD_2D, numElements);
+typedef double* SymmOperator3D_t @dim(6, NUM_QUAD_3D, numElements);
+
+typedef double* DLocal_t   @dim(NUM_DOFS, numElements);
+typedef double* DLocal1D_t @dim(NUM_DOFS_1D, numElements);
+typedef double* DLocal2D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+typedef double* DLocal3D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+
+typedef double* QLocal_t   @dim(NUM_QUAD, numElements);
+typedef double* QLocal1D_t @dim(NUM_QUAD_1D, numElements);
+typedef double* QLocal2D_t @dim(NUM_QUAD_1D, NUM_QUAD_1D, numElements);
+typedef double* QLocal3D_t @dim(NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
+
+#if VDIM_ORDERING == ORDERING_BY_VDIM
+typedef double* DVLocal_t   @dim(NUM_VDIM, NUM_DOFS, numElements);
+typedef double* DVLocal1D_t @dim(NUM_VDIM, NUM_DOFS_1D, numElements);
+typedef double* DVLocal2D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+typedef double* DVLocal3D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+
+typedef double* QVLocal_t   @dim(NUM_VDIM, NUM_QUAD, numElements);
+typedef double* QVLocal1D_t @dim(NUM_VDIM, NUM_QUAD_1D, numElements);
+typedef double* QVLocal2D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
+typedef double* QVLocal3D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
+#else
+typedef double* DVLocal_t   @dim(NUM_VDIM, NUM_DOFS, numElements) @dimOrder(2,0,1);
+typedef double* DVLocal1D_t @dim(NUM_VDIM, NUM_DOFS_1D, numElements) @dimOrder(2,0,1);
+typedef double* DVLocal2D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, numElements) @dimOrder(3,0,1,2);
+typedef double* DVLocal3D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements) @dimOrder(4,0,1,2,3);
+
+typedef double* QVLocal_t   @dim(NUM_VDIM, NUM_QUAD, numElements) @dimOrder(2,0,1);
+typedef double* QVLocal1D_t @dim(NUM_VDIM, NUM_QUAD_1D, numElements) @dimOrder(2,0,1);
+typedef double* QVLocal2D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, numElements) @dimOrder(3,0,1,2);
+typedef double* QVLocal3D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements) @dimOrder(4,0,1,2,3);
+#endif
+
+typedef int* DLocalMap_t   @dim(NUM_DOFS, numElements);
+typedef int* DLocalMap1D_t @dim(NUM_DOFS_1D, numElements);
+typedef int* DLocalMap2D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, numElements);
+typedef int* DLocalMap3D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);

backends/occa/diffusion/simplex/cpu.okl

@@ -0,0 +1,168 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        @restrict const double *quadWeights,
+                        @restrict const Jacobian2D_t J,
+                        COEFF_ARGS
+                        @restrict SymmOperator2D_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+      const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
+
+      oper(0, q, e) =  c_detJ * (J12*J12 + J22*J22); // (1,1)
+      oper(1, q, e) = -c_detJ * (J12*J11 + J22*J21); // (1,2) + (2,1)
+      oper(2, q, e) =  c_detJ * (J11*J11 + J21*J21); // (2,2)
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD2D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD2D_t quadToDofD,
+                       @restrict const SymmOperator2D_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double r_sol[NUM_DOFS];
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        r_sol[d] = 0;
+      }
+
+      for (int q = 0; q < NUM_QUAD; ++q) {
+        double gradX = 0, gradY = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          const double s = solIn(d, e);
+          gradX += s * quadToDofD(0, d, q);
+          gradY += s * quadToDofD(1, d, q);
+        }
+        const double O11 = oper(0, q, e);
+        const double O12 = oper(1, q, e);
+        const double O22 = oper(2, q, e);
+
+        const double gradX2 = (O11 * gradX) + (O12 * gradY);
+        const double gradY2 = (O12 * gradX) + (O22 * gradY);
+
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          r_sol[d] += ((gradX2 * quadToDofD(0, d, q)) +
+                       (gradY2 * quadToDofD(1, d, q)));
+        }
+      }
+
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        solOut(d, e) += r_sol[d];
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian3D_t J,
+                        COEFF_ARGS
+                        @restrict SymmOperator3D_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+      const double c_detJ = quadWeights[q] * COEFF / detJ;
+
+      // adj(J)
+      const double A11 = (J22 * J33) - (J23 * J32);
+      const double A12 = (J23 * J31) - (J21 * J33);
+      const double A13 = (J21 * J32) - (J22 * J31);
+
+      const double A21 = (J13 * J32) - (J12 * J33);
+      const double A22 = (J11 * J33) - (J13 * J31);
+      const double A23 = (J12 * J31) - (J11 * J32);
+
+      const double A31 = (J12 * J23) - (J13 * J22);
+      const double A32 = (J13 * J21) - (J11 * J23);
+      const double A33 = (J11 * J22) - (J12 * J21);
+
+      // adj(J)^Tadj(J)
+      oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
+      oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2) + (2,1)
+      oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3) + (3,1)
+      oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
+      oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3) + (3,2)
+      oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD3D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD3D_t quadToDofD,
+                       @restrict const SymmOperator3D_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double r_sol[NUM_DOFS];
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        r_sol[d] = 0;
+      }
+
+      for (int q = 0; q < NUM_QUAD; ++q) {
+        double gradX = 0, gradY = 0, gradZ = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          const double s = solIn(d, e);
+          gradX += s * quadToDofD(0, d, q);
+          gradY += s * quadToDofD(1, d, q);
+          gradZ += s * quadToDofD(2, d, q);
+        }
+        const double O11 = oper(0, q, e);
+        const double O12 = oper(1, q, e);
+        const double O13 = oper(2, q, e);
+        const double O22 = oper(3, q, e);
+        const double O23 = oper(4, q, e);
+        const double O33 = oper(5, q, e);
+
+        const double gradX2 = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
+        const double gradY2 = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
+        const double gradZ2 = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
+
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          r_sol[d] += ((gradX2 * quadToDofD(0, d, q)) +
+                       (gradY2 * quadToDofD(1, d, q)) +
+                       (gradZ2 * quadToDofD(2, d, q)));
+        }
+      }
+
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        solOut(d, e) += r_sol[d];
+      }
+    }
+  }
+}
+//======================================

backends/occa/diffusion/simplex/gpuHighOrder.okl

@@ -0,0 +1,182 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian2D_t J,
+                        COEFF_ARGS
+                        SymmOperator2D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD; q += A2_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+            const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
+
+            oper(0, q, e) =  c_detJ * (J12*J12 + J22*J22); // (1,1)
+            oper(1, q, e) = -c_detJ * (J12*J11 + J22*J21); // (1,2) + (2,1)
+            oper(2, q, e) =  c_detJ * (J11*J11 + J21*J21); // (2,2)
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD2D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD2D_t quadToDofD,
+                       @restrict const SymmOperator2D_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_gradX[NUM_QUAD];
+    @shared double s_gradY[NUM_QUAD];
+
+    for (int qOff = 0; qOff < M2_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M2_INNER_BATCH) {
+        double gradX = 0, gradY = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          const double s = solIn(d, e);
+          gradX += s * quadToDofD(0, d, q);
+          gradY += s * quadToDofD(1, d, q);
+        }
+        const double O11 = oper(0, q, e);
+        const double O12 = oper(1, q, e);
+        const double O22 = oper(2, q, e);
+
+        s_gradX[q] = (O11 * gradX) + (O12 * gradY);
+        s_gradY[q] = (O12 * gradX) + (O22 * gradY);
+      }
+    }
+
+    for (int dOff = 0; dOff < M2_INNER_BATCH; ++dOff) {
+      for (int d = dOff; d < NUM_DOFS; d += M2_INNER_BATCH) {
+        double r_sol = 0;
+        for (int q = 0; q < NUM_QUAD; ++q) {
+          // FIXME: s_gradX and s_gradY are @shared used outside of @inner
+          r_sol += ((s_gradX[q] * quadToDofD(0, d, q)) +
+                    (s_gradY[q] * quadToDofD(1, d, q)));
+        }
+        solOut(d, e) += r_sol;
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian3D_t J,
+                        COEFF_ARGS
+                        SymmOperator3D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD; q += A3_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+            const double c_detJ = quadWeights[q] * COEFF / detJ;
+
+            // adj(J)
+            const double A11 = (J22 * J33) - (J23 * J32);
+            const double A12 = (J23 * J31) - (J21 * J33);
+            const double A13 = (J21 * J32) - (J22 * J31);
+
+            const double A21 = (J13 * J32) - (J12 * J33);
+            const double A22 = (J11 * J33) - (J13 * J31);
+            const double A23 = (J12 * J31) - (J11 * J32);
+
+            const double A31 = (J12 * J23) - (J13 * J22);
+            const double A32 = (J13 * J21) - (J11 * J23);
+            const double A33 = (J11 * J22) - (J12 * J21);
+
+            // adj(J)^Tadj(J)
+            oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
+            oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2) + (2,1)
+            oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3) + (3,1)
+            oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
+            oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3) + (3,2)
+            oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD3D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD3D_t quadToDofD,
+                       @restrict const SymmOperator3D_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_gradX[NUM_QUAD];
+    @shared double s_gradY[NUM_QUAD];
+    @shared double s_gradZ[NUM_QUAD];
+
+    for (int qOff = 0; qOff < M3_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M3_INNER_BATCH) {
+        double gradX = 0, gradY = 0, gradZ = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          const double s = solIn(d, e);
+          gradX += s * quadToDofD(0, d, q);
+          gradY += s * quadToDofD(1, d, q);
+          gradZ += s * quadToDofD(2, d, q);
+        }
+        const double O11 = oper(0, q, e);
+        const double O12 = oper(1, q, e);
+        const double O13 = oper(2, q, e);
+        const double O22 = oper(3, q, e);
+        const double O23 = oper(4, q, e);
+        const double O33 = oper(5, q, e);
+
+        s_gradX[q] = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
+        s_gradY[q] = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
+        s_gradZ[q] = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
+      }
+    }
+
+    for (int dOff = 0; dOff < M3_INNER_BATCH; ++dOff) {
+      for (int d = dOff; d < NUM_DOFS; d += M3_INNER_BATCH) {
+        double r_sol = 0;
+        for (int q = 0; q < NUM_QUAD; ++q) {
+          r_sol += ((s_gradX[q] * quadToDofD(0, d, q)) +
+                    (s_gradY[q] * quadToDofD(1, d, q)) +
+                    (s_gradZ[q] * quadToDofD(2, d, q)));
+        }
+        solOut(d, e) += r_sol;
+      }
+    }
+  }
+}
+//======================================

backends/occa/diffusion/tensor/cpu.okl

@@ -0,0 +1,370 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void Assemble1D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian1D_t J,
+                        COEFF_ARGS
+                        @restrict SymmOperator1D_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
+      oper(q, e) = quadWeights[q] * COEFF / J(q, e);
+    }
+  }
+}
+
+@kernel void MultAdd1D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator1D_t oper,
+                       @restrict const DLocal1D_t solIn,
+                       @restrict DLocal1D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double grad[NUM_QUAD_1D];
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        grad[qx] = 0;
+      }
+      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+        const double s = solIn(dx, e);
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          grad[qx] += s * dofToQuadD(qx, dx);
+        }
+      }
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        grad[qx] *= oper(qx, e);
+      }
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        const double gradX = grad[qx];
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          solOut(dx, e) += gradX * quadToDofD(dx, qx);
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian2D_t J,
+                        COEFF_ARGS
+                        @restrict SymmOperator2D_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_2D; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+      const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
+
+      oper(0, q, e) =  c_detJ * (J21*J21 + J22*J22); // (1,1)
+      oper(1, q, e) = -c_detJ * (J21*J11 + J22*J12); // (1,2), (2,1)
+      oper(2, q, e) =  c_detJ * (J11*J11 + J12*J12); // (2,2)
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator2D_t oper,
+                       @restrict const DLocal2D_t solIn,
+                       @restrict DLocal2D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double grad[NUM_QUAD_1D][NUM_QUAD_1D][2];
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          grad[qy][qx][0] = 0;
+          grad[qy][qx][1] = 0;
+        }
+      }
+
+      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+        double gradX[NUM_QUAD_1D][2];
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          gradX[qx][0] = 0;
+          gradX[qx][1] = 0;
+        }
+
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          const double s = solIn(dx, dy, e);
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            gradX[qx][0] += s * dofToQuad(qx, dx);
+            gradX[qx][1] += s * dofToQuadD(qx, dx);
+          }
+        }
+
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          const double wy  = dofToQuad(qy, dy);
+          const double wDy = dofToQuadD(qy, dy);
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            grad[qy][qx][0] += gradX[qx][1] * wy;
+            grad[qy][qx][1] += gradX[qx][0] * wDy;
+          }
+        }
+      }
+
+      // Calculate Dxy, xDy in plane
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          const int q = QUAD_2D_ID(qx, qy);
+          const double O11 = oper(0, q, e);
+          const double O12 = oper(1, q, e);
+          const double O22 = oper(2, q, e);
+
+          const double gradX = grad[qy][qx][0];
+          const double gradY = grad[qy][qx][1];
+
+          grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
+          grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
+        }
+      }
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        double gradX[NUM_DOFS_1D][2];
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          gradX[dx][0] = 0;
+          gradX[dx][1] = 0;
+        }
+
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          const double gX = grad[qy][qx][0];
+          const double gY = grad[qy][qx][1];
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double wx  = quadToDof(dx, qx);
+            const double wDx = quadToDofD(dx, qx);
+            gradX[dx][0] += gX * wDx;
+            gradX[dx][1] += gY * wx;
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          const double wy  = quadToDof(dy, qy);
+          const double wDy = quadToDofD(dy, qy);
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            solOut(dx, dy, e) += ((gradX[dx][0] * wy) +
+                                  (gradX[dx][1] * wDy));
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian3D_t J,
+                        COEFF_ARGS
+                        @restrict SymmOperator3D_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_3D; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+      const double c_detJ = quadWeights[q] * COEFF / detJ;
+
+      // adj(J)
+      const double A11 = (J22 * J33) - (J23 * J32);
+      const double A12 = (J23 * J31) - (J21 * J33);
+      const double A13 = (J21 * J32) - (J22 * J31);
+
+      const double A21 = (J13 * J32) - (J12 * J33);
+      const double A22 = (J11 * J33) - (J13 * J31);
+      const double A23 = (J12 * J31) - (J11 * J32);
+
+      const double A31 = (J12 * J23) - (J13 * J22);
+      const double A32 = (J13 * J21) - (J11 * J23);
+      const double A33 = (J11 * J22) - (J12 * J21);
+
+      // adj(J)^Tadj(J)
+      oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
+      oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2), (2,1)
+      oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3), (3,1)
+      oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
+      oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3), (3,2)
+      oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator3D_t oper,
+                       @restrict const DLocal3D_t solIn,
+                       @restrict DLocal3D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double grad[NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D][4];
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            grad[qz][qy][qx][0] = 0;
+            grad[qz][qy][qx][1] = 0;
+            grad[qz][qy][qx][2] = 0;
+          }
+        }
+      }
+
+      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+        double gradXY[NUM_QUAD_1D][NUM_QUAD_1D][4];
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            gradXY[qy][qx][0] = 0;
+            gradXY[qy][qx][1] = 0;
+            gradXY[qy][qx][2] = 0;
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          double gradX[NUM_QUAD_1D][2];
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            gradX[qx][0] = 0;
+            gradX[qx][1] = 0;
+          }
+
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double s = solIn(dx, dy, dz, e);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              gradX[qx][0] += s * dofToQuad(qx, dx);
+              gradX[qx][1] += s * dofToQuadD(qx, dx);
+            }
+          }
+
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            const double wy  = dofToQuad(qy, dy);
+            const double wDy = dofToQuadD(qy, dy);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              const double wx  = gradX[qx][0];
+              const double wDx = gradX[qx][1];
+              gradXY[qy][qx][0] += wDx * wy;
+              gradXY[qy][qx][1] += wx  * wDy;
+              gradXY[qy][qx][2] += wx  * wy;
+            }
+          }
+        }
+
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          const double wz  = dofToQuad(qz, dz);
+          const double wDz = dofToQuadD(qz, dz);
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              grad[qz][qy][qx][0] += gradXY[qy][qx][0] * wz;
+              grad[qz][qy][qx][1] += gradXY[qy][qx][1] * wz;
+              grad[qz][qy][qx][2] += gradXY[qy][qx][2] * wDz;
+            }
+          }
+        }
+      }
+
+      // Calculate Dxyz, xDyz, xyDz in plane
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            const int q = QUAD_3D_ID(qx, qy, qz);
+            const double O11 = oper(0, q, e);
+            const double O12 = oper(1, q, e);
+            const double O13 = oper(2, q, e);
+            const double O22 = oper(3, q, e);
+            const double O23 = oper(4, q, e);
+            const double O33 = oper(5, q, e);
+
+            const double gradX = grad[qz][qy][qx][0];
+            const double gradY = grad[qz][qy][qx][1];
+            const double gradZ = grad[qz][qy][qx][2];
+
+            grad[qz][qy][qx][0] = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
+            grad[qz][qy][qx][1] = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
+            grad[qz][qy][qx][2] = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
+          }
+        }
+      }
+
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        double gradXY[NUM_DOFS_1D][NUM_DOFS_1D][4];
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            gradXY[dy][dx][0] = 0;
+            gradXY[dy][dx][1] = 0;
+            gradXY[dy][dx][2] = 0;
+          }
+        }
+
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          double gradX[NUM_DOFS_1D][4];
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            gradX[dx][0] = 0;
+            gradX[dx][1] = 0;
+            gradX[dx][2] = 0;
+          }
+
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            const double gX = grad[qz][qy][qx][0];
+            const double gY = grad[qz][qy][qx][1];
+            const double gZ = grad[qz][qy][qx][2];
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              const double wx  = quadToDof(dx, qx);
+              const double wDx = quadToDofD(dx, qx);
+              gradX[dx][0] += gX * wDx;
+              gradX[dx][1] += gY * wx;
+              gradX[dx][2] += gZ * wx;
+            }
+          }
+
+          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+            const double wy  = quadToDof(dy, qy);
+            const double wDy = quadToDofD(dy, qy);
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              gradXY[dy][dx][0] += gradX[dx][0] * wy;
+              gradXY[dy][dx][1] += gradX[dx][1] * wDy;
+              gradXY[dy][dx][2] += gradX[dx][2] * wy;
+            }
+          }
+        }
+
+        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+          const double wz  = quadToDof(dz, qz);
+          const double wDz = quadToDofD(dz, qz);
+          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              solOut(dx, dy, dz, e) += ((gradXY[dy][dx][0] * wz) +
+                                        (gradXY[dy][dx][1] * wz) +
+                                        (gradXY[dy][dx][2] * wDz));
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/diffusion/tensor/gpuHighOrder.okl

@@ -0,0 +1,435 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void Assemble1D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian1D_t J,
+                        COEFF_ARGS
+                        SymmOperator1D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A1_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A1_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
+          oper(q, e) = quadWeights[q] * COEFF / J(q, e);
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd1D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator1D_t oper,
+                       @restrict const DLocal1D_t solIn,
+                       @restrict DLocal1D_t solOut) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
+    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    @exclusive double grad[NUM_QUAD_1D];
+
+    for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+      for (int i = el; i < NUM_QUAD_DOFS_1D; i += M1_INNER_ELEMENT_BATCH) {
+        s_dofToQuadD[i] = dofToQuadD[i];
+        s_quadToDofD[i] = quadToDofD[i];
+      }
+    }
+
+    for (int b = 0; b < M1_OUTER_ELEMENT_BATCH; ++b) {
+      for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+        const int e = eOff + b*M1_INNER_ELEMENT_BATCH + el;
+        if (e < numElements) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            grad[qx] = 0;
+          }
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double s = solIn(dx, e);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              grad[qx] += s * s_dofToQuadD(qx, dx);
+            }
+          }
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            grad[qx] *= oper(qx, e);
+          }
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            double s = 0;
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              s += grad[qx] * s_quadToDofD(dx, qx);
+            }
+            solOut(dx, e) += s;
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian2D_t J,
+                        COEFF_ARGS
+                        SymmOperator2D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD_2D; q += A2_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+            const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
+
+            oper(0, q, e) =  c_detJ * (J21*J21 + J22*J22); // (1,1)
+            oper(1, q, e) = -c_detJ * (J21*J11 + J22*J12); // (1,2), (2,1)
+            oper(2, q, e) =  c_detJ * (J11*J11 + J12*J12); // (2,2)
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator2D_t oper,
+                       @restrict const DLocal2D_t solIn,
+                       @restrict DLocal2D_t solOut) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    // Store xy planes in shared memory
+    @shared double s_xy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_xDy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_grad[2 * NUM_QUAD_2D] @dim(2, NUM_QUAD_1D, NUM_QUAD_1D);
+
+    @exclusive double r_x[NUM_MAX_1D];
+    @exclusive double r_y[NUM_QUAD_1D];
+
+    for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+      for (int id = x; id < NUM_QUAD_DOFS_1D; id += NUM_MAX_1D) {
+        s_dofToQuad[id]  = dofToQuad[id];
+        s_dofToQuadD[id] = dofToQuadD[id];
+        s_quadToDof[id]  = quadToDof[id];
+        s_quadToDofD[id] = quadToDofD[id];
+      }
+    }
+
+    for (int e = eOff; e < (eOff + M2_ELEMENT_BATCH); ++e) {
+      if (e < numElements) {
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx; @inner) {
+          if (dx < NUM_DOFS_1D) {
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              s_xy(dx, qy) = 0;
+              s_xDy(dx, qy) = 0;
+            }
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              r_x[dy] = solIn(dx, dy, e);
+            }
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              double xy = 0;
+              double xDy = 0;
+              for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+                xy  += r_x[dy] * s_dofToQuad(qy, dy);
+                xDy += r_x[dy] * s_dofToQuadD(qy, dy);
+              }
+              s_xy(dx, qy)  = xy;
+              s_xDy(dx, qy) = xDy;
+            }
+          }
+        }
+        for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+          if (qy < NUM_QUAD_1D) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              double gradX = 0, gradY = 0;
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                gradX += s_xy(dx, qy)  * s_dofToQuadD(qx, dx);
+                gradY += s_xDy(dx, qy) * s_dofToQuad(qx, dx);
+              }
+
+              const int q = QUAD_2D_ID(qx, qy);
+              const double O11 = oper(0, q, e);
+              const double O12 = oper(1, q, e);
+              const double O22 = oper(2, q, e);
+
+              s_grad(0, qx, qy) = (O11 * gradX) + (O12 * gradY);
+              s_grad(1, qx, qy) = (O12 * gradX) + (O22 * gradY);
+            }
+          }
+        }
+
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx; @inner) {
+          if (qx < NUM_QUAD_1D) {
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              s_xy(dy, qx) = 0;
+              s_xDy(dy, qx) = 0;
+            }
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              r_x[qy] = s_grad(0, qx, qy);
+              r_y[qy] = s_grad(1, qx, qy);
+            }
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              double xy  = 0;
+              double xDy = 0;
+              for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+                xy  += r_x[qy] * s_quadToDof(dy, qy);
+                xDy += r_y[qy] * s_quadToDofD(dy, qy);
+              }
+              s_xy(dy, qx) = xy;
+              s_xDy(dy, qx) = xDy;
+            }
+          }
+        }
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx; @inner) {
+          if (dx < NUM_DOFS_1D) {
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              double s = 0;
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                s += ((s_xy(dy, qx) * s_quadToDofD(dx, qx)) +
+                      (s_xDy(dy, qx) * s_quadToDof(dx, qx)));
+              }
+              solOut(dx, dy, e) += s;
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian3D_t J,
+                        COEFF_ARGS
+                        SymmOperator3D_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD_3D; q += A3_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+            const double c_detJ = quadWeights[q] * COEFF / detJ;
+
+            // adj(J)
+            const double A11 = (J22 * J33) - (J23 * J32);
+            const double A12 = (J23 * J31) - (J21 * J33);
+            const double A13 = (J21 * J32) - (J22 * J31);
+
+            const double A21 = (J13 * J32) - (J12 * J33);
+            const double A22 = (J11 * J33) - (J13 * J31);
+            const double A23 = (J12 * J31) - (J11 * J32);
+
+            const double A31 = (J12 * J23) - (J13 * J22);
+            const double A32 = (J13 * J21) - (J11 * J23);
+            const double A33 = (J11 * J22) - (J12 * J21);
+
+            // adj(J)^Tadj(J)
+            oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
+            oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2), (2,1)
+            oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3), (3,1)
+            oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
+            oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3), (3,2)
+            oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const SymmOperator3D_t oper,
+                       @restrict const DLocal3D_t solIn,
+                       @restrict DLocal3D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    // Store xy planes in shared memory
+    @shared double s_z[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
+    @shared double s_Dz[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
+    @shared double s_xyDz[NUM_QUAD_2D] @dim(NUM_QUAD_1D, NUM_QUAD_1D);
+
+    // Store z axis as registers
+    @exclusive double r_qz[NUM_QUAD_1D];
+    @exclusive double r_qDz[NUM_QUAD_1D];
+    @exclusive double r_dDxyz[NUM_DOFS_1D];
+    @exclusive double r_dxDyz[NUM_DOFS_1D];
+    @exclusive double r_dxyDz[NUM_DOFS_1D];
+
+    for (int y = 0; y < NUM_MAX_1D; ++y; @inner) {
+      for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+        const int id = (y * NUM_MAX_1D) + x;
+        // Fetch Q <--> D maps
+        if (id < NUM_QUAD_DOFS_1D) {
+          s_dofToQuad[id]  = dofToQuad[id];
+          s_dofToQuadD[id] = dofToQuadD[id];
+          s_quadToDof[id]  = quadToDof[id];
+          s_quadToDofD[id] = quadToDofD[id];
+        }
+        // Initialize our Z axis
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          r_qz[qz] = 0;
+          r_qDz[qz] = 0;
+        }
+        // Initialize our solution updates in the Z axis
+        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+          r_dDxyz[dz] = 0;
+          r_dxDyz[dz] = 0;
+          r_dxyDz[dz] = 0;
+        }
+      }
+    }
+
+    for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+      for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+        if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+          for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+            const double s = solIn(dx, dy, dz, e);
+            // Calculate D -> Q in the Z axis
+            for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+              r_qz[qz]  += s * s_dofToQuad(qz, dz);
+              r_qDz[qz] += s * s_dofToQuadD(qz, dz);
+            }
+          }
+        }
+      }
+    }
+    // For each xy plane
+    for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+      // Fill xy plane at given z position
+      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+            s_z(dx, dy)  = r_qz[qz];
+            s_Dz(dx, dy) = r_qDz[qz];
+          }
+        }
+      }
+      // Calculate Dxyz, xDyz, xyDz in plane
+      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
+            double Dxyz = 0;
+            double xDyz = 0;
+            double xyDz = 0;
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              const double wy  = s_dofToQuad(qy, dy);
+              const double wDy = s_dofToQuadD(qy, dy);
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                const double wx  = s_dofToQuad(qx, dx);
+                const double wDx = s_dofToQuadD(qx, dx);
+                const double z  = s_z(dx, dy);
+                const double Dz = s_Dz(dx, dy);
+                Dxyz += wDx * wy  * z;
+                xDyz += wx  * wDy * z;
+                xyDz += wx  * wy  * Dz;
+              }
+            }
+
+            const int q = QUAD_3D_ID(qx, qy, qz);
+            const double O11 = oper(0, q, e);
+            const double O12 = oper(1, q, e);
+            const double O13 = oper(2, q, e);
+            const double O22 = oper(3, q, e);
+            const double O23 = oper(4, q, e);
+            const double O33 = oper(5, q, e);
+
+            const double qDxyz = (O11 * Dxyz) + (O12 * xDyz) + (O13 * xyDz);
+            const double qxDyz = (O12 * Dxyz) + (O22 * xDyz) + (O23 * xyDz);
+            const double qxyDz = (O13 * Dxyz) + (O23 * xDyz) + (O33 * xyDz);
+
+            for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+              const double wz  = s_quadToDof(dz, qz);
+              const double wDz = s_quadToDofD(dz, qz);
+              r_dDxyz[dz] += wz  * qDxyz;
+              r_dxDyz[dz] += wz  * qxDyz;
+              r_dxyDz[dz] += wDz * qxyDz;
+            }
+          }
+        }
+      }
+      @barrier("s_z_s_Dz_sync_1");
+    }
+    // Iterate over xy planes to compute solution
+    for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+      // Place xy plane in shared memory
+      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
+            s_z(qx, qy)    = r_dDxyz[dz];
+            s_Dz(qx, qy)   = r_dxDyz[dz];
+            s_xyDz(qx, qy) = r_dxyDz[dz];
+          }
+        }
+      }
+      // Finalize solution in xy plane
+      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+            double solZ = 0;
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              const double wy  = s_quadToDof(dy, qy);
+              const double wDy = s_quadToDofD(dy, qy);
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                const double wx  = s_quadToDof(dx, qx);
+                const double wDx = s_quadToDofD(dx, qx);
+                const double Dxyz = s_z(qx, qy);
+                const double xDyz = s_Dz(qx, qy);
+                const double xyDz = s_xyDz(qx, qy);
+                solZ += ((wDx * wy  * Dxyz) +
+                         (wx  * wDy * xDyz) +
+                         (wx  * wy  * xyDz));
+              }
+            }
+            solOut(dx, dy, dz, e) += solZ;
+          }
+        }
+      }
+      @barrier("s_z_s_Dz_s_xyDz_sync_1");
+    }
+  }
+}
+//======================================

backends/occa/engine.cpp

@@ -0,0 +1,167 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "backend.hpp"
+#include "url_handler.hpp"
+#include "bilinearform.hpp"
+#include "../../general/array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+bool Engine::fileOpenerRegistered = false;
+
+void Engine::Init(const std::string &engine_spec)
+{
+   //
+   // Initialize inherited fields
+   //
+   memory_resources[0] = NULL;
+   workers_weights[0]= 1.0;
+   workers_mem_res[0] = 0;
+
+   //
+   // Initialize the OCCA engine
+   //
+   ::occa::properties props(engine_spec);
+   device = new ::occa::device[1];
+   device[0].setup(props);
+
+   okl_path = "mfem-occa://";
+   if (!fileOpenerRegistered)
+   {
+      // The directories from "MFEM_OCCA_OKL_PATH", if any, have the highest
+      // priority.
+      FileOpener *fo = new FileOpener("mfem-occa://", "MFEM_OCCA_OKL_PATH");
+      // Next in priority is the source path, if it exists.
+      std::string mfem_src_prefix = mfem::GetSourcePath();
+      fo->AddDir(mfem_src_prefix + "/backends/occa");
+      // And last in priority is the install path, if it exists.
+      std::string mfem_install_prefix = mfem::GetInstallPath();
+      fo->AddDir(mfem_install_prefix + "/lib/mfem/occa");
+      ::occa::io::fileOpener::add(fo);
+      fileOpenerRegistered = true;
+   }
+   // std::cout << "OCCA device properties:\n" << device[0].properties();
+
+   force_cuda_aware_mpi = false;
+}
+
+Engine::Engine(const std::string &engine_spec)
+   : mfem::Engine(NULL, 1, 1)
+{
+   Init(engine_spec);
+}
+
+#ifdef MFEM_USE_MPI
+Engine::Engine(MPI_Comm _comm, const std::string &engine_spec)
+   : mfem::Engine(NULL, 1, 1)
+{
+   comm = _comm;
+   Init(engine_spec);
+}
+#endif
+
+bool Engine::CheckEngine(const mfem::Engine *engine) const
+{
+   return (engine != NULL && util::Is<const Engine>(engine) != NULL &&
+           *util::As<const Engine>(engine) == *this);
+}
+
+bool Engine::CheckLayout(const PLayout *layout) const
+{
+   return (layout != NULL && util::Is<const Layout>(layout) != NULL &&
+           layout->As<Layout>().OccaEngine() == *this);
+}
+
+bool Engine::CheckArray(const PArray *array) const
+{
+   return (array != NULL && util::Is<const Array>(array) != NULL &&
+           array->As<Array>().OccaEngine() == *this);
+}
+
+bool Engine::CheckVector(const PVector *vector) const
+{
+   return (vector != NULL && util::Is<const Vector>(vector) != NULL &&
+           vector->As<Vector>().OccaEngine() == *this);
+}
+
+bool Engine::CheckFESpace(const PFiniteElementSpace *fes) const
+{
+   return (fes != NULL && util::Is<const FiniteElementSpace>(fes) != NULL &&
+           fes->As<FiniteElementSpace>().OccaEngine() == *this);
+}
+
+DLayout Engine::MakeLayout(std::size_t size) const
+{
+   return DLayout(new Layout(*this, size));
+}
+
+DLayout Engine::MakeLayout(const mfem::Array<std::size_t> &offsets) const
+{
+   MFEM_ASSERT(offsets.Size() == 2,
+               "multiple workers are not supported yet");
+   return DLayout(new Layout(*this, offsets.Last()));
+}
+
+DArray Engine::MakeArray(PLayout &layout, std::size_t item_size) const
+{
+   return DArray(new Array(layout.As<Layout>(), item_size));
+}
+
+DVector Engine::MakeVector(PLayout &layout, int type_id) const
+{
+   MFEM_ASSERT(type_id == ScalarId<double>::value, "type_id " << type_id
+               << " is not supported");
+   return DVector(new Vector(layout.As<Layout>()));
+}
+
+DFiniteElementSpace Engine::MakeFESpace(mfem::FiniteElementSpace &fespace) const
+{
+   return DFiniteElementSpace(new FiniteElementSpace(*this, fespace));
+}
+
+DBilinearForm Engine::MakeBilinearForm(mfem::BilinearForm &bf) const
+{
+   return DBilinearForm(new BilinearForm(*this, bf));
+}
+
+void Engine::AssembleLinearForm(LinearForm &l_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+}
+
+mfem::Operator *Engine::MakeOperator(const MixedBilinearForm &mbl_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+   return NULL;
+}
+
+mfem::Operator *Engine::MakeOperator(const NonlinearForm &nl_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+   return NULL;
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/engine.hpp

@@ -0,0 +1,144 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_ENGINE_HPP
+#define MFEM_BACKENDS_OCCA_ENGINE_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "../base/backend.hpp"
+#include <occa.hpp>
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Engine : public mfem::Engine
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // mfem::Backend *backend;
+#ifdef MFEM_USE_MPI
+   // MPI_Comm comm;
+#endif
+   // int num_mem_res;
+   // int num_workers;
+   // MemoryResource **memory_resources;
+   // double *workers_weights;
+   // int *workers_mem_res;
+
+   static bool fileOpenerRegistered;
+   /// An array of OCCA devices. Currently only a single device is supported.
+   ::occa::device *device;
+   std::string okl_path;
+   bool force_cuda_aware_mpi;
+
+   void Init(const std::string &engine_spec);
+
+public:
+   /// TODO: doxygen
+   Engine(const std::string &engine_spec);
+
+#ifdef MFEM_USE_MPI
+   /// TODO: doxygen
+   Engine(MPI_Comm comm, const std::string &engine_spec);
+#endif
+
+   /// TODO: doxygen
+   virtual ~Engine() { delete [] device; }
+
+   /**
+       @name OCCA specific interface, used by other objects in the OCCA backend
+    */
+   ///@{
+
+   /// Get the associated OCCA device.
+   ::occa::device GetDevice(int idx = 0) const { return device[idx]; }
+
+   /// TODO: doxygen
+   const std::string &GetOklPath() const { return okl_path; }
+
+   /// OCCA device memory allocation.
+   ::occa::memory Alloc(std::size_t bytes) const
+   { return GetDevice().malloc(bytes); }
+
+   /// Two mfem::occa::Engine%s are equal if they use the same OCCA device.
+   bool operator==(const Engine &other) const
+   { return GetDevice() == other.GetDevice(); }
+
+   /// TODO: doxygen
+   bool CheckEngine(const mfem::Engine *e) const;
+
+   /// TODO: doxygen
+   bool CheckLayout(const PLayout *layout) const;
+
+   /// TODO: doxygen
+   bool CheckArray(const PArray *array) const;
+
+   /// TODO: doxygen
+   bool CheckVector(const PVector *vector) const;
+
+   /// TODO: doxygen
+   bool CheckFESpace(const PFiniteElementSpace *fes) const;
+
+#ifdef MFEM_USE_MPI
+   void SetForceCudaAwareMPI(bool force = true)
+   { force_cuda_aware_mpi = force; }
+
+   bool GetForceCudaAwareMPI() const { return force_cuda_aware_mpi; }
+#endif
+
+   ///@}
+   // End: OCCA specific interface
+
+   /**
+       @name Virtual interface: finite element data structures and algorithms
+    */
+   ///@{
+
+   virtual DLayout MakeLayout(std::size_t size) const;
+   virtual DLayout MakeLayout(const mfem::Array<std::size_t> &offsets) const;
+
+   virtual DArray MakeArray(PLayout &layout, std::size_t item_size) const;
+
+   virtual DVector MakeVector(PLayout &layout,
+                              int type_id = ScalarId<double>::value) const;
+
+   virtual DFiniteElementSpace MakeFESpace(mfem::FiniteElementSpace &
+                                           fespace) const;
+
+   virtual DBilinearForm MakeBilinearForm(mfem::BilinearForm &bf) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual void AssembleLinearForm(LinearForm &l_form) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual mfem::Operator *MakeOperator(const MixedBilinearForm &mbl_form) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual mfem::Operator *MakeOperator(const NonlinearForm &nl_form) const;
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_ENGINE_HPP

backends/occa/fespace.cpp

@@ -0,0 +1,532 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "backend.hpp"
+#include "fespace.hpp"
+#include "interpolation.hpp"
+
+#ifdef MFEM_USE_MPI
+#ifdef OMPI_RELEASE_VERSION
+#include <mpi-ext.h> // Check for cuda support
+#endif
+#endif
+
+namespace mfem
+{
+
+namespace occa
+{
+
+FiniteElementSpace::FiniteElementSpace(const Engine &e,
+                                       mfem::FiniteElementSpace &fespace)
+   : PFiniteElementSpace(e, fespace),
+     e_layout(new Layout(e, 0)) // resized in SetupLocalGlobalMaps()
+{
+   vdim     = fespace.GetVDim();
+   ordering = fespace.GetOrdering();
+
+   SetupLocalGlobalMaps();
+   SetupOperators(); // calls virtual methods of 'fes'
+   SetupKernels();
+}
+
+FiniteElementSpace::~FiniteElementSpace()
+{
+   delete restrictionOp;
+   delete prolongationOp;
+}
+
+void FiniteElementSpace::SetupLocalGlobalMaps()
+{
+   const int elements = fes->GetNE();
+
+   if (elements == 0) { return; }
+
+   // Assuming of finite elements are the same.
+   const mfem::FiniteElement &fe = *fes->GetFE(0);
+   const mfem::TensorBasisElement *el =
+      dynamic_cast<const mfem::TensorBasisElement*>(&fe);
+
+   const mfem::Table &e2dTable = fes->GetElementToDofTable();
+   const int *elementMap = e2dTable.GetJ();
+
+   globalDofs = fes->GetNDofs();
+   localDofs  = fe.GetDof();
+
+   e_layout->OccaResize(e2dTable.Size_of_connections());
+
+   int *elementDofMap = new int[localDofs];
+   if (el)
+   {
+      ::memcpy(elementDofMap,
+               el->GetDofMap().GetData(),
+               localDofs * sizeof(int));
+   }
+   else
+   {
+      for (int i = 0; i < localDofs; ++i)
+      {
+         elementDofMap[i] = i;
+      }
+   }
+
+   // Allocate device offsets and indices
+   globalToLocalOffsets.allocate(GetDevice(),
+                                 globalDofs + 1);
+   globalToLocalIndices.allocate(GetDevice(),
+                                 localDofs, elements);
+   localToGlobalMap.allocate(GetDevice(),
+                             localDofs, elements);
+
+   int *offsets = globalToLocalOffsets.ptr();
+   int *indices = globalToLocalIndices.ptr();
+   int *l2gMap  = localToGlobalMap.ptr();
+
+   // We'll be keeping a count of how many local nodes point
+   //   to its global dof
+   for (int i = 0; i <= globalDofs; ++i)
+   {
+      offsets[i] = 0;
+   }
+
+   for (int e = 0; e < elements; ++e)
+   {
+      MFEM_ASSERT(e2dTable.RowSize(e) == localDofs, "");
+      for (int d = 0; d < localDofs; ++d)
+      {
+         const int gid = elementMap[localDofs*e + d];
+         ++offsets[gid + 1];
+      }
+   }
+   // Aggregate to find offsets for each global dof
+   for (int i = 1; i <= globalDofs; ++i)
+   {
+      offsets[i] += offsets[i - 1];
+   }
+   // For each global dof, fill in all local nodes that point
+   //   to it
+   for (int e = 0; e < elements; ++e)
+   {
+      for (int d = 0; d < localDofs; ++d)
+      {
+         const int gid = elementMap[localDofs*e + elementDofMap[d]];
+         const int lid = localDofs*e + d;
+         indices[offsets[gid]++] = lid;
+         l2gMap[lid] = gid;
+      }
+   }
+   // We shifted the offsets vector by 1 by using it
+   //   as a counter. Now we shift it back.
+   for (int i = globalDofs; i > 0; --i)
+   {
+      offsets[i] = offsets[i - 1];
+   }
+   offsets[0] = 0;
+
+   delete [] elementDofMap;
+
+   globalToLocalOffsets.keepInDevice();
+   globalToLocalIndices.keepInDevice();
+   localToGlobalMap.keepInDevice();
+}
+
+void FiniteElementSpace::SetupOperators() const
+{
+   // Construct 'restrictionOp' and 'prolongationOp'.
+
+   prolongationOp = restrictionOp = NULL;
+
+   const mfem::SparseMatrix *R = fes->GetRestrictionMatrix();
+   const mfem::Operator *P = fes->GetProlongationMatrix();
+
+   if (!P) { return; }
+
+   Layout &v_layout = OccaVLayout();
+   Layout &t_layout = OccaTrueVLayout();
+
+   // Assuming R has one entry per row equal to 1.
+   MFEM_ASSERT(R->Finalized(), "");
+   const int tdofs = R->Height();
+   MFEM_ASSERT(tdofs == (int)t_layout.Size(), "");
+   MFEM_ASSERT(tdofs == R->GetI()[tdofs], "");
+   ::occa::array<int> ltdof_ldof(GetDevice(), tdofs, R->GetJ());
+   ltdof_ldof.keepInDevice();
+
+   restrictionOp = new RestrictionOperator(v_layout, t_layout, ltdof_ldof);
+
+   const mfem::SparseMatrix *pmat = dynamic_cast<const mfem::SparseMatrix*>(P);
+   if (pmat)
+   {
+      const mfem::SparseMatrix *pmatT = Transpose(*pmat);
+
+      OccaSparseMatrix *occaP  =
+         CreateMappedSparseMatrix(t_layout, v_layout, *pmat);
+      OccaSparseMatrix *occaPT =
+         CreateMappedSparseMatrix(v_layout, t_layout, *pmatT);
+
+      prolongationOp = new ProlongationOperator(*occaP, *occaPT);
+
+      delete occaPT;
+      delete occaP;
+   }
+#ifdef MFEM_USE_MPI
+   else if (fes->Conforming() && dynamic_cast<ParFiniteElementSpace*>(fes))
+   {
+      ParFiniteElementSpace *pfes = static_cast<ParFiniteElementSpace*>(fes);
+      prolongationOp = new OccaConformingProlongation(*this, *pfes,
+                                                      ltdof_ldof.memory());
+   }
+#endif
+   else
+   {
+      prolongationOp = new ProlongationOperator(t_layout, v_layout, P);
+   }
+}
+
+void FiniteElementSpace::SetupKernels()
+{
+   ::occa::properties props("defines: {"
+                            "  TILESIZE: 256,"
+                            "}");
+   props["defines/NUM_VDIM"] = vdim;
+
+   props["defines/ORDERING_BY_NODES"] = 0;
+   props["defines/ORDERING_BY_VDIM"]  = 1;
+   props["defines/VDIM_ORDERING"] = (int) (ordering == Ordering::byVDIM);
+
+   ::occa::device device = GetDevice();
+   const std::string &okl_path = OccaEngine().GetOklPath();
+   globalToLocalKernel = device.buildKernel(okl_path + "fespace.okl",
+                                            "GlobalToLocal",
+                                            props);
+   localToGlobalKernel = device.buildKernel(okl_path + "fespace.okl",
+                                            "LocalToGlobal",
+                                            props);
+}
+
+
+#ifdef MFEM_USE_MPI
+
+OccaConformingProlongation::OccaConformingProlongation(
+   const FiniteElementSpace &ofes, const mfem::ParFiniteElementSpace &pfes,
+   ::occa::memory ltdof_ldof_)
+
+   : Operator(ofes.OccaTrueVLayout(), ofes.OccaVLayout()),
+     shr_ltdof(ofes.OccaEngine()),
+     ext_ldof(ofes.OccaEngine()),
+     shr_buf(shr_ltdof.OccaLayout(), sizeof(double)),
+     ext_buf(ext_ldof.OccaLayout(), sizeof(double)),
+     shr_buf_offsets(NULL), ext_buf_offsets(NULL),
+     ltdof_ldof(ltdof_ldof_),
+     gc(pfes.GroupComm())
+{
+   MFEM_ASSERT(pfes.Conforming(), "internal error");
+
+   const Engine &engine = ofes.OccaEngine();
+   const std::string &okl_path = engine.GetOklPath();
+   ::occa::device device = engine.GetDevice();
+
+   {
+      Table nbr_ltdof;
+      gc.GetNeighborLTDofTable(nbr_ltdof);
+      shr_ltdof.OccaResize(nbr_ltdof.Size_of_connections(), sizeof(int));
+      shr_ltdof.OccaPush(nbr_ltdof.GetJ());
+      shr_buf.OccaResize(&shr_ltdof.OccaLayout(), sizeof(double));
+      shr_buf_offsets = nbr_ltdof.GetI();
+      {
+         mfem::Array<int> shr_ltdof(nbr_ltdof.GetJ(),
+                                    nbr_ltdof.Size_of_connections());
+         mfem::Array<int> unique_ltdof(shr_ltdof);
+         unique_ltdof.Sort();
+         unique_ltdof.Unique();
+         // Note: the next loop modifies the J array of nbr_ltdof
+         for (int i = 0; i < shr_ltdof.Size(); i++)
+         {
+            shr_ltdof[i] = unique_ltdof.FindSorted(shr_ltdof[i]);
+            MFEM_ASSERT(shr_ltdof[i] != -1, "internal error");
+         }
+         Table unique_shr;
+         Transpose(shr_ltdof, unique_shr, unique_ltdof.Size());
+
+         unq_ltdof = device.malloc(unique_ltdof.Size()*sizeof(int),
+                                   unique_ltdof.GetData());
+         unq_shr_i = device.malloc((unique_shr.Size()+1)*sizeof(int),
+                                   unique_shr.GetI());
+         unq_shr_j = device.malloc(unique_shr.Size_of_connections()*sizeof(int),
+                                   unique_shr.GetJ());
+      }
+      delete [] nbr_ltdof.GetJ();
+      nbr_ltdof.LoseData();
+   }
+   {
+      Table nbr_ldof;
+      gc.GetNeighborLDofTable(nbr_ldof);
+      ext_ldof.OccaResize(nbr_ldof.Size_of_connections(), sizeof(int));
+      ext_ldof.OccaPush(nbr_ldof.GetJ());
+      ext_buf.OccaResize(&ext_ldof.OccaLayout(), sizeof(double));
+      ext_buf_offsets = nbr_ldof.GetI();
+      delete [] nbr_ldof.GetJ();
+      nbr_ldof.LoseData();
+   }
+   host_shr_buf = NULL;
+   host_ext_buf = NULL;
+   // If the device has a separate memory space (e.g. CUDA device) and the MPI
+   // library does not support buffers in that separate memory space, we
+   // allocate separate host buffers to use for MPI communication.
+   if (device.hasSeparateMemorySpace())
+   {
+      bool need_host_buf = true;
+      if (device.mode() == "CUDA")
+      {
+#ifdef MPIX_CUDA_AWARE_SUPPORT
+         need_host_buf = !MPIX_Query_cuda_support();
+#endif
+         if (engine.GetForceCudaAwareMPI()) { need_host_buf = false; }
+         if (gc.GetGroupTopology().MyRank() == 0)
+         {
+            mfem::out << "\nOccaConformingProlongation: CUDA-aware MPI: "
+                      << (need_host_buf ? "NO" : "YES") << "\n\n";
+         }
+      }
+      if (need_host_buf)
+      {
+         host_shr_buf = new char[shr_buf.OccaMem().size()];
+         host_ext_buf = new char[ext_buf.OccaMem().size()];
+      }
+   }
+
+   ExtractSubVector = device.buildKernel(okl_path + "mappings.okl",
+                                         "ExtractSubVector",
+                                         "defines: { TILESIZE: 256 }");
+   SetSubVector = device.buildKernel(okl_path + "mappings.okl",
+                                     "SetSubVector",
+                                     "defines: { TILESIZE: 256 }");
+   AddSubVector = device.buildKernel(okl_path + "mappings.okl",
+                                     "AddSubVector",
+                                     "defines: { TILESIZE: 256 }");
+
+   const GroupTopology &gtopo = gc.GetGroupTopology();
+   int req_counter = 0;
+   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
+   {
+      const int send_offset = shr_buf_offsets[nbr];
+      const int send_size = shr_buf_offsets[nbr+1] - send_offset;
+      if (send_size > 0) { req_counter++; }
+
+      const int recv_offset = ext_buf_offsets[nbr];
+      const int recv_size = ext_buf_offsets[nbr+1] - recv_offset;
+      if (recv_size > 0) { req_counter++; }
+   }
+   requests = new MPI_Request[req_counter];
+}
+
+OccaConformingProlongation::~OccaConformingProlongation()
+{
+   delete [] requests;
+   delete [] host_ext_buf;
+   delete [] host_shr_buf;
+   delete [] ext_buf_offsets;
+   delete [] shr_buf_offsets;
+}
+
+void OccaConformingProlongation::BcastBeginCopy(const ::occa::memory &src,
+                                                std::size_t item_size) const
+{
+   // shr_buf[i] = src[shr_ltdof[i]]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (shr_ltdof.Size() == 0) { return; }
+   ExtractSubVector((int)shr_ltdof.Size(), shr_ltdof.OccaMem(), src,
+                    shr_buf.OccaMem());
+   // If the above kernel is executed asynchronously, wait for it to complete:
+   shr_buf.OccaMem().getDevice().finish();
+   if (host_shr_buf)
+   {
+      shr_buf.OccaMem().copyTo(host_shr_buf);
+   }
+}
+
+void OccaConformingProlongation::BcastLocalCopy(const ::occa::memory &src,
+                                                ::occa::memory &dst,
+                                                std::size_t item_size) const
+{
+   // dst[ltdof_ldof[i]] = src[i]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (ltdof_ldof.size<int>() == 0) { return; }
+   SetSubVector((int)ltdof_ldof.size<int>(), ltdof_ldof, src, dst);
+}
+
+void OccaConformingProlongation::BcastEndCopy(::occa::memory &dst,
+                                              std::size_t item_size) const
+{
+   // dst[ext_ldof[i]] = ext_buf[i]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (ext_ldof.Size() == 0) { return; }
+   if (host_ext_buf)
+   {
+      ext_buf.OccaMem().copyFrom(host_ext_buf);
+   }
+   SetSubVector((int)ext_ldof.Size(), ext_ldof.OccaMem(),
+                ext_buf.OccaMem(), dst);
+}
+
+void OccaConformingProlongation::ReduceBeginCopy(const ::occa::memory &src,
+                                                 std::size_t item_size) const
+{
+   // ext_buf[i] = src[ext_ldof[i]]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (ext_ldof.Size() == 0) { return; }
+   ExtractSubVector((int)ext_ldof.Size(), ext_ldof.OccaMem(), src,
+                    ext_buf.OccaMem());
+   // If the above kernel is executed asynchronously, wait for it to complete:
+   ext_buf.OccaMem().getDevice().finish();
+   if (host_ext_buf)
+   {
+      ext_buf.OccaMem().copyTo(host_ext_buf);
+   }
+}
+
+void OccaConformingProlongation::ReduceLocalCopy(const ::occa::memory &src,
+                                                 ::occa::memory &dst,
+                                                 std::size_t item_size) const
+{
+   // dst[i] = src[ltdof_ldof[i]]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (ltdof_ldof.size<int>() == 0) { return; }
+   ExtractSubVector((int)ltdof_ldof.size<int>(), ltdof_ldof, src, dst);
+}
+
+void OccaConformingProlongation::ReduceEndAssemble(::occa::memory &dst,
+                                                   std::size_t item_size) const
+{
+   // dst[shr_ltdof[i]] += shr_buf[i]
+   MFEM_ASSERT(item_size == sizeof(double), "");
+   if (unq_ltdof.size<int>() == 0) { return; }
+   if (host_shr_buf)
+   {
+      shr_buf.OccaMem().copyFrom(host_shr_buf);
+   }
+   AddSubVector((int)unq_ltdof.size<int>(), unq_ltdof, unq_shr_i, unq_shr_j,
+                shr_buf.OccaMem(), dst);
+}
+
+void OccaConformingProlongation::Mult_(const Vector &x, Vector &y) const
+{
+   const GroupTopology &gtopo = gc.GetGroupTopology();
+
+   BcastBeginCopy(x.OccaMem(), sizeof(double)); // copy to 'shr_buf'
+
+   int req_counter = 0;
+   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
+   {
+      const int send_offset = shr_buf_offsets[nbr];
+      const int send_size = shr_buf_offsets[nbr+1] - send_offset;
+      if (send_size > 0)
+      {
+         void *send_buf;
+         if (host_shr_buf)
+         {
+            send_buf = host_shr_buf + send_offset*sizeof(double);
+         }
+         else
+         {
+            send_buf = (shr_buf.OccaMem() + send_offset*sizeof(double)).ptr();
+         }
+         MPI_Isend(send_buf, send_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
+                   41822, gtopo.GetComm(), &requests[req_counter++]);
+      }
+
+      const int recv_offset = ext_buf_offsets[nbr];
+      const int recv_size = ext_buf_offsets[nbr+1] - recv_offset;
+      if (recv_size > 0)
+      {
+         void *recv_buf;
+         if (host_ext_buf)
+         {
+            recv_buf = host_ext_buf + recv_offset*sizeof(double);
+         }
+         else
+         {
+            recv_buf = (ext_buf.OccaMem() + recv_offset*sizeof(double)).ptr();
+         }
+         MPI_Irecv(recv_buf, recv_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
+                   41822, gtopo.GetComm(), &requests[req_counter++]);
+      }
+   }
+
+   BcastLocalCopy(x.OccaMem(), y.OccaMem(), sizeof(double));
+
+   MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
+
+   BcastEndCopy(y.OccaMem(), sizeof(double)); // copy from 'ext_buf'
+}
+
+void OccaConformingProlongation::MultTranspose_(const Vector &x,
+                                                Vector &y) const
+{
+   const GroupTopology &gtopo = gc.GetGroupTopology();
+
+   ReduceBeginCopy(x.OccaMem(), sizeof(double)); // copy to 'ext_buf'
+
+   int req_counter = 0;
+   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
+   {
+      const int send_offset = ext_buf_offsets[nbr];
+      const int send_size = ext_buf_offsets[nbr+1] - send_offset;
+      if (send_size > 0)
+      {
+         void *send_buf;
+         if (host_ext_buf)
+         {
+            send_buf = host_ext_buf + send_offset*sizeof(double);
+         }
+         else
+         {
+            send_buf = (ext_buf.OccaMem() + send_offset*sizeof(double)).ptr();
+         }
+         MPI_Isend(send_buf, send_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
+                   41823, gtopo.GetComm(), &requests[req_counter++]);
+      }
+
+      const int recv_offset = shr_buf_offsets[nbr];
+      const int recv_size = shr_buf_offsets[nbr+1] - recv_offset;
+      if (recv_size > 0)
+      {
+         void *recv_buf;
+         if (host_shr_buf)
+         {
+            recv_buf = host_shr_buf + recv_offset*sizeof(double);
+         }
+         else
+         {
+            recv_buf = (shr_buf.OccaMem() + recv_offset*sizeof(double)).ptr();
+         }
+         MPI_Irecv(recv_buf, recv_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
+                   41823, gtopo.GetComm(), &requests[req_counter++]);
+      }
+   }
+
+   ReduceLocalCopy(x.OccaMem(), y.OccaMem(), sizeof(double));
+
+   MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
+
+   ReduceEndAssemble(y.OccaMem(), sizeof(double)); // assemble from 'shr_buf'
+}
+
+#endif // MFEM_USE_MPI
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/fespace.hpp

@@ -0,0 +1,210 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_FE_SPACE_HPP
+#define MFEM_BACKENDS_OCCA_FE_SPACE_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "engine.hpp"
+#include "operator.hpp"
+#include "../../fem/fem.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+/// TODO: doxygen
+class FiniteElementSpace : public mfem::PFiniteElementSpace
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // mfem::FiniteElementSpace *fes;
+
+   SharedPtr<Layout> e_layout;
+
+   ::occa::array<int> globalToLocalOffsets;
+   ::occa::array<int> globalToLocalIndices;
+   ::occa::array<int> localToGlobalMap;
+   ::occa::kernel globalToLocalKernel, localToGlobalKernel;
+
+   mfem::Ordering::Type ordering;
+
+   int globalDofs, localDofs;
+   int vdim;
+
+   mutable Operator *prolongationOp, *restrictionOp;
+
+   void SetupLocalGlobalMaps();
+   void SetupOperators() const; // calls virtual methods of 'fes' !!!
+   void SetupKernels();
+
+public:
+   /// TODO: doxygen
+   FiniteElementSpace(const Engine &e, mfem::FiniteElementSpace &fespace);
+
+   /// Virtual destructor
+   virtual ~FiniteElementSpace();
+
+   /// TODO: doxygen
+   const Engine &OccaEngine() const { return engine->As<Engine>(); }
+
+   /// TODO: doxygen
+   ::occa::device GetDevice(int idx = 0) const
+   { return OccaEngine().GetDevice(idx); }
+
+   mfem::Mesh* GetMesh() const { return fes->GetMesh(); }
+
+   Layout &OccaVLayout() const
+   { return *fes->GetVLayout().As<Layout>(); }
+
+   Layout &OccaTrueVLayout() const
+   { return *fes->GetTrueVLayout().As<Layout>(); }
+
+   Layout &OccaEVLayout() { return *e_layout; }
+
+   bool hasTensorBasis() const
+   { return dynamic_cast<const mfem::TensorBasisElement*>(fes->GetFE(0)); }
+
+   mfem::Ordering::Type GetOrdering() const { return ordering; }
+
+   int GetGlobalDofs() const { return globalDofs; }
+   int GetLocalDofs() const { return localDofs; }
+
+   int GetDim() const { return fes->GetMesh()->Dimension(); }
+   int GetVDim() const { return vdim; }
+
+   int GetVSize() const { return globalDofs * vdim; }
+   int GetTrueVSize() const { return fes->GetTrueVSize(); }
+   int GetGlobalVSize() const { return globalDofs*vdim; /* FIXME: MPI */ }
+   int GetGlobalTrueVSize() const { return fes->GetTrueVSize(); }
+
+   int GetNE() const { return fes->GetNE(); }
+
+   const mfem::FiniteElementCollection *FEColl() const
+   { return fes->FEColl(); }
+   const mfem::FiniteElement *GetFE(const int idx) const
+   { return fes->GetFE(idx); }
+
+   virtual const mfem::Operator *GetProlongationOperator() const
+   { return prolongationOp; }
+
+   virtual const mfem::Operator *GetRestrictionOperator() const
+   { return restrictionOp; }
+
+   virtual const mfem::Operator *GetInterpolationOperator(
+      const mfem::QuadratureSpace &qspace) const
+   { return NULL; /* FIXME */ }
+
+   virtual const mfem::Operator *GetGradientOperator(
+      const mfem::QuadratureSpace &qspace) const
+   { return NULL; /* FIXME */ }
+
+   const ::occa::array<int> GetLocalToGlobalMap() const
+   { return localToGlobalMap; }
+
+   /// L-vector to E-vector
+   void GlobalToLocal(const Vector &globalVec, Vector &localVec) const
+   {
+      globalToLocalKernel(globalDofs,
+                          localDofs * fes->GetNE(),
+                          globalToLocalOffsets,
+                          globalToLocalIndices,
+                          globalVec.OccaMem(), localVec.OccaMem());
+   }
+
+   /// E-vector to L-vector, transpose of GlobalToLocal
+   void LocalToGlobal(const Vector &localVec, Vector &globalVec) const
+   {
+      localToGlobalKernel(globalDofs,
+                          localDofs * fes->GetNE(),
+                          globalToLocalOffsets,
+                          globalToLocalIndices,
+                          localVec.OccaMem(), globalVec.OccaMem());
+   }
+};
+
+
+#ifdef MFEM_USE_MPI
+
+/// OCCA version of mfem::ConformingProlongationOperator
+class OccaConformingProlongation : public Operator
+{
+protected:
+   // size(shr_buf)=size(shr_ltdof)
+   // size(ext_buf)=size(ext_ldof)
+   Array shr_ltdof, ext_ldof;
+   mutable Array shr_buf, ext_buf;
+   mutable char *host_shr_buf, *host_ext_buf;
+   // Offsets into {shr,ext}_buf; size is num. neighbors, i.e.
+   // gc.GetGroupTopology().GetNumNeighbors():
+   int *shr_buf_offsets, *ext_buf_offsets;
+
+   ::occa::memory ltdof_ldof; // shared with the restriction operator
+
+   ::occa::memory unq_ltdof; // enumeration of the unique ltdofs in shr_ltdof
+   ::occa::memory unq_shr_i, unq_shr_j;
+
+   ::occa::kernel ExtractSubVector, SetSubVector, AddSubVector;
+
+   MPI_Request *requests;
+
+   const GroupCommunicator &gc;
+
+   // Kernel: copy ltdofs from 'src' to 'shr_buf' - prepare for send.
+   //         shr_buf[i] = src[shr_ltdof[i]]
+   void BcastBeginCopy(const ::occa::memory &src, std::size_t item_size) const;
+   // Kernel: copy ltdofs from 'src' to ldofs in 'dst'.
+   //         dst[ltdof_ldof[i]] = src[i]
+   void BcastLocalCopy(const ::occa::memory &src, ::occa::memory &dst,
+                       std::size_t item_size) const;
+   // Kernel: copy ext. dofs from 'ext_buf' to 'dst' - after recv.
+   //         dst[ext_ldof[i]] = ext_buf[i]
+   void BcastEndCopy(::occa::memory &dst, std::size_t item_size) const;
+
+   // Kernel: copy ext. dofs from 'src' to 'ext_buf' - prepare for send.
+   //         ext_buf[i] = src[ext_ldof[i]]
+   void ReduceBeginCopy(const ::occa::memory &src, std::size_t item_size) const;
+   // Kernel: copy owned ldofs from 'src' to ltdofs in 'dst'.
+   //         dst[i] = src[ltdof_ldof[i]]
+   void ReduceLocalCopy(const ::occa::memory &src, ::occa::memory &dst,
+                       std::size_t item_size) const;
+   // Kernel: assemble dofs from 'shr_buf' into to 'dst' - after recv.
+   //         dst[shr_ltdof[i]] += shr_buf[i]
+   void ReduceEndAssemble(::occa::memory &dst, std::size_t item_size) const;
+
+public:
+   OccaConformingProlongation(const FiniteElementSpace &ofes,
+                              const mfem::ParFiniteElementSpace &pfes,
+                              ::occa::memory ltdof_ldof_);
+
+   virtual ~OccaConformingProlongation();
+
+   // overrides
+   virtual void Mult_(const Vector &x, Vector &y) const;
+   virtual void MultTranspose_(const Vector &x, Vector &y) const;
+};
+
+#endif // MFEM_USE_MPI
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_FE_SPACE_HPP

backends/occa/fespace.okl

@@ -0,0 +1,67 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  TILESIZE : Tilesize for iterating over entries
+  ================================================
+*/
+
+#if VDIM_ORDERING == ORDERING_BY_VDIM
+typedef double *Global_t @dim(NUM_VDIM, globalEntries);
+typedef double *Local_t  @dim(NUM_VDIM, localEntries);
+#else
+typedef double *Global_t @dim(NUM_VDIM, globalEntries) @dimOrder(1, 0);
+typedef double *Local_t  @dim(NUM_VDIM, localEntries)  @dimOrder(1, 0);
+#endif
+
+@kernel void GlobalToLocal(const int globalEntries,
+                           const int localEntries,
+                           @restrict const int * offsets,
+                           @restrict const int * indices,
+                           @restrict const Global_t globalX,
+                           @restrict Local_t localX) {
+
+  for (int i = 0; i < globalEntries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < globalEntries) {
+      const int offset = offsets[i];
+      const int nextOffset = offsets[i + 1];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        const double dofValue = globalX(v, i);
+        for (int j = offset; j < nextOffset; ++j) {
+          localX(v, indices[j]) = dofValue;
+        }
+      }
+    }
+  }
+}
+
+@kernel void LocalToGlobal(const int globalEntries,
+                           const int localEntries,
+                           @restrict const int * offsets,
+                           @restrict const int * indices,
+                           @restrict const Local_t localX,
+                           @restrict Global_t globalX) {
+
+  for (int i = 0; i < globalEntries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < globalEntries) {
+      const int offset = offsets[i];
+      const int nextOffset = offsets[i + 1];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        double dofValue = 0;
+        for (int j = offset; j < nextOffset; ++j) {
+          dofValue += localX(v, indices[j]);
+        }
+        globalX(v, i) = dofValue;
+      }
+    }
+  }
+}

backends/occa/geometry.okl

@@ -0,0 +1,181 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef STORE_JACOBIAN
+#  define STORE_JACOBIAN 1
+#endif
+
+#ifndef STORE_JACOBIAN_INV
+#  define STORE_JACOBIAN_INV 1
+#endif
+
+#ifndef STORE_JACOBIAN_DET
+#  define STORE_JACOBIAN_DET 1
+#endif
+
+typedef double* Local1D_t @dim(1, NUM_DOFS, numElements);
+typedef double* Local2D_t @dim(2, NUM_DOFS, numElements);
+typedef double* Local3D_t @dim(3, NUM_DOFS, numElements);
+
+typedef double* QLocal_t @dim(NUM_QUAD, numElements);
+
+typedef double* DofToQuadD1D_t @dim(NUM_QUAD, NUM_DOFS);
+typedef double* DofToQuadD2D_t @dim(2, NUM_QUAD, NUM_DOFS);
+typedef double* DofToQuadD3D_t @dim(3, NUM_QUAD, NUM_DOFS);
+
+typedef double* Jacobian1D_t @dim(NUM_QUAD, numElements);
+typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD, numElements);
+typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
+
+@kernel void InitGeometryInfo1D(const int numElements,
+                                @restrict const DofToQuadD1D_t dofToQuadD,
+                                @restrict const Local1D_t nodes,
+                                @restrict Jacobian1D_t J,
+                                @restrict Jacobian1D_t invJ,
+                                @restrict QLocal_t detJ) {
+
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_nodes[NUM_DOFS];
+
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
+        s_nodes[d] = nodes(0, d, e);
+      }
+    }
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      double J11 = 0;
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const double wx = dofToQuadD(q, d);
+        J11 += wx * s_nodes[d];
+      }
+#if STORE_JACOBIAN
+      J(q, e) = J11;
+#endif
+#if STORE_JACOBIAN_INV
+      invJ(q, e) = 1.0 / J11;
+#endif
+#if STORE_JACOBIAN_DET
+      detJ(q, e) = J11;
+#endif
+    }
+  }
+}
+
+@kernel void InitGeometryInfo2D(const int numElements,
+                                @restrict const DofToQuadD2D_t dofToQuadD,
+                                @restrict const Local2D_t nodes,
+                                @restrict Jacobian2D_t J,
+                                @restrict Jacobian2D_t invJ,
+                                @restrict QLocal_t detJ) {
+
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_nodes[2 * NUM_DOFS] @dim(2, NUM_DOFS);
+
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
+        s_nodes(0, d) = nodes(0, d, e);
+        s_nodes(1, d) = nodes(1, d, e);
+      }
+    }
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      double J11 = 0, J12 = 0;
+      double J21 = 0, J22 = 0;
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const double wx = dofToQuadD(0, q, d);
+        const double wy = dofToQuadD(1, q, d);
+        const double x = s_nodes(0, d);
+        const double y = s_nodes(1, d);
+        J11 += (wx * x); J12 += (wx * y);
+        J21 += (wy * x); J22 += (wy * y);
+      }
+#if STORE_JACOBIAN_INV || STORE_JACOBIAN_DET
+      const double r_detJ = (J11 * J22) - (J12 * J21);
+#endif
+#if STORE_JACOBIAN
+      J(0, 0, q, e) = J11; J(1, 0, q, e) = J12;
+      J(0, 1, q, e) = J21; J(1, 1, q, e) = J22;
+#endif
+#if STORE_JACOBIAN_INV
+      const double r_idetJ = 1.0 / r_detJ;
+      invJ(0, 0, q, e) =  J22 * r_idetJ;
+      invJ(1, 0, q, e) = -J12 * r_idetJ;
+
+      invJ(0, 1, q, e) = -J21 * r_idetJ;
+      invJ(1, 1, q, e) =  J11 * r_idetJ;
+#endif
+#if STORE_JACOBIAN_DET
+      detJ(q, e) = r_detJ;
+#endif
+    }
+  }
+}
+
+@kernel void InitGeometryInfo3D(const int numElements,
+                                @restrict const DofToQuadD3D_t dofToQuadD,
+                                @restrict const Local3D_t nodes,
+                                @restrict Jacobian3D_t J,
+                                @restrict Jacobian3D_t invJ,
+                                @restrict QLocal_t detJ) {
+
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_nodes[3 * NUM_DOFS] @dim(3, NUM_DOFS);
+
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
+        s_nodes(0, d) = nodes(0, d, e);
+        s_nodes(1, d) = nodes(1, d, e);
+        s_nodes(2, d) = nodes(2, d, e);
+      }
+    }
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      double J11 = 0, J12 = 0, J13 = 0;
+      double J21 = 0, J22 = 0, J23 = 0;
+      double J31 = 0, J32 = 0, J33 = 0;
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const double wx = dofToQuadD(0, q, d);
+        const double wy = dofToQuadD(1, q, d);
+        const double wz = dofToQuadD(2, q, d);
+        const double x = s_nodes(0, d);
+        const double y = s_nodes(1, d);
+        const double z = s_nodes(2, d);
+        J11 += (wx * x); J12 += (wx * y); J13 += (wx * z);
+        J21 += (wy * x); J22 += (wy * y); J23 += (wy * z);
+        J31 += (wz * x); J32 += (wz * y); J33 += (wz * z);
+      }
+#if STORE_JACOBIAN_INV || STORE_JACOBIAN_DET
+      const double r_detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                             (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+#endif
+#if STORE_JACOBIAN
+      J(0, 0, q, e) = J11; J(1, 0, q, e) = J12; J(2, 0, q, e) = J13;
+      J(0, 1, q, e) = J21; J(1, 1, q, e) = J22; J(2, 1, q, e) = J23;
+      J(0, 2, q, e) = J31; J(1, 2, q, e) = J32; J(2, 2, q, e) = J33;
+#endif
+#if STORE_JACOBIAN_INV
+      const double r_idetJ = 1.0 / r_detJ;
+      invJ(0, 0, q, e) = r_idetJ * ((J22 * J33) - (J23 * J32));
+      invJ(1, 0, q, e) = r_idetJ * ((J32 * J13) - (J33 * J12));
+      invJ(2, 0, q, e) = r_idetJ * ((J12 * J23) - (J13 * J22));
+
+      invJ(0, 1, q, e) = r_idetJ * ((J23 * J31) - (J21 * J33));
+      invJ(1, 1, q, e) = r_idetJ * ((J33 * J11) - (J31 * J13));
+      invJ(2, 1, q, e) = r_idetJ * ((J13 * J21) - (J11 * J23));
+
+      invJ(0, 2, q, e) = r_idetJ * ((J21 * J32) - (J22 * J31));
+      invJ(1, 2, q, e) = r_idetJ * ((J31 * J12) - (J32 * J11));
+      invJ(2, 2, q, e) = r_idetJ * ((J11 * J22) - (J12 * J21));
+#endif
+#if STORE_JACOBIAN_DET
+      detJ(q, e) = r_detJ;
+#endif
+    }
+  }
+}

backends/occa/gridfunc.cpp

@@ -0,0 +1,86 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "gridfunc.hpp"
+#include "bilininteg.hpp"
+#include "../../fem/gridfunc.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+std::map<std::string, ::occa::kernel> gridFunctionKernels;
+
+::occa::kernel GetGridFunctionKernel(::occa::device device,
+                                     FiniteElementSpace &fespace,
+                                     const mfem::IntegrationRule &ir)
+{
+   const int numQuad = ir.GetNPoints();
+
+   const FiniteElement &fe = *(fespace.GetFE(0));
+   const int dim  = fe.GetDim();
+   const int vdim = fespace.GetVDim();
+
+   std::stringstream ss;
+   ss << ::occa::hash(device)
+      << "FEColl : " << fespace.FEColl()->Name()
+      << "Quad: "    << numQuad
+      << "Dim: "     << dim
+      << "VDim: "    << vdim;
+   std::string hash = ss.str();
+
+   // Kernel defines
+   ::occa::properties props;
+   props["defines/NUM_VDIM"] = vdim;
+
+   SetProperties(fespace, ir, props);
+
+   ::occa::kernel kernel = gridFunctionKernels[hash];
+   if (!kernel.isInitialized())
+   {
+      const std::string &okl_path = fespace.OccaEngine().GetOklPath();
+      kernel = device.buildKernel(okl_path + "gridfunc.okl",
+                                  stringWithDim("GridFuncToQuad", dim),
+                                  props);
+   }
+   return kernel;
+}
+
+void ToQuad(const IntegrationRule &ir, FiniteElementSpace &fespace, Vector &gf,
+            Vector &quadValues)
+{
+   const Engine &engine = fespace.OccaEngine();
+   ::occa::device device = engine.GetDevice();
+
+   OccaDofQuadMaps &maps = OccaDofQuadMaps::Get(device, fespace, ir);
+
+   const int elements = fespace.GetNE();
+   const int numQuad  = ir.GetNPoints();
+   quadValues.OccaResize(numQuad * elements, sizeof(double));
+
+   ::occa::kernel g2qKernel = GetGridFunctionKernel(device, fespace, ir);
+   g2qKernel(elements,
+             maps.dofToQuad,
+             fespace.GetLocalToGlobalMap(),
+             gf.OccaMem(),
+             quadValues.OccaMem());
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/gridfunc.hpp

@@ -0,0 +1,55 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_GRID_FUNC_HPP
+#define MFEM_BACKENDS_OCCA_GRID_FUNC_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "vector.hpp"
+#include "fespace.hpp"
+
+namespace mfem
+{
+
+class IntegrationRule;
+
+namespace occa
+{
+
+// TODO: make this object part of the backend or the engine.
+extern std::map<std::string, ::occa::kernel> gridFunctionKernels;
+
+// TODO: make this a method of the backend or the engine.
+::occa::kernel GetGridFunctionKernel(::occa::device device,
+                                     FiniteElementSpace &fespace,
+                                     const mfem::IntegrationRule &ir);
+
+// ToQuad version without the deprecated class.
+//
+// FIXME: This is the action of a global B matrix, mapping L-vector to Q-vector,
+//        so it should be made into an operator that can be constructed by the
+//        FE space class. A batched version, where only a subset of the elements
+//        are processed should be defined as well.
+//
+//        The abstract operator construction method in the FE space class is:
+//           PFiniteElementSpace::GetInterpolationOperator(...)
+void ToQuad(const IntegrationRule &ir, FiniteElementSpace &ofespace, Vector &gf,
+            Vector &quadValues);
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_GRID_FUNC_HPP

backends/occa/gridfunc.okl

@@ -0,0 +1,26 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+#ifdef USING_TENSOR_OPS
+#  ifdef OCCA_USING_CPU
+#    include "mfem-occa://gridfunc/tensor/cpu.okl"
+#  else
+#    include "mfem-occa://gridfunc/tensor/gpuHighOrder.okl"
+#  endif
+#else
+#  ifdef OCCA_USING_CPU
+#    include "mfem-occa://gridfunc/simplex/cpu.okl"
+#  else
+#    include "mfem-occa://gridfunc/simplex/gpuHighOrder.okl"
+#  endif
+#endif

backends/occa/gridfunc/simplex/cpu.okl

@@ -0,0 +1,63 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void GridFuncToQuad2D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal_t out) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const int gid = l2gMap(d, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          const double r_gf = gf[v + gid*NUM_VDIM];
+          double r_out = 0;
+          for (int q = 0; q < NUM_QUAD; ++q) {
+            r_out += r_gf * dofToQuad(d, q);
+          }
+          out(v, d, e) = r_out;
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void GridFuncToQuad3D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal_t out) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        const int gid = l2gMap(d, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          const double r_gf = gf[v + gid*NUM_VDIM];
+          double r_out = 0;
+          for (int q = 0; q < NUM_QUAD; ++q) {
+            r_out += r_gf * dofToQuad(d, q);
+          }
+          out(v, d, e) = r_out;
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/gridfunc/simplex/gpuHighOrder.okl

@@ -0,0 +1,79 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void GridFuncToQuad2D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal_t out) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_gf[NUM_VDIM][NUM_DOFS];
+
+    for (int dOff = 0; dOff < M2_INNER_BATCH; ++dOff; @inner) {
+      for (int d = dOff; d < NUM_DOFS; d += M2_INNER_BATCH) {
+        const int gid = l2gMap(d, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          s_gf[v][d] = gf[v + gid*NUM_VDIM]];
+        }
+      }
+    }
+
+    for (int qOff = 0; qOff < M2_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M2_INNER_BATCH) {
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          double r_out = 0;
+          for (int d = 0; d < NUM_DOFS; ++d) {
+            r_out += s_gf[v][d] * dofToQuad(d, q);
+          }
+          out(v, q, e) = r_out;
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void GridFuncToQuad3D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal_t out) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_gf[NUM_VDIM][NUM_DOFS];
+
+    for (int dOff = 0; dOff < M3_INNER_BATCH; ++dOff; @inner) {
+      for (int d = dOff; d < NUM_DOFS; d += M3_INNER_BATCH) {
+        const int gid = l2gMap(d, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          s_gf[v][d] = gf[v + gid*NUM_VDIM]];
+        }
+      }
+    }
+
+    for (int qOff = 0; qOff < M3_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M3_INNER_BATCH) {
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          double r_out = 0;
+          for (int d = 0; d < NUM_DOFS; ++d) {
+            r_out += s_gf[v][d] * dofToQuad(d, q);
+          }
+          out(v, q, e) = r_out;
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/gridfunc/tensor/cpu.okl

@@ -0,0 +1,188 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void GridFuncToQuad1D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap1D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal1D_t out) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double r_out[NUM_VDIM][NUM_QUAD_1D];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          r_out[v][qx] = 0;
+        }
+      }
+      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+        const int gid = l2gMap(dx, e);
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          const double r_gf = gf[v + gid*NUM_VDIM];
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            r_out[v][qx] += r_gf * dofToQuad(qx, dx);
+          }
+        }
+      }
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          out(v, qx, e) = r_out[v][qx];
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void GridFuncToQuad2D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap2D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal2D_t out) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double out_xy[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            out_xy[v][qy][qx] = 0;
+          }
+        }
+      }
+
+      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+        double out_x[NUM_VDIM][NUM_QUAD_1D];
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            out_x[v][qy] = 0;
+          }
+        }
+
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          const int gid = l2gMap(dx, dy, e);
+          for (int v = 0; v < NUM_VDIM; ++v) {
+            const double r_gf = gf[v + gid*NUM_VDIM];
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              out_x[v][qy] += r_gf * dofToQuad(qy, dx);
+            }
+          }
+        }
+
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            const double d2q = dofToQuad(qy, dy);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              out_xy[v][qy][qx] += d2q * out_x[v][qx];
+            }
+          }
+        }
+      }
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          for (int v = 0; v < NUM_VDIM; ++v) {
+            out(v, qx, qy, e) = out_xy[v][qy][qx];
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void GridFuncToQuad3D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap3D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QVLocal3D_t out) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double out_xyz[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D];
+      for (int v = 0; v < NUM_VDIM; ++v) {
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              out_xyz[v][qz][qy][qx] = 0;
+            }
+          }
+        }
+      }
+
+      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+        double out_xy[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D];
+        for (int v = 0; v < NUM_VDIM; ++v) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              out_xy[v][qy][qx] = 0;
+            }
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          double out_x[NUM_VDIM][NUM_QUAD_1D];
+          for (int v = 0; v < NUM_VDIM; ++v) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              out_x[v][qx] = 0;
+            }
+          }
+
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const int gid = l2gMap(dx, dy, dz, e);
+            for (int v = 0; v < NUM_VDIM; ++v) {
+              const double r_gf = gf[v + gid*NUM_VDIM];
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                out_x[v][qx] += r_gf * dofToQuad(qx, dx);
+              }
+            }
+          }
+
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            const double wy = dofToQuad(qy, dy);
+            for (int v = 0; v < NUM_VDIM; ++v) {
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                out_xy[v][qy][qx] += wy * out_x[v][qx];
+              }
+            }
+          }
+        }
+
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          const double wz = dofToQuad(qz, dz);
+          for (int v = 0; v < NUM_VDIM; ++v) {
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                out_xyz[v][qz][qy][qx] += wz * out_xy[v][qy][qx];
+              }
+            }
+          }
+        }
+      }
+
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            for (int v = 0; v < NUM_VDIM; ++v) {
+              out(v, qx, qy, qz, e) = out_xyz[v][qz][qy][qx];
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/gridfunc/tensor/gpuHighOrder.okl

@@ -0,0 +1,183 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void GridFuncToQuad1D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap1D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QLocal1D_t out) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @exclusive double r_out[NUM_QUAD_1D];
+
+    for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+      for (int i = el; i < NUM_QUAD_DOFS_1D; i += M1_INNER_ELEMENT_BATCH) {
+        s_dofToQuad[i] = dofToQuad[i];
+      }
+    }
+
+    for (int b = 0; b < M1_OUTER_ELEMENT_BATCH; ++b) {
+      for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+        const int e = eOff + b*M1_INNER_ELEMENT_BATCH + el;
+        if (e < numElements) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            r_out[qx] = 0;
+          }
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double r_gf = gf[l2gMap(dx, e)];
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              r_out[qx] += r_gf * s_dofToQuad(qx, dx);
+            }
+          }
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            out(qx, e) = r_out[qx];
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void GridFuncToQuad2D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap2D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QLocal2D_t out) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+
+    // Store xy planes in shared memory
+    @shared double s_xy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+      for (int id = x; id < NUM_QUAD_DOFS_1D; id += NUM_MAX_1D) {
+        s_dofToQuad[id] = dofToQuad[id];
+      }
+    }
+
+    for (int e = eOff; e < (eOff + M2_ELEMENT_BATCH); ++e) {
+      if (e < numElements) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if (dx < NUM_DOFS_1D) {
+            double r_x[NUM_DOFS_1D];
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              r_x[dy] = gf[l2gMap(dx, dy, e)];
+            }
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              double xy = 0;
+              for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+                xy += r_x[dy] * s_dofToQuad(qy, dy);
+              }
+              s_xy(dx, qy) = xy;
+            }
+          }
+        }
+        for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+          if (qy < NUM_QUAD_1D) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              double val = 0;
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                val += s_xy(dx, qy) * s_dofToQuad(qx, dx);
+              }
+              out(qx, qy, e) = val;
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void GridFuncToQuad3D(const int numElements,
+                              @restrict const DofToQuad_t dofToQuad,
+                              @restrict const DLocalMap3D_t l2gMap,
+                              @restrict const double * gf,
+                              @restrict QLocal3D_t out) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+
+    // Store xy planes in shared memory
+    @shared double s_z[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
+
+    // Store z axis as registers
+    @exclusive double r_qz[NUM_QUAD_1D];
+
+    for (int y = 0; y < NUM_MAX_1D; ++y; @inner) {
+      for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+        const int id = (y * NUM_MAX_1D) + x;
+        // Fetch Q <--> D maps
+        if (id < NUM_QUAD_DOFS_1D) {
+          s_dofToQuad[id] = dofToQuad[id];
+        }
+        // Initialize our Z axis
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          r_qz[qz] = 0;
+        }
+      }
+    }
+
+    for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+      for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+        if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+          for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+            const double val = gf[l2gMap(dx, dy, dz, e)];
+            // Calculate D -> Q in the Z axis
+            for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+              r_qz[qz] += val * s_dofToQuad(qz, dz);
+            }
+          }
+        }
+      }
+    }
+    // For each xy plane
+    for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+      // Fill xy plane at given z position
+      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+            s_z(dx, dy) = r_qz[qz];
+          }
+        }
+      }
+      // Calculate Dxyz, xDyz, xyDz in plane
+      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
+            double val = 0;
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              const double wy = s_dofToQuad(qy, dy);
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                const double wx = s_dofToQuad(qx, dx);
+                val += wx * wy * s_z(dx, dy);
+              }
+            }
+            out(qx, qy, qz, e) = val;
+          }
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/interpolation.cpp

@@ -0,0 +1,119 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "interpolation.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+RestrictionOperator::RestrictionOperator(Layout &in_layout, Layout &out_layout,
+                                         ::occa::array<int> indices)
+   : Operator(in_layout, out_layout)
+{
+   trueIndices = indices;
+
+   ::occa::device device = in_layout.OccaEngine().GetDevice();
+   const std::string &okl_path = in_layout.OccaEngine().GetOklPath();
+   multOp = device.buildKernel(okl_path + "mappings.okl",
+                               "ExtractSubVector",
+                               "defines: { TILESIZE: 256 }");
+
+   multTransposeOp = device.buildKernel(okl_path + "mappings.okl",
+                                        "SetSubVector",
+                                        "defines: { TILESIZE: 256 }");
+}
+
+void RestrictionOperator::Mult_(const Vector &x, Vector &y) const
+{
+   // y[i] = x[trueIndices[i]]
+   multOp(height, trueIndices, x.OccaMem(), y.OccaMem());
+}
+
+void RestrictionOperator::MultTranspose_(const Vector &x, Vector &y) const
+{
+   y.OccaFill<double>(0.0);
+   // y[trueIndices[i]] = x[i]
+   multTransposeOp(height, trueIndices, x.OccaMem(), y.OccaMem());
+}
+
+ProlongationOperator::ProlongationOperator(OccaSparseMatrix &multOp_,
+                                           OccaSparseMatrix &multTransposeOp_)
+   : Operator(multOp_),
+     pmat(NULL),
+     multOp(multOp_),
+     multTransposeOp(multTransposeOp_)
+{ }
+
+ProlongationOperator::ProlongationOperator(Layout &in_layout,
+                                           Layout &out_layout,
+                                           const mfem::Operator *pmat_)
+   : Operator(in_layout, out_layout),
+     pmat(pmat_),
+     multOp(*this),
+     multTransposeOp(*this)
+{ }
+
+void ProlongationOperator::Mult_(const Vector &x, Vector &y) const
+{
+   MFEM_VERIFY(pmat == NULL, "");
+   multOp.Mult_(x, y);
+}
+
+void ProlongationOperator::MultTranspose_(const Vector &x, Vector &y) const
+{
+   MFEM_VERIFY(pmat == NULL, "");
+   multTransposeOp.Mult_(x, y);
+}
+
+void ProlongationOperator::Mult(const mfem::Vector &x, mfem::Vector &y) const
+{
+   if (pmat)
+   {
+      // FIXME: create an OCCA version of 'pmat'
+      x.Pull();
+      y.Pull(false);
+      pmat->Mult(x, y);
+      y.Push();
+   }
+   else
+   {
+      multOp.Mult(x, y);
+   }
+}
+
+void ProlongationOperator::MultTranspose(const mfem::Vector &x,
+                                         mfem::Vector &y) const
+{
+   if (pmat)
+   {
+      // FIXME: create an OCCA version of 'pmat'
+      x.Pull();
+      y.Pull(false);
+      pmat->MultTranspose(x, y);
+      y.Push();
+   }
+   else
+   {
+      multTransposeOp.Mult(x, y);
+   }
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/interpolation.hpp

@@ -0,0 +1,73 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_INTERPOLATION_HPP
+#define MFEM_BACKENDS_OCCA_INTERPOLATION_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include <occa.hpp>
+#include "vector.hpp"
+#include "engine.hpp"
+#include "sparsemat.hpp"
+#include "../../fem/fem.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class RestrictionOperator : public Operator
+{
+protected:
+   ::occa::array<int> trueIndices; // ldof = trueIndices[ltdof]
+   ::occa::kernel multOp, multTransposeOp;
+
+public:
+   RestrictionOperator(Layout &in_layout, Layout &out_layout,
+                       ::occa::array<int> indices);
+
+   // overrides
+   virtual void Mult_(const Vector &x, Vector &y) const;
+   virtual void MultTranspose_(const Vector &x, Vector &y) const;
+};
+
+class ProlongationOperator : public Operator
+{
+protected:
+   const mfem::Operator *pmat;
+   OccaSparseMatrix multOp, multTransposeOp;
+
+public:
+   ProlongationOperator(OccaSparseMatrix &multOp_,
+                        OccaSparseMatrix &multTransposeOp_);
+
+   ProlongationOperator(Layout &in_layout, Layout &out_layout,
+                        const mfem::Operator *pmat_);
+
+   // overrides
+   virtual void Mult_(const Vector &x, Vector &y) const;
+   virtual void MultTranspose_(const Vector &x, Vector &y) const;
+
+   // overrides
+   virtual void Mult(const mfem::Vector &x, mfem::Vector &y) const;
+   virtual void MultTranspose(const mfem::Vector &x, mfem::Vector &y) const;
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_INTERPOLATION_HPP

backends/occa/layout.cpp

@@ -0,0 +1,40 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "layout.hpp"
+#include "../../general/array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+void Layout::Resize(std::size_t new_size)
+{
+   size = new_size;
+}
+
+void Layout::Resize(const Array<std::size_t> &offsets)
+{
+   MFEM_ASSERT(offsets.Size() == 2,
+               "multiple workers are not supported yet");
+   size = offsets.Last();
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/layout.hpp

@@ -0,0 +1,67 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_LAYOUT_HPP
+#define MFEM_BACKENDS_OCCA_LAYOUT_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "../base/layout.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Layout : public PLayout
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // SharedPtr<const mfem::Engine> engine;
+   // std::size_t size;
+
+public:
+   Layout(const Engine &e, std::size_t s = 0) : PLayout(e, s) { }
+
+   const Engine &OccaEngine() const
+   { return *static_cast<const Engine *>(engine.Get()); }
+
+   void OccaResize(std::size_t new_size) { size = new_size; }
+
+   virtual ~Layout() { }
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   /// Resize the layout
+   virtual void Resize(std::size_t new_size);
+
+   /// Resize the layout based on the given worker offsets
+   virtual void Resize(const Array<std::size_t> &offsets);
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_LAYOUT_HPP

backends/occa/mappings.okl

@@ -0,0 +1,76 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  TILESIZE : Tilesize for iterating over entries
+  ================================================
+*/
+
+@kernel void ExtractSubVector(const int entries,
+                              @restrict const int *indices,
+                              @restrict const double *in,
+                              @restrict double *out) {
+
+  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < entries) {
+      out[i] = in[indices[i]]; // indices can be repeated
+    }
+  }
+}
+
+@kernel void SetSubVector(const int entries,
+                          @restrict const int *indices,
+                          @restrict const double *in,
+                          @restrict double *out) {
+
+  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < entries) {
+      out[indices[i]] = in[i]; // indices CANNOT be repeated
+    }
+  }
+}
+
+@kernel void AddSubVector(const int num_unique_dst_indices,
+                          @restrict const int *unique_dst_indices,
+                          @restrict const int *unique_to_src_offsets,
+                          @restrict const int *unique_to_src_indices,
+                          @restrict const double *src,
+                          @restrict double *dst) {
+
+  for (int i = 0; i < num_unique_dst_indices; ++i;
+       @tile(TILESIZE, @outer, @inner)) {
+
+    if (i < num_unique_dst_indices) {
+      const int dst_idx = unique_dst_indices[i];
+      double sum = dst[dst_idx];
+      const int end = unique_to_src_offsets[i+1];
+      for (int j = unique_to_src_offsets[i]; j != end; ++j) {
+        sum += src[unique_to_src_indices[j]];
+      }
+      dst[dst_idx] = sum;
+    }
+  }
+}
+
+@kernel void MapSubVector(const int entries,
+                          @restrict const int *indices,
+                          @restrict const double *in,
+                          @restrict double *out) {
+
+  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < entries) {
+      const int fromIdx = indices[2*i + 0]; // fromIdx indices can be repeated
+      const int toIdx   = indices[2*i + 1]; // toIdx indices CANNOT be repeated
+      out[toIdx] = in[fromIdx];
+    }
+  }
+}

backends/occa/mass/simplex/cpu.okl

@@ -0,0 +1,122 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian2D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+      oper(q, e) = quadWeights[q] * COEFF * ((J11 * J22) - (J21 * J12));
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD2D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD2D_t quadToDofD,
+                       @restrict const QLocal_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double r_sol[NUM_DOFS];
+
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        r_sol[d] = 0;
+      }
+
+      for (int q = 0; q < NUM_QUAD; ++q) {
+        double s = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          s += solIn(d, e) * quadToDof(d, q);
+        }
+        s *= oper(q, e);
+
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          r_sol[d] += (s * quadToDof(d, q));
+        }
+      }
+
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        solOut(d, e) += r_sol[d];
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian3D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+      oper(q, e) = quadWeights[q] * COEFF * detJ;
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD3D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD3D_t quadToDofD,
+                       @restrict const QLocal_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double r_sol[NUM_DOFS];
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        r_sol[d] = 0;
+      }
+
+      for (int q = 0; q < NUM_QUAD; ++q) {
+        double s = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          s += solIn(d, e) * quadToDof(d, q);
+        }
+        s *= oper(q, e);
+
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          r_sol[d] += (s * quadToDof(d, q));
+        }
+      }
+
+      for (int d = 0; d < NUM_DOFS; ++d) {
+        solOut(d, e) += r_sol[d];
+      }
+    }
+  }
+}
+//======================================

backends/occa/mass/simplex/gpuHighOrder.okl

@@ -0,0 +1,129 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian2D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD; q += A2_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+            oper(q, e) = quadWeights[q] * COEFF * ((J11 * J22) - (J21 * J12));
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD2D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD2D_t quadToDofD,
+                       @restrict const QLocal_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_sol[NUM_QUAD];
+
+    for (int qOff = 0; qOff < M2_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M2_INNER_BATCH) {
+        double s = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          s += solIn(d, e) * dofToQuad(d, q);
+        }
+        s_sol[q] = s * oper(q, e);
+      }
+    }
+
+    for (int dOff = 0; dOff < M2_INNER_BATCH; ++dOff) {
+      for (int d = dOff; d < NUM_DOFS; d += M2_INNER_BATCH) {
+        double r_sol = 0;
+        for (int q = 0; q < NUM_QUAD; ++q) {
+          r_sol += (s_sol[q] * quadToDof(d, q));
+        }
+        solOut(d, e) += r_sol;
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian3D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD; q += A3_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+            oper(q, e) = quadWeights[q] * COEFF * detJ;
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuadD3D_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDofD3D_t quadToDofD,
+                       @restrict const QLocal_t oper,
+                       @restrict const DLocal_t solIn,
+                       @restrict DLocal_t solOut) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    @shared double s_sol[NUM_QUAD];
+
+    for (int qOff = 0; qOff < M3_INNER_BATCH; ++qOff; @inner) {
+      for (int q = qOff; q < NUM_QUAD; q += M3_INNER_BATCH) {
+        double s = 0;
+        for (int d = 0; d < NUM_DOFS; ++d) {
+          s += solIn(d, e) * dofToQuad(d, q);
+        }
+        s_sol[q] = s * oper(q, e);
+      }
+    }
+
+    for (int dOff = 0; dOff < M3_INNER_BATCH; ++dOff) {
+      for (int d = dOff; d < NUM_DOFS; d += M3_INNER_BATCH) {
+        double r_sol = 0;
+        for (int q = 0; q < NUM_QUAD; ++q) {
+          r_sol += (s_sol[q] * quadToDof(d, q));
+        }
+        solOut(d, e) += r_sol;
+      }
+    }
+  }
+}
+//======================================

backends/occa/mass/tensor/cpu.okl

@@ -0,0 +1,281 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void Assemble1D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian1D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
+      const double detJ = J(q, e);
+      oper(q, e) = quadWeights[q] * COEFF * detJ;
+    }
+  }
+}
+
+@kernel void MultAdd1D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal1D_t oper,
+                       @restrict const DLocal1D_t solIn,
+                       @restrict DLocal1D_t solOut) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double sol_x[NUM_QUAD_1D];
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        sol_x[qx] = 0;
+      }
+      // sol_x{qx} = dofToQuad{qx,dx} * sol{dx}
+      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+        const double s = solIn(dx, e);
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          sol_x[qx] += s * dofToQuad(qx, dx);
+        }
+      }
+      // sol_x{q} *= oper{q}
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        sol_x[qx] *= oper(qx, e);
+      }
+      // sol{dx} = quadToDof{dx,qx} * sol_x{qx}
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          solOut(dx, e) += sol_x[qx] * quadToDof(dx, qx);
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian2D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_2D; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+      const double detJ = ((J11 * J22) - (J21 * J12));
+
+      oper(q, e) = quadWeights[q] * COEFF * detJ;
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal2D_t oper,
+                       @restrict const DLocal2D_t solIn,
+                       @restrict DLocal2D_t solOut) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double sol_xy[NUM_QUAD_1D][NUM_QUAD_1D];
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          sol_xy[qy][qx] = 0;
+        }
+      }
+
+      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+        double sol_x[NUM_QUAD_1D];
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          sol_x[qy] = 0;
+        }
+
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          const double s = solIn(dx, dy, e);
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_x[qx] += dofToQuad(qx, dx) * s;
+          }
+        }
+
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          const double d2q = dofToQuad(qy, dy);
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_xy[qy][qx] += d2q * sol_x[qx];
+          }
+        }
+      }
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          sol_xy[qy][qx] *= oper(qx, qy, e);
+        }
+      }
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        double sol_x[NUM_DOFS_1D];
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          sol_x[dx] = 0;
+        }
+
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          const double s = sol_xy[qy][qx];
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            sol_x[dx] += quadToDof(dx, qx) * s;
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          const double q2d = quadToDof(dy, qy);
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            solOut(dx, dy, e) += q2d * sol_x[dx];
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian3D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_3D; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+      oper(q, e) = quadWeights[q] * COEFF * detJ;
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal3D_t oper,
+                       @restrict const DLocal3D_t solIn,
+                       @restrict DLocal3D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double sol_xyz[NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D];
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_xyz[qz][qy][qx] = 0;
+          }
+        }
+      }
+
+      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+        double sol_xy[NUM_QUAD_1D][NUM_QUAD_1D];
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_xy[qy][qx] = 0;
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          double sol_x[NUM_QUAD_1D];
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_x[qx] = 0;
+          }
+
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double s = solIn(dx, dy, dz, e);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              sol_x[qx] += dofToQuad(qx, dx) * s;
+            }
+          }
+
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            const double wy = dofToQuad(qy, dy);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              sol_xy[qy][qx] += wy * sol_x[qx];
+            }
+          }
+        }
+
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          const double wz = dofToQuad(qz, dz);
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
+            }
+          }
+        }
+      }
+
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_xyz[qz][qy][qx] *= oper(qx, qy, qz, e);
+          }
+        }
+      }
+
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        double sol_xy[NUM_DOFS_1D][NUM_DOFS_1D];
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            sol_xy[dy][dx] = 0;
+          }
+        }
+
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          double sol_x[NUM_DOFS_1D];
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            sol_x[dx] = 0;
+          }
+
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            const double s = sol_xyz[qz][qy][qx];
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              sol_x[dx] += quadToDof(dx, qx) * s;
+            }
+          }
+
+          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+            const double wy = quadToDof(dy, qy);
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              sol_xy[dy][dx] += wy * sol_x[dx];
+            }
+          }
+        }
+
+        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+          const double wz = quadToDof(dz, qz);
+          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              solOut(dx, dy, dz, e) += wz * sol_xy[dy][dx];
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================

backends/occa/mass/tensor/gpuHighOrder.okl

@@ -0,0 +1,341 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void Assemble1D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian1D_t J,
+                        COEFF_ARGS
+                        QLocal_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A1_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A1_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
+          oper(q, e) = quadWeights[q] * COEFF * J(q, e);
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd1D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal1D_t oper,
+                       @restrict const DLocal1D_t solIn,
+                       @restrict DLocal1D_t solOut) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    @exclusive double r_sol[NUM_QUAD_1D];
+
+    for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+      for (int i = el; i < NUM_QUAD_DOFS_1D; i += M1_INNER_ELEMENT_BATCH) {
+        s_dofToQuad[i] = dofToQuad[i];
+        s_quadToDof[i] = quadToDof[i];
+      }
+    }
+
+    for (int b = 0; b < M1_OUTER_ELEMENT_BATCH; ++b) {
+      for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
+        const int e = eOff + b*M1_INNER_ELEMENT_BATCH + el;
+        if (e < numElements) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            r_sol[qx] = 0;
+          }
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            const double s = solIn(dx, e);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              r_sol[qx] += s * s_dofToQuad(qx, dx);
+            }
+          }
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            r_sol[qx] *= oper(qx, e);
+          }
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            double s = 0;
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              s += r_sol[qx] * s_quadToDof(dx, qx);
+            }
+            solOut(dx, e) += s;
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian2D_t J,
+                        COEFF_ARGS
+                        QLocal_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD_2D; q += A2_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+            oper(q, e) = quadWeights[q] * COEFF * ((J11 * J22) - (J21 * J12));
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal2D_t oper,
+                       @restrict const DLocal2D_t solIn,
+                       @restrict DLocal2D_t solOut) {
+  // Iterate over elements
+  for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    // Store xy planes in @shared memory
+    @shared double s_xy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+    @shared double s_xy2[NUM_QUAD_2D] @dim(NUM_QUAD_1D, NUM_QUAD_1D);
+
+    @exclusive double r_x[NUM_MAX_1D];
+
+    for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+      for (int id = x; id < NUM_QUAD_DOFS_1D; id += NUM_MAX_1D) {
+        s_dofToQuad[id]  = dofToQuad[id];
+        s_quadToDof[id]  = quadToDof[id];
+      }
+    }
+
+    for (int e = eOff; e < (eOff + M2_ELEMENT_BATCH); ++e) {
+      if (e < numElements) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if (dx < NUM_DOFS_1D) {
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              s_xy(dx, qy) = 0;
+            }
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              r_x[dy] = solIn(dx, dy, e);
+            }
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              double xy = 0;
+              for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+                xy += r_x[dy] * s_dofToQuad(qy, dy);
+              }
+              s_xy(dx, qy) = xy;
+            }
+          }
+        }
+        for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+          if (qy < NUM_QUAD_1D) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              double s = 0;
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                s += s_xy(dx, qy) * s_dofToQuad(qx, dx);
+              }
+              s_xy2(qx, qy) = s * oper(qx, qy, e);
+            }
+          }
+        }
+
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if (qx < NUM_QUAD_1D) {
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              s_xy(dy, qx) = 0;
+            }
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              r_x[qy] = s_xy2(qx, qy);
+            }
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              double s = 0;
+              for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+                s += r_x[qy] * s_quadToDof(dy, qy);
+              }
+              s_xy(dy, qx) = s;
+            }
+          }
+        }
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if (dx < NUM_DOFS_1D) {
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              double s = 0;
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                s += (s_xy(dy, qx) * s_quadToDof(dx, qx));
+              }
+              solOut(dx, dy, e) += s;
+            }
+          }
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        const double *quadWeights,
+                        const Jacobian3D_t J,
+                        COEFF_ARGS
+                        QLocal_t oper) {
+  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
+    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
+      if (e < numElements) {
+        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
+          for (int q = qOff; q < NUM_QUAD_3D; q += A3_QUAD_BATCH) {
+            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+            oper(q, e) = quadWeights[q] * COEFF * detJ;
+          }
+        }
+      }
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal3D_t oper,
+                       @restrict const DLocal3D_t solIn,
+                       @restrict DLocal3D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    // Store dof <--> quad mappings
+    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
+    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
+
+    // Store xy planes in @shared memory
+    @shared double s_xy[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
+
+    // Store z axis as registers
+    @exclusive double r_z[NUM_QUAD_1D];
+    @exclusive double r_z2[NUM_DOFS_1D];
+
+    for (int y = 0; y < NUM_MAX_1D; ++y; @inner) {
+      for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
+        const int id = (y * NUM_MAX_1D) + x;
+        // Fetch Q <--> D maps
+        if (id < NUM_QUAD_DOFS_1D) {
+          s_dofToQuad[id]  = dofToQuad[id];
+          s_quadToDof[id]  = quadToDof[id];
+        }
+        // Initialize our Z axis
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          r_z[qz] = 0;
+        }
+        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+          r_z2[dz] = 0;
+        }
+      }
+    }
+
+    for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+      for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+        if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+          for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+            const double s = solIn(dx, dy, dz, e);
+            // Calculate D -> Q in the Z axis
+            for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+              r_z[qz] += s * s_dofToQuad(qz, dz);
+            }
+          }
+        }
+      }
+    }
+    // For each xy plane
+    for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+      // Fill xy plane at given z position
+      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+            s_xy(dx, dy) = r_z[qz];
+          }
+        }
+      }
+      // Calculate Dxyz, xDyz, xyDz in plane
+      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
+            double s = 0;
+            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+              const double wy = s_dofToQuad(qy, dy);
+              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+                const double wx = s_dofToQuad(qx, dx);
+                s += wx * wy * s_xy(dx, dy);
+              }
+            }
+
+            s *= oper(qx, qy, qz, e);
+
+            for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+              const double wz  = s_quadToDof(dz, qz);
+              r_z2[dz] += wz * s;
+            }
+          }
+        }
+      }
+      @barrier("s_xy_sync_1");
+    }
+    // Iterate over xy planes to compute solution
+    for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+      // Place xy plane in @shared memory
+      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
+        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
+          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
+            s_xy(qx, qy) = r_z2[dz];
+          }
+        }
+      }
+      // Finalize solution in xy plane
+      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
+        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
+          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
+            double solZ = 0;
+            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+              const double wy = s_quadToDof(dy, qy);
+              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+                const double wx = s_quadToDof(dx, qx);
+                solZ += wx * wy * s_xy(qx, qy);
+              }
+            }
+            solOut(dx, dy, dz, e) += solZ;
+          }
+        }
+      }
+      @barrier("s_xy_sync_2");
+    }
+  }
+}
+//======================================

backends/occa/operator.cpp

@@ -0,0 +1,142 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "operator.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+// FIXME: move this object to the Backend?
+::occa::kernelBuilder OccaConstrainedOperator::mapDofBuilder =
+   ::occa::linalg::customLinearMethod(
+      "vector_map_dofs",
+
+      "const int idx = v2[i];"
+      "v0[idx] = v1[idx];",
+
+      "defines: {"
+      "  VTYPE0: 'double',"
+      "  VTYPE1: 'double',"
+      "  VTYPE2: 'int',"
+      "  TILESIZE: 128,"
+      "}");
+
+// FIXME: move this object to the Backend?
+::occa::kernelBuilder OccaConstrainedOperator::clearDofBuilder =
+   ::occa::linalg::customLinearMethod(
+      "vector_clear_dofs",
+
+      "v0[v1[i]] = 0.0;",
+
+      "defines: {"
+      "  VTYPE0: 'double',"
+      "  VTYPE1: 'int',"
+      "  TILESIZE: 128,"
+      "}");
+
+OccaConstrainedOperator::OccaConstrainedOperator(
+   mfem::Operator *A_,
+   const mfem::Array<int> &constraintList_,
+   bool own_A_)
+
+   : Operator(A_->InLayout()->As<Layout>()),
+     z(OutLayout_()),
+     w(OutLayout_()),
+     mfem_z((z.DontDelete(), z)),
+     mfem_w((w.DontDelete(), w))
+{
+   Setup(OutLayout_().OccaEngine().GetDevice(), A_, constraintList_, own_A_);
+}
+
+void OccaConstrainedOperator::Setup(::occa::device device_,
+                                    mfem::Operator *A_,
+                                    const mfem::Array<int> &constraintList_,
+                                    bool own_A_)
+{
+   device = device_;
+
+   A = A_;
+   own_A = own_A_;
+
+   constraintIndices = constraintList_.Size();
+   if (constraintList_.Size() > 0)
+   {
+      constraintList = constraintList_.Get_PArray()->As<Array>().OccaMem();
+   }
+   else
+   {
+      // constraintList is not used
+   }
+}
+
+void OccaConstrainedOperator::EliminateRHS(const Vector &x, Vector &b) const
+{
+   ::occa::kernel mapDofs = mapDofBuilder.build(device);
+
+   w.OccaFill(0.0);
+
+   if (constraintIndices)
+   {
+      mapDofs(constraintIndices, w.OccaMem(), x.OccaMem(), constraintList);
+   }
+
+   A->Mult(mfem_w, mfem_z);
+
+   b.Axpby<double>(1.0, b, -1.0, z);
+
+   if (constraintIndices)
+   {
+      mapDofs(constraintIndices, b.OccaMem(), x.OccaMem(), constraintList);
+   }
+}
+
+void OccaConstrainedOperator::Mult_(const Vector &x, Vector &y) const
+{
+   mfem::Vector mfem_y(y);
+   if (constraintIndices == 0)
+   {
+      A->Mult(x.Wrap(), mfem_y);
+      return;
+   }
+
+   ::occa::kernel mapDofs   = mapDofBuilder.build(device);
+   ::occa::kernel clearDofs = clearDofBuilder.build(device);
+
+   // z.OccaAssign(x); // z = x
+   // Is Axpy faster than DtoD copy on Volta?
+   z.Axpby(1.0, x, 0.0, x);
+
+   clearDofs(constraintIndices, z.OccaMem(), constraintList);
+
+   A->Mult(mfem_z, mfem_y);
+
+   mapDofs(constraintIndices, y.OccaMem(), x.OccaMem(), constraintList);
+}
+
+OccaConstrainedOperator::~OccaConstrainedOperator()
+{
+   if (own_A)
+   {
+      delete A;
+   }
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/operator.hpp

@@ -0,0 +1,127 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_OPERATOR_HPP
+#define MFEM_BACKENDS_OCCA_OPERATOR_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "vector.hpp"
+#include "../../linalg/operator.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class Operator : public mfem::Operator
+{
+public:
+   /// Creare an operator with the same dimensions as @a orig.
+   Operator(const Operator &orig)
+      : mfem::Operator(orig) { }
+
+   Operator(Layout &layout)
+      : mfem::Operator(layout) { }
+
+   Operator(Layout &in_layout, Layout &out_layout)
+      : mfem::Operator(in_layout, out_layout) { }
+
+   Layout &InLayout_() const { return in_layout->As<Layout>(); }
+
+   Layout &OutLayout_() const { return out_layout->As<Layout>(); }
+
+   virtual void Mult_(const Vector &x, Vector &y) const = 0;
+
+   virtual void MultTranspose_(const Vector &x, Vector &y) const
+   { MFEM_ABORT("method is not supported"); }
+
+   // override
+   virtual void Mult(const mfem::Vector &x, mfem::Vector &y) const
+   {
+      Mult_(x.Get_PVector()->As<Vector>(),
+            y.Get_PVector()->As<Vector>());
+   }
+
+   // override
+   virtual void MultTranspose(const mfem::Vector &x, mfem::Vector &y) const
+   {
+      MultTranspose_(x.Get_PVector()->As<Vector>(),
+                     y.Get_PVector()->As<Vector>());
+   }
+};
+
+
+class OccaConstrainedOperator : public Operator
+{
+protected:
+   ::occa::device device;
+
+   mfem::Operator *A;              //< The unconstrained Operator.
+   bool own_A;                     //< Ownership flag for A.
+   ::occa::memory constraintList;  //< List of constrained indices/dofs.
+   int constraintIndices;
+   mutable Vector z, w;       //< Auxiliary vectors.
+   mutable mfem::Vector mfem_z, mfem_w; // Wrap z, w
+
+   static ::occa::kernelBuilder mapDofBuilder, clearDofBuilder;
+
+public:
+   /** @brief Constructor from a general Operator and a list of essential
+       indices/dofs.
+
+       Specify the unconstrained operator @a *A and a @a list of indices to
+       constrain, i.e. each entry @a list[i] represents an essential-dof. If the
+       ownership flag @a own_A is true, the operator @a *A will be destroyed
+       when this object is destroyed. */
+   OccaConstrainedOperator(mfem::Operator *A_,
+                           const mfem::Array<int> &constraintList_,
+                           bool own_A_ = false);
+
+   void Setup(::occa::device device_,
+              mfem::Operator *A_,
+              const mfem::Array<int> &constraintList_,
+              bool own_A_ = false);
+
+   /** @brief Eliminate "essential boundary condition" values specified in @a x
+       from the given right-hand side @a b.
+
+       Performs the following steps:
+
+       z = A((0,x_b));  b_i -= z_i;  b_b = x_b;
+
+       where the "_b" subscripts denote the essential (boundary) indices/dofs of
+       the vectors, and "_i" -- the rest of the entries. */
+   void EliminateRHS(const Vector &x, Vector &b) const;
+
+   /** @brief Constrained operator action.
+
+       Performs the following steps:
+
+       z = A((x_i,0));  y_i = z_i;  y_b = x_b;
+
+       where the "_b" subscripts denote the essential (boundary) indices/dofs of
+       the vectors, and "_i" -- the rest of the entries. */
+   virtual void Mult_(const Vector &x, Vector &y) const;
+
+   // Destructor: destroys the unconstrained Operator @a A if @a own_A is true.
+   virtual ~OccaConstrainedOperator();
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_OPERATOR_HPP

backends/occa/sparse.okl

@@ -0,0 +1,57 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+/*
+  ---[ Defines Known At Compile-Time ]------------
+  TILESIZE : Tilesize for iterating over dofs
+  ================================================
+*/
+
+@kernel void Mult(const int entries,
+                  @restrict const int *offsets,
+                  @restrict const int *indices,
+                  @restrict const double *weights,
+                  @restrict const double *in,
+                  @restrict double *out) {
+
+  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < entries) {
+      const int offset = offsets[i];
+      const int nextOffset = offsets[i + 1];
+      double value = 0;
+      for (int j = offset; j < nextOffset; ++j) {
+        value += weights[j] * in[indices[j]];
+      }
+      out[i] = value;
+    }
+  }
+}
+
+@kernel void MappedMult(const int entries,
+                        @restrict const int *offsets,
+                        @restrict const int *indices,
+                        @restrict const double *weights,
+                        @restrict const int *outIndices,
+                        @restrict const double *in,
+                        @restrict double *out) {
+
+  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
+    if (i < entries) {
+      const int offset = offsets[i];
+      const int nextOffset = offsets[i + 1];
+      double value = 0;
+      for (int j = offset; j < nextOffset; ++j) {
+        value += weights[j] * in[indices[j]];
+      }
+      out[outIndices[i]] = value;
+    }
+  }
+}

backends/occa/sparsemat.cpp

@@ -0,0 +1,221 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "sparsemat.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                                   const mfem::SparseMatrix &m,
+                                   const ::occa::properties &props)
+   : Operator(in_layout, out_layout)
+{
+   Setup(in_layout.OccaEngine().GetDevice(), m, props);
+}
+
+OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                                   ::occa::array<int> offsets_,
+                                   ::occa::array<int> indices_,
+                                   ::occa::array<double> weights_,
+                                   ::occa::array<int> reorderIndices_,
+                                   ::occa::array<int> mappedIndices_,
+                                   const ::occa::properties &props)
+   : Operator(in_layout, out_layout),
+     offsets(offsets_),
+     indices(indices_),
+     weights(weights_),
+     reorderIndices(reorderIndices_),
+     mappedIndices(mappedIndices_)
+{
+   SetupKernel(in_layout.OccaEngine().GetDevice(), props);
+}
+
+void OccaSparseMatrix::Setup(::occa::device device, const mfem::SparseMatrix &m,
+                             const ::occa::properties &props)
+{
+   Setup(device, m, ::occa::array<int>(), ::occa::array<int>(), props);
+}
+
+void OccaSparseMatrix::Setup(::occa::device device, const SparseMatrix &m,
+                             ::occa::array<int> reorderIndices_,
+                             ::occa::array<int> mappedIndices_,
+                             const ::occa::properties &props)
+{
+   MFEM_ASSERT(m.Finalized(), "");
+   MFEM_ASSERT(m.Height() == height, "");
+   MFEM_ASSERT(m.Width() == width, "");
+
+   const int nnz = m.GetI()[height];
+   offsets.allocate(device,
+                    height + 1, m.GetI());
+   indices.allocate(device,
+                    nnz, m.GetJ());
+   weights.allocate(device,
+                    nnz, m.GetData());
+
+   offsets.keepInDevice();
+   indices.keepInDevice();
+   weights.keepInDevice();
+
+   reorderIndices = reorderIndices_;
+   mappedIndices  = mappedIndices_;
+
+   SetupKernel(device, props);
+}
+
+void OccaSparseMatrix::SetupKernel(::occa::device device,
+                                   const ::occa::properties &props)
+{
+   const bool hasOutIndices = mappedIndices.isInitialized();
+
+   const ::occa::properties defaultProps("defines: {"
+                                         "  TILESIZE: 256,"
+                                         "}");
+
+   const std::string &okl_path = InLayout_().OccaEngine().GetOklPath();
+   mapKernel = device.buildKernel(okl_path + "mappings.okl",
+                                  "MapSubVector",
+                                  defaultProps + props);
+
+   multKernel = device.buildKernel(okl_path + "sparse.okl",
+                                   hasOutIndices ? "MappedMult" : "Mult",
+                                   defaultProps + props);
+}
+
+void OccaSparseMatrix::Mult_(const Vector &x, Vector &y) const
+{
+   if (reorderIndices.isInitialized() ||
+       mappedIndices.isInitialized())
+   {
+      if (reorderIndices.isInitialized())
+      {
+         mapKernel((int) (reorderIndices.size() / 2),
+                   reorderIndices,
+                   x.OccaMem(), y.OccaMem());
+      }
+      if (mappedIndices.isInitialized())
+      {
+         multKernel((int) (mappedIndices.size()),
+                    offsets, indices, weights,
+                    mappedIndices,
+                    x.OccaMem(), y.OccaMem());
+      }
+   }
+   else
+   {
+      multKernel((int) height,
+                 offsets, indices, weights,
+                 x.OccaMem(), y.OccaMem());
+   }
+}
+
+
+OccaSparseMatrix* CreateMappedSparseMatrix(Layout &in_layout,
+                                           Layout &out_layout,
+                                           const mfem::SparseMatrix &m,
+                                           const ::occa::properties &props)
+{
+   const int mHeight = m.Height();
+   // const int mWidth  = m.Width();
+
+   // Count indices that are only reordered (true dofs)
+   const int *I = m.GetI();
+   const int *J = m.GetJ();
+   const double *D = m.GetData();
+
+   int trueCount = 0;
+   for (int i = 0; i < mHeight; ++i)
+   {
+      trueCount += ((I[i + 1] - I[i]) == 1);
+   }
+   const int dupCount = (mHeight - trueCount);
+
+   // Create the reordering map for entries that aren't modified (true dofs)
+   ::occa::device device(in_layout.OccaEngine().GetDevice());
+   ::occa::array<int> reorderIndices(device,
+                                     2 * trueCount);
+   ::occa::array<int> mappedIndices, offsets, indices;
+   ::occa::array<double> weights;
+
+   if (dupCount)
+   {
+      mappedIndices.allocate(device,
+                             dupCount);
+   }
+   int trueIdx = 0, dupIdx = 0;
+   for (int i = 0; i < mHeight; ++i)
+   {
+      const int i1 = I[i];
+      if ((I[i + 1] - i1) == 1)
+      {
+         reorderIndices[trueIdx++] = J[i1];
+         reorderIndices[trueIdx++] = i;
+      }
+      else
+      {
+         mappedIndices[dupIdx++] = i;
+      }
+   }
+   reorderIndices.keepInDevice();
+
+   if (dupCount)
+   {
+      mappedIndices.keepInDevice();
+
+      // Extract sparse matrix without reordered identity
+      const int dupNnz = I[mHeight] - trueCount;
+
+      offsets.allocate(device,
+                       dupCount + 1);
+      indices.allocate(device,
+                       dupNnz);
+      weights.allocate(device,
+                       dupNnz);
+
+      int nnz = 0;
+      offsets[0] = 0;
+      for (int i = 0; i < dupCount; ++i)
+      {
+         const int idx = mappedIndices[i];
+         const int offStart = I[idx];
+         const int offEnd   = I[idx + 1];
+         offsets[i + 1] = offsets[i] + (offEnd - offStart);
+         for (int j = offStart; j < offEnd; ++j)
+         {
+            indices[nnz] = J[j];
+            weights[nnz] = D[j];
+            ++nnz;
+         }
+      }
+
+      offsets.keepInDevice();
+      indices.keepInDevice();
+      weights.keepInDevice();
+   }
+
+   return new OccaSparseMatrix(in_layout, out_layout,
+                               offsets, indices, weights,
+                               reorderIndices, mappedIndices,
+                               props);
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/sparsemat.hpp

@@ -0,0 +1,89 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_SPARSE_MAT_HPP
+#define MFEM_BACKENDS_OCCA_SPARSE_MAT_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include <occa.hpp>
+#include "vector.hpp"
+#include "engine.hpp"
+#include "operator.hpp"
+#include "../../linalg/sparsemat.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+/// TODO: doxygen
+class OccaSparseMatrix : public Operator
+{
+protected:
+   ::occa::array<int> offsets, indices;
+   ::occa::array<double> weights;
+   ::occa::array<int> reorderIndices, mappedIndices;
+   ::occa::kernel mapKernel, multKernel;
+
+   void Setup(::occa::device device, const mfem::SparseMatrix &m,
+              const ::occa::properties &props);
+
+   void Setup(::occa::device device, const mfem::SparseMatrix &m,
+              ::occa::array<int> reorderIndices_,
+              ::occa::array<int> mappedIndices_,
+              const ::occa::properties &props);
+
+   void SetupKernel(::occa::device device,
+                    const ::occa::properties &props);
+
+public:
+   /// Construct an empty OccaSparseMatrix.
+   OccaSparseMatrix(const Operator &orig)
+      : Operator(orig) { }
+
+   // Implicitly defined copy constructor.
+
+   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                    const mfem::SparseMatrix &m,
+                    const ::occa::properties &props = ::occa::properties());
+
+   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
+                    ::occa::array<int> offsets_,
+                    ::occa::array<int> indices_,
+                    ::occa::array<double> weights_,
+                    ::occa::array<int> reorderIndices_,
+                    ::occa::array<int> mappedIndices_,
+                    const ::occa::properties &props = ::occa::properties());
+
+   const ::occa::array<int> &GetReorderIndices() const
+   { return reorderIndices; }
+
+   // override
+   virtual void Mult_(const Vector &x, Vector &y) const;
+};
+
+
+/// TODO: doxygen
+OccaSparseMatrix* CreateMappedSparseMatrix(
+   Layout &in_layout, Layout &out_layout,
+   const mfem::SparseMatrix &m,
+   const ::occa::properties &props = ::occa::properties());
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_SPARSE_MAT_HPP

backends/occa/url_handler.cpp

@@ -0,0 +1,81 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "url_handler.hpp"
+#include "../../general/error.hpp"
+#include <cstdlib>
+#include <sys/stat.h>
+
+namespace mfem
+{
+
+namespace occa
+{
+
+FileOpener::FileOpener(const std::string &prefix,
+                       const std::string &env_variable)
+   : pfx(prefix)
+{
+   const char *env_path = getenv(env_variable.c_str());
+   if (!env_path) { return; }
+   std::string path(env_path);
+   for (std::size_t start = 0, end; start < path.size(); start = end + 1)
+   {
+      end = path.find(':', start);
+      if (end == std::string::npos)
+      {
+         AddDir(path.substr(start, end));
+         break;
+      }
+      AddDir(path.substr(start, end - start));
+   }
+}
+
+bool FileOpener::AddDir(const std::string &dir)
+{
+   if (dir.size() == 0 || dir[0] != '/') { return false; }
+   struct stat dir_stat;
+   if (stat(dir.c_str(), &dir_stat)) { return false; }
+   if (!S_ISDIR(dir_stat.st_mode)) { return false; }
+   paths.push_back(dir + (*dir.rbegin() == '/' ? "" : "/"));
+   return true;
+}
+
+bool FileOpener::handles(const std::string &filename)
+{
+   return filename.size() >= pfx.size() &&
+          filename.compare(0, pfx.size(), pfx) == 0;
+}
+
+std::string FileOpener::expand(const std::string &filename)
+{
+   std::string sfx(filename.substr(pfx.size()));
+   for (std::size_t i = 0; i < paths.size(); i++)
+   {
+      std::string file = paths[i] + sfx;
+      struct stat file_stat;
+      if (stat(file.c_str(), &file_stat) == 0 && S_ISREG(file_stat.st_mode))
+      {
+         return file;
+      }
+   }
+   MFEM_ABORT("invalid url: " << filename);
+   return sfx;
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/url_handler.hpp

@@ -0,0 +1,47 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_URL_HANDLER_HPP
+#define MFEM_BACKENDS_OCCA_URL_HANDLER_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include <occa.hpp>
+
+namespace mfem
+{
+
+namespace occa
+{
+
+class FileOpener : public ::occa::io::fileOpener
+{
+protected:
+   std::string pfx; // prefix, e.g. "mfem://"
+   std::vector<std::string> paths; // paths to search for prefix replacement
+
+public:
+   FileOpener(const std::string &prefix, const std::string &env_variable);
+
+   bool AddDir(const std::string &dir);
+
+   virtual bool handles(const std::string &filename);
+   virtual std::string expand(const std::string &filename);
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_URL_HANDLER_HPP

backends/occa/utils.okl

@@ -0,0 +1,22 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+typedef double* Local_t @dim(numDofs, numElements);
+
+@kernel void InitLocalVector(const int numElements,
+                             const int numDofs,
+                             @restrict Local_t sol) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int d = 0; d < numDofs; ++d; @inner) {
+      sol(d, e) = 0;
+    }
+  }
+}

backends/occa/vector.cpp

@@ -0,0 +1,196 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include "vector.hpp"
+#include "../../linalg/vector.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+PVector *Vector::DoVectorClone(bool copy_data, void **buffer,
+                               int buffer_type_id) const
+{
+   MFEM_ASSERT(buffer_type_id == ScalarId<double>::value, "");
+   Vector *new_vector = new Vector(OccaLayout());
+   if (copy_data)
+   {
+      new_vector->slice.copyFrom(slice);
+   }
+   if (buffer)
+   {
+      *buffer = new_vector->GetBuffer();
+   }
+   return new_vector;
+}
+
+void Vector::DoDotProduct(const PVector &x, void *result,
+                          int result_type_id) const
+{
+   // Can be called when Size() == 0, e.g. when an MPI-parallel vector has a
+   // local size of 0.
+
+   MFEM_ASSERT(result_type_id == ScalarId<double>::value, "");
+   double *res = (double *)result;
+   const Vector &xp = x.As<Vector>();
+   MFEM_ASSERT(this->Size() == xp.Size(), "");
+   *res = ::occa::linalg::dot<double, double, double>(this->slice, xp.slice);
+
+#ifdef MFEM_USE_MPI
+   double local_dot = *res;
+   if (IsParallel())
+   {
+      MPI_Allreduce(&local_dot, res, 1, MPI_DOUBLE, MPI_SUM,
+                    OccaEngine().GetComm());
+   }
+#endif
+}
+
+void Vector::DoAxpby(const void *a, const PVector &x,
+                     const void *b, const PVector &y,
+                     int ab_type_id)
+{
+   //
+   // TODO: move all kernel builders to class mfem::occa::Backend
+   //
+   static ::occa::kernelBuilder axpby1_builder =
+      ::occa::linalg::customLinearMethod(
+         "mfem_occa_axpby1",
+         "v0[i] = c0 * v1[i];",
+         "defines: {"
+         "  CTYPE0: 'double',"
+         "  VTYPE0: 'double',"
+         "  VTYPE1: 'double',"
+         "  TILESIZE: '128',"
+         "}");
+
+   static ::occa::kernelBuilder axpby2_builder =
+      ::occa::linalg::customLinearMethod(
+         "mfem_occa_axpby2",
+         "v0[i] = c0 * v0[i] + c1 * v1[i];",
+         "defines: {"
+         "  CTYPE0: 'double',"
+         "  CTYPE1: 'double',"
+         "  VTYPE0: 'double',"
+         "  VTYPE1: 'double',"
+         "  TILESIZE: '128',"
+         "}");
+
+   static ::occa::kernelBuilder axpby3_builder =
+      ::occa::linalg::customLinearMethod(
+         "mfem_occa_axpby3",
+         "v0[i] = c0 * v1[i] + c1 * v2[i];",
+         "defines: {"
+         "  CTYPE0: 'double',"
+         "  CTYPE1: 'double',"
+         "  VTYPE0: 'double',"
+         "  VTYPE1: 'double',"
+         "  VTYPE2: 'double',"
+         "  TILESIZE: '128',"
+         "}");
+
+   // called only when Size() != 0
+
+   MFEM_ASSERT(ab_type_id == ScalarId<double>::value, "");
+   const double da = *static_cast<const double *>(a);
+   const double db = *static_cast<const double *>(b);
+   MFEM_ASSERT(da == 0.0 || dynamic_cast<const Vector *>(&x) != NULL,
+               "invalid Vector x");
+   MFEM_ASSERT(db == 0.0 || dynamic_cast<const Vector *>(&y) != NULL,
+               "invalid Vector y");
+   const Vector *xp = static_cast<const Vector *>(&x);
+   const Vector *yp = static_cast<const Vector *>(&y);
+
+   MFEM_ASSERT(da == 0.0 || this->Size() == xp->Size(), "");
+   MFEM_ASSERT(db == 0.0 || this->Size() == yp->Size(), "");
+
+   if (da == 0.0)
+   {
+      if (db == 0.0)
+      {
+         OccaFill(da);
+      }
+      else
+      {
+         if (this->slice == yp->slice)
+         {
+            // *this *= db
+            ::occa::linalg::operator_mult_eq(slice, db);
+         }
+         else
+         {
+            // *this = db * y
+            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice());
+            kernel((int)Size(), db, slice, yp->slice);
+         }
+      }
+   }
+   else
+   {
+      if (db == 0.0)
+      {
+         if (this->slice == xp->slice)
+         {
+            // *this *= da
+            ::occa::linalg::operator_mult_eq(slice, da);
+         }
+         else
+         {
+            // *this = da * x
+            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice());
+            kernel((int)Size(), da, slice, xp->slice);
+         }
+      }
+      else
+      {
+         MFEM_ASSERT(xp->slice != yp->slice, "invalid input");
+         if (this->slice == xp->slice)
+         {
+            // *this = da * (*this) + db * y
+            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice());
+            kernel((int)Size(), da, db, slice, yp->slice);
+         }
+         else if (this->slice == yp->slice)
+         {
+            // *this = da * x + db * (*this)
+            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice());
+            kernel((int)Size(), db, da, slice, xp->slice);
+         }
+         else
+         {
+            // *this = da * x + db * y
+            ::occa::kernel kernel = axpby3_builder.build(slice.getDevice());
+            kernel((int)Size(), da, db, slice, xp->slice, yp->slice);
+         }
+      }
+   }
+}
+
+mfem::Vector Vector::Wrap()
+{
+   return mfem::Vector(*this);
+}
+
+const mfem::Vector Vector::Wrap() const
+{
+   return mfem::Vector(*const_cast<Vector*>(this));
+}
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/vector.hpp

@@ -0,0 +1,83 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_OCCA_VECTOR_HPP
+#define MFEM_BACKENDS_OCCA_VECTOR_HPP
+
+#include "../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#include <occa.hpp>
+#include "../base/vector.hpp"
+#include "array.hpp"
+
+namespace mfem
+{
+
+namespace occa
+{
+
+// FIXME: Once XL fixes this code quirk we can remove this #ifdef switch
+#ifdef __ibmxl__
+class Vector : public Array, public PVector
+#else
+class Vector : virtual public Array, public PVector
+#endif
+{
+protected:
+   //
+   // Inherited fields
+   //
+   // DLayout layout;
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   virtual PVector *DoVectorClone(bool copy_data, void **buffer,
+                                  int buffer_type_id) const;
+
+   virtual void DoDotProduct(const PVector &x, void *result,
+                             int result_type_id) const;
+
+   virtual void DoAxpby(const void *a, const PVector &x,
+                        const void *b, const PVector &y,
+                        int ab_type_id);
+
+   ///@}
+   // End: Virtual interface
+
+public:
+   Vector(const Engine &e)
+      : PArray(*(new Layout(e, 0))), Array(e), PVector(*layout)
+   { }
+
+   Vector(Layout &lt)
+      : PArray(lt), Array(lt, sizeof(double)), PVector(lt)
+   { }
+
+   mfem::Vector Wrap();
+
+   const mfem::Vector Wrap() const;
+
+#if defined(MFEM_USE_MPI)
+   bool IsParallel() const { return (OccaEngine().GetComm() != MPI_COMM_NULL); }
+#endif
+};
+
+} // namespace mfem::occa
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
+
+#endif // MFEM_BACKENDS_OCCA_VECTOR_HPP

backends/occa/vmass/tensor/cpu.okl

@@ -0,0 +1,321 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "mfem-occa://defines.okl"
+
+//---[ 1D ]-----------------------------
+@kernel void Assemble1D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian1D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
+      oper(q, e) = quadWeights[q] * COEFF * J(q, e);
+    }
+  }
+}
+
+@kernel void MultAdd1D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal_t oper,
+                       @restrict const DVLocal1D_t solIn,
+                       @restrict DVLocal1D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double sol_x[1][NUM_QUAD_1D];
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        sol_x[0][qx] = 0;
+      }
+      // sol_x{qx} = dofToQuad{qx,dx} * sol{dx}
+      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          sol_x[0][qx] += dofToQuad(qx, dx) * solIn(0, dx, e);
+        }
+      }
+      // sol_x{q} *= oper{q}
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        sol_x[0][qx] *= oper(qx, e);
+      }
+      // sol{dx} = quadToDof{dx,qx} * sol_x{qx}
+      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          solOut(0, dx, e) += sol_x[0][qx] * quadToDof(dx, qx);
+        }
+      }
+    }
+  }
+}
+//======================================
+
+
+//---[ 2D ]-----------------------------
+@kernel void Assemble2D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian2D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_2D; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
+
+      oper(q, e) = quadWeights[q] * COEFF * ((J11 * J22) - (J21 * J12));
+    }
+  } // e
+}
+
+@kernel void MultAdd2D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal_t oper,
+                       @restrict const DVLocal2D_t solIn,
+                       @restrict DVLocal2D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy=0; dummy<1; ++dummy; @inner) {
+      double sol_xy[2][NUM_QUAD_1D][NUM_QUAD_1D];
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          sol_xy[0][qx][qy] = 0;
+          sol_xy[1][qx][qy] = 0;
+        }
+      }
+
+      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+        double sol_x[2][NUM_QUAD_1D];
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          sol_x[0][qy] = 0;
+          sol_x[1][qy] = 0;
+        }
+
+        // sol_x{vd, dx, qy} = dofToQuad{qy, dy} * sol{vd, dx, dy}
+        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            sol_x[0][qy] += dofToQuad(qy, dx) * solIn(0, dx, dy, e);
+            sol_x[1][qy] += dofToQuad(qy, dx) * solIn(1, dx, dy, e);
+          }
+        }
+
+        // sol_xy{qx, qy} = dofToQuad{qx, dx} * sol_x{dx, qy}
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          const double d2q = dofToQuad(qy, dy);
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_xy[0][qx][qy] += d2q * sol_x[0][qx];
+            sol_xy[1][qx][qy] += d2q * sol_x[1][qx];
+          }
+        }
+      } // dy
+
+      // sol_xy{qx, qy} = sol_xy{q} *= oper{q, e}
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          const int q = QUAD_2D_ID(qx, qy);
+          sol_xy[0][qx][qy] *= oper(q, e);
+          sol_xy[1][qx][qy] *= oper(q, e);
+        }
+      }
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+        double sol_x[2][NUM_DOFS_1D];
+        for (int dx = 0; dx < NUM_QUAD_1D; ++dx) {
+          sol_x[0][dx] = 0;
+          sol_x[1][dx] = 0;
+        }
+
+        // sol_x{qx, dy} = quadToDof{dy, qy} * sol_xy{qx, qy}
+        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            sol_x[0][dx] += quadToDof(dx, qx) * sol_xy[0][qx][qy];
+            sol_x[1][dx] += quadToDof(dx, qx) * sol_xy[1][qx][qy];
+          }
+        }
+
+        // sol{dx, dy, e} = quadToDof{dx, qx} * sol_x{qx, dy}
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          const double q2d = quadToDof(dy, qy);
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            solOut(0, dx, dy, e) += q2d * sol_x[0][dx];
+            solOut(1, dx, dy, e) += q2d * sol_x[1][dx];
+          }
+        }
+      }
+    } // dummy
+  } // e
+}
+//======================================
+
+
+//---[ 3D ]-----------------------------
+@kernel void Assemble3D(const int numElements,
+                        @restrict const double * quadWeights,
+                        @restrict const Jacobian3D_t J,
+                        COEFF_ARGS
+                        @restrict QLocal_t oper) {
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int q = 0; q < NUM_QUAD_3D; ++q; @inner) {
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
+
+      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
+                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
+
+      oper(q, e) = quadWeights[q] * COEFF * detJ;
+    }
+  }
+}
+
+@kernel void MultAdd3D(const int numElements,
+                       @restrict const DofToQuad_t dofToQuad,
+                       @restrict const DofToQuad_t dofToQuadD,
+                       @restrict const QuadToDof_t quadToDof,
+                       @restrict const QuadToDof_t quadToDofD,
+                       @restrict const QLocal_t oper,
+                       @restrict const DVLocal3D_t solIn,
+                       @restrict DVLocal3D_t solOut) {
+  // Iterate over elements
+  for (int e = 0; e < numElements; ++e; @outer) {
+    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
+      double sol_xyz[3][NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D];
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_xyz[0][qz][qy][qx] = 0;
+            sol_xyz[1][qz][qy][qx] = 0;
+            sol_xyz[2][qz][qy][qx] = 0;
+          }
+        }
+      }
+
+      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+        double sol_xy[3][NUM_QUAD_1D][NUM_QUAD_1D];
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_xy[0][qy][qx] = 0;
+            sol_xy[1][qy][qx] = 0;
+            sol_xy[2][qy][qx] = 0;
+          }
+        }
+
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          double sol_x[3][NUM_QUAD_1D];
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            sol_x[0][qx] = 0;
+            sol_x[1][qx] = 0;
+            sol_x[2][qx] = 0;
+          }
+
+          // sol_x{qx} = dofToQuad{qx, dx} * sol{dx, dy, dz, e}
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              sol_x[0][qx] += dofToQuad(qx, dx) * solIn(0, dx, dy, dz, e);
+              sol_x[1][qx] += dofToQuad(qx, dx) * solIn(1, dx, dy, dz, e);
+              sol_x[2][qx] += dofToQuad(qx, dx) * solIn(2, dx, dy, dz, e);
+            }
+          }
+
+          // sol_xy{qx, qy} = dofToQuad{qy, dy} * sol_x{dx}
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            const double wy = dofToQuad(qy, dy);
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              sol_xy[0][qy][qx] += wy * sol_x[0][qx];
+              sol_xy[1][qy][qx] += wy * sol_x[1][qx];
+              sol_xy[2][qy][qx] += wy * sol_x[2][qx];
+            }
+          }
+        } // dy
+
+        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+          const double wz = dofToQuad(qz, dz);
+          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+              sol_xyz[0][qz][qy][qx] += wz * sol_xy[0][qy][qx];
+              sol_xyz[1][qz][qy][qx] += wz * sol_xy[1][qy][qx];
+              sol_xyz[2][qz][qy][qx] += wz * sol_xy[2][qy][qx];
+            }
+          }
+        }
+      } // dz
+
+      // sol_xyz{qz, qy, qx} *= oper{q, e}
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            const int q = QUAD_3D_ID(qx, qy, qz);
+            sol_xyz[0][qz][qy][qx] *= oper(q, e);
+            sol_xyz[1][qz][qy][qx] *= oper(q, e);
+            sol_xyz[2][qz][qy][qx] *= oper(q, e);
+          }
+        }
+      }
+
+      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
+        double sol_xy[3][NUM_DOFS_1D][NUM_DOFS_1D];
+        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            sol_xy[0][dy][dx] = 0;
+            sol_xy[1][dy][dx] = 0;
+            sol_xy[2][dy][dx] = 0;
+          }
+        }
+
+        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
+          double sol_x[3][NUM_DOFS_1D];
+          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+            sol_x[0][dx] = 0;
+            sol_x[1][dx] = 0;
+            sol_x[2][dx] = 0;
+          }
+
+          // sol_x{dx} = quadToDof{dx, qx} * sol_xyz{qz, qy, qx}
+          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              sol_x[0][dx] += quadToDof(dx, qx) * sol_xyz[0][qz][qy][qx];
+              sol_x[1][dx] += quadToDof(dx, qx) * sol_xyz[1][qz][qy][qx];
+              sol_x[2][dx] += quadToDof(dx, qx) * sol_xyz[2][qz][qy][qx];
+            }
+          }
+
+          // sol_xy{dy, dx} = quadToDof{dy, qy} * sol_x{dx}
+          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+            const double wy = quadToDof(dy, qy);
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              sol_xy[0][dy][dx] += wy * sol_x[0][dx];
+              sol_xy[1][dy][dx] += wy * sol_x[1][dx];
+              sol_xy[2][dy][dx] += wy * sol_x[2][dx];
+            }
+          }
+        } // qy
+
+        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
+          const double wz = quadToDof(dz, qz);
+          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
+              solOut(0, dx, dy, dz, e) += wz * sol_xy[0][dy][dx];
+              solOut(1, dx, dy, dz, e) += wz * sol_xy[1][dy][dx];
+              solOut(2, dx, dy, dz, e) += wz * sol_xy[2][dy][dx];
+            }
+          }
+        }
+      } // qz
+    } // dummy
+  } // e
+
+}
+//======================================

config/config.hpp

@@ -40,3 +40,11 @@
 #error Building with PUMI (MFEM_USE_PUMI=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
 #endif // MFEM_USE_MPI not defined
+
+// Macro that returns its first arg when MFEM_USE_BACKENDS is defined, and its
+// second arg if it is not defined.
+#ifdef MFEM_USE_BACKENDS
+#define MFEM_IF_BACKENDS(x,y) x
+#else
+#define MFEM_IF_BACKENDS(x,y) y
+#endif

config/config.hpp.in

@@ -33,6 +33,12 @@
 // Description of the git commit used to build MFEM.
 // #define MFEM_GIT_STRING "@MFEM_GIT_STRING@"
 
+// The absolute path of the MFEM source prefix
+// #define MFEM_SOURCE_DIR "@MFEM_SOURCE_DIR@"
+
+// The absolute path of the MFEM installation prefix
+// #define MFEM_INSTALL_DIR "@MFEM_INSTALL_DIR@"
+
 // Build the parallel MFEM library.
 // Requires an MPI compiler, and the libraries HYPRE and METIS.
 // #define MFEM_USE_MPI
@@ -112,6 +118,12 @@
 // Enable MFEM functionality based on the PUMI library
 // #define MFEM_USE_PUMI
 
+// Enable the use of MFEM backends.
+// #define MFEM_USE_BACKENDS
+
+// Enable the OCCA backend.
+// #define MFEM_USE_OCCA
+
 // Windows specific options
 #ifdef _WIN32
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)

config/config.mk.in

@@ -13,6 +13,8 @@
 MFEM_VERSION         = @MFEM_VERSION@
 MFEM_VERSION_STRING  = @MFEM_VERSION_STRING@
 MFEM_GIT_STRING      = @MFEM_GIT_STRING@
+MFEM_SOURCE_DIR      = @MFEM_SOURCE_DIR@
+MFEM_INSTALL_DIR     = @MFEM_INSTALL_DIR@
 MFEM_USE_MPI         = @MFEM_USE_MPI@
 MFEM_USE_METIS       = @MFEM_USE_METIS@
 MFEM_USE_METIS_5     = @MFEM_USE_METIS_5@
@@ -38,6 +40,8 @@ MFEM_USE_MPFR        = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE       = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT     = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI        = @MFEM_USE_PUMI@
+MFEM_USE_BACKENDS    = @MFEM_USE_BACKENDS@
+MFEM_USE_OCCA        = @MFEM_USE_OCCA@
 
 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@

config/defaults.mk

@@ -83,6 +83,9 @@ MFEM_MPI_NP = 4
 # in config.mk and config.hpp.
 
 MFEM_USE_MPI         = NO
+# FIXME: add MFEM_USE_BACKENDS, MFEM_USE_OCCA to the CMake build system
+MFEM_USE_BACKENDS    = YES
+MFEM_USE_OCCA        = YES
 MFEM_USE_METIS       = $(MFEM_USE_MPI)
 MFEM_USE_METIS_5     = NO
 MFEM_DEBUG           = NO
@@ -279,6 +282,10 @@ PUMI_OPT = -I$(PUMI_DIR)/include
 PUMI_LIB = -L$(PUMI_DIR)/lib -lpumi -lcrv -lma -lmds -lapf -lpcu -lgmi -lparma\
    -llion -lmth -lapf_zoltan -lspr
 
+OCCA_DIR = @MFEM_DIR@/../occa
+OCCA_OPT = -I$(OCCA_DIR)/include
+OCCA_LIB = -Wl,-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca
+
 # If YES, enable some informational messages
 VERBOSE = NO
 

doc/CodeDocumentation.conf.in

@@ -760,6 +760,8 @@ WARN_LOGFILE           =
 
 INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/mfem.hpp \
+                         @MFEM_SOURCE_DIR@/backends/base \
+                         @MFEM_SOURCE_DIR@/backends/occa \
                          @MFEM_SOURCE_DIR@/config \
                          @MFEM_SOURCE_DIR@/general \
                          @MFEM_SOURCE_DIR@/linalg \

examples/ex16d.cpp

@@ -0,0 +1,441 @@
+//                                MFEM Example 16
+//
+// Compile with: make ex16
+//
+// Sample runs:  ex16
+//               ex16 -m ../data/inline-tri.mesh
+//               ex16 -m ../data/disc-nurbs.mesh -tf 2
+//               ex16 -s 1 -a 0.0 -k 1.0
+//               ex16 -s 2 -a 1.0 -k 0.0
+//               ex16 -s 3 -a 0.5 -k 0.5 -o 4
+//               ex16 -s 14 -dt 1.0e-4 -tf 4.0e-2 -vs 40
+//               ex16 -m ../data/fichera-q2.mesh
+//               ex16 -m ../data/escher.mesh
+//               ex16 -m ../data/beam-tet.mesh -tf 10 -dt 0.1
+//               ex16 -m ../data/amr-quad.mesh -o 4 -r 0
+//               ex16 -m ../data/amr-hex.mesh -o 2 -r 0
+//
+// Description:  This example solves a time dependent nonlinear heat equation
+//               problem of the form du/dt = C(u), with a non-linear diffusion
+//               operator C(u) = \nabla \cdot (\kappa + \alpha u) \nabla u.
+//
+//               The example demonstrates the use of nonlinear operators (the
+//               class ConductionOperator defining C(u)), as well as their
+//               implicit time integration. Note that implementing the method
+//               ConductionOperator::ImplicitSolve is the only requirement for
+//               high-order implicit (SDIRK) time integration.
+//
+//               We recommend viewing examples 2, 9 and 10 before viewing this
+//               example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+/** After spatial discretization, the conduction model can be written as:
+ *
+ *     du/dt = M^{-1}(-Ku)
+ *
+ *  where u is the vector representing the temperature, M is the mass matrix,
+ *  and K is the diffusion operator with diffusivity depending on u:
+ *  (\kappa + \alpha u).
+ *
+ *  Class ConductionOperator represents the right-hand side of the above ODE.
+ */
+class ConductionOperator : public TimeDependentOperator
+{
+protected:
+   FiniteElementSpace &fespace;
+   Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
+
+   BilinearForm M;
+   BilinearForm K;
+
+   GridFunction u_alpha_gf;
+   GridFunctionCoefficient u_coeff;
+
+   OperatorHandle Moper, Koper;
+   Operator *T; // T = M + dt K
+   double current_dt;
+
+   CGSolver M_solver; // Krylov solver for inverting the mass matrix M
+   // FIXME: add the preconditioner
+   // DSmoother M_prec;  // Preconditioner for the mass matrix M
+
+   CGSolver T_solver; // Implicit solver for T = M + dt K
+   // FIXME: add the preconditioner
+   // DSmoother T_prec;  // Preconditioner for the implicit solver
+
+   double alpha, kappa;
+
+   mutable Vector z; // auxiliary vector
+
+public:
+   ConductionOperator(FiniteElementSpace &f, const char *oper_spec,
+                      double alpha, double kappa, const Vector &u);
+
+   virtual void Mult(const Vector &u, Vector &du_dt) const;
+   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
+       This is the only requirement for high-order SDIRK implicit integration.*/
+   virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);
+
+   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
+   void SetParameters(const Vector &u);
+
+   virtual ~ConductionOperator();
+};
+
+
+class TimeDerivativeOperator : public Operator
+{
+   Operator *Moper;
+   Operator *Koper;
+   mutable Vector Kdu;
+   const double dt;
+
+public:
+   // FIXME: Sparse matrices should be changed to have PLayouts
+   // allocated so that this constructor works even when Moper and
+   // Koper were not created from engines.
+   TimeDerivativeOperator(Operator *_Moper, const double _dt, Operator *_Koper)
+      : Operator(*_Koper->InLayout(), *_Moper->OutLayout()),
+        Moper(_Moper),
+        Koper(_Koper),
+        Kdu(_Moper->OutLayout()),
+        dt(_dt) { }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      Moper->Mult(x, y);
+      Koper->Mult(x, Kdu);
+
+      y.Axpby(1.0, y, dt, Kdu);
+   }
+};
+
+double InitialTemperature(const Vector &x);
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int ref_levels = 2;
+   int order = 2;
+   int ode_solver_type = 3;
+   double t_final = 0.5;
+   double dt = 1.0e-2;
+   double alpha = 1.0e-2;
+   double kappa = 0.5;
+   bool visualization = true;
+   bool visit = false;
+   int vis_steps = 5;
+   const char *oper_spec = "representation: 'partial'";
+   const char *occa_spec = "mode: 'Serial'";
+
+   int precision = 8;
+   cout.precision(precision);
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ref_levels, "-r", "--refine",
+                  "Number of times to refine the mesh uniformly.");
+   args.AddOption(&order, "-o", "--order",
+                  "Order (degree) of the finite elements.");
+   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
+                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
+                  "\t   11 - Forward Euler, 12 - RK2, 13 - RK3 SSP, 14 - RK4.");
+   args.AddOption(&t_final, "-tf", "--t-final",
+                  "Final time; start time is 0.");
+   args.AddOption(&dt, "-dt", "--time-step",
+                  "Time step.");
+   args.AddOption(&alpha, "-a", "--alpha",
+                  "Alpha coefficient.");
+   args.AddOption(&kappa, "-k", "--kappa",
+                  "Kappa coefficient offset.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
+                  "--no-visit-datafiles",
+                  "Save data files for VisIt (visit.llnl.gov) visualization.");
+   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
+                  "Visualize every n-th timestep.");
+   args.AddOption(&oper_spec, "-s", "--oper-spec", "Operator specification");
+   args.AddOption(&occa_spec, "-os", "--occa-spec", "OCCA engine specification");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // Examples for OCCA specifications:
+   //   - CPU (serial): "mode: 'Serial'"
+   //   - CUDA GPU: "mode: 'CUDA', device_id: 0"
+   //   - OpenMP on CPUs: "mode: 'OpenMP', threads: 4"
+   //   - OpenCL on device 0: "mode: 'OpenCL', device_id: 0, platform_id: 0"
+
+   SharedPtr<Engine> engine(new mfem::occa::Engine(occa_spec));
+
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   mesh->SetEngine(*engine);
+   int dim = mesh->Dimension();
+
+   // 3. Define the ODE solver used for time integration. Several implicit
+   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
+   //    explicit Runge-Kutta methods are available.
+   ODESolver *ode_solver;
+   switch (ode_solver_type)
+   {
+      // Implicit L-stable methods
+      case 1:  ode_solver = new BackwardEulerSolver; break;
+      case 2:  ode_solver = new SDIRK23Solver(2); break;
+      case 3:  ode_solver = new SDIRK33Solver; break;
+      // Explicit methods
+      case 11: ode_solver = new ForwardEulerSolver; break;
+      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 13: ode_solver = new RK3SSPSolver; break;
+      case 14: ode_solver = new RK4Solver; break;
+      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }
+
+   // 4. Refine the mesh to increase the resolution. In this example we do
+   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
+   //    command-line parameter.
+   for (int lev = 0; lev < ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 5. Define the vector finite element space representing the current and the
+   //    initial temperature, u_ref.
+   H1_FECollection fe_coll(order, dim);
+   FiniteElementSpace fespace(mesh, &fe_coll);
+
+   int fe_size = fespace.GetTrueVSize();
+   cout << "Number of temperature unknowns: " << fe_size << endl;
+
+   GridFunction u_gf(&fespace);
+
+   // 6. Set the initial conditions for u. All boundaries are considered
+   //    natural. This computes this on the host, so pull/push is needed.
+   u_gf.Pull();
+   FunctionCoefficient u_0(InitialTemperature);
+   u_gf.ProjectCoefficient(u_0);
+   u_gf.Push();
+   Vector u;
+   u_gf.GetTrueDofs(u);
+
+   // 7. Initialize the conduction operator and the visualization.
+   ConductionOperator oper(fespace, oper_spec, alpha, kappa, u);
+
+   u_gf.SetFromTrueDofs(u);
+   {
+      ofstream omesh("ex16.mesh");
+      omesh.precision(precision);
+      mesh->Print(omesh);
+      ofstream osol("ex16-init.gf");
+      osol.precision(precision);
+      u_gf.Pull(); // pull back to host before saving
+      u_gf.Save(osol);
+   }
+
+   VisItDataCollection visit_dc("Example16", mesh);
+   visit_dc.RegisterField("temperature", &u_gf);
+   if (visit)
+   {
+      visit_dc.SetCycle(0);
+      visit_dc.SetTime(0.0);
+      visit_dc.Save();
+   }
+
+   socketstream sout;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      sout.open(vishost, visport);
+      if (!sout)
+      {
+         cout << "Unable to connect to GLVis server at "
+              << vishost << ':' << visport << endl;
+         visualization = false;
+         cout << "GLVis visualization disabled.\n";
+      }
+      else
+      {
+         sout.precision(precision);
+         sout << "solution\n" << *mesh << u_gf;
+         sout << "pause\n";
+         sout << flush;
+         cout << "GLVis visualization paused."
+              << " Press space (in the GLVis window) to resume it.\n";
+      }
+   }
+
+   // 8. Perform time-integration (looping over the time iterations, ti, with a
+   //    time-step dt).
+   ode_solver->Init(oper);
+   double t = 0.0;
+
+   bool last_step = false;
+   for (int ti = 1; !last_step; ti++)
+   {
+      if (t + dt >= t_final - dt/2)
+      {
+         last_step = true;
+      }
+
+      ode_solver->Step(u, t, dt);
+
+      if (last_step || (ti % vis_steps) == 0)
+      {
+         cout << "step " << ti << ", t = " << t << endl;
+
+         // u_gf and u are both on the device at this point.
+         u_gf.SetFromTrueDofs(u);
+         if (visualization)
+         {
+            u_gf.Pull(); // pull back to host before saving
+            sout << "solution\n" << *mesh << u_gf << flush;
+         }
+
+         if (visit)
+         {
+            visit_dc.SetCycle(ti);
+            visit_dc.SetTime(t);
+            visit_dc.Save();
+         }
+      }
+      oper.SetParameters(u);
+   }
+
+   // 9. Save the final solution. This output can be viewed later using GLVis:
+   //    "glvis -m ex16.mesh -g ex16-final.gf".
+   {
+      ofstream osol("ex16-final.gf");
+      osol.precision(precision);
+      u_gf.Pull(); // pull back to host before saving
+      u_gf.Save(osol);
+   }
+
+   // 10. Free the used memory.
+   delete ode_solver;
+   delete mesh;
+
+   return 0;
+}
+
+ConductionOperator::ConductionOperator(FiniteElementSpace &f,
+                                       const char *oper_spec, double al,
+                                       double kap, const Vector &u)
+   : TimeDependentOperator(*f.GetTrueVLayout()), fespace(f), M(&fespace),
+     K(&fespace), u_alpha_gf(&f), u_coeff(&u_alpha_gf), Moper(oper_spec),
+     Koper(oper_spec), T(NULL), current_dt(0.0), z(f.GetTrueVLayout())
+{
+   const double rel_tol = 1e-8;
+
+   M.AddDomainIntegrator(new MassIntegrator());
+   M.Assemble();
+   M.FormSystemMatrix(ess_tdof_list, Moper);
+
+   K.AddDomainIntegrator(new DiffusionIntegrator(u_coeff));
+
+   M_solver.iterative_mode = false;
+   M_solver.SetRelTol(rel_tol);
+   M_solver.SetAbsTol(0.0);
+   M_solver.SetMaxIter(200);
+   M_solver.SetPrintLevel(0);
+   // M_solver.SetPreconditioner(M_prec);
+   M_solver.SetOperator(*Moper.Ptr());
+
+   alpha = al;
+   kappa = kap;
+
+   T_solver.iterative_mode = false;
+   T_solver.SetRelTol(rel_tol);
+   T_solver.SetAbsTol(0.0);
+   T_solver.SetMaxIter(200);
+   T_solver.SetPrintLevel(0);
+   // T_solver.SetPreconditioner(T_prec);
+
+   SetParameters(u);
+}
+
+void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
+{
+   // Compute:
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
+   Koper.Ptr()->Mult(u, z);
+   z.Axpby(-1.0, z, 0.0, z);
+   M_solver.Mult(z, du_dt);
+}
+
+void ConductionOperator::ImplicitSolve(const double dt,
+                                       const Vector &u, Vector &du_dt)
+{
+   // Solve the equation:
+   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
+   // for du_dt
+   if (!T)
+   {
+      T = new TimeDerivativeOperator(Moper.Ptr(), dt, Koper.Ptr());
+      current_dt = dt;
+      T_solver.SetOperator(*T);
+   }
+   MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt
+   Koper.Ptr()->Mult(u, z);
+   z.Axpby(-1.0, z, 0.0, z);
+   T_solver.Mult(z, du_dt);
+}
+
+void ConductionOperator::SetParameters(const Vector &u)
+{
+   u_alpha_gf.SetFromTrueDofs(u);
+   u_alpha_gf.Pull();
+   for (int i = 0; i < u_alpha_gf.Size(); i++)
+   {
+      u_alpha_gf(i) = kappa + alpha*u_alpha_gf(i);
+   }
+   u_alpha_gf.Push();
+
+   // Reassemble after changing u_alpha_gf (and hence u_coeff)...
+   K.Assemble();
+   K.FormSystemMatrix(ess_tdof_list, Koper);
+
+   delete T;
+   T = NULL; // re-compute T on the next ImplicitSolve
+}
+
+ConductionOperator::~ConductionOperator()
+{
+   delete T;
+}
+
+double InitialTemperature(const Vector &x)
+{
+   if (x.Norml2() < 0.5)
+   {
+      return 2.0;
+   }
+   else
+   {
+      return 1.0;
+   }
+}

examples/ex1d.cpp

@@ -0,0 +1,185 @@
+
+#include <mfem.hpp>
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *spec = "cpu";
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   int ref_levels = -1;
+   bool static_cond = false;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&spec, "-s", "--spec",
+                  "Compute resource specification.");
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&ref_levels, "-r", "--refine-levels",
+                  "Number of uniform refinements to apply to the mesh.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+#ifdef MFEM_USE_BACKENDS
+   /// Engine *engine = EngineDepot.Select(spec);
+
+   // string occa_spec("mode: 'Serial'");
+   string occa_spec("mode: 'CUDA', device_id: 0");
+   // string occa_spec("mode: 'OpenMP', threads: 4");
+   // string occa_spec("mode: 'OpenCL', device_id: 0, platform_id: 0");
+
+   // The following flag affects only 'Serial' and 'OpenMP' modes.
+   // In 'CUDA' mode, '-O3' affects only host code.
+   // In 'OpenCL' mode, adding '-O3' breaks compilation.
+   // occa_spec += ", kernel: { compiler_flags: '-O3' }";
+
+   SharedPtr<Engine> engine(new mfem::occa::Engine(occa_spec));
+#endif
+
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
+   //    the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+#ifdef MFEM_USE_BACKENDS
+   mesh->SetEngine(*engine);
+#endif
+   int dim = mesh->Dimension();
+
+   // 3. Refine the mesh to increase the resolution. In this example we do
+   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
+   //    largest number that gives a final mesh with no more than 50,000
+   //    elements.
+   {
+      ref_levels = ref_levels >= 0 ? ref_levels :
+         (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 4. Define a finite element space on the mesh. Here we use continuous
+   //    Lagrange finite elements of the specified order. If order < 1, we
+   //    instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (mesh->GetNodes())
+   {
+      fec = mesh->GetNodes()->OwnFEC();
+      cout << "Using isoparametric FEs: " << fec->Name() << endl;
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   cout << "Number of finite element unknowns: "
+        << fespace->GetTrueVSize() << endl;
+
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
+   //    In this example, the boundary conditions are defined by marking all
+   //    the boundary attributes from the mesh as essential (Dirichlet) and
+   //    converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
+   //    the basis functions in the finite element fespace.
+   LinearForm *b = new LinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   // 7. Define the solution vector x as a finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   GridFunction x(fespace);
+   x.Fill(0.0);
+
+   // 8. Set up the bilinear form a(.,.) on the finite element space
+   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //    domain integrator.
+   BilinearForm *a = new BilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 9. Assemble the bilinear form and the corresponding linear system,
+   //    applying any necessary transformations such as: eliminating boundary
+   //    conditions, applying conforming constraints for non-conforming AMR,
+   //    static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   OperatorHandle A(Operator::ANY_TYPE);
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   cout << "Size of linear system: " << A.Ptr()->Height() << endl;
+
+   // 10. Solve the system A X = B with CG.
+   tic_toc.Clear();
+   tic_toc.Start();
+   const int print_level = 3;
+   CG(*A.Ptr(), B, X, print_level, 1000, 1e-12, 0.0);
+   tic_toc.Stop();
+   cout << "CG time: " << tic_toc.RealTime() << " sec." << endl;
+
+   // 11. Recover the solution as a finite element grid function.
+   a->RecoverFEMSolution(X, *b, x);
+   x.Pull();
+
+   // 12. Save the refined mesh and the solution. This output can be viewed
+   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
+   ofstream mesh_ofs("refined.mesh");
+   mesh_ofs.precision(8);
+   mesh->Print(mesh_ofs);
+   ofstream sol_ofs("sol.gf");
+   sol_ofs.precision(8);
+   x.Save(sol_ofs);
+
+   // 13. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *mesh << x << flush;
+   }
+
+   // 14. Free the used memory.
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete mesh;
+
+   return 0;
+}

examples/ex1pd.cpp

@@ -0,0 +1,273 @@
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int ser_ref_levels = -1;
+   int par_ref_levels = -1;
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+#ifdef MFEM_USE_BACKENDS
+   /// Engine *engine = EngineDepot.Select(spec);
+
+   // string occa_spec("mode: 'Serial'");
+   string occa_spec;
+   {
+      stringstream occa_spec_ss;
+      occa_spec_ss << "mode: 'CUDA', device_id: 0";
+      // const int nGPUs = 4;
+      // occa_spec_ss << "mode: 'CUDA', device_id: " << (myid % nGPUs);
+      occa_spec = occa_spec_ss.str();
+   }
+   // string occa_spec("mode: 'OpenMP', threads: 4");
+   // string occa_spec("mode: 'OpenCL', device_id: 0, platform_id: 0");
+
+   SharedPtr<Engine> engine(new mfem::occa::Engine(MPI_COMM_WORLD, occa_spec));
+#endif
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+#ifdef MFEM_USE_BACKENDS
+   mesh->SetEngine(*engine);
+#endif
+   int dim = mesh->Dimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 10,000 elements.
+   {
+      int ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+      ref_levels = ser_ref_levels >= 0 ? ser_ref_levels : ref_levels;
+      if (myid == 0)
+      {
+         cout << "Serial refinement levels: " << ref_levels << endl;
+      }
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   {
+      par_ref_levels = par_ref_levels >= 0 ? par_ref_levels : 2;
+      if (myid == 0)
+      {
+         cout << "Parallel refinement levels: " << par_ref_levels << endl;
+      }
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh->UniformRefinement();
+      }
+   }
+   pmesh->PrintInfo(cout);
+   if (myid == 0) { cout << endl; }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+      if (myid == 0)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (1,phi_i) where phi_i are the basis functions in fespace.
+   ParLinearForm *b = new ParLinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   ParGridFunction x(fespace);
+   x.Fill(0.0);
+
+   // 10. Set up the parallel bilinear form a(.,.) on the finite element space
+   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //     domain integrator.
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   OperatorHandle A(Operator::ANY_TYPE);
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   CGSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+   pcg->SetRelTol(1e-6);
+   pcg->SetAbsTol(0.0);
+   pcg->SetMaxIter(1000);
+   pcg->SetPrintLevel(3);
+   pcg->SetOperator(*A.Ptr());
+
+   // Run one CG iteration to make sure all kernels are loaded before measuring
+   // time.
+   if (myid == 0)
+   {
+      cout << "Running 1 CG iteration to load all kernels ..." << flush;
+   }
+   {
+      Vector X2(X);
+      pcg->SetMaxIter(1);
+      pcg->SetPrintLevel(-1);
+      pcg->Mult(B, X2);
+      pcg->SetMaxIter(1000);
+      pcg->SetPrintLevel(3);
+   }
+   if (myid == 0)
+   {
+      cout << " done." << endl;
+   }
+
+   double start_time = MPI_Wtime();
+   pcg->Mult(B, X);
+   double end_time = MPI_Wtime();
+   double loc_time = end_time - start_time;
+   double max_time, min_time;
+   MPI_Allreduce(&loc_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
+   MPI_Allreduce(&loc_time, &min_time, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);
+   if (myid == 0)
+   {
+      cout << "CG time: " << max_time << " sec (min: " << min_time << " sec)\n"
+           << "DOFs/sec in CG: "
+           << 1e-6*size*pcg->GetNumIterations()/max_time << " ("
+           << 1e-6*size*pcg->GetNumIterations()/min_time << ") million.\n"
+           << endl;
+   }
+
+   // 13. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a->RecoverFEMSolution(X, *b, x);
+   x.Pull();
+
+   // 14. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 15. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x << flush;
+   }
+
+   // 16. Free the used memory.
+   delete pcg;
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}

fem/bilinearform.cpp

@@ -74,6 +74,12 @@ BilinearForm::BilinearForm (FiniteElementSpace * f)
    hybridization = NULL;
    precompute_sparsity = 0;
    diag_policy = DIAG_KEEP;
+#ifdef MFEM_USE_BACKENDS
+   if (fes->GetVLayout()->HasEngine())
+   {
+      dev_ext = fes->GetVLayout()->GetEngine().MakeBilinearForm(*this);
+   }
+#endif
 }
 
 BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
@@ -126,6 +132,14 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
 void BilinearForm::EnableStaticCondensation()
 {
    delete static_cond;
+#ifdef MFEM_USE_BACKENDS
+   if (fes->GetVLayout()->HasEngine())
+   {
+      static_cond = NULL;
+      MFEM_WARNING("Engine interface does not support static condensation yet");
+      return;
+   }
+#endif
    static_cond = new StaticCondensation(fes);
    if (static_cond->ReducesTrueVSize())
    {
@@ -145,6 +159,15 @@ void BilinearForm::EnableHybridization(FiniteElementSpace *constr_space,
                                        const Array<int> &ess_tdof_list)
 {
    delete hybridization;
+#ifdef MFEM_USE_BACKENDS
+   if (fes->GetVLayout()->HasEngine())
+   {
+      delete constr_integ;
+      hybridization = NULL;
+      MFEM_WARNING("Engine interface does not support hybridization yet");
+      return;
+   }
+#endif
    hybridization = new Hybridization(fes, constr_space);
    hybridization->SetConstraintIntegrator(constr_integ);
    hybridization->Init(ess_tdof_list);
@@ -313,7 +336,15 @@ void BilinearForm::Assemble (int skip_zeros)
    Mesh *mesh = fes -> GetMesh();
    DenseMatrix elmat, *elmat_p;
 
-   int i;
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      // TODO: push the 'skip_zeros' as a parameter to 'dev_ext'
+
+      const bool assembly_done = dev_ext->Assemble();
+      if (assembly_done) { return; }
+   }
+#endif
 
    if (mat == NULL)
    {
@@ -331,7 +362,7 @@ void BilinearForm::Assemble (int skip_zeros)
 
    if (dbfi.Size())
    {
-      for (i = 0; i < fes -> GetNE(); i++)
+      for (int i = 0; i < fes -> GetNE(); i++)
       {
          fes->GetElementVDofs(i, vdofs);
          if (element_matrices)
@@ -388,7 +419,7 @@ void BilinearForm::Assemble (int skip_zeros)
          }
       }
 
-      for (i = 0; i < fes -> GetNBE(); i++)
+      for (int i = 0; i < fes -> GetNBE(); i++)
       {
          const int bdr_attr = mesh->GetBdrAttribute(i);
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
@@ -426,7 +457,7 @@ void BilinearForm::Assemble (int skip_zeros)
       Array<int> vdofs2;
 
       int nfaces = mesh->GetNumFaces();
-      for (i = 0; i < nfaces; i++)
+      for (int i = 0; i < nfaces; i++)
       {
          tr = mesh -> GetInteriorFaceTransformations (i);
          if (tr != NULL)
@@ -471,7 +502,7 @@ void BilinearForm::Assemble (int skip_zeros)
          }
       }
 
-      for (i = 0; i < fes -> GetNBE(); i++)
+      for (int i = 0; i < fes -> GetNBE(); i++)
       {
          const int bdr_attr = mesh->GetBdrAttribute(i);
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
@@ -643,9 +674,91 @@ void BilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
    }
 }
 
+void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
+                                    Vector &x, Vector &b,
+                                    OperatorHandle &A, Vector &X, Vector &B,
+                                    int copy_interior)
+{
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      MFEM_VERIFY(!static_cond && !hybridization, "");
+      dev_ext->FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
+   }
+   else
+#endif
+   {
+      if (A.Type() == Operator::MFEM_SPARSEMAT || A.Type() == Operator::ANY_TYPE)
+      {
+         SparseMatrix A_sm;
+         FormLinearSystem(ess_tdof_list, x, b, A_sm, X, B, copy_interior);
+         if (static_cond)
+         {
+            A.Reset(&static_cond->GetMatrix(), false);
+         }
+         else if (hybridization)
+         {
+            A.Reset(&hybridization->GetMatrix(), false);
+         }
+         else
+         {
+            A.Reset(mat, false);
+         }
+      }
+      else
+      {
+         MFEM_ABORT("Operator::Type is not supported: type_id = " << A.Type());
+      }
+   }
+}
+
+void BilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                    OperatorHandle &A)
+{
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      MFEM_VERIFY(!static_cond && !hybridization, "");
+      dev_ext->FormSystemMatrix(ess_tdof_list, A);
+   }
+   else
+#endif
+   {
+      if (A.Type() == Operator::MFEM_SPARSEMAT || A.Type() == Operator::ANY_TYPE)
+      {
+         SparseMatrix A_sm;
+         FormSystemMatrix(ess_tdof_list, A_sm);
+         if (static_cond)
+         {
+            A.Reset(&static_cond->GetMatrix(), false);
+         }
+         else if (hybridization)
+         {
+            A.Reset(&hybridization->GetMatrix(), false);
+         }
+         else
+         {
+            A.Reset(mat, false);
+         }
+      }
+      else
+      {
+         MFEM_ABORT("Operator::Type is not supported: type_id = " << A.Type());
+      }
+   }
+}
+
 void BilinearForm::RecoverFEMSolution(const Vector &X,
                                       const Vector &b, Vector &x)
 {
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      dev_ext->RecoverFEMSolution(X, b, x);
+      return;
+   }
+#endif
+
    const SparseMatrix *P = fes->GetConformingProlongation();
    if (!P) // conforming space
    {
@@ -734,7 +847,8 @@ void BilinearForm::ComputeElementMatrices()
 }
 
 void BilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                                        const Vector &sol, Vector &rhs, DiagonalPolicy dpolicy)
+                                        const Vector &sol, Vector &rhs,
+                                        DiagonalPolicy dpolicy)
 {
    Array<int> ess_dofs, conf_ess_dofs;
    fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);

fem/bilinearform.hpp

@@ -38,6 +38,11 @@ protected:
    /// FE space on which the form lives.
    FiniteElementSpace *fes;
 
+#ifdef MFEM_USE_BACKENDS
+   /// Device/Engine extension (smart shared pointer)
+   DBilinearForm dev_ext;
+#endif
+
    /// Indicates the Mesh::sequence corresponding to the current state of the
    /// BilinearForm.
    long sequence;
@@ -285,6 +290,37 @@ public:
    /// Form the linear system matrix A, see FormLinearSystem() for details.
    void FormSystemMatrix(const Array<int> &ess_tdof_list, SparseMatrix &A);
 
+   /** Form the linear system @a A @a X = @a B, corresponding to the bilinear
+       form and the r.h.s. linear form (vector) @a b, by applying any necessary
+       transformations such as: eliminating boundary conditions; applying
+       conforming constraints for non-conforming AMR; parallel assembly; static
+       condensation; hybridization.
+
+       The GridFunction-size vector @a x must contain the essential b.c. The
+       BilinearForm and the LinearForm-size vector @a b must be assembled.
+
+       The vector @a X is initialized with a suitable initial guess: when using
+       hybridization, the vector @a X is set to zero; otherwise, the essential
+       entries of @a X are set to the corresponding b.c. and all other entries
+       are set to zero (if @a copy_interior == 0) or copied from @a x (if
+       @a copy_interior != 0).
+
+       This method can be called multiple times (with the same @a ess_tdof_list
+       array) to initialize different right-hand sides and boundary condition
+       values.
+
+       After solving the linear system, the finite element solution @a x can be
+       recovered by calling RecoverFEMSolution() (with the same vectors @a X,
+       @a b, and @a x). */
+   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
+                                 Vector &x, Vector &b,
+                                 OperatorHandle &A, Vector &X, Vector &B,
+                                 int copy_interior = 0);
+
+   /// Form the linear system matrix @a A, see FormLinearSystem() for details.
+   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                 OperatorHandle &A);
+
    /// Recover the solution of a linear system formed with FormLinearSystem().
    /** Call this method after solving a linear system constructed using the
        FormLinearSystem() method to recover the solution as a GridFunction-size

fem/bilininteg.cpp

@@ -361,6 +361,28 @@ void MixedScalarVectorIntegrator::AssembleElementMatrix2(
    }
 }
 
+
+const IntegrationRule &DiffusionIntegrator::GetRule(
+   const FiniteElement &trial_fe, const FiniteElement &test_fe)
+{
+   int order;
+   if (trial_fe.Space() == FunctionSpace::Pk)
+   {
+      order = trial_fe.GetOrder() + test_fe.GetOrder() - 2;
+   }
+   else
+   {
+      // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
+      order = trial_fe.GetOrder() + test_fe.GetOrder() + trial_fe.GetDim() - 1;
+   }
+
+   if (trial_fe.Space() == FunctionSpace::rQk)
+   {
+      return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
+   }
+   return IntRules.Get(trial_fe.GetGeomType(), order);
+}
+
 void DiffusionIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
   DenseMatrix &elmat )
@@ -383,25 +405,7 @@ void DiffusionIntegrator::AssembleElementMatrix
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int order;
-      if (el.Space() == FunctionSpace::Pk)
-      {
-         order = 2*el.GetOrder() - 2;
-      }
-      else
-         // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
-      {
-         order = 2*el.GetOrder() + dim - 1;
-      }
-
-      if (el.Space() == FunctionSpace::rQk)
-      {
-         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
-      }
-      else
-      {
-         ir = &IntRules.Get(el.GetGeomType(), order);
-      }
+      ir = &GetRule(el, el);
    }
 
    elmat = 0.0;
@@ -461,24 +465,7 @@ void DiffusionIntegrator::AssembleElementMatrix2(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int order;
-      if (trial_fe.Space() == FunctionSpace::Pk)
-      {
-         order = trial_fe.GetOrder() + test_fe.GetOrder() - 2;
-      }
-      else
-      {
-         order = trial_fe.GetOrder() + test_fe.GetOrder() + dim - 1;
-      }
-
-      if (trial_fe.Space() == FunctionSpace::rQk)
-      {
-         ir = &RefinedIntRules.Get(trial_fe.GetGeomType(), order);
-      }
-      else
-      {
-         ir = &IntRules.Get(trial_fe.GetGeomType(), order);
-      }
+      ir = &GetRule(trial_fe, test_fe);
    }
 
    elmat = 0.0;
@@ -717,6 +704,22 @@ double DiffusionIntegrator::ComputeFluxEnergy
 }
 
 
+const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
+                                               const FiniteElement &test_fe,
+                                               ElementTransformation &Trans,
+                                               int coeff_order)
+{
+   // int order = trial_fe.GetOrder() + test_fe.GetOrder();
+   int order = trial_fe.GetOrder() + test_fe.GetOrder() +
+               Trans.OrderW() + coeff_order;
+
+   if (trial_fe.Space() == FunctionSpace::rQk)
+   {
+      return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
+   }
+   return IntRules.Get(trial_fe.GetGeomType(), order);
+}
+
 void MassIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
   DenseMatrix &elmat )
@@ -734,17 +737,7 @@ void MassIntegrator::AssembleElementMatrix
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      // int order = 2 * el.GetOrder();
-      int order = 2 * el.GetOrder() + Trans.OrderW();
-
-      if (el.Space() == FunctionSpace::rQk)
-      {
-         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
-      }
-      else
-      {
-         ir = &IntRules.Get(el.GetGeomType(), order);
-      }
+      ir = &GetRule(el, el, Trans);
    }
 
    elmat = 0.0;
@@ -783,9 +776,7 @@ void MassIntegrator::AssembleElementMatrix2(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
-
-      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+      ir = &GetRule(trial_fe, test_fe, Trans);
    }
 
    elmat = 0.0;
@@ -980,16 +971,7 @@ void VectorMassIntegrator::AssembleElementMatrix
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int order = 2 * el.GetOrder() + Trans.OrderW() + Q_order;
-
-      if (el.Space() == FunctionSpace::rQk)
-      {
-         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
-      }
-      else
-      {
-         ir = &IntRules.Get(el.GetGeomType(), order);
-      }
+      ir = &MassIntegrator::GetRule(el, el, Trans, Q_order);
    }
 
    elmat = 0.0;
@@ -1065,17 +1047,7 @@ void VectorMassIntegrator::AssembleElementMatrix2(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int order = (trial_fe.GetOrder() + test_fe.GetOrder() +
-                   Trans.OrderW() + Q_order);
-
-      if (trial_fe.Space() == FunctionSpace::rQk)
-      {
-         ir = &RefinedIntRules.Get(trial_fe.GetGeomType(), order);
-      }
-      else
-      {
-         ir = &IntRules.Get(trial_fe.GetGeomType(), order);
-      }
+      ir = &MassIntegrator::GetRule(trial_fe, test_fe, Trans, Q_order);
    }
 
    elmat = 0.0;

fem/bilininteg.hpp

@@ -1622,6 +1622,13 @@ public:
    virtual double ComputeFluxEnergy(const FiniteElement &fluxelem,
                                     ElementTransformation &Trans,
                                     Vector &flux, Vector *d_energy = NULL);
+
+   virtual const char *Name() const { return "diffusion"; }
+
+   virtual Coefficient *GetScalarCoefficient() const { return Q; }
+
+   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
+                                         const FiniteElement &test_fe);
 };
 
 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
@@ -1649,6 +1656,15 @@ public:
                                        const FiniteElement &test_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
+
+   virtual const char *Name() const { return "mass"; }
+
+   virtual Coefficient *GetScalarCoefficient() const { return Q; }
+
+   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
+                                         const FiniteElement &test_fe,
+                                         ElementTransformation &Trans,
+                                         int coeff_order = 0);
 };
 
 class BoundaryMassIntegrator : public MassIntegrator

fem/fe_coll.cpp

@@ -165,8 +165,10 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
                                 BasisType::GetType(name[3]));
    }
    else if (!strncmp(name, "L2_T", 4))
+   {
       fec = new L2_FECollection(atoi(name + 10), atoi(name + 6),
                                 atoi(name + 4));
+   }
    else if (!strncmp(name, "L2_", 3))
    {
       fec = new L2_FECollection(atoi(name + 7), atoi(name + 3));

fem/fespace.cpp

@@ -61,7 +61,7 @@ FiniteElementSpace::FiniteElementSpace()
      fdofs(NULL), bdofs(NULL),
      elem_dof(NULL), bdrElem_dof(NULL),
      NURBSext(NULL), own_ext(false),
-     cP(NULL), cR(NULL), cP_is_set(false),
+     cP(NULL), cR(NULL),
      Th(Operator::ANY_TYPE),
      sequence(0)
 { }
@@ -377,6 +377,12 @@ void FiniteElementSpace::GetEssentialTrueDofs(const Array<int> &bdr_attr_is_ess,
    {
       R->BooleanMult(ess_vdofs, ess_tdofs);
    }
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      ess_tdof_list.SetEngine(dev_ext->GetEngine());
+   }
+#endif
    MarkerToList(ess_tdofs, ess_tdof_list);
 }
 
@@ -389,12 +395,14 @@ void FiniteElementSpace::MarkerToList(const Array<int> &marker,
    {
       if (marker[i]) { num_marked++; }
    }
+   list.Resize(num_marked);
+   list.Pull(false);
    list.SetSize(0);
-   list.Reserve(num_marked);
    for (int i = 0; i < marker.Size(); i++)
    {
       if (marker[i]) { list.Append(i); }
    }
+   list.Push();
 }
 
 // static method
@@ -578,11 +586,12 @@ void FiniteElementSpace::BuildConformingInterpolation() const
 #ifdef MFEM_USE_MPI
    MFEM_VERIFY(dynamic_cast<const ParFiniteElementSpace*>(this) == NULL,
                "This method should not be used with a ParFiniteElementSpace!");
+   MFEM_VERIFY(Nonconforming() &&
+               dynamic_cast<const ParMesh*>(mesh) == NULL,
+               "This method should be used only with serial non-conforming"
+               " meshes!");
 #endif
 
-   if (cP_is_set) { return; }
-   cP_is_set = true;
-
    // For each slave DOF, the dependency matrix will contain a row that
    // expresses the slave DOF as a linear combination of its immediate master
    // DOFs. Rows of independent DOFs will remain empty.
@@ -642,9 +651,27 @@ void FiniteElementSpace::BuildConformingInterpolation() const
    if (n_true_dofs == ndofs)
    {
       cP = cR = NULL; // will be treated as identities
+#ifdef MFEM_USE_BACKENDS
+      t_layout = v_layout;
+#endif
       return;
    }
 
+#ifdef MFEM_USE_BACKENDS
+   if (t_layout != v_layout)
+   {
+      t_layout->Resize(n_true_dofs*vdim);
+   }
+   else if (mesh->HasEngine())
+   {
+      t_layout = mesh->GetEngine().MakeLayout(n_true_dofs*vdim);
+   }
+   else
+   {
+      t_layout.Reset(new PLayout(n_true_dofs*vdim));
+   }
+#endif
+
    // create the conforming restriction matrix cR
    int *cR_J;
    {
@@ -757,27 +784,6 @@ void FiniteElementSpace::MakeVDimMatrix(SparseMatrix &mat) const
    delete vmat;
 }
 
-
-const SparseMatrix* FiniteElementSpace::GetConformingProlongation() const
-{
-   if (Conforming()) { return NULL; }
-   if (!cP_is_set) { BuildConformingInterpolation(); }
-   return cP;
-}
-
-const SparseMatrix* FiniteElementSpace::GetConformingRestriction() const
-{
-   if (Conforming()) { return NULL; }
-   if (!cP_is_set) { BuildConformingInterpolation(); }
-   return cR;
-}
-
-int FiniteElementSpace::GetNConformingDofs() const
-{
-   const SparseMatrix* P = GetConformingProlongation();
-   return P ? (P->Width() / vdim) : ndofs;
-}
-
 SparseMatrix *FiniteElementSpace::RefinementMatrix_main(
    const int coarse_ndofs, const Table &coarse_elem_dof,
    const DenseTensor &localP) const
@@ -1102,13 +1108,25 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
    sequence = mesh->GetSequence();
    Th.SetType(Operator::ANY_TYPE);
 
+#ifdef MFEM_USE_BACKENDS
+   if (mesh->HasEngine())
+   {
+      v_layout = mesh->GetEngine().MakeLayout(0);
+   }
+   else
+   {
+      v_layout.Reset(new PLayout(0));
+   }
+   t_layout = v_layout;
+#endif
+
    const NURBSFECollection *nurbs_fec =
       dynamic_cast<const NURBSFECollection *>(fec);
    if (nurbs_fec)
    {
       if (!mesh->NURBSext)
       {
-         mfem_error("FiniteElementSpace::FiniteElementSpace :\n"
+         mfem_error("FiniteElementSpace::Constructor :\n"
                     "   NURBS FE space requires NURBS mesh.");
       }
 
@@ -1124,7 +1142,6 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
       }
       UpdateNURBS();
       cP = cR = NULL;
-      cP_is_set = false;
    }
    else
    {
@@ -1132,7 +1149,23 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
       own_ext = 0;
       Construct();
    }
-   BuildElementToDofTable();
+   BuildElementToDofTable(); // calls virtual method: GetElementDofs()
+
+#ifdef MFEM_USE_BACKENDS
+   if (mesh->HasEngine())
+   {
+      // If constructing a parallel space, we postpone the construction of the
+      // dev_ext until we have t_layout constructed.
+      // TODO: currently, if we are constructing a serial space on a parallel
+      //       mesh then dev_ext will remain NULL.
+#ifdef MFEM_USE_MPI
+      if (dynamic_cast<ParMesh*>(mesh) == NULL)
+#endif
+      {
+         dev_ext = mesh->GetEngine().MakeFESpace(*this);
+      }
+   }
+#endif
 }
 
 NURBSExtension *FiniteElementSpace::StealNURBSext()
@@ -1160,10 +1193,17 @@ void FiniteElementSpace::UpdateNURBS()
    ndofs = NURBSext->GetNDof();
    elem_dof = NURBSext->GetElementDofTable();
    bdrElem_dof = NURBSext->GetBdrElementDofTable();
+
+#ifdef MFEM_USE_BACKENDS
+   v_layout->Resize(GetVSize());
+   // t_layout is the same as v_layout (in serial)
+#endif
 }
 
 void FiniteElementSpace::Construct()
 {
+   // called in parallel by ParFiniteElementSpace::Update()
+
    // This method should be used only for non-NURBS spaces.
    MFEM_ASSERT(!NURBSext, "internal error");
 
@@ -1188,7 +1228,6 @@ void FiniteElementSpace::Construct()
    fdofs = NULL;
    cP = NULL;
    cR = NULL;
-   cP_is_set = false;
    // Th is initialized/destroyed before this method is called.
 
    if (mesh->Dimension() == 3 && mesh->GetNE())
@@ -1226,6 +1265,24 @@ void FiniteElementSpace::Construct()
 
    ndofs = nvdofs + nedofs + nfdofs + nbdofs;
 
+#ifdef MFEM_USE_BACKENDS
+   v_layout->Resize(GetVSize());
+   // t_layout is the same as v_layout, or it is set/updated in
+   // BuildConformingInterpolation(), or in ParConstruct().
+#endif
+
+   if (Nonconforming())
+   {
+#ifdef MFEM_USE_MPI
+      if (dynamic_cast<ParMesh*>(mesh) == NULL)
+#endif
+      {
+         BuildConformingInterpolation();
+      }
+      // If creating/updating a serial space on a parallel mesh then the
+      // conforming interpolation, cP, will remain NULL!
+   }
+
    // Do not build elem_dof Table here: in parallel it has to be constructed
    // later.
 }
@@ -1635,6 +1692,10 @@ FiniteElementSpace::~FiniteElementSpace()
 
 void FiniteElementSpace::Destroy()
 {
+   // called in parallel by ParFiniteElementSpace::Update()
+
+   // For now, do not reset dev_ext and/or v_layout, t_layout
+
    delete cR;
    delete cP;
    Th.Clear();
@@ -1750,6 +1811,10 @@ void FiniteElementSpace::Update(bool want_transform)
    Construct();
    BuildElementToDofTable();
 
+#ifdef MFEM_USE_BACKENDS
+   // TODO: update dev_ext ...
+#endif
+
    if (want_transform)
    {
       // calculate appropriate GridFunction transformation
@@ -1774,7 +1839,7 @@ void FiniteElementSpace::Update(bool want_transform)
 
          case Mesh::DEREFINE:
          {
-            BuildConformingInterpolation();
+            // BuildConformingInterpolation(); // called by Construct() now
             Th.Reset(DerefinementMatrix(old_ndofs, old_elem_dof));
             if (cP && cR)
             {

fem/fespace.hpp

@@ -16,6 +16,11 @@
 #include "../linalg/sparsemat.hpp"
 #include "../mesh/mesh.hpp"
 #include "fe_coll.hpp"
+
+#ifdef MFEM_USE_BACKENDS
+#include "../backends/base/backend.hpp"
+#endif
+
 #include <iostream>
 
 namespace mfem
@@ -72,6 +77,12 @@ protected:
    /// Associated FE collection (not owned).
    const FiniteElementCollection *fec;
 
+#ifdef MFEM_USE_BACKENDS
+   DFiniteElementSpace dev_ext;
+   DLayout v_layout;
+   mutable DLayout t_layout;
+#endif
+
    /// %Vector dimension (number of unknowns per degree of freedom).
    int vdim;
 
@@ -100,7 +111,6 @@ protected:
    mutable SparseMatrix *cP; // owned
    /// Conforming restriction matrix such that cR.cP=I.
    mutable SparseMatrix *cR; // owned
-   mutable bool cP_is_set;
 
    /// Transformation to apply to GridFunctions after space Update().
    OperatorHandle Th;
@@ -180,6 +190,7 @@ protected:
                                    DenseTensor &localP) const;
 
    /// Help function for constructors + Load().
+   /** Indirectly calls the virtual method GetElementDofs(). */
    void Constructor(Mesh *mesh, NURBSExtension *ext,
                     const FiniteElementCollection *fec,
                     int vdim = 1, int ordering = Ordering::byNODES);
@@ -230,8 +241,8 @@ public:
    bool Conforming() const { return mesh->Conforming(); }
    bool Nonconforming() const { return mesh->Nonconforming(); }
 
-   const SparseMatrix *GetConformingProlongation() const;
-   const SparseMatrix *GetConformingRestriction() const;
+   const SparseMatrix *GetConformingProlongation() const { return cP; }
+   const SparseMatrix *GetConformingRestriction() const { return cR; }
 
    virtual const Operator *GetProlongationMatrix() const
    { return GetConformingProlongation(); }
@@ -255,11 +266,27 @@ public:
    /// Return the number of vector true (conforming) dofs.
    virtual int GetTrueVSize() const { return GetConformingVSize(); }
 
+#ifdef MFEM_USE_BACKENDS
+   /// TODO: doxygen
+   const DLayout &GetVLayout() { return v_layout; }
+
+   /// TODO: doxygen
+   const DLayout &GetTrueVLayout() { return t_layout; }
+
+   /// TODO: doxygen
+   PFiniteElementSpace *Get_PFESpace() { return dev_ext.Get(); }
+
+   /// TODO: doxygen
+   const PFiniteElementSpace *Get_PFESpace() const { return dev_ext.Get(); }
+#endif
+
    /// Returns the number of conforming ("true") degrees of freedom
    /// (if the space is on a nonconforming mesh with hanging nodes).
-   int GetNConformingDofs() const;
+   int GetNConformingDofs() const
+   { return cP ? (cP->Width() / vdim) : ndofs; }
 
-   int GetConformingVSize() const { return vdim * GetNConformingDofs(); }
+   int GetConformingVSize() const
+   { return cP ? cP->Width() : GetVSize(); }
 
    /// Return the ordering method.
    inline Ordering::Type GetOrdering() const { return ordering; }
@@ -411,12 +438,16 @@ public:
    /** Get a list of essential true dofs, ess_tdof_list, corresponding to the
        boundary attributes marked in the array bdr_attr_is_ess.
        For spaces with 'vdim' > 1, the 'component' parameter can be used
-       to restricts the marked tDOFs to the specified component. */
+       to restricts the marked tDOFs to the specified component.
+
+       If the FE space is linked to an Engine (as defined by its Mesh), the out
+       @a ess_tdof_list will be transferred to the engine as well. */
    virtual void GetEssentialTrueDofs(const Array<int> &bdr_attr_is_ess,
                                      Array<int> &ess_tdof_list,
                                      int component = -1);
 
    /// Convert a Boolean marker array to a list containing all marked indices.
+   /** This method supports device arrays for the output parameter, @a list. */
    static void MarkerToList(const Array<int> &marker, Array<int> &list);
 
    /** Convert an array of indices (list) to a Boolean marker array where all

fem/gridfunc.cpp

@@ -33,6 +33,9 @@ GridFunction::GridFunction(Mesh *m, std::istream &input)
    fes = new FiniteElementSpace;
    fec = fes->Load(m, input);
 
+   Resize(MFEM_IF_BACKENDS(fes->GetVLayout(), fes->GetVSize()));
+   Pull(false);
+
    skip_comment_lines(input, '#');
    istream::int_type next_char = input.peek();
    if (next_char == 'N') // First letter of "NURBS_patches"
@@ -56,6 +59,7 @@ GridFunction::GridFunction(Mesh *m, std::istream &input)
       Vector::Load(input, fes->GetVSize());
    }
    sequence = fes->GetSequence();
+   Push();
 }
 
 GridFunction::GridFunction(Mesh *m, GridFunction *gf_array[], int num_pieces)
@@ -309,18 +313,28 @@ int GridFunction::VectorDim() const
    return fes->GetVDim()*fes->GetMesh()->SpaceDimension();
 }
 
-void GridFunction::GetTrueDofs(Vector &tv) const
+void GridFunction::GetTrueDofs(Vector &tv)
 {
    const SparseMatrix *R = fes->GetRestrictionMatrix();
    if (!R)
    {
       // R is identity -> make tv a reference to *this
-      tv.NewDataAndSize(data, size);
+      tv.MakeRef(*this);
    }
    else
    {
-      tv.SetSize(R->Height());
-      R->Mult(*this, tv);
+#ifdef MFEM_USE_BACKENDS
+      if (fes->GetMesh()->HasEngine())
+      {
+         tv.Resize(fes->GetTrueVLayout());
+         fes->Get_PFESpace()->GetRestrictionOperator()->Mult(*this, tv);
+      }
+      else
+#endif
+      {
+         tv.SetSize(R->Height());
+         R->Mult(*this, tv);
+      }
    }
 }
 
@@ -330,14 +344,21 @@ void GridFunction::SetFromTrueDofs(const Vector &tv)
    const SparseMatrix *cP = fes->GetConformingProlongation();
    if (!cP)
    {
-      if (tv.GetData() != data)
-      {
-         *this = tv;
-      }
+      Assign(tv);
    }
    else
    {
-      cP->Mult(tv, *this);
+#ifdef MFEM_USE_BACKENDS
+      if (fes->GetMesh()->HasEngine())
+      {
+         Resize(fes->GetTrueVLayout());
+         fes->Get_PFESpace()->GetProlongationOperator()->Mult(tv, *this);
+      }
+      else
+#endif
+      {
+         cP->Mult(tv, *this);
+      }
    }
 }
 
@@ -1431,6 +1452,7 @@ void GridFunction::ProjectCoefficient(VectorCoefficient &vcoeff)
    Array<int> vdofs;
    Vector vals;
 
+   Pull(false);
    for (i = 0; i < fes->GetNE(); i++)
    {
       fes->GetElementVDofs(i, vdofs);
@@ -1438,6 +1460,7 @@ void GridFunction::ProjectCoefficient(VectorCoefficient &vcoeff)
       fes->GetFE(i)->Project(vcoeff, *fes->GetElementTransformation(i), vals);
       SetSubVector(vdofs, vals);
    }
+   Push();
 }
 
 void GridFunction::ProjectCoefficient(

fem/gridfunc.hpp

@@ -61,6 +61,8 @@ protected:
        degree of freedom. */
    void ProjectDiscCoefficient(VectorCoefficient &coeff, Array<int> &dof_attr);
 
+   // There is a method with the same name in the base class Vector.
+   // This version of Destroy does not call Vector::Destroy.
    void Destroy();
 
 public:
@@ -72,7 +74,8 @@ public:
       : Vector(orig), fes(orig.fes), fec(NULL), sequence(orig.sequence) { }
 
    /// Construct a GridFunction associated with the FiniteElementSpace @a *f.
-   GridFunction(FiniteElementSpace *f) : Vector(f->GetVSize())
+   GridFunction(FiniteElementSpace *f)
+      : Vector(MFEM_IF_BACKENDS(f->GetVLayout(), f->GetVSize()))
    { fes = f; fec = NULL; sequence = f->GetSequence(); }
 
    /// Construct a GridFunction using previously allocated array @a data.
@@ -81,8 +84,7 @@ public:
        for externally allocated array, the pointer @a data can be NULL. The data
        array can be replaced later using the method SetData().
     */
-   GridFunction(FiniteElementSpace *f, double *data) : Vector(data, f->GetVSize())
-   { fes = f; fec = NULL; sequence = f->GetSequence(); }
+   inline GridFunction(FiniteElementSpace *f, double *data);
 
    /// Construct a GridFunction on the given Mesh, using the data from @a input.
    /** The content of @a input should be in the format created by the method
@@ -110,7 +112,7 @@ public:
 
    /// @brief Extract the true-dofs from the GridFunction. If all dofs are true,
    /// then `tv` will be set to point to the data of `*this`.
-   void GetTrueDofs(Vector &tv) const;
+   void GetTrueDofs(Vector &tv);
 
    /// Shortcut for calling GetTrueDofs() with GetTrueVector() as argument.
    void SetTrueVector() { GetTrueDofs(GetTrueVector()); }
@@ -467,7 +469,7 @@ public:
    { vdim = vdim_; SetSize(vdim*qspace->GetSize()); }
 
    /// Get the QuadratureSpace ownership flag.
-   bool OwnsSpace() { return own_qspace; }
+   bool OwnsSpace() const { return own_qspace; }
 
    /// Set the QuadratureSpace ownership flag.
    void SetOwnsSpace(bool own) { own_qspace = own; }
@@ -564,7 +566,21 @@ GridFunction *Extrude1DGridFunction(Mesh *mesh, Mesh *mesh2d,
                                     GridFunction *sol, const int ny);
 
 
-// Inline methods
+// Inline methods: class GridFunction
+
+inline GridFunction::GridFunction(FiniteElementSpace *f, double *data)
+   : Vector(data, f->GetVSize())
+{
+#ifdef MFEM_USE_BACKENDS
+   MFEM_ASSERT(f->GetVLayout()->HasEngine() == false, "not supported");
+#endif
+   fes = f;
+   fec = NULL;
+   sequence = f->GetSequence();
+}
+
+
+// Inline methods: class QuadratureFunction
 
 inline void QuadratureFunction::SetSpace(QuadratureSpace *qspace_, int vdim_)
 {

fem/linearform.cpp

@@ -54,12 +54,14 @@ void LinearForm::Assemble()
    ElementTransformation *eltrans;
    Vector elemvect;
 
-   int i;
+   // FIXME: Use the Engine when set
 
+   // Assemble on host
+   Pull(false); // copy_data = false
    Vector::operator=(0.0);
 
    if (dlfi.Size())
-      for (i = 0; i < fes -> GetNE(); i++)
+      for (int i = 0; i < fes -> GetNE(); i++)
       {
          fes -> GetElementVDofs (i, vdofs);
          eltrans = fes -> GetElementTransformation (i);
@@ -73,7 +75,7 @@ void LinearForm::Assemble()
    AssembleDelta();
 
    if (blfi.Size())
-      for (i = 0; i < fes -> GetNBE(); i++)
+      for (int i = 0; i < fes -> GetNBE(); i++)
       {
          fes -> GetBdrElementVDofs (i, vdofs);
          eltrans = fes -> GetBdrElementTransformation (i);
@@ -110,7 +112,7 @@ void LinearForm::Assemble()
          }
       }
 
-      for (i = 0; i < mesh->GetNBE(); i++)
+      for (int i = 0; i < mesh->GetNBE(); i++)
       {
          const int bdr_attr = mesh->GetBdrAttribute(i);
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
@@ -131,11 +133,16 @@ void LinearForm::Assemble()
          }
       }
    }
+
+   Push();
 }
 
 void LinearForm::Update(FiniteElementSpace *f, Vector &v, int v_offset)
 {
    fes = f;
+#ifdef MFEM_USE_BACKENDS
+   MFEM_VERIFY(f->GetMesh()->HasEngine() == false, "not supported yet");
+#endif
    NewDataAndSize((double *)v + v_offset, fes->GetVSize());
    ResetDeltaLocations();
 }

fem/linearform.hpp

@@ -53,8 +53,8 @@ private:
 
 public:
    /// Creates linear form associated with FE space *f.
-   LinearForm (FiniteElementSpace * f) : Vector (f -> GetVSize())
-   { fes = f; }
+   LinearForm(FiniteElementSpace *f)
+      : Vector(MFEM_IF_BACKENDS(f->GetVLayout(), f->GetVSize())), fes(f) { }
 
    LinearForm() { fes = NULL; }
 
@@ -87,10 +87,13 @@ public:
    /// Assembles delta functions of the linear form
    void AssembleDelta();
 
-   void Update() { SetSize(fes->GetVSize()); ResetDeltaLocations(); }
+   void Update()
+   {
+      Resize(MFEM_IF_BACKENDS(fes->GetVLayout(), fes->GetVSize()));
+      ResetDeltaLocations();
+   }
 
-   void Update(FiniteElementSpace *f)
-   { fes = f; SetSize(f->GetVSize()); ResetDeltaLocations(); }
+   void Update(FiniteElementSpace *f) { fes = f; Update(); }
 
    void Update(FiniteElementSpace *f, Vector &v, int v_offset);
 

fem/nonlininteg.hpp

@@ -39,6 +39,10 @@ public:
    /// Prescribe a fixed IntegrationRule to use.
    void SetIntegrationRule(const IntegrationRule &irule) { IntRule = &irule; }
 
+   /** @brief Get the prescribed IntegrationRule, if set by SetIntRule() or
+       SetIntegrationRule(). */
+   const IntegrationRule *GetIntRule() const { return IntRule; }
+
    /// Perform the local action of the NonlinearFormIntegrator
    virtual void AssembleElementVector(const FiniteElement &el,
                                       ElementTransformation &Tr,
@@ -68,6 +72,13 @@ public:
                                    ElementTransformation &Tr,
                                    const Vector &elfun);
 
+   /// Return a string that identifies the integrator.
+   virtual const char *Name() const { return "(undefined)"; }
+
+   /** If the integrator uses a scalar Coefficient, return a pointer to it.
+       Otherwise return NULL. */
+   virtual Coefficient *GetScalarCoefficient() const { return NULL; }
+
    virtual ~NonlinearFormIntegrator() { }
 };
 

fem/pbilinearform.cpp

@@ -284,84 +284,104 @@ void ParBilinearForm::FormLinearSystem(
    const Array<int> &ess_tdof_list, Vector &x, Vector &b,
    OperatorHandle &A, Vector &X, Vector &B, int copy_interior)
 {
-   // Finish the matrix assembly and perform BC elimination, storing the
-   // eliminated part of the matrix.
-   FormSystemMatrix(ess_tdof_list, A);
-
-   const Operator &P = *pfes->GetProlongationMatrix();
-   const SparseMatrix &R = *pfes->GetRestrictionMatrix();
-
-   // Transform the system and perform the elimination in B, based on the
-   // essential BC values from x. Restrict the BC part of x in X, and set the
-   // non-BC part to zero. Since there is no good initial guess for the Lagrange
-   // multipliers, set X = 0.0 for hybridization.
-   if (static_cond)
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
    {
-      // Schur complement reduction to the exposed dofs
-      static_cond->ReduceSystem(x, b, X, B, copy_interior);
-   }
-   else if (hybridization)
-   {
-      // Reduction to the Lagrange multipliers system
-      HypreParVector true_X(pfes), true_B(pfes);
-      P.MultTranspose(b, true_B);
-      R.Mult(x, true_X);
-      p_mat.EliminateBC(p_mat_e, ess_tdof_list, true_X, true_B);
-      R.MultTranspose(true_B, b);
-      hybridization->ReduceRHS(true_B, B);
-      X.SetSize(B.Size());
-      X = 0.0;
+      MFEM_VERIFY(!static_cond && !hybridization, "");
+      dev_ext->FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
    }
    else
+#endif
    {
-      // Variational restriction with P
-      X.SetSize(pfes->TrueVSize());
-      B.SetSize(X.Size());
-      P.MultTranspose(b, B);
-      R.Mult(x, X);
-      p_mat.EliminateBC(p_mat_e, ess_tdof_list, X, B);
-      if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
+      // Finish the matrix assembly and perform BC elimination, storing the
+      // eliminated part of the matrix.
+      FormSystemMatrix(ess_tdof_list, A);
+
+      const Operator &P = *pfes->GetProlongationMatrix();
+      const SparseMatrix &R = *pfes->GetRestrictionMatrix();
+
+      // Transform the system and perform the elimination in B, based on the
+      // essential BC values from x. Restrict the BC part of x in X, and set the
+      // non-BC part to zero. Since there is no good initial guess for the Lagrange
+      // multipliers, set X = 0.0 for hybridization.
+      if (static_cond)
+      {
+         // Schur complement reduction to the exposed dofs
+         static_cond->ReduceSystem(x, b, X, B, copy_interior);
+      }
+      else if (hybridization)
+      {
+         // Reduction to the Lagrange multipliers system
+         HypreParVector true_X(pfes), true_B(pfes);
+         P.MultTranspose(b, true_B);
+         R.Mult(x, true_X);
+         p_mat.EliminateBC(p_mat_e, ess_tdof_list, true_X, true_B);
+         R.MultTranspose(true_B, b);
+         hybridization->ReduceRHS(true_B, B);
+         X.SetSize(B.Size());
+         X = 0.0;
+      }
+      else
+      {
+         // Variational restriction with P
+         X.SetSize(pfes->TrueVSize());
+         B.SetSize(X.Size());
+         P.MultTranspose(b, B);
+         R.Mult(x, X);
+         p_mat.EliminateBC(p_mat_e, ess_tdof_list, X, B);
+         if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
+      }
    }
 }
 
 void ParBilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
                                        OperatorHandle &A)
 {
-   // Finish the matrix assembly and perform BC elimination, storing the
-   // eliminated part of the matrix.
-   if (static_cond)
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
    {
-      if (!static_cond->HasEliminatedBC())
-      {
-         static_cond->SetEssentialTrueDofs(ess_tdof_list);
-         static_cond->Finalize();
-         static_cond->EliminateReducedTrueDofs(Matrix::DIAG_ONE);
-      }
-      static_cond->GetParallelMatrix(A);
+      MFEM_VERIFY(!static_cond && !hybridization, "");
+      dev_ext->FormSystemMatrix(ess_tdof_list, A);
    }
    else
+#endif
    {
-      if (mat)
+      // Finish the matrix assembly and perform BC elimination, storing the
+      // eliminated part of the matrix.
+      if (static_cond)
       {
-         const int remove_zeros = 0;
-         Finalize(remove_zeros);
-         MFEM_VERIFY(p_mat.Ptr() == NULL && p_mat_e.Ptr() == NULL,
-                     "The ParBilinearForm must be updated with Update() before "
-                     "re-assembling the ParBilinearForm.");
-         ParallelAssemble(p_mat, mat);
-         delete mat;
-         mat = NULL;
-         delete mat_e;
-         mat_e = NULL;
-         p_mat_e.EliminateRowsCols(p_mat, ess_tdof_list);
-      }
-      if (hybridization)
-      {
-         hybridization->GetParallelMatrix(A);
+         if (!static_cond->HasEliminatedBC())
+         {
+            static_cond->SetEssentialTrueDofs(ess_tdof_list);
+            static_cond->Finalize();
+            static_cond->EliminateReducedTrueDofs(Matrix::DIAG_ONE);
+         }
+         static_cond->GetParallelMatrix(A);
       }
       else
       {
-         A = p_mat;
+         if (mat)
+         {
+            const int remove_zeros = 0;
+            Finalize(remove_zeros);
+            MFEM_VERIFY(p_mat.Ptr() == NULL && p_mat_e.Ptr() == NULL,
+                        "The ParBilinearForm must be updated with Update() before "
+                        "re-assembling the ParBilinearForm.");
+            ParallelAssemble(p_mat, mat);
+            delete mat;
+            mat = NULL;
+            delete mat_e;
+            mat_e = NULL;
+            p_mat_e.EliminateRowsCols(p_mat, ess_tdof_list);
+         }
+         if (hybridization)
+         {
+            hybridization->GetParallelMatrix(A);
+         }
+         else
+         {
+            A = p_mat;
+         }
       }
    }
 }
@@ -369,6 +389,14 @@ void ParBilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
 void ParBilinearForm::RecoverFEMSolution(
    const Vector &X, const Vector &b, Vector &x)
 {
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      dev_ext->RecoverFEMSolution(X, b, x);
+      return;
+   }
+#endif
+
    const Operator &P = *pfes->GetProlongationMatrix();
 
    if (static_cond)

fem/pbilinearform.hpp

@@ -145,30 +145,10 @@ public:
    virtual const Operator *GetRestriction() const
    { return pfes->GetRestrictionMatrix(); }
 
-   /** Form the linear system A X = B, corresponding to the current bilinear
-       form and b(.), by applying any necessary transformations such as:
-       eliminating boundary conditions; applying conforming constraints for
-       non-conforming AMR; parallel assembly; static condensation;
-       hybridization.
-
-       The ParGridFunction-size vector x must contain the essential b.c. The
-       ParBilinearForm and the ParLinearForm-size vector b must be assembled.
-
-       The vector X is initialized with a suitable initial guess: when using
-       hybridization, the vector X is set to zero; otherwise, the essential
-       entries of X are set to the corresponding b.c. and all other entries are
-       set to zero (copy_interior == 0) or copied from x (copy_interior != 0).
-
-       This method can be called multiple times (with the same ess_tdof_list
-       array) to initialize different right-hand sides and boundary condition
-       values.
-
-       After solving the linear system, the finite element solution x can be
-       recovered by calling RecoverFEMSolution (with the same vectors X, b, and
-       x). */
-   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
-                         OperatorHandle &A, Vector &X, Vector &B,
-                         int copy_interior = 0);
+   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
+                                 Vector &x, Vector &b,
+                                 OperatorHandle &A, Vector &X, Vector &B,
+                                 int copy_interior = 0);
 
    /** Version of the method FormLinearSystem() where the system matrix is
        returned in the variable @a A, of type OpType, holding a *reference* to
@@ -187,8 +167,8 @@ public:
       A.MakeRef(*A_ptr);
    }
 
-   /// Form the linear system matrix @a A, see FormLinearSystem() for details.
-   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
+   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                 OperatorHandle &A);
 
    /** Version of the method FormSystemMatrix() where the system matrix is
        returned in the variable @a A, of type OpType, holding a *reference* to

fem/pfespace.cpp

@@ -117,16 +117,23 @@ void ParFiniteElementSpace::ParInit(ParMesh *pm)
       UpdateNURBS();
    }
 
-   Construct(); // parallel version of Construct().
+   ParConstruct();
 
    // Apply the ldof_signs to the elem_dof Table
    if (Conforming() && !NURBSext)
    {
       ApplyLDofSigns(*elem_dof);
    }
+
+#ifdef MFEM_USE_BACKENDS
+   if (pmesh->HasEngine())
+   {
+      dev_ext = pmesh->GetEngine().MakeFESpace(*this);
+   }
+#endif
 }
 
-void ParFiniteElementSpace::Construct()
+void ParFiniteElementSpace::ParConstruct()
 {
    if (NURBSext)
    {
@@ -169,6 +176,23 @@ void ParFiniteElementSpace::Construct()
       // to overlap its communication with processing between this constructor
       // and the point where the P matrix is actually needed.
    }
+
+#ifdef MFEM_USE_BACKENDS
+   // Now that we have the local true dof size (ltdof_size), we need to
+   // initialize/update t_layout.
+   if (t_layout != v_layout)
+   {
+      t_layout->Resize(ltdof_size);
+   }
+   else if (pmesh->HasEngine())
+   {
+      t_layout = pmesh->GetEngine().MakeLayout(ltdof_size);
+   }
+   else
+   {
+      t_layout.Reset(new PLayout(ltdof_size));
+   }
+#endif
 }
 
 void ParFiniteElementSpace::GetGroupComm(
@@ -644,6 +668,12 @@ void ParFiniteElementSpace::GetEssentialTrueDofs(const Array<int>
       if (bool(true_ess_dofs[i]) != bool(true_ess_dofs2[i])) { counter++; }
    }
    MFEM_VERIFY(counter == 0, "internal MFEM error: counter = " << counter);
+#endif
+#ifdef MFEM_USE_BACKENDS
+   if (dev_ext)
+   {
+      ess_tdof_list.SetEngine(dev_ext->GetEngine());
+   }
 #endif
    MarkerToList(true_ess_dofs, ess_tdof_list);
 }
@@ -1154,7 +1184,7 @@ void ParFiniteElementSpace::GetGhostEdgeDofs(const MeshId &edge_id,
    dofs.SetSize(2*nv + ne);
 
    int V[2], ghost = pncmesh->GetNVertices();
-   pmesh->pncmesh->GetEdgeVertices(edge_id, V);
+   pncmesh->GetEdgeVertices(edge_id, V);
 
    for (int i = 0; i < 2; i++)
    {
@@ -1183,7 +1213,7 @@ void ParFiniteElementSpace::GetGhostFaceDofs(const MeshId &face_id,
    dofs.SetSize(4*nv + 4*ne + nf);
 
    int V[4], E[4], Eo[4];
-   pmesh->pncmesh->GetFaceVerticesEdges(face_id, V, E, Eo);
+   pncmesh->GetFaceVerticesEdges(face_id, V, E, Eo);
 
    int offset = 0;
    for (int i = 0; i < 4; i++)
@@ -2190,7 +2220,6 @@ ParFiniteElementSpace::RebalanceMatrix(int old_ndofs,
                           ? old_dof_offsets[0] : old_dof_offsets[MyRank];
 
    // send old DOFs of elements we used to own
-   ParNCMesh* pncmesh = pmesh->pncmesh;
    pncmesh->SendRebalanceDofs(old_ndofs, *old_elem_dof, old_offset, this);
 
    Array<int> dofs;
@@ -2304,7 +2333,7 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
    Array<int> dofs, old_dofs, old_vdofs;
    Vector row;
 
-   ParNCMesh* pncmesh = pmesh->pncmesh;
+   ParNCMesh* pncmesh = this->pncmesh;
    int geom = pncmesh->GetElementGeometry();
    int ldof = fec->FiniteElementForGeometry(geom)->GetDof();
 
@@ -2515,7 +2544,7 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
    return R;
 }
 
-void ParFiniteElementSpace::Destroy()
+void ParFiniteElementSpace::ParDestroy()
 {
    ldof_group.DeleteAll();
    ldof_ltdof.DeleteAll();
@@ -2596,14 +2625,18 @@ void ParFiniteElementSpace::Update(bool want_transform)
       Swap(dof_offsets, old_dof_offsets);
    }
 
-   Destroy();
+   ParDestroy();
    FiniteElementSpace::Destroy(); // calls Th.Clear()
 
    FiniteElementSpace::Construct();
-   Construct();
+   ParConstruct();
 
    BuildElementToDofTable();
 
+#ifdef MFEM_USE_BACKENDS
+   // TODO: update dev_ext ...
+#endif
+
    if (want_transform)
    {
       // calculate appropriate GridFunction transformation

fem/pfespace.hpp

@@ -88,8 +88,8 @@ private:
    // Auxiliary method used in constructors
    void ParInit(ParMesh *pm);
 
-   void Construct();
-   void Destroy();
+   void ParConstruct();
+   void ParDestroy();
 
    // ldof_type = 0 : DOFs communicator, otherwise VDOFs communicator
    void GetGroupComm(GroupCommunicator &gcomm, int ldof_type,
@@ -334,8 +334,8 @@ public:
    void LoseDofOffsets() { dof_offsets.LoseData(); }
    void LoseTrueDofOffsets() { tdof_offsets.LoseData(); }
 
-   bool Conforming() const { return pmesh->pncmesh == NULL; }
-   bool Nonconforming() const { return pmesh->pncmesh != NULL; }
+   bool Conforming() const { return pncmesh == NULL; }
+   bool Nonconforming() const { return pncmesh != NULL; }
 
    // Transfer parallel true-dof data from coarse_fes, defined on a coarse mesh,
    // to this FE space, defined on a refined mesh. See full documentation in the
@@ -354,7 +354,7 @@ public:
       old_dof_offsets.DeleteAll();
    }
 
-   virtual ~ParFiniteElementSpace() { Destroy(); }
+   virtual ~ParFiniteElementSpace() { ParDestroy(); }
 
    // Obsolete, kept for backward compatibility
    int TrueVSize() const { return ltdof_size; }

fem/pgridfunc.cpp

@@ -138,7 +138,7 @@ void ParGridFunction::AddDistribute(double a, const Vector *tv)
    pfes->Dof_TrueDof_Matrix()->Mult(a, *tv, 1.0, *this);
 }
 
-HypreParVector *ParGridFunction::GetTrueDofs() const
+HypreParVector *ParGridFunction::GetTrueDofs()
 {
    HypreParVector *tv = pfes->NewTrueDofVector();
    GetTrueDofs(*tv);

fem/pgridfunc.hpp

@@ -145,7 +145,7 @@ public:
    using GridFunction::GetTrueDofs;
 
    /// Returns the true dofs in a new HypreParVector
-   HypreParVector *GetTrueDofs() const;
+   HypreParVector *GetTrueDofs();
 
    /// Returns the vector averaged on the true dofs.
    void ParallelAverage(Vector &tv) const;

fem/staticcond.cpp

@@ -443,14 +443,16 @@ void StaticCondensation::ConvertMarkerToReducedTrueDofs(
    const int nedofs = tr_fes->GetVSize();
    const SparseMatrix *R = fes->GetRestrictionMatrix();
    Array<int> ess_dof_marker;
+   Array<const int> ess_dof_marker_const;
    if (!R)
    {
-      ess_dof_marker.MakeRef(ess_tdof_marker);
+      ess_dof_marker_const.MakeConstRef(ess_tdof_marker);
    }
    else
    {
       ess_dof_marker.SetSize(fes->GetVSize());
       R->BooleanMultTranspose(ess_tdof_marker, ess_dof_marker);
+      ess_dof_marker_const.MakeConstRef(ess_dof_marker);
    }
    const SparseMatrix *tr_R = tr_fes->GetRestrictionMatrix();
    Array<int> ess_rdof_marker;
@@ -465,7 +467,7 @@ void StaticCondensation::ConvertMarkerToReducedTrueDofs(
    }
    for (int i = 0; i < nedofs; i++)
    {
-      ess_rdof_marker[i] = ess_dof_marker[rdof_edof[i]];
+      ess_rdof_marker[i] = ess_dof_marker_const[rdof_edof[i]];
    }
    if (tr_R)
    {

general/array.cpp

@@ -18,48 +18,6 @@
 namespace mfem
 {
 
-BaseArray::BaseArray(int asize, int ainc, int elementsize)
-{
-   if (asize > 0)
-   {
-      data = new char[asize * elementsize];
-      size = allocsize = asize;
-   }
-   else
-   {
-      data = 0;
-      size = allocsize = 0;
-   }
-   inc = ainc;
-}
-
-BaseArray::~BaseArray()
-{
-   if (allocsize > 0)
-   {
-      delete [] (char*)data;
-   }
-}
-
-void BaseArray::GrowSize(int minsize, int elementsize)
-{
-   void *p;
-   int nsize = (inc > 0) ? abs(allocsize) + inc : 2 * abs(allocsize);
-   if (nsize < minsize) { nsize = minsize; }
-
-   p = new char[nsize * elementsize];
-   if (size > 0)
-   {
-      memcpy(p, data, size * elementsize);
-   }
-   if (allocsize > 0)
-   {
-      delete [] (char*)data;
-   }
-   data = p;
-   allocsize = nsize;
-}
-
 template <class T>
 void Array<T>::Print(std::ostream &out, int width) const
 {
@@ -86,7 +44,7 @@ void Array<T>::Save(std::ostream &out, int fmt) const
    }
    for (int i = 0; i < size; i++)
    {
-      out << operator[](i) << '\n';
+      out << this->operator[](i) << '\n';
    }
 }
 
@@ -97,11 +55,11 @@ void Array<T>::Load(std::istream &in, int fmt)
    {
       int new_size;
       in >> new_size;
-      SetSize(new_size);
+      this->SetSize(new_size);
    }
    for (int i = 0; i < size; i++)
    {
-      in >> operator[](i);
+      in >> this->operator[](i);
    }
 }
 
@@ -110,11 +68,11 @@ T Array<T>::Max() const
 {
    MFEM_ASSERT(size > 0, "Array is empty with size " << size);
 
-   T max = operator[](0);
+   T max = this->operator[](0);
    for (int i = 1; i < size; i++)
-      if (max < operator[](i))
+      if (max < this->operator[](i))
       {
-         max = operator[](i);
+         max = this->operator[](i);
       }
 
    return max;
@@ -125,11 +83,11 @@ T Array<T>::Min() const
 {
    MFEM_ASSERT(size > 0, "Array is empty with size " << size);
 
-   T min = operator[](0);
+   T min = this->operator[](0);
    for (int i = 1; i < size; i++)
-      if (operator[](i) < min)
+      if (this->operator[](i) < min)
       {
-         min = operator[](i);
+         min = this->operator[](i);
       }
 
    return min;
@@ -142,8 +100,8 @@ void Array<T>::PartialSum()
    T sum = static_cast<T>(0);
    for (int i = 0; i < size; i++)
    {
-      sum+=operator[](i);
-      operator[](i) = sum;
+      sum += this->operator[](i);
+      this->operator[](i) = sum;
    }
 }
 
@@ -154,7 +112,7 @@ T Array<T>::Sum()
    T sum = static_cast<T>(0);
    for (int i = 0; i < size; i++)
    {
-      sum+=operator[](i);
+      sum += this->operator[](i);
    }
 
    return sum;
@@ -163,10 +121,10 @@ T Array<T>::Sum()
 template <class T>
 int Array<T>::IsSorted()
 {
-   T val_prev = operator[](0), val;
+   T val_prev = this->operator[](0), val;
    for (int i = 1; i < size; i++)
    {
-      val=operator[](i);
+      val = this->operator[](i);
       if (val < val_prev)
       {
          return 0;

general/communication.cpp

@@ -513,6 +513,78 @@ void GroupCommunicator::SetLTDofTable(const Array<int> &ldof_ltdof)
    group_ltdof.ShiftUpI();
 }
 
+void GroupCommunicator::GetNeighborLTDofTable(Table &nbr_ltdof) const
+{
+   nbr_ltdof.MakeI(nbr_send_groups.Size());
+   for (int nbr = 1; nbr < nbr_send_groups.Size(); nbr++)
+   {
+      const int num_send_groups = nbr_send_groups.RowSize(nbr);
+      if (num_send_groups > 0)
+      {
+         const int *grp_list = nbr_send_groups.GetRow(nbr);
+         for (int i = 0; i < num_send_groups; i++)
+         {
+            const int group = grp_list[i];
+            const int nltdofs = group_ltdof.RowSize(group);
+            nbr_ltdof.AddColumnsInRow(nbr, nltdofs);
+         }
+      }
+   }
+   nbr_ltdof.MakeJ();
+   for (int nbr = 1; nbr < nbr_send_groups.Size(); nbr++)
+   {
+      const int num_send_groups = nbr_send_groups.RowSize(nbr);
+      if (num_send_groups > 0)
+      {
+         const int *grp_list = nbr_send_groups.GetRow(nbr);
+         for (int i = 0; i < num_send_groups; i++)
+         {
+            const int group = grp_list[i];
+            const int nltdofs = group_ltdof.RowSize(group);
+            const int *ltdofs = group_ltdof.GetRow(group);
+            nbr_ltdof.AddConnections(nbr, ltdofs, nltdofs);
+         }
+      }
+   }
+   nbr_ltdof.ShiftUpI();
+}
+
+void GroupCommunicator::GetNeighborLDofTable(Table &nbr_ldof) const
+{
+   nbr_ldof.MakeI(nbr_recv_groups.Size());
+   for (int nbr = 1; nbr < nbr_recv_groups.Size(); nbr++)
+   {
+      const int num_recv_groups = nbr_recv_groups.RowSize(nbr);
+      if (num_recv_groups > 0)
+      {
+         const int *grp_list = nbr_recv_groups.GetRow(nbr);
+         for (int i = 0; i < num_recv_groups; i++)
+         {
+            const int group = grp_list[i];
+            const int nldofs = group_ldof.RowSize(group);
+            nbr_ldof.AddColumnsInRow(nbr, nldofs);
+         }
+      }
+   }
+   nbr_ldof.MakeJ();
+   for (int nbr = 1; nbr < nbr_recv_groups.Size(); nbr++)
+   {
+      const int num_recv_groups = nbr_recv_groups.RowSize(nbr);
+      if (num_recv_groups > 0)
+      {
+         const int *grp_list = nbr_recv_groups.GetRow(nbr);
+         for (int i = 0; i < num_recv_groups; i++)
+         {
+            const int group = grp_list[i];
+            const int nldofs = group_ldof.RowSize(group);
+            const int *ldofs = group_ldof.GetRow(group);
+            nbr_ldof.AddConnections(nbr, ldofs, nldofs);
+         }
+      }
+   }
+   nbr_ldof.ShiftUpI();
+}
+
 template <class T>
 T *GroupCommunicator::CopyGroupToBuffer(const T *ldata, T *buf, int group,
                                         int layout) const

general/communication.hpp

@@ -179,6 +179,12 @@ public:
    /// Get a const reference to the associated GroupTopology object
    const GroupTopology &GetGroupTopology() const { return gtopo; }
 
+   // TODO: doxygen
+   void GetNeighborLTDofTable(Table &nbr_ltdof) const;
+
+   // TODO: doxygen
+   void GetNeighborLDofTable(Table &nbr_ldof) const;
+
    /** @brief Data structure on which we define reduce operations.
 
      The data is associated with (and the operation is performed on) one group

general/globals.cpp

@@ -23,6 +23,19 @@ OutStream out(std::cout);
 OutStream err(std::cerr);
 
 
+/// TODO: doxygen
+const char *GetSourcePath()
+{
+   return MFEM_SOURCE_DIR;
+}
+
+/// TODO: doxygen
+const char *GetInstallPath()
+{
+   return MFEM_INSTALL_DIR;
+}
+
+
 std::string MakeParFilename(const std::string &prefix, const int myid,
                             const std::string suffix, const int width)
 {

general/globals.hpp

@@ -69,6 +69,13 @@ extern OutStream out;
 extern OutStream err;
 
 
+/// TODO: doxygen
+extern const char *GetSourcePath();
+
+/// TODO: doxygen
+extern const char *GetInstallPath();
+
+
 /** @brief Construct a string of the form "<prefix><myid><suffix>" where the
     integer @a myid is padded with leading zeros to be at least @a width digits
     long. */

general/scalars.hpp

@@ -0,0 +1,51 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_SCALARS_HPP
+#define MFEM_SCALARS_HPP
+
+#include "../config/config.hpp"
+#include <complex>
+
+namespace mfem
+{
+
+/// Map a scalar type to a scalar type Id.
+template <typename T> struct ScalarId;
+
+template<> struct ScalarId<double>
+{
+   static const int value = 0;
+};
+
+template<> struct ScalarId<std::complex<double> >
+{
+   static const int value = 1;
+};
+
+
+/// Basic operations on scalars
+template <typename T> struct ScalarOps;
+
+template <> struct ScalarOps<double>
+{
+   static inline const double &conj(const double &a) { return a; }
+};
+
+template <> struct ScalarOps<std::complex<double> >
+{
+   static inline std::complex<double> conj(const std::complex<double> &a)
+   { return std::conj(a); }
+};
+
+} // namespace mfem
+
+#endif // MFEM_SCALARS_HPP