updating GetElementBaseGeometry based on axom changes

axom::sidre::SidreLength was also renamed as axom::sidre::IndexType.

CMakeLists.txt

@@ -243,7 +243,7 @@ endif()
 
 # Axom/Sidre
 if (MFEM_USE_SIDRE)
-  find_package(Axom REQUIRED Sidre SLIC axom_utils)
+  find_package(Axom REQUIRED Sidre SLIC axom_core)
 endif()
 
 # PUMI

backends/all.hpp

@@ -0,0 +1,31 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_ALL_HPP
+#define MFEM_BACKENDS_ALL_HPP
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_BACKENDS
+
+#include "base/backend.hpp"
+
+#ifdef MFEM_USE_OCCA
+#include "occa/backend.hpp"
+#endif
+
+#ifdef MFEM_USE_KERNELS
+#include "kernels/kernels.hpp"
+#endif
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_ALL_HPP

backends/base/array.hpp

@@ -0,0 +1,228 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_ARRAY_HPP
+#define MFEM_BACKENDS_BASE_ARRAY_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "layout.hpp"
+#include "utils.hpp"
+
+namespace mfem
+{
+
+/// Extension to the template class Array<T>
+class PArray : public RefCounted
+{
+protected:
+   /// Layout with shared ownership (smart pointer)
+   DLayout layout;
+
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new array (in @a *clone) of the same dynamic
+       type as this array using the same layout and ItemSize().
+
+       Set @a *clone to NULL if allocation fails.
+
+       If @a copy_data is true, the contents of this array is copied to the new
+       array; otherwise, the new array remains uninitialized.
+
+       If @a buffer is not NULL, return the array data of the newly created
+       object (in @a *buffer) , if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   virtual PArray *DoClone(bool copy_data, void **buffer,
+                           std::size_t item_size) const = 0;
+
+   /// Resize the array, reallocating its data if necessary.
+   /** If @a buffer is not NULL, return the array data (in @a *buffer), if it
+       is stored as a contiguous array on the host; otherwise, set @a *buffer to
+       NULL. Returns 0 on success and non-zero otherwise, e.g. if memory
+       allocation fails.
+
+       If the @a new_layout is not supported, a non-zero error code will be
+       returned.
+
+       The @a new_layout has to be valid, i.e. new_layout != NULL and
+       new_layout->HasEngine() == true.
+
+       @note If reallocation is performed, the previous content of the array is
+       NOT copied to the new location. */
+   virtual int DoResize(PLayout &new_layout, void **buffer,
+                        std::size_t item_size) = 0;
+
+   /** @brief Get access to the contents of the array in host memory, as a
+       contiguous array. */
+   /** If the array data is stored as a contiguous array in host memory, return
+       a pointer to it. Otherwise, copy the data to @a buffer (if @a buffer is
+       not NULL) and return @a buffer.
+       @note If not NULL, @a buffer is assumed to be of size greater than or
+       equal to Size(). */
+   virtual void *DoPullData(void *buffer, std::size_t item_size) = 0;
+
+   /** @brief Set all entries of the array to the (single) value pointed to by
+       @a value_ptr. */
+   virtual void DoFill(const void *value_ptr, std::size_t item_size) = 0;
+
+   /** @brief Set all Size() entries of the array from the given contiguous
+       array, @a src_buffer, on the host. */
+   virtual void DoPushData(const void *src_buffer, std::size_t item_size) = 0;
+
+   /// Copy the data from @a src to @a *this.
+   /** Both arrays must have the same dynamic type, layout, and item_size. */
+   virtual void DoAssign(const PArray &src, std::size_t item_size) = 0;
+
+   /// TODO
+   virtual void DoMakeRefOffset(const PArray &src,
+                                const std::size_t offset,
+                                const std::size_t size,
+                                const std::size_t item_size) = 0;
+
+   ///@}
+   // End: Virtual interface
+
+public:
+   /** @brief The @a layout parameter will be reference counted and therefore it
+       should be dynamically allocated. */
+   /** The @a layout must be valid in the sense that layout != NULL and
+       layout->HasEngine() == true. */
+   PArray(PLayout &p_layout)
+      : layout(&p_layout)
+   {
+      MFEM_ASSERT(layout && layout->HasEngine(), "invalid layout");
+   }
+
+   virtual ~PArray() { }
+
+   /// Get the current size of the array.
+   std::size_t Size() const { return layout->Size(); }
+
+   /// Get the current layout of the array.
+   PLayout &GetLayout() const { return *layout; }
+
+   /// TODO
+   /// Note: we cannot use static_cast for class PArray.
+   template <typename derived_t>
+   derived_t &As() { return dynamic_cast<derived_t&>(*this); }
+
+   /// TODO
+   /// Note: we cannot use static_cast for class PArray.
+   template <typename derived_t>
+   const derived_t &As() const { return dynamic_cast<const derived_t&>(*this); }
+
+
+   // TODO: Error handling ... handle errors at the Engine level, at the class
+   //       level, or at the method level?
+
+   // TODO: Asynchronous execution interface ...
+
+
+   /**
+       @name Public virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new array (in @a *clone) of the same dynamic
+       type as this array using the same layout and ItemSize().
+
+       Set @a *clone to NULL if allocation fails.
+
+       If @a copy_data is true, the contents of this array is copied to the new
+       array; otherwise, the new array remains uninitialized.
+
+       If @a buffer is not NULL, return the array data of the newly created
+       object (in @a *buffer) , if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   template <typename T>
+   DArray Clone(bool copy_data, T **buffer) const
+   { return DArray(DoClone(copy_data, (void**)buffer, sizeof(T))); }
+
+   /// Resize the array, reallocating its data if necessary.
+   /** If @a buffer is not NULL, return the array data (in @a *buffer), if it
+       is stored as a contiguous array on the host; otherwise, set @a *buffer to
+       NULL. Returns 0 on success and non-zero otherwise, e.g. if memory
+       allocation fails.
+
+       If the @a new_layout is not supported, a non-zero error code will be
+       returned.
+
+       The @a new_layout has to be valid, i.e. new_layout != NULL and
+       new_layout->HasEngine() == true.
+
+       @note If reallocation is performed, the previous content of the array is
+       NOT copied to the new location. */
+   template <typename T>
+   int Resize(PLayout &new_layout, T **buffer)
+   { return DoResize(new_layout, (void**)buffer, sizeof(T)); }
+
+   /// Shortcut for Resize(*layout, buffer).
+   /** This method is useful for updating the array after its layout is changed
+       externally. */
+   template <typename T>
+   int Update(T **buffer)
+   { return DoResize(*layout, (void**)buffer, sizeof(T)); }
+
+   /// Shortcut for layout->Resize(new_size) followed by Update()
+   template <typename T>
+   int Resize(std::size_t new_size, T **buffer)
+   { layout->Resize(new_size); return Update(buffer); }
+
+   /** @brief Get access to the contents of the array in host memory, as a
+       contiguous array. */
+   /** If the array data is stored as a contiguous array in host memory, return
+       a pointer to it. Otherwise, copy the data to @a buffer (if @a buffer is
+       not NULL) and return @a buffer.
+       @note If not NULL, @a buffer is assumed to be of size greater than or
+       equal to Size(). */
+   template <typename T>
+   T *PullData(T *buffer)
+   { return Size() ? (T*)DoPullData((void*)buffer, sizeof(T)) : NULL; }
+
+   /** @brief Set all entries of the array to the (single) value pointed to by
+       @a value_ptr. */
+   template <typename T>
+   void Fill(const T &value) { if (Size()) { DoFill(&value, sizeof(T)); } }
+
+   /** @brief Set all Size() entries of the array from the given contiguous
+       array, @a src_buffer, on the host. */
+   template <typename T>
+   void PushData(const T *src_buffer)
+   { if (Size()) { DoPushData(src_buffer, sizeof(T)); } }
+
+   /// Copy the data from @a src to @a *this.
+   /** Both arrays must have the same dynamic type, layout, and entry type. */
+   template <typename T>
+   void Assign(const PArray &src) { if (Size()) { DoAssign(src, sizeof(T)); } }
+
+   /// TODO
+   template <typename T>
+   void MakeRefOffset(const PArray &src,
+                      const std::size_t offset,
+                      const std::size_t size)
+   {
+      DoMakeRefOffset(src, offset, size, sizeof(T));
+   }
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_ARRAY_HPP

backends/base/backend.hpp

@@ -0,0 +1,57 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_BACKEND_HPP
+#define MFEM_BACKENDS_BASE_BACKEND_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "memory_resource.hpp"
+#include "engine.hpp"
+#include "array.hpp"
+#include "vector.hpp"
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+
+#include <string>
+
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#endif
+
+namespace mfem
+{
+
+/// TODO
+class Backend
+{
+public:
+   /// TODO
+   virtual ~Backend() { }
+
+   /// TODO
+   virtual bool Supports(const std::string &engine_spec) const = 0;
+
+   /// TODO
+   virtual Engine *Create(const std::string &engine_spec) = 0;
+
+#ifdef MFEM_USE_MPI
+   /// TODO
+   virtual Engine *Create(MPI_Comm comm, const std::string &engine_spec) = 0;
+#endif
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_BACKEND_HPP

backends/base/bilinearform.hpp

@@ -0,0 +1,72 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_BILINEARFORM_HPP
+#define MFEM_BACKENDS_BASE_BILINEARFORM_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "engine.hpp"
+
+namespace mfem
+{
+
+class Vector;
+class OperatorHandle;
+class BilinearForm;
+
+/// TODO: doxygen
+class PBilinearForm : public RefCounted
+{
+protected:
+   /// Engine with shared ownership
+   SharedPtr<const Engine> engine;
+   /// Not owned.
+   BilinearForm *bform;
+
+public:
+   /// TODO: doxygen
+   PBilinearForm(const Engine &e, BilinearForm &bf)
+      : engine(&e), bform(&bf) { }
+
+   /// Virtual destructor
+   virtual ~PBilinearForm() { }
+
+   /// Get the associated Engine
+   const Engine &GetEngine() const { return *engine; }
+
+   /// Assemble the PBilinearForm.
+   /** This method is called from the method BilinearForm::Assemble() of the
+       associated BilinearForm #bform.
+       @returns True, if the host assembly should be skipped. */
+   virtual bool Assemble() = 0;
+
+   /// TODO: doxygen
+   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                 OperatorHandle &A) = 0;
+
+   /// TODO: doxygen
+   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
+                                 Vector &x, Vector &b,
+                                 OperatorHandle &A, Vector &X, Vector &B,
+                                 int copy_interior) = 0;
+
+   /// TODO: doxygen
+   virtual void RecoverFEMSolution(const Vector &X, const Vector &b,
+                                   Vector &x) = 0;
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_BILINEARFORM_HPP

backends/base/engine.cpp

@@ -0,0 +1,49 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "engine.hpp"
+#include "fespace.hpp"
+#include "bilinearform.hpp"
+
+namespace mfem
+{
+
+Engine::Engine(Backend *b, int n_mem, int n_workers)
+   : backend(b),
+#ifdef MFEM_USE_MPI
+     comm(MPI_COMM_NULL),
+#endif
+     num_mem_res(n_mem),
+     num_workers(n_workers),
+     memory_resources(new MemoryResource*[num_mem_res]()),
+     workers_weights(new double[num_workers]()),
+     workers_mem_res(new int[num_workers]())
+{
+   // Note: all arrays are value-initialized with zeros.
+}
+
+Engine::~Engine()
+{
+   delete [] workers_mem_res;
+   delete [] workers_weights;
+   for (int i = 0; i < num_mem_res; i++)
+   {
+      delete memory_resources[i];
+   }
+   delete [] memory_resources;
+}
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS

backends/base/engine.hpp

@@ -0,0 +1,190 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_ENGINE_HPP
+#define MFEM_BACKENDS_BASE_ENGINE_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "../../general/scalars.hpp"
+#include "memory_resource.hpp"
+#include "smart_pointers.hpp"
+#include "utils.hpp"
+
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#endif
+
+namespace mfem
+{
+
+// Forward declarations.
+class Backend;
+template <typename T> class Array;
+class Operator;
+class FiniteElementSpace;
+class LinearForm;
+class BilinearForm;
+class MixedBilinearForm;
+class NonlinearForm;
+
+
+/// In parallel, each MPI rank will usually create a single engine.
+class Engine : public RefCounted
+{
+protected:
+   Backend *backend; ///< Backend that created the engine. Not owned.
+
+#ifdef MFEM_USE_MPI
+   MPI_Comm comm; ///< Associated MPI communicator (may be MPI_COMM_NULL).
+#endif
+
+   /// Number of memory resources used by the Engine.
+   int num_mem_res;
+   /// Number of workers used by the Engine.
+   int num_workers;
+
+   /// Memory resources used by the engine - array of pointers.
+   /** Both the array and the entries are owned. */
+   MemoryResource **memory_resources;
+
+   /// Relative computational speed of the workers. Owned.
+   double *workers_weights;
+
+   /// For each worker, which memory resource it uses.
+   int *workers_mem_res;
+
+public:
+   /// TODO: doxygen
+   Engine(Backend *b, int n_mem, int n_workers);
+
+   /// TODO: doxygen
+   virtual ~Engine();
+
+
+   /**
+       @name Machine resources interface
+    */
+   ///@{
+
+#ifdef MFEM_USE_MPI
+   /// Get the associated MPI_Comm
+   MPI_Comm GetComm() const { return comm; }
+#endif
+
+   /// TODO
+   int GetNumMemRes() const { return num_mem_res; }
+
+   /// TODO
+   MemoryResource &GetMemRes(int idx) const { return *memory_resources[idx]; }
+
+   /// TODO
+   int GetNumWorkers() const { return num_workers; }
+
+   /// TODO
+   const double *GetWorkersWeights() const { return workers_weights; }
+
+   /// TODO
+   const int *GetWorkersMemRes() const { return workers_mem_res; }
+
+   ///@}
+   // End: Machine resources interface
+
+   /// TODO
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   /// TODO
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+
+
+   // TODO: Error handling ... handle errors at the Engine level, at the class
+   //       level, or at the method level?
+
+
+   /**
+       @name Virtual interface: finite element data structures and algorithms
+    */
+   ///@{
+
+   // TODO: Asynchronous execution in this class ...
+
+   /// Allocate and return a new layout for the given @a size.
+   /** The layout decomposition (in the case of multiple workers) is determined
+       automatically by the Engine using a deterministic algorithm: calls to
+       this method with the same @a size will produce the same result, as long
+       as the Engine remains unmodified between the calls.
+
+       The returned object is allocated with operator new and must be
+       deallocated by the caller.
+
+       TODO: Returns NULL if memory allocation fails?
+   */
+   virtual DLayout MakeLayout(std::size_t size) const = 0;
+
+   /// Allocate and return a new layout for the given worker decomposition.
+   /** The returned object is allocated with operator new and must be
+       deallocated by the caller.
+
+       TODO: Returns NULL if memory allocation fails?
+
+       The @a offsets should satisfy: offsets.Size() == number of workers + 1,
+       offsets[0] == 0, and offsets[i] <= offsets[i+1], for i: 0 <= i < number
+       of workers. */
+   virtual DLayout MakeLayout(const Array<std::size_t> &offsets) const = 0;
+
+   // Note: There may be other ways to construct layouts in the future, e.g.
+   //       block-vector layouts, or multi-vector layouts.
+
+   /// TODO
+   virtual DArray MakeArray(PLayout &layout, std::size_t item_size) const = 0;
+
+   /// Allocate and return a new vector using the given @a layout.
+   /** The returned object is a smart pointer that will automatically deallocate
+       the vector.
+
+       TODO: Produce an error if memory allocation fails?
+
+       Only layouts returned by this Engine are guaranteed to be supported.
+       Using a type that is not supported will produce an error. */
+   virtual DVector MakeVector(PLayout &layout,
+                              int type_id = ScalarId<double>::value) const = 0;
+
+   /// TODO: doxygen
+   virtual DFiniteElementSpace MakeFESpace(FiniteElementSpace &fes) const = 0;
+
+   /// TODO: doxygen
+   virtual DBilinearForm MakeBilinearForm(BilinearForm &bf) const = 0;
+
+
+   // Question: How do we construct coefficients?
+
+
+   /// FIXME - What will the actual parameters be?
+   virtual void AssembleLinearForm(LinearForm &l_form) const = 0;
+
+   /// FIXME - What will the actual parameters be?
+   virtual Operator *MakeOperator(const MixedBilinearForm &mbl_form) const = 0;
+
+   /// FIXME - What will the actual parameters be?
+   virtual Operator *MakeOperator(const NonlinearForm &nl_form) const = 0;
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_ENGINE_HPP

backends/base/fespace.hpp

@@ -0,0 +1,98 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_FE_SPACE_HPP
+#define MFEM_BACKENDS_BASE_FE_SPACE_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "engine.hpp"
+#include "utils.hpp"
+
+namespace mfem
+{
+
+class FiniteElementSpace;
+class QuadratureSpace;
+
+/// TODO: doxygen
+class PFiniteElementSpace : public RefCounted
+{
+protected:
+   /// Engine with shared ownership
+   SharedPtr<const Engine> engine;
+   /// Not owned.
+   mfem::FiniteElementSpace *fes;
+
+public:
+   /// TODO: doxygen
+   PFiniteElementSpace(const Engine &e, mfem::FiniteElementSpace &fespace)
+      : engine(&e), fes(&fespace) { }
+
+   /// Virtual destructor
+   virtual ~PFiniteElementSpace() { }
+
+   /// Get the associated engine
+   const Engine &GetEngine() const { return *engine; }
+
+   /// Return the associated mfem::FiniteElementSpace
+   mfem::FiniteElementSpace *GetFESpace() const { return fes; }
+
+   /// TODO
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   /// TODO
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+
+
+   /**
+       @name Virtual interface: finite element space functionality
+    */
+   ///@{
+
+   /// TODO
+   /** Return the operator mapping T-vectors to L-vectors. If a NULL pointer is
+       returned then the mapping is the idenity. */
+   virtual const mfem::Operator *GetProlongationOperator() const = 0;
+
+   /// TODO
+   /** Return the operator mapping L-vectors to T-vectors that extracts the
+       subset of all true dofs, i.e. no assembly is performed. If a NULL pointer
+       is returned then the mapping is the idenity. */
+   virtual const mfem::Operator *GetRestrictionOperator() const = 0;
+
+   /// TODO
+   /** Return the operator mapping L-vectors to Q-vectors that evaluates the
+       values of a GridFunction as a QuadratureFunction on the given
+       QuadratureSpace. If the returned pointer is NULL, then the mapping is the
+       identity. */
+   virtual const mfem::Operator *GetInterpolationOperator(
+      const mfem::QuadratureSpace &qspace) const = 0;
+
+   /// TODO
+   /** Return the operator mapping L-vectors to Q-vectors that evaluates the
+       _reference element_ gradients of a GridFunction as a QuadratureFunction
+       on the given QuadratureSpace. */
+   virtual const mfem::Operator *GetGradientOperator(
+      const mfem::QuadratureSpace &qspace) const = 0;
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_FE_SPACE_HPP

backends/base/layout.hpp

@@ -0,0 +1,110 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_LAYOUT_HPP
+#define MFEM_BACKENDS_BASE_LAYOUT_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "smart_pointers.hpp"
+#include "engine.hpp"
+
+namespace mfem
+{
+
+/// Polymorphic layout (array/vector layout descriptor)
+class PLayout : public RefCounted
+{
+protected:
+   /// Engine with shared ownership
+   SharedPtr<const Engine> engine;
+   std::size_t size;
+
+   template <typename DObject>
+   struct Maker
+   {
+      template <typename entry_t>
+      static DObject MakeNew(PLayout &layout);
+   };
+
+public:
+   explicit PLayout(std::size_t s = 0) : engine(NULL), size(s) { }
+
+   explicit PLayout(const Engine &e, std::size_t s = 0)
+      : engine(&e), size(s) { }
+
+   virtual ~PLayout() { }
+
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   /// Resize the layout
+   virtual void Resize(std::size_t new_size) { size = new_size; }
+
+   /// Resize the layout based on the given worker offsets
+   virtual void Resize(const Array<std::size_t> &offsets)
+   { MFEM_ABORT("method not supported"); }
+
+   ///@}
+   // End: Virtual interface
+
+   /// Layouts without engine cannot create DArray, DVector, etc.
+   bool HasEngine() const { return engine != NULL; }
+
+   /// TODO: doxygen
+   const Engine &GetEngine() const { return *engine; }
+
+   /// TODO: doxygen
+   std::size_t Size() const { return size; }
+
+   /// TODO
+   /** Useful in backends for down-casting to a backend-specific layout type.
+
+       When MFEM_DEBUG=YES, performs a type check using dynamic_cast. */
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   /// TODO
+   /** Useful in backends for down-casting to a backend-specific layout type.
+
+       When MFEM_DEBUG=YES, performs a type check using dynamic_cast. */
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+
+   /// TODO: doxygen
+   template <typename DObject, typename entry_t>
+   DObject Make()
+   {
+      MFEM_ASSERT(HasEngine(), "this method requires an Engine");
+      return Maker<DObject>::template MakeNew<entry_t>(*this);
+   }
+};
+
+template <> struct PLayout::Maker<DArray>
+{
+   template <typename entry_t> static DArray MakeNew(PLayout &layout)
+   { return layout.GetEngine().MakeArray(layout, sizeof(entry_t)); }
+};
+
+template <> struct PLayout::Maker<DVector>
+{
+   template <typename entry_t> static DVector MakeNew(PLayout &layout)
+   { return layout.GetEngine().MakeVector(layout, ScalarId<entry_t>::value); }
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_LAYOUT_HPP

backends/base/memory_resource.cpp

@@ -0,0 +1,59 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "memory_resource.hpp"
+#include "../../general/error.hpp"
+
+#include <cstdlib>
+#include <cstring>
+#include <cerrno>
+
+namespace mfem
+{
+
+void *NewDeleteMemoryResource::DoAllocate(std::size_t bytes,
+                                          std::size_t alignment)
+{
+   void *p = ::operator new[](bytes);
+   MFEM_VERIFY(!alignment || (std::size_t)(p) % alignment == 0,
+               "invalid alignment");
+   return p;
+}
+
+void NewDeleteMemoryResource::DoDeallocate(void *p, std::size_t bytes,
+                                           std::size_t alignment)
+{
+   ::operator delete[](p);
+}
+
+
+void *AlignedMemoryResource::DoAllocate(std::size_t bytes,
+                                        std::size_t alignment)
+{
+   void *p;
+   if (!alignment) { alignment = sizeof(long double); }
+   MFEM_VERIFY(posix_memalign(&p, alignment, bytes) == 0,
+               "error in posix_memalign(): " << strerror(errno));
+   return p;
+}
+
+void AlignedMemoryResource::DoDeallocate(void *p, std::size_t bytes,
+                                         std::size_t alignment)
+{
+   free(p);
+}
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS

backends/base/memory_resource.hpp

@@ -0,0 +1,70 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP
+#define MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include <cstddef>
+
+namespace mfem
+{
+
+/// Polymorphic memory resource. Similar to C++17's std::pmr::memory_resource.
+class MemoryResource
+{
+protected:
+   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment) = 0;
+   virtual void DoDeallocate(void* p, std::size_t bytes,
+                             std::size_t alignment) = 0;
+
+public:
+   // Implicitly defined default & copy constructors
+
+   /// Virtual destructor.
+   virtual ~MemoryResource() { }
+
+   /// If alignment == 0, use default alignment.
+   void *Allocate(std::size_t bytes, std::size_t alignment = 0)
+   { return DoAllocate(bytes, alignment); }
+
+   /// If alignment == 0, use default alignment.
+   void Deallocate(void *p, std::size_t bytes, std::size_t alignment = 0)
+   { DoDeallocate(p, bytes, alignment); }
+};
+
+
+/** @brief Dynamic host memory resource using operator new[](std::size_t) for
+    allocation and operator delete[](void*) for deallocation. */
+class NewDeleteMemoryResource : public MemoryResource
+{
+protected:
+   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment);
+   virtual void DoDeallocate(void *p, std::size_t bytes, std::size_t alignment);
+};
+
+
+/** @brief Dynamic host memory resource using posix_memalign() for aligned
+    allocation and free() for deallocation. */
+class AlignedMemoryResource : public MemoryResource
+{
+protected:
+   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment);
+   virtual void DoDeallocate(void *p, std::size_t bytes, std::size_t alignment);
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP

backends/base/smart_pointers.hpp

@@ -0,0 +1,234 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_SMART_POINTERS_HPP
+#define MFEM_BACKENDS_BASE_SMART_POINTERS_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "utils.hpp"
+#include "../../general/error.hpp"
+#include <cstddef>
+
+// #define MFEM_TRACE_SHARED_PTR
+#ifdef MFEM_TRACE_SHARED_PTR
+#include "../../general/globals.hpp"
+#endif
+
+namespace mfem
+{
+
+/// Base class for classes with simple reference counting.
+/** Reference counting is performed by the class SharedPtr. */
+class RefCounted
+{
+private:
+   mutable unsigned ref_count;
+
+   /// Only class SharedPtr can access ref_count.
+   template <typename T> friend class SharedPtr;
+
+public:
+   RefCounted() : ref_count(0) { }
+
+   /** @brief Prevent SharedPtr objects from deleting this object by
+       incrementing the reference counter by one. */
+   void DontDelete() const { ++ref_count; }
+};
+
+
+/** @brief Smart pointer class that manages objects of type T derived from class
+    RefCounted. */
+/** This class is generally meant to work with dynamically allocated object,
+    specifically objects allocated with operator new(). It will invoke operator
+    delete() to destroy the managed object when its reference counter reaches
+    zero. This behavior can be overriden by calling RefCounted::DontDelete() to
+    ensure that an object will not be deleted by a SharedPtr that holds a
+    pointer to it.
+    @note This class is NOT thread-safe and does not support circular ownership.
+ */
+template <typename T>
+class SharedPtr
+{
+public:
+   typedef T stored_type;
+
+private:
+   T *ptr;
+
+   void Init(T *new_ptr)
+   {
+      ptr = new_ptr;
+      if (ptr) { ++ptr->RefCounted::ref_count; }
+#ifdef MFEM_TRACE_SHARED_PTR
+#elif 0
+      mfem::out << "  [" << _MFEM_FUNC_NAME << "]: ptr = " << ptr;
+      if (ptr)
+      {
+         mfem::out << ", new ref_count = " << ptr->RefCounted::ref_count;
+      }
+      mfem::out << '\n';
+#endif
+   }
+   void Destroy()
+   {
+      MFEM_ASSERT(!ptr || ptr->RefCounted::ref_count >= 1, "invalid use");
+      if (ptr && --ptr->RefCounted::ref_count == 0) { delete ptr; }
+#ifdef MFEM_TRACE_SHARED_PTR
+#elif 0
+      mfem::out << "  [" << _MFEM_FUNC_NAME << "]: ptr = " << ptr;
+      if (ptr)
+      {
+         mfem::out << ", new ref_count = " << ptr->RefCounted::ref_count;
+      }
+      mfem::out << '\n';
+#endif
+   }
+
+public:
+   SharedPtr() : ptr(NULL)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]: ptr = " << ptr << '\n';
+#endif
+   }
+
+   SharedPtr(const SharedPtr &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Init(other.ptr);
+   }
+
+   template <typename U>
+   SharedPtr(const SharedPtr<U> &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Init(other.Get());
+   }
+
+   explicit SharedPtr(T *p)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Init(p);
+   }
+
+   ~SharedPtr()
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Destroy();
+   }
+
+   SharedPtr &operator=(const SharedPtr &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Reset(other.ptr); return *this;
+   }
+
+   template <typename U>
+   SharedPtr &operator=(const SharedPtr<U> &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Reset(other.Get()); return *this;
+   }
+
+   T &operator*() const { return *ptr; }
+   T *operator->() const { return ptr; }
+
+   operator bool() const { return ptr; }
+   bool operator!() const { return !ptr; }
+
+   template <typename U>
+   bool operator==(const SharedPtr<U> &other) const
+   { return ptr == other.Get(); }
+   template <typename U>
+   bool operator!=(const SharedPtr<U> &other) const
+   { return ptr != other.Get(); }
+
+   // Comparison to any type convertible to void *, e.g. the type of NULL.
+   template <typename U>
+   bool operator==(const U &p) const { return ptr == (void*) p; }
+   template <typename U>
+   bool operator!=(const U &p) const { return ptr != (void*) p; }
+
+   T *Get() const { return ptr; }
+
+   /// TODO
+   template <typename derived_t>
+   derived_t *As() const { return util::As<derived_t>(ptr); }
+
+   unsigned UseCount() const { return ptr ? ptr->RefCounted::ref_count : 0; }
+
+   void Reset()
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      Destroy();
+      ptr = NULL;
+   }
+
+   /// The type U* needs to be implicitly convertible to T*
+   template <typename U>
+   void Reset(U *new_ptr)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      if (ptr != new_ptr) { Destroy(); Init(new_ptr); }
+   }
+
+   void Swap(SharedPtr &other)
+   {
+#ifdef MFEM_TRACE_SHARED_PTR
+      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
+#endif
+      std::swap(ptr, other.ptr);
+   }
+};
+
+
+template <class T>
+inline void Swap(SharedPtr<T> &a, SharedPtr<T> &b) { a.Swap(b); }
+
+
+class PLayout;
+typedef SharedPtr<PLayout> DLayout;
+
+class PArray;
+typedef SharedPtr<PArray> DArray;
+
+class PVector;
+typedef SharedPtr<PVector> DVector;
+
+class PFiniteElementSpace;
+typedef SharedPtr<PFiniteElementSpace> DFiniteElementSpace;
+
+class PBilinearForm;
+typedef SharedPtr<PBilinearForm> DBilinearForm;
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_SMART_POINTERS_HPP

backends/base/utils.hpp

@@ -0,0 +1,52 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_UTILS_HPP
+#define MFEM_BACKENDS_BASE_UTILS_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "../../general/error.hpp"
+
+namespace mfem
+{
+
+namespace util
+{
+
+//
+// Inline methods
+//
+
+/// TODO: doxygen
+template <typename derived_t, typename base_t>
+inline derived_t *As(base_t *base_obj)
+{
+   MFEM_ASSERT(dynamic_cast<derived_t*>(base_obj) != NULL,
+               "invalid object type");
+   return static_cast<derived_t*>(base_obj);
+}
+
+/// TODO: doxygen
+template <typename derived_t, typename base_t>
+inline derived_t *Is(base_t *base_obj)
+{
+   return dynamic_cast<derived_t*>(base_obj);
+}
+
+} // namespace mfem::util
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_UTILS_HPP

backends/base/vector.hpp

@@ -0,0 +1,153 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_BASE_VECTOR_HPP
+#define MFEM_BACKENDS_BASE_VECTOR_HPP
+
+#include "../../config/config.hpp"
+#ifdef MFEM_USE_BACKENDS
+
+#include "../../general/scalars.hpp"
+#include "array.hpp"
+
+namespace mfem
+{
+
+/// Polymorphic vector - array of scalars.
+class PVector : virtual public PArray
+{
+protected:
+   /**
+       @name Virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new vector of the same dynamic type as this
+       vector using the same layout with entries specified by @a buffer_type_id
+       which should be a constant defined by the `value` field in a
+       specialization of the template class mfem::ScalarId.
+
+       Returns NULL if allocation fails.
+
+       If @a copy_data is true, the contents of this vector is copied to the new
+       vector; otherwise, the new vector remains uninitialized.
+
+       If @a buffer is not NULL, return the vector data of the newly created
+       object (in @a *buffer), if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   virtual PVector *DoVectorClone(bool copy_data, void **buffer,
+                                  int buffer_type_id) const = 0;
+
+   /** @brief Compute and return the dot product of @a *this and @a x. In the
+       case of an MPI-parallel vector, the result must be the MPI-global dot
+       product. */
+   /** Both vectors must have the same dynamic type and layout. */
+   virtual void DoDotProduct(const PVector &x, void *result,
+                             int result_type_id) const = 0;
+
+   // TODO: add reduction operations: min, max, sum
+
+   /// Perform the operation @a *this = @a a @a x + @a b @a y.
+   /** Rules:
+       - the dynamic type of both @a x and @a y is the same as that of @a *this
+       - if @a a == 0, neither @a x nor its data are accessed
+       - if @a b == 0, neither @a y nor its data are accessed
+       - @a x's data is never the same as @a y's data, unless @a a == 0, or
+         @a b == 0
+       - @a x's data or @a y's data may be the same as the data of @a *this
+       - all accessed vectors, @a x, @a y, and @a *this have the same layout. */
+   virtual void DoAxpby(const void *a, const PVector &x,
+                        const void *b, const PVector &y,
+                        int ab_type_id) = 0;
+
+   ///@}
+   // End: Virtual interface
+
+public:
+   /** @brief Create a PVector. */
+   /** The @a layout must be valid in the sense that layout != NULL and
+       layout->HasEngine() == true. */
+   PVector(PLayout &p_layout)
+      : PArray(p_layout) { }
+
+   template <typename derived_t>
+   derived_t &As() { return *util::As<derived_t>(this); }
+
+   template <typename derived_t>
+   const derived_t &As() const { return *util::As<const derived_t>(this); }
+
+
+   // TODO: Error handling ... handle errors at the Engine level, at the class
+   //       level, or at the method level?
+
+   // TODO: Asynchronous execution interface ...
+
+   // TODO: Multi-vector interface ...
+
+
+   /**
+       @name Public virtual interface
+    */
+   ///@{
+
+   /** @brief Create and return a new vector of the same dynamic type as this
+       vector using the same layout with entries of type @a scalar_t.
+
+       If @a copy_data is true, the contents of this vector is copied to the new
+       vector; otherwise, the new vector remains uninitialized.
+
+       If @a buffer is not NULL, return the vector data of the newly created
+       object (in @a *buffer) , if it is stored as a contiguous array on the
+       host; otherwise, set @a *buffer to NULL. */
+   template <typename scalar_t>
+   DVector Clone(bool copy_data, scalar_t **buffer) const
+   {
+      return DVector(DoVectorClone(copy_data, (void**)buffer,
+                                   ScalarId<scalar_t>::value));
+   }
+
+   /** @brief Compute and return the dot product of @a *this and @a x. In the
+       case of an MPI-parallel vector, the result must be the MPI-global dot
+       product. */
+   /** Both vectors must have the same dynamic type and layout. */
+   template <typename scalar_t>
+   scalar_t DotProduct(const PVector &x) const
+   {
+      scalar_t result;
+      DoDotProduct(x, &result, ScalarId<scalar_t>::value);
+      return result;
+   }
+
+   // TODO: add reduction operations: min, max, sum
+
+   /// Perform the operation @a *this = @a a @a x + @a b @a y.
+   /** Rules:
+       - the dynamic type of both @a x and @a y is the same as that of @a *this
+       - if @a a == 0, neither @a x nor its data are accessed
+       - if @a b == 0, neither @a y nor its data are accessed
+       - @a x's data is never the same as @a y's data, unless @a a == 0, or
+         @a b == 0
+       - @a x's data or @a y's data may be the same as the data of @a *this
+       - all accessed vectors, @a x, @a y, and @a *this have the same layout. */
+   template <typename scalar_t>
+   void Axpby(const scalar_t &a, const PVector &x,
+              const scalar_t &b, const PVector &y)
+   { if (Size()) { DoAxpby(&a, x, &b, y, ScalarId<scalar_t>::value); } }
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_BACKENDS
+
+#endif // MFEM_BACKENDS_BASE_VECTOR_HPP

backends/kernels/config/config.cpp

@@ -0,0 +1,264 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+
+// ***************************************************************************
+bool isNvidiaCudaMpsDaemonRunning(void)
+{
+   const char *command="pidof -s nvidia-cuda-mps-control>/dev/null";
+   return system(command)==0;
+}
+
+#ifdef __NVCC__
+// ***************************************************************************
+void computeCapabilityOfTheDevice(const int mpi_rank,
+                                  const CUdevice cuDevice,
+                                  const int device)
+{
+   char name[128];
+   int major, minor;
+   cuDeviceGetName(name, 128, cuDevice);
+   cuDeviceComputeCapability(&major, &minor, device);
+   dbg("\n\033[32m[computeCapabilityOfTheDevice] Rank_%d => Device_%d (%s:sm_%d.%d)\033[m\n",
+       mpi_rank, device, name, major, minor);
+}
+#endif
+
+// ***************************************************************************
+static bool isTux(void)
+{
+   char hostname[1024];
+   hostname[1023] = '\0';
+   gethostname(hostname, 1023);
+   if (strncmp("tux", hostname, 3)==0) { return true; }
+   return false;
+}
+
+// ***************************************************************************
+#ifdef __NVCC__
+__attribute__((unused))
+static void printDevProp(cudaDeviceProp devProp)
+{
+   printf("Major revision number:         %d\n",  devProp.major);
+   printf("Minor revision number:         %d\n",  devProp.minor);
+   printf("Name:                          %s\n",  devProp.name);
+   printf("Total global memory:           %u\n",  devProp.totalGlobalMem);
+   printf("Total shared memory per block: %u\n",  devProp.sharedMemPerBlock);
+   printf("Total registers per block:     %d\n",  devProp.regsPerBlock);
+   printf("Warp size:                     %d\n",  devProp.warpSize);
+   printf("Maximum memory pitch:          %u\n",  devProp.memPitch);
+   printf("Maximum threads per block:     %d\n",  devProp.maxThreadsPerBlock);
+   for (int i = 0; i < 3; ++i)
+   {
+      printf("Maximum dimension %d of block:  %d\n", i, devProp.maxThreadsDim[i]);
+   }
+   for (int i = 0; i < 3; ++i)
+   {
+      printf("Maximum dimension %d of grid:   %d\n", i, devProp.maxGridSize[i]);
+   }
+   printf("Clock rate:                    %d\n",  devProp.clockRate);
+   printf("Total constant memory:         %u\n",  devProp.totalConstMem);
+   printf("Texture alignment:             %u\n",  devProp.textureAlignment);
+   printf("Concurrent copy and execution: %s\n",
+          (devProp.deviceOverlap ? "Yes" : "No"));
+   printf("Number of multiprocessors:     %d\n",  devProp.multiProcessorCount);
+   printf("Kernel execution timeout:      %s\n",
+          (devProp.kernelExecTimeoutEnabled ? "Yes" : "No"));
+}
+#endif
+
+// ***************************************************************************
+// *   Setup
+// ***************************************************************************
+void config::Setup(const int _mpi_rank,
+                   const int _mpi_size,
+                   const bool _cuda,
+                   const bool _dcg,
+                   const bool _uvm,
+                   const bool _aware,
+                   const bool _share,
+                   const bool _occa,
+                   const bool _hcpo,
+                   const bool _sync,
+                   const bool _dot,
+                   const int rs_levels)
+{
+   mpi_rank=_mpi_rank;
+   mpi_size=_mpi_size;
+
+   // Look if we are on a Tux machine
+   const bool tux = isTux();
+   if (tux && Root())
+   {
+      printf("\033[32m[Setup] \033[1mTux\033[m\n");
+   }
+
+   // On Tux machines, use the MPIX_Query_cuda_support
+   // Otherwise, assume there is a support
+   //aware = tux?(MPIX_Query_cuda_support()==1)?true:false:true;
+
+   // On Tux machines, look for MPS
+   mps = tux?isNvidiaCudaMpsDaemonRunning():false;
+   if (tux && Mps() && Root())
+   {
+      printf("\033[32m[Setup] \033[32;1mMPS daemon\033[m\033[m\n");
+   }
+   if (tux && !Mps() && Root())
+   {
+      printf("\033[32m[Setup] \033[31;1mNo MPS daemon\033[m\n");
+   }
+
+#ifdef __NVCC__
+   // Get the number of devices with compute capability greater or equal to 2.0
+   // Can be changed wuth CUDA_VISIBLE_DEVICES
+   cuCheck(cudaGetDeviceCount(&gpu_count));
+#endif
+
+   cuda=_cuda;
+   if (Root())
+   {
+      if (Cuda())
+      {
+         printf("\033[32m[Setup] \033[1mCUDA Kernels\033[m\n");
+      }
+      else
+      {
+         printf("\033[32m[Setup] \033[31;1mCPU Kernels\033[m\n");
+      }
+   }
+
+   dcg=_dcg; // CG on device
+   uvm=_uvm;
+   aware=_aware;
+   share=_share;
+   share_env=getenv("SHR");
+   occa=_occa;
+   hcpo=_hcpo;
+   sync=_sync;
+
+#if defined(__NVCC__)
+
+   // __NVVP__ warning output
+#if defined(__NVVP__)
+   if (Root())
+   {
+      printf("\033[32m[Setup] \033[31;1m__NVVP__\033[m\n");
+   }
+#endif // __NVVP__
+
+   // MFEM_DEBUG warning output
+#if defined(MFEM_DEBUG)
+   if (Root())
+   {
+      printf("\033[32m[Setup] \033[31;1mMFEM_DEBUG\033[m\n");
+   }
+#endif
+
+   // Check for Enforced Kernel Synchronization
+   if (Sync() && Root())
+   {
+      printf("\033[32m[Setup] \033[31;1mEnforced Kernel Synchronization!\033[m\n");
+   }
+
+   // Check if MPI is CUDA aware
+   if (Root())
+      printf("\033[32m[Setup] MPI %s CUDA aware\033[m\n",
+             aware?"\033[1mIS":"is \033[31;1mNOT\033[32m");
+
+   if (Root())
+   {
+      printf("\033[32m[Setup] CUDA device count: %i\033[m\n", gpu_count);
+   }
+
+   // Initializes the driver API
+   // Must be called before any other function from the driver API
+   // Currently, the Flags parameter must be 0.
+   const unsigned int Flags = 0; // parameter must be 0
+   cuInit(Flags);
+
+   // Returns properties for the selected device
+   const int device = Mps()?0:(mpi_rank%gpu_count);
+   // Check if we have enough devices for all ranks
+   assert(device<gpu_count);
+
+   // Get a handle to our compute device
+   cuDeviceGet(&cuDevice,device);
+   computeCapabilityOfTheDevice(mpi_rank,cuDevice,device);
+
+   // Get the properties of the device
+   struct cudaDeviceProp properties;
+   cudaGetDeviceProperties(&properties, device);
+#if defined(MFEM_DEBUG)
+   if (Root())
+   {
+      printDevProp(properties);
+   }
+#endif // MFEM_DEBUG
+   maxXGridSize=properties.maxGridSize[0];
+   maxXThreadsDim=properties.maxThreadsDim[0];
+   //printf("\033[32m[Setup] maxXGridSize: %d\033[m\n", maxXGridSize);
+   //printf("\033[32m[Setup] maxXThreadsDim: %d\033[m\n", maxXThreadsDim);
+
+   // Create our context
+   cuCtxCreate(&cuContext, CU_CTX_SCHED_AUTO, cuDevice);
+   hStream=new CUstream;
+   cuStreamCreate(hStream, CU_STREAM_DEFAULT);
+#endif // __NVCC__
+   if (_dot)
+   {
+      assert(false);
+      exit(0);
+   }
+}
+
+// ***************************************************************************
+bool config::IAmAlone()
+{
+   if (Occa()) { return false; }
+   return mpi_size==1;
+}
+
+// ***************************************************************************
+bool config::GeomNeedsUpdate(const int sequence)
+{
+   if (Occa()) { return true; }
+   assert(sequence==0);
+   return (sequence!=0);
+}
+
+// ***************************************************************************
+bool config::DoHostConformingProlongationOperator()
+{
+   if (Occa()) { return true; }
+   return (Cuda())?hcpo:true;
+}
+
+} // kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/config/config.hpp

@@ -0,0 +1,125 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+
+#ifndef MFEM_BACKENDS_KERNELS_CONFIG_HPP
+#define MFEM_BACKENDS_KERNELS_CONFIG_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#ifdef __NVCC__
+#include <cuda.h>
+#endif // __NVCC__
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+#ifndef __NVCC__
+typedef int CUstream;
+#endif
+
+// ***************************************************************************
+// * Configuration class for KERNELS
+// ***************************************************************************
+class config
+{
+private:
+   // *************************************************************************
+   int mpi_rank=0;
+   int mpi_size=0;
+   bool aware=false;
+   //  ************************************************************************
+   bool mps=false;
+   int gpu_count=0;
+   int maxXGridSize=0;
+   int maxXThreadsDim=0;
+#ifdef __NVCC__
+   CUdevice cuDevice;
+   CUcontext cuContext;
+   CUstream *hStream;
+#endif
+   // *************************************************************************
+   bool cuda=false;
+   bool dcg=false;
+   bool uvm=false;
+   bool share=false;
+   bool share_env=false;
+   // *************************************************************************
+   bool occa=false;
+   bool hcpo=false;
+   bool sync=false;
+   bool nvvp=false;
+   // *************************************************************************
+private:
+   config()
+#ifdef __NVCC__
+      :cuda(true)
+#endif
+   {}
+   config(config const&);
+   void operator=(config const&);
+   // *************************************************************************
+public:
+   static config& Get()
+   {
+      static config config_singleton;
+      return config_singleton;
+   }
+   // *************************************************************************
+   void Setup(const int,const int,
+              const bool cuda, const bool dcg,
+              const bool uvm, const bool aware,
+              const bool share, const bool occa, const bool hcpo,
+              const bool sync, const bool dot, const int rs_levels);
+   // *************************************************************************
+   bool IAmAlone();
+   bool GeomNeedsUpdate(const int);
+   bool DoHostConformingProlongationOperator();
+   // *************************************************************************
+   inline int Rank() { return mpi_rank; }
+   inline int Size() { return mpi_size; }
+   inline bool Root() { return mpi_rank==0; }
+   inline bool Aware() { return aware; }
+   // *************************************************************************
+   inline bool Mps() { return mps; }
+   // *************************************************************************
+   inline bool Uvm() { return uvm; }
+   inline bool Cuda() { return cuda; }
+   inline bool Dcg() { return dcg; }
+   inline bool Share() { return share && !share_env; }
+   inline bool ShareEnv() { return share_env; }
+   inline bool Occa() { return occa; }
+   inline bool Hcpo() { return hcpo; }
+   inline bool Sync() { return sync; }
+   inline bool Nvvp(bool toggle=false) { return toggle?nvvp=!nvvp:nvvp; }
+   inline int MaxXGridSize() { return maxXGridSize; }
+   inline int MaxXThreadsDim() { return maxXThreadsDim; }
+   // *************************************************************************
+#ifdef __NVCC__
+   inline CUstream *Stream() { return hStream; }
+#endif
+};
+
+} // kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_CONFIG_HPP

backends/kernels/config/dbg.cpp

@@ -0,0 +1,95 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+// DBG *************************************************************************
+void kdbg(const char *format,...)
+{
+   va_list args;
+   va_start(args, format);
+   fflush(stdout);
+   vfprintf(stdout,format,args);
+   fflush(stdout);
+   va_end(args);
+}
+
+// *****************************************************************************
+const char *strrnchr(const char *s, const unsigned char c, int n)
+{
+   size_t len = strlen(s);
+   char *p = (char*)s+len-1;
+   for (; n; n--,p--,len--)
+   {
+      for (; len; p--,len--)
+         if (*p==c) { break; }
+      if (!len) { return NULL; }
+      if (n==1) { return p; }
+   }
+   return NULL;
+}
+
+//*****************************************************************************
+uint8_t chk8(const char *bfr)
+{
+   unsigned int chk = 0;
+   size_t len = strlen(bfr);
+   for (; len; len--,bfr++)
+   {
+      chk += *bfr;
+   }
+   return (uint8_t) chk;
+}
+
+// *****************************************************************************
+// * file: __FILENAME__, line: __LINE__, func: __FUNCTION__
+// * header will display the file, line & func
+// * nargs: number of arguments that follow
+// *****************************************************************************
+void kdbge(const char *file, const int line, const char *func,
+           const bool header, const int nargs, ...)
+{
+   static bool env_ini = false;
+   static bool env_dbg = false;
+   if (!env_ini) { env_dbg = getenv("DBG"); env_ini = true; }
+   if (!env_dbg) { return; }
+   assert(nargs>0);
+   const uint8_t color = 17 + chk8(file)%216;
+   fflush(stdout);
+   fprintf(stdout,"\033[38;5;%dm",color);
+   if (header)
+   {
+      fprintf(stdout,"\n%24s\b\b\b\b:\033[2m%3d\033[22m: %s: \033[1m",
+              file, line, func);
+   }
+   else
+   {
+      fprintf(stdout,"\033[1m");
+   }
+   va_list args;
+   va_start(args,nargs);
+   const char *format=va_arg(args,const char*);
+   assert(format);
+   vfprintf(stdout,format,args);
+   va_end(args);
+   fprintf(stdout,"\033[m");
+   fflush(stdout);
+}
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/config/dbg.hpp

@@ -0,0 +1,47 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_DBG_HPP
+#define MFEM_BACKENDS_KERNELS_DBG_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+// ****************************************************************************
+void kdbg(const char *,...);
+
+// *****************************************************************************
+void kdbge(const char*, const int, const char*, const bool, const int, ...);
+
+// *****************************************************************************
+const char *strrnchr(const char*, const unsigned char, const int);
+
+// *****************************************************************************
+uint8_t chk8(const char*);
+
+// *****************************************************************************
+#define NX_ARGS(z,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,X,...) X
+#define NB_ARGS(...) NX_ARGS(,##__VA_ARGS__,\
+                             16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0)
+
+// *****************************************************************************
+#define __FILENAME__ ({const char *f=strrnchr(__FILE__,'/',2);f?f+1:__FILE__;})
+
+// *****************************************************************************
+#define _F_L_F_ __FILENAME__,__LINE__,__FUNCTION__
+
+// *****************************************************************************
+#define dbp(...) kdbge(_F_L_F_,false, NB_ARGS(__VA_ARGS__),__VA_ARGS__)
+#define dbg(...) kdbge(_F_L_F_, true, NB_ARGS(__VA_ARGS__),__VA_ARGS__)
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_DBG_HPP

backends/kernels/config/nvvp.cpp

@@ -0,0 +1,114 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+// *****************************************************************************
+void mfem::kernels::push_flf(const char *file, const int line, const char *func)
+{
+   static bool env_ini = false;
+   static bool env_dbg = false;
+   if (!env_ini) { env_dbg = getenv("DBG"); env_ini = true; }
+   if (!env_dbg) { return; }
+   const uint8_t color = 17 + chk8(file)%216;
+   fflush(stdout);
+   fprintf(stdout,"\033[38;5;%dm",color);
+   fprintf(stdout,"\n%24s\b\b\b\b:\033[2m%3d\033[22m: %s", file, line, func);
+   fprintf(stdout,"\033[m");
+   fflush(stdout);
+}
+
+#if defined(__NVCC__) and defined(__NVVP__)
+
+// *****************************************************************************
+static const uint32_t legacy_colors[] =
+{
+   0x000000, 0x000080, 0x00008B, 0x0000CD, 0x0000FF, 0x006400, 0x008000,
+   0x008080, 0x008B8B, 0x00BFFF, 0x00CED1, 0x00FA9A, 0x00FF00, 0x00FF00,
+   0x00FF7F, 0x00FFFF, 0x00FFFF, 0x191970, 0x1E90FF, 0x20B2AA, 0x228B22,
+   0x2E8B57, 0x2F4F4F, 0x32CD32, 0x3CB371, 0x40E0D0, 0x4169E1, 0x4682B4,
+   0x483D8B, 0x48D1CC, 0x4B0082, 0x556B2F, 0x5F9EA0, 0x6495ED, 0x663399,
+   0x66CDAA, 0x696969, 0x6A5ACD, 0x6B8E23, 0x708090, 0x778899, 0x7B68EE,
+   0x7CFC00, 0x7F0000, 0x7F007F, 0x7FFF00, 0x7FFFD4, 0x808000, 0x808080,
+   0x87CEEB, 0x87CEFA, 0x8A2BE2, 0x8B0000, 0x8B008B, 0x8B4513, 0x8FBC8F,
+   0x90EE90, 0x9370DB, 0x9400D3, 0x98FB98, 0x9932CC, 0x9ACD32, 0xA020F0,
+   0xA0522D, 0xA52A2A, 0xA9A9A9, 0xADD8E6, 0xADFF2F, 0xAFEEEE, 0xB03060,
+   0xB0C4DE, 0xB0E0E6, 0xB22222, 0xB8860B, 0xBA55D3, 0xBC8F8F, 0xBDB76B,
+   0xBEBEBE, 0xC0C0C0, 0xC71585, 0xCD5C5C, 0xCD853F, 0xD2691E, 0xD2B48C,
+   0xD3D3D3, 0xD8BFD8, 0xDA70D6, 0xDAA520, 0xDB7093, 0xDC143C, 0xDCDCDC,
+   0xDDA0DD, 0xDEB887, 0xE0FFFF, 0xE6E6FA, 0xE9967A, 0xEE82EE, 0xEEE8AA,
+   0xF08080, 0xF0E68C, 0xF0F8FF, 0xF0FFF0, 0xF0FFFF, 0xF4A460, 0xF5DEB3,
+   0xF5F5DC, 0xF5F5F5, 0xF5FFFA, 0xF8F8FF, 0xFA8072, 0xFAEBD7, 0xFAF0E6,
+   0xFAFAD2, 0xFDF5E6, 0xFF0000, 0xFF00FF, 0xFF00FF, 0xFF1493, 0xFF4500,
+   0xFF6347, 0xFF69B4, 0xFF7F50, 0xFF8C00, 0xFFA07A, 0xFFA500, 0xFFB6C1,
+   0xFFC0CB, 0xFFD700, 0xFFDAB9, 0xFFDEAD, 0xFFE4B5, 0xFFE4C4, 0xFFE4E1,
+   0xFFEBCD, 0xFFEFD5, 0xFFF0F5, 0xFFF5EE, 0xFFF8DC, 0xFFFACD, 0xFFFAF0,
+   0xFFFAFA, 0xFFFF00, 0xFFFFE0, 0xFFFFF0, 0xFFFFFF
+};
+
+// *****************************************************************************
+static const int nb_colors = sizeof(legacy_colors)/sizeof(uint32_t);
+
+// *****************************************************************************
+static int kNvtxAttrPushEx(const char *ascii, const int color)
+{
+   if (!mfem::rconfig::Get().Nvvp()) { return 0; }
+   const int color_id = color%nb_colors;
+   nvtxEventAttributes_t eAttrib = {0};
+   eAttrib.version = NVTX_VERSION;
+   eAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
+   eAttrib.colorType = NVTX_COLOR_ARGB;
+   eAttrib.color = legacy_colors[color_id];
+   eAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
+   eAttrib.message.ascii = ascii;
+   return nvtxRangePushEx(&eAttrib);
+}
+
+// *****************************************************************************
+int kNvtxRangePushEx(const char *function, const char *file, const int line,
+                     const int color)
+{
+   if (!mfem::rconfig::Get().Nvvp()) { return 0; }
+   const size_t size = 2048;
+   static char marker[size];
+   const int nb_of_char_printed =
+      snprintf(marker,size, "%s@%s:%d",function,file,line);
+   assert(nb_of_char_printed>=0);
+   return rNvtxAttrPushEx(marker,color);
+}
+
+// *****************************************************************************
+int kNvtxRangePushEx(const char *ascii, const int color)
+{
+   return kNvtxAttrPushEx(ascii,color);
+}
+
+// ***************************************************************************
+int kNvtxSyncPop(void)  // Enforce Kernel Synchronization
+{
+   if (!mfem::rconfig::Get().Nvvp()) { return 0; }
+   kNvtxAttrPushEx("EKS", Yellow);
+   cudaStreamSynchronize(0);//*mfem::rconfig::Get().Stream());
+   nvtxRangePop();
+   return nvtxRangePop();
+}
+
+#endif // defined(__NVCC__) and defined(__NVVP__)
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/config/nvvp.hpp

@@ -0,0 +1,111 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+
+#ifndef MFEM_BACKENDS_KERNELS_NVVP_HPP
+#define MFEM_BACKENDS_KERNELS_NVVP_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// *****************************************************************************
+// en.wikipedia.org/wiki/Web_colors#Hex_triplet
+typedef enum
+{
+   Black, NavyBlue, DarkBlue, MediumBlue, Blue, DarkGreen, WebGreen, Teal,
+   DarkCyan, DeepSkyBlue, DarkTurquoise, MediumSpringGreen, Green, Lime,
+   SpringGreen, Aqua, Cyan, MidnightBlue, DodgerBlue, LightSeaGreen,
+   ForestGreen, SeaGreen, DarkSlateGray, LimeGreen, MediumSeaGreen,
+   Turquoise, RoyalBlue, SteelBlue, DarkSlateBlue, MediumTurquoise, Indigo,
+   DarkOliveGreen, CadetBlue, Cornflower, RebeccaPurple, MediumAquamarine,
+   DimGray, SlateBlue, OliveDrab, SlateGray, LightSlateGray,
+   MediumSlateBlue, LawnGreen, WebMaroon, WebPurple, Chartreuse,
+   Aquamarine, Olive, WebGray, SkyBlue, LightSkyBlue, BlueViolet, DarkRed,
+   DarkMagenta, SaddleBrown, DarkSeaGreen, LightGreen, MediumPurple,
+   DarkViolet, PaleGreen, DarkOrchid, YellowGreen, Purple, Sienna, Brown,
+   DarkGray, LightBlue, GreenYellow, PaleTurquoise, Maroon,
+   LightSteelBlue, PowderBlue, Firebrick, DarkGoldenrod, MediumOrchid,
+   RosyBrown, DarkKhaki, Gray, Silver, MediumVioletRed, IndianRed, Peru,
+   Chocolate, Tan, LightGray, Thistle, Orchid, Goldenrod, PaleVioletRed,
+   Crimson, Gainsboro, Plum, Burlywood, LightCyan, Lavender, DarkSalmon,
+   Violet, PaleGoldenrod, LightCoral, Khaki, AliceBlue, Honeydew, Azure,
+   SandyBrown, Wheat, Beige, WhiteSmoke, MintCream, GhostWhite, Salmon,
+   AntiqueWhite, Linen, LightGoldenrod, OldLace, Red, Fuchsia, Magenta,
+   DeepPink, OrangeRed, Tomato, HotPink, Coral, DarkOrange, LightSalmon,
+   Orange, LightPink, Pink, Gold, PeachPuff, NavajoWhite, Moccasin,
+   Bisque, MistyRose, BlanchedAlmond, PapayaWhip, LavenderBlush, Seashell,
+   Cornsilk, LemonChiffon, FloralWhite, Snow, Yellow, LightYellow, Ivory,
+   White
+} colors;
+
+// *****************************************************************************
+#if defined(__NVCC__) and defined(__NVVP__)
+
+#include <cuda.h>
+#include <nvToolsExt.h>
+#include <cudaProfiler.h>
+
+// *****************************************************************************
+NVTX_DECLSPEC int NVTX_API rNvtxRangePushEx(const char*,const char*,const int,
+                                            const int);
+NVTX_DECLSPEC int NVTX_API rNvtxRangePushEx(const char*, const int);
+NVTX_DECLSPEC int NVTX_API rNvtxSyncPop(void);
+
+// *****************************************************************************
+#define nvtx_pop(...) (mfem::rconfig::Get().Sync())?rNvtxSyncPop():nvtxRangePop();
+
+// *****************************************************************************
+#define PUSH2(ascii,color) rNvtxRangePushEx(#ascii,color);
+#define PUSH1(color) rNvtxRangePushEx(__PRETTY_FUNCTION__, __FILE__, __LINE__,color);
+#define PUSH0() rNvtxRangePushEx(__PRETTY_FUNCTION__, __FILE__, __LINE__,Lime);
+
+// *****************************************************************************
+#define LPAREN (
+#define COMMA_IF_PARENS(...) ,
+#define EXPAND(...) __VA_ARGS__
+#define PUSH(a0,a1,a2,a3,a4,a5,a,...) a
+#define CHOOSE(...) EXPAND(PUSH LPAREN \
+      __VA_ARGS__ COMMA_IF_PARENS \
+      __VA_ARGS__ COMMA_IF_PARENS __VA_ARGS__ (),  \
+      PUSH2, impossible, PUSH2, PUSH1, PUSH0, PUSH1, ))
+#define nvtx_push(...) CHOOSE(__VA_ARGS__)(__VA_ARGS__)
+
+#else // __NVCC__ && _NVVP__ ***************************************************
+
+// *****************************************************************************
+void push_flf(const char *file, const int line, const char *func);
+
+// *****************************************************************************
+#define nvtx_pop(...)
+#define nvtx_push(...) mfem::kernels::push_flf(__FILENAME__,__LINE__,__FUNCTION__)
+#define cuProfilerStart(...)
+#define cuProfilerStop(...)
+
+#endif // defined(__NVCC__) and defined(__NVVP__)
+
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_NVVP_HPP

backends/kernels/engine/backend.cpp

@@ -0,0 +1,53 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// *****************************************************************************
+bool Backend::Supports(const std::string &engine_spec) const
+{
+   nvtx_push();
+   nvtx_pop();
+   return true;
+}
+
+// *****************************************************************************
+mfem::Engine *Create(const std::string &engine_spec)
+{
+   nvtx_push();
+   nvtx_pop();
+   return new Engine(engine_spec);
+}
+
+// *****************************************************************************
+#ifdef MFEM_USE_MPI
+mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec)
+{
+   nvtx_push();
+   nvtx_pop();
+   return new Engine(comm, engine_spec);
+}
+#endif
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/engine/backend.hpp

@@ -0,0 +1,45 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_BACKEND_HPP
+#define MFEM_BACKENDS_KERNELS_BACKEND_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// *****************************************************************************
+class Backend : public mfem::Backend
+{
+public:
+   virtual ~Backend();
+
+   virtual bool Supports(const std::string &engine_spec) const;
+
+   virtual mfem::Engine *Create(const std::string &engine_spec);
+
+#ifdef MFEM_USE_MPI
+   virtual mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec);
+#endif
+};
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_BACKEND_HPP

backends/kernels/engine/device.cpp

@@ -0,0 +1,54 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// *****************************************************************************
+device::device() {}
+
+// *****************************************************************************
+device::~device() {}
+
+// *****************************************************************************
+bool device::hasSeparateMemorySpace()
+{
+   if (mfem::kernels::config::Get().Cuda())
+   {
+      return true;
+   }
+   return false;
+}
+
+// *****************************************************************************
+memory device::malloc(const std::size_t bytes,
+                      const void *src)
+{
+   nvtx_push();
+   assert(src==NULL);
+   dbg("bytes=%d",bytes);
+   nvtx_pop();
+   return memory(bytes,src);
+}
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/engine/device.hpp

@@ -0,0 +1,42 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mnfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_DEVICE_HPP
+#define MFEM_BACKENDS_KERNELS_DEVICE_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+// **************************************************************************
+class memory;
+
+// **************************************************************************
+class device
+{
+public:
+   device();
+   ~device();
+   bool hasSeparateMemorySpace();
+   kernels::memory malloc(const std::size_t,const void * = NULL);
+};
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_DEVICE_HPP

backends/kernels/engine/engine.cpp

@@ -0,0 +1,190 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// *****************************************************************************
+void Engine::Init(const std::string &engine_spec)
+{
+   nvtx_push();
+   memory_resources[0] = NULL;
+   workers_weights[0] = 1.0;
+   workers_mem_res[0] = 0;
+   dev = new device();
+
+   bool cuda = false;
+   const bool uvm = false;
+
+   if (engine_spec.find("cpu")!=std::string::npos)
+   {
+      dbg("CPU engine");
+      cuda = false;
+   }
+   if (engine_spec.find("gpu")!=std::string::npos)
+   {
+      dbg("GPU engine");
+      cuda = true;
+   }
+
+   kernels::config::Get().Setup(world_rank,
+                                world_size,
+                                cuda,
+                                false, // CG on device
+                                uvm,
+                                false, // MPI CUDA aware
+                                false, // share
+                                false, // occa
+                                false, // hcpo
+                                false, // sync
+                                false, // dot
+                                0); // rp_levels
+
+   nvtx_pop();
+}
+
+// *****************************************************************************
+Engine::Engine(const std::string &engine_spec) : mfem::Engine(NULL, 1, 1)
+{
+   nvtx_push();
+   Init(engine_spec);
+   nvtx_pop();
+}
+
+// *****************************************************************************
+#ifdef MFEM_USE_MPI
+Engine::Engine(MPI_Comm _comm,
+               const std::string &engine_spec)
+   : mfem::Engine(NULL, 1, 1),
+     comm(_comm)
+{
+   nvtx_push();
+   Init(engine_spec);
+   nvtx_pop();
+}
+
+Engine::Engine(const MPI_Session *_mpi,
+               const std::string &engine_spec)
+   : mfem::Engine(NULL, 1, 1),
+     comm(MPI_COMM_WORLD),
+     mpi(_mpi),
+     world_rank(mpi->WorldRank()),
+     world_size(mpi->WorldSize())
+{
+   nvtx_push();
+   Init(engine_spec);
+   nvtx_pop();
+}
+#endif
+
+// *****************************************************************************
+DLayout Engine::MakeLayout(std::size_t size) const
+{
+   nvtx_push();
+   const DLayout layout = DLayout(new kernels::Layout(*this, size));
+   nvtx_pop();
+   return layout;
+}
+
+// *****************************************************************************
+DLayout Engine::MakeLayout(const mfem::Array<std::size_t> &offsets) const
+{
+   nvtx_push();
+   MFEM_ASSERT(offsets.Size() == 2,
+               "multiple workers are not supported yet");
+   const DLayout layout = DLayout(new kernels::Layout(*this, offsets.Last()));
+   nvtx_pop();
+   return layout;
+}
+
+// *****************************************************************************
+DArray Engine::MakeArray(PLayout &layout, std::size_t item_size) const
+{
+   nvtx_push();
+   const DArray array = DArray(new kernels::Array(layout.As<Layout>(), item_size));
+   nvtx_pop();
+   return array;
+}
+
+// *****************************************************************************
+DVector Engine::MakeVector(PLayout &layout, int type_id) const
+{
+   nvtx_push();
+   const DVector vector(new kernels::Vector(layout.As<Layout>()));
+   nvtx_pop();
+   return vector;
+}
+
+// *****************************************************************************
+#ifdef MFEM_USE_MPI
+DFiniteElementSpace Engine::MakeFESpace(mfem::ParFiniteElementSpace &pfes) const
+{
+   nvtx_push();
+   const DFiniteElementSpace dfes(new kFiniteElementSpace(*this, pfes));
+   nvtx_pop();
+   return dfes;
+}
+#endif
+
+// *****************************************************************************
+DFiniteElementSpace Engine::MakeFESpace(mfem::FiniteElementSpace &fes) const
+{
+   nvtx_push();
+   DFiniteElementSpace dfes(new kFiniteElementSpace(*this, fes));
+   nvtx_pop();
+   return dfes;
+}
+
+// *****************************************************************************
+DBilinearForm Engine::MakeBilinearForm(mfem::BilinearForm &bf) const
+{
+   nvtx_push();
+   const DBilinearForm dbf(new BilinearForm(*this, bf));
+   nvtx_pop();
+   return dbf;
+}
+
+// *****************************************************************************
+void Engine::AssembleLinearForm(LinearForm &l_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+}
+
+// *****************************************************************************
+mfem::Operator *Engine::MakeOperator(const MixedBilinearForm &mbl_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+   return NULL;
+}
+
+// *****************************************************************************
+mfem::Operator *Engine::MakeOperator(const NonlinearForm &nl_form) const
+{
+   /// FIXME - What will the actual parameters be?
+   MFEM_ABORT("FIXME");
+   return NULL;
+}
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/engine/engine.hpp

@@ -0,0 +1,98 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_ENGINE_HPP
+#define MFEM_BACKENDS_KERNELS_ENGINE_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+class Engine : public mfem::Engine
+{
+protected:
+   kernels::device *dev=NULL;
+#ifdef MFEM_USE_MPI
+   const MPI_Comm comm = MPI_COMM_NULL;
+   const MPI_Session *mpi;
+   const int world_rank = 0;
+   const int world_size = 1;
+
+#endif
+
+   void Init(const std::string &engine_spec);
+
+public:
+   Engine(const std::string &engine_spec);
+
+#ifdef MFEM_USE_MPI
+   Engine(MPI_Comm, const std::string&);
+   Engine(const MPI_Session*, const std::string&);
+#endif
+
+   virtual ~Engine() { }
+
+   /**
+       @name KERNELS specific interface, used by other objects in the KERNELS backend
+    */
+   ///@{
+   kernels::device GetDevice(int idx = 0) const { return *dev; }
+
+   ///@}
+   // End: KERNELS specific interface
+
+   /**
+       @name Virtual interface: finite element data structures and algorithms
+    */
+   ///@{
+
+   virtual DLayout MakeLayout(std::size_t) const;
+
+   virtual DLayout MakeLayout(const mfem::Array<std::size_t>&) const;
+
+   virtual DArray MakeArray(PLayout&, std::size_t) const;
+
+   virtual DVector MakeVector(PLayout&,
+                              int type_id = ScalarId<double>::value) const;
+
+#ifdef MFEM_USE_MPI
+   virtual DFiniteElementSpace MakeFESpace(mfem::ParFiniteElementSpace &) const;
+#endif
+
+   virtual DFiniteElementSpace MakeFESpace(mfem::FiniteElementSpace&) const;
+
+   virtual DBilinearForm MakeBilinearForm(mfem::BilinearForm&) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual void AssembleLinearForm(LinearForm&) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual mfem::Operator *MakeOperator(const MixedBilinearForm&) const;
+
+   /// FIXME - What will the actual parameters be?
+   virtual mfem::Operator *MakeOperator(const NonlinearForm&) const;
+
+   ///@}
+   // End: Virtual interface
+};
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_ENGINE_HPP

backends/kernels/engine/memory.cpp

@@ -0,0 +1,129 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// **************************************************************************
+memory::memory(const std::size_t _bytes, const void *src):
+   bytes(_bytes),
+   data((char*)mfem::kernels::kmalloc<char>::operator new (bytes))
+{
+   nvtx_push();
+   if (src)
+   {
+      data=(char*)src;
+   }
+   nvtx_pop();
+}
+
+// **************************************************************************
+kernels::device memory::getDevice()
+{
+   return kernels::device();
+}
+
+// **************************************************************************
+size_t memory::size() const
+{
+   return bytes;
+}
+
+// *****************************************************************************
+void memory::copyFrom(memory &src, size_t b) const
+{
+   nvtx_push();
+   //memcpy(data,src,b);
+   mfem::kernels::kmemcpy::rHtoD(data,src,b);
+   nvtx_pop();
+}
+void memory::copyFrom(memory &src)
+{
+   nvtx_push();
+   //memcpy(data,src,bytes);
+   mfem::kernels::kmemcpy::rHtoD(data,src,bytes);
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void memory::copyFrom(const void *src, size_t b) const
+{
+   nvtx_push();
+   //memcpy(data,src,b);
+   mfem::kernels::kmemcpy::rHtoD(data,src,b);
+   nvtx_pop();
+}
+void memory::copyFrom(const void *src)
+{
+   nvtx_push();
+   //memcpy(data,src,bytes);
+   assert(data);
+   assert(src);
+   mfem::kernels::kmemcpy::rHtoD(data,src,bytes);
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void memory::copyTo(void *dest, size_t b) const
+{
+   nvtx_push();
+   //memcpy(dest,data,b);
+   mfem::kernels::kmemcpy::rDtoH(dest,data,b);
+   nvtx_pop();
+}
+void memory::copyTo(void *dest)
+{
+   nvtx_push();
+   //memcpy(dest,data,bytes);
+   mfem::kernels::kmemcpy::rDtoH(dest,data,bytes);
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void* memory::ptr() const
+{
+   return (void*)data;
+}
+
+// *****************************************************************************
+memory memory::slice(const size_t offset,
+                     const int bytes) const
+{
+   nvtx_push();
+   assert(bytes>0);
+   memory m = memory(bytes,NULL);
+   dbg("\n\033[31;1;7m[memory::slice] rDtoD!");
+   mfem::kernels::kmemcpy::rDtoD(m.data,data+offset,bytes);
+   nvtx_pop();
+   return m;
+}
+
+// *****************************************************************************
+bool memory::operator == (const memory &m)
+{
+   nvtx_push();
+   nvtx_pop();
+   return (ptr() == m.ptr()) && (size() == m.size());
+}
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/engine/memory.hpp

@@ -0,0 +1,71 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_MEMORY_HPP
+#define MFEM_BACKENDS_KERNELS_MEMORY_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+class device;
+
+// ***************************************************************************
+class memory
+{
+public:
+   std::size_t bytes;
+   char *data;
+public:
+   memory(const std::size_t =0, const void* =NULL);
+
+   size_t size() const;
+
+   kernels::device getDevice();
+
+   void copyTo(void*);
+   void copyTo(void*, size_t)const;
+
+   void copyFrom(memory&);
+   void copyFrom(memory&, size_t)const;
+
+   void copyFrom(const void*);
+   void copyFrom(const void*, size_t)const;
+
+   void* ptr() const;
+
+   inline operator double* () { return (double*)data; }
+
+   inline operator const double* () const { return (const double*)data; }
+
+   memory slice(const size_t offset,
+                const int bytes = -1) const;
+
+   inline char* operator[](const size_t i)
+   {
+      return data+i;
+   }
+
+   bool operator == (const memory &);
+};
+
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_MEMORY_HPP

backends/kernels/fem/bilinearform.cpp

@@ -0,0 +1,129 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// *****************************************************************************
+bool BilinearForm::Assemble()
+{
+   nvtx_push();
+   if (kbform == NULL) { InitKBilinearForm(); }
+   kbform->Assemble();
+   nvtx_pop();
+   return true; // --> host assembly is not needed
+}
+
+// *****************************************************************************
+void BilinearForm::FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
+                                    mfem::OperatorHandle &A)
+{
+   nvtx_push();//assert(false);// ex1pd comes here, Laghos dont
+   if (A.Type() == mfem::Operator::ANY_TYPE)
+   {
+      mfem::Operator *Aout = NULL;
+      kbform->FormOperator(ess_tdof_list, Aout);
+      A.Reset(Aout);
+   }
+   else
+   {
+      MFEM_ABORT("Operator::Type is not supported, type = " << A.Type());
+   }
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void BilinearForm::FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
+                                    mfem::Vector &x, mfem::Vector &b,
+                                    mfem::OperatorHandle &A,
+                                    mfem::Vector &X, mfem::Vector &B,
+                                    int copy_interior)
+{
+   nvtx_push();//assert(false); // ex1pd comes here, Laghos does not
+   FormSystemMatrix(ess_tdof_list, A);
+   kbform->InitRHS(ess_tdof_list, x, b, A.Ptr(), X, B, copy_interior);
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void BilinearForm::RecoverFEMSolution(const mfem::Vector &X,
+                                      const mfem::Vector &b,
+                                      mfem::Vector &x)
+{
+   nvtx_push();
+   kbform->KernelsRecoverFEMSolution(X, b, x);
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void BilinearForm::InitKBilinearForm()
+{
+   nvtx_push();
+   // Init 'kbform' using 'bform'
+   MFEM_ASSERT(bform != NULL, "");
+   MFEM_ASSERT(kbform == NULL, "");
+
+   kFiniteElementSpace &ofes =
+      bform->FESpace()->Get_PFESpace()->As<kFiniteElementSpace>();
+   kbform = new kBilinearForm(&ofes);
+
+   dbg(", transfer domain integrators");
+   mfem::Array<mfem::BilinearFormIntegrator*> &dbfi = *bform->GetDBFI();
+   for (int i = 0; i < dbfi.Size(); i++)
+   {
+      std::string integ_name(dbfi[i]->Name());
+      dbg(", integ_name: %s",integ_name.c_str());
+      Coefficient *scal_coeff = dbfi[i]->GetScalarCoefficient();
+      ConstantCoefficient *const_coeff =
+         dynamic_cast<ConstantCoefficient*>(scal_coeff);
+      // TODO: other types of coefficients ...
+      double val = const_coeff ? const_coeff->constant : 1.0;
+      KernelsCoefficient coeff(kbform->engine(), val);
+      KernelsIntegrator *integ = NULL;
+
+      if (integ_name == "mass")
+      {
+         //MFEM_ABORT("BilinearFormIntegrator does not define Name()");
+         integ = new KernelsMassIntegrator(coeff);
+      }
+      else if (integ_name == "diffusion")
+      {
+         //assert(false);
+         integ = new KernelsDiffusionIntegrator(coeff);
+      }
+      else
+      {
+         MFEM_ABORT("BilinearFormIntegrator [Name() = " << integ_name
+                    << "] is not supported");
+      }
+
+      const mfem::IntegrationRule *ir = dbfi[i]->GetIntRule();
+      if (ir) { integ->SetIntegrationRule(*ir); }
+
+      kbform->AddDomainIntegrator(integ);
+   }
+   nvtx_pop();
+   // TODO: other types of integrators ...
+}
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/fem/bilinearform.hpp

@@ -0,0 +1,68 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_BILINEAR_FORM_HPP
+#define MFEM_BACKENDS_KERNELS_BILINEAR_FORM_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+/// TODO: doxygen
+class BilinearForm : public mfem::PBilinearForm
+{
+protected:
+   kBilinearForm *kbform;
+public:
+   /// TODO: doxygen
+   BilinearForm(const Engine &e,
+                mfem::BilinearForm &bf)
+      : mfem::PBilinearForm(e, bf),
+        kbform(NULL) { }
+
+   /// Virtual destructor
+   virtual ~BilinearForm() { }
+
+   /// Assemble the PBilinearForm.
+   /** This method is called from the method mfem::BilinearForm::Assemble() of
+       the associated mfem::BilinearForm, #bform.
+       @returns True, if the host assembly should NOT be performed. */
+   virtual bool Assemble();
+
+   virtual void FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
+                                 mfem::OperatorHandle &A);
+
+   virtual void FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
+                                 mfem::Vector &x, mfem::Vector &b,
+                                 mfem::OperatorHandle &A,
+                                 mfem::Vector &X, mfem::Vector &B,
+                                 int copy_interior);
+
+   virtual void RecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
+                                   mfem::Vector &x);
+
+protected:
+   // Called from Assemble() if kbform is NULL to initialize kbform.
+   void InitKBilinearForm();
+};
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_BILINEAR_FORM_HPP

backends/kernels/fem/bilinintDiffusion.cpp

@@ -0,0 +1,117 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// *****************************************************************************
+KernelsDiffusionIntegrator::KernelsDiffusionIntegrator(const KernelsCoefficient
+                                                       &coeff_)
+   :
+   KernelsIntegrator(coeff_.KernelsEngine()),
+   coeff(coeff_),
+   assembledOperator(*(new Layout(coeff_.KernelsEngine(), 0)))
+{
+   nvtx_push();
+   coeff.SetName("COEFF");
+   nvtx_pop();
+}
+
+// *****************************************************************************
+KernelsDiffusionIntegrator::~KernelsDiffusionIntegrator() {}
+
+// *****************************************************************************
+std::string KernelsDiffusionIntegrator::GetName()
+{
+   return "DiffusionIntegrator";
+}
+
+// *****************************************************************************
+void KernelsDiffusionIntegrator::SetupIntegrationRule()
+{
+   nvtx_push();
+   const FiniteElement &trialFE = *(trialFESpace->GetFE(0));
+   const FiniteElement &testFE  = *(testFESpace->GetFE(0));
+   ir = &mfem::DiffusionIntegrator::GetRule(trialFE, testFE);
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void KernelsDiffusionIntegrator::Setup()
+{
+   nvtx_push();
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void KernelsDiffusionIntegrator::Assemble()
+{
+   nvtx_push();
+   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
+   const int dim = mesh->Dimension();
+   const int dims = fe.GetDim();
+   assert(dim==dims);
+
+   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+   const int elements = trialFESpace->GetNE();
+   assert(elements==mesh->GetNE());
+
+   const int quadraturePoints = ir->GetNPoints();
+   const int quad1D = IntRules.Get(Geometry::SEGMENT,ir->GetOrder()).GetNPoints();
+
+   kGeometry *geo = GetGeometry(kGeometry::Jacobian);
+   assert(geo);
+
+   assembledOperator.Resize<double>(symmDims * quadraturePoints * elements,NULL);
+   rDiffusionAssemble(dim,
+                      quad1D,
+                      mesh->GetNE(),
+                      maps->quadWeights,
+                      geo->J,
+                      1.0,//COEFF
+                      (double*)assembledOperator.KernelsMem().ptr());
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void KernelsDiffusionIntegrator::MultAdd(Vector &x, Vector &y)
+{
+   nvtx_push();
+   const int dim = mesh->Dimension();
+   const int quad1D = IntRules.Get(Geometry::SEGMENT,ir->GetOrder()).GetNPoints();
+   const int dofs1D = trialFESpace->GetFE(0)->GetOrder() + 1;
+   rDiffusionMultAdd(dim,
+                     dofs1D,
+                     quad1D,
+                     mesh->GetNE(),
+                     maps->dofToQuad,
+                     maps->dofToQuadD,
+                     maps->quadToDof,
+                     maps->quadToDofD,
+                     (double*)assembledOperator.KernelsMem().ptr(),
+                     (const double*)x.KernelsMem().ptr(),
+                     (double*)y.KernelsMem().ptr());
+   nvtx_pop();
+}
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/fem/bilinintDiffusion.hpp

@@ -0,0 +1,45 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_BILIN_INTEG_DIFFUSION_HPP
+#define MFEM_BACKENDS_KERNELS_BILIN_INTEG_DIFFUSION_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+class KernelsDiffusionIntegrator : public KernelsIntegrator
+{
+private:
+   KernelsCoefficient coeff;
+   Vector assembledOperator;
+public:
+   KernelsDiffusionIntegrator(const KernelsCoefficient &coeff_);
+   virtual ~KernelsDiffusionIntegrator();
+   virtual std::string GetName();
+   virtual void SetupIntegrationRule();
+   virtual void Setup();
+   virtual void Assemble();
+   virtual void MultAdd(Vector &x, Vector &y);
+};
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_BILIN_INTEG_DIFFUSION_HPP

backends/kernels/fem/bilinintMass.cpp

@@ -0,0 +1,111 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// *****************************************************************************
+KernelsMassIntegrator::KernelsMassIntegrator(const mfem::Engine &ng) :
+   KernelsIntegrator(ng.As<kernels::Engine>()),
+   engine(ng),
+   coeff(ng.As<kernels::Engine>(),1.0)
+{
+   nvtx_push();
+   nvtx_pop();
+}
+
+// *****************************************************************************
+KernelsMassIntegrator::KernelsMassIntegrator(const KernelsCoefficient &coeff_) :
+   KernelsIntegrator(coeff_.KernelsEngine()),
+   engine(coeff_.KernelsEngine()),
+   coeff(coeff_)
+{
+   nvtx_push();
+   coeff.SetName("COEFF");
+   nvtx_pop();
+}
+
+KernelsMassIntegrator::~KernelsMassIntegrator() {}
+
+std::string KernelsMassIntegrator::GetName()
+{
+   return "MassIntegrator";
+}
+
+// *****************************************************************************
+void KernelsMassIntegrator::SetupIntegrationRule()
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace->GetFE(0));
+   const mfem::FiniteElement &testFE  = *(testFESpace->GetFE(0));
+   mfem::ElementTransformation &T = *trialFESpace->GetElementTransformation(0);
+   ir = &mfem::MassIntegrator::GetRule(trialFE, testFE, T);
+}
+
+// *****************************************************************************
+void KernelsMassIntegrator::Setup()
+{
+   nvtx_push();
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void KernelsMassIntegrator::Assemble()
+{
+   nvtx_push();
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void KernelsMassIntegrator::SetOperator(mfem::Vector &v)
+{
+   nvtx_push();
+   op = v;
+   op.Resize(engine.MakeLayout(v.Size()));
+   op.PushData(v.GetData());
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void KernelsMassIntegrator::MultAdd(kernels::Vector &x,
+                                    kernels::Vector &y)
+{
+   nvtx_push();
+   const int dim = mesh->Dimension();
+   const int quad1D = IntRules.Get(Geometry::SEGMENT,ir->GetOrder()).GetNPoints();
+   const int dofs1D = trialFESpace->GetFE(0)->GetOrder() + 1;
+   kernels::Vector kop = op.Get_PVector()->As<kernels::Vector>();
+   rMassMultAdd(dim,
+                dofs1D,
+                quad1D,
+                mesh->GetNE(),
+                maps->dofToQuad,
+                maps->dofToQuadD,
+                maps->quadToDof,
+                maps->quadToDofD,
+                (const double*)kop.KernelsMem().ptr(),
+                (const double*)x.KernelsMem().ptr(),
+                (double*)y.KernelsMem().ptr());
+   nvtx_pop();
+}
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/fem/bilinintMass.hpp

@@ -0,0 +1,48 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_BILIN_INTEG_MASS_HPP
+#define MFEM_BACKENDS_KERNELS_BILIN_INTEG_MASS_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+class KernelsMassIntegrator : public KernelsIntegrator
+{
+private:
+   const mfem::Engine& engine;
+   KernelsCoefficient coeff;
+   mfem::Vector op;
+public:
+   KernelsMassIntegrator(const mfem::Engine&);
+   KernelsMassIntegrator(const KernelsCoefficient&);
+   virtual ~KernelsMassIntegrator();
+   virtual std::string GetName();
+   virtual void SetupIntegrationRule();
+   virtual void Setup();
+   virtual void Assemble();
+   void SetOperator(mfem::Vector &v);
+   virtual void MultAdd(Vector &x, Vector &y);
+};
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_BILIN_INTEG_MASS_HPP

backends/kernels/fem/bilinintVMass.cpp

@@ -0,0 +1,99 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+// *****************************************************************************
+KernelsVectorMassIntegrator::KernelsVectorMassIntegrator(
+   const KernelsCoefficient &
+   coeff_)
+   :
+   KernelsIntegrator(coeff_.KernelsEngine()),
+   coeff(coeff_),
+   assembledOperator(*(new Layout(coeff_.KernelsEngine(), 0)))
+{
+   coeff.SetName("COEFF");
+}
+
+KernelsVectorMassIntegrator::~KernelsVectorMassIntegrator() {}
+
+// *****************************************************************************
+std::string KernelsVectorMassIntegrator::GetName()
+{
+   return "VectorMassIntegrator";
+}
+
+// *****************************************************************************
+void KernelsVectorMassIntegrator::SetupIntegrationRule()
+{
+   const mfem::FiniteElement &trialFE = *(trialFESpace->GetFE(0));
+   const mfem::FiniteElement &testFE  = *(testFESpace->GetFE(0));
+   mfem::ElementTransformation &T = *trialFESpace->GetElementTransformation(0);
+   ir = &mfem::MassIntegrator::GetRule(trialFE, testFE, T);
+}
+
+// *****************************************************************************
+void KernelsVectorMassIntegrator::Setup()
+{
+   assert(false);/*
+ ::kernels::properties kernelProps = props;
+
+   coeff.Setup(*this, kernelProps);
+
+   // Setup assemble and mult kernels
+   assembleKernel = GetAssembleKernel(kernelProps);
+   multKernel     = GetMultAddKernel(kernelProps);*/
+}
+
+// *****************************************************************************
+void KernelsVectorMassIntegrator::Assemble()
+{
+   assert(false);/*
+const int elements = trialFESpace->GetNE();
+   const int quadraturePoints = ir->GetNPoints();
+
+   KernelsGeometry geom = GetGeometry(KernelsGeometry::Jacobian);
+
+   assembledOperator.Resize<double>(quadraturePoints * elements, NULL);
+
+   assembleKernel((int) mesh->GetNE(),
+                  maps.quadWeights,
+                  geom.J,
+                  coeff,
+                  assembledOperator.KernelsMem());*/
+}
+
+// *****************************************************************************
+void KernelsVectorMassIntegrator::MultAdd(Vector &x, Vector &y)
+{
+   assert(false);/*
+ multKernel((int) mesh->GetNE(),
+              maps.dofToQuad,
+              maps.dofToQuadD,
+              maps.quadToDof,
+              maps.quadToDofD,
+              assembledOperator.KernelsMem(),
+              x.KernelsMem(), y.KernelsMem());*/
+}
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/fem/bilinintVMass.hpp

@@ -0,0 +1,45 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_BILIN_INTEG_VMASS_HPP
+#define MFEM_BACKENDS_KERNELS_BILIN_INTEG_VMASS_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+class KernelsVectorMassIntegrator : public KernelsIntegrator
+{
+private:
+   KernelsCoefficient coeff;
+   Vector assembledOperator;
+public:
+   KernelsVectorMassIntegrator(const KernelsCoefficient &coeff_);
+   virtual ~KernelsVectorMassIntegrator();
+   virtual std::string GetName();
+   virtual void SetupIntegrationRule();
+   virtual void Setup();
+   virtual void Assemble();
+   virtual void MultAdd(Vector &x, Vector &y);
+};
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_BILIN_INTEG_VMASS_HPP

backends/kernels/fem/bilininteg.cpp

@@ -0,0 +1,129 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// *****************************************************************************
+KernelsIntegrator::KernelsIntegrator(const kernels::Engine &e)
+   : engine(&e),
+     bform(),
+     mesh(),
+     rtrialFESpace(),
+     rtestFESpace(),
+     trialFESpace(),
+     testFESpace(),
+     itype(DomainIntegrator),
+     ir(NULL),
+     hasTensorBasis(false) { nvtx_push(); nvtx_pop();}
+
+// *****************************************************************************
+KernelsIntegrator::~KernelsIntegrator() {}
+
+// *****************************************************************************
+void KernelsIntegrator::SetupMaps()
+{
+   nvtx_push();
+   maps = kDofQuadMaps::Get(*rtrialFESpace->GetFESpace(),
+                            *rtestFESpace->GetFESpace(),
+                            *ir);
+
+   mapsTranspose = kDofQuadMaps::Get(*rtestFESpace->GetFESpace(),
+                                     *rtrialFESpace->GetFESpace(),
+                                     *ir);
+   nvtx_pop();
+}
+
+// *****************************************************************************
+kFiniteElementSpace& KernelsIntegrator::GetTrialKernelsFESpace() const
+{
+   return *rtrialFESpace;
+}
+
+kFiniteElementSpace& KernelsIntegrator::GetTestKernelsFESpace() const
+{
+   return *rtestFESpace;
+}
+
+mfem::FiniteElementSpace& KernelsIntegrator::GetTrialFESpace() const
+{
+   return *trialFESpace;
+}
+
+mfem::FiniteElementSpace& KernelsIntegrator::GetTestFESpace() const
+{
+   return *testFESpace;
+}
+
+void KernelsIntegrator::SetIntegrationRule(const mfem::IntegrationRule &ir_)
+{
+   ir = &ir_;
+}
+
+const mfem::IntegrationRule& KernelsIntegrator::GetIntegrationRule() const
+{
+   assert(ir);
+   return *ir;
+}
+
+kDofQuadMaps *KernelsIntegrator::GetDofQuadMaps()
+{
+   return maps;
+}
+
+// *****************************************************************************
+void KernelsIntegrator::SetupIntegrator(kBilinearForm &bform_,
+                                        const KernelsIntegratorType itype_)
+{
+   nvtx_push();
+   //MFEM_ASSERT(engine == &bform_.KernelsEngine(), "");
+   bform     = &bform_;
+   mesh      = &(bform_.GetMesh());
+
+   rtrialFESpace = &(bform_.GetTrialKernelsFESpace());
+   rtestFESpace  = &(bform_.GetTestKernelsFESpace());
+
+   trialFESpace = &(bform_.GetTrialFESpace());
+   testFESpace  = &(bform_.GetTestFESpace());
+
+   hasTensorBasis = rtrialFESpace->hasTensorBasis();
+
+   itype = itype_;
+
+   if (ir == NULL)
+   {
+      SetupIntegrationRule();
+   }
+   SetupMaps();
+   Setup();
+   nvtx_pop();
+}
+
+// *****************************************************************************
+kGeometry *KernelsIntegrator::GetGeometry(const int flags)
+{
+   dbg("GetGeometry");
+   return kGeometry::Get(*rtrialFESpace, *ir);
+}
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/fem/bilininteg.hpp

@@ -0,0 +1,82 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_BILIN_INTEG_HPP
+#define MFEM_BACKENDS_KERNELS_BILIN_INTEG_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+class KernelsIntegrator
+{
+protected:
+   SharedPtr<const Engine> engine;
+
+   kernels::kBilinearForm *bform;
+   mfem::Mesh *mesh;
+
+   kFiniteElementSpace *rtrialFESpace;
+   kFiniteElementSpace *rtestFESpace;
+
+   mfem::FiniteElementSpace *trialFESpace;
+   mfem::FiniteElementSpace *testFESpace;
+
+   KernelsIntegratorType itype;
+
+   const IntegrationRule *ir;
+   bool hasTensorBasis;
+   kernels::kDofQuadMaps *maps;
+   kDofQuadMaps *mapsTranspose;
+
+public:
+   KernelsIntegrator(const kernels::Engine &e);
+   virtual ~KernelsIntegrator();
+   const Engine &KernelsEngine() const { assert(engine); return *engine; }
+   kernels::device GetDevice(int idx = 0) const
+   { return engine->GetDevice(idx); }
+   virtual std::string GetName() = 0;
+   kFiniteElementSpace& GetTrialKernelsFESpace() const;
+   kFiniteElementSpace& GetTestKernelsFESpace() const;
+   mfem::FiniteElementSpace& GetTrialFESpace() const;
+   mfem::FiniteElementSpace& GetTestFESpace() const;
+   void SetIntegrationRule(const mfem::IntegrationRule &ir_);
+   const mfem::IntegrationRule& GetIntegrationRule() const;
+   kDofQuadMaps* GetDofQuadMaps();
+   void SetupMaps();
+   virtual void SetupIntegrationRule() = 0;
+   virtual void SetupIntegrator(kBilinearForm &bform_,
+                                const KernelsIntegratorType itype_);
+   virtual void Setup() = 0;
+   virtual void Assemble() = 0;
+   /// This method works on E-vectors!
+   virtual void MultAdd(Vector &x, Vector &y) = 0;
+   virtual void MultTransposeAdd(Vector &x, Vector &y)
+   {
+      mfem_error("KernelsIntegrator::MultTransposeAdd() is not overloaded!");
+   }
+   kGeometry *GetGeometry(const int flags = (kGeometry::Jacobian    |
+                                             kGeometry::JacobianInv |
+                                             kGeometry::JacobianDet));
+};
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_BILIN_INTEG_HPP

backends/kernels/fem/coefficient.cpp

@@ -0,0 +1,275 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+//---[ Parameter ]------------
+KernelsParameter::~KernelsParameter() {}
+
+void KernelsParameter::Setup(KernelsIntegrator &integ) {nvtx_push(); nvtx_pop();}
+
+//====================================
+
+
+//---[ Include Parameter ]------------
+KernelsIncludeParameter::KernelsIncludeParameter(const std::string &filename_) :
+   filename(filename_) {}
+
+KernelsParameter* KernelsIncludeParameter::Clone()
+{
+   return new KernelsIncludeParameter(filename);
+}
+
+void KernelsIncludeParameter::Setup(KernelsIntegrator &integ)
+{
+}
+//====================================
+
+
+//---[ Source Parameter ]------------
+KernelsSourceParameter::KernelsSourceParameter(const std::string &source_) :
+   source(source_) {}
+
+KernelsParameter* KernelsSourceParameter::Clone()
+{
+   return new KernelsSourceParameter(source);
+}
+
+void KernelsSourceParameter::Setup(KernelsIntegrator &integ)
+{
+}
+//====================================
+
+//---[ Vector Parameter ]-------
+KernelsVectorParameter::KernelsVectorParameter(const std::string &name_,
+                                               Vector &v_,
+                                               const bool useRestrict_) :
+   name(name_),
+   v(v_),
+   useRestrict(useRestrict_),
+   attr("") {}
+
+KernelsVectorParameter::KernelsVectorParameter(const std::string &name_,
+                                               Vector &v_,
+                                               const std::string &attr_,
+                                               const bool useRestrict_) :
+   name(name_),
+   v(v_),
+   useRestrict(useRestrict_),
+   attr(attr_) {}
+
+KernelsParameter* KernelsVectorParameter::Clone()
+{
+   return new KernelsVectorParameter(name, v, attr, useRestrict);
+}
+
+void KernelsVectorParameter::Setup(KernelsIntegrator &integ)
+{
+}
+
+//====================================
+
+
+//---[ GridFunction Parameter ]-------
+KernelsGridFunctionParameter::KernelsGridFunctionParameter(
+   const std::string &name_,
+   KernelsGridFunction &gf_,
+   const bool useRestrict_)
+   : name(name_),
+     gf(gf_),
+     gfQuad(*(new Layout(gf_.KernelsLayout().KernelsEngine(), 0))),
+     useRestrict(useRestrict_) {}
+
+KernelsParameter* KernelsGridFunctionParameter::Clone()
+{
+   KernelsGridFunctionParameter *param =
+      new KernelsGridFunctionParameter(name, gf, useRestrict);
+   param->gfQuad.MakeRef(gfQuad);
+   return param;
+}
+
+void KernelsGridFunctionParameter::Setup(KernelsIntegrator &integ)
+{
+   nvtx_push();
+   nvtx_pop();
+}
+//====================================
+
+
+//---[ Coefficient ]------------------
+KernelsCoefficient::KernelsCoefficient(const double value) :
+   engine(NULL),
+   integ(NULL),
+   name("COEFF")
+{
+   nvtx_push();
+   nvtx_pop();
+}
+//---[ Coefficient ]------------------
+KernelsCoefficient::KernelsCoefficient(const Engine &e, const double value) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   nvtx_push();
+   nvtx_pop();
+}
+
+KernelsCoefficient::KernelsCoefficient(const Engine &e,
+                                       const std::string &source) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   nvtx_push();
+   nvtx_pop();
+}
+
+KernelsCoefficient::KernelsCoefficient(const Engine &e, const char *source) :
+   engine(&e),
+   integ(NULL),
+   name("COEFF")
+{
+   nvtx_push();
+   nvtx_pop();
+}
+
+KernelsCoefficient::KernelsCoefficient(const KernelsCoefficient &coeff) :
+   engine(coeff.engine),
+   integ(NULL),
+   name(coeff.name)
+{
+   nvtx_push();
+   const int paramCount = (int) coeff.params.size();
+   for (int i = 0; i < paramCount; ++i)
+   {
+      params.push_back(coeff.params[i]->Clone());
+   }
+   nvtx_pop();
+}
+
+KernelsCoefficient::~KernelsCoefficient()
+{
+   const int paramCount = (int) params.size();
+   for (int i = 0; i < paramCount; ++i)
+   {
+      delete params[i];
+   }
+}
+
+KernelsCoefficient& KernelsCoefficient::SetName(const std::string &name_)
+{
+   name = name_;
+   return *this;
+}
+
+void KernelsCoefficient::Setup(KernelsIntegrator &integ_)
+{
+   nvtx_push();
+   integ = &integ_;
+   nvtx_pop();
+}
+
+KernelsCoefficient& KernelsCoefficient::Add(KernelsParameter *param)
+{
+   nvtx_push();
+   params.push_back(param);
+   nvtx_pop();
+   return *this;
+}
+
+KernelsCoefficient& KernelsCoefficient::IncludeHeader(const std::string
+                                                      &filename)
+{
+   return Add(new KernelsIncludeParameter(filename));
+}
+
+KernelsCoefficient& KernelsCoefficient::IncludeSource(const std::string &source)
+{
+   return Add(new KernelsSourceParameter(source));
+}
+
+KernelsCoefficient& KernelsCoefficient::AddVector(const std::string &name_,
+                                                  Vector &v,
+                                                  const bool useRestrict)
+{
+   return Add(new KernelsVectorParameter(name_, v, useRestrict));
+}
+
+KernelsCoefficient& KernelsCoefficient::AddVector(const std::string &name_,
+                                                  Vector &v,
+                                                  const std::string &attr,
+                                                  const bool useRestrict)
+{
+   return Add(new KernelsVectorParameter(name_, v, attr, useRestrict));
+}
+
+KernelsCoefficient& KernelsCoefficient::AddGridFunction(
+   const std::string &name_,
+   KernelsGridFunction &gf,
+   const bool useRestrict)
+{
+   return Add(new KernelsGridFunctionParameter(name_, gf, useRestrict));
+}
+
+bool KernelsCoefficient::IsConstant()
+{
+   assert(false);
+   return true;
+}
+
+double KernelsCoefficient::GetConstantValue()
+{
+   if (!IsConstant())
+   {
+      mfem_error("KernelsCoefficient is not constant");
+   }
+   assert(false);
+   return 1.0;
+}
+
+Vector KernelsCoefficient::Eval()
+{
+   if (integ == NULL)
+   {
+      mfem_error("KernelsCoefficient requires a Setup() call before Eval()");
+   }
+
+   mfem::FiniteElementSpace &fespace = integ->GetTrialFESpace();
+   const mfem::IntegrationRule &ir   = integ->GetIntegrationRule();
+
+   const int elements = fespace.GetNE();
+   const int numQuad  = ir.GetNPoints();
+
+   Vector quadCoeff(*(new Layout(KernelsEngine(), numQuad * elements)));
+   Eval(quadCoeff);
+   return quadCoeff;
+}
+
+void KernelsCoefficient::Eval(Vector &quadCoeff)
+{
+   assert(false);
+}
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/fem/coefficient.hpp

@@ -0,0 +1,256 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_COEFFICIENT_HPP
+#define MFEM_BACKENDS_KERNELS_COEFFICIENT_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+class KernelsIntegrator;
+
+
+class KernelsParameter
+{
+public:
+   virtual ~KernelsParameter();
+
+   virtual KernelsParameter* Clone() = 0;
+
+   virtual void Setup(KernelsIntegrator &integ);
+
+   //virtual ::kernels::kernelArg KernelArgs();
+};
+
+
+//---[ Include Parameter ]------------
+class KernelsIncludeParameter : public KernelsParameter
+{
+private:
+   std::string filename;
+
+public:
+   KernelsIncludeParameter(const std::string &filename_);
+
+   virtual KernelsParameter* Clone();
+
+   virtual void Setup(KernelsIntegrator &integ);
+};
+//====================================
+
+
+//---[ Source Parameter ]------------
+class KernelsSourceParameter : public KernelsParameter
+{
+private:
+   std::string source;
+
+public:
+   KernelsSourceParameter(const std::string &filename_);
+
+   virtual KernelsParameter* Clone();
+
+   virtual void Setup(KernelsIntegrator &integ);
+};
+//====================================
+
+
+//---[ Define Parameter ]------------
+template <class TM>
+class KernelsDefineParameter : public KernelsParameter
+{
+private:
+   const std::string name;
+   TM value;
+
+public:
+   KernelsDefineParameter(const std::string &name_,
+                          const TM &value_) :
+      name(name_),
+      value(value_) {}
+
+   virtual KernelsParameter* Clone()
+   {
+      return new KernelsDefineParameter(name, value);
+   }
+
+   virtual void Setup(KernelsIntegrator &integ)
+   {
+   }
+};
+//====================================
+
+
+//---[ Variable Parameter ]-----------
+template <class TM>
+class KernelsVariableParameter : public KernelsParameter
+{
+private:
+   const std::string name;
+   const TM &value;
+
+public:
+   KernelsVariableParameter(const std::string &name_,
+                            const TM &value_) :
+      name(name_),
+      value(value_) {}
+
+   virtual KernelsParameter* Clone()
+   {
+      return new KernelsVariableParameter(name, value);
+   }
+
+   virtual void Setup(KernelsIntegrator &integ)
+   {
+      assert(false);
+   }
+
+
+};
+//====================================
+
+
+//---[ Vector Parameter ]-------
+class KernelsVectorParameter : public KernelsParameter
+{
+private:
+   const std::string name;
+   Vector v;
+   bool useRestrict;
+   std::string attr;
+
+public:
+   KernelsVectorParameter(const std::string &name_,
+                          Vector &v_,
+                          const bool useRestrict_ = false);
+
+   KernelsVectorParameter(const std::string &name_,
+                          Vector &v_,
+                          const std::string &attr_,
+                          const bool useRestrict_ = false);
+
+   virtual KernelsParameter* Clone();
+
+   virtual void Setup(KernelsIntegrator &integ);
+};
+//====================================
+
+
+//---[ GridFunction Parameter ]-------
+class KernelsGridFunctionParameter : public KernelsParameter
+{
+private:
+   const std::string name;
+   KernelsGridFunction &gf;
+   Vector gfQuad;
+   bool useRestrict;
+
+public:
+   KernelsGridFunctionParameter(const std::string &name_,
+                                KernelsGridFunction &gf_,
+                                const bool useRestrict_ = false);
+
+   virtual KernelsParameter* Clone();
+
+   virtual void Setup(KernelsIntegrator &integ);
+};
+//====================================
+
+
+//---[ Coefficient ]------------------
+// [MISSING]
+// Needs to know about the integrator's
+//   - fespace
+//   - ir
+// Step where parameters that need the ir get called for setup
+// For example, GridFunction (d, e) -> (q, e)
+class KernelsCoefficient
+{
+private:
+   SharedPtr<const Engine> engine;
+
+   KernelsIntegrator *integ;
+
+   std::string name;
+
+   std::vector<KernelsParameter*> params;
+
+public:
+   KernelsCoefficient(const double value = 1.0);
+   KernelsCoefficient(const Engine &e, const double value = 1.0);
+   KernelsCoefficient(const Engine &e, const std::string &source);
+   KernelsCoefficient(const Engine &e, const char *source);
+   ~KernelsCoefficient();
+
+   KernelsCoefficient(const KernelsCoefficient &coeff);
+
+   const Engine &KernelsEngine() const { return *engine; }
+
+   kernels::device GetDevice(int idx = 0) const
+   { return engine->GetDevice(idx); }
+
+   KernelsCoefficient& SetName(const std::string &name_);
+
+   void Setup(KernelsIntegrator &integ_);
+
+   KernelsCoefficient& Add(KernelsParameter *param);
+
+   KernelsCoefficient& IncludeHeader(const std::string &filename);
+   KernelsCoefficient& IncludeSource(const std::string &source);
+
+   template <class TM>
+   KernelsCoefficient& AddDefine(const std::string &name_, const TM &value)
+   {
+      return Add(new KernelsDefineParameter<TM>(name_, value));
+   }
+
+   template <class TM>
+   KernelsCoefficient& AddVariable(const std::string &name_, const TM &value)
+   {
+      return Add(new KernelsVariableParameter<TM>(name_, value));
+   }
+
+   KernelsCoefficient& AddVector(const std::string &name_,
+                                 Vector &v,
+                                 const bool useRestrict = false);
+
+
+   KernelsCoefficient& AddVector(const std::string &name_,
+                                 Vector &v,
+                                 const std::string &attr,
+                                 const bool useRestrict = false);
+
+   KernelsCoefficient& AddGridFunction(const std::string &name_,
+                                       KernelsGridFunction &gf,
+                                       const bool useRestrict = false);
+
+   bool IsConstant();
+   double GetConstantValue();
+
+   Vector Eval();
+   void Eval(Vector &quadCoeff);
+
+   //operator ::kernels::kernelArg ();
+};
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_COEFFICIENT_HPP

backends/kernels/fem/doftoquad.cpp

@@ -0,0 +1,320 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// ***************************************************************************
+// * kDofQuadMaps
+// ***************************************************************************
+static std::map<std::string, kDofQuadMaps* > AllDofQuadMaps;
+
+// ***************************************************************************
+kDofQuadMaps::~kDofQuadMaps() {}
+
+// *****************************************************************************
+void kDofQuadMaps::delkDofQuadMaps()
+{
+   for (std::map<std::string,
+        kDofQuadMaps*>::iterator itr = AllDofQuadMaps.begin();
+        itr != AllDofQuadMaps.end();
+        itr++)
+   {
+      delete itr->second;
+   }
+}
+
+// *****************************************************************************
+kDofQuadMaps* kDofQuadMaps::Get(const FiniteElementSpace& fes,
+                                const IntegrationRule& ir,
+                                const bool transpose)
+{
+   return Get(*fes.GetFE(0), *fes.GetFE(0), ir, transpose);
+}
+
+kDofQuadMaps* kDofQuadMaps::Get(const FiniteElementSpace& trialFES,
+                                const FiniteElementSpace& testFES,
+                                const IntegrationRule& ir,
+                                const bool transpose)
+{
+   return Get(*trialFES.GetFE(0), *testFES.GetFE(0), ir, transpose);
+}
+
+kDofQuadMaps* kDofQuadMaps::Get(const FiniteElement& trialFE,
+                                const FiniteElement& testFE,
+                                const IntegrationRule& ir,
+                                const bool transpose)
+{
+   return GetTensorMaps(trialFE, testFE, ir, transpose);
+}
+
+// ***************************************************************************
+kDofQuadMaps* kDofQuadMaps::GetTensorMaps(const FiniteElement& trialFE,
+                                          const FiniteElement& testFE,
+                                          const IntegrationRule& ir,
+                                          const bool transpose)
+{
+   const TensorBasisElement& trialTFE =
+      dynamic_cast<const TensorBasisElement&>(trialFE);
+   const TensorBasisElement& testTFE =
+      dynamic_cast<const TensorBasisElement&>(testFE);
+   std::stringstream ss;
+   ss << "TensorMap:"
+      << " O1:"  << trialFE.GetOrder()
+      << " O2:"  << testFE.GetOrder()
+      << " BT1:" << trialTFE.GetBasisType()
+      << " BT2:" << testTFE.GetBasisType()
+      << " Q:"   << ir.GetNPoints();
+   std::string hash = ss.str();
+   // If we've already made the dof-quad maps, reuse them
+   if (AllDofQuadMaps.find(hash)!=AllDofQuadMaps.end())
+   {
+      return AllDofQuadMaps[hash];
+   }
+   // Otherwise, build them
+   kDofQuadMaps *maps = new kDofQuadMaps();
+   AllDofQuadMaps[hash]=maps;
+   maps->hash = hash;
+   nvtx_push();
+   const kDofQuadMaps* trialMaps = GetD2QTensorMaps(trialFE, ir);
+   const kDofQuadMaps* testMaps  = GetD2QTensorMaps(testFE, ir, true);
+   maps->dofToQuad   = trialMaps->dofToQuad;
+   maps->dofToQuadD  = trialMaps->dofToQuadD;
+   maps->quadToDof   = testMaps->dofToQuad;
+   maps->quadToDofD  = testMaps->dofToQuadD;
+   maps->quadWeights = testMaps->quadWeights;
+   nvtx_pop();
+   return maps;
+}
+
+// ***************************************************************************
+kDofQuadMaps* kDofQuadMaps::GetD2QTensorMaps(const FiniteElement& fe,
+                                             const IntegrationRule& ir,
+                                             const bool transpose)
+{
+   const IntegrationRule& ir1D = IntRules.Get(Geometry::SEGMENT,ir.GetOrder());
+
+   const int dims = fe.GetDim();
+   const int order = fe.GetOrder();
+   const int numDofs = order + 1;
+   const int numQuad1D = ir1D.GetNPoints();
+   const int numQuad2D = numQuad1D * numQuad1D;
+   const int numQuad3D = numQuad2D * numQuad1D;
+   const int numQuad =
+      (dims == 1) ? numQuad1D :
+      (dims == 2) ? numQuad2D :
+      (dims == 3) ? numQuad3D : 0;
+   assert(numQuad > 0);
+
+   std::stringstream ss;
+   ss << "D2QTensorMap:"
+      << " dims:" << dims
+      << " order:" << order
+      << " numDofs:" << numDofs
+      << " numQuad1D:" << numQuad1D
+      << " transpose:"  << (transpose?"true":"false");
+   std::string hash = ss.str();
+
+   if (AllDofQuadMaps.find(hash)!=AllDofQuadMaps.end())
+   {
+      return AllDofQuadMaps[hash];
+   }
+
+   kDofQuadMaps *maps = new kDofQuadMaps();
+   AllDofQuadMaps[hash]=maps;
+   maps->hash = hash;
+   nvtx_push();
+
+   maps->dofToQuad.allocate( numQuad1D, numDofs, 1, 1, transpose);
+   maps->dofToQuadD.allocate(numQuad1D, numDofs, 1, 1, transpose);
+   const int dim0 = maps->dofToQuad.dim()[0];
+   const int dim1 = maps->dofToQuad.dim()[1];
+
+   if (transpose) // Initialize quad weights only for transpose
+   {
+      maps->quadWeights.allocate(numQuad);
+   }
+   mfem::Vector d2q(numDofs);
+   mfem::Vector d2qD(numDofs);
+   mfem::Array<double> quadWeights1D(numQuad1D);
+   mfem::Array<double> dofToQuad(numQuad1D*numDofs);
+   mfem::Array<double> dofToQuadD(numQuad1D*numDofs);
+   const TensorBasisElement& tbe = dynamic_cast<const TensorBasisElement&>(fe);
+   const Poly_1D::Basis& basis = tbe.GetBasis1D();
+
+   for (int q = 0; q < numQuad1D; ++q)
+   {
+      const IntegrationPoint& ip = ir1D.IntPoint(q);
+      if (transpose)
+      {
+         quadWeights1D[q] = ip.weight;
+      }
+      basis.Eval(ip.x, d2q, d2qD);
+      for (int d = 0; d < numDofs; ++d)
+      {
+         const double w = d2q[d];
+         const double wD = d2qD[d];
+         const int idx = dim0*q + dim1*d;
+         dofToQuad[idx] = w;
+         dofToQuadD[idx] = wD;
+      }
+   }
+   if (transpose)
+   {
+      mfem::Array<double> quadWeights(numQuad);
+      for (int q = 0; q < numQuad; ++q)
+      {
+         const int qx = q % numQuad1D;
+         const int qz = q / numQuad2D;
+         const int qy = (q - qz*numQuad2D) / numQuad1D;
+         double w = quadWeights1D[qx];
+         if (dims > 1) { w *= quadWeights1D[qy]; }
+         if (dims > 2) { w *= quadWeights1D[qz]; }
+         quadWeights[q] = w;
+      }
+      maps->quadWeights = quadWeights;
+   }
+   maps->dofToQuad = dofToQuad;
+   maps->dofToQuadD = dofToQuadD;
+   nvtx_pop();
+   return maps;
+}
+
+// ***************************************************************************
+kDofQuadMaps* kDofQuadMaps::GetSimplexMaps(const FiniteElement& fe,
+                                           const IntegrationRule& ir,
+                                           const bool transpose)
+{
+   return GetSimplexMaps(fe, fe, ir, transpose);
+}
+
+// *****************************************************************************
+kDofQuadMaps* kDofQuadMaps::GetSimplexMaps(const FiniteElement& trialFE,
+                                           const FiniteElement& testFE,
+                                           const IntegrationRule& ir,
+                                           const bool transpose)
+{
+   std::stringstream ss;
+   ss << "SimplexMap:"
+      << " O1:" << trialFE.GetOrder()
+      << " O2:" << testFE.GetOrder()
+      << " Q:"  << ir.GetNPoints();
+   std::string hash = ss.str();
+   // If we've already made the dof-quad maps, reuse them
+   if (AllDofQuadMaps.find(hash)!=AllDofQuadMaps.end())
+   {
+      return AllDofQuadMaps[hash];
+   }
+   nvtx_push();
+   kDofQuadMaps *maps = new kDofQuadMaps();
+   AllDofQuadMaps[hash]=maps;
+   maps->hash = hash;
+   const kDofQuadMaps* trialMaps = GetD2QSimplexMaps(trialFE, ir);
+   const kDofQuadMaps* testMaps  = GetD2QSimplexMaps(testFE, ir, true);
+   maps->dofToQuad   = trialMaps->dofToQuad;
+   maps->dofToQuadD  = trialMaps->dofToQuadD;
+   maps->quadToDof   = testMaps->dofToQuad;
+   maps->quadToDofD  = testMaps->dofToQuadD;
+   maps->quadWeights = testMaps->quadWeights;
+   nvtx_pop();
+   return maps;
+}
+
+// ***************************************************************************
+kDofQuadMaps* kDofQuadMaps::GetD2QSimplexMaps(const FiniteElement& fe,
+                                              const IntegrationRule& ir,
+                                              const bool transpose)
+{
+   const int dims = fe.GetDim();
+   const int numDofs = fe.GetDof();
+   const int numQuad = ir.GetNPoints();
+
+   std::stringstream ss ;
+   ss << "D2QSimplexMap:"
+      << " Dim:" << dims
+      << " numDofs:" << numDofs
+      << " numQuad:" << numQuad
+      << " transpose:" << (transpose?"true":"false");
+   std::string hash = ss.str();
+
+   if (AllDofQuadMaps.find(hash)!=AllDofQuadMaps.end())
+   {
+      return AllDofQuadMaps[hash];
+   }
+
+   kDofQuadMaps* maps = new kDofQuadMaps();
+   AllDofQuadMaps[hash]=maps;
+   maps->hash = hash;
+   nvtx_push();
+
+   maps->dofToQuad.allocate( numQuad, numDofs,       1, 1, transpose);
+   maps->dofToQuadD.allocate(   dims, numQuad, numDofs, 1, transpose);
+   const int dim0 = maps->dofToQuad.dim()[0];
+   const int dim1 = maps->dofToQuad.dim()[1];
+   const int dim0D = maps->dofToQuadD.dim()[0];
+   const int dim1D = maps->dofToQuadD.dim()[1];
+   const int dim2D = maps->dofToQuadD.dim()[2];
+
+   if (transpose) // Initialize quad weights only for transpose
+   {
+      maps->quadWeights.allocate(numQuad);
+   }
+   mfem::Vector d2q(numDofs);
+   mfem::DenseMatrix d2qD(numDofs, dims);
+   mfem::Array<double> quadWeights(numQuad);
+   mfem::Array<double> dofToQuad(numQuad*numDofs);
+   mfem::Array<double> dofToQuadD(dims*numQuad*numDofs);
+
+   for (int q = 0; q < numQuad; ++q)
+   {
+      const IntegrationPoint& ip = ir.IntPoint(q);
+      if (transpose)
+      {
+         quadWeights[q] = ip.weight;
+      }
+      fe.CalcShape(ip, d2q);
+      fe.CalcDShape(ip, d2qD);
+      for (int d = 0; d < numDofs; ++d)
+      {
+         const double w = d2q[d];
+         const int idx = dim0*q + dim1*d;
+         dofToQuad[idx] = w;
+         for (int dim = 0; dim < dims; ++dim)
+         {
+            const double wD = d2qD(d, dim);
+            const int idxD = dim0D*dim + dim1D*q + dim2D*d;
+            dofToQuadD[idxD] = wD;
+         }
+      }
+   }
+   if (transpose)
+   {
+      maps->quadWeights = quadWeights;
+   }
+   maps->dofToQuad = dofToQuad;
+   maps->dofToQuadD = dofToQuadD;
+   nvtx_pop();
+   return maps;
+}
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/fem/doftoquad.hpp

@@ -0,0 +1,74 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_BILIN_DQM_HPP
+#define MFEM_BACKENDS_KERNELS_BILIN_DQM_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// ***************************************************************************
+// * kDofQuadMaps
+// ***************************************************************************
+class kDofQuadMaps
+{
+private:
+   std::string hash;
+public:
+   kernels::array<double, false> dofToQuad, dofToQuadD; // B
+   kernels::array<double, false> quadToDof, quadToDofD; // B^T
+   kernels::array<double> quadWeights;
+public:
+   ~kDofQuadMaps();
+   static void delkDofQuadMaps();
+   static kDofQuadMaps* Get(const mfem::FiniteElementSpace&,
+                            const mfem::IntegrationRule&,
+                            const bool = false);
+   static kDofQuadMaps* Get(const mfem::FiniteElementSpace&,
+                            const mfem::FiniteElementSpace&,
+                            const mfem::IntegrationRule&,
+                            const bool = false);
+   static kDofQuadMaps* Get(const mfem::FiniteElement&,
+                            const mfem::FiniteElement&,
+                            const mfem::IntegrationRule&,
+                            const bool = false);
+   static kDofQuadMaps* GetTensorMaps(const mfem::FiniteElement&,
+                                      const mfem::FiniteElement&,
+                                      const mfem::IntegrationRule&,
+                                      const bool = false);
+   static kDofQuadMaps* GetD2QTensorMaps(const mfem::FiniteElement&,
+                                         const mfem::IntegrationRule&,
+                                         const bool = false);
+   static kDofQuadMaps* GetSimplexMaps(const mfem::FiniteElement&,
+                                       const mfem::IntegrationRule&,
+                                       const bool = false);
+   static kDofQuadMaps* GetSimplexMaps(const mfem::FiniteElement&,
+                                       const mfem::FiniteElement&,
+                                       const mfem::IntegrationRule&,
+                                       const bool = false);
+   static kDofQuadMaps* GetD2QSimplexMaps(const mfem::FiniteElement&,
+                                          const mfem::IntegrationRule&,
+                                          const bool = false);
+};
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_BILIN_DQM_HPP

backends/kernels/fem/fespace.cpp

@@ -0,0 +1,222 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+
+// **************************************************************************
+kFiniteElementSpace::
+kFiniteElementSpace(const Engine& e,
+                    mfem::FiniteElementSpace& fespace)
+   : PFiniteElementSpace(e, fespace),
+     e_layout(e, 0),
+     globalDofs(fes->GetNDofs()),
+     localDofs(GetFE(0)->GetDof()),
+     vdim(fespace.GetVDim()),
+     ordering(fespace.GetOrdering()),
+     offsets(globalDofs+1),
+     indices(localDofs, GetNE()),
+     map(localDofs, GetNE())
+{
+   nvtx_push(PowderBlue);
+   dbg("\033[7m[kFiniteElementSpace]");
+   const mfem::FiniteElement& fe = *(fes->GetFE(0));
+   const mfem::TensorBasisElement* el =
+      dynamic_cast<const mfem::TensorBasisElement*>(&fe);
+   if( !el ) return; //RYAN -- engines does not support non-tensor basis yet
+   const mfem::Array<int>& dof_map = el->GetDofMap();
+   const bool dof_map_is_identity = (dof_map.Size()==0);
+
+   const Table& e2dTable = fes->GetElementToDofTable();
+   const int* elementMap = e2dTable.GetJ();
+   const int elements = GetNE();
+   mfem::Array<int> h_offsets(globalDofs+1);
+
+   const int e_size = localDofs * elements * fes->GetVDim();
+   e_layout.Resize(e_size);
+   dbg("\033[7m[kFiniteElementSpace] e_size/fes->GetVDim()=%d",
+       e_size/fes->GetVDim());
+
+   // We'll be keeping a count of how many local nodes point to its global dof
+   for (int i = 0; i <= globalDofs; ++i)
+   {
+      h_offsets[i] = 0;
+   }
+   for (int e = 0; e < elements; ++e)
+   {
+      for (int d = 0; d < localDofs; ++d)
+      {
+         const int gid = elementMap[localDofs*e + d];
+         ++h_offsets[gid + 1];
+      }
+   }
+   dbg("Aggregate to find offsets for each global dof");
+   for (int i = 1; i <= globalDofs; ++i)
+   {
+      h_offsets[i] += h_offsets[i - 1];
+   }
+
+   mfem::Array<int> h_indices(localDofs*elements);
+   mfem::Array<int> h_map(localDofs*elements);
+   // For each global dof, fill in all local nodes that point to it
+   for (int e = 0; e < elements; ++e)
+   {
+      for (int d = 0; d < localDofs; ++d)
+      {
+         const int did = dof_map_is_identity?d:dof_map[d];
+         const int gid = elementMap[localDofs*e + did];
+         const int lid = localDofs*e + d;
+         h_indices[h_offsets[gid]++] = lid;
+         h_map[lid] = gid;
+      }
+   }
+
+   // We shifted the offsets vector by 1 by using it as a counter
+   // Now we shift it back.
+   for (int i = globalDofs; i > 0; --i)
+   {
+      h_offsets[i] = h_offsets[i - 1];
+   }
+   h_offsets[0] = 0;
+
+   dbg("offsets, indices copy");
+   offsets = h_offsets;
+   indices = h_indices;   map = h_map;
+
+   if (kernels::config::Get().IAmAlone())
+   {
+      dbg("\033[7mIAmAlone: Switching to IdentityOperator!");
+      restrictionOp = new IdentityOperator(KernelsTrueVLayout());
+      prolongationOp = new IdentityOperator(KernelsTrueVLayout());
+      nvtx_pop();
+      return;
+   }
+
+   Layout &v_layout = KernelsVLayout();
+   Layout &t_layout = KernelsTrueVLayout();
+
+   dbg("\033[7mR");
+   const mfem::SparseMatrix* R = fes->GetRestrictionMatrix();
+   dbg("\033[7mP");
+   const kConformingProlongationOperator *P =
+      new kConformingProlongationOperator(t_layout,v_layout,this->GetParFESpace());
+
+   assert(R);
+   assert(P);
+
+   dbg("locals");
+   const int mHeight = R->Height();
+   const int* I = R->GetI();
+   const int* J = R->GetJ();
+   int trueCount = 0;
+
+   dbg("trueCount for-loop");
+   for (int i = 0; i < mHeight; ++i)
+   {
+      trueCount += ((I[i + 1] - I[i]) == 1);
+   }
+
+   dbg("h_reorderIndices");
+   mfem::Array<int> h_reorderIndices(2*trueCount);
+   for (int i = 0, trueIdx=0; i < mHeight; ++i)
+   {
+      if ((I[i + 1] - I[i]) == 1)
+      {
+         h_reorderIndices[trueIdx++] = J[I[i]];
+         h_reorderIndices[trueIdx++] = i;
+      }
+   }
+
+   dbg("reorderIndices");
+   reorderIndices = ::new kernels::array<int>(2*trueCount);
+   dbg("*=h");
+   *reorderIndices = h_reorderIndices;
+
+   dbg("\033[7mRestrictionOperator asserts");
+   dbg("\033[7mRestrictionOperator R->Width()=%d",R->Width());
+   dbg("\033[7mRestrictionOperator R->Height()=%d",R->Height());
+   //const kernels::Engine &engine = KernelsEngine();
+   //kernels::Layout width = *engine.MakeLayout(R->Width()).As<kernels::Layout>();
+   //kernels::Layout height = *engine.MakeLayout(R->Height()).As<kernels::Layout>();
+   //assert(R->InLayout().As<Layout>()->Size()==(std::size_t)R->Width());
+   //assert(R->OutLayout().As<Layout>()->Size()==(std::size_t)R->Height());
+   dbg("\033[7mRestrictionOperator new");
+
+   restrictionOp = new kernels::RestrictionOperator(v_layout,t_layout,
+                                                    reorderIndices);
+   //(width, height, reorderIndices);
+
+   dbg("\033[7mProlongationOperator");
+   prolongationOp = new kernels::ProlongationOperator(P);
+   dbg("done");
+   nvtx_pop();
+}
+
+// **************************************************************************
+kFiniteElementSpace::~kFiniteElementSpace()
+{
+   delete restrictionOp;
+   delete prolongationOp;
+}
+
+// **************************************************************************
+void kFiniteElementSpace::GlobalToLocal(const Vector& globalVec,
+                                        Vector& localVec) const
+{
+   nvtx_push(PowderBlue);
+   const int vdim = GetVDim();
+   const int localEntries = localDofs * GetNE();
+   const bool vdim_ordering = ordering == Ordering::byVDIM;
+   rGlobalToLocal(vdim,
+                  vdim_ordering,
+                  globalDofs,
+                  localEntries,
+                  offsets,
+                  indices,
+                  (const double*)globalVec.KernelsMem().ptr(),
+                  (double*)localVec.KernelsMem().ptr());
+   nvtx_pop();
+}
+
+// **************************************************************************
+void kFiniteElementSpace::LocalToGlobal(const Vector& localVec,
+                                        Vector& globalVec) const
+{
+   nvtx_push(PowderBlue);
+   const int vdim = GetVDim();
+   const int localEntries = localDofs * GetNE();
+   const bool vdim_ordering = ordering == Ordering::byVDIM;
+   rLocalToGlobal(vdim,
+                  vdim_ordering,
+                  globalDofs,
+                  localEntries,
+                  offsets,
+                  indices,
+                  (const double*)localVec.KernelsMem().ptr(),
+                  (double*)globalVec.KernelsMem().ptr());
+   nvtx_pop();
+}
+
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/fem/fespace.hpp

@@ -0,0 +1,130 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_FESPACE_HPP
+#define MFEM_BACKENDS_KERNELS_FESPACE_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// *****************************************************************************
+class kFiniteElementSpace : public mfem::PFiniteElementSpace
+{
+protected:
+   Layout e_layout;
+   int globalDofs, localDofs;
+   int vdim;
+   mfem::Ordering::Type ordering;
+   kernels::array<int> offsets;
+   kernels::array<int> indices,*reorderIndices;
+   kernels::array<int> map;
+   mfem::Operator *restrictionOp, *prolongationOp;
+public:
+   /// TODO: doxygen
+   kFiniteElementSpace(const Engine&, mfem::FiniteElementSpace&);
+
+   /// Virtual destructor
+   virtual ~kFiniteElementSpace();
+
+   /// TODO: doxygen
+   const kernels::Engine &KernelsEngine() const
+   { return *static_cast<const Engine *>(engine.Get()); }
+
+   /// TODO: doxygen
+   kernels::device GetDevice(int idx = 0) const
+   { return KernelsEngine().GetDevice(idx); }
+
+   mfem::Mesh* GetMesh() const { return fes->GetMesh(); }
+
+   mfem::FiniteElementSpace* GetFESpace() const { return fes; }
+
+#ifdef MFEM_USE_MPI
+   mfem::ParFiniteElementSpace& GetParFESpace() const
+   {
+      return *static_cast<ParFiniteElementSpace*>(fes);
+   }
+#endif
+
+   Layout &KernelsVLayout() const
+   { return *fes->GetVLayout().As<Layout>(); }
+
+   Layout &KernelsTrueVLayout() const
+   { return *fes->GetTrueVLayout().As<Layout>(); }
+
+   Layout &KernelsEVLayout() { return e_layout; }
+
+#ifdef MFEM_USE_MPI
+   bool isDistributed() const
+   {
+      return (KernelsEngine().GetComm() != MPI_COMM_NULL);
+   }
+#else
+   bool isDistributed() const { return false; }
+#endif
+
+   bool hasTensorBasis() const
+   {
+      return dynamic_cast<const mfem::TensorBasisElement*>(fes->GetFE(0));
+   }
+
+   mfem::Ordering::Type GetOrdering() const { return ordering; }
+
+   int GetGlobalDofs() const { return globalDofs; }
+   int GetLocalDofs() const { return localDofs; }
+
+   int GetDim() const { return fes->GetMesh()->Dimension(); }
+   int GetVDim() const { return vdim; }
+
+   int GetVSize() const { return globalDofs * vdim; }
+   int GetTrueVSize() const { return fes->GetTrueVSize(); }
+   int GetGlobalVSize() const { return globalDofs*vdim; /* FIXME: MPI */ }
+   int GetGlobalTrueVSize() const { return fes->GetTrueVSize(); }
+
+   int GetNE() const { return fes->GetNE(); }
+
+   const mfem::FiniteElementCollection* FEColl() const
+   { return fes->FEColl(); }
+
+   const mfem::FiniteElement* GetFE(const int idx) const
+   { return fes->GetFE(idx); }
+
+   const mfem::Operator* GetRestrictionOperator() const { return restrictionOp; }
+   const mfem::Operator* GetProlongationOperator() const { return prolongationOp; }
+
+   virtual const mfem::Operator *GetInterpolationOperator(
+      const mfem::QuadratureSpace &qspace) const
+   { return NULL; /* FIXME */ }
+
+   virtual const mfem::Operator *GetGradientOperator(
+      const mfem::QuadratureSpace &qspace) const
+   { return NULL; /* FIXME */ }
+
+   const kernels::array<int> GetLocalToGlobalMap() const
+   { return map; }
+
+   void GlobalToLocal(const kernels::Vector &global, kernels::Vector &local) const;
+   void LocalToGlobal(const kernels::Vector &local, kernels::Vector &global) const;
+
+};
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_FESPACE_HPP

backends/kernels/fem/geom.cpp

@@ -0,0 +1,223 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+// *****************************************************************************
+static kGeometry *geom=NULL;
+
+// ***************************************************************************
+// * ~ kGeometry
+// ***************************************************************************
+kGeometry::~kGeometry()
+{
+   nvtx_push();
+   free(geom->meshNodes);
+   free(geom->J);
+   free(geom->invJ);
+   free(geom->detJ);
+   delete[] geom;
+   nvtx_pop();
+}
+
+// *****************************************************************************
+// * kGeometry Get: use this one to fetch nodes from vector Sx
+// *****************************************************************************
+kGeometry* kGeometry::Get(kFiniteElementSpace& fes,
+                          const IntegrationRule& ir,
+                          const kvector& Sx)
+{
+   nvtx_push();
+   const Mesh *mesh = fes.GetFESpace()->GetMesh();
+   const GridFunction *nodes = mesh->GetNodes();
+   const mfem::FiniteElementSpace *fespace = nodes->FESpace();
+   const mfem::FiniteElement *fe = fespace->GetFE(0);
+   const int dims     = fe->GetDim();
+   const int numDofs  = fe->GetDof();
+   const int numQuad  = ir.GetNPoints();
+   const int elements = fespace->GetNE();
+   const int ndofs    = fespace->GetNDofs();
+   const kDofQuadMaps* maps = kDofQuadMaps::GetSimplexMaps(*fe, ir);
+   nvtx_push();
+   rNodeCopyByVDim(elements,numDofs,ndofs,dims,geom->eMap,Sx,geom->meshNodes);
+   nvtx_pop();
+   nvtx_push();
+   rIniGeom(dims,numDofs,numQuad,elements,
+            maps->dofToQuadD,
+            geom->meshNodes,
+            geom->J,
+            geom->invJ,
+            geom->detJ);
+   nvtx_pop();
+   nvtx_pop();
+   return geom;
+}
+
+
+// *****************************************************************************
+kGeometry* kGeometry::Get(kFiniteElementSpace& fes,
+                          const IntegrationRule& ir)
+{
+   nvtx_push();
+   Mesh& mesh = *(fes.GetFESpace()->GetMesh());
+   const bool geom_to_allocate =
+      (!geom) || config::Get().GeomNeedsUpdate(mesh.GetSequence());
+   if (geom_to_allocate)
+   {
+      dbg("geom_to_allocate: new kGeometry");
+      geom = new kGeometry();
+   }
+   if (!mesh.GetNodes()) { mesh.SetCurvature(1, false, -1, Ordering::byVDIM); }
+   GridFunction& nodes = *(mesh.GetNodes());
+   const mfem::FiniteElementSpace& fespace = *(nodes.FESpace());
+   const mfem::FiniteElement& fe = *(fespace.GetFE(0));
+   const int dims     = fe.GetDim();
+   const int elements = fespace.GetNE();
+   const int numDofs  = fe.GetDof();
+   const int numQuad  = ir.GetNPoints();
+   const bool orderedByNODES = (fespace.GetOrdering() == Ordering::byNODES);
+   dbg("orderedByNODES: %s", orderedByNODES?"true":"false");
+
+   if (orderedByNODES)
+   {
+      dbg("orderedByNODES, ReorderByVDim");
+      ReorderByVDim(nodes);
+   }
+   const int asize = dims*numDofs*elements;
+   mfem::Array<double> meshNodes(asize);
+   const Table& e2dTable = fespace.GetElementToDofTable();
+   const int* elementMap = e2dTable.GetJ();
+   mfem::Array<int> eMap(numDofs*elements);
+   {
+      nvtx_push();
+      for (int e = 0; e < elements; ++e)
+      {
+         for (int d = 0; d < numDofs; ++d)
+         {
+            const int lid = d+numDofs*e;
+            const int gid = elementMap[lid];
+            eMap[lid]=gid;
+            for (int v = 0; v < dims; ++v)
+            {
+               const int moffset = v+dims*lid;
+               const int xoffset = v+dims*gid;
+               meshNodes[moffset] = nodes[xoffset];
+            }
+         }
+      }
+      nvtx_pop();
+   }
+   if (geom_to_allocate)
+   {
+      geom->meshNodes.allocate(dims, numDofs, elements);
+      geom->eMap.allocate(numDofs, elements);
+   }
+   {
+      nvtx_push();
+      geom->meshNodes = meshNodes;
+      geom->eMap = eMap;
+      nvtx_pop();
+   }
+
+   // Reorder the original gf back
+   if (orderedByNODES)
+   {
+      dbg("Reorder the original gf back");
+      ReorderByNodes(nodes);
+   }
+
+   if (geom_to_allocate)
+   {
+      dbg("geom_to_allocate: J, invJ & detJ");
+      geom->J.allocate(dims, dims, numQuad, elements);
+      geom->invJ.allocate(dims, dims, numQuad, elements);
+      geom->detJ.allocate(numQuad, elements);
+   }
+
+   const kDofQuadMaps* maps = kDofQuadMaps::GetSimplexMaps(fe, ir);
+   assert(maps);
+   {
+      dbg("dims=%d, numDofs=%d, numQuad=%d, elements=%d",dims,numDofs,numQuad,
+          elements);
+      nvtx_push(rIniGeom,SteelBlue);
+      rIniGeom(dims,numDofs,numQuad,elements,
+               maps->dofToQuadD,
+               geom->meshNodes,
+               geom->J,
+               geom->invJ,
+               geom->detJ);
+      nvtx_pop();
+   }
+   nvtx_pop();
+   return geom;
+}
+
+// ***************************************************************************
+void kGeometry::ReorderByVDim(GridFunction& nodes)
+{
+   nvtx_push();
+   const mfem::FiniteElementSpace *fes=nodes.FESpace();
+   const int size = nodes.Size();
+   const int vdim = fes->GetVDim();
+   const int ndofs = fes->GetNDofs();
+   double *data = nodes.GetData();
+   double *temp = new double[size];
+   int k=0;
+   for (int d = 0; d < ndofs; d++)
+      for (int v = 0; v < vdim; v++)
+      {
+         temp[k++] = data[d+v*ndofs];
+      }
+   for (int i = 0; i < size; i++)
+   {
+      data[i] = temp[i];
+   }
+   delete [] temp;
+   nvtx_pop();
+}
+
+// ***************************************************************************
+void kGeometry::ReorderByNodes(GridFunction& nodes)
+{
+   nvtx_push();
+   const mfem::FiniteElementSpace *fes=nodes.FESpace();
+   const int size = nodes.Size();
+   const int vdim = fes->GetVDim();
+   const int ndofs = fes->GetNDofs();
+   double *data = nodes.GetData();
+   double *temp = new double[size];
+   int k = 0;
+   for (int j = 0; j < ndofs; j++)
+      for (int i = 0; i < vdim; i++)
+      {
+         temp[j+i*ndofs] = data[k++];
+      }
+   for (int i = 0; i < size; i++)
+   {
+      data[i] = temp[i];
+   }
+   delete [] temp;
+   nvtx_pop();
+}
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/fem/geom.hpp

@@ -0,0 +1,53 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_BILIN_GEOM_HPP
+#define MFEM_BACKENDS_KERNELS_BILIN_GEOM_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// ***************************************************************************
+// * kGeometry
+// ***************************************************************************
+class kGeometry
+{
+public:
+   ~kGeometry();
+   kernels::array<int> eMap;
+   kernels::array<double> meshNodes;
+   kernels::array<double> J, invJ, detJ;
+   static const int Jacobian    = (1 << 0);
+   static const int JacobianInv = (1 << 1);
+   static const int JacobianDet = (1 << 2);
+   static kGeometry* Get(kFiniteElementSpace&,
+                         const IntegrationRule&);
+   static kGeometry* Get(kFiniteElementSpace&,
+                         const IntegrationRule&,
+                         const kvector&);
+   static void ReorderByVDim(GridFunction& nodes);
+   static void ReorderByNodes(GridFunction& nodes);
+};
+
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_BILIN_GEOM_HPP

backends/kernels/fem/gridfunc.cpp

@@ -0,0 +1,132 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+KernelsGridFunction::KernelsGridFunction(kFiniteElementSpace &f)
+   : PArray(f.KernelsVLayout()),
+     Array(f.KernelsVLayout(), sizeof(double)),
+     Vector(f.KernelsVLayout()),
+     fes(f),
+     sequence(0) {nvtx_push(); nvtx_pop();}
+
+KernelsGridFunction::KernelsGridFunction(const KernelsGridFunction &v)
+   : PArray(v),
+     Array(v),
+     Vector(v),
+     fes(v.fes),
+     sequence(v.sequence) {nvtx_push(); nvtx_pop();}
+
+KernelsGridFunction& KernelsGridFunction::operator = (double value)
+{
+   Fill(value);
+   return *this;
+}
+
+KernelsGridFunction& KernelsGridFunction::operator = (const Vector &v)
+{
+   Assign<double>(v);
+   return *this;
+}
+
+KernelsGridFunction& KernelsGridFunction::operator = (const KernelsGridFunction
+                                                      &v)
+{
+   Assign<double>(v);
+   return *this;
+}
+
+void KernelsGridFunction::GetTrueDofs(Vector &v)
+{
+   nvtx_push();
+   const mfem::Operator *R = fes.GetRestrictionOperator();
+   if (!R)
+   {
+      v.MakeRef(*this);
+   }
+   else
+   {
+      v.Resize<double>(R->OutLayout(), NULL);
+      mfem::Vector mfem_v(v);
+      R->Mult(this->Wrap(), mfem_v);
+   }
+   nvtx_pop();
+}
+
+void KernelsGridFunction::SetFromTrueDofs(Vector &v)
+{
+   nvtx_push();
+   const mfem::Operator *P = fes.GetProlongationOperator();
+   if (!P)
+   {
+      MakeRef(v);
+   }
+   else
+   {
+      Resize<double>(P->OutLayout(), NULL);
+      mfem::Vector mfem_this(*this);
+      P->Mult(v.Wrap(), mfem_this);
+   }
+   nvtx_pop();
+}
+
+mfem::FiniteElementSpace* KernelsGridFunction::GetFESpace()
+{
+   return fes.GetFESpace();
+}
+
+const mfem::FiniteElementSpace* KernelsGridFunction::GetFESpace() const
+{
+   return fes.GetFESpace();
+}
+
+void KernelsGridFunction::ToQuad(const IntegrationRule &ir, Vector &quadValues)
+{
+   nvtx_push();
+   const Engine &engine = KernelsLayout().KernelsEngine();
+   kernels::device device = engine.GetDevice();
+
+   const int elements = fes.GetNE();
+   const int numQuad  = ir.GetNPoints();
+   quadValues.Resize<double>(*(new Layout(engine, numQuad * elements)), NULL);
+   assert(false);
+   nvtx_pop();
+}
+
+void KernelsGridFunction::Distribute(const Vector &v)
+{
+   nvtx_push();
+   if (fes.isDistributed())
+   {
+      mfem::Vector mfem_this(*this);
+      fes.GetProlongationOperator()->Mult(v.Wrap(), mfem_this);
+   }
+   else
+   {
+      *this = v;
+   }
+   nvtx_pop();
+}
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/fem/gridfunc.hpp

@@ -0,0 +1,63 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_GRID_FUNC_HPP
+#define MFEM_BACKENDS_KERNELS_GRID_FUNC_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+class IntegrationRule;
+class GridFunction;
+
+namespace kernels
+{
+
+class KernelsIntegrator;
+class KernelsDofQuadMaps;
+
+class KernelsGridFunction : public Vector
+{
+protected:
+   kFiniteElementSpace &fes;
+   long sequence;
+   kvector v;
+public:
+
+   KernelsGridFunction(kFiniteElementSpace&);
+
+   KernelsGridFunction(const KernelsGridFunction &gf);
+
+   KernelsGridFunction& operator = (double value);
+   KernelsGridFunction& operator = (const Vector &v);
+   KernelsGridFunction& operator = (const KernelsGridFunction &gf);
+
+   void GetTrueDofs(Vector &v);
+   void SetFromTrueDofs(Vector &v);
+
+   mfem::FiniteElementSpace* GetFESpace();
+   const mfem::FiniteElementSpace* GetFESpace() const;
+
+   void ToQuad(const mfem::IntegrationRule &ir, Vector &quadValues);
+
+   void Distribute(const Vector &v);
+};
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_GRID_FUNC_HPP

backends/kernels/fem/kbilinearform.cpp

@@ -0,0 +1,383 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// *****************************************************************************
+kBilinearForm::kBilinearForm(kFiniteElementSpace *kfes) :
+   Operator(kfes->KernelsVLayout()),
+   localX(kfes->KernelsEVLayout()),
+   localY(kfes->KernelsEVLayout())
+{
+   nvtx_push();
+   Init(kfes->KernelsEngine(), kfes, kfes);
+   nvtx_pop();
+}
+
+// *****************************************************************************
+kBilinearForm::kBilinearForm(kFiniteElementSpace *kTrialFes_,
+                             kFiniteElementSpace *kTestFes_) :
+   Operator(kTrialFes_->KernelsVLayout(), kTestFes_->KernelsVLayout()),
+   localX(kTrialFes_->KernelsEVLayout()),
+   localY(kTestFes_->KernelsEVLayout())
+{
+   nvtx_push();
+   Init(kTrialFes_->KernelsEngine(), kTrialFes_, kTestFes_);
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void kBilinearForm::Init(const Engine &e,
+                         kFiniteElementSpace *kTrialFes_,
+                         kFiniteElementSpace *kTestFes_)
+{
+   nvtx_push();
+   ng.Reset(&e);
+   kTrialFes = kTrialFes_;
+   mTrialFes  = kTrialFes_->GetFESpace();
+
+   kTestFes = kTestFes_;
+   mTestFes  = kTestFes_->GetFESpace();
+
+   mesh = mTrialFes->GetMesh();
+   nvtx_pop();
+}
+
+// *****************************************************************************
+kFiniteElementSpace& kBilinearForm::GetTrialKernelsFESpace() const
+{
+   nvtx_push();
+   assert(kTrialFes);
+   nvtx_pop();
+   return *kTrialFes;
+}
+
+// *****************************************************************************
+kFiniteElementSpace& kBilinearForm::GetTestKernelsFESpace() const
+{
+   nvtx_push();
+   assert(kTestFes);
+   nvtx_pop();
+   return *kTestFes;
+}
+
+// *****************************************************************************
+mfem::FiniteElementSpace& kBilinearForm::GetTrialFESpace() const
+{
+   nvtx_push();
+   assert(mTrialFes);
+   nvtx_pop();
+   return *mTrialFes;
+}
+
+// *****************************************************************************
+mfem::FiniteElementSpace& kBilinearForm::GetTestFESpace() const
+{
+   nvtx_push();
+   assert(mTestFes);
+   nvtx_pop();
+   return *mTestFes;
+}
+
+int64_t kBilinearForm::GetTrialNDofs() const
+{
+   nvtx_push(); nvtx_pop();
+   return mTrialFes->GetNDofs();
+}
+
+int64_t kBilinearForm::GetTestNDofs() const
+{
+   return mTestFes->GetNDofs();
+}
+
+int64_t kBilinearForm::GetTrialVDim() const
+{
+   return mTrialFes->GetVDim();
+}
+
+int64_t kBilinearForm::GetTestVDim() const
+{
+   return mTestFes->GetVDim();
+}
+
+const FiniteElement& kBilinearForm::GetTrialFE(const int i) const
+{
+   return *(mTrialFes->GetFE(i));
+}
+
+const FiniteElement& kBilinearForm::GetTestFE(const int i) const
+{
+   return *(mTestFes->GetFE(i));
+}
+
+// Adds new Domain Integrator.
+void kBilinearForm::AddDomainIntegrator(KernelsIntegrator *integrator)
+{
+   nvtx_push();
+   AddIntegrator(integrator, DomainIntegrator);
+   nvtx_pop();
+}
+
+// Adds new Boundary Integrator.
+void kBilinearForm::AddBoundaryIntegrator(KernelsIntegrator *integrator)
+{
+   nvtx_push();
+   AddIntegrator(integrator, BoundaryIntegrator);
+   nvtx_pop();
+}
+
+// Adds new interior Face Integrator.
+void kBilinearForm::AddInteriorFaceIntegrator(KernelsIntegrator
+                                              *integrator)
+{
+   nvtx_push();
+   AddIntegrator(integrator, InteriorFaceIntegrator);
+   nvtx_pop();
+}
+
+// Adds new boundary Face Integrator.
+void kBilinearForm::AddBoundaryFaceIntegrator(KernelsIntegrator
+                                              *integrator)
+{
+   nvtx_push();
+   AddIntegrator(integrator, BoundaryFaceIntegrator);
+   nvtx_pop();
+}
+
+// Adds Integrator based on KernelsIntegratorType
+void kBilinearForm::AddIntegrator(KernelsIntegrator *integrator,
+                                  const KernelsIntegratorType itype)
+{
+   nvtx_push();
+   if (integrator == NULL)
+   {
+      std::stringstream error_ss;
+      error_ss << "kBilinearForm::";
+      switch (itype)
+      {
+         case DomainIntegrator      : error_ss << "AddDomainIntegrator";       break;
+         case BoundaryIntegrator    : error_ss << "AddBoundaryIntegrator";     break;
+         case InteriorFaceIntegrator: error_ss << "AddInteriorFaceIntegrator"; break;
+         case BoundaryFaceIntegrator: error_ss << "AddBoundaryFaceIntegrator"; break;
+      }
+      error_ss << " (...):\n"
+               << "  Integrator is NULL";
+      const std::string error = error_ss.str();
+      mfem_error(error.c_str());
+   }
+   integrator->SetupIntegrator(*this, itype);
+   integrators.push_back(integrator);
+   nvtx_pop();
+}
+
+const mfem::Operator* kBilinearForm::GetTrialProlongation() const
+{
+   return kTrialFes->GetProlongationOperator();
+}
+
+const mfem::Operator* kBilinearForm::GetTestProlongation() const
+{
+   return kTestFes->GetProlongationOperator();
+}
+
+const mfem::Operator* kBilinearForm::GetTrialRestriction() const
+{
+   return kTrialFes->GetRestrictionOperator();
+}
+
+const mfem::Operator* kBilinearForm::GetTestRestriction() const
+{
+   return kTestFes->GetRestrictionOperator();
+}
+
+// *****************************************************************************
+void kBilinearForm::Assemble()
+{
+   nvtx_push();
+   const int integratorCount = (int) integrators.size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->Assemble();
+   }
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void kBilinearForm::FormLinearSystem(const mfem::Array<int>
+                                     &constraintList,
+                                     mfem::Vector &x, mfem::Vector &b,
+                                     mfem::Operator *&Aout,
+                                     mfem::Vector &X, mfem::Vector &B,
+                                     int copy_interior)
+{
+   nvtx_push();
+   assert(false);
+   /*
+   FormOperator(constraintList, Aout);
+   InitRHS(constraintList, x, b, Aout, X, B, copy_interior);
+   */
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void kBilinearForm::FormOperator(const mfem::Array<int> &constraintList,
+                                 mfem::Operator *&Aout)
+{
+   nvtx_push();
+   const mfem::Operator *trialP = GetTrialProlongation();
+   const mfem::Operator *testP  = GetTestProlongation();
+   mfem::Operator *rap = this;
+   if (trialP)
+   {
+      dbg("\033[7mnew RAPOperator");
+      rap = new RAPOperator(*testP, *this, *trialP);
+   }
+   dbg("\033[7mnew kConstrainedOperator");
+   dbg("\033[7mrap->Height()=%d",rap->Height());
+   dbg("\033[7mrap->Width()=%d",rap->Width());
+   Aout = new kConstrainedOperator(rap, constraintList, rap != this);
+   dbg("\033[7mdone");
+   //assert(false);
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void kBilinearForm::InitRHS(const mfem::Array<int> &constraintList,
+                            mfem::Vector &x, mfem::Vector &b,
+                            mfem::Operator *A,
+                            mfem::Vector &X, mfem::Vector &B,
+                            int copy_interior)
+{
+   nvtx_push(); //assert(false);// ex1pd comes here, Laghos dont
+
+   const mfem::Operator *P = GetTrialProlongation();
+   const mfem::Operator *R = GetTrialRestriction();
+   if (P)
+   {
+      // Variational restriction with P
+      B.Resize(P->InLayout());
+      P->MultTranspose(b, B);
+      X.Resize(R->OutLayout());
+      R->Mult(x, X);
+   }
+   else
+   {
+      // rap, X and B point to the same data as this, x and b
+      assert(false);
+      X.MakeRef(x);
+      B.MakeRef(b);
+   }
+
+   if (!copy_interior && constraintList.Size() > 0)
+   {
+      //assert(false);
+      const Array &constrList = constraintList.Get_PArray()->As<Array>();
+      Vector subvec(constrList.KernelsLayout());
+      vector_get_subvector(constraintList.Size(),
+                           (double*)subvec.KernelsMem().ptr(),
+                           (double*)X.Get_PVector()->As<Vector>().KernelsMem().ptr(),
+                           (int*)constrList.KernelsMem().ptr());
+      X.Fill(0.0);
+      vector_set_subvector(constraintList.Size(),
+                           (double*)X.Get_PVector()->As<Vector>().KernelsMem().ptr(),
+                           (double*)subvec.KernelsMem().ptr(),
+                           (int*)constrList.KernelsMem().ptr());
+   }
+
+   kConstrainedOperator *cA = dynamic_cast<kConstrainedOperator*>(A);
+   if (cA)
+   {
+      cA->EliminateRHS(X.Get_PVector()->As<Vector>(),
+                       B.Get_PVector()->As<Vector>());
+   }
+   else
+   {
+      mfem_error("kBilinearForm::InitRHS expects an kConstrainedOperator");
+   }
+   nvtx_pop();
+}
+
+
+// Matrix vector multiplication ************************************************
+void kBilinearForm::Mult_(const kernels::Vector &x,
+                          kernels::Vector &y) const
+{
+   dbg("\033[7mkBilinearForm::Mult_");
+   kTrialFes->GlobalToLocal(x, localX);
+   localY.Fill<double>(0.0);
+   const int integratorCount = (int) integrators.size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->MultAdd(localX, localY);
+   }
+   kTestFes->LocalToGlobal(localY, y);
+   nvtx_pop();
+}
+
+// Matrix transpose vector multiplication **************************************
+void kBilinearForm::MultTranspose_(const kernels::Vector &x,
+                                   kernels::Vector &y) const
+{
+   nvtx_push();
+   kTestFes->GlobalToLocal(x, localX);
+   localY.Fill<double>(0.0);
+   const int integratorCount = (int) integrators.size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->MultTransposeAdd(localX, localY);
+   }
+   kTrialFes->LocalToGlobal(localY, y);
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void kBilinearForm::KernelsRecoverFEMSolution(const mfem::Vector &X,
+                                              const mfem::Vector &b,
+                                              mfem::Vector &x)
+{
+   nvtx_push();
+   const mfem::Operator *P = this->GetTrialProlongation();
+   if (P)
+   {
+      // Apply conforming prolongation
+      x.Resize(P->OutLayout());
+      P->Mult(X, x);
+   }
+   // Otherwise X and x point to the same data
+   nvtx_pop();
+}
+
+// Frees memory bilinear form **************************************************
+kBilinearForm::~kBilinearForm()
+{
+   // Make sure all integrators free their data
+   IntegratorVector::iterator it = integrators.begin();
+   while (it != integrators.end())
+   {
+      delete *it;
+      ++it;
+   }
+}
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/fem/kbilinearform.hpp

@@ -0,0 +1,134 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_BILINEAR_HPP
+#define MFEM_BACKENDS_KERNELS_BILINEAR_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+enum KernelsIntegratorType
+{
+   DomainIntegrator       = 0,
+   BoundaryIntegrator     = 1,
+   InteriorFaceIntegrator = 2,
+   BoundaryFaceIntegrator = 3
+};
+
+class KernelsIntegrator;
+
+// *****************************************************************************
+// * Class for bilinear form - "Matrix" with associated FE space and
+// * BLFIntegrators.
+// *****************************************************************************
+class kBilinearForm : public Operator
+{
+   friend class KernelsIntegrator;
+protected:
+   typedef std::vector<KernelsIntegrator*> IntegratorVector;
+   SharedPtr<const Engine> ng;
+   // State information
+   mutable mfem::Mesh *mesh;
+   mutable kFiniteElementSpace *kTrialFes;
+   mutable mfem::FiniteElementSpace *mTrialFes;
+   mutable kFiniteElementSpace *kTestFes;
+   mutable mfem::FiniteElementSpace *mTestFes;
+   IntegratorVector integrators;
+   // The input and output vectors are mapped to local nodes for efficient
+   // operations. In other words, they are E-vectors.
+   // The size is: (number of elements) * (nodes in element) * (vector dim)
+   mutable kernels::Vector localX, localY;
+public:
+   kBilinearForm(kFiniteElementSpace *ofespace_);
+   kBilinearForm(kFiniteElementSpace*, kFiniteElementSpace*);
+   void Init(const Engine&, kFiniteElementSpace*, kFiniteElementSpace*);
+   const Engine &engine() const { return *ng; }
+   device GetDevice(int idx = 0) const { return ng->GetDevice(idx); }
+
+   // Useful mesh Information
+   int BaseGeom() const { return mesh->GetElementBaseGeometry(0); }
+   int GetDim() const { return mesh->Dimension(); }
+   int64_t GetNE() const { return mesh->GetNE(); }
+   mfem::Mesh& GetMesh() const { return *mesh; }
+
+   kFiniteElementSpace& GetTrialKernelsFESpace() const;
+   kFiniteElementSpace& GetTestKernelsFESpace() const;
+   mfem::FiniteElementSpace& GetTrialFESpace() const;
+   mfem::FiniteElementSpace& GetTestFESpace() const;
+
+   // Useful FE information
+   int64_t GetTrialNDofs() const;
+   int64_t GetTestNDofs() const;
+   int64_t GetTrialVDim() const;
+   int64_t GetTestVDim() const;
+   const mfem::FiniteElement& GetTrialFE(const int i) const;
+   const mfem::FiniteElement& GetTestFE(const int i) const;
+
+   // Adds new Domain Integrator.
+   void AddDomainIntegrator(KernelsIntegrator*);
+   // Adds new Boundary Integrator.
+   void AddBoundaryIntegrator(KernelsIntegrator*);
+   // Adds new interior Face Integrator.
+   void AddInteriorFaceIntegrator(KernelsIntegrator*);
+   // Adds new boundary Face Integrator.
+   void AddBoundaryFaceIntegrator(KernelsIntegrator*);
+   // Adds Integrator based on KernelsIntegratorType
+   void AddIntegrator(KernelsIntegrator*, const KernelsIntegratorType );
+
+   virtual const mfem::Operator *GetTrialProlongation() const;
+   virtual const mfem::Operator *GetTestProlongation() const;
+
+   virtual const mfem::Operator *GetTrialRestriction() const;
+   virtual const mfem::Operator *GetTestRestriction() const;
+
+   // Assembles the form i.e. sums over all domain/bdr integrators.
+   virtual void Assemble();
+
+   void FormLinearSystem(const mfem::Array<int> &constraintList,
+                         mfem::Vector &x, mfem::Vector &b,
+                         mfem::Operator *&Aout,
+                         mfem::Vector &X, mfem::Vector &B,
+                         int copy_interior = 0);
+
+   void FormOperator(const mfem::Array<int> &constraintList,
+                     mfem::Operator *&Aout);
+
+   void InitRHS(const mfem::Array<int> &constraintList,
+                mfem::Vector &x, mfem::Vector &b,
+                mfem::Operator *Aout,
+                mfem::Vector &X, mfem::Vector &B,
+                int copy_interior = 0);
+
+   // overrides
+   virtual void Mult_(const Vector &x, Vector &y) const;
+   virtual void MultTranspose_(const Vector &x, Vector &y) const;
+
+   void KernelsRecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
+                                  mfem::Vector &x);
+
+   // Destroys bilinear form.
+   ~kBilinearForm();
+};
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_BILINEAR_HPP

backends/kernels/general/array.cpp

@@ -0,0 +1,165 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// *****************************************************************************
+PArray *Array::DoClone(bool copy_data, void **buffer,
+                       std::size_t item_size) const
+{
+   nvtx_push();
+   Array *new_array = new Array(KernelsLayout(), item_size);
+   if (copy_data)
+   {
+      new_array->slice.copyFrom(slice);
+   }
+   if (buffer)
+   {
+      *buffer = new_array->GetBuffer();
+   }
+   nvtx_pop();
+   return new_array;
+}
+
+// *****************************************************************************
+int Array::DoResize(PLayout &new_layout, void **buffer,
+                    std::size_t item_size)
+{
+   nvtx_push();
+   MFEM_ASSERT(dynamic_cast<Layout *>(&new_layout) != NULL,
+               "new_layout is not an KERNELS Layout");
+   Layout *lt = static_cast<Layout *>(&new_layout);
+   layout.Reset(lt); // Reset() checks if the pointer is the same
+   int err = ResizeData(lt, item_size);
+   if (!err && buffer)
+   {
+      *buffer = GetBuffer();
+   }
+   nvtx_pop();
+   return err;
+}
+
+// *****************************************************************************
+int Array::ResizeData(const Layout *lt, std::size_t item_size)
+{
+   nvtx_push();
+   const std::size_t new_bytes = lt->Size()*item_size;
+   dbg("data.size()=%d, slice.size()=%d & new_bytes=%d", data.size(), slice.size(),
+       new_bytes);
+   if (data.size() < new_bytes )
+   {
+      dbg("Alloc");
+      data = lt->Alloc(new_bytes);
+      slice = data;
+      // If memory allocation fails - an exception is thrown.
+   }
+   else if (slice.size() != new_bytes)
+   {
+      dbg("Slice");
+      slice = data.slice(0, new_bytes);
+   }
+   nvtx_pop();
+   return 0;
+}
+
+// *****************************************************************************
+void *Array::DoPullData(void *buffer, std::size_t item_size)
+{
+   nvtx_push();
+   if (!slice.getDevice().hasSeparateMemorySpace())
+   {
+      nvtx_pop();
+      return slice.ptr();
+   }
+   if (buffer)
+   {
+      slice.copyTo(buffer);
+   }
+   nvtx_pop();
+   return buffer;
+}
+
+// *****************************************************************************
+void Array::DoFill(const void *value_ptr, std::size_t item_size)
+{
+   nvtx_push();
+   switch (item_size)
+   {
+      case sizeof(int8_t):
+         KernelsFill((const int8_t *)value_ptr);
+         break;
+      case sizeof(int16_t):
+         KernelsFill((const int16_t *)value_ptr);
+         break;
+      case sizeof(int32_t):
+         KernelsFill((const int32_t *)value_ptr);
+         break;
+      case sizeof(double):
+         KernelsFill((const double *)value_ptr);
+         break;
+      default:
+         MFEM_ABORT("item_size = " << item_size << " is not supported");
+   }
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void Array::DoPushData(const void *src_buffer, std::size_t item_size)
+{
+   nvtx_push();
+   if (slice.getDevice().hasSeparateMemorySpace() || slice.ptr() != src_buffer)
+   {
+      slice.copyFrom(src_buffer);
+   }
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void Array::DoAssign(const PArray &src, std::size_t item_size)
+{
+   nvtx_push();
+   const kernels::Array *source = dynamic_cast<const kernels::Array*>(&src);
+   MFEM_ASSERT(source != NULL, "invalid source Array type");
+   MFEM_ASSERT(Size() == source->Size(), "");
+   slice.copyFrom(source->slice);
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void Array::DoMakeRefOffset(const PArray &src,
+                            const std::size_t offset,
+                            const std::size_t size,
+                            const std::size_t item_size)
+{
+   nvtx_push();
+   layout->Resize(size);
+   const kernels::Array &ksrc = src.As<const kernels::Array>();
+   const std::size_t bytes_size = size * item_size;
+   const std::size_t bytes_offset = offset * item_size;
+   memory m = memory(bytes_size,ksrc.data[bytes_offset]);
+   data = slice = m;
+   nvtx_pop();
+}
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/general/array.hpp

@@ -0,0 +1,110 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_ARRAY_HPP
+#define MFEM_BACKENDS_KERNELS_ARRAY_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+class Array : public virtual mfem::PArray
+{
+protected:
+
+   mutable kernels::memory data, slice;
+
+   // Virtual interface ********************************************************
+   virtual PArray *DoClone(bool copy_data, void **buffer,
+                           std::size_t item_size) const;
+
+   virtual int DoResize(PLayout &new_layout, void **buffer,
+                        std::size_t item_size);
+
+   virtual void *DoPullData(void *buffer, std::size_t item_size);
+
+   virtual void DoFill(const void *value_ptr, std::size_t item_size);
+
+   virtual void DoPushData(const void *src_buffer, std::size_t item_size);
+
+   virtual void DoAssign(const PArray &src, std::size_t item_size);
+
+   virtual void DoMakeRefOffset(const PArray &src,
+                                const std::size_t offset,
+                                const std::size_t size,
+                                const std::size_t item_size);
+
+   // Auxiliary methods ********************************************************
+
+   inline void *GetBuffer() const;
+
+   int ResizeData(const Layout *lt, std::size_t item_size);
+
+   template <typename T>
+   inline void KernelsFill(const T *val_ptr)
+   {
+      nvtx_push();
+      kernels::linalg::operator_eq<T>(slice, *val_ptr);
+      nvtx_pop();
+   }
+
+public:
+   Array(Layout &lt, std::size_t item_size)
+      : PArray(lt),
+        data(lt.Alloc(lt.Size()*item_size)),
+        slice(data)
+   { }
+
+   virtual ~Array() { }
+
+   inline void MakeRef(Array &master);
+
+   Layout &KernelsLayout() const
+   { return *static_cast<Layout*>(layout.Get()); }
+
+   kernels::memory &KernelsMem() { return slice; }
+
+   const kernels::memory &KernelsMem() const { return slice; }
+};
+
+
+//
+// Inline methods
+//
+
+inline void *Array::GetBuffer() const
+{
+   if (!slice.getDevice().hasSeparateMemorySpace())
+   {
+      return slice.ptr();
+   }
+   return NULL;
+}
+
+inline void Array::MakeRef(Array &master)
+{
+   layout = master.layout;
+   data = master.data;
+   slice = master.slice;
+}
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_ARRAY_HPP

backends/kernels/general/commd.cpp

@@ -0,0 +1,512 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+#ifdef MFEM_USE_MPI
+
+// ***************************************************************************
+// * kCommD
+// ***************************************************************************
+kCommD::kCommD(ParFiniteElementSpace &pfes):
+   GroupCommunicator(pfes.GroupComm()),
+   d_group_ldof(group_ldof),
+   d_group_ltdof(group_ltdof),
+   d_group_buf(NULL) {nvtx_push(); comm_lock=0; nvtx_pop();}
+
+
+// ***************************************************************************
+// * ~kCommD
+// ***************************************************************************
+kCommD::~kCommD() { }
+
+
+#ifdef __NVCC__
+// ***************************************************************************
+// * kCopyFromTable
+// ***************************************************************************
+template <class T> static 
+ __global__
+void k_CopyGroupToBuffer(T *buf,const T *data,const int *dofs)
+{
+   const int j = blockDim.x * blockIdx.x + threadIdx.x;
+   const int idx = dofs[j];
+   buf[j]=data[idx];
+}
+
+// ***************************************************************************
+// ***************************************************************************
+template <class T> static
+T *d_CopyGroupToBuffer_k(const T *d_ldata,T *d_buf,
+                         const ktable &d_dofs,
+                         const int group)
+{
+   nvtx_push(PapayaWhip);
+   const int ndofs = d_dofs.RowSize(group);
+   const int *dofs = d_dofs.GetRow(group);
+   k_CopyGroupToBuffer<<<ndofs,1>>>(d_buf,d_ldata,dofs);
+   nvtx_pop();
+   return d_buf + ndofs;
+}
+
+// ***************************************************************************
+// * d_CopyGroupToBuffer
+// ***************************************************************************
+template <class T>
+T *kCommD::d_CopyGroupToBuffer(const T *d_ldata, T *d_buf,
+                               int group, int layout) const
+{
+   if (layout==2) // master
+   {
+      return d_CopyGroupToBuffer_k(d_ldata,d_buf,d_group_ltdof,group);
+   }
+   if (layout==0) // slave
+   {
+      return d_CopyGroupToBuffer_k(d_ldata,d_buf,d_group_ldof,group);
+   }
+   assert(false);
+   return 0;
+}
+
+// ***************************************************************************
+// * k_CopyGroupFromBuffer
+// ***************************************************************************
+template <class T> static __global__
+void k_CopyGroupFromBuffer(const T *buf,T *data,const int *dofs)
+{
+   const int j = blockDim.x * blockIdx.x + threadIdx.x;
+   const int idx = dofs[j];
+   data[idx]=buf[j];
+}
+
+// ***************************************************************************
+// * d_CopyGroupFromBuffer
+// ***************************************************************************
+template <class T>
+const T *kCommD::d_CopyGroupFromBuffer(const T *d_buf, T *d_ldata,
+                                       int group, int layout) const
+{
+   nvtx_push(Gold);
+   assert(layout==0);
+   const int ndofs = d_group_ldof.RowSize(group);
+   const int *dofs = d_group_ldof.GetRow(group);
+   k_CopyGroupFromBuffer<<<ndofs,1>>>(d_buf,d_ldata,dofs);
+   nvtx_pop();
+   return d_buf + ndofs;
+}
+
+// ***************************************************************************
+// * kAtomicAdd
+// ***************************************************************************
+template <class T>
+static __global__ void kAtomicAdd(T* adrs, const int* dofs,T *value)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   const int idx = dofs[i];
+   adrs[idx] += value[i];
+}
+template __global__ void kAtomicAdd<int>(int*, const int*, int*);
+template __global__ void kAtomicAdd<double>(double*, const int*, double*);
+
+// ***************************************************************************
+// * ReduceGroupFromBuffer
+// ***************************************************************************
+template <class T>
+const T *kCommD::d_ReduceGroupFromBuffer(const T *d_buf, T *d_ldata,
+                                         int group, int layout,
+                                         void (*Op)(OpData<T>)) const
+{
+   nvtx_push(PaleGoldenrod);
+   dbg("\t[d_ReduceGroupFromBuffer]");
+   OpData<T> opd;
+   opd.ldata = d_ldata;
+   opd.nldofs = group_ldof.RowSize(group);
+   opd.nb = 1;
+   opd.buf = const_cast<T*>(d_buf);
+   dbg("\t\t[d_ReduceGroupFromBuffer] layout 2");
+   opd.ldofs = const_cast<int*>(d_group_ltdof.GetRow(group));
+   assert(opd.nb == 1);
+   // this is the operation to perform: opd.ldata[opd.ldofs[i]] += opd.buf[i];
+   // mfem/general/communication.cpp, line 1008
+   kAtomicAdd<<<opd.nldofs,1>>>(opd.ldata,opd.ldofs,opd.buf);
+   dbg("\t\t[d_ReduceGroupFromBuffer] done");
+   nvtx_pop();
+   return d_buf + opd.nldofs;
+}
+
+
+// ***************************************************************************
+// * d_BcastBegin
+// ***************************************************************************
+template <class T>
+void kCommD::d_BcastBegin(T *d_ldata, int layout)
+{
+   MFEM_VERIFY(comm_lock == 0, "object is already in use");
+   if (group_buf_size == 0) { return; }
+
+   nvtx_push(Moccasin);
+   assert(layout==2);
+   const int rnk = mfem::kernels::config::Get().Rank();
+   dbg("[%d-d_BcastBegin]",rnk);
+   int request_counter = 0;
+   nvtx_push(alloc,Moccasin);
+   group_buf.SetSize(group_buf_size*sizeof(T));
+   T *buf = (T *)group_buf.GetData();
+   if (!d_group_buf)
+   {
+      nvtx_push(alloc,Purple);
+      d_group_buf = mfem::kernels::kmalloc<T>::operator new (group_buf_size);
+      dbg("[%d-d_ReduceBegin] d_buf cuMemAlloc\033[m",rnk);
+      nvtx_pop();
+   }
+   T *d_buf = (T*)d_group_buf;
+   nvtx_pop();
+   for (int nbr = 1; nbr < nbr_send_groups.Size(); nbr++)
+   {
+      const int num_send_groups = nbr_send_groups.RowSize(nbr);
+      if (num_send_groups > 0)
+      {
+         T *buf_start = buf;
+         T *d_buf_start = d_buf;
+         const int *grp_list = nbr_send_groups.GetRow(nbr);
+         for (int i = 0; i < num_send_groups; i++)
+         {
+            T *d_buf_ini = d_buf;
+            assert(layout==2);
+            d_buf = d_CopyGroupToBuffer(d_ldata, d_buf, grp_list[i], 2);
+            buf += d_buf - d_buf_ini;
+         }
+         if (!mfem::kernels::config::Get().Aware())
+         {
+            nvtx_push(BcastBegin:DtoH,Red);
+            mfem::kernels::kmemcpy::rDtoH(buf_start,d_buf_start,(buf-buf_start)*sizeof(T));
+            nvtx_pop();
+         }
+
+         // make sure the device has finished
+         if (mfem::kernels::config::Get().Aware())
+         {
+            nvtx_push(sync,Lime);
+            cudaStreamSynchronize(0);//*rconfig::Get().Stream());
+            nvtx_pop();
+         }
+
+         nvtx_push(MPI_Isend,Orange);
+         if (mfem::kernels::config::Get().Aware())
+            MPI_Isend(d_buf_start,
+                      buf - buf_start,
+                      MPITypeMap<T>::mpi_type,
+                      gtopo.GetNeighborRank(nbr),
+                      40822,
+                      gtopo.GetComm(),
+                      &requests[request_counter]);
+         else
+            MPI_Isend(buf_start,
+                      buf - buf_start,
+                      MPITypeMap<T>::mpi_type,
+                      gtopo.GetNeighborRank(nbr),
+                      40822,
+                      gtopo.GetComm(),
+                      &requests[request_counter]);
+         nvtx_pop();
+         request_marker[request_counter] = -1; // mark as send request
+         request_counter++;
+      }
+
+      const int num_recv_groups = nbr_recv_groups.RowSize(nbr);
+      if (num_recv_groups > 0)
+      {
+         const int *grp_list = nbr_recv_groups.GetRow(nbr);
+         int recv_size = 0;
+         for (int i = 0; i < num_recv_groups; i++)
+         {
+            recv_size += group_ldof.RowSize(grp_list[i]);
+         }
+         nvtx_push(MPI_Irecv,Orange);
+         if (mfem::kernels::config::Get().Aware())
+            MPI_Irecv(d_buf,
+                      recv_size,
+                      MPITypeMap<T>::mpi_type,
+                      gtopo.GetNeighborRank(nbr),
+                      40822,
+                      gtopo.GetComm(),
+                      &requests[request_counter]);
+         else
+            MPI_Irecv(buf,
+                      recv_size,
+                      MPITypeMap<T>::mpi_type,
+                      gtopo.GetNeighborRank(nbr),
+                      40822,
+                      gtopo.GetComm(),
+                      &requests[request_counter]);
+         nvtx_pop();
+         request_marker[request_counter] = nbr;
+         request_counter++;
+         buf_offsets[nbr] = buf - (T*)group_buf.GetData();
+         buf += recv_size;
+         d_buf += recv_size;
+      }
+   }
+   assert(buf - (T*)group_buf.GetData() == group_buf_size);
+   comm_lock = 1; // 1 - locked for Bcast
+   num_requests = request_counter;
+   dbg("[%d-d_BcastBegin] done",rnk);
+   nvtx_pop();
+}
+
+// ***************************************************************************
+// * d_BcastEnd
+// ***************************************************************************
+template <class T>
+void kCommD::d_BcastEnd(T *d_ldata, int layout)
+{
+   if (comm_lock == 0) { return; }
+   nvtx_push(PeachPuff);
+   const int rnk = mfem::kernels::config::Get().Rank();
+   dbg("[%d-d_BcastEnd]",rnk);
+   // The above also handles the case (group_buf_size == 0).
+   assert(comm_lock == 1);
+   // copy the received data from the buffer to d_ldata, as it arrives
+   int idx;
+   nvtx_push(MPI_Waitany,Orange);
+   while (MPI_Waitany(num_requests, requests, &idx, MPI_STATUS_IGNORE),
+          idx != MPI_UNDEFINED)
+   {
+      nvtx_pop();
+      int nbr = request_marker[idx];
+      if (nbr == -1) { continue; } // skip send requests
+
+      const int num_recv_groups = nbr_recv_groups.RowSize(nbr);
+      if (num_recv_groups > 0)
+      {
+         const int *grp_list = nbr_recv_groups.GetRow(nbr);
+         int recv_size = 0;
+         for (int i = 0; i < num_recv_groups; i++)
+         {
+            recv_size += group_ldof.RowSize(grp_list[i]);
+         }
+         const T *buf = (T*)group_buf.GetData() + buf_offsets[nbr];
+         const T *d_buf = (T*)d_group_buf + buf_offsets[nbr];
+         if (!mfem::kernels::config::Get().Aware())
+         {
+            nvtx_push(BcastEnd:HtoD,Red);
+            mfem::kernels::kmemcpy::rHtoD((void*)d_buf,buf,recv_size*sizeof(T));
+            nvtx_pop();
+         }
+         for (int i = 0; i < num_recv_groups; i++)
+         {
+            d_buf = d_CopyGroupFromBuffer(d_buf, d_ldata, grp_list[i], layout);
+         }
+      }
+   }
+   comm_lock = 0; // 0 - no lock
+   num_requests = 0;
+   dbg("[%d-d_BcastEnd] done",rnk);
+   nvtx_pop();
+}
+
+// ***************************************************************************
+// * d_ReduceBegin
+// ***************************************************************************
+template <class T>
+void kCommD::d_ReduceBegin(const T *d_ldata)
+{
+   MFEM_VERIFY(comm_lock == 0, "object is already in use");
+   if (group_buf_size == 0) { return; }
+   nvtx_push(PapayaWhip);
+   const int rnk = mfem::kernels::config::Get().Rank();
+   dbg("[%d-d_ReduceBegin]",rnk);
+
+   int request_counter = 0;
+   group_buf.SetSize(group_buf_size*sizeof(T));
+   T *buf = (T *)group_buf.GetData();
+   if (!d_group_buf)
+   {
+      d_group_buf = mfem::kernels::kmalloc<T>::operator new (group_buf_size);
+   }
+   T *d_buf = (T*)d_group_buf;
+   for (int nbr = 1; nbr < nbr_send_groups.Size(); nbr++)
+   {
+      const int num_send_groups = nbr_recv_groups.RowSize(nbr);
+      if (num_send_groups > 0)
+      {
+         T *buf_start = buf;
+         T *d_buf_start = d_buf;
+         const int *grp_list = nbr_recv_groups.GetRow(nbr);
+         for (int i = 0; i < num_send_groups; i++)
+         {
+            T *d_buf_ini = d_buf;
+            d_buf = d_CopyGroupToBuffer(d_ldata, d_buf, grp_list[i], 0);
+            buf += d_buf - d_buf_ini;
+         }
+         dbg("[%d-d_ReduceBegin] MPI_Isend",rnk);
+         if (!mfem::kernels::config::Get().Aware())
+         {
+            nvtx_push(ReduceBegin:DtoH,Red);
+            mfem::kernels::kmemcpy::rDtoH(buf_start,d_buf_start,(buf-buf_start)*sizeof(T));
+            nvtx_pop();
+         }
+         // make sure the device has finished
+         if (mfem::kernels::config::Get().Aware())
+         {
+            nvtx_push(sync,Lime);
+            cudaStreamSynchronize(0);//*rconfig::Get().Stream());
+            nvtx_pop();
+         }
+         nvtx_push(MPI_Isend,Orange);
+         if (mfem::kernels::config::Get().Aware())
+            MPI_Isend(d_buf_start,
+                      buf - buf_start,
+                      MPITypeMap<T>::mpi_type,
+                      gtopo.GetNeighborRank(nbr),
+                      43822,
+                      gtopo.GetComm(),
+                      &requests[request_counter]);
+         else
+            MPI_Isend(buf_start,
+                      buf - buf_start,
+                      MPITypeMap<T>::mpi_type,
+                      gtopo.GetNeighborRank(nbr),
+                      43822,
+                      gtopo.GetComm(),
+                      &requests[request_counter]);
+         nvtx_pop();
+         request_marker[request_counter] = -1; // mark as send request
+         request_counter++;
+      }
+
+      // In Reduce operation: send_groups <--> recv_groups
+      const int num_recv_groups = nbr_send_groups.RowSize(nbr);
+      if (num_recv_groups > 0)
+      {
+         const int *grp_list = nbr_send_groups.GetRow(nbr);
+         int recv_size = 0;
+         for (int i = 0; i < num_recv_groups; i++)
+         {
+            recv_size += group_ldof.RowSize(grp_list[i]);
+         }
+         dbg("[%d-d_ReduceBegin] MPI_Irecv",rnk);
+         nvtx_push(MPI_Irecv,Orange);
+         if (mfem::kernels::config::Get().Aware())
+            MPI_Irecv(d_buf,
+                      recv_size,
+                      MPITypeMap<T>::mpi_type,
+                      gtopo.GetNeighborRank(nbr),
+                      43822,
+                      gtopo.GetComm(),
+                      &requests[request_counter]);
+         else
+            MPI_Irecv(buf,
+                      recv_size,
+                      MPITypeMap<T>::mpi_type,
+                      gtopo.GetNeighborRank(nbr),
+                      43822,
+                      gtopo.GetComm(),
+                      &requests[request_counter]);
+         nvtx_pop();
+         request_marker[request_counter] = nbr;
+         request_counter++;
+         buf_offsets[nbr] = buf - (T*)group_buf.GetData();
+         buf += recv_size;
+         d_buf += recv_size;
+      }
+   }
+   assert(buf - (T*)group_buf.GetData() == group_buf_size);
+   comm_lock = 2;
+   num_requests = request_counter;
+   dbg("[%d-d_ReduceBegin] done",rnk);
+   nvtx_pop();
+}
+
+// ***************************************************************************
+// * d_ReduceEnd
+// ***************************************************************************
+template <class T>
+void kCommD::d_ReduceEnd(T *d_ldata, int layout,
+                         void (*Op)(OpData<T>))
+{
+   if (comm_lock == 0) { return; }
+   nvtx_push(LavenderBlush);
+   const int rnk = mfem::kernels::config::Get().Rank();
+   dbg("[%d-d_ReduceEnd]",rnk);
+   // The above also handles the case (group_buf_size == 0).
+   assert(comm_lock == 2);
+
+   nvtx_push(MPI_Waitall,Orange);
+   MPI_Waitall(num_requests, requests, MPI_STATUSES_IGNORE);
+   nvtx_pop();
+   for (int nbr = 1; nbr < nbr_send_groups.Size(); nbr++)
+   {
+      // In Reduce operation: send_groups <--> recv_groups
+      const int num_recv_groups = nbr_send_groups.RowSize(nbr);
+      if (num_recv_groups > 0)
+      {
+         const int *grp_list = nbr_send_groups.GetRow(nbr);
+         int recv_size = 0;
+         for (int i = 0; i < num_recv_groups; i++)
+         {
+            recv_size += group_ldof.RowSize(grp_list[i]);
+         }
+         const T *buf = (T*)group_buf.GetData() + buf_offsets[nbr];
+         assert(d_group_buf);
+         const T *d_buf = (T*)d_group_buf + buf_offsets[nbr];
+         if (!mfem::kernels::config::Get().Aware())
+         {
+            nvtx_push(ReduceEnd:HtoD,Red);
+            mfem::kernels::kmemcpy::rHtoD((void*)d_buf,buf,recv_size*sizeof(T));
+            nvtx_pop();
+         }
+         for (int i = 0; i < num_recv_groups; i++)
+         {
+            d_buf = d_ReduceGroupFromBuffer(d_buf, d_ldata, grp_list[i], layout, Op);
+         }
+      }
+   }
+   comm_lock = 0; // 0 - no lock
+   num_requests = 0;
+   dbg("[%d-d_ReduceEnd] end",rnk);
+   nvtx_pop();
+}
+
+// ***************************************************************************
+// * instantiate kCommD::Bcast and Reduce for doubles
+// ***************************************************************************
+template void kCommD::d_BcastBegin<double>(double*, int);
+template void kCommD::d_BcastEnd<double>(double*, int);
+template void kCommD::d_ReduceBegin<double>(const double *);
+template void kCommD::d_ReduceEnd<double>(double*,int,void (*)(OpData<double>));
+#else // __NVCC__
+template <class T> void kCommD::d_ReduceBegin(const T*) {}
+template <class T> void kCommD::d_ReduceEnd(T*,int,void (*Op)(OpData<T>)) {}
+template <class T> void kCommD::d_BcastBegin(T*, int) {}
+template <class T> void kCommD::d_BcastEnd(T*, int) {}
+template void kCommD::d_BcastBegin<double>(double*, int);
+template void kCommD::d_BcastEnd<double>(double*, int);
+template void kCommD::d_ReduceBegin<double>(const double *);
+template void kCommD::d_ReduceEnd<double>(double*,int,void (*)(OpData<double>));
+#endif // __NVCC__
+
+#endif // MFEM_USE_MPI
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/general/commd.hpp

@@ -0,0 +1,70 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+
+#ifndef MFEM_BACKENDS_KERNELS_COMMD_HPP
+#define MFEM_BACKENDS_KERNELS_COMMD_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#endif
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+#ifdef MFEM_USE_MPI
+
+// ***************************************************************************
+// * First communicator, buf goes on the device
+// ***************************************************************************
+class kCommD : public GroupCommunicator, public kmemcpy
+{
+private:
+   ktable d_group_ldof;
+   ktable d_group_ltdof;
+   void *d_group_buf;
+   int comm_lock; // 0 - no lock, 1 - locked for Bcast, 2 - locked for Reduce
+   int num_requests;
+public:
+   kCommD(ParFiniteElementSpace&);
+   ~kCommD();
+
+   template <class T> T *d_CopyGroupToBuffer(const T*,T*,int,int) const;
+   template <class T> const T *d_CopyGroupFromBuffer(const T*, T*,int, int) const;
+   template <class T> const T *d_ReduceGroupFromBuffer(const T*,T*,int,int,
+                                                       void (*)(OpData<T>)) const;
+
+   template <class T> void d_BcastBegin(T*,int);
+   template <class T> void d_BcastEnd(T*, int);
+
+   template <class T> void d_ReduceBegin(const T*);
+   template <class T> void d_ReduceEnd(T*,int,void (*)(OpData<T>));
+};
+
+#endif
+
+} // namespace mfem::kernels
+
+} // mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_COMMD_HPP

backends/kernels/general/karray.hpp

@@ -0,0 +1,208 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+
+#ifndef MFEM_BACKENDS_KERNELS_RARRAY_HPP
+#define MFEM_BACKENDS_KERNELS_RARRAY_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+template <class T, bool xyz = true> class array;
+
+// Partial Specializations for xyz==TRUE *************************************
+template <class T> class array<T,true> : public kmalloc<T>
+{
+private:
+   T* data = NULL;
+   size_t sz=0;
+   size_t d[4]= {0};
+public:
+   array():data(NULL),sz(0),d{0,0,0,0} {}
+   array(const size_t x) {allocate(x);}
+   array(const size_t x,const size_t y) {allocate(x,y);}
+   array(const array<T,true> &r)
+   {
+      nvtx_push();
+      allocate(r.d[0], r.d[1], r.d[2], r.d[3]);
+      //memcpy(data,r.GetData(),r.bytes());
+      kmemcpy::rDtoD(data,r.GetData(),r.bytes());
+      nvtx_pop();
+   }
+   array& operator=(const array<T,true> &r)
+   {
+      nvtx_push();
+      allocate(r.d[0], r.d[1], r.d[2], r.d[3]);
+      //memcpy(data,r.GetData(),r.bytes());
+      kmemcpy::rDtoD(data,r.GetData(),r.bytes());
+      nvtx_pop();
+      return *this;
+   }
+   array& operator=(mfem::Array<T> &a)
+   {
+      nvtx_push();
+      kmemcpy::rHtoD(data,a.GetData(),a.Size()*sizeof(T));
+      nvtx_pop();
+      return *this;
+   }
+   array& operator=(const mfem::Array<T> &a)
+   {
+      nvtx_push();
+      kmemcpy::rHtoD(data,a.GetData(),a.Size()*sizeof(T));
+      nvtx_pop();
+      return *this;
+   }
+   ~array() {/*dbp("\033[32m[~i");*/ kmalloc<T>::operator delete (data);}
+   inline size_t* dim() { return &d[0]; }
+   inline T* ptr() { return data; }
+   inline const T* GetData() const { return data; }
+   inline const T* ptr() const { return data; }
+   inline operator T* () { return data; }
+   inline operator const T* () const { return data; }
+   double operator* (const array& a) const { return vector_dot(sz, data, a.data); }
+   inline size_t size() const { return sz; }
+   inline size_t Size() const { return sz; }
+   inline size_t bytes() const { return size()*sizeof(T); }
+   void allocate(const size_t X, const size_t Y =1,
+                 const size_t Z =1, const size_t D =1,
+                 const bool transposed = false)
+   {
+      nvtx_push();
+      d[0]=X; d[1]=Y; d[2]=Z; d[3]=D;
+      sz=d[0]*d[1]*d[2]*d[3];
+      dbp("\033[32m[i");
+      data=(T*) kmalloc<T>::operator new (sz);
+      nvtx_pop();
+   }
+   inline bool isInitialized(void)const {return true;}
+   inline T& operator[](const size_t x) { return data[x]; }
+   inline T& operator()(const size_t x, const size_t y)
+   {
+      return data[x + d[0]*y];
+   }
+   inline T& operator()(const size_t x, const size_t y, const size_t z)
+   {
+      return data[x + d[0]*(y + d[1]*z)];
+   }
+   void Print(std::ostream& out= std::cout, int width = 8) const
+   {
+      T *h_data = (double*) ::malloc(bytes());
+      kmemcpy::rDtoH(h_data,data,bytes());
+      for (size_t i=0; i<sz; i+=1)
+         if (sizeof(T)==8) { printf("\n\t[%ld] %.15e",i,h_data[i]); }
+         else { printf("\n\t[%ld] %d",i,h_data[i]); }
+      free(h_data);
+   }
+};
+
+// Partial Specializations for xyz==FALSE ************************************
+template <class T> class array<T,false> : public kmalloc<T>
+{
+private:
+   static const int DIM = 4;
+   T* data = NULL;
+   size_t sz=0;
+   size_t d[DIM]= {0};
+public:
+   array():data(NULL),sz(0),d{0,0,0,0} {}
+   array(const size_t d0) {allocate(d0);}
+   array(const array<T,false> &r)
+   {
+      nvtx_push();
+      allocate(r.d[0], r.d[1], r.d[2], r.d[3]);
+      //memcpy(data,r.GetData(),r.bytes());
+      kmemcpy::rDtoD(data,r.GetData(),r.bytes());
+      nvtx_pop();
+   }
+   array& operator=(const array<T,true> &r)
+   {
+      nvtx_push();
+      allocate(r.d[0], r.d[1], r.d[2], r.d[3]);
+      //memcpy(data,r.GetData(),r.bytes());
+      kmemcpy::rDtoD(data,r.GetData(),r.bytes());
+      nvtx_pop();
+      return *this;
+   }
+   array& operator=(mfem::Array<T> &a)
+   {
+      nvtx_push();
+      kmemcpy::rHtoD(data,a.GetData(),a.Size()*sizeof(T));
+      nvtx_pop();
+      return *this;
+   }
+   ~array() {dbp("\033[32m[~I"); kmalloc<T>::operator delete (data);}
+   inline size_t* dim() { return &d[0]; }
+   inline T* ptr() { return data; }
+   inline T* GetData() const { return data; }
+   inline const T* ptr() const { return data; }
+   inline operator T* () { return data; }
+   inline operator const T* () const { return data; }
+   double operator* (const array& a) const { return vector_dot(sz, data, a.data); }
+   inline size_t size() const { return sz; }
+   inline size_t Size() const { return sz; }
+   inline size_t bytes() const { return size()*sizeof(T); }
+   void allocate(const size_t X, const size_t Y =1,
+                 const size_t Z =1, const size_t D =1,
+                 const bool transposed = false)
+   {
+      nvtx_push();
+      d[0]=X; d[1]=Y; d[2]=Z; d[3]=D;
+      sz=d[0]*d[1]*d[2]*d[3];
+      dbp("\033[32m[I");
+      assert(sz>0);
+      data=(T*) kmalloc<T>::operator new (sz);
+      if (transposed) { std::swap(d[0],d[1]); }
+      for (size_t i=1,b=d[0]; i<DIM; std::swap(d[i],b),++i)
+      {
+         d[i]*=d[i-1];
+      }
+      d[0]=1;
+      if (transposed) { std::swap(d[0],d[1]); }
+      nvtx_pop();
+   }
+   inline bool isInitialized(void)const {return true;}
+   inline T& operator[](const size_t x) { return data[x]; }
+   inline T& operator()(const size_t x, const size_t y)
+   {
+      return data[d[0]*x + d[1]*y];
+   }
+   inline T& operator()(const size_t x, const size_t y, const size_t z)
+   {
+      return data[d[0]*x + d[1]*y + d[2]*z];
+   }
+   void Print(std::ostream& out= std::cout, int width = 8) const
+   {
+      T *h_data = (double*) ::malloc(bytes());
+      kmemcpy::rDtoH(h_data,data,bytes());
+      for (size_t i=0; i<sz; i+=1)
+         if (sizeof(T)==8) { printf("\n\t[%ld] %.15e",i,h_data[i]); }
+         else { printf("\n\t[%ld] %d",i,h_data[i]); }
+      free(h_data);
+   }
+};
+
+} // kernels
+
+} // mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_RARRAY_HPP

backends/kernels/general/layout.cpp

@@ -0,0 +1,47 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+// **************************************************************************
+void Layout::Resize(std::size_t new_size)
+{
+   nvtx_push();
+   dbg("size=%d, new_size=%d",size,new_size);
+   size = new_size;
+   nvtx_pop();
+}
+
+// **************************************************************************
+void Layout::Resize(const mfem::Array<std::size_t> &offsets)
+{
+   nvtx_push();
+   dbg("Array");
+   MFEM_ASSERT(offsets.Size() == 2,
+               "multiple workers are not supported yet");
+   assert(offsets.Size() == 2);
+   size = offsets.Last();
+   nvtx_pop();
+}
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/general/layout.hpp

@@ -0,0 +1,52 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BACKENDS_KERNELS_LAYOUT_HPP
+#define MFEM_BACKENDS_KERNELS_LAYOUT_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+class Layout : public mfem::PLayout
+{
+protected:
+
+public:
+   Layout(const Engine &e, std::size_t s = 0) : PLayout(e, s) { }
+
+   const Engine &KernelsEngine() const
+   { return *static_cast<const Engine *>(engine.Get()); }
+
+   kernels::memory Alloc(std::size_t bytes) const
+   { return KernelsEngine().GetDevice().malloc(bytes); }
+
+   virtual ~Layout() { }
+
+   /// Resize the layout
+   virtual void Resize(std::size_t new_size);
+
+   /// Resize the layout based on the given worker offsets
+   virtual void Resize(const Array<std::size_t> &offsets);
+};
+
+} // namespace mfem::kernels
+
+} // namespace mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_LAYOUT_HPP

backends/kernels/general/malloc.hpp

@@ -0,0 +1,100 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+
+#ifndef MFEM_BACKENDS_KERNELS_MALLOC_HPP
+#define MFEM_BACKENDS_KERNELS_MALLOC_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// ***************************************************************************
+template<class T> struct kmalloc: public kmemcpy
+{
+
+   // *************************************************************************
+   inline void* operator new (size_t n, bool lock_page = false)
+   {
+      dbp("+]\033[m");
+      if (!config::Get().Cuda())
+      {
+         return ::new T[n];
+      }
+#ifdef __NVCC__
+      void *ptr = NULL;
+      nvtx_push(new,Purple);
+      if (!config::Get().Uvm())
+      {
+         //dbg("\033[31;1m>cuMemAlloc");
+         if (lock_page) { checkCudaErrors(cuMemHostAlloc(&ptr, n*sizeof(T),CU_MEMHOSTALLOC_PORTABLE)); }
+         else
+         {
+            //assert(n>0); // DevExtension<>::SetEngine does a 'InitLayout(*e.MakeLayout(0));'
+            if (n==0) { n=1; }
+            checkCudaErrors(cuMemAlloc((CUdeviceptr*)&ptr, n*sizeof(T)));
+         }
+      }
+      else
+      {
+         //dbg("\033[31;1m>cuMemAllocManaged");
+         checkCudaErrors(cuMemAllocManaged((CUdeviceptr*)&ptr, n*sizeof(T),
+                                           CU_MEM_ATTACH_GLOBAL));
+      }
+      nvtx_pop();
+      return ptr;
+#else
+      // We come here when the user requests a manager,
+      // but has compiled the code without NVCC
+      assert(false);
+      return ::new T[n];
+#endif // __NVCC__
+   }
+
+   // ***************************************************************************
+   inline void operator delete (void *ptr)
+   {
+      dbp("-]\033[m");
+      if (!config::Get().Cuda())
+      {
+         if (ptr)
+         {
+            ::delete[] static_cast<T*>(ptr);
+         }
+      }
+#ifdef __NVCC__
+      else
+      {
+         nvtx_push(delete,Fuchsia);
+         cuMemFree((CUdeviceptr)ptr); // or cuMemFreeHost if page_locked was used
+         nvtx_pop();
+      }
+#endif // __NVCC__
+      ptr = nullptr;
+   }
+};
+
+} // kernels
+
+} // mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_MALLOC_HPP

backends/kernels/general/memcpy.cpp

@@ -0,0 +1,120 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// *************************************************************************
+void* kmemcpy::rHtoH(void *dest, const void *src, std::size_t bytes,
+                     const bool async)
+{
+   //nvtx_push();
+   dbg("=\033[m");
+   if (bytes==0) { return dest; }
+   assert(src); assert(dest);
+   std::memcpy(dest,src,bytes);
+   //nvtx_pop();
+   return dest;
+}
+
+// *************************************************************************
+void* kmemcpy::rHtoD(void *dest, const void *src, std::size_t bytes,
+                     const bool async)
+{
+   //nvtx_push();
+   dbg(">\033[m");
+   if (bytes==0)
+   {
+      //nvtx_pop();
+      return dest;
+   }
+   assert(src); assert(dest);
+   if (!config::Get().Cuda())
+   {
+      return std::memcpy(dest,src,bytes);
+   }
+#ifdef __NVCC__
+   if (!config::Get().Uvm())
+   {
+      checkCudaErrors(cuMemcpyHtoD((CUdeviceptr)dest,src,bytes));
+   }
+   else { checkCudaErrors(cuMemcpy((CUdeviceptr)dest,(CUdeviceptr)src,bytes)); }
+#endif
+   //nvtx_pop();
+   return dest;
+}
+
+// ***************************************************************************
+void* kmemcpy::rDtoH(void *dest, const void *src, std::size_t bytes,
+                     const bool async)
+{
+   //nvtx_push();
+   dbg("<\033[m");
+   if (bytes==0) { /*nvtx_pop();*/ return dest; }
+   assert(src); assert(dest);
+   if (!config::Get().Cuda()) { /*nvtx_pop();*/ return std::memcpy(dest,src,bytes); }
+#ifdef __NVCC__
+   if (!config::Get().Uvm())
+   {
+      checkCudaErrors(cuMemcpyDtoH(dest,(CUdeviceptr)src,bytes));
+   }
+   else { checkCudaErrors(cuMemcpy((CUdeviceptr)dest,(CUdeviceptr)src,bytes)); }
+#endif
+   //nvtx_pop();
+   return dest;
+}
+
+// ***************************************************************************
+void* kmemcpy::rDtoD(void *dest, const void *src, std::size_t bytes,
+                     const bool async)
+{
+   //nvtx_push();
+   dbg("=\033[m");
+   if (bytes==0) { /*nvtx_pop();*/ return dest; }
+   assert(src); assert(dest);
+   if (!config::Get().Cuda()) { /*nvtx_pop();*/ return std::memcpy(dest,src,bytes); }
+#ifdef __NVCC__
+   if (!config::Get().Uvm())
+   {
+      if (!async)
+      {
+         checkCudaErrors(cuMemcpyDtoD((CUdeviceptr)dest,(CUdeviceptr)src,bytes));
+      }
+      else
+      {
+         const CUstream s = *config::Get().Stream();
+         checkCudaErrors(cuMemcpyDtoDAsync((CUdeviceptr)dest,(CUdeviceptr)src,bytes,s));
+      }
+   }
+   else { checkCudaErrors(cuMemcpy((CUdeviceptr)dest,(CUdeviceptr)src,bytes)); }
+#endif
+   //nvtx_pop();
+   return dest;
+}
+
+} // kernels
+
+} // mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/general/memcpy.hpp

@@ -0,0 +1,45 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+
+#ifndef MFEM_BACKENDS_KERNELS_MEMCPY_HPP
+#define MFEM_BACKENDS_KERNELS_MEMCPY_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// ***************************************************************************
+struct kmemcpy
+{
+   static void* rHtoH(void*, const void*, std::size_t, const bool =false);
+   static void* rHtoD(void*, const void*, std::size_t, const bool =false);
+   static void* rDtoH(void*, const void*, std::size_t, const bool =false);
+   static void* rDtoD(void*, const void*, std::size_t, const bool =false);
+};
+
+} // kernels
+
+} // mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_MEMCPY_HPP

backends/kernels/general/table.cpp

@@ -0,0 +1,45 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+// ***************************************************************************
+ktable::ktable(const mfem::Table &table)
+{
+   nvtx_push();
+   size = table.Size();
+   assert(size > 0);
+   const int nnz = table.GetI()[size];
+   I = new int[size+1];
+   J = (int*) operator new (nnz);
+   rHtoH(I,table.GetI(),sizeof(int)*(size+1));
+   if (nnz>0)
+   {
+      assert(table.GetJ());
+      rHtoD(J,table.GetJ(),sizeof(int)*nnz);
+   }
+   nvtx_pop();
+}
+
+} // kernels
+
+} // mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)

backends/kernels/general/table.hpp

@@ -0,0 +1,49 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+
+#ifndef MFEM_BACKENDS_KERNELS_TABLE_HPP
+#define MFEM_BACKENDS_KERNELS_TABLE_HPP
+
+#include "../../../config/config.hpp"
+#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+class ktable : public kmalloc<int>
+{
+private:
+   int size = 0;
+   int *I = NULL;
+   int *J = NULL;
+public:
+   ktable(const mfem::Table&);
+   inline int Size() {return size;}
+   int RowSize(int i) const { return I[i+1]-I[i]; }
+   const int *GetRow(int i) const { return J+I[i]; }
+   int *GetRow(int i) { return J+I[i]; }
+};
+
+} // kernels
+
+} // mfem
+
+#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_KERNELS)
+
+#endif // MFEM_BACKENDS_KERNELS_RTABLE_HPP

backends/kernels/kernels.hpp

@@ -0,0 +1,109 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+#ifndef MFEM_BACKENDS_KERNELS_HPP
+#define MFEM_BACKENDS_KERNELS_HPP
+
+// *****************************************************************************
+#define __TEMPLATES__
+
+// stdincs *********************************************************************
+#include <math.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <assert.h>
+#include <assert.h>
+#include <sys/time.h>
+#include <unistd.h>
+#include <string.h>
+
+// MPI *************************************************************************
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#include <mpi-ext.h>
+#endif
+
+// __NVCC__ ********************************************************************
+#ifdef __NVCC__
+#include <cuda.h>
+#include <helper_cuda.h>
+#include <helper_functions.h>
+#define cuCheck checkCudaErrors
+#include "cuda_runtime_api.h"
+#endif
+
+// MFEM ************************************************************************
+#include "../../config/config.hpp"
+#include "../../general/array.hpp"
+#include "../../fem/gridfunc.hpp"
+#include "../../general/communication.hpp"
+#include "../../fem/fem.hpp"
+#include "../../fem/fespace.hpp"
+#include "../../fem/pfespace.hpp"
+#include "../../fem/bilinearform.hpp"
+#include "../../linalg/operator.hpp"
+
+// backends ********************************************************************
+#include "../base/backend.hpp"
+
+// kernels *********************************************************************
+#include "kernels/blas/blas.hpp"
+#include "kernels/diffusion/diffusion.hpp"
+#include "kernels/geom/geom.hpp"
+#include "kernels/mapping/mapping.hpp"
+#include "kernels/mass/mass.hpp"
+#include "kernels/quad/quad.hpp"
+
+// engine **********************************************************************
+#include "engine/memory.hpp"
+#include "engine/device.hpp"
+#include "engine/engine.hpp"
+#include "engine/backend.hpp"
+
+// config **********************************************************************
+#include "config/dbg.hpp"
+#include "config/nvvp.hpp"
+#include "config/config.hpp"
+
+// general *********************************************************************
+#include "linalg/linalg.hpp"
+#include "general/layout.hpp"
+#include "general/array.hpp"
+#include "general/memcpy.hpp"
+#include "general/malloc.hpp"
+#include "general/karray.hpp"
+#include "general/table.hpp"
+#include "general/commd.hpp"
+
+// linalg **********************************************************************
+#include "linalg/vector.hpp"
+#include "linalg/kvector.hpp"
+#include "linalg/operator.hpp"
+#include "linalg/constrained.hpp"
+#include "linalg/sparsemat.hpp"
+#include "linalg/conform.hpp"
+#include "linalg/restrict.hpp"
+#include "linalg/prolong.hpp"
+
+// fem *************************************************************************
+#include "fem/fespace.hpp"
+#include "fem/gridfunc.hpp"
+#include "fem/kbilinearform.hpp"
+#include "fem/bilinearform.hpp"
+#include "fem/coefficient.hpp"
+#include "fem/doftoquad.hpp"
+#include "fem/geom.hpp"
+#include "fem/bilininteg.hpp"
+#include "fem/bilinintDiffusion.hpp"
+#include "fem/bilinintMass.hpp"
+#include "fem/bilinintVMass.hpp"
+
+#endif // MFEM_BACKENDS_KERNELS_HPP
+

backends/kernels/kernels/blas/axpy.cpp

@@ -0,0 +1,42 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+#ifdef __NVCC__
+extern "C" kernel
+void vector_axpy0(const int N,
+                  const double alpha,
+                  double* __restrict v0,
+                  const double* __restrict v1)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < N) { v0[i] += alpha * v1[i]; }
+}
+#endif
+
+void vector_axpy(const int N,
+                 const double alpha,
+                 double* __restrict v0,
+                 const double* __restrict v1)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(vector_axpy,N,alpha,v0,v1);
+#else
+   forall(i,N,v0[i] += alpha * v1[i];);
+#endif
+   nvtx_pop();
+}

backends/kernels/kernels/blas/axpy2.cpp

@@ -0,0 +1,47 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+#ifdef __NVCC__
+extern "C" kernel
+void vector_axpby0(const int N,
+                   const double alpha,
+                   const double beta,
+                   double* __restrict v0,
+                   const double* __restrict v1)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < N) { v0[i] = alpha * v0[i] + beta * v1[i]; }
+}
+#endif
+
+// *****************************************************************************
+// * v0 = alpha * (*this) + beta * v1
+// *****************************************************************************
+void vector_axpby(const int N,
+                  const double alpha,
+                  const double beta,
+                  double* __restrict v0,
+                  const double* __restrict v1)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(vector_axpby,N,alpha,beta,v0,v1);
+#else
+   forall(i,N,v0[i] = alpha * v0[i] + beta * v1[i];);
+#endif
+   nvtx_pop();
+}

backends/kernels/kernels/blas/axpy3.cpp

@@ -0,0 +1,49 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+#ifdef __NVCC__
+extern "C" kernel
+void vector_axpby3_gpu0(const int N,
+                        const double alpha,
+                        const double beta,
+                        double* __restrict v0,
+                        const double* __restrict v1,
+                        const double* __restrict v2)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < N) { v0[i] = alpha * v1[i] + beta * v2[i]; }
+}
+#endif
+
+// *****************************************************************************
+// * v0 = alpha * (*this) + beta * v1
+// *****************************************************************************
+void vector_axpby3(const int N,
+                   const double alpha,
+                   const double beta,
+                   double* __restrict v0,
+                   const double* __restrict v1,
+                   const double* __restrict v2)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(vector_axpby3_gpu,N,alpha,beta,v0,v1,v2);
+#else
+   forall(i,N,v0[i] = alpha * v1[i] + beta * v2[i];);
+#endif
+   nvtx_pop();
+}

backends/kernels/kernels/blas/blas.hpp

@@ -0,0 +1,45 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#ifndef MFEM_KERNELS_BLAS
+#define MFEM_KERNELS_BLAS
+
+// **** BLAS1 ******************************************************************
+void vector_neg(const int, double*);
+void vector_op_eq(const int, const double, double*);
+void vector_op_set(const int, const double*, double*);
+void vector_xpay(const int, const double, double*, const double*,const double*);
+void vector_xsy(const int, double*, const double*, const double*);
+void vector_axpy(const int, const double, double*, const double*);
+void vector_axpby(const int, const double,const double, double*, const double*);
+void vector_axpby3(const int, const double,const double, double*, const double*,
+                   const double*);
+void vector_map_dofs(const int, double*, const double*, const int*);
+template <class T>
+void vector_map_add_dofs(const int, T*, const T*, const int*);
+void vector_clear_dofs(const int, double*, const int*);
+void vector_vec_sub(const int, double*, const double*);
+void vector_vec_add(const int, double*, const double*);
+void vector_vec_mul(const int, double*, const double);
+void vector_set_subvector(const int, double*, const double*,
+                          const int*);
+void vector_get_subvector(const int, double*, const double*,
+                          const int*);
+void vector_set_subvector_const(const int, const double, double*,
+                                const int*);
+double vector_dot(const int, const double*, const double*);
+double vector_min(const int, const double*);
+
+#endif // MFEM_KERNELS_BLAS

backends/kernels/kernels/blas/vadd.cpp

@@ -0,0 +1,41 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+#ifdef __NVCC__
+extern "C" kernel
+void vector_vec_add0(const int N,
+                     double* __restrict v0,
+                     const double* __restrict v1)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < N) { v0[i] += v1[i]; }
+}
+#endif
+
+// *****************************************************************************
+void vector_vec_add(const int N,
+                    double* __restrict v0,
+                    const double* __restrict v1)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(vector_vec_add,N,v0,v1);
+#else
+   forall(i,N,v0[i] += v1[i];);
+#endif
+   nvtx_pop();
+}

backends/kernels/kernels/blas/vclear_dofs.cpp

@@ -0,0 +1,40 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+#ifdef __NVCC__
+extern "C" kernel
+void vector_clear_dofs0(const int N,
+                        double* __restrict v0,
+                        const int* __restrict v1)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < N) { v0[v1[i]] = 0.0; }
+}
+#endif
+
+void vector_clear_dofs(const int N,
+                       double* __restrict v0,
+                       const int* __restrict v1)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(vector_clear_dofs,N,v0,v1);
+#else
+   forall(i,N,v0[v1[i]] = 0.0;);
+#endif
+   nvtx_pop();
+}

backends/kernels/kernels/blas/vdot.cpp

@@ -0,0 +1,69 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+// *****************************************************************************
+#ifdef __NVCC__
+__inline__ __device__ double4 operator*(double4 a, double4 b)
+{
+   return make_double4(a.x*b.x, a.y*b.y, a.z*b.z, a.w*b.w);
+}
+#include <cub/cub.cuh>
+
+// *****************************************************************************
+static double cub_vector_dot(const int N,
+                             const double* __restrict vec1,
+                             const double* __restrict vec2)
+{
+   static double *h_dot = NULL;
+   if (!h_dot) { h_dot = (double*)mfem::kernels::kmalloc<double>::operator new (1,true); }
+   static double *d_dot = NULL;
+   if (!d_dot) { d_dot=(double*)mfem::kernels::kmalloc<double>::operator new (1); }
+   static void *d_storage = NULL;
+   static size_t storage_bytes = 0;
+   if (!d_storage)
+   {
+      cub::DeviceReduce::Dot(d_storage, storage_bytes, vec1, vec2, d_dot, N);
+      d_storage = mfem::kernels::kmalloc<char>::operator new (storage_bytes);
+   }
+   cub::DeviceReduce::Dot(d_storage, storage_bytes, vec1, vec2, d_dot, N);
+   mfem::kernels::kmemcpy::rDtoH(h_dot,d_dot,sizeof(double));
+   return *h_dot;
+}
+#endif // __NVCC__
+
+// *****************************************************************************
+double vector_dot(const int N,
+                  const double* __restrict vec1,
+                  const double* __restrict vec2)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   if (mfem::kernels::config::Get().Cuda())
+   {
+      const double result = cub_vector_dot(N,vec1,vec2);
+      nvtx_pop();
+      return result;
+   }
+#endif
+   double dot=0.0;
+   for (int i=0; i<N; i+=1)
+   {
+      dot += vec1[i]*vec2[i];
+   }
+   nvtx_pop();
+   return dot;
+}

backends/kernels/kernels/blas/vget_subvector.cpp

@@ -0,0 +1,52 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+#ifdef __NVCC__
+extern "C" kernel
+void vector_get_subvector0(const int N,
+                           double* __restrict v0,
+                           const double* __restrict v1,
+                           const int* __restrict v2)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < N)
+   {
+      const int dof_i = v2[i];
+      v0[i] = dof_i >= 0 ? v1[dof_i] : -v1[-dof_i-1];
+   }
+}
+#endif
+
+// *****************************************************************************
+void vector_get_subvector(const int N,
+                          double* __restrict v0,
+                          const double* __restrict v1,
+                          const int* __restrict v2)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(vector_get_subvector,N,v0,v1,v2);
+#else
+   forall(i,N,
+   {
+      const int dof_i = v2[i];
+      v0[i] = dof_i >= 0 ? v1[dof_i] : -v1[-dof_i-1];
+   });
+#endif
+   nvtx_pop();
+}
+

backends/kernels/kernels/blas/vmap_dofs.cpp

@@ -0,0 +1,47 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+#ifdef __NVCC__
+extern "C" kernel
+void vector_map_dofs0(const int N,
+                      double* __restrict v0,
+                      const double* __restrict v1,
+                      const int* v2)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < N)
+   {
+      const int idx = v2[i];
+      v0[idx] = v1[idx];
+   }
+}
+#endif
+
+// *****************************************************************************
+void vector_map_dofs(const int N,
+                     double* __restrict v0,
+                     const double* __restrict v1,
+                     const int* v2)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(vector_map_dofs,N,v0,v1,v2);
+#else
+   forall(i,N, { const int idx = v2[i]; v0[idx] = v1[idx]; });
+#endif
+   nvtx_pop();
+}

backends/kernels/kernels/blas/vmin.cpp

@@ -0,0 +1,62 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+// *****************************************************************************
+#ifdef __NVCC__
+#include <cub/cub.cuh>
+
+// *****************************************************************************
+static double cub_vector_min(const int N,
+                             const double* __restrict vec)
+{
+   static double *h_min = NULL;
+   if (!h_min) { h_min = (double*)mfem::kernels::kmalloc<double>::operator new (1,true); }
+   static double *d_min = NULL;
+   if (!d_min) { d_min=(double*)mfem::kernels::kmalloc<double>::operator new (1); }
+   static void *d_storage = NULL;
+   static size_t storage_bytes = 0;
+   if (!d_storage)
+   {
+      cub::DeviceReduce::Min(d_storage, storage_bytes, vec, d_min, N);
+      d_storage = mfem::kernels::kmalloc<char>::operator new (storage_bytes);
+   }
+   cub::DeviceReduce::Min(d_storage, storage_bytes, vec, d_min, N);
+   mfem::kernels::kmemcpy::rDtoH(h_min,d_min,sizeof(double));
+   return *h_min;
+}
+#endif // __NVCC__
+
+
+// *****************************************************************************
+double vector_min(const int N,
+                  const double* __restrict vec)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   if (mfem::kernels::config::Get().Cuda())
+   {
+      const double result = cub_vector_min(N,vec);
+      nvtx_pop();
+      return result;
+   }
+#endif
+   ReduceDecl(Min,red,vec[0]);
+   ReduceForall(i,N,red.min(vec[i]););
+   nvtx_pop();
+   return red;
+}
+

backends/kernels/kernels/blas/vmul.cpp

@@ -0,0 +1,42 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+// *****************************************************************************
+#ifdef __NVCC__
+extern "C" kernel
+void vector_vec_mul0(const int N,
+                     double* __restrict v0,
+                     const double d)
+{
+   const int i = blockDim.x*blockIdx.x+threadIdx.x;
+   if (i < N) { v0[i]*=d; }
+}
+#endif
+
+// *****************************************************************************
+void vector_vec_mul(const int N,
+                    double* __restrict v0,
+                    const double d)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(vector_vec_mul,N,v0,d);
+#else
+   forall(i,N,v0[i]*=d;);
+#endif
+   nvtx_pop();
+}

backends/kernels/kernels/blas/vneg.cpp

@@ -0,0 +1,39 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+#ifdef __NVCC__
+extern "C" kernel
+void vector_neg0(const int N,
+                 double* __restrict vec)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < N) { vec[i] *= -1.0; }
+}
+#endif
+
+// *****************************************************************************
+void vector_neg(const int N,
+                double* __restrict vec)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(vector_neg,N,vec);
+#else
+   forall(i,N,vec[i] *= -1.0;);
+#endif
+   nvtx_pop();
+}

backends/kernels/kernels/blas/vopeq.cpp

@@ -0,0 +1,41 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+#ifdef __NVCC__
+// *****************************************************************************
+extern "C" kernel void vector_op_eq0(const int N,
+                                     const double c0,
+                                     double* __restrict v0)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < N) { v0[i] = c0; }
+}
+#endif
+
+// *****************************************************************************
+void vector_op_eq(const int N,
+                  const double c0,
+                  double* __restrict v0)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(vector_op_eq,N,c0,v0);
+#else
+   forall(i,N,v0[i] = c0;);
+#endif
+   nvtx_pop();
+}

backends/kernels/kernels/blas/vopeq_d.cu

@@ -0,0 +1,33 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+// *****************************************************************************
+extern "C" static __global__ void d_vector_op_eq0(const int N,
+                                       const double c0,
+                                       double* __restrict v0){
+  const int i = blockDim.x * blockIdx.x + threadIdx.x;
+  if (i < N) v0[i] = c0;
+}
+
+// *****************************************************************************
+extern "C" __global__ void d_vector_op_eq(const int N,
+                                          const double c0,
+                                          double* __restrict v0){
+  const size_t blockSize = 128;
+  const size_t gridSize = (N+blockSize-1)/blockSize;
+  d_vector_op_eq0<<<gridSize,blockSize>>>(N,c0,v0);
+}

backends/kernels/kernels/blas/vset.cpp

@@ -0,0 +1,41 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+#ifdef __NVCC__
+// *****************************************************************************
+extern "C" kernel void vector_op_set0(const int N,
+                                      const double* __restrict x,
+                                      double* __restrict y)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < N) { y[i] = x[i]; }
+}
+#endif
+
+// *****************************************************************************
+void vector_op_set(const int N,
+                   const double* __restrict x,
+                   double* __restrict y)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(vector_op_set,N,x,y);
+#else
+   forall(i,N, y[i] = x[i];);
+#endif
+   nvtx_pop();
+}

backends/kernels/kernels/blas/vset_subvector.cpp

@@ -0,0 +1,58 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+#ifdef __NVCC__
+// *****************************************************************************
+extern "C" kernel
+void vector_set_subvector0(const int N,
+                           double* __restrict v0,
+                           const double* __restrict v1,
+                           const int* __restrict v2)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < N)
+   {
+      const int dof_i = v2[i];
+      const bool tst = dof_i >= 0;
+      const int idx = tst?dof_i:-dof_i-1;
+      const double value = tst?v1[i]:-v1[i];
+      v0[idx]=value;
+   }
+}
+#endif
+
+// *****************************************************************************
+void vector_set_subvector(const int N,
+                          double* __restrict v0,
+                          const double* __restrict v1,
+                          const int* __restrict v2)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(vector_set_subvector,N,v0,v1,v2);
+#else
+   forall(i,N,
+   {
+      const int dof_i = v2[i];
+      const bool tst = dof_i >= 0;
+      const int idx = tst?dof_i:-dof_i-1;
+      const double value = tst?v1[i]:-v1[i];
+      v0[idx]=value;
+   });
+#endif
+   nvtx_pop();
+}

backends/kernels/kernels/blas/vset_subvector_const.cpp

@@ -0,0 +1,66 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+#ifdef __NVCC__
+// *****************************************************************************
+extern "C" kernel
+void vector_set_subvector_const0(const int N,
+                                 const double value,
+                                 double* __restrict data,
+                                 const int* __restrict tdofs)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i >= N) { return; }
+   const int dof_i = tdofs[i];
+   data[dof_i] = value;
+   if (dof_i >= 0)
+   {
+      data[dof_i] = value;
+   }
+   else
+   {
+      data[-dof_i-1] = -value;
+   }
+}
+#endif
+
+// *****************************************************************************
+void vector_set_subvector_const(const int N,
+                                const double value,
+                                double* __restrict data,
+                                const int* __restrict tdofs)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(vector_set_subvector_const,N,value,data,tdofs);
+#else
+   forall(i,N,
+   {
+      const int dof_i = tdofs[i];
+      data[dof_i] = value;
+      if (dof_i >= 0)
+      {
+         data[dof_i] = value;
+      }
+      else
+      {
+         data[-dof_i-1] = -value;
+      }
+   });
+#endif
+   nvtx_pop();
+}

backends/kernels/kernels/blas/vsub.cpp

@@ -0,0 +1,42 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+#ifdef __NVCC__
+// *****************************************************************************
+extern "C" kernel
+void vector_vec_sub0(const int N,
+                     double* __restrict v0,
+                     const double* __restrict v1)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < N) { v0[i] -= v1[i]; }
+}
+#endif
+
+// *****************************************************************************
+void vector_vec_sub(const int N,
+                    double* __restrict v0,
+                    const double* __restrict v1)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(vector_vec_sub,N,v0,v1);
+#else
+   forall(i,N, { v0[i] -= v1[i]; });
+#endif
+   nvtx_pop();
+}

backends/kernels/kernels/blas/xpay.cpp

@@ -0,0 +1,46 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+#ifdef __NVCC__
+// *****************************************************************************
+extern "C" kernel
+void vector_xpay0(const int N,
+                  const double c0,
+                  double* __restrict v0,
+                  const double* __restrict v1,
+                  const double* __restrict v2)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < N) { v0[i] = v1[i] + (c0 * v2[i]); }
+}
+#endif
+
+// *****************************************************************************
+void vector_xpay(const int N,
+                 const double c0,
+                 double* __restrict v0,
+                 const double* __restrict v1,
+                 const double* __restrict v2)
+{
+   nvtx_push(xpay,Cyan);
+#ifdef __NVCC__
+   cuKer(vector_xpay,N,c0,v0,v1,v2);
+#else
+   forall(i,N, { v0[i] = v1[i] + (c0 * v2[i]); });
+#endif
+   nvtx_pop();
+}

backends/kernels/kernels/blas/xsy.cpp

@@ -0,0 +1,44 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+#ifdef __NVCC__
+// *****************************************************************************
+extern "C" kernel
+void vector_xsy0(const int N,
+                 double* __restrict v0,
+                 const double* __restrict v1,
+                 const double* __restrict v2)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < N) { v0[i] = v1[i]-v2[i]; }
+}
+#endif
+
+// *****************************************************************************
+void vector_xsy(const int N,
+                double* __restrict v0,
+                const double* __restrict v1,
+                const double* __restrict v2)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(vector_xsy,N,v0,v1,v2);
+#else
+   forall(i,N, {v0[i] = v1[i]-v2[i];});
+#endif
+   nvtx_pop();
+}

backends/kernels/kernels/diffusion/assemble.cpp

@@ -0,0 +1,85 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+// *****************************************************************************
+extern "C" kernel
+void rDiffusionAssemble2D0(const int numElements,
+                           const int NUM_QUAD_2D,
+                           const double COEFF,
+                           const double* quadWeights,
+                           const double* J,
+                           double* __restrict oper)
+{
+#ifdef __NVCC__
+   const int e = blockDim.x * blockIdx.x + threadIdx.x;
+   if (e < numElements)
+#else
+   forall(e,numElements,
+#endif
+   {
+      for (int q = 0; q < NUM_QUAD_2D; ++q)
+      {
+         const double J11 = J[ijklNM(0,0,q,e,2,NUM_QUAD_2D)];
+         const double J12 = J[ijklNM(1,0,q,e,2,NUM_QUAD_2D)];
+         const double J21 = J[ijklNM(0,1,q,e,2,NUM_QUAD_2D)];
+         const double J22 = J[ijklNM(1,1,q,e,2,NUM_QUAD_2D)];
+         const double c_detJ = quadWeights[q] * COEFF / ((J11*J22)-(J21*J12));
+         oper[ijkNM(0,q,e,3,NUM_QUAD_2D)]/*(0, q, e)*/ =  c_detJ *
+         (J21*J21 + J22*J22); // (1,1)
+         oper[ijkNM(1,q,e,3,NUM_QUAD_2D)]/*(1, q, e)*/ = -c_detJ *
+         (J21*J11 + J22*J12); // (1,2), (2,1)
+         oper[ijkNM(2,q,e,3,NUM_QUAD_2D)]/*(2, q, e)*/ =  c_detJ *
+         (J11*J11 + J12*J12); // (2,2)
+      }
+   }
+#ifndef __NVCC__
+   );
+#endif
+}
+// *****************************************************************************
+static void rDiffusionAssemble2D(const int numElements,
+                                 const int NUM_QUAD_2D,
+                                 const double COEFF,
+                                 const double* quadWeights,
+                                 const double* J,
+                                 double* __restrict oper)
+{
+   nvtx_push();
+   cuKer(rDiffusionAssemble2D,numElements,NUM_QUAD_2D,COEFF,quadWeights,J,oper);
+   nvtx_pop();
+}
+
+// *****************************************************************************
+void rDiffusionAssemble(const int dim,
+                        const int NUM_QUAD_1D,
+                        const int numElements,
+                        const double* quadWeights,
+                        const double* J,
+                        const double COEFF,
+                        double* __restrict oper)
+{
+   nvtx_push();
+   if (dim==1) { assert(false); }
+   if (dim==2) { rDiffusionAssemble2D(numElements,
+                                      NUM_QUAD_1D*NUM_QUAD_1D,
+                                      COEFF,
+                                      quadWeights,
+                                      J,
+                                      oper); }
+   if (dim==3) { assert(false); }
+   nvtx_pop();
+}

backends/kernels/kernels/diffusion/diffusion.hpp

@@ -0,0 +1,42 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#ifndef MFEM_KERNELS_DIFFUSION
+#define MFEM_KERNELS_DIFFUSION
+
+// *****************************************************************************
+#define restrict __restrict__
+
+// *****************************************************************************
+void rDiffusionAssemble(const int dim,
+                        const int NUM_QUAD,
+                        const int numElements,
+                        const double* quadWeights,
+                        const double* J,
+                        const double COEFF,
+                        double* restrict oper);
+void rDiffusionMultAdd(const int dim,
+                       const int NUM_DOFS_1D,
+                       const int NUM_QUAD_1D,
+                       const int numElements,
+                       const double* restrict dofToQuad,
+                       const double* restrict dofToQuadD,
+                       const double* restrict quadToDof,
+                       const double* restrict quadToDofD,
+                       const double* restrict op,
+                       const double* restrict x,
+                       double* restrict y);
+
+#endif // MFEM_KERNELS_DIFFUSION

backends/kernels/kernels/diffusion/multadd.cpp

@@ -0,0 +1,182 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+#define QUAD_2D_ID(X, Y) (X + ((Y) * NUM_QUAD_1D))
+
+// *****************************************************************************
+template<const int NUM_DOFS_1D,
+         const int NUM_QUAD_1D> kernel
+void rDiffusionMultAdd2D(const int numElements,
+                         const double* __restrict dofToQuad,
+                         const double* __restrict dofToQuadD,
+                         const double* __restrict quadToDof,
+                         const double* __restrict quadToDofD,
+                         const double* __restrict oper,
+                         const double* __restrict solIn,
+                         double* __restrict solOut)
+{
+#ifdef __NVCC__
+   const int e = blockDim.x * blockIdx.x + threadIdx.x;
+   if (e < numElements)
+#else
+   forall(e,numElements,
+#endif
+   {
+      double grad[NUM_QUAD_1D][NUM_QUAD_1D][2];
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy)
+      {
+         for (int qx = 0; qx < NUM_QUAD_1D; ++qx)
+         {
+            grad[qy][qx][0] = 0;
+            grad[qy][qx][1] = 0;
+         }
+      }
+
+      for (int dy = 0; dy < NUM_DOFS_1D; ++dy)
+      {
+         double gradX[NUM_QUAD_1D][2];
+         for (int qx = 0; qx < NUM_QUAD_1D; ++qx)
+         {
+            gradX[qx][0] = 0;
+            gradX[qx][1] = 0;
+         }
+
+         for (int dx = 0; dx < NUM_DOFS_1D; ++dx)
+         {
+            const double s = solIn[ijkN(dx,dy,e,NUM_DOFS_1D)];
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx)
+            {
+               gradX[qx][0] += s * dofToQuad[ijN(qx,dx,NUM_QUAD_1D)];
+               gradX[qx][1] += s * dofToQuadD[ijN(qx,dx,NUM_QUAD_1D)];
+            }
+         }
+
+         for (int qy = 0; qy < NUM_QUAD_1D; ++qy)
+         {
+            const double wy  = dofToQuad[ijN(qy,dy,NUM_QUAD_1D)];
+            const double wDy = dofToQuadD[ijN(qy,dy,NUM_QUAD_1D)];
+            for (int qx = 0; qx < NUM_QUAD_1D; ++qx)
+            {
+               grad[qy][qx][0] += gradX[qx][1] * wy;
+               grad[qy][qx][1] += gradX[qx][0] * wDy;
+            }
+         }
+      }
+
+      // Calculate Dxy, xDy in plane
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy)
+      {
+         for (int qx = 0; qx < NUM_QUAD_1D; ++qx)
+         {
+            const int q = QUAD_2D_ID(qx, qy);
+            const double O11 = oper[ijkNM(0,q,e,3,NUM_QUAD_1D)];//(0, q, e);
+            const double O12 = oper[ijkNM(1,q,e,3,NUM_QUAD_1D)];//(1, q, e);
+            const double O22 = oper[ijkNM(2,q,e,3,NUM_QUAD_1D)];//(2, q, e);
+
+            const double gradX = grad[qy][qx][0];
+            const double gradY = grad[qy][qx][1];
+
+            grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
+            grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
+         }
+      }
+
+      for (int qy = 0; qy < NUM_QUAD_1D; ++qy)
+      {
+         double gradX[NUM_DOFS_1D][2];
+         for (int dx = 0; dx < NUM_DOFS_1D; ++dx)
+         {
+            gradX[dx][0] = 0;
+            gradX[dx][1] = 0;
+         }
+
+         for (int qx = 0; qx < NUM_QUAD_1D; ++qx)
+         {
+            const double gX = grad[qy][qx][0];
+            const double gY = grad[qy][qx][1];
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx)
+            {
+               const double wx  = quadToDof[ijN(dx,qx,NUM_DOFS_1D)];
+               const double wDx = quadToDofD[ijN(dx,qx,NUM_DOFS_1D)];
+               gradX[dx][0] += gX * wDx;
+               gradX[dx][1] += gY * wx;
+            }
+         }
+
+         for (int dy = 0; dy < NUM_DOFS_1D; ++dy)
+         {
+            const double wy  = quadToDof[ijN(dy,qy,NUM_DOFS_1D)];
+            const double wDy = quadToDofD[ijN(dy,qy,NUM_DOFS_1D)];
+            for (int dx = 0; dx < NUM_DOFS_1D; ++dx)
+            {
+               solOut[ijkN(dx,dy,e,NUM_DOFS_1D)] += ((gradX[dx][0] * wy) +
+                                                     (gradX[dx][1] * wDy));
+            }
+         }
+      }
+   }
+#ifndef __NVCC__
+          );
+#endif
+}
+
+// *****************************************************************************
+typedef void (*fDiffusionMultAdd)(const int numElements,
+                                  const double* __restrict dofToQuad,
+                                  const double* __restrict dofToQuadD,
+                                  const double* __restrict quadToDof,
+                                  const double* __restrict quadToDofD,
+                                  const double* __restrict oper,
+                                  const double* __restrict solIn,
+                                  double* __restrict solOut);
+
+// *****************************************************************************
+void rDiffusionMultAdd(const int DIM,
+                       const int NUM_DOFS_1D,
+                       const int NUM_QUAD_1D,
+                       const int numElements,
+                       const double* dofToQuad,
+                       const double* dofToQuadD,
+                       const double* quadToDof,
+                       const double* quadToDofD,
+                       const double* op,
+                       const double* x,
+                       double* __restrict y)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   const int blck = 256;
+   const int grid = (numElements+blck-1)/blck;
+#endif
+   const unsigned int id = (DIM<<16)|((NUM_DOFS_1D-1)<<8)|(NUM_QUAD_1D>>1);
+   static std::unordered_map<unsigned int, fDiffusionMultAdd> call = {
+    {0x20001,&rDiffusionMultAdd2D<1,2>},    {0x20101,&rDiffusionMultAdd2D<2,2>},
+    {0x20102,&rDiffusionMultAdd2D<2,4>},    {0x20202,&rDiffusionMultAdd2D<3,4>},
+    {0x20203,&rDiffusionMultAdd2D<3,6>},    {0x20303,&rDiffusionMultAdd2D<4,6>},
+    //{0x30001,&rDiffusionMultAdd3D<1,2>},    {0x30101,&rDiffusionMultAdd3D<2,2>},
+    //{0x30102,&rDiffusionMultAdd3D<2,4>},    {0x30202,&rDiffusionMultAdd3D<3,4>},
+   };
+   if(!call[id]){
+      printf("\n[rDiffusionMultAdd] id \033[33m0x%X\033[m ",id);
+      fflush(stdout);
+   }
+   assert(call[id]);
+   call0(dummy,id,grid,blck,
+         numElements,dofToQuad,dofToQuadD,quadToDof,quadToDofD,op,x,y);
+  
+   nvtx_pop();
+}

backends/kernels/kernels/geom/geom.cpp

@@ -0,0 +1,101 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+// *****************************************************************************
+template<const int NUM_DOFS,
+         const int NUM_QUAD> kernel
+void rIniGeom2D(const int,
+                const double*,const double*,
+                double*,double*,double*);
+
+// *****************************************************************************
+template<const int NUM_DOFS,
+         const int NUM_QUAD> kernel
+void rIniGeom3D(const int,
+                const double*,const double*,
+                double*,double*,double*);
+
+// *****************************************************************************
+typedef void (*fIniGeom)(const int,const double*,const double*,
+                         double*, double*, double*);
+
+// *****************************************************************************
+void rIniGeom(const int DIM,
+              const int NUM_DOFS,
+              const int NUM_QUAD,
+              const int numElements,
+              const double* dofToQuadD,
+              const double* nodes,
+              double* J,
+              double* invJ,
+              double* detJ){
+   nvtx_push();
+#ifdef __NVCC__
+   const int blck = CUDA_BLOCK_SIZE;
+   const int grid = (numElements+blck-1)/blck;
+#endif
+   const unsigned int dofs1D = IROOT(DIM,NUM_DOFS);
+   const unsigned int quad1D = IROOT(DIM,NUM_QUAD);
+   const unsigned int id = (DIM<<4)|(dofs1D-2);
+   assert(LOG2(DIM)<=4);
+   assert(LOG2(dofs1D-2)<=4);
+   assert(quad1D==2*(dofs1D-1));
+   static std::unordered_map<unsigned int, fIniGeom> call = {
+      // 2D
+      {0x20,&rIniGeom2D<2*2,(2*2-2)*(2*2-2)>},
+      {0x21,&rIniGeom2D<3*3,(3*2-2)*(3*2-2)>},
+      {0x22,&rIniGeom2D<4*4,(4*2-2)*(4*2-2)>},
+      {0x23,&rIniGeom2D<5*5,(5*2-2)*(5*2-2)>},
+      {0x24,&rIniGeom2D<6*6,(6*2-2)*(6*2-2)>},
+      {0x25,&rIniGeom2D<7*7,(7*2-2)*(7*2-2)>},
+      {0x26,&rIniGeom2D<8*8,(8*2-2)*(8*2-2)>},
+      {0x27,&rIniGeom2D<9*9,(9*2-2)*(9*2-2)>},
+      {0x28,&rIniGeom2D<10*10,(10*2-2)*(10*2-2)>},
+      {0x29,&rIniGeom2D<11*11,(11*2-2)*(11*2-2)>},
+      {0x2A,&rIniGeom2D<12*12,(12*2-2)*(12*2-2)>},
+      {0x2B,&rIniGeom2D<13*13,(13*2-2)*(13*2-2)>},
+      {0x2C,&rIniGeom2D<14*14,(14*2-2)*(14*2-2)>},
+      {0x2D,&rIniGeom2D<15*15,(15*2-2)*(15*2-2)>},
+      {0x2E,&rIniGeom2D<16*16,(16*2-2)*(16*2-2)>},
+      {0x2F,&rIniGeom2D<17*17,(17*2-2)*(17*2-2)>},
+      // 3D
+      {0x30,&rIniGeom3D<2*2*2,2*2*2>},
+      {0x31,&rIniGeom3D<3*3*3,4*4*4>},
+      {0x32,&rIniGeom3D<4*4*4,6*6*6>},
+      {0x33,&rIniGeom3D<5*5*5,8*8*8>},
+      {0x34,&rIniGeom3D<6*6*6,10*10*10>},
+      {0x35,&rIniGeom3D<7*7*7,12*12*12>},
+      {0x36,&rIniGeom3D<8*8*8,14*14*14>},
+      {0x37,&rIniGeom3D<9*9*9,16*16*16>},
+      {0x38,&rIniGeom3D<10*10*10,18*18*18>},
+      {0x39,&rIniGeom3D<11*11*11,20*20*20>},
+      {0x3A,&rIniGeom3D<12*12*12,22*22*22>},
+      {0x3B,&rIniGeom3D<13*13*13,24*24*24>},
+      {0x3C,&rIniGeom3D<14*14*14,26*26*26>},
+      {0x3D,&rIniGeom3D<15*15*15,28*28*28>},
+      {0x3E,&rIniGeom3D<16*16*16,30*30*30>},
+      {0x3F,&rIniGeom3D<17*17*17,32*32*32>},
+   };
+   if (!call[id]){
+      printf("\n[rIniGeom] id \033[33m0x%X\033[m ",id);
+      fflush(stdout);
+   }
+   assert(call[id]);
+   call0(dummy,id,grid,blck,
+        numElements,dofToQuadD,nodes,J,invJ,detJ);
+   nvtx_pop();
+}

backends/kernels/kernels/geom/geom.hpp

@@ -0,0 +1,39 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#ifndef MFEM_KERNELS_GEOM
+#define MFEM_KERNELS_GEOM
+
+// *****************************************************************************
+void rNodeCopyByVDim(const int elements,
+                     const int numDofs,
+                     const int ndofs,
+                     const int dims,
+                     const int* eMap,
+                     const double* Sx,
+                     double* nodes);
+
+// *****************************************************************************
+void rIniGeom(const int dim,
+              const int nDofs,
+              const int nQuads,
+              const int nzones,
+              const double* dofToQuadD,
+              const double* nodes,
+              double* J,
+              double* invJ,
+              double* detJ);
+
+#endif // MFEM_KERNELS_GEOM

backends/kernels/kernels/geom/geom1D.cpp

@@ -0,0 +1,61 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#if 0
+#include "../kernels.hpp"
+
+// *****************************************************************************
+void rIniGeom1D(const int NUM_DOFS,
+                const int NUM_QUAD,
+                const int numElements,
+                const double* __restrict__ dofToQuadD,
+                const double* __restrict__ nodes,
+                double* __restrict__ J,
+                double* __restrict__ invJ,
+                double* __restrict__ detJ){
+#ifdef __NVCC__
+   const int e = blockDim.x * blockIdx.x + threadIdx.x;
+   if (e < numElements)
+#else
+   forall(e,numElements,
+#endif
+   {
+      double s_nodes[NUM_DOFS];
+      for (int q = 0; q < NUM_QUAD; ++q)
+      {
+         for (int d = q; d < NUM_DOFS; d += NUM_QUAD)
+         {
+            s_nodes[d] = nodes[ijkN(0,d,e,NUM_QUAD)];
+         }
+      }
+      for (int q = 0; q < NUM_QUAD; ++q)
+      {
+         double J11 = 0;
+         for (int d = 0; d < NUM_DOFS; ++d)
+         {
+            const double wx = dofToQuadD[ijN(q,d,NUM_DOFS)];
+            J11 += wx * s_nodes[d];
+         }
+         J[ijN(q,e,NUM_QUAD)] = J11;
+         invJ[ijN(q, e,NUM_QUAD)] = 1.0 / J11;
+         detJ[ijN(q, e,NUM_QUAD)] = J11;
+      }
+   }
+#ifndef __NVCC__
+           );
+#endif
+}
+
+#endif

backends/kernels/kernels/geom/geom2D.cpp

@@ -0,0 +1,89 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+// *****************************************************************************
+template<const int NUM_DOFS,
+         const int NUM_QUAD> kernel
+void rIniGeom2D(const int numElements,
+                const double* __restrict dofToQuadD,
+                const double* __restrict nodes,
+                double* __restrict J,
+                double* __restrict invJ,
+                double* __restrict detJ){
+#ifdef __NVCC__
+   const int e = blockDim.x * blockIdx.x + threadIdx.x;
+   if (e < numElements)
+#else
+   forall(e,numElements,
+#endif
+   {
+      double s_nodes[2 * NUM_DOFS];
+      for (int q = 0; q < NUM_QUAD; ++q)
+      {
+         for (int d = q; d < NUM_DOFS; d +=NUM_QUAD)
+         {
+            s_nodes[ijN(0,d,2)] = nodes[ijkNM(0,d,e,2,NUM_DOFS)];
+            s_nodes[ijN(1,d,2)] = nodes[ijkNM(1,d,e,2,NUM_DOFS)];
+         }
+      }
+      for (int q = 0; q < NUM_QUAD; ++q)
+      {
+         double J11 = 0; double J12 = 0;
+         double J21 = 0; double J22 = 0;
+         for (int d = 0; d < NUM_DOFS; ++d)
+         {
+            const double wx = dofToQuadD[ijkNM(0,q,d,2,NUM_QUAD)];
+            const double wy = dofToQuadD[ijkNM(1,q,d,2,NUM_QUAD)];
+            const double x = s_nodes[ijN(0,d,2)];
+            const double y = s_nodes[ijN(1,d,2)];
+            J11 += (wx * x); J12 += (wx * y);
+            J21 += (wy * x); J22 += (wy * y);
+         }
+         const double r_detJ = (J11 * J22)-(J12 * J21);
+         J[ijklNM(0, 0, q, e,2,NUM_QUAD)] = J11;
+         J[ijklNM(1, 0, q, e,2,NUM_QUAD)] = J12;
+         J[ijklNM(0, 1, q, e,2,NUM_QUAD)] = J21;
+         J[ijklNM(1, 1, q, e,2,NUM_QUAD)] = J22;
+         const double r_idetJ = 1.0 / r_detJ;
+         invJ[ijklNM(0, 0, q, e,2,NUM_QUAD)] =  J22 * r_idetJ;
+         invJ[ijklNM(1, 0, q, e,2,NUM_QUAD)] = -J12 * r_idetJ;
+         invJ[ijklNM(0, 1, q, e,2,NUM_QUAD)] = -J21 * r_idetJ;
+         invJ[ijklNM(1, 1, q, e,2,NUM_QUAD)] =  J11 * r_idetJ;
+         detJ[ijN(q, e,NUM_QUAD)] = r_detJ;
+      }
+   }
+#ifndef __NVCC__
+          );
+#endif
+}
+
+template kernel void rIniGeom2D<4,4>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom2D<9,16>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom2D<16,36>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom2D<25,64>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom2D<36,100>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom2D<49,144>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom2D<64,196>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom2D<81,256>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom2D<100,324>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom2D<121,400>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom2D<144,484>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom2D<169,576>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom2D<196,676>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom2D<225,784>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom2D<256,900>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom2D<289,1024>(int, double const*, double const*, double*, double*, double*);

backends/kernels/kernels/geom/geom3D.cpp

@@ -0,0 +1,109 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+// *****************************************************************************
+template<const int NUM_DOFS,
+         const int NUM_QUAD> kernel
+void rIniGeom3D(const int numElements,
+                const double* __restrict dofToQuadD,
+                const double* __restrict nodes,
+                double* __restrict J,
+                double* __restrict invJ,
+                double* __restrict detJ){
+#ifdef __NVCC__
+   const int e = blockDim.x * blockIdx.x + threadIdx.x;
+   if (e < numElements)
+#else
+   forall(e,numElements,
+#endif
+   {
+      double s_nodes[3*NUM_DOFS];
+      for (int q = 0; q < NUM_QUAD; ++q)
+      {
+         for (int d = q; d < NUM_DOFS; d += NUM_QUAD)
+         {
+            s_nodes[ijN(0,d,3)] = nodes[ijkNM(0, d, e,3,NUM_DOFS)];
+            s_nodes[ijN(1,d,3)] = nodes[ijkNM(1, d, e,3,NUM_DOFS)];
+            s_nodes[ijN(2,d,3)] = nodes[ijkNM(2, d, e,3,NUM_DOFS)];
+         }
+      }
+      for (int q = 0; q < NUM_QUAD; ++q)
+      {
+         double J11 = 0; double J12 = 0; double J13 = 0;
+         double J21 = 0; double J22 = 0; double J23 = 0;
+         double J31 = 0; double J32 = 0; double J33 = 0;
+         for (int d = 0; d < NUM_DOFS; ++d)
+         {
+            const double wx = dofToQuadD[ijkNM(0, q, d,3,NUM_QUAD)];
+            const double wy = dofToQuadD[ijkNM(1, q, d,3,NUM_QUAD)];
+            const double wz = dofToQuadD[ijkNM(2, q, d,3,NUM_QUAD)];
+            const double x = s_nodes[ijN(0, d,3)];
+            const double y = s_nodes[ijN(1, d,3)];
+            const double z = s_nodes[ijN(2, d,3)];
+            J11 += (wx * x); J12 += (wx * y); J13 += (wx * z);
+            J21 += (wy * x); J22 += (wy * y); J23 += (wy * z);
+            J31 += (wz * x); J32 += (wz * y); J33 += (wz * z);
+         }
+         const double r_detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) +
+                                (J13 * J21 * J32) -
+                                (J13 * J22 * J31)-(J12 * J21 * J33)-(J11 * J23 * J32));
+         J[ijklNM(0, 0, q, e,3,NUM_QUAD)] = J11;
+         J[ijklNM(1, 0, q, e,3,NUM_QUAD)] = J12;
+         J[ijklNM(2, 0, q, e,3,NUM_QUAD)] = J13;
+         J[ijklNM(0, 1, q, e,3,NUM_QUAD)] = J21;
+         J[ijklNM(1, 1, q, e,3,NUM_QUAD)] = J22;
+         J[ijklNM(2, 1, q, e,3,NUM_QUAD)] = J23;
+         J[ijklNM(0, 2, q, e,3,NUM_QUAD)] = J31;
+         J[ijklNM(1, 2, q, e,3,NUM_QUAD)] = J32;
+         J[ijklNM(2, 2, q, e,3,NUM_QUAD)] = J33;
+
+         const double r_idetJ = 1.0 / r_detJ;
+         invJ[ijklNM(0, 0, q, e,3,NUM_QUAD)] = r_idetJ * ((J22 * J33)-(J23 * J32));
+         invJ[ijklNM(1, 0, q, e,3,NUM_QUAD)] = r_idetJ * ((J32 * J13)-(J33 * J12));
+         invJ[ijklNM(2, 0, q, e,3,NUM_QUAD)] = r_idetJ * ((J12 * J23)-(J13 * J22));
+
+         invJ[ijklNM(0, 1, q, e,3,NUM_QUAD)] = r_idetJ * ((J23 * J31)-(J21 * J33));
+         invJ[ijklNM(1, 1, q, e,3,NUM_QUAD)] = r_idetJ * ((J33 * J11)-(J31 * J13));
+         invJ[ijklNM(2, 1, q, e,3,NUM_QUAD)] = r_idetJ * ((J13 * J21)-(J11 * J23));
+
+         invJ[ijklNM(0, 2, q, e,3,NUM_QUAD)] = r_idetJ * ((J21 * J32)-(J22 * J31));
+         invJ[ijklNM(1, 2, q, e,3,NUM_QUAD)] = r_idetJ * ((J31 * J12)-(J32 * J11));
+         invJ[ijklNM(2, 2, q, e,3,NUM_QUAD)] = r_idetJ * ((J11 * J22)-(J12 * J21));
+         detJ[ijN(q, e,NUM_QUAD)] = r_detJ;
+      }
+   }
+#ifndef __NVCC__
+           );
+#endif
+}
+
+template kernel void rIniGeom3D<8,8>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom3D<27,64>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom3D<64,216>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom3D<125,512>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom3D<216,1000>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom3D<343,1728>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom3D<512,2744>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom3D<729,4096>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom3D<1000,5832>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom3D<1331,8000>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom3D<1728,10648>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom3D<2197,13824>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom3D<2744,17576>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom3D<3375,21952>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom3D<4096,27000>(int, double const*, double const*, double*, double*, double*);
+template kernel void rIniGeom3D<4913,32768>(int, double const*, double const*, double*, double*, double*);

backends/kernels/kernels/geom/nodecpy.cpp

@@ -0,0 +1,70 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+// *****************************************************************************
+#ifdef __NVCC__
+extern "C" kernel
+#endif
+void rNodeCopyByVDim0(const int elements,
+                      const int numDofs,
+                      const int ndofs,
+                      const int dims,
+                      const int* eMap,
+                      const double* Sx,
+                      double* nodes)
+{
+#ifdef __NVCC__
+   const int e = blockDim.x * blockIdx.x + threadIdx.x;
+   if (e < elements)
+#else
+   nvtx_push();
+   forall(e,elements,
+#endif
+   {
+      for (int dof = 0; dof < numDofs; ++dof)
+      {
+         const int lid = dof+numDofs*e;
+         const int gid = eMap[lid];
+         for (int v = 0; v < dims; ++v)
+         {
+            const int moffset = v+dims*lid;
+            const int voffset = gid+v*ndofs;
+            nodes[moffset] = Sx[voffset];
+         }
+      }
+   }
+#ifndef __NVCC__
+   );
+   nvtx_pop();
+#endif
+}
+
+// *****************************************************************************
+void rNodeCopyByVDim(const int elements,
+                     const int numDofs,
+                     const int ndofs,
+                     const int dims,
+                     const int* eMap,
+                     const double* Sx,
+                     double* nodes)
+{
+#ifdef __NVCC__
+   cuKer(rNodeCopyByVDim,elements,numDofs,ndofs,dims,eMap,Sx,nodes);
+#else
+   rNodeCopyByVDim0(elements,numDofs,ndofs,dims,eMap,Sx,nodes);
+#endif
+}

backends/kernels/kernels/include/forall.hpp

@@ -0,0 +1,108 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#ifndef MFEM_KERNELS_FORALL
+#define MFEM_KERNELS_FORALL
+
+// *****************************************************************************
+#define ELEMENT_BATCH 10
+#define M2_ELEMENT_BATCH 32
+#define A2_ELEMENT_BATCH 1
+#define A2_QUAD_BATCH 1
+
+// *****************************************************************************
+#ifdef __RAJA__ // *************************************************************
+//#warning RAJA KERNELS, WITH NVCC
+#define sync
+#define share
+#define kernel
+const int CUDA_BLOCK_SIZE = 256;
+#define cu_device __device__
+#define cu_exec RAJA::cuda_exec<CUDA_BLOCK_SIZE>
+#define cu_reduce RAJA::cuda_reduce<CUDA_BLOCK_SIZE>
+#define sq_device __host__
+#define sq_exec RAJA::seq_exec
+#define sq_reduce RAJA::seq_reduce
+#define ReduceDecl(type,var,ini) \
+  RAJA::Reduce ## type<sq_reduce, RAJA::Real_type> var(ini);
+#define ReduceForall(i,max,body) \
+  RAJA::forall<sq_exec>(0,max,[=]sq_device(RAJA::Index_type i) {body});
+#define forall(i,max,body)                                              \
+   if (mfem::config::Get().Cuda())                                      \
+    RAJA::forall<cu_exec>(0,max,[=]cu_device(RAJA::Index_type i) {body}); \
+  else                                                                  \
+    RAJA::forall<sq_exec>(0,max,[=]sq_device(RAJA::Index_type i) {body});
+#define forallS(i,max,step,body) {assert(false);forall(i,max,body)}
+#define call0(name,id,grid,blck,...) call[id](__VA_ARGS__)
+#define cuKerGBS(name,grid,block,end,...) name ## 0(end,__VA_ARGS__)
+#define cuKer(name,end,...) name ## 0(end,__VA_ARGS__)
+
+
+// *****************************************************************************
+#else // KERNELS on GPU, CUDA Kernel launches  *********************************
+#ifdef __NVCC__
+//#warning GPU KERNELS, WITH NVCC direct launch
+#define kernel __global__
+#define share __shared__
+#define sync __syncthreads();
+const int CUDA_BLOCK_SIZE = 256;
+#define cuKer(name,end,...) name ## 0<<<((end+256-1)/256),256>>>(end,__VA_ARGS__)
+#define cuLaunchKer(name,args) {                                      \
+    cuLaunchKernel(name ## 0,                                         \
+                   ((end+256-1)/256),1,1,                             \
+                   256,1,1,                                           \
+                   0,0,                                               \
+                   args);                                             \
+      }
+#define cuKerGBS(name,grid,block,end,...) name ## 0<<<grid,block>>>(end,__VA_ARGS__)
+#define call0p(name,id,grid,blck,...)                               \
+  printf("\033[32;1m[call0] name=%s grid:%d, block:%d\033[m\n",#name,grid,blck); \
+  call[id]<<<grid,blck>>>(__VA_ARGS__)
+#define call0(name,id,grid,blck,...) call[id]<<<grid,blck>>>(__VA_ARGS__)
+#define ReduceDecl(type,var,ini) double var=ini;
+#define ReduceForall(i,max,body) 
+
+// *****************************************************************************
+#else // KERNELS on CPU ********************************************************
+//#warning NO RAJA, NO NVCC
+#define sync
+#define share
+#define kernel
+class ReduceSum{
+public:
+  double s;
+public:
+  inline ReduceSum(double d):s(d){}
+  inline operator double() { return s; }
+  inline ReduceSum& operator +=(const double d) { return *this=(s+d); }
+};
+class ReduceMin{
+public:
+  double m;
+public:
+  inline ReduceMin(double d):m(d){}
+  inline operator double() { return m; }
+  inline ReduceMin& min(const double d) { return *this=(m<d)?m:d; }
+};
+#define ReduceDecl(type,var,ini) Reduce##type var(ini);
+#define forall(i,max,body) for(int i=0;i<max;i++){body}
+#define forallS(i,max,step,body) for(int i=0;i<max;i+=step){body}
+#define ReduceForall(i,max,body) forall(i,max,body)
+#define call0(name,id,grid,blck,...) call[id](__VA_ARGS__)
+#define cuKer(name,...) name ## 0(__VA_ARGS__)
+#define cuKerGBS(name,grid,block,end,...) name ## 0(end,__VA_ARGS__)
+#endif //__NVCC__
+#endif // __RAJA__
+#endif // MFEM_KERNELS_FORALL

backends/kernels/kernels/include/offsets.hpp

@@ -0,0 +1,35 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#ifndef MFEM_KERNELS_KERNEL_OFFSETS
+#define MFEM_KERNELS_KERNEL_OFFSETS
+
+// Offsets *********************************************************************
+#define   ijN(i,j,N) (i)+(N)*(j)
+#define  ijkN(i,j,k,N) (i)+(N)*((j)+(N)*(k))
+#define ijklN(i,j,k,l,N) (i)+(N)*((j)+(N)*((k)+(N)*(l)))
+
+#define    ijNMt(i,j,N,M,t) (t)?((i)+(N)*(j)):((j)+(M)*(i))
+#define    ijkNM(i,j,k,N,M) (i)+(N)*((j)+(M)*(k))
+#define   _ijkNM(i,j,k,N,M) (j)+(N)*((k)+(M)*(i))
+#define   ijklNM(i,j,k,l,N,M) (i)+(N)*((j)+(N)*((k)+(M)*(l)))
+#define  _ijklNM(i,j,k,l,N,M) (j)+(N)*((k)+(N)*((l)+(M)*(i)))
+#define   ijklmNM(i,j,k,l,m,N,M) (i)+(N)*((j)+(N)*((k)+(M)*((l)+(M)*(m))))
+#define __ijklmNM(i,j,k,l,m,N,M) (k)+(M)*((l)+(M)*((m)+(N*N)*((i)+(N)*j)))
+
+#define _ijklmNM(i,j,k,l,m,N,M) (j)+(N)*((k)+(N)*((l)+(N)*((m)+(M)*(i))))
+#define ijklmnNM(i,j,k,l,m,n,N,M) (i)+(N)*((j)+(N)*((k)+(M)*((l)+(M)*((m)+(M)*(n)))))
+
+#endif // MFEM_KERNELS_KERNEL_OFFSETS

backends/kernels/kernels/kernels.hpp

@@ -0,0 +1,60 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#ifndef MFEM_KERNELS_KERNELS_KERNELS
+#define MFEM_KERNELS_KERNELS_KERNELS
+
+// *****************************************************************************
+#define __TEMPLATES__
+
+// *****************************************************************************
+#include <math.h>
+#include <stdarg.h>
+#include <string.h>
+#include <assert.h>
+#include <unordered_map>
+
+// *****************************************************************************
+#define LOG2(X) ((unsigned) (8*sizeof(unsigned long long)-__builtin_clzll((X))))
+#define ISQRT(N) static_cast<unsigned>(sqrt(static_cast<float>(N)))
+#define ICBRT(N) static_cast<unsigned>(cbrt(static_cast<float>(N)))
+#define IROOT(D,N) ((D==1)?N:(D==2)?ISQRT(N):(D==3)?ICBRT(N):0)
+
+// *****************************************************************************
+#ifdef __NVCC__
+#include <cuda.h>
+#include <helper_cuda.h>
+#include <helper_functions.h>
+#endif // __NVCC__
+
+// *****************************************************************************
+#ifdef __RAJA__
+#include <cuda.h>
+#include "RAJA/RAJA.hpp"
+#include "RAJA/policy/cuda.hpp"
+#endif
+
+// *****************************************************************************
+#include "../config/dbg.hpp"
+#include "../config/nvvp.hpp"
+#include "../config/config.hpp"
+#include "../general/memcpy.hpp"
+#include "../general/malloc.hpp"
+
+// *****************************************************************************
+#include "include/forall.hpp"
+#include "include/offsets.hpp"
+
+#endif // MFEM_KERNELS_KERNELS_KERNELS

backends/kernels/kernels/mapping/globalToLocal.cpp

@@ -0,0 +1,64 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+// *****************************************************************************
+extern "C" kernel
+void rGlobalToLocal0(const int globalEntries,
+                     const int NUM_VDIM,
+                     const bool VDIM_ORDERING,                     
+                     const int localEntries,
+                     const int* __restrict offsets,
+                     const int* __restrict indices,
+                     const double* __restrict globalX,
+                     double* __restrict localX) {
+#ifdef __NVCC__
+  const int i = blockDim.x * blockIdx.x + threadIdx.x;
+  if (i < globalEntries)
+#else
+     forall(i,globalEntries,
+#endif
+  {
+    const int offset = offsets[i];
+    const int nextOffset = offsets[i+1];
+    for (int v = 0; v < NUM_VDIM; ++v) {
+      const int g_offset = ijNMt(v,i,NUM_VDIM,globalEntries,VDIM_ORDERING);
+      const double dofValue = globalX[g_offset];
+      for (int j = offset; j < nextOffset; ++j) {
+        const int l_offset = ijNMt(v,indices[j],NUM_VDIM,localEntries,VDIM_ORDERING);
+        localX[l_offset] = dofValue;
+      }
+    }
+  }
+#ifndef __NVCC__
+            );
+#endif
+}
+
+// *****************************************************************************
+void rGlobalToLocal(const int NUM_VDIM,
+                    const bool VDIM_ORDERING,
+                    const int globalEntries,
+                    const int localEntries,
+                    const int* __restrict offsets,
+                    const int* __restrict indices,
+                    const double* __restrict globalX,
+                    double* __restrict localX) {
+  nvtx_push(Lime);
+  cuKer(rGlobalToLocal,globalEntries,NUM_VDIM,VDIM_ORDERING,
+        localEntries,offsets,indices,globalX,localX);
+  nvtx_pop();
+}

backends/kernels/kernels/mapping/localToGlobal.cpp

@@ -0,0 +1,69 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+// *****************************************************************************
+extern "C" kernel
+void rLocalToGlobal0(const int globalEntries,
+                     const int NUM_VDIM,
+                     const bool VDIM_ORDERING,
+                     const int localEntries,
+                     const int* offsets,
+                     const int* indices,
+                     const double* localX,
+                     double* __restrict globalX)
+{
+#ifdef __NVCC__
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < globalEntries)
+#else
+   forall(i,globalEntries,
+#endif
+   {
+      const int offset = offsets[i];
+      const int nextOffset = offsets[i + 1];
+      for (int v = 0; v < NUM_VDIM; ++v)
+      {
+         double dofValue = 0;
+         for (int j = offset; j < nextOffset; ++j)
+         {
+            const int l_offset = ijNMt(v,indices[j],NUM_VDIM,localEntries,VDIM_ORDERING);
+            dofValue += localX[l_offset];
+         }
+         const int g_offset = ijNMt(v,i,NUM_VDIM,globalEntries,VDIM_ORDERING);
+         globalX[g_offset] = dofValue;
+      }
+   }
+#ifndef __NVCC__
+   );
+#endif
+}
+
+// *****************************************************************************
+void rLocalToGlobal(const int NUM_VDIM,
+                    const bool VDIM_ORDERING,
+                    const int globalEntries,
+                    const int localEntries,
+                    const int* offsets,
+                    const int* indices,
+                    const double* localX,
+                    double* globalX)
+{
+   nvtx_push();
+   cuKer(rLocalToGlobal,globalEntries,NUM_VDIM,VDIM_ORDERING,
+         localEntries,offsets,indices,localX,globalX);
+   nvtx_pop();
+}

backends/kernels/kernels/mapping/mapping.cpp

@@ -0,0 +1,104 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+// *****************************************************************************
+#ifdef __NVCC__
+extern "C" kernel
+void rSetSubVector0(const int N,
+                    const int* indices,
+                    const double* __restrict in,
+                    double* __restrict out)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < N) { out[indices[i]] = in[i]; }
+}
+#endif
+void rSetSubVector(const int N,
+                   const int* indices,
+                   const double* in,
+                   double* out)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(rSetSubVector,N,indices,in,out);
+#else
+   forall(i,N,out[indices[i]] = in[i];);
+#endif
+   nvtx_pop();
+}
+
+// *****************************************************************************
+#ifdef __NVCC__
+extern "C" kernel
+void rMapSubVector0(const int N,
+                    const int* indices,
+                    const double* __restrict in,
+                    double* __restrict out)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < N)
+   {
+      const int fromIdx = indices[2*i + 0];
+      const int toIdx   = indices[2*i + 1];
+      out[toIdx] = in[fromIdx];
+   }
+}
+#endif
+void rMapSubVector(const int N,
+                   const int* indices,
+                   const double* in,
+                   double* out)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(rMapSubVector,N,indices,in,out);
+#else
+   forall(i,N,
+   {
+      const int fromIdx = indices[2*i + 0];
+      const int toIdx   = indices[2*i + 1];
+      out[toIdx] = in[fromIdx];
+   });
+#endif
+   nvtx_pop();
+}
+
+// *****************************************************************************
+#ifdef __NVCC__
+extern "C" kernel
+void rExtractSubVector0(const int N,
+                        const int* indices,
+                        const double* __restrict in,
+                        double* __restrict out)
+{
+   const int i = blockDim.x * blockIdx.x + threadIdx.x;
+   if (i < N) { out[i] = in[indices[i]]; }
+}
+#endif
+void rExtractSubVector(const int N,
+                       const int* indices,
+                       const double* in,
+                       double* out)
+{
+   nvtx_push();
+#ifdef __NVCC__
+   cuKer(rExtractSubVector,N,indices,in,out);
+#else
+   forall(i,N,out[i] = in[indices[i]];);
+#endif
+   nvtx_pop();
+}

backends/kernels/kernels/mapping/mapping.hpp

@@ -0,0 +1,57 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#ifndef MFEM_KERNELS_MAPPING
+#define MFEM_KERNELS_MAPPING
+
+// *****************************************************************************
+void rSetSubVector(const int entries,
+                   const int* indices,
+                   const double* in,
+                   double* out);
+
+// *****************************************************************************
+void rMapSubVector(const int entries,
+                   const int* indices,
+                   const double* in,
+                   double* out);
+
+// *****************************************************************************
+void rExtractSubVector(const int entries,
+                       const int* indices,
+                       const double* in,
+                       double* out);
+
+// *****************************************************************************
+void rGlobalToLocal(const int NUM_VDIM,
+                    const bool VDIM_ORDERING,
+                    const int globalEntries,
+                    const int localEntries,
+                    const int* offsets,
+                    const int* indices,
+                    const double* globalX,
+                    double* localX);
+
+// *****************************************************************************
+void rLocalToGlobal(const int NUM_VDIM,
+                    const bool VDIM_ORDERING,
+                    const int globalEntries,
+                    const int localEntries,
+                    const int* offsets,
+                    const int* indices,
+                    const double* localX,
+                    double* globalX);
+
+#endif // MFEM_KERNELS_MAPPING

backends/kernels/kernels/mass/assemble.cpp

@@ -0,0 +1,39 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+#include "../kernels.hpp"
+
+// *****************************************************************************
+void rMassAssemble2D(const int, const int, const double,
+                     const double*, const double*, double*);
+void rMassAssemble3D(const int, const int, const double,
+                     const double*, const double*, double*);
+
+// *****************************************************************************
+void rMassAssemble(const int dim,
+                   const int NUM_QUAD,
+                   const int numElements,
+                   const double* quadWeights,
+                   const double* J,
+                   const double COEFF,
+                   double* oper)
+{
+   nvtx_push(Lime);
+   //assert(false);
+   if (dim==1) { assert(false); }
+   if (dim==2) { rMassAssemble2D(numElements,NUM_QUAD,COEFF,quadWeights,J,oper); }
+   if (dim==3) { rMassAssemble3D(numElements,NUM_QUAD,COEFF,quadWeights,J,oper); }
+   nvtx_pop();
+}