Interior DoF for Cubic Quadrilateral Elements Fixed and currently implemented for Quadrant 1 Mesh

Add dimension checks in the Get and Set methods of IntegrationPoint

CHANGELOG

@@ -13,6 +13,13 @@ Version 3.4.1 (development)
 - Added support for reading linear and quadratic 2D quadrilateral and triangular
   Cubit meshes.
 
+- The tetrahedral mesh refinement algorithm in serial and in parallel now
+  follows precisely the paper:
+     D. Arnold, A. Mukherjee, and L. Pouly, "Locally Adapted Tetrahedral Meshes
+     Using Bisection", SIAM J. Sci. Comput., 22(2), 431–448.
+  This guarantees that the shape regularity of the elements will be preserved
+  under refinement.
+
 
 Version 3.4, released on May 29, 2018
 =====================================

backends/all.hpp

@@ -1,27 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_ALL_HPP
-#define MFEM_BACKENDS_ALL_HPP
-
-#include "../config/config.hpp"
-
-#ifdef MFEM_USE_BACKENDS
-
-#include "base/backend.hpp"
-
-#ifdef MFEM_USE_OCCA
-#include "occa/backend.hpp"
-#endif
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_ALL_HPP

backends/base/array.hpp

@@ -1,213 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_ARRAY_HPP
-#define MFEM_BACKENDS_BASE_ARRAY_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "layout.hpp"
-#include "utils.hpp"
-
-namespace mfem
-{
-
-/// Extension to the template class Array<T>
-class PArray : public RefCounted
-{
-protected:
-   /// Layout with shared ownership (smart pointer)
-   DLayout layout;
-
-
-   /**
-       @name Virtual interface
-    */
-   ///@{
-
-   /** @brief Create and return a new array (in @a *clone) of the same dynamic
-       type as this array using the same layout and ItemSize().
-
-       Set @a *clone to NULL if allocation fails.
-
-       If @a copy_data is true, the contents of this array is copied to the new
-       array; otherwise, the new array remains uninitialized.
-
-       If @a buffer is not NULL, return the array data of the newly created
-       object (in @a *buffer) , if it is stored as a contiguous array on the
-       host; otherwise, set @a *buffer to NULL. */
-   virtual PArray *DoClone(bool copy_data, void **buffer,
-                           std::size_t item_size) const = 0;
-
-   /// Resize the array, reallocating its data if necessary.
-   /** If @a buffer is not NULL, return the array data (in @a *buffer), if it
-       is stored as a contiguous array on the host; otherwise, set @a *buffer to
-       NULL. Returns 0 on success and non-zero otherwise, e.g. if memory
-       allocation fails.
-
-       If the @a new_layout is not supported, a non-zero error code will be
-       returned.
-
-       The @a new_layout has to be valid, i.e. new_layout != NULL and
-       new_layout->HasEngine() == true.
-
-       @note If reallocation is performed, the previous content of the array is
-       NOT copied to the new location. */
-   virtual int DoResize(PLayout &new_layout, void **buffer,
-                        std::size_t item_size) = 0;
-
-   /** @brief Get access to the contents of the array in host memory, as a
-       contiguous array. */
-   /** If the array data is stored as a contiguous array in host memory, return
-       a pointer to it. Otherwise, copy the data to @a buffer (if @a buffer is
-       not NULL) and return @a buffer.
-       @note If not NULL, @a buffer is assumed to be of size greater than or
-       equal to Size(). */
-   virtual void *DoPullData(void *buffer, std::size_t item_size) = 0;
-
-   /** @brief Set all entries of the array to the (single) value pointed to by
-       @a value_ptr. */
-   virtual void DoFill(const void *value_ptr, std::size_t item_size) = 0;
-
-   /** @brief Set all Size() entries of the array from the given contiguous
-       array, @a src_buffer, on the host. */
-   virtual void DoPushData(const void *src_buffer, std::size_t item_size) = 0;
-
-   /// Copy the data from @a src to @a *this.
-   /** Both arrays must have the same dynamic type, layout, and item_size. */
-   virtual void DoAssign(const PArray &src, std::size_t item_size) = 0;
-
-   ///@}
-   // End: Virtual interface
-
-public:
-   /** @brief The @a layout parameter will be reference counted and therefore it
-       should be dynamically allocated. */
-   /** The @a layout must be valid in the sense that layout != NULL and
-       layout->HasEngine() == true. */
-   PArray(PLayout &p_layout)
-      : layout(&p_layout)
-   {
-      MFEM_ASSERT(layout && layout->HasEngine(), "invalid layout");
-   }
-
-   virtual ~PArray() { }
-
-   /// Get the current size of the array.
-   std::size_t Size() const { return layout->Size(); }
-
-   /// Get the current layout of the array.
-   PLayout &GetLayout() const { return *layout; }
-
-   /// TODO
-   /// Note: we cannot use static_cast for class PArray.
-   template <typename derived_t>
-   derived_t &As() { return dynamic_cast<derived_t&>(*this); }
-
-   /// TODO
-   /// Note: we cannot use static_cast for class PArray.
-   template <typename derived_t>
-   const derived_t &As() const { return dynamic_cast<const derived_t&>(*this); }
-
-
-   // TODO: Error handling ... handle errors at the Engine level, at the class
-   //       level, or at the method level?
-
-   // TODO: Asynchronous execution interface ...
-
-
-   /**
-       @name Public virtual interface
-    */
-   ///@{
-
-   /** @brief Create and return a new array (in @a *clone) of the same dynamic
-       type as this array using the same layout and ItemSize().
-
-       Set @a *clone to NULL if allocation fails.
-
-       If @a copy_data is true, the contents of this array is copied to the new
-       array; otherwise, the new array remains uninitialized.
-
-       If @a buffer is not NULL, return the array data of the newly created
-       object (in @a *buffer) , if it is stored as a contiguous array on the
-       host; otherwise, set @a *buffer to NULL. */
-   template <typename T>
-   DArray Clone(bool copy_data, T **buffer) const
-   { return DArray(DoClone(copy_data, (void**)buffer, sizeof(T))); }
-
-   /// Resize the array, reallocating its data if necessary.
-   /** If @a buffer is not NULL, return the array data (in @a *buffer), if it
-       is stored as a contiguous array on the host; otherwise, set @a *buffer to
-       NULL. Returns 0 on success and non-zero otherwise, e.g. if memory
-       allocation fails.
-
-       If the @a new_layout is not supported, a non-zero error code will be
-       returned.
-
-       The @a new_layout has to be valid, i.e. new_layout != NULL and
-       new_layout->HasEngine() == true.
-
-       @note If reallocation is performed, the previous content of the array is
-       NOT copied to the new location. */
-   template <typename T>
-   int Resize(PLayout &new_layout, T **buffer)
-   { return DoResize(new_layout, (void**)buffer, sizeof(T)); }
-
-   /// Shortcut for Resize(*layout, buffer).
-   /** This method is useful for updating the array after its layout is changed
-       externally. */
-   template <typename T>
-   int Update(T **buffer)
-   { return DoResize(*layout, (void**)buffer, sizeof(T)); }
-
-   /// Shortcut for layout->Resize(new_size) followed by Update()
-   template <typename T>
-   int Resize(std::size_t new_size, T **buffer)
-   { layout->Resize(new_size); return Update(buffer); }
-
-   /** @brief Get access to the contents of the array in host memory, as a
-       contiguous array. */
-   /** If the array data is stored as a contiguous array in host memory, return
-       a pointer to it. Otherwise, copy the data to @a buffer (if @a buffer is
-       not NULL) and return @a buffer.
-       @note If not NULL, @a buffer is assumed to be of size greater than or
-       equal to Size(). */
-   template <typename T>
-   T *PullData(T *buffer)
-   { return Size() ? (T*)DoPullData((void*)buffer, sizeof(T)) : NULL; }
-
-   /** @brief Set all entries of the array to the (single) value pointed to by
-       @a value_ptr. */
-   template <typename T>
-   void Fill(const T &value) { if (Size()) { DoFill(&value, sizeof(T)); } }
-
-   /** @brief Set all Size() entries of the array from the given contiguous
-       array, @a src_buffer, on the host. */
-   template <typename T>
-   void PushData(const T *src_buffer)
-   { if (Size()) { DoPushData(src_buffer, sizeof(T)); } }
-
-   /// Copy the data from @a src to @a *this.
-   /** Both arrays must have the same dynamic type, layout, and entry type. */
-   template <typename T>
-   void Assign(const PArray &src) { if (Size()) { DoAssign(src, sizeof(T)); } }
-
-   ///@}
-   // End: Virtual interface
-};
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_ARRAY_HPP

backends/base/backend.hpp

@@ -1,57 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_BACKEND_HPP
-#define MFEM_BACKENDS_BASE_BACKEND_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "memory_resource.hpp"
-#include "engine.hpp"
-#include "array.hpp"
-#include "vector.hpp"
-#include "fespace.hpp"
-#include "bilinearform.hpp"
-
-#include <string>
-
-#ifdef MFEM_USE_MPI
-#include <mpi.h>
-#endif
-
-namespace mfem
-{
-
-/// TODO
-class Backend
-{
-public:
-   /// TODO
-   virtual ~Backend() { }
-
-   /// TODO
-   virtual bool Supports(const std::string &engine_spec) const = 0;
-
-   /// TODO
-   virtual Engine *Create(const std::string &engine_spec) = 0;
-
-#ifdef MFEM_USE_MPI
-   /// TODO
-   virtual Engine *Create(MPI_Comm comm, const std::string &engine_spec) = 0;
-#endif
-};
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_BACKEND_HPP

backends/base/bilinearform.hpp

@@ -1,72 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_BILINEARFORM_HPP
-#define MFEM_BACKENDS_BASE_BILINEARFORM_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "engine.hpp"
-
-namespace mfem
-{
-
-class Vector;
-class OperatorHandle;
-class BilinearForm;
-
-/// TODO: doxygen
-class PBilinearForm : public RefCounted
-{
-protected:
-   /// Engine with shared ownership
-   SharedPtr<const Engine> engine;
-   /// Not owned.
-   BilinearForm *bform;
-
-public:
-   /// TODO: doxygen
-   PBilinearForm(const Engine &e, BilinearForm &bf)
-      : engine(&e), bform(&bf) { }
-
-   /// Virtual destructor
-   virtual ~PBilinearForm() { }
-
-   /// Get the associated Engine
-   const Engine &GetEngine() const { return *engine; }
-
-   /// Assemble the PBilinearForm.
-   /** This method is called from the method BilinearForm::Assemble() of the
-       associated BilinearForm #bform.
-       @returns True, if the host assembly should be skipped. */
-   virtual bool Assemble() = 0;
-
-   /// TODO: doxygen
-   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
-                                 OperatorHandle &A) = 0;
-
-   /// TODO: doxygen
-   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
-                                 Vector &x, Vector &b,
-                                 OperatorHandle &A, Vector &X, Vector &B,
-                                 int copy_interior) = 0;
-
-   /// TODO: doxygen
-   virtual void RecoverFEMSolution(const Vector &X, const Vector &b,
-                                   Vector &x) = 0;
-};
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_BILINEARFORM_HPP

backends/base/engine.cpp

@@ -1,49 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "engine.hpp"
-#include "fespace.hpp"
-#include "bilinearform.hpp"
-
-namespace mfem
-{
-
-Engine::Engine(Backend *b, int n_mem, int n_workers)
-   : backend(b),
-#ifdef MFEM_USE_MPI
-     comm(MPI_COMM_NULL),
-#endif
-     num_mem_res(n_mem),
-     num_workers(n_workers),
-     memory_resources(new MemoryResource*[num_mem_res]()),
-     workers_weights(new double[num_workers]()),
-     workers_mem_res(new int[num_workers]())
-{
-   // Note: all arrays are value-initialized with zeros.
-}
-
-Engine::~Engine()
-{
-   delete [] workers_mem_res;
-   delete [] workers_weights;
-   for (int i = 0; i < num_mem_res; i++)
-   {
-      delete memory_resources[i];
-   }
-   delete [] memory_resources;
-}
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS

backends/base/engine.hpp

@@ -1,190 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_ENGINE_HPP
-#define MFEM_BACKENDS_BASE_ENGINE_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "../../general/scalars.hpp"
-#include "memory_resource.hpp"
-#include "smart_pointers.hpp"
-#include "utils.hpp"
-
-#ifdef MFEM_USE_MPI
-#include <mpi.h>
-#endif
-
-namespace mfem
-{
-
-// Forward declarations.
-class Backend;
-template <typename T> class Array;
-class Operator;
-class FiniteElementSpace;
-class LinearForm;
-class BilinearForm;
-class MixedBilinearForm;
-class NonlinearForm;
-
-
-/// In parallel, each MPI rank will usually create a single engine.
-class Engine : public RefCounted
-{
-protected:
-   Backend *backend; ///< Backend that created the engine. Not owned.
-
-#ifdef MFEM_USE_MPI
-   MPI_Comm comm; ///< Associated MPI communicator (may be MPI_COMM_NULL).
-#endif
-
-   /// Number of memory resources used by the Engine.
-   int num_mem_res;
-   /// Number of workers used by the Engine.
-   int num_workers;
-
-   /// Memory resources used by the engine - array of pointers.
-   /** Both the array and the entries are owned. */
-   MemoryResource **memory_resources;
-
-   /// Relative computational speed of the workers. Owned.
-   double *workers_weights;
-
-   /// For each worker, which memory resource it uses.
-   int *workers_mem_res;
-
-public:
-   /// TODO: doxygen
-   Engine(Backend *b, int n_mem, int n_workers);
-
-   /// TODO: doxygen
-   virtual ~Engine();
-
-
-   /**
-       @name Machine resources interface
-    */
-   ///@{
-
-#ifdef MFEM_USE_MPI
-   /// Get the associated MPI_Comm
-   MPI_Comm GetComm() const { return comm; }
-#endif
-
-   /// TODO
-   int GetNumMemRes() const { return num_mem_res; }
-
-   /// TODO
-   MemoryResource &GetMemRes(int idx) const { return *memory_resources[idx]; }
-
-   /// TODO
-   int GetNumWorkers() const { return num_workers; }
-
-   /// TODO
-   const double *GetWorkersWeights() const { return workers_weights; }
-
-   /// TODO
-   const int *GetWorkersMemRes() const { return workers_mem_res; }
-
-   ///@}
-   // End: Machine resources interface
-
-   /// TODO
-   template <typename derived_t>
-   derived_t &As() { return *util::As<derived_t>(this); }
-
-   /// TODO
-   template <typename derived_t>
-   const derived_t &As() const { return *util::As<const derived_t>(this); }
-
-
-   // TODO: Error handling ... handle errors at the Engine level, at the class
-   //       level, or at the method level?
-
-
-   /**
-       @name Virtual interface: finite element data structures and algorithms
-    */
-   ///@{
-
-   // TODO: Asynchronous execution in this class ...
-
-   /// Allocate and return a new layout for the given @a size.
-   /** The layout decomposition (in the case of multiple workers) is determined
-       automatically by the Engine using a deterministic algorithm: calls to
-       this method with the same @a size will produce the same result, as long
-       as the Engine remains unmodified between the calls.
-
-       The returned object is allocated with operator new and must be
-       deallocated by the caller.
-
-       TODO: Returns NULL if memory allocation fails?
-   */
-   virtual DLayout MakeLayout(std::size_t size) const = 0;
-
-   /// Allocate and return a new layout for the given worker decomposition.
-   /** The returned object is allocated with operator new and must be
-       deallocated by the caller.
-
-       TODO: Returns NULL if memory allocation fails?
-
-       The @a offsets should satisfy: offsets.Size() == number of workers + 1,
-       offsets[0] == 0, and offsets[i] <= offsets[i+1], for i: 0 <= i < number
-       of workers. */
-   virtual DLayout MakeLayout(const Array<std::size_t> &offsets) const = 0;
-
-   // Note: There may be other ways to construct layouts in the future, e.g.
-   //       block-vector layouts, or multi-vector layouts.
-
-   /// TODO
-   virtual DArray MakeArray(PLayout &layout, std::size_t item_size) const = 0;
-
-   /// Allocate and return a new vector using the given @a layout.
-   /** The returned object is a smart pointer that will automatically deallocate
-       the vector.
-
-       TODO: Produce an error if memory allocation fails?
-
-       Only layouts returned by this Engine are guaranteed to be supported.
-       Using a type that is not supported will produce an error. */
-   virtual DVector MakeVector(PLayout &layout,
-                              int type_id = ScalarId<double>::value) const = 0;
-
-   /// TODO: doxygen
-   virtual DFiniteElementSpace MakeFESpace(FiniteElementSpace &fes) const = 0;
-
-   /// TODO: doxygen
-   virtual DBilinearForm MakeBilinearForm(BilinearForm &bf) const = 0;
-
-
-   // Question: How do we construct coefficients?
-
-
-   /// FIXME - What will the actual parameters be?
-   virtual void AssembleLinearForm(LinearForm &l_form) const = 0;
-
-   /// FIXME - What will the actual parameters be?
-   virtual Operator *MakeOperator(const MixedBilinearForm &mbl_form) const = 0;
-
-   /// FIXME - What will the actual parameters be?
-   virtual Operator *MakeOperator(const NonlinearForm &nl_form) const = 0;
-
-   ///@}
-   // End: Virtual interface
-};
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_ENGINE_HPP

backends/base/fespace.hpp

@@ -1,98 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_FE_SPACE_HPP
-#define MFEM_BACKENDS_BASE_FE_SPACE_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "engine.hpp"
-#include "utils.hpp"
-
-namespace mfem
-{
-
-class FiniteElementSpace;
-class QuadratureSpace;
-
-/// TODO: doxygen
-class PFiniteElementSpace : public RefCounted
-{
-protected:
-   /// Engine with shared ownership
-   SharedPtr<const Engine> engine;
-   /// Not owned.
-   mfem::FiniteElementSpace *fes;
-
-public:
-   /// TODO: doxygen
-   PFiniteElementSpace(const Engine &e, mfem::FiniteElementSpace &fespace)
-      : engine(&e), fes(&fespace) { }
-
-   /// Virtual destructor
-   virtual ~PFiniteElementSpace() { }
-
-   /// Get the associated engine
-   const Engine &GetEngine() const { return *engine; }
-
-   /// Return the associated mfem::FiniteElementSpace
-   mfem::FiniteElementSpace *GetFESpace() const { return fes; }
-
-   /// TODO
-   template <typename derived_t>
-   derived_t &As() { return *util::As<derived_t>(this); }
-
-   /// TODO
-   template <typename derived_t>
-   const derived_t &As() const { return *util::As<const derived_t>(this); }
-
-
-   /**
-       @name Virtual interface: finite element space functionality
-    */
-   ///@{
-
-   /// TODO
-   /** Return the operator mapping T-vectors to L-vectors. If a NULL pointer is
-       returned then the mapping is the idenity. */
-   virtual const mfem::Operator *GetProlongationOperator() const = 0;
-
-   /// TODO
-   /** Return the operator mapping L-vectors to T-vectors that extracts the
-       subset of all true dofs, i.e. no assembly is performed. If a NULL pointer
-       is returned then the mapping is the idenity. */
-   virtual const mfem::Operator *GetRestrictionOperator() const = 0;
-
-   /// TODO
-   /** Return the operator mapping L-vectors to Q-vectors that evaluates the
-       values of a GridFunction as a QuadratureFunction on the given
-       QuadratureSpace. If the returned pointer is NULL, then the mapping is the
-       identity. */
-   virtual const mfem::Operator *GetInterpolationOperator(
-      const mfem::QuadratureSpace &qspace) const = 0;
-
-   /// TODO
-   /** Return the operator mapping L-vectors to Q-vectors that evaluates the
-       _reference element_ gradients of a GridFunction as a QuadratureFunction
-       on the given QuadratureSpace. */
-   virtual const mfem::Operator *GetGradientOperator(
-      const mfem::QuadratureSpace &qspace) const = 0;
-
-   ///@}
-   // End: Virtual interface
-};
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_FE_SPACE_HPP

backends/base/layout.hpp

@@ -1,110 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_LAYOUT_HPP
-#define MFEM_BACKENDS_BASE_LAYOUT_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "smart_pointers.hpp"
-#include "engine.hpp"
-
-namespace mfem
-{
-
-/// Polymorphic layout (array/vector layout descriptor)
-class PLayout : public RefCounted
-{
-protected:
-   /// Engine with shared ownership
-   SharedPtr<const Engine> engine;
-   std::size_t size;
-
-   template <typename DObject>
-   struct Maker
-   {
-      template <typename entry_t>
-      static DObject MakeNew(PLayout &layout);
-   };
-
-public:
-   explicit PLayout(std::size_t s = 0) : engine(NULL), size(s) { }
-
-   explicit PLayout(const Engine &e, std::size_t s = 0)
-      : engine(&e), size(s) { }
-
-   virtual ~PLayout() { }
-
-   /**
-       @name Virtual interface
-    */
-   ///@{
-
-   /// Resize the layout
-   virtual void Resize(std::size_t new_size) { size = new_size; }
-
-   /// Resize the layout based on the given worker offsets
-   virtual void Resize(const Array<std::size_t> &offsets)
-   { MFEM_ABORT("method not supported"); }
-
-   ///@}
-   // End: Virtual interface
-
-   /// Layouts without engine cannot create DArray, DVector, etc.
-   bool HasEngine() const { return engine != NULL; }
-
-   /// TODO: doxygen
-   const Engine &GetEngine() const { return *engine; }
-
-   /// TODO: doxygen
-   std::size_t Size() const { return size; }
-
-   /// TODO
-   /** Useful in backends for down-casting to a backend-specific layout type.
-
-       When MFEM_DEBUG=YES, performs a type check using dynamic_cast. */
-   template <typename derived_t>
-   derived_t &As() { return *util::As<derived_t>(this); }
-
-   /// TODO
-   /** Useful in backends for down-casting to a backend-specific layout type.
-
-       When MFEM_DEBUG=YES, performs a type check using dynamic_cast. */
-   template <typename derived_t>
-   const derived_t &As() const { return *util::As<const derived_t>(this); }
-
-   /// TODO: doxygen
-   template <typename DObject, typename entry_t>
-   DObject Make()
-   {
-      MFEM_ASSERT(HasEngine(), "this method requires an Engine");
-      return Maker<DObject>::template MakeNew<entry_t>(*this);
-   }
-};
-
-template <> struct PLayout::Maker<DArray>
-{
-   template <typename entry_t> static DArray MakeNew(PLayout &layout)
-   { return layout.GetEngine().MakeArray(layout, sizeof(entry_t)); }
-};
-
-template <> struct PLayout::Maker<DVector>
-{
-   template <typename entry_t> static DVector MakeNew(PLayout &layout)
-   { return layout.GetEngine().MakeVector(layout, ScalarId<entry_t>::value); }
-};
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_LAYOUT_HPP

backends/base/memory_resource.cpp

@@ -1,59 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "memory_resource.hpp"
-#include "../../general/error.hpp"
-
-#include <cstdlib>
-#include <cstring>
-#include <cerrno>
-
-namespace mfem
-{
-
-void *NewDeleteMemoryResource::DoAllocate(std::size_t bytes,
-                                          std::size_t alignment)
-{
-   void *p = ::operator new[](bytes);
-   MFEM_VERIFY(!alignment || (std::size_t)(p) % alignment == 0,
-               "invalid alignment");
-   return p;
-}
-
-void NewDeleteMemoryResource::DoDeallocate(void *p, std::size_t bytes,
-                                           std::size_t alignment)
-{
-   ::operator delete[](p);
-}
-
-
-void *AlignedMemoryResource::DoAllocate(std::size_t bytes,
-                                        std::size_t alignment)
-{
-   void *p;
-   if (!alignment) { alignment = sizeof(long double); }
-   MFEM_VERIFY(posix_memalign(&p, alignment, bytes) == 0,
-               "error in posix_memalign(): " << strerror(errno));
-   return p;
-}
-
-void AlignedMemoryResource::DoDeallocate(void *p, std::size_t bytes,
-                                         std::size_t alignment)
-{
-   free(p);
-}
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS

backends/base/memory_resource.hpp

@@ -1,70 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP
-#define MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include <cstddef>
-
-namespace mfem
-{
-
-/// Polymorphic memory resource. Similar to C++17's std::pmr::memory_resource.
-class MemoryResource
-{
-protected:
-   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment) = 0;
-   virtual void DoDeallocate(void* p, std::size_t bytes,
-                             std::size_t alignment) = 0;
-
-public:
-   // Implicitly defined default & copy constructors
-
-   /// Virtual destructor.
-   virtual ~MemoryResource() { }
-
-   /// If alignment == 0, use default alignment.
-   void *Allocate(std::size_t bytes, std::size_t alignment = 0)
-   { return DoAllocate(bytes, alignment); }
-
-   /// If alignment == 0, use default alignment.
-   void Deallocate(void *p, std::size_t bytes, std::size_t alignment = 0)
-   { DoDeallocate(p, bytes, alignment); }
-};
-
-
-/** @brief Dynamic host memory resource using operator new[](std::size_t) for
-    allocation and operator delete[](void*) for deallocation. */
-class NewDeleteMemoryResource : public MemoryResource
-{
-protected:
-   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment);
-   virtual void DoDeallocate(void *p, std::size_t bytes, std::size_t alignment);
-};
-
-
-/** @brief Dynamic host memory resource using posix_memalign() for aligned
-    allocation and free() for deallocation. */
-class AlignedMemoryResource : public MemoryResource
-{
-protected:
-   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment);
-   virtual void DoDeallocate(void *p, std::size_t bytes, std::size_t alignment);
-};
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP

backends/base/smart_pointers.hpp

@@ -1,234 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_SMART_POINTERS_HPP
-#define MFEM_BACKENDS_BASE_SMART_POINTERS_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "utils.hpp"
-#include "../../general/error.hpp"
-#include <cstddef>
-
-// #define MFEM_TRACE_SHARED_PTR
-#ifdef MFEM_TRACE_SHARED_PTR
-#include "../../general/globals.hpp"
-#endif
-
-namespace mfem
-{
-
-/// Base class for classes with simple reference counting.
-/** Reference counting is performed by the class SharedPtr. */
-class RefCounted
-{
-private:
-   mutable unsigned ref_count;
-
-   /// Only class SharedPtr can access ref_count.
-   template <typename T> friend class SharedPtr;
-
-public:
-   RefCounted() : ref_count(0) { }
-
-   /** @brief Prevent SharedPtr objects from deleting this object by
-       incrementing the reference counter by one. */
-   void DontDelete() const { ++ref_count; }
-};
-
-
-/** @brief Smart pointer class that manages objects of type T derived from class
-    RefCounted. */
-/** This class is generally meant to work with dynamically allocated object,
-    specifically objects allocated with operator new(). It will invoke operator
-    delete() to destroy the managed object when its reference counter reaches
-    zero. This behavior can be overriden by calling RefCounted::DontDelete() to
-    ensure that an object will not be deleted by a SharedPtr that holds a
-    pointer to it.
-    @note This class is NOT thread-safe and does not support circular ownership.
- */
-template <typename T>
-class SharedPtr
-{
-public:
-   typedef T stored_type;
-
-private:
-   T *ptr;
-
-   void Init(T *new_ptr)
-   {
-      ptr = new_ptr;
-      if (ptr) { ++ptr->RefCounted::ref_count; }
-#ifdef MFEM_TRACE_SHARED_PTR
-#elif 0
-      mfem::out << "  [" << _MFEM_FUNC_NAME << "]: ptr = " << ptr;
-      if (ptr)
-      {
-         mfem::out << ", new ref_count = " << ptr->RefCounted::ref_count;
-      }
-      mfem::out << '\n';
-#endif
-   }
-   void Destroy()
-   {
-      MFEM_ASSERT(!ptr || ptr->RefCounted::ref_count >= 1, "invalid use");
-      if (ptr && --ptr->RefCounted::ref_count == 0) { delete ptr; }
-#ifdef MFEM_TRACE_SHARED_PTR
-#elif 0
-      mfem::out << "  [" << _MFEM_FUNC_NAME << "]: ptr = " << ptr;
-      if (ptr)
-      {
-         mfem::out << ", new ref_count = " << ptr->RefCounted::ref_count;
-      }
-      mfem::out << '\n';
-#endif
-   }
-
-public:
-   SharedPtr() : ptr(NULL)
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]: ptr = " << ptr << '\n';
-#endif
-   }
-
-   SharedPtr(const SharedPtr &other)
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      Init(other.ptr);
-   }
-
-   template <typename U>
-   SharedPtr(const SharedPtr<U> &other)
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      Init(other.Get());
-   }
-
-   explicit SharedPtr(T *p)
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      Init(p);
-   }
-
-   ~SharedPtr()
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      Destroy();
-   }
-
-   SharedPtr &operator=(const SharedPtr &other)
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      Reset(other.ptr); return *this;
-   }
-
-   template <typename U>
-   SharedPtr &operator=(const SharedPtr<U> &other)
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      Reset(other.Get()); return *this;
-   }
-
-   T &operator*() const { return *ptr; }
-   T *operator->() const { return ptr; }
-
-   operator bool() const { return ptr; }
-   bool operator!() const { return !ptr; }
-
-   template <typename U>
-   bool operator==(const SharedPtr<U> &other) const
-   { return ptr == other.Get(); }
-   template <typename U>
-   bool operator!=(const SharedPtr<U> &other) const
-   { return ptr != other.Get(); }
-
-   // Comparison to any type convertible to void *, e.g. the type of NULL.
-   template <typename U>
-   bool operator==(const U &p) const { return ptr == (void*) p; }
-   template <typename U>
-   bool operator!=(const U &p) const { return ptr != (void*) p; }
-
-   T *Get() const { return ptr; }
-
-   /// TODO
-   template <typename derived_t>
-   derived_t *As() const { return util::As<derived_t>(ptr); }
-
-   unsigned UseCount() const { return ptr ? ptr->RefCounted::ref_count : 0; }
-
-   void Reset()
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      Destroy();
-      ptr = NULL;
-   }
-
-   /// The type U* needs to be implicitly convertible to T*
-   template <typename U>
-   void Reset(U *new_ptr)
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      if (ptr != new_ptr) { Destroy(); Init(new_ptr); }
-   }
-
-   void Swap(SharedPtr &other)
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      std::swap(ptr, other.ptr);
-   }
-};
-
-
-template <class T>
-inline void Swap(SharedPtr<T> &a, SharedPtr<T> &b) { a.Swap(b); }
-
-
-class PLayout;
-typedef SharedPtr<PLayout> DLayout;
-
-class PArray;
-typedef SharedPtr<PArray> DArray;
-
-class PVector;
-typedef SharedPtr<PVector> DVector;
-
-class PFiniteElementSpace;
-typedef SharedPtr<PFiniteElementSpace> DFiniteElementSpace;
-
-class PBilinearForm;
-typedef SharedPtr<PBilinearForm> DBilinearForm;
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_SMART_POINTERS_HPP

backends/base/utils.hpp

@@ -1,52 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_UTILS_HPP
-#define MFEM_BACKENDS_BASE_UTILS_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "../../general/error.hpp"
-
-namespace mfem
-{
-
-namespace util
-{
-
-//
-// Inline methods
-//
-
-/// TODO: doxygen
-template <typename derived_t, typename base_t>
-inline derived_t *As(base_t *base_obj)
-{
-   MFEM_ASSERT(dynamic_cast<derived_t*>(base_obj) != NULL,
-               "invalid object type");
-   return static_cast<derived_t*>(base_obj);
-}
-
-/// TODO: doxygen
-template <typename derived_t, typename base_t>
-inline derived_t *Is(base_t *base_obj)
-{
-   return dynamic_cast<derived_t*>(base_obj);
-}
-
-} // namespace mfem::util
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_UTILS_HPP

backends/base/vector.hpp

@@ -1,153 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_VECTOR_HPP
-#define MFEM_BACKENDS_BASE_VECTOR_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "../../general/scalars.hpp"
-#include "array.hpp"
-
-namespace mfem
-{
-
-/// Polymorphic vector - array of scalars.
-class PVector : virtual public PArray
-{
-protected:
-   /**
-       @name Virtual interface
-    */
-   ///@{
-
-   /** @brief Create and return a new vector of the same dynamic type as this
-       vector using the same layout with entries specified by @a buffer_type_id
-       which should be a constant defined by the `value` field in a
-       specialization of the template class mfem::ScalarId.
-
-       Returns NULL if allocation fails.
-
-       If @a copy_data is true, the contents of this vector is copied to the new
-       vector; otherwise, the new vector remains uninitialized.
-
-       If @a buffer is not NULL, return the vector data of the newly created
-       object (in @a *buffer), if it is stored as a contiguous array on the
-       host; otherwise, set @a *buffer to NULL. */
-   virtual PVector *DoVectorClone(bool copy_data, void **buffer,
-                                  int buffer_type_id) const = 0;
-
-   /** @brief Compute and return the dot product of @a *this and @a x. In the
-       case of an MPI-parallel vector, the result must be the MPI-global dot
-       product. */
-   /** Both vectors must have the same dynamic type and layout. */
-   virtual void DoDotProduct(const PVector &x, void *result,
-                             int result_type_id) const = 0;
-
-   // TODO: add reduction operations: min, max, sum
-
-   /// Perform the operation @a *this = @a a @a x + @a b @a y.
-   /** Rules:
-       - the dynamic type of both @a x and @a y is the same as that of @a *this
-       - if @a a == 0, neither @a x nor its data are accessed
-       - if @a b == 0, neither @a y nor its data are accessed
-       - @a x's data is never the same as @a y's data, unless @a a == 0, or
-         @a b == 0
-       - @a x's data or @a y's data may be the same as the data of @a *this
-       - all accessed vectors, @a x, @a y, and @a *this have the same layout. */
-   virtual void DoAxpby(const void *a, const PVector &x,
-                        const void *b, const PVector &y,
-                        int ab_type_id) = 0;
-
-   ///@}
-   // End: Virtual interface
-
-public:
-   /** @brief Create a PVector. */
-   /** The @a layout must be valid in the sense that layout != NULL and
-       layout->HasEngine() == true. */
-   PVector(PLayout &p_layout)
-      : PArray(p_layout) { }
-
-   template <typename derived_t>
-   derived_t &As() { return *util::As<derived_t>(this); }
-
-   template <typename derived_t>
-   const derived_t &As() const { return *util::As<const derived_t>(this); }
-
-
-   // TODO: Error handling ... handle errors at the Engine level, at the class
-   //       level, or at the method level?
-
-   // TODO: Asynchronous execution interface ...
-
-   // TODO: Multi-vector interface ...
-
-
-   /**
-       @name Public virtual interface
-    */
-   ///@{
-
-   /** @brief Create and return a new vector of the same dynamic type as this
-       vector using the same layout with entries of type @a scalar_t.
-
-       If @a copy_data is true, the contents of this vector is copied to the new
-       vector; otherwise, the new vector remains uninitialized.
-
-       If @a buffer is not NULL, return the vector data of the newly created
-       object (in @a *buffer) , if it is stored as a contiguous array on the
-       host; otherwise, set @a *buffer to NULL. */
-   template <typename scalar_t>
-   DVector Clone(bool copy_data, scalar_t **buffer) const
-   {
-      return DVector(DoVectorClone(copy_data, (void**)buffer,
-                                   ScalarId<scalar_t>::value));
-   }
-
-   /** @brief Compute and return the dot product of @a *this and @a x. In the
-       case of an MPI-parallel vector, the result must be the MPI-global dot
-       product. */
-   /** Both vectors must have the same dynamic type and layout. */
-   template <typename scalar_t>
-   scalar_t DotProduct(const PVector &x) const
-   {
-      scalar_t result;
-      DoDotProduct(x, &result, ScalarId<scalar_t>::value);
-      return result;
-   }
-
-   // TODO: add reduction operations: min, max, sum
-
-   /// Perform the operation @a *this = @a a @a x + @a b @a y.
-   /** Rules:
-       - the dynamic type of both @a x and @a y is the same as that of @a *this
-       - if @a a == 0, neither @a x nor its data are accessed
-       - if @a b == 0, neither @a y nor its data are accessed
-       - @a x's data is never the same as @a y's data, unless @a a == 0, or
-         @a b == 0
-       - @a x's data or @a y's data may be the same as the data of @a *this
-       - all accessed vectors, @a x, @a y, and @a *this have the same layout. */
-   template <typename scalar_t>
-   void Axpby(const scalar_t &a, const PVector &x,
-              const scalar_t &b, const PVector &y)
-   { if (Size()) { DoAxpby(&a, x, &b, y, ScalarId<scalar_t>::value); } }
-
-   ///@}
-   // End: Virtual interface
-};
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_VECTOR_HPP

backends/occa/DiffusionIntegrator.okl

@@ -1,66 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-/*
-  ---[ Defines Known At Compile-Time ]------------
-  ELEMENT_BATCH   : How many elements are in each
-  .                   computation batch
-  NUM_DOFS_1D     : Dofs in the 1D segments
-  NUM_DOFS_2D     : Dofs in the 2D faces
-  NUM_DOFS_3D     : Dofs in the 3D domain
-  NUM_QUAD_1D     : Dofs in the 1D segments
-  NUM_QUAD_2D     : Dofs in the 2D faces
-  NUM_QUAD_3D     : Dofs in the 3D domain
-  NUM_MAX_1D      : max(NUM_QUAD_1D, NUM_DOFS_1D)
-  NUM_QUAD_DOFS_1D: NUM_QUAD_1D * NUM_DOFS_1D
-  COEFF_ARGS      : Code that passes required arguments to the kernel
-  COEFF           : Code that computes the coefficient
-  ================================================
-
-  [MISSING]
-  - Add support to auto-pick @dim and use @idxOrder on stack arrays
-  |   double a[2][2];
-  |   a[0][1]; <-- regular index
-  |   a(0,1);  <-- uses @idxOrder a[0][1] or a[1][0]
-  - Add support for @idxOrder to change indexing order after allocation
-  |   double a[2][2] @idxOrder(0,1);
-  |   a(0,1) -> a[1][0]
-  |   @set(a, idxOrder(1,0));
-  |   a(0,1) -> a[0][1]
-  - Add support to iterate over loop depending on mode
-  |   for(i; @inner) {
-  |     for(0 < j < N) {} <-- ++j or j += block?
-  |   }
-*/
-
-#include "mfem-occa://defines.okl"
-
-#if USING_TENSOR_OPS
-#  ifdef OCCA_USING_GPU
-#    if USING_LOW_ORDER
-#      include "mfem-occa://diffusion/tensor/gpuHighOrder.okl"
-#    else
-#      include "mfem-occa://diffusion/tensor/gpuHighOrder.okl"
-#    endif
-#  else
-#    include "mfem-occa://diffusion/tensor/cpu.okl"
-#  endif
-#else
-#  ifdef OCCA_USING_GPU
-#    if USING_LOW_ORDER
-#      include "mfem-occa://diffusion/simplex/gpuHighOrder.okl"
-#    else
-#      include "mfem-occa://diffusion/simplex/gpuHighOrder.okl"
-#    endif
-#  else
-#    include "mfem-occa://diffusion/simplex/cpu.okl"
-#  endif
-#endif

backends/occa/MassIntegrator.okl

@@ -1,66 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-/*
-  ---[ Defines Known At Compile-Time ]------------
-  ELEMENT_BATCH   : How many elements are in each
-  .                   computation batch
-  NUM_DOFS_1D     : Dofs in the 1D segments
-  NUM_DOFS_2D     : Dofs in the 2D faces
-  NUM_DOFS_3D     : Dofs in the 3D domain
-  NUM_QUAD_1D     : Dofs in the 1D segments
-  NUM_QUAD_2D     : Dofs in the 2D faces
-  NUM_QUAD_3D     : Dofs in the 3D domain
-  NUM_MAX_1D      : max(NUM_QUAD_1D, NUM_DOFS_1D)
-  NUM_QUAD_DOFS_1D: NUM_QUAD_1D * NUM_DOFS_1D
-  COEFF_ARGS      : Code that passes required arguments to the kernel
-  COEFF           : Code that computes the coefficient
-  ================================================
-
-  [MISSING]
-  - Add support to auto-pick @dim and use @idxOrder on stack arrays
-  |   double a[2][2];
-  |   a[0][1]; <-- regular index
-  |   a(0,1);  <-- uses @idxOrder a[0][1] or a[1][0]
-  - Add support for @idxOrder to change indexing order after allocation
-  |   double a[2][2] @idxOrder(0,1);
-  |   a(0,1) -> a[1][0]
-  |   @set(a, idxOrder(1,0));
-  |   a(0,1) -> a[0][1]
-  - Add support to iterate over loop depending on mode
-  |   for(i; @inner) {
-  |     for(0 < j < N) {} <-- ++j or j += block?
-  |   }
-*/
-
-#include "mfem-occa://defines.okl"
-
-#if USING_TENSOR_OPS
-#  ifdef OCCA_USING_GPU
-#    if USING_LOW_ORDER
-#      include "mfem-occa://mass/tensor/gpuHighOrder.okl"
-#    else
-#      include "mfem-occa://mass/tensor/gpuHighOrder.okl"
-#    endif
-#  else
-#    include "mfem-occa://mass/tensor/cpu.okl"
-#  endif
-#else
-#  ifdef OCCA_USING_GPU
-#    if USING_LOW_ORDER
-#      include "mfem-occa://mass/simplex/gpuHighOrder.okl"
-#    else
-#      include "mfem-occa://mass/simplex/gpuHighOrder.okl"
-#    endif
-#  else
-#    include "mfem-occa://mass/simplex/cpu.okl"
-#  endif
-#endif

backends/occa/VectorMassIntegrator.okl

@@ -1,38 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-/*
-  ---[ Defines Known At Compile-Time ]------------
-  ELEMENT_BATCH   : How many elements are in each
-  .                   computation batch
-  NUM_DOFS_1D     : Dofs in the 1D segments
-  NUM_DOFS_2D     : Dofs in the 2D faces
-  NUM_DOFS_3D     : Dofs in the 3D domain
-  NUM_QUAD_1D     : Dofs in the 1D segments
-  NUM_QUAD_2D     : Dofs in the 2D faces
-  NUM_QUAD_3D     : Dofs in the 3D domain
-  NUM_MAX_1D      : max(NUM_QUAD_1D, NUM_DOFS_1D)
-  NUM_QUAD_DOFS_1D: NUM_QUAD_1D * NUM_DOFS_1D
-  CONST_COEFF     : If the coefficient is constant, pass it
-  .                   as a define
-  ================================================
-
-  [MISSING]
-  See kernels/DiffusionIntegrator.okl
-*/
-
-#include "mfem-occa://defines.okl"
-
-#if USING_TENSOR_OPS
-#  ifndef OCCA_USING_GPU
-#    include "mfem-occa://vmass/tensor/cpu.okl"
-#  endif
-#endif

backends/occa/array.cpp

@@ -1,121 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "array.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-PArray *Array::DoClone(bool copy_data, void **buffer,
-                       std::size_t item_size) const
-{
-   Array *new_array = new Array(OccaLayout(), item_size);
-   if (copy_data)
-   {
-      new_array->slice.copyFrom(slice);
-   }
-   if (buffer)
-   {
-      *buffer = new_array->GetBuffer();
-   }
-   return new_array;
-}
-
-int Array::DoResize(PLayout &new_layout, void **buffer,
-                    std::size_t item_size)
-{
-   MFEM_ASSERT(dynamic_cast<Layout *>(&new_layout) != NULL,
-               "new_layout is not an OCCA Layout");
-   Layout *lt = static_cast<Layout *>(&new_layout);
-   int err = OccaResize(lt, item_size);
-   if (!err && buffer)
-   {
-      *buffer = GetBuffer();
-   }
-   return err;
-}
-
-void *Array::DoPullData(void *buffer, std::size_t item_size)
-{
-   // called only when Size() != 0
-
-   if (!slice.getDevice().hasSeparateMemorySpace())
-   {
-      return slice.ptr();
-   }
-   if (buffer)
-   {
-      slice.copyTo(buffer);
-   }
-   return buffer;
-}
-
-void Array::DoFill(const void *value_ptr, std::size_t item_size)
-{
-   // called only when Size() != 0
-
-   switch (item_size)
-   {
-      case sizeof(int8_t):
-         OccaFill(*(const int8_t *)value_ptr);
-         break;
-      case sizeof(int16_t):
-         OccaFill(*(const int16_t *)value_ptr);
-         break;
-      case sizeof(int32_t):
-         OccaFill(*(const int32_t *)value_ptr);
-         break;
-      // case sizeof(int64_t):
-      //    OccaFill(*(const int64_t *)value_ptr);
-      //    break;
-      case sizeof(double):
-         OccaFill(*(const double *)value_ptr);
-         break;
-      // case sizeof(::occa::double2):
-      //    OccaFill(*(const ::occa::double2 *)value_ptr);
-      //    break;
-      default:
-         MFEM_ABORT("item_size = " << item_size << " is not supported");
-   }
-}
-
-void Array::DoPushData(const void *src_buffer, std::size_t item_size)
-{
-   // called only when Size() != 0
-
-   if (slice.getDevice().hasSeparateMemorySpace() || slice.ptr() != src_buffer)
-   {
-      slice.copyFrom(src_buffer);
-   }
-}
-
-void Array::DoAssign(const PArray &src, std::size_t item_size)
-{
-   // called only when Size() != 0
-
-   // Note: static_cast can not be used here since PArray is a virtual base
-   //       class.
-   const Array *source = dynamic_cast<const Array *>(&src);
-   MFEM_ASSERT(source != NULL, "invalid source Array type");
-   OccaAssign(*source);
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/array.hpp

@@ -1,175 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_ARRAY_HPP
-#define MFEM_BACKENDS_OCCA_ARRAY_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include <occa.hpp>
-#include "layout.hpp"
-#include "../base/array.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class Array : public virtual PArray
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // DLayout layout;
-
-   // Always true: Size()*item_size == slice.size() <= data.size()
-   mutable ::occa::memory data, slice;
-
-   //
-   // Virtual interface
-   //
-
-   virtual PArray *DoClone(bool copy_data, void **buffer,
-                           std::size_t item_size) const;
-
-   virtual int DoResize(PLayout &new_layout, void **buffer,
-                        std::size_t item_size);
-
-   virtual void *DoPullData(void *buffer, std::size_t item_size);
-
-   virtual void DoFill(const void *value_ptr, std::size_t item_size);
-
-   virtual void DoPushData(const void *src_buffer, std::size_t item_size);
-
-   virtual void DoAssign(const PArray &src, std::size_t item_size);
-
-   //
-   // Auxiliary methods
-   //
-
-   inline void *GetBuffer() const;
-
-public:
-   Array(const Engine &e)
-      : PArray(*(new Layout(e, 0))),
-        data(e.Alloc(0)),
-        slice(data)
-   { }
-
-   Array(Layout &lt, std::size_t item_size)
-      : PArray(lt),
-        data(lt.OccaEngine().Alloc(lt.Size()*item_size)),
-        slice(data)
-   { }
-
-   virtual ~Array() { }
-
-   inline void MakeRef(Array &master);
-
-   Layout &OccaLayout() const { return layout->As<Layout>(); }
-
-   const Engine &OccaEngine() const { return OccaLayout().OccaEngine(); }
-
-   ::occa::memory &OccaMem() { return slice; }
-   const ::occa::memory &OccaMem() const { return slice; }
-
-   inline int OccaResize(Layout *lt, std::size_t item_size);
-
-   inline int OccaResize(std::size_t new_size, std::size_t item_size);
-
-   template <typename T>
-   inline void OccaFill(const T val);
-
-   inline void OccaAssign(const Array &src);
-
-   inline void OccaPush(const void *src);
-};
-
-
-//
-// Inline methods
-//
-
-inline void *Array::GetBuffer() const
-{
-   if (!slice.getDevice().hasSeparateMemorySpace())
-   {
-      return slice.ptr();
-   }
-   return NULL;
-}
-
-inline int Array::OccaResize(Layout *lt, std::size_t item_size)
-{
-   layout.Reset(lt); // Reset() checks if the pointer is the same
-   const std::size_t new_bytes = lt->Size()*item_size;
-   if (data.size() < new_bytes ||
-       data.getDevice() != lt->OccaEngine().GetDevice())
-   {
-      data = lt->OccaEngine().Alloc(new_bytes);
-      slice = data;
-      // If memory allocation fails - an exception is thrown.
-   }
-   else if (slice.size() != new_bytes)
-   {
-      slice = data.slice(0, new_bytes);
-   }
-   return 0;
-}
-
-inline void Array::MakeRef(Array &master)
-{
-   layout = master.layout;
-   data = master.data;
-   slice = master.slice;
-}
-
-inline int Array::OccaResize(std::size_t new_size, std::size_t item_size)
-{
-   Layout &ol = OccaLayout();
-   ol.OccaResize(new_size);
-   return OccaResize(&ol, item_size);
-}
-
-template <typename T>
-inline void Array::OccaFill(const T val)
-{
-   ::occa::linalg::operator_eq<T>(slice, val);
-}
-
-inline void Array::OccaAssign(const Array &src)
-{
-   if (slice != src.slice && slice.size() != 0)
-   {
-      MFEM_ASSERT(slice.size() == src.slice.size(), "");
-      slice.copyFrom(src.slice);
-   }
-}
-
-inline void Array::OccaPush(const void *src)
-{
-   if (slice.size() != 0)
-   {
-      slice.copyFrom(src);
-   }
-}
-
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_ARRAY_HPP

backends/occa/backend.cpp

@@ -1,47 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "backend.hpp"
-#include "engine.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-bool Backend::Supports(const std::string &engine_spec) const
-{
-   // TODO: check if 'engine_spec' is valid OCCA string.
-   return true;
-}
-
-mfem::Engine *Create(const std::string &engine_spec)
-{
-   return new Engine(engine_spec);
-}
-
-#ifdef MFEM_USE_MPI
-mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec)
-{
-   return new Engine(comm, engine_spec);
-}
-#endif
-
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/backend.hpp

@@ -1,49 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_BACKEND_HPP
-#define MFEM_BACKENDS_OCCA_BACKEND_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-// Only the Backend and Engine classes should be exposed through "backend.hpp"
-#include "../base/backend.hpp"
-#include "engine.hpp"
-#include <occa.hpp>
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class Backend : public mfem::Backend
-{
-public:
-   virtual ~Backend();
-
-   virtual bool Supports(const std::string &engine_spec) const;
-
-   virtual mfem::Engine *Create(const std::string &engine_spec);
-
-#ifdef MFEM_USE_MPI
-   virtual mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec);
-#endif
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_BACKEND_HPP

backends/occa/bilinearform.cpp

@@ -1,538 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "backend.hpp"
-#include "bilininteg.hpp"
-#include "../../fem/bilinearform.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-OccaBilinearForm::OccaBilinearForm(FiniteElementSpace *ofespace_) :
-   Operator(ofespace_->OccaVLayout()),
-   localX(ofespace_->OccaEVLayout()),
-   localY(ofespace_->OccaEVLayout())
-{
-   Init(ofespace_->OccaEngine(), ofespace_, ofespace_);
-}
-
-OccaBilinearForm::OccaBilinearForm(FiniteElementSpace *otrialFESpace_,
-                                   FiniteElementSpace *otestFESpace_) :
-   Operator(otrialFESpace_->OccaVLayout(),
-            otestFESpace_->OccaVLayout()),
-   localX(otrialFESpace_->OccaEVLayout()),
-   localY(otestFESpace_->OccaEVLayout())
-{
-   Init(otrialFESpace_->OccaEngine(), otrialFESpace_, otestFESpace_);
-}
-
-void OccaBilinearForm::Init(const Engine &e,
-                            FiniteElementSpace *otrialFESpace_,
-                            FiniteElementSpace *otestFESpace_)
-{
-   engine.Reset(&e);
-
-   otrialFESpace = otrialFESpace_;
-   trialFESpace  = otrialFESpace_->GetFESpace();
-
-   otestFESpace = otestFESpace_;
-   testFESpace  = otestFESpace_->GetFESpace();
-
-   mesh = trialFESpace->GetMesh();
-
-   const int elements = GetNE();
-
-   const int trialVDim = trialFESpace->GetVDim();
-
-   const int trialLocalDofs = otrialFESpace->GetLocalDofs();
-   const int testLocalDofs  = otestFESpace->GetLocalDofs();
-
-   // First-touch policy when running with OpenMP
-   if (GetDevice().mode() == "OpenMP")
-   {
-      const std::string &okl_path = OccaEngine().GetOklPath();
-      ::occa::kernel initLocalKernel =
-         GetDevice().buildKernel(okl_path + "utils.okl",
-                                 "InitLocalVector");
-
-      const std::size_t sd = sizeof(double);
-      const uint64_t trialEntries = sd * (elements * trialLocalDofs);
-      const uint64_t testEntries  = sd * (elements * testLocalDofs);
-      for (int v = 0; v < trialVDim; ++v)
-      {
-         const uint64_t trialOffset = v * trialEntries;
-         const uint64_t testOffset  = v * testEntries;
-
-         initLocalKernel(elements, trialLocalDofs,
-                         localX.OccaMem().slice(trialOffset, trialEntries));
-         initLocalKernel(elements, testLocalDofs,
-                         localY.OccaMem().slice(testOffset, testEntries));
-      }
-   }
-}
-
-int OccaBilinearForm::BaseGeom() const
-{
-   return mesh->GetElementBaseGeometry();
-}
-
-int OccaBilinearForm::GetDim() const
-{
-   return mesh->Dimension();
-}
-
-int64_t OccaBilinearForm::GetNE() const
-{
-   return mesh->GetNE();
-}
-
-Mesh& OccaBilinearForm::GetMesh() const
-{
-   return *mesh;
-}
-
-FiniteElementSpace& OccaBilinearForm::GetTrialOccaFESpace() const
-{
-   return *otrialFESpace;
-}
-
-FiniteElementSpace& OccaBilinearForm::GetTestOccaFESpace() const
-{
-   return *otestFESpace;
-}
-
-mfem::FiniteElementSpace& OccaBilinearForm::GetTrialFESpace() const
-{
-   return *trialFESpace;
-}
-
-mfem::FiniteElementSpace& OccaBilinearForm::GetTestFESpace() const
-{
-   return *testFESpace;
-}
-
-int64_t OccaBilinearForm::GetTrialNDofs() const
-{
-   return trialFESpace->GetNDofs();
-}
-
-int64_t OccaBilinearForm::GetTestNDofs() const
-{
-   return testFESpace->GetNDofs();
-}
-
-int64_t OccaBilinearForm::GetTrialVDim() const
-{
-   return trialFESpace->GetVDim();
-}
-
-int64_t OccaBilinearForm::GetTestVDim() const
-{
-   return testFESpace->GetVDim();
-}
-
-const FiniteElement& OccaBilinearForm::GetTrialFE(const int i) const
-{
-   return *(trialFESpace->GetFE(i));
-}
-
-const FiniteElement& OccaBilinearForm::GetTestFE(const int i) const
-{
-   return *(testFESpace->GetFE(i));
-}
-
-// Adds new Domain Integrator.
-void OccaBilinearForm::AddDomainIntegrator(OccaIntegrator *integrator,
-                                           const ::occa::properties &props)
-{
-   AddIntegrator(integrator, props, DomainIntegrator);
-}
-
-// Adds new Boundary Integrator.
-void OccaBilinearForm::AddBoundaryIntegrator(OccaIntegrator *integrator,
-                                             const ::occa::properties &props)
-{
-   AddIntegrator(integrator, props, BoundaryIntegrator);
-}
-
-// Adds new interior Face Integrator.
-void OccaBilinearForm::AddInteriorFaceIntegrator(OccaIntegrator *integrator,
-                                                 const ::occa::properties &props)
-{
-   AddIntegrator(integrator, props, InteriorFaceIntegrator);
-}
-
-// Adds new boundary Face Integrator.
-void OccaBilinearForm::AddBoundaryFaceIntegrator(OccaIntegrator *integrator,
-                                                 const ::occa::properties &props)
-{
-   AddIntegrator(integrator, props, BoundaryFaceIntegrator);
-}
-
-// Adds Integrator based on OccaIntegratorType
-void OccaBilinearForm::AddIntegrator(OccaIntegrator *integrator,
-                                     const ::occa::properties &props,
-                                     const OccaIntegratorType itype)
-{
-   if (integrator == NULL)
-   {
-      std::stringstream error_ss;
-      error_ss << "OccaBilinearForm::";
-      switch (itype)
-      {
-         case DomainIntegrator      : error_ss << "AddDomainIntegrator";       break;
-         case BoundaryIntegrator    : error_ss << "AddBoundaryIntegrator";     break;
-         case InteriorFaceIntegrator: error_ss << "AddInteriorFaceIntegrator"; break;
-         case BoundaryFaceIntegrator: error_ss << "AddBoundaryFaceIntegrator"; break;
-      }
-      error_ss << " (...):\n"
-               << "  Integrator is NULL";
-      const std::string error = error_ss.str();
-      mfem_error(error.c_str());
-   }
-   integrator->SetupIntegrator(*this, baseKernelProps + props, itype);
-   integrators.push_back(integrator);
-}
-
-const mfem::Operator* OccaBilinearForm::GetTrialProlongation() const
-{
-   return otrialFESpace->GetProlongationOperator();
-}
-
-const mfem::Operator* OccaBilinearForm::GetTestProlongation() const
-{
-   return otestFESpace->GetProlongationOperator();
-}
-
-const mfem::Operator* OccaBilinearForm::GetTrialRestriction() const
-{
-   return otrialFESpace->GetRestrictionOperator();
-}
-
-const mfem::Operator* OccaBilinearForm::GetTestRestriction() const
-{
-   return otestFESpace->GetRestrictionOperator();
-}
-
-void OccaBilinearForm::Assemble()
-{
-   // [MISSING] Find geometric information that is needed by intergrators
-   //             to share between integrators.
-   const int integratorCount = (int) integrators.size();
-   for (int i = 0; i < integratorCount; ++i)
-   {
-      integrators[i]->Assemble();
-   }
-}
-
-void OccaBilinearForm::FormLinearSystem(const mfem::Array<int> &constraintList,
-                                        mfem::Vector &x, mfem::Vector &b,
-                                        mfem::Operator *&Aout,
-                                        mfem::Vector &X, mfem::Vector &B,
-                                        int copy_interior)
-{
-   FormOperator(constraintList, Aout);
-   InitRHS(constraintList, x, b, Aout, X, B, copy_interior);
-}
-
-void OccaBilinearForm::FormOperator(const mfem::Array<int> &constraintList,
-                                    mfem::Operator *&Aout)
-{
-   const mfem::Operator *trialP = GetTrialProlongation();
-   const mfem::Operator *testP  = GetTestProlongation();
-   mfem::Operator *rap = this;
-
-   if (trialP)
-   {
-      rap = new RAPOperator(*testP, *this, *trialP);
-   }
-
-   Aout = new OccaConstrainedOperator(rap, constraintList,
-                                      rap != this);
-}
-
-void OccaBilinearForm::InitRHS(const mfem::Array<int> &constraintList,
-                               mfem::Vector &x, mfem::Vector &b,
-                               mfem::Operator *A,
-                               mfem::Vector &X, mfem::Vector &B,
-                               int copy_interior)
-{
-   // FIXME: move these kernels to the Backend?
-   static ::occa::kernelBuilder get_subvector_builder =
-      ::occa::linalg::customLinearMethod(
-         "vector_get_subvector",
-
-         "const int dof_i = v2[i];"
-         "v0[i] = dof_i >= 0 ? v1[dof_i] : -v1[-dof_i - 1];",
-
-         "defines: {"
-         "  VTYPE0: 'double',"
-         "  VTYPE1: 'double',"
-         "  VTYPE2: 'int',"
-         "  TILESIZE: 128,"
-         "}");
-
-   static ::occa::kernelBuilder set_subvector_builder =
-      ::occa::linalg::customLinearMethod(
-         "vector_set_subvector",
-         "const int dof_i = v2[i];"
-         "if (dof_i >= 0) { v0[dof_i]      = v1[i]; }"
-         "else            { v0[-dof_i - 1] = -v1[i]; }",
-
-         "defines: {"
-         "  VTYPE0: 'double',"
-         "  VTYPE1: 'double',"
-         "  VTYPE2: 'int',"
-         "  TILESIZE: 128,"
-         "}");
-
-   const mfem::Operator *P = GetTrialProlongation();
-   const mfem::Operator *R = GetTrialRestriction();
-
-   if (P)
-   {
-      // Variational restriction with P
-      B.Resize(P->InLayout());
-      P->MultTranspose(b, B);
-      X.Resize(R->OutLayout());
-      R->Mult(x, X);
-   }
-   else
-   {
-      // rap, X and B point to the same data as this, x and b
-      X.MakeRef(x);
-      B.MakeRef(b);
-   }
-
-   if (!copy_interior && constraintList.Size() > 0)
-   {
-      ::occa::kernel get_subvector_kernel =
-         get_subvector_builder.build(GetDevice());
-      ::occa::kernel set_subvector_kernel =
-         set_subvector_builder.build(GetDevice());
-
-      const Array &constrList = constraintList.Get_PArray()->As<Array>();
-      Vector subvec(constrList.OccaLayout());
-
-      get_subvector_kernel(constraintList.Size(),
-                           subvec.OccaMem(),
-                           X.Get_PVector()->As<Vector>().OccaMem(),
-                           constrList.OccaMem());
-
-      X.Fill(0.0);
-
-      set_subvector_kernel(constraintList.Size(),
-                           X.Get_PVector()->As<Vector>().OccaMem(),
-                           subvec.OccaMem(),
-                           constrList.OccaMem());
-   }
-
-   // FIXME: add case for HypreParMatrix here
-   OccaConstrainedOperator *cA = dynamic_cast<OccaConstrainedOperator*>(A);
-   if (cA)
-   {
-      cA->EliminateRHS(X.Get_PVector()->As<Vector>(),
-                       B.Get_PVector()->As<Vector>());
-   }
-   else
-   {
-      mfem_error("OccaBilinearForm::InitRHS expects an OccaConstrainedOperator");
-   }
-}
-
-// Matrix vector multiplication.
-void OccaBilinearForm::Mult_(const Vector &x, Vector &y) const
-{
-   otrialFESpace->GlobalToLocal(x, localX);
-   localY.OccaFill<double>(0.0);
-
-   const int integratorCount = (int) integrators.size();
-   for (int i = 0; i < integratorCount; ++i)
-   {
-      integrators[i]->MultAdd(localX, localY);
-   }
-
-   otestFESpace->LocalToGlobal(localY, y);
-}
-
-// Matrix transpose vector multiplication.
-void OccaBilinearForm::MultTranspose_(const Vector &x, Vector &y) const
-{
-   otestFESpace->GlobalToLocal(x, localX);
-   localY.OccaFill<double>(0.0);
-
-   const int integratorCount = (int) integrators.size();
-   for (int i = 0; i < integratorCount; ++i)
-   {
-      integrators[i]->MultTransposeAdd(localX, localY);
-   }
-
-   otrialFESpace->LocalToGlobal(localY, y);
-}
-
-void OccaBilinearForm::OccaRecoverFEMSolution(const mfem::Vector &X,
-                                              const mfem::Vector &b,
-                                              mfem::Vector &x)
-{
-   const mfem::Operator *P = this->GetTrialProlongation();
-   if (P)
-   {
-      // Apply conforming prolongation
-      x.Resize(P->OutLayout());
-      P->Mult(X, x);
-   }
-   // Otherwise X and x point to the same data
-}
-
-// Frees memory bilinear form.
-OccaBilinearForm::~OccaBilinearForm()
-{
-   // Make sure all integrators free their data
-   IntegratorVector::iterator it = integrators.begin();
-   while (it != integrators.end())
-   {
-      delete *it;
-      ++it;
-   }
-}
-
-
-void BilinearForm::InitOccaBilinearForm()
-{
-   // Init 'obform' using 'bform'
-   MFEM_ASSERT(bform != NULL, "");
-   MFEM_ASSERT(obform == NULL, "");
-
-   FiniteElementSpace &ofes =
-      bform->FESpace()->Get_PFESpace()->As<FiniteElementSpace>();
-   obform = new OccaBilinearForm(&ofes);
-
-   // Transfer domain integrators
-   mfem::Array<mfem::BilinearFormIntegrator*> &dbfi = *bform->GetDBFI();
-   for (int i = 0; i < dbfi.Size(); i++)
-   {
-      std::string integ_name(dbfi[i]->Name());
-      Coefficient *scal_coeff = dbfi[i]->GetScalarCoefficient();
-      ConstantCoefficient *const_coeff =
-         dynamic_cast<ConstantCoefficient*>(scal_coeff);
-      GridFunctionCoefficient *gridfunc_coeff =
-         dynamic_cast<GridFunctionCoefficient*>(scal_coeff);
-      // TODO: other types of coefficients ...
-
-      OccaCoefficient *ocoeff = NULL;
-      if (const_coeff)
-      {
-         ocoeff = new OccaCoefficient(obform->OccaEngine(),
-                                      const_coeff->constant);
-      }
-      else if (gridfunc_coeff)
-      {
-         ocoeff = new OccaCoefficient(obform->OccaEngine(),
-                                      *gridfunc_coeff->GetGridFunction(), true);
-      }
-      else if (!scal_coeff)
-      {
-         ocoeff = new OccaCoefficient(obform->OccaEngine(), 1.0);
-      }
-      else
-      {
-         MFEM_ABORT("Coefficient type not supported");
-      }
-
-      OccaIntegrator *ointeg = NULL;
-      if (integ_name == "(undefined)")
-      {
-         MFEM_ABORT("BilinearFormIntegrator does not define Name()");
-      }
-      else if (integ_name == "mass")
-      {
-         ointeg = new OccaMassIntegrator(*ocoeff);
-      }
-      else if (integ_name == "diffusion")
-      {
-         ointeg = new OccaDiffusionIntegrator(*ocoeff);
-      }
-      else
-      {
-         MFEM_ABORT("BilinearFormIntegrator [Name() = " << integ_name
-                    << "] is not supported");
-      }
-
-      // NOTE: The integrators copy ocoeff, so it can be deleted here so there
-      //       is no memory leak.
-      delete ocoeff;
-
-      const mfem::IntegrationRule *ir = dbfi[i]->GetIntRule();
-      if (ir) { ointeg->SetIntegrationRule(*ir); }
-
-      obform->AddDomainIntegrator(ointeg);
-   }
-
-   // TODO: other types of integrators ...
-}
-
-bool BilinearForm::Assemble()
-{
-   if (obform == NULL) { InitOccaBilinearForm(); }
-
-   obform->Assemble();
-
-   return true; // --> host assembly is not needed
-}
-
-void BilinearForm::FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
-                                    mfem::OperatorHandle &A)
-{
-   if (A.Type() == mfem::Operator::ANY_TYPE)
-   {
-      mfem::Operator *Aout = NULL;
-      obform->FormOperator(ess_tdof_list, Aout);
-      A.Reset(Aout);
-   }
-   else
-   {
-      MFEM_ABORT("Operator::Type is not supported, type = " << A.Type());
-   }
-}
-
-void BilinearForm::FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
-                                    mfem::Vector &x, mfem::Vector &b,
-                                    mfem::OperatorHandle &A,
-                                    mfem::Vector &X, mfem::Vector &B,
-                                    int copy_interior)
-{
-   FormSystemMatrix(ess_tdof_list, A);
-   obform->InitRHS(ess_tdof_list, x, b, A.Ptr(), X, B, copy_interior);
-}
-
-void BilinearForm::RecoverFEMSolution(const mfem::Vector &X,
-                                      const mfem::Vector &b,
-                                      mfem::Vector &x)
-{
-   obform->OccaRecoverFEMSolution(X, b, x);
-}
-
-BilinearForm::~BilinearForm()
-{
-   delete obform;
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/bilinearform.hpp

@@ -1,213 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP
-#define MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "fespace.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-enum OccaIntegratorType
-{
-   DomainIntegrator       = 0,
-   BoundaryIntegrator     = 1,
-   InteriorFaceIntegrator = 2,
-   BoundaryFaceIntegrator = 3
-};
-
-class OccaIntegrator;
-
-
-/** Class for bilinear form - "Matrix" with associated FE space and
-    BLFIntegrators. */
-class OccaBilinearForm : public Operator
-{
-   friend class OccaIntegrator;
-
-protected:
-   typedef std::vector<OccaIntegrator*> IntegratorVector;
-
-   SharedPtr<const Engine> engine;
-
-   // State information
-   mutable mfem::Mesh *mesh;
-
-   mutable FiniteElementSpace *otrialFESpace;
-   mutable mfem::FiniteElementSpace *trialFESpace;
-
-   mutable FiniteElementSpace *otestFESpace;
-   mutable mfem::FiniteElementSpace *testFESpace;
-
-   IntegratorVector integrators;
-
-   // Device data
-   ::occa::properties baseKernelProps;
-
-   // The input and output vectors are mapped to local nodes for efficient
-   // operations. In other words, they are E-vectors.
-   // The size is: (number of elements) * (nodes in element) * (vector dim)
-   mutable Vector localX, localY;
-
-public:
-   OccaBilinearForm(FiniteElementSpace *ofespace_);
-
-   OccaBilinearForm(FiniteElementSpace *otrialFESpace_,
-                    FiniteElementSpace *otestFESpace_);
-
-   void Init(const Engine &e,
-             FiniteElementSpace *otrialFESpace_,
-             FiniteElementSpace *otestFESpace_);
-
-   const Engine &OccaEngine() const { return *engine; }
-
-   ::occa::device GetDevice(int idx = 0) const
-   { return engine->GetDevice(idx); }
-
-   // Useful mesh Information
-   int BaseGeom() const;
-   int GetDim() const;
-   int64_t GetNE() const;
-
-   mfem::Mesh& GetMesh() const;
-
-   FiniteElementSpace& GetTrialOccaFESpace() const;
-   FiniteElementSpace& GetTestOccaFESpace() const;
-
-   mfem::FiniteElementSpace& GetTrialFESpace() const;
-   mfem::FiniteElementSpace& GetTestFESpace() const;
-
-   // Useful FE information
-   int64_t GetTrialNDofs() const;
-   int64_t GetTestNDofs() const;
-
-   int64_t GetTrialVDim() const;
-   int64_t GetTestVDim() const;
-
-   const mfem::FiniteElement& GetTrialFE(const int i) const;
-   const mfem::FiniteElement& GetTestFE(const int i) const;
-
-   // Adds new Domain Integrator.
-   void AddDomainIntegrator(OccaIntegrator *integrator,
-                            const ::occa::properties &props =
-                               ::occa::properties());
-
-   // Adds new Boundary Integrator.
-   void AddBoundaryIntegrator(OccaIntegrator *integrator,
-                              const ::occa::properties &props =
-                                 ::occa::properties());
-
-   // Adds new interior Face Integrator.
-   void AddInteriorFaceIntegrator(OccaIntegrator *integrator,
-                                  const ::occa::properties &props =
-                                     ::occa::properties());
-
-   // Adds new boundary Face Integrator.
-   void AddBoundaryFaceIntegrator(OccaIntegrator *integrator,
-                                  const ::occa::properties &props =
-                                     ::occa::properties());
-
-   // Adds Integrator based on OccaIntegratorType
-   void AddIntegrator(OccaIntegrator *integrator,
-                      const ::occa::properties &props,
-                      const OccaIntegratorType itype);
-
-   virtual const mfem::Operator *GetTrialProlongation() const;
-   virtual const mfem::Operator *GetTestProlongation() const;
-
-   virtual const mfem::Operator *GetTrialRestriction() const;
-   virtual const mfem::Operator *GetTestRestriction() const;
-
-   // Assembles the form i.e. sums over all domain/bdr integrators.
-   virtual void Assemble();
-
-   void FormLinearSystem(const mfem::Array<int> &constraintList,
-                         mfem::Vector &x, mfem::Vector &b,
-                         mfem::Operator *&Aout,
-                         mfem::Vector &X, mfem::Vector &B,
-                         int copy_interior = 0);
-
-   void FormOperator(const mfem::Array<int> &constraintList,
-                     mfem::Operator *&Aout);
-
-   void InitRHS(const mfem::Array<int> &constraintList,
-                mfem::Vector &x, mfem::Vector &b,
-                mfem::Operator *Aout,
-                mfem::Vector &X, mfem::Vector &B,
-                int copy_interior = 0);
-
-   // overrides
-   virtual void Mult_(const Vector &x, Vector &y) const;
-   virtual void MultTranspose_(const Vector &x, Vector &y) const;
-
-   void OccaRecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
-                               mfem::Vector &x);
-
-   // Destroys bilinear form.
-   ~OccaBilinearForm();
-};
-
-
-/// TODO: doxygen
-class BilinearForm : public mfem::PBilinearForm
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // SharedPtr<const mfem::Engine> engine;
-   // mfem::BilinearForm *bform;
-   OccaBilinearForm *obform;
-
-   // Called from Assemble() if obform is NULL to initialize obform.
-   void InitOccaBilinearForm();
-
-public:
-   /// TODO: doxygen
-   BilinearForm(const Engine &e, mfem::BilinearForm &bf)
-      : mfem::PBilinearForm(e, bf), obform(NULL) { }
-
-   /// Virtual destructor
-   virtual ~BilinearForm();
-
-   /// Assemble the PBilinearForm.
-   /** This method is called from the method mfem::BilinearForm::Assemble() of
-       the associated mfem::BilinearForm, #bform.
-       @returns True, if the host assembly should NOT be performed. */
-   virtual bool Assemble();
-
-   virtual void FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
-                                 mfem::OperatorHandle &A);
-
-   virtual void FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
-                                 mfem::Vector &x, mfem::Vector &b,
-                                 mfem::OperatorHandle &A,
-                                 mfem::Vector &X, mfem::Vector &B,
-                                 int copy_interior);
-
-   virtual void RecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
-                                   mfem::Vector &x);
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP

backends/occa/bilininteg.cpp

@@ -1,954 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "bilininteg.hpp"
-#include "../../fem/fem.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-std::map<std::string, OccaDofQuadMaps> OccaDofQuadMaps::AllDofQuadMaps;
-
-OccaGeometry OccaGeometry::Get(::occa::device device,
-                               FiniteElementSpace &ofespace,
-                               const mfem::IntegrationRule &ir,
-                               const int flags)
-{
-   OccaGeometry geom;
-
-   mfem::Mesh &mesh = *(ofespace.GetMesh());
-   if (!mesh.GetNodes())
-   {
-      mesh.SetCurvature(1, false, -1, mfem::Ordering::byVDIM);
-   }
-   mfem::GridFunction &nodes = *(mesh.GetNodes());
-   const mfem::FiniteElementSpace &fespace = *(nodes.FESpace());
-   const mfem::FiniteElement &fe = *(fespace.GetFE(0));
-
-   const int dims     = fe.GetDim();
-   const int elements = fespace.GetNE();
-   const int numDofs  = fe.GetDof();
-   const int numQuad  = ir.GetNPoints();
-
-   MFEM_ASSERT(dims == mesh.SpaceDimension(), "");
-
-   geom.meshNodes.allocate(device,
-                           dims, numDofs, elements);
-
-   const mfem::Table &e2dTable = fespace.GetElementToDofTable();
-   const int *elementMap = e2dTable.GetJ();
-   nodes.Pull();
-   for (int e = 0; e < elements; ++e)
-   {
-      for (int dof = 0; dof < numDofs; ++dof)
-      {
-         const int gid = elementMap[dof + numDofs*e];
-         for (int dim = 0; dim < dims; ++dim)
-         {
-            geom.meshNodes(dim, dof, e) = nodes[fespace.DofToVDof(gid,dim)];
-         }
-      }
-   }
-   geom.meshNodes.keepInDevice();
-
-   if (flags & Jacobian)
-   {
-      geom.J.allocate(device,
-                      dims, dims, numQuad, elements);
-   }
-   else
-   {
-      geom.J.allocate(device, 1);
-   }
-   if (flags & JacobianInv)
-   {
-      geom.invJ.allocate(device,
-                         dims, dims, numQuad, elements);
-   }
-   else
-   {
-      geom.invJ.allocate(device, 1);
-   }
-   if (flags & JacobianDet)
-   {
-      geom.detJ.allocate(device,
-                         numQuad, elements);
-   }
-   else
-   {
-      geom.detJ.allocate(device, 1);
-   }
-
-   geom.J.stopManaging();
-   geom.invJ.stopManaging();
-   geom.detJ.stopManaging();
-
-   OccaDofQuadMaps &maps = OccaDofQuadMaps::GetSimplexMaps(device, fe, ir);
-
-   ::occa::properties props;
-   props["defines/NUM_DOFS"] = numDofs;
-   props["defines/NUM_QUAD"] = numQuad;
-   props["defines/STORE_JACOBIAN"]     = (flags & Jacobian);
-   props["defines/STORE_JACOBIAN_INV"] = (flags & JacobianInv);
-   props["defines/STORE_JACOBIAN_DET"] = (flags & JacobianDet);
-
-   const std::string &okl_path = ofespace.OccaEngine().GetOklPath();
-   ::occa::kernel init = device.buildKernel(okl_path + "geometry.okl",
-                                            stringWithDim("InitGeometryInfo",
-                                                          fe.GetDim()),
-                                            props);
-   init(elements,
-        maps.dofToQuadD,
-        geom.meshNodes,
-        geom.J, geom.invJ, geom.detJ);
-
-   return geom;
-}
-
-OccaDofQuadMaps::OccaDofQuadMaps() :
-   hash() {}
-
-OccaDofQuadMaps::OccaDofQuadMaps(const OccaDofQuadMaps &maps)
-{
-   *this = maps;
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::operator = (const OccaDofQuadMaps &maps)
-{
-   hash = maps.hash;
-   dofToQuad   = maps.dofToQuad;
-   dofToQuadD  = maps.dofToQuadD;
-   quadToDof   = maps.quadToDof;
-   quadToDofD  = maps.quadToDofD;
-   quadWeights = maps.quadWeights;
-   return *this;
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
-                                      const FiniteElementSpace &fespace,
-                                      const mfem::IntegrationRule &ir,
-                                      const bool transpose)
-{
-   return Get(device,
-              *fespace.GetFE(0),
-              *fespace.GetFE(0),
-              ir,
-              transpose);
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
-                                      const mfem::FiniteElement &fe,
-                                      const mfem::IntegrationRule &ir,
-                                      const bool transpose)
-{
-   return Get(device, fe, fe, ir, transpose);
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
-                                      const FiniteElementSpace &trialFESpace,
-                                      const FiniteElementSpace &testFESpace,
-                                      const mfem::IntegrationRule &ir,
-                                      const bool transpose)
-{
-   return Get(device,
-              *trialFESpace.GetFE(0),
-              *testFESpace.GetFE(0),
-              ir,
-              transpose);
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
-                                      const mfem::FiniteElement &trialFE,
-                                      const mfem::FiniteElement &testFE,
-                                      const mfem::IntegrationRule &ir,
-                                      const bool transpose)
-{
-   return (dynamic_cast<const mfem::TensorBasisElement*>(&trialFE)
-           ? GetTensorMaps(device, trialFE, testFE, ir, transpose)
-           : GetSimplexMaps(device, trialFE, testFE, ir, transpose));
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::GetTensorMaps(::occa::device device,
-                                                const mfem::FiniteElement &fe,
-                                                const mfem::IntegrationRule &ir,
-                                                const bool transpose)
-{
-   return GetTensorMaps(device,
-                        fe, fe,
-                        ir, transpose);
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::GetTensorMaps(::occa::device device,
-                                                const mfem::FiniteElement &trialFE,
-                                                const mfem::FiniteElement &testFE,
-                                                const mfem::IntegrationRule &ir,
-                                                const bool transpose)
-{
-   const mfem::TensorBasisElement &trialTFE =
-      dynamic_cast<const mfem::TensorBasisElement&>(trialFE);
-   const mfem::TensorBasisElement &testTFE =
-      dynamic_cast<const mfem::TensorBasisElement&>(testFE);
-
-   std::stringstream ss;
-   ss << ::occa::hash(device)
-      << "Tensor"
-      << "O1:"  << trialFE.GetOrder()
-      << "O2:"  << testFE.GetOrder()
-      << "BT1:" << trialTFE.GetBasisType()
-      << "BT2:" << testTFE.GetBasisType()
-      << "Q:"   << ir.GetNPoints();
-   std::string hash = ss.str();
-
-   // If we've already made the dof-quad maps, reuse them
-   OccaDofQuadMaps &maps = AllDofQuadMaps[hash];
-   if (!maps.hash.size())
-   {
-      // Create the dof-quad maps
-      maps.hash = hash;
-
-      OccaDofQuadMaps trialMaps = GetD2QTensorMaps(device, trialFE, ir);
-      OccaDofQuadMaps testMaps  = GetD2QTensorMaps(device, testFE , ir, true);
-
-      maps.dofToQuad   = trialMaps.dofToQuad;
-      maps.dofToQuadD  = trialMaps.dofToQuadD;
-      maps.quadToDof   = testMaps.dofToQuad;
-      maps.quadToDofD  = testMaps.dofToQuadD;
-      maps.quadWeights = testMaps.quadWeights;
-   }
-   return maps;
-}
-
-OccaDofQuadMaps OccaDofQuadMaps::GetD2QTensorMaps(::occa::device device,
-                                                  const mfem::FiniteElement &fe,
-                                                  const mfem::IntegrationRule &ir,
-                                                  const bool transpose)
-{
-   const mfem::TensorBasisElement &tfe =
-      dynamic_cast<const mfem::TensorBasisElement&>(fe);
-
-   const mfem::Poly_1D::Basis &basis = tfe.GetBasis1D();
-   const int order = fe.GetOrder();
-   // [MISSING] Get 1D dofs
-   const int dofs = order + 1;
-   const int dims = fe.GetDim();
-
-   // Create the dof -> quadrature point map
-   const mfem::IntegrationRule &ir1D =
-      mfem::IntRules.Get(mfem::Geometry::SEGMENT, ir.GetOrder());
-   const int quadPoints = ir1D.GetNPoints();
-   const int quadPoints2D = quadPoints*quadPoints;
-   const int quadPoints3D = quadPoints2D*quadPoints;
-   const int quadPointsND = ((dims == 1) ? quadPoints :
-                             ((dims == 2) ? quadPoints2D : quadPoints3D));
-
-   OccaDofQuadMaps maps;
-   // Initialize the dof -> quad mapping
-   maps.dofToQuad.allocate(device,
-                           quadPoints, dofs);
-   maps.dofToQuadD.allocate(device,
-                            quadPoints, dofs);
-
-   double *quadWeights1DData = NULL;
-
-   if (transpose)
-   {
-      maps.dofToQuad.reindex(1,0);
-      maps.dofToQuadD.reindex(1,0);
-      // Initialize quad weights only for transpose
-      maps.quadWeights.allocate(device,
-                                quadPointsND);
-      quadWeights1DData = new double[quadPoints];
-   }
-
-   mfem::Vector d2q(dofs);
-   mfem::Vector d2qD(dofs);
-   for (int q = 0; q < quadPoints; ++q)
-   {
-      const mfem::IntegrationPoint &ip = ir1D.IntPoint(q);
-      basis.Eval(ip.x, d2q, d2qD);
-      if (transpose)
-      {
-         quadWeights1DData[q] = ip.weight;
-      }
-      for (int d = 0; d < dofs; ++d)
-      {
-         maps.dofToQuad(q, d)  = d2q[d];
-         maps.dofToQuadD(q, d) = d2qD[d];
-      }
-   }
-
-   maps.dofToQuad.keepInDevice();
-   maps.dofToQuadD.keepInDevice();
-
-   if (transpose)
-   {
-      for (int q = 0; q < quadPointsND; ++q)
-      {
-         const int qx = q % quadPoints;
-         const int qz = q / quadPoints2D;
-         const int qy = (q - qz*quadPoints2D) / quadPoints;
-         double w = quadWeights1DData[qx];
-         if (dims > 1)
-         {
-            w *= quadWeights1DData[qy];
-         }
-         if (dims > 2)
-         {
-            w *= quadWeights1DData[qz];
-         }
-         maps.quadWeights[q] = w;
-      }
-      maps.quadWeights.keepInDevice();
-      delete [] quadWeights1DData;
-   }
-
-   return maps;
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::GetSimplexMaps(::occa::device device,
-                                                 const mfem::FiniteElement &fe,
-                                                 const mfem::IntegrationRule &ir,
-                                                 const bool transpose)
-{
-   return GetSimplexMaps(device,
-                         fe, fe,
-                         ir, transpose);
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::GetSimplexMaps(::occa::device device,
-                                                 const mfem::FiniteElement &trialFE,
-                                                 const mfem::FiniteElement &testFE,
-                                                 const mfem::IntegrationRule &ir,
-                                                 const bool transpose)
-{
-   std::stringstream ss;
-   ss << ::occa::hash(device)
-      << "Simplex"
-      << "O1:" << trialFE.GetOrder()
-      << "O2:" << testFE.GetOrder()
-      << "Q:"  << ir.GetNPoints();
-   std::string hash = ss.str();
-
-   // If we've already made the dof-quad maps, reuse them
-   OccaDofQuadMaps &maps = AllDofQuadMaps[hash];
-   if (!maps.hash.size())
-   {
-      // Create the dof-quad maps
-      maps.hash = hash;
-
-      OccaDofQuadMaps trialMaps = GetD2QSimplexMaps(device, trialFE, ir);
-      OccaDofQuadMaps testMaps  = GetD2QSimplexMaps(device, testFE , ir, true);
-
-      maps.dofToQuad   = trialMaps.dofToQuad;
-      maps.dofToQuadD  = trialMaps.dofToQuadD;
-      maps.quadToDof   = testMaps.dofToQuad;
-      maps.quadToDofD  = testMaps.dofToQuadD;
-      maps.quadWeights = testMaps.quadWeights;
-   }
-   return maps;
-}
-
-OccaDofQuadMaps OccaDofQuadMaps::GetD2QSimplexMaps(::occa::device device,
-                                                   const mfem::FiniteElement &fe,
-                                                   const mfem::IntegrationRule &ir,
-                                                   const bool transpose)
-{
-   const int dims = fe.GetDim();
-   const int numDofs = fe.GetDof();
-   const int numQuad = ir.GetNPoints();
-
-   OccaDofQuadMaps maps;
-   // Initialize the dof -> quad mapping
-   maps.dofToQuad.allocate(device,
-                           numQuad, numDofs);
-   maps.dofToQuadD.allocate(device,
-                            dims, numQuad, numDofs);
-
-   if (transpose)
-   {
-      maps.dofToQuad.reindex(1,0);
-      maps.dofToQuadD.reindex(1,0);
-      // Initialize quad weights only for transpose
-      maps.quadWeights.allocate(device,
-                                numQuad);
-   }
-
-   mfem::Vector d2q(numDofs);
-   mfem::DenseMatrix d2qD(numDofs, dims);
-   for (int q = 0; q < numQuad; ++q)
-   {
-      const mfem::IntegrationPoint &ip = ir.IntPoint(q);
-      if (transpose)
-      {
-         maps.quadWeights[q] = ip.weight;
-      }
-      fe.CalcShape(ip, d2q);
-      fe.CalcDShape(ip, d2qD);
-      for (int d = 0; d < numDofs; ++d)
-      {
-         const double w = d2q[d];
-         maps.dofToQuad(q, d) = w;
-         for (int dim = 0; dim < dims; ++dim)
-         {
-            const double wD = d2qD(d, dim);
-            maps.dofToQuadD(dim, q, d) = wD;
-         }
-      }
-   }
-
-   maps.dofToQuad.keepInDevice();
-   maps.dofToQuadD.keepInDevice();
-   if (transpose)
-   {
-      maps.quadWeights.keepInDevice();
-   }
-
-   return maps;
-}
-
-//---[ Integrator Defines ]-----------
-std::string stringWithDim(const std::string &s, const int dim)
-{
-   std::string ret = s;
-   ret += ('0' + (char) dim);
-   ret += 'D';
-   return ret;
-}
-
-int closestWarpBatchTo(const int value)
-{
-   return ((value + 31) / 32) * 32;
-}
-
-int closestMultipleWarpBatch(const int multiple, const int maxSize)
-{
-   if (multiple > maxSize)
-   {
-      return maxSize;
-   }
-   int batch = (32 / multiple);
-   int minDiff = 32 - (multiple * batch);
-   for (int i = 64; i <= maxSize; i += 32)
-   {
-      const int newDiff = i - (multiple * (i / multiple));
-      if (newDiff < minDiff)
-      {
-         batch = (i / multiple);
-         minDiff = newDiff;
-      }
-   }
-   return batch;
-}
-
-void SetProperties(FiniteElementSpace &fespace,
-                   const mfem::IntegrationRule &ir,
-                   ::occa::properties &props)
-{
-   SetProperties(fespace, fespace, ir, props);
-}
-
-void SetProperties(FiniteElementSpace &trialFESpace,
-                   FiniteElementSpace &testFESpace,
-                   const mfem::IntegrationRule &ir,
-                   ::occa::properties &props)
-{
-   props["defines/TRIAL_VDIM"] = trialFESpace.GetVDim();
-   props["defines/TEST_VDIM"]  = testFESpace.GetVDim();
-   props["defines/NUM_DIM"]    = trialFESpace.GetDim();
-
-   if (trialFESpace.hasTensorBasis())
-   {
-      SetTensorProperties(trialFESpace, testFESpace, ir, props);
-   }
-   else
-   {
-      SetSimplexProperties(trialFESpace, testFESpace, ir, props);
-   }
-}
-
-void SetTensorProperties(FiniteElementSpace &fespace,
-                         const mfem::IntegrationRule &ir,
-                         ::occa::properties &props)
-{
-   SetTensorProperties(fespace, fespace, ir, props);
-}
-
-void SetTensorProperties(FiniteElementSpace &trialFESpace,
-                         FiniteElementSpace &testFESpace,
-                         const mfem::IntegrationRule &ir,
-                         ::occa::properties &props)
-{
-   const mfem::FiniteElement &trialFE = *(trialFESpace.GetFE(0));
-   const mfem::FiniteElement &testFE = *(testFESpace.GetFE(0));
-
-   const mfem::IntegrationRule &ir1D =
-      mfem::IntRules.Get(mfem::Geometry::SEGMENT, ir.GetOrder());
-
-   const int trialDofs = trialFE.GetDof();
-   const int testDofs  = testFE.GetDof();
-   const int numQuad  = ir.GetNPoints();
-
-   const int trialDofs1D = trialFE.GetOrder() + 1;
-   const int testDofs1D  = testFE.GetOrder() + 1;
-   const int quad1D  = ir1D.GetNPoints();
-   int trialDofsND = trialDofs1D;
-   int testDofsND  = testDofs1D;
-   int quadND  = quad1D;
-
-   const bool trialByVDIM = (trialFESpace.GetOrdering() == mfem::Ordering::byVDIM);
-   const bool testByVDIM  = (testFESpace.GetOrdering()  == mfem::Ordering::byVDIM);
-
-   props["defines/ORDERING_BY_NODES"] = 0;
-   props["defines/ORDERING_BY_VDIM"]  = 1;
-   props["defines/VDIM_ORDERING"]  = (int) trialByVDIM;
-   props["defines/TRIAL_ORDERING"] = (int) trialByVDIM;
-   props["defines/TEST_ORDERING"]  = (int) testByVDIM;
-
-   props["defines/USING_TENSOR_OPS"] = 1;
-   props["defines/NUM_DOFS"]  = trialDofs;
-   props["defines/NUM_QUAD"]  = numQuad;
-
-   props["defines/TRIAL_DOFS"] = trialDofs;
-   props["defines/TEST_DOFS"]  = testDofs;
-
-   for (int d = 1; d <= 3; ++d)
-   {
-      if (d > 1)
-      {
-         trialDofsND *= trialDofs1D;
-         testDofsND  *= testDofs1D;
-         quadND *= quad1D;
-      }
-      props["defines"][stringWithDim("NUM_DOFS_", d)] = trialDofsND;
-      props["defines"][stringWithDim("NUM_QUAD_", d)] = quadND;
-
-      props["defines"][stringWithDim("TRIAL_DOFS_", d)] = trialDofsND;
-      props["defines"][stringWithDim("TEST_DOFS_" , d)] = testDofsND;
-   }
-
-   // 1D Defines
-   const int m1InnerBatch = 32 * ((quad1D + 31) / 32);
-   props["defines/A1_ELEMENT_BATCH"]       = closestMultipleWarpBatch(quad1D, 512);
-   props["defines/M1_OUTER_ELEMENT_BATCH"] = closestMultipleWarpBatch(m1InnerBatch,
-                                                                      512);
-   props["defines/M1_INNER_ELEMENT_BATCH"] = m1InnerBatch;
-
-   // 2D Defines
-   props["defines/A2_ELEMENT_BATCH"] = 1;
-   props["defines/A2_QUAD_BATCH"]    = 1;
-   props["defines/M2_ELEMENT_BATCH"] = 32;
-
-   // 3D Defines
-   const int a3QuadBatch = closestMultipleWarpBatch(quadND, 512);
-   props["defines/A3_ELEMENT_BATCH"] = closestMultipleWarpBatch(a3QuadBatch, 512);
-   props["defines/A3_QUAD_BATCH"]    = a3QuadBatch;
-}
-
-void SetSimplexProperties(FiniteElementSpace &fespace,
-                          const mfem::IntegrationRule &ir,
-                          ::occa::properties &props)
-{
-   SetSimplexProperties(fespace, fespace, ir, props);
-}
-
-void SetSimplexProperties(FiniteElementSpace &trialFESpace,
-                          FiniteElementSpace &testFESpace,
-                          const mfem::IntegrationRule &ir,
-                          ::occa::properties &props)
-{
-   const mfem::FiniteElement &trialFE = *(trialFESpace.GetFE(0));
-   const mfem::FiniteElement &testFE = *(testFESpace.GetFE(0));
-
-   const int trialDofs = trialFE.GetDof();
-   const int testDofs  = testFE.GetDof();
-   const int numQuad = ir.GetNPoints();
-   const int maxDQ   = std::max(std::max(trialDofs, testDofs), numQuad);
-
-   const bool trialByVDIM = (trialFESpace.GetOrdering() == mfem::Ordering::byVDIM);
-   const bool testByVDIM  = (testFESpace.GetOrdering()  == mfem::Ordering::byVDIM);
-
-   props["defines/ORDERING_BY_NODES"] = 0;
-   props["defines/ORDERING_BY_VDIM"]  = 1;
-   props["defines/VDIM_ORDERING"]  = (int) trialByVDIM;
-   props["defines/TRIAL_ORDERING"] = (int) trialByVDIM;
-   props["defines/TEST_ORDERING"]  = (int) testByVDIM;
-
-   props["defines/USING_TENSOR_OPS"] = 0;
-   props["defines/NUM_DOFS"]  = trialDofs;
-   props["defines/NUM_QUAD"]  = numQuad;
-
-   props["defines/TRIAL_DOFS"] = trialDofs;
-   props["defines/TEST_DOFS"]  = testDofs;
-
-   // 2D Defines
-   const int quadBatch = closestWarpBatchTo(numQuad);
-   props["defines/A2_ELEMENT_BATCH"] = closestMultipleWarpBatch(quadBatch, 2048);
-   props["defines/A2_QUAD_BATCH"]    = quadBatch;
-   props["defines/M2_INNER_BATCH"]   = closestWarpBatchTo(maxDQ);
-
-   // 3D Defines
-   props["defines/A3_ELEMENT_BATCH"] = closestMultipleWarpBatch(quadBatch, 2048);
-   props["defines/A3_QUAD_BATCH"]    = quadBatch;
-   props["defines/M3_INNER_BATCH"]   = closestWarpBatchTo(maxDQ);
-}
-
-
-//---[ Base Integrator ]--------------
-OccaIntegrator::OccaIntegrator(const Engine &e)
-   : engine(&e),
-     bform(),
-     mesh(),
-     otrialFESpace(),
-     otestFESpace(),
-     trialFESpace(),
-     testFESpace(),
-     itype(DomainIntegrator),
-     ir(NULL),
-     hasTensorBasis(false) { }
-
-OccaIntegrator::~OccaIntegrator() {}
-
-void OccaIntegrator::SetupMaps()
-{
-   maps = OccaDofQuadMaps::Get(GetDevice(),
-                               *otrialFESpace,
-                               *otestFESpace,
-                               *ir);
-
-   mapsTranspose = OccaDofQuadMaps::Get(GetDevice(),
-                                        *otestFESpace,
-                                        *otrialFESpace,
-                                        *ir);
-}
-
-FiniteElementSpace& OccaIntegrator::GetTrialOccaFESpace() const
-{
-   return *otrialFESpace;
-}
-
-FiniteElementSpace& OccaIntegrator::GetTestOccaFESpace() const
-{
-   return *otestFESpace;
-}
-
-mfem::FiniteElementSpace& OccaIntegrator::GetTrialFESpace() const
-{
-   return *trialFESpace;
-}
-
-mfem::FiniteElementSpace& OccaIntegrator::GetTestFESpace() const
-{
-   return *testFESpace;
-}
-
-void OccaIntegrator::SetIntegrationRule(const mfem::IntegrationRule &ir_)
-{
-   ir = &ir_;
-}
-
-const mfem::IntegrationRule& OccaIntegrator::GetIntegrationRule() const
-{
-   return *ir;
-}
-
-OccaDofQuadMaps& OccaIntegrator::GetDofQuadMaps()
-{
-   return maps;
-}
-
-void OccaIntegrator::SetupIntegrator(OccaBilinearForm &bform_,
-                                     const ::occa::properties &props_,
-                                     const OccaIntegratorType itype_)
-{
-   MFEM_ASSERT(engine == &bform_.OccaEngine(), "");
-   bform     = &bform_;
-   mesh      = &(bform_.GetMesh());
-
-   otrialFESpace = &(bform_.GetTrialOccaFESpace());
-   otestFESpace  = &(bform_.GetTestOccaFESpace());
-
-   trialFESpace = &(bform_.GetTrialFESpace());
-   testFESpace  = &(bform_.GetTestFESpace());
-
-   hasTensorBasis = otrialFESpace->hasTensorBasis();
-
-   props = props_;
-   itype = itype_;
-
-   if (ir == NULL)
-   {
-      SetupIntegrationRule();
-   }
-
-   SetupMaps();
-
-   SetProperties(*otrialFESpace,
-                 *otestFESpace,
-                 *ir,
-                 props);
-
-   Setup();
-}
-
-OccaGeometry OccaIntegrator::GetGeometry(const int flags)
-{
-   return OccaGeometry::Get(GetDevice(), *otrialFESpace, *ir, flags);
-}
-
-::occa::kernel OccaIntegrator::GetAssembleKernel(const ::occa::properties
-                                                 &props)
-{
-   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
-   return GetKernel(stringWithDim("Assemble", fe.GetDim()),
-                    props);
-}
-
-::occa::kernel OccaIntegrator::GetMultAddKernel(const ::occa::properties &props)
-{
-   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
-   return GetKernel(stringWithDim("MultAdd", fe.GetDim()),
-                    props);
-}
-
-::occa::kernel OccaIntegrator::GetKernel(const std::string &kernelName,
-                                         const ::occa::properties &props)
-{
-   const std::string filename = GetName() + ".okl";
-   const std::string &okl_path = OccaEngine().GetOklPath();
-   return GetDevice().buildKernel(okl_path + filename,
-                                  kernelName,
-                                  props);
-}
-//====================================
-
-
-//---[ Diffusion Integrator ]---------
-OccaDiffusionIntegrator::OccaDiffusionIntegrator(const OccaCoefficient &coeff_)
-   :
-   OccaIntegrator(coeff_.OccaEngine()),
-   coeff(coeff_),
-   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
-{
-   coeff.SetName("COEFF");
-}
-
-OccaDiffusionIntegrator::~OccaDiffusionIntegrator() {}
-
-
-std::string OccaDiffusionIntegrator::GetName()
-{
-   return "DiffusionIntegrator";
-}
-
-void OccaDiffusionIntegrator::SetupIntegrationRule()
-{
-   const FiniteElement &trialFE = *(trialFESpace->GetFE(0));
-   const FiniteElement &testFE  = *(testFESpace->GetFE(0));
-   ir = &mfem::DiffusionIntegrator::GetRule(trialFE, testFE);
-}
-
-void OccaDiffusionIntegrator::Setup()
-{
-   ::occa::properties kernelProps = props;
-
-   coeff.Setup(*this, kernelProps);
-
-   // Setup assemble and mult kernels
-   assembleKernel = GetAssembleKernel(kernelProps);
-   multKernel     = GetMultAddKernel(kernelProps);
-}
-
-void OccaDiffusionIntegrator::Assemble()
-{
-   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
-
-   const int dims = fe.GetDim();
-   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
-
-   const int elements = trialFESpace->GetNE();
-   const int quadraturePoints = ir->GetNPoints();
-
-   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
-
-   assembledOperator.OccaResize(symmDims * quadraturePoints * elements,
-                                sizeof(double));
-
-   assembleKernel((int) mesh->GetNE(),
-                  maps.quadWeights,
-                  geom.J,
-                  coeff,
-                  assembledOperator.OccaMem());
-}
-
-void OccaDiffusionIntegrator::MultAdd(Vector &x, Vector &y)
-{
-   // Note: x and y are E-vectors
-
-   multKernel((int) mesh->GetNE(),
-              maps.dofToQuad,
-              maps.dofToQuadD,
-              maps.quadToDof,
-              maps.quadToDofD,
-              assembledOperator.OccaMem(),
-              x.OccaMem(), y.OccaMem());
-}
-//====================================
-
-
-//---[ Mass Integrator ]--------------
-OccaMassIntegrator::OccaMassIntegrator(const OccaCoefficient &coeff_) :
-   OccaIntegrator(coeff_.OccaEngine()),
-   coeff(coeff_),
-   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
-{
-   coeff.SetName("COEFF");
-}
-
-OccaMassIntegrator::~OccaMassIntegrator() {}
-
-std::string OccaMassIntegrator::GetName()
-{
-   return "MassIntegrator";
-}
-
-void OccaMassIntegrator::SetupIntegrationRule()
-{
-   const mfem::FiniteElement &trialFE = *(trialFESpace->GetFE(0));
-   const mfem::FiniteElement &testFE  = *(testFESpace->GetFE(0));
-   mfem::ElementTransformation &T = *trialFESpace->GetElementTransformation(0);
-   ir = &mfem::MassIntegrator::GetRule(trialFE, testFE, T);
-}
-
-void OccaMassIntegrator::Setup()
-{
-   ::occa::properties kernelProps = props;
-
-   coeff.Setup(*this, kernelProps);
-
-   // Setup assemble and mult kernels
-   assembleKernel = GetAssembleKernel(kernelProps);
-   multKernel     = GetMultAddKernel(kernelProps);
-}
-
-void OccaMassIntegrator::Assemble()
-{
-   if (assembledOperator.Size())
-   {
-      return;
-   }
-
-   const int elements = trialFESpace->GetNE();
-   const int quadraturePoints = ir->GetNPoints();
-
-   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
-
-   assembledOperator.Resize<double>(quadraturePoints * elements, NULL);
-
-   assembleKernel((int) mesh->GetNE(),
-                  maps.quadWeights,
-                  geom.J,
-                  coeff,
-                  assembledOperator.OccaMem());
-}
-
-void OccaMassIntegrator::SetOperator(Vector &v)
-{
-   assembledOperator = v;
-}
-
-void OccaMassIntegrator::MultAdd(Vector &x, Vector &y)
-{
-   multKernel((int) mesh->GetNE(),
-              maps.dofToQuad,
-              maps.dofToQuadD,
-              maps.quadToDof,
-              maps.quadToDofD,
-              assembledOperator.OccaMem(),
-              x.OccaMem(), y.OccaMem());
-}
-//====================================
-
-
-//---[ Vector Mass Integrator ]--------------
-OccaVectorMassIntegrator::OccaVectorMassIntegrator(const OccaCoefficient &
-                                                   coeff_)
-   :
-   OccaIntegrator(coeff_.OccaEngine()),
-   coeff(coeff_),
-   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
-{
-   coeff.SetName("COEFF");
-}
-
-OccaVectorMassIntegrator::~OccaVectorMassIntegrator() {}
-
-std::string OccaVectorMassIntegrator::GetName()
-{
-   return "VectorMassIntegrator";
-}
-
-void OccaVectorMassIntegrator::SetupIntegrationRule()
-{
-   const mfem::FiniteElement &trialFE = *(trialFESpace->GetFE(0));
-   const mfem::FiniteElement &testFE  = *(testFESpace->GetFE(0));
-   mfem::ElementTransformation &T = *trialFESpace->GetElementTransformation(0);
-   ir = &mfem::MassIntegrator::GetRule(trialFE, testFE, T);
-}
-
-void OccaVectorMassIntegrator::Setup()
-{
-   ::occa::properties kernelProps = props;
-
-   coeff.Setup(*this, kernelProps);
-
-   // Setup assemble and mult kernels
-   assembleKernel = GetAssembleKernel(kernelProps);
-   multKernel     = GetMultAddKernel(kernelProps);
-}
-
-void OccaVectorMassIntegrator::Assemble()
-{
-   const int elements = trialFESpace->GetNE();
-   const int quadraturePoints = ir->GetNPoints();
-
-   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
-
-   assembledOperator.Resize<double>(quadraturePoints * elements, NULL);
-
-   assembleKernel((int) mesh->GetNE(),
-                  maps.quadWeights,
-                  geom.J,
-                  coeff,
-                  assembledOperator.OccaMem());
-}
-
-void OccaVectorMassIntegrator::MultAdd(Vector &x, Vector &y)
-{
-   multKernel((int) mesh->GetNE(),
-              maps.dofToQuad,
-              maps.dofToQuadD,
-              maps.quadToDof,
-              maps.quadToDofD,
-              assembledOperator.OccaMem(),
-              x.OccaMem(), y.OccaMem());
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/bilininteg.hpp

@@ -1,323 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP
-#define MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "fespace.hpp"
-#include "bilinearform.hpp"
-#include "coefficient.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class OccaGeometry
-{
-public:
-   ::occa::array<double> meshNodes;
-   ::occa::array<double> J, invJ, detJ;
-
-   // byVDIM  -> [x y z x y z x y z]
-   // byNodes -> [x x x y y y z z z]
-   static const int Jacobian    = (1 << 0);
-   static const int JacobianInv = (1 << 1);
-   static const int JacobianDet = (1 << 2);
-
-   static OccaGeometry Get(::occa::device device,
-                           FiniteElementSpace &ofespace,
-                           const IntegrationRule &ir,
-                           const int flags = (Jacobian    |
-                                              JacobianInv |
-                                              JacobianDet));
-};
-
-class OccaDofQuadMaps
-{
-private:
-   // Reuse dof-quad maps
-   static std::map<std::string, OccaDofQuadMaps> AllDofQuadMaps;
-   std::string hash;
-
-public:
-   // Local stiffness matrices (B and B^T operators)
-   ::occa::array<double, ::occa::dynamic> dofToQuad, dofToQuadD; // B
-   ::occa::array<double, ::occa::dynamic> quadToDof, quadToDofD; // B^T
-   ::occa::array<double> quadWeights;
-
-   OccaDofQuadMaps();
-   OccaDofQuadMaps(const OccaDofQuadMaps &maps);
-   OccaDofQuadMaps& operator = (const OccaDofQuadMaps &maps);
-
-   // [[x y] [x y] [x y]]
-   // [[x y z] [x y z] [x y z]]
-   // mfem::GridFunction* mfem::Mesh::GetNodes() { return Nodes; }
-
-   // mfem::FiniteElementSpace *Nodes->FESpace()
-   // 25
-   // 1D [x x x x x x]
-   // 2D [x y x y x y]
-   // GetVdim()
-   // 3D ordering == byVDIM  -> [x y z x y z x y z x y z x y z x y z]
-   //    ordering == byNODES -> [x x x x x x y y y y y y z z z z z z]
-   static OccaDofQuadMaps& Get(::occa::device device,
-                               const FiniteElementSpace &fespace,
-                               const mfem::IntegrationRule &ir,
-                               const bool transpose = false);
-
-   static OccaDofQuadMaps& Get(::occa::device device,
-                               const mfem::FiniteElement &fe,
-                               const mfem::IntegrationRule &ir,
-                               const bool transpose = false);
-
-   static OccaDofQuadMaps& Get(::occa::device device,
-                               const FiniteElementSpace &trialFESpace,
-                               const FiniteElementSpace &testFESpace,
-                               const mfem::IntegrationRule &ir,
-                               const bool transpose = false);
-
-   static OccaDofQuadMaps& Get(::occa::device device,
-                               const mfem::FiniteElement &trialFE,
-                               const mfem::FiniteElement &testFE,
-                               const mfem::IntegrationRule &ir,
-                               const bool transpose = false);
-
-   static OccaDofQuadMaps& GetTensorMaps(::occa::device device,
-                                         const mfem::FiniteElement &fe,
-                                         const mfem::IntegrationRule &ir,
-                                         const bool transpose = false);
-
-   static OccaDofQuadMaps& GetTensorMaps(::occa::device device,
-                                         const mfem::FiniteElement &trialFE,
-                                         const mfem::FiniteElement &testFE,
-                                         const mfem::IntegrationRule &ir,
-                                         const bool transpose = false);
-
-   static OccaDofQuadMaps GetD2QTensorMaps(::occa::device device,
-                                           const mfem::FiniteElement &fe,
-                                           const mfem::IntegrationRule &ir,
-                                           const bool transpose = false);
-
-   static OccaDofQuadMaps& GetSimplexMaps(::occa::device device,
-                                          const mfem::FiniteElement &fe,
-                                          const mfem::IntegrationRule &ir,
-                                          const bool transpose = false);
-
-   static OccaDofQuadMaps& GetSimplexMaps(::occa::device device,
-                                          const mfem::FiniteElement &trialFE,
-                                          const mfem::FiniteElement &testFE,
-                                          const mfem::IntegrationRule &ir,
-                                          const bool transpose = false);
-
-   static OccaDofQuadMaps GetD2QSimplexMaps(::occa::device device,
-                                            const mfem::FiniteElement &fe,
-                                            const mfem::IntegrationRule &ir,
-                                            const bool transpose = false);
-};
-
-//---[ Define Methods ]---------------
-std::string stringWithDim(const std::string &s, const int dim);
-int closestWarpBatch(const int multiple, const int maxSize);
-
-void SetProperties(FiniteElementSpace &fespace,
-                   const mfem::IntegrationRule &ir,
-                   ::occa::properties &props);
-
-void SetProperties(FiniteElementSpace &trialFESpace,
-                   FiniteElementSpace &testFESpace,
-                   const mfem::IntegrationRule &ir,
-                   ::occa::properties &props);
-
-void SetTensorProperties(FiniteElementSpace &fespace,
-                         const mfem::IntegrationRule &ir,
-                         ::occa::properties &props);
-
-void SetTensorProperties(FiniteElementSpace &trialFESpace,
-                         FiniteElementSpace &testFESpace,
-                         const IntegrationRule &ir,
-                         ::occa::properties &props);
-
-void SetSimplexProperties(FiniteElementSpace &fespace,
-                          const IntegrationRule &ir,
-                          ::occa::properties &props);
-
-void SetSimplexProperties(FiniteElementSpace &trialFESpace,
-                          FiniteElementSpace &testFESpace,
-                          const IntegrationRule &ir,
-                          ::occa::properties &props);
-
-//---[ Base Integrator ]--------------
-class OccaIntegrator
-{
-protected:
-   SharedPtr<const Engine> engine;
-
-   OccaBilinearForm *bform;
-   mfem::Mesh *mesh;
-
-   FiniteElementSpace *otrialFESpace;
-   FiniteElementSpace *otestFESpace;
-
-   mfem::FiniteElementSpace *trialFESpace;
-   mfem::FiniteElementSpace *testFESpace;
-
-   ::occa::properties props;
-   OccaIntegratorType itype;
-
-   const IntegrationRule *ir;
-   bool hasTensorBasis;
-   OccaDofQuadMaps maps;
-   OccaDofQuadMaps mapsTranspose;
-
-public:
-   OccaIntegrator(const Engine &e);
-   virtual ~OccaIntegrator();
-
-   const Engine &OccaEngine() const { return *engine; }
-
-   ::occa::device GetDevice(int idx = 0) const
-   { return engine->GetDevice(idx); }
-
-   virtual std::string GetName() = 0;
-
-   FiniteElementSpace& GetTrialOccaFESpace() const;
-   FiniteElementSpace& GetTestOccaFESpace() const;
-
-   mfem::FiniteElementSpace& GetTrialFESpace() const;
-   mfem::FiniteElementSpace& GetTestFESpace() const;
-
-   void SetIntegrationRule(const mfem::IntegrationRule &ir_);
-   const mfem::IntegrationRule& GetIntegrationRule() const;
-
-   OccaDofQuadMaps& GetDofQuadMaps();
-
-   void SetupMaps();
-
-   virtual void SetupIntegrationRule() = 0;
-
-   virtual void SetupIntegrator(OccaBilinearForm &bform_,
-                                const ::occa::properties &props_,
-                                const OccaIntegratorType itype_);
-
-   virtual void Setup() = 0;
-
-   virtual void Assemble() = 0;
-   /// This method works on E-vectors!
-   virtual void MultAdd(Vector &x, Vector &y) = 0;
-
-   virtual void MultTransposeAdd(Vector &x, Vector &y)
-   {
-      mfem_error("OccaIntegrator::MultTransposeAdd() is not overloaded!");
-   }
-
-   OccaGeometry GetGeometry(const int flags = (OccaGeometry::Jacobian    |
-                                               OccaGeometry::JacobianInv |
-                                               OccaGeometry::JacobianDet));
-
-   ::occa::kernel GetAssembleKernel(const ::occa::properties &props);
-   ::occa::kernel GetMultAddKernel(const ::occa::properties &props);
-
-   ::occa::kernel GetKernel(const std::string &kernelName,
-                            const ::occa::properties &props);
-};
-//====================================
-
-
-//---[ Diffusion Integrator ]---------
-class OccaDiffusionIntegrator : public OccaIntegrator
-{
-private:
-   OccaCoefficient coeff;
-
-   ::occa::kernel assembleKernel, multKernel;
-
-   Vector assembledOperator;
-
-public:
-   OccaDiffusionIntegrator(const OccaCoefficient &coeff_);
-   virtual ~OccaDiffusionIntegrator();
-
-   virtual std::string GetName();
-
-   virtual void SetupIntegrationRule();
-
-   virtual void Setup();
-
-   virtual void Assemble();
-   virtual void MultAdd(Vector &x, Vector &y);
-};
-//====================================
-
-
-//---[ Mass Integrator ]--------------
-class OccaMassIntegrator : public OccaIntegrator
-{
-private:
-   OccaCoefficient coeff;
-
-   ::occa::kernel assembleKernel, multKernel;
-
-   Vector assembledOperator;
-
-public:
-   OccaMassIntegrator(const OccaCoefficient &coeff_);
-   virtual ~OccaMassIntegrator();
-
-   virtual std::string GetName();
-
-   virtual void SetupIntegrationRule();
-
-   virtual void Setup();
-
-   virtual void Assemble();
-   void SetOperator(Vector &v);
-
-   virtual void MultAdd(Vector &x, Vector &y);
-};
-//====================================
-
-//---[ Vector Mass Integrator ]--------------
-class OccaVectorMassIntegrator : public OccaIntegrator
-{
-private:
-   OccaCoefficient coeff;
-
-   ::occa::kernel assembleKernel, multKernel;
-
-   Vector assembledOperator;
-
-public:
-   OccaVectorMassIntegrator(const OccaCoefficient &coeff_);
-   virtual ~OccaVectorMassIntegrator();
-
-   virtual std::string GetName();
-
-   virtual void SetupIntegrationRule();
-
-   virtual void Setup();
-
-   virtual void Assemble();
-
-   virtual void MultAdd(Vector &x, Vector &y);
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP

backends/occa/coefficient.cpp

@@ -1,357 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "coefficient.hpp"
-#include "bilininteg.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-//---[ Parameter ]------------
-OccaParameter::~OccaParameter() {}
-
-void OccaParameter::Setup(OccaIntegrator &integ,
-                          ::occa::properties &props) {}
-
-::occa::kernelArg OccaParameter::KernelArgs()
-{
-   return ::occa::kernelArg();
-}
-//====================================
-
-
-//---[ Include Parameter ]------------
-OccaIncludeParameter::OccaIncludeParameter(const std::string &filename_) :
-   filename(filename_) {}
-
-OccaParameter* OccaIncludeParameter::Clone()
-{
-   return new OccaIncludeParameter(filename);
-}
-
-void OccaIncludeParameter::Setup(OccaIntegrator &integ,
-                                 ::occa::properties &props)
-{
-   props["headers"].asArray() += "#include " + filename;
-}
-//====================================
-
-
-//---[ Source Parameter ]------------
-OccaSourceParameter::OccaSourceParameter(const std::string &source_) :
-   source(source_) {}
-
-OccaParameter* OccaSourceParameter::Clone()
-{
-   return new OccaSourceParameter(source);
-}
-
-void OccaSourceParameter::Setup(OccaIntegrator &integ,
-                                ::occa::properties &props)
-{
-   props["headers"].asArray() += source;
-}
-//====================================
-
-//---[ Vector Parameter ]-------
-OccaVectorParameter::OccaVectorParameter(const std::string &name_,
-                                         Vector &v_,
-                                         const bool useRestrict_) :
-   name(name_),
-   v(v_),
-   useRestrict(useRestrict_),
-   attr("") {}
-
-OccaVectorParameter::OccaVectorParameter(const std::string &name_,
-                                         Vector &v_,
-                                         const std::string &attr_,
-                                         const bool useRestrict_) :
-   name(name_),
-   v(v_),
-   useRestrict(useRestrict_),
-   attr(attr_) {}
-
-OccaParameter* OccaVectorParameter::Clone()
-{
-   return new OccaVectorParameter(name, v, attr, useRestrict);
-}
-
-void OccaVectorParameter::Setup(OccaIntegrator &integ,
-                                ::occa::properties &props)
-{
-   std::string &args = (props["defines/COEFF_ARGS"]
-                        .asString()
-                        .string());
-   args += "const double *";
-   if (useRestrict)
-   {
-      args += " restrict ";
-   }
-   args += name;
-   if (attr.size())
-   {
-      args += ' ';
-      args += attr;
-   }
-   args += ",\n";
-}
-
-::occa::kernelArg OccaVectorParameter::KernelArgs()
-{
-   return ::occa::kernelArg(v.OccaMem());
-}
-//====================================
-
-
-//---[ GridFunction Parameter ]-------
-OccaGridFunctionParameter::OccaGridFunctionParameter(const std::string &name_,
-                                                     const Engine &e,
-                                                     mfem::GridFunction &gf_,
-                                                     const bool useRestrict_)
-   : name(name_),
-     gf(gf_),
-     gfQuad(e),
-     useRestrict(useRestrict_) {}
-
-OccaParameter* OccaGridFunctionParameter::Clone()
-{
-   OccaGridFunctionParameter *param =
-      new OccaGridFunctionParameter(name, gfQuad.OccaEngine(), gf, useRestrict);
-   param->gfQuad.MakeRef(gfQuad);
-   return param;
-}
-
-void OccaGridFunctionParameter::Setup(OccaIntegrator &integ,
-                                      ::occa::properties &props)
-{
-
-   std::string &args = (props["defines/COEFF_ARGS"]
-                        .asString()
-                        .string());
-   if (useRestrict)
-   {
-      args += "@restrict ";
-   }
-   args += "const double *";
-   args += name;
-   args += " @dim(NUM_QUAD, numElements),\n";
-
-   FiniteElementSpace &f = gf.FESpace()->Get_PFESpace()->As<FiniteElementSpace>();
-   ToQuad(integ.GetIntegrationRule(), f, gf.Get_PVector()->As<Vector>(), gfQuad);
-}
-
-::occa::kernelArg OccaGridFunctionParameter::KernelArgs()
-{
-   return gfQuad.OccaMem();
-}
-//====================================
-
-
-//---[ Coefficient ]------------------
-OccaCoefficient::OccaCoefficient(const Engine &e, const double value) :
-   engine(&e),
-   integ(NULL),
-   name("COEFF")
-{
-   coeffValue = value;
-}
-
-OccaCoefficient::OccaCoefficient(const Engine &e, mfem::GridFunction &gf,
-                                 const bool useRestrict) :
-   engine(&e),
-   integ(NULL),
-   name("COEFF")
-{
-   coeffValue = "(u(q, e))";
-   AddGridFunction("u", gf, useRestrict);
-}
-
-OccaCoefficient::OccaCoefficient(const Engine &e, const std::string &source) :
-   engine(&e),
-   integ(NULL),
-   name("COEFF")
-{
-   coeffValue = source;
-}
-
-OccaCoefficient::OccaCoefficient(const Engine &e, const char *source) :
-   engine(&e),
-   integ(NULL),
-   name("COEFF")
-{
-   coeffValue = source;
-}
-
-OccaCoefficient::OccaCoefficient(const OccaCoefficient &coeff) :
-   engine(coeff.engine),
-   integ(NULL),
-   name(coeff.name),
-   coeffValue(coeff.coeffValue)
-{
-
-   const int paramCount = (int) coeff.params.size();
-   for (int i = 0; i < paramCount; ++i)
-   {
-      params.push_back(coeff.params[i]->Clone());
-   }
-}
-
-OccaCoefficient::~OccaCoefficient()
-{
-   const int paramCount = (int) params.size();
-   for (int i = 0; i < paramCount; ++i)
-   {
-      delete params[i];
-   }
-}
-
-OccaCoefficient& OccaCoefficient::SetName(const std::string &name_)
-{
-   name = name_;
-   return *this;
-}
-
-void OccaCoefficient::Setup(OccaIntegrator &integ_,
-                            ::occa::properties &props_)
-{
-   integ = &integ_;
-
-   const int paramCount = (int) params.size();
-   props_["defines"][name + "_ARGS"] = "";
-   for (int i = 0; i < paramCount; ++i)
-   {
-      params[i]->Setup(integ_, props_);
-   }
-   props_["defines"][name] = coeffValue;
-
-   props = props_;
-}
-
-OccaCoefficient& OccaCoefficient::Add(OccaParameter *param)
-{
-   params.push_back(param);
-   return *this;
-}
-
-OccaCoefficient& OccaCoefficient::IncludeHeader(const std::string &filename)
-{
-   return Add(new OccaIncludeParameter(filename));
-}
-
-OccaCoefficient& OccaCoefficient::IncludeSource(const std::string &source)
-{
-   return Add(new OccaSourceParameter(source));
-}
-
-OccaCoefficient& OccaCoefficient::AddVector(const std::string &name_,
-                                            Vector &v,
-                                            const bool useRestrict)
-{
-   return Add(new OccaVectorParameter(name_, v, useRestrict));
-}
-
-OccaCoefficient& OccaCoefficient::AddVector(const std::string &name_,
-                                            Vector &v,
-                                            const std::string &attr,
-                                            const bool useRestrict)
-{
-   return Add(new OccaVectorParameter(name_, v, attr, useRestrict));
-}
-
-OccaCoefficient& OccaCoefficient::AddGridFunction(const std::string &name_,
-                                                  mfem::GridFunction &gf,
-                                                  const bool useRestrict)
-{
-   MFEM_ASSERT(engine->CheckVector(gf.Get_PVector()) &&
-               engine->CheckFESpace(gf.FESpace()->Get_PFESpace()),
-               "invalid device GridFunction");
-   return Add(new OccaGridFunctionParameter(name_, *engine, gf, useRestrict));
-}
-
-bool OccaCoefficient::IsConstant()
-{
-   return coeffValue.isNumber();
-}
-
-double OccaCoefficient::GetConstantValue()
-{
-   if (!IsConstant())
-   {
-      mfem_error("OccaCoefficient is not constant");
-   }
-   return coeffValue.number();
-}
-
-Vector OccaCoefficient::Eval()
-{
-   if (integ == NULL)
-   {
-      mfem_error("OccaCoefficient requires a Setup() call before Eval()");
-   }
-
-   mfem::FiniteElementSpace &fespace = integ->GetTrialFESpace();
-   const mfem::IntegrationRule &ir   = integ->GetIntegrationRule();
-
-   const int elements = fespace.GetNE();
-   const int numQuad  = ir.GetNPoints();
-
-   Vector quadCoeff(*(new Layout(OccaEngine(), numQuad * elements)));
-   Eval(quadCoeff);
-   return quadCoeff;
-}
-
-void OccaCoefficient::Eval(Vector &quadCoeff)
-{
-   const std::string &okl_path = OccaEngine().GetOklPath();
-   static ::occa::kernelBuilder builder =
-      ::occa::kernelBuilder::fromFile(okl_path + "coefficient.okl",
-                                      "CoefficientEval");
-
-   if (integ == NULL)
-   {
-      mfem_error("OccaCoefficient requires a Setup() call before Eval()");
-   }
-
-   const int elements = integ->GetTrialFESpace().GetNE();
-
-   ::occa::properties kernelProps = props;
-   if (name != "COEFF")
-   {
-      kernelProps["defines/COEFF"]      = name;
-      kernelProps["defines/COEFF_ARGS"] = name + "_ARGS";
-   }
-
-   ::occa::kernel evalKernel = builder.build(GetDevice(), kernelProps);
-   evalKernel(elements, *this, quadCoeff.OccaMem());
-}
-
-OccaCoefficient::operator ::occa::kernelArg ()
-{
-   ::occa::kernelArg kArg;
-   const int paramCount = (int) params.size();
-   for (int i = 0; i < paramCount; ++i)
-   {
-      kArg.add(params[i]->KernelArgs());
-   }
-   return kArg;
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/coefficient.hpp

@@ -1,287 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_COEFFICIENT_HPP
-#define MFEM_BACKENDS_OCCA_COEFFICIENT_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "vector.hpp"
-#include "gridfunc.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class OccaIntegrator;
-
-
-class OccaParameter
-{
-public:
-   virtual ~OccaParameter();
-
-   virtual OccaParameter* Clone() = 0;
-
-   virtual void Setup(OccaIntegrator &integ,
-                      ::occa::properties &props);
-
-   virtual ::occa::kernelArg KernelArgs();
-};
-
-
-//---[ Include Parameter ]------------
-class OccaIncludeParameter : public OccaParameter
-{
-private:
-   std::string filename;
-
-public:
-   OccaIncludeParameter(const std::string &filename_);
-
-   virtual OccaParameter* Clone();
-
-   virtual void Setup(OccaIntegrator &integ,
-                      ::occa::properties &props);
-};
-//====================================
-
-
-//---[ Source Parameter ]------------
-class OccaSourceParameter : public OccaParameter
-{
-private:
-   std::string source;
-
-public:
-   OccaSourceParameter(const std::string &filename_);
-
-   virtual OccaParameter* Clone();
-
-   virtual void Setup(OccaIntegrator &integ,
-                      ::occa::properties &props);
-};
-//====================================
-
-
-//---[ Define Parameter ]------------
-template <class TM>
-class OccaDefineParameter : public OccaParameter
-{
-private:
-   const std::string name;
-   TM value;
-
-public:
-   OccaDefineParameter(const std::string &name_,
-                       const TM &value_) :
-      name(name_),
-      value(value_) {}
-
-   virtual OccaParameter* Clone()
-   {
-      return new OccaDefineParameter(name, value);
-   }
-
-   virtual void Setup(OccaIntegrator &integ,
-                      ::occa::properties &props)
-   {
-      props["defines"][name] = value;
-   }
-};
-//====================================
-
-
-//---[ Variable Parameter ]-----------
-template <class TM>
-class OccaVariableParameter : public OccaParameter
-{
-private:
-   const std::string name;
-   const TM &value;
-
-public:
-   OccaVariableParameter(const std::string &name_,
-                         const TM &value_) :
-      name(name_),
-      value(value_) {}
-
-   virtual OccaParameter* Clone()
-   {
-      return new OccaVariableParameter(name, value);
-   }
-
-   virtual void Setup(OccaIntegrator &integ,
-                      ::occa::properties &props)
-   {
-      std::string &args = (props["defines/COEFF_ARGS"]
-                           .asString()
-                           .string());
-      // const TM name,\n"
-      args += "const ";
-      args += ::occa::primitiveinfo<TM>::name;
-      args += ' ';
-      args += name;
-      args += ",\n";
-   }
-
-   virtual ::occa::kernelArg KernelArgs()
-   {
-      return ::occa::kernelArg(value);
-   }
-};
-//====================================
-
-
-//---[ Vector Parameter ]-------
-class OccaVectorParameter : public OccaParameter
-{
-private:
-   const std::string name;
-   Vector v;
-   bool useRestrict;
-   std::string attr;
-
-public:
-   OccaVectorParameter(const std::string &name_,
-                       Vector &v_,
-                       const bool useRestrict_ = false);
-
-   OccaVectorParameter(const std::string &name_,
-                       Vector &v_,
-                       const std::string &attr_,
-                       const bool useRestrict_ = false);
-
-   virtual OccaParameter* Clone();
-
-   virtual void Setup(OccaIntegrator &integ,
-                      ::occa::properties &props);
-
-   virtual ::occa::kernelArg KernelArgs();
-};
-//====================================
-
-
-//---[ GridFunction Parameter ]-------
-class OccaGridFunctionParameter : public OccaParameter
-{
-private:
-   const std::string name;
-   mfem::GridFunction &gf;
-   Vector gfQuad;
-   bool useRestrict;
-
-public:
-   OccaGridFunctionParameter(const std::string &name_,
-                             const Engine &e,
-                             mfem::GridFunction &gf_,
-                             const bool useRestrict_ = false);
-
-   virtual OccaParameter* Clone();
-
-   virtual void Setup(OccaIntegrator &integ,
-                      ::occa::properties &props);
-
-   virtual ::occa::kernelArg KernelArgs();
-};
-//====================================
-
-
-//---[ Coefficient ]------------------
-// [MISSING]
-// Needs to know about the integrator's
-//   - fespace
-//   - ir
-// Step where parameters that need the ir get called for setup
-// For example, GridFunction (d, e) -> (q, e)
-class OccaCoefficient
-{
-private:
-   SharedPtr<const Engine> engine;
-
-   OccaIntegrator *integ;
-
-   std::string name;
-   ::occa::json coeffValue;
-
-   ::occa::properties props;
-   std::vector<OccaParameter*> params;
-
-public:
-   OccaCoefficient(const Engine &e, const double value = 1.0);
-   OccaCoefficient(const Engine &e, mfem::GridFunction &gf,
-                   const bool useRestrict = false);
-   OccaCoefficient(const Engine &e, const std::string &source);
-   OccaCoefficient(const Engine &e, const char *source);
-   ~OccaCoefficient();
-
-   OccaCoefficient(const OccaCoefficient &coeff);
-
-   const Engine &OccaEngine() const { return *engine; }
-
-   ::occa::device GetDevice(int idx = 0) const
-   { return engine->GetDevice(idx); }
-
-   OccaCoefficient& SetName(const std::string &name_);
-
-   void Setup(OccaIntegrator &integ_,
-              ::occa::properties &props_);
-
-   OccaCoefficient& Add(OccaParameter *param);
-
-   OccaCoefficient& IncludeHeader(const std::string &filename);
-   OccaCoefficient& IncludeSource(const std::string &source);
-
-   template <class TM>
-   OccaCoefficient& AddDefine(const std::string &name_, const TM &value)
-   {
-      return Add(new OccaDefineParameter<TM>(name_, value));
-   }
-
-   template <class TM>
-   OccaCoefficient& AddVariable(const std::string &name_, const TM &value)
-   {
-      return Add(new OccaVariableParameter<TM>(name_, value));
-   }
-
-   OccaCoefficient& AddVector(const std::string &name_,
-                              Vector &v,
-                              const bool useRestrict = false);
-
-
-   OccaCoefficient& AddVector(const std::string &name_,
-                              Vector &v,
-                              const std::string &attr,
-                              const bool useRestrict = false);
-
-   OccaCoefficient& AddGridFunction(const std::string &name_,
-                                    mfem::GridFunction &gf,
-                                    const bool useRestrict = false);
-
-   bool IsConstant();
-   double GetConstantValue();
-
-   Vector Eval();
-   void Eval(Vector &quadCoeff);
-
-   operator ::occa::kernelArg ();
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_COEFFICIENT_HPP

backends/occa/defines.okl

@@ -1,40 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_OCCA_DEFINES
-#define MFEM_OCCA_DEFINES
-
-#ifndef USING_TENSOR_OPS
-#  define USING_TENSOR_OPS 0
-#endif
-
-#ifdef OCCA_USING_GPU
-#  define GPU_ORDER_2(I0, I1)         @dimOrder(I0, I1)
-#  define GPU_ORDER_3(I0, I1, I2)     @dimOrder(I0, I1, I2)
-#  define GPU_ORDER_4(I0, I1, I2, I3) @dimOrder(I0, I1, I2, I3)
-#else
-#  define GPU_ORDER_2(I0, I1)         @dimOrder(0, 1)
-#  define GPU_ORDER_3(I0, I1, I2)     @dimOrder(0, 1, 2)
-#  define GPU_ORDER_4(I0, I1, I2, I3) @dimOrder(0, 1, 2, 3)
-#endif
-
-#ifndef COEFF
-#  define COEFF 1.0
-#  define COEFF_ARGS
-#endif
-
-#if USING_TENSOR_OPS
-#  include "mfem-occa://defines/tensor.okl"
-#else
-#  include "mfem-occa://defines/simplex.okl"
-#endif
-
-#endif

backends/occa/defines/simplex.okl

@@ -1,40 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#define USING_LOW_ORDER 1
-#define USING_HI_ORDER  0
-
-typedef double* DofToQuad_t @dim(NUM_QUAD, NUM_DOFS);
-typedef double* DofToQuadD2D_t @dim(2, NUM_QUAD, NUM_DOFS);
-typedef double* DofToQuadD3D_t @dim(3, NUM_QUAD, NUM_DOFS);
-
-typedef double* QuadToDof_t @dim(NUM_DOFS, NUM_QUAD);
-typedef double* QuadToDofD2D_t @dim(2, NUM_DOFS, NUM_QUAD);
-typedef double* QuadToDofD3D_t @dim(3, NUM_DOFS, NUM_QUAD);
-
-typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD, numElements);
-typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
-
-typedef double* SymmOperator2D_t @dim(3, NUM_QUAD, numElements);
-typedef double* SymmOperator3D_t @dim(6, NUM_QUAD, numElements);
-
-typedef double* DLocal_t @dim(NUM_DOFS, numElements);
-typedef double* QLocal_t @dim(NUM_QUAD, numElements);
-
-#if VDIM_ORDERING == ORDERING_BY_VDIM
-typedef double* DVLocal_t @dim(NUM_VDIM, NUM_DOFS, numElements);
-typedef double* QVLocal_t @dim(NUM_VDIM, NUM_QUAD, numElements);
-#else
-typedef double* DVLocal_t @dim(NUM_VDIM, NUM_DOFS, numElements) @dimOrder(2,0,1);
-typedef double* QVLocal_t @dim(NUM_VDIM, NUM_QUAD, numElements) @dimOrder(2,0,1);
-#endif
-
-typedef int* DLocalMap_t @dim(NUM_DOFS, numElements);

backends/occa/defines/tensor.okl

@@ -1,85 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#if NUM_QUAD_1D < NUM_DOFS_1D
-#  define NUM_MAX_1D NUM_DOFS_1D
-#else
-#  define NUM_MAX_1D NUM_QUAD_1D
-#endif
-
-#define NUM_MAX_2D (NUM_MAX_1D * NUM_MAX_1D)
-
-#define NUM_QUAD_DOFS_1D (NUM_QUAD_1D * NUM_DOFS_1D)
-
-#define QUAD_2D_ID(X, Y) (X + ((Y) * NUM_QUAD_1D))
-#define DOFS_2D_ID(X, Y) (X + ((Y) * NUM_DOFS_1D))
-
-#define QUAD_3D_ID(X, Y, Z) (X + ((Y) * NUM_QUAD_1D) + ((Z) * NUM_QUAD_2D))
-#define DOFS_3D_ID(X, Y, Z) (X + ((Y) * NUM_DOFS_1D) + ((Z) * NUM_DOFS_2D))
-
-#if NUM_MAX_1D < 8
-#  define USING_LOW_ORDER 1
-#  define USING_HI_ORDER  0
-#else
-#  define USING_LOW_ORDER 0
-#  define USING_HI_ORDER  1
-#endif
-
-#define M1_ELEMENT_BATCHES (M1_OUTER_ELEMENT_BATCH * M1_INNER_ELEMENT_BATCH)
-
-typedef double* DofToQuad_t @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-typedef double* QuadToDof_t @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-
-typedef double* Jacobian_t   @dim(NUM_DIM, NUM_DIM, numElements);
-typedef double* Jacobian1D_t @dim(NUM_QUAD_1D, numElements);
-typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD_2D, numElements);
-typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD_3D, numElements);
-
-typedef double* SymmOperator1D_t @dim(NUM_QUAD_1D, numElements);
-typedef double* SymmOperator2D_t @dim(3, NUM_QUAD_2D, numElements);
-typedef double* SymmOperator3D_t @dim(6, NUM_QUAD_3D, numElements);
-
-typedef double* DLocal_t   @dim(NUM_DOFS, numElements);
-typedef double* DLocal1D_t @dim(NUM_DOFS_1D, numElements);
-typedef double* DLocal2D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, numElements);
-typedef double* DLocal3D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
-
-typedef double* QLocal_t   @dim(NUM_QUAD, numElements);
-typedef double* QLocal1D_t @dim(NUM_QUAD_1D, numElements);
-typedef double* QLocal2D_t @dim(NUM_QUAD_1D, NUM_QUAD_1D, numElements);
-typedef double* QLocal3D_t @dim(NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
-
-#if VDIM_ORDERING == ORDERING_BY_VDIM
-typedef double* DVLocal_t   @dim(NUM_VDIM, NUM_DOFS, numElements);
-typedef double* DVLocal1D_t @dim(NUM_VDIM, NUM_DOFS_1D, numElements);
-typedef double* DVLocal2D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
-typedef double* DVLocal3D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
-
-typedef double* QVLocal_t   @dim(NUM_VDIM, NUM_QUAD, numElements);
-typedef double* QVLocal1D_t @dim(NUM_VDIM, NUM_QUAD_1D, numElements);
-typedef double* QVLocal2D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
-typedef double* QVLocal3D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
-#else
-typedef double* DVLocal_t   @dim(NUM_VDIM, NUM_DOFS, numElements) @dimOrder(2,0,1);
-typedef double* DVLocal1D_t @dim(NUM_VDIM, NUM_DOFS_1D, numElements) @dimOrder(2,0,1);
-typedef double* DVLocal2D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, numElements) @dimOrder(3,0,1,2);
-typedef double* DVLocal3D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements) @dimOrder(4,0,1,2,3);
-
-typedef double* QVLocal_t   @dim(NUM_VDIM, NUM_QUAD, numElements) @dimOrder(2,0,1);
-typedef double* QVLocal1D_t @dim(NUM_VDIM, NUM_QUAD_1D, numElements) @dimOrder(2,0,1);
-typedef double* QVLocal2D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, numElements) @dimOrder(3,0,1,2);
-typedef double* QVLocal3D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements) @dimOrder(4,0,1,2,3);
-#endif
-
-typedef int* DLocalMap_t   @dim(NUM_DOFS, numElements);
-typedef int* DLocalMap1D_t @dim(NUM_DOFS_1D, numElements);
-typedef int* DLocalMap2D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, numElements);
-typedef int* DLocalMap3D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);

backends/occa/diffusion/simplex/cpu.okl

@@ -1,168 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        @restrict const double *quadWeights,
-                        @restrict const Jacobian2D_t J,
-                        COEFF_ARGS
-                        @restrict SymmOperator2D_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-
-      const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
-
-      oper(0, q, e) =  c_detJ * (J12*J12 + J22*J22); // (1,1)
-      oper(1, q, e) = -c_detJ * (J12*J11 + J22*J21); // (1,2) + (2,1)
-      oper(2, q, e) =  c_detJ * (J11*J11 + J21*J21); // (2,2)
-    }
-  }
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuadD2D_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDofD2D_t quadToDofD,
-                       @restrict const SymmOperator2D_t oper,
-                       @restrict const DLocal_t solIn,
-                       @restrict DLocal_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double r_sol[NUM_DOFS];
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        r_sol[d] = 0;
-      }
-
-      for (int q = 0; q < NUM_QUAD; ++q) {
-        double gradX = 0, gradY = 0;
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          const double s = solIn(d, e);
-          gradX += s * quadToDofD(0, d, q);
-          gradY += s * quadToDofD(1, d, q);
-        }
-        const double O11 = oper(0, q, e);
-        const double O12 = oper(1, q, e);
-        const double O22 = oper(2, q, e);
-
-        const double gradX2 = (O11 * gradX) + (O12 * gradY);
-        const double gradY2 = (O12 * gradX) + (O22 * gradY);
-
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          r_sol[d] += ((gradX2 * quadToDofD(0, d, q)) +
-                       (gradY2 * quadToDofD(1, d, q)));
-        }
-      }
-
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        solOut(d, e) += r_sol[d];
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian3D_t J,
-                        COEFF_ARGS
-                        @restrict SymmOperator3D_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-      const double c_detJ = quadWeights[q] * COEFF / detJ;
-
-      // adj(J)
-      const double A11 = (J22 * J33) - (J23 * J32);
-      const double A12 = (J23 * J31) - (J21 * J33);
-      const double A13 = (J21 * J32) - (J22 * J31);
-
-      const double A21 = (J13 * J32) - (J12 * J33);
-      const double A22 = (J11 * J33) - (J13 * J31);
-      const double A23 = (J12 * J31) - (J11 * J32);
-
-      const double A31 = (J12 * J23) - (J13 * J22);
-      const double A32 = (J13 * J21) - (J11 * J23);
-      const double A33 = (J11 * J22) - (J12 * J21);
-
-      // adj(J)^Tadj(J)
-      oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
-      oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2) + (2,1)
-      oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3) + (3,1)
-      oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
-      oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3) + (3,2)
-      oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuadD3D_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDofD3D_t quadToDofD,
-                       @restrict const SymmOperator3D_t oper,
-                       @restrict const DLocal_t solIn,
-                       @restrict DLocal_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double r_sol[NUM_DOFS];
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        r_sol[d] = 0;
-      }
-
-      for (int q = 0; q < NUM_QUAD; ++q) {
-        double gradX = 0, gradY = 0, gradZ = 0;
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          const double s = solIn(d, e);
-          gradX += s * quadToDofD(0, d, q);
-          gradY += s * quadToDofD(1, d, q);
-          gradZ += s * quadToDofD(2, d, q);
-        }
-        const double O11 = oper(0, q, e);
-        const double O12 = oper(1, q, e);
-        const double O13 = oper(2, q, e);
-        const double O22 = oper(3, q, e);
-        const double O23 = oper(4, q, e);
-        const double O33 = oper(5, q, e);
-
-        const double gradX2 = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
-        const double gradY2 = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
-        const double gradZ2 = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
-
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          r_sol[d] += ((gradX2 * quadToDofD(0, d, q)) +
-                       (gradY2 * quadToDofD(1, d, q)) +
-                       (gradZ2 * quadToDofD(2, d, q)));
-        }
-      }
-
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        solOut(d, e) += r_sol[d];
-      }
-    }
-  }
-}
-//======================================

backends/occa/diffusion/simplex/gpuHighOrder.okl

@@ -1,182 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        const double *quadWeights,
-                        const Jacobian2D_t J,
-                        COEFF_ARGS
-                        SymmOperator2D_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
-          for (int q = qOff; q < NUM_QUAD; q += A2_QUAD_BATCH) {
-            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-
-            const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
-
-            oper(0, q, e) =  c_detJ * (J12*J12 + J22*J22); // (1,1)
-            oper(1, q, e) = -c_detJ * (J12*J11 + J22*J21); // (1,2) + (2,1)
-            oper(2, q, e) =  c_detJ * (J11*J11 + J21*J21); // (2,2)
-          }
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuadD2D_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDofD2D_t quadToDofD,
-                       @restrict const SymmOperator2D_t oper,
-                       @restrict const DLocal_t solIn,
-                       @restrict DLocal_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_gradX[NUM_QUAD];
-    @shared double s_gradY[NUM_QUAD];
-
-    for (int qOff = 0; qOff < M2_INNER_BATCH; ++qOff; @inner) {
-      for (int q = qOff; q < NUM_QUAD; q += M2_INNER_BATCH) {
-        double gradX = 0, gradY = 0;
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          const double s = solIn(d, e);
-          gradX += s * quadToDofD(0, d, q);
-          gradY += s * quadToDofD(1, d, q);
-        }
-        const double O11 = oper(0, q, e);
-        const double O12 = oper(1, q, e);
-        const double O22 = oper(2, q, e);
-
-        s_gradX[q] = (O11 * gradX) + (O12 * gradY);
-        s_gradY[q] = (O12 * gradX) + (O22 * gradY);
-      }
-    }
-
-    for (int dOff = 0; dOff < M2_INNER_BATCH; ++dOff) {
-      for (int d = dOff; d < NUM_DOFS; d += M2_INNER_BATCH) {
-        double r_sol = 0;
-        for (int q = 0; q < NUM_QUAD; ++q) {
-          // FIXME: s_gradX and s_gradY are @shared used outside of @inner
-          r_sol += ((s_gradX[q] * quadToDofD(0, d, q)) +
-                    (s_gradY[q] * quadToDofD(1, d, q)));
-        }
-        solOut(d, e) += r_sol;
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        const double *quadWeights,
-                        const Jacobian3D_t J,
-                        COEFF_ARGS
-                        SymmOperator3D_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
-          for (int q = qOff; q < NUM_QUAD; q += A3_QUAD_BATCH) {
-            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-            const double c_detJ = quadWeights[q] * COEFF / detJ;
-
-            // adj(J)
-            const double A11 = (J22 * J33) - (J23 * J32);
-            const double A12 = (J23 * J31) - (J21 * J33);
-            const double A13 = (J21 * J32) - (J22 * J31);
-
-            const double A21 = (J13 * J32) - (J12 * J33);
-            const double A22 = (J11 * J33) - (J13 * J31);
-            const double A23 = (J12 * J31) - (J11 * J32);
-
-            const double A31 = (J12 * J23) - (J13 * J22);
-            const double A32 = (J13 * J21) - (J11 * J23);
-            const double A33 = (J11 * J22) - (J12 * J21);
-
-            // adj(J)^Tadj(J)
-            oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
-            oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2) + (2,1)
-            oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3) + (3,1)
-            oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
-            oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3) + (3,2)
-            oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
-          }
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuadD3D_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDofD3D_t quadToDofD,
-                       @restrict const SymmOperator3D_t oper,
-                       @restrict const DLocal_t solIn,
-                       @restrict DLocal_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_gradX[NUM_QUAD];
-    @shared double s_gradY[NUM_QUAD];
-    @shared double s_gradZ[NUM_QUAD];
-
-    for (int qOff = 0; qOff < M3_INNER_BATCH; ++qOff; @inner) {
-      for (int q = qOff; q < NUM_QUAD; q += M3_INNER_BATCH) {
-        double gradX = 0, gradY = 0, gradZ = 0;
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          const double s = solIn(d, e);
-          gradX += s * quadToDofD(0, d, q);
-          gradY += s * quadToDofD(1, d, q);
-          gradZ += s * quadToDofD(2, d, q);
-        }
-        const double O11 = oper(0, q, e);
-        const double O12 = oper(1, q, e);
-        const double O13 = oper(2, q, e);
-        const double O22 = oper(3, q, e);
-        const double O23 = oper(4, q, e);
-        const double O33 = oper(5, q, e);
-
-        s_gradX[q] = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
-        s_gradY[q] = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
-        s_gradZ[q] = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
-      }
-    }
-
-    for (int dOff = 0; dOff < M3_INNER_BATCH; ++dOff) {
-      for (int d = dOff; d < NUM_DOFS; d += M3_INNER_BATCH) {
-        double r_sol = 0;
-        for (int q = 0; q < NUM_QUAD; ++q) {
-          r_sol += ((s_gradX[q] * quadToDofD(0, d, q)) +
-                    (s_gradY[q] * quadToDofD(1, d, q)) +
-                    (s_gradZ[q] * quadToDofD(2, d, q)));
-        }
-        solOut(d, e) += r_sol;
-      }
-    }
-  }
-}
-//======================================

backends/occa/diffusion/tensor/cpu.okl

@@ -1,370 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 1D ]-----------------------------
-@kernel void Assemble1D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian1D_t J,
-                        COEFF_ARGS
-                        @restrict SymmOperator1D_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
-      oper(q, e) = quadWeights[q] * COEFF / J(q, e);
-    }
-  }
-}
-
-@kernel void MultAdd1D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const SymmOperator1D_t oper,
-                       @restrict const DLocal1D_t solIn,
-                       @restrict DLocal1D_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double grad[NUM_QUAD_1D];
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        grad[qx] = 0;
-      }
-      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-        const double s = solIn(dx, e);
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          grad[qx] += s * dofToQuadD(qx, dx);
-        }
-      }
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        grad[qx] *= oper(qx, e);
-      }
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        const double gradX = grad[qx];
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          solOut(dx, e) += gradX * quadToDofD(dx, qx);
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian2D_t J,
-                        COEFF_ARGS
-                        @restrict SymmOperator2D_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_2D; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-
-      const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
-
-      oper(0, q, e) =  c_detJ * (J21*J21 + J22*J22); // (1,1)
-      oper(1, q, e) = -c_detJ * (J21*J11 + J22*J12); // (1,2), (2,1)
-      oper(2, q, e) =  c_detJ * (J11*J11 + J12*J12); // (2,2)
-    }
-  }
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const SymmOperator2D_t oper,
-                       @restrict const DLocal2D_t solIn,
-                       @restrict DLocal2D_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double grad[NUM_QUAD_1D][NUM_QUAD_1D][2];
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          grad[qy][qx][0] = 0;
-          grad[qy][qx][1] = 0;
-        }
-      }
-
-      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-        double gradX[NUM_QUAD_1D][2];
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          gradX[qx][0] = 0;
-          gradX[qx][1] = 0;
-        }
-
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          const double s = solIn(dx, dy, e);
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            gradX[qx][0] += s * dofToQuad(qx, dx);
-            gradX[qx][1] += s * dofToQuadD(qx, dx);
-          }
-        }
-
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          const double wy  = dofToQuad(qy, dy);
-          const double wDy = dofToQuadD(qy, dy);
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            grad[qy][qx][0] += gradX[qx][1] * wy;
-            grad[qy][qx][1] += gradX[qx][0] * wDy;
-          }
-        }
-      }
-
-      // Calculate Dxy, xDy in plane
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          const int q = QUAD_2D_ID(qx, qy);
-          const double O11 = oper(0, q, e);
-          const double O12 = oper(1, q, e);
-          const double O22 = oper(2, q, e);
-
-          const double gradX = grad[qy][qx][0];
-          const double gradY = grad[qy][qx][1];
-
-          grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
-          grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
-        }
-      }
-
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        double gradX[NUM_DOFS_1D][2];
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          gradX[dx][0] = 0;
-          gradX[dx][1] = 0;
-        }
-
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          const double gX = grad[qy][qx][0];
-          const double gY = grad[qy][qx][1];
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            const double wx  = quadToDof(dx, qx);
-            const double wDx = quadToDofD(dx, qx);
-            gradX[dx][0] += gX * wDx;
-            gradX[dx][1] += gY * wx;
-          }
-        }
-
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          const double wy  = quadToDof(dy, qy);
-          const double wDy = quadToDofD(dy, qy);
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            solOut(dx, dy, e) += ((gradX[dx][0] * wy) +
-                                  (gradX[dx][1] * wDy));
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian3D_t J,
-                        COEFF_ARGS
-                        @restrict SymmOperator3D_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_3D; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-      const double c_detJ = quadWeights[q] * COEFF / detJ;
-
-      // adj(J)
-      const double A11 = (J22 * J33) - (J23 * J32);
-      const double A12 = (J23 * J31) - (J21 * J33);
-      const double A13 = (J21 * J32) - (J22 * J31);
-
-      const double A21 = (J13 * J32) - (J12 * J33);
-      const double A22 = (J11 * J33) - (J13 * J31);
-      const double A23 = (J12 * J31) - (J11 * J32);
-
-      const double A31 = (J12 * J23) - (J13 * J22);
-      const double A32 = (J13 * J21) - (J11 * J23);
-      const double A33 = (J11 * J22) - (J12 * J21);
-
-      // adj(J)^Tadj(J)
-      oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
-      oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2), (2,1)
-      oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3), (3,1)
-      oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
-      oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3), (3,2)
-      oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const SymmOperator3D_t oper,
-                       @restrict const DLocal3D_t solIn,
-                       @restrict DLocal3D_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double grad[NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D][4];
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            grad[qz][qy][qx][0] = 0;
-            grad[qz][qy][qx][1] = 0;
-            grad[qz][qy][qx][2] = 0;
-          }
-        }
-      }
-
-      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-        double gradXY[NUM_QUAD_1D][NUM_QUAD_1D][4];
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            gradXY[qy][qx][0] = 0;
-            gradXY[qy][qx][1] = 0;
-            gradXY[qy][qx][2] = 0;
-          }
-        }
-
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          double gradX[NUM_QUAD_1D][2];
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            gradX[qx][0] = 0;
-            gradX[qx][1] = 0;
-          }
-
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            const double s = solIn(dx, dy, dz, e);
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              gradX[qx][0] += s * dofToQuad(qx, dx);
-              gradX[qx][1] += s * dofToQuadD(qx, dx);
-            }
-          }
-
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            const double wy  = dofToQuad(qy, dy);
-            const double wDy = dofToQuadD(qy, dy);
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              const double wx  = gradX[qx][0];
-              const double wDx = gradX[qx][1];
-              gradXY[qy][qx][0] += wDx * wy;
-              gradXY[qy][qx][1] += wx  * wDy;
-              gradXY[qy][qx][2] += wx  * wy;
-            }
-          }
-        }
-
-        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-          const double wz  = dofToQuad(qz, dz);
-          const double wDz = dofToQuadD(qz, dz);
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              grad[qz][qy][qx][0] += gradXY[qy][qx][0] * wz;
-              grad[qz][qy][qx][1] += gradXY[qy][qx][1] * wz;
-              grad[qz][qy][qx][2] += gradXY[qy][qx][2] * wDz;
-            }
-          }
-        }
-      }
-
-      // Calculate Dxyz, xDyz, xyDz in plane
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            const int q = QUAD_3D_ID(qx, qy, qz);
-            const double O11 = oper(0, q, e);
-            const double O12 = oper(1, q, e);
-            const double O13 = oper(2, q, e);
-            const double O22 = oper(3, q, e);
-            const double O23 = oper(4, q, e);
-            const double O33 = oper(5, q, e);
-
-            const double gradX = grad[qz][qy][qx][0];
-            const double gradY = grad[qz][qy][qx][1];
-            const double gradZ = grad[qz][qy][qx][2];
-
-            grad[qz][qy][qx][0] = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
-            grad[qz][qy][qx][1] = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
-            grad[qz][qy][qx][2] = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
-          }
-        }
-      }
-
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        double gradXY[NUM_DOFS_1D][NUM_DOFS_1D][4];
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            gradXY[dy][dx][0] = 0;
-            gradXY[dy][dx][1] = 0;
-            gradXY[dy][dx][2] = 0;
-          }
-        }
-
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          double gradX[NUM_DOFS_1D][4];
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            gradX[dx][0] = 0;
-            gradX[dx][1] = 0;
-            gradX[dx][2] = 0;
-          }
-
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            const double gX = grad[qz][qy][qx][0];
-            const double gY = grad[qz][qy][qx][1];
-            const double gZ = grad[qz][qy][qx][2];
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              const double wx  = quadToDof(dx, qx);
-              const double wDx = quadToDofD(dx, qx);
-              gradX[dx][0] += gX * wDx;
-              gradX[dx][1] += gY * wx;
-              gradX[dx][2] += gZ * wx;
-            }
-          }
-
-          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-            const double wy  = quadToDof(dy, qy);
-            const double wDy = quadToDofD(dy, qy);
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              gradXY[dy][dx][0] += gradX[dx][0] * wy;
-              gradXY[dy][dx][1] += gradX[dx][1] * wDy;
-              gradXY[dy][dx][2] += gradX[dx][2] * wy;
-            }
-          }
-        }
-
-        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-          const double wz  = quadToDof(dz, qz);
-          const double wDz = quadToDofD(dz, qz);
-          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              solOut(dx, dy, dz, e) += ((gradXY[dy][dx][0] * wz) +
-                                        (gradXY[dy][dx][1] * wz) +
-                                        (gradXY[dy][dx][2] * wDz));
-            }
-          }
-        }
-      }
-    }
-  }
-}
-//======================================

backends/occa/diffusion/tensor/gpuHighOrder.okl

@@ -1,435 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 1D ]-----------------------------
-@kernel void Assemble1D(const int numElements,
-                        const double *quadWeights,
-                        const Jacobian1D_t J,
-                        COEFF_ARGS
-                        SymmOperator1D_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A1_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A1_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
-          oper(q, e) = quadWeights[q] * COEFF / J(q, e);
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd1D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const SymmOperator1D_t oper,
-                       @restrict const DLocal1D_t solIn,
-                       @restrict DLocal1D_t solOut) {
-  // Iterate over elements
-  for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
-    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-
-    @exclusive double grad[NUM_QUAD_1D];
-
-    for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
-      for (int i = el; i < NUM_QUAD_DOFS_1D; i += M1_INNER_ELEMENT_BATCH) {
-        s_dofToQuadD[i] = dofToQuadD[i];
-        s_quadToDofD[i] = quadToDofD[i];
-      }
-    }
-
-    for (int b = 0; b < M1_OUTER_ELEMENT_BATCH; ++b) {
-      for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
-        const int e = eOff + b*M1_INNER_ELEMENT_BATCH + el;
-        if (e < numElements) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            grad[qx] = 0;
-          }
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            const double s = solIn(dx, e);
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              grad[qx] += s * s_dofToQuadD(qx, dx);
-            }
-          }
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            grad[qx] *= oper(qx, e);
-          }
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            double s = 0;
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              s += grad[qx] * s_quadToDofD(dx, qx);
-            }
-            solOut(dx, e) += s;
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        const double *quadWeights,
-                        const Jacobian2D_t J,
-                        COEFF_ARGS
-                        SymmOperator2D_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
-          for (int q = qOff; q < NUM_QUAD_2D; q += A2_QUAD_BATCH) {
-            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-
-            const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
-
-            oper(0, q, e) =  c_detJ * (J21*J21 + J22*J22); // (1,1)
-            oper(1, q, e) = -c_detJ * (J21*J11 + J22*J12); // (1,2), (2,1)
-            oper(2, q, e) =  c_detJ * (J11*J11 + J12*J12); // (2,2)
-          }
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const SymmOperator2D_t oper,
-                       @restrict const DLocal2D_t solIn,
-                       @restrict DLocal2D_t solOut) {
-  // Iterate over elements
-  for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
-    // Store dof <--> quad mappings
-    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-
-    // Store xy planes in shared memory
-    @shared double s_xy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-    @shared double s_xDy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-    @shared double s_grad[2 * NUM_QUAD_2D] @dim(2, NUM_QUAD_1D, NUM_QUAD_1D);
-
-    @exclusive double r_x[NUM_MAX_1D];
-    @exclusive double r_y[NUM_QUAD_1D];
-
-    for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
-      for (int id = x; id < NUM_QUAD_DOFS_1D; id += NUM_MAX_1D) {
-        s_dofToQuad[id]  = dofToQuad[id];
-        s_dofToQuadD[id] = dofToQuadD[id];
-        s_quadToDof[id]  = quadToDof[id];
-        s_quadToDofD[id] = quadToDofD[id];
-      }
-    }
-
-    for (int e = eOff; e < (eOff + M2_ELEMENT_BATCH); ++e) {
-      if (e < numElements) {
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx; @inner) {
-          if (dx < NUM_DOFS_1D) {
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              s_xy(dx, qy) = 0;
-              s_xDy(dx, qy) = 0;
-            }
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              r_x[dy] = solIn(dx, dy, e);
-            }
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              double xy = 0;
-              double xDy = 0;
-              for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-                xy  += r_x[dy] * s_dofToQuad(qy, dy);
-                xDy += r_x[dy] * s_dofToQuadD(qy, dy);
-              }
-              s_xy(dx, qy)  = xy;
-              s_xDy(dx, qy) = xDy;
-            }
-          }
-        }
-        for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
-          if (qy < NUM_QUAD_1D) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              double gradX = 0, gradY = 0;
-              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-                gradX += s_xy(dx, qy)  * s_dofToQuadD(qx, dx);
-                gradY += s_xDy(dx, qy) * s_dofToQuad(qx, dx);
-              }
-
-              const int q = QUAD_2D_ID(qx, qy);
-              const double O11 = oper(0, q, e);
-              const double O12 = oper(1, q, e);
-              const double O22 = oper(2, q, e);
-
-              s_grad(0, qx, qy) = (O11 * gradX) + (O12 * gradY);
-              s_grad(1, qx, qy) = (O12 * gradX) + (O22 * gradY);
-            }
-          }
-        }
-
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx; @inner) {
-          if (qx < NUM_QUAD_1D) {
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              s_xy(dy, qx) = 0;
-              s_xDy(dy, qx) = 0;
-            }
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              r_x[qy] = s_grad(0, qx, qy);
-              r_y[qy] = s_grad(1, qx, qy);
-            }
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              double xy  = 0;
-              double xDy = 0;
-              for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-                xy  += r_x[qy] * s_quadToDof(dy, qy);
-                xDy += r_y[qy] * s_quadToDofD(dy, qy);
-              }
-              s_xy(dy, qx) = xy;
-              s_xDy(dy, qx) = xDy;
-            }
-          }
-        }
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx; @inner) {
-          if (dx < NUM_DOFS_1D) {
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              double s = 0;
-              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-                s += ((s_xy(dy, qx) * s_quadToDofD(dx, qx)) +
-                      (s_xDy(dy, qx) * s_quadToDof(dx, qx)));
-              }
-              solOut(dx, dy, e) += s;
-            }
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        const double *quadWeights,
-                        const Jacobian3D_t J,
-                        COEFF_ARGS
-                        SymmOperator3D_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
-          for (int q = qOff; q < NUM_QUAD_3D; q += A3_QUAD_BATCH) {
-            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-            const double c_detJ = quadWeights[q] * COEFF / detJ;
-
-            // adj(J)
-            const double A11 = (J22 * J33) - (J23 * J32);
-            const double A12 = (J23 * J31) - (J21 * J33);
-            const double A13 = (J21 * J32) - (J22 * J31);
-
-            const double A21 = (J13 * J32) - (J12 * J33);
-            const double A22 = (J11 * J33) - (J13 * J31);
-            const double A23 = (J12 * J31) - (J11 * J32);
-
-            const double A31 = (J12 * J23) - (J13 * J22);
-            const double A32 = (J13 * J21) - (J11 * J23);
-            const double A33 = (J11 * J22) - (J12 * J21);
-
-            // adj(J)^Tadj(J)
-            oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
-            oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2), (2,1)
-            oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3), (3,1)
-            oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
-            oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3), (3,2)
-            oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
-          }
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const SymmOperator3D_t oper,
-                       @restrict const DLocal3D_t solIn,
-                       @restrict DLocal3D_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    // Store dof <--> quad mappings
-    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-
-    // Store xy planes in shared memory
-    @shared double s_z[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
-    @shared double s_Dz[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
-    @shared double s_xyDz[NUM_QUAD_2D] @dim(NUM_QUAD_1D, NUM_QUAD_1D);
-
-    // Store z axis as registers
-    @exclusive double r_qz[NUM_QUAD_1D];
-    @exclusive double r_qDz[NUM_QUAD_1D];
-    @exclusive double r_dDxyz[NUM_DOFS_1D];
-    @exclusive double r_dxDyz[NUM_DOFS_1D];
-    @exclusive double r_dxyDz[NUM_DOFS_1D];
-
-    for (int y = 0; y < NUM_MAX_1D; ++y; @inner) {
-      for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
-        const int id = (y * NUM_MAX_1D) + x;
-        // Fetch Q <--> D maps
-        if (id < NUM_QUAD_DOFS_1D) {
-          s_dofToQuad[id]  = dofToQuad[id];
-          s_dofToQuadD[id] = dofToQuadD[id];
-          s_quadToDof[id]  = quadToDof[id];
-          s_quadToDofD[id] = quadToDofD[id];
-        }
-        // Initialize our Z axis
-        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-          r_qz[qz] = 0;
-          r_qDz[qz] = 0;
-        }
-        // Initialize our solution updates in the Z axis
-        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-          r_dDxyz[dz] = 0;
-          r_dxDyz[dz] = 0;
-          r_dxyDz[dz] = 0;
-        }
-      }
-    }
-
-    for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
-      for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-        if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
-          for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-            const double s = solIn(dx, dy, dz, e);
-            // Calculate D -> Q in the Z axis
-            for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-              r_qz[qz]  += s * s_dofToQuad(qz, dz);
-              r_qDz[qz] += s * s_dofToQuadD(qz, dz);
-            }
-          }
-        }
-      }
-    }
-    // For each xy plane
-    for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-      // Fill xy plane at given z position
-      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
-        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
-            s_z(dx, dy)  = r_qz[qz];
-            s_Dz(dx, dy) = r_qDz[qz];
-          }
-        }
-      }
-      // Calculate Dxyz, xDyz, xyDz in plane
-      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
-        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
-          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
-            double Dxyz = 0;
-            double xDyz = 0;
-            double xyDz = 0;
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              const double wy  = s_dofToQuad(qy, dy);
-              const double wDy = s_dofToQuadD(qy, dy);
-              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-                const double wx  = s_dofToQuad(qx, dx);
-                const double wDx = s_dofToQuadD(qx, dx);
-                const double z  = s_z(dx, dy);
-                const double Dz = s_Dz(dx, dy);
-                Dxyz += wDx * wy  * z;
-                xDyz += wx  * wDy * z;
-                xyDz += wx  * wy  * Dz;
-              }
-            }
-
-            const int q = QUAD_3D_ID(qx, qy, qz);
-            const double O11 = oper(0, q, e);
-            const double O12 = oper(1, q, e);
-            const double O13 = oper(2, q, e);
-            const double O22 = oper(3, q, e);
-            const double O23 = oper(4, q, e);
-            const double O33 = oper(5, q, e);
-
-            const double qDxyz = (O11 * Dxyz) + (O12 * xDyz) + (O13 * xyDz);
-            const double qxDyz = (O12 * Dxyz) + (O22 * xDyz) + (O23 * xyDz);
-            const double qxyDz = (O13 * Dxyz) + (O23 * xDyz) + (O33 * xyDz);
-
-            for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-              const double wz  = s_quadToDof(dz, qz);
-              const double wDz = s_quadToDofD(dz, qz);
-              r_dDxyz[dz] += wz  * qDxyz;
-              r_dxDyz[dz] += wz  * qxDyz;
-              r_dxyDz[dz] += wDz * qxyDz;
-            }
-          }
-        }
-      }
-      @barrier("s_z_s_Dz_sync_1");
-    }
-    // Iterate over xy planes to compute solution
-    for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-      // Place xy plane in shared memory
-      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
-        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
-          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
-            s_z(qx, qy)    = r_dDxyz[dz];
-            s_Dz(qx, qy)   = r_dxDyz[dz];
-            s_xyDz(qx, qy) = r_dxyDz[dz];
-          }
-        }
-      }
-      // Finalize solution in xy plane
-      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
-        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
-            double solZ = 0;
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              const double wy  = s_quadToDof(dy, qy);
-              const double wDy = s_quadToDofD(dy, qy);
-              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-                const double wx  = s_quadToDof(dx, qx);
-                const double wDx = s_quadToDofD(dx, qx);
-                const double Dxyz = s_z(qx, qy);
-                const double xDyz = s_Dz(qx, qy);
-                const double xyDz = s_xyDz(qx, qy);
-                solZ += ((wDx * wy  * Dxyz) +
-                         (wx  * wDy * xDyz) +
-                         (wx  * wy  * xyDz));
-              }
-            }
-            solOut(dx, dy, dz, e) += solZ;
-          }
-        }
-      }
-      @barrier("s_z_s_Dz_s_xyDz_sync_1");
-    }
-  }
-}
-//======================================

backends/occa/engine.cpp

@@ -1,167 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "backend.hpp"
-#include "url_handler.hpp"
-#include "bilinearform.hpp"
-#include "../../general/array.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-bool Engine::fileOpenerRegistered = false;
-
-void Engine::Init(const std::string &engine_spec)
-{
-   //
-   // Initialize inherited fields
-   //
-   memory_resources[0] = NULL;
-   workers_weights[0]= 1.0;
-   workers_mem_res[0] = 0;
-
-   //
-   // Initialize the OCCA engine
-   //
-   ::occa::properties props(engine_spec);
-   device = new ::occa::device[1];
-   device[0].setup(props);
-
-   okl_path = "mfem-occa://";
-   if (!fileOpenerRegistered)
-   {
-      // The directories from "MFEM_OCCA_OKL_PATH", if any, have the highest
-      // priority.
-      FileOpener *fo = new FileOpener("mfem-occa://", "MFEM_OCCA_OKL_PATH");
-      // Next in priority is the source path, if it exists.
-      std::string mfem_src_prefix = mfem::GetSourcePath();
-      fo->AddDir(mfem_src_prefix + "/backends/occa");
-      // And last in priority is the install path, if it exists.
-      std::string mfem_install_prefix = mfem::GetInstallPath();
-      fo->AddDir(mfem_install_prefix + "/lib/mfem/occa");
-      ::occa::io::fileOpener::add(fo);
-      fileOpenerRegistered = true;
-   }
-   // std::cout << "OCCA device properties:\n" << device[0].properties();
-
-   force_cuda_aware_mpi = false;
-}
-
-Engine::Engine(const std::string &engine_spec)
-   : mfem::Engine(NULL, 1, 1)
-{
-   Init(engine_spec);
-}
-
-#ifdef MFEM_USE_MPI
-Engine::Engine(MPI_Comm _comm, const std::string &engine_spec)
-   : mfem::Engine(NULL, 1, 1)
-{
-   comm = _comm;
-   Init(engine_spec);
-}
-#endif
-
-bool Engine::CheckEngine(const mfem::Engine *engine) const
-{
-   return (engine != NULL && util::Is<const Engine>(engine) != NULL &&
-           *util::As<const Engine>(engine) == *this);
-}
-
-bool Engine::CheckLayout(const PLayout *layout) const
-{
-   return (layout != NULL && util::Is<const Layout>(layout) != NULL &&
-           layout->As<Layout>().OccaEngine() == *this);
-}
-
-bool Engine::CheckArray(const PArray *array) const
-{
-   return (array != NULL && util::Is<const Array>(array) != NULL &&
-           array->As<Array>().OccaEngine() == *this);
-}
-
-bool Engine::CheckVector(const PVector *vector) const
-{
-   return (vector != NULL && util::Is<const Vector>(vector) != NULL &&
-           vector->As<Vector>().OccaEngine() == *this);
-}
-
-bool Engine::CheckFESpace(const PFiniteElementSpace *fes) const
-{
-   return (fes != NULL && util::Is<const FiniteElementSpace>(fes) != NULL &&
-           fes->As<FiniteElementSpace>().OccaEngine() == *this);
-}
-
-DLayout Engine::MakeLayout(std::size_t size) const
-{
-   return DLayout(new Layout(*this, size));
-}
-
-DLayout Engine::MakeLayout(const mfem::Array<std::size_t> &offsets) const
-{
-   MFEM_ASSERT(offsets.Size() == 2,
-               "multiple workers are not supported yet");
-   return DLayout(new Layout(*this, offsets.Last()));
-}
-
-DArray Engine::MakeArray(PLayout &layout, std::size_t item_size) const
-{
-   return DArray(new Array(layout.As<Layout>(), item_size));
-}
-
-DVector Engine::MakeVector(PLayout &layout, int type_id) const
-{
-   MFEM_ASSERT(type_id == ScalarId<double>::value, "type_id " << type_id
-               << " is not supported");
-   return DVector(new Vector(layout.As<Layout>()));
-}
-
-DFiniteElementSpace Engine::MakeFESpace(mfem::FiniteElementSpace &fespace) const
-{
-   return DFiniteElementSpace(new FiniteElementSpace(*this, fespace));
-}
-
-DBilinearForm Engine::MakeBilinearForm(mfem::BilinearForm &bf) const
-{
-   return DBilinearForm(new BilinearForm(*this, bf));
-}
-
-void Engine::AssembleLinearForm(LinearForm &l_form) const
-{
-   /// FIXME - What will the actual parameters be?
-   MFEM_ABORT("FIXME");
-}
-
-mfem::Operator *Engine::MakeOperator(const MixedBilinearForm &mbl_form) const
-{
-   /// FIXME - What will the actual parameters be?
-   MFEM_ABORT("FIXME");
-   return NULL;
-}
-
-mfem::Operator *Engine::MakeOperator(const NonlinearForm &nl_form) const
-{
-   /// FIXME - What will the actual parameters be?
-   MFEM_ABORT("FIXME");
-   return NULL;
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/engine.hpp

@@ -1,144 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_ENGINE_HPP
-#define MFEM_BACKENDS_OCCA_ENGINE_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "../base/backend.hpp"
-#include <occa.hpp>
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class Engine : public mfem::Engine
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // mfem::Backend *backend;
-#ifdef MFEM_USE_MPI
-   // MPI_Comm comm;
-#endif
-   // int num_mem_res;
-   // int num_workers;
-   // MemoryResource **memory_resources;
-   // double *workers_weights;
-   // int *workers_mem_res;
-
-   static bool fileOpenerRegistered;
-   /// An array of OCCA devices. Currently only a single device is supported.
-   ::occa::device *device;
-   std::string okl_path;
-   bool force_cuda_aware_mpi;
-
-   void Init(const std::string &engine_spec);
-
-public:
-   /// TODO: doxygen
-   Engine(const std::string &engine_spec);
-
-#ifdef MFEM_USE_MPI
-   /// TODO: doxygen
-   Engine(MPI_Comm comm, const std::string &engine_spec);
-#endif
-
-   /// TODO: doxygen
-   virtual ~Engine() { delete [] device; }
-
-   /**
-       @name OCCA specific interface, used by other objects in the OCCA backend
-    */
-   ///@{
-
-   /// Get the associated OCCA device.
-   ::occa::device GetDevice(int idx = 0) const { return device[idx]; }
-
-   /// TODO: doxygen
-   const std::string &GetOklPath() const { return okl_path; }
-
-   /// OCCA device memory allocation.
-   ::occa::memory Alloc(std::size_t bytes) const
-   { return GetDevice().malloc(bytes); }
-
-   /// Two mfem::occa::Engine%s are equal if they use the same OCCA device.
-   bool operator==(const Engine &other) const
-   { return GetDevice() == other.GetDevice(); }
-
-   /// TODO: doxygen
-   bool CheckEngine(const mfem::Engine *e) const;
-
-   /// TODO: doxygen
-   bool CheckLayout(const PLayout *layout) const;
-
-   /// TODO: doxygen
-   bool CheckArray(const PArray *array) const;
-
-   /// TODO: doxygen
-   bool CheckVector(const PVector *vector) const;
-
-   /// TODO: doxygen
-   bool CheckFESpace(const PFiniteElementSpace *fes) const;
-
-#ifdef MFEM_USE_MPI
-   void SetForceCudaAwareMPI(bool force = true)
-   { force_cuda_aware_mpi = force; }
-
-   bool GetForceCudaAwareMPI() const { return force_cuda_aware_mpi; }
-#endif
-
-   ///@}
-   // End: OCCA specific interface
-
-   /**
-       @name Virtual interface: finite element data structures and algorithms
-    */
-   ///@{
-
-   virtual DLayout MakeLayout(std::size_t size) const;
-   virtual DLayout MakeLayout(const mfem::Array<std::size_t> &offsets) const;
-
-   virtual DArray MakeArray(PLayout &layout, std::size_t item_size) const;
-
-   virtual DVector MakeVector(PLayout &layout,
-                              int type_id = ScalarId<double>::value) const;
-
-   virtual DFiniteElementSpace MakeFESpace(mfem::FiniteElementSpace &
-                                           fespace) const;
-
-   virtual DBilinearForm MakeBilinearForm(mfem::BilinearForm &bf) const;
-
-   /// FIXME - What will the actual parameters be?
-   virtual void AssembleLinearForm(LinearForm &l_form) const;
-
-   /// FIXME - What will the actual parameters be?
-   virtual mfem::Operator *MakeOperator(const MixedBilinearForm &mbl_form) const;
-
-   /// FIXME - What will the actual parameters be?
-   virtual mfem::Operator *MakeOperator(const NonlinearForm &nl_form) const;
-
-   ///@}
-   // End: Virtual interface
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_ENGINE_HPP

backends/occa/fespace.cpp

@@ -1,532 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "backend.hpp"
-#include "fespace.hpp"
-#include "interpolation.hpp"
-
-#ifdef MFEM_USE_MPI
-#ifdef OMPI_RELEASE_VERSION
-#include <mpi-ext.h> // Check for cuda support
-#endif
-#endif
-
-namespace mfem
-{
-
-namespace occa
-{
-
-FiniteElementSpace::FiniteElementSpace(const Engine &e,
-                                       mfem::FiniteElementSpace &fespace)
-   : PFiniteElementSpace(e, fespace),
-     e_layout(new Layout(e, 0)) // resized in SetupLocalGlobalMaps()
-{
-   vdim     = fespace.GetVDim();
-   ordering = fespace.GetOrdering();
-
-   SetupLocalGlobalMaps();
-   SetupOperators(); // calls virtual methods of 'fes'
-   SetupKernels();
-}
-
-FiniteElementSpace::~FiniteElementSpace()
-{
-   delete restrictionOp;
-   delete prolongationOp;
-}
-
-void FiniteElementSpace::SetupLocalGlobalMaps()
-{
-   const int elements = fes->GetNE();
-
-   if (elements == 0) { return; }
-
-   // Assuming of finite elements are the same.
-   const mfem::FiniteElement &fe = *fes->GetFE(0);
-   const mfem::TensorBasisElement *el =
-      dynamic_cast<const mfem::TensorBasisElement*>(&fe);
-
-   const mfem::Table &e2dTable = fes->GetElementToDofTable();
-   const int *elementMap = e2dTable.GetJ();
-
-   globalDofs = fes->GetNDofs();
-   localDofs  = fe.GetDof();
-
-   e_layout->OccaResize(e2dTable.Size_of_connections());
-
-   int *elementDofMap = new int[localDofs];
-   if (el)
-   {
-      ::memcpy(elementDofMap,
-               el->GetDofMap().GetData(),
-               localDofs * sizeof(int));
-   }
-   else
-   {
-      for (int i = 0; i < localDofs; ++i)
-      {
-         elementDofMap[i] = i;
-      }
-   }
-
-   // Allocate device offsets and indices
-   globalToLocalOffsets.allocate(GetDevice(),
-                                 globalDofs + 1);
-   globalToLocalIndices.allocate(GetDevice(),
-                                 localDofs, elements);
-   localToGlobalMap.allocate(GetDevice(),
-                             localDofs, elements);
-
-   int *offsets = globalToLocalOffsets.ptr();
-   int *indices = globalToLocalIndices.ptr();
-   int *l2gMap  = localToGlobalMap.ptr();
-
-   // We'll be keeping a count of how many local nodes point
-   //   to its global dof
-   for (int i = 0; i <= globalDofs; ++i)
-   {
-      offsets[i] = 0;
-   }
-
-   for (int e = 0; e < elements; ++e)
-   {
-      MFEM_ASSERT(e2dTable.RowSize(e) == localDofs, "");
-      for (int d = 0; d < localDofs; ++d)
-      {
-         const int gid = elementMap[localDofs*e + d];
-         ++offsets[gid + 1];
-      }
-   }
-   // Aggregate to find offsets for each global dof
-   for (int i = 1; i <= globalDofs; ++i)
-   {
-      offsets[i] += offsets[i - 1];
-   }
-   // For each global dof, fill in all local nodes that point
-   //   to it
-   for (int e = 0; e < elements; ++e)
-   {
-      for (int d = 0; d < localDofs; ++d)
-      {
-         const int gid = elementMap[localDofs*e + elementDofMap[d]];
-         const int lid = localDofs*e + d;
-         indices[offsets[gid]++] = lid;
-         l2gMap[lid] = gid;
-      }
-   }
-   // We shifted the offsets vector by 1 by using it
-   //   as a counter. Now we shift it back.
-   for (int i = globalDofs; i > 0; --i)
-   {
-      offsets[i] = offsets[i - 1];
-   }
-   offsets[0] = 0;
-
-   delete [] elementDofMap;
-
-   globalToLocalOffsets.keepInDevice();
-   globalToLocalIndices.keepInDevice();
-   localToGlobalMap.keepInDevice();
-}
-
-void FiniteElementSpace::SetupOperators() const
-{
-   // Construct 'restrictionOp' and 'prolongationOp'.
-
-   prolongationOp = restrictionOp = NULL;
-
-   const mfem::SparseMatrix *R = fes->GetRestrictionMatrix();
-   const mfem::Operator *P = fes->GetProlongationMatrix();
-
-   if (!P) { return; }
-
-   Layout &v_layout = OccaVLayout();
-   Layout &t_layout = OccaTrueVLayout();
-
-   // Assuming R has one entry per row equal to 1.
-   MFEM_ASSERT(R->Finalized(), "");
-   const int tdofs = R->Height();
-   MFEM_ASSERT(tdofs == (int)t_layout.Size(), "");
-   MFEM_ASSERT(tdofs == R->GetI()[tdofs], "");
-   ::occa::array<int> ltdof_ldof(GetDevice(), tdofs, R->GetJ());
-   ltdof_ldof.keepInDevice();
-
-   restrictionOp = new RestrictionOperator(v_layout, t_layout, ltdof_ldof);
-
-   const mfem::SparseMatrix *pmat = dynamic_cast<const mfem::SparseMatrix*>(P);
-   if (pmat)
-   {
-      const mfem::SparseMatrix *pmatT = Transpose(*pmat);
-
-      OccaSparseMatrix *occaP  =
-         CreateMappedSparseMatrix(t_layout, v_layout, *pmat);
-      OccaSparseMatrix *occaPT =
-         CreateMappedSparseMatrix(v_layout, t_layout, *pmatT);
-
-      prolongationOp = new ProlongationOperator(*occaP, *occaPT);
-
-      delete occaPT;
-      delete occaP;
-   }
-#ifdef MFEM_USE_MPI
-   else if (fes->Conforming() && dynamic_cast<ParFiniteElementSpace*>(fes))
-   {
-      ParFiniteElementSpace *pfes = static_cast<ParFiniteElementSpace*>(fes);
-      prolongationOp = new OccaConformingProlongation(*this, *pfes,
-                                                      ltdof_ldof.memory());
-   }
-#endif
-   else
-   {
-      prolongationOp = new ProlongationOperator(t_layout, v_layout, P);
-   }
-}
-
-void FiniteElementSpace::SetupKernels()
-{
-   ::occa::properties props("defines: {"
-                            "  TILESIZE: 256,"
-                            "}");
-   props["defines/NUM_VDIM"] = vdim;
-
-   props["defines/ORDERING_BY_NODES"] = 0;
-   props["defines/ORDERING_BY_VDIM"]  = 1;
-   props["defines/VDIM_ORDERING"] = (int) (ordering == Ordering::byVDIM);
-
-   ::occa::device device = GetDevice();
-   const std::string &okl_path = OccaEngine().GetOklPath();
-   globalToLocalKernel = device.buildKernel(okl_path + "fespace.okl",
-                                            "GlobalToLocal",
-                                            props);
-   localToGlobalKernel = device.buildKernel(okl_path + "fespace.okl",
-                                            "LocalToGlobal",
-                                            props);
-}
-
-
-#ifdef MFEM_USE_MPI
-
-OccaConformingProlongation::OccaConformingProlongation(
-   const FiniteElementSpace &ofes, const mfem::ParFiniteElementSpace &pfes,
-   ::occa::memory ltdof_ldof_)
-
-   : Operator(ofes.OccaTrueVLayout(), ofes.OccaVLayout()),
-     shr_ltdof(ofes.OccaEngine()),
-     ext_ldof(ofes.OccaEngine()),
-     shr_buf(shr_ltdof.OccaLayout(), sizeof(double)),
-     ext_buf(ext_ldof.OccaLayout(), sizeof(double)),
-     shr_buf_offsets(NULL), ext_buf_offsets(NULL),
-     ltdof_ldof(ltdof_ldof_),
-     gc(pfes.GroupComm())
-{
-   MFEM_ASSERT(pfes.Conforming(), "internal error");
-
-   const Engine &engine = ofes.OccaEngine();
-   const std::string &okl_path = engine.GetOklPath();
-   ::occa::device device = engine.GetDevice();
-
-   {
-      Table nbr_ltdof;
-      gc.GetNeighborLTDofTable(nbr_ltdof);
-      shr_ltdof.OccaResize(nbr_ltdof.Size_of_connections(), sizeof(int));
-      shr_ltdof.OccaPush(nbr_ltdof.GetJ());
-      shr_buf.OccaResize(&shr_ltdof.OccaLayout(), sizeof(double));
-      shr_buf_offsets = nbr_ltdof.GetI();
-      {
-         mfem::Array<int> shr_ltdof(nbr_ltdof.GetJ(),
-                                    nbr_ltdof.Size_of_connections());
-         mfem::Array<int> unique_ltdof(shr_ltdof);
-         unique_ltdof.Sort();
-         unique_ltdof.Unique();
-         // Note: the next loop modifies the J array of nbr_ltdof
-         for (int i = 0; i < shr_ltdof.Size(); i++)
-         {
-            shr_ltdof[i] = unique_ltdof.FindSorted(shr_ltdof[i]);
-            MFEM_ASSERT(shr_ltdof[i] != -1, "internal error");
-         }
-         Table unique_shr;
-         Transpose(shr_ltdof, unique_shr, unique_ltdof.Size());
-
-         unq_ltdof = device.malloc(unique_ltdof.Size()*sizeof(int),
-                                   unique_ltdof.GetData());
-         unq_shr_i = device.malloc((unique_shr.Size()+1)*sizeof(int),
-                                   unique_shr.GetI());
-         unq_shr_j = device.malloc(unique_shr.Size_of_connections()*sizeof(int),
-                                   unique_shr.GetJ());
-      }
-      delete [] nbr_ltdof.GetJ();
-      nbr_ltdof.LoseData();
-   }
-   {
-      Table nbr_ldof;
-      gc.GetNeighborLDofTable(nbr_ldof);
-      ext_ldof.OccaResize(nbr_ldof.Size_of_connections(), sizeof(int));
-      ext_ldof.OccaPush(nbr_ldof.GetJ());
-      ext_buf.OccaResize(&ext_ldof.OccaLayout(), sizeof(double));
-      ext_buf_offsets = nbr_ldof.GetI();
-      delete [] nbr_ldof.GetJ();
-      nbr_ldof.LoseData();
-   }
-   host_shr_buf = NULL;
-   host_ext_buf = NULL;
-   // If the device has a separate memory space (e.g. CUDA device) and the MPI
-   // library does not support buffers in that separate memory space, we
-   // allocate separate host buffers to use for MPI communication.
-   if (device.hasSeparateMemorySpace())
-   {
-      bool need_host_buf = true;
-      if (device.mode() == "CUDA")
-      {
-#ifdef MPIX_CUDA_AWARE_SUPPORT
-         need_host_buf = !MPIX_Query_cuda_support();
-#endif
-         if (engine.GetForceCudaAwareMPI()) { need_host_buf = false; }
-         if (gc.GetGroupTopology().MyRank() == 0)
-         {
-            mfem::out << "\nOccaConformingProlongation: CUDA-aware MPI: "
-                      << (need_host_buf ? "NO" : "YES") << "\n\n";
-         }
-      }
-      if (need_host_buf)
-      {
-         host_shr_buf = new char[shr_buf.OccaMem().size()];
-         host_ext_buf = new char[ext_buf.OccaMem().size()];
-      }
-   }
-
-   ExtractSubVector = device.buildKernel(okl_path + "mappings.okl",
-                                         "ExtractSubVector",
-                                         "defines: { TILESIZE: 256 }");
-   SetSubVector = device.buildKernel(okl_path + "mappings.okl",
-                                     "SetSubVector",
-                                     "defines: { TILESIZE: 256 }");
-   AddSubVector = device.buildKernel(okl_path + "mappings.okl",
-                                     "AddSubVector",
-                                     "defines: { TILESIZE: 256 }");
-
-   const GroupTopology &gtopo = gc.GetGroupTopology();
-   int req_counter = 0;
-   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
-   {
-      const int send_offset = shr_buf_offsets[nbr];
-      const int send_size = shr_buf_offsets[nbr+1] - send_offset;
-      if (send_size > 0) { req_counter++; }
-
-      const int recv_offset = ext_buf_offsets[nbr];
-      const int recv_size = ext_buf_offsets[nbr+1] - recv_offset;
-      if (recv_size > 0) { req_counter++; }
-   }
-   requests = new MPI_Request[req_counter];
-}
-
-OccaConformingProlongation::~OccaConformingProlongation()
-{
-   delete [] requests;
-   delete [] host_ext_buf;
-   delete [] host_shr_buf;
-   delete [] ext_buf_offsets;
-   delete [] shr_buf_offsets;
-}
-
-void OccaConformingProlongation::BcastBeginCopy(const ::occa::memory &src,
-                                                std::size_t item_size) const
-{
-   // shr_buf[i] = src[shr_ltdof[i]]
-   MFEM_ASSERT(item_size == sizeof(double), "");
-   if (shr_ltdof.Size() == 0) { return; }
-   ExtractSubVector((int)shr_ltdof.Size(), shr_ltdof.OccaMem(), src,
-                    shr_buf.OccaMem());
-   // If the above kernel is executed asynchronously, wait for it to complete:
-   shr_buf.OccaMem().getDevice().finish();
-   if (host_shr_buf)
-   {
-      shr_buf.OccaMem().copyTo(host_shr_buf);
-   }
-}
-
-void OccaConformingProlongation::BcastLocalCopy(const ::occa::memory &src,
-                                                ::occa::memory &dst,
-                                                std::size_t item_size) const
-{
-   // dst[ltdof_ldof[i]] = src[i]
-   MFEM_ASSERT(item_size == sizeof(double), "");
-   if (ltdof_ldof.size<int>() == 0) { return; }
-   SetSubVector((int)ltdof_ldof.size<int>(), ltdof_ldof, src, dst);
-}
-
-void OccaConformingProlongation::BcastEndCopy(::occa::memory &dst,
-                                              std::size_t item_size) const
-{
-   // dst[ext_ldof[i]] = ext_buf[i]
-   MFEM_ASSERT(item_size == sizeof(double), "");
-   if (ext_ldof.Size() == 0) { return; }
-   if (host_ext_buf)
-   {
-      ext_buf.OccaMem().copyFrom(host_ext_buf);
-   }
-   SetSubVector((int)ext_ldof.Size(), ext_ldof.OccaMem(),
-                ext_buf.OccaMem(), dst);
-}
-
-void OccaConformingProlongation::ReduceBeginCopy(const ::occa::memory &src,
-                                                 std::size_t item_size) const
-{
-   // ext_buf[i] = src[ext_ldof[i]]
-   MFEM_ASSERT(item_size == sizeof(double), "");
-   if (ext_ldof.Size() == 0) { return; }
-   ExtractSubVector((int)ext_ldof.Size(), ext_ldof.OccaMem(), src,
-                    ext_buf.OccaMem());
-   // If the above kernel is executed asynchronously, wait for it to complete:
-   ext_buf.OccaMem().getDevice().finish();
-   if (host_ext_buf)
-   {
-      ext_buf.OccaMem().copyTo(host_ext_buf);
-   }
-}
-
-void OccaConformingProlongation::ReduceLocalCopy(const ::occa::memory &src,
-                                                 ::occa::memory &dst,
-                                                 std::size_t item_size) const
-{
-   // dst[i] = src[ltdof_ldof[i]]
-   MFEM_ASSERT(item_size == sizeof(double), "");
-   if (ltdof_ldof.size<int>() == 0) { return; }
-   ExtractSubVector((int)ltdof_ldof.size<int>(), ltdof_ldof, src, dst);
-}
-
-void OccaConformingProlongation::ReduceEndAssemble(::occa::memory &dst,
-                                                   std::size_t item_size) const
-{
-   // dst[shr_ltdof[i]] += shr_buf[i]
-   MFEM_ASSERT(item_size == sizeof(double), "");
-   if (unq_ltdof.size<int>() == 0) { return; }
-   if (host_shr_buf)
-   {
-      shr_buf.OccaMem().copyFrom(host_shr_buf);
-   }
-   AddSubVector((int)unq_ltdof.size<int>(), unq_ltdof, unq_shr_i, unq_shr_j,
-                shr_buf.OccaMem(), dst);
-}
-
-void OccaConformingProlongation::Mult_(const Vector &x, Vector &y) const
-{
-   const GroupTopology &gtopo = gc.GetGroupTopology();
-
-   BcastBeginCopy(x.OccaMem(), sizeof(double)); // copy to 'shr_buf'
-
-   int req_counter = 0;
-   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
-   {
-      const int send_offset = shr_buf_offsets[nbr];
-      const int send_size = shr_buf_offsets[nbr+1] - send_offset;
-      if (send_size > 0)
-      {
-         void *send_buf;
-         if (host_shr_buf)
-         {
-            send_buf = host_shr_buf + send_offset*sizeof(double);
-         }
-         else
-         {
-            send_buf = (shr_buf.OccaMem() + send_offset*sizeof(double)).ptr();
-         }
-         MPI_Isend(send_buf, send_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
-                   41822, gtopo.GetComm(), &requests[req_counter++]);
-      }
-
-      const int recv_offset = ext_buf_offsets[nbr];
-      const int recv_size = ext_buf_offsets[nbr+1] - recv_offset;
-      if (recv_size > 0)
-      {
-         void *recv_buf;
-         if (host_ext_buf)
-         {
-            recv_buf = host_ext_buf + recv_offset*sizeof(double);
-         }
-         else
-         {
-            recv_buf = (ext_buf.OccaMem() + recv_offset*sizeof(double)).ptr();
-         }
-         MPI_Irecv(recv_buf, recv_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
-                   41822, gtopo.GetComm(), &requests[req_counter++]);
-      }
-   }
-
-   BcastLocalCopy(x.OccaMem(), y.OccaMem(), sizeof(double));
-
-   MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
-
-   BcastEndCopy(y.OccaMem(), sizeof(double)); // copy from 'ext_buf'
-}
-
-void OccaConformingProlongation::MultTranspose_(const Vector &x,
-                                                Vector &y) const
-{
-   const GroupTopology &gtopo = gc.GetGroupTopology();
-
-   ReduceBeginCopy(x.OccaMem(), sizeof(double)); // copy to 'ext_buf'
-
-   int req_counter = 0;
-   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
-   {
-      const int send_offset = ext_buf_offsets[nbr];
-      const int send_size = ext_buf_offsets[nbr+1] - send_offset;
-      if (send_size > 0)
-      {
-         void *send_buf;
-         if (host_ext_buf)
-         {
-            send_buf = host_ext_buf + send_offset*sizeof(double);
-         }
-         else
-         {
-            send_buf = (ext_buf.OccaMem() + send_offset*sizeof(double)).ptr();
-         }
-         MPI_Isend(send_buf, send_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
-                   41823, gtopo.GetComm(), &requests[req_counter++]);
-      }
-
-      const int recv_offset = shr_buf_offsets[nbr];
-      const int recv_size = shr_buf_offsets[nbr+1] - recv_offset;
-      if (recv_size > 0)
-      {
-         void *recv_buf;
-         if (host_shr_buf)
-         {
-            recv_buf = host_shr_buf + recv_offset*sizeof(double);
-         }
-         else
-         {
-            recv_buf = (shr_buf.OccaMem() + recv_offset*sizeof(double)).ptr();
-         }
-         MPI_Irecv(recv_buf, recv_size, MPI_DOUBLE, gtopo.GetNeighborRank(nbr),
-                   41823, gtopo.GetComm(), &requests[req_counter++]);
-      }
-   }
-
-   ReduceLocalCopy(x.OccaMem(), y.OccaMem(), sizeof(double));
-
-   MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
-
-   ReduceEndAssemble(y.OccaMem(), sizeof(double)); // assemble from 'shr_buf'
-}
-
-#endif // MFEM_USE_MPI
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/fespace.hpp

@@ -1,210 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_FE_SPACE_HPP
-#define MFEM_BACKENDS_OCCA_FE_SPACE_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "engine.hpp"
-#include "operator.hpp"
-#include "../../fem/fem.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-/// TODO: doxygen
-class FiniteElementSpace : public mfem::PFiniteElementSpace
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // SharedPtr<const mfem::Engine> engine;
-   // mfem::FiniteElementSpace *fes;
-
-   SharedPtr<Layout> e_layout;
-
-   ::occa::array<int> globalToLocalOffsets;
-   ::occa::array<int> globalToLocalIndices;
-   ::occa::array<int> localToGlobalMap;
-   ::occa::kernel globalToLocalKernel, localToGlobalKernel;
-
-   mfem::Ordering::Type ordering;
-
-   int globalDofs, localDofs;
-   int vdim;
-
-   mutable Operator *prolongationOp, *restrictionOp;
-
-   void SetupLocalGlobalMaps();
-   void SetupOperators() const; // calls virtual methods of 'fes' !!!
-   void SetupKernels();
-
-public:
-   /// TODO: doxygen
-   FiniteElementSpace(const Engine &e, mfem::FiniteElementSpace &fespace);
-
-   /// Virtual destructor
-   virtual ~FiniteElementSpace();
-
-   /// TODO: doxygen
-   const Engine &OccaEngine() const { return engine->As<Engine>(); }
-
-   /// TODO: doxygen
-   ::occa::device GetDevice(int idx = 0) const
-   { return OccaEngine().GetDevice(idx); }
-
-   mfem::Mesh* GetMesh() const { return fes->GetMesh(); }
-
-   Layout &OccaVLayout() const
-   { return *fes->GetVLayout().As<Layout>(); }
-
-   Layout &OccaTrueVLayout() const
-   { return *fes->GetTrueVLayout().As<Layout>(); }
-
-   Layout &OccaEVLayout() { return *e_layout; }
-
-   bool hasTensorBasis() const
-   { return dynamic_cast<const mfem::TensorBasisElement*>(fes->GetFE(0)); }
-
-   mfem::Ordering::Type GetOrdering() const { return ordering; }
-
-   int GetGlobalDofs() const { return globalDofs; }
-   int GetLocalDofs() const { return localDofs; }
-
-   int GetDim() const { return fes->GetMesh()->Dimension(); }
-   int GetVDim() const { return vdim; }
-
-   int GetVSize() const { return globalDofs * vdim; }
-   int GetTrueVSize() const { return fes->GetTrueVSize(); }
-   int GetGlobalVSize() const { return globalDofs*vdim; /* FIXME: MPI */ }
-   int GetGlobalTrueVSize() const { return fes->GetTrueVSize(); }
-
-   int GetNE() const { return fes->GetNE(); }
-
-   const mfem::FiniteElementCollection *FEColl() const
-   { return fes->FEColl(); }
-   const mfem::FiniteElement *GetFE(const int idx) const
-   { return fes->GetFE(idx); }
-
-   virtual const mfem::Operator *GetProlongationOperator() const
-   { return prolongationOp; }
-
-   virtual const mfem::Operator *GetRestrictionOperator() const
-   { return restrictionOp; }
-
-   virtual const mfem::Operator *GetInterpolationOperator(
-      const mfem::QuadratureSpace &qspace) const
-   { return NULL; /* FIXME */ }
-
-   virtual const mfem::Operator *GetGradientOperator(
-      const mfem::QuadratureSpace &qspace) const
-   { return NULL; /* FIXME */ }
-
-   const ::occa::array<int> GetLocalToGlobalMap() const
-   { return localToGlobalMap; }
-
-   /// L-vector to E-vector
-   void GlobalToLocal(const Vector &globalVec, Vector &localVec) const
-   {
-      globalToLocalKernel(globalDofs,
-                          localDofs * fes->GetNE(),
-                          globalToLocalOffsets,
-                          globalToLocalIndices,
-                          globalVec.OccaMem(), localVec.OccaMem());
-   }
-
-   /// E-vector to L-vector, transpose of GlobalToLocal
-   void LocalToGlobal(const Vector &localVec, Vector &globalVec) const
-   {
-      localToGlobalKernel(globalDofs,
-                          localDofs * fes->GetNE(),
-                          globalToLocalOffsets,
-                          globalToLocalIndices,
-                          localVec.OccaMem(), globalVec.OccaMem());
-   }
-};
-
-
-#ifdef MFEM_USE_MPI
-
-/// OCCA version of mfem::ConformingProlongationOperator
-class OccaConformingProlongation : public Operator
-{
-protected:
-   // size(shr_buf)=size(shr_ltdof)
-   // size(ext_buf)=size(ext_ldof)
-   Array shr_ltdof, ext_ldof;
-   mutable Array shr_buf, ext_buf;
-   mutable char *host_shr_buf, *host_ext_buf;
-   // Offsets into {shr,ext}_buf; size is num. neighbors, i.e.
-   // gc.GetGroupTopology().GetNumNeighbors():
-   int *shr_buf_offsets, *ext_buf_offsets;
-
-   ::occa::memory ltdof_ldof; // shared with the restriction operator
-
-   ::occa::memory unq_ltdof; // enumeration of the unique ltdofs in shr_ltdof
-   ::occa::memory unq_shr_i, unq_shr_j;
-
-   ::occa::kernel ExtractSubVector, SetSubVector, AddSubVector;
-
-   MPI_Request *requests;
-
-   const GroupCommunicator &gc;
-
-   // Kernel: copy ltdofs from 'src' to 'shr_buf' - prepare for send.
-   //         shr_buf[i] = src[shr_ltdof[i]]
-   void BcastBeginCopy(const ::occa::memory &src, std::size_t item_size) const;
-   // Kernel: copy ltdofs from 'src' to ldofs in 'dst'.
-   //         dst[ltdof_ldof[i]] = src[i]
-   void BcastLocalCopy(const ::occa::memory &src, ::occa::memory &dst,
-                       std::size_t item_size) const;
-   // Kernel: copy ext. dofs from 'ext_buf' to 'dst' - after recv.
-   //         dst[ext_ldof[i]] = ext_buf[i]
-   void BcastEndCopy(::occa::memory &dst, std::size_t item_size) const;
-
-   // Kernel: copy ext. dofs from 'src' to 'ext_buf' - prepare for send.
-   //         ext_buf[i] = src[ext_ldof[i]]
-   void ReduceBeginCopy(const ::occa::memory &src, std::size_t item_size) const;
-   // Kernel: copy owned ldofs from 'src' to ltdofs in 'dst'.
-   //         dst[i] = src[ltdof_ldof[i]]
-   void ReduceLocalCopy(const ::occa::memory &src, ::occa::memory &dst,
-                       std::size_t item_size) const;
-   // Kernel: assemble dofs from 'shr_buf' into to 'dst' - after recv.
-   //         dst[shr_ltdof[i]] += shr_buf[i]
-   void ReduceEndAssemble(::occa::memory &dst, std::size_t item_size) const;
-
-public:
-   OccaConformingProlongation(const FiniteElementSpace &ofes,
-                              const mfem::ParFiniteElementSpace &pfes,
-                              ::occa::memory ltdof_ldof_);
-
-   virtual ~OccaConformingProlongation();
-
-   // overrides
-   virtual void Mult_(const Vector &x, Vector &y) const;
-   virtual void MultTranspose_(const Vector &x, Vector &y) const;
-};
-
-#endif // MFEM_USE_MPI
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_FE_SPACE_HPP

backends/occa/fespace.okl

@@ -1,67 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-/*
-  ---[ Defines Known At Compile-Time ]------------
-  TILESIZE : Tilesize for iterating over entries
-  ================================================
-*/
-
-#if VDIM_ORDERING == ORDERING_BY_VDIM
-typedef double *Global_t @dim(NUM_VDIM, globalEntries);
-typedef double *Local_t  @dim(NUM_VDIM, localEntries);
-#else
-typedef double *Global_t @dim(NUM_VDIM, globalEntries) @dimOrder(1, 0);
-typedef double *Local_t  @dim(NUM_VDIM, localEntries)  @dimOrder(1, 0);
-#endif
-
-@kernel void GlobalToLocal(const int globalEntries,
-                           const int localEntries,
-                           @restrict const int * offsets,
-                           @restrict const int * indices,
-                           @restrict const Global_t globalX,
-                           @restrict Local_t localX) {
-
-  for (int i = 0; i < globalEntries; ++i; @tile(TILESIZE, @outer, @inner)) {
-    if (i < globalEntries) {
-      const int offset = offsets[i];
-      const int nextOffset = offsets[i + 1];
-      for (int v = 0; v < NUM_VDIM; ++v) {
-        const double dofValue = globalX(v, i);
-        for (int j = offset; j < nextOffset; ++j) {
-          localX(v, indices[j]) = dofValue;
-        }
-      }
-    }
-  }
-}
-
-@kernel void LocalToGlobal(const int globalEntries,
-                           const int localEntries,
-                           @restrict const int * offsets,
-                           @restrict const int * indices,
-                           @restrict const Local_t localX,
-                           @restrict Global_t globalX) {
-
-  for (int i = 0; i < globalEntries; ++i; @tile(TILESIZE, @outer, @inner)) {
-    if (i < globalEntries) {
-      const int offset = offsets[i];
-      const int nextOffset = offsets[i + 1];
-      for (int v = 0; v < NUM_VDIM; ++v) {
-        double dofValue = 0;
-        for (int j = offset; j < nextOffset; ++j) {
-          dofValue += localX(v, indices[j]);
-        }
-        globalX(v, i) = dofValue;
-      }
-    }
-  }
-}

backends/occa/geometry.okl

@@ -1,181 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef STORE_JACOBIAN
-#  define STORE_JACOBIAN 1
-#endif
-
-#ifndef STORE_JACOBIAN_INV
-#  define STORE_JACOBIAN_INV 1
-#endif
-
-#ifndef STORE_JACOBIAN_DET
-#  define STORE_JACOBIAN_DET 1
-#endif
-
-typedef double* Local1D_t @dim(1, NUM_DOFS, numElements);
-typedef double* Local2D_t @dim(2, NUM_DOFS, numElements);
-typedef double* Local3D_t @dim(3, NUM_DOFS, numElements);
-
-typedef double* QLocal_t @dim(NUM_QUAD, numElements);
-
-typedef double* DofToQuadD1D_t @dim(NUM_QUAD, NUM_DOFS);
-typedef double* DofToQuadD2D_t @dim(2, NUM_QUAD, NUM_DOFS);
-typedef double* DofToQuadD3D_t @dim(3, NUM_QUAD, NUM_DOFS);
-
-typedef double* Jacobian1D_t @dim(NUM_QUAD, numElements);
-typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD, numElements);
-typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
-
-@kernel void InitGeometryInfo1D(const int numElements,
-                                @restrict const DofToQuadD1D_t dofToQuadD,
-                                @restrict const Local1D_t nodes,
-                                @restrict Jacobian1D_t J,
-                                @restrict Jacobian1D_t invJ,
-                                @restrict QLocal_t detJ) {
-
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_nodes[NUM_DOFS];
-
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
-        s_nodes[d] = nodes(0, d, e);
-      }
-    }
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      double J11 = 0;
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        const double wx = dofToQuadD(q, d);
-        J11 += wx * s_nodes[d];
-      }
-#if STORE_JACOBIAN
-      J(q, e) = J11;
-#endif
-#if STORE_JACOBIAN_INV
-      invJ(q, e) = 1.0 / J11;
-#endif
-#if STORE_JACOBIAN_DET
-      detJ(q, e) = J11;
-#endif
-    }
-  }
-}
-
-@kernel void InitGeometryInfo2D(const int numElements,
-                                @restrict const DofToQuadD2D_t dofToQuadD,
-                                @restrict const Local2D_t nodes,
-                                @restrict Jacobian2D_t J,
-                                @restrict Jacobian2D_t invJ,
-                                @restrict QLocal_t detJ) {
-
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_nodes[2 * NUM_DOFS] @dim(2, NUM_DOFS);
-
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
-        s_nodes(0, d) = nodes(0, d, e);
-        s_nodes(1, d) = nodes(1, d, e);
-      }
-    }
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      double J11 = 0, J12 = 0;
-      double J21 = 0, J22 = 0;
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        const double wx = dofToQuadD(0, q, d);
-        const double wy = dofToQuadD(1, q, d);
-        const double x = s_nodes(0, d);
-        const double y = s_nodes(1, d);
-        J11 += (wx * x); J12 += (wx * y);
-        J21 += (wy * x); J22 += (wy * y);
-      }
-#if STORE_JACOBIAN_INV || STORE_JACOBIAN_DET
-      const double r_detJ = (J11 * J22) - (J12 * J21);
-#endif
-#if STORE_JACOBIAN
-      J(0, 0, q, e) = J11; J(1, 0, q, e) = J12;
-      J(0, 1, q, e) = J21; J(1, 1, q, e) = J22;
-#endif
-#if STORE_JACOBIAN_INV
-      const double r_idetJ = 1.0 / r_detJ;
-      invJ(0, 0, q, e) =  J22 * r_idetJ;
-      invJ(1, 0, q, e) = -J12 * r_idetJ;
-
-      invJ(0, 1, q, e) = -J21 * r_idetJ;
-      invJ(1, 1, q, e) =  J11 * r_idetJ;
-#endif
-#if STORE_JACOBIAN_DET
-      detJ(q, e) = r_detJ;
-#endif
-    }
-  }
-}
-
-@kernel void InitGeometryInfo3D(const int numElements,
-                                @restrict const DofToQuadD3D_t dofToQuadD,
-                                @restrict const Local3D_t nodes,
-                                @restrict Jacobian3D_t J,
-                                @restrict Jacobian3D_t invJ,
-                                @restrict QLocal_t detJ) {
-
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_nodes[3 * NUM_DOFS] @dim(3, NUM_DOFS);
-
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
-        s_nodes(0, d) = nodes(0, d, e);
-        s_nodes(1, d) = nodes(1, d, e);
-        s_nodes(2, d) = nodes(2, d, e);
-      }
-    }
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      double J11 = 0, J12 = 0, J13 = 0;
-      double J21 = 0, J22 = 0, J23 = 0;
-      double J31 = 0, J32 = 0, J33 = 0;
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        const double wx = dofToQuadD(0, q, d);
-        const double wy = dofToQuadD(1, q, d);
-        const double wz = dofToQuadD(2, q, d);
-        const double x = s_nodes(0, d);
-        const double y = s_nodes(1, d);
-        const double z = s_nodes(2, d);
-        J11 += (wx * x); J12 += (wx * y); J13 += (wx * z);
-        J21 += (wy * x); J22 += (wy * y); J23 += (wy * z);
-        J31 += (wz * x); J32 += (wz * y); J33 += (wz * z);
-      }
-#if STORE_JACOBIAN_INV || STORE_JACOBIAN_DET
-      const double r_detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                             (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-#endif
-#if STORE_JACOBIAN
-      J(0, 0, q, e) = J11; J(1, 0, q, e) = J12; J(2, 0, q, e) = J13;
-      J(0, 1, q, e) = J21; J(1, 1, q, e) = J22; J(2, 1, q, e) = J23;
-      J(0, 2, q, e) = J31; J(1, 2, q, e) = J32; J(2, 2, q, e) = J33;
-#endif
-#if STORE_JACOBIAN_INV
-      const double r_idetJ = 1.0 / r_detJ;
-      invJ(0, 0, q, e) = r_idetJ * ((J22 * J33) - (J23 * J32));
-      invJ(1, 0, q, e) = r_idetJ * ((J32 * J13) - (J33 * J12));
-      invJ(2, 0, q, e) = r_idetJ * ((J12 * J23) - (J13 * J22));
-
-      invJ(0, 1, q, e) = r_idetJ * ((J23 * J31) - (J21 * J33));
-      invJ(1, 1, q, e) = r_idetJ * ((J33 * J11) - (J31 * J13));
-      invJ(2, 1, q, e) = r_idetJ * ((J13 * J21) - (J11 * J23));
-
-      invJ(0, 2, q, e) = r_idetJ * ((J21 * J32) - (J22 * J31));
-      invJ(1, 2, q, e) = r_idetJ * ((J31 * J12) - (J32 * J11));
-      invJ(2, 2, q, e) = r_idetJ * ((J11 * J22) - (J12 * J21));
-#endif
-#if STORE_JACOBIAN_DET
-      detJ(q, e) = r_detJ;
-#endif
-    }
-  }
-}

backends/occa/gridfunc.cpp

@@ -1,86 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "gridfunc.hpp"
-#include "bilininteg.hpp"
-#include "../../fem/gridfunc.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-std::map<std::string, ::occa::kernel> gridFunctionKernels;
-
-::occa::kernel GetGridFunctionKernel(::occa::device device,
-                                     FiniteElementSpace &fespace,
-                                     const mfem::IntegrationRule &ir)
-{
-   const int numQuad = ir.GetNPoints();
-
-   const FiniteElement &fe = *(fespace.GetFE(0));
-   const int dim  = fe.GetDim();
-   const int vdim = fespace.GetVDim();
-
-   std::stringstream ss;
-   ss << ::occa::hash(device)
-      << "FEColl : " << fespace.FEColl()->Name()
-      << "Quad: "    << numQuad
-      << "Dim: "     << dim
-      << "VDim: "    << vdim;
-   std::string hash = ss.str();
-
-   // Kernel defines
-   ::occa::properties props;
-   props["defines/NUM_VDIM"] = vdim;
-
-   SetProperties(fespace, ir, props);
-
-   ::occa::kernel kernel = gridFunctionKernels[hash];
-   if (!kernel.isInitialized())
-   {
-      const std::string &okl_path = fespace.OccaEngine().GetOklPath();
-      kernel = device.buildKernel(okl_path + "gridfunc.okl",
-                                  stringWithDim("GridFuncToQuad", dim),
-                                  props);
-   }
-   return kernel;
-}
-
-void ToQuad(const IntegrationRule &ir, FiniteElementSpace &fespace, Vector &gf,
-            Vector &quadValues)
-{
-   const Engine &engine = fespace.OccaEngine();
-   ::occa::device device = engine.GetDevice();
-
-   OccaDofQuadMaps &maps = OccaDofQuadMaps::Get(device, fespace, ir);
-
-   const int elements = fespace.GetNE();
-   const int numQuad  = ir.GetNPoints();
-   quadValues.OccaResize(numQuad * elements, sizeof(double));
-
-   ::occa::kernel g2qKernel = GetGridFunctionKernel(device, fespace, ir);
-   g2qKernel(elements,
-             maps.dofToQuad,
-             fespace.GetLocalToGlobalMap(),
-             gf.OccaMem(),
-             quadValues.OccaMem());
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/gridfunc.hpp

@@ -1,55 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_GRID_FUNC_HPP
-#define MFEM_BACKENDS_OCCA_GRID_FUNC_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "vector.hpp"
-#include "fespace.hpp"
-
-namespace mfem
-{
-
-class IntegrationRule;
-
-namespace occa
-{
-
-// TODO: make this object part of the backend or the engine.
-extern std::map<std::string, ::occa::kernel> gridFunctionKernels;
-
-// TODO: make this a method of the backend or the engine.
-::occa::kernel GetGridFunctionKernel(::occa::device device,
-                                     FiniteElementSpace &fespace,
-                                     const mfem::IntegrationRule &ir);
-
-// ToQuad version without the deprecated class.
-//
-// FIXME: This is the action of a global B matrix, mapping L-vector to Q-vector,
-//        so it should be made into an operator that can be constructed by the
-//        FE space class. A batched version, where only a subset of the elements
-//        are processed should be defined as well.
-//
-//        The abstract operator construction method in the FE space class is:
-//           PFiniteElementSpace::GetInterpolationOperator(...)
-void ToQuad(const IntegrationRule &ir, FiniteElementSpace &ofespace, Vector &gf,
-            Vector &quadValues);
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_GRID_FUNC_HPP

backends/occa/gridfunc.okl

@@ -1,26 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-#ifdef USING_TENSOR_OPS
-#  ifdef OCCA_USING_CPU
-#    include "mfem-occa://gridfunc/tensor/cpu.okl"
-#  else
-#    include "mfem-occa://gridfunc/tensor/gpuHighOrder.okl"
-#  endif
-#else
-#  ifdef OCCA_USING_CPU
-#    include "mfem-occa://gridfunc/simplex/cpu.okl"
-#  else
-#    include "mfem-occa://gridfunc/simplex/gpuHighOrder.okl"
-#  endif
-#endif

backends/occa/gridfunc/simplex/cpu.okl

@@ -1,63 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 2D ]-----------------------------
-@kernel void GridFuncToQuad2D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QVLocal_t out) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        const int gid = l2gMap(d, e);
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          const double r_gf = gf[v + gid*NUM_VDIM];
-          double r_out = 0;
-          for (int q = 0; q < NUM_QUAD; ++q) {
-            r_out += r_gf * dofToQuad(d, q);
-          }
-          out(v, d, e) = r_out;
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void GridFuncToQuad3D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QVLocal_t out) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        const int gid = l2gMap(d, e);
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          const double r_gf = gf[v + gid*NUM_VDIM];
-          double r_out = 0;
-          for (int q = 0; q < NUM_QUAD; ++q) {
-            r_out += r_gf * dofToQuad(d, q);
-          }
-          out(v, d, e) = r_out;
-        }
-      }
-    }
-  }
-}
-//======================================

backends/occa/gridfunc/simplex/gpuHighOrder.okl

@@ -1,79 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 2D ]-----------------------------
-@kernel void GridFuncToQuad2D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QVLocal_t out) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_gf[NUM_VDIM][NUM_DOFS];
-
-    for (int dOff = 0; dOff < M2_INNER_BATCH; ++dOff; @inner) {
-      for (int d = dOff; d < NUM_DOFS; d += M2_INNER_BATCH) {
-        const int gid = l2gMap(d, e);
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          s_gf[v][d] = gf[v + gid*NUM_VDIM]];
-        }
-      }
-    }
-
-    for (int qOff = 0; qOff < M2_INNER_BATCH; ++qOff; @inner) {
-      for (int q = qOff; q < NUM_QUAD; q += M2_INNER_BATCH) {
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          double r_out = 0;
-          for (int d = 0; d < NUM_DOFS; ++d) {
-            r_out += s_gf[v][d] * dofToQuad(d, q);
-          }
-          out(v, q, e) = r_out;
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void GridFuncToQuad3D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QVLocal_t out) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_gf[NUM_VDIM][NUM_DOFS];
-
-    for (int dOff = 0; dOff < M3_INNER_BATCH; ++dOff; @inner) {
-      for (int d = dOff; d < NUM_DOFS; d += M3_INNER_BATCH) {
-        const int gid = l2gMap(d, e);
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          s_gf[v][d] = gf[v + gid*NUM_VDIM]];
-        }
-      }
-    }
-
-    for (int qOff = 0; qOff < M3_INNER_BATCH; ++qOff; @inner) {
-      for (int q = qOff; q < NUM_QUAD; q += M3_INNER_BATCH) {
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          double r_out = 0;
-          for (int d = 0; d < NUM_DOFS; ++d) {
-            r_out += s_gf[v][d] * dofToQuad(d, q);
-          }
-          out(v, q, e) = r_out;
-        }
-      }
-    }
-  }
-}
-//======================================

backends/occa/gridfunc/tensor/cpu.okl

@@ -1,188 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 1D ]-----------------------------
-@kernel void GridFuncToQuad1D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap1D_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QVLocal1D_t out) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double r_out[NUM_VDIM][NUM_QUAD_1D];
-      for (int v = 0; v < NUM_VDIM; ++v) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          r_out[v][qx] = 0;
-        }
-      }
-      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-        const int gid = l2gMap(dx, e);
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          const double r_gf = gf[v + gid*NUM_VDIM];
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            r_out[v][qx] += r_gf * dofToQuad(qx, dx);
-          }
-        }
-      }
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          out(v, qx, e) = r_out[v][qx];
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 2D ]-----------------------------
-@kernel void GridFuncToQuad2D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap2D_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QVLocal2D_t out) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double out_xy[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D];
-      for (int v = 0; v < NUM_VDIM; ++v) {
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            out_xy[v][qy][qx] = 0;
-          }
-        }
-      }
-
-      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-        double out_x[NUM_VDIM][NUM_QUAD_1D];
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            out_x[v][qy] = 0;
-          }
-        }
-
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          const int gid = l2gMap(dx, dy, e);
-          for (int v = 0; v < NUM_VDIM; ++v) {
-            const double r_gf = gf[v + gid*NUM_VDIM];
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              out_x[v][qy] += r_gf * dofToQuad(qy, dx);
-            }
-          }
-        }
-
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            const double d2q = dofToQuad(qy, dy);
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              out_xy[v][qy][qx] += d2q * out_x[v][qx];
-            }
-          }
-        }
-      }
-
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          for (int v = 0; v < NUM_VDIM; ++v) {
-            out(v, qx, qy, e) = out_xy[v][qy][qx];
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void GridFuncToQuad3D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap3D_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QVLocal3D_t out) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double out_xyz[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D];
-      for (int v = 0; v < NUM_VDIM; ++v) {
-        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              out_xyz[v][qz][qy][qx] = 0;
-            }
-          }
-        }
-      }
-
-      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-        double out_xy[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D];
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              out_xy[v][qy][qx] = 0;
-            }
-          }
-        }
-
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          double out_x[NUM_VDIM][NUM_QUAD_1D];
-          for (int v = 0; v < NUM_VDIM; ++v) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              out_x[v][qx] = 0;
-            }
-          }
-
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            const int gid = l2gMap(dx, dy, dz, e);
-            for (int v = 0; v < NUM_VDIM; ++v) {
-              const double r_gf = gf[v + gid*NUM_VDIM];
-              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-                out_x[v][qx] += r_gf * dofToQuad(qx, dx);
-              }
-            }
-          }
-
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            const double wy = dofToQuad(qy, dy);
-            for (int v = 0; v < NUM_VDIM; ++v) {
-              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-                out_xy[v][qy][qx] += wy * out_x[v][qx];
-              }
-            }
-          }
-        }
-
-        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-          const double wz = dofToQuad(qz, dz);
-          for (int v = 0; v < NUM_VDIM; ++v) {
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-                out_xyz[v][qz][qy][qx] += wz * out_xy[v][qy][qx];
-              }
-            }
-          }
-        }
-      }
-
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            for (int v = 0; v < NUM_VDIM; ++v) {
-              out(v, qx, qy, qz, e) = out_xyz[v][qz][qy][qx];
-            }
-          }
-        }
-      }
-    }
-  }
-}
-//======================================

backends/occa/gridfunc/tensor/gpuHighOrder.okl

@@ -1,183 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 1D ]-----------------------------
-@kernel void GridFuncToQuad1D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap1D_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QLocal1D_t out) {
-  // Iterate over elements
-  for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
-    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @exclusive double r_out[NUM_QUAD_1D];
-
-    for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
-      for (int i = el; i < NUM_QUAD_DOFS_1D; i += M1_INNER_ELEMENT_BATCH) {
-        s_dofToQuad[i] = dofToQuad[i];
-      }
-    }
-
-    for (int b = 0; b < M1_OUTER_ELEMENT_BATCH; ++b) {
-      for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
-        const int e = eOff + b*M1_INNER_ELEMENT_BATCH + el;
-        if (e < numElements) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            r_out[qx] = 0;
-          }
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            const double r_gf = gf[l2gMap(dx, e)];
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              r_out[qx] += r_gf * s_dofToQuad(qx, dx);
-            }
-          }
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            out(qx, e) = r_out[qx];
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 2D ]-----------------------------
-@kernel void GridFuncToQuad2D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap2D_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QLocal2D_t out) {
-  // Iterate over elements
-  for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
-    // Store dof <--> quad mappings
-    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-
-    // Store xy planes in shared memory
-    @shared double s_xy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-
-    for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
-      for (int id = x; id < NUM_QUAD_DOFS_1D; id += NUM_MAX_1D) {
-        s_dofToQuad[id] = dofToQuad[id];
-      }
-    }
-
-    for (int e = eOff; e < (eOff + M2_ELEMENT_BATCH); ++e) {
-      if (e < numElements) {
-        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-          if (dx < NUM_DOFS_1D) {
-            double r_x[NUM_DOFS_1D];
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              r_x[dy] = gf[l2gMap(dx, dy, e)];
-            }
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              double xy = 0;
-              for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-                xy += r_x[dy] * s_dofToQuad(qy, dy);
-              }
-              s_xy(dx, qy) = xy;
-            }
-          }
-        }
-        for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
-          if (qy < NUM_QUAD_1D) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              double val = 0;
-              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-                val += s_xy(dx, qy) * s_dofToQuad(qx, dx);
-              }
-              out(qx, qy, e) = val;
-            }
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void GridFuncToQuad3D(const int numElements,
-                              @restrict const DofToQuad_t dofToQuad,
-                              @restrict const DLocalMap3D_t l2gMap,
-                              @restrict const double * gf,
-                              @restrict QLocal3D_t out) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    // Store dof <--> quad mappings
-    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-
-    // Store xy planes in shared memory
-    @shared double s_z[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
-
-    // Store z axis as registers
-    @exclusive double r_qz[NUM_QUAD_1D];
-
-    for (int y = 0; y < NUM_MAX_1D; ++y; @inner) {
-      for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
-        const int id = (y * NUM_MAX_1D) + x;
-        // Fetch Q <--> D maps
-        if (id < NUM_QUAD_DOFS_1D) {
-          s_dofToQuad[id] = dofToQuad[id];
-        }
-        // Initialize our Z axis
-        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-          r_qz[qz] = 0;
-        }
-      }
-    }
-
-    for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
-      for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-        if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
-          for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-            const double val = gf[l2gMap(dx, dy, dz, e)];
-            // Calculate D -> Q in the Z axis
-            for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-              r_qz[qz] += val * s_dofToQuad(qz, dz);
-            }
-          }
-        }
-      }
-    }
-    // For each xy plane
-    for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-      // Fill xy plane at given z position
-      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
-        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
-            s_z(dx, dy) = r_qz[qz];
-          }
-        }
-      }
-      // Calculate Dxyz, xDyz, xyDz in plane
-      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
-        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
-          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
-            double val = 0;
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              const double wy = s_dofToQuad(qy, dy);
-              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-                const double wx = s_dofToQuad(qx, dx);
-                val += wx * wy * s_z(dx, dy);
-              }
-            }
-            out(qx, qy, qz, e) = val;
-          }
-        }
-      }
-    }
-  }
-}
-//======================================

backends/occa/interpolation.cpp

@@ -1,119 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "interpolation.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-RestrictionOperator::RestrictionOperator(Layout &in_layout, Layout &out_layout,
-                                         ::occa::array<int> indices)
-   : Operator(in_layout, out_layout)
-{
-   trueIndices = indices;
-
-   ::occa::device device = in_layout.OccaEngine().GetDevice();
-   const std::string &okl_path = in_layout.OccaEngine().GetOklPath();
-   multOp = device.buildKernel(okl_path + "mappings.okl",
-                               "ExtractSubVector",
-                               "defines: { TILESIZE: 256 }");
-
-   multTransposeOp = device.buildKernel(okl_path + "mappings.okl",
-                                        "SetSubVector",
-                                        "defines: { TILESIZE: 256 }");
-}
-
-void RestrictionOperator::Mult_(const Vector &x, Vector &y) const
-{
-   // y[i] = x[trueIndices[i]]
-   multOp(height, trueIndices, x.OccaMem(), y.OccaMem());
-}
-
-void RestrictionOperator::MultTranspose_(const Vector &x, Vector &y) const
-{
-   y.OccaFill<double>(0.0);
-   // y[trueIndices[i]] = x[i]
-   multTransposeOp(height, trueIndices, x.OccaMem(), y.OccaMem());
-}
-
-ProlongationOperator::ProlongationOperator(OccaSparseMatrix &multOp_,
-                                           OccaSparseMatrix &multTransposeOp_)
-   : Operator(multOp_),
-     pmat(NULL),
-     multOp(multOp_),
-     multTransposeOp(multTransposeOp_)
-{ }
-
-ProlongationOperator::ProlongationOperator(Layout &in_layout,
-                                           Layout &out_layout,
-                                           const mfem::Operator *pmat_)
-   : Operator(in_layout, out_layout),
-     pmat(pmat_),
-     multOp(*this),
-     multTransposeOp(*this)
-{ }
-
-void ProlongationOperator::Mult_(const Vector &x, Vector &y) const
-{
-   MFEM_VERIFY(pmat == NULL, "");
-   multOp.Mult_(x, y);
-}
-
-void ProlongationOperator::MultTranspose_(const Vector &x, Vector &y) const
-{
-   MFEM_VERIFY(pmat == NULL, "");
-   multTransposeOp.Mult_(x, y);
-}
-
-void ProlongationOperator::Mult(const mfem::Vector &x, mfem::Vector &y) const
-{
-   if (pmat)
-   {
-      // FIXME: create an OCCA version of 'pmat'
-      x.Pull();
-      y.Pull(false);
-      pmat->Mult(x, y);
-      y.Push();
-   }
-   else
-   {
-      multOp.Mult(x, y);
-   }
-}
-
-void ProlongationOperator::MultTranspose(const mfem::Vector &x,
-                                         mfem::Vector &y) const
-{
-   if (pmat)
-   {
-      // FIXME: create an OCCA version of 'pmat'
-      x.Pull();
-      y.Pull(false);
-      pmat->MultTranspose(x, y);
-      y.Push();
-   }
-   else
-   {
-      multTransposeOp.Mult(x, y);
-   }
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/interpolation.hpp

@@ -1,73 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_INTERPOLATION_HPP
-#define MFEM_BACKENDS_OCCA_INTERPOLATION_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include <occa.hpp>
-#include "vector.hpp"
-#include "engine.hpp"
-#include "sparsemat.hpp"
-#include "../../fem/fem.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class RestrictionOperator : public Operator
-{
-protected:
-   ::occa::array<int> trueIndices; // ldof = trueIndices[ltdof]
-   ::occa::kernel multOp, multTransposeOp;
-
-public:
-   RestrictionOperator(Layout &in_layout, Layout &out_layout,
-                       ::occa::array<int> indices);
-
-   // overrides
-   virtual void Mult_(const Vector &x, Vector &y) const;
-   virtual void MultTranspose_(const Vector &x, Vector &y) const;
-};
-
-class ProlongationOperator : public Operator
-{
-protected:
-   const mfem::Operator *pmat;
-   OccaSparseMatrix multOp, multTransposeOp;
-
-public:
-   ProlongationOperator(OccaSparseMatrix &multOp_,
-                        OccaSparseMatrix &multTransposeOp_);
-
-   ProlongationOperator(Layout &in_layout, Layout &out_layout,
-                        const mfem::Operator *pmat_);
-
-   // overrides
-   virtual void Mult_(const Vector &x, Vector &y) const;
-   virtual void MultTranspose_(const Vector &x, Vector &y) const;
-
-   // overrides
-   virtual void Mult(const mfem::Vector &x, mfem::Vector &y) const;
-   virtual void MultTranspose(const mfem::Vector &x, mfem::Vector &y) const;
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_INTERPOLATION_HPP

backends/occa/layout.cpp

@@ -1,40 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "layout.hpp"
-#include "../../general/array.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-void Layout::Resize(std::size_t new_size)
-{
-   size = new_size;
-}
-
-void Layout::Resize(const Array<std::size_t> &offsets)
-{
-   MFEM_ASSERT(offsets.Size() == 2,
-               "multiple workers are not supported yet");
-   size = offsets.Last();
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/layout.hpp

@@ -1,67 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_LAYOUT_HPP
-#define MFEM_BACKENDS_OCCA_LAYOUT_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "../base/layout.hpp"
-#include "engine.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class Layout : public PLayout
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // SharedPtr<const mfem::Engine> engine;
-   // std::size_t size;
-
-public:
-   Layout(const Engine &e, std::size_t s = 0) : PLayout(e, s) { }
-
-   const Engine &OccaEngine() const
-   { return *static_cast<const Engine *>(engine.Get()); }
-
-   void OccaResize(std::size_t new_size) { size = new_size; }
-
-   virtual ~Layout() { }
-
-   /**
-       @name Virtual interface
-    */
-   ///@{
-
-   /// Resize the layout
-   virtual void Resize(std::size_t new_size);
-
-   /// Resize the layout based on the given worker offsets
-   virtual void Resize(const Array<std::size_t> &offsets);
-
-   ///@}
-   // End: Virtual interface
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_LAYOUT_HPP

backends/occa/mappings.okl

@@ -1,76 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-/*
-  ---[ Defines Known At Compile-Time ]------------
-  TILESIZE : Tilesize for iterating over entries
-  ================================================
-*/
-
-@kernel void ExtractSubVector(const int entries,
-                              @restrict const int *indices,
-                              @restrict const double *in,
-                              @restrict double *out) {
-
-  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
-    if (i < entries) {
-      out[i] = in[indices[i]]; // indices can be repeated
-    }
-  }
-}
-
-@kernel void SetSubVector(const int entries,
-                          @restrict const int *indices,
-                          @restrict const double *in,
-                          @restrict double *out) {
-
-  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
-    if (i < entries) {
-      out[indices[i]] = in[i]; // indices CANNOT be repeated
-    }
-  }
-}
-
-@kernel void AddSubVector(const int num_unique_dst_indices,
-                          @restrict const int *unique_dst_indices,
-                          @restrict const int *unique_to_src_offsets,
-                          @restrict const int *unique_to_src_indices,
-                          @restrict const double *src,
-                          @restrict double *dst) {
-
-  for (int i = 0; i < num_unique_dst_indices; ++i;
-       @tile(TILESIZE, @outer, @inner)) {
-
-    if (i < num_unique_dst_indices) {
-      const int dst_idx = unique_dst_indices[i];
-      double sum = dst[dst_idx];
-      const int end = unique_to_src_offsets[i+1];
-      for (int j = unique_to_src_offsets[i]; j != end; ++j) {
-        sum += src[unique_to_src_indices[j]];
-      }
-      dst[dst_idx] = sum;
-    }
-  }
-}
-
-@kernel void MapSubVector(const int entries,
-                          @restrict const int *indices,
-                          @restrict const double *in,
-                          @restrict double *out) {
-
-  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
-    if (i < entries) {
-      const int fromIdx = indices[2*i + 0]; // fromIdx indices can be repeated
-      const int toIdx   = indices[2*i + 1]; // toIdx indices CANNOT be repeated
-      out[toIdx] = in[fromIdx];
-    }
-  }
-}

backends/occa/mass/simplex/cpu.okl

@@ -1,122 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian2D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-
-      oper(q, e) = quadWeights[q] * COEFF * ((J11 * J22) - (J21 * J12));
-    }
-  }
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuadD2D_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDofD2D_t quadToDofD,
-                       @restrict const QLocal_t oper,
-                       @restrict const DLocal_t solIn,
-                       @restrict DLocal_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double r_sol[NUM_DOFS];
-
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        r_sol[d] = 0;
-      }
-
-      for (int q = 0; q < NUM_QUAD; ++q) {
-        double s = 0;
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          s += solIn(d, e) * quadToDof(d, q);
-        }
-        s *= oper(q, e);
-
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          r_sol[d] += (s * quadToDof(d, q));
-        }
-      }
-
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        solOut(d, e) += r_sol[d];
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian3D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-      oper(q, e) = quadWeights[q] * COEFF * detJ;
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuadD3D_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDofD3D_t quadToDofD,
-                       @restrict const QLocal_t oper,
-                       @restrict const DLocal_t solIn,
-                       @restrict DLocal_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double r_sol[NUM_DOFS];
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        r_sol[d] = 0;
-      }
-
-      for (int q = 0; q < NUM_QUAD; ++q) {
-        double s = 0;
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          s += solIn(d, e) * quadToDof(d, q);
-        }
-        s *= oper(q, e);
-
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          r_sol[d] += (s * quadToDof(d, q));
-        }
-      }
-
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        solOut(d, e) += r_sol[d];
-      }
-    }
-  }
-}
-//======================================

backends/occa/mass/simplex/gpuHighOrder.okl

@@ -1,129 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian2D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
-          for (int q = qOff; q < NUM_QUAD; q += A2_QUAD_BATCH) {
-            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-
-            oper(q, e) = quadWeights[q] * COEFF * ((J11 * J22) - (J21 * J12));
-          }
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuadD2D_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDofD2D_t quadToDofD,
-                       @restrict const QLocal_t oper,
-                       @restrict const DLocal_t solIn,
-                       @restrict DLocal_t solOut) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_sol[NUM_QUAD];
-
-    for (int qOff = 0; qOff < M2_INNER_BATCH; ++qOff; @inner) {
-      for (int q = qOff; q < NUM_QUAD; q += M2_INNER_BATCH) {
-        double s = 0;
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          s += solIn(d, e) * dofToQuad(d, q);
-        }
-        s_sol[q] = s * oper(q, e);
-      }
-    }
-
-    for (int dOff = 0; dOff < M2_INNER_BATCH; ++dOff) {
-      for (int d = dOff; d < NUM_DOFS; d += M2_INNER_BATCH) {
-        double r_sol = 0;
-        for (int q = 0; q < NUM_QUAD; ++q) {
-          r_sol += (s_sol[q] * quadToDof(d, q));
-        }
-        solOut(d, e) += r_sol;
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian3D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
-          for (int q = qOff; q < NUM_QUAD; q += A3_QUAD_BATCH) {
-            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-            oper(q, e) = quadWeights[q] * COEFF * detJ;
-          }
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuadD3D_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDofD3D_t quadToDofD,
-                       @restrict const QLocal_t oper,
-                       @restrict const DLocal_t solIn,
-                       @restrict DLocal_t solOut) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_sol[NUM_QUAD];
-
-    for (int qOff = 0; qOff < M3_INNER_BATCH; ++qOff; @inner) {
-      for (int q = qOff; q < NUM_QUAD; q += M3_INNER_BATCH) {
-        double s = 0;
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          s += solIn(d, e) * dofToQuad(d, q);
-        }
-        s_sol[q] = s * oper(q, e);
-      }
-    }
-
-    for (int dOff = 0; dOff < M3_INNER_BATCH; ++dOff) {
-      for (int d = dOff; d < NUM_DOFS; d += M3_INNER_BATCH) {
-        double r_sol = 0;
-        for (int q = 0; q < NUM_QUAD; ++q) {
-          r_sol += (s_sol[q] * quadToDof(d, q));
-        }
-        solOut(d, e) += r_sol;
-      }
-    }
-  }
-}
-//======================================

backends/occa/mass/tensor/cpu.okl

@@ -1,281 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 1D ]-----------------------------
-@kernel void Assemble1D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian1D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
-      const double detJ = J(q, e);
-      oper(q, e) = quadWeights[q] * COEFF * detJ;
-    }
-  }
-}
-
-@kernel void MultAdd1D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal1D_t oper,
-                       @restrict const DLocal1D_t solIn,
-                       @restrict DLocal1D_t solOut) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double sol_x[NUM_QUAD_1D];
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        sol_x[qx] = 0;
-      }
-      // sol_x{qx} = dofToQuad{qx,dx} * sol{dx}
-      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-        const double s = solIn(dx, e);
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          sol_x[qx] += s * dofToQuad(qx, dx);
-        }
-      }
-      // sol_x{q} *= oper{q}
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        sol_x[qx] *= oper(qx, e);
-      }
-      // sol{dx} = quadToDof{dx,qx} * sol_x{qx}
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          solOut(dx, e) += sol_x[qx] * quadToDof(dx, qx);
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian2D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_2D; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-      const double detJ = ((J11 * J22) - (J21 * J12));
-
-      oper(q, e) = quadWeights[q] * COEFF * detJ;
-    }
-  }
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal2D_t oper,
-                       @restrict const DLocal2D_t solIn,
-                       @restrict DLocal2D_t solOut) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double sol_xy[NUM_QUAD_1D][NUM_QUAD_1D];
-
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          sol_xy[qy][qx] = 0;
-        }
-      }
-
-      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-        double sol_x[NUM_QUAD_1D];
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          sol_x[qy] = 0;
-        }
-
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          const double s = solIn(dx, dy, e);
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_x[qx] += dofToQuad(qx, dx) * s;
-          }
-        }
-
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          const double d2q = dofToQuad(qy, dy);
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_xy[qy][qx] += d2q * sol_x[qx];
-          }
-        }
-      }
-
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          sol_xy[qy][qx] *= oper(qx, qy, e);
-        }
-      }
-
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        double sol_x[NUM_DOFS_1D];
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          sol_x[dx] = 0;
-        }
-
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          const double s = sol_xy[qy][qx];
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            sol_x[dx] += quadToDof(dx, qx) * s;
-          }
-        }
-
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          const double q2d = quadToDof(dy, qy);
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            solOut(dx, dy, e) += q2d * sol_x[dx];
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian3D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_3D; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-      oper(q, e) = quadWeights[q] * COEFF * detJ;
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal3D_t oper,
-                       @restrict const DLocal3D_t solIn,
-                       @restrict DLocal3D_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double sol_xyz[NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D];
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_xyz[qz][qy][qx] = 0;
-          }
-        }
-      }
-
-      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-        double sol_xy[NUM_QUAD_1D][NUM_QUAD_1D];
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_xy[qy][qx] = 0;
-          }
-        }
-
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          double sol_x[NUM_QUAD_1D];
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_x[qx] = 0;
-          }
-
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            const double s = solIn(dx, dy, dz, e);
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              sol_x[qx] += dofToQuad(qx, dx) * s;
-            }
-          }
-
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            const double wy = dofToQuad(qy, dy);
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              sol_xy[qy][qx] += wy * sol_x[qx];
-            }
-          }
-        }
-
-        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-          const double wz = dofToQuad(qz, dz);
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
-            }
-          }
-        }
-      }
-
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_xyz[qz][qy][qx] *= oper(qx, qy, qz, e);
-          }
-        }
-      }
-
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        double sol_xy[NUM_DOFS_1D][NUM_DOFS_1D];
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            sol_xy[dy][dx] = 0;
-          }
-        }
-
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          double sol_x[NUM_DOFS_1D];
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            sol_x[dx] = 0;
-          }
-
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            const double s = sol_xyz[qz][qy][qx];
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              sol_x[dx] += quadToDof(dx, qx) * s;
-            }
-          }
-
-          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-            const double wy = quadToDof(dy, qy);
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              sol_xy[dy][dx] += wy * sol_x[dx];
-            }
-          }
-        }
-
-        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-          const double wz = quadToDof(dz, qz);
-          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              solOut(dx, dy, dz, e) += wz * sol_xy[dy][dx];
-            }
-          }
-        }
-      }
-    }
-  }
-}
-//======================================

backends/occa/mass/tensor/gpuHighOrder.okl

@@ -1,341 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 1D ]-----------------------------
-@kernel void Assemble1D(const int numElements,
-                        const double *quadWeights,
-                        const Jacobian1D_t J,
-                        COEFF_ARGS
-                        QLocal_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A1_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A1_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
-          oper(q, e) = quadWeights[q] * COEFF * J(q, e);
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd1D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal1D_t oper,
-                       @restrict const DLocal1D_t solIn,
-                       @restrict DLocal1D_t solOut) {
-  // Iterate over elements
-  for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
-    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-
-    @exclusive double r_sol[NUM_QUAD_1D];
-
-    for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
-      for (int i = el; i < NUM_QUAD_DOFS_1D; i += M1_INNER_ELEMENT_BATCH) {
-        s_dofToQuad[i] = dofToQuad[i];
-        s_quadToDof[i] = quadToDof[i];
-      }
-    }
-
-    for (int b = 0; b < M1_OUTER_ELEMENT_BATCH; ++b) {
-      for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
-        const int e = eOff + b*M1_INNER_ELEMENT_BATCH + el;
-        if (e < numElements) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            r_sol[qx] = 0;
-          }
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            const double s = solIn(dx, e);
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              r_sol[qx] += s * s_dofToQuad(qx, dx);
-            }
-          }
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            r_sol[qx] *= oper(qx, e);
-          }
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            double s = 0;
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              s += r_sol[qx] * s_quadToDof(dx, qx);
-            }
-            solOut(dx, e) += s;
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        const double *quadWeights,
-                        const Jacobian2D_t J,
-                        COEFF_ARGS
-                        QLocal_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
-          for (int q = qOff; q < NUM_QUAD_2D; q += A2_QUAD_BATCH) {
-            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-
-            oper(q, e) = quadWeights[q] * COEFF * ((J11 * J22) - (J21 * J12));
-          }
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal2D_t oper,
-                       @restrict const DLocal2D_t solIn,
-                       @restrict DLocal2D_t solOut) {
-  // Iterate over elements
-  for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
-    // Store dof <--> quad mappings
-    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-
-    // Store xy planes in @shared memory
-    @shared double s_xy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-    @shared double s_xy2[NUM_QUAD_2D] @dim(NUM_QUAD_1D, NUM_QUAD_1D);
-
-    @exclusive double r_x[NUM_MAX_1D];
-
-    for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
-      for (int id = x; id < NUM_QUAD_DOFS_1D; id += NUM_MAX_1D) {
-        s_dofToQuad[id]  = dofToQuad[id];
-        s_quadToDof[id]  = quadToDof[id];
-      }
-    }
-
-    for (int e = eOff; e < (eOff + M2_ELEMENT_BATCH); ++e) {
-      if (e < numElements) {
-        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-          if (dx < NUM_DOFS_1D) {
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              s_xy(dx, qy) = 0;
-            }
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              r_x[dy] = solIn(dx, dy, e);
-            }
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              double xy = 0;
-              for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-                xy += r_x[dy] * s_dofToQuad(qy, dy);
-              }
-              s_xy(dx, qy) = xy;
-            }
-          }
-        }
-        for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
-          if (qy < NUM_QUAD_1D) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              double s = 0;
-              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-                s += s_xy(dx, qy) * s_dofToQuad(qx, dx);
-              }
-              s_xy2(qx, qy) = s * oper(qx, qy, e);
-            }
-          }
-        }
-
-        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
-          if (qx < NUM_QUAD_1D) {
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              s_xy(dy, qx) = 0;
-            }
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              r_x[qy] = s_xy2(qx, qy);
-            }
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              double s = 0;
-              for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-                s += r_x[qy] * s_quadToDof(dy, qy);
-              }
-              s_xy(dy, qx) = s;
-            }
-          }
-        }
-        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-          if (dx < NUM_DOFS_1D) {
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              double s = 0;
-              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-                s += (s_xy(dy, qx) * s_quadToDof(dx, qx));
-              }
-              solOut(dx, dy, e) += s;
-            }
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        const double *quadWeights,
-                        const Jacobian3D_t J,
-                        COEFF_ARGS
-                        QLocal_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
-          for (int q = qOff; q < NUM_QUAD_3D; q += A3_QUAD_BATCH) {
-            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-            oper(q, e) = quadWeights[q] * COEFF * detJ;
-          }
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal3D_t oper,
-                       @restrict const DLocal3D_t solIn,
-                       @restrict DLocal3D_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    // Store dof <--> quad mappings
-    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-
-    // Store xy planes in @shared memory
-    @shared double s_xy[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
-
-    // Store z axis as registers
-    @exclusive double r_z[NUM_QUAD_1D];
-    @exclusive double r_z2[NUM_DOFS_1D];
-
-    for (int y = 0; y < NUM_MAX_1D; ++y; @inner) {
-      for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
-        const int id = (y * NUM_MAX_1D) + x;
-        // Fetch Q <--> D maps
-        if (id < NUM_QUAD_DOFS_1D) {
-          s_dofToQuad[id]  = dofToQuad[id];
-          s_quadToDof[id]  = quadToDof[id];
-        }
-        // Initialize our Z axis
-        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-          r_z[qz] = 0;
-        }
-        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-          r_z2[dz] = 0;
-        }
-      }
-    }
-
-    for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
-      for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-        if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
-          for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-            const double s = solIn(dx, dy, dz, e);
-            // Calculate D -> Q in the Z axis
-            for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-              r_z[qz] += s * s_dofToQuad(qz, dz);
-            }
-          }
-        }
-      }
-    }
-    // For each xy plane
-    for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-      // Fill xy plane at given z position
-      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
-        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
-            s_xy(dx, dy) = r_z[qz];
-          }
-        }
-      }
-      // Calculate Dxyz, xDyz, xyDz in plane
-      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
-        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
-          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
-            double s = 0;
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              const double wy = s_dofToQuad(qy, dy);
-              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-                const double wx = s_dofToQuad(qx, dx);
-                s += wx * wy * s_xy(dx, dy);
-              }
-            }
-
-            s *= oper(qx, qy, qz, e);
-
-            for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-              const double wz  = s_quadToDof(dz, qz);
-              r_z2[dz] += wz * s;
-            }
-          }
-        }
-      }
-      @barrier("s_xy_sync_1");
-    }
-    // Iterate over xy planes to compute solution
-    for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-      // Place xy plane in @shared memory
-      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
-        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
-          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
-            s_xy(qx, qy) = r_z2[dz];
-          }
-        }
-      }
-      // Finalize solution in xy plane
-      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
-        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
-            double solZ = 0;
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              const double wy = s_quadToDof(dy, qy);
-              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-                const double wx = s_quadToDof(dx, qx);
-                solZ += wx * wy * s_xy(qx, qy);
-              }
-            }
-            solOut(dx, dy, dz, e) += solZ;
-          }
-        }
-      }
-      @barrier("s_xy_sync_2");
-    }
-  }
-}
-//======================================

backends/occa/operator.cpp

@@ -1,142 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "operator.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-// FIXME: move this object to the Backend?
-::occa::kernelBuilder OccaConstrainedOperator::mapDofBuilder =
-   ::occa::linalg::customLinearMethod(
-      "vector_map_dofs",
-
-      "const int idx = v2[i];"
-      "v0[idx] = v1[idx];",
-
-      "defines: {"
-      "  VTYPE0: 'double',"
-      "  VTYPE1: 'double',"
-      "  VTYPE2: 'int',"
-      "  TILESIZE: 128,"
-      "}");
-
-// FIXME: move this object to the Backend?
-::occa::kernelBuilder OccaConstrainedOperator::clearDofBuilder =
-   ::occa::linalg::customLinearMethod(
-      "vector_clear_dofs",
-
-      "v0[v1[i]] = 0.0;",
-
-      "defines: {"
-      "  VTYPE0: 'double',"
-      "  VTYPE1: 'int',"
-      "  TILESIZE: 128,"
-      "}");
-
-OccaConstrainedOperator::OccaConstrainedOperator(
-   mfem::Operator *A_,
-   const mfem::Array<int> &constraintList_,
-   bool own_A_)
-
-   : Operator(A_->InLayout()->As<Layout>()),
-     z(OutLayout_()),
-     w(OutLayout_()),
-     mfem_z((z.DontDelete(), z)),
-     mfem_w((w.DontDelete(), w))
-{
-   Setup(OutLayout_().OccaEngine().GetDevice(), A_, constraintList_, own_A_);
-}
-
-void OccaConstrainedOperator::Setup(::occa::device device_,
-                                    mfem::Operator *A_,
-                                    const mfem::Array<int> &constraintList_,
-                                    bool own_A_)
-{
-   device = device_;
-
-   A = A_;
-   own_A = own_A_;
-
-   constraintIndices = constraintList_.Size();
-   if (constraintList_.Size() > 0)
-   {
-      constraintList = constraintList_.Get_PArray()->As<Array>().OccaMem();
-   }
-   else
-   {
-      // constraintList is not used
-   }
-}
-
-void OccaConstrainedOperator::EliminateRHS(const Vector &x, Vector &b) const
-{
-   ::occa::kernel mapDofs = mapDofBuilder.build(device);
-
-   w.OccaFill(0.0);
-
-   if (constraintIndices)
-   {
-      mapDofs(constraintIndices, w.OccaMem(), x.OccaMem(), constraintList);
-   }
-
-   A->Mult(mfem_w, mfem_z);
-
-   b.Axpby<double>(1.0, b, -1.0, z);
-
-   if (constraintIndices)
-   {
-      mapDofs(constraintIndices, b.OccaMem(), x.OccaMem(), constraintList);
-   }
-}
-
-void OccaConstrainedOperator::Mult_(const Vector &x, Vector &y) const
-{
-   mfem::Vector mfem_y(y);
-   if (constraintIndices == 0)
-   {
-      A->Mult(x.Wrap(), mfem_y);
-      return;
-   }
-
-   ::occa::kernel mapDofs   = mapDofBuilder.build(device);
-   ::occa::kernel clearDofs = clearDofBuilder.build(device);
-
-   // z.OccaAssign(x); // z = x
-   // Is Axpy faster than DtoD copy on Volta?
-   z.Axpby(1.0, x, 0.0, x);
-
-   clearDofs(constraintIndices, z.OccaMem(), constraintList);
-
-   A->Mult(mfem_z, mfem_y);
-
-   mapDofs(constraintIndices, y.OccaMem(), x.OccaMem(), constraintList);
-}
-
-OccaConstrainedOperator::~OccaConstrainedOperator()
-{
-   if (own_A)
-   {
-      delete A;
-   }
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/operator.hpp

@@ -1,127 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_OPERATOR_HPP
-#define MFEM_BACKENDS_OCCA_OPERATOR_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "vector.hpp"
-#include "../../linalg/operator.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class Operator : public mfem::Operator
-{
-public:
-   /// Creare an operator with the same dimensions as @a orig.
-   Operator(const Operator &orig)
-      : mfem::Operator(orig) { }
-
-   Operator(Layout &layout)
-      : mfem::Operator(layout) { }
-
-   Operator(Layout &in_layout, Layout &out_layout)
-      : mfem::Operator(in_layout, out_layout) { }
-
-   Layout &InLayout_() const { return in_layout->As<Layout>(); }
-
-   Layout &OutLayout_() const { return out_layout->As<Layout>(); }
-
-   virtual void Mult_(const Vector &x, Vector &y) const = 0;
-
-   virtual void MultTranspose_(const Vector &x, Vector &y) const
-   { MFEM_ABORT("method is not supported"); }
-
-   // override
-   virtual void Mult(const mfem::Vector &x, mfem::Vector &y) const
-   {
-      Mult_(x.Get_PVector()->As<Vector>(),
-            y.Get_PVector()->As<Vector>());
-   }
-
-   // override
-   virtual void MultTranspose(const mfem::Vector &x, mfem::Vector &y) const
-   {
-      MultTranspose_(x.Get_PVector()->As<Vector>(),
-                     y.Get_PVector()->As<Vector>());
-   }
-};
-
-
-class OccaConstrainedOperator : public Operator
-{
-protected:
-   ::occa::device device;
-
-   mfem::Operator *A;              //< The unconstrained Operator.
-   bool own_A;                     //< Ownership flag for A.
-   ::occa::memory constraintList;  //< List of constrained indices/dofs.
-   int constraintIndices;
-   mutable Vector z, w;       //< Auxiliary vectors.
-   mutable mfem::Vector mfem_z, mfem_w; // Wrap z, w
-
-   static ::occa::kernelBuilder mapDofBuilder, clearDofBuilder;
-
-public:
-   /** @brief Constructor from a general Operator and a list of essential
-       indices/dofs.
-
-       Specify the unconstrained operator @a *A and a @a list of indices to
-       constrain, i.e. each entry @a list[i] represents an essential-dof. If the
-       ownership flag @a own_A is true, the operator @a *A will be destroyed
-       when this object is destroyed. */
-   OccaConstrainedOperator(mfem::Operator *A_,
-                           const mfem::Array<int> &constraintList_,
-                           bool own_A_ = false);
-
-   void Setup(::occa::device device_,
-              mfem::Operator *A_,
-              const mfem::Array<int> &constraintList_,
-              bool own_A_ = false);
-
-   /** @brief Eliminate "essential boundary condition" values specified in @a x
-       from the given right-hand side @a b.
-
-       Performs the following steps:
-
-       z = A((0,x_b));  b_i -= z_i;  b_b = x_b;
-
-       where the "_b" subscripts denote the essential (boundary) indices/dofs of
-       the vectors, and "_i" -- the rest of the entries. */
-   void EliminateRHS(const Vector &x, Vector &b) const;
-
-   /** @brief Constrained operator action.
-
-       Performs the following steps:
-
-       z = A((x_i,0));  y_i = z_i;  y_b = x_b;
-
-       where the "_b" subscripts denote the essential (boundary) indices/dofs of
-       the vectors, and "_i" -- the rest of the entries. */
-   virtual void Mult_(const Vector &x, Vector &y) const;
-
-   // Destructor: destroys the unconstrained Operator @a A if @a own_A is true.
-   virtual ~OccaConstrainedOperator();
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_OPERATOR_HPP

backends/occa/sparse.okl

@@ -1,57 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-/*
-  ---[ Defines Known At Compile-Time ]------------
-  TILESIZE : Tilesize for iterating over dofs
-  ================================================
-*/
-
-@kernel void Mult(const int entries,
-                  @restrict const int *offsets,
-                  @restrict const int *indices,
-                  @restrict const double *weights,
-                  @restrict const double *in,
-                  @restrict double *out) {
-
-  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
-    if (i < entries) {
-      const int offset = offsets[i];
-      const int nextOffset = offsets[i + 1];
-      double value = 0;
-      for (int j = offset; j < nextOffset; ++j) {
-        value += weights[j] * in[indices[j]];
-      }
-      out[i] = value;
-    }
-  }
-}
-
-@kernel void MappedMult(const int entries,
-                        @restrict const int *offsets,
-                        @restrict const int *indices,
-                        @restrict const double *weights,
-                        @restrict const int *outIndices,
-                        @restrict const double *in,
-                        @restrict double *out) {
-
-  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
-    if (i < entries) {
-      const int offset = offsets[i];
-      const int nextOffset = offsets[i + 1];
-      double value = 0;
-      for (int j = offset; j < nextOffset; ++j) {
-        value += weights[j] * in[indices[j]];
-      }
-      out[outIndices[i]] = value;
-    }
-  }
-}

backends/occa/sparsemat.cpp

@@ -1,221 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "sparsemat.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                                   const mfem::SparseMatrix &m,
-                                   const ::occa::properties &props)
-   : Operator(in_layout, out_layout)
-{
-   Setup(in_layout.OccaEngine().GetDevice(), m, props);
-}
-
-OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                                   ::occa::array<int> offsets_,
-                                   ::occa::array<int> indices_,
-                                   ::occa::array<double> weights_,
-                                   ::occa::array<int> reorderIndices_,
-                                   ::occa::array<int> mappedIndices_,
-                                   const ::occa::properties &props)
-   : Operator(in_layout, out_layout),
-     offsets(offsets_),
-     indices(indices_),
-     weights(weights_),
-     reorderIndices(reorderIndices_),
-     mappedIndices(mappedIndices_)
-{
-   SetupKernel(in_layout.OccaEngine().GetDevice(), props);
-}
-
-void OccaSparseMatrix::Setup(::occa::device device, const mfem::SparseMatrix &m,
-                             const ::occa::properties &props)
-{
-   Setup(device, m, ::occa::array<int>(), ::occa::array<int>(), props);
-}
-
-void OccaSparseMatrix::Setup(::occa::device device, const SparseMatrix &m,
-                             ::occa::array<int> reorderIndices_,
-                             ::occa::array<int> mappedIndices_,
-                             const ::occa::properties &props)
-{
-   MFEM_ASSERT(m.Finalized(), "");
-   MFEM_ASSERT(m.Height() == height, "");
-   MFEM_ASSERT(m.Width() == width, "");
-
-   const int nnz = m.GetI()[height];
-   offsets.allocate(device,
-                    height + 1, m.GetI());
-   indices.allocate(device,
-                    nnz, m.GetJ());
-   weights.allocate(device,
-                    nnz, m.GetData());
-
-   offsets.keepInDevice();
-   indices.keepInDevice();
-   weights.keepInDevice();
-
-   reorderIndices = reorderIndices_;
-   mappedIndices  = mappedIndices_;
-
-   SetupKernel(device, props);
-}
-
-void OccaSparseMatrix::SetupKernel(::occa::device device,
-                                   const ::occa::properties &props)
-{
-   const bool hasOutIndices = mappedIndices.isInitialized();
-
-   const ::occa::properties defaultProps("defines: {"
-                                         "  TILESIZE: 256,"
-                                         "}");
-
-   const std::string &okl_path = InLayout_().OccaEngine().GetOklPath();
-   mapKernel = device.buildKernel(okl_path + "mappings.okl",
-                                  "MapSubVector",
-                                  defaultProps + props);
-
-   multKernel = device.buildKernel(okl_path + "sparse.okl",
-                                   hasOutIndices ? "MappedMult" : "Mult",
-                                   defaultProps + props);
-}
-
-void OccaSparseMatrix::Mult_(const Vector &x, Vector &y) const
-{
-   if (reorderIndices.isInitialized() ||
-       mappedIndices.isInitialized())
-   {
-      if (reorderIndices.isInitialized())
-      {
-         mapKernel((int) (reorderIndices.size() / 2),
-                   reorderIndices,
-                   x.OccaMem(), y.OccaMem());
-      }
-      if (mappedIndices.isInitialized())
-      {
-         multKernel((int) (mappedIndices.size()),
-                    offsets, indices, weights,
-                    mappedIndices,
-                    x.OccaMem(), y.OccaMem());
-      }
-   }
-   else
-   {
-      multKernel((int) height,
-                 offsets, indices, weights,
-                 x.OccaMem(), y.OccaMem());
-   }
-}
-
-
-OccaSparseMatrix* CreateMappedSparseMatrix(Layout &in_layout,
-                                           Layout &out_layout,
-                                           const mfem::SparseMatrix &m,
-                                           const ::occa::properties &props)
-{
-   const int mHeight = m.Height();
-   // const int mWidth  = m.Width();
-
-   // Count indices that are only reordered (true dofs)
-   const int *I = m.GetI();
-   const int *J = m.GetJ();
-   const double *D = m.GetData();
-
-   int trueCount = 0;
-   for (int i = 0; i < mHeight; ++i)
-   {
-      trueCount += ((I[i + 1] - I[i]) == 1);
-   }
-   const int dupCount = (mHeight - trueCount);
-
-   // Create the reordering map for entries that aren't modified (true dofs)
-   ::occa::device device(in_layout.OccaEngine().GetDevice());
-   ::occa::array<int> reorderIndices(device,
-                                     2 * trueCount);
-   ::occa::array<int> mappedIndices, offsets, indices;
-   ::occa::array<double> weights;
-
-   if (dupCount)
-   {
-      mappedIndices.allocate(device,
-                             dupCount);
-   }
-   int trueIdx = 0, dupIdx = 0;
-   for (int i = 0; i < mHeight; ++i)
-   {
-      const int i1 = I[i];
-      if ((I[i + 1] - i1) == 1)
-      {
-         reorderIndices[trueIdx++] = J[i1];
-         reorderIndices[trueIdx++] = i;
-      }
-      else
-      {
-         mappedIndices[dupIdx++] = i;
-      }
-   }
-   reorderIndices.keepInDevice();
-
-   if (dupCount)
-   {
-      mappedIndices.keepInDevice();
-
-      // Extract sparse matrix without reordered identity
-      const int dupNnz = I[mHeight] - trueCount;
-
-      offsets.allocate(device,
-                       dupCount + 1);
-      indices.allocate(device,
-                       dupNnz);
-      weights.allocate(device,
-                       dupNnz);
-
-      int nnz = 0;
-      offsets[0] = 0;
-      for (int i = 0; i < dupCount; ++i)
-      {
-         const int idx = mappedIndices[i];
-         const int offStart = I[idx];
-         const int offEnd   = I[idx + 1];
-         offsets[i + 1] = offsets[i] + (offEnd - offStart);
-         for (int j = offStart; j < offEnd; ++j)
-         {
-            indices[nnz] = J[j];
-            weights[nnz] = D[j];
-            ++nnz;
-         }
-      }
-
-      offsets.keepInDevice();
-      indices.keepInDevice();
-      weights.keepInDevice();
-   }
-
-   return new OccaSparseMatrix(in_layout, out_layout,
-                               offsets, indices, weights,
-                               reorderIndices, mappedIndices,
-                               props);
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/sparsemat.hpp

@@ -1,89 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_SPARSE_MAT_HPP
-#define MFEM_BACKENDS_OCCA_SPARSE_MAT_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include <occa.hpp>
-#include "vector.hpp"
-#include "engine.hpp"
-#include "operator.hpp"
-#include "../../linalg/sparsemat.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-/// TODO: doxygen
-class OccaSparseMatrix : public Operator
-{
-protected:
-   ::occa::array<int> offsets, indices;
-   ::occa::array<double> weights;
-   ::occa::array<int> reorderIndices, mappedIndices;
-   ::occa::kernel mapKernel, multKernel;
-
-   void Setup(::occa::device device, const mfem::SparseMatrix &m,
-              const ::occa::properties &props);
-
-   void Setup(::occa::device device, const mfem::SparseMatrix &m,
-              ::occa::array<int> reorderIndices_,
-              ::occa::array<int> mappedIndices_,
-              const ::occa::properties &props);
-
-   void SetupKernel(::occa::device device,
-                    const ::occa::properties &props);
-
-public:
-   /// Construct an empty OccaSparseMatrix.
-   OccaSparseMatrix(const Operator &orig)
-      : Operator(orig) { }
-
-   // Implicitly defined copy constructor.
-
-   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                    const mfem::SparseMatrix &m,
-                    const ::occa::properties &props = ::occa::properties());
-
-   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                    ::occa::array<int> offsets_,
-                    ::occa::array<int> indices_,
-                    ::occa::array<double> weights_,
-                    ::occa::array<int> reorderIndices_,
-                    ::occa::array<int> mappedIndices_,
-                    const ::occa::properties &props = ::occa::properties());
-
-   const ::occa::array<int> &GetReorderIndices() const
-   { return reorderIndices; }
-
-   // override
-   virtual void Mult_(const Vector &x, Vector &y) const;
-};
-
-
-/// TODO: doxygen
-OccaSparseMatrix* CreateMappedSparseMatrix(
-   Layout &in_layout, Layout &out_layout,
-   const mfem::SparseMatrix &m,
-   const ::occa::properties &props = ::occa::properties());
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_SPARSE_MAT_HPP

backends/occa/url_handler.cpp

@@ -1,81 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "url_handler.hpp"
-#include "../../general/error.hpp"
-#include <cstdlib>
-#include <sys/stat.h>
-
-namespace mfem
-{
-
-namespace occa
-{
-
-FileOpener::FileOpener(const std::string &prefix,
-                       const std::string &env_variable)
-   : pfx(prefix)
-{
-   const char *env_path = getenv(env_variable.c_str());
-   if (!env_path) { return; }
-   std::string path(env_path);
-   for (std::size_t start = 0, end; start < path.size(); start = end + 1)
-   {
-      end = path.find(':', start);
-      if (end == std::string::npos)
-      {
-         AddDir(path.substr(start, end));
-         break;
-      }
-      AddDir(path.substr(start, end - start));
-   }
-}
-
-bool FileOpener::AddDir(const std::string &dir)
-{
-   if (dir.size() == 0 || dir[0] != '/') { return false; }
-   struct stat dir_stat;
-   if (stat(dir.c_str(), &dir_stat)) { return false; }
-   if (!S_ISDIR(dir_stat.st_mode)) { return false; }
-   paths.push_back(dir + (*dir.rbegin() == '/' ? "" : "/"));
-   return true;
-}
-
-bool FileOpener::handles(const std::string &filename)
-{
-   return filename.size() >= pfx.size() &&
-          filename.compare(0, pfx.size(), pfx) == 0;
-}
-
-std::string FileOpener::expand(const std::string &filename)
-{
-   std::string sfx(filename.substr(pfx.size()));
-   for (std::size_t i = 0; i < paths.size(); i++)
-   {
-      std::string file = paths[i] + sfx;
-      struct stat file_stat;
-      if (stat(file.c_str(), &file_stat) == 0 && S_ISREG(file_stat.st_mode))
-      {
-         return file;
-      }
-   }
-   MFEM_ABORT("invalid url: " << filename);
-   return sfx;
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/url_handler.hpp

@@ -1,47 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_URL_HANDLER_HPP
-#define MFEM_BACKENDS_OCCA_URL_HANDLER_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include <occa.hpp>
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class FileOpener : public ::occa::io::fileOpener
-{
-protected:
-   std::string pfx; // prefix, e.g. "mfem://"
-   std::vector<std::string> paths; // paths to search for prefix replacement
-
-public:
-   FileOpener(const std::string &prefix, const std::string &env_variable);
-
-   bool AddDir(const std::string &dir);
-
-   virtual bool handles(const std::string &filename);
-   virtual std::string expand(const std::string &filename);
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_URL_HANDLER_HPP

backends/occa/utils.okl

@@ -1,22 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-typedef double* Local_t @dim(numDofs, numElements);
-
-@kernel void InitLocalVector(const int numElements,
-                             const int numDofs,
-                             @restrict Local_t sol) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int d = 0; d < numDofs; ++d; @inner) {
-      sol(d, e) = 0;
-    }
-  }
-}

backends/occa/vector.cpp

@@ -1,196 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "vector.hpp"
-#include "../../linalg/vector.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-PVector *Vector::DoVectorClone(bool copy_data, void **buffer,
-                               int buffer_type_id) const
-{
-   MFEM_ASSERT(buffer_type_id == ScalarId<double>::value, "");
-   Vector *new_vector = new Vector(OccaLayout());
-   if (copy_data)
-   {
-      new_vector->slice.copyFrom(slice);
-   }
-   if (buffer)
-   {
-      *buffer = new_vector->GetBuffer();
-   }
-   return new_vector;
-}
-
-void Vector::DoDotProduct(const PVector &x, void *result,
-                          int result_type_id) const
-{
-   // Can be called when Size() == 0, e.g. when an MPI-parallel vector has a
-   // local size of 0.
-
-   MFEM_ASSERT(result_type_id == ScalarId<double>::value, "");
-   double *res = (double *)result;
-   const Vector &xp = x.As<Vector>();
-   MFEM_ASSERT(this->Size() == xp.Size(), "");
-   *res = ::occa::linalg::dot<double, double, double>(this->slice, xp.slice);
-
-#ifdef MFEM_USE_MPI
-   double local_dot = *res;
-   if (IsParallel())
-   {
-      MPI_Allreduce(&local_dot, res, 1, MPI_DOUBLE, MPI_SUM,
-                    OccaEngine().GetComm());
-   }
-#endif
-}
-
-void Vector::DoAxpby(const void *a, const PVector &x,
-                     const void *b, const PVector &y,
-                     int ab_type_id)
-{
-   //
-   // TODO: move all kernel builders to class mfem::occa::Backend
-   //
-   static ::occa::kernelBuilder axpby1_builder =
-      ::occa::linalg::customLinearMethod(
-         "mfem_occa_axpby1",
-         "v0[i] = c0 * v1[i];",
-         "defines: {"
-         "  CTYPE0: 'double',"
-         "  VTYPE0: 'double',"
-         "  VTYPE1: 'double',"
-         "  TILESIZE: '128',"
-         "}");
-
-   static ::occa::kernelBuilder axpby2_builder =
-      ::occa::linalg::customLinearMethod(
-         "mfem_occa_axpby2",
-         "v0[i] = c0 * v0[i] + c1 * v1[i];",
-         "defines: {"
-         "  CTYPE0: 'double',"
-         "  CTYPE1: 'double',"
-         "  VTYPE0: 'double',"
-         "  VTYPE1: 'double',"
-         "  TILESIZE: '128',"
-         "}");
-
-   static ::occa::kernelBuilder axpby3_builder =
-      ::occa::linalg::customLinearMethod(
-         "mfem_occa_axpby3",
-         "v0[i] = c0 * v1[i] + c1 * v2[i];",
-         "defines: {"
-         "  CTYPE0: 'double',"
-         "  CTYPE1: 'double',"
-         "  VTYPE0: 'double',"
-         "  VTYPE1: 'double',"
-         "  VTYPE2: 'double',"
-         "  TILESIZE: '128',"
-         "}");
-
-   // called only when Size() != 0
-
-   MFEM_ASSERT(ab_type_id == ScalarId<double>::value, "");
-   const double da = *static_cast<const double *>(a);
-   const double db = *static_cast<const double *>(b);
-   MFEM_ASSERT(da == 0.0 || dynamic_cast<const Vector *>(&x) != NULL,
-               "invalid Vector x");
-   MFEM_ASSERT(db == 0.0 || dynamic_cast<const Vector *>(&y) != NULL,
-               "invalid Vector y");
-   const Vector *xp = static_cast<const Vector *>(&x);
-   const Vector *yp = static_cast<const Vector *>(&y);
-
-   MFEM_ASSERT(da == 0.0 || this->Size() == xp->Size(), "");
-   MFEM_ASSERT(db == 0.0 || this->Size() == yp->Size(), "");
-
-   if (da == 0.0)
-   {
-      if (db == 0.0)
-      {
-         OccaFill(da);
-      }
-      else
-      {
-         if (this->slice == yp->slice)
-         {
-            // *this *= db
-            ::occa::linalg::operator_mult_eq(slice, db);
-         }
-         else
-         {
-            // *this = db * y
-            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice());
-            kernel((int)Size(), db, slice, yp->slice);
-         }
-      }
-   }
-   else
-   {
-      if (db == 0.0)
-      {
-         if (this->slice == xp->slice)
-         {
-            // *this *= da
-            ::occa::linalg::operator_mult_eq(slice, da);
-         }
-         else
-         {
-            // *this = da * x
-            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice());
-            kernel((int)Size(), da, slice, xp->slice);
-         }
-      }
-      else
-      {
-         MFEM_ASSERT(xp->slice != yp->slice, "invalid input");
-         if (this->slice == xp->slice)
-         {
-            // *this = da * (*this) + db * y
-            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice());
-            kernel((int)Size(), da, db, slice, yp->slice);
-         }
-         else if (this->slice == yp->slice)
-         {
-            // *this = da * x + db * (*this)
-            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice());
-            kernel((int)Size(), db, da, slice, xp->slice);
-         }
-         else
-         {
-            // *this = da * x + db * y
-            ::occa::kernel kernel = axpby3_builder.build(slice.getDevice());
-            kernel((int)Size(), da, db, slice, xp->slice, yp->slice);
-         }
-      }
-   }
-}
-
-mfem::Vector Vector::Wrap()
-{
-   return mfem::Vector(*this);
-}
-
-const mfem::Vector Vector::Wrap() const
-{
-   return mfem::Vector(*const_cast<Vector*>(this));
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/vector.hpp

@@ -1,83 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_VECTOR_HPP
-#define MFEM_BACKENDS_OCCA_VECTOR_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include <occa.hpp>
-#include "../base/vector.hpp"
-#include "array.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-// FIXME: Once XL fixes this code quirk we can remove this #ifdef switch
-#ifdef __ibmxl__
-class Vector : public Array, public PVector
-#else
-class Vector : virtual public Array, public PVector
-#endif
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // DLayout layout;
-
-   /**
-       @name Virtual interface
-    */
-   ///@{
-
-   virtual PVector *DoVectorClone(bool copy_data, void **buffer,
-                                  int buffer_type_id) const;
-
-   virtual void DoDotProduct(const PVector &x, void *result,
-                             int result_type_id) const;
-
-   virtual void DoAxpby(const void *a, const PVector &x,
-                        const void *b, const PVector &y,
-                        int ab_type_id);
-
-   ///@}
-   // End: Virtual interface
-
-public:
-   Vector(const Engine &e)
-      : PArray(*(new Layout(e, 0))), Array(e), PVector(*layout)
-   { }
-
-   Vector(Layout &lt)
-      : PArray(lt), Array(lt, sizeof(double)), PVector(lt)
-   { }
-
-   mfem::Vector Wrap();
-
-   const mfem::Vector Wrap() const;
-
-#if defined(MFEM_USE_MPI)
-   bool IsParallel() const { return (OccaEngine().GetComm() != MPI_COMM_NULL); }
-#endif
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_VECTOR_HPP

backends/occa/vmass/tensor/cpu.okl

@@ -1,321 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 1D ]-----------------------------
-@kernel void Assemble1D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian1D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
-      oper(q, e) = quadWeights[q] * COEFF * J(q, e);
-    }
-  }
-}
-
-@kernel void MultAdd1D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal_t oper,
-                       @restrict const DVLocal1D_t solIn,
-                       @restrict DVLocal1D_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double sol_x[1][NUM_QUAD_1D];
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        sol_x[0][qx] = 0;
-      }
-      // sol_x{qx} = dofToQuad{qx,dx} * sol{dx}
-      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          sol_x[0][qx] += dofToQuad(qx, dx) * solIn(0, dx, e);
-        }
-      }
-      // sol_x{q} *= oper{q}
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        sol_x[0][qx] *= oper(qx, e);
-      }
-      // sol{dx} = quadToDof{dx,qx} * sol_x{qx}
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          solOut(0, dx, e) += sol_x[0][qx] * quadToDof(dx, qx);
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian2D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_2D; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-
-      oper(q, e) = quadWeights[q] * COEFF * ((J11 * J22) - (J21 * J12));
-    }
-  } // e
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal_t oper,
-                       @restrict const DVLocal2D_t solIn,
-                       @restrict DVLocal2D_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy=0; dummy<1; ++dummy; @inner) {
-      double sol_xy[2][NUM_QUAD_1D][NUM_QUAD_1D];
-
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          sol_xy[0][qx][qy] = 0;
-          sol_xy[1][qx][qy] = 0;
-        }
-      }
-
-      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-        double sol_x[2][NUM_QUAD_1D];
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          sol_x[0][qy] = 0;
-          sol_x[1][qy] = 0;
-        }
-
-        // sol_x{vd, dx, qy} = dofToQuad{qy, dy} * sol{vd, dx, dy}
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            sol_x[0][qy] += dofToQuad(qy, dx) * solIn(0, dx, dy, e);
-            sol_x[1][qy] += dofToQuad(qy, dx) * solIn(1, dx, dy, e);
-          }
-        }
-
-        // sol_xy{qx, qy} = dofToQuad{qx, dx} * sol_x{dx, qy}
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          const double d2q = dofToQuad(qy, dy);
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_xy[0][qx][qy] += d2q * sol_x[0][qx];
-            sol_xy[1][qx][qy] += d2q * sol_x[1][qx];
-          }
-        }
-      } // dy
-
-      // sol_xy{qx, qy} = sol_xy{q} *= oper{q, e}
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          const int q = QUAD_2D_ID(qx, qy);
-          sol_xy[0][qx][qy] *= oper(q, e);
-          sol_xy[1][qx][qy] *= oper(q, e);
-        }
-      }
-
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        double sol_x[2][NUM_DOFS_1D];
-        for (int dx = 0; dx < NUM_QUAD_1D; ++dx) {
-          sol_x[0][dx] = 0;
-          sol_x[1][dx] = 0;
-        }
-
-        // sol_x{qx, dy} = quadToDof{dy, qy} * sol_xy{qx, qy}
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            sol_x[0][dx] += quadToDof(dx, qx) * sol_xy[0][qx][qy];
-            sol_x[1][dx] += quadToDof(dx, qx) * sol_xy[1][qx][qy];
-          }
-        }
-
-        // sol{dx, dy, e} = quadToDof{dx, qx} * sol_x{qx, dy}
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          const double q2d = quadToDof(dy, qy);
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            solOut(0, dx, dy, e) += q2d * sol_x[0][dx];
-            solOut(1, dx, dy, e) += q2d * sol_x[1][dx];
-          }
-        }
-      }
-    } // dummy
-  } // e
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        @restrict const double * quadWeights,
-                        @restrict const Jacobian3D_t J,
-                        COEFF_ARGS
-                        @restrict QLocal_t oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_3D; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-      oper(q, e) = quadWeights[q] * COEFF * detJ;
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       @restrict const DofToQuad_t dofToQuad,
-                       @restrict const DofToQuad_t dofToQuadD,
-                       @restrict const QuadToDof_t quadToDof,
-                       @restrict const QuadToDof_t quadToDofD,
-                       @restrict const QLocal_t oper,
-                       @restrict const DVLocal3D_t solIn,
-                       @restrict DVLocal3D_t solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double sol_xyz[3][NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D];
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_xyz[0][qz][qy][qx] = 0;
-            sol_xyz[1][qz][qy][qx] = 0;
-            sol_xyz[2][qz][qy][qx] = 0;
-          }
-        }
-      }
-
-      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-        double sol_xy[3][NUM_QUAD_1D][NUM_QUAD_1D];
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_xy[0][qy][qx] = 0;
-            sol_xy[1][qy][qx] = 0;
-            sol_xy[2][qy][qx] = 0;
-          }
-        }
-
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          double sol_x[3][NUM_QUAD_1D];
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            sol_x[0][qx] = 0;
-            sol_x[1][qx] = 0;
-            sol_x[2][qx] = 0;
-          }
-
-          // sol_x{qx} = dofToQuad{qx, dx} * sol{dx, dy, dz, e}
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              sol_x[0][qx] += dofToQuad(qx, dx) * solIn(0, dx, dy, dz, e);
-              sol_x[1][qx] += dofToQuad(qx, dx) * solIn(1, dx, dy, dz, e);
-              sol_x[2][qx] += dofToQuad(qx, dx) * solIn(2, dx, dy, dz, e);
-            }
-          }
-
-          // sol_xy{qx, qy} = dofToQuad{qy, dy} * sol_x{dx}
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            const double wy = dofToQuad(qy, dy);
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              sol_xy[0][qy][qx] += wy * sol_x[0][qx];
-              sol_xy[1][qy][qx] += wy * sol_x[1][qx];
-              sol_xy[2][qy][qx] += wy * sol_x[2][qx];
-            }
-          }
-        } // dy
-
-        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-          const double wz = dofToQuad(qz, dz);
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              sol_xyz[0][qz][qy][qx] += wz * sol_xy[0][qy][qx];
-              sol_xyz[1][qz][qy][qx] += wz * sol_xy[1][qy][qx];
-              sol_xyz[2][qz][qy][qx] += wz * sol_xy[2][qy][qx];
-            }
-          }
-        }
-      } // dz
-
-      // sol_xyz{qz, qy, qx} *= oper{q, e}
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            const int q = QUAD_3D_ID(qx, qy, qz);
-            sol_xyz[0][qz][qy][qx] *= oper(q, e);
-            sol_xyz[1][qz][qy][qx] *= oper(q, e);
-            sol_xyz[2][qz][qy][qx] *= oper(q, e);
-          }
-        }
-      }
-
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        double sol_xy[3][NUM_DOFS_1D][NUM_DOFS_1D];
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            sol_xy[0][dy][dx] = 0;
-            sol_xy[1][dy][dx] = 0;
-            sol_xy[2][dy][dx] = 0;
-          }
-        }
-
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          double sol_x[3][NUM_DOFS_1D];
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            sol_x[0][dx] = 0;
-            sol_x[1][dx] = 0;
-            sol_x[2][dx] = 0;
-          }
-
-          // sol_x{dx} = quadToDof{dx, qx} * sol_xyz{qz, qy, qx}
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              sol_x[0][dx] += quadToDof(dx, qx) * sol_xyz[0][qz][qy][qx];
-              sol_x[1][dx] += quadToDof(dx, qx) * sol_xyz[1][qz][qy][qx];
-              sol_x[2][dx] += quadToDof(dx, qx) * sol_xyz[2][qz][qy][qx];
-            }
-          }
-
-          // sol_xy{dy, dx} = quadToDof{dy, qy} * sol_x{dx}
-          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-            const double wy = quadToDof(dy, qy);
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              sol_xy[0][dy][dx] += wy * sol_x[0][dx];
-              sol_xy[1][dy][dx] += wy * sol_x[1][dx];
-              sol_xy[2][dy][dx] += wy * sol_x[2][dx];
-            }
-          }
-        } // qy
-
-        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-          const double wz = quadToDof(dz, qz);
-          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              solOut(0, dx, dy, dz, e) += wz * sol_xy[0][dy][dx];
-              solOut(1, dx, dy, dz, e) += wz * sol_xy[1][dy][dx];
-              solOut(2, dx, dy, dz, e) += wz * sol_xy[2][dy][dx];
-            }
-          }
-        }
-      } // qz
-    } // dummy
-  } // e
-
-}
-//======================================

config/config.hpp

@@ -40,11 +40,3 @@
 #error Building with PUMI (MFEM_USE_PUMI=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
 #endif // MFEM_USE_MPI not defined
-
-// Macro that returns its first arg when MFEM_USE_BACKENDS is defined, and its
-// second arg if it is not defined.
-#ifdef MFEM_USE_BACKENDS
-#define MFEM_IF_BACKENDS(x,y) x
-#else
-#define MFEM_IF_BACKENDS(x,y) y
-#endif

config/config.hpp.in

@@ -33,12 +33,6 @@
 // Description of the git commit used to build MFEM.
 // #define MFEM_GIT_STRING "@MFEM_GIT_STRING@"
 
-// The absolute path of the MFEM source prefix
-// #define MFEM_SOURCE_DIR "@MFEM_SOURCE_DIR@"
-
-// The absolute path of the MFEM installation prefix
-// #define MFEM_INSTALL_DIR "@MFEM_INSTALL_DIR@"
-
 // Build the parallel MFEM library.
 // Requires an MPI compiler, and the libraries HYPRE and METIS.
 // #define MFEM_USE_MPI
@@ -118,12 +112,6 @@
 // Enable MFEM functionality based on the PUMI library
 // #define MFEM_USE_PUMI
 
-// Enable the use of MFEM backends.
-// #define MFEM_USE_BACKENDS
-
-// Enable the OCCA backend.
-// #define MFEM_USE_OCCA
-
 // Windows specific options
 #ifdef _WIN32
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)

config/config.mk.in

@@ -13,8 +13,6 @@
 MFEM_VERSION         = @MFEM_VERSION@
 MFEM_VERSION_STRING  = @MFEM_VERSION_STRING@
 MFEM_GIT_STRING      = @MFEM_GIT_STRING@
-MFEM_SOURCE_DIR      = @MFEM_SOURCE_DIR@
-MFEM_INSTALL_DIR     = @MFEM_INSTALL_DIR@
 MFEM_USE_MPI         = @MFEM_USE_MPI@
 MFEM_USE_METIS       = @MFEM_USE_METIS@
 MFEM_USE_METIS_5     = @MFEM_USE_METIS_5@
@@ -40,8 +38,6 @@ MFEM_USE_MPFR        = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE       = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT     = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI        = @MFEM_USE_PUMI@
-MFEM_USE_BACKENDS    = @MFEM_USE_BACKENDS@
-MFEM_USE_OCCA        = @MFEM_USE_OCCA@
 
 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@

config/defaults.mk

@@ -83,9 +83,6 @@ MFEM_MPI_NP = 4
 # in config.mk and config.hpp.
 
 MFEM_USE_MPI         = NO
-# FIXME: add MFEM_USE_BACKENDS, MFEM_USE_OCCA to the CMake build system
-MFEM_USE_BACKENDS    = YES
-MFEM_USE_OCCA        = YES
 MFEM_USE_METIS       = $(MFEM_USE_MPI)
 MFEM_USE_METIS_5     = NO
 MFEM_DEBUG           = NO
@@ -282,10 +279,6 @@ PUMI_OPT = -I$(PUMI_DIR)/include
 PUMI_LIB = -L$(PUMI_DIR)/lib -lpumi -lcrv -lma -lmds -lapf -lpcu -lgmi -lparma\
    -llion -lmth -lapf_zoltan -lspr
 
-OCCA_DIR = @MFEM_DIR@/../occa
-OCCA_OPT = -I$(OCCA_DIR)/include
-OCCA_LIB = -Wl,-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca
-
 # If YES, enable some informational messages
 VERBOSE = NO
 

data/mixed-ele-meshes/annulus-trans-24.mesh

@@ -0,0 +1,218 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+74
+2 3 0 1 2 3
+2 3 1 5 6 2
+2 3 5 8 9 6
+2 3 8 11 12 9
+2 3 11 14 15 12
+2 3 14 17 18 15
+2 3 17 20 21 18
+2 3 20 23 24 21
+2 3 23 26 27 24
+2 3 26 29 30 27
+2 3 29 32 33 30
+2 3 32 35 36 33
+2 3 35 38 39 36
+2 3 38 41 42 39
+2 3 41 44 45 42
+2 3 44 47 48 45
+2 3 47 50 51 48
+2 3 50 53 54 51
+2 3 53 56 57 54
+2 3 56 59 60 57
+2 3 59 62 63 60
+2 3 62 65 66 63
+2 3 65 68 69 66
+2 3 68 71 72 69
+2 3 71 74 75 72
+1 2 2 3 4
+1 2 6 2 7
+1 2 9 6 10
+1 2 12 9 13
+1 2 15 12 16
+1 2 18 15 19
+1 2 21 18 22
+1 2 24 21 25
+1 2 27 24 28
+1 2 30 27 31
+1 2 33 30 34
+1 2 36 33 37
+1 2 39 36 40
+1 2 42 39 43
+1 2 45 42 46
+1 2 48 45 49
+1 2 51 48 52
+1 2 54 51 55
+1 2 57 54 58
+1 2 60 57 61
+1 2 63 60 64
+1 2 66 63 67
+1 2 69 66 70
+1 2 72 69 73
+1 2 75 72 76
+1 2 2 4 7
+1 2 6 7 10
+1 2 9 10 13
+1 2 12 13 16
+1 2 15 16 19
+1 2 18 19 22
+1 2 21 22 25
+1 2 24 25 28
+1 2 27 28 31
+1 2 30 31 34
+1 2 33 34 37
+1 2 36 37 40
+1 2 39 40 43
+1 2 42 43 46
+1 2 45 46 49
+1 2 48 49 52
+1 2 51 52 55
+1 2 54 55 58
+1 2 57 58 61
+1 2 60 61 64
+1 2 63 64 67
+1 2 66 67 70
+1 2 69 70 73
+1 2 72 73 76
+
+boundary
+53
+1 1 0 1
+1 1 1 5
+1 1 5 8
+1 1 8 11
+1 1 11 14
+1 1 14 17
+1 1 17 20
+1 1 20 23
+1 1 23 26
+1 1 26 29
+1 1 29 32
+1 1 32 35
+1 1 35 38
+1 1 38 41
+1 1 41 44
+1 1 44 47
+1 1 47 50
+1 1 50 53
+1 1 53 56
+1 1 56 59
+1 1 59 62
+1 1 62 65
+1 1 65 68
+1 1 68 71
+1 1 71 74
+1 1 74 75
+1 1 75 76
+1 1 76 73
+1 1 73 70
+1 1 70 67
+1 1 67 64
+1 1 64 61
+1 1 61 58
+1 1 58 55
+1 1 55 52
+1 1 52 49
+1 1 49 46
+1 1 46 43
+1 1 43 40
+1 1 40 37
+1 1 37 34
+1 1 34 31
+1 1 31 28
+1 1 28 25
+1 1 25 22
+1 1 22 19
+1 1 19 16
+1 1 16 13
+1 1 13 10
+1 1 10 7
+1 1 7 4
+1 1 4 3
+1 1 3 0
+
+vertices
+77
+2
+3.9788735773 0.0
+3.84329674785 1.02980825986
+2.88247256089 0.772356194895
+2.98415518297 0.0
+1.97241688113 0.259673608685
+3.44580559639 1.98943678865
+2.58435419729 1.49207759149
+1.83799993026 0.761324498753
+2.81348848799 2.81348848799
+2.11011636599 2.11011636599
+1.57832632157 1.21109238238
+1.98943678865 3.44580559639
+1.49207759149 2.58435419729
+1.21109238238 1.57832632157
+1.02980825986 3.84329674785
+0.772356194895 2.88247256089
+0.761324498753 1.83799993026
+2.43635739532e-16 3.9788735773
+1.82726804649e-16 2.98415518297
+0.259673608685 1.97241688113
+-1.02980825986 3.84329674785
+-0.772356194895 2.88247256089
+-0.259673608685 1.97241688113
+-1.98943678865 3.44580559639
+-1.49207759149 2.58435419729
+-0.761324498753 1.83799993026
+-2.81348848799 2.81348848799
+-2.11011636599 2.11011636599
+-1.21109238238 1.57832632157
+-3.44580559639 1.98943678865
+-2.58435419729 1.49207759149
+-1.57832632157 1.21109238238
+-3.84329674785 1.02980825986
+-2.88247256089 0.772356194895
+-1.83799993026 0.761324498753
+-3.9788735773 4.87271479065e-16
+-2.98415518297 3.65453609299e-16
+-1.97241688113 0.259673608685
+-3.84329674785 -1.02980825986
+-2.88247256089 -0.772356194895
+-1.97241688113 -0.259673608685
+-3.44580559639 -1.98943678865
+-2.58435419729 -1.49207759149
+-1.83799993026 -0.761324498753
+-2.81348848799 -2.81348848799
+-2.11011636599 -2.11011636599
+-1.57832632157 -1.21109238238
+-1.98943678865 -3.44580559639
+-1.49207759149 -2.58435419729
+-1.21109238238 -1.57832632157
+-1.02980825986 -3.84329674785
+-0.772356194895 -2.88247256089
+-0.761324498753 -1.83799993026
+-7.30907218597e-16 -3.9788735773
+-5.48180413948e-16 -2.98415518297
+-0.259673608685 -1.97241688113
+1.02980825986 -3.84329674785
+0.772356194895 -2.88247256089
+0.259673608685 -1.97241688113
+1.98943678865 -3.44580559639
+1.49207759149 -2.58435419729
+0.761324498753 -1.83799993026
+2.81348848799 -2.81348848799
+2.11011636599 -2.11011636599
+1.21109238238 -1.57832632157
+3.44580559639 -1.98943678865
+2.58435419729 -1.49207759149
+1.57832632157 -1.21109238238
+3.84329674785 -1.02980825986
+2.88247256089 -0.772356194895
+1.83799993026 -0.761324498753
+3.9788735773 -9.7454295813e-16
+2.98415518297 -7.30907218597e-16
+1.97241688113 -0.259673608685
+3.84329674785 1.02980825986
+2.88247256089 0.772356194895
+1.97241688113 0.259673608685

data/mixed-ele-meshes/annulus-trans-6.mesh

@@ -0,0 +1,74 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+20
+2 3 0 1 2 3
+2 3 1 5 6 2
+2 3 5 8 9 6
+2 3 8 11 12 9
+2 3 11 14 15 12
+2 3 14 17 18 15
+2 3 17 20 21 18
+1 2 2 3 4
+1 2 6 2 7
+1 2 9 6 10
+1 2 12 9 13
+1 2 15 12 16
+1 2 18 15 19
+1 2 21 18 22
+1 2 2 4 7
+1 2 6 7 10
+1 2 9 10 13
+1 2 12 13 16
+1 2 15 16 19
+1 2 18 19 22
+
+boundary
+17
+1 1 0 1
+1 1 1 5
+1 1 5 8
+1 1 8 11
+1 1 11 14
+1 1 14 17
+1 1 17 20
+1 1 20 21
+1 1 21 22
+1 1 22 19
+1 1 19 16
+1 1 16 13
+1 1 13 10
+1 1 10 7
+1 1 7 4
+1 1 4 3
+1 1 3 0
+
+vertices
+23
+2
+1.11408460164 0.0
+0.557042300822 0.964825566988
+0.417781725616 0.723619175241
+0.835563451232 0.0
+0.482412783494 0.278521150411
+-0.557042300822 0.964825566988
+-0.417781725616 0.723619175241
+3.41090035345e-17 0.557042300822
+-1.11408460164 1.36436014138e-16
+-0.835563451232 1.02327010604e-16
+-0.482412783494 0.278521150411
+-0.557042300822 -0.964825566988
+-0.417781725616 -0.723619175241
+-0.482412783494 -0.278521150411
+0.557042300822 -0.964825566988
+0.417781725616 -0.723619175241
+-1.02327010604e-16 -0.557042300822
+1.11408460164 -2.72872028276e-16
+0.835563451232 -2.04654021207e-16
+0.482412783494 -0.278521150411
+0.557042300822 0.964825566988
+0.417781725616 0.723619175241
+0.482412783494 0.278521150411

data/mixed-ele-meshes/circInSquare.py

@@ -0,0 +1,924 @@
+#Title:circInSquare.py
+#Author:T. M. McManus
+#Date:10-7-18
+#Purpose: Fill a circular sector with triangles and a bounding region,
+#defined by 3 nodes, with quads.  Then reflect/preserve QuadI twice to
+#create a complete disc bounded in a square.
+
+import scipy as sp
+import argparse
+import sys
+import subprocess
+import time
+
+parser=argparse.ArgumentParser(description='Fill a circular sector with triangles and a bounding region,\
+defined by 3 nodes, with quads.  Then reflect/preserve QuadI twice to create a complete disc bounded in a square.'
+,epilog='Sample run: python circInSquare.py -r 1 -e 2 -n 8 -g ../../../glvis/glvis')
+
+parser.add_argument('-r','--circRad', nargs='?',const=1, default = 1.0, type=float, help='Radius of circle')
+parser.add_argument('-e','--edgeLength', nargs='?',const=1,default=2.0,type=float,help='Edge-length of bounding square')
+parser.add_argument('-n','--numEdges',nargs='?',const=1,default=6,type=int,help='n-gon approximation of internal circle')
+parser.add_argument('-o','--outputFile',nargs='?',const=1,default='circInSquare', help='Output file name.')
+parser.add_argument('-g','--glvis',nargs='?',const=1,default='',type=str,help='Abs. or rel. path of glvis binary.')
+args=parser.parse_args()
+
+r=args.circRad 
+edgeLength=args.edgeLength
+numEdges=args.numEdges
+outputName=args.outputFile
+glvis=args.glvis
+
+visMesh=False;
+
+if glvis!='':
+    visMesh=True
+
+if r >= edgeLength:
+    print("Circle radius must be less than bounding square edge length")
+    sys.exit(1)
+
+if sp.mod(numEdges,2) != 0:
+    print("Currently this mixed element generator only supports an even numbers of edges.")
+    sys.exit(1)
+
+    
+#The basic idea:
+#1. Construct topology for regions
+#2. Combine topologies
+#3. Construct boundary
+#4. Construct geometry for regions
+#5. Combine geometries
+#6. Output
+
+def eleMatCirc(numEdges):
+    
+    nNodesSeq=sp.zeros([numEdges])
+    nNodesSeq[0]=3
+    if numEdges != 1:
+        for n in range(1,numEdges):
+            nNodesSeq[n]=nNodesSeq[n-1]+(2+n)
+            
+    numCircNodesTot =int(((numEdges+1)*(numEdges+2))/2)
+    
+    b=range(numCircNodesTot)
+    row_size=1
+    A=sp.zeros([numEdges+1,numEdges+1])
+    start=0;stop=1;
+    for m in range(numEdges+1):
+        if m==0:
+            A[m,range(m+1)]=b[0:1]
+            start=0
+            stop=1
+        else:
+            start=stop
+            stop=stop+m+1
+            A[m,range(m+1)]=b[start:stop]
+
+    M=sp.ones([numEdges**2,5])
+    m_row=0
+    for m in range(numEdges):
+        if m==0:
+            M[0,:]=[1,2,0,1,2]
+            m_row+=1
+        else:
+           holder=sp.size(sp.nonzero(A[m,:]))
+           for n in range(holder):
+               if n!=holder-1:
+                   M[m_row,:]=[1,2,A[m,n],A[m,n+1],A[m+1,n+1]]
+                   m_row+=1
+                   M[m_row,:]=[1,2,A[m,n],A[m+1,n],A[m+1,n+1]]
+                   m_row+=1
+               else:
+                   M[m_row,:]=[1,2,A[m,n],A[m+1,n],A[m+1,n+1]]
+                   m_row+=1
+            
+    return M.astype(int),numCircNodesTot
+
+def eleMatQuad(numEdges):
+    S0=numEdges*(numEdges+1)/(2.0)
+    A=sp.linspace(S0,(S0+(numEdges+1)**2)-1,(numEdges+1)**2)
+    A=A.reshape([numEdges+1,numEdges+1])
+    quadNode=sp.delete(A,-1,1)
+    quadNode=sp.delete(quadNode,-1,0)
+    quadNode=quadNode.flatten()
+    M=sp.zeros([numEdges**2,6])
+    for n in range(numEdges**2):
+        M[n,:]=[2,3,quadNode[n],quadNode[n]+1,quadNode[n]+numEdges+2,quadNode[n]+numEdges+1]
+    return M.astype(int)
+
+def boundMatTot(numEdges):
+    triS1=sp.zeros(numEdges+1)
+    triS3=sp.zeros(numEdges+1)
+    quadS1=sp.zeros(numEdges)
+    quadS2=sp.zeros(numEdges-1)
+    quadS3=sp.zeros(numEdges)
+
+    triS1[0]=0;
+    triS3[0]=0;
+    for n in range(1,numEdges+1):
+        triS1[n]=triS1[n-1]+n
+        triS3[n]=triS1[n]+n
+    ref1=triS3
+    
+    triS3=sp.flipud(triS3)
+    quadS1[0]=triS1[-1]+numEdges+1
+    quadS3[0]=triS1[-1]+2*numEdges+1
+
+    for n in range(1,numEdges):
+        quadS1[n]=quadS1[n-1]+(numEdges+1)
+        quadS3[n]=quadS3[n-1]+(numEdges+1)    
+    ref2=quadS3
+    xAxisRootRef=sp.concatenate([triS1.copy(),quadS1],axis=0)
+    quadS3=sp.flipud(quadS3)
+    quadS2=range(int(quadS1[-1]+1),int(quadS3[0]),1)
+    STOT=sp.concatenate([triS1,quadS1,quadS2,quadS3,triS3],axis=0)
+
+    filler=sp.zeros(1)
+    filler[0]=quadS3[0]
+    fillerFirst=sp.zeros(1)
+    fillerFirst[0]=quadS1[-1]
+    sTotRef=sp.concatenate([triS1,quadS1,quadS2,filler],axis=0)
+    newsTotRef=sp.concatenate([fillerFirst,quadS2,filler],axis=0)
+    boundMat=sp.zeros([STOT.size-1,4])
+    boundMatRef=sp.zeros([sTotRef.size-1,4])
+    new_boundMat_ref=sp.zeros([newsTotRef.size-1,4])
+    
+    for n in range(STOT.size-1):
+        boundMat[n,:]=[1,1,STOT[n],STOT[n+1]]
+    for n in range(sTotRef.size-1):
+        boundMatRef[n,:]=[1,1,sTotRef[n],sTotRef[n+1]]
+    for n in range(newsTotRef.size-1):
+        new_boundMat_ref[n,:]=[1,1,newsTotRef[n],newsTotRef[n+1]]
+        
+    ref=sp.concatenate([ref1,ref2],axis=0).astype(int)
+    return boundMat.astype(int),ref,boundMatRef.astype(int),xAxisRootRef.astype(int),new_boundMat_ref.astype(int)
+
+def vertMatCirc(numEdges):
+    r_o=sp.linspace(0,r,numEdges+1)
+    counter=0
+    vertMat=sp.zeros([numCircNodesTot,2])
+    for m in range(numEdges+1):
+        theta=sp.linspace(0,sp.pi/2.0,m+1)
+        for n in range(sp.size(theta)):
+            vertMat[counter,:]=[r_o[m]*sp.cos(theta[n]),r_o[m]*sp.sin(theta[n])]
+            counter+=1
+    return vertMat
+
+def vertMatQuad(numEdges):
+
+    theta=sp.linspace(0,sp.pi/2.0,numEdges+1)
+    AX=sp.zeros([numEdges+1,numEdges+1])
+    AY=sp.zeros([numEdges+1,numEdges+1])
+    AX[0,:]=r*sp.cos(theta)
+    AY[0,:]=r*sp.sin(theta)
+
+    vertLinSpace=sp.linspace(0,edgeLength,(numEdges/2)+1)
+    horzLineSpace=sp.linspace(edgeLength,0,(numEdges/2)+1)
+
+    #Assigning node locations along the boundary
+    vertCount=0
+    horzCount=1
+    for n in range(numEdges+1):
+        if n < (numEdges/2):
+            AX[-1,n]=edgeLength
+            AY[-1,n]=vertLinSpace[vertCount]
+            vertCount+=1
+        elif n == int(numEdges/2):
+            AX[-1,n]=edgeLength
+            AY[-1,n]=edgeLength
+        else:
+            AX[-1,n]=horzLineSpace[horzCount]
+            AY[-1,n]=edgeLength
+            horzCount+=1
+
+    #Linearly spacing nodes between the inner/outer boundaries
+    #One could then smooth this via r-based adaptivity
+    for col in range(numEdges+1):
+        for row in range(1,numEdges):
+            AX[row,col]=sp.linspace(AX[0,col],AX[-1,col],numEdges+1)[row]
+            AY[row,col]=sp.linspace(AY[0,col],AY[-1,col],numEdges+1)[row]
+
+    AX=sp.delete(AX,0,0)
+    AY=sp.delete(AY,0,0)
+    AX=AX.flatten()
+    AY=AY.flatten()
+    AX_reshape = AX.flatten()
+    
+    numQuadNodesTot=numEdges*(numEdges+1)
+    vertMat=sp.zeros([numQuadNodesTot,2])
+    for n in range(numQuadNodesTot):
+        vertMat[n,:]=[AX[n],AY[n]]    
+    return vertMat
+
+def orient(A):
+    aOrient=sp.zeros([A.shape[0],A.shape[1]])
+    triCounter=0
+    quadCounter=0
+    #Determine the number of triangle and quad elments in the given element matrix
+    for n in range(A.shape[0]):
+        if A[n,1]==2:
+            triCounter+=1
+        else:
+            quadCounter+=1
+    edgeMatTotal=sp.zeros([3*triCounter+4*quadCounter,2])
+    counter=0
+    for n in range(A.shape[0]):
+        detected=0
+        if A[n,1]==2:
+            for m in range(edgeMatTotal.shape[0]):
+                if detected != 1:
+                    if edgeMatTotal[m,0]==A[n,2] and edgeMatTotal[m,1]==A[n,3]:
+                        aOrient[n,:]=[1,2,A[n,2],A[n,4],A[n,3],0]
+                        detected=1
+                        #print("reorder:[{} {} {}] to [{} {} {}]".format(A[n,2],A[n,3],A[n,4],int(aOrient[n,2]),int(aOrient[n,3]),int(aOrient[n,4])))
+                    elif edgeMatTotal[m,0]==A[n,4] and edgeMatTotal[m,1]==A[n,2]:
+                        aOrient[n,:]=[1,2,A[n,2],A[n,4],A[n,3],0]
+                        detected=1
+                    else:
+                        aOrient[n,:]=A[n,:]
+
+            edgeMatTotal[counter,:]=[aOrient[n,2],aOrient[n,3]]
+            counter+=1
+            edgeMatTotal[counter,:]=[aOrient[n,3],aOrient[n,4]]
+            counter+=1
+            edgeMatTotal[counter,:]=[aOrient[n,4],aOrient[n,2]]
+            counter+=1
+        else:
+            for m in range(edgeMatTotal.shape[0]):
+                if detected != 1: 
+                    if edgeMatTotal[m,0]==A[n,2] and edgeMatTotal[m,1]==A[n,3]:
+                        aOrient[n,:]=[2,3,A[n,2],A[n,5],A[n,4],A[n,3]]
+                        detected=1
+                        #print("reorder:[{} {} {} {}] to [{} {} {} {}]".format(A[n,2],A[n,3],A[n,4],A[n,5],int(aOrient[n,2]),int(aOrient[n,3]),int(aOrient[n,4]),int(aOrient[n,5])))
+                    elif edgeMatTotal[m,0]==A[n,5] and edgeMatTotal[m,1]==A[n,2]:
+                        aOrient[n,:]=[2,3,A[n,2],A[n,5],A[n,4],A[n,3]]
+                        detected=1
+                    else:
+                        aOrient[n,:]=A[n,:]
+            edgeMatTotal[counter,:]=[aOrient[n,2],aOrient[n,3]]
+            counter+=1
+            edgeMatTotal[counter,:]=[aOrient[n,3],aOrient[n,4]]
+            counter+=1
+            edgeMatTotal[counter,:]=[aOrient[n,4],aOrient[n,5]]
+            counter+=1
+            edgeMatTotal[counter,:]=[aOrient[n,5],aOrient[n,2]]
+            counter+=1
+    return aOrient.astype(int)
+
+def gVis(_glvis,_meshFile):
+
+    if(_glvis==''):
+        print("Failure: Set glvis location via -g switch")
+        sys.exit(1)
+
+    colFuncFileName=_meshFile.replace('.mesh','.gf')
+    glvsScriptFileName=_meshFile.replace('.mesh','.glvs')
+    imageFileName=_meshFile.replace('.mesh','.png')
+    
+    #Create Coloring Function for mesh
+    _colFuncCommand=_glvis+ ' -m '+ _meshFile +' -sc -k q'
+    args=_colFuncCommand.split()
+    p=subprocess.Popen(args)#Create 'GLVis_coloring.gf'
+
+    _renameCommand='mv GLVis_coloring.gf {}'.format(colFuncFileName)
+    args=_renameCommand.split()
+    p=subprocess.Popen(args)
+
+    #Glvis script template
+    f=open(glvsScriptFileName,'w')
+    f.write('window 0 0 800 800\n'+'\n')
+    f.write('solution {} {}\n'.format(_meshFile,colFuncFileName)+'\n')
+    f.write('{\n'+'perspective off\n'+'zoom 1.5\n'+'keys gAeeRM\n'+'solution {} {} screenshot {}\n'.format(_meshFile,colFuncFileName,imageFileName)+'keys q\n'+'}\n')
+    f.close()
+
+    _runGlvisCommand=_glvis+' -run {}'.format(glvsScriptFileName)
+    args=_runGlvisCommand.split()
+    p=subprocess.Popen(args)
+    p.wait()
+    
+    return 0
+def quadInterDof(_edge,_linEleMat,_linVertMatRound):
+    _state=False
+    for n in range(_linEleMat.shape[0]):
+        if _linEleMat[n,1]==3:
+            if sp.any(_edge[0]==_linEleMat[n,2:6]) and sp.any(_edge[1]==_linEleMat[n,2:6]):
+                print("{} is possibly in {}".format(_edge,_linEleMat[n,2:6]))
+                _n1Loc=sp.where(_edge[0]==_linEleMat[n,2:6])[0][0]
+                _n2Loc=sp.where(_edge[1]==_linEleMat[n,2:6])[0][0]
+                if _n1Loc==sp.mod(_n2Loc+1,4) or _n1Loc==sp.mod(_n2Loc-1,4):
+                    _state=True
+                    xcent=(_linVertMatRound[_linEleMat[n,2],0]+_linVertMatRound[_linEleMat[n,3],0]+_linVertMatRound[_linEleMat[n,4],0]+_linVertMatRound[_linEleMat[n,5],0])/4.0
+                    ycent=(_linVertMatRound[_linEleMat[n,2],1]+_linVertMatRound[_linEleMat[n,3],1]+_linVertMatRound[_linEleMat[n,4],1]+_linVertMatRound[_linEleMat[n,5],1])/4.0
+                    _interDof=sp.zeros(2)
+                    _interDof[0]=sp.round_((_linVertMatRound[_edge[0],0]+_linVertMatRound[_edge[1],0]+xcent)/3.0,5)
+                    _interDof[1]=sp.round_((_linVertMatRound[_edge[0],1]+_linVertMatRound[_edge[1],1]+ycent)/3.0,5)
+                    print("dof loc is {},{}".format(_interDof[0],_interDof[1]))
+                    return(_state,_interDof[0],_interDof[1])
+
+    return(_state,0,0)
+
+[eleMatTriHolder,numCircNodesTot]=eleMatCirc(numEdges) #Construct tri element matrix for the region inside circular sector
+eleMatQuadHolder=eleMatQuad(numEdges) #Construct quad element matrix for region outside the circular sector
+
+#Combining eleMatTriHolder and eleMatQuadHolder
+linEleMat=sp.zeros([eleMatTriHolder.shape[0]+eleMatQuadHolder.shape[0],6])
+counter=0
+for n in range(eleMatTriHolder.shape[0]):
+    linEleMat[n,[0,1,2,3,4]]=eleMatTriHolder[n,:]
+    counter+=1
+for n in range(eleMatQuadHolder.shape[0]):
+    linEleMat[counter+n,:]=eleMatQuadHolder[n,:]
+
+linEleMat=linEleMat.astype(int)
+linBoundMat=boundMatTot(numEdges)[0] #Construct the boundary
+vertMatCircHolder = vertMatCirc(numEdges) #Construct vertex matrix for triang region
+vertMatQuadHolder = vertMatQuad(numEdges) #Construct vertex matrix for the quad region
+
+
+#Combining the two vertex matrices in Quadrant I (q1)
+linVertMat=sp.zeros([vertMatCircHolder.shape[0]+vertMatQuadHolder.shape[0],2])
+counter=0
+for n in range(vertMatCircHolder.shape[0]):
+    linVertMat[n,:]=vertMatCircHolder[n,:]
+    counter+=1
+for n in range(vertMatQuadHolder.shape[0]):
+    linVertMat[counter+n,:]=vertMatQuadHolder[n,:]
+
+#Outputting P1/Q1 mesh to a .mesh file
+g=open(outputName+'Lin.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(linEleMat.shape[0]))
+for n in range(linEleMat.shape[0]):
+    if linEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4],linEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(linBoundMat.shape[0]))
+for n in range(linBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(linBoundMat[n,0],linBoundMat[n,1],linBoundMat[n,2],linBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(linVertMat.shape[0])+'2\n')
+for n in range(linVertMat.shape[0]):
+    g.write('{} {}\n'.format(linVertMat[n,0],linVertMat[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'Lin.mesh')
+
+#Quadratic (P2/Q2) Element Generation
+
+#1.)Create Edge list from previously generated linear elements
+edgeMat=sp.zeros([3*eleMatTriHolder.shape[0]+4*eleMatQuadHolder.shape[0],2])
+linEleMat=orient(linEleMat)#Make sure that element orientation is in agreement with MFEM requirements
+
+counter=0
+for n in range(linEleMat.shape[0]):
+    if linEleMat[n,1]==2:
+        edgeMat[counter,:]=[linEleMat[n,2],linEleMat[n,3]]
+        counter+=1
+        edgeMat[counter,:]=[linEleMat[n,3],linEleMat[n,4]]
+        counter+=1
+        edgeMat[counter,:]=[linEleMat[n,4],linEleMat[n,2]]
+        counter+=1
+    else:
+        edgeMat[counter,:]=[linEleMat[n,2],linEleMat[n,3]]
+        counter+=1
+        edgeMat[counter,:]=[linEleMat[n,3],linEleMat[n,4]]
+        counter+=1
+        edgeMat[counter,:]=[linEleMat[n,4],linEleMat[n,5]]
+        counter+=1
+        edgeMat[counter,:]=[linEleMat[n,5],linEleMat[n,2]]
+        counter+=1
+
+#Remove duplicates
+holder=[]
+for n in range(edgeMat.shape[0]):
+    counter=0
+    for m in range(edgeMat.shape[0]):
+        if edgeMat[n,0]==edgeMat[m,0] and edgeMat[n,1]==edgeMat[m,1] and m!=n:
+            holder.append([n,m])
+        elif edgeMat[n,1]==edgeMat[m,0] and edgeMat[n,0]==edgeMat[m,1] and m!=n:
+            holder.append([n,m])
+
+removeIndices=sp.zeros(len(holder))
+for n in range(len(holder)):
+    if holder[n][0]>holder[n][1]:
+        removeIndices[n]=holder[n][0]
+    else:
+        removeIndices[n]=holder[n][1]
+removeIndices=sp.unique(removeIndices).astype(int)
+edgeMat=sp.delete(edgeMat,removeIndices,0)
+edgeMat=edgeMat.astype(int)
+
+edgeDofMat=sp.zeros([edgeMat.shape[0],2])#These will be the new DoFs that appear after the Element Vertices within the .mesh file
+linVertMatRound=sp.round_(linVertMat,5) 
+ 
+counter=0
+
+for n in edgeMat:
+    if linVertMatRound[n[0],1] == linVertMatRound[n[1],1]:
+        xmid=(linVertMatRound[n[0],0]+linVertMatRound[n[1],0])/2.0
+        ymid=linVertMatRound[n[0],1]
+        edgeDofMat[counter,:]=[xmid,ymid]
+    elif linVertMatRound[n[0],0] == linVertMatRound[n[1],0]:
+        xmid=linVertMatRound[n[0],0]
+        ymid=(linVertMatRound[n[0],1]+linVertMatRound[n[1],1])/2.0
+        edgeDofMat[counter,:]=[xmid,ymid]
+    else:
+        r0=sp.sqrt(linVertMatRound[n[0],0]**2+linVertMatRound[n[0],1]**2)
+        r1=sp.sqrt(linVertMatRound[n[1],0]**2+linVertMatRound[n[1],1]**2)
+        rmid = (r0+r1)/2.0 #should not be needed
+        xmidOld=(linVertMatRound[n[0],0]+linVertMatRound[n[1],0])/2.0
+        ymidOld=(linVertMatRound[n[0],1]+linVertMatRound[n[1],1])/2.0
+        midtheta=sp.arctan(ymidOld/xmidOld)
+        xmid=rmid*sp.cos(midtheta)
+        ymid=rmid*sp.sin(midtheta)
+        edgeDofMat[counter,:]=[xmid,ymid]
+    counter+=1
+edgeDofMat = sp.round_(edgeDofMat,5)
+
+#Determine midpoints of all Q1 elements:
+quadCentroidLoc=sp.zeros([eleMatQuadHolder.shape[0],2])
+for n in range(eleMatQuadHolder.shape[0]):
+    quadCentroidLoc[n,0]=(linVertMatRound[eleMatQuadHolder[n,2],0]+linVertMatRound[eleMatQuadHolder[n,3],0]+linVertMatRound[eleMatQuadHolder[n,4],0]+linVertMatRound[eleMatQuadHolder[n,5],0])/4.0
+    quadCentroidLoc[n,1]=(linVertMatRound[eleMatQuadHolder[n,2],1]+linVertMatRound[eleMatQuadHolder[n,3],1]+linVertMatRound[eleMatQuadHolder[n,4],1]+linVertMatRound[eleMatQuadHolder[n,5],1])/4.0
+
+quadCentroidLoc = sp.round_(quadCentroidLoc,5)
+
+#3.)Populate nodes section
+g=open(outputName+'Quad.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(linEleMat.shape[0]))
+for n in range(linEleMat.shape[0]):
+    if linEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4],linEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(linBoundMat.shape[0]))
+for n in range(linBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(linBoundMat[n,0],linBoundMat[n,1],linBoundMat[n,2],linBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(linVertMat.shape[0]))
+g.write('\n'+'nodes'+'\n'+'FiniteElementSpace'+'\n'+'FiniteElementCollection: H1_2D_P2'+'\n'+'VDim: 2'+'\n'+'Ordering: 1' +'\n\n')
+for n in range(linVertMatRound.shape[0]):
+    g.write('{} {}\n'.format(linVertMatRound[n,0],linVertMatRound[n,1]))
+for n in range(edgeDofMat.shape[0]):
+    g.write('{} {}\n'.format(edgeDofMat[n,0],edgeDofMat[n,1]))
+for n in range(quadCentroidLoc.shape[0]):
+    g.write('{} {}\n'.format(quadCentroidLoc[n,0],quadCentroidLoc[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'Quad.mesh')
+
+#Cubic (P3/Q3) Element Generation
+
+cubeDofMat=sp.zeros([2*edgeMat.shape[0],2])#These will be the new DoFs that appear after the Element Vertices within the .mesh file
+
+counter=0
+for n in edgeMat: #Here DoF ordering matters. 
+    if linVertMatRound[n[0],1] == linVertMatRound[n[1],1]:
+        xmid=(linVertMatRound[n[0],0]+linVertMatRound[n[1],0])/2.0
+        ymid=linVertMatRound[n[0],1]
+        xmid1=(linVertMatRound[n[0],0]+xmid)/2.0
+        ymid1=linVertMatRound[n[0],1]
+        xmid2=(linVertMatRound[n[1],0]+xmid)/2.0
+        ymid2=linVertMatRound[n[0],1]
+        if n[0] > n[1]:
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+        else:
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+            
+    elif linVertMatRound[n[0],0] == linVertMatRound[n[1],0]:
+        xmid=linVertMatRound[n[0],0]
+        ymid=(linVertMatRound[n[0],1]+linVertMatRound[n[1],1])/2.0
+        xmid1=linVertMatRound[n[0],0]
+        ymid1=(linVertMatRound[n[0],1]+ymid)/2.0
+        xmid2=linVertMatRound[n[0],0]
+        ymid2=(linVertMatRound[n[1],1]+ymid)/2.0
+        if n[0] > n[1]:
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+        else:
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+    else:
+        r0=sp.sqrt(linVertMatRound[n[0],0]**2+linVertMatRound[n[0],1]**2)
+        r1=sp.sqrt(linVertMatRound[n[1],0]**2+linVertMatRound[n[1],1]**2)
+        rmid = (r0+r1)/2.0 #should not be needed
+        xmidOld=(linVertMatRound[n[0],0]+linVertMatRound[n[1],0])/2.0
+        ymidOld=(linVertMatRound[n[0],1]+linVertMatRound[n[1],1])/2.0
+        midtheta=sp.arctan(ymidOld/xmidOld)
+        xmid=rmid*sp.cos(midtheta)
+        ymid=rmid*sp.sin(midtheta)
+        xmid1=(linVertMatRound[n[0],0]+xmid)/2.0
+        ymid1=(linVertMatRound[n[0],1]+ymid)/2.0
+        xmid2=(linVertMatRound[n[1],0]+xmid)/2.0
+        ymid2=(linVertMatRound[n[1],1]+ymid)/2.0
+        if n[0] > n[1]:
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+        else:
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+            
+cubeDofMat = sp.round_(cubeDofMat,5)
+
+triCentroidLoc=sp.zeros([eleMatTriHolder.shape[0],2])
+
+for n in range(eleMatTriHolder.shape[0]):
+    triCentroidLoc[n,0]=(linVertMatRound[eleMatTriHolder[n,2],0]+linVertMatRound[eleMatTriHolder[n,3],0]+linVertMatRound[eleMatTriHolder[n,4],0])/3.0
+    triCentroidLoc[n,1]=(linVertMatRound[eleMatTriHolder[n,2],1]+linVertMatRound[eleMatTriHolder[n,3],1]+linVertMatRound[eleMatTriHolder[n,4],1])/3.0
+
+quadCentroidLocCubic=sp.zeros([4*eleMatQuadHolder.shape[0],2])
+
+counter=0
+for n in range(eleMatQuadHolder.shape[0]):
+    xcent=quadCentroidLoc[n,0];ycent=quadCentroidLoc[n,1]
+    a=eleMatQuadHolder[n,2:6]
+    aMinIndex=sp.where(a[:]==a.min())[0][0]
+    dof0=0.5*sp.array([xcent+linVertMatRound[a[aMinIndex],0],ycent+linVertMatRound[a[aMinIndex],1]])
+    quadCentroidLocCubic[counter,:]=dof0
+    counter+=1
+    if aMinIndex==0:
+        aLeft=-1
+        aRight=1
+        aLast=2
+    else:
+        aLeft=aMinIndex-1
+        aRight=aMinIndex+1
+        aLast=sp.delete(a,[aMinIndex,aLeft,aRight])[0]
+    edge1=[a[aMinIndex], a[aLeft]]
+    edge2=[a[aMinIndex], a[aRight]]
+    edge1Index=0
+    edge2Index=0
+    edgeCounter=0
+    for edge in edgeMat:
+        if(edge[0]==edge1[0] and edge[1]==edge1[1]) or (edge[1]==edge1[0] and edge[0]==edge1[1]):
+            edge1Index=edgeCounter
+        if(edge[0]==edge2[0] and edge[1]==edge2[1]) or (edge[1]==edge2[0] and edge[0]==edge2[1]):
+            edge2Index=edgeCounter
+        edgeCounter+=1
+
+    if (edge1Index > edge2Index):
+        dof1=0.5*sp.array([xcent+linVertMatRound[a[aLeft],0],ycent+linVertMatRound[a[aLeft],1]])
+        quadCentroidLocCubic[counter,:]=dof1
+        counter+=1
+        dof2=0.5*sp.array([xcent+linVertMatRound[a[aRight],0],ycent+linVertMatRound[a[aRight],1]])
+        quadCentroidLocCubic[counter,:]=dof2
+        counter+=1
+        dof3=0.5*sp.array([xcent+linVertMatRound[a[aLast],0],ycent+linVertMatRound[a[aLast],1]])
+        quadCentroidLocCubic[counter,:]=dof3
+        counter+=1
+    else:
+        dof1=0.5*sp.array([xcent+linVertMatRound[a[aRight],0],ycent+linVertMatRound[a[aRight],1]])
+        quadCentroidLocCubic[counter,:]=dof1
+        counter+=1
+        dof2=0.5*sp.array([xcent+linVertMatRound[a[aLeft],0],ycent+linVertMatRound[a[aLeft],1]])
+        quadCentroidLocCubic[counter,:]=dof2
+        counter+=1
+        dof3=0.5*sp.array([xcent+linVertMatRound[a[aLast],0],ycent+linVertMatRound[a[aLast],1]])
+        quadCentroidLocCubic[counter,:]=dof3
+        counter+=1
+        
+truCentroidLoc=sp.round_(triCentroidLoc,5)
+    
+#3.)Populate nodes section
+g=open(outputName+'Cub.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(linEleMat.shape[0]))
+for n in range(linEleMat.shape[0]):
+    if linEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4],linEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(linBoundMat.shape[0]))
+for n in range(linBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(linBoundMat[n,0],linBoundMat[n,1],linBoundMat[n,2],linBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(linVertMat.shape[0]))
+g.write('\n'+'nodes'+'\n'+'FiniteElementSpace'+'\n'+'FiniteElementCollection: H1_2D_P3'+'\n'+'VDim: 2'+'\n'+'Ordering: 1' +'\n\n')
+for n in range(linVertMatRound.shape[0]):
+    g.write('{} {}\n'.format(linVertMatRound[n,0],linVertMatRound[n,1]))
+for n in range(cubeDofMat.shape[0]):
+    g.write('{} {}\n'.format(cubeDofMat[n,0],cubeDofMat[n,1]))
+for n in range(triCentroidLoc.shape[0]):
+    g.write('{} {}\n'.format(triCentroidLoc[n,0],triCentroidLoc[n,1]))
+for n in range(quadCentroidLocCubic.shape[0]):
+    g.write('{} {}\n'.format(quadCentroidLocCubic[n,0],quadCentroidLocCubic[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'Cub.mesh')
+#raw_input()
+#'Reflecting' topology about one of its edges and append it to itself
+upperPlaneEleMat = sp.zeros([2*linEleMat.shape[0],6])
+for n in range(linEleMat.shape[0]):
+    upperPlaneEleMat[n,:]=linEleMat[n,:]
+    
+#Create ele_mat_holder.shape[0]x2 matrix for mapping
+refEdge=boundMatTot(numEdges)[1]
+q1NumNodes=linVertMat.shape[0]
+
+mapping = sp.zeros([q1NumNodes])
+counter=0
+for n in range(q1NumNodes):
+     if (sp.any(refEdge == n)):
+         mapping[n]=n
+     else:
+         mapping[n]=counter+q1NumNodes
+         counter+=1
+
+mapping=mapping.astype(int)
+#Implement mapping
+
+counter=0
+for n in range(linEleMat.shape[0],2*linEleMat.shape[0]):
+    upperPlaneEleMat[n,0]=linEleMat[counter,0]
+    upperPlaneEleMat[n,1]=linEleMat[counter,1]
+    upperPlaneEleMat[n,2]=mapping[linEleMat[counter,2]]
+    upperPlaneEleMat[n,3]=mapping[linEleMat[counter,3]]
+    upperPlaneEleMat[n,4]=mapping[linEleMat[counter,4]]
+    upperPlaneEleMat[n,5]=mapping[linEleMat[counter,5]]
+    counter+=1
+    
+upperPlaneEleMat = upperPlaneEleMat.astype(int)
+
+#Reflecting boundary matrix
+origBound=boundMatTot(numEdges)[2]
+upperPlaneBoundMat=sp.zeros([2*origBound.shape[0],4])
+for n in range(origBound.shape[0]):
+    upperPlaneBoundMat[n,:]=origBound[n,:]
+counter=0
+newOrigBound=origBound.copy()
+newOrigBound[:,2]=sp.flipud(origBound[:,3])
+newOrigBound[:,3]=sp.flipud(origBound[:,2])
+for n in range(newOrigBound.shape[0],upperPlaneBoundMat.shape[0]):
+    upperPlaneBoundMat[n,0]=newOrigBound[counter,0]
+    upperPlaneBoundMat[n,1]=newOrigBound[counter,1]
+    upperPlaneBoundMat[n,2]=mapping[newOrigBound[counter,2]]
+    upperPlaneBoundMat[n,3]=mapping[newOrigBound[counter,3]]
+    counter+=1
+upperPlaneBoundMat=upperPlaneBoundMat.astype(int)
+
+#Reflecting vertex matrix about the y-axis and appending it to itself
+upperPlaneNumNodes=q1NumNodes+(q1NumNodes-refEdge.shape[0])
+upperPlaneVertMat = sp.zeros([upperPlaneNumNodes,2])
+for n in range(linVertMat.shape[0]):
+    upperPlaneVertMat[n,:]=linVertMat[n,:]
+counter=0
+for n in range(linVertMat.shape[0],upperPlaneNumNodes):
+        upperPlaneVertMat[n,0]=-1.0*linVertMat[sp.where(mapping==n)[0][0],0]
+        upperPlaneVertMat[n,1]=linVertMat[sp.where(mapping==n)[0][0],1]
+        counter+=1
+
+upperPlaneEleMat=orient(upperPlaneEleMat)
+g=open(outputName+'UpperPlaneLin.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(upperPlaneEleMat.shape[0]))
+for n in range(upperPlaneEleMat.shape[0]):
+    if upperPlaneEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(upperPlaneEleMat[n,0],upperPlaneEleMat[n,1],upperPlaneEleMat[n,2],upperPlaneEleMat[n,3],upperPlaneEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(upperPlaneEleMat[n,0],upperPlaneEleMat[n,1],upperPlaneEleMat[n,2],upperPlaneEleMat[n,3],upperPlaneEleMat[n,4],upperPlaneEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(upperPlaneBoundMat.shape[0]))
+for n in range(upperPlaneBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(upperPlaneBoundMat[n,0],upperPlaneBoundMat[n,1],upperPlaneBoundMat[n,2],upperPlaneBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(upperPlaneVertMat.shape[0])+'2\n')
+for n in range(upperPlaneVertMat.shape[0]):
+    g.write('{} {}\n'.format(upperPlaneVertMat[n,0],upperPlaneVertMat[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'UpperPlaneLin.mesh')
+
+#'Reflecting' topology about one of its edges and append it to itself
+wholePlaneEleMat = sp.zeros([2*upperPlaneEleMat.shape[0],6])
+for n in range(upperPlaneEleMat.shape[0]):
+    wholePlaneEleMat[n,:]=upperPlaneEleMat[n,:]
+
+quad1Edge=boundMatTot(numEdges)[3]
+newRefEdge=sp.zeros(2*quad1Edge.shape[0]-1)
+for n in range(quad1Edge.shape[0]):
+    newRefEdge[n]=quad1Edge[n]
+counter=0
+for n in range(quad1Edge.shape[0],newRefEdge.shape[0]):
+    newRefEdge[n]=mapping[quad1Edge[counter]]
+    counter+=1
+newRefEdge=sp.unique(newRefEdge)
+newRefEdge=newRefEdge.astype(int)
+newTotNumNodes=upperPlaneVertMat.shape[0]
+
+newMapping=sp.zeros([newTotNumNodes])
+counter=0
+for n in range(newTotNumNodes):
+    if (sp.any(newRefEdge == n)):
+        newMapping[n]=n
+    else:
+        newMapping[n]=counter+newTotNumNodes
+        counter+=1
+newMapping=newMapping.astype(int)
+
+counter=0
+for n in range(upperPlaneEleMat.shape[0],2*upperPlaneEleMat.shape[0]):
+    wholePlaneEleMat[n,0]=upperPlaneEleMat[counter,0]
+    wholePlaneEleMat[n,1]=upperPlaneEleMat[counter,1]
+    wholePlaneEleMat[n,2]=newMapping[upperPlaneEleMat[counter,2]]
+    wholePlaneEleMat[n,3]=newMapping[upperPlaneEleMat[counter,3]]
+    wholePlaneEleMat[n,4]=newMapping[upperPlaneEleMat[counter,4]]
+    wholePlaneEleMat[n,5]=newMapping[upperPlaneEleMat[counter,5]]
+    counter+=1
+
+wholePlaneEleMat=wholePlaneEleMat.astype(int)
+
+#Reflecting boundary matrix
+newOrigBoundQuad1=boundMatTot(numEdges)[4]
+newFirstBoundMatHolder=sp.zeros([2*newOrigBoundQuad1.shape[0],4])
+for n in range(newOrigBoundQuad1.shape[0]):
+    newFirstBoundMatHolder[n,:]=newOrigBoundQuad1[n,:]
+
+newNewOrigBoundQuad1=newOrigBoundQuad1.copy()
+newNewOrigBoundQuad1[:,2]=sp.flipud(newOrigBoundQuad1[:,3])
+newNewOrigBoundQuad1[:,3]=sp.flipud(newOrigBoundQuad1[:,2])
+counter=0
+for n in range(newOrigBoundQuad1.shape[0],newFirstBoundMatHolder.shape[0]):
+    newFirstBoundMatHolder[n,0]=newNewOrigBoundQuad1[counter,0]
+    newFirstBoundMatHolder[n,1]=newNewOrigBoundQuad1[counter,1]
+    newFirstBoundMatHolder[n,2]=mapping[newNewOrigBoundQuad1[counter,2]]
+    newFirstBoundMatHolder[n,3]=mapping[newNewOrigBoundQuad1[counter,3]]
+    counter+=1
+
+upperQuadMat=newFirstBoundMatHolder.copy()
+wholePlaneBoundMat=sp.zeros([2*upperQuadMat.shape[0],4])
+for n in range(upperQuadMat.shape[0]):
+    wholePlaneBoundMat[n,:]=upperQuadMat[n,:]
+
+counter=0
+newNewOrigBound=upperQuadMat.copy()
+newNewOrigBound[:,2]=sp.flipud(upperQuadMat[:,3])
+newNewOrigBound[:,3]=sp.flipud(upperQuadMat[:,2])
+newNewOrigBound=newNewOrigBound.astype(int)
+for n in range(newNewOrigBound.shape[0],wholePlaneBoundMat.shape[0]):
+    wholePlaneBoundMat[n,0]=newNewOrigBound[counter,0]
+    wholePlaneBoundMat[n,1]=newNewOrigBound[counter,1]
+    wholePlaneBoundMat[n,2]=newMapping[newNewOrigBound[counter,2]]
+    wholePlaneBoundMat[n,3]=newMapping[newNewOrigBound[counter,3]]
+    counter+=1
+wholePlaneBoundMat=wholePlaneBoundMat.astype(int)
+
+wholePlaneNumNodes=newTotNumNodes+(newTotNumNodes-newRefEdge.shape[0])
+wholePlaneVertMat = sp.zeros([wholePlaneNumNodes,2])
+for n in range(upperPlaneVertMat.shape[0]):
+    wholePlaneVertMat[n,:]=upperPlaneVertMat[n,:]
+counter=0
+for n in range(upperPlaneVertMat.shape[0],wholePlaneNumNodes):
+        wholePlaneVertMat[n,0]=upperPlaneVertMat[sp.where(newMapping==n)[0][0],0]
+        wholePlaneVertMat[n,1]=-1.0*upperPlaneVertMat[sp.where(newMapping==n)[0][0],1]
+        counter+=1
+
+g=open(outputName+'WholePlaneLin.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(wholePlaneEleMat.shape[0]))
+for n in range(wholePlaneEleMat.shape[0]):
+    if wholePlaneEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(wholePlaneEleMat[n,0],wholePlaneEleMat[n,1],wholePlaneEleMat[n,2],wholePlaneEleMat[n,3],wholePlaneEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(wholePlaneEleMat[n,0],wholePlaneEleMat[n,1],wholePlaneEleMat[n,2],wholePlaneEleMat[n,3],wholePlaneEleMat[n,4],wholePlaneEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(wholePlaneBoundMat.shape[0]))
+for n in range(wholePlaneBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(wholePlaneBoundMat[n,0],wholePlaneBoundMat[n,1],wholePlaneBoundMat[n,2],wholePlaneBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(wholePlaneVertMat.shape[0])+'2\n')
+for n in range(wholePlaneVertMat.shape[0]):
+    g.write('{} {}\n'.format(wholePlaneVertMat[n,0],wholePlaneVertMat[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'WholePlaneLin.mesh')
+
+#1.)Create Edge list from elements
+wholePlaneEleMat=orient(wholePlaneEleMat)
+triCounter=0;quadCounter=0;
+for n in range(wholePlaneEleMat.shape[0]):
+    if wholePlaneEleMat[n,1]==2:
+        triCounter+=1
+    else:
+        quadCounter+=1
+edgeMat=sp.zeros([3*triCounter+4*quadCounter,2])
+counter=0
+for n in range(wholePlaneEleMat.shape[0]):
+    if wholePlaneEleMat[n,1]==2:
+        edgeMat[counter,:]=[wholePlaneEleMat[n,2],wholePlaneEleMat[n,3]]
+        counter+=1
+        edgeMat[counter,:]=[wholePlaneEleMat[n,3],wholePlaneEleMat[n,4]]
+        counter+=1
+        edgeMat[counter,:]=[wholePlaneEleMat[n,4],wholePlaneEleMat[n,2]]
+        counter+=1
+    else:
+        edgeMat[counter,:]=[wholePlaneEleMat[n,2],wholePlaneEleMat[n,3]]
+        counter+=1
+        edgeMat[counter,:]=[wholePlaneEleMat[n,3],wholePlaneEleMat[n,4]]
+        counter+=1
+        edgeMat[counter,:]=[wholePlaneEleMat[n,4],wholePlaneEleMat[n,5]]
+        counter+=1
+        edgeMat[counter,:]=[wholePlaneEleMat[n,5],wholePlaneEleMat[n,2]]
+        counter+=1
+
+#Remove duplicates
+holder=[]
+for n in range(edgeMat.shape[0]):
+    counter=0
+    for m in range(edgeMat.shape[0]):
+        if edgeMat[n,0]==edgeMat[m,0] and edgeMat[n,1]==edgeMat[m,1] and m!=n:
+            holder.append([n,m])
+        elif edgeMat[n,1]==edgeMat[m,0] and edgeMat[n,0]==edgeMat[m,1] and m!=n:
+            holder.append([n,m])
+removeIndices=sp.zeros(len(holder))
+for n in range(len(holder)):
+    if holder[n][0]>holder[n][1]:
+        removeIndices[n]=holder[n][0]
+    else:
+        removeIndices[n]=holder[n][1]
+removeIndices=sp.unique(removeIndices).astype(int)
+edgeMat=sp.delete(edgeMat,removeIndices,0)
+edgeMat=edgeMat.astype(int)
+
+edgeDofMat=sp.zeros([edgeMat.shape[0],2])
+
+wholePlaneVertMatRound=sp.round_(wholePlaneVertMat,5)
+
+counter=0
+for n in edgeMat:
+    if wholePlaneVertMatRound[n[0],1] == wholePlaneVertMatRound[n[1],1]:
+        xmid=(wholePlaneVertMatRound[n[0],0]+wholePlaneVertMatRound[n[1],0])/2.0
+        ymid=wholePlaneVertMatRound[n[0],1]
+        edgeDofMat[counter,:]=[xmid,ymid]
+    elif wholePlaneVertMatRound[n[0],0] == wholePlaneVertMatRound[n[1],0]:
+        xmid=wholePlaneVertMatRound[n[0],0]
+        ymid=(wholePlaneVertMatRound[n[0],1]+wholePlaneVertMatRound[n[1],1])/2.0
+        edgeDofMat[counter,:]=[xmid,ymid]
+    else:
+        r0=sp.sqrt(wholePlaneVertMatRound[n[0],0]**2+wholePlaneVertMatRound[n[0],1]**2)
+        r1=sp.sqrt(wholePlaneVertMatRound[n[1],0]**2+wholePlaneVertMatRound[n[1],1]**2)
+        rmid = (r0+r1)/2.0 #should not be needed
+        xmidOld=(wholePlaneVertMatRound[n[0],0]+wholePlaneVertMatRound[n[1],0])/2.0
+        ymidOld=(wholePlaneVertMatRound[n[0],1]+wholePlaneVertMatRound[n[1],1])/2.0
+        midtheta=sp.arctan2(ymidOld,xmidOld)
+        xmid=rmid*sp.cos(midtheta)
+        ymid=rmid*sp.sin(midtheta)
+        edgeDofMat[counter,:]=[xmid,ymid]
+    counter+=1
+edgeDofMat = sp.round_(edgeDofMat,5)
+
+#2.)Create correct dof locations
+#Determine midpoints of all quads:
+quadCentroidLoc=sp.zeros([quadCounter,2])
+counter=0
+for n in range(wholePlaneEleMat.shape[0]):
+    if wholePlaneEleMat[n,1]==3:
+        quadCentroidLoc[counter,0]=(wholePlaneVertMatRound[wholePlaneEleMat[n,2],0]+wholePlaneVertMatRound[wholePlaneEleMat[n,3],0]+wholePlaneVertMatRound[wholePlaneEleMat[n,4],0]+wholePlaneVertMatRound[wholePlaneEleMat[n,5],0])/4.0
+        quadCentroidLoc[counter,1]=(wholePlaneVertMatRound[wholePlaneEleMat[n,2],1]+wholePlaneVertMatRound[wholePlaneEleMat[n,3],1]+wholePlaneVertMatRound[wholePlaneEleMat[n,4],1]+wholePlaneVertMatRound[wholePlaneEleMat[n,5],1])/4.0
+        counter+=1
+
+quadCentroidLoc = sp.round_(quadCentroidLoc,5)
+
+#3.)Populate nodes section
+g=open(outputName+'WholePlaneQuad.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(wholePlaneEleMat.shape[0]))
+for n in range(wholePlaneEleMat.shape[0]):
+    if wholePlaneEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(wholePlaneEleMat[n,0],wholePlaneEleMat[n,1],wholePlaneEleMat[n,2],wholePlaneEleMat[n,3],wholePlaneEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(wholePlaneEleMat[n,0],wholePlaneEleMat[n,1],wholePlaneEleMat[n,2],wholePlaneEleMat[n,3],wholePlaneEleMat[n,4],wholePlaneEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(wholePlaneBoundMat.shape[0]))
+for n in range(wholePlaneBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(wholePlaneBoundMat[n,0],wholePlaneBoundMat[n,1],wholePlaneBoundMat[n,2],wholePlaneBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(wholePlaneVertMat.shape[0]))
+g.write('\n'+'nodes'+'\n'+'FiniteElementSpace'+'\n'+'FiniteElementCollection: H1_2D_P2'+'\n'+'VDim: 2'+'\n'+'Ordering: 1' +'\n\n')
+for n in range(wholePlaneVertMatRound.shape[0]):
+    g.write('{} {}\n'.format(wholePlaneVertMatRound[n,0],wholePlaneVertMatRound[n,1]))
+for n in range(edgeDofMat.shape[0]):
+    g.write('{} {}\n'.format(edgeDofMat[n,0],edgeDofMat[n,1]))
+for n in range(quadCentroidLoc.shape[0]):
+    g.write('{} {}\n'.format(quadCentroidLoc[n,0],quadCentroidLoc[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'WholePlaneQuad.mesh')

data/mixed-ele-meshes/comp_tri_quad_4_edges.mesh

@@ -0,0 +1,264 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+128
+1 2 0 1 2
+1 2 1 2 4
+1 2 1 3 4
+1 2 2 4 5
+1 2 3 4 7
+1 2 3 6 7
+1 2 4 5 8
+1 2 4 7 8
+1 2 5 8 9
+1 2 6 7 11
+1 2 6 10 11
+1 2 7 8 12
+1 2 7 11 12
+1 2 8 9 13
+1 2 8 12 13
+1 2 9 13 14
+2 3 10 15 16 11
+2 3 11 16 17 12
+2 3 12 17 18 13
+2 3 13 18 19 14
+2 3 15 20 21 16
+2 3 16 21 22 17
+2 3 17 22 23 18
+2 3 18 23 24 19
+2 3 20 25 26 21
+2 3 21 26 27 22
+2 3 22 27 28 23
+2 3 23 28 29 24
+2 3 25 30 31 26
+2 3 26 31 32 27
+2 3 27 32 33 28
+2 3 28 33 34 29
+1 2 0 35 2
+1 2 35 2 37
+1 2 35 36 37
+1 2 2 37 5
+1 2 36 37 39
+1 2 36 38 39
+1 2 37 5 40
+1 2 37 39 40
+1 2 5 40 9
+1 2 38 39 42
+1 2 38 41 42
+1 2 39 40 43
+1 2 39 42 43
+1 2 40 9 44
+1 2 40 43 44
+1 2 9 44 14
+2 3 41 45 46 42
+2 3 42 46 47 43
+2 3 43 47 48 44
+2 3 44 48 19 14
+2 3 45 49 50 46
+2 3 46 50 51 47
+2 3 47 51 52 48
+2 3 48 52 24 19
+2 3 49 53 54 50
+2 3 50 54 55 51
+2 3 51 55 56 52
+2 3 52 56 29 24
+2 3 53 57 58 54
+2 3 54 58 59 55
+2 3 55 59 60 56
+2 3 56 60 34 29
+1 2 0 1 61
+1 2 1 61 62
+1 2 1 3 62
+1 2 61 62 63
+1 2 3 62 64
+1 2 3 6 64
+1 2 62 63 65
+1 2 62 64 65
+1 2 63 65 66
+1 2 6 64 67
+1 2 6 10 67
+1 2 64 65 68
+1 2 64 67 68
+1 2 65 66 69
+1 2 65 68 69
+1 2 66 69 70
+2 3 10 15 71 67
+2 3 67 71 72 68
+2 3 68 72 73 69
+2 3 69 73 74 70
+2 3 15 20 75 71
+2 3 71 75 76 72
+2 3 72 76 77 73
+2 3 73 77 78 74
+2 3 20 25 79 75
+2 3 75 79 80 76
+2 3 76 80 81 77
+2 3 77 81 82 78
+2 3 25 30 83 79
+2 3 79 83 84 80
+2 3 80 84 85 81
+2 3 81 85 86 82
+1 2 0 35 61
+1 2 35 61 87
+1 2 35 36 87
+1 2 61 87 63
+1 2 36 87 88
+1 2 36 38 88
+1 2 87 63 89
+1 2 87 88 89
+1 2 63 89 66
+1 2 38 88 90
+1 2 38 41 90
+1 2 88 89 91
+1 2 88 90 91
+1 2 89 66 92
+1 2 89 91 92
+1 2 66 92 70
+2 3 41 45 93 90
+2 3 90 93 94 91
+2 3 91 94 95 92
+2 3 92 95 74 70
+2 3 45 49 96 93
+2 3 93 96 97 94
+2 3 94 97 98 95
+2 3 95 98 78 74
+2 3 49 53 99 96
+2 3 96 99 100 97
+2 3 97 100 101 98
+2 3 98 101 82 78
+2 3 53 102 103 99
+2 3 99 103 104 100
+2 3 100 104 105 101
+2 3 101 105 86 82
+
+boundary
+16
+1 1 30 31
+1 1 31 32
+1 1 32 33
+1 1 33 34
+1 1 34 60
+1 1 60 59
+1 1 59 58
+1 1 58 57
+1 1 102 103
+1 1 103 104
+1 1 104 105
+1 1 105 86
+1 1 86 85
+1 1 85 84
+1 1 84 83
+1 1 83 30
+
+vertices
+106
+2
+0.0 0.0
+0.125 0.0
+7.65404249467e-18 0.125
+0.25 0.0
+0.176776695297 0.176776695297
+1.53080849893e-17 0.25
+0.375 0.0
+0.324759526419 0.1875
+0.1875 0.324759526419
+2.2962127484e-17 0.375
+0.5 0.0
+0.461939766256 0.191341716183
+0.353553390593 0.353553390593
+0.191341716183 0.461939766256
+3.06161699787e-17 0.5
+0.625 0.0
+0.596454824692 0.268506287137
+0.515165042945 0.515165042945
+0.268506287137 0.596454824692
+2.2962127484e-17 0.625
+0.75 0.0
+0.730969883128 0.345670858091
+0.676776695297 0.676776695297
+0.345670858091 0.730969883128
+1.53080849893e-17 0.75
+0.875 0.0
+0.865484941564 0.422835429046
+0.838388347648 0.838388347648
+0.422835429046 0.865484941564
+7.65404249467e-18 0.875
+1.0 0.0
+1.0 0.5
+1.0 1.0
+0.5 1.0
+0.0 1.0
+-0.125 0.0
+-0.25 0.0
+-0.176776695297 0.176776695297
+-0.375 0.0
+-0.324759526419 0.1875
+-0.1875 0.324759526419
+-0.5 0.0
+-0.461939766256 0.191341716183
+-0.353553390593 0.353553390593
+-0.191341716183 0.461939766256
+-0.625 0.0
+-0.596454824692 0.268506287137
+-0.515165042945 0.515165042945
+-0.268506287137 0.596454824692
+-0.75 0.0
+-0.730969883128 0.345670858091
+-0.676776695297 0.676776695297
+-0.345670858091 0.730969883128
+-0.875 0.0
+-0.865484941564 0.422835429046
+-0.838388347648 0.838388347648
+-0.422835429046 0.865484941564
+-1.0 0.0
+-1.0 0.5
+-1.0 1.0
+-0.5 1.0
+7.65404249467e-18 -0.125
+0.176776695297 -0.176776695297
+1.53080849893e-17 -0.25
+0.324759526419 -0.1875
+0.1875 -0.324759526419
+2.2962127484e-17 -0.375
+0.461939766256 -0.191341716183
+0.353553390593 -0.353553390593
+0.191341716183 -0.461939766256
+3.06161699787e-17 -0.5
+0.596454824692 -0.268506287137
+0.515165042945 -0.515165042945
+0.268506287137 -0.596454824692
+2.2962127484e-17 -0.625
+0.730969883128 -0.345670858091
+0.676776695297 -0.676776695297
+0.345670858091 -0.730969883128
+1.53080849893e-17 -0.75
+0.865484941564 -0.422835429046
+0.838388347648 -0.838388347648
+0.422835429046 -0.865484941564
+7.65404249467e-18 -0.875
+1.0 -0.5
+1.0 -1.0
+0.5 -1.0
+0.0 -1.0
+-0.176776695297 -0.176776695297
+-0.324759526419 -0.1875
+-0.1875 -0.324759526419
+-0.461939766256 -0.191341716183
+-0.353553390593 -0.353553390593
+-0.191341716183 -0.461939766256
+-0.596454824692 -0.268506287137
+-0.515165042945 -0.515165042945
+-0.268506287137 -0.596454824692
+-0.730969883128 -0.345670858091
+-0.676776695297 -0.676776695297
+-0.345670858091 -0.730969883128
+-0.865484941564 -0.422835429046
+-0.838388347648 -0.838388347648
+-0.422835429046 -0.865484941564
+-1.0 -0.0
+-1.0 -0.5
+-1.0 -1.0
+-0.5 -1.0

data/mixed-ele-meshes/spec-3-mixed-type.mesh

@@ -0,0 +1,72 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+#
+
+dimension
+2
+
+elements
+17
+1 2 0 1 2
+1 2 0 2 3
+1 2 0 3 4
+2 3 0 4 5 6
+2 3 0 6 7 1
+1 2 7 8 1
+1 2 1 8 9
+2 3 1 9 10 2
+1 2 2 10 11
+2 3 2 11 12 3
+1 2 3 12 13
+2 3 3 13 14 4
+1 2 4 14 15
+1 2 4 15 5
+1 2 5 16 6 
+1 2 6 16 17
+1 2 6 17 7
+
+boundary
+12
+1 1 7 8
+1 1 8 9
+1 1 9 10
+1 1 10 11
+1 1 11 12
+1 1 12 13
+1 1 13 14
+1 1 14 15
+1 1 15 5
+1 1 5 16
+1 1 16 17
+1 1 17 7
+
+vertices
+18
+2
+0 0
+1 0
+0.5 0.866025
+-0.5 0.866025
+-1 0
+-1 -1
+0 -1
+1 -1
+1.866025 -0.5
+1.866025 0.5
+1.366025 1.366025
+0.5 1.866025
+-0.5 1.866025
+-1.366025 1.366025
+-1.866025 0.5
+-1.866025 -0.5
+-0.5 -1.866025
+0.5 -1.866025

data/mixed-ele-meshes/spec-3-trans-and-ref.mesh

@@ -0,0 +1,362 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+154
+2 3 0 1 2 3
+2 3 1 5 6 2
+2 3 5 8 9 6
+2 3 8 11 12 9
+2 3 11 14 15 12
+2 3 14 17 18 15
+2 3 17 20 21 18
+2 3 20 23 24 21
+2 3 23 26 27 24
+2 3 26 29 30 27
+2 3 29 32 33 30
+2 3 32 35 36 33
+2 3 35 38 39 36
+2 3 38 41 42 39
+2 3 41 44 45 42
+2 3 44 47 48 45
+2 3 47 50 51 48
+2 3 50 53 54 51
+2 3 53 56 57 54
+2 3 56 59 60 57
+2 3 59 62 63 60
+2 3 62 65 66 63
+2 3 65 68 69 66
+2 3 68 71 72 69
+2 3 71 74 75 72
+2 3 74 77 78 75
+1 2 2 3 4
+1 2 6 2 7
+1 2 9 6 10
+1 2 12 9 13
+1 2 15 12 16
+1 2 18 15 19
+1 2 21 18 22
+1 2 24 21 25
+1 2 27 24 28
+1 2 30 27 31
+1 2 33 30 34
+1 2 36 33 37
+1 2 39 36 40
+1 2 42 39 43
+1 2 45 42 46
+1 2 48 45 49
+1 2 51 48 52
+1 2 54 51 55
+1 2 57 54 58
+1 2 60 57 61
+1 2 63 60 64
+1 2 66 63 67
+1 2 69 66 70
+1 2 72 69 73
+1 2 75 72 76
+1 2 78 75 79
+1 2 2 4 7
+1 2 6 7 10
+1 2 9 10 13
+1 2 12 13 16
+1 2 15 16 19
+1 2 18 19 22
+1 2 21 22 25
+1 2 24 25 28
+1 2 27 28 31
+1 2 30 31 34
+1 2 33 34 37
+1 2 36 37 40
+1 2 39 40 43
+1 2 42 43 46
+1 2 45 46 49
+1 2 48 49 52
+1 2 51 52 55
+1 2 54 55 58
+1 2 57 58 61
+1 2 60 61 64
+1 2 63 64 67
+1 2 66 67 70
+1 2 69 70 73
+1 2 72 73 76
+1 2 75 76 79
+2 3 80 81 82 83
+2 3 81 84 85 82
+2 3 84 86 87 85
+2 3 86 88 89 87
+2 3 88 90 91 89
+2 3 90 92 93 91
+2 3 92 94 95 93
+2 3 94 96 97 95
+2 3 96 98 99 97
+2 3 98 100 101 99
+2 3 100 102 103 101
+2 3 102 104 105 103
+2 3 104 106 107 105
+2 3 106 108 109 107
+2 3 108 110 111 109
+2 3 110 112 113 111
+2 3 112 114 115 113
+2 3 114 116 117 115
+2 3 116 118 119 117
+2 3 118 120 121 119
+2 3 120 122 123 121
+2 3 122 124 125 123
+2 3 124 126 127 125
+2 3 126 128 129 127
+2 3 128 130 131 129
+2 3 130 132 133 131
+1 2 82 83 4
+1 2 85 82 7
+1 2 87 85 10
+1 2 89 87 13
+1 2 91 89 16
+1 2 93 91 19
+1 2 95 93 22
+1 2 97 95 25
+1 2 99 97 28
+1 2 101 99 31
+1 2 103 101 34
+1 2 105 103 37
+1 2 107 105 40
+1 2 109 107 43
+1 2 111 109 46
+1 2 113 111 49
+1 2 115 113 52
+1 2 117 115 55
+1 2 119 117 58
+1 2 121 119 61
+1 2 123 121 64
+1 2 125 123 67
+1 2 127 125 70
+1 2 129 127 73
+1 2 131 129 76
+1 2 133 131 79
+1 2 82 4 7
+1 2 85 7 10
+1 2 87 10 13
+1 2 89 13 16
+1 2 91 16 19
+1 2 93 19 22
+1 2 95 22 25
+1 2 97 25 28
+1 2 99 28 31
+1 2 101 31 34
+1 2 103 34 37
+1 2 105 37 40
+1 2 107 40 43
+1 2 109 43 46
+1 2 111 46 49
+1 2 113 49 52
+1 2 115 52 55
+1 2 117 55 58
+1 2 119 58 61
+1 2 121 61 64
+1 2 123 64 67
+1 2 125 67 70
+1 2 127 70 73
+1 2 129 73 76
+1 2 131 76 79
+
+boundary
+60
+1 1 0 1
+1 1 1 5
+1 1 5 8
+1 1 8 11
+1 1 11 14
+1 1 14 17
+1 1 17 20
+1 1 20 23
+1 1 23 26
+1 1 26 29
+1 1 29 32
+1 1 32 35
+1 1 35 38
+1 1 38 41
+1 1 41 44
+1 1 44 47
+1 1 47 50
+1 1 50 53
+1 1 53 56
+1 1 56 59
+1 1 59 62
+1 1 62 65
+1 1 65 68
+1 1 68 71
+1 1 71 74
+1 1 74 77
+1 1 77 78
+1 1 78 79
+1 1 79 133
+1 1 133 132
+1 1 132 130
+1 1 130 128
+1 1 128 126
+1 1 126 124
+1 1 124 122
+1 1 122 120
+1 1 120 118
+1 1 118 116
+1 1 116 114
+1 1 114 112
+1 1 112 110
+1 1 110 108
+1 1 108 106
+1 1 106 104
+1 1 104 102
+1 1 102 100
+1 1 100 98
+1 1 98 96
+1 1 96 94
+1 1 94 92
+1 1 92 90
+1 1 90 88
+1 1 88 86
+1 1 86 84
+1 1 84 81
+1 1 81 80
+1 1 80 83
+1 1 83 4
+1 1 4 3
+1 1 3 0
+
+vertices
+134
+2
+0.0 0.0
+1.0 0.0
+1.0 1.0
+0.0 1.0
+0.5 1.86602540378
+2.0 0.0
+2.0 1.0
+1.5 1.86602540378
+3.0 0.0
+3.0 1.0
+2.5 1.86602540378
+4.0 0.0
+4.0 1.0
+3.5 1.86602540378
+5.0 0.0
+5.0 1.0
+4.5 1.86602540378
+6.0 0.0
+6.0 1.0
+5.5 1.86602540378
+7.0 0.0
+7.0 1.0
+6.5 1.86602540378
+8.0 0.0
+8.0 1.0
+7.5 1.86602540378
+9.0 0.0
+9.0 1.0
+8.5 1.86602540378
+10.0 0.0
+10.0 1.0
+9.5 1.86602540378
+11.0 0.0
+11.0 1.0
+10.5 1.86602540378
+12.0 0.0
+12.0 1.0
+11.5 1.86602540378
+13.0 0.0
+13.0 1.0
+12.5 1.86602540378
+14.0 0.0
+14.0 1.0
+13.5 1.86602540378
+15.0 0.0
+15.0 1.0
+14.5 1.86602540378
+16.0 0.0
+16.0 1.0
+15.5 1.86602540378
+17.0 0.0
+17.0 1.0
+16.5 1.86602540378
+18.0 0.0
+18.0 1.0
+17.5 1.86602540378
+19.0 0.0
+19.0 1.0
+18.5 1.86602540378
+20.0 0.0
+20.0 1.0
+19.5 1.86602540378
+21.0 0.0
+21.0 1.0
+20.5 1.86602540378
+22.0 0.0
+22.0 1.0
+21.5 1.86602540378
+23.0 0.0
+23.0 1.0
+22.5 1.86602540378
+24.0 0.0
+24.0 1.0
+23.5 1.86602540378
+25.0 0.0
+25.0 1.0
+24.5 1.86602540378
+26.0 0.0
+26.0 1.0
+25.5 1.86602540378
+0.0 3.73205080757
+1.0 3.73205080757
+1.0 2.73205080757
+0.0 2.73205080757
+2.0 3.73205080757
+2.0 2.73205080757
+3.0 3.73205080757
+3.0 2.73205080757
+4.0 3.73205080757
+4.0 2.73205080757
+5.0 3.73205080757
+5.0 2.73205080757
+6.0 3.73205080757
+6.0 2.73205080757
+7.0 3.73205080757
+7.0 2.73205080757
+8.0 3.73205080757
+8.0 2.73205080757
+9.0 3.73205080757
+9.0 2.73205080757
+10.0 3.73205080757
+10.0 2.73205080757
+11.0 3.73205080757
+11.0 2.73205080757
+12.0 3.73205080757
+12.0 2.73205080757
+13.0 3.73205080757
+13.0 2.73205080757
+14.0 3.73205080757
+14.0 2.73205080757
+15.0 3.73205080757
+15.0 2.73205080757
+16.0 3.73205080757
+16.0 2.73205080757
+17.0 3.73205080757
+17.0 2.73205080757
+18.0 3.73205080757
+18.0 2.73205080757
+19.0 3.73205080757
+19.0 2.73205080757
+20.0 3.73205080757
+20.0 2.73205080757
+21.0 3.73205080757
+21.0 2.73205080757
+22.0 3.73205080757
+22.0 2.73205080757
+23.0 3.73205080757
+23.0 2.73205080757
+24.0 3.73205080757
+24.0 2.73205080757
+25.0 3.73205080757
+25.0 2.73205080757
+26.0 3.73205080757
+26.0 2.73205080757

data/mixed-ele-meshes/spec-3-type-1-trans-5.mesh

@@ -0,0 +1,66 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+17
+2 3 0 1 2 3
+2 3 1 5 6 2
+2 3 5 8 9 6
+2 3 8 11 12 9
+2 3 11 14 15 12
+2 3 14 17 18 15
+1 2 2 3 4
+1 2 6 2 7
+1 2 9 6 10
+1 2 12 9 13
+1 2 15 12 16
+1 2 18 15 19
+1 2 2 4 7
+1 2 6 7 10
+1 2 9 10 13
+1 2 12 13 16
+1 2 15 16 19
+
+boundary
+15
+1 1 0 1
+1 1 1 5
+1 1 5 8
+1 1 8 11
+1 1 11 14
+1 1 14 17
+1 1 17 18
+1 1 18 19
+1 1 19 16
+1 1 16 13
+1 1 13 10
+1 1 10 7
+1 1 7 4
+1 1 4 3
+1 1 3 0
+
+vertices
+20
+2
+0.0 0.0
+1.0 0.0
+1.0 1.0
+0.0 1.0
+0.5 1.86602540378
+2.0 0.0
+2.0 1.0
+1.5 1.86602540378
+3.0 0.0
+3.0 1.0
+2.5 1.86602540378
+4.0 0.0
+4.0 1.0
+3.5 1.86602540378
+5.0 0.0
+5.0 1.0
+4.5 1.86602540378
+6.0 0.0
+6.0 1.0
+5.5 1.86602540378

data/mixed-ele-meshes/spec-3-type-1.mesh

@@ -0,0 +1,45 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+#
+
+dimension
+2
+
+elements
+5
+1 2 0 1 2
+1 2 0 2 3
+1 2 0 3 4
+2 3 0 4 5 6
+2 3 0 6 7 1
+
+boundary
+7
+1 1 1 2
+1 1 2 3
+1 1 3 4
+1 1 4 5
+1 1 5 6
+1 1 6 7
+1 1 7 1
+
+vertices
+8
+2
+0 0
+1 0
+0.5 0.866025
+-0.5 0.866025
+-1 0
+-1 -1
+0 -1
+1 -1

data/mixed-ele-meshes/spec-3-type-2.mesh

@@ -0,0 +1,45 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+#
+
+dimension
+2
+
+elements
+5
+1 2 0 1 2
+1 2 0 2 3
+1 2 0 5 6
+2 3 0 3 4 5
+2 3 0 6 7 1
+
+boundary
+7
+1 1 1 2
+1 1 2 3
+1 1 3 4
+1 1 4 5
+1 1 5 6
+1 1 6 7
+1 1 7 1
+
+vertices
+8
+2
+0 0
+0.866025 0.5
+0 1
+-0.866025 0.5
+-1.366025 -0.36602500
+-0.5 -0.866025
+0.5 -0.866025
+1.366025 -0.36602500

data/mixed-ele-meshes/tri_circ_sect_4.mesh

@@ -0,0 +1,57 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+16
+1 2 0 1 2
+1 2 1 2 4
+1 2 1 3 4
+1 2 2 4 5
+1 2 3 4 7
+1 2 3 6 7
+1 2 4 5 8
+1 2 4 7 8
+1 2 5 8 9
+1 2 6 7 11
+1 2 6 10 11
+1 2 7 8 12
+1 2 7 11 12
+1 2 8 9 13
+1 2 8 12 13
+1 2 9 13 14
+
+boundary
+12
+1 1 0 1
+1 1 1 3
+1 1 3 6
+1 1 6 10
+1 1 10 11
+1 1 11 12
+1 1 12 13
+1 1 13 14
+1 1 14 9
+1 1 9 5
+1 1 5 2
+1 1 2 0
+
+vertices
+15
+2
+0.0 0.0
+0.25 0.0
+1.53080849893e-17 0.25
+0.5 0.0
+0.353553390593 0.353553390593
+3.06161699787e-17 0.5
+0.75 0.0
+0.649519052838 0.375
+0.375 0.649519052838
+4.5924254968e-17 0.75
+1.0 0.0
+0.923879532511 0.382683432365
+0.707106781187 0.707106781187
+0.382683432365 0.923879532511
+6.12323399574e-17 1.0

data/mixed-ele-meshes/tri_circ_sect_8.mesh

@@ -0,0 +1,147 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+64
+1 2 0 1 2
+1 2 1 2 4
+1 2 1 3 4
+1 2 2 4 5
+1 2 3 4 7
+1 2 3 6 7
+1 2 4 5 8
+1 2 4 7 8
+1 2 5 8 9
+1 2 6 7 11
+1 2 6 10 11
+1 2 7 8 12
+1 2 7 11 12
+1 2 8 9 13
+1 2 8 12 13
+1 2 9 13 14
+1 2 10 11 16
+1 2 10 15 16
+1 2 11 12 17
+1 2 11 16 17
+1 2 12 13 18
+1 2 12 17 18
+1 2 13 14 19
+1 2 13 18 19
+1 2 14 19 20
+1 2 15 16 22
+1 2 15 21 22
+1 2 16 17 23
+1 2 16 22 23
+1 2 17 18 24
+1 2 17 23 24
+1 2 18 19 25
+1 2 18 24 25
+1 2 19 20 26
+1 2 19 25 26
+1 2 20 26 27
+1 2 21 22 29
+1 2 21 28 29
+1 2 22 23 30
+1 2 22 29 30
+1 2 23 24 31
+1 2 23 30 31
+1 2 24 25 32
+1 2 24 31 32
+1 2 25 26 33
+1 2 25 32 33
+1 2 26 27 34
+1 2 26 33 34
+1 2 27 34 35
+1 2 28 29 37
+1 2 28 36 37
+1 2 29 30 38
+1 2 29 37 38
+1 2 30 31 39
+1 2 30 38 39
+1 2 31 32 40
+1 2 31 39 40
+1 2 32 33 41
+1 2 32 40 41
+1 2 33 34 42
+1 2 33 41 42
+1 2 34 35 43
+1 2 34 42 43
+1 2 35 43 44
+
+boundary
+24
+1 1 0 1
+1 1 1 3
+1 1 3 6
+1 1 6 10
+1 1 10 15
+1 1 15 21
+1 1 21 28
+1 1 28 36
+1 1 36 37
+1 1 37 38
+1 1 38 39
+1 1 39 40
+1 1 40 41
+1 1 41 42
+1 1 42 43
+1 1 43 44
+1 1 44 35
+1 1 35 27
+1 1 27 20
+1 1 20 14
+1 1 14 9
+1 1 9 5
+1 1 5 2
+1 1 2 0
+
+vertices
+45
+2
+0.0 0.0
+0.125 0.0
+7.65404249467e-18 0.125
+0.25 0.0
+0.176776695297 0.176776695297
+1.53080849893e-17 0.25
+0.375 0.0
+0.324759526419 0.1875
+0.1875 0.324759526419
+2.2962127484e-17 0.375
+0.5 0.0
+0.461939766256 0.191341716183
+0.353553390593 0.353553390593
+0.191341716183 0.461939766256
+3.06161699787e-17 0.5
+0.625 0.0
+0.594410322684 0.193135621484
+0.505635621484 0.367365782683
+0.367365782683 0.505635621484
+0.193135621484 0.594410322684
+3.82702124734e-17 0.625
+0.75 0.0
+0.724444369717 0.194114283827
+0.649519052838 0.375
+0.53033008589 0.53033008589
+0.375 0.649519052838
+0.194114283827 0.724444369717
+4.5924254968e-17 0.75
+0.875 0.0
+0.853061923159 0.194705817212
+0.788347759415 0.379648271728
+0.68410254716 0.545553576626
+0.545553576626 0.68410254716
+0.379648271728 0.788347759415
+0.194705817212 0.853061923159
+5.35782974627e-17 0.875
+1.0 0.0
+0.980785280403 0.195090322016
+0.923879532511 0.382683432365
+0.831469612303 0.55557023302
+0.707106781187 0.707106781187
+0.55557023302 0.831469612303
+0.382683432365 0.923879532511
+0.195090322016 0.980785280403
+6.12323399574e-17 1.0

data/mixed-ele-meshes/tri_quad_2nd_order.mesh

@@ -0,0 +1,73 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+8
+1 2 0 1 2
+1 2 1 4 2
+1 2 1 3 4
+1 2 4 5 2
+2 3 3 6 7 4
+2 3 8 5 4 7
+2 3 6 9 10 7
+2 3 11 8 7 10
+
+boundary
+10
+1 1 0 1
+1 1 1 3
+1 1 3 6
+1 1 6 9
+1 1 9 10
+1 1 10 11
+1 1 11 8
+1 1 8 5
+1 1 5 2
+1 1 2 0
+
+vertices
+12
+
+nodes
+FiniteElementSpace
+FiniteElementCollection: Quadratic
+VDim: 2
+Ordering: 1
+
+0.0 0.0
+0.5 0.0
+0.0 0.5
+1.0 0.0
+0.70711 0.70711
+0.0 1.0
+1.5 0.0
+1.35355 1.35355
+0.0 1.5
+2.0 0.0
+2.0 2.0
+0.0 2.0
+0.25 0.0
+0.35355 0.35355
+0.0 0.25
+0.64714 0.37909
+0.37909 0.64714
+0.75 0.0
+0.92388 0.38269
+0.38269 0.92388
+0.0 0.75
+1.25 0.0
+1.54238 0.73161
+1.03033 1.03033
+0.0 1.25
+0.73161 1.54238
+1.75 0.0
+2.0 1.0
+1.67678 1.67678
+0.0 1.75
+1.0 2.0
+1.14017 0.51516
+0.51516 1.14017
+1.71339 0.83839
+0.83839 1.71339

data/mixed-ele-meshes/whole_plane_tri_quad_2nd_order.mesh

@@ -0,0 +1,567 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+128
+1 2 0 1 2
+1 2 1 4 2
+1 2 1 3 4
+1 2 2 4 5
+1 2 3 7 4
+1 2 3 6 7
+1 2 4 8 5
+1 2 4 7 8
+1 2 5 8 9
+1 2 6 11 7
+1 2 6 10 11
+1 2 7 12 8
+1 2 7 11 12
+1 2 8 13 9
+1 2 8 12 13
+1 2 9 13 14
+2 3 10 15 16 11
+2 3 11 16 17 12
+2 3 12 17 18 13
+2 3 13 18 19 14
+2 3 15 20 21 16
+2 3 16 21 22 17
+2 3 17 22 23 18
+2 3 18 23 24 19
+2 3 20 25 26 21
+2 3 21 26 27 22
+2 3 22 27 28 23
+2 3 23 28 29 24
+2 3 25 30 31 26
+2 3 26 31 32 27
+2 3 27 32 33 28
+2 3 28 33 34 29
+1 2 0 2 35
+1 2 35 2 37
+1 2 35 37 36
+1 2 2 5 37
+1 2 36 37 39
+1 2 36 39 38
+1 2 37 5 40
+1 2 37 40 39
+1 2 5 9 40
+1 2 38 39 42
+1 2 38 42 41
+1 2 39 40 43
+1 2 39 43 42
+1 2 40 9 44
+1 2 40 44 43
+1 2 9 14 44
+2 3 41 42 46 45
+2 3 42 43 47 46
+2 3 43 44 48 47
+2 3 44 14 19 48
+2 3 45 46 50 49
+2 3 46 47 51 50
+2 3 47 48 52 51
+2 3 48 19 24 52
+2 3 49 50 54 53
+2 3 50 51 55 54
+2 3 51 52 56 55
+2 3 52 24 29 56
+2 3 53 54 58 57
+2 3 54 55 59 58
+2 3 55 56 60 59
+2 3 56 29 34 60
+1 2 0 61 1
+1 2 1 61 62
+1 2 1 62 3
+1 2 61 63 62
+1 2 3 62 64
+1 2 3 64 6
+1 2 62 63 65
+1 2 62 65 64
+1 2 63 66 65
+1 2 6 64 67
+1 2 6 67 10
+1 2 64 65 68
+1 2 64 68 67
+1 2 65 66 69
+1 2 65 69 68
+1 2 66 70 69
+2 3 10 67 71 15
+2 3 67 68 72 71
+2 3 68 69 73 72
+2 3 69 70 74 73
+2 3 15 71 75 20
+2 3 71 72 76 75
+2 3 72 73 77 76
+2 3 73 74 78 77
+2 3 20 75 79 25
+2 3 75 76 80 79
+2 3 76 77 81 80
+2 3 77 78 82 81
+2 3 25 79 83 30
+2 3 79 80 84 83
+2 3 80 81 85 84
+2 3 81 82 86 85
+1 2 0 35 61
+1 2 35 87 61
+1 2 35 36 87
+1 2 61 87 63
+1 2 36 88 87
+1 2 36 38 88
+1 2 87 89 63
+1 2 87 88 89
+1 2 63 89 66
+1 2 38 90 88
+1 2 38 41 90
+1 2 88 91 89
+1 2 88 90 91
+1 2 89 92 66
+1 2 89 91 92
+1 2 66 92 70
+2 3 41 45 93 90
+2 3 90 93 94 91
+2 3 91 94 95 92
+2 3 92 95 74 70
+2 3 45 49 96 93
+2 3 93 96 97 94
+2 3 94 97 98 95
+2 3 95 98 78 74
+2 3 49 53 99 96
+2 3 96 99 100 97
+2 3 97 100 101 98
+2 3 98 101 82 78
+2 3 53 102 103 99
+2 3 99 103 104 100
+2 3 100 104 105 101
+2 3 101 105 86 82
+
+boundary
+16
+1 1 30 31
+1 1 31 32
+1 1 32 33
+1 1 33 34
+1 1 34 60
+1 1 60 59
+1 1 59 58
+1 1 58 57
+1 1 102 103
+1 1 103 104
+1 1 104 105
+1 1 105 86
+1 1 86 85
+1 1 85 84
+1 1 84 83
+1 1 83 30
+
+vertices
+106
+
+nodes
+FiniteElementSpace
+FiniteElementCollection: H1_2D_P2
+VDim: 2
+Ordering: 1
+
+0.0 0.0
+0.25 0.0
+0.0 0.25
+0.5 0.0
+0.35355 0.35355
+0.0 0.5
+0.75 0.0
+0.64952 0.375
+0.375 0.64952
+0.0 0.75
+1.0 0.0
+0.92388 0.38268
+0.70711 0.70711
+0.38268 0.92388
+0.0 1.0
+1.25 0.0
+1.19291 0.53701
+1.03033 1.03033
+0.53701 1.19291
+0.0 1.25
+1.5 0.0
+1.46194 0.69134
+1.35355 1.35355
+0.69134 1.46194
+0.0 1.5
+1.75 0.0
+1.73097 0.84567
+1.67678 1.67678
+0.84567 1.73097
+0.0 1.75
+2.0 0.0
+2.0 1.0
+2.0 2.0
+1.0 2.0
+0.0 2.0
+-0.25 0.0
+-0.5 0.0
+-0.35355 0.35355
+-0.75 0.0
+-0.64952 0.375
+-0.375 0.64952
+-1.0 0.0
+-0.92388 0.38268
+-0.70711 0.70711
+-0.38268 0.92388
+-1.25 0.0
+-1.19291 0.53701
+-1.03033 1.03033
+-0.53701 1.19291
+-1.5 0.0
+-1.46194 0.69134
+-1.35355 1.35355
+-0.69134 1.46194
+-1.75 0.0
+-1.73097 0.84567
+-1.67678 1.67678
+-0.84567 1.73097
+-2.0 0.0
+-2.0 1.0
+-2.0 2.0
+-1.0 2.0
+0.0 -0.25
+0.35355 -0.35355
+0.0 -0.5
+0.64952 -0.375
+0.375 -0.64952
+0.0 -0.75
+0.92388 -0.38268
+0.70711 -0.70711
+0.38268 -0.92388
+0.0 -1.0
+1.19291 -0.53701
+1.03033 -1.03033
+0.53701 -1.19291
+0.0 -1.25
+1.46194 -0.69134
+1.35355 -1.35355
+0.69134 -1.46194
+0.0 -1.5
+1.73097 -0.84567
+1.67678 -1.67678
+0.84567 -1.73097
+0.0 -1.75
+2.0 -1.0
+2.0 -2.0
+1.0 -2.0
+0.0 -2.0
+-0.35355 -0.35355
+-0.64952 -0.375
+-0.375 -0.64952
+-0.92388 -0.38268
+-0.70711 -0.70711
+-0.38268 -0.92388
+-1.19291 -0.53701
+-1.03033 -1.03033
+-0.53701 -1.19291
+-1.46194 -0.69134
+-1.35355 -1.35355
+-0.69134 -1.46194
+-1.73097 -0.84567
+-1.67678 -1.67678
+-0.84567 -1.73097
+-2.0 -0.0
+-2.0 -1.0
+-2.0 -2.0
+-1.0 -2.0
+0.125 0.0
+0.17678 0.17678
+0.0 0.125
+0.32357 0.18954
+0.18954 0.32357
+0.375 0.0
+0.46194 0.19134
+0.19134 0.46194
+0.0 0.375
+0.59418 0.19384
+0.50569 0.36729
+0.625 0.0
+0.72444 0.19411
+0.36729 0.50569
+0.19384 0.59418
+0.53033 0.53033
+0.19411 0.72444
+0.0 0.625
+0.85299 0.19501
+0.78835 0.37964
+0.875 0.0
+0.98079 0.19509
+0.68405 0.54563
+0.54563 0.68405
+0.83147 0.55557
+0.37964 0.78835
+0.19501 0.85299
+0.55557 0.83147
+0.19509 0.98079
+0.0 0.875
+1.125 0.0
+1.24928 0.27462
+1.05851 0.4599
+1.13007 0.79668
+0.86872 0.86872
+0.79668 1.13007
+0.4599 1.05851
+0.27462 1.24928
+0.0 1.125
+1.375 0.0
+1.51779 0.35426
+1.32748 0.6142
+1.42864 1.03762
+1.19194 1.19194
+1.03762 1.42864
+0.6142 1.32748
+0.35426 1.51779
+0.0 1.375
+1.625 0.0
+1.78629 0.43396
+1.59649 0.76852
+1.72721 1.2785
+1.51516 1.51516
+1.2785 1.72721
+0.76852 1.59649
+0.43396 1.78629
+0.0 1.625
+1.875 0.0
+2.0 0.5
+1.8655 0.92284
+2.0 1.5
+1.83839 1.83839
+1.5 2.0
+0.92284 1.8655
+0.5 2.0
+0.0 1.875
+-0.17678 0.17678
+-0.125 0.0
+-0.18954 0.32357
+-0.32357 0.18954
+-0.46194 0.19134
+-0.375 0.0
+-0.19134 0.46194
+-0.50569 0.36729
+-0.59418 0.19384
+-0.72444 0.19411
+-0.625 0.0
+-0.19384 0.59418
+-0.36729 0.50569
+-0.53033 0.53033
+-0.19411 0.72444
+-0.78835 0.37964
+-0.85299 0.19501
+-0.98079 0.19509
+-0.875 0.0
+-0.54563 0.68405
+-0.68405 0.54563
+-0.83147 0.55557
+-0.19501 0.85299
+-0.37964 0.78835
+-0.55557 0.83147
+-0.19509 0.98079
+-1.05851 0.4599
+-1.24928 0.27462
+-1.125 0.0
+-0.86872 0.86872
+-1.13007 0.79668
+-0.4599 1.05851
+-0.79668 1.13007
+-0.27462 1.24928
+-1.32748 0.6142
+-1.51779 0.35426
+-1.375 0.0
+-1.19194 1.19194
+-1.42864 1.03762
+-0.6142 1.32748
+-1.03762 1.42864
+-0.35426 1.51779
+-1.59649 0.76852
+-1.78629 0.43396
+-1.625 0.0
+-1.51516 1.51516
+-1.72721 1.2785
+-0.76852 1.59649
+-1.2785 1.72721
+-0.43396 1.78629
+-1.8655 0.92284
+-2.0 0.5
+-1.875 0.0
+-1.83839 1.83839
+-2.0 1.5
+-0.92284 1.8655
+-1.5 2.0
+-0.5 2.0
+0.0 -0.125
+0.17678 -0.17678
+0.18954 -0.32357
+0.32357 -0.18954
+0.46194 -0.19134
+0.0 -0.375
+0.19134 -0.46194
+0.50569 -0.36729
+0.59418 -0.19384
+0.72444 -0.19411
+0.19384 -0.59418
+0.36729 -0.50569
+0.53033 -0.53033
+0.0 -0.625
+0.19411 -0.72444
+0.78835 -0.37964
+0.85299 -0.19501
+0.98079 -0.19509
+0.54563 -0.68405
+0.68405 -0.54563
+0.83147 -0.55557
+0.19501 -0.85299
+0.37964 -0.78835
+0.55557 -0.83147
+0.0 -0.875
+0.19509 -0.98079
+1.05851 -0.4599
+1.24928 -0.27462
+0.86872 -0.86872
+1.13007 -0.79668
+0.4599 -1.05851
+0.79668 -1.13007
+0.0 -1.125
+0.27462 -1.24928
+1.32748 -0.6142
+1.51779 -0.35426
+1.19194 -1.19194
+1.42864 -1.03762
+0.6142 -1.32748
+1.03762 -1.42864
+0.0 -1.375
+0.35426 -1.51779
+1.59649 -0.76852
+1.78629 -0.43396
+1.51516 -1.51516
+1.72721 -1.2785
+0.76852 -1.59649
+1.2785 -1.72721
+0.0 -1.625
+0.43396 -1.78629
+1.8655 -0.92284
+2.0 -0.5
+1.83839 -1.83839
+2.0 -1.5
+0.92284 -1.8655
+1.5 -2.0
+0.0 -1.875
+0.5 -2.0
+-0.17678 -0.17678
+-0.32357 -0.18954
+-0.18954 -0.32357
+-0.46194 -0.19134
+-0.19134 -0.46194
+-0.59418 -0.19384
+-0.50569 -0.36729
+-0.72444 -0.19411
+-0.36729 -0.50569
+-0.19384 -0.59418
+-0.53033 -0.53033
+-0.19411 -0.72444
+-0.85299 -0.19501
+-0.78835 -0.37964
+-0.98079 -0.19509
+-0.68405 -0.54563
+-0.54563 -0.68405
+-0.83147 -0.55557
+-0.37964 -0.78835
+-0.19501 -0.85299
+-0.55557 -0.83147
+-0.19509 -0.98079
+-1.24928 -0.27462
+-1.05851 -0.4599
+-1.13007 -0.79668
+-0.86872 -0.86872
+-0.79668 -1.13007
+-0.4599 -1.05851
+-0.27462 -1.24928
+-1.51779 -0.35426
+-1.32748 -0.6142
+-1.42864 -1.03762
+-1.19194 -1.19194
+-1.03762 -1.42864
+-0.6142 -1.32748
+-0.35426 -1.51779
+-1.78629 -0.43396
+-1.59649 -0.76852
+-1.72721 -1.2785
+-1.51516 -1.51516
+-1.2785 -1.72721
+-0.76852 -1.59649
+-0.43396 -1.78629
+-1.875 0.0
+-2.0 -0.5
+-1.8655 -0.92284
+-2.0 -1.5
+-1.83839 -1.83839
+-1.5 -2.0
+-0.92284 -1.8655
+-0.5 -2.0
+1.0917 0.22992
+0.96356 0.66428
+0.66428 0.96356
+0.22992 1.0917
+1.35121 0.30709
+1.25968 0.90306
+0.90306 1.25968
+0.30709 1.35121
+1.61073 0.38425
+1.55581 1.14184
+1.14184 1.55581
+0.38425 1.61073
+1.87024 0.46142
+1.85194 1.38061
+1.38061 1.85194
+0.46142 1.87024
+-1.0917 0.22992
+-0.96356 0.66428
+-0.66428 0.96356
+-0.22992 1.0917
+-1.35121 0.30709
+-1.25968 0.90306
+-0.90306 1.25968
+-0.30709 1.35121
+-1.61073 0.38425
+-1.55581 1.14184
+-1.14184 1.55581
+-0.38425 1.61073
+-1.87024 0.46142
+-1.85194 1.38061
+-1.38061 1.85194
+-0.46142 1.87024
+1.0917 -0.22992
+0.96356 -0.66428
+0.66428 -0.96356
+0.22992 -1.0917
+1.35121 -0.30709
+1.25968 -0.90306
+0.90306 -1.25968
+0.30709 -1.35121
+1.61073 -0.38425
+1.55581 -1.14184
+1.14184 -1.55581
+0.38425 -1.61073
+1.87024 -0.46142
+1.85194 -1.38061
+1.38061 -1.85194
+0.46142 -1.87024
+-1.0917 -0.22992
+-0.96356 -0.66428
+-0.66428 -0.96356
+-0.22992 -1.0917
+-1.35121 -0.30709
+-1.25968 -0.90306
+-0.90306 -1.25968
+-0.30709 -1.35121
+-1.61073 -0.38425
+-1.55581 -1.14184
+-1.14184 -1.55581
+-0.38425 -1.61073
+-1.87024 -0.46142
+-1.85194 -1.38061
+-1.38061 -1.85194
+-0.46142 -1.87024

doc/CodeDocumentation.conf.in

@@ -760,8 +760,6 @@ WARN_LOGFILE           =
 
 INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/mfem.hpp \
-                         @MFEM_SOURCE_DIR@/backends/base \
-                         @MFEM_SOURCE_DIR@/backends/occa \
                          @MFEM_SOURCE_DIR@/config \
                          @MFEM_SOURCE_DIR@/general \
                          @MFEM_SOURCE_DIR@/linalg \

examples/ex1.cpp

@@ -80,8 +80,8 @@ int main(int argc, char *argv[])
    //    largest number that gives a final mesh with no more than 50,000
    //    elements.
    {
-      int ref_levels =
-         (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
+     int ref_levels = 0; 
+       //(int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
          mesh->UniformRefinement();

examples/ex13p.cpp

@@ -110,6 +110,7 @@ int main(int argc, char *argv[])
    {
       pmesh->UniformRefinement();
    }
+   pmesh->ReorientTetMesh();
 
    // 6. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Nedelec finite elements of the specified order.

examples/ex16d.cpp

@@ -1,441 +0,0 @@
-//                                MFEM Example 16
-//
-// Compile with: make ex16
-//
-// Sample runs:  ex16
-//               ex16 -m ../data/inline-tri.mesh
-//               ex16 -m ../data/disc-nurbs.mesh -tf 2
-//               ex16 -s 1 -a 0.0 -k 1.0
-//               ex16 -s 2 -a 1.0 -k 0.0
-//               ex16 -s 3 -a 0.5 -k 0.5 -o 4
-//               ex16 -s 14 -dt 1.0e-4 -tf 4.0e-2 -vs 40
-//               ex16 -m ../data/fichera-q2.mesh
-//               ex16 -m ../data/escher.mesh
-//               ex16 -m ../data/beam-tet.mesh -tf 10 -dt 0.1
-//               ex16 -m ../data/amr-quad.mesh -o 4 -r 0
-//               ex16 -m ../data/amr-hex.mesh -o 2 -r 0
-//
-// Description:  This example solves a time dependent nonlinear heat equation
-//               problem of the form du/dt = C(u), with a non-linear diffusion
-//               operator C(u) = \nabla \cdot (\kappa + \alpha u) \nabla u.
-//
-//               The example demonstrates the use of nonlinear operators (the
-//               class ConductionOperator defining C(u)), as well as their
-//               implicit time integration. Note that implementing the method
-//               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration.
-//
-//               We recommend viewing examples 2, 9 and 10 before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-/** After spatial discretization, the conduction model can be written as:
- *
- *     du/dt = M^{-1}(-Ku)
- *
- *  where u is the vector representing the temperature, M is the mass matrix,
- *  and K is the diffusion operator with diffusivity depending on u:
- *  (\kappa + \alpha u).
- *
- *  Class ConductionOperator represents the right-hand side of the above ODE.
- */
-class ConductionOperator : public TimeDependentOperator
-{
-protected:
-   FiniteElementSpace &fespace;
-   Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
-
-   BilinearForm M;
-   BilinearForm K;
-
-   GridFunction u_alpha_gf;
-   GridFunctionCoefficient u_coeff;
-
-   OperatorHandle Moper, Koper;
-   Operator *T; // T = M + dt K
-   double current_dt;
-
-   CGSolver M_solver; // Krylov solver for inverting the mass matrix M
-   // FIXME: add the preconditioner
-   // DSmoother M_prec;  // Preconditioner for the mass matrix M
-
-   CGSolver T_solver; // Implicit solver for T = M + dt K
-   // FIXME: add the preconditioner
-   // DSmoother T_prec;  // Preconditioner for the implicit solver
-
-   double alpha, kappa;
-
-   mutable Vector z; // auxiliary vector
-
-public:
-   ConductionOperator(FiniteElementSpace &f, const char *oper_spec,
-                      double alpha, double kappa, const Vector &u);
-
-   virtual void Mult(const Vector &u, Vector &du_dt) const;
-   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
-       This is the only requirement for high-order SDIRK implicit integration.*/
-   virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);
-
-   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
-   void SetParameters(const Vector &u);
-
-   virtual ~ConductionOperator();
-};
-
-
-class TimeDerivativeOperator : public Operator
-{
-   Operator *Moper;
-   Operator *Koper;
-   mutable Vector Kdu;
-   const double dt;
-
-public:
-   // FIXME: Sparse matrices should be changed to have PLayouts
-   // allocated so that this constructor works even when Moper and
-   // Koper were not created from engines.
-   TimeDerivativeOperator(Operator *_Moper, const double _dt, Operator *_Koper)
-      : Operator(*_Koper->InLayout(), *_Moper->OutLayout()),
-        Moper(_Moper),
-        Koper(_Koper),
-        Kdu(_Moper->OutLayout()),
-        dt(_dt) { }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      Moper->Mult(x, y);
-      Koper->Mult(x, Kdu);
-
-      y.Axpby(1.0, y, dt, Kdu);
-   }
-};
-
-double InitialTemperature(const Vector &x);
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int ref_levels = 2;
-   int order = 2;
-   int ode_solver_type = 3;
-   double t_final = 0.5;
-   double dt = 1.0e-2;
-   double alpha = 1.0e-2;
-   double kappa = 0.5;
-   bool visualization = true;
-   bool visit = false;
-   int vis_steps = 5;
-   const char *oper_spec = "representation: 'partial'";
-   const char *occa_spec = "mode: 'Serial'";
-
-   int precision = 8;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
-                  "\t   11 - Forward Euler, 12 - RK2, 13 - RK3 SSP, 14 - RK4.");
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time step.");
-   args.AddOption(&alpha, "-a", "--alpha",
-                  "Alpha coefficient.");
-   args.AddOption(&kappa, "-k", "--kappa",
-                  "Kappa coefficient offset.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.AddOption(&oper_spec, "-s", "--oper-spec", "Operator specification");
-   args.AddOption(&occa_spec, "-os", "--occa-spec", "OCCA engine specification");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // Examples for OCCA specifications:
-   //   - CPU (serial): "mode: 'Serial'"
-   //   - CUDA GPU: "mode: 'CUDA', device_id: 0"
-   //   - OpenMP on CPUs: "mode: 'OpenMP', threads: 4"
-   //   - OpenCL on device 0: "mode: 'OpenCL', device_id: 0, platform_id: 0"
-
-   SharedPtr<Engine> engine(new mfem::occa::Engine(occa_spec));
-
-   // 2. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   mesh->SetEngine(*engine);
-   int dim = mesh->Dimension();
-
-   // 3. Define the ODE solver used for time integration. Several implicit
-   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
-   //    explicit Runge-Kutta methods are available.
-   ODESolver *ode_solver;
-   switch (ode_solver_type)
-   {
-      // Implicit L-stable methods
-      case 1:  ode_solver = new BackwardEulerSolver; break;
-      case 2:  ode_solver = new SDIRK23Solver(2); break;
-      case 3:  ode_solver = new SDIRK33Solver; break;
-      // Explicit methods
-      case 11: ode_solver = new ForwardEulerSolver; break;
-      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 13: ode_solver = new RK3SSPSolver; break;
-      case 14: ode_solver = new RK4Solver; break;
-      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
-      // Implicit A-stable methods (not L-stable)
-      case 22: ode_solver = new ImplicitMidpointSolver; break;
-      case 23: ode_solver = new SDIRK23Solver; break;
-      case 24: ode_solver = new SDIRK34Solver; break;
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         delete mesh;
-         return 3;
-   }
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
-   //    command-line parameter.
-   for (int lev = 0; lev < ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 5. Define the vector finite element space representing the current and the
-   //    initial temperature, u_ref.
-   H1_FECollection fe_coll(order, dim);
-   FiniteElementSpace fespace(mesh, &fe_coll);
-
-   int fe_size = fespace.GetTrueVSize();
-   cout << "Number of temperature unknowns: " << fe_size << endl;
-
-   GridFunction u_gf(&fespace);
-
-   // 6. Set the initial conditions for u. All boundaries are considered
-   //    natural. This computes this on the host, so pull/push is needed.
-   u_gf.Pull();
-   FunctionCoefficient u_0(InitialTemperature);
-   u_gf.ProjectCoefficient(u_0);
-   u_gf.Push();
-   Vector u;
-   u_gf.GetTrueDofs(u);
-
-   // 7. Initialize the conduction operator and the visualization.
-   ConductionOperator oper(fespace, oper_spec, alpha, kappa, u);
-
-   u_gf.SetFromTrueDofs(u);
-   {
-      ofstream omesh("ex16.mesh");
-      omesh.precision(precision);
-      mesh->Print(omesh);
-      ofstream osol("ex16-init.gf");
-      osol.precision(precision);
-      u_gf.Pull(); // pull back to host before saving
-      u_gf.Save(osol);
-   }
-
-   VisItDataCollection visit_dc("Example16", mesh);
-   visit_dc.RegisterField("temperature", &u_gf);
-   if (visit)
-   {
-      visit_dc.SetCycle(0);
-      visit_dc.SetTime(0.0);
-      visit_dc.Save();
-   }
-
-   socketstream sout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      if (!sout)
-      {
-         cout << "Unable to connect to GLVis server at "
-              << vishost << ':' << visport << endl;
-         visualization = false;
-         cout << "GLVis visualization disabled.\n";
-      }
-      else
-      {
-         sout.precision(precision);
-         sout << "solution\n" << *mesh << u_gf;
-         sout << "pause\n";
-         sout << flush;
-         cout << "GLVis visualization paused."
-              << " Press space (in the GLVis window) to resume it.\n";
-      }
-   }
-
-   // 8. Perform time-integration (looping over the time iterations, ti, with a
-   //    time-step dt).
-   ode_solver->Init(oper);
-   double t = 0.0;
-
-   bool last_step = false;
-   for (int ti = 1; !last_step; ti++)
-   {
-      if (t + dt >= t_final - dt/2)
-      {
-         last_step = true;
-      }
-
-      ode_solver->Step(u, t, dt);
-
-      if (last_step || (ti % vis_steps) == 0)
-      {
-         cout << "step " << ti << ", t = " << t << endl;
-
-         // u_gf and u are both on the device at this point.
-         u_gf.SetFromTrueDofs(u);
-         if (visualization)
-         {
-            u_gf.Pull(); // pull back to host before saving
-            sout << "solution\n" << *mesh << u_gf << flush;
-         }
-
-         if (visit)
-         {
-            visit_dc.SetCycle(ti);
-            visit_dc.SetTime(t);
-            visit_dc.Save();
-         }
-      }
-      oper.SetParameters(u);
-   }
-
-   // 9. Save the final solution. This output can be viewed later using GLVis:
-   //    "glvis -m ex16.mesh -g ex16-final.gf".
-   {
-      ofstream osol("ex16-final.gf");
-      osol.precision(precision);
-      u_gf.Pull(); // pull back to host before saving
-      u_gf.Save(osol);
-   }
-
-   // 10. Free the used memory.
-   delete ode_solver;
-   delete mesh;
-
-   return 0;
-}
-
-ConductionOperator::ConductionOperator(FiniteElementSpace &f,
-                                       const char *oper_spec, double al,
-                                       double kap, const Vector &u)
-   : TimeDependentOperator(*f.GetTrueVLayout()), fespace(f), M(&fespace),
-     K(&fespace), u_alpha_gf(&f), u_coeff(&u_alpha_gf), Moper(oper_spec),
-     Koper(oper_spec), T(NULL), current_dt(0.0), z(f.GetTrueVLayout())
-{
-   const double rel_tol = 1e-8;
-
-   M.AddDomainIntegrator(new MassIntegrator());
-   M.Assemble();
-   M.FormSystemMatrix(ess_tdof_list, Moper);
-
-   K.AddDomainIntegrator(new DiffusionIntegrator(u_coeff));
-
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(rel_tol);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(200);
-   M_solver.SetPrintLevel(0);
-   // M_solver.SetPreconditioner(M_prec);
-   M_solver.SetOperator(*Moper.Ptr());
-
-   alpha = al;
-   kappa = kap;
-
-   T_solver.iterative_mode = false;
-   T_solver.SetRelTol(rel_tol);
-   T_solver.SetAbsTol(0.0);
-   T_solver.SetMaxIter(200);
-   T_solver.SetPrintLevel(0);
-   // T_solver.SetPreconditioner(T_prec);
-
-   SetParameters(u);
-}
-
-void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
-{
-   // Compute:
-   //    du_dt = M^{-1}*-K(u)
-   // for du_dt
-   Koper.Ptr()->Mult(u, z);
-   z.Axpby(-1.0, z, 0.0, z);
-   M_solver.Mult(z, du_dt);
-}
-
-void ConductionOperator::ImplicitSolve(const double dt,
-                                       const Vector &u, Vector &du_dt)
-{
-   // Solve the equation:
-   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt
-   if (!T)
-   {
-      T = new TimeDerivativeOperator(Moper.Ptr(), dt, Koper.Ptr());
-      current_dt = dt;
-      T_solver.SetOperator(*T);
-   }
-   MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt
-   Koper.Ptr()->Mult(u, z);
-   z.Axpby(-1.0, z, 0.0, z);
-   T_solver.Mult(z, du_dt);
-}
-
-void ConductionOperator::SetParameters(const Vector &u)
-{
-   u_alpha_gf.SetFromTrueDofs(u);
-   u_alpha_gf.Pull();
-   for (int i = 0; i < u_alpha_gf.Size(); i++)
-   {
-      u_alpha_gf(i) = kappa + alpha*u_alpha_gf(i);
-   }
-   u_alpha_gf.Push();
-
-   // Reassemble after changing u_alpha_gf (and hence u_coeff)...
-   K.Assemble();
-   K.FormSystemMatrix(ess_tdof_list, Koper);
-
-   delete T;
-   T = NULL; // re-compute T on the next ImplicitSolve
-}
-
-ConductionOperator::~ConductionOperator()
-{
-   delete T;
-}
-
-double InitialTemperature(const Vector &x)
-{
-   if (x.Norml2() < 0.5)
-   {
-      return 2.0;
-   }
-   else
-   {
-      return 1.0;
-   }
-}

examples/ex1d.cpp

@@ -1,185 +0,0 @@
-
-#include <mfem.hpp>
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *spec = "cpu";
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-   int ref_levels = -1;
-   bool static_cond = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&spec, "-s", "--spec",
-                  "Compute resource specification.");
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&ref_levels, "-r", "--refine-levels",
-                  "Number of uniform refinements to apply to the mesh.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-#ifdef MFEM_USE_BACKENDS
-   /// Engine *engine = EngineDepot.Select(spec);
-
-   // string occa_spec("mode: 'Serial'");
-   string occa_spec("mode: 'CUDA', device_id: 0");
-   // string occa_spec("mode: 'OpenMP', threads: 4");
-   // string occa_spec("mode: 'OpenCL', device_id: 0, platform_id: 0");
-
-   // The following flag affects only 'Serial' and 'OpenMP' modes.
-   // In 'CUDA' mode, '-O3' affects only host code.
-   // In 'OpenCL' mode, adding '-O3' breaks compilation.
-   // occa_spec += ", kernel: { compiler_flags: '-O3' }";
-
-   SharedPtr<Engine> engine(new mfem::occa::Engine(occa_spec));
-#endif
-
-   // 2. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-#ifdef MFEM_USE_BACKENDS
-   mesh->SetEngine(*engine);
-#endif
-   int dim = mesh->Dimension();
-
-   // 3. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 50,000
-   //    elements.
-   {
-      ref_levels = ref_levels >= 0 ? ref_levels :
-         (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 4. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order. If order < 1, we
-   //    instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (mesh->GetNodes())
-   {
-      fec = mesh->GetNodes()->OwnFEC();
-      cout << "Using isoparametric FEs: " << fec->Name() << endl;
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-   cout << "Number of finite element unknowns: "
-        << fespace->GetTrueVSize() << endl;
-
-   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (mesh->bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   LinearForm *b = new LinearForm(fespace);
-   ConstantCoefficient one(1.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->Assemble();
-
-   // 7. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(fespace);
-   x.Fill(0.0);
-
-   // 8. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   BilinearForm *a = new BilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 9. Assemble the bilinear form and the corresponding linear system,
-   //    applying any necessary transformations such as: eliminating boundary
-   //    conditions, applying conforming constraints for non-conforming AMR,
-   //    static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   OperatorHandle A(Operator::ANY_TYPE);
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   cout << "Size of linear system: " << A.Ptr()->Height() << endl;
-
-   // 10. Solve the system A X = B with CG.
-   tic_toc.Clear();
-   tic_toc.Start();
-   const int print_level = 3;
-   CG(*A.Ptr(), B, X, print_level, 1000, 1e-12, 0.0);
-   tic_toc.Stop();
-   cout << "CG time: " << tic_toc.RealTime() << " sec." << endl;
-
-   // 11. Recover the solution as a finite element grid function.
-   a->RecoverFEMSolution(X, *b, x);
-   x.Pull();
-
-   // 12. Save the refined mesh and the solution. This output can be viewed
-   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh->Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-   // 13. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x << flush;
-   }
-
-   // 14. Free the used memory.
-   delete a;
-   delete b;
-   delete fespace;
-   if (order > 0) { delete fec; }
-   delete mesh;
-
-   return 0;
-}

examples/ex1pd.cpp

@@ -1,273 +0,0 @@
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int ser_ref_levels = -1;
-   int par_ref_levels = -1;
-   int order = 1;
-   bool static_cond = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-#ifdef MFEM_USE_BACKENDS
-   /// Engine *engine = EngineDepot.Select(spec);
-
-   // string occa_spec("mode: 'Serial'");
-   string occa_spec;
-   {
-      stringstream occa_spec_ss;
-      occa_spec_ss << "mode: 'CUDA', device_id: 0";
-      // const int nGPUs = 4;
-      // occa_spec_ss << "mode: 'CUDA', device_id: " << (myid % nGPUs);
-      occa_spec = occa_spec_ss.str();
-   }
-   // string occa_spec("mode: 'OpenMP', threads: 4");
-   // string occa_spec("mode: 'OpenCL', device_id: 0, platform_id: 0");
-
-   SharedPtr<Engine> engine(new mfem::occa::Engine(MPI_COMM_WORLD, occa_spec));
-#endif
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-#ifdef MFEM_USE_BACKENDS
-   mesh->SetEngine(*engine);
-#endif
-   int dim = mesh->Dimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 10,000 elements.
-   {
-      int ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
-      ref_levels = ser_ref_levels >= 0 ? ser_ref_levels : ref_levels;
-      if (myid == 0)
-      {
-         cout << "Serial refinement levels: " << ref_levels << endl;
-      }
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   {
-      par_ref_levels = par_ref_levels >= 0 ? par_ref_levels : 2;
-      if (myid == 0)
-      {
-         cout << "Parallel refinement levels: " << par_ref_levels << endl;
-      }
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh->UniformRefinement();
-      }
-   }
-   pmesh->PrintInfo(cout);
-   if (myid == 0) { cout << endl; }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (pmesh->GetNodes())
-   {
-      fec = pmesh->GetNodes()->OwnFEC();
-      if (myid == 0)
-      {
-         cout << "Using isoparametric FEs: " << fec->Name() << endl;
-      }
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh->bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm *b = new ParLinearForm(fespace);
-   ConstantCoefficient one(1.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   ParGridFunction x(fespace);
-   x.Fill(0.0);
-
-   // 10. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   OperatorHandle A(Operator::ANY_TYPE);
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   CGSolver *pcg = new CGSolver(MPI_COMM_WORLD);
-   pcg->SetRelTol(1e-6);
-   pcg->SetAbsTol(0.0);
-   pcg->SetMaxIter(1000);
-   pcg->SetPrintLevel(3);
-   pcg->SetOperator(*A.Ptr());
-
-   // Run one CG iteration to make sure all kernels are loaded before measuring
-   // time.
-   if (myid == 0)
-   {
-      cout << "Running 1 CG iteration to load all kernels ..." << flush;
-   }
-   {
-      Vector X2(X);
-      pcg->SetMaxIter(1);
-      pcg->SetPrintLevel(-1);
-      pcg->Mult(B, X2);
-      pcg->SetMaxIter(1000);
-      pcg->SetPrintLevel(3);
-   }
-   if (myid == 0)
-   {
-      cout << " done." << endl;
-   }
-
-   double start_time = MPI_Wtime();
-   pcg->Mult(B, X);
-   double end_time = MPI_Wtime();
-   double loc_time = end_time - start_time;
-   double max_time, min_time;
-   MPI_Allreduce(&loc_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
-   MPI_Allreduce(&loc_time, &min_time, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);
-   if (myid == 0)
-   {
-      cout << "CG time: " << max_time << " sec (min: " << min_time << " sec)\n"
-           << "DOFs/sec in CG: "
-           << 1e-6*size*pcg->GetNumIterations()/max_time << " ("
-           << 1e-6*size*pcg->GetNumIterations()/min_time << ") million.\n"
-           << endl;
-   }
-
-   // 13. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-   x.Pull();
-
-   // 14. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 15. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 16. Free the used memory.
-   delete pcg;
-   delete a;
-   delete b;
-   delete fespace;
-   if (order > 0) { delete fec; }
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}

fem/bilinearform.cpp

@@ -74,12 +74,6 @@ BilinearForm::BilinearForm (FiniteElementSpace * f)
    hybridization = NULL;
    precompute_sparsity = 0;
    diag_policy = DIAG_KEEP;
-#ifdef MFEM_USE_BACKENDS
-   if (fes->GetVLayout()->HasEngine())
-   {
-      dev_ext = fes->GetVLayout()->GetEngine().MakeBilinearForm(*this);
-   }
-#endif
 }
 
 BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
@@ -132,14 +126,6 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
 void BilinearForm::EnableStaticCondensation()
 {
    delete static_cond;
-#ifdef MFEM_USE_BACKENDS
-   if (fes->GetVLayout()->HasEngine())
-   {
-      static_cond = NULL;
-      MFEM_WARNING("Engine interface does not support static condensation yet");
-      return;
-   }
-#endif
    static_cond = new StaticCondensation(fes);
    if (static_cond->ReducesTrueVSize())
    {
@@ -159,15 +145,6 @@ void BilinearForm::EnableHybridization(FiniteElementSpace *constr_space,
                                        const Array<int> &ess_tdof_list)
 {
    delete hybridization;
-#ifdef MFEM_USE_BACKENDS
-   if (fes->GetVLayout()->HasEngine())
-   {
-      delete constr_integ;
-      hybridization = NULL;
-      MFEM_WARNING("Engine interface does not support hybridization yet");
-      return;
-   }
-#endif
    hybridization = new Hybridization(fes, constr_space);
    hybridization->SetConstraintIntegrator(constr_integ);
    hybridization->Init(ess_tdof_list);
@@ -336,15 +313,7 @@ void BilinearForm::Assemble (int skip_zeros)
    Mesh *mesh = fes -> GetMesh();
    DenseMatrix elmat, *elmat_p;
 
-#ifdef MFEM_USE_BACKENDS
-   if (dev_ext)
-   {
-      // TODO: push the 'skip_zeros' as a parameter to 'dev_ext'
-
-      const bool assembly_done = dev_ext->Assemble();
-      if (assembly_done) { return; }
-   }
-#endif
+   int i;
 
    if (mat == NULL)
    {
@@ -362,7 +331,7 @@ void BilinearForm::Assemble (int skip_zeros)
 
    if (dbfi.Size())
    {
-      for (int i = 0; i < fes -> GetNE(); i++)
+      for (i = 0; i < fes -> GetNE(); i++)
       {
          fes->GetElementVDofs(i, vdofs);
          if (element_matrices)
@@ -419,7 +388,7 @@ void BilinearForm::Assemble (int skip_zeros)
          }
       }
 
-      for (int i = 0; i < fes -> GetNBE(); i++)
+      for (i = 0; i < fes -> GetNBE(); i++)
       {
          const int bdr_attr = mesh->GetBdrAttribute(i);
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
@@ -457,7 +426,7 @@ void BilinearForm::Assemble (int skip_zeros)
       Array<int> vdofs2;
 
       int nfaces = mesh->GetNumFaces();
-      for (int i = 0; i < nfaces; i++)
+      for (i = 0; i < nfaces; i++)
       {
          tr = mesh -> GetInteriorFaceTransformations (i);
          if (tr != NULL)
@@ -502,7 +471,7 @@ void BilinearForm::Assemble (int skip_zeros)
          }
       }
 
-      for (int i = 0; i < fes -> GetNBE(); i++)
+      for (i = 0; i < fes -> GetNBE(); i++)
       {
          const int bdr_attr = mesh->GetBdrAttribute(i);
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
@@ -674,91 +643,9 @@ void BilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
    }
 }
 
-void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
-                                    Vector &x, Vector &b,
-                                    OperatorHandle &A, Vector &X, Vector &B,
-                                    int copy_interior)
-{
-#ifdef MFEM_USE_BACKENDS
-   if (dev_ext)
-   {
-      MFEM_VERIFY(!static_cond && !hybridization, "");
-      dev_ext->FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
-   }
-   else
-#endif
-   {
-      if (A.Type() == Operator::MFEM_SPARSEMAT || A.Type() == Operator::ANY_TYPE)
-      {
-         SparseMatrix A_sm;
-         FormLinearSystem(ess_tdof_list, x, b, A_sm, X, B, copy_interior);
-         if (static_cond)
-         {
-            A.Reset(&static_cond->GetMatrix(), false);
-         }
-         else if (hybridization)
-         {
-            A.Reset(&hybridization->GetMatrix(), false);
-         }
-         else
-         {
-            A.Reset(mat, false);
-         }
-      }
-      else
-      {
-         MFEM_ABORT("Operator::Type is not supported: type_id = " << A.Type());
-      }
-   }
-}
-
-void BilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
-                                    OperatorHandle &A)
-{
-#ifdef MFEM_USE_BACKENDS
-   if (dev_ext)
-   {
-      MFEM_VERIFY(!static_cond && !hybridization, "");
-      dev_ext->FormSystemMatrix(ess_tdof_list, A);
-   }
-   else
-#endif
-   {
-      if (A.Type() == Operator::MFEM_SPARSEMAT || A.Type() == Operator::ANY_TYPE)
-      {
-         SparseMatrix A_sm;
-         FormSystemMatrix(ess_tdof_list, A_sm);
-         if (static_cond)
-         {
-            A.Reset(&static_cond->GetMatrix(), false);
-         }
-         else if (hybridization)
-         {
-            A.Reset(&hybridization->GetMatrix(), false);
-         }
-         else
-         {
-            A.Reset(mat, false);
-         }
-      }
-      else
-      {
-         MFEM_ABORT("Operator::Type is not supported: type_id = " << A.Type());
-      }
-   }
-}
-
 void BilinearForm::RecoverFEMSolution(const Vector &X,
                                       const Vector &b, Vector &x)
 {
-#ifdef MFEM_USE_BACKENDS
-   if (dev_ext)
-   {
-      dev_ext->RecoverFEMSolution(X, b, x);
-      return;
-   }
-#endif
-
    const SparseMatrix *P = fes->GetConformingProlongation();
    if (!P) // conforming space
    {
@@ -847,8 +734,7 @@ void BilinearForm::ComputeElementMatrices()
 }
 
 void BilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                                        const Vector &sol, Vector &rhs,
-                                        DiagonalPolicy dpolicy)
+                                        const Vector &sol, Vector &rhs, DiagonalPolicy dpolicy)
 {
    Array<int> ess_dofs, conf_ess_dofs;
    fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);

fem/bilinearform.hpp

@@ -38,11 +38,6 @@ protected:
    /// FE space on which the form lives.
    FiniteElementSpace *fes;
 
-#ifdef MFEM_USE_BACKENDS
-   /// Device/Engine extension (smart shared pointer)
-   DBilinearForm dev_ext;
-#endif
-
    /// Indicates the Mesh::sequence corresponding to the current state of the
    /// BilinearForm.
    long sequence;
@@ -290,37 +285,6 @@ public:
    /// Form the linear system matrix A, see FormLinearSystem() for details.
    void FormSystemMatrix(const Array<int> &ess_tdof_list, SparseMatrix &A);
 
-   /** Form the linear system @a A @a X = @a B, corresponding to the bilinear
-       form and the r.h.s. linear form (vector) @a b, by applying any necessary
-       transformations such as: eliminating boundary conditions; applying
-       conforming constraints for non-conforming AMR; parallel assembly; static
-       condensation; hybridization.
-
-       The GridFunction-size vector @a x must contain the essential b.c. The
-       BilinearForm and the LinearForm-size vector @a b must be assembled.
-
-       The vector @a X is initialized with a suitable initial guess: when using
-       hybridization, the vector @a X is set to zero; otherwise, the essential
-       entries of @a X are set to the corresponding b.c. and all other entries
-       are set to zero (if @a copy_interior == 0) or copied from @a x (if
-       @a copy_interior != 0).
-
-       This method can be called multiple times (with the same @a ess_tdof_list
-       array) to initialize different right-hand sides and boundary condition
-       values.
-
-       After solving the linear system, the finite element solution @a x can be
-       recovered by calling RecoverFEMSolution() (with the same vectors @a X,
-       @a b, and @a x). */
-   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
-                                 Vector &x, Vector &b,
-                                 OperatorHandle &A, Vector &X, Vector &B,
-                                 int copy_interior = 0);
-
-   /// Form the linear system matrix @a A, see FormLinearSystem() for details.
-   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
-                                 OperatorHandle &A);
-
    /// Recover the solution of a linear system formed with FormLinearSystem().
    /** Call this method after solving a linear system constructed using the
        FormLinearSystem() method to recover the solution as a GridFunction-size

fem/bilininteg.cpp

@@ -361,28 +361,6 @@ void MixedScalarVectorIntegrator::AssembleElementMatrix2(
    }
 }
 
-
-const IntegrationRule &DiffusionIntegrator::GetRule(
-   const FiniteElement &trial_fe, const FiniteElement &test_fe)
-{
-   int order;
-   if (trial_fe.Space() == FunctionSpace::Pk)
-   {
-      order = trial_fe.GetOrder() + test_fe.GetOrder() - 2;
-   }
-   else
-   {
-      // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
-      order = trial_fe.GetOrder() + test_fe.GetOrder() + trial_fe.GetDim() - 1;
-   }
-
-   if (trial_fe.Space() == FunctionSpace::rQk)
-   {
-      return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
-   }
-   return IntRules.Get(trial_fe.GetGeomType(), order);
-}
-
 void DiffusionIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
   DenseMatrix &elmat )
@@ -405,7 +383,25 @@ void DiffusionIntegrator::AssembleElementMatrix
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      ir = &GetRule(el, el);
+      int order;
+      if (el.Space() == FunctionSpace::Pk)
+      {
+         order = 2*el.GetOrder() - 2;
+      }
+      else
+         // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
+      {
+         order = 2*el.GetOrder() + dim - 1;
+      }
+
+      if (el.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(el.GetGeomType(), order);
+      }
    }
 
    elmat = 0.0;
@@ -465,7 +461,24 @@ void DiffusionIntegrator::AssembleElementMatrix2(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      ir = &GetRule(trial_fe, test_fe);
+      int order;
+      if (trial_fe.Space() == FunctionSpace::Pk)
+      {
+         order = trial_fe.GetOrder() + test_fe.GetOrder() - 2;
+      }
+      else
+      {
+         order = trial_fe.GetOrder() + test_fe.GetOrder() + dim - 1;
+      }
+
+      if (trial_fe.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(trial_fe.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+      }
    }
 
    elmat = 0.0;
@@ -704,22 +717,6 @@ double DiffusionIntegrator::ComputeFluxEnergy
 }
 
 
-const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
-                                               const FiniteElement &test_fe,
-                                               ElementTransformation &Trans,
-                                               int coeff_order)
-{
-   // int order = trial_fe.GetOrder() + test_fe.GetOrder();
-   int order = trial_fe.GetOrder() + test_fe.GetOrder() +
-               Trans.OrderW() + coeff_order;
-
-   if (trial_fe.Space() == FunctionSpace::rQk)
-   {
-      return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
-   }
-   return IntRules.Get(trial_fe.GetGeomType(), order);
-}
-
 void MassIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
   DenseMatrix &elmat )
@@ -737,7 +734,17 @@ void MassIntegrator::AssembleElementMatrix
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      ir = &GetRule(el, el, Trans);
+      // int order = 2 * el.GetOrder();
+      int order = 2 * el.GetOrder() + Trans.OrderW();
+
+      if (el.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(el.GetGeomType(), order);
+      }
    }
 
    elmat = 0.0;
@@ -776,7 +783,9 @@ void MassIntegrator::AssembleElementMatrix2(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      ir = &GetRule(trial_fe, test_fe, Trans);
+      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
+
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
    }
 
    elmat = 0.0;
@@ -971,7 +980,16 @@ void VectorMassIntegrator::AssembleElementMatrix
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      ir = &MassIntegrator::GetRule(el, el, Trans, Q_order);
+      int order = 2 * el.GetOrder() + Trans.OrderW() + Q_order;
+
+      if (el.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(el.GetGeomType(), order);
+      }
    }
 
    elmat = 0.0;
@@ -1047,7 +1065,17 @@ void VectorMassIntegrator::AssembleElementMatrix2(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      ir = &MassIntegrator::GetRule(trial_fe, test_fe, Trans, Q_order);
+      int order = (trial_fe.GetOrder() + test_fe.GetOrder() +
+                   Trans.OrderW() + Q_order);
+
+      if (trial_fe.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(trial_fe.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+      }
    }
 
    elmat = 0.0;

fem/bilininteg.hpp

@@ -1622,13 +1622,6 @@ public:
    virtual double ComputeFluxEnergy(const FiniteElement &fluxelem,
                                     ElementTransformation &Trans,
                                     Vector &flux, Vector *d_energy = NULL);
-
-   virtual const char *Name() const { return "diffusion"; }
-
-   virtual Coefficient *GetScalarCoefficient() const { return Q; }
-
-   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
-                                         const FiniteElement &test_fe);
 };
 
 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
@@ -1656,15 +1649,6 @@ public:
                                        const FiniteElement &test_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
-
-   virtual const char *Name() const { return "mass"; }
-
-   virtual Coefficient *GetScalarCoefficient() const { return Q; }
-
-   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
-                                         const FiniteElement &test_fe,
-                                         ElementTransformation &Trans,
-                                         int coeff_order = 0);
 };
 
 class BoundaryMassIntegrator : public MassIntegrator