Interior DoF for Cubic Quadrilateral Elements Fixed and currently implemented for Quadrant 1 Mesh

Add dimension checks in the Get and Set methods of IntegrationPoint

.gitignore

@@ -112,6 +112,15 @@ examples/petsc/deformed.*
 examples/petsc/velocity.*
 examples/petsc/elastic_energy.*
 
+examples/pumi/ex1
+examples/pumi/ex[126]p
+
+examples/pumi/refined.mesh
+examples/pumi/sol.gf
+examples/pumi/mesh.*
+examples/pumi/sol.*
+examples/pumi/displaced.mesh
+
 miniapps/electromagnetics/volta
 miniapps/electromagnetics/tesla
 miniapps/electromagnetics/maxwell

CHANGELOG

@@ -8,11 +8,30 @@
                                http://mfem.org
 
 
-Version 3.3.3 (development)
+Version 3.4.1 (development)
 ===========================
+- Added support for reading linear and quadratic 2D quadrilateral and triangular
+  Cubit meshes.
+
+- The tetrahedral mesh refinement algorithm in serial and in parallel now
+  follows precisely the paper:
+     D. Arnold, A. Mukherjee, and L. Pouly, "Locally Adapted Tetrahedral Meshes
+     Using Bisection", SIAM J. Sci. Comput., 22(2), 431–448.
+  This guarantees that the shape regularity of the elements will be preserved
+  under refinement.
+
+
+Version 3.4, released on May 29, 2018
+=====================================
+
+More general and efficient mesh adaptivity
+------------------------------------------
+- Added support for PUMI, the Parallel Unstructured Mesh Infrastructure from
+  https://scorec.rpi.edu/pumi. PUMI is an unstructured, distributed mesh data
+  management system that is capable of handling general non-manifold models and
+  effectively supports automated adaptive analysis. PUMI enables for the first
+  time support for parallel unstructured modifications of MFEM meshes.
 
-More efficient non-conforming adaptive mesh refinement
-------------------------------------------------------
 - Significantly reduced MPI communication in the construction of the parallel
   prolongation matrix in ParFiniteElementSpace, for much improved parallel
   scaling of non-conforming AMR on hundreds of thousands of MPI tasks. The
@@ -81,6 +100,11 @@ New and updated examples and miniapps
   NURBS meshes in the miniapps/nurbs directory. Currently the directory contains
   variable order NURBS versions of examples 1, 1p and 11p.
 
+- Added PUMI versions of examples ex1, ex1p, ex2 and ex6p in a new examples/pumi
+  directory. The new examples demonstrate the PUMI APIs for parallel and serial
+  mesh loading (ex1 and ex1p), applying BCs using classification (ex2), and
+  performing parallel mesh adaptation (ex6p).
+
 - Added two new miniapps related to DataCollection I/O in miniapps/tools:
   load-dc.cpp can be used to visualize fields saved via DataCollection classes;
   convert-dc.cpp demonstrates how to convert between MFEM's different concrete

CMakeLists.txt

@@ -45,7 +45,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 3.3.3)
+set(${PROJECT_NAME}_VERSION 3.4.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -139,7 +139,7 @@ if (MFEM_USE_MPI)
     set(PETSC_INCLUDE_DIRS ${PETSC_INCLUDES})
   endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU PETSC STRUMPACK)
+  set(PKGS_NEED_MPI SUPERLU PETSC STRUMPACK PUMI)
   foreach(PKG IN LISTS PKGS_NEED_MPI)
     if (MFEM_USE_${PKG})
       message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -246,6 +246,22 @@ if (MFEM_USE_SIDRE)
   find_package(Axom REQUIRED Sidre SLIC axom_utils)
 endif()
 
+# PUMI
+if (MFEM_USE_PUMI)
+  # If PUMI_DIR was specified, only link to that directory,
+  # i.e. don't link to another installation in /usr/lib by mistake
+  find_package(SCOREC 2.1.0 REQUIRED OPTIONAL_COMPONENTS gmi_sim
+    CONFIG PATHS ${PUMI_DIR} NO_DEFAULT_PATH)
+  if (SCOREC_FOUND)
+    # Define a header file with the MFEM_USE_SIMMETRIX preprocessor variable
+    set(MFEM_USE_SIMMETRIX ${SCOREC_gmi_sim_FOUND})
+    set(PUMI_FOUND ${SCOREC_FOUND})
+    get_target_property(PUMI_INCLUDE_DIRS
+      SCOREC::apf INTERFACE_INCLUDE_DIRECTORIES)
+    set(PUMI_LIBRARIES SCOREC::core)
+  endif()
+endif()
+
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -270,8 +286,8 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
-    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT GECKO GNUTLS NETCDF MPFR POSIXCLOCKS
-    MFEMBacktrace ZLIB)
+    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT GECKO GNUTLS NETCDF MPFR PUMI
+    POSIXCLOCKS MFEMBacktrace ZLIB)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")

CONTRIBUTING.md

@@ -1,3 +1,15 @@
+<p align="center">
+<a href="http://mfem.org/"><img alt="mfem" src="http://mfem.org/img/logo-300.png"></a>
+</p>
+
+<p align="center">
+<a href="https://github.com/mfem/mfem/blob/master/COPYRIGHT"><img alt="License" src="https://img.shields.io/badge/License-LGPL--2.1-brightgreen.svg"></a>
+<a href="https://travis-ci.org/mfem/mfem"><img alt="Build Status" src="https://travis-ci.org/mfem/mfem.svg?branch=master"></a>
+<a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
+<a href="http://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
+</p>
+
+
 # How to Contribute
 
 The MFEM team welcomes contributions at all levels: bugfixes; code
@@ -16,6 +28,7 @@ See the [Quick Summary](#quick-summary) section for the main highlights of our
 GitHub workflow. For more details, consult the following sections and refer
 back to them before issuing pull requests:
 
+- [Code Overview](#code-overview)
 - [GitHub Workflow](#github-workflow)
   - [MFEM Organization](#mfem-organization)
   - [New Feature Development](#new-feature-development)
@@ -53,6 +66,94 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
 - Don't hesitate to [contact us](#contact-information) if you have any questions.
 
 
+### Code Overview
+
+- The MFEM library uses object-orient design principles which reflect, in code,
+  the independent mathematical concepts of meshing, linear algebra and finite
+  element spaces and operators.
+
+- The MFEM source code has the following structure:
+  ```
+  .
+  ├── config
+  │   └── cmake
+  │       └── modules
+  ├── data
+  ├── doc
+  │   └── web
+  │       └── examples
+  ├── examples
+  │   ├── petsc
+  │   ├── pumi
+  │   └── sundials
+  ├── fem
+  ├── general
+  ├── linalg
+  ├── mesh
+  └── miniapps
+      ├── common
+      ├── electromagnetics
+      ├── meshing
+      ├── nurbs
+      ├── performance
+      └── tools
+  ```
+
+- The main directories are `fem/`, `mesh/` and `linalg/` containing the C++
+  classes implementing the finite element, mesh and linear algebra concepts
+  respectively.
+
+- The main mesh classes are:
+  + [`Mesh`](http://mfem.github.io/doxygen/html/classmfem_1_1Mesh.html)
+  + [`NCMesh`](http://mfem.github.io/doxygen/html/classmfem_1_1NCMesh.html)
+  + [`Element`](http://mfem.github.io/doxygen/html/classmfem_1_1Element.html)
+  + [`ElementTransformation`](http://mfem.github.io/doxygen/html/classmfem_1_1ElementTransformation.html)
+
+- The main finite element classes are:
+  + [`FiniteElement`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementCollection`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementSpace`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElementSpace.html)
+  + [`GridFunction`](http://mfem.github.io/doxygen/html/classmfem_1_1GridFunction.html)
+  + [`BilinearFormIntegrator`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html)
+  + [`LinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1MixedBilinearForm.html)
+
+- The main linear algebra classes and sources are
+  + [`Operator`](http://mfem.github.io/doxygen/html/classmfem_1_1Operator.html) and [`BilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html)
+  + [`Vector`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearForm.html)
+  + [`DenseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1SparseMatrix.html)
+  + Sparse [smoothers](http://mfem.github.io/doxygen/html/sparsesmoothers_8hpp.html) and linear [solvers](http://mfem.github.io/doxygen/html/solvers_8hpp.html)
+
+- Parallel MPI objects in MFEM inherit their serial counterparts, so a parallel
+  mesh for example is just a serial mesh on each task plus the information on
+  shared geometric entities between different tasks. The parallel source files
+  have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
+
+- The main parallel classes are
+  + [`ParMesh`](http://mfem.github.io/doxygen/html/solvers_8hpp.html)
+  + [`ParNCMesh`](http://mfem.github.io/doxygen/html/classmfem_1_1ParMesh.html)
+  + [`ParFiniteElementSpace`](http://mfem.github.io/doxygen/html/classmfem_1_1ParFiniteElementSpace.html)
+  + [`ParGridFunction`](http://mfem.github.io/doxygen/html/classmfem_1_1ParGridFunction.html)
+  + [`ParBilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1ParLinearForm.html)
+  + [`HypreParMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParVector.html)
+  + [`HypreSolver`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreSolver.html) and other [hypre classes](http://mfem.github.io/doxygen/html/hypre_8hpp.html)
+
+- The `general/` directory contains C++ classes that serve as utilities for
+  communication, error handling, arrays, (Boolean) tables, timing, etc.
+
+- The `config/` directory contains build-related files, both for the plain
+  Makefile and the CMake build options.
+
+- The `doc/` directory contains configuration for the Doxygen code documentation
+  that can either be build locally, or browsed online at
+  http://mfem.github.io/doxygen/html/index.html.
+
+- The `data/` directory contains a collection of small mesh files, that are used
+  in the simple example codes and more fully-featured mini applications in the
+  `examples/` and `miniapps/` directories.
+
+- See also the [code overview](http://mfem.org/code-overview/) section on the
+  MFEM website.
+
 ## GitHub Workflow
 
 The GitHub organization, https://github.com/mfem, is the main developer hub for
@@ -122,7 +223,7 @@ will allow us to reach you directly with project announcements.
   # Work on "feature-dev", add local commits
   # ...
 
-  # One time only) push the branch to github and setup your local
+  # (One time only) push the branch to github and setup your local
   # branch to track the github branch (for "git pull"):
   git push -u origin feature-dev
 
@@ -332,7 +433,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
     - [ ] `CHANGELOG`
     - [ ] `makefile`
     - [ ] `CMakeLists.txt`
-    - [ ] `doc/CodeDocumentation.conf`
+    - [ ] `doc/CodeDocumentation.conf.in`
 - [ ] (LLNL only) Make sure all `README.html` files in the source repo are up to date.
 - [ ] Tag the repository:
 

INSTALL

@@ -32,6 +32,9 @@ following package managers:
 - OpenHPC, http://openhpc.community
 - Homebrew/Science, https://github.com/Homebrew/homebrew-science
 
+We also recommend downloading and building the MFEM-based GLVis visualization
+tool which can be used to visualize the meshes and solution in MFEM's examples
+and miniapps. See http://glvis.org and http://mfem.org/building.
 
 Quick start with GNU make
 =========================
@@ -352,6 +355,13 @@ MFEM_USE_GZSTREAM = YES/NO
    before attempting to use it with MFEM.
    When enabled, this option uses the ZLIB_* library options, see below.
 
+MFEM_USE_PUMI = YES/NO
+   Enable the usage of PUMI (https://scorec.rpi.edu/pumi/) in MFEM. The Parallel
+   Unstructured Mesh Infrastructure (PUMI) is an unstructured, distributed mesh
+   data management system that is capable of handling general non-manifold
+   models and effectively supports automated adaptive analysis. PUMI enables
+   support for parallel unstructured mesh modifications in MFEM.
+
 MFEM_BUILD_TAG = (any value)
    An optional tag to characterize the build. Exported to config/config.mk.
    Can be used to identify the MFEM build from other makefiles.
@@ -461,6 +471,10 @@ The specific libraries and their options are:
        https://support.hdfgroup.org/HDF5 (HDF5)
   Options: CONDUIT_OPT, CONDUIT_LIB.
 
+- PUMI, used when MFEM_USE_PUMI = YES.
+  URL: https://scorec.rpi.edu/pumi
+  Options: PUMI_OPT, PUMI_LIB.
+
 - MPFR (optional), used when MFEM_USE_MPFR = YES.
   URL: http://mpfr.org, it depends on the GMP library: https://gmplib.org
   Options: MPFR_OPT, MPFR_LIB.
@@ -593,6 +607,7 @@ MFEM_USE_GNUTLS
 MFEM_USE_NETCDF
 MFEM_USE_MPFR
 MFEM_USE_GZSTREAM
+MFEM_USE_PUMI
 
 The following options are CMake specific:
 
@@ -638,6 +653,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - MPFR
  - LIBUNWIND
  - POSIXCLOCKS
+ - PUMI
 
 The following built-in CMake packages are also used:
 

README

@@ -12,11 +12,15 @@ to enable the research and development of scalable finite element discretization
 and solver algorithms through general finite element abstractions, accurate and
 flexible visualization, and tight integration with the hypre library.
 
-For building instructions, see the file INSTALL, or type "make help". Copyright
-information and licensing restrictions can be found in the file COPYRIGHT.
+* For building instructions, see the file INSTALL, or type "make help".
 
-The best starting point for new users interested in MFEM's features is the
-interactive documentation in examples/README.html.
+* Copyright and licensing information can be found in the file COPYRIGHT.
+
+* The best starting point for new users interested in MFEM's features is the
+  interactive documentation in examples/README.html.
+
+* Developers interested in contributing to the library, should read the
+  instructions and documentation in the CONTRIBUTING.md file.
 
 Conceptually, MFEM can be viewed as a finite element toolbox that provides the
 building blocks for developing finite element algorithms in a manner similar to
@@ -56,8 +60,8 @@ time integrators, etc.
 For examples of using MFEM, see the examples/ and miniapps/ directories, as well
 as the OpenGL visualization tool GLVis which is available at http://glvis.org.
 
-This project is released under the LGPL v2.1 license. See LICENSE file for full
-details.
+This project is released under the LGPL v2.1 license with static linking
+exception. See files COPYRIGHT and LICENSE file for full details.
 
 LLNL Release Number: LLNL-CODE-443211
 DOI: 10.11578/dc.20171025.1248

backends/all.hpp

@@ -1,31 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_ALL_HPP
-#define MFEM_BACKENDS_ALL_HPP
-
-#include "../config/config.hpp"
-
-#ifdef MFEM_USE_BACKENDS
-
-#include "base/backend.hpp"
-
-#ifdef MFEM_USE_OCCA
-#include "occa/backend.hpp"
-#endif
-
-#ifdef MFEM_USE_OMP
-#include "omp/backend.hpp"
-#endif
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_ALL_HPP

backends/base/array.hpp

@@ -1,215 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_ARRAY_HPP
-#define MFEM_BACKENDS_BASE_ARRAY_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "layout.hpp"
-#include "utils.hpp"
-
-namespace mfem
-{
-
-/// Extension to the template class Array<T>
-class PArray : public RefCounted
-{
-protected:
-   /// Layout with shared ownership (smart pointer)
-   DLayout layout;
-
-
-   /**
-       @name Virtual interface
-    */
-   ///@{
-
-   virtual void *DoGetData() const = 0;
-
-   /** @brief Create and return a new array (in @a *clone) of the same dynamic
-       type as this array using the same layout and ItemSize().
-
-       Set @a *clone to NULL if allocation fails.
-
-       If @a copy_data is true, the contents of this array is copied to the new
-       array; otherwise, the new array remains uninitialized.
-
-       If @a buffer is not NULL, return the array data of the newly created
-       object (in @a *buffer) , if it is stored as a contiguous array on the
-       host; otherwise, set @a *buffer to NULL. */
-   virtual PArray *DoClone(bool copy_data, void **buffer,
-                           std::size_t item_size) const = 0;
-
-   /// Resize the array, reallocating its data if necessary.
-   /** If @a buffer is not NULL, return the array data (in @a *buffer), if it
-       is stored as a contiguous array on the host; otherwise, set @a *buffer to
-       NULL. Returns 0 on success and non-zero otherwise, e.g. if memory
-       allocation fails.
-
-       If the @a new_layout is not supported, a non-zero error code will be
-       returned.
-
-       The @a new_layout has to be valid, i.e. new_layout != NULL and
-       new_layout->HasEngine() == true.
-
-       @note If reallocation is performed, the previous content of the array is
-       NOT copied to the new location. */
-   virtual int DoResize(PLayout &new_layout, void **buffer,
-                        std::size_t item_size) = 0;
-
-   /** @brief Get access to the contents of the array in host memory, as a
-       contiguous array. */
-   /** If the array data is stored as a contiguous array in host memory, return
-       a pointer to it. Otherwise, copy the data to @a buffer (if @a buffer is
-       not NULL) and return @a buffer.
-       @note If not NULL, @a buffer is assumed to be of size greater than or
-       equal to Size(). */
-   virtual void *DoPullData(void *buffer, std::size_t item_size) = 0;
-
-   /** @brief Set all entries of the array to the (single) value pointed to by
-       @a value_ptr. */
-   virtual void DoFill(const void *value_ptr, std::size_t item_size) = 0;
-
-   /** @brief Set all Size() entries of the array from the given contiguous
-       array, @a src_buffer, on the host. */
-   virtual void DoPushData(const void *src_buffer, std::size_t item_size) = 0;
-
-   /// Copy the data from @a src to @a *this.
-   /** Both arrays must have the same dynamic type, layout, and item_size. */
-   virtual void DoAssign(const PArray &src, std::size_t item_size) = 0;
-
-   ///@}
-   // End: Virtual interface
-
-public:
-   /** @brief The @a layout parameter will be reference counted and therefore it
-       should be dynamically allocated. */
-   /** The @a layout must be valid in the sense that layout != NULL and
-       layout->HasEngine() == true. */
-   PArray(PLayout &p_layout)
-      : layout(&p_layout)
-   {
-      MFEM_ASSERT(layout && layout->HasEngine(), "invalid layout");
-   }
-
-   virtual ~PArray() { }
-
-   /// Get the current size of the array.
-   std::size_t Size() const { return layout->Size(); }
-
-   /// Get the current layout of the array.
-   PLayout &GetLayout() const { return *layout; }
-
-   /// TODO
-   template <typename derived_t>
-   derived_t &As() { return dynamic_cast<derived_t&>(*this); }
-
-   /// TODO
-   template <typename derived_t>
-   const derived_t &As() const { return dynamic_cast<const derived_t&>(*this); }
-
-   // TODO: Error handling ... handle errors at the Engine level, at the class
-   //       level, or at the method level?
-
-   // TODO: Asynchronous execution interface ...
-
-
-   /**
-       @name Public virtual interface
-    */
-   ///@{
-
-   template <typename T=void>
-   T* GetData() const { return (T*) DoGetData(); }
-
-   /** @brief Create and return a new array (in @a *clone) of the same dynamic
-       type as this array using the same layout and ItemSize().
-
-       Set @a *clone to NULL if allocation fails.
-
-       If @a copy_data is true, the contents of this array is copied to the new
-       array; otherwise, the new array remains uninitialized.
-
-       If @a buffer is not NULL, return the array data of the newly created
-       object (in @a *buffer) , if it is stored as a contiguous array on the
-       host; otherwise, set @a *buffer to NULL. */
-   template <typename T>
-   DArray Clone(bool copy_data, T **buffer) const
-   { return DArray(DoClone(copy_data, (void**)buffer, sizeof(T))); }
-
-   /// Resize the array, reallocating its data if necessary.
-   /** If @a buffer is not NULL, return the array data (in @a *buffer), if it
-       is stored as a contiguous array on the host; otherwise, set @a *buffer to
-       NULL. Returns 0 on success and non-zero otherwise, e.g. if memory
-       allocation fails.
-
-       If the @a new_layout is not supported, a non-zero error code will be
-       returned.
-
-       The @a new_layout has to be valid, i.e. new_layout != NULL and
-       new_layout->HasEngine() == true.
-
-       @note If reallocation is performed, the previous content of the array is
-       NOT copied to the new location. */
-   template <typename T>
-   int Resize(PLayout &new_layout, T **buffer)
-   { return DoResize(new_layout, (void**)buffer, sizeof(T)); }
-
-   /// Shortcut for Resize(*layout, buffer).
-   /** This method is useful for updating the array after its layout is changed
-       externally. */
-   template <typename T>
-   int Update(T **buffer)
-   { return DoResize(*layout, (void**)buffer, sizeof(T)); }
-
-   /// Shortcut for layout->Resize(new_size) followed by Update()
-   template <typename T>
-   int Resize(std::size_t new_size, T **buffer)
-   { layout->Resize(new_size); return Update(buffer); }
-
-   /** @brief Get access to the contents of the array in host memory, as a
-       contiguous array. */
-   /** If the array data is stored as a contiguous array in host memory, return
-       a pointer to it. Otherwise, copy the data to @a buffer (if @a buffer is
-       not NULL) and return @a buffer.
-       @note If not NULL, @a buffer is assumed to be of size greater than or
-       equal to Size(). */
-   template <typename T>
-   T *PullData(T *buffer)
-   { return Size() ? (T*)DoPullData((void*)buffer, sizeof(T)) : NULL; }
-
-   /** @brief Set all entries of the array to the (single) value pointed to by
-       @a value_ptr. */
-   template <typename T>
-   void Fill(const T &value) { if (Size()) { DoFill(&value, sizeof(T)); } }
-
-   /** @brief Set all Size() entries of the array from the given contiguous
-       array, @a src_buffer, on the host. */
-   template <typename T>
-   void PushData(const T *src_buffer)
-   { if (Size()) { DoPushData(src_buffer, sizeof(T)); } }
-
-   /// Copy the data from @a src to @a *this.
-   /** Both arrays must have the same dynamic type, layout, and entry type. */
-   template <typename T>
-   void Assign(const PArray &src) { if (Size()) { DoAssign(src, sizeof(T)); } }
-
-   ///@}
-   // End: Virtual interface
-};
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_ARRAY_HPP

backends/base/backend.hpp

@@ -1,57 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_BACKEND_HPP
-#define MFEM_BACKENDS_BASE_BACKEND_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "memory_resource.hpp"
-#include "engine.hpp"
-#include "array.hpp"
-#include "vector.hpp"
-#include "fespace.hpp"
-#include "bilinearform.hpp"
-
-#include <string>
-
-#ifdef MFEM_USE_MPI
-#include <mpi.h>
-#endif
-
-namespace mfem
-{
-
-/// TODO
-class Backend
-{
-public:
-   /// TODO
-   virtual ~Backend() { }
-
-   /// TODO
-   virtual bool Supports(const std::string &engine_spec) const = 0;
-
-   /// TODO
-   virtual Engine *Create(const std::string &engine_spec) = 0;
-
-#ifdef MFEM_USE_MPI
-   /// TODO
-   virtual Engine *Create(MPI_Comm comm, const std::string &engine_spec) = 0;
-#endif
-};
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_BACKEND_HPP

backends/base/bilinearform.hpp

@@ -1,72 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_BILINEARFORM_HPP
-#define MFEM_BACKENDS_BASE_BILINEARFORM_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "engine.hpp"
-
-namespace mfem
-{
-
-class Vector;
-class OperatorHandle;
-class BilinearForm;
-
-/// TODO: doxygen
-class PBilinearForm : public RefCounted
-{
-protected:
-   /// Engine with shared ownership
-   SharedPtr<const Engine> engine;
-   /// Not owned.
-   BilinearForm *bform;
-
-public:
-   /// TODO: doxygen
-   PBilinearForm(const Engine &e, BilinearForm &bf)
-      : engine(&e), bform(&bf) { }
-
-   /// Virtual destructor
-   virtual ~PBilinearForm() { }
-
-   /// Get the associated Engine
-   const Engine &GetEngine() const { return *engine; }
-
-   /// Assemble the PBilinearForm.
-   /** This method is called from the method BilinearForm::Assemble() of the
-       associated BilinearForm #bform.
-       @returns True, if the host assembly should be skipped. */
-   virtual bool Assemble() = 0;
-
-   /// TODO: doxygen
-   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
-                                 OperatorHandle &A) = 0;
-
-   /// TODO: doxygen
-   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
-                                 Vector &x, Vector &b,
-                                 OperatorHandle &A, Vector &X, Vector &B,
-                                 int copy_interior) = 0;
-
-   /// TODO: doxygen
-   virtual void RecoverFEMSolution(const Vector &X, const Vector &b,
-                                   Vector &x) = 0;
-};
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_BILINEARFORM_HPP

backends/base/engine.cpp

@@ -1,29 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "engine.hpp"
-#include "fespace.hpp"
-#include "bilinearform.hpp"
-
-namespace mfem
-{
-
-DFiniteElementSpace Engine::MakeFESpace(FiniteElementSpace &fes) const
-{
-   return DFiniteElementSpace(new PFiniteElementSpace(*this, fes));
-}
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS

backends/base/engine.hpp

@@ -1,210 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_ENGINE_HPP
-#define MFEM_BACKENDS_BASE_ENGINE_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "../../general/scalars.hpp"
-#include "memory_resource.hpp"
-#include "smart_pointers.hpp"
-#include "utils.hpp"
-
-#ifdef MFEM_USE_MPI
-#include <mpi.h>
-#endif
-
-namespace mfem
-{
-
-// Forward declarations.
-class Backend;
-template <typename T> class Array;
-class Vector;
-class Operator;
-class FiniteElementSpace;
-class LinearForm;
-class BilinearForm;
-class MixedBilinearForm;
-class NonlinearForm;
-
-
-/// In parallel, each MPI rank will usually create a single engine.
-class Engine : public RefCounted
-{
-protected:
-   Backend *backend; ///< Backend that created the engine. Not owned.
-
-#ifdef MFEM_USE_MPI
-   MPI_Comm comm; ///< Associated MPI communicator (may be MPI_COMM_NULL).
-#endif
-
-   /// Number of memory resources used by the Engine.
-   int num_mem_res;
-   /// Number of workers used by the Engine.
-   int num_workers;
-
-   /// Memory resources used by the engine - array of pointers.
-   /** Both the array and the entries are owned. */
-   MemoryResource **memory_resources;
-
-   /// Relative computational speed of the workers. Owned.
-   double *workers_weights;
-
-   /// For each worker, which memory resource it uses.
-   int *workers_mem_res;
-
-public:
-   /// TODO: doxygen
-   Engine(Backend *b, int n_mem, int n_workers)
-      : backend(b),
-#ifdef MFEM_USE_MPI
-        comm(MPI_COMM_NULL),
-#endif
-        num_mem_res(n_mem),
-        num_workers(n_workers),
-        memory_resources(new MemoryResource*[num_mem_res]()),
-        workers_weights(new double[num_workers]()),
-        workers_mem_res(new int[num_workers]())
-   { /* Note: all arrays are value-initialized with zeros. */ }
-
-   /// TODO: doxygen
-   virtual ~Engine()
-   {
-      delete [] workers_mem_res;
-      delete [] workers_weights;
-      for (int i = 0; i < num_mem_res; i++)
-      {
-         delete memory_resources[i];
-      }
-      delete [] memory_resources;
-   }
-
-
-   /**
-       @name Machine resources interface
-    */
-   ///@{
-
-#ifdef MFEM_USE_MPI
-   /// Get the associated MPI_Comm
-   MPI_Comm GetComm() const { return comm; }
-#endif
-
-   /// TODO
-   int GetNumMemRes() const { return num_mem_res; }
-
-   /// TODO
-   MemoryResource &GetMemRes(int idx) const { return *memory_resources[idx]; }
-
-   /// TODO
-   int GetNumWorkers() const { return num_workers; }
-
-   /// TODO
-   const double *GetWorkersWeights() const { return workers_weights; }
-
-   /// TODO
-   const int *GetWorkersMemRes() const { return workers_mem_res; }
-
-   ///@}
-   // End: Machine resources interface
-
-   /// TODO
-   template <typename derived_t>
-   derived_t &As() { *util::As<derived_t>(this); }
-
-   /// TODO
-   template <typename derived_t>
-   const derived_t &As() const { *util::As<const derived_t>(this); }
-
-
-   // TODO: Error handling ... handle errors at the Engine level, at the class
-   //       level, or at the method level?
-
-
-   /**
-       @name Virtual interface: finite element data structures and algorithms
-    */
-   ///@{
-
-   // TODO: Asynchronous execution in this class ...
-
-   /// Allocate and return a new layout for the given @a size.
-   /** The layout decomposition is determined automatically by the Engine using
-       a deterministic algorithm: calls to this method with the same @a size
-       will produce the same result, as long as the Engine remains unmodified
-       between the calls.
-
-       The returned object is allocated with operator new and must be
-       deallocated by the caller.
-
-       TODO: Returns NULL if memory allocation fails?
-   */
-   virtual DLayout MakeLayout(std::size_t size) const = 0;
-
-   /// Allocate and return a new layout for the given worker decomposition.
-   /** The returned object is allocated with operator new and must be
-       deallocated by the caller.
-
-       TODO: Returns NULL if memory allocation fails?
-
-       The @a offsets should satisfy: offsets.Size() == number of workers + 1,
-       offsets[0] == 0, and offsets[i] <= offsets[i+1], for i: 0 <= i < number
-       of workers. */
-   virtual DLayout MakeLayout(const Array<std::size_t> &offsets) const = 0;
-
-   // Note: There may be other ways to construct layouts in the future, e.g.
-   //       block-vector layouts, or multi-vector layouts.
-
-   /// TODO
-   virtual DArray MakeArray(PLayout &layout, std::size_t item_size) const = 0;
-
-   /// Allocate and return a new vector using the given @a layout.
-   /** The returned object is a smart pointer that will automatically deallocate
-       the vector.
-
-       TODO: Produce an error if memory allocation fails?
-
-       Only layouts returned by this Engine are guaranteed to be supported.
-       Using a type that is not supported will produce an error. */
-   virtual DVector MakeVector(PLayout &layout,
-                              int type_id = ScalarId<double>::value) const = 0;
-
-   /// TODO: doxygen
-   virtual DFiniteElementSpace MakeFESpace(FiniteElementSpace &fes) const;
-
-   /// TODO: doxygen
-   virtual DBilinearForm MakeBilinearForm(BilinearForm &bf) const = 0;
-
-
-   // Question: How do we construct coefficients?
-
-
-   /// FIXME - What will the actual parameters be?
-   virtual void AssembleLinearForm(LinearForm &l_form) const = 0;
-
-   /// FIXME - What will the actual parameters be?
-   virtual Operator *MakeOperator(const MixedBilinearForm &mbl_form) const = 0;
-
-   /// FIXME - What will the actual parameters be?
-   virtual Operator *MakeOperator(const NonlinearForm &nl_form) const = 0;
-
-   ///@}
-   // End: Virtual interface
-};
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_ENGINE_HPP

backends/base/fespace.hpp

@@ -1,61 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_FE_SPACE_HPP
-#define MFEM_BACKENDS_BASE_FE_SPACE_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "engine.hpp"
-#include "utils.hpp"
-
-namespace mfem
-{
-
-class FiniteElementSpace;
-
-/// TODO: doxygen
-class PFiniteElementSpace : public RefCounted
-{
-protected:
-   /// Engine with shared ownership
-   SharedPtr<const Engine> engine;
-   /// Not owned.
-   FiniteElementSpace *fes;
-
-public:
-   /// TODO: doxygen
-   PFiniteElementSpace(const Engine &e, FiniteElementSpace &fespace)
-      : engine(&e), fes(&fespace) { }
-
-   /// Virtual destructor
-   virtual ~PFiniteElementSpace() { }
-
-   /// Get the associated engine
-   const Engine &GetEngine() const { return *engine; }
-
-   mfem::FiniteElementSpace* GetFESpace() const { return fes; }
-
-   /// TODO
-   template <typename derived_t>
-   derived_t &As() { return *util::As<derived_t>(this); }
-
-   /// TODO
-   template <typename derived_t>
-   const derived_t &As() const { return *util::As<const derived_t>(this); }
-};
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_FE_SPACE_HPP

backends/base/layout.hpp

@@ -1,104 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_LAYOUT_HPP
-#define MFEM_BACKENDS_BASE_LAYOUT_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "smart_pointers.hpp"
-#include "engine.hpp"
-
-namespace mfem
-{
-
-/// Polymorphic layout (array/vector layout descriptor)
-class PLayout : public RefCounted
-{
-protected:
-   /// Engine with shared ownership
-   SharedPtr<const Engine> engine;
-   std::size_t size;
-
-   template <typename DObject>
-   struct Maker
-   {
-      template <typename entry_t>
-      static DObject MakeNew(PLayout &layout);
-   };
-
-public:
-   explicit PLayout(std::size_t s = 0) : engine(NULL), size(s) { }
-
-   explicit PLayout(const Engine &e, std::size_t s = 0)
-      : engine(&e), size(s) { }
-
-   virtual ~PLayout() { }
-
-   /**
-       @name Virtual interface
-    */
-   ///@{
-
-   /// Resize the layout
-   virtual void Resize(std::size_t new_size) { size = new_size; }
-
-   /// Resize the layout based on the given worker offsets
-   virtual void Resize(const Array<std::size_t> &offsets)
-   { MFEM_ABORT("method not supported"); }
-
-   ///@}
-   // End: Virtual interface
-
-   /// Layouts without engine cannot create DArray, DVector, etc.
-   bool HasEngine() const { return engine != NULL; }
-
-   /// TODO: doxygen
-   const Engine &GetEngine() const { return *engine; }
-
-   /// TODO: doxygen
-   std::size_t Size() const { return size; }
-
-   /// TODO
-   template <typename derived_t>
-   derived_t &As() { return *util::As<derived_t>(this); }
-
-   /// TODO
-   template <typename derived_t>
-   const derived_t &As() const { return *util::As<const derived_t>(this); }
-
-   /// TODO: doxygen
-   template <typename DObject, typename entry_t>
-   DObject Make()
-   {
-      MFEM_ASSERT(HasEngine(), "this method requires an Engine");
-      return Maker<DObject>::template MakeNew<entry_t>(*this);
-   }
-};
-
-template <> struct PLayout::Maker<DArray>
-{
-   template <typename entry_t> static DArray MakeNew(PLayout &layout)
-   { return layout.GetEngine().MakeArray(layout, sizeof(entry_t)); }
-};
-
-template <> struct PLayout::Maker<DVector>
-{
-   template <typename entry_t> static DVector MakeNew(PLayout &layout)
-   { return layout.GetEngine().MakeVector(layout, ScalarId<entry_t>::value); }
-};
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_LAYOUT_HPP

backends/base/memory_resource.cpp

@@ -1,59 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "memory_resource.hpp"
-#include "../../general/error.hpp"
-
-#include <cstdlib>
-#include <cstring>
-#include <cerrno>
-
-namespace mfem
-{
-
-void *NewDeleteMemoryResource::DoAllocate(std::size_t bytes,
-                                          std::size_t alignment)
-{
-   void *p = ::operator new[](bytes);
-   MFEM_VERIFY(!alignment || (std::size_t)(p) % alignment == 0,
-               "invalid alignment");
-   return p;
-}
-
-void NewDeleteMemoryResource::DoDeallocate(void *p, std::size_t bytes,
-                                           std::size_t alignment)
-{
-   ::operator delete[](p);
-}
-
-
-void *AlignedMemoryResource::DoAllocate(std::size_t bytes,
-                                        std::size_t alignment)
-{
-   void *p;
-   if (!alignment) { alignment = sizeof(long double); }
-   MFEM_VERIFY(posix_memalign(&p, alignment, bytes) == 0,
-               "error in posix_memalign(): " << strerror(errno));
-   return p;
-}
-
-void AlignedMemoryResource::DoDeallocate(void *p, std::size_t bytes,
-                                         std::size_t alignment)
-{
-   free(p);
-}
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS

backends/base/memory_resource.hpp

@@ -1,70 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP
-#define MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include <cstddef>
-
-namespace mfem
-{
-
-/// Polymorphic memory resource. Similar to C++17's std::pmr::memory_resource.
-class MemoryResource
-{
-protected:
-   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment) = 0;
-   virtual void DoDeallocate(void* p, std::size_t bytes,
-                             std::size_t alignment) = 0;
-
-public:
-   // Implicitly defined default & copy constructors
-
-   /// Virtual destructor.
-   virtual ~MemoryResource() { }
-
-   /// If alignment == 0, use default alignment.
-   void *Allocate(std::size_t bytes, std::size_t alignment = 0)
-   { return DoAllocate(bytes, alignment); }
-
-   /// If alignment == 0, use default alignment.
-   void Deallocate(void *p, std::size_t bytes, std::size_t alignment = 0)
-   { DoDeallocate(p, bytes, alignment); }
-};
-
-
-/** @brief Dynamic host memory resource using operator new[](std::size_t) for
-    allocation and operator delete[](void*) for deallocation. */
-class NewDeleteMemoryResource : public MemoryResource
-{
-protected:
-   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment);
-   virtual void DoDeallocate(void *p, std::size_t bytes, std::size_t alignment);
-};
-
-
-/** @brief Dynamic host memory resource using posix_memalign() for aligned
-    allocation and free() for deallocation. */
-class AlignedMemoryResource : public MemoryResource
-{
-protected:
-   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment);
-   virtual void DoDeallocate(void *p, std::size_t bytes, std::size_t alignment);
-};
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_MEMORY_RESOURCE_HPP

backends/base/smart_pointers.hpp

@@ -1,233 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_SMART_POINTERS_HPP
-#define MFEM_BACKENDS_BASE_SMART_POINTERS_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "utils.hpp"
-#include "../../general/error.hpp"
-#include <cstddef>
-
-// #define MFEM_TRACE_SHARED_PTR
-#ifdef MFEM_TRACE_SHARED_PTR
-#include "../../general/globals.hpp"
-#endif
-
-namespace mfem
-{
-
-/// Base class for classes with simple reference counting.
-/** Reference counting is performed by the class SharedPtr. */
-class RefCounted
-{
-private:
-   mutable unsigned ref_count;
-
-   /// Only class SharedPtr can access ref_count.
-   template <typename T> friend class SharedPtr;
-
-public:
-   RefCounted() : ref_count(0) { }
-
-   /** @brief Prevent SharedPtr objects from deleting this object by
-       incrementing the reference counter by one. */
-   void DontDelete() const { ++ref_count; }
-};
-
-
-/** @brief Smart pointer class that manages objects of type T derived from class
-    RefCounted. */
-/** This class is generally meant to work with dynamically allocated object,
-    specifically objects allocated with operator new(). It will invoke operator
-    delete() to destroy the managed object when its reference counter reaches
-    zero. This behavior can be overriden by calling RefCounted::DontDelete() to
-    ensure that an object will not be deleted by a SharedPtr that holds a
-    pointer to it.
-    @note This class is NOT thread-safe and does not support circular ownership.
- */
-template <typename T>
-class SharedPtr
-{
-public:
-   typedef T stored_type;
-
-private:
-   T *ptr;
-
-   void Init(T *new_ptr)
-   {
-      ptr = new_ptr;
-      if (ptr) { ++ptr->RefCounted::ref_count; }
-#ifdef MFEM_TRACE_SHARED_PTR
-#elif 0
-      mfem::out << "  [" << _MFEM_FUNC_NAME << "]: ptr = " << ptr;
-      if (ptr)
-      {
-         mfem::out << ", new ref_count = " << ptr->RefCounted::ref_count;
-      }
-      mfem::out << '\n';
-#endif
-   }
-   void Destroy()
-   {
-      MFEM_ASSERT(!ptr || ptr->RefCounted::ref_count >= 1, "invalid use");
-      if (ptr && --ptr->RefCounted::ref_count == 0) { delete ptr; }
-#ifdef MFEM_TRACE_SHARED_PTR
-#elif 0
-      mfem::out << "  [" << _MFEM_FUNC_NAME << "]: ptr = " << ptr;
-      if (ptr)
-      {
-         mfem::out << ", new ref_count = " << ptr->RefCounted::ref_count;
-      }
-      mfem::out << '\n';
-#endif
-   }
-
-public:
-   SharedPtr() : ptr(NULL)
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]: ptr = " << ptr << '\n';
-#endif
-   }
-
-   SharedPtr(const SharedPtr &other)
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      Init(other.ptr);
-   }
-
-   template <typename U>
-   SharedPtr(const SharedPtr<U> &other)
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      Init(other.Get());
-   }
-
-   explicit SharedPtr(T *p)
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      Init(p);
-   }
-
-   ~SharedPtr()
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      Destroy();
-   }
-
-   SharedPtr &operator=(const SharedPtr &other)
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      Reset(other.ptr); return *this;
-   }
-
-   template <typename U>
-   SharedPtr &operator=(const SharedPtr<U> &other)
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      Reset(other.Get()); return *this;
-   }
-
-   T &operator*() const { return *ptr; }
-   T *operator->() const { return ptr; }
-
-   operator bool() const { return ptr; }
-   bool operator!() const { return !ptr; }
-
-   template <typename U>
-   bool operator==(const SharedPtr<U> &other) const
-   { return ptr == other.Ptr(); }
-   template <typename U>
-   bool operator!=(const SharedPtr<U> &other) const
-   { return ptr != other.Ptr(); }
-
-   template <typename U>
-   bool operator==(const U &p) const { return ptr == (void*) p; }
-   template <typename U>
-   bool operator!=(const U &p) const { return ptr != (void*) p; }
-
-   T *Get() const { return ptr; }
-
-   /// TODO
-   template <typename derived_t>
-   derived_t *As() const { return util::As<derived_t>(ptr); }
-
-   unsigned UseCount() const { return ptr ? ptr->RefCounted::ref_count : 0; }
-
-   void Reset()
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      Destroy();
-      ptr = NULL;
-   }
-
-   /// The type U* needs to be implicitly convertible to T*
-   template <typename U>
-   void Reset(U *new_ptr)
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      if (ptr != new_ptr) { Destroy(); Init(new_ptr); }
-   }
-
-   void Swap(SharedPtr &other)
-   {
-#ifdef MFEM_TRACE_SHARED_PTR
-      mfem::out << '[' << _MFEM_FUNC_NAME << "]\n";
-#endif
-      std::swap(ptr, other.ptr);
-   }
-};
-
-
-template <class T>
-inline void Swap(SharedPtr<T> &a, SharedPtr<T> &b) { a.Swap(b); }
-
-
-class PLayout;
-typedef SharedPtr<PLayout> DLayout;
-
-class PArray;
-typedef SharedPtr<PArray> DArray;
-
-class PVector;
-typedef SharedPtr<PVector> DVector;
-
-class PFiniteElementSpace;
-typedef SharedPtr<PFiniteElementSpace> DFiniteElementSpace;
-
-class PBilinearForm;
-typedef SharedPtr<PBilinearForm> DBilinearForm;
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_SMART_POINTERS_HPP

backends/base/utils.hpp

@@ -1,52 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_UTILS_HPP
-#define MFEM_BACKENDS_BASE_UTILS_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "../../general/error.hpp"
-
-namespace mfem
-{
-
-namespace util
-{
-
-//
-// Inline methods
-//
-
-/// TODO: doxygen
-template <typename derived_t, typename base_t>
-inline derived_t *As(base_t *base_obj)
-{
-   MFEM_ASSERT(dynamic_cast<derived_t*>(base_obj) != NULL,
-               "invalid object type");
-   return static_cast<derived_t*>(base_obj);
-}
-
-/// TODO: doxygen
-template <typename derived_t, typename base_t>
-inline derived_t *Is(base_t *base_obj)
-{
-   return dynamic_cast<derived_t*>(base_obj);
-}
-
-} // namespace mfem::util
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_UTILS_HPP

backends/base/vector.hpp

@@ -1,153 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_BASE_VECTOR_HPP
-#define MFEM_BACKENDS_BASE_VECTOR_HPP
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_BACKENDS
-
-#include "../../general/scalars.hpp"
-#include "array.hpp"
-
-namespace mfem
-{
-
-/// Polymorphic vector - array of scalars.
-class PVector : virtual public PArray
-{
-protected:
-   /**
-       @name Virtual interface
-    */
-   ///@{
-
-   /** @brief Create and return a new vector of the same dynamic type as this
-       vector using the same layout with entries specified by @a buffer_type_id
-       which should be a constant defined by the `value` field in a
-       specialization of the template class mfem::ScalarId.
-
-       Returns NULL if allocation fails.
-
-       If @a copy_data is true, the contents of this vector is copied to the new
-       vector; otherwise, the new vector remains uninitialized.
-
-       If @a buffer is not NULL, return the vector data of the newly created
-       object (in @a *buffer), if it is stored as a contiguous array on the
-       host; otherwise, set @a *buffer to NULL. */
-   virtual PVector *DoVectorClone(bool copy_data, void **buffer,
-                                  int buffer_type_id) const = 0;
-
-   /** @brief Compute and return the dot product of @a *this and @a x. In the
-       case of an MPI-parallel vector, the result must be the MPI-global dot
-       product. */
-   /** Both vectors must have the same dynamic type and layout. */
-   virtual void DoDotProduct(const PVector &x, void *result,
-                             int result_type_id) const = 0;
-
-   // TODO: add reduction operations: min, max, sum
-
-   /// Perform the operation @a *this = @a a @a x + @a b @a y.
-   /** Rules:
-       - the dynamic type of both @a x and @a y is the same as that of @a *this
-       - if @a a == 0, neither @a x nor its data are accessed
-       - if @a b == 0, neither @a y nor its data are accessed
-       - @a x's data is never the same as @a y's data, unless @a a == 0, or
-         @a b == 0
-       - @a x's data or @a y's data may be the same as the data of @a *this
-       - all accessed vectors, @a x, @a y, and @a *this have the same layout. */
-   virtual void DoAxpby(const void *a, const PVector &x,
-                        const void *b, const PVector &y,
-                        int ab_type_id) = 0;
-
-   ///@}
-   // End: Virtual interface
-
-public:
-   /** @brief Create a PVector. */
-   /** The @a layout must be valid in the sense that layout != NULL and
-       layout->HasEngine() == true. */
-   PVector(PLayout &p_layout)
-      : PArray(p_layout) { }
-
-   template <typename derived_t>
-   derived_t &As() { return *util::As<derived_t>(this); }
-
-   template <typename derived_t>
-   const derived_t &As() const { return *util::As<const derived_t>(this); }
-
-
-   // TODO: Error handling ... handle errors at the Engine level, at the class
-   //       level, or at the method level?
-
-   // TODO: Asynchronous execution interface ...
-
-   // TODO: Multi-vector interface ...
-
-
-   /**
-       @name Public virtual interface
-    */
-   ///@{
-
-   /** @brief Create and return a new vector of the same dynamic type as this
-       vector using the same layout with entries of type @a scalar_t.
-
-       If @a copy_data is true, the contents of this vector is copied to the new
-       vector; otherwise, the new vector remains uninitialized.
-
-       If @a buffer is not NULL, return the vector data of the newly created
-       object (in @a *buffer) , if it is stored as a contiguous array on the
-       host; otherwise, set @a *buffer to NULL. */
-   template <typename scalar_t>
-   DVector Clone(bool copy_data, scalar_t **buffer) const
-   {
-      return DVector(DoVectorClone(copy_data, (void**)buffer,
-                                   ScalarId<scalar_t>::value));
-   }
-
-   /** @brief Compute and return the dot product of @a *this and @a x. In the
-       case of an MPI-parallel vector, the result must be the MPI-global dot
-       product. */
-   /** Both vectors must have the same dynamic type and layout. */
-   template <typename scalar_t>
-   scalar_t DotProduct(const PVector &x) const
-   {
-      scalar_t result;
-      DoDotProduct(x, &result, ScalarId<scalar_t>::value);
-      return result;
-   }
-
-   // TODO: add reduction operations: min, max, sum
-
-   /// Perform the operation @a *this = @a a @a x + @a b @a y.
-   /** Rules:
-       - the dynamic type of both @a x and @a y is the same as that of @a *this
-       - if @a a == 0, neither @a x nor its data are accessed
-       - if @a b == 0, neither @a y nor its data are accessed
-       - @a x's data is never the same as @a y's data, unless @a a == 0, or
-         @a b == 0
-       - @a x's data or @a y's data may be the same as the data of @a *this
-       - all accessed vectors, @a x, @a y, and @a *this have the same layout. */
-   template <typename scalar_t>
-   void Axpby(const scalar_t &a, const PVector &x,
-              const scalar_t &b, const PVector &y)
-   { if (Size()) { DoAxpby(&a, x, &b, y, ScalarId<scalar_t>::value); } }
-
-   ///@}
-   // End: Virtual interface
-};
-
-} // namespace mfem
-
-#endif // MFEM_USE_BACKENDS
-
-#endif // MFEM_BACKENDS_BASE_VECTOR_HPP

backends/occa/DiffusionIntegrator.okl

@@ -1,66 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-/*
-  ---[ Defines Known At Compile-Time ]------------
-  ELEMENT_BATCH   : How many elements are in each
-  .                   computation batch
-  NUM_DOFS_1D     : Dofs in the 1D segments
-  NUM_DOFS_2D     : Dofs in the 2D faces
-  NUM_DOFS_3D     : Dofs in the 3D domain
-  NUM_QUAD_1D     : Dofs in the 1D segments
-  NUM_QUAD_2D     : Dofs in the 2D faces
-  NUM_QUAD_3D     : Dofs in the 3D domain
-  NUM_MAX_1D      : max(NUM_QUAD_1D, NUM_DOFS_1D)
-  NUM_QUAD_DOFS_1D: NUM_QUAD_1D * NUM_DOFS_1D
-  COEFF_ARGS      : Code that passes required arguments to the kernel
-  COEFF           : Code that computes the coefficient
-  ================================================
-
-  [MISSING]
-  - Add support to auto-pick @dim and use @idxOrder on stack arrays
-  |   double a[2][2];
-  |   a[0][1]; <-- regular index
-  |   a(0,1);  <-- uses @idxOrder a[0][1] or a[1][0]
-  - Add support for @idxOrder to change indexing order after allocation
-  |   double a[2][2] @idxOrder(0,1);
-  |   a(0,1) -> a[1][0]
-  |   @set(a, idxOrder(1,0));
-  |   a(0,1) -> a[0][1]
-  - Add support to iterate over loop depending on mode
-  |   for(i; @inner) {
-  |     for(0 < j < N) {} <-- ++j or j += block?
-  |   }
-*/
-
-#include "mfem-occa://defines.okl"
-
-#if USING_TENSOR_OPS
-#  ifdef OCCA_USING_GPU
-#    if USING_LOW_ORDER
-#      include "mfem-occa://diffusion/tensor/gpuHighOrder.okl"
-#    else
-#      include "mfem-occa://diffusion/tensor/gpuHighOrder.okl"
-#    endif
-#  else
-#    include "mfem-occa://diffusion/tensor/cpu.okl"
-#  endif
-#else
-#  ifdef OCCA_USING_GPU
-#    if USING_LOW_ORDER
-#      include "mfem-occa://diffusion/simplex/gpuHighOrder.okl"
-#    else
-#      include "mfem-occa://diffusion/simplex/gpuHighOrder.okl"
-#    endif
-#  else
-#    include "mfem-occa://diffusion/simplex/cpu.okl"
-#  endif
-#endif

backends/occa/array.cpp

@@ -1,123 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "array.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-PArray *Array::DoClone(bool copy_data, void **buffer,
-                       std::size_t item_size) const
-{
-   Array *new_array = new Array(OccaLayout(), item_size);
-   if (copy_data)
-   {
-      new_array->slice.copyFrom(slice);
-   }
-   if (buffer)
-   {
-      *buffer = new_array->GetBuffer();
-   }
-   return new_array;
-}
-
-int Array::DoResize(PLayout &new_layout, void **buffer,
-                    std::size_t item_size)
-{
-   MFEM_ASSERT(dynamic_cast<Layout *>(&new_layout) != NULL,
-               "new_layout is not an OCCA Layout");
-   Layout *lt = static_cast<Layout *>(&new_layout);
-   layout.Reset(lt); // Reset() checks if the pointer is the same
-   int err = ResizeData(lt, item_size);
-   if (!err && buffer)
-   {
-      *buffer = GetBuffer();
-   }
-   return err;
-}
-
-void *Array::DoPullData(void *buffer, std::size_t item_size)
-{
-   // called only when Size() != 0
-
-   if (!slice.getDevice().hasSeparateMemorySpace())
-   {
-      return slice.ptr();
-   }
-   if (buffer)
-   {
-      slice.copyTo(buffer);
-   }
-   return buffer;
-}
-
-void Array::DoFill(const void *value_ptr, std::size_t item_size)
-{
-   // called only when Size() != 0
-
-   switch (item_size)
-   {
-      case sizeof(int8_t):
-         OccaFill((const int8_t *)value_ptr);
-         break;
-      case sizeof(int16_t):
-         OccaFill((const int16_t *)value_ptr);
-         break;
-      case sizeof(int32_t):
-         OccaFill((const int32_t *)value_ptr);
-         break;
-      // case sizeof(int64_t):
-      //    OccaFill((const int64_t *)value_ptr);
-      //    break;
-      case sizeof(double):
-         OccaFill((const double *)value_ptr);
-         break;
-      // case sizeof(::occa::double2):
-      //    OccaFill((const ::occa::double2 *)value_ptr);
-      //    break;
-      default:
-         MFEM_ABORT("item_size = " << item_size << " is not supported");
-   }
-}
-
-void Array::DoPushData(const void *src_buffer, std::size_t item_size)
-{
-   // called only when Size() != 0
-
-   if (slice.getDevice().hasSeparateMemorySpace() || slice.ptr() != src_buffer)
-   {
-      slice.copyFrom(src_buffer);
-   }
-}
-
-void Array::DoAssign(const PArray &src, std::size_t item_size)
-{
-   // called only when Size() != 0
-
-   // Note: static_cast can not be used here since PArray is a virtual base
-   //       class.
-   const Array *source = dynamic_cast<const Array *>(&src);
-   MFEM_ASSERT(source != NULL, "invalid source Array type");
-   MFEM_ASSERT(Size() == source->Size(), "");
-   slice.copyFrom(source->slice);
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/array.hpp

@@ -1,133 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_ARRAY_HPP
-#define MFEM_BACKENDS_OCCA_ARRAY_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include <occa.hpp>
-#include "layout.hpp"
-#include "../base/array.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class Array : public virtual PArray
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // DLayout layout;
-
-   // Always true: Size()*item_size == slice.size() <= data.size()
-   mutable ::occa::memory data, slice;
-
-   //
-   // Virtual interface
-   //
-
-   virtual void *DoGetData() const { return GetBuffer(); }
-
-   virtual PArray *DoClone(bool copy_data, void **buffer,
-                           std::size_t item_size) const;
-
-   virtual int DoResize(PLayout &new_layout, void **buffer,
-                        std::size_t item_size);
-
-   virtual void *DoPullData(void *buffer, std::size_t item_size);
-
-   virtual void DoFill(const void *value_ptr, std::size_t item_size);
-
-   virtual void DoPushData(const void *src_buffer, std::size_t item_size);
-
-   virtual void DoAssign(const PArray &src, std::size_t item_size);
-
-   //
-   // Auxiliary methods
-   //
-
-   inline void *GetBuffer() const;
-
-   inline int ResizeData(const Layout *lt, std::size_t item_size);
-
-   template <typename T>
-   inline void OccaFill(const T *val_ptr)
-   { ::occa::linalg::operator_eq<T>(slice, *val_ptr); }
-
-public:
-   Array(Layout &lt, std::size_t item_size)
-      : PArray(lt),
-        data(lt.Alloc(lt.Size()*item_size)),
-        slice(data)
-   { }
-
-   virtual ~Array() { }
-
-   inline void MakeRef(Array &master);
-
-   Layout &OccaLayout() const
-   { return *static_cast<Layout *>(layout.Get()); }
-
-   ::occa::memory &OccaMem() { return slice; }
-   const ::occa::memory &OccaMem() const { return slice; }
-};
-
-
-//
-// Inline methods
-//
-
-inline void *Array::GetBuffer() const
-{
-   if (!slice.getDevice().hasSeparateMemorySpace())
-   {
-      return slice.ptr();
-   }
-   return NULL;
-}
-
-inline int Array::ResizeData(const Layout *lt, std::size_t item_size)
-{
-   const std::size_t new_bytes = lt->Size()*item_size;
-   if (data.size() < new_bytes ||
-       data.getDHandle() != lt->OccaEngine().GetDevice().getDHandle())
-   {
-      data = lt->Alloc(new_bytes);
-      slice = data;
-      // If memory allocation fails - an exception is thrown.
-   }
-   else if (slice.size() != new_bytes)
-   {
-      slice = data.slice(0, new_bytes);
-   }
-   return 0;
-}
-
-inline void Array::MakeRef(Array &master)
-{
-   layout = master.layout;
-   data = master.data;
-   slice = master.slice;
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_ARRAY_HPP

backends/occa/backend.cpp

@@ -1,47 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "backend.hpp"
-#include "engine.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-bool Backend::Supports(const std::string &engine_spec) const
-{
-   // TODO: check if 'engine_spec' is valid OCCA string.
-   return true;
-}
-
-mfem::Engine *Create(const std::string &engine_spec)
-{
-   return new Engine(engine_spec);
-}
-
-#ifdef MFEM_USE_MPI
-mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec)
-{
-   return new Engine(comm, engine_spec);
-}
-#endif
-
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/backend.hpp

@@ -1,49 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_BACKEND_HPP
-#define MFEM_BACKENDS_OCCA_BACKEND_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-// Only the Backend and Engine classes should be exposed through "backend.hpp"
-#include "../base/backend.hpp"
-#include "engine.hpp"
-#include <occa.hpp>
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class Backend : public mfem::Backend
-{
-public:
-   virtual ~Backend();
-
-   virtual bool Supports(const std::string &engine_spec) const;
-
-   virtual mfem::Engine *Create(const std::string &engine_spec);
-
-#ifdef MFEM_USE_MPI
-   virtual mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec);
-#endif
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_BACKEND_HPP

backends/occa/bilinearform.cpp

@@ -1,514 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "backend.hpp"
-#include "bilininteg.hpp"
-#include "../../fem/bilinearform.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-OccaBilinearForm::OccaBilinearForm(FiniteElementSpace *ofespace_) :
-   Operator(ofespace_->OccaVLayout()),
-   localX((ofespace_->OccaEVLayout().DontDelete(), ofespace_->OccaEVLayout())),
-   localY((ofespace_->OccaEVLayout().DontDelete(), ofespace_->OccaEVLayout()))
-{
-   Init(ofespace_->OccaEngine(), ofespace_, ofespace_);
-}
-
-OccaBilinearForm::OccaBilinearForm(FiniteElementSpace *otrialFESpace_,
-                                   FiniteElementSpace *otestFESpace_) :
-   Operator(otrialFESpace_->OccaVLayout(),
-            otestFESpace_->OccaVLayout()),
-   localX((otrialFESpace_->OccaEVLayout().DontDelete(), otrialFESpace_->OccaEVLayout())),
-   localY((otestFESpace_->OccaEVLayout().DontDelete(), otestFESpace_->OccaEVLayout()))
-{
-   Init(otrialFESpace_->OccaEngine(), otrialFESpace_, otestFESpace_);
-}
-
-void OccaBilinearForm::Init(const Engine &e,
-                            FiniteElementSpace *otrialFESpace_,
-                            FiniteElementSpace *otestFESpace_)
-{
-   engine.Reset(&e);
-
-   otrialFESpace = otrialFESpace_;
-   trialFESpace  = otrialFESpace_->GetFESpace();
-
-   otestFESpace = otestFESpace_;
-   testFESpace  = otestFESpace_->GetFESpace();
-
-   mesh = trialFESpace->GetMesh();
-
-   const int elements = GetNE();
-
-   const int trialVDim = trialFESpace->GetVDim();
-
-   const int trialLocalDofs = otrialFESpace->GetLocalDofs();
-   const int testLocalDofs  = otestFESpace->GetLocalDofs();
-
-   // First-touch policy when running with OpenMP
-   if (GetDevice().mode() == "OpenMP")
-   {
-      const std::string &okl_path = OccaEngine().GetOklPath();
-      const std::string &okl_defines = OccaEngine().GetOklDefines();
-      ::occa::kernel initLocalKernel =
-         GetDevice().buildKernel(okl_path + "utils.okl",
-                                 "InitLocalVector",
-                                 okl_defines);
-
-      const std::size_t sd = sizeof(double);
-      const uint64_t trialEntries = sd * (elements * trialLocalDofs);
-      const uint64_t testEntries  = sd * (elements * testLocalDofs);
-      for (int v = 0; v < trialVDim; ++v)
-      {
-         const uint64_t trialOffset = v * trialEntries;
-         const uint64_t testOffset  = v * testEntries;
-
-         initLocalKernel(elements, trialLocalDofs,
-                         localX.OccaMem().slice(trialOffset, trialEntries));
-         initLocalKernel(elements, testLocalDofs,
-                         localY.OccaMem().slice(testOffset, testEntries));
-      }
-   }
-}
-
-int OccaBilinearForm::BaseGeom() const
-{
-   return mesh->GetElementBaseGeometry();
-}
-
-int OccaBilinearForm::GetDim() const
-{
-   return mesh->Dimension();
-}
-
-int64_t OccaBilinearForm::GetNE() const
-{
-   return mesh->GetNE();
-}
-
-Mesh& OccaBilinearForm::GetMesh() const
-{
-   return *mesh;
-}
-
-FiniteElementSpace& OccaBilinearForm::GetTrialOccaFESpace() const
-{
-   return *otrialFESpace;
-}
-
-FiniteElementSpace& OccaBilinearForm::GetTestOccaFESpace() const
-{
-   return *otestFESpace;
-}
-
-mfem::FiniteElementSpace& OccaBilinearForm::GetTrialFESpace() const
-{
-   return *trialFESpace;
-}
-
-mfem::FiniteElementSpace& OccaBilinearForm::GetTestFESpace() const
-{
-   return *testFESpace;
-}
-
-int64_t OccaBilinearForm::GetTrialNDofs() const
-{
-   return trialFESpace->GetNDofs();
-}
-
-int64_t OccaBilinearForm::GetTestNDofs() const
-{
-   return testFESpace->GetNDofs();
-}
-
-int64_t OccaBilinearForm::GetTrialVDim() const
-{
-   return trialFESpace->GetVDim();
-}
-
-int64_t OccaBilinearForm::GetTestVDim() const
-{
-   return testFESpace->GetVDim();
-}
-
-const FiniteElement& OccaBilinearForm::GetTrialFE(const int i) const
-{
-   return *(trialFESpace->GetFE(i));
-}
-
-const FiniteElement& OccaBilinearForm::GetTestFE(const int i) const
-{
-   return *(testFESpace->GetFE(i));
-}
-
-// Adds new Domain Integrator.
-void OccaBilinearForm::AddDomainIntegrator(OccaIntegrator *integrator,
-                                           const ::occa::properties &props)
-{
-   AddIntegrator(integrator, props, DomainIntegrator);
-}
-
-// Adds new Boundary Integrator.
-void OccaBilinearForm::AddBoundaryIntegrator(OccaIntegrator *integrator,
-                                             const ::occa::properties &props)
-{
-   AddIntegrator(integrator, props, BoundaryIntegrator);
-}
-
-// Adds new interior Face Integrator.
-void OccaBilinearForm::AddInteriorFaceIntegrator(OccaIntegrator *integrator,
-                                                 const ::occa::properties &props)
-{
-   AddIntegrator(integrator, props, InteriorFaceIntegrator);
-}
-
-// Adds new boundary Face Integrator.
-void OccaBilinearForm::AddBoundaryFaceIntegrator(OccaIntegrator *integrator,
-                                                 const ::occa::properties &props)
-{
-   AddIntegrator(integrator, props, BoundaryFaceIntegrator);
-}
-
-// Adds Integrator based on OccaIntegratorType
-void OccaBilinearForm::AddIntegrator(OccaIntegrator *integrator,
-                                     const ::occa::properties &props,
-                                     const OccaIntegratorType itype)
-{
-   if (integrator == NULL)
-   {
-      std::stringstream error_ss;
-      error_ss << "OccaBilinearForm::";
-      switch (itype)
-      {
-         case DomainIntegrator      : error_ss << "AddDomainIntegrator";       break;
-         case BoundaryIntegrator    : error_ss << "AddBoundaryIntegrator";     break;
-         case InteriorFaceIntegrator: error_ss << "AddInteriorFaceIntegrator"; break;
-         case BoundaryFaceIntegrator: error_ss << "AddBoundaryFaceIntegrator"; break;
-      }
-      error_ss << " (...):\n"
-               << "  Integrator is NULL";
-      const std::string error = error_ss.str();
-      mfem_error(error.c_str());
-   }
-   integrator->SetupIntegrator(*this, baseKernelProps + props, itype);
-   integrators.push_back(integrator);
-}
-
-const mfem::Operator* OccaBilinearForm::GetTrialProlongation() const
-{
-   return otrialFESpace->GetProlongationOperator();
-}
-
-const mfem::Operator* OccaBilinearForm::GetTestProlongation() const
-{
-   return otestFESpace->GetProlongationOperator();
-}
-
-const mfem::Operator* OccaBilinearForm::GetTrialRestriction() const
-{
-   return otrialFESpace->GetRestrictionOperator();
-}
-
-const mfem::Operator* OccaBilinearForm::GetTestRestriction() const
-{
-   return otestFESpace->GetRestrictionOperator();
-}
-
-void OccaBilinearForm::Assemble()
-{
-   // [MISSING] Find geometric information that is needed by intergrators
-   //             to share between integrators.
-   const int integratorCount = (int) integrators.size();
-   for (int i = 0; i < integratorCount; ++i)
-   {
-      integrators[i]->Assemble();
-   }
-}
-
-void OccaBilinearForm::FormLinearSystem(const mfem::Array<int> &constraintList,
-                                        mfem::Vector &x, mfem::Vector &b,
-                                        mfem::Operator *&Aout,
-                                        mfem::Vector &X, mfem::Vector &B,
-                                        int copy_interior)
-{
-   FormOperator(constraintList, Aout);
-   InitRHS(constraintList, x, b, Aout, X, B, copy_interior);
-}
-
-void OccaBilinearForm::FormOperator(const mfem::Array<int> &constraintList,
-                                    mfem::Operator *&Aout)
-{
-   const mfem::Operator *trialP = GetTrialProlongation();
-   const mfem::Operator *testP  = GetTestProlongation();
-   mfem::Operator *rap = this;
-
-   if (trialP)
-   {
-      rap = new RAPOperator(*testP, *this, *trialP);
-   }
-
-   Aout = new OccaConstrainedOperator(rap, constraintList,
-                                      rap != this);
-}
-
-void OccaBilinearForm::InitRHS(const mfem::Array<int> &constraintList,
-                               mfem::Vector &x, mfem::Vector &b,
-                               mfem::Operator *A,
-                               mfem::Vector &X, mfem::Vector &B,
-                               int copy_interior)
-{
-   const std::string okl_defines = OccaEngine().GetOklDefines();
-
-   // FIXME: move these kernels to the Backend?
-   static ::occa::kernelBuilder get_subvector_builder =
-      ::occa::linalg::customLinearMethod(
-         "vector_get_subvector",
-
-         "const int dof_i = v2[i];"
-         "v0[i] = dof_i >= 0 ? v1[dof_i] : -v1[-dof_i - 1];",
-
-         "defines: {"
-         "  VTYPE0: 'double',"
-         "  VTYPE1: 'double',"
-         "  VTYPE2: 'int',"
-         "  TILESIZE: 128,"
-         "}" + okl_defines);
-
-   static ::occa::kernelBuilder set_subvector_builder =
-      ::occa::linalg::customLinearMethod(
-         "vector_set_subvector",
-         "const int dof_i = v2[i];"
-         "if (dof_i >= 0) { v0[dof_i]      = v1[i]; }"
-         "else            { v0[-dof_i - 1] = -v1[i]; }",
-
-         "defines: {"
-         "  VTYPE0: 'double',"
-         "  VTYPE1: 'double',"
-         "  VTYPE2: 'int',"
-         "  TILESIZE: 128,"
-         "}" + okl_defines);
-
-   const mfem::Operator *P = GetTrialProlongation();
-   const mfem::Operator *R = GetTrialRestriction();
-
-   if (P)
-   {
-      // Variational restriction with P
-      B.Resize(P->InLayout());
-      P->MultTranspose(b, B);
-      X.Resize(R->OutLayout());
-      R->Mult(x, X);
-   }
-   else
-   {
-      // rap, X and B point to the same data as this, x and b
-      X.MakeRef(x);
-      B.MakeRef(b);
-   }
-
-   if (!copy_interior && constraintList.Size() > 0)
-   {
-      ::occa::kernel get_subvector_kernel =
-         get_subvector_builder.build(GetDevice());
-      ::occa::kernel set_subvector_kernel =
-         set_subvector_builder.build(GetDevice());
-
-      const Array &constrList = constraintList.Get_PArray()->As<Array>();
-      Vector subvec(constrList.OccaLayout());
-
-      get_subvector_kernel(constraintList.Size(),
-                           subvec.OccaMem(),
-                           X.Get_PVector()->As<Vector>().OccaMem(),
-                           constrList.OccaMem());
-
-      X.Fill(0.0);
-
-      set_subvector_kernel(constraintList.Size(),
-                           X.Get_PVector()->As<Vector>().OccaMem(),
-                           subvec.OccaMem(),
-                           constrList.OccaMem());
-   }
-
-   OccaConstrainedOperator *cA = dynamic_cast<OccaConstrainedOperator*>(A);
-   if (cA)
-   {
-      cA->EliminateRHS(X.Get_PVector()->As<Vector>(),
-                       B.Get_PVector()->As<Vector>());
-   }
-   else
-   {
-      mfem_error("OccaBilinearForm::InitRHS expects an OccaConstrainedOperator");
-   }
-}
-
-// Matrix vector multiplication.
-void OccaBilinearForm::Mult_(const Vector &x, Vector &y) const
-{
-   otrialFESpace->GlobalToLocal(x, localX);
-   localY.Fill<double>(0.0);
-
-   const int integratorCount = (int) integrators.size();
-   for (int i = 0; i < integratorCount; ++i)
-   {
-      integrators[i]->MultAdd(localX, localY);
-   }
-
-   otestFESpace->LocalToGlobal(localY, y);
-}
-
-// Matrix transpose vector multiplication.
-void OccaBilinearForm::MultTranspose_(const Vector &x, Vector &y) const
-{
-   otestFESpace->GlobalToLocal(x, localX);
-   localY.Fill<double>(0.0);
-
-   const int integratorCount = (int) integrators.size();
-   for (int i = 0; i < integratorCount; ++i)
-   {
-      integrators[i]->MultTransposeAdd(localX, localY);
-   }
-
-   otrialFESpace->LocalToGlobal(localY, y);
-}
-
-void OccaBilinearForm::OccaRecoverFEMSolution(const mfem::Vector &X,
-                                              const mfem::Vector &b,
-                                              mfem::Vector &x)
-{
-   const mfem::Operator *P = this->GetTrialProlongation();
-   if (P)
-   {
-      // Apply conforming prolongation
-      x.Resize(P->OutLayout());
-      P->Mult(X, x);
-   }
-   // Otherwise X and x point to the same data
-}
-
-// Frees memory bilinear form.
-OccaBilinearForm::~OccaBilinearForm()
-{
-   // Make sure all integrators free their data
-   IntegratorVector::iterator it = integrators.begin();
-   while (it != integrators.end())
-   {
-      delete *it;
-      ++it;
-   }
-}
-
-
-void BilinearForm::InitOccaBilinearForm()
-{
-   // Init 'obform' using 'bform'
-   MFEM_ASSERT(bform != NULL, "");
-   MFEM_ASSERT(obform == NULL, "");
-
-   FiniteElementSpace &ofes =
-      bform->FESpace()->Get_PFESpace()->As<FiniteElementSpace>();
-   obform = new OccaBilinearForm(&ofes);
-
-   // Transfer domain integrators
-   mfem::Array<mfem::BilinearFormIntegrator*> &dbfi = *bform->GetDBFI();
-   for (int i = 0; i < dbfi.Size(); i++)
-   {
-      std::string integ_name(dbfi[i]->Name());
-      Coefficient *scal_coeff = dbfi[i]->GetScalarCoefficient();
-      ConstantCoefficient *const_coeff =
-         dynamic_cast<ConstantCoefficient*>(scal_coeff);
-      // TODO: other types of coefficients ...
-      double val = const_coeff ? const_coeff->constant : 1.0;
-      OccaCoefficient ocoeff(obform->OccaEngine(), val);
-
-      OccaIntegrator *ointeg = NULL;
-
-      if (integ_name == "(undefined)")
-      {
-         MFEM_ABORT("BilinearFormIntegrator does not define Name()");
-      }
-      else if (integ_name == "diffusion")
-      {
-         ointeg = new OccaDiffusionIntegrator(ocoeff);
-      }
-      else
-      {
-         MFEM_ABORT("BilinearFormIntegrator [Name() = " << integ_name
-                    << "] is not supported");
-      }
-
-      const mfem::IntegrationRule *ir = dbfi[i]->GetIntRule();
-      if (ir) { ointeg->SetIntegrationRule(*ir); }
-
-      obform->AddDomainIntegrator(ointeg);
-   }
-
-   // TODO: other types of integrators ...
-}
-
-bool BilinearForm::Assemble()
-{
-   if (obform == NULL) { InitOccaBilinearForm(); }
-
-   obform->Assemble();
-
-   return true; // --> host assembly is not needed
-}
-
-void BilinearForm::FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
-                                    mfem::OperatorHandle &A)
-{
-   if (A.Type() == mfem::Operator::ANY_TYPE)
-   {
-      mfem::Operator *Aout = NULL;
-      obform->FormOperator(ess_tdof_list, Aout);
-      A.Reset(Aout);
-   }
-   else
-   {
-      MFEM_ABORT("Operator::Type is not supported, type = " << A.Type());
-   }
-}
-
-void BilinearForm::FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
-                                    mfem::Vector &x, mfem::Vector &b,
-                                    mfem::OperatorHandle &A,
-                                    mfem::Vector &X, mfem::Vector &B,
-                                    int copy_interior)
-{
-   FormSystemMatrix(ess_tdof_list, A);
-   obform->InitRHS(ess_tdof_list, x, b, A.Ptr(), X, B, copy_interior);
-}
-
-void BilinearForm::RecoverFEMSolution(const mfem::Vector &X,
-                                      const mfem::Vector &b,
-                                      mfem::Vector &x)
-{
-   obform->OccaRecoverFEMSolution(X, b, x);
-}
-
-BilinearForm::~BilinearForm()
-{
-   delete obform;
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/bilinearform.hpp

@@ -1,213 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP
-#define MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "fespace.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-enum OccaIntegratorType
-{
-   DomainIntegrator       = 0,
-   BoundaryIntegrator     = 1,
-   InteriorFaceIntegrator = 2,
-   BoundaryFaceIntegrator = 3
-};
-
-class OccaIntegrator;
-
-
-/** Class for bilinear form - "Matrix" with associated FE space and
-    BLFIntegrators. */
-class OccaBilinearForm : public Operator
-{
-   friend class OccaIntegrator;
-
-protected:
-   typedef std::vector<OccaIntegrator*> IntegratorVector;
-
-   SharedPtr<const Engine> engine;
-
-   // State information
-   mutable mfem::Mesh *mesh;
-
-   mutable FiniteElementSpace *otrialFESpace;
-   mutable mfem::FiniteElementSpace *trialFESpace;
-
-   mutable FiniteElementSpace *otestFESpace;
-   mutable mfem::FiniteElementSpace *testFESpace;
-
-   IntegratorVector integrators;
-
-   // Device data
-   ::occa::properties baseKernelProps;
-
-   // The input and output vectors are mapped to local nodes for efficient
-   // operations. In other words, they are E-vectors.
-   // The size is: (number of elements) * (nodes in element) * (vector dim)
-   mutable Vector localX, localY;
-
-public:
-   OccaBilinearForm(FiniteElementSpace *ofespace_);
-
-   OccaBilinearForm(FiniteElementSpace *otrialFESpace_,
-                    FiniteElementSpace *otestFESpace_);
-
-   void Init(const Engine &e,
-             FiniteElementSpace *otrialFESpace_,
-             FiniteElementSpace *otestFESpace_);
-
-   const Engine &OccaEngine() const { return *engine; }
-
-   ::occa::device GetDevice(int idx = 0) const
-   { return engine->GetDevice(idx); }
-
-   // Useful mesh Information
-   int BaseGeom() const;
-   int GetDim() const;
-   int64_t GetNE() const;
-
-   mfem::Mesh& GetMesh() const;
-
-   FiniteElementSpace& GetTrialOccaFESpace() const;
-   FiniteElementSpace& GetTestOccaFESpace() const;
-
-   mfem::FiniteElementSpace& GetTrialFESpace() const;
-   mfem::FiniteElementSpace& GetTestFESpace() const;
-
-   // Useful FE information
-   int64_t GetTrialNDofs() const;
-   int64_t GetTestNDofs() const;
-
-   int64_t GetTrialVDim() const;
-   int64_t GetTestVDim() const;
-
-   const mfem::FiniteElement& GetTrialFE(const int i) const;
-   const mfem::FiniteElement& GetTestFE(const int i) const;
-
-   // Adds new Domain Integrator.
-   void AddDomainIntegrator(OccaIntegrator *integrator,
-                            const ::occa::properties &props =
-                               ::occa::properties());
-
-   // Adds new Boundary Integrator.
-   void AddBoundaryIntegrator(OccaIntegrator *integrator,
-                              const ::occa::properties &props =
-                                 ::occa::properties());
-
-   // Adds new interior Face Integrator.
-   void AddInteriorFaceIntegrator(OccaIntegrator *integrator,
-                                  const ::occa::properties &props =
-                                     ::occa::properties());
-
-   // Adds new boundary Face Integrator.
-   void AddBoundaryFaceIntegrator(OccaIntegrator *integrator,
-                                  const ::occa::properties &props =
-                                     ::occa::properties());
-
-   // Adds Integrator based on OccaIntegratorType
-   void AddIntegrator(OccaIntegrator *integrator,
-                      const ::occa::properties &props,
-                      const OccaIntegratorType itype);
-
-   virtual const mfem::Operator *GetTrialProlongation() const;
-   virtual const mfem::Operator *GetTestProlongation() const;
-
-   virtual const mfem::Operator *GetTrialRestriction() const;
-   virtual const mfem::Operator *GetTestRestriction() const;
-
-   // Assembles the form i.e. sums over all domain/bdr integrators.
-   virtual void Assemble();
-
-   void FormLinearSystem(const mfem::Array<int> &constraintList,
-                         mfem::Vector &x, mfem::Vector &b,
-                         mfem::Operator *&Aout,
-                         mfem::Vector &X, mfem::Vector &B,
-                         int copy_interior = 0);
-
-   void FormOperator(const mfem::Array<int> &constraintList,
-                     mfem::Operator *&Aout);
-
-   void InitRHS(const mfem::Array<int> &constraintList,
-                mfem::Vector &x, mfem::Vector &b,
-                mfem::Operator *Aout,
-                mfem::Vector &X, mfem::Vector &B,
-                int copy_interior = 0);
-
-   // overrides
-   virtual void Mult_(const Vector &x, Vector &y) const;
-   virtual void MultTranspose_(const Vector &x, Vector &y) const;
-
-   void OccaRecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
-                               mfem::Vector &x);
-
-   // Destroys bilinear form.
-   ~OccaBilinearForm();
-};
-
-
-/// TODO: doxygen
-class BilinearForm : public mfem::PBilinearForm
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // SharedPtr<const mfem::Engine> engine;
-   // mfem::BilinearForm *bform;
-   OccaBilinearForm *obform;
-
-   // Called from Assemble() if obform is NULL to initialize obform.
-   void InitOccaBilinearForm();
-
-public:
-   /// TODO: doxygen
-   BilinearForm(const Engine &e, mfem::BilinearForm &bf)
-      : mfem::PBilinearForm(e, bf), obform(NULL) { }
-
-   /// Virtual destructor
-   virtual ~BilinearForm();
-
-   /// Assemble the PBilinearForm.
-   /** This method is called from the method mfem::BilinearForm::Assemble() of
-       the associated mfem::BilinearForm, #bform.
-       @returns True, if the host assembly should NOT be performed. */
-   virtual bool Assemble();
-
-   virtual void FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
-                                 mfem::OperatorHandle &A);
-
-   virtual void FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
-                                 mfem::Vector &x, mfem::Vector &b,
-                                 mfem::OperatorHandle &A,
-                                 mfem::Vector &X, mfem::Vector &B,
-                                 int copy_interior);
-
-   virtual void RecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
-                                   mfem::Vector &x);
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_BILINEAR_FORM_HPP

backends/occa/bilininteg.cpp

@@ -1,956 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "bilininteg.hpp"
-#include "../../fem/fem.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-std::map<std::string, OccaDofQuadMaps> OccaDofQuadMaps::AllDofQuadMaps;
-
-OccaGeometry OccaGeometry::Get(::occa::device device,
-                               FiniteElementSpace &ofespace,
-                               const mfem::IntegrationRule &ir,
-                               const int flags)
-{
-   OccaGeometry geom;
-
-   mfem::Mesh &mesh = *(ofespace.GetMesh());
-   if (!mesh.GetNodes())
-   {
-      mesh.SetCurvature(1, false, -1, mfem::Ordering::byVDIM);
-   }
-   mfem::GridFunction &nodes = *(mesh.GetNodes());
-   const mfem::FiniteElementSpace &fespace = *(nodes.FESpace());
-   const mfem::FiniteElement &fe = *(fespace.GetFE(0));
-
-   const int dims     = fe.GetDim();
-   const int elements = fespace.GetNE();
-   const int numDofs  = fe.GetDof();
-   const int numQuad  = ir.GetNPoints();
-
-   MFEM_ASSERT(dims == mesh.SpaceDimension(), "");
-
-   geom.meshNodes.allocate(device,
-                           dims, numDofs, elements);
-
-   const mfem::Table &e2dTable = fespace.GetElementToDofTable();
-   const int *elementMap = e2dTable.GetJ();
-   nodes.Pull();
-   for (int e = 0; e < elements; ++e)
-   {
-      for (int dof = 0; dof < numDofs; ++dof)
-      {
-         const int gid = elementMap[dof + numDofs*e];
-         for (int dim = 0; dim < dims; ++dim)
-         {
-            geom.meshNodes(dim, dof, e) = nodes[fespace.DofToVDof(gid,dim)];
-         }
-      }
-   }
-   geom.meshNodes.keepInDevice();
-
-   if (flags & Jacobian)
-   {
-      geom.J.allocate(device,
-                      dims, dims, numQuad, elements);
-   }
-   else
-   {
-      geom.J.allocate(device, 1);
-   }
-   if (flags & JacobianInv)
-   {
-      geom.invJ.allocate(device,
-                         dims, dims, numQuad, elements);
-   }
-   else
-   {
-      geom.invJ.allocate(device, 1);
-   }
-   if (flags & JacobianDet)
-   {
-      geom.detJ.allocate(device,
-                         numQuad, elements);
-   }
-   else
-   {
-      geom.detJ.allocate(device, 1);
-   }
-
-   geom.J.stopManaging();
-   geom.invJ.stopManaging();
-   geom.detJ.stopManaging();
-
-   OccaDofQuadMaps &maps = OccaDofQuadMaps::GetSimplexMaps(device, fe, ir);
-
-   ::occa::properties props;
-   props["defines/NUM_DOFS"] = numDofs;
-   props["defines/NUM_QUAD"] = numQuad;
-   props["defines/STORE_JACOBIAN"]     = (flags & Jacobian);
-   props["defines/STORE_JACOBIAN_INV"] = (flags & JacobianInv);
-   props["defines/STORE_JACOBIAN_DET"] = (flags & JacobianDet);
-
-   const std::string &okl_path = ofespace.OccaEngine().GetOklPath();
-   const std::string &okl_defines = ofespace.OccaEngine().GetOklDefines();
-   ::occa::kernel init = device.buildKernel(okl_path + "geometry.okl",
-                                            stringWithDim("InitGeometryInfo",
-                                                          fe.GetDim()),
-                                            props + okl_defines);
-   init(elements,
-        maps.dofToQuadD,
-        geom.meshNodes,
-        geom.J, geom.invJ, geom.detJ);
-
-   return geom;
-}
-
-OccaDofQuadMaps::OccaDofQuadMaps() :
-   hash() {}
-
-OccaDofQuadMaps::OccaDofQuadMaps(const OccaDofQuadMaps &maps)
-{
-   *this = maps;
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::operator = (const OccaDofQuadMaps &maps)
-{
-   hash = maps.hash;
-   dofToQuad   = maps.dofToQuad;
-   dofToQuadD  = maps.dofToQuadD;
-   quadToDof   = maps.quadToDof;
-   quadToDofD  = maps.quadToDofD;
-   quadWeights = maps.quadWeights;
-   return *this;
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
-                                      const FiniteElementSpace &fespace,
-                                      const mfem::IntegrationRule &ir,
-                                      const bool transpose)
-{
-   return Get(device,
-              *fespace.GetFE(0),
-              *fespace.GetFE(0),
-              ir,
-              transpose);
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
-                                      const mfem::FiniteElement &fe,
-                                      const mfem::IntegrationRule &ir,
-                                      const bool transpose)
-{
-   return Get(device, fe, fe, ir, transpose);
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
-                                      const FiniteElementSpace &trialFESpace,
-                                      const FiniteElementSpace &testFESpace,
-                                      const mfem::IntegrationRule &ir,
-                                      const bool transpose)
-{
-   return Get(device,
-              *trialFESpace.GetFE(0),
-              *testFESpace.GetFE(0),
-              ir,
-              transpose);
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::Get(::occa::device device,
-                                      const mfem::FiniteElement &trialFE,
-                                      const mfem::FiniteElement &testFE,
-                                      const mfem::IntegrationRule &ir,
-                                      const bool transpose)
-{
-   return (dynamic_cast<const mfem::TensorBasisElement*>(&trialFE)
-           ? GetTensorMaps(device, trialFE, testFE, ir, transpose)
-           : GetSimplexMaps(device, trialFE, testFE, ir, transpose));
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::GetTensorMaps(::occa::device device,
-                                                const mfem::FiniteElement &fe,
-                                                const mfem::IntegrationRule &ir,
-                                                const bool transpose)
-{
-   return GetTensorMaps(device,
-                        fe, fe,
-                        ir, transpose);
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::GetTensorMaps(::occa::device device,
-                                                const mfem::FiniteElement &trialFE,
-                                                const mfem::FiniteElement &testFE,
-                                                const mfem::IntegrationRule &ir,
-                                                const bool transpose)
-{
-   const mfem::TensorBasisElement &trialTFE =
-      dynamic_cast<const mfem::TensorBasisElement&>(trialFE);
-   const mfem::TensorBasisElement &testTFE =
-      dynamic_cast<const mfem::TensorBasisElement&>(testFE);
-
-   std::stringstream ss;
-   ss << ::occa::hash(device)
-      << "Tensor"
-      << "O1:"  << trialFE.GetOrder()
-      << "O2:"  << testFE.GetOrder()
-      << "BT1:" << trialTFE.GetBasisType()
-      << "BT2:" << testTFE.GetBasisType()
-      << "Q:"   << ir.GetNPoints();
-   std::string hash = ss.str();
-
-   // If we've already made the dof-quad maps, reuse them
-   OccaDofQuadMaps &maps = AllDofQuadMaps[hash];
-   if (!maps.hash.size())
-   {
-      // Create the dof-quad maps
-      maps.hash = hash;
-
-      OccaDofQuadMaps trialMaps = GetD2QTensorMaps(device, trialFE, ir);
-      OccaDofQuadMaps testMaps  = GetD2QTensorMaps(device, testFE , ir, true);
-
-      maps.dofToQuad   = trialMaps.dofToQuad;
-      maps.dofToQuadD  = trialMaps.dofToQuadD;
-      maps.quadToDof   = testMaps.dofToQuad;
-      maps.quadToDofD  = testMaps.dofToQuadD;
-      maps.quadWeights = testMaps.quadWeights;
-   }
-   return maps;
-}
-
-OccaDofQuadMaps OccaDofQuadMaps::GetD2QTensorMaps(::occa::device device,
-                                                  const mfem::FiniteElement &fe,
-                                                  const mfem::IntegrationRule &ir,
-                                                  const bool transpose)
-{
-   const mfem::TensorBasisElement &tfe =
-      dynamic_cast<const mfem::TensorBasisElement&>(fe);
-
-   const mfem::Poly_1D::Basis &basis = tfe.GetBasis1D();
-   const int order = fe.GetOrder();
-   // [MISSING] Get 1D dofs
-   const int dofs = order + 1;
-   const int dims = fe.GetDim();
-
-   // Create the dof -> quadrature point map
-   const mfem::IntegrationRule &ir1D =
-      mfem::IntRules.Get(mfem::Geometry::SEGMENT, ir.GetOrder());
-   const int quadPoints = ir1D.GetNPoints();
-   const int quadPoints2D = quadPoints*quadPoints;
-   const int quadPoints3D = quadPoints2D*quadPoints;
-   const int quadPointsND = ((dims == 1) ? quadPoints :
-                             ((dims == 2) ? quadPoints2D : quadPoints3D));
-
-   OccaDofQuadMaps maps;
-   // Initialize the dof -> quad mapping
-   maps.dofToQuad.allocate(device,
-                           quadPoints, dofs);
-   maps.dofToQuadD.allocate(device,
-                            quadPoints, dofs);
-
-   double *quadWeights1DData = NULL;
-
-   if (transpose)
-   {
-      maps.dofToQuad.reindex(1,0);
-      maps.dofToQuadD.reindex(1,0);
-      // Initialize quad weights only for transpose
-      maps.quadWeights.allocate(device,
-                                quadPointsND);
-      quadWeights1DData = new double[quadPoints];
-   }
-
-   mfem::Vector d2q(dofs);
-   mfem::Vector d2qD(dofs);
-   for (int q = 0; q < quadPoints; ++q)
-   {
-      const mfem::IntegrationPoint &ip = ir1D.IntPoint(q);
-      basis.Eval(ip.x, d2q, d2qD);
-      if (transpose)
-      {
-         quadWeights1DData[q] = ip.weight;
-      }
-      for (int d = 0; d < dofs; ++d)
-      {
-         maps.dofToQuad(q, d)  = d2q[d];
-         maps.dofToQuadD(q, d) = d2qD[d];
-      }
-   }
-
-   maps.dofToQuad.keepInDevice();
-   maps.dofToQuadD.keepInDevice();
-
-   if (transpose)
-   {
-      for (int q = 0; q < quadPointsND; ++q)
-      {
-         const int qx = q % quadPoints;
-         const int qz = q / quadPoints2D;
-         const int qy = (q - qz*quadPoints2D) / quadPoints;
-         double w = quadWeights1DData[qx];
-         if (dims > 1)
-         {
-            w *= quadWeights1DData[qy];
-         }
-         if (dims > 2)
-         {
-            w *= quadWeights1DData[qz];
-         }
-         maps.quadWeights[q] = w;
-      }
-      maps.quadWeights.keepInDevice();
-      delete [] quadWeights1DData;
-   }
-
-   return maps;
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::GetSimplexMaps(::occa::device device,
-                                                 const mfem::FiniteElement &fe,
-                                                 const mfem::IntegrationRule &ir,
-                                                 const bool transpose)
-{
-   return GetSimplexMaps(device,
-                         fe, fe,
-                         ir, transpose);
-}
-
-OccaDofQuadMaps& OccaDofQuadMaps::GetSimplexMaps(::occa::device device,
-                                                 const mfem::FiniteElement &trialFE,
-                                                 const mfem::FiniteElement &testFE,
-                                                 const mfem::IntegrationRule &ir,
-                                                 const bool transpose)
-{
-   std::stringstream ss;
-   ss << ::occa::hash(device)
-      << "Simplex"
-      << "O1:" << trialFE.GetOrder()
-      << "O2:" << testFE.GetOrder()
-      << "Q:"  << ir.GetNPoints();
-   std::string hash = ss.str();
-
-   // If we've already made the dof-quad maps, reuse them
-   OccaDofQuadMaps &maps = AllDofQuadMaps[hash];
-   if (!maps.hash.size())
-   {
-      // Create the dof-quad maps
-      maps.hash = hash;
-
-      OccaDofQuadMaps trialMaps = GetD2QSimplexMaps(device, trialFE, ir);
-      OccaDofQuadMaps testMaps  = GetD2QSimplexMaps(device, testFE , ir, true);
-
-      maps.dofToQuad   = trialMaps.dofToQuad;
-      maps.dofToQuadD  = trialMaps.dofToQuadD;
-      maps.quadToDof   = testMaps.dofToQuad;
-      maps.quadToDofD  = testMaps.dofToQuadD;
-      maps.quadWeights = testMaps.quadWeights;
-   }
-   return maps;
-}
-
-OccaDofQuadMaps OccaDofQuadMaps::GetD2QSimplexMaps(::occa::device device,
-                                                   const mfem::FiniteElement &fe,
-                                                   const mfem::IntegrationRule &ir,
-                                                   const bool transpose)
-{
-   const int dims = fe.GetDim();
-   const int numDofs = fe.GetDof();
-   const int numQuad = ir.GetNPoints();
-
-   OccaDofQuadMaps maps;
-   // Initialize the dof -> quad mapping
-   maps.dofToQuad.allocate(device,
-                           numQuad, numDofs);
-   maps.dofToQuadD.allocate(device,
-                            dims, numQuad, numDofs);
-
-   if (transpose)
-   {
-      maps.dofToQuad.reindex(1,0);
-      maps.dofToQuadD.reindex(1,0);
-      // Initialize quad weights only for transpose
-      maps.quadWeights.allocate(device,
-                                numQuad);
-   }
-
-   mfem::Vector d2q(numDofs);
-   mfem::DenseMatrix d2qD(numDofs, dims);
-   for (int q = 0; q < numQuad; ++q)
-   {
-      const mfem::IntegrationPoint &ip = ir.IntPoint(q);
-      if (transpose)
-      {
-         maps.quadWeights[q] = ip.weight;
-      }
-      fe.CalcShape(ip, d2q);
-      fe.CalcDShape(ip, d2qD);
-      for (int d = 0; d < numDofs; ++d)
-      {
-         const double w = d2q[d];
-         maps.dofToQuad(q, d) = w;
-         for (int dim = 0; dim < dims; ++dim)
-         {
-            const double wD = d2qD(d, dim);
-            maps.dofToQuadD(dim, q, d) = wD;
-         }
-      }
-   }
-
-   maps.dofToQuad.keepInDevice();
-   maps.dofToQuadD.keepInDevice();
-   if (transpose)
-   {
-      maps.quadWeights.keepInDevice();
-   }
-
-   return maps;
-}
-
-//---[ Integrator Defines ]-----------
-std::string stringWithDim(const std::string &s, const int dim)
-{
-   std::string ret = s;
-   ret += ('0' + (char) dim);
-   ret += 'D';
-   return ret;
-}
-
-int closestWarpBatchTo(const int value)
-{
-   return ((value + 31) / 32) * 32;
-}
-
-int closestMultipleWarpBatch(const int multiple, const int maxSize)
-{
-   if (multiple > maxSize)
-   {
-      return maxSize;
-   }
-   int batch = (32 / multiple);
-   int minDiff = 32 - (multiple * batch);
-   for (int i = 64; i <= maxSize; i += 32)
-   {
-      const int newDiff = i - (multiple * (i / multiple));
-      if (newDiff < minDiff)
-      {
-         batch = (i / multiple);
-         minDiff = newDiff;
-      }
-   }
-   return batch;
-}
-
-void SetProperties(FiniteElementSpace &fespace,
-                   const mfem::IntegrationRule &ir,
-                   ::occa::properties &props)
-{
-   SetProperties(fespace, fespace, ir, props);
-}
-
-void SetProperties(FiniteElementSpace &trialFESpace,
-                   FiniteElementSpace &testFESpace,
-                   const mfem::IntegrationRule &ir,
-                   ::occa::properties &props)
-{
-   props["defines/TRIAL_VDIM"] = trialFESpace.GetVDim();
-   props["defines/TEST_VDIM"]  = testFESpace.GetVDim();
-   props["defines/NUM_DIM"]    = trialFESpace.GetDim();
-
-   if (trialFESpace.hasTensorBasis())
-   {
-      SetTensorProperties(trialFESpace, testFESpace, ir, props);
-   }
-   else
-   {
-      SetSimplexProperties(trialFESpace, testFESpace, ir, props);
-   }
-}
-
-void SetTensorProperties(FiniteElementSpace &fespace,
-                         const mfem::IntegrationRule &ir,
-                         ::occa::properties &props)
-{
-   SetTensorProperties(fespace, fespace, ir, props);
-}
-
-void SetTensorProperties(FiniteElementSpace &trialFESpace,
-                         FiniteElementSpace &testFESpace,
-                         const mfem::IntegrationRule &ir,
-                         ::occa::properties &props)
-{
-   const mfem::FiniteElement &trialFE = *(trialFESpace.GetFE(0));
-   const mfem::FiniteElement &testFE = *(testFESpace.GetFE(0));
-
-   const mfem::IntegrationRule &ir1D =
-      mfem::IntRules.Get(mfem::Geometry::SEGMENT, ir.GetOrder());
-
-   const int trialDofs = trialFE.GetDof();
-   const int testDofs  = testFE.GetDof();
-   const int numQuad  = ir.GetNPoints();
-
-   const int trialDofs1D = trialFE.GetOrder() + 1;
-   const int testDofs1D  = testFE.GetOrder() + 1;
-   const int quad1D  = ir1D.GetNPoints();
-   int trialDofsND = trialDofs1D;
-   int testDofsND  = testDofs1D;
-   int quadND  = quad1D;
-
-   const bool trialByVDIM = (trialFESpace.GetOrdering() == mfem::Ordering::byVDIM);
-   const bool testByVDIM  = (testFESpace.GetOrdering()  == mfem::Ordering::byVDIM);
-
-   props["defines/ORDERING_BY_NODES"] = 0;
-   props["defines/ORDERING_BY_VDIM"]  = 1;
-   props["defines/VDIM_ORDERING"]  = (int) trialByVDIM;
-   props["defines/TRIAL_ORDERING"] = (int) trialByVDIM;
-   props["defines/TEST_ORDERING"]  = (int) testByVDIM;
-
-   props["defines/USING_TENSOR_OPS"] = 1;
-   props["defines/NUM_DOFS"]  = trialDofs;
-   props["defines/NUM_QUAD"]  = numQuad;
-
-   props["defines/TRIAL_DOFS"] = trialDofs;
-   props["defines/TEST_DOFS"]  = testDofs;
-
-   for (int d = 1; d <= 3; ++d)
-   {
-      if (d > 1)
-      {
-         trialDofsND *= trialDofs1D;
-         testDofsND  *= testDofs1D;
-         quadND *= quad1D;
-      }
-      props["defines"][stringWithDim("NUM_DOFS_", d)] = trialDofsND;
-      props["defines"][stringWithDim("NUM_QUAD_", d)] = quadND;
-
-      props["defines"][stringWithDim("TRIAL_DOFS_", d)] = trialDofsND;
-      props["defines"][stringWithDim("TEST_DOFS_" , d)] = testDofsND;
-   }
-
-   // 1D Defines
-   const int m1InnerBatch = 32 * ((quad1D + 31) / 32);
-   props["defines/A1_ELEMENT_BATCH"]       = closestMultipleWarpBatch(quad1D, 512);
-   props["defines/M1_OUTER_ELEMENT_BATCH"] = closestMultipleWarpBatch(m1InnerBatch,
-                                                                      512);
-   props["defines/M1_INNER_ELEMENT_BATCH"] = m1InnerBatch;
-
-   // 2D Defines
-   props["defines/A2_ELEMENT_BATCH"] = 1;
-   props["defines/A2_QUAD_BATCH"]    = 1;
-   props["defines/M2_ELEMENT_BATCH"] = 32;
-
-   // 3D Defines
-   const int a3QuadBatch = closestMultipleWarpBatch(quadND, 512);
-   props["defines/A3_ELEMENT_BATCH"] = closestMultipleWarpBatch(a3QuadBatch, 512);
-   props["defines/A3_QUAD_BATCH"]    = a3QuadBatch;
-}
-
-void SetSimplexProperties(FiniteElementSpace &fespace,
-                          const mfem::IntegrationRule &ir,
-                          ::occa::properties &props)
-{
-   SetSimplexProperties(fespace, fespace, ir, props);
-}
-
-void SetSimplexProperties(FiniteElementSpace &trialFESpace,
-                          FiniteElementSpace &testFESpace,
-                          const mfem::IntegrationRule &ir,
-                          ::occa::properties &props)
-{
-   const mfem::FiniteElement &trialFE = *(trialFESpace.GetFE(0));
-   const mfem::FiniteElement &testFE = *(testFESpace.GetFE(0));
-
-   const int trialDofs = trialFE.GetDof();
-   const int testDofs  = testFE.GetDof();
-   const int numQuad = ir.GetNPoints();
-   const int maxDQ   = std::max(std::max(trialDofs, testDofs), numQuad);
-
-   const bool trialByVDIM = (trialFESpace.GetOrdering() == mfem::Ordering::byVDIM);
-   const bool testByVDIM  = (testFESpace.GetOrdering()  == mfem::Ordering::byVDIM);
-
-   props["defines/ORDERING_BY_NODES"] = 0;
-   props["defines/ORDERING_BY_VDIM"]  = 1;
-   props["defines/VDIM_ORDERING"]  = (int) trialByVDIM;
-   props["defines/TRIAL_ORDERING"] = (int) trialByVDIM;
-   props["defines/TEST_ORDERING"]  = (int) testByVDIM;
-
-   props["defines/USING_TENSOR_OPS"] = 0;
-   props["defines/NUM_DOFS"]  = trialDofs;
-   props["defines/NUM_QUAD"]  = numQuad;
-
-   props["defines/TRIAL_DOFS"] = trialDofs;
-   props["defines/TEST_DOFS"]  = testDofs;
-
-   // 2D Defines
-   const int quadBatch = closestWarpBatchTo(numQuad);
-   props["defines/A2_ELEMENT_BATCH"] = closestMultipleWarpBatch(quadBatch, 2048);
-   props["defines/A2_QUAD_BATCH"]    = quadBatch;
-   props["defines/M2_INNER_BATCH"]   = closestWarpBatchTo(maxDQ);
-
-   // 3D Defines
-   props["defines/A3_ELEMENT_BATCH"] = closestMultipleWarpBatch(quadBatch, 2048);
-   props["defines/A3_QUAD_BATCH"]    = quadBatch;
-   props["defines/M3_INNER_BATCH"]   = closestWarpBatchTo(maxDQ);
-}
-
-
-//---[ Base Integrator ]--------------
-OccaIntegrator::OccaIntegrator(const Engine &e)
-   : engine(&e),
-     bform(),
-     mesh(),
-     otrialFESpace(),
-     otestFESpace(),
-     trialFESpace(),
-     testFESpace(),
-     itype(DomainIntegrator),
-     ir(NULL),
-     hasTensorBasis(false) { }
-
-OccaIntegrator::~OccaIntegrator() {}
-
-void OccaIntegrator::SetupMaps()
-{
-   maps = OccaDofQuadMaps::Get(GetDevice(),
-                               *otrialFESpace,
-                               *otestFESpace,
-                               *ir);
-
-   mapsTranspose = OccaDofQuadMaps::Get(GetDevice(),
-                                        *otestFESpace,
-                                        *otrialFESpace,
-                                        *ir);
-}
-
-FiniteElementSpace& OccaIntegrator::GetTrialOccaFESpace() const
-{
-   return *otrialFESpace;
-}
-
-FiniteElementSpace& OccaIntegrator::GetTestOccaFESpace() const
-{
-   return *otestFESpace;
-}
-
-mfem::FiniteElementSpace& OccaIntegrator::GetTrialFESpace() const
-{
-   return *trialFESpace;
-}
-
-mfem::FiniteElementSpace& OccaIntegrator::GetTestFESpace() const
-{
-   return *testFESpace;
-}
-
-void OccaIntegrator::SetIntegrationRule(const mfem::IntegrationRule &ir_)
-{
-   ir = &ir_;
-}
-
-const mfem::IntegrationRule& OccaIntegrator::GetIntegrationRule() const
-{
-   return *ir;
-}
-
-OccaDofQuadMaps& OccaIntegrator::GetDofQuadMaps()
-{
-   return maps;
-}
-
-void OccaIntegrator::SetupIntegrator(OccaBilinearForm &bform_,
-                                     const ::occa::properties &props_,
-                                     const OccaIntegratorType itype_)
-{
-   MFEM_ASSERT(engine == &bform_.OccaEngine(), "");
-   bform     = &bform_;
-   mesh      = &(bform_.GetMesh());
-
-   otrialFESpace = &(bform_.GetTrialOccaFESpace());
-   otestFESpace  = &(bform_.GetTestOccaFESpace());
-
-   trialFESpace = &(bform_.GetTrialFESpace());
-   testFESpace  = &(bform_.GetTestFESpace());
-
-   hasTensorBasis = otrialFESpace->hasTensorBasis();
-
-   props = props_;
-   itype = itype_;
-
-   if (ir == NULL)
-   {
-      SetupIntegrationRule();
-   }
-
-   SetupMaps();
-
-   SetProperties(*otrialFESpace,
-                 *otestFESpace,
-                 *ir,
-                 props);
-
-   Setup();
-}
-
-OccaGeometry OccaIntegrator::GetGeometry(const int flags)
-{
-   return OccaGeometry::Get(GetDevice(), *otrialFESpace, *ir, flags);
-}
-
-::occa::kernel OccaIntegrator::GetAssembleKernel(const ::occa::properties
-                                                 &props)
-{
-   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
-   return GetKernel(stringWithDim("Assemble", fe.GetDim()),
-                    props);
-}
-
-::occa::kernel OccaIntegrator::GetMultAddKernel(const ::occa::properties &props)
-{
-   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
-   return GetKernel(stringWithDim("MultAdd", fe.GetDim()),
-                    props);
-}
-
-::occa::kernel OccaIntegrator::GetKernel(const std::string &kernelName,
-                                         const ::occa::properties &props)
-{
-   const std::string filename = GetName() + ".okl";
-   const std::string &okl_path = OccaEngine().GetOklPath();
-   const std::string &okl_defines = OccaEngine().GetOklDefines();
-   return GetDevice().buildKernel(okl_path + filename,
-                                  kernelName,
-                                  props + okl_defines);
-}
-//====================================
-
-
-//---[ Diffusion Integrator ]---------
-OccaDiffusionIntegrator::OccaDiffusionIntegrator(const OccaCoefficient &coeff_)
-   :
-   OccaIntegrator(coeff_.OccaEngine()),
-   coeff(coeff_),
-   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
-{
-   coeff.SetName("COEFF");
-}
-
-OccaDiffusionIntegrator::~OccaDiffusionIntegrator() {}
-
-
-std::string OccaDiffusionIntegrator::GetName()
-{
-   return "DiffusionIntegrator";
-}
-
-void OccaDiffusionIntegrator::SetupIntegrationRule()
-{
-   const FiniteElement &trialFE = *(trialFESpace->GetFE(0));
-   const FiniteElement &testFE  = *(testFESpace->GetFE(0));
-   ir = &mfem::DiffusionIntegrator::GetRule(trialFE, testFE);
-}
-
-void OccaDiffusionIntegrator::Setup()
-{
-   ::occa::properties kernelProps = props;
-
-   coeff.Setup(*this, kernelProps);
-
-   // Setup assemble and mult kernels
-   assembleKernel = GetAssembleKernel(kernelProps);
-   multKernel     = GetMultAddKernel(kernelProps);
-}
-
-void OccaDiffusionIntegrator::Assemble()
-{
-   const mfem::FiniteElement &fe = *(trialFESpace->GetFE(0));
-
-   const int dims = fe.GetDim();
-   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
-
-   const int elements = trialFESpace->GetNE();
-   const int quadraturePoints = ir->GetNPoints();
-
-   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
-
-   assembledOperator.Resize<double>(symmDims * quadraturePoints * elements,
-                                    NULL);
-
-   assembleKernel((int) mesh->GetNE(),
-                  maps.quadWeights,
-                  geom.J,
-                  coeff,
-                  assembledOperator.OccaMem());
-}
-
-void OccaDiffusionIntegrator::MultAdd(Vector &x, Vector &y)
-{
-   // Note: x and y are E-vectors
-
-   multKernel((int) mesh->GetNE(),
-              maps.dofToQuad,
-              maps.dofToQuadD,
-              maps.quadToDof,
-              maps.quadToDofD,
-              assembledOperator.OccaMem(),
-              x.OccaMem(), y.OccaMem());
-}
-//====================================
-
-
-//---[ Mass Integrator ]--------------
-OccaMassIntegrator::OccaMassIntegrator(const OccaCoefficient &coeff_) :
-   OccaIntegrator(coeff_.OccaEngine()),
-   coeff(coeff_),
-   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
-{
-   coeff.SetName("COEFF");
-}
-
-OccaMassIntegrator::~OccaMassIntegrator() {}
-
-std::string OccaMassIntegrator::GetName()
-{
-   return "MassIntegrator";
-}
-
-void OccaMassIntegrator::SetupIntegrationRule()
-{
-   const mfem::FiniteElement &trialFE = *(trialFESpace->GetFE(0));
-   const mfem::FiniteElement &testFE  = *(testFESpace->GetFE(0));
-   mfem::ElementTransformation &T = *trialFESpace->GetElementTransformation(0);
-   ir = &mfem::MassIntegrator::GetRule(trialFE, testFE, T);
-}
-
-void OccaMassIntegrator::Setup()
-{
-   ::occa::properties kernelProps = props;
-
-   coeff.Setup(*this, kernelProps);
-
-   // Setup assemble and mult kernels
-   assembleKernel = GetAssembleKernel(kernelProps);
-   multKernel     = GetMultAddKernel(kernelProps);
-}
-
-void OccaMassIntegrator::Assemble()
-{
-   if (assembledOperator.Size())
-   {
-      return;
-   }
-
-   const int elements = trialFESpace->GetNE();
-   const int quadraturePoints = ir->GetNPoints();
-
-   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
-
-   assembledOperator.Resize<double>(quadraturePoints * elements, NULL);
-
-   assembleKernel((int) mesh->GetNE(),
-                  maps.quadWeights,
-                  geom.J,
-                  coeff,
-                  assembledOperator.OccaMem());
-}
-
-void OccaMassIntegrator::SetOperator(Vector &v)
-{
-   assembledOperator = v;
-}
-
-void OccaMassIntegrator::MultAdd(Vector &x, Vector &y)
-{
-   multKernel((int) mesh->GetNE(),
-              maps.dofToQuad,
-              maps.dofToQuadD,
-              maps.quadToDof,
-              maps.quadToDofD,
-              assembledOperator.OccaMem(),
-              x.OccaMem(), y.OccaMem());
-}
-//====================================
-
-
-//---[ Vector Mass Integrator ]--------------
-OccaVectorMassIntegrator::OccaVectorMassIntegrator(const OccaCoefficient &
-                                                   coeff_)
-   :
-   OccaIntegrator(coeff_.OccaEngine()),
-   coeff(coeff_),
-   assembledOperator(*(new Layout(coeff_.OccaEngine(), 0)))
-{
-   coeff.SetName("COEFF");
-}
-
-OccaVectorMassIntegrator::~OccaVectorMassIntegrator() {}
-
-std::string OccaVectorMassIntegrator::GetName()
-{
-   return "VectorMassIntegrator";
-}
-
-void OccaVectorMassIntegrator::SetupIntegrationRule()
-{
-   const mfem::FiniteElement &trialFE = *(trialFESpace->GetFE(0));
-   const mfem::FiniteElement &testFE  = *(testFESpace->GetFE(0));
-   mfem::ElementTransformation &T = *trialFESpace->GetElementTransformation(0);
-   ir = &mfem::MassIntegrator::GetRule(trialFE, testFE, T);
-}
-
-void OccaVectorMassIntegrator::Setup()
-{
-   ::occa::properties kernelProps = props;
-
-   coeff.Setup(*this, kernelProps);
-
-   // Setup assemble and mult kernels
-   assembleKernel = GetAssembleKernel(kernelProps);
-   multKernel     = GetMultAddKernel(kernelProps);
-}
-
-void OccaVectorMassIntegrator::Assemble()
-{
-   const int elements = trialFESpace->GetNE();
-   const int quadraturePoints = ir->GetNPoints();
-
-   OccaGeometry geom = GetGeometry(OccaGeometry::Jacobian);
-
-   assembledOperator.Resize<double>(quadraturePoints * elements, NULL);
-
-   assembleKernel((int) mesh->GetNE(),
-                  maps.quadWeights,
-                  geom.J,
-                  coeff,
-                  assembledOperator.OccaMem());
-}
-
-void OccaVectorMassIntegrator::MultAdd(Vector &x, Vector &y)
-{
-   multKernel((int) mesh->GetNE(),
-              maps.dofToQuad,
-              maps.dofToQuadD,
-              maps.quadToDof,
-              maps.quadToDofD,
-              assembledOperator.OccaMem(),
-              x.OccaMem(), y.OccaMem());
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/bilininteg.hpp

@@ -1,323 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP
-#define MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "fespace.hpp"
-#include "bilinearform.hpp"
-#include "coefficient.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class OccaGeometry
-{
-public:
-   ::occa::array<double> meshNodes;
-   ::occa::array<double> J, invJ, detJ;
-
-   // byVDIM  -> [x y z x y z x y z]
-   // byNodes -> [x x x y y y z z z]
-   static const int Jacobian    = (1 << 0);
-   static const int JacobianInv = (1 << 1);
-   static const int JacobianDet = (1 << 2);
-
-   static OccaGeometry Get(::occa::device device,
-                           FiniteElementSpace &ofespace,
-                           const IntegrationRule &ir,
-                           const int flags = (Jacobian    |
-                                              JacobianInv |
-                                              JacobianDet));
-};
-
-class OccaDofQuadMaps
-{
-private:
-   // Reuse dof-quad maps
-   static std::map<std::string, OccaDofQuadMaps> AllDofQuadMaps;
-   std::string hash;
-
-public:
-   // Local stiffness matrices (B and B^T operators)
-   ::occa::array<double, ::occa::dynamic> dofToQuad, dofToQuadD; // B
-   ::occa::array<double, ::occa::dynamic> quadToDof, quadToDofD; // B^T
-   ::occa::array<double> quadWeights;
-
-   OccaDofQuadMaps();
-   OccaDofQuadMaps(const OccaDofQuadMaps &maps);
-   OccaDofQuadMaps& operator = (const OccaDofQuadMaps &maps);
-
-   // [[x y] [x y] [x y]]
-   // [[x y z] [x y z] [x y z]]
-   // mfem::GridFunction* mfem::Mesh::GetNodes() { return Nodes; }
-
-   // mfem::FiniteElementSpace *Nodes->FESpace()
-   // 25
-   // 1D [x x x x x x]
-   // 2D [x y x y x y]
-   // GetVdim()
-   // 3D ordering == byVDIM  -> [x y z x y z x y z x y z x y z x y z]
-   //    ordering == byNODES -> [x x x x x x y y y y y y z z z z z z]
-   static OccaDofQuadMaps& Get(::occa::device device,
-                               const FiniteElementSpace &fespace,
-                               const mfem::IntegrationRule &ir,
-                               const bool transpose = false);
-
-   static OccaDofQuadMaps& Get(::occa::device device,
-                               const mfem::FiniteElement &fe,
-                               const mfem::IntegrationRule &ir,
-                               const bool transpose = false);
-
-   static OccaDofQuadMaps& Get(::occa::device device,
-                               const FiniteElementSpace &trialFESpace,
-                               const FiniteElementSpace &testFESpace,
-                               const mfem::IntegrationRule &ir,
-                               const bool transpose = false);
-
-   static OccaDofQuadMaps& Get(::occa::device device,
-                               const mfem::FiniteElement &trialFE,
-                               const mfem::FiniteElement &testFE,
-                               const mfem::IntegrationRule &ir,
-                               const bool transpose = false);
-
-   static OccaDofQuadMaps& GetTensorMaps(::occa::device device,
-                                         const mfem::FiniteElement &fe,
-                                         const mfem::IntegrationRule &ir,
-                                         const bool transpose = false);
-
-   static OccaDofQuadMaps& GetTensorMaps(::occa::device device,
-                                         const mfem::FiniteElement &trialFE,
-                                         const mfem::FiniteElement &testFE,
-                                         const mfem::IntegrationRule &ir,
-                                         const bool transpose = false);
-
-   static OccaDofQuadMaps GetD2QTensorMaps(::occa::device device,
-                                           const mfem::FiniteElement &fe,
-                                           const mfem::IntegrationRule &ir,
-                                           const bool transpose = false);
-
-   static OccaDofQuadMaps& GetSimplexMaps(::occa::device device,
-                                          const mfem::FiniteElement &fe,
-                                          const mfem::IntegrationRule &ir,
-                                          const bool transpose = false);
-
-   static OccaDofQuadMaps& GetSimplexMaps(::occa::device device,
-                                          const mfem::FiniteElement &trialFE,
-                                          const mfem::FiniteElement &testFE,
-                                          const mfem::IntegrationRule &ir,
-                                          const bool transpose = false);
-
-   static OccaDofQuadMaps GetD2QSimplexMaps(::occa::device device,
-                                            const mfem::FiniteElement &fe,
-                                            const mfem::IntegrationRule &ir,
-                                            const bool transpose = false);
-};
-
-//---[ Define Methods ]---------------
-std::string stringWithDim(const std::string &s, const int dim);
-int closestWarpBatch(const int multiple, const int maxSize);
-
-void SetProperties(FiniteElementSpace &fespace,
-                   const mfem::IntegrationRule &ir,
-                   ::occa::properties &props);
-
-void SetProperties(FiniteElementSpace &trialFESpace,
-                   FiniteElementSpace &testFESpace,
-                   const mfem::IntegrationRule &ir,
-                   ::occa::properties &props);
-
-void SetTensorProperties(FiniteElementSpace &fespace,
-                         const mfem::IntegrationRule &ir,
-                         ::occa::properties &props);
-
-void SetTensorProperties(FiniteElementSpace &trialFESpace,
-                         FiniteElementSpace &testFESpace,
-                         const IntegrationRule &ir,
-                         ::occa::properties &props);
-
-void SetSimplexProperties(FiniteElementSpace &fespace,
-                          const IntegrationRule &ir,
-                          ::occa::properties &props);
-
-void SetSimplexProperties(FiniteElementSpace &trialFESpace,
-                          FiniteElementSpace &testFESpace,
-                          const IntegrationRule &ir,
-                          ::occa::properties &props);
-
-//---[ Base Integrator ]--------------
-class OccaIntegrator
-{
-protected:
-   SharedPtr<const Engine> engine;
-
-   OccaBilinearForm *bform;
-   mfem::Mesh *mesh;
-
-   FiniteElementSpace *otrialFESpace;
-   FiniteElementSpace *otestFESpace;
-
-   mfem::FiniteElementSpace *trialFESpace;
-   mfem::FiniteElementSpace *testFESpace;
-
-   ::occa::properties props;
-   OccaIntegratorType itype;
-
-   const IntegrationRule *ir;
-   bool hasTensorBasis;
-   OccaDofQuadMaps maps;
-   OccaDofQuadMaps mapsTranspose;
-
-public:
-   OccaIntegrator(const Engine &e);
-   virtual ~OccaIntegrator();
-
-   const Engine &OccaEngine() const { return *engine; }
-
-   ::occa::device GetDevice(int idx = 0) const
-   { return engine->GetDevice(idx); }
-
-   virtual std::string GetName() = 0;
-
-   FiniteElementSpace& GetTrialOccaFESpace() const;
-   FiniteElementSpace& GetTestOccaFESpace() const;
-
-   mfem::FiniteElementSpace& GetTrialFESpace() const;
-   mfem::FiniteElementSpace& GetTestFESpace() const;
-
-   void SetIntegrationRule(const mfem::IntegrationRule &ir_);
-   const mfem::IntegrationRule& GetIntegrationRule() const;
-
-   OccaDofQuadMaps& GetDofQuadMaps();
-
-   void SetupMaps();
-
-   virtual void SetupIntegrationRule() = 0;
-
-   virtual void SetupIntegrator(OccaBilinearForm &bform_,
-                                const ::occa::properties &props_,
-                                const OccaIntegratorType itype_);
-
-   virtual void Setup() = 0;
-
-   virtual void Assemble() = 0;
-   /// This method works on E-vectors!
-   virtual void MultAdd(Vector &x, Vector &y) = 0;
-
-   virtual void MultTransposeAdd(Vector &x, Vector &y)
-   {
-      mfem_error("OccaIntegrator::MultTransposeAdd() is not overloaded!");
-   }
-
-   OccaGeometry GetGeometry(const int flags = (OccaGeometry::Jacobian    |
-                                               OccaGeometry::JacobianInv |
-                                               OccaGeometry::JacobianDet));
-
-   ::occa::kernel GetAssembleKernel(const ::occa::properties &props);
-   ::occa::kernel GetMultAddKernel(const ::occa::properties &props);
-
-   ::occa::kernel GetKernel(const std::string &kernelName,
-                            const ::occa::properties &props);
-};
-//====================================
-
-
-//---[ Diffusion Integrator ]---------
-class OccaDiffusionIntegrator : public OccaIntegrator
-{
-private:
-   OccaCoefficient coeff;
-
-   ::occa::kernel assembleKernel, multKernel;
-
-   Vector assembledOperator;
-
-public:
-   OccaDiffusionIntegrator(const OccaCoefficient &coeff_);
-   virtual ~OccaDiffusionIntegrator();
-
-   virtual std::string GetName();
-
-   virtual void SetupIntegrationRule();
-
-   virtual void Setup();
-
-   virtual void Assemble();
-   virtual void MultAdd(Vector &x, Vector &y);
-};
-//====================================
-
-
-//---[ Mass Integrator ]--------------
-class OccaMassIntegrator : public OccaIntegrator
-{
-private:
-   OccaCoefficient coeff;
-
-   ::occa::kernel assembleKernel, multKernel;
-
-   Vector assembledOperator;
-
-public:
-   OccaMassIntegrator(const OccaCoefficient &coeff_);
-   virtual ~OccaMassIntegrator();
-
-   virtual std::string GetName();
-
-   virtual void SetupIntegrationRule();
-
-   virtual void Setup();
-
-   virtual void Assemble();
-   void SetOperator(Vector &v);
-
-   virtual void MultAdd(Vector &x, Vector &y);
-};
-//====================================
-
-//---[ Vector Mass Integrator ]--------------
-class OccaVectorMassIntegrator : public OccaIntegrator
-{
-private:
-   OccaCoefficient coeff;
-
-   ::occa::kernel assembleKernel, multKernel;
-
-   Vector assembledOperator;
-
-public:
-   OccaVectorMassIntegrator(const OccaCoefficient &coeff_);
-   virtual ~OccaVectorMassIntegrator();
-
-   virtual std::string GetName();
-
-   virtual void SetupIntegrationRule();
-
-   virtual void Setup();
-
-   virtual void Assemble();
-
-   virtual void MultAdd(Vector &x, Vector &y);
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_BILIN_INTEG_HPP

backends/occa/coefficient.cpp

@@ -1,344 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "coefficient.hpp"
-#include "bilininteg.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-//---[ Parameter ]------------
-OccaParameter::~OccaParameter() {}
-
-void OccaParameter::Setup(OccaIntegrator &integ,
-                          ::occa::properties &props) {}
-
-::occa::kernelArg OccaParameter::KernelArgs()
-{
-   return ::occa::kernelArg();
-}
-//====================================
-
-
-//---[ Include Parameter ]------------
-OccaIncludeParameter::OccaIncludeParameter(const std::string &filename_) :
-   filename(filename_) {}
-
-OccaParameter* OccaIncludeParameter::Clone()
-{
-   return new OccaIncludeParameter(filename);
-}
-
-void OccaIncludeParameter::Setup(OccaIntegrator &integ,
-                                 ::occa::properties &props)
-{
-   props["headers"].asArray() += "#include " + filename;
-}
-//====================================
-
-
-//---[ Source Parameter ]------------
-OccaSourceParameter::OccaSourceParameter(const std::string &source_) :
-   source(source_) {}
-
-OccaParameter* OccaSourceParameter::Clone()
-{
-   return new OccaSourceParameter(source);
-}
-
-void OccaSourceParameter::Setup(OccaIntegrator &integ,
-                                ::occa::properties &props)
-{
-   props["headers"].asArray() += source;
-}
-//====================================
-
-//---[ Vector Parameter ]-------
-OccaVectorParameter::OccaVectorParameter(const std::string &name_,
-                                         Vector &v_,
-                                         const bool useRestrict_) :
-   name(name_),
-   v(v_),
-   useRestrict(useRestrict_),
-   attr("") {}
-
-OccaVectorParameter::OccaVectorParameter(const std::string &name_,
-                                         Vector &v_,
-                                         const std::string &attr_,
-                                         const bool useRestrict_) :
-   name(name_),
-   v(v_),
-   useRestrict(useRestrict_),
-   attr(attr_) {}
-
-OccaParameter* OccaVectorParameter::Clone()
-{
-   return new OccaVectorParameter(name, v, attr, useRestrict);
-}
-
-void OccaVectorParameter::Setup(OccaIntegrator &integ,
-                                ::occa::properties &props)
-{
-   std::string &args = (props["defines/COEFF_ARGS"]
-                        .asString()
-                        .string());
-   args += "const double *";
-   if (useRestrict)
-   {
-      args += " restrict ";
-   }
-   args += name;
-   if (attr.size())
-   {
-      args += ' ';
-      args += attr;
-   }
-   args += ",\n";
-}
-
-::occa::kernelArg OccaVectorParameter::KernelArgs()
-{
-   return ::occa::kernelArg(v.OccaMem());
-}
-//====================================
-
-
-//---[ GridFunction Parameter ]-------
-OccaGridFunctionParameter::OccaGridFunctionParameter(const std::string &name_,
-                                                     OccaGridFunction &gf_,
-                                                     const bool useRestrict_)
-   : name(name_),
-     gf(gf_),
-     gfQuad(*(new Layout(gf_.OccaLayout().OccaEngine(), 0))),
-     useRestrict(useRestrict_) {}
-
-OccaParameter* OccaGridFunctionParameter::Clone()
-{
-   OccaGridFunctionParameter *param =
-      new OccaGridFunctionParameter(name, gf, useRestrict);
-   param->gfQuad.MakeRef(gfQuad);
-   return param;
-}
-
-void OccaGridFunctionParameter::Setup(OccaIntegrator &integ,
-                                      ::occa::properties &props)
-{
-
-   std::string &args = (props["defines/COEFF_ARGS"]
-                        .asString()
-                        .string());
-   args += "const double *";
-   if (useRestrict)
-   {
-      args += " restrict ";
-   }
-   args += name;
-   args += " @dim(NUM_QUAD, numElements),\n";
-
-   gf.ToQuad(integ.GetIntegrationRule(), gfQuad);
-}
-
-::occa::kernelArg OccaGridFunctionParameter::KernelArgs()
-{
-   return gfQuad.OccaMem();
-}
-//====================================
-
-
-//---[ Coefficient ]------------------
-OccaCoefficient::OccaCoefficient(const Engine &e, const double value) :
-   engine(&e),
-   integ(NULL),
-   name("COEFF")
-{
-   coeffValue = value;
-}
-
-OccaCoefficient::OccaCoefficient(const Engine &e, const std::string &source) :
-   engine(&e),
-   integ(NULL),
-   name("COEFF")
-{
-   coeffValue = source;
-}
-
-OccaCoefficient::OccaCoefficient(const Engine &e, const char *source) :
-   engine(&e),
-   integ(NULL),
-   name("COEFF")
-{
-   coeffValue = source;
-}
-
-OccaCoefficient::OccaCoefficient(const OccaCoefficient &coeff) :
-   engine(coeff.engine),
-   integ(NULL),
-   name(coeff.name),
-   coeffValue(coeff.coeffValue)
-{
-
-   const int paramCount = (int) coeff.params.size();
-   for (int i = 0; i < paramCount; ++i)
-   {
-      params.push_back(coeff.params[i]->Clone());
-   }
-}
-
-OccaCoefficient::~OccaCoefficient()
-{
-   const int paramCount = (int) params.size();
-   for (int i = 0; i < paramCount; ++i)
-   {
-      delete params[i];
-   }
-}
-
-OccaCoefficient& OccaCoefficient::SetName(const std::string &name_)
-{
-   name = name_;
-   return *this;
-}
-
-void OccaCoefficient::Setup(OccaIntegrator &integ_,
-                            ::occa::properties &props_)
-{
-   integ = &integ_;
-
-   const int paramCount = (int) params.size();
-   props_["defines"][name + "_ARGS"] = "";
-   for (int i = 0; i < paramCount; ++i)
-   {
-      params[i]->Setup(integ_, props_);
-   }
-   props_["defines"][name] = coeffValue;
-
-   props = props_;
-}
-
-OccaCoefficient& OccaCoefficient::Add(OccaParameter *param)
-{
-   params.push_back(param);
-   return *this;
-}
-
-OccaCoefficient& OccaCoefficient::IncludeHeader(const std::string &filename)
-{
-   return Add(new OccaIncludeParameter(filename));
-}
-
-OccaCoefficient& OccaCoefficient::IncludeSource(const std::string &source)
-{
-   return Add(new OccaSourceParameter(source));
-}
-
-OccaCoefficient& OccaCoefficient::AddVector(const std::string &name_,
-                                            Vector &v,
-                                            const bool useRestrict)
-{
-   return Add(new OccaVectorParameter(name_, v, useRestrict));
-}
-
-OccaCoefficient& OccaCoefficient::AddVector(const std::string &name_,
-                                            Vector &v,
-                                            const std::string &attr,
-                                            const bool useRestrict)
-{
-   return Add(new OccaVectorParameter(name_, v, attr, useRestrict));
-}
-
-OccaCoefficient& OccaCoefficient::AddGridFunction(const std::string &name_,
-                                                  OccaGridFunction &gf,
-                                                  const bool useRestrict)
-{
-   return Add(new OccaGridFunctionParameter(name_, gf, useRestrict));
-}
-
-bool OccaCoefficient::IsConstant()
-{
-   return coeffValue.isNumber();
-}
-
-double OccaCoefficient::GetConstantValue()
-{
-   if (!IsConstant())
-   {
-      mfem_error("OccaCoefficient is not constant");
-   }
-   return coeffValue.number();
-}
-
-Vector OccaCoefficient::Eval()
-{
-   if (integ == NULL)
-   {
-      mfem_error("OccaCoefficient requires a Setup() call before Eval()");
-   }
-
-   mfem::FiniteElementSpace &fespace = integ->GetTrialFESpace();
-   const mfem::IntegrationRule &ir   = integ->GetIntegrationRule();
-
-   const int elements = fespace.GetNE();
-   const int numQuad  = ir.GetNPoints();
-
-   Vector quadCoeff(*(new Layout(OccaEngine(), numQuad * elements)));
-   Eval(quadCoeff);
-   return quadCoeff;
-}
-
-void OccaCoefficient::Eval(Vector &quadCoeff)
-{
-   const std::string &okl_path = OccaEngine().GetOklPath();
-   const std::string &okl_defines = OccaEngine().GetOklDefines();
-   static ::occa::kernelBuilder builder =
-      ::occa::kernelBuilder::fromFile(okl_path + "coefficient.okl",
-                                      "CoefficientEval", okl_defines);
-
-   if (integ == NULL)
-   {
-      mfem_error("OccaCoefficient requires a Setup() call before Eval()");
-   }
-
-   const int elements = integ->GetTrialFESpace().GetNE();
-
-   ::occa::properties kernelProps = props;
-   if (name != "COEFF")
-   {
-      kernelProps["defines/COEFF"]      = name;
-      kernelProps["defines/COEFF_ARGS"] = name + "_ARGS";
-   }
-   kernelProps += okl_defines;
-
-   ::occa::kernel evalKernel = builder.build(GetDevice(), kernelProps);
-   evalKernel(elements, *this, quadCoeff.OccaMem());
-}
-
-OccaCoefficient::operator ::occa::kernelArg ()
-{
-   ::occa::kernelArg kArg;
-   const int paramCount = (int) params.size();
-   for (int i = 0; i < paramCount; ++i)
-   {
-      kArg.add(params[i]->KernelArgs());
-   }
-   return kArg;
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/coefficient.hpp

@@ -1,284 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_COEFFICIENT_HPP
-#define MFEM_BACKENDS_OCCA_COEFFICIENT_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "vector.hpp"
-#include "gridfunc.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class OccaIntegrator;
-
-
-class OccaParameter
-{
-public:
-   virtual ~OccaParameter();
-
-   virtual OccaParameter* Clone() = 0;
-
-   virtual void Setup(OccaIntegrator &integ,
-                      ::occa::properties &props);
-
-   virtual ::occa::kernelArg KernelArgs();
-};
-
-
-//---[ Include Parameter ]------------
-class OccaIncludeParameter : public OccaParameter
-{
-private:
-   std::string filename;
-
-public:
-   OccaIncludeParameter(const std::string &filename_);
-
-   virtual OccaParameter* Clone();
-
-   virtual void Setup(OccaIntegrator &integ,
-                      ::occa::properties &props);
-};
-//====================================
-
-
-//---[ Source Parameter ]------------
-class OccaSourceParameter : public OccaParameter
-{
-private:
-   std::string source;
-
-public:
-   OccaSourceParameter(const std::string &filename_);
-
-   virtual OccaParameter* Clone();
-
-   virtual void Setup(OccaIntegrator &integ,
-                      ::occa::properties &props);
-};
-//====================================
-
-
-//---[ Define Parameter ]------------
-template <class TM>
-class OccaDefineParameter : public OccaParameter
-{
-private:
-   const std::string name;
-   TM value;
-
-public:
-   OccaDefineParameter(const std::string &name_,
-                       const TM &value_) :
-      name(name_),
-      value(value_) {}
-
-   virtual OccaParameter* Clone()
-   {
-      return new OccaDefineParameter(name, value);
-   }
-
-   virtual void Setup(OccaIntegrator &integ,
-                      ::occa::properties &props)
-   {
-      props["defines"][name] = value;
-   }
-};
-//====================================
-
-
-//---[ Variable Parameter ]-----------
-template <class TM>
-class OccaVariableParameter : public OccaParameter
-{
-private:
-   const std::string name;
-   const TM &value;
-
-public:
-   OccaVariableParameter(const std::string &name_,
-                         const TM &value_) :
-      name(name_),
-      value(value_) {}
-
-   virtual OccaParameter* Clone()
-   {
-      return new OccaVariableParameter(name, value);
-   }
-
-   virtual void Setup(OccaIntegrator &integ,
-                      ::occa::properties &props)
-   {
-      std::string &args = (props["defines/COEFF_ARGS"]
-                           .asString()
-                           .string());
-      // const TM name,\n"
-      args += "const ";
-      args += ::occa::primitiveinfo<TM>::name;
-      args += ' ';
-      args += name;
-      args += ",\n";
-   }
-
-   virtual ::occa::kernelArg KernelArgs()
-   {
-      return ::occa::kernelArg(value);
-   }
-};
-//====================================
-
-
-//---[ Vector Parameter ]-------
-class OccaVectorParameter : public OccaParameter
-{
-private:
-   const std::string name;
-   Vector v;
-   bool useRestrict;
-   std::string attr;
-
-public:
-   OccaVectorParameter(const std::string &name_,
-                       Vector &v_,
-                       const bool useRestrict_ = false);
-
-   OccaVectorParameter(const std::string &name_,
-                       Vector &v_,
-                       const std::string &attr_,
-                       const bool useRestrict_ = false);
-
-   virtual OccaParameter* Clone();
-
-   virtual void Setup(OccaIntegrator &integ,
-                      ::occa::properties &props);
-
-   virtual ::occa::kernelArg KernelArgs();
-};
-//====================================
-
-
-//---[ GridFunction Parameter ]-------
-class OccaGridFunctionParameter : public OccaParameter
-{
-private:
-   const std::string name;
-   OccaGridFunction &gf;
-   Vector gfQuad;
-   bool useRestrict;
-
-public:
-   OccaGridFunctionParameter(const std::string &name_,
-                             OccaGridFunction &gf_,
-                             const bool useRestrict_ = false);
-
-   virtual OccaParameter* Clone();
-
-   virtual void Setup(OccaIntegrator &integ,
-                      ::occa::properties &props);
-
-   virtual ::occa::kernelArg KernelArgs();
-};
-//====================================
-
-
-//---[ Coefficient ]------------------
-// [MISSING]
-// Needs to know about the integrator's
-//   - fespace
-//   - ir
-// Step where parameters that need the ir get called for setup
-// For example, GridFunction (d, e) -> (q, e)
-class OccaCoefficient
-{
-private:
-   SharedPtr<const Engine> engine;
-
-   OccaIntegrator *integ;
-
-   std::string name;
-   ::occa::json coeffValue;
-
-   ::occa::properties props;
-   std::vector<OccaParameter*> params;
-
-public:
-   OccaCoefficient(const Engine &e, const double value = 1.0);
-   OccaCoefficient(const Engine &e, const std::string &source);
-   OccaCoefficient(const Engine &e, const char *source);
-   ~OccaCoefficient();
-
-   OccaCoefficient(const OccaCoefficient &coeff);
-
-   const Engine &OccaEngine() const { return *engine; }
-
-   ::occa::device GetDevice(int idx = 0) const
-   { return engine->GetDevice(idx); }
-
-   OccaCoefficient& SetName(const std::string &name_);
-
-   void Setup(OccaIntegrator &integ_,
-              ::occa::properties &props_);
-
-   OccaCoefficient& Add(OccaParameter *param);
-
-   OccaCoefficient& IncludeHeader(const std::string &filename);
-   OccaCoefficient& IncludeSource(const std::string &source);
-
-   template <class TM>
-   OccaCoefficient& AddDefine(const std::string &name_, const TM &value)
-   {
-      return Add(new OccaDefineParameter<TM>(name_, value));
-   }
-
-   template <class TM>
-   OccaCoefficient& AddVariable(const std::string &name_, const TM &value)
-   {
-      return Add(new OccaVariableParameter<TM>(name_, value));
-   }
-
-   OccaCoefficient& AddVector(const std::string &name_,
-                              Vector &v,
-                              const bool useRestrict = false);
-
-
-   OccaCoefficient& AddVector(const std::string &name_,
-                              Vector &v,
-                              const std::string &attr,
-                              const bool useRestrict = false);
-
-   OccaCoefficient& AddGridFunction(const std::string &name_,
-                                    OccaGridFunction &gf,
-                                    const bool useRestrict = false);
-
-   bool IsConstant();
-   double GetConstantValue();
-
-   Vector Eval();
-   void Eval(Vector &quadCoeff);
-
-   operator ::occa::kernelArg ();
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_COEFFICIENT_HPP

backends/occa/defines.okl

@@ -1,40 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_OCCA_DEFINES
-#define MFEM_OCCA_DEFINES
-
-#ifndef USING_TENSOR_OPS
-#  define USING_TENSOR_OPS 0
-#endif
-
-#ifdef OCCA_USING_GPU
-#  define GPU_ORDER_2(I0, I1)         @dimOrder(I0, I1)
-#  define GPU_ORDER_3(I0, I1, I2)     @dimOrder(I0, I1, I2)
-#  define GPU_ORDER_4(I0, I1, I2, I3) @dimOrder(I0, I1, I2, I3)
-#else
-#  define GPU_ORDER_2(I0, I1)         @dimOrder(0, 1)
-#  define GPU_ORDER_3(I0, I1, I2)     @dimOrder(0, 1, 2)
-#  define GPU_ORDER_4(I0, I1, I2, I3) @dimOrder(0, 1, 2, 3)
-#endif
-
-#ifndef COEFF
-#  define COEFF 1.0
-#  define COEFF_ARGS
-#endif
-
-#if USING_TENSOR_OPS
-#  include "mfem-occa://defines/tensor.okl"
-#else
-#  include "mfem-occa://defines/simplex.okl"
-#endif
-
-#endif

backends/occa/defines/simplex.okl

@@ -1,40 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#define USING_LOW_ORDER 1
-#define USING_HI_ORDER  0
-
-typedef double* DofToQuad_t @dim(NUM_QUAD, NUM_DOFS);
-typedef double* DofToQuadD2D_t @dim(2, NUM_QUAD, NUM_DOFS);
-typedef double* DofToQuadD3D_t @dim(3, NUM_QUAD, NUM_DOFS);
-
-typedef double* QuadToDof_t @dim(NUM_DOFS, NUM_QUAD);
-typedef double* QuadToDofD2D_t @dim(2, NUM_DOFS, NUM_QUAD);
-typedef double* QuadToDofD3D_t @dim(3, NUM_DOFS, NUM_QUAD);
-
-typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD, numElements);
-typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
-
-typedef double* SymmOperator2D_t @dim(3, NUM_QUAD, numElements);
-typedef double* SymmOperator3D_t @dim(6, NUM_QUAD, numElements);
-
-typedef double* DLocal_t @dim(NUM_DOFS, numElements);
-typedef double* QLocal_t @dim(NUM_QUAD, numElements);
-
-#if VDIM_ORDERING == ORDERING_BY_VDIM
-typedef double* DVLocal_t @dim(NUM_VDIM, NUM_DOFS, numElements);
-typedef double* QVLocal_t @dim(NUM_VDIM, NUM_QUAD, numElements);
-#else
-typedef double* DVLocal_t @dim(NUM_VDIM, NUM_DOFS, numElements) @dimOrder(2,0,1);
-typedef double* QVLocal_t @dim(NUM_VDIM, NUM_QUAD, numElements) @dimOrder(2,0,1);
-#endif
-
-typedef int* DLocalMap_t @dim(NUM_DOFS, numElements);

backends/occa/defines/tensor.okl

@@ -1,85 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#if NUM_QUAD_1D < NUM_DOFS_1D
-#  define NUM_MAX_1D NUM_DOFS_1D
-#else
-#  define NUM_MAX_1D NUM_QUAD_1D
-#endif
-
-#define NUM_MAX_2D (NUM_MAX_1D * NUM_MAX_1D)
-
-#define NUM_QUAD_DOFS_1D (NUM_QUAD_1D * NUM_DOFS_1D)
-
-#define QUAD_2D_ID(X, Y) (X + ((Y) * NUM_QUAD_1D))
-#define DOFS_2D_ID(X, Y) (X + ((Y) * NUM_DOFS_1D))
-
-#define QUAD_3D_ID(X, Y, Z) (X + ((Y) * NUM_QUAD_1D) + ((Z) * NUM_QUAD_2D))
-#define DOFS_3D_ID(X, Y, Z) (X + ((Y) * NUM_DOFS_1D) + ((Z) * NUM_DOFS_2D))
-
-#if NUM_MAX_1D < 8
-#  define USING_LOW_ORDER 1
-#  define USING_HI_ORDER  0
-#else
-#  define USING_LOW_ORDER 0
-#  define USING_HI_ORDER  1
-#endif
-
-#define M1_ELEMENT_BATCHES (M1_OUTER_ELEMENT_BATCH * M1_INNER_ELEMENT_BATCH)
-
-typedef double* DofToQuad_t @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-typedef double* QuadToDof_t @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-
-typedef double* Jacobian_t   @dim(NUM_DIM, NUM_DIM, numElements);
-typedef double* Jacobian1D_t @dim(NUM_QUAD_1D, numElements);
-typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD_2D, numElements);
-typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD_3D, numElements);
-
-typedef double* SymmOperator1D_t @dim(NUM_QUAD_1D, numElements);
-typedef double* SymmOperator2D_t @dim(3, NUM_QUAD_2D, numElements);
-typedef double* SymmOperator3D_t @dim(6, NUM_QUAD_3D, numElements);
-
-typedef double* DLocal_t   @dim(NUM_DOFS, numElements);
-typedef double* DLocal1D_t @dim(NUM_DOFS_1D, numElements);
-typedef double* DLocal2D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, numElements);
-typedef double* DLocal3D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
-
-typedef double* QLocal_t   @dim(NUM_QUAD, numElements);
-typedef double* QLocal1D_t @dim(NUM_QUAD_1D, numElements);
-typedef double* QLocal2D_t @dim(NUM_QUAD_1D, NUM_QUAD_1D, numElements);
-typedef double* QLocal3D_t @dim(NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
-
-#if VDIM_ORDERING == ORDERING_BY_VDIM
-typedef double* DVLocal_t   @dim(NUM_VDIM, NUM_DOFS, numElements);
-typedef double* DVLocal1D_t @dim(NUM_VDIM, NUM_DOFS_1D, numElements);
-typedef double* DVLocal2D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
-typedef double* DVLocal3D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);
-
-typedef double* QVLocal_t   @dim(NUM_VDIM, NUM_QUAD, numElements);
-typedef double* QVLocal1D_t @dim(NUM_VDIM, NUM_QUAD_1D, numElements);
-typedef double* QVLocal2D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
-typedef double* QVLocal3D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements);
-#else
-typedef double* DVLocal_t   @dim(NUM_VDIM, NUM_DOFS, numElements) @dimOrder(2,0,1);
-typedef double* DVLocal1D_t @dim(NUM_VDIM, NUM_DOFS_1D, numElements) @dimOrder(2,0,1);
-typedef double* DVLocal2D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, numElements) @dimOrder(3,0,1,2);
-typedef double* DVLocal3D_t @dim(NUM_VDIM, NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements) @dimOrder(4,0,1,2,3);
-
-typedef double* QVLocal_t   @dim(NUM_VDIM, NUM_QUAD, numElements) @dimOrder(2,0,1);
-typedef double* QVLocal1D_t @dim(NUM_VDIM, NUM_QUAD_1D, numElements) @dimOrder(2,0,1);
-typedef double* QVLocal2D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, numElements) @dimOrder(3,0,1,2);
-typedef double* QVLocal3D_t @dim(NUM_VDIM, NUM_QUAD_1D, NUM_QUAD_1D, NUM_QUAD_1D, numElements) @dimOrder(4,0,1,2,3);
-#endif
-
-typedef int* DLocalMap_t   @dim(NUM_DOFS, numElements);
-typedef int* DLocalMap1D_t @dim(NUM_DOFS_1D, numElements);
-typedef int* DLocalMap2D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, numElements);
-typedef int* DLocalMap3D_t @dim(NUM_DOFS_1D, NUM_DOFS_1D, NUM_DOFS_1D, numElements);

backends/occa/diffusion/simplex/cpu.okl

@@ -1,168 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        const double * restrict quadWeights,
-                        const Jacobian2D_t restrict J,
-                        COEFF_ARGS
-                        SymmOperator2D_t restrict oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-
-      const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
-
-      oper(0, q, e) =  c_detJ * (J12*J12 + J22*J22); // (1,1)
-      oper(1, q, e) = -c_detJ * (J12*J11 + J22*J21); // (1,2) + (2,1)
-      oper(2, q, e) =  c_detJ * (J11*J11 + J21*J21); // (2,2)
-    }
-  }
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuadD2D_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDofD2D_t restrict quadToDofD,
-                       const SymmOperator2D_t restrict oper,
-                       const DLocal_t restrict solIn,
-                       DLocal_t restrict solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double r_sol[NUM_DOFS];
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        r_sol[d] = 0;
-      }
-
-      for (int q = 0; q < NUM_QUAD; ++q) {
-        double gradX = 0, gradY = 0;
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          const double s = solIn(d, e);
-          gradX += s * quadToDofD(0, d, q);
-          gradY += s * quadToDofD(1, d, q);
-        }
-        const double O11 = oper(0, q, e);
-        const double O12 = oper(1, q, e);
-        const double O22 = oper(2, q, e);
-
-        const double gradX2 = (O11 * gradX) + (O12 * gradY);
-        const double gradY2 = (O12 * gradX) + (O22 * gradY);
-
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          r_sol[d] += ((gradX2 * quadToDofD(0, d, q)) +
-                       (gradY2 * quadToDofD(1, d, q)));
-        }
-      }
-
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        solOut(d, e) += r_sol[d];
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        const double * restrict quadWeights,
-                        const Jacobian3D_t restrict J,
-                        COEFF_ARGS
-                        SymmOperator3D_t restrict oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-      const double c_detJ = quadWeights[q] * COEFF / detJ;
-
-      // adj(J)
-      const double A11 = (J22 * J33) - (J23 * J32);
-      const double A12 = (J23 * J31) - (J21 * J33);
-      const double A13 = (J21 * J32) - (J22 * J31);
-
-      const double A21 = (J13 * J32) - (J12 * J33);
-      const double A22 = (J11 * J33) - (J13 * J31);
-      const double A23 = (J12 * J31) - (J11 * J32);
-
-      const double A31 = (J12 * J23) - (J13 * J22);
-      const double A32 = (J13 * J21) - (J11 * J23);
-      const double A33 = (J11 * J22) - (J12 * J21);
-
-      // adj(J)^Tadj(J)
-      oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
-      oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2) + (2,1)
-      oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3) + (3,1)
-      oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
-      oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3) + (3,2)
-      oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuadD3D_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDofD3D_t restrict quadToDofD,
-                       const SymmOperator3D_t restrict oper,
-                       const DLocal_t restrict solIn,
-                       DLocal_t restrict solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double r_sol[NUM_DOFS];
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        r_sol[d] = 0;
-      }
-
-      for (int q = 0; q < NUM_QUAD; ++q) {
-        double gradX = 0, gradY = 0, gradZ = 0;
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          const double s = solIn(d, e);
-          gradX += s * quadToDofD(0, d, q);
-          gradY += s * quadToDofD(1, d, q);
-          gradZ += s * quadToDofD(2, d, q);
-        }
-        const double O11 = oper(0, q, e);
-        const double O12 = oper(1, q, e);
-        const double O13 = oper(2, q, e);
-        const double O22 = oper(3, q, e);
-        const double O23 = oper(4, q, e);
-        const double O33 = oper(5, q, e);
-
-        const double gradX2 = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
-        const double gradY2 = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
-        const double gradZ2 = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
-
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          r_sol[d] += ((gradX2 * quadToDofD(0, d, q)) +
-                       (gradY2 * quadToDofD(1, d, q)) +
-                       (gradZ2 * quadToDofD(2, d, q)));
-        }
-      }
-
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        solOut(d, e) += r_sol[d];
-      }
-    }
-  }
-}
-//======================================

backends/occa/diffusion/simplex/gpuHighOrder.okl

@@ -1,182 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        const double *quadWeights,
-                        const Jacobian2D_t J,
-                        COEFF_ARGS
-                        SymmOperator2D_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
-          for (int q = qOff; q < NUM_QUAD; q += A2_QUAD_BATCH) {
-            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-
-            const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
-
-            oper(0, q, e) =  c_detJ * (J12*J12 + J22*J22); // (1,1)
-            oper(1, q, e) = -c_detJ * (J12*J11 + J22*J21); // (1,2) + (2,1)
-            oper(2, q, e) =  c_detJ * (J11*J11 + J21*J21); // (2,2)
-          }
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuadD2D_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDofD2D_t restrict quadToDofD,
-                       const SymmOperator2D_t restrict oper,
-                       const DLocal_t restrict solIn,
-                       DLocal_t restrict solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_gradX[NUM_QUAD];
-    @shared double s_gradY[NUM_QUAD];
-
-    for (int qOff = 0; qOff < M2_INNER_BATCH; ++qOff; @inner) {
-      for (int q = qOff; q < NUM_QUAD; q += M2_INNER_BATCH) {
-        double gradX = 0, gradY = 0;
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          const double s = solIn(d, e);
-          gradX += s * quadToDofD(0, d, q);
-          gradY += s * quadToDofD(1, d, q);
-        }
-        const double O11 = oper(0, q, e);
-        const double O12 = oper(1, q, e);
-        const double O22 = oper(2, q, e);
-
-        s_gradX[q] = (O11 * gradX) + (O12 * gradY);
-        s_gradY[q] = (O12 * gradX) + (O22 * gradY);
-      }
-    }
-
-    for (int dOff = 0; dOff < M2_INNER_BATCH; ++dOff) {
-      for (int d = dOff; d < NUM_DOFS; d += M2_INNER_BATCH) {
-        double r_sol = 0;
-        for (int q = 0; q < NUM_QUAD; ++q) {
-          // FIXME: s_gradX and s_gradY are @shared used outside of @inner
-          r_sol += ((s_gradX[q] * quadToDofD(0, d, q)) +
-                    (s_gradY[q] * quadToDofD(1, d, q)));
-        }
-        solOut(d, e) += r_sol;
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        const double *quadWeights,
-                        const Jacobian3D_t J,
-                        COEFF_ARGS
-                        SymmOperator3D_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
-          for (int q = qOff; q < NUM_QUAD; q += A3_QUAD_BATCH) {
-            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-            const double c_detJ = quadWeights[q] * COEFF / detJ;
-
-            // adj(J)
-            const double A11 = (J22 * J33) - (J23 * J32);
-            const double A12 = (J23 * J31) - (J21 * J33);
-            const double A13 = (J21 * J32) - (J22 * J31);
-
-            const double A21 = (J13 * J32) - (J12 * J33);
-            const double A22 = (J11 * J33) - (J13 * J31);
-            const double A23 = (J12 * J31) - (J11 * J32);
-
-            const double A31 = (J12 * J23) - (J13 * J22);
-            const double A32 = (J13 * J21) - (J11 * J23);
-            const double A33 = (J11 * J22) - (J12 * J21);
-
-            // adj(J)^Tadj(J)
-            oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
-            oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2) + (2,1)
-            oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3) + (3,1)
-            oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
-            oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3) + (3,2)
-            oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
-          }
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuadD3D_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDofD3D_t restrict quadToDofD,
-                       const SymmOperator3D_t restrict oper,
-                       const DLocal_t restrict solIn,
-                       DLocal_t restrict solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_gradX[NUM_QUAD];
-    @shared double s_gradY[NUM_QUAD];
-    @shared double s_gradZ[NUM_QUAD];
-
-    for (int qOff = 0; qOff < M3_INNER_BATCH; ++qOff; @inner) {
-      for (int q = qOff; q < NUM_QUAD; q += M3_INNER_BATCH) {
-        double gradX = 0, gradY = 0, gradZ = 0;
-        for (int d = 0; d < NUM_DOFS; ++d) {
-          const double s = solIn(d, e);
-          gradX += s * quadToDofD(0, d, q);
-          gradY += s * quadToDofD(1, d, q);
-          gradZ += s * quadToDofD(2, d, q);
-        }
-        const double O11 = oper(0, q, e);
-        const double O12 = oper(1, q, e);
-        const double O13 = oper(2, q, e);
-        const double O22 = oper(3, q, e);
-        const double O23 = oper(4, q, e);
-        const double O33 = oper(5, q, e);
-
-        s_gradX[q] = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
-        s_gradY[q] = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
-        s_gradZ[q] = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
-      }
-    }
-
-    for (int dOff = 0; dOff < M3_INNER_BATCH; ++dOff) {
-      for (int d = dOff; d < NUM_DOFS; d += M3_INNER_BATCH) {
-        double r_sol = 0;
-        for (int q = 0; q < NUM_QUAD; ++q) {
-          r_sol += ((s_gradX[q] * quadToDofD(0, d, q)) +
-                    (s_gradY[q] * quadToDofD(1, d, q)) +
-                    (s_gradZ[q] * quadToDofD(2, d, q)));
-        }
-        solOut(d, e) += r_sol;
-      }
-    }
-  }
-}
-//======================================

backends/occa/diffusion/tensor/cpu.okl

@@ -1,370 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 1D ]-----------------------------
-@kernel void Assemble1D(const int numElements,
-                        const double * restrict quadWeights,
-                        const Jacobian1D_t restrict J,
-                        COEFF_ARGS
-                        SymmOperator1D_t restrict oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
-      oper(q, e) = quadWeights[q] * COEFF / J(q, e);
-    }
-  }
-}
-
-@kernel void MultAdd1D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuad_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDof_t restrict quadToDofD,
-                       const SymmOperator1D_t restrict oper,
-                       const DLocal1D_t restrict solIn,
-                       DLocal1D_t restrict solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double grad[NUM_QUAD_1D];
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        grad[qx] = 0;
-      }
-      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-        const double s = solIn(dx, e);
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          grad[qx] += s * dofToQuadD(qx, dx);
-        }
-      }
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        grad[qx] *= oper(qx, e);
-      }
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        const double gradX = grad[qx];
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          solOut(dx, e) += gradX * quadToDofD(dx, qx);
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        const double * restrict quadWeights,
-                        const Jacobian2D_t restrict J,
-                        COEFF_ARGS
-                        SymmOperator2D_t restrict oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_2D; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-
-      const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
-
-      oper(0, q, e) =  c_detJ * (J21*J21 + J22*J22); // (1,1)
-      oper(1, q, e) = -c_detJ * (J21*J11 + J22*J12); // (1,2), (2,1)
-      oper(2, q, e) =  c_detJ * (J11*J11 + J12*J12); // (2,2)
-    }
-  }
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuad_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDof_t restrict quadToDofD,
-                       const SymmOperator2D_t restrict oper,
-                       const DLocal2D_t restrict solIn,
-                       DLocal2D_t restrict solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double grad[NUM_QUAD_1D][NUM_QUAD_1D][2];
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          grad[qy][qx][0] = 0;
-          grad[qy][qx][1] = 0;
-        }
-      }
-
-      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-        double gradX[NUM_QUAD_1D][2];
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          gradX[qx][0] = 0;
-          gradX[qx][1] = 0;
-        }
-
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          const double s = solIn(dx, dy, e);
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            gradX[qx][0] += s * dofToQuad(qx, dx);
-            gradX[qx][1] += s * dofToQuadD(qx, dx);
-          }
-        }
-
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          const double wy  = dofToQuad(qy, dy);
-          const double wDy = dofToQuadD(qy, dy);
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            grad[qy][qx][0] += gradX[qx][1] * wy;
-            grad[qy][qx][1] += gradX[qx][0] * wDy;
-          }
-        }
-      }
-
-      // Calculate Dxy, xDy in plane
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          const int q = QUAD_2D_ID(qx, qy);
-          const double O11 = oper(0, q, e);
-          const double O12 = oper(1, q, e);
-          const double O22 = oper(2, q, e);
-
-          const double gradX = grad[qy][qx][0];
-          const double gradY = grad[qy][qx][1];
-
-          grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
-          grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
-        }
-      }
-
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        double gradX[NUM_DOFS_1D][2];
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          gradX[dx][0] = 0;
-          gradX[dx][1] = 0;
-        }
-
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          const double gX = grad[qy][qx][0];
-          const double gY = grad[qy][qx][1];
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            const double wx  = quadToDof(dx, qx);
-            const double wDx = quadToDofD(dx, qx);
-            gradX[dx][0] += gX * wDx;
-            gradX[dx][1] += gY * wx;
-          }
-        }
-
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          const double wy  = quadToDof(dy, qy);
-          const double wDy = quadToDofD(dy, qy);
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            solOut(dx, dy, e) += ((gradX[dx][0] * wy) +
-                                  (gradX[dx][1] * wDy));
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        const double * restrict quadWeights,
-                        const Jacobian3D_t restrict J,
-                        COEFF_ARGS
-                        SymmOperator3D_t restrict oper) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int q = 0; q < NUM_QUAD_3D; ++q; @inner) {
-      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-      const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                           (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-      const double c_detJ = quadWeights[q] * COEFF / detJ;
-
-      // adj(J)
-      const double A11 = (J22 * J33) - (J23 * J32);
-      const double A12 = (J23 * J31) - (J21 * J33);
-      const double A13 = (J21 * J32) - (J22 * J31);
-
-      const double A21 = (J13 * J32) - (J12 * J33);
-      const double A22 = (J11 * J33) - (J13 * J31);
-      const double A23 = (J12 * J31) - (J11 * J32);
-
-      const double A31 = (J12 * J23) - (J13 * J22);
-      const double A32 = (J13 * J21) - (J11 * J23);
-      const double A33 = (J11 * J22) - (J12 * J21);
-
-      // adj(J)^Tadj(J)
-      oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
-      oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2), (2,1)
-      oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3), (3,1)
-      oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
-      oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3), (3,2)
-      oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuad_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDof_t restrict quadToDofD,
-                       const SymmOperator3D_t restrict oper,
-                       const DLocal3D_t restrict solIn,
-                       DLocal3D_t restrict solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double grad[NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D][4];
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            grad[qz][qy][qx][0] = 0;
-            grad[qz][qy][qx][1] = 0;
-            grad[qz][qy][qx][2] = 0;
-          }
-        }
-      }
-
-      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-        double gradXY[NUM_QUAD_1D][NUM_QUAD_1D][4];
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            gradXY[qy][qx][0] = 0;
-            gradXY[qy][qx][1] = 0;
-            gradXY[qy][qx][2] = 0;
-          }
-        }
-
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          double gradX[NUM_QUAD_1D][2];
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            gradX[qx][0] = 0;
-            gradX[qx][1] = 0;
-          }
-
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            const double s = solIn(dx, dy, dz, e);
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              gradX[qx][0] += s * dofToQuad(qx, dx);
-              gradX[qx][1] += s * dofToQuadD(qx, dx);
-            }
-          }
-
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            const double wy  = dofToQuad(qy, dy);
-            const double wDy = dofToQuadD(qy, dy);
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              const double wx  = gradX[qx][0];
-              const double wDx = gradX[qx][1];
-              gradXY[qy][qx][0] += wDx * wy;
-              gradXY[qy][qx][1] += wx  * wDy;
-              gradXY[qy][qx][2] += wx  * wy;
-            }
-          }
-        }
-
-        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-          const double wz  = dofToQuad(qz, dz);
-          const double wDz = dofToQuadD(qz, dz);
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              grad[qz][qy][qx][0] += gradXY[qy][qx][0] * wz;
-              grad[qz][qy][qx][1] += gradXY[qy][qx][1] * wz;
-              grad[qz][qy][qx][2] += gradXY[qy][qx][2] * wDz;
-            }
-          }
-        }
-      }
-
-      // Calculate Dxyz, xDyz, xyDz in plane
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            const int q = QUAD_3D_ID(qx, qy, qz);
-            const double O11 = oper(0, q, e);
-            const double O12 = oper(1, q, e);
-            const double O13 = oper(2, q, e);
-            const double O22 = oper(3, q, e);
-            const double O23 = oper(4, q, e);
-            const double O33 = oper(5, q, e);
-
-            const double gradX = grad[qz][qy][qx][0];
-            const double gradY = grad[qz][qy][qx][1];
-            const double gradZ = grad[qz][qy][qx][2];
-
-            grad[qz][qy][qx][0] = (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
-            grad[qz][qy][qx][1] = (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
-            grad[qz][qy][qx][2] = (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
-          }
-        }
-      }
-
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        double gradXY[NUM_DOFS_1D][NUM_DOFS_1D][4];
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            gradXY[dy][dx][0] = 0;
-            gradXY[dy][dx][1] = 0;
-            gradXY[dy][dx][2] = 0;
-          }
-        }
-
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          double gradX[NUM_DOFS_1D][4];
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            gradX[dx][0] = 0;
-            gradX[dx][1] = 0;
-            gradX[dx][2] = 0;
-          }
-
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            const double gX = grad[qz][qy][qx][0];
-            const double gY = grad[qz][qy][qx][1];
-            const double gZ = grad[qz][qy][qx][2];
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              const double wx  = quadToDof(dx, qx);
-              const double wDx = quadToDofD(dx, qx);
-              gradX[dx][0] += gX * wDx;
-              gradX[dx][1] += gY * wx;
-              gradX[dx][2] += gZ * wx;
-            }
-          }
-
-          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-            const double wy  = quadToDof(dy, qy);
-            const double wDy = quadToDofD(dy, qy);
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              gradXY[dy][dx][0] += gradX[dx][0] * wy;
-              gradXY[dy][dx][1] += gradX[dx][1] * wDy;
-              gradXY[dy][dx][2] += gradX[dx][2] * wy;
-            }
-          }
-        }
-
-        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-          const double wz  = quadToDof(dz, qz);
-          const double wDz = quadToDofD(dz, qz);
-          for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-            for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-              solOut(dx, dy, dz, e) += ((gradXY[dy][dx][0] * wz) +
-                                        (gradXY[dy][dx][1] * wz) +
-                                        (gradXY[dy][dx][2] * wDz));
-            }
-          }
-        }
-      }
-    }
-  }
-}
-//======================================

backends/occa/diffusion/tensor/gpuHighOrder.okl

@@ -1,433 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 1D ]-----------------------------
-@kernel void Assemble1D(const int numElements,
-                        const double *quadWeights,
-                        const Jacobian1D_t J,
-                        COEFF_ARGS
-                        SymmOperator1D_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A1_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A1_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int q = 0; q < NUM_QUAD_1D; ++q; @inner) {
-          oper(q, e) = quadWeights[q] * COEFF / J(q, e);
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd1D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuad_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDof_t restrict quadToDofD,
-                       const SymmOperator1D_t restrict oper,
-                       const DLocal1D_t restrict solIn,
-                       DLocal1D_t restrict solOut) {
-  // Iterate over elements
-  for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
-    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-
-    @exclusive double grad[NUM_QUAD_1D];
-
-    for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
-      for (int i = el; i < NUM_QUAD_DOFS_1D; i += M1_INNER_ELEMENT_BATCH) {
-        s_dofToQuadD[i] = dofToQuadD[i];
-        s_quadToDofD[i] = quadToDofD[i];
-      }
-    }
-
-    for (int b = 0; b < M1_OUTER_ELEMENT_BATCH; ++b) {
-      for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
-        const int e = eOff + b*M1_INNER_ELEMENT_BATCH + el;
-        if (e < numElements) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            grad[qx] = 0;
-          }
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            const double s = solIn(dx, e);
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              grad[qx] += s * s_dofToQuadD(qx, dx);
-            }
-          }
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            grad[qx] *= oper(qx, e);
-          }
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            double s = 0;
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              s += grad[qx] * s_quadToDofD(dx, qx);
-            }
-            solOut(dx, e) += s;
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 2D ]-----------------------------
-@kernel void Assemble2D(const int numElements,
-                        const double *quadWeights,
-                        const Jacobian2D_t J,
-                        COEFF_ARGS
-                        SymmOperator2D_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A2_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A2_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int qOff = 0; qOff < A2_QUAD_BATCH; ++qOff; @inner) {
-          for (int q = qOff; q < NUM_QUAD_2D; q += A2_QUAD_BATCH) {
-            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
-            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
-
-            const double c_detJ = quadWeights[q] * COEFF / ((J11 * J22) - (J21 * J12));
-
-            oper(0, q, e) =  c_detJ * (J21*J21 + J22*J22); // (1,1)
-            oper(1, q, e) = -c_detJ * (J21*J11 + J22*J12); // (1,2), (2,1)
-            oper(2, q, e) =  c_detJ * (J11*J11 + J12*J12); // (2,2)
-          }
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd2D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuad_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDof_t restrict quadToDofD,
-                       const SymmOperator2D_t restrict oper,
-                       const DLocal2D_t restrict solIn,
-                       DLocal2D_t restrict solOut) {
-  // Iterate over elements
-  for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
-    // Store dof <--> quad mappings
-    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-
-    // Store xy planes in shared memory
-    @shared double s_xy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-    @shared double s_xDy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-    @shared double s_grad[2 * NUM_QUAD_2D] @dim(2, NUM_QUAD_1D, NUM_QUAD_1D);
-
-    @exclusive double r_x[NUM_MAX_1D];
-    @exclusive double r_y[NUM_QUAD_1D];
-
-    for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
-      for (int id = x; id < NUM_QUAD_DOFS_1D; id += NUM_MAX_1D) {
-        s_dofToQuad[id]  = dofToQuad[id];
-        s_dofToQuadD[id] = dofToQuadD[id];
-        s_quadToDof[id]  = quadToDof[id];
-        s_quadToDofD[id] = quadToDofD[id];
-      }
-    }
-
-    for (int e = eOff; e < (eOff + M2_ELEMENT_BATCH); ++e) {
-      if (e < numElements) {
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx; @inner) {
-          if (dx < NUM_DOFS_1D) {
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              s_xy(dx, qy) = 0;
-              s_xDy(dx, qy) = 0;
-            }
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              r_x[dy] = solIn(dx, dy, e);
-            }
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              double xy = 0;
-              double xDy = 0;
-              for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-                xy  += r_x[dy] * s_dofToQuad(qy, dy);
-                xDy += r_x[dy] * s_dofToQuadD(qy, dy);
-              }
-              s_xy(dx, qy)  = xy;
-              s_xDy(dx, qy) = xDy;
-            }
-          }
-        }
-        for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
-          if (qy < NUM_QUAD_1D) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              double gradX = 0, gradY = 0;
-              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-                gradX += s_xy(dx, qy)  * s_dofToQuadD(qx, dx);
-                gradY += s_xDy(dx, qy) * s_dofToQuad(qx, dx);
-              }
-
-              const int q = QUAD_2D_ID(qx, qy);
-              const double O11 = oper(0, q, e);
-              const double O12 = oper(1, q, e);
-              const double O22 = oper(2, q, e);
-
-              s_grad(0, qx, qy) = (O11 * gradX) + (O12 * gradY);
-              s_grad(1, qx, qy) = (O12 * gradX) + (O22 * gradY);
-            }
-          }
-        }
-
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx; @inner) {
-          if (qx < NUM_QUAD_1D) {
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              s_xy(dy, qx) = 0;
-              s_xDy(dy, qx) = 0;
-            }
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              r_x[qy] = s_grad(0, qx, qy);
-              r_y[qy] = s_grad(1, qx, qy);
-            }
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              double xy  = 0;
-              double xDy = 0;
-              for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-                xy  += r_x[qy] * s_quadToDof(dy, qy);
-                xDy += r_y[qy] * s_quadToDofD(dy, qy);
-              }
-              s_xy(dy, qx) = xy;
-              s_xDy(dy, qx) = xDy;
-            }
-          }
-        }
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx; @inner) {
-          if (dx < NUM_DOFS_1D) {
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              double s = 0;
-              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-                s += ((s_xy(dy, qx) * s_quadToDofD(dx, qx)) +
-                      (s_xDy(dy, qx) * s_quadToDof(dx, qx)));
-              }
-              solOut(dx, dy, e) += s;
-            }
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void Assemble3D(const int numElements,
-                        const double *quadWeights,
-                        const Jacobian3D_t J,
-                        COEFF_ARGS
-                        SymmOperator3D_t oper) {
-  for (int eOff = 0; eOff < numElements; eOff += A3_ELEMENT_BATCH; @outer) {
-    for (int e = eOff; e < (eOff + A3_ELEMENT_BATCH); ++e; @inner) {
-      if (e < numElements) {
-        for (int qOff = 0; qOff < A3_QUAD_BATCH; ++qOff; @inner) {
-          for (int q = qOff; q < NUM_QUAD_3D; q += A3_QUAD_BATCH) {
-            const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
-            const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
-            const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
-
-            const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                                 (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-
-            const double c_detJ = quadWeights[q] * COEFF / detJ;
-
-            // adj(J)
-            const double A11 = (J22 * J33) - (J23 * J32);
-            const double A12 = (J23 * J31) - (J21 * J33);
-            const double A13 = (J21 * J32) - (J22 * J31);
-
-            const double A21 = (J13 * J32) - (J12 * J33);
-            const double A22 = (J11 * J33) - (J13 * J31);
-            const double A23 = (J12 * J31) - (J11 * J32);
-
-            const double A31 = (J12 * J23) - (J13 * J22);
-            const double A32 = (J13 * J21) - (J11 * J23);
-            const double A33 = (J11 * J22) - (J12 * J21);
-
-            // adj(J)^Tadj(J)
-            oper(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
-            oper(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2), (2,1)
-            oper(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3), (3,1)
-            oper(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
-            oper(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3), (3,2)
-            oper(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
-          }
-        }
-      }
-    }
-  }
-}
-
-@kernel void MultAdd3D(const int numElements,
-                       const DofToQuad_t restrict dofToQuad,
-                       const DofToQuad_t restrict dofToQuadD,
-                       const QuadToDof_t restrict quadToDof,
-                       const QuadToDof_t restrict quadToDofD,
-                       const SymmOperator3D_t restrict oper,
-                       const DLocal3D_t restrict solIn,
-                       DLocal3D_t restrict solOut) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    // Store dof <--> quad mappings
-    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @shared double s_dofToQuadD[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @shared double s_quadToDof[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-    @shared double s_quadToDofD[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-
-    // Store xy planes in shared memory
-    @shared double s_z[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
-    @shared double s_Dz[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
-    @shared double s_xyDz[NUM_QUAD_2D] @dim(NUM_QUAD_1D, NUM_QUAD_1D);
-
-    // Store z axis as registers
-    @exclusive double r_qz[NUM_QUAD_1D];
-    @exclusive double r_qDz[NUM_QUAD_1D];
-    @exclusive double r_dDxyz[NUM_DOFS_1D];
-    @exclusive double r_dxDyz[NUM_DOFS_1D];
-    @exclusive double r_dxyDz[NUM_DOFS_1D];
-
-    for (int y = 0; y < NUM_MAX_1D; ++y; @inner) {
-      for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
-        const int id = (y * NUM_MAX_1D) + x;
-        // Fetch Q <--> D maps
-        if (id < NUM_QUAD_DOFS_1D) {
-          s_dofToQuad[id]  = dofToQuad[id];
-          s_dofToQuadD[id] = dofToQuadD[id];
-          s_quadToDof[id]  = quadToDof[id];
-          s_quadToDofD[id] = quadToDofD[id];
-        }
-        // Initialize our Z axis
-        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-          r_qz[qz] = 0;
-          r_qDz[qz] = 0;
-        }
-        // Initialize our solution updates in the Z axis
-        for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-          r_dDxyz[dz] = 0;
-          r_dxDyz[dz] = 0;
-          r_dxyDz[dz] = 0;
-        }
-      }
-    }
-
-    for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
-      for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-        if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
-          for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-            const double s = solIn(dx, dy, dz, e);
-            // Calculate D -> Q in the Z axis
-            for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-              r_qz[qz]  += s * s_dofToQuad(qz, dz);
-              r_qDz[qz] += s * s_dofToQuadD(qz, dz);
-            }
-          }
-        }
-      }
-    }
-    // For each xy plane
-    for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-      // Fill xy plane at given z position
-      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
-        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
-            s_z(dx, dy)  = r_qz[qz];
-            s_Dz(dx, dy) = r_qDz[qz];
-          }
-        }
-      }
-      // Calculate Dxyz, xDyz, xyDz in plane
-      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
-        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
-          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
-            double Dxyz = 0;
-            double xDyz = 0;
-            double xyDz = 0;
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              const double wy  = s_dofToQuad(qy, dy);
-              const double wDy = s_dofToQuadD(qy, dy);
-              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-                const double wx  = s_dofToQuad(qx, dx);
-                const double wDx = s_dofToQuadD(qx, dx);
-                const double z  = s_z(dx, dy);
-                const double Dz = s_Dz(dx, dy);
-                Dxyz += wDx * wy  * z;
-                xDyz += wx  * wDy * z;
-                xyDz += wx  * wy  * Dz;
-              }
-            }
-
-            const int q = QUAD_3D_ID(qx, qy, qz);
-            const double O11 = oper(0, q, e);
-            const double O12 = oper(1, q, e);
-            const double O13 = oper(2, q, e);
-            const double O22 = oper(3, q, e);
-            const double O23 = oper(4, q, e);
-            const double O33 = oper(5, q, e);
-
-            const double qDxyz = (O11 * Dxyz) + (O12 * xDyz) + (O13 * xyDz);
-            const double qxDyz = (O12 * Dxyz) + (O22 * xDyz) + (O23 * xyDz);
-            const double qxyDz = (O13 * Dxyz) + (O23 * xDyz) + (O33 * xyDz);
-
-            for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-              const double wz  = s_quadToDof(dz, qz);
-              const double wDz = s_quadToDofD(dz, qz);
-              r_dDxyz[dz] += wz  * qDxyz;
-              r_dxDyz[dz] += wz  * qxDyz;
-              r_dxyDz[dz] += wDz * qxyDz;
-            }
-          }
-        }
-      }
-    }
-    // Iterate over xy planes to compute solution
-    for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-      // Place xy plane in shared memory
-      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
-        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
-          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
-            s_z(qx, qy)    = r_dDxyz[dz];
-            s_Dz(qx, qy)   = r_dxDyz[dz];
-            s_xyDz(qx, qy) = r_dxyDz[dz];
-          }
-        }
-      }
-      // Finalize solution in xy plane
-      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
-        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
-            double solZ = 0;
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              const double wy  = s_quadToDof(dy, qy);
-              const double wDy = s_quadToDofD(dy, qy);
-              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-                const double wx  = s_quadToDof(dx, qx);
-                const double wDx = s_quadToDofD(dx, qx);
-                const double Dxyz = s_z(qx, qy);
-                const double xDyz = s_Dz(qx, qy);
-                const double xyDz = s_xyDz(qx, qy);
-                solZ += ((wDx * wy  * Dxyz) +
-                         (wx  * wDy * xDyz) +
-                         (wx  * wy  * xyDz));
-              }
-            }
-            solOut(dx, dy, dz, e) += solZ;
-          }
-        }
-      }
-    }
-  }
-}
-//======================================

backends/occa/engine.cpp

@@ -1,140 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "backend.hpp"
-#include "url_handler.hpp"
-#include "bilinearform.hpp"
-#include "../../general/array.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-bool Engine::fileOpenerRegistered = false;
-
-void Engine::Init(const std::string &engine_spec)
-{
-   //
-   // Initialize inherited fields
-   //
-   memory_resources[0] = NULL;
-   workers_weights[0]= 1.0;
-   workers_mem_res[0] = 0;
-
-   //
-   // Initialize the OCCA engine
-   //
-   ::occa::properties props(engine_spec);
-   device = new ::occa::device[1];
-   device[0].setup(props);
-
-   okl_path = "mfem-occa://";
-   // okl_defines = "...";
-   if (!fileOpenerRegistered)
-   {
-      // The directories from "MFEM_OCCA_OKL_PATH", if any, have the highest
-      // priority.
-      FileOpener *fo = new FileOpener("mfem-occa://", "MFEM_OCCA_OKL_PATH");
-      // Next in priority is the source path, if it exists.
-      std::string mfem_src_prefix = mfem::GetSourcePath();
-      fo->AddDir(mfem_src_prefix + "/backends/occa");
-      // And last in priority is the install path, if it exists.
-      std::string mfem_install_prefix = mfem::GetInstallPath();
-      fo->AddDir(mfem_install_prefix + "/lib/mfem/occa");
-      ::occa::io::fileOpener::add(fo);
-      fileOpenerRegistered = true;
-   }
-}
-
-Engine::Engine(const std::string &engine_spec)
-   : mfem::Engine(NULL, 1, 1)
-{
-   Init(engine_spec);
-}
-
-#ifdef MFEM_USE_MPI
-Engine::Engine(MPI_Comm _comm, const std::string &engine_spec)
-   : mfem::Engine(NULL, 1, 1)
-{
-   comm = _comm;
-   Init(engine_spec);
-}
-#endif
-
-DLayout Engine::MakeLayout(std::size_t size) const
-{
-   return DLayout(new Layout(*this, size));
-}
-
-DLayout Engine::MakeLayout(const mfem::Array<std::size_t> &offsets) const
-{
-   MFEM_ASSERT(offsets.Size() == 2,
-               "multiple workers are not supported yet");
-   return DLayout(new Layout(*this, offsets.Last()));
-}
-
-DArray Engine::MakeArray(PLayout &layout, std::size_t item_size) const
-{
-   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
-               "invalid input layout");
-   Layout *lt = static_cast<Layout *>(&layout);
-   return DArray(new Array(*lt, item_size));
-}
-
-DVector Engine::MakeVector(PLayout &layout, int type_id) const
-{
-   MFEM_ASSERT(type_id == ScalarId<double>::value, "invalid type_id");
-   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
-               "invalid input layout");
-   Layout *lt = static_cast<Layout *>(&layout);
-   return DVector(new Vector(*lt));
-}
-
-DFiniteElementSpace Engine::MakeFESpace(mfem::FiniteElementSpace &fespace) const
-{
-   return DFiniteElementSpace(new FiniteElementSpace(*this, fespace));
-}
-
-DBilinearForm Engine::MakeBilinearForm(mfem::BilinearForm &bf) const
-{
-   return DBilinearForm(new BilinearForm(*this, bf));
-}
-
-void Engine::AssembleLinearForm(LinearForm &l_form) const
-{
-   /// FIXME - What will the actual parameters be?
-   MFEM_ABORT("FIXME");
-}
-
-mfem::Operator *Engine::MakeOperator(const MixedBilinearForm &mbl_form) const
-{
-   /// FIXME - What will the actual parameters be?
-   MFEM_ABORT("FIXME");
-   return NULL;
-}
-
-mfem::Operator *Engine::MakeOperator(const NonlinearForm &nl_form) const
-{
-   /// FIXME - What will the actual parameters be?
-   MFEM_ABORT("FIXME");
-   return NULL;
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/engine.hpp

@@ -1,111 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_ENGINE_HPP
-#define MFEM_BACKENDS_OCCA_ENGINE_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "../base/backend.hpp"
-#include <occa.hpp>
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class Engine : public mfem::Engine
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // mfem::Backend *backend;
-#ifdef MFEM_USE_MPI
-   // MPI_Comm comm;
-#endif
-   // int num_mem_res;
-   // int num_workers;
-   // MemoryResource **memory_resources;
-   // double *workers_weights;
-   // int *workers_mem_res;
-
-   static bool fileOpenerRegistered;
-   ::occa::device *device; // An array of OCCA devices
-   std::string okl_path, okl_defines;
-
-   void Init(const std::string &engine_spec);
-
-public:
-   Engine(const std::string &engine_spec);
-
-#ifdef MFEM_USE_MPI
-   Engine(MPI_Comm comm, const std::string &engine_spec);
-#endif
-
-   virtual ~Engine() { delete [] device; }
-
-   /**
-       @name OCCA specific interface, used by other objects in the OCCA backend
-    */
-   ///@{
-
-   ::occa::device GetDevice(int idx = 0) const { return device[idx]; }
-
-   /// TODO: doxygen
-   const std::string &GetOklPath() const { return okl_path; }
-
-   /// TODO: doxygen
-   const std::string &GetOklDefines() const { return okl_defines; }
-
-   ///@}
-   // End: OCCA specific interface
-
-   /**
-       @name Virtual interface: finite element data structures and algorithms
-    */
-   ///@{
-
-   virtual DLayout MakeLayout(std::size_t size) const;
-   virtual DLayout MakeLayout(const mfem::Array<std::size_t> &offsets) const;
-
-   virtual DArray MakeArray(PLayout &layout, std::size_t item_size) const;
-
-   virtual DVector MakeVector(PLayout &layout,
-                              int type_id = ScalarId<double>::value) const;
-
-   virtual DFiniteElementSpace MakeFESpace(mfem::FiniteElementSpace &
-                                           fespace) const;
-
-   virtual DBilinearForm MakeBilinearForm(mfem::BilinearForm &bf) const;
-
-   /// FIXME - What will the actual parameters be?
-   virtual void AssembleLinearForm(LinearForm &l_form) const;
-
-   /// FIXME - What will the actual parameters be?
-   virtual mfem::Operator *MakeOperator(const MixedBilinearForm &mbl_form) const;
-
-   /// FIXME - What will the actual parameters be?
-   virtual mfem::Operator *MakeOperator(const NonlinearForm &nl_form) const;
-
-   ///@}
-   // End: Virtual interface
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_ENGINE_HPP

backends/occa/fespace.cpp

@@ -1,174 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "backend.hpp"
-#include "fespace.hpp"
-#include "interpolation.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-FiniteElementSpace::FiniteElementSpace(const Engine &e,
-                                       mfem::FiniteElementSpace &fespace)
-   : PFiniteElementSpace(e, fespace),
-     e_layout(e, 0) // resized in SetupLocalGlobalMaps()
-{
-   vdim     = fespace.GetVDim();
-   ordering = fespace.GetOrdering();
-
-   SetupLocalGlobalMaps();
-   SetupOperators();
-   SetupKernels();
-}
-
-FiniteElementSpace::~FiniteElementSpace()
-{
-   delete [] elementDofMap;
-   delete [] elementDofMapInverse;
-   delete restrictionOp;
-   delete prolongationOp;
-}
-
-void FiniteElementSpace::SetupLocalGlobalMaps()
-{
-   const mfem::FiniteElement &fe = *(fes->GetFE(0));
-   const mfem::TensorBasisElement *el =
-      dynamic_cast<const mfem::TensorBasisElement*>(&fe);
-
-   const mfem::Table &e2dTable = fes->GetElementToDofTable();
-   const int *elementMap = e2dTable.GetJ();
-   const int elements = fes->GetNE();
-
-   globalDofs = fes->GetNDofs();
-   localDofs  = fe.GetDof();
-
-   e_layout.Resize(localDofs * elements * fes->GetVDim());
-
-   elementDofMap        = new int[localDofs];
-   elementDofMapInverse = new int[localDofs];
-   if (el)
-   {
-      ::memcpy(elementDofMap,
-               el->GetDofMap().GetData(),
-               localDofs * sizeof(int));
-   }
-   else
-   {
-      for (int i = 0; i < localDofs; ++i)
-      {
-         elementDofMap[i] = i;
-      }
-   }
-   for (int i = 0; i < localDofs; ++i)
-   {
-      elementDofMapInverse[elementDofMap[i]] = i;
-   }
-
-   // Allocate device offsets and indices
-   globalToLocalOffsets.allocate(GetDevice(),
-                                 globalDofs + 1);
-   globalToLocalIndices.allocate(GetDevice(),
-                                 localDofs, elements);
-   localToGlobalMap.allocate(GetDevice(),
-                             localDofs, elements);
-
-   int *offsets = globalToLocalOffsets.ptr();
-   int *indices = globalToLocalIndices.ptr();
-   int *l2gMap  = localToGlobalMap.ptr();
-
-   // We'll be keeping a count of how many local nodes point
-   //   to its global dof
-   for (int i = 0; i <= globalDofs; ++i)
-   {
-      offsets[i] = 0;
-   }
-
-   for (int e = 0; e < elements; ++e)
-   {
-      for (int d = 0; d < localDofs; ++d)
-      {
-         const int gid = elementMap[localDofs*e + d];
-         ++offsets[gid + 1];
-      }
-   }
-   // Aggregate to find offsets for each global dof
-   for (int i = 1; i <= globalDofs; ++i)
-   {
-      offsets[i] += offsets[i - 1];
-   }
-   // For each global dof, fill in all local nodes that point
-   //   to it
-   for (int e = 0; e < elements; ++e)
-   {
-      for (int d = 0; d < localDofs; ++d)
-      {
-         const int gid = elementMap[localDofs*e + elementDofMap[d]];
-         const int lid = localDofs*e + d;
-         indices[offsets[gid]++] = lid;
-         l2gMap[lid] = gid;
-      }
-   }
-   // We shifted the offsets vector by 1 by using it
-   //   as a counter. Now we shift it back.
-   for (int i = globalDofs; i > 0; --i)
-   {
-      offsets[i] = offsets[i - 1];
-   }
-   offsets[0] = 0;
-
-   globalToLocalOffsets.keepInDevice();
-   globalToLocalIndices.keepInDevice();
-   localToGlobalMap.keepInDevice();
-}
-
-void FiniteElementSpace::SetupOperators()
-{
-   const mfem::SparseMatrix *R = fes->GetRestrictionMatrix();
-   const mfem::Operator *P = fes->GetProlongationMatrix();
-   CreateRPOperators(OccaVLayout(), OccaTrueVLayout(),
-                     R, P,
-                     restrictionOp,
-                     prolongationOp);
-}
-
-void FiniteElementSpace::SetupKernels()
-{
-   ::occa::properties props("defines: {"
-                            "  TILESIZE: 256,"
-                            "}");
-   props["defines/NUM_VDIM"] = vdim;
-
-   props["defines/ORDERING_BY_NODES"] = 0;
-   props["defines/ORDERING_BY_VDIM"]  = 1;
-   props["defines/VDIM_ORDERING"] = (int) (ordering == Ordering::byVDIM);
-
-   ::occa::device device = GetDevice();
-   const std::string &okl_path = OccaEngine().GetOklPath();
-   const std::string &okl_defines = OccaEngine().GetOklDefines();
-   globalToLocalKernel = device.buildKernel(okl_path + "fespace.okl",
-                                            "GlobalToLocal",
-                                            props + okl_defines);
-   localToGlobalKernel = device.buildKernel(okl_path + "fespace.okl",
-                                            "LocalToGlobal",
-                                            props + okl_defines);
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/fespace.hpp

@@ -1,146 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_FE_SPACE_HPP
-#define MFEM_BACKENDS_OCCA_FE_SPACE_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "engine.hpp"
-#include "operator.hpp"
-#include "../../fem/fem.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-/// TODO: doxygen
-class FiniteElementSpace : public mfem::PFiniteElementSpace
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // SharedPtr<const mfem::Engine> engine;
-   // mfem::FiniteElementSpace *fes;
-
-   Layout e_layout;
-
-   int *elementDofMap;
-   int *elementDofMapInverse;
-
-   ::occa::array<int> globalToLocalOffsets;
-   ::occa::array<int> globalToLocalIndices;
-   ::occa::array<int> localToGlobalMap;
-   ::occa::kernel globalToLocalKernel, localToGlobalKernel;
-
-   mfem::Ordering::Type ordering;
-
-   int globalDofs, localDofs;
-   int vdim;
-
-   mfem::Operator *restrictionOp, *prolongationOp;
-
-   void SetupLocalGlobalMaps();
-   void SetupOperators();
-   void SetupKernels();
-
-public:
-   /// TODO: doxygen
-   FiniteElementSpace(const Engine &e, mfem::FiniteElementSpace &fespace);
-
-   /// Virtual destructor
-   virtual ~FiniteElementSpace();
-
-   /// TODO: doxygen
-   const Engine &OccaEngine() const
-   { return *static_cast<const Engine *>(engine.Get()); }
-
-   /// TODO: doxygen
-   ::occa::device GetDevice(int idx = 0) const
-   { return OccaEngine().GetDevice(idx); }
-
-   mfem::Mesh* GetMesh() const { return fes->GetMesh(); }
-
-   Layout &OccaVLayout() const
-   { return *fes->GetVLayout().As<Layout>(); }
-
-   Layout &OccaTrueVLayout() const
-   { return *fes->GetTrueVLayout().As<Layout>(); }
-
-   Layout &OccaEVLayout() { return e_layout; }
-
-#ifdef MFEM_USE_MPI
-   bool isDistributed() const { return (OccaEngine().GetComm() != MPI_COMM_NULL); }
-#else
-   bool isDistributed() const { return false; }
-#endif
-
-   bool hasTensorBasis() const
-   { return dynamic_cast<const mfem::TensorBasisElement*>(fes->GetFE(0)); }
-
-   mfem::Ordering::Type GetOrdering() const { return ordering; }
-
-   int GetGlobalDofs() const { return globalDofs; }
-   int GetLocalDofs() const { return localDofs; }
-
-   int GetDim() const { return fes->GetMesh()->Dimension(); }
-   int GetVDim() const { return vdim; }
-
-   int GetVSize() const { return globalDofs * vdim; }
-   int GetTrueVSize() const { return fes->GetTrueVSize(); }
-   int GetGlobalVSize() const { return globalDofs*vdim; /* FIXME: MPI */ }
-   int GetGlobalTrueVSize() const { return fes->GetTrueVSize(); }
-
-   int GetNE() const { return fes->GetNE(); }
-
-   const mfem::FiniteElementCollection* FEColl() const
-   { return fes->FEColl(); }
-   const mfem::FiniteElement* GetFE(const int idx) const
-   { return fes->GetFE(idx); }
-
-   const int* GetElementDofMap() const { return elementDofMap; }
-   const int* GetElementDofMapInverse() const { return elementDofMapInverse; }
-
-   const mfem::Operator* GetRestrictionOperator() { return restrictionOp; }
-   const mfem::Operator* GetProlongationOperator() { return prolongationOp; }
-
-   const ::occa::array<int> GetLocalToGlobalMap() const
-   { return localToGlobalMap; }
-
-   void GlobalToLocal(const Vector &globalVec, Vector &localVec) const
-   {
-      globalToLocalKernel(globalDofs,
-                          localDofs * fes->GetNE(),
-                          globalToLocalOffsets,
-                          globalToLocalIndices,
-                          globalVec.OccaMem(), localVec.OccaMem());
-   }
-   void LocalToGlobal(const Vector &localVec, Vector &globalVec) const
-   {
-      localToGlobalKernel(globalDofs,
-                          localDofs * fes->GetNE(),
-                          globalToLocalOffsets,
-                          globalToLocalIndices,
-                          localVec.OccaMem(), globalVec.OccaMem());
-   }
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_FE_SPACE_HPP

backends/occa/fespace.okl

@@ -1,67 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-/*
-  ---[ Defines Known At Compile-Time ]------------
-  TILESIZE : Tilesize for iterating over entries
-  ================================================
-*/
-
-#if VDIM_ORDERING == ORDERING_BY_VDIM
-typedef double *Global_t @dim(NUM_VDIM, globalEntries);
-typedef double *Local_t  @dim(NUM_VDIM, localEntries);
-#else
-typedef double *Global_t @dim(NUM_VDIM, globalEntries) @dimOrder(1, 0);
-typedef double *Local_t  @dim(NUM_VDIM, localEntries)  @dimOrder(1, 0);
-#endif
-
-@kernel void GlobalToLocal(const int globalEntries,
-                           const int localEntries,
-                           const int * restrict offsets,
-                           const int * restrict indices,
-                           const Global_t restrict globalX,
-                           Local_t restrict localX) {
-
-  for (int i = 0; i < globalEntries; ++i; @tile(TILESIZE, @outer, @inner)) {
-    if (i < globalEntries) {
-      const int offset = offsets[i];
-      const int nextOffset = offsets[i + 1];
-      for (int v = 0; v < NUM_VDIM; ++v) {
-        const double dofValue = globalX(v, i);
-        for (int j = offset; j < nextOffset; ++j) {
-          localX(v, indices[j]) = dofValue;
-        }
-      }
-    }
-  }
-}
-
-@kernel void LocalToGlobal(const int globalEntries,
-                           const int localEntries,
-                           const int * restrict offsets,
-                           const int * restrict indices,
-                           const Local_t restrict localX,
-                           Global_t restrict globalX) {
-
-  for (int i = 0; i < globalEntries; ++i; @tile(TILESIZE, @outer, @inner)) {
-    if (i < globalEntries) {
-      const int offset = offsets[i];
-      const int nextOffset = offsets[i + 1];
-      for (int v = 0; v < NUM_VDIM; ++v) {
-        double dofValue = 0;
-        for (int j = offset; j < nextOffset; ++j) {
-          dofValue += localX(v, indices[j]);
-        }
-        globalX(v, i) = dofValue;
-      }
-    }
-  }
-}

backends/occa/geometry.okl

@@ -1,181 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef STORE_JACOBIAN
-#  define STORE_JACOBIAN 1
-#endif
-
-#ifndef STORE_JACOBIAN_INV
-#  define STORE_JACOBIAN_INV 1
-#endif
-
-#ifndef STORE_JACOBIAN_DET
-#  define STORE_JACOBIAN_DET 1
-#endif
-
-typedef double* Local1D_t @dim(1, NUM_DOFS, numElements);
-typedef double* Local2D_t @dim(2, NUM_DOFS, numElements);
-typedef double* Local3D_t @dim(3, NUM_DOFS, numElements);
-
-typedef double* QLocal_t @dim(NUM_QUAD, numElements);
-
-typedef double* DofToQuadD1D_t @dim(NUM_QUAD, NUM_DOFS);
-typedef double* DofToQuadD2D_t @dim(2, NUM_QUAD, NUM_DOFS);
-typedef double* DofToQuadD3D_t @dim(3, NUM_QUAD, NUM_DOFS);
-
-typedef double* Jacobian1D_t @dim(NUM_QUAD, numElements);
-typedef double* Jacobian2D_t @dim(2, 2, NUM_QUAD, numElements);
-typedef double* Jacobian3D_t @dim(3, 3, NUM_QUAD, numElements);
-
-@kernel void InitGeometryInfo1D(const int numElements,
-                                const DofToQuadD1D_t restrict dofToQuadD,
-                                const Local1D_t restrict nodes,
-                                Jacobian1D_t restrict J,
-                                Jacobian1D_t restrict invJ,
-                                QLocal_t restrict detJ) {
-
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_nodes[NUM_DOFS];
-
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
-        s_nodes[d] = nodes(0, d, e);
-      }
-    }
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      double J11 = 0;
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        const double wx = dofToQuadD(q, d);
-        J11 += wx * s_nodes[d];
-      }
-#if STORE_JACOBIAN
-      J(q, e) = J11;
-#endif
-#if STORE_JACOBIAN_INV
-      invJ(q, e) = 1.0 / J11;
-#endif
-#if STORE_JACOBIAN_DET
-      detJ(q, e) = J11;
-#endif
-    }
-  }
-}
-
-@kernel void InitGeometryInfo2D(const int numElements,
-                                const DofToQuadD2D_t restrict dofToQuadD,
-                                const Local2D_t restrict nodes,
-                                Jacobian2D_t restrict J,
-                                Jacobian2D_t restrict invJ,
-                                QLocal_t restrict detJ) {
-
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_nodes[2 * NUM_DOFS] @dim(2, NUM_DOFS);
-
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
-        s_nodes(0, d) = nodes(0, d, e);
-        s_nodes(1, d) = nodes(1, d, e);
-      }
-    }
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      double J11 = 0, J12 = 0;
-      double J21 = 0, J22 = 0;
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        const double wx = dofToQuadD(0, q, d);
-        const double wy = dofToQuadD(1, q, d);
-        const double x = s_nodes(0, d);
-        const double y = s_nodes(1, d);
-        J11 += (wx * x); J12 += (wx * y);
-        J21 += (wy * x); J22 += (wy * y);
-      }
-#if STORE_JACOBIAN_INV || STORE_JACOBIAN_DET
-      const double r_detJ = (J11 * J22) - (J12 * J21);
-#endif
-#if STORE_JACOBIAN
-      J(0, 0, q, e) = J11; J(1, 0, q, e) = J12;
-      J(0, 1, q, e) = J21; J(1, 1, q, e) = J22;
-#endif
-#if STORE_JACOBIAN_INV
-      const double r_idetJ = 1.0 / r_detJ;
-      invJ(0, 0, q, e) =  J22 * r_idetJ;
-      invJ(1, 0, q, e) = -J12 * r_idetJ;
-
-      invJ(0, 1, q, e) = -J21 * r_idetJ;
-      invJ(1, 1, q, e) =  J11 * r_idetJ;
-#endif
-#if STORE_JACOBIAN_DET
-      detJ(q, e) = r_detJ;
-#endif
-    }
-  }
-}
-
-@kernel void InitGeometryInfo3D(const int numElements,
-                                const DofToQuadD3D_t restrict dofToQuadD,
-                                const Local3D_t restrict nodes,
-                                Jacobian3D_t restrict J,
-                                Jacobian3D_t restrict invJ,
-                                QLocal_t restrict detJ) {
-
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_nodes[3 * NUM_DOFS] @dim(3, NUM_DOFS);
-
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      for (int d = q; d < NUM_DOFS; d += NUM_QUAD) {
-        s_nodes(0, d) = nodes(0, d, e);
-        s_nodes(1, d) = nodes(1, d, e);
-        s_nodes(2, d) = nodes(2, d, e);
-      }
-    }
-    for (int q = 0; q < NUM_QUAD; ++q; @inner) {
-      double J11 = 0, J12 = 0, J13 = 0;
-      double J21 = 0, J22 = 0, J23 = 0;
-      double J31 = 0, J32 = 0, J33 = 0;
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        const double wx = dofToQuadD(0, q, d);
-        const double wy = dofToQuadD(1, q, d);
-        const double wz = dofToQuadD(2, q, d);
-        const double x = s_nodes(0, d);
-        const double y = s_nodes(1, d);
-        const double z = s_nodes(2, d);
-        J11 += (wx * x); J12 += (wx * y); J13 += (wx * z);
-        J21 += (wy * x); J22 += (wy * y); J23 += (wy * z);
-        J31 += (wz * x); J32 += (wz * y); J33 += (wz * z);
-      }
-#if STORE_JACOBIAN_INV || STORE_JACOBIAN_DET
-      const double r_detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
-                             (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
-#endif
-#if STORE_JACOBIAN
-      J(0, 0, q, e) = J11; J(1, 0, q, e) = J12; J(2, 0, q, e) = J13;
-      J(0, 1, q, e) = J21; J(1, 1, q, e) = J22; J(2, 1, q, e) = J23;
-      J(0, 2, q, e) = J31; J(1, 2, q, e) = J32; J(2, 2, q, e) = J33;
-#endif
-#if STORE_JACOBIAN_INV
-      const double r_idetJ = 1.0 / r_detJ;
-      invJ(0, 0, q, e) = r_idetJ * ((J22 * J33) - (J23 * J32));
-      invJ(1, 0, q, e) = r_idetJ * ((J32 * J13) - (J33 * J12));
-      invJ(2, 0, q, e) = r_idetJ * ((J12 * J23) - (J13 * J22));
-
-      invJ(0, 1, q, e) = r_idetJ * ((J23 * J31) - (J21 * J33));
-      invJ(1, 1, q, e) = r_idetJ * ((J33 * J11) - (J31 * J13));
-      invJ(2, 1, q, e) = r_idetJ * ((J13 * J21) - (J11 * J23));
-
-      invJ(0, 2, q, e) = r_idetJ * ((J21 * J32) - (J22 * J31));
-      invJ(1, 2, q, e) = r_idetJ * ((J31 * J12) - (J32 * J11));
-      invJ(2, 2, q, e) = r_idetJ * ((J11 * J22) - (J12 * J21));
-#endif
-#if STORE_JACOBIAN_DET
-      detJ(q, e) = r_detJ;
-#endif
-    }
-  }
-}

backends/occa/gridfunc.cpp

@@ -1,195 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "gridfunc.hpp"
-#include "bilininteg.hpp"
-#include "../../fem/gridfunc.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-std::map<std::string, ::occa::kernel> gridFunctionKernels;
-
-::occa::kernel GetGridFunctionKernel(::occa::device device,
-                                     FiniteElementSpace &fespace,
-                                     const mfem::IntegrationRule &ir)
-{
-   const int numQuad = ir.GetNPoints();
-
-   const FiniteElement &fe = *(fespace.GetFE(0));
-   const int dim  = fe.GetDim();
-   const int vdim = fespace.GetVDim();
-
-   std::stringstream ss;
-   ss << ::occa::hash(device)
-      << "FEColl : " << fespace.FEColl()->Name()
-      << "Quad: "    << numQuad
-      << "Dim: "     << dim
-      << "VDim: "    << vdim;
-   std::string hash = ss.str();
-
-   // Kernel defines
-   ::occa::properties props;
-   props["defines/NUM_VDIM"] = vdim;
-
-   SetProperties(fespace, ir, props);
-
-   ::occa::kernel kernel = gridFunctionKernels[hash];
-   if (!kernel.isInitialized())
-   {
-      const std::string &okl_path = fespace.OccaEngine().GetOklPath();
-      kernel = device.buildKernel(okl_path + "gridfunc.okl",
-                                  stringWithDim("GridFuncToQuad", dim),
-                                  props);
-   }
-   return kernel;
-}
-
-// OccaGridFunction::OccaGridFunction() :
-//    Vector(),
-//    ofespace(NULL),
-//    sequence(0) {}
-
-OccaGridFunction::OccaGridFunction(FiniteElementSpace *ofespace_)
-   : PArray(ofespace_->OccaVLayout()),
-     Array(ofespace_->OccaVLayout(), sizeof(double)),
-     Vector(ofespace_->OccaVLayout()),
-     ofespace(ofespace_),
-     sequence(0) {}
-
-// OccaGridFunction::OccaGridFunction(OccaFiniteElementSpace *ofespace_,
-//                                    OccaVectorRef ref) :
-//    OccaVector(ref),
-//    ofespace(ofespace_),
-//    sequence(0) {}
-
-OccaGridFunction::OccaGridFunction(const OccaGridFunction &v)
-   : PArray(v),
-     Array(v),
-     Vector(v),
-     ofespace(v.ofespace),
-     sequence(v.sequence) {}
-
-OccaGridFunction& OccaGridFunction::operator = (double value)
-{
-   Fill(value);
-   return *this;
-}
-
-OccaGridFunction& OccaGridFunction::operator = (const Vector &v)
-{
-   Assign<double>(v);
-   return *this;
-}
-
-// OccaGridFunction& OccaGridFunction::operator = (const OccaVectorRef &v)
-// {
-//    OccaVector::operator = (v);
-//    return *this;
-// }
-
-OccaGridFunction& OccaGridFunction::operator = (const OccaGridFunction &v)
-{
-   Assign<double>(v);
-   return *this;
-}
-
-// void OccaGridFunction::SetGridFunction(mfem::GridFunction &gf)
-// {
-//    Vector v = *this;
-//    gf.MakeRef(ofespace->GetFESpace(), v, 0);
-//    // Make gf the owner of the data
-//    v.Swap(gf);
-// }
-
-void OccaGridFunction::GetTrueDofs(Vector &v)
-{
-   const mfem::Operator *R = ofespace->GetRestrictionOperator();
-   if (!R)
-   {
-      v.MakeRef(*this);
-   }
-   else
-   {
-      v.Resize<double>(R->OutLayout(), NULL);
-      mfem::Vector mfem_v(v);
-      R->Mult(this->Wrap(), mfem_v);
-   }
-}
-
-void OccaGridFunction::SetFromTrueDofs(Vector &v)
-{
-   const mfem::Operator *P = ofespace->GetProlongationOperator();
-   if (!P)
-   {
-      MakeRef(v);
-   }
-   else
-   {
-      Resize<double>(P->OutLayout(), NULL);
-      mfem::Vector mfem_this(*this);
-      P->Mult(v.Wrap(), mfem_this);
-   }
-}
-
-mfem::FiniteElementSpace* OccaGridFunction::GetFESpace()
-{
-   return ofespace->GetFESpace();
-}
-
-const mfem::FiniteElementSpace* OccaGridFunction::GetFESpace() const
-{
-   return ofespace->GetFESpace();
-}
-
-void OccaGridFunction::ToQuad(const IntegrationRule &ir, Vector &quadValues)
-{
-   const Engine &engine = OccaLayout().OccaEngine();
-   ::occa::device device = engine.GetDevice();
-
-   OccaDofQuadMaps &maps = OccaDofQuadMaps::Get(device, *ofespace, ir);
-
-   const int elements = ofespace->GetNE();
-   const int numQuad  = ir.GetNPoints();
-   quadValues.Resize<double>(*(new Layout(engine, numQuad * elements)), NULL);
-
-   ::occa::kernel g2qKernel = GetGridFunctionKernel(device, *ofespace, ir);
-   g2qKernel(elements,
-             maps.dofToQuad,
-             ofespace->GetLocalToGlobalMap(),
-             this->OccaMem(),
-             quadValues.OccaMem());
-}
-
-void OccaGridFunction::Distribute(const Vector &v)
-{
-   if (ofespace->isDistributed())
-   {
-      mfem::Vector mfem_this(*this);
-      ofespace->GetProlongationOperator()->Mult(v.Wrap(), mfem_this);
-   }
-   else
-   {
-      *this = v;
-   }
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/gridfunc.hpp

@@ -1,83 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_GRID_FUNC_HPP
-#define MFEM_BACKENDS_OCCA_GRID_FUNC_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "vector.hpp"
-#include "fespace.hpp"
-
-namespace mfem
-{
-
-class IntegrationRule;
-class GridFunction;
-
-namespace occa
-{
-
-class OccaIntegrator;
-class OccaDofQuadMaps;
-
-// TODO: make this object part of the backend or the engine.
-extern std::map<std::string, ::occa::kernel> gridFunctionKernels;
-
-// TODO: make this a method of the backend or the engine.
-::occa::kernel GetGridFunctionKernel(::occa::device device,
-                                     FiniteElementSpace &fespace,
-                                     const mfem::IntegrationRule &ir);
-
-class OccaGridFunction : public Vector
-{
-protected:
-   FiniteElementSpace *ofespace;
-   long sequence;
-
-   ::occa::kernel gridFuncToQuad[3];
-
-public:
-   // OccaGridFunction();
-
-   OccaGridFunction(FiniteElementSpace *ofespace_);
-
-   // OccaGridFunction(FiniteElementSpace *ofespace_,
-   //                  OccaVectorRef ref);
-
-   OccaGridFunction(const OccaGridFunction &gf);
-
-   OccaGridFunction& operator = (double value);
-   OccaGridFunction& operator = (const Vector &v);
-   // OccaGridFunction& operator = (const OccaVectorRef &v);
-   OccaGridFunction& operator = (const OccaGridFunction &gf);
-
-   // void SetGridFunction(mfem::GridFunction &gf);
-
-   void GetTrueDofs(Vector &v);
-   void SetFromTrueDofs(Vector &v);
-
-   mfem::FiniteElementSpace* GetFESpace();
-   const mfem::FiniteElementSpace* GetFESpace() const;
-
-   void ToQuad(const mfem::IntegrationRule &ir, Vector &quadValues);
-
-   void Distribute(const Vector &v);
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_GRID_FUNC_HPP

backends/occa/gridfunc.okl

@@ -1,26 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-#if USING_TENSOR_OPS
-#  if OCCA_USING_CPU
-#    include "mfem-occa://gridfunc/tensor/cpu.okl"
-#  else
-#    include "mfem-occa://gridfunc/tensor/gpuHighOrder.okl"
-#  endif
-#else
-#  if OCCA_USING_CPU
-#    include "mfem-occa://gridfunc/simplex/cpu.okl"
-#  else
-#    include "mfem-occa://gridfunc/simplex/gpuHighOrder.okl"
-#  endif
-#endif

backends/occa/gridfunc/simplex/cpu.okl

@@ -1,63 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 2D ]-----------------------------
-@kernel void GridFuncToQuad2D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap_t restrict l2gMap,
-                              const double * restrict gf,
-                              QVLocal_t restrict out) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        const int gid = l2gMap(d, e);
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          const double r_gf = gf[v + gid*NUM_VDIM];
-          double r_out = 0;
-          for (int q = 0; q < NUM_QUAD; ++q) {
-            r_out += r_gf * dofToQuad(d, q);
-          }
-          out(v, d, e) = r_out;
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void GridFuncToQuad3D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap_t restrict l2gMap,
-                              const double * restrict gf,
-                              QVLocal_t restrict out) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      for (int d = 0; d < NUM_DOFS; ++d) {
-        const int gid = l2gMap(d, e);
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          const double r_gf = gf[v + gid*NUM_VDIM];
-          double r_out = 0;
-          for (int q = 0; q < NUM_QUAD; ++q) {
-            r_out += r_gf * dofToQuad(d, q);
-          }
-          out(v, d, e) = r_out;
-        }
-      }
-    }
-  }
-}
-//======================================

backends/occa/gridfunc/simplex/gpuHighOrder.okl

@@ -1,79 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 2D ]-----------------------------
-@kernel void GridFuncToQuad2D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap_t restrict l2gMap,
-                              const double * restrict gf,
-                              QVLocal_t restrict out) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_gf[NUM_VDIM][NUM_DOFS];
-
-    for (int dOff = 0; dOff < M2_INNER_BATCH; ++dOff; @inner) {
-      for (int d = dOff; d < NUM_DOFS; d += M2_INNER_BATCH) {
-        const int gid = l2gMap(d, e);
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          s_gf[v][d] = gf[v + gid*NUM_VDIM]];
-        }
-      }
-    }
-
-    for (int qOff = 0; qOff < M2_INNER_BATCH; ++qOff; @inner) {
-      for (int q = qOff; q < NUM_QUAD; q += M2_INNER_BATCH) {
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          double r_out = 0;
-          for (int d = 0; d < NUM_DOFS; ++d) {
-            r_out += s_gf[v][d] * dofToQuad(d, q);
-          }
-          out(v, q, e) = r_out;
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void GridFuncToQuad3D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap_t restrict l2gMap,
-                              const double * restrict gf,
-                              QVLocal_t restrict out) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    @shared double s_gf[NUM_VDIM][NUM_DOFS];
-
-    for (int dOff = 0; dOff < M3_INNER_BATCH; ++dOff; @inner) {
-      for (int d = dOff; d < NUM_DOFS; d += M3_INNER_BATCH) {
-        const int gid = l2gMap(d, e);
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          s_gf[v][d] = gf[v + gid*NUM_VDIM]];
-        }
-      }
-    }
-
-    for (int qOff = 0; qOff < M3_INNER_BATCH; ++qOff; @inner) {
-      for (int q = qOff; q < NUM_QUAD; q += M3_INNER_BATCH) {
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          double r_out = 0;
-          for (int d = 0; d < NUM_DOFS; ++d) {
-            r_out += s_gf[v][d] * dofToQuad(d, q);
-          }
-          out(v, q, e) = r_out;
-        }
-      }
-    }
-  }
-}
-//======================================

backends/occa/gridfunc/tensor/cpu.okl

@@ -1,188 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 1D ]-----------------------------
-@kernel void GridFuncToQuad1D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap1D_t restrict l2gMap,
-                              const double * restrict gf,
-                              QVLocal1D_t restrict out) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double r_out[NUM_VDIM][NUM_QUAD_1D];
-      for (int v = 0; v < NUM_VDIM; ++v) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          r_out[v][qx] = 0;
-        }
-      }
-      for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-        const int gid = l2gMap(dx, e);
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          const double r_gf = gf[v + gid*NUM_VDIM];
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            r_out[v][qx] += r_gf * dofToQuad(qx, dx);
-          }
-        }
-      }
-      for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          out(v, qx, e) = r_out[v][qx];
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 2D ]-----------------------------
-@kernel void GridFuncToQuad2D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap2D_t restrict l2gMap,
-                              const double * restrict gf,
-                              QVLocal2D_t restrict out) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double out_xy[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D];
-      for (int v = 0; v < NUM_VDIM; ++v) {
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            out_xy[v][qy][qx] = 0;
-          }
-        }
-      }
-
-      for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-        double out_x[NUM_VDIM][NUM_QUAD_1D];
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            out_x[v][qy] = 0;
-          }
-        }
-
-        for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-          const int gid = l2gMap(dx, dy, e);
-          for (int v = 0; v < NUM_VDIM; ++v) {
-            const double r_gf = gf[v + gid*NUM_VDIM];
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              out_x[v][qy] += r_gf * dofToQuad(qy, dx);
-            }
-          }
-        }
-
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            const double d2q = dofToQuad(qy, dy);
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              out_xy[v][qy][qx] += d2q * out_x[v][qx];
-            }
-          }
-        }
-      }
-
-      for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-        for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-          for (int v = 0; v < NUM_VDIM; ++v) {
-            out(v, qx, qy, e) = out_xy[v][qy][qx];
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void GridFuncToQuad3D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap3D_t restrict l2gMap,
-                              const double * restrict gf,
-                              QVLocal3D_t restrict out) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int dummy = 0; dummy < 1; ++dummy; @inner) {
-      double out_xyz[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D][NUM_QUAD_1D];
-      for (int v = 0; v < NUM_VDIM; ++v) {
-        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              out_xyz[v][qz][qy][qx] = 0;
-            }
-          }
-        }
-      }
-
-      for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-        double out_xy[NUM_VDIM][NUM_QUAD_1D][NUM_QUAD_1D];
-        for (int v = 0; v < NUM_VDIM; ++v) {
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              out_xy[v][qy][qx] = 0;
-            }
-          }
-        }
-
-        for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-          double out_x[NUM_VDIM][NUM_QUAD_1D];
-          for (int v = 0; v < NUM_VDIM; ++v) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              out_x[v][qx] = 0;
-            }
-          }
-
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            const int gid = l2gMap(dx, dy, dz, e);
-            for (int v = 0; v < NUM_VDIM; ++v) {
-              const double r_gf = gf[v + gid*NUM_VDIM];
-              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-                out_x[v][qx] += r_gf * dofToQuad(qx, dx);
-              }
-            }
-          }
-
-          for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-            const double wy = dofToQuad(qy, dy);
-            for (int v = 0; v < NUM_VDIM; ++v) {
-              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-                out_xy[v][qy][qx] += wy * out_x[v][qx];
-              }
-            }
-          }
-        }
-
-        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-          const double wz = dofToQuad(qz, dz);
-          for (int v = 0; v < NUM_VDIM; ++v) {
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-                out_xyz[v][qz][qy][qx] += wz * out_xy[v][qy][qx];
-              }
-            }
-          }
-        }
-      }
-
-      for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-        for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            for (int v = 0; v < NUM_VDIM; ++v) {
-              out(v, qx, qy, qz, e) = out_xyz[v][qz][qy][qx];
-            }
-          }
-        }
-      }
-    }
-  }
-}
-//======================================

backends/occa/gridfunc/tensor/gpuHighOrder.okl

@@ -1,183 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "mfem-occa://defines.okl"
-
-//---[ 1D ]-----------------------------
-@kernel void GridFuncToQuad1D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap1D_t restrict l2gMap,
-                              const double * restrict gf,
-                              QLocal1D_t restrict out) {
-  // Iterate over elements
-  for (int eOff = 0; eOff < numElements; eOff += M1_ELEMENT_BATCHES; @outer) {
-    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-    @exclusive double r_out[NUM_QUAD_1D];
-
-    for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
-      for (int i = el; i < NUM_QUAD_DOFS_1D; i += M1_INNER_ELEMENT_BATCH) {
-        s_dofToQuad[i] = dofToQuad[i];
-      }
-    }
-
-    for (int b = 0; b < M1_OUTER_ELEMENT_BATCH; ++b) {
-      for (int el = 0; el < M1_INNER_ELEMENT_BATCH; ++el; @inner) {
-        const int e = eOff + b*M1_INNER_ELEMENT_BATCH + el;
-        if (e < numElements) {
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            r_out[qx] = 0;
-          }
-          for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-            const double r_gf = gf[l2gMap(dx, e)];
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              r_out[qx] += r_gf * s_dofToQuad(qx, dx);
-            }
-          }
-          for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-            out(qx, e) = r_out[qx];
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 2D ]-----------------------------
-@kernel void GridFuncToQuad2D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap2D_t restrict l2gMap,
-                              const double * restrict gf,
-                              QLocal2D_t restrict out) {
-  // Iterate over elements
-  for (int eOff = 0; eOff < numElements; eOff += M2_ELEMENT_BATCH; @outer) {
-    // Store dof <--> quad mappings
-    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-
-    // Store xy planes in shared memory
-    @shared double s_xy[NUM_QUAD_DOFS_1D] @dim(NUM_DOFS_1D, NUM_QUAD_1D);
-
-    for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
-      for (int id = x; id < NUM_QUAD_DOFS_1D; id += NUM_MAX_1D) {
-        s_dofToQuad[id] = dofToQuad[id];
-      }
-    }
-
-    for (int e = eOff; e < (eOff + M2_ELEMENT_BATCH); ++e) {
-      if (e < numElements) {
-        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-          if (dx < NUM_DOFS_1D) {
-            double r_x[NUM_DOFS_1D];
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              r_x[dy] = gf[l2gMap(dx, dy, e)];
-            }
-            for (int qy = 0; qy < NUM_QUAD_1D; ++qy) {
-              double xy = 0;
-              for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-                xy += r_x[dy] * s_dofToQuad(qy, dy);
-              }
-              s_xy(dx, qy) = xy;
-            }
-          }
-        }
-        for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
-          if (qy < NUM_QUAD_1D) {
-            for (int qx = 0; qx < NUM_QUAD_1D; ++qx) {
-              double val = 0;
-              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-                val += s_xy(dx, qy) * s_dofToQuad(qx, dx);
-              }
-              out(qx, qy, e) = val;
-            }
-          }
-        }
-      }
-    }
-  }
-}
-//======================================
-
-
-//---[ 3D ]-----------------------------
-@kernel void GridFuncToQuad3D(const int numElements,
-                              const DofToQuad_t restrict dofToQuad,
-                              const DLocalMap3D_t restrict l2gMap,
-                              const double * restrict gf,
-                              QLocal3D_t restrict out) {
-  // Iterate over elements
-  for (int e = 0; e < numElements; ++e; @outer) {
-    // Store dof <--> quad mappings
-    @shared double s_dofToQuad[NUM_QUAD_DOFS_1D] @dim(NUM_QUAD_1D, NUM_DOFS_1D);
-
-    // Store xy planes in shared memory
-    @shared double s_z[NUM_MAX_2D] @dim(NUM_MAX_1D, NUM_MAX_1D);
-
-    // Store z axis as registers
-    @exclusive double r_qz[NUM_QUAD_1D];
-
-    for (int y = 0; y < NUM_MAX_1D; ++y; @inner) {
-      for (int x = 0; x < NUM_MAX_1D; ++x; @inner) {
-        const int id = (y * NUM_MAX_1D) + x;
-        // Fetch Q <--> D maps
-        if (id < NUM_QUAD_DOFS_1D) {
-          s_dofToQuad[id] = dofToQuad[id];
-        }
-        // Initialize our Z axis
-        for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-          r_qz[qz] = 0;
-        }
-      }
-    }
-
-    for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
-      for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-        if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
-          for (int dz = 0; dz < NUM_DOFS_1D; ++dz) {
-            const double val = gf[l2gMap(dx, dy, dz, e)];
-            // Calculate D -> Q in the Z axis
-            for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-              r_qz[qz] += val * s_dofToQuad(qz, dz);
-            }
-          }
-        }
-      }
-    }
-    // For each xy plane
-    for (int qz = 0; qz < NUM_QUAD_1D; ++qz) {
-      // Fill xy plane at given z position
-      for (int dy = 0; dy < NUM_MAX_1D; ++dy; @inner) {
-        for (int dx = 0; dx < NUM_MAX_1D; ++dx; @inner) {
-          if ((dx < NUM_DOFS_1D) && (dy < NUM_DOFS_1D)) {
-            s_z(dx, dy) = r_qz[qz];
-          }
-        }
-      }
-      // Calculate Dxyz, xDyz, xyDz in plane
-      for (int qy = 0; qy < NUM_MAX_1D; ++qy; @inner) {
-        for (int qx = 0; qx < NUM_MAX_1D; ++qx; @inner) {
-          if ((qx < NUM_QUAD_1D) && (qy < NUM_QUAD_1D)) {
-            double val = 0;
-            for (int dy = 0; dy < NUM_DOFS_1D; ++dy) {
-              const double wy = s_dofToQuad(qy, dy);
-              for (int dx = 0; dx < NUM_DOFS_1D; ++dx) {
-                const double wx = s_dofToQuad(qx, dx);
-                val += wx * wy * s_z(dx, dy);
-              }
-            }
-            out(qx, qy, qz, e) = val;
-          }
-        }
-      }
-    }
-  }
-}
-//======================================

backends/occa/interpolation.cpp

@@ -1,162 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "interpolation.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-void CreateRPOperators(Layout &v_layout, Layout &t_layout,
-                       const mfem::SparseMatrix *R, const mfem::Operator *P,
-                       mfem::Operator *&OccaR, mfem::Operator *&OccaP)
-{
-   if (!P)
-   {
-      OccaR = new IdentityOperator(t_layout);
-      OccaP = new IdentityOperator(t_layout);
-      return;
-   }
-
-   const mfem::SparseMatrix *pmat = dynamic_cast<const mfem::SparseMatrix*>(P);
-   ::occa::device device = v_layout.OccaEngine().GetDevice();
-
-   if (R)
-   {
-      OccaSparseMatrix *occaR =
-         CreateMappedSparseMatrix(v_layout, t_layout, *R);
-      ::occa::array<int> reorderIndices = occaR->reorderIndices;
-      delete occaR;
-
-      OccaR = new RestrictionOperator(v_layout, t_layout, reorderIndices);
-   }
-
-   if (pmat)
-   {
-      const mfem::SparseMatrix *pmatT = Transpose(*pmat);
-
-      OccaSparseMatrix *occaP  =
-         CreateMappedSparseMatrix(t_layout, v_layout, *pmat);
-      OccaSparseMatrix *occaPT =
-         CreateMappedSparseMatrix(v_layout, t_layout, *pmatT);
-
-      OccaP = new ProlongationOperator(*occaP, *occaPT);
-   }
-   else
-   {
-      OccaP = new ProlongationOperator(t_layout, v_layout, P);
-   }
-}
-
-RestrictionOperator::RestrictionOperator(Layout &in_layout, Layout &out_layout,
-                                         ::occa::array<int> indices) :
-   Operator(in_layout, out_layout)
-{
-
-   entries     = indices.size() / 2;
-   trueIndices = indices;
-
-   // FIXME: paths ...
-   ::occa::device device = in_layout.OccaEngine().GetDevice();
-   const std::string &okl_path = in_layout.OccaEngine().GetOklPath();
-   const std::string &okl_defines = in_layout.OccaEngine().GetOklDefines();
-   multOp = device.buildKernel(okl_path + "mappings.okl",
-                               "ExtractSubVector",
-                               "defines: { TILESIZE: 256 }" + okl_defines);
-
-   multTransposeOp = device.buildKernel(okl_path + "mappings.okl",
-                                        "SetSubVector",
-                                        "defines: { TILESIZE: 256 }" +
-                                        okl_defines);
-}
-
-void RestrictionOperator::Mult_(const Vector &x, Vector &y) const
-{
-   multOp(entries, trueIndices, x.OccaMem(), y.OccaMem());
-}
-
-void RestrictionOperator::MultTranspose_(const Vector &x, Vector &y) const
-{
-   y.Fill<double>(0.0);
-   multTransposeOp(entries, trueIndices, x.OccaMem(), y.OccaMem());
-}
-
-ProlongationOperator::ProlongationOperator(OccaSparseMatrix &multOp_,
-                                           OccaSparseMatrix &multTransposeOp_) :
-   Operator(multOp_),
-   pmat(NULL),
-   multOp(multOp_),
-   multTransposeOp(multTransposeOp_) {}
-
-ProlongationOperator::ProlongationOperator(Layout &in_layout,
-                                           Layout &out_layout,
-                                           const mfem::Operator *pmat_) :
-   Operator(in_layout, out_layout),
-   pmat(pmat_),
-   multOp(*this),
-   multTransposeOp(*this)
-{ }
-
-void ProlongationOperator::Mult_(const Vector &x, Vector &y) const
-{
-   MFEM_VERIFY(pmat == NULL, "");
-   multOp.Mult_(x, y);
-}
-
-void ProlongationOperator::MultTranspose_(const Vector &x, Vector &y) const
-{
-   MFEM_VERIFY(pmat == NULL, "");
-   multTransposeOp.Mult_(x, y);
-}
-
-void ProlongationOperator::Mult(const mfem::Vector &x, mfem::Vector &y) const
-{
-   if (pmat)
-   {
-      // FIXME: create an OCCA version of 'pmat'
-      x.Pull();
-      y.Pull(false);
-      pmat->Mult(x, y);
-      y.Push();
-   }
-   else
-   {
-      multOp.Mult(x, y);
-   }
-}
-
-void ProlongationOperator::MultTranspose(const mfem::Vector &x,
-                                         mfem::Vector &y) const
-{
-   if (pmat)
-   {
-      // FIXME: create an OCCA version of 'pmat'
-      x.Pull();
-      y.Pull(false);
-      pmat->MultTranspose(x, y);
-      y.Push();
-   }
-   else
-   {
-      multTransposeOp.Mult(x, y);
-   }
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/interpolation.hpp

@@ -1,79 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_INTERPOLATION_HPP
-#define MFEM_BACKENDS_OCCA_INTERPOLATION_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include <occa.hpp>
-#include "vector.hpp"
-#include "engine.hpp"
-#include "sparsemat.hpp"
-#include "../../fem/fem.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-// [MISSING] Proper destructors
-void CreateRPOperators(Layout &v_layout, Layout &t_layout,
-                       const mfem::SparseMatrix *R, const mfem::Operator *P,
-                       mfem::Operator *&OccaR, mfem::Operator *&OccaP);
-
-class RestrictionOperator : public Operator
-{
-protected:
-   int entries;
-   ::occa::array<int> trueIndices;
-   ::occa::kernel multOp, multTransposeOp;
-
-public:
-   RestrictionOperator(Layout &in_layout, Layout &out_layout,
-                       ::occa::array<int> indices);
-
-   // overrides
-   virtual void Mult_(const Vector &x, Vector &y) const;
-   virtual void MultTranspose_(const Vector &x, Vector &y) const;
-};
-
-class ProlongationOperator : public Operator
-{
-protected:
-   const mfem::Operator *pmat;
-   OccaSparseMatrix multOp, multTransposeOp;
-
-public:
-   ProlongationOperator(OccaSparseMatrix &multOp_,
-                        OccaSparseMatrix &multTransposeOp_);
-
-   ProlongationOperator(Layout &in_layout, Layout &out_layout,
-                        const mfem::Operator *pmat_);
-
-   // overrides
-   virtual void Mult_(const Vector &x, Vector &y) const;
-   virtual void MultTranspose_(const Vector &x, Vector &y) const;
-
-   // overrides
-   virtual void Mult(const mfem::Vector &x, mfem::Vector &y) const;
-   virtual void MultTranspose(const mfem::Vector &x, mfem::Vector &y) const;
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_INTERPOLATION_HPP

backends/occa/layout.cpp

@@ -1,40 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "layout.hpp"
-#include "../../general/array.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-void Layout::Resize(std::size_t new_size)
-{
-   size = new_size;
-}
-
-void Layout::Resize(const Array<std::size_t> &offsets)
-{
-   MFEM_ASSERT(offsets.Size() == 2,
-               "multiple workers are not supported yet");
-   size = offsets.Last();
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/layout.hpp

@@ -1,68 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_LAYOUT_HPP
-#define MFEM_BACKENDS_OCCA_LAYOUT_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "../base/layout.hpp"
-#include "engine.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class Layout : public PLayout
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // SharedPtr<const mfem::Engine> engine;
-   // std::size_t size;
-
-public:
-   Layout(const Engine &e, std::size_t s = 0) : PLayout(e, s) { }
-
-   const Engine &OccaEngine() const
-   { return *static_cast<const Engine *>(engine.Get()); }
-
-   ::occa::memory Alloc(std::size_t bytes) const
-   { return OccaEngine().GetDevice().malloc(bytes); }
-
-   virtual ~Layout() { }
-
-   /**
-       @name Virtual interface
-    */
-   ///@{
-
-   /// Resize the layout
-   virtual void Resize(std::size_t new_size);
-
-   /// Resize the layout based on the given worker offsets
-   virtual void Resize(const Array<std::size_t> &offsets);
-
-   ///@}
-   // End: Virtual interface
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_LAYOUT_HPP

backends/occa/mappings.okl

@@ -1,54 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-/*
-  ---[ Defines Known At Compile-Time ]------------
-  TILESIZE : Tilesize for iterating over entries
-  ================================================
-*/
-
-@kernel void ExtractSubVector(const int entries,
-                              const int * restrict indices,
-                              const double * restrict in,
-                              double * restrict out) {
-
-  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
-    if (i < entries) {
-      out[i] = in[indices[i]];
-    }
-  }
-}
-
-@kernel void SetSubVector(const int entries,
-                          const int * restrict indices,
-                          const double * restrict in,
-                          double * restrict out) {
-
-  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
-    if (i < entries) {
-      out[indices[i]] = in[i];
-    }
-  }
-}
-
-@kernel void MapSubVector(const int entries,
-                          const int * restrict indices,
-                          const double * restrict in,
-                          double * restrict out) {
-
-  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
-    if (i < entries) {
-      const int fromIdx = indices[2*i + 0];
-      const int toIdx   = indices[2*i + 1];
-      out[toIdx] = in[fromIdx];
-    }
-  }
-}

backends/occa/operator.cpp

@@ -1,135 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "operator.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-// FIXME: move this object to the Backend?
-::occa::kernelBuilder OccaConstrainedOperator::mapDofBuilder =
-   ::occa::linalg::customLinearMethod(
-      "vector_map_dofs",
-
-      "const int idx = v2[i];"
-      "v0[idx] = v1[idx];",
-
-      "defines: {"
-      "  VTYPE0: 'double',"
-      "  VTYPE1: 'double',"
-      "  VTYPE2: 'int',"
-      "  TILESIZE: 128,"
-      "}");
-
-// FIXME: move this object to the Backend?
-::occa::kernelBuilder OccaConstrainedOperator::clearDofBuilder =
-   ::occa::linalg::customLinearMethod(
-      "vector_clear_dofs",
-
-      "v0[v1[i]] = 0.0;",
-
-      "defines: {"
-      "  VTYPE0: 'double',"
-      "  VTYPE1: 'int',"
-      "  TILESIZE: 128,"
-      "}");
-
-OccaConstrainedOperator::OccaConstrainedOperator(
-   mfem::Operator *A_,
-   const mfem::Array<int> &constraintList_,
-   bool own_A_)
-
-   : Operator(A_->InLayout()->As<Layout>()),
-     z(OutLayout_()),
-     w(OutLayout_()),
-     mfem_z((z.DontDelete(), z)),
-     mfem_w((w.DontDelete(), w))
-{
-   Setup(OutLayout_().OccaEngine().GetDevice(), A_, constraintList_, own_A_);
-}
-
-void OccaConstrainedOperator::Setup(::occa::device device_,
-                                    mfem::Operator *A_,
-                                    const mfem::Array<int> &constraintList_,
-                                    bool own_A_)
-{
-   device = device_;
-
-   A = A_;
-   own_A = own_A_;
-
-   constraintIndices = constraintList_.Size();
-   constraintList = constraintList_.Get_PArray()->As<Array>().OccaMem();
-}
-
-void OccaConstrainedOperator::EliminateRHS(const Vector &x, Vector &b) const
-{
-   const std::string &okl_defines = InLayout_().OccaEngine().GetOklDefines();
-   ::occa::kernel mapDofs = mapDofBuilder.build(device, okl_defines);
-
-   w.Fill<double>(0.0);
-
-   if (constraintIndices)
-   {
-      mapDofs(constraintIndices, w.OccaMem(), x.OccaMem(), constraintList);
-   }
-
-   A->Mult(mfem_w, mfem_z);
-
-   b.Axpby<double>(1.0, b, -1.0, z);
-
-   if (constraintIndices)
-   {
-      mapDofs(constraintIndices, b.OccaMem(), x.OccaMem(), constraintList);
-   }
-}
-
-void OccaConstrainedOperator::Mult_(const Vector &x, Vector &y) const
-{
-   mfem::Vector mfem_y(y);
-   if (constraintIndices == 0)
-   {
-      A->Mult(x.Wrap(), mfem_y);
-      return;
-   }
-
-   const std::string &okl_defines = InLayout_().OccaEngine().GetOklDefines();
-   ::occa::kernel mapDofs   = mapDofBuilder.build(device, okl_defines);
-   ::occa::kernel clearDofs = clearDofBuilder.build(device, okl_defines);
-
-   z.Assign<double>(x); // z = x
-
-   clearDofs(constraintIndices, z.OccaMem(), constraintList);
-
-   A->Mult(mfem_z, mfem_y);
-
-   mapDofs(constraintIndices, y.OccaMem(), x.OccaMem(), constraintList);
-}
-
-OccaConstrainedOperator::~OccaConstrainedOperator()
-{
-   if (own_A)
-   {
-      delete A;
-   }
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/operator.hpp

@@ -1,129 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_OPERATOR_HPP
-#define MFEM_BACKENDS_OCCA_OPERATOR_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "vector.hpp"
-#include "../../linalg/operator.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class Operator : public mfem::Operator
-{
-public:
-   /// Creare an operator with the same dimensions as @a orig.
-   Operator(const Operator &orig)
-      : mfem::Operator(orig) { }
-
-   Operator(Layout &layout)
-      : mfem::Operator(layout) { }
-
-   Operator(Layout &in_layout, Layout &out_layout)
-      : mfem::Operator(in_layout, out_layout) { }
-
-   Layout &InLayout_() const
-   { return *static_cast<Layout*>(in_layout.Get()); }
-
-   Layout &OutLayout_() const
-   { return *static_cast<Layout*>(out_layout.Get()); }
-
-   virtual void Mult_(const Vector &x, Vector &y) const = 0;
-
-   virtual void MultTranspose_(const Vector &x, Vector &y) const
-   { MFEM_ABORT("method is not supported"); }
-
-   // override
-   virtual void Mult(const mfem::Vector &x, mfem::Vector &y) const
-   {
-      Mult_(x.Get_PVector()->As<Vector>(),
-            y.Get_PVector()->As<Vector>());
-   }
-
-   // override
-   virtual void MultTranspose(const mfem::Vector &x, mfem::Vector &y) const
-   {
-      MultTranspose_(x.Get_PVector()->As<Vector>(),
-                     y.Get_PVector()->As<Vector>());
-   }
-};
-
-
-class OccaConstrainedOperator : public Operator
-{
-protected:
-   ::occa::device device;
-
-   mfem::Operator *A;              //< The unconstrained Operator.
-   bool own_A;                     //< Ownership flag for A.
-   ::occa::memory constraintList;  //< List of constrained indices/dofs.
-   int constraintIndices;
-   mutable Vector z, w;       //< Auxiliary vectors.
-   mutable mfem::Vector mfem_z, mfem_w; // Wrap z, w
-
-   static ::occa::kernelBuilder mapDofBuilder, clearDofBuilder;
-
-public:
-   /** @brief Constructor from a general Operator and a list of essential
-       indices/dofs.
-
-       Specify the unconstrained operator @a *A and a @a list of indices to
-       constrain, i.e. each entry @a list[i] represents an essential-dof. If the
-       ownership flag @a own_A is true, the operator @a *A will be destroyed
-       when this object is destroyed. */
-   OccaConstrainedOperator(mfem::Operator *A_,
-                           const mfem::Array<int> &constraintList_,
-                           bool own_A_ = false);
-
-   void Setup(::occa::device device_,
-              mfem::Operator *A_,
-              const mfem::Array<int> &constraintList_,
-              bool own_A_ = false);
-
-   /** @brief Eliminate "essential boundary condition" values specified in @a x
-       from the given right-hand side @a b.
-
-       Performs the following steps:
-
-       z = A((0,x_b));  b_i -= z_i;  b_b = x_b;
-
-       where the "_b" subscripts denote the essential (boundary) indices/dofs of
-       the vectors, and "_i" -- the rest of the entries. */
-   void EliminateRHS(const Vector &x, Vector &b) const;
-
-   /** @brief Constrained operator action.
-
-       Performs the following steps:
-
-       z = A((x_i,0));  y_i = z_i;  y_b = x_b;
-
-       where the "_b" subscripts denote the essential (boundary) indices/dofs of
-       the vectors, and "_i" -- the rest of the entries. */
-   virtual void Mult_(const Vector &x, Vector &y) const;
-
-   // Destructor: destroys the unconstrained Operator @a A if @a own_A is true.
-   virtual ~OccaConstrainedOperator();
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_OPERATOR_HPP

backends/occa/sparse.okl

@@ -1,57 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-/*
-  ---[ Defines Known At Compile-Time ]------------
-  TILESIZE : Tilesize for iterating over dofs
-  ================================================
-*/
-
-@kernel void Mult(const int entries,
-                  const int * restrict offsets,
-                  const int * restrict indices,
-                  const double * restrict weights,
-                  const double * restrict in,
-                  double * restrict out) {
-
-  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
-    if (i < entries) {
-      const int offset = offsets[i];
-      const int nextOffset = offsets[i + 1];
-      double value = 0;
-      for (int j = offset; j < nextOffset; ++j) {
-        value += weights[j] * in[indices[j]];
-      }
-      out[i] = value;
-    }
-  }
-}
-
-@kernel void MappedMult(const int entries,
-                        const int * restrict offsets,
-                        const int * restrict indices,
-                        const double * restrict weights,
-                        const int * restrict outIndices,
-                        const double * restrict in,
-                        double * restrict out) {
-
-  for (int i = 0; i < entries; ++i; @tile(TILESIZE, @outer, @inner)) {
-    if (i < entries) {
-      const int offset = offsets[i];
-      const int nextOffset = offsets[i + 1];
-      double value = 0;
-      for (int j = offset; j < nextOffset; ++j) {
-        value += weights[j] * in[indices[j]];
-      }
-      out[outIndices[i]] = value;
-    }
-  }
-}

backends/occa/sparsemat.cpp

@@ -1,248 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "sparsemat.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                                   const mfem::SparseMatrix &m,
-                                   const ::occa::properties &props) :
-   Operator(in_layout, out_layout)
-{
-
-   Setup(in_layout.OccaEngine().GetDevice(), m, props);
-}
-
-OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                                   const mfem::SparseMatrix &m,
-                                   ::occa::array<int> reorderIndices_,
-                                   ::occa::array<int> mappedIndices_,
-                                   const ::occa::properties &props) :
-   Operator(in_layout, out_layout)
-{
-
-   Setup(in_layout.OccaEngine().GetDevice(), m,
-         reorderIndices, mappedIndices_, props);
-}
-
-OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                                   ::occa::array<int> offsets_,
-                                   ::occa::array<int> indices_,
-                                   ::occa::array<double> weights_,
-                                   const ::occa::properties &props) :
-   Operator(in_layout, out_layout),
-   offsets(offsets_),
-   indices(indices_),
-   weights(weights_)
-{
-
-   SetupKernel(in_layout.OccaEngine().GetDevice(), props);
-}
-
-OccaSparseMatrix::OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                                   ::occa::array<int> offsets_,
-                                   ::occa::array<int> indices_,
-                                   ::occa::array<double> weights_,
-                                   ::occa::array<int> reorderIndices_,
-                                   ::occa::array<int> mappedIndices_,
-                                   const ::occa::properties &props) :
-   Operator(in_layout, out_layout),
-   offsets(offsets_),
-   indices(indices_),
-   weights(weights_),
-   reorderIndices(reorderIndices_),
-   mappedIndices(mappedIndices_)
-{
-
-   SetupKernel(in_layout.OccaEngine().GetDevice(), props);
-}
-
-void OccaSparseMatrix::Setup(::occa::device device, const mfem::SparseMatrix &m,
-                             const ::occa::properties &props)
-{
-   Setup(device, m, ::occa::array<int>(), ::occa::array<int>(), props);
-}
-
-void OccaSparseMatrix::Setup(::occa::device device, const SparseMatrix &m,
-                             ::occa::array<int> reorderIndices_,
-                             ::occa::array<int> mappedIndices_,
-                             const ::occa::properties &props)
-{
-
-   const int nnz = m.GetI()[height];
-   offsets.allocate(device,
-                    height + 1, m.GetI());
-   indices.allocate(device,
-                    nnz, m.GetJ());
-   weights.allocate(device,
-                    nnz, m.GetData());
-
-   offsets.keepInDevice();
-   indices.keepInDevice();
-   weights.keepInDevice();
-
-   reorderIndices = reorderIndices_;
-   mappedIndices  = mappedIndices_;
-
-   SetupKernel(device, props);
-}
-
-void OccaSparseMatrix::SetupKernel(::occa::device device,
-                                   const ::occa::properties &props)
-{
-
-   const bool hasOutIndices = mappedIndices.isInitialized();
-
-   const ::occa::properties defaultProps("defines: {"
-                                         "  TILESIZE: 256,"
-                                         "}");
-
-   const std::string &okl_path = InLayout_().OccaEngine().GetOklPath();
-   const std::string &okl_defines = InLayout_().OccaEngine().GetOklDefines();
-   mapKernel = device.buildKernel(okl_path + "mappings.okl",
-                                  "MapSubVector",
-                                  defaultProps + props + okl_defines);
-
-   multKernel = device.buildKernel(okl_path + "sparse.okl",
-                                   hasOutIndices ? "MappedMult" : "Mult",
-                                   defaultProps + props + okl_defines);
-}
-
-void OccaSparseMatrix::Mult_(const Vector &x, Vector &y) const
-{
-   if (reorderIndices.isInitialized() ||
-       mappedIndices.isInitialized())
-   {
-      if (reorderIndices.isInitialized())
-      {
-         mapKernel((int) (reorderIndices.size() / 2),
-                   reorderIndices,
-                   x.OccaMem(), y.OccaMem());
-      }
-      if (mappedIndices.isInitialized())
-      {
-         multKernel((int) (mappedIndices.size()),
-                    offsets, indices, weights,
-                    mappedIndices,
-                    x.OccaMem(), y.OccaMem());
-      }
-   }
-   else
-   {
-      multKernel((int) height,
-                 offsets, indices, weights,
-                 x.OccaMem(), y.OccaMem());
-   }
-}
-
-
-OccaSparseMatrix* CreateMappedSparseMatrix(Layout &in_layout,
-                                           Layout &out_layout,
-                                           const mfem::SparseMatrix &m,
-                                           const ::occa::properties &props)
-{
-   const int mHeight = m.Height();
-   // const int mWidth  = m.Width();
-
-   // Count indices that are only reordered (true dofs)
-   const int *I = m.GetI();
-   const int *J = m.GetJ();
-   const double *D = m.GetData();
-
-   int trueCount = 0;
-   for (int i = 0; i < mHeight; ++i)
-   {
-      trueCount += ((I[i + 1] - I[i]) == 1);
-   }
-   const int dupCount = (mHeight - trueCount);
-
-   // Create the reordering map for entries that aren't modified (true dofs)
-   ::occa::device device(in_layout.OccaEngine().GetDevice());
-   ::occa::array<int> reorderIndices(device,
-                                     2 * trueCount);
-   ::occa::array<int> mappedIndices, offsets, indices;
-   ::occa::array<double> weights;
-
-   if (dupCount)
-   {
-      mappedIndices.allocate(device,
-                             dupCount);
-   }
-   int trueIdx = 0, dupIdx = 0;
-   for (int i = 0; i < mHeight; ++i)
-   {
-      const int i1 = I[i];
-      if ((I[i + 1] - i1) == 1)
-      {
-         reorderIndices[trueIdx++] = J[i1];
-         reorderIndices[trueIdx++] = i;
-      }
-      else
-      {
-         mappedIndices[dupIdx++] = i;
-      }
-   }
-   reorderIndices.keepInDevice();
-
-   if (dupCount)
-   {
-      mappedIndices.keepInDevice();
-
-      // Extract sparse matrix without reordered identity
-      const int dupNnz = I[mHeight] - trueCount;
-
-      offsets.allocate(device,
-                       dupCount + 1);
-      indices.allocate(device,
-                       dupNnz);
-      weights.allocate(device,
-                       dupNnz);
-
-      int nnz = 0;
-      offsets[0] = 0;
-      for (int i = 0; i < dupCount; ++i)
-      {
-         const int idx = mappedIndices[i];
-         const int offStart = I[idx];
-         const int offEnd   = I[idx + 1];
-         offsets[i + 1] = offsets[i] + (offEnd - offStart);
-         for (int j = offStart; j < offEnd; ++j)
-         {
-            indices[nnz] = J[j];
-            weights[nnz] = D[j];
-            ++nnz;
-         }
-      }
-
-      offsets.keepInDevice();
-      indices.keepInDevice();
-      weights.keepInDevice();
-   }
-
-   return new OccaSparseMatrix(in_layout, out_layout,
-                               offsets, indices, weights,
-                               reorderIndices, mappedIndices,
-                               props);
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/sparsemat.hpp

@@ -1,95 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_SPARSE_MAT_HPP
-#define MFEM_BACKENDS_OCCA_SPARSE_MAT_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include <occa.hpp>
-#include "vector.hpp"
-#include "engine.hpp"
-#include "operator.hpp"
-#include "../../linalg/sparsemat.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-/// TODO: doxygen
-class OccaSparseMatrix : public Operator
-{
-public:
-   ::occa::array<int> offsets, indices;
-   ::occa::array<double> weights;
-   ::occa::array<int> reorderIndices, mappedIndices;
-   ::occa::kernel mapKernel, multKernel;
-
-   /// Construct an empty OccaSparseMatrix.
-   OccaSparseMatrix(const Operator &orig)
-      : Operator(orig) { }
-
-   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                    const mfem::SparseMatrix &m,
-                    const ::occa::properties &props = ::occa::properties());
-
-   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                    const mfem::SparseMatrix &m,
-                    ::occa::array<int> reorderIndices_,
-                    ::occa::array<int> mappedIndices_,
-                    const ::occa::properties &props = ::occa::properties());
-
-   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                    ::occa::array<int> offsets_,
-                    ::occa::array<int> indices_,
-                    ::occa::array<double> weights_,
-                    const ::occa::properties &props = ::occa::properties());
-
-   OccaSparseMatrix(Layout &in_layout, Layout &out_layout,
-                    ::occa::array<int> offsets_,
-                    ::occa::array<int> indices_,
-                    ::occa::array<double> weights_,
-                    ::occa::array<int> reorderIndices_,
-                    ::occa::array<int> mappedIndices_,
-                    const ::occa::properties &props = ::occa::properties());
-
-   void Setup(::occa::device device, const mfem::SparseMatrix &m,
-              const ::occa::properties &props);
-
-   void Setup(::occa::device device, const mfem::SparseMatrix &m,
-              ::occa::array<int> reorderIndices_,
-              ::occa::array<int> mappedIndices_,
-              const ::occa::properties &props);
-
-   void SetupKernel(::occa::device device,
-                    const ::occa::properties &props);
-
-   // override
-   virtual void Mult_(const Vector &x, Vector &y) const;
-};
-
-
-/// TODO: doxygen
-OccaSparseMatrix* CreateMappedSparseMatrix(
-   Layout &in_layout, Layout &out_layout,
-   const mfem::SparseMatrix &m,
-   const ::occa::properties &props = ::occa::properties());
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_SPARSE_MAT_HPP

backends/occa/url_handler.cpp

@@ -1,81 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "url_handler.hpp"
-#include "../../general/error.hpp"
-#include <cstdlib>
-#include <sys/stat.h>
-
-namespace mfem
-{
-
-namespace occa
-{
-
-FileOpener::FileOpener(const std::string &prefix,
-                       const std::string &env_variable)
-   : pfx(prefix)
-{
-   const char *env_path = getenv(env_variable.c_str());
-   if (!env_path) { return; }
-   std::string path(env_path);
-   for (std::size_t start = 0, end; start < path.size(); start = end + 1)
-   {
-      end = path.find(':', start);
-      if (end == std::string::npos)
-      {
-         AddDir(path.substr(start, end));
-         break;
-      }
-      AddDir(path.substr(start, end - start));
-   }
-}
-
-bool FileOpener::AddDir(const std::string &dir)
-{
-   if (dir.size() == 0 || dir[0] != '/') { return false; }
-   struct stat dir_stat;
-   if (stat(dir.c_str(), &dir_stat)) { return false; }
-   if (!S_ISDIR(dir_stat.st_mode)) { return false; }
-   paths.push_back(dir + (*dir.rbegin() == '/' ? "" : "/"));
-   return true;
-}
-
-bool FileOpener::handles(const std::string &filename)
-{
-   return filename.size() >= pfx.size() &&
-          filename.compare(0, pfx.size(), pfx) == 0;
-}
-
-std::string FileOpener::expand(const std::string &filename)
-{
-   std::string sfx(filename.substr(pfx.size()));
-   for (std::size_t i = 0; i < paths.size(); i++)
-   {
-      std::string file = paths[i] + sfx;
-      struct stat file_stat;
-      if (stat(file.c_str(), &file_stat) == 0 && S_ISREG(file_stat.st_mode))
-      {
-         return file;
-      }
-   }
-   MFEM_ABORT("invalid url: " << filename);
-   return sfx;
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/url_handler.hpp

@@ -1,47 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_URL_HANDLER_HPP
-#define MFEM_BACKENDS_OCCA_URL_HANDLER_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include <occa.hpp>
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class FileOpener : public ::occa::io::fileOpener
-{
-protected:
-   std::string pfx; // prefix, e.g. "mfem://"
-   std::vector<std::string> paths; // paths to search for prefix replacement
-
-public:
-   FileOpener(const std::string &prefix, const std::string &env_variable);
-
-   bool AddDir(const std::string &dir);
-
-   virtual bool handles(const std::string &filename);
-   virtual std::string expand(const std::string &filename);
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_URL_HANDLER_HPP

backends/occa/utils.okl

@@ -1,22 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-typedef double* Local_t @dim(numDofs, numElements);
-
-@kernel void InitLocalVector(const int numElements,
-                             const int numDofs,
-                             Local_t restrict sol) {
-  for (int e = 0; e < numElements; ++e; @outer) {
-    for (int d = 0; d < numDofs; ++d; @inner) {
-      sol(d, e) = 0;
-    }
-  }
-}

backends/occa/vector.cpp

@@ -1,204 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include "vector.hpp"
-#include "../../linalg/vector.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-PVector *Vector::DoVectorClone(bool copy_data, void **buffer,
-                               int buffer_type_id) const
-{
-   MFEM_ASSERT(buffer_type_id == ScalarId<double>::value, "");
-   Vector *new_vector = new Vector(OccaLayout());
-   if (copy_data)
-   {
-      new_vector->slice.copyFrom(slice);
-   }
-   if (buffer)
-   {
-      *buffer = new_vector->GetBuffer();
-   }
-   return new_vector;
-}
-
-void Vector::DoDotProduct(const PVector &x, void *result,
-                          int result_type_id) const
-{
-   // Can be called when Size() == 0, e.g. when an MPI-parallel vector has a
-   // local size of 0.
-
-   MFEM_ASSERT(result_type_id == ScalarId<double>::value, "");
-   double *res = (double *)result;
-   MFEM_ASSERT(dynamic_cast<const Vector *>(&x) != NULL, "invalid Vector type");
-   const Vector *xp = static_cast<const Vector *>(&x);
-   MFEM_ASSERT(this->Size() == xp->Size(), "");
-   *res = ::occa::linalg::dot<double, double, double>(this->slice, xp->slice);
-
-#ifdef MFEM_USE_MPI
-   double local_dot = *res;
-   if (IsParallel())
-   {
-      MPI_Allreduce(&local_dot, res, 1, MPI_DOUBLE, MPI_SUM,
-                    OccaLayout().OccaEngine().GetComm());
-   }
-#endif
-}
-
-void Vector::DoAxpby(const void *a, const PVector &x,
-                     const void *b, const PVector &y,
-                     int ab_type_id)
-{
-   const std::string &okl_defines = OccaLayout().OccaEngine().GetOklDefines();
-
-   //
-   // TODO: move all kernel builders to class mfem::occa::Backend
-   //
-   static ::occa::kernelBuilder axpby1_builder =
-      ::occa::linalg::customLinearMethod(
-         "mfem_occa_axpby1",
-         "v0[i] = c0 * v1[i];",
-         "defines: {"
-         "  CTYPE0: 'double',"
-         "  VTYPE0: 'double',"
-         "  VTYPE1: 'double',"
-         "  TILESIZE: '128',"
-         "}");
-
-   static ::occa::kernelBuilder axpby2_builder =
-      ::occa::linalg::customLinearMethod(
-         "mfem_occa_axpby2",
-         "v0[i] = c0 * v0[i] + c1 * v1[i];",
-         "defines: {"
-         "  CTYPE0: 'double',"
-         "  CTYPE1: 'double',"
-         "  VTYPE0: 'double',"
-         "  VTYPE1: 'double',"
-         "  TILESIZE: '128',"
-         "}");
-
-   static ::occa::kernelBuilder axpby3_builder =
-      ::occa::linalg::customLinearMethod(
-         "mfem_occa_axpby3",
-         "v0[i] = c0 * v1[i] + c1 * v2[i];",
-         "defines: {"
-         "  CTYPE0: 'double',"
-         "  CTYPE1: 'double',"
-         "  VTYPE0: 'double',"
-         "  VTYPE1: 'double',"
-         "  VTYPE2: 'double',"
-         "  TILESIZE: '128',"
-         "}");
-
-   // called only when Size() != 0
-
-   MFEM_ASSERT(ab_type_id == ScalarId<double>::value, "");
-   const double da = *static_cast<const double *>(a);
-   const double db = *static_cast<const double *>(b);
-   MFEM_ASSERT(da == 0.0 || dynamic_cast<const Vector *>(&x) != NULL,
-               "invalid Vector x");
-   MFEM_ASSERT(db == 0.0 || dynamic_cast<const Vector *>(&y) != NULL,
-               "invalid Vector y");
-   const Vector *xp = static_cast<const Vector *>(&x);
-   const Vector *yp = static_cast<const Vector *>(&y);
-
-   MFEM_ASSERT(da == 0.0 || this->Size() == xp->Size(), "");
-   MFEM_ASSERT(db == 0.0 || this->Size() == yp->Size(), "");
-
-   if (da == 0.0)
-   {
-      if (db == 0.0)
-      {
-         OccaFill(&da);
-      }
-      else
-      {
-         if (this->slice == yp->slice)
-         {
-            // *this *= db
-            ::occa::linalg::operator_mult_eq(slice, db);
-         }
-         else
-         {
-            // *this = db * y
-            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice(),
-                                                         okl_defines);
-            kernel((int)Size(), db, slice, yp->slice);
-         }
-      }
-   }
-   else
-   {
-      if (db == 0.0)
-      {
-         if (this->slice == xp->slice)
-         {
-            // *this *= da
-            ::occa::linalg::operator_mult_eq(slice, da);
-         }
-         else
-         {
-            // *this = da * x
-            ::occa::kernel kernel = axpby1_builder.build(slice.getDevice(),
-                                                         okl_defines);
-            kernel((int)Size(), da, slice, xp->slice);
-         }
-      }
-      else
-      {
-         MFEM_ASSERT(xp->slice != yp->slice, "invalid input");
-         if (this->slice == xp->slice)
-         {
-            // *this = da * (*this) + db * y
-            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice(),
-                                                         okl_defines);
-            kernel((int)Size(), da, db, slice, yp->slice);
-         }
-         else if (this->slice == yp->slice)
-         {
-            // *this = da * x + db * (*this)
-            ::occa::kernel kernel = axpby2_builder.build(slice.getDevice(),
-                                                         okl_defines);
-            kernel((int)Size(), db, da, slice, xp->slice);
-         }
-         else
-         {
-            // *this = da * x + db * y
-            ::occa::kernel kernel = axpby3_builder.build(slice.getDevice(),
-                                                         okl_defines);
-            kernel((int)Size(), da, db, slice, xp->slice, yp->slice);
-         }
-      }
-   }
-}
-
-mfem::Vector Vector::Wrap()
-{
-   return mfem::Vector(*this);
-}
-
-const mfem::Vector Vector::Wrap() const
-{
-   return mfem::Vector(*const_cast<Vector*>(this));
-}
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)

backends/occa/vector.hpp

@@ -1,74 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OCCA_VECTOR_HPP
-#define MFEM_BACKENDS_OCCA_VECTOR_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#include <occa.hpp>
-#include "../base/vector.hpp"
-#include "array.hpp"
-
-namespace mfem
-{
-
-namespace occa
-{
-
-class Vector : virtual public Array, public PVector
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // DLayout layout;
-
-   /**
-       @name Virtual interface
-    */
-   ///@{
-
-   virtual PVector *DoVectorClone(bool copy_data, void **buffer,
-                                  int buffer_type_id) const;
-
-   virtual void DoDotProduct(const PVector &x, void *result,
-                             int result_type_id) const;
-
-   virtual void DoAxpby(const void *a, const PVector &x,
-                        const void *b, const PVector &y,
-                        int ab_type_id);
-
-   ///@}
-   // End: Virtual interface
-
-public:
-   Vector(Layout &lt)
-      : PArray(lt), Array(lt, sizeof(double)), PVector(lt)
-   { }
-
-   mfem::Vector Wrap();
-
-   const mfem::Vector Wrap() const;
-
-#if defined(MFEM_USE_MPI)
-   bool IsParallel() const { return (OccaLayout().OccaEngine().GetComm() != MPI_COMM_NULL); }
-#endif
-};
-
-} // namespace mfem::occa
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OCCA)
-
-#endif // MFEM_BACKENDS_OCCA_VECTOR_HPP

backends/omp/adiffusioninteg.cpp

@@ -1,675 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) &&               \
-   defined(MFEM_USE_OMP) &&                     \
-   defined(MFEM_USE_ACROTENSOR)
-
-#include "adiffusioninteg.hpp"
-
-namespace mfem
-{
-
-namespace omp
-{
-
-PAIntegrator::PAIntegrator(Coefficient &q, FiniteElementSpace &f)
-{
-   Q = &q;
-   ofes = &f;
-   fes = ofes->GetFESpace();
-   onGPU = (ofes->OmpEngine().ExecTarget() == Device);
-   fe = fes->GetFE(0);
-   tfe = dynamic_cast<const TensorBasisElement*>(fe);
-   if (tfe)
-   {
-      tDofMap = tfe->GetDofMap();
-   }
-   else
-   {
-      tDofMap.SetSize(nDof);
-      for (int i = 0; i < nDof; ++i)
-      {
-         tDofMap[i] = i;
-      }
-   }
-
-   nElem  = fes->GetNE();
-   GeomType = fe->GetGeomType();
-   FEOrder = fe->GetOrder();
-   nDim    = fe->GetDim();
-   nDof   = fe->GetDof();
-
-   ElementTransformation *Trans = fes->GetElementTransformation(0);
-   int irorder = 2*fe->GetOrder() + Trans->OrderW();
-   ir = &IntRules.Get(GeomType, irorder);
-   nQuad = ir->GetNPoints();
-   hasTensorBasis = tfe ? true : false;
-
-   if (nDim > 3)
-   {
-      mfem_error("AcroIntegrator tensor computations don't support dim > 3.");
-   }
-}
-
-PAIntegrator::~PAIntegrator()
-{
-
-}
-
-AcroDiffusionIntegrator::AcroDiffusionIntegrator(Coefficient &q, FiniteElementSpace &f) :
-   PAIntegrator(q,f)
-{
-   if (onGPU)
-   {
-      //TE.SetExecutorType("OneOutPerThread");
-      TE.SetExecutorType("Cuda");
-      //TODO:  Set to an existing cuda context if one exists
-   }
-   else
-   {
-      TE.SetExecutorType("CPUInterpreted");
-   }
-
-   const IntegrationRule *ir1D = &IntRules.Get(Geometry::SEGMENT, ir->GetOrder());
-   nDof1D = FEOrder + 1;
-   nQuad1D = ir1D->GetNPoints();
-
-   if (hasTensorBasis)
-   {
-      H1_FECollection fec(FEOrder,1);
-      const FiniteElement *fe1D = fec.FiniteElementForGeometry(Geometry::SEGMENT);
-      mfem::Vector eval(nDof1D);
-      DenseMatrix deval(nDof1D,1);
-      B.Init(nQuad1D, nDof1D);
-      G.Init(nQuad1D, nDof1D);
-      std::vector<int> wdims(nDim, nQuad1D);
-      W.Init(wdims);
-
-      mfem::Vector w(nQuad1D);
-      for (int k = 0; k < nQuad1D; ++k)
-      {
-         const IntegrationPoint &ip = ir1D->IntPoint(k);
-         fe1D->CalcShape(ip, eval);
-         fe1D->CalcDShape(ip, deval);
-
-         B(k,0) = eval(0);
-         B(k,nDof1D-1) = eval(1);
-         G(k,0) = deval(0,0);
-         G(k,nDof1D-1) = deval(1,0);
-         for (int i = 1; i < nDof1D-1; ++i)
-         {
-            B(k,i) = eval(i+1);
-            G(k,i) = deval(i+1,0);
-         }
-         w(k) = ip.weight;
-      }
-
-      if (nDim == 1)
-      {
-         for (int k1 = 0; k1 < nQuad1D; ++k1)
-         {
-            W(k1) = w(k1);
-         }
-      }
-      else if (nDim == 2)
-      {
-         for (int k1 = 0; k1 < nQuad1D; ++k1)
-         {
-            for (int k2 = 0; k2 < nQuad1D; ++k2)
-            {
-               W(k1,k2) = w(k1)*w(k2);
-            }
-         }
-      }
-      else if (nDim == 3)
-      {
-         for (int k1 = 0; k1 < nQuad1D; ++k1)
-         {
-            for (int k2 = 0; k2 < nQuad1D; ++k2)
-            {
-               for (int k3 = 0; k3 < nQuad1D; ++k3)
-               {
-                  W(k1,k2,k3) = w(k1)*w(k2)*w(k3);
-               }
-            }
-         }
-      }
-   }
-   else
-   {
-      mfem::Vector eval(nDof);
-      DenseMatrix deval(nDof,nDim);
-      G.Init(nQuad, nDof,nDim);
-      W.Init(nQuad);
-      for (int k = 0; k < nQuad; ++k)
-      {
-         const IntegrationPoint &ip = ir->IntPoint(k);
-         fe->CalcDShape(ip, deval);
-         for (int i = 0; i < nDof; ++i)
-         {
-            for (int d = 0; d < nDim; ++d)
-            {
-               G(k,i,d) = deval(i,d);
-            }
-         }
-         W(k) = ip.weight;
-      }
-   }
-
-   if (onGPU)
-   {
-      B.MapToGPU();
-      G.MapToGPU();
-      W.MapToGPU();
-   }
-
-   // Assemble in the constructor!
-   BatchedPartialAssemble();
-}
-
-
-AcroDiffusionIntegrator::~AcroDiffusionIntegrator()
-{
-   for (int i = 0; i < Btil.Size(); i++) delete Btil[i];
-}
-
-
-void AcroDiffusionIntegrator::ComputeBTilde()
-{
-   Btil.SetSize(nDim);
-   for (int d = 0; d < nDim; ++d)
-   {
-      Btil[d] = new acro::Tensor(nDim, nDim, nQuad1D, nDof1D, nDof1D);
-      for (int m = 0; m < nDim; ++m)
-      {
-         for (int n = 0; n < nDim; ++n)
-         {
-            acro::Tensor &BGM = (m == d) ? G : B;
-            acro::Tensor &BGN = (n == d) ? G : B;
-            for (int k = 0; k < nQuad1D; ++k)
-            {
-               for (int i = 0; i < nDof1D; ++i)
-               {
-                  for (int j = 0; j < nDof1D; ++j)
-                  {
-                     (*Btil[d])(m, n, k, i, j) = BGM(k,i)*BGN(k,j);
-                  }
-               }
-            }
-         }
-      }
-   }
-}
-
-
-void AcroDiffusionIntegrator::BatchedPartialAssemble()
-{
-   //Initilze the tensors
-   acro::Tensor J,Jinv,Jdet,C;
-   if (hasTensorBasis)
-   {
-      const IntegrationRule *ir1D = &IntRules.Get(Geometry::SEGMENT, ir->GetOrder());
-      IntegrationPoint ip;
-      if (nDim == 1)
-      {
-         D.Init(nElem, nDim, nDim, nQuad1D);
-         J.Init(nElem, nQuad1D, nDim, nDim);
-         Jinv.Init(nElem, nQuad1D, nDim, nDim);
-         Jdet.Init(nElem, nQuad1D);
-         C.Init(nElem, nQuad1D);
-
-         for (int e = 0; e < nElem; ++e)
-         {
-            ElementTransformation *Trans = fes->GetElementTransformation(e);
-            for (int k1 = 0; k1 < nQuad1D; ++k1)
-            {
-               ip.x = ir1D->IntPoint(k1).x;
-               ip.y = 0.0;
-               ip.z = 0.0;
-               Trans->SetIntPoint(&ip);
-               C(e,k1) = Q->Eval(*Trans, ip);
-               const DenseMatrix &JMat = Trans->Jacobian();
-               for (int m = 0; m < nDim; ++m)
-               {
-                  for (int n = 0; n < nDim; ++n)
-                  {
-                     J(e,k1,m,n) = JMat.Elem(m,n);
-                  }
-               }
-            }
-         }
-      }
-      else if (nDim == 2)
-      {
-         D.Init(nElem, nDim, nDim, nQuad1D, nQuad1D);
-         J.Init(nElem, nQuad1D, nQuad1D, nDim, nDim);
-         Jinv.Init(nElem, nQuad1D, nQuad1D, nDim, nDim);
-         Jdet.Init(nElem, nQuad1D, nQuad1D);
-         C.Init(nElem, nQuad1D, nQuad1D);
-
-         for (int e = 0; e < nElem; ++e)
-         {
-            ElementTransformation *Trans = fes->GetElementTransformation(e);
-            for (int k1 = 0; k1 < nQuad1D; ++k1)
-            {
-               for (int k2 = 0; k2 < nQuad1D; ++k2)
-               {
-                  ip.x = ir1D->IntPoint(k1).x;
-                  ip.y = ir1D->IntPoint(k2).y;
-                  ip.z = 0.0;
-                  Trans->SetIntPoint(&ip);
-                  C(e,k1,k2) = Q->Eval(*Trans, ip);
-                  const DenseMatrix &JMat = Trans->Jacobian();
-                  for (int m = 0; m < nDim; ++m)
-                  {
-                     for (int n = 0; n < nDim; ++n)
-                     {
-                        J(e,k1,k2,m,n) = JMat.Elem(m,n);
-                     }
-                  }
-               }
-            }
-         }
-      }
-      else if (nDim == 3)
-      {
-         D.Init(nElem, nDim, nDim, nQuad1D, nQuad1D, nQuad1D);
-         J.Init(nElem, nQuad1D, nQuad1D, nQuad1D, nDim, nDim);
-         Jinv.Init(nElem, nQuad1D, nQuad1D, nQuad1D, nDim, nDim);
-         Jdet.Init(nElem, nQuad1D, nQuad1D, nQuad1D);
-         C.Init(nElem, nQuad1D, nQuad1D, nQuad1D);
-
-         for (int e = 0; e < nElem; ++e)
-         {
-            ElementTransformation *Trans = fes->GetElementTransformation(e);
-            for (int k1 = 0; k1 < nQuad1D; ++k1)
-            {
-               for (int k2 = 0; k2 < nQuad1D; ++k2)
-               {
-                  for (int k3 = 0; k3 < nQuad1D; ++k3)
-                  {
-                     ip.x = ir1D->IntPoint(k1).x;
-                     ip.y = ir1D->IntPoint(k2).y;
-                     ip.z = ir1D->IntPoint(k3).z;
-                     Trans->SetIntPoint(&ip);
-                     C(e,k1,k2,k3) = Q->Eval(*Trans, ip);
-                     const DenseMatrix &JMat = Trans->Jacobian();
-                     for (int m = 0; m < nDim; ++m)
-                     {
-                        for (int n = 0; n < nDim; ++n)
-                        {
-                           J(e,k1,k2,k3,m,n) = JMat.Elem(m,n);
-                        }
-                     }
-                  }
-               }
-            }
-         }
-      }
-   }
-   else
-   {
-      D.Init(nElem, nDim, nDim, nQuad);
-      J.Init(nElem, nQuad, nDim, nDim);
-      Jinv.Init(nElem, nQuad, nDim, nDim);
-      Jdet.Init(nElem, nQuad);
-      C.Init(nElem, nQuad);
-
-      for (int e = 0; e < nElem; ++e)
-      {
-         ElementTransformation *Trans = fes->GetElementTransformation(e);
-         for (int k = 0; k < nQuad; ++k)
-         {
-            const IntegrationPoint &ip = ir->IntPoint(k);
-            Trans->SetIntPoint(&ip);
-            C(e,k) = Q->Eval(*Trans, ip);
-            const DenseMatrix &JMat = Trans->Jacobian();
-            for (int m = 0; m < nDim; ++m)
-            {
-               for (int n = 0; n < nDim; ++n)
-               {
-                  J(e,k,m,n) = JMat.Elem(m,n);
-               }
-            }
-         }
-      }
-   }
-
-   TE.BatchMatrixInvDet(Jinv, Jdet, J);
-
-   if (hasTensorBasis)
-   {
-      if (nDim == 1)
-      {
-         TE("D_e_m_n_k = W_k C_e_k Jdet_e_k Jinv_e_k_m_j Jinv_e_k_n_j",
-            D, W, C, Jdet, Jinv, Jinv);
-      }
-      else if (nDim == 2)
-      {
-         TE("D_e_m_n_k1_k2 = W_k1_k2 C_e_k1_k2 Jdet_e_k1_k2 Jinv_e_k1_k2_m_j Jinv_e_k1_k2_n_j",
-            D, W, C, Jdet, Jinv, Jinv);
-      }
-      else if (nDim == 3)
-      {
-         TE("D_e_m_n_k1_k2_k3 = W_k1_k2_k3 C_e_k1_k2_k3 Jdet_e_k1_k2_k3 Jinv_e_k1_k2_k3_n_j Jinv_e_k1_k2_k3_m_j",
-            D, W, C, Jdet, Jinv, Jinv);
-      }
-   }
-   else
-   {
-      TE("D_e_m_n_k = W_k C_e_k Jdet_e_k Jinv_e_k_m_j Jinv_e_k_n_j",
-         D, W, C, Jdet, Jinv, Jinv);
-   }
-}
-
-
-void AcroDiffusionIntegrator::BatchedAssembleElementMatrices(DenseTensor &elmats)
-{
-   if (hasTensorBasis && Btil.Size() == 0)
-   {
-      ComputeBTilde();
-   }
-
-   if (!D.IsInitialized())
-   {
-      BatchedPartialAssemble();
-   }
-
-   if (!S.IsInitialized())
-   {
-      if (hasTensorBasis)
-      {
-         if (nDim == 1)
-         {
-            S.Init(nElem, nDof1D, nDof1D);
-         }
-         else if (nDim == 2)
-         {
-            S.Init(nElem, nDof1D, nDof1D, nDof1D, nDof1D);
-         }
-         else if (nDim == 3)
-         {
-            S.Init(nElem, nDof1D, nDof1D, nDof1D, nDof1D, nDof1D, nDof1D);
-         }
-      }
-      else
-      {
-         S.Init(nElem, nDof, nDof);
-      }
-      if (onGPU) {S.SwitchToGPU();}
-   }
-
-
-   if (hasTensorBasis) {
-      if (nDim == 1) {
-         TE("S_e_i1_j1 = Btil_m_n_k1_i1_j1 D_e_m_n_k1",
-            S, *Btil[0], D);
-      }
-      else if (nDim == 2)
-      {
-         TE("S_e_i1_i2_j1_j2 = Btil1_m_n_k1_i1_j1 Btil2_m_n_k2_i2_j2 D_e_m_n_k1_k2",
-            S, *Btil[0], *Btil[1], D);
-      }
-      else if (nDim == 3)
-      {
-         TE("S_e_i1_i2_i3_j1_j2_j3 = Btil1_m_n_k1_i1_j1 Btil2_m_n_k2_i2_j2 Btil3_m_n_k3_i3_j3 D_e_m_n_k1_k2_k3",
-            S, *Btil[0], *Btil[1], *Btil[2], D);
-      }
-   }
-   else
-   {
-      TE("S_e_i_j = G_k_i_m G_k_i_n D_e_m_n_k",
-         S, G, G, D);
-   }
-
-   S.MoveFromGPU();
-   for (int e = 0; e < nElem; ++e)
-   {
-      for (int ei = 0; ei < nDof; ++ei)
-      {
-         for (int ej = 0; ej < nDof; ++ej)
-         {
-            elmats(tDofMap[ei], tDofMap[ej], e) = S[e*nDof*nDof + ei*nDof + ej];
-         }
-      }
-   }
-}
-
-
-void AcroDiffusionIntegrator::ComputeElementMatrices(Vector &elmats)
-{
-   if (hasTensorBasis && Btil.Size() == 0)
-   {
-      ComputeBTilde();
-   }
-
-   if (!D.IsInitialized())
-   {
-      BatchedPartialAssemble();
-   }
-
-   if (!S.IsInitialized())
-   {
-      if (hasTensorBasis)
-      {
-         if (nDim == 1)
-         {
-            S.Init(nElem, nDof1D, nDof1D);
-         }
-         else if (nDim == 2)
-         {
-            S.Init(nElem, nDof1D, nDof1D, nDof1D, nDof1D);
-         }
-         else if (nDim == 3)
-         {
-            S.Init(nElem, nDof1D, nDof1D, nDof1D, nDof1D, nDof1D, nDof1D);
-         }
-      }
-      else
-      {
-         S.Init(nElem, nDof, nDof);
-      }
-      if (onGPU) {S.SwitchToGPU();}
-   }
-
-
-   if (hasTensorBasis) {
-      if (nDim == 1) {
-         TE("S_e_i1_j1 += Btil_m_n_k1_i1_j1 D_e_m_n_k1",
-            S, *Btil[0], D);
-      }
-      else if (nDim == 2)
-      {
-         TE("S_e_i1_i2_j1_j2 += Btil1_m_n_k1_i1_j1 Btil2_m_n_k2_i2_j2 D_e_m_n_k1_k2",
-            S, *Btil[0], *Btil[1], D);
-      }
-      else if (nDim == 3)
-      {
-         TE("S_e_i1_i2_i3_j1_j2_j3 += Btil1_m_n_k1_i1_j1 Btil2_m_n_k2_i2_j2 Btil3_m_n_k3_i3_j3 D_e_m_n_k1_k2_k3",
-            S, *Btil[0], *Btil[1], *Btil[2], D);
-      }
-   }
-   else
-   {
-      TE("S_e_i_j += G_k_i_m G_k_i_n D_e_m_n_k",
-         S, G, G, D);
-   }
-
-   S.MoveFromGPU();
-
-   double *edata = elmats.GetData<double>();
-   for (int e = 0; e < nElem; ++e)
-   {
-      const int e_offset = e * nDof * nDof;
-      for (int ei = 0; ei < nDof; ++ei)
-      {
-         const int offset = e_offset + ei * tDofMap[ei] * nDof;
-         for (int ej = 0; ej < nDof; ++ej)
-         {
-            const int index = offset + tDofMap[ej];
-            edata[index] = S[e*nDof*nDof + ei*nDof + ej];
-         }
-      }
-   }
-}
-
-void AcroDiffusionIntegrator::ReassembleOperator()
-{
-   BatchedPartialAssemble();
-}
-
-void AcroDiffusionIntegrator::PAMult(const Vector &x, Vector &y)
-{
-   MFEM_ASSERT(hasTensorBasis,"AcroDiffusionIntegrator PAMult on simplices not supported");
-
-   if (!U.IsInitialized())
-   {
-      // NOTE: x and y are already sized for the fespace in the constructor
-      double *Xptr = const_cast<double*>(x.GetData<double>());
-      double *Yptr = y.GetData<double>();
-      if (nDim == 1) {
-         X.Init(nElem,nDof1D,Xptr,Xptr,onGPU);
-         Y.Init(nElem,nDof1D,Yptr,Yptr,onGPU);
-         U.Init(nDim, nElem, nQuad1D);
-         Z.Init(nDim, nElem, nQuad1D);
-         if (onGPU)
-         {
-            U.SwitchToGPU();
-            Z.SwitchToGPU();
-         }
-      }
-      else if (nDim == 2)
-      {
-         X.Init(nElem,nDof1D,nDof1D,Xptr,Xptr,onGPU);
-         Y.Init(nElem,nDof1D,nDof1D,Yptr,Yptr,onGPU);
-         U.Init(nDim, nElem, nQuad1D, nQuad1D);
-         Z.Init(nDim, nElem, nQuad1D, nQuad1D);
-         T1.Init(nElem,nDof1D,nQuad1D);
-         if (onGPU)
-         {
-            U.SwitchToGPU();
-            Z.SwitchToGPU();
-            T1.SwitchToGPU();
-         }
-      }
-      else if (nDim == 3)
-      {
-         X.Init(nElem,nDof1D,nDof1D,nDof1D,Xptr,Xptr,onGPU);
-         Y.Init(nElem,nDof1D,nDof1D,nDof1D,Yptr,Yptr,onGPU);
-         U.Init(nDim, nElem, nQuad1D, nQuad1D, nQuad1D);
-         Z.Init(nDim, nElem, nQuad1D, nQuad1D, nQuad1D);
-         T1.Init(nElem, nDof1D, nQuad1D, nQuad1D);
-         T2.Init(nElem, nDof1D, nDof1D, nQuad1D);
-         if (onGPU)
-         {
-            U.SwitchToGPU();
-            Z.SwitchToGPU();
-            T1.SwitchToGPU();
-            T2.SwitchToGPU();
-         }
-      }
-   }
-   else
-   {
-      // NOTE: x and y are already sized for the fespace in the constructor
-      double *Xptr = const_cast<double*>(x.GetData<double>());
-      double *Yptr = y.GetData<double>();
-      X.Retarget(Xptr,Xptr);
-      Y.Retarget(Yptr,Yptr);
-   }
-
-   acro::SliceTensor U1,U2,U3,Z1,Z2,Z3;
-   if (nDim == 1)
-   {
-      TE("U_n_e_k1 = G_k1_i1 X_e_i1", U, G, X);
-      TE("Z_m_e_k1 = D_e_m_n_k1 U_n_e_k1", Z, D, U);
-      TE("Y_e_i1 = G_k1_i1 Z_m_e_k1", Y, G, Z);
-   }
-   else if (nDim == 2)
-   {
-      U1.SliceInit(U, 0); U2.SliceInit(U, 1);
-      Z1.SliceInit(Z, 0); Z2.SliceInit(Z, 1);
-
-      //U1_e_k1_k2 = G_k1_i1 B_k2_i2 X_e_i1_i2
-      TE("BX_e_i1_k2 = B_k2_i2 X_e_i2_i1", T1, B, X);
-      TE("U1_e_k1_k2 = G_k1_i1 BX_e_i1_k2", U1, G, T1);
-
-      //U2_e_k1_k2 = B_k1_i1 G_k2_i2 X_e_i1_i2
-      TE("GX_e_i1_k2 = G_k2_i2 X_e_i2_i1", T1, G, X);
-      TE("U2_e_k1_k2 = B_k1_i1 GX_e_i1_k2", U2, B, T1);
-
-      TE("Z_m_e_k1_k2 = D_e_m_n_k1_k2 U_n_e_k1_k2", Z, D, U);
-
-      //Y_e_i1_i2 = G_k1_i1 B_k2_i2 Z1_e_k1_k2
-      TE("BZ1_e_i2_k1 = B_k2_i2 Z1_e_k1_k2", T1, B, Z1);
-      TE("Y_e_i2_i1 = G_k1_i1 BZ1_e_i2_k1", Y, G, T1);
-
-      //Y_e_i1_i2 += B_k1_i1 G_k2_i2 Z2_e_k1_k2
-      TE("GZ2_e_i2_k1 = G_k2_i2 Z2_e_k1_k2", T1, G, Z2);
-      TE("Y_e_i2_i1 += B_k1_i1 GZ2_e_i2_k1", Y, B, T1);
-   }
-   else if (nDim == 3)
-   {
-      U1.SliceInit(U, 0); U2.SliceInit(U, 1); U3.SliceInit(U, 2);
-      Z1.SliceInit(Z, 0); Z2.SliceInit(Z, 1); Z3.SliceInit(Z, 2);
-
-      TE.BeginMultiKernelLaunch();
-      //U1_e_k1_k2_k3 = G_k1_i1 B_k2_i2 B_k3_i3 X_e_i1_i2_i3
-      TE("T2_e_i1_i2_k3 = B_k3_i3 X_e_i1_i2_i3", T2, B, X);
-      TE("T1_e_i1_k2_k3 = B_k2_i2 T2_e_i1_i2_k3", T1, B, T2);
-      TE("U1_e_k1_k2_k3 = G_k1_i1 T1_e_i1_k2_k3", U1, G, T1);
-
-      //U2_e_k1_k2_k3 = B_k1_i1 G_k2_i2 B_k3_i3 X_e_i1_i2_i3
-      TE("T1_e_i1_k2_k3 = G_k2_i2 T2_e_i1_i2_k3", T1, G, T2);
-      TE("U2_e_k1_k2_k3 = B_k1_i1 T1_e_i1_k2_k3", U2, B, T1);
-
-      //U3_e_k1_k2_k3 = B_k1_i1 B_k2_i2 G_k3_i3 X_e_i1_i2_i3
-      TE("T2_e_i1_i2_k3 = G_k3_i3 X_e_i1_i2_i3", T2, G, X);
-      TE("T1_e_i1_k2_k3 = B_k2_i2 T2_e_i1_i2_k3", T1, B, T2);
-      TE("U3_e_k1_k2_k3 = B_k1_i1 T1_e_i1_k2_k3", U3, B, T1);
-
-      TE("Z_m_e_k1_k2_k3 = D_e_m_n_k1_k2_k3 U_n_e_k1_k2_k3", Z, D, U);
-
-      //Y_e_i1_i2_i3 =  G_k1_i1 B_k2_i2 B_k3_i3 Z1_e_k1_k2_k3
-      TE("T1_e_i3_k1_k2 = B_k3_i3 Z1_e_k1_k2_k3", T1, B, Z1);
-      TE("T2_e_i2_i3_k1 = B_k2_i2 T1_e_i3_k1_k2", T2, B, T1);
-      TE("Y_e_i1_i2_i3 = G_k1_i1 T2_e_i2_i3_k1", Y, G, T2);
-
-      //Y_e_i1_i2_i3 +=  B_k1_i1 G_k2_i2 B_k3_i3 Z2_e_k1_k2_k3
-      TE("T1_e_i3_k1_k2 = B_k3_i3 Z2_e_k1_k2_k3", T1, B, Z2);
-      TE("T2_e_i2_i3_k1 = G_k2_i2 T1_e_i3_k1_k2", T2, G, T1);
-      TE("Y_e_i1_i2_i3 += B_k1_i1 T2_e_i2_i3_k1", Y, B, T2);
-
-      //Y_e_i1_i2_i3 +=  B_k1_i1 B_k2_i2 G_k3_i3 Z3_e_k1_k2_k3
-      TE("T1_e_i3_k1_k2 = G_k3_i3 Z3_e_k1_k2_k3", T1, G, Z3);
-      TE("T2_e_i2_i3_k1 = B_k2_i2 T1_e_i3_k1_k2", T2, B, T1);
-      TE("Y_e_i1_i2_i3 += B_k1_i1 T2_e_i2_i3_k1", Y, B, T2);
-      TE.EndMultiKernelLaunch();
-   }
-}
-
-void AcroDiffusionIntegrator::MultAdd(const Vector &x, Vector &y) const
-{
-   const_cast<AcroDiffusionIntegrator*>(this)->PAMult(x, y);
-}
-
-void AcroDiffusionIntegrator::MultTransposeAdd(const Vector &x, Vector &y) const
-{
-   mfem_error("Not supported");
-}
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif

backends/omp/adiffusioninteg.hpp

@@ -1,95 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OMP_ADIFFUSIONINTEG_HPP
-#define MFEM_BACKENDS_OMP_ADIFFUSIONINTEG_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) &&               \
-   defined(MFEM_USE_OMP) &&                     \
-   defined(MFEM_USE_ACROTENSOR)
-
-#include "../../fem/bilininteg.hpp"
-#include "../../fem/fem.hpp"
-#include "vector.hpp"
-#include "fespace.hpp"
-#include "bilinearform.hpp"
-#include "AcroTensor.hpp"
-
-namespace mfem
-{
-
-namespace omp
-{
-
-class PAIntegrator : public TensorBilinearFormIntegrator
-{
-protected:
-   Coefficient *Q;
-   FiniteElementSpace *ofes;
-   mfem::FiniteElementSpace *fes;
-   const FiniteElement *fe;
-   const TensorBasisElement *tfe;
-   const IntegrationRule *ir;
-   mfem::Array<int> tDofMap;
-   int GeomType;
-   int FEOrder;
-   bool onGPU;
-   bool hasTensorBasis;
-   int nDim;
-   int nElem;
-   int nDof;
-   int nQuad;
-
-public:
-   PAIntegrator(Coefficient &q, FiniteElementSpace &f);
-   virtual ~PAIntegrator();
-};
-
-class AcroDiffusionIntegrator : public PAIntegrator
-{
-private:
-   acro::TensorEngine TE;
-   int nDof1D;
-   int nQuad1D;
-
-   acro::Tensor B, G;         //Basis and dbasis evaluated on the quad points
-   acro::Tensor W;            //Integration weights
-   mfem::Array<acro::Tensor*> Btil; //Btilde used to compute stiffness matrix
-   acro::Tensor D;            //Product of integration weight, physical consts, and element shape info
-   acro::Tensor S;            //The assembled local stiffness matrices
-   acro::Tensor U, Z, T1, T2; //Intermediate computations for tensor product partial assembly
-   acro::Tensor X, Y;
-
-   void ComputeBTilde();
-
-public:
-   AcroDiffusionIntegrator(BilinearFormIntegrator *integ);
-   AcroDiffusionIntegrator(Coefficient &q, FiniteElementSpace &f);
-   virtual ~AcroDiffusionIntegrator();
-
-   void BatchedPartialAssemble();
-   void BatchedAssembleElementMatrices(DenseTensor &elmats);
-   void ComputeElementMatrices(Vector &elmats);
-   void PAMult(const Vector &x, Vector &y);
-   virtual void MultTransposeAdd(const Vector &x, Vector &y) const;
-   virtual void MultAdd(const Vector &x, Vector &y) const;
-   virtual void ReassembleOperator();
-};
-
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif
-
-#endif

backends/omp/array.cpp

@@ -1,128 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#include <cstring>
-#include "array.hpp"
-
-namespace mfem
-{
-
-namespace omp
-{
-
-PArray *Array::DoClone(bool copy_data, void **buffer,
-                       std::size_t item_size) const
-{
-   Array *new_array = new Array(OmpLayout(), item_size);
-   if (copy_data)
-   {
-      if (!ComputeOnDevice())
-         std::memcpy(new_array->GetData<void>(), data, bytes);
-      else
-      {
-         char *new_data = new_array->GetData<char>();
-         const bool use_target = ComputeOnDevice();
-         const bool use_parallel = Size() > 1000;
-#pragma omp target teams distribute parallel for       \
-   if (target: use_target) if (parallel: use_parallel) \
-   is_device_ptr(new_data)
-         for (std::size_t i = 0; i < bytes; i++) new_data[i] = data[i];
-      }
-   }
-   if (buffer)
-   {
-      *buffer = new_array->GetData<void>();
-   }
-   return new_array;
-}
-
-int Array::DoResize(PLayout &new_layout, void **buffer,
-                    std::size_t item_size)
-{
-   MFEM_ASSERT(dynamic_cast<Layout *>(&new_layout) != NULL,
-               "new_layout is not an OMP Layout");
-   Layout *lt = static_cast<Layout *>(&new_layout);
-   layout.Reset(lt); // Reset() checks if the pointer is the same
-   int err = ResizeData(lt, item_size);
-   if (!err && buffer)
-   {
-      *buffer = GetData<void>();
-   }
-   return err;
-}
-
-void *Array::DoPullData(void *buffer, std::size_t item_size)
-{
-   // called only when Size() != 0
-
-   if (!IsUnifiedMemory() && ComputeOnDevice() && (buffer != NULL))
-   {
-#pragma omp target update from(data)
-      std::memcpy(buffer, data, bytes);
-   }
-   else
-   {
-      buffer = data;
-   }
-
-   return buffer;
-}
-
-void Array::DoFill(const void *value_ptr, std::size_t item_size)
-{
-   // called only when Size() != 0
-
-   switch (item_size)
-   {
-   case sizeof(int):
-      OmpFill((const int *)value_ptr);
-      break;
-   case sizeof(double):
-      OmpFill((const double *)value_ptr);
-      break;
-   default:
-      MFEM_ABORT("item_size = " << item_size << " is not supported");
-   }
-}
-
-void Array::DoPushData(const void *src_buffer, std::size_t item_size)
-{
-   // called only when Size() != 0
-
-   std::memcpy(data, (char *) src_buffer, bytes);
-
-   if ((!IsUnifiedMemory() && ComputeOnDevice()) && (data != src_buffer))
-   {
-#pragma omp target update to(data)
-   }
-}
-
-void Array::DoAssign(const PArray &src, std::size_t item_size)
-{
-   // called only when Size() != 0
-
-   // Note: static_cast can not be used here since PArray is a virtual base
-   //       class.
-   const Array *source = dynamic_cast<const Array *>(&src);
-   MFEM_ASSERT(source != NULL, "invalid source Array type");
-   MFEM_ASSERT(Size() == source->Size(), "");
-   // All arrays from this engine are of the same type, so we can simply check *this and assume the same is used in src.
-   DoPushData(source->GetData<void>(), item_size);
-}
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/array.hpp

@@ -1,143 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OMP_ARRAY_HPP
-#define MFEM_BACKENDS_OMP_ARRAY_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#include "layout.hpp"
-#include "../base/array.hpp"
-
-namespace mfem
-{
-
-namespace omp
-{
-
-class Array : public virtual mfem::PArray
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // DLayout layout;
-
-   bool own_data;
-   std::size_t bytes;
-   char *data;
-
-   //
-   // Virtual interface
-   //
-
-   virtual void *DoGetData() const { return (void *) data; }
-
-   virtual PArray *DoClone(bool copy_data, void **buffer,
-                           std::size_t item_size) const;
-
-   virtual int DoResize(PLayout &new_layout, void **buffer,
-                        std::size_t item_size);
-
-   virtual void *DoPullData(void *buffer, std::size_t item_size);
-
-   virtual void DoFill(const void *value_ptr, std::size_t item_size);
-
-   virtual void DoPushData(const void *src_buffer, std::size_t item_size);
-
-   virtual void DoAssign(const PArray &src, std::size_t item_size);
-
-   //
-   // Auxiliary methods
-   //
-
-   inline int ResizeData(const Layout *lt, std::size_t item_size);
-
-   inline bool IsUnifiedMemory() const { return OmpLayout().OmpEngine().UnifiedMemory(); }
-
-   template <typename T>
-   void OmpFill(const T *pval)
-   {
-      T *ptr = (T*) data;
-      T val = *pval;
-      const bool use_target = ComputeOnDevice();
-      const bool use_parallel = (use_target || layout->Size() > 1000);
-      const std::size_t size = layout->Size();
-#pragma omp target teams distribute parallel for        \
-   if (target: use_target)                              \
-   if (parallel: use_parallel) map (to: ptr, val)
-      for (int i = 0; i < size; i++) ptr[i] = val;
-   }
-
-public:
-   Array(Layout &lt, std::size_t item_size)
-      : PArray(lt),
-        own_data(true),
-        bytes(lt.Size() * item_size),
-        data(static_cast<char *>(lt.Alloc(bytes)))
-   {
-#pragma omp target enter data map(alloc:data[:bytes]) if (!IsUnifiedMemory() && ComputeOnDevice())
-   }
-
-   Array(const Array &array)
-      : PArray(array.GetLayout()),
-        own_data(false),
-        bytes(array.bytes),
-        data(array.data) { }
-
-   inline bool ComputeOnDevice() const { return (OmpLayout().OmpEngine().ExecTarget() == Device); }
-
-   virtual ~Array()
-   {
-#pragma omp target exit data map(delete:data[:bytes]) if (!IsUnifiedMemory() && ComputeOnDevice())
-      if (own_data) layout->As<Layout>().Dealloc(data);
-   }
-
-   inline void MakeRef(Array &master);
-
-   Layout &OmpLayout() const
-   { return *static_cast<Layout *>(layout.Get()); }
-};
-
-
-//
-// Inline methods
-//
-
-inline int Array::ResizeData(const Layout *lt, std::size_t item_size)
-{
-   const std::size_t new_bytes = lt->Size() * item_size;
-   if (bytes < new_bytes)
-   {
-#pragma omp target exit data map(delete:data)
-      OmpLayout().Dealloc(data);
-      data = static_cast<char *>(OmpLayout().Alloc(new_bytes));
-      MFEM_VERIFY(data != NULL, "");
-      // If memory allocation fails - an exception is thrown.
-#pragma omp target enter data map(alloc:data[:new_bytes])
-   }
-   return 0;
-}
-
-inline void Array::MakeRef(Array &master)
-{
-   layout = master.layout;
-   data = master.data;
-}
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#endif // MFEM_BACKENDS_OMP_ARRAY_HPP

backends/omp/backend.cpp

@@ -1,46 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#include "backend.hpp"
-#include "engine.hpp"
-
-namespace mfem
-{
-
-namespace omp
-{
-
-bool Backend::Supports(const std::string &engine_spec) const
-{
-   return true;
-}
-
-mfem::Engine *Create(const std::string &engine_spec)
-{
-   return new Engine(engine_spec);
-}
-
-#ifdef MFEM_USE_MPI
-mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec)
-{
-   return new Engine(comm, engine_spec);
-}
-#endif
-
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/backend.hpp

@@ -1,48 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OMP_BACKEND_HPP
-#define MFEM_BACKENDS_OMP_BACKEND_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-// Only the Backend and Engine classes should be exposed through "backend.hpp"
-#include "../base/backend.hpp"
-#include "engine.hpp"
-
-namespace mfem
-{
-
-namespace omp
-{
-
-class Backend : public mfem::Backend
-{
-public:
-   virtual ~Backend();
-
-   virtual bool Supports(const std::string &engine_spec) const;
-
-   virtual mfem::Engine *Create(const std::string &engine_spec);
-
-#ifdef MFEM_USE_MPI
-   virtual mfem::Engine *Create(MPI_Comm comm, const std::string &engine_spec);
-#endif
-};
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#endif // MFEM_BACKENDS_OMP_BACKEND_HPP

backends/omp/bilinearform.cpp

@@ -1,399 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#include "backend.hpp"
-#include "bilinearform.hpp"
-#include "adiffusioninteg.hpp"
-
-namespace mfem
-{
-
-namespace omp
-{
-
-BilinearForm::~BilinearForm()
-{
-   // Make sure all integrators free their data
-   for (int i = 0; i < tbfi.Size(); i++) delete tbfi[i];
-
-   delete element_matrices;
-}
-
-void BilinearForm::TransferIntegrators()
-{
-   mfem::Array<mfem::BilinearFormIntegrator*> &dbfi = *bform->GetDBFI();
-   for (int i = 0; i < dbfi.Size(); i++)
-   {
-      std::string integ_name(dbfi[i]->Name());
-      Coefficient *scal_coeff = dbfi[i]->GetScalarCoefficient();
-      // ConstantCoefficient *const_coeff =
-      //    dynamic_cast<ConstantCoefficient*>(scal_coeff);
-      // // TODO: other types of coefficients ...
-      // double val = const_coeff ? const_coeff->constant : 1.0;
-
-      if (integ_name == "(undefined)")
-      {
-         MFEM_ABORT("BilinearFormIntegrator does not define Name()");
-      }
-      else if (integ_name == "diffusion")
-      {
-         switch (OmpEngine().IntegType())
-         {
-         case Acrotensor:
-            tbfi.Append(new AcroDiffusionIntegrator(*scal_coeff, bform->FESpace()->Get_PFESpace()->As<FiniteElementSpace>()));
-            break;
-         default:
-            mfem_error("integrator is not supported for any MultType");
-            break;
-         }
-      }
-      else
-      {
-         MFEM_ABORT("BilinearFormIntegrator [Name() = " << integ_name
-                    << "] is not supported");
-      }
-   }
-}
-
-void BilinearForm::InitRHS(const mfem::Array<int> &ess_tdof_list,
-                           mfem::Vector &mfem_x, mfem::Vector &mfem_b,
-                           mfem::OperatorHandle &A,
-                           mfem::Vector &mfem_X, mfem::Vector &mfem_B,
-                           int copy_interior) const
-{
-   const mfem::Operator *P = GetProlongation();
-   const mfem::Operator *R = GetRestriction();
-
-   if (P)
-   {
-      // Variational restriction with P
-      mfem_B.Resize(P->InLayout());
-      P->MultTranspose(mfem_b, mfem_B);
-      mfem_X.Resize(R->OutLayout());
-      R->Mult(mfem_x, mfem_X);
-   }
-   else
-   {
-      // rap, X and B point to the same data as this, x and b
-      mfem_X.MakeRef(mfem_x);
-      mfem_B.MakeRef(mfem_b);
-   }
-
-   if (A.Type() != mfem::Operator::ANY_TYPE)
-   {
-      A.EliminateBC(mat_e, ess_tdof_list, mfem_X, mfem_B);
-   }
-
-   if (!copy_interior && ess_tdof_list.Size() > 0)
-   {
-      Vector &X = mfem_X.Get_PVector()->As<Vector>();
-      const Array &constraint_list = ess_tdof_list.Get_PArray()->As<Array>();
-
-      double *X_data = X.GetData<double>();
-      const int* constraint_data = constraint_list.GetData<int>();
-
-      Vector subvec(constraint_list.OmpLayout());
-      double *subvec_data = subvec.GetData<double>();
-
-      const std::size_t num_constraint = constraint_list.Size();
-      const bool use_target = constraint_list.ComputeOnDevice();
-      const bool use_parallel = (use_target || num_constraint > 1000);
-
-      // This operation is a general version of mfem::Vector::SetSubVectorComplement()
-      // {
-#pragma omp target teams distribute parallel for        \
-   map(to: subvec_data, constraint_data, X_data)        \
-   if (target: use_target)                              \
-   if (parallel: use_parallel)
-      for (std::size_t i = 0; i < num_constraint; i++) subvec_data[i] = X_data[constraint_data[i]];
-
-      X.Fill(0.0);
-
-#pragma omp target teams distribute parallel for        \
-   map(to: X_data, constraint_data, subvec_data)        \
-   if (target: use_target)                              \
-   if (parallel: use_parallel)
-      for (std::size_t i = 0; i < num_constraint; i++) X_data[constraint_data[i]] = subvec_data[i];
-      // }
-   }
-
-   if (A.Type() == mfem::Operator::ANY_TYPE)
-   {
-      ConstrainedOperator *A_constrained = static_cast<ConstrainedOperator*>(A.Ptr());
-      A_constrained->EliminateRHS(mfem_X, mfem_B);
-   }
-}
-
-
-bool BilinearForm::Assemble()
-{
-   if (!has_assembled)
-   {
-      TransferIntegrators();
-      has_assembled = true;
-   }
-
-   return true;
-}
-
-void BilinearForm::ComputeElementMatrices()
-{
-   // Only called if performing full assembly
-   const int nelements = trial_fes->GetFESpace()->GetNE();
-   const int trial_ndofs = trial_fes->GetFESpace()->GetFE(0)->GetDof() * trial_fes->GetFESpace()->GetVDim();
-   const int test_ndofs = test_fes->GetFESpace()->GetFE(0)->GetDof() * test_fes->GetFESpace()->GetVDim();
-   const std::size_t length = nelements * trial_ndofs * test_ndofs;
-
-   if (!element_matrices) element_matrices = new mfem::Vector(*(new Layout(OmpEngine(), length)));
-   else element_matrices->Push();
-
-   element_matrices->Fill(0.0);
-   Vector &elmats = element_matrices->Get_PVector()->As<Vector>();
-
-   tbfi[0]->ComputeElementMatrices(elmats);
-
-   if (tbfi.Size() > 1)
-   {
-      for (int k = 1; k < tbfi.Size(); k++)
-      {
-         tbfi[k]->ComputeElementMatrices(elmats);
-      }
-   }
-}
-
-void BilinearForm::FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
-                                    mfem::OperatorHandle &A)
-{
-   if (A.Type() == mfem::Operator::ANY_TYPE)
-   {
-      // FIXME: Support different test and trial spaces (MixedBilinearForm)
-      const mfem::Operator *P = GetProlongation();
-
-      mfem::Operator *rap = this;
-      if (P != NULL) rap = new mfem::RAPOperator(*P, *this, *P);
-
-      A.Reset(new ConstrainedOperator(rap, ess_tdof_list, (rap != this)));
-
-      return;
-   }
-   else
-   {
-      // ASSUMPTION: some sort of sparse matrix
-      // Compute the local matrices (stored in bform->element_matrices
-      ComputeElementMatrices();
-      bform->AllocateMatrix();
-      mfem::SparseMatrix &mat = bform->SpMat();
-
-      element_matrices->Pull();
-      double *data = element_matrices->GetData();
-
-      const bool skip_zeros = true;
-      mfem::Array<int> tr_vdofs, te_vdofs;
-      for (int i = 0; i < trial_fes->GetFESpace()->GetNE(); i++)
-      {
-         trial_fes->GetFESpace()->GetElementVDofs(i, tr_vdofs);
-         test_fes->GetFESpace()->GetElementVDofs(i, te_vdofs);
-         const mfem::DenseMatrix elmat(data, te_vdofs.Size(), tr_vdofs.Size());
-         mat.AddSubMatrix(te_vdofs, tr_vdofs, elmat, skip_zeros);
-         data += tr_vdofs.Size() * te_vdofs.Size();
-      }
-   }
-
-   if (A.Type() == mfem::Operator::MFEM_SPARSEMAT)
-   {
-      // This works because the FormSystemMatrix call with an explicit
-      // SparseMatrix doesnt call the backend version... This might
-      // change in the future.
-      bform->FormSystemMatrix(ess_tdof_list, static_cast<mfem::SparseMatrix&>(*A.Ptr()));
-   }
-#ifdef MFEM_USE_MPI
-   else if (A.Type() == mfem::Operator::Hypre_ParCSR)
-   {
-      mfem::SparseMatrix &mat = bform->SpMat();
-      mfem::ParBilinearForm *pbform = dynamic_cast<mfem::ParBilinearForm*>(bform);
-
-      const bool skip_zeros = false;
-      mat.Finalize(skip_zeros);
-
-      // -------- FOR SOME VERY AGGREVATING REASON THIS DOESN'T WORK ---------
-      // mfem::ParFiniteElementSpace *pfes = pbform->ParFESpace();
-      // OperatorHandle dA(Operator::Hypre_ParCSR);
-      // // construct a parallel block-diagonal matrix 'A' based on 'a'
-      // dA.MakeSquareBlockDiag(pfes->GetComm(), *engine->MakeLayout(pfes->GlobalTrueVSize()),
-      //                        pfes->GetDofOffsets(), &mat);
-      // OperatorHandle Ph(pfes->Dof_TrueDof_Matrix());
-      // A.MakePtAP(dA, Ph);
-      // A.SetOperatorOwner(false);
-      // -------- BUT THIS DOES ---------
-      pbform->ParallelAssemble(A, &mat);
-      A.SetOperatorOwner(false);
-      // ---------------------
-      mat.Clear();
-      mat_e.Clear();
-      std::cout << "operator size (FormSystemMatrix): " << A.Ptr()->InLayout()->Size() << " " << A.Ptr()->OutLayout()->Size() << std::endl;
-
-      mat_e.EliminateRowsCols(A, ess_tdof_list);
-   }
-#endif
-   else
-   {
-      MFEM_ABORT("Operator::Type is not supported, type = " << A.Type());
-   }
-}
-
-void BilinearForm::FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
-                                    mfem::Vector &x, mfem::Vector &b,
-                                    mfem::OperatorHandle &A, mfem::Vector &X, mfem::Vector &B,
-                                    int copy_interior)
-{
-   FormSystemMatrix(ess_tdof_list, A);
-   std::cout << "operator size (FormLinearSystem 1): " << A.Ptr()->InLayout()->Size() << " " << A.Ptr()->OutLayout()->Size() << std::endl;
-   InitRHS(ess_tdof_list, x, b, A, X, B, copy_interior);
-}
-
-void BilinearForm::RecoverFEMSolution(const mfem::Vector &X, const mfem::Vector &b,
-                                      mfem::Vector &x)
-{
-   const mfem::Operator *P = GetProlongation();
-   if (P)
-   {
-      // Apply conforming prolongation
-      x.Resize(P->OutLayout());
-      P->Mult(X, x);
-   }
-   // Otherwise X and x point to the same data
-}
-
-void BilinearForm::Mult(const mfem::Vector &x, mfem::Vector &y) const
-{
-   trial_fes->ToEVector(x.Get_PVector()->As<Vector>(), x_local);
-
-   y_local.Fill<double>(0.0);
-   for (int i = 0; i < tbfi.Size(); i++) tbfi[i]->MultAdd(x_local, y_local);
-
-   test_fes->ToLVector(y_local, y.Get_PVector()->As<Vector>());
-}
-
-void BilinearForm::MultTranspose(const mfem::Vector &x, mfem::Vector &y) const
-{ mfem_error("mfem::omp::BilinearForm::MultTranspose() is not supported!"); }
-
-
-ConstrainedOperator::ConstrainedOperator(mfem::Operator *A_,
-                                         const mfem::Array<int> &constraint_list_,
-                                         bool own_A_)
-   : Operator(A_->InLayout()->As<Layout>()),
-     A(A_),
-     own_A(own_A_),
-     // FIXME: @dudouit1 has a general fix for this
-     constraint_list(constraint_list_.Get_PArray()->As<Array>()),
-     z(OutLayout()->As<Layout>()),
-     w(OutLayout()->As<Layout>()),
-     mfem_z((z.DontDelete(), z)),
-     mfem_w((w.DontDelete(), w)) { }
-
-void ConstrainedOperator::EliminateRHS(const mfem::Vector &mfem_x, mfem::Vector &mfem_b) const
-{
-   w.Fill<double>(0.0);
-
-   const Vector &x = mfem_x.Get_PVector()->As<Vector>();
-   Vector &b = mfem_b.Get_PVector()->As<Vector>();
-
-   const double *x_data = x.GetData<double>();
-   double *b_data = b.GetData<double>();
-   double *w_data = w.GetData<double>();
-   const int* constraint_data = constraint_list.GetData<int>();
-
-   const std::size_t num_constraint = constraint_list.Size();
-   const bool use_target = constraint_list.ComputeOnDevice();
-   const bool use_parallel = (use_target || num_constraint > 1000);
-
-   if (num_constraint > 0)
-   {
-#pragma omp target teams distribute parallel for             \
-   map(to: w_data, constraint_data, x_data)                  \
-   if (target: use_target)                                   \
-   if (parallel: use_parallel)
-      for (std::size_t i = 0; i < num_constraint; i++)
-         w_data[constraint_data[i]] = x_data[constraint_data[i]];
-   }
-
-   A->Mult(mfem_w, mfem_z);
-
-   b.Axpby<double>(1.0, b, -1.0, z);
-
-   if (num_constraint > 0)
-   {
-#pragma omp target teams distribute parallel for        \
-   map(to: b_data, constraint_data, x_data)             \
-   if (target: use_target)                              \
-   if (parallel: use_parallel)
-      for (std::size_t i = 0; i < num_constraint; i++)
-         b_data[constraint_data[i]] = x_data[constraint_data[i]];
-   }
-}
-
-void ConstrainedOperator::Mult(const mfem::Vector &mfem_x, mfem::Vector &mfem_y) const
-{
-   if (constraint_list.Size() == 0)
-   {
-      A->Mult(mfem_x, mfem_y);
-      return;
-   }
-
-   const Vector &x = mfem_x.Get_PVector()->As<Vector>();
-   Vector &y = mfem_y.Get_PVector()->As<Vector>();
-
-   const double *x_data = x.GetData<double>();
-   double *y_data = y.GetData<double>();
-   double *z_data = z.GetData<double>();
-   const int* constraint_data = constraint_list.GetData<int>();
-
-   const std::size_t num_constraint = constraint_list.Size();
-   const bool use_target = constraint_list.ComputeOnDevice();
-   const bool use_parallel = (use_target || num_constraint > 1000);
-
-   z.Assign<double>(x); // z = x
-
-   // z[constraint_list] = 0.0
-#pragma omp target teams distribute parallel for             \
-   map(to: z_data, constraint_data)                          \
-   if (target: use_target)                                   \
-   if (parallel: use_parallel)
-   for (std::size_t i = 0; i < num_constraint; i++)
-      z_data[constraint_data[i]] = 0.0;
-
-   // y = A * z
-   A->Mult(mfem_z, mfem_y);
-
-   // y[constraint_list] = x[constraint_list]
-#pragma omp target teams distribute parallel for             \
-   map(to: y_data, constraint_data, x_data)                  \
-   if (target: use_target)                                   \
-   if (parallel: use_parallel)
-   for (std::size_t i = 0; i < num_constraint; i++)
-      y_data[constraint_data[i]] = x_data[constraint_data[i]];
-}
-
-// Destructor: destroys the unconstrained Operator @a A if @a own_A is true.
-ConstrainedOperator::~ConstrainedOperator()
-{
-   if (own_A) delete A;
-}
-
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/bilinearform.hpp

@@ -1,176 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OMP_BILINEARFORM_HPP
-#define MFEM_BACKENDS_OMP_BILINEARFORM_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#include "fespace.hpp"
-#include "array.hpp"
-#include "vector.hpp"
-#include "../../fem/bilininteg.hpp"
-
-namespace mfem
-{
-
-namespace omp
-{
-
-class TensorBilinearFormIntegrator
-{
-public:
-   virtual ~TensorBilinearFormIntegrator() { }
-
-   virtual void ReassembleOperator() = 0;
-
-   virtual void ComputeElementMatrices(Vector &element_matrices)
-   { mfem_error("TensorBilinaerFormIntegrator::ComputeElementMatrices is not overloaded"); }
-
-   virtual void MultAdd(const Vector &x, Vector &y) const = 0;
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   { y.Fill<double>(0.0); MultAdd(x, y); }
-};
-
-/// TODO: doxygen
-class BilinearForm : public mfem::PBilinearForm, public mfem::Operator
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // SharedPtr<const mfem::Engine> engine;
-   // mfem::BilinearForm *bform;
-
-   mfem::Array<TensorBilinearFormIntegrator*> tbfi;
-   bool has_assembled;
-
-   mutable FiniteElementSpace *trial_fes, *test_fes;
-
-   mutable Vector x_local, y_local;
-
-   mfem::Vector *element_matrices;
-   OperatorHandle mat_e;
-
-   void TransferIntegrators();
-
-   void ComputeElementMatrices();
-
-   void InitRHS(const mfem::Array<int> &constraint_list,
-                mfem::Vector &mfem_x, mfem::Vector &mfem_b,
-                mfem::OperatorHandle &A,
-                mfem::Vector &mfem_X, mfem::Vector &mfem_B,
-                int copy_interior = 0) const;
-
-public:
-   /// TODO: doxygen
-   BilinearForm(const Engine &e, mfem::BilinearForm &bf)
-      : mfem::PBilinearForm(e, bf),
-        // FIXME: for mixed bilinear forms
-        mfem::Operator(*bf.FESpace()->GetVLayout().As<Layout>()),
-        tbfi(),
-        has_assembled(false),
-        trial_fes(&bf.FESpace()->Get_PFESpace()->As<FiniteElementSpace>()),
-        test_fes(&bf.FESpace()->Get_PFESpace()->As<FiniteElementSpace>()),
-        x_local(trial_fes->GetELayout()),
-        y_local(test_fes->GetELayout()),
-        element_matrices(NULL),
-        mat_e() { }
-
-   /// Virtual destructor
-   virtual ~BilinearForm();
-
-   /// Return the engine as an OpenMP engine
-   const Engine &OmpEngine() { return static_cast<const Engine&>(*engine); }
-
-   /** @brief Prolongation operator from linear algebra (linear system) vectors,
-       to input vectors for the operator. `NULL` means identity. */
-   virtual const Operator *GetProlongation() const { return trial_fes->GetProlongation(); }
-
-   /** @brief Restriction operator from input vectors for the operator to linear
-       algebra (linear system) vectors. `NULL` means identity. */
-   virtual const Operator *GetRestriction() const  { return test_fes->GetRestriction(); }
-
-   /// Assemble the PBilinearForm.
-   /** This method is called from the method BilinearForm::Assemble() of the
-       associated BilinearForm #bform.
-       @returns True, if the host assembly should be skipped. */
-   virtual bool Assemble();
-
-   /// TODO: doxygen
-   virtual void FormSystemMatrix(const mfem::Array<int> &ess_tdof_list,
-                                 mfem::OperatorHandle &A);
-
-   /// TODO: doxygen
-   virtual void FormLinearSystem(const mfem::Array<int> &ess_tdof_list,
-                                 mfem::Vector &x, mfem::Vector &b,
-                                 mfem::OperatorHandle &A, mfem::Vector &mfem_X, mfem::Vector &mfem_B,
-                                 int copy_interior);
-
-   /// TODO: doxygen
-   virtual void RecoverFEMSolution(const mfem::Vector &mfem_X, const mfem::Vector &mfem_b,
-                                   mfem::Vector &mfem_x);
-
-   /// Operator application: `y=A(x)`.
-   virtual void Mult(const mfem::Vector &mfem_x, mfem::Vector &mfem_y) const;
-
-   /** @brief Action of the transpose operator: `y=A^t(x)`. The default behavior
-       in class Operator is to generate an error. */
-   virtual void MultTranspose(const mfem::Vector &mfem_x, mfem::Vector &mfem_y) const;
-};
-
-class ConstrainedOperator : public mfem::Operator
-{
-   const mfem::Operator *A;
-   const bool own_A;
-   const Array constraint_list;
-   mutable Vector z, w;
-   mutable mfem::Vector mfem_z, mfem_w;
-
-public:
-   ConstrainedOperator(mfem::Operator *A_,
-                       const mfem::Array<int> &constraint_list_,
-                       bool own_A_ = false);
-
-   // Destructor: destroys the unconstrained Operator @a A if @a own_A is true.
-   virtual ~ConstrainedOperator();
-
-   /** @brief Eliminate "essential boundary condition" values specified in @a x
-       from the given right-hand side @a b.
-
-       Performs the following steps:
-
-       z = A((0,x_b));  b_i -= z_i;  b_b = x_b;
-
-       where the "_b" subscripts denote the essential (boundary) indices/dofs of
-       the vectors, and "_i" -- the rest of the entries. */
-   void EliminateRHS(const mfem::Vector &mfem_x, mfem::Vector &mfem_b) const;
-
-   /** @brief Constrained operator action.
-
-       Performs the following steps:
-
-       z = A((x_i,0));  y_i = z_i;  y_b = x_b;
-
-       where the "_b" subscripts denote the essential (boundary) indices/dofs of
-       the vectors, and "_i" -- the rest of the entries. */
-   virtual void Mult(const mfem::Vector &mfem_x, mfem::Vector &mfem_y) const;
-};
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#endif // MFEM_BACKENDS_OMP_BILINEAR_FORM_HPP

backends/omp/engine.cpp

@@ -1,253 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#include "engine.hpp"
-#include "array.hpp"
-#include "layout.hpp"
-#include "vector.hpp"
-#include "fespace.hpp"
-#include "bilinearform.hpp"
-#include "memory_resource.hpp"
-
-#include <map>
-
-namespace mfem
-{
-
-namespace omp
-{
-
-typedef std::map<std::string, std::string> keyval_pair_t;
-
-template<typename T, typename P>
-static T remove_if(T beg, T end, P pred)
-{
-   T dest = beg;
-   for (T itr = beg;itr != end; ++itr)
-      if (!pred(*itr))
-         *(dest++) = *itr;
-   return dest;
-}
-
-void parse_token(const std::string &token, std::string &key, std::string &val)
-{
-   std::size_t sep = token.find_first_of(':');
-   if (sep > token.size()) mfem_error("Parse error");
-
-   key = token.substr(0, sep);
-   key.erase(mfem::omp::remove_if(key.begin(), key.end(), isspace), key.end());
-   key.erase(std::remove(key.begin(), key.end(), '\''), key.end());
-
-   val = token.substr(sep+1);
-   val.erase(mfem::omp::remove_if(val.begin(), val.end(), isspace), val.end());
-   val.erase(std::remove(val.begin(), val.end(), '\''), val.end());
-}
-
-keyval_pair_t parse_engine_spec(const std::string &engine_spec)
-{
-   keyval_pair_t map;
-   std::size_t token_extent = 0;
-   std::string key, val;
-   while (token_extent < engine_spec.size())
-   {
-      const std::string remaining(engine_spec, token_extent);
-
-      std::size_t next_comma = remaining.find_first_of(',');
-      if (next_comma == std::string::npos) next_comma = engine_spec.size() - 1;
-
-      const std::string token(remaining, 0, next_comma);
-      parse_token(token, key, val);
-
-      map[key] = val;
-      token_extent += next_comma+1;
-   }
-   return map;
-}
-
-void Engine::Init(const std::string &engine_spec)
-{
-   keyval_pair_t tokens(parse_engine_spec(engine_spec));
-   keyval_pair_t::iterator it;
-
-   it = tokens.find("exec_target");
-   if (it != tokens.end())
-   {
-      if (!std::strncmp(it->second.data(), "device", 6))
-      {
-         exec_target = Device;
-         device_number = 0;
-      }
-      else if (!std::strncmp(it->second.data(), "host", 4))
-      {
-         exec_target = Host;
-         device_number = -1;
-      }
-      else
-      {
-         mfem_error("Parse error. Possible values for exec_target are: ['host', 'device']");
-      }
-   }
-   else
-   {
-      // Default to host if not specified
-      mfem::out << "Did not specify exec_target. Defaulting to host..." << std::endl;
-      exec_target = Host;
-      device_number = -1;
-   }
-
-   it = tokens.find("mem_type");
-   if (it != tokens.end())
-   {
-      if (!std::strncmp(it->second.data(), "unified", 7))
-      {
-#if defined(MFEM_USE_CUDAUM)
-         memory_resources[0] = new UnifiedMemoryResource();
-         unified_memory = true;
-#else
-         mfem_error("Have not compiled support for CUDA unified memory.");
-#endif
-      }
-      else if (!std::strncmp(it->second.data(), "separate", 4))
-      {
-         memory_resources[0] = new NewDeleteMemoryResource();
-         unified_memory = false;
-      }
-      else
-      {
-         mfem_error("Parse error. Possible values for mem_type are: ['separate', 'unified']");
-      }
-   }
-   else {
-      if (exec_target == Device)
-      {
-#if defined(MFEM_USE_CUDAUM)
-         mfem::out << "Did not specify mem_type in engine spec. Defaulting to unified memory..." << std::endl;
-         // Default to unified memory
-         memory_resources[0] = new UnifiedMemoryResource();
-         unified_memory = true;
-#else
-         mfem::out << "Did not specify mem_type in engine spec. Defaulting to standard host memory..." << std::endl;
-         memory_resources[0] = new NewDeleteMemoryResource();
-         unified_memory = false;
-#endif
-      }
-      else
-      {
-         mfem::out << "Did not specify mem_type in engine spec. Defaulting to standard host memory..." << std::endl;
-         memory_resources[0] = new NewDeleteMemoryResource();
-         unified_memory = false;
-      }
-   }
-
-   it = tokens.find("mult_engine");
-   if (it != tokens.end())
-   {
-      if (!std::strncmp(it->second.data(), "acrotensor", 10))
-      {
-         mult_type = Acrotensor;
-      }
-      else
-      {
-         mfem_error("Parse error. Possible values for mem_type are: ['acrotensor'].");
-      }
-   }
-   else
-   {
-      mfem::out << "Did not specify mult_engine in engine spec. Defaulting to Acrotensor..." << std::endl;
-#ifndef MFEM_USE_ACROTENSOR
-      mfem_error("Must compile with Acrotensor support");
-#endif
-      mult_type = Acrotensor;
-   }
-}
-
-Engine::Engine(const std::string &engine_spec)
-   : mfem::Engine(NULL, 1, 1)
-{
-   Init(engine_spec);
-}
-
-#ifdef MFEM_USE_MPI
-Engine::Engine(MPI_Comm _comm, const std::string &engine_spec)
-   : mfem::Engine(NULL, 1, 1)
-{
-   comm = _comm;
-   Init(engine_spec);
-}
-#endif
-
-DLayout Engine::MakeLayout(std::size_t size) const
-{
-   return DLayout(new Layout(*this, size));
-}
-
-DLayout Engine::MakeLayout(const mfem::Array<std::size_t> &offsets) const
-{
-   MFEM_ASSERT(offsets.Size() == 2,
-               "multiple workers are not supported yet");
-   return DLayout(new Layout(*this, offsets.Last()));
-}
-
-DArray Engine::MakeArray(PLayout &layout, std::size_t item_size) const
-{
-   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
-               "invalid input layout");
-   Layout *lt = static_cast<Layout *>(&layout);
-   return DArray(new Array(*lt, item_size));
-}
-
-DVector Engine::MakeVector(PLayout &layout, int type_id) const
-{
-   MFEM_ASSERT(type_id == ScalarId<double>::value, "invalid type_id");
-   MFEM_ASSERT(dynamic_cast<Layout *>(&layout) != NULL,
-               "invalid input layout");
-   Layout *lt = static_cast<Layout *>(&layout);
-   return DVector(new Vector(*lt));
-}
-
-DFiniteElementSpace Engine::MakeFESpace(mfem::FiniteElementSpace &fespace) const
-{
-   return DFiniteElementSpace(new FiniteElementSpace(*this, fespace));
-}
-
-DBilinearForm Engine::MakeBilinearForm(mfem::BilinearForm &bf) const
-{
-   return DBilinearForm(new BilinearForm(*this, bf));
-}
-
-void Engine::AssembleLinearForm(LinearForm &l_form) const
-{
-   /// FIXME - What will the actual parameters be?
-   MFEM_ABORT("FIXME");
-}
-
-mfem::Operator *Engine::MakeOperator(const MixedBilinearForm &mbl_form) const
-{
-   /// FIXME - What will the actual parameters be?
-   MFEM_ABORT("FIXME");
-   return NULL;
-}
-
-mfem::Operator *Engine::MakeOperator(const NonlinearForm &nl_form) const
-{
-   /// FIXME - What will the actual parameters be?
-   MFEM_ABORT("FIXME");
-   return NULL;
-}
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/engine.hpp

@@ -1,123 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OMP_ENGINE_HPP
-#define MFEM_BACKENDS_OMP_ENGINE_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#include "../base/engine.hpp"
-
-namespace mfem
-{
-
-namespace omp
-{
-
-enum ExecutionTarget { Host, Device };
-
-enum IntegratorType { Acrotensor };
-
-class Engine : public mfem::Engine
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // mfem::Backend *backend;
-#ifdef MFEM_USE_MPI
-   // MPI_Comm comm;
-#endif
-   // int num_mem_res;
-   // int num_workers;
-   // MemoryResource **memory_resources;
-   // double *workers_weights;
-   // int *workers_mem_res;
-
-   enum ExecutionTarget exec_target;
-   bool unified_memory;
-   int device_number;
-   IntegratorType mult_type;
-
-   void Init(const std::string &engine_spec);
-
-public:
-   Engine(const std::string &engine_spec);
-
-#ifdef MFEM_USE_MPI
-   Engine(MPI_Comm comm, const std::string &engine_spec);
-#endif
-
-   virtual ~Engine() { }
-
-   /**
-      @name OMP specific interface, used by other objects in the OMP backend
-   */
-   ///@{
-
-   IntegratorType IntegType() const { return mult_type; }
-
-   ExecutionTarget ExecTarget() const { return exec_target; }
-
-   inline bool UnifiedMemory() const { return unified_memory; }
-
-   void* Malloc(std::size_t bytes) const
-   {
-      return memory_resources[0]->Allocate(bytes, 16);
-   }
-
-   void Dealloc(void *ptr, std::size_t bytes = 0) const
-   {
-      memory_resources[0]->Deallocate(ptr, bytes);
-   }
-
-   ///@}
-   // End: OMP specific interface
-
-   /**
-      @name Virtual interface: finite element data structures and algorithms
-   */
-   ///@{
-
-   virtual DLayout MakeLayout(std::size_t size) const;
-   virtual DLayout MakeLayout(const mfem::Array<std::size_t> &offsets) const;
-
-   virtual DArray MakeArray(PLayout &layout, std::size_t item_size) const;
-
-   virtual DVector MakeVector(PLayout &layout,
-                              int type_id = ScalarId<double>::value) const;
-
-   virtual DFiniteElementSpace MakeFESpace(mfem::FiniteElementSpace &
-                                           fespace) const;
-
-   virtual DBilinearForm MakeBilinearForm(mfem::BilinearForm &bf) const;
-
-   /// FIXME - What will the actual parameters be?
-   virtual void AssembleLinearForm(LinearForm &l_form) const;
-
-   /// FIXME - What will the actual parameters be?
-   virtual mfem::Operator *MakeOperator(const MixedBilinearForm &mbl_form) const;
-
-   /// FIXME - What will the actual parameters be?
-   virtual mfem::Operator *MakeOperator(const NonlinearForm &nl_form) const;
-
-   ///@}
-   // End: Virtual interface
-};
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#endif // MFEM_BACKENDS_OMP_ENGINE_HPP

backends/omp/fespace.cpp

@@ -1,237 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#include "fespace.hpp"
-
-namespace mfem
-{
-
-namespace omp
-{
-
-FiniteElementSpace::FiniteElementSpace(const Engine &e,
-                                       mfem::FiniteElementSpace &fespace)
-   : PFiniteElementSpace(e, fespace),
-     e_layout(e, 0),
-     tensor_offsets(NULL),
-     tensor_indices(NULL),
-     prolongation(NULL),
-     restriction(NULL)
-{
-   std::size_t lsize = 0;
-   for (int e = 0; e < fespace.GetNE(); e++) { lsize += fespace.GetFE(e)->GetDof(); }
-   e_layout.Resize(lsize);
-   // The e_layout will be stored inside multiple shared DLayout objects
-   e_layout.DontDelete();
-}
-
-void FiniteElementSpace::BuildDofMaps()
-{
-   mfem::FiniteElementSpace *mfem_fes = GetFESpace();
-
-   const int local_size = GetELayout().Size();
-   const int global_size = mfem_fes->GetVLayout()->Size();
-   const int vdim = mfem_fes->GetVDim();
-
-   // Now we can allocate and fill the global map
-   tensor_offsets = new mfem::Array<int>(*(new Layout(OmpEngine(), global_size + 1)));
-   tensor_indices = new mfem::Array<int>(*(new Layout(OmpEngine(), local_size)));
-
-   mfem::Array<int> &offsets = *tensor_offsets;
-   mfem::Array<int> &indices = *tensor_indices;
-
-   mfem::Array<int> global_map(local_size);
-   mfem::Array<int> elem_vdof;
-
-   int offset = 0;
-   for (int e = 0; e < mfem_fes->GetNE(); e++)
-   {
-      const FiniteElement *fe = mfem_fes->GetFE(e);
-      const int dofs = fe->GetDof();
-      const TensorBasisElement *tfe = dynamic_cast<const TensorBasisElement *>(fe);
-      const mfem::Array<int> &dof_map = tfe->GetDofMap();
-
-      mfem_fes->GetElementVDofs(e, elem_vdof);
-
-      for (int vd = 0; vd < vdim; vd++)
-         for (int i = 0; i < dofs; i++)
-         {
-            global_map[offset + dofs*vd + i] = elem_vdof[dofs*vd + dof_map[i]];
-         }
-      offset += dofs * vdim;
-   }
-
-   // global_map[i] = index in global vector for local dof i
-   // NOTE: multiple i values will yield same global_map[i] for shared DOF.
-
-   // We want to now invert this map so we have indices[j] = (local dof for global dof j).
-
-   // Zero the offset vector
-   offsets = 0;
-
-   // Keep track of how many local dof point to its global dof
-   // Count how many times each dof gets hit
-   for (int i = 0; i < local_size; i++)
-   {
-      const int g = global_map[i];
-      ++offsets[g + 1];
-   }
-   // Aggregate the offsets
-   for (int i = 1; i <= global_size; i++)
-   {
-      offsets[i] += offsets[i - 1];
-   }
-
-   for (int i = 0; i < local_size; i++)
-   {
-      const int g = global_map[i];
-      indices[offsets[g]++] = i;
-   }
-
-   // Shift the offset vector back by one, since it was used as a
-   // counter above.
-   for (int i = global_size; i > 0; i--)
-   {
-      offsets[i] = offsets[i - 1];
-   }
-   offsets[0] = 0;
-
-   offsets.Push();
-   indices.Push();
-}
-
-/// Convert an E vector to L vector
-void FiniteElementSpace::ToLVector(const Vector &e_vector, Vector &l_vector)
-{
-   if (tensor_indices == NULL) BuildDofMaps();
-
-   if (l_vector.Size() != (std::size_t) GetFESpace()->GetVSize())
-   {
-      l_vector.Resize<double>(GetFESpace()->GetVLayout(), NULL);
-   }
-
-   const int lsize = l_vector.Size();
-   const int *offsets = tensor_offsets->Get_PArray()->As<Array>().GetData<int>();
-   const int *indices = tensor_indices->Get_PArray()->As<Array>().GetData<int>();
-
-   const double *e_data = e_vector.GetData<double>();
-   double *l_data = l_vector.GetData<double>();
-
-   const bool use_target = l_vector.ComputeOnDevice();
-   const bool use_parallel = (use_target || lsize > 1000);
-
-#pragma omp target teams distribute parallel for        \
-   map (to: offsets, indices, l_data, e_data)           \
-   if (target: use_target)                              \
-   if (parallel: use_parallel)
-   for (int i = 0; i < lsize; i++)
-   {
-      const int offset = offsets[i];
-      const int next_offset = offsets[i + 1];
-      double dof_value = 0;
-      for (int j = offset; j < next_offset; j++)
-      {
-         dof_value += e_data[indices[j]];
-      }
-      l_data[i] = dof_value;
-   }
-}
-
-/// Covert an L vector to E vector
-void FiniteElementSpace::ToEVector(const Vector &l_vector, Vector &e_vector)
-{
-   if (tensor_indices == NULL) BuildDofMaps();
-
-   if (e_vector.Size() != (std::size_t) e_layout.Size())
-   {
-      e_vector.Resize<double>(GetELayout(), NULL);
-   }
-
-   const int lsize = l_vector.Size();
-   const int *offsets = tensor_offsets->Get_PArray()->As<Array>().GetData<int>();
-   const int *indices = tensor_indices->Get_PArray()->As<Array>().GetData<int>();
-
-   const double *l_data = l_vector.GetData<double>();
-   double *e_data = e_vector.GetData<double>();
-
-   const bool use_target = l_vector.ComputeOnDevice();
-   const bool use_parallel = (use_target || lsize > 1000);
-
-#pragma omp target teams distribute parallel for         \
-   map (to: offsets, indices, l_data, e_data)            \
-   if (target: use_target)                               \
-   if (parallel: use_parallel)
-   for (int i = 0; i < lsize; i++)
-   {
-      const int offset = offsets[i];
-      const int next_offset = offsets[i + 1];
-      const double dof_value = l_data[i];
-      for (int j = offset; j < next_offset; j++)
-      {
-         e_data[indices[j]] = dof_value;
-      }
-   }
-}
-
-/// Get the finite element space prolongation matrix
-const Operator *FiniteElementSpace::GetProlongation() const
-{
-   // FIXME: This relies on unified memory if using a device other than the CPU
-   if (!prolongation)
-   {
-      Layout &v_layout = GetVLayout();
-      Layout &t_layout = GetTrueVLayout();
-
-      const mfem::Operator *op = GetFESpace()->GetProlongationMatrix();
-      if (!op)
-      {
-         prolongation = new mfem::IdentityOperator(t_layout);
-      }
-      else
-      {
-         prolongation = new BackendOperator(t_layout, v_layout, op);
-      }
-   }
-   return prolongation;
-}
-
-/// Get the finite element space restriction matrix
-const Operator *FiniteElementSpace::GetRestriction() const
-{
-   // FIXME: This relies on unified memory if using a device other than the CPU
-   if (!restriction)
-   {
-      Layout &v_layout = GetVLayout();
-      Layout &t_layout = GetTrueVLayout();
-
-      const mfem::Operator *op = GetFESpace()->GetRestrictionMatrix();
-      if (!op)
-      {
-         restriction = new mfem::IdentityOperator(t_layout);
-      }
-      else
-      {
-         restriction = new BackendOperator(v_layout, t_layout, op);
-      }
-   }
-   return restriction;
-}
-
-
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/fespace.hpp

@@ -1,106 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OMP_FESPACE_HPP
-#define MFEM_BACKENDS_OMP_FESPACE_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#include "engine.hpp"
-#include "array.hpp"
-#include "vector.hpp"
-#include "../../fem/fem.hpp"
-
-namespace mfem
-{
-
-namespace omp
-{
-
-/*
-  Wraps an mfem::Operator that does not contain layout information.
- */
-class BackendOperator : public mfem::Operator
-{
-   const mfem::Operator *op;
-
-public:
-   BackendOperator(Layout &in_layout, Layout &out_layout,
-                   const mfem::Operator *op_) : Operator(in_layout, out_layout), op(op_) { }
-
-   virtual void Mult(const mfem::Vector &x, mfem::Vector &y) const { op->Mult(x, y); }
-   virtual void MultTranspose(const mfem::Vector &x, mfem::Vector &y) const { op->MultTranspose(x, y); }
-};
-
-/// TODO: doxygen
-class FiniteElementSpace : public mfem::PFiniteElementSpace
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // SharedPtr<const mfem::Engine> engine;
-   // mfem::FiniteElementSpace *fes;
-
-   Layout e_layout;
-
-   mfem::Array<int> *tensor_offsets, *tensor_indices;
-
-   mutable mfem::Operator *prolongation, *restriction;
-
-   void BuildDofMaps();
-
-public:
-   /// Nearly-empty class that stores a pointer to a mfem::FiniteElementSpace instance and the engine
-   FiniteElementSpace(const Engine &e, mfem::FiniteElementSpace &fespace);
-
-   /// Virtual destructor
-   virtual ~FiniteElementSpace()
-   {
-      delete tensor_offsets;
-      delete tensor_indices;
-      delete prolongation;
-      delete restriction;
-   }
-
-   Layout &GetELayout() { return e_layout; }
-
-   Layout &GetVLayout() const
-   { return *fes->GetVLayout().As<Layout>(); }
-
-   Layout &GetTrueVLayout() const
-   { return *fes->GetTrueVLayout().As<Layout>(); }
-
-   /// Return the engine as an OpenMP engine
-   const Engine &OmpEngine() { return static_cast<const Engine&>(*engine); }
-
-   /// Convert an E vector to L vector
-   void ToLVector(const Vector &e_vector, Vector &l_vector);
-
-   /// Covert an L vector to E vector
-   void ToEVector(const Vector &l_vector, Vector &e_vector);
-
-   /// Get the finite element space prolongation matrix
-   const Operator *GetProlongation() const;
-
-   /// Get the finite element space restriction matrix
-   const Operator *GetRestriction() const;
-
-};
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#endif // MFEM_BACKENDS_OMP_FESPACE_HPP

backends/omp/layout.cpp

@@ -1,40 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#include "layout.hpp"
-#include "../../general/array.hpp"
-
-namespace mfem
-{
-
-namespace omp
-{
-
-void Layout::Resize(std::size_t new_size)
-{
-   size = new_size;
-}
-
-void Layout::Resize(const Array<std::size_t> &offsets)
-{
-   MFEM_ASSERT(offsets.Size() == 2,
-               "multiple workers are not supported yet");
-   size = offsets.Last();
-}
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/layout.hpp

@@ -1,71 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OMP_LAYOUT_HPP
-#define MFEM_BACKENDS_OMP_LAYOUT_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#include "../base/layout.hpp"
-#include "engine.hpp"
-
-namespace mfem
-{
-
-namespace omp
-{
-
-class Layout : public mfem::PLayout
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // SharedPtr<const mfem::Engine> engine;
-   // std::size_t size;
-
-public:
-   Layout(const Engine &e, std::size_t s = 0) : PLayout(e, s) { }
-
-   const Engine &OmpEngine() const
-   { return *static_cast<const Engine *>(engine.Get()); }
-
-   void *Alloc(std::size_t bytes) const
-   { return OmpEngine().Malloc(bytes); }
-
-   void Dealloc(void *ptr) const
-   { return OmpEngine().Dealloc(ptr); }
-
-   virtual ~Layout() { }
-
-   /**
-      @name Virtual interface
-   */
-   ///@{
-
-   /// Resize the layout
-   virtual void Resize(std::size_t new_size);
-
-   /// Resize the layout based on the given worker offsets
-   virtual void Resize(const Array<std::size_t> &offsets);
-
-   ///@}
-   // End: Virtual interface
-};
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#endif // MFEM_BACKENDS_OMP_LAYOUT_HPP

backends/omp/memory_resource.cpp

@@ -1,57 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#include "memory_resource.hpp"
-#include "../../general/error.hpp"
-
-#ifdef MFEM_USE_CUDAUM
-#include "cuda_runtime.h"
-#include "cuda.h"
-#endif
-
-namespace mfem
-{
-
-namespace omp
-{
-
-#ifdef MFEM_USE_CUDAUM
-void *UnifiedMemoryResource::DoAllocate(std::size_t bytes,
-                                        std::size_t alignment)
-{
-   void *p = NULL;
-   if (bytes > 0)
-   {
-      cudaError_t ret = cudaMallocManaged(&p, bytes);
-      MFEM_VERIFY(ret == cudaSuccess, "");
-   }
-   return p;
-}
-
-void UnifiedMemoryResource::DoDeallocate(void *p, std::size_t bytes,
-                                         std::size_t alignment)
-{
-   if (p != NULL)
-   {
-      cudaFree(p);
-   }
-}
-#endif
-
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/memory_resource.hpp

@@ -1,44 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OMP_MEMORY_RESOURCE_HPP
-#define MFEM_BACKENDS_OMP_MEMORY_RESOURCE_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#include "../../backends/base/memory_resource.hpp"
-
-namespace mfem
-{
-
-namespace omp
-{
-
-/// Polymorphic memory resource. Similar to C++17's std::pmr::memory_resource.
-
-#ifdef MFEM_USE_CUDAUM
-/** @brief Memory resource using unified memory. */
-class UnifiedMemoryResource : public MemoryResource
-{
-protected:
-   virtual void *DoAllocate(std::size_t bytes, std::size_t alignment);
-   virtual void DoDeallocate(void *p, std::size_t bytes, std::size_t alignment);
-};
-#endif
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#endif // MFEM_BACKENDS_OMP_MEMORY_RESOURCE_HPP

backends/omp/vector.cpp

@@ -1,205 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#include "vector.hpp"
-#include "../../linalg/vector.hpp"
-
-namespace mfem
-{
-
-namespace omp
-{
-
-PVector *Vector::DoVectorClone(bool copy_data, void **buffer,
-                               int buffer_type_id) const
-{
-   MFEM_ASSERT(buffer_type_id == ScalarId<double>::value, "");
-   Vector *new_vector = new Vector(OmpLayout());
-   if (copy_data)
-   {
-      const std::size_t total_size = sizeof(double) * OmpLayout().Size();
-      if (!ComputeOnDevice())
-         std::memcpy(new_vector->GetData<void>(), data, total_size);
-      else
-      {
-         char *new_data = new_vector->GetData<char>();
-#pragma omp target teams distribute parallel for is_device_ptr(new_data)
-         for (std::size_t i = 0; i < total_size; i++) new_data[i] = data[i];
-      }
-   }
-   if (buffer)
-   {
-      *buffer = new_vector->GetData<void>();
-   }
-   return new_vector;
-}
-
-void Vector::DoDotProduct(const PVector &x, void *result,
-                          int result_type_id) const
-{
-   // Can be called when Size() == 0, e.g. when an MPI-parallel vector has a
-   // local size of 0.
-
-   MFEM_ASSERT(result_type_id == ScalarId<double>::value, "");
-   double *res = (double *)result;
-   double local_dot = 0.;
-   MFEM_ASSERT(dynamic_cast<const Vector *>(&x) != NULL, "invalid Vector type");
-   const Vector *xp = static_cast<const Vector *>(&x);
-   MFEM_ASSERT(this->Size() == xp->Size(), "");
-
-   const double *ptr = GetData<double>();
-   const double *xptr = xp->GetData<double>();
-   const std::size_t size = Size();
-
-   if (!ComputeOnDevice())
-   {
-      for (std::size_t i = 0; i < size; i++) local_dot += ptr[i] * xptr[i];
-   }
-   else
-   {
-#pragma omp target teams distribute parallel for map(to: ptr, xptr) reduction(+:local_dot)
-      for (std::size_t i = 0; i < size; i++) local_dot += ptr[i] * xptr[i];
-   }
-
-   *res = local_dot;
-#ifdef MFEM_USE_MPI
-   MPI_Comm comm = OmpLayout().OmpEngine().GetComm();
-   if (comm != MPI_COMM_NULL)
-   {
-      MPI_Allreduce(&local_dot, res, 1, MPI_DOUBLE, MPI_SUM, comm);
-   }
-#endif
-}
-
-void Vector::DoAxpby(const void *a, const PVector &x,
-                     const void *b, const PVector &y,
-                     int ab_type_id)
-{
-   // called only when Size() != 0
-   MFEM_ASSERT(ab_type_id == ScalarId<double>::value, "");
-
-   const double da = *static_cast<const double *>(a);
-   const double db = *static_cast<const double *>(b);
-   MFEM_ASSERT(da == 0.0 || dynamic_cast<const Vector *>(&x) != NULL,
-               "invalid Vector x");
-   MFEM_ASSERT(db == 0.0 || dynamic_cast<const Vector *>(&y) != NULL,
-               "invalid Vector y");
-   const Vector *xp = static_cast<const Vector *>(&x);
-   const Vector *yp = static_cast<const Vector *>(&y);
-
-   MFEM_ASSERT(da == 0.0 || this->Size() == xp->Size(), "");
-   MFEM_ASSERT(db == 0.0 || this->Size() == yp->Size(), "");
-
-   const std::size_t size = Size();
-   const std::size_t critical_size = 1000;
-   const double *xd = xp->GetData<double>();
-   const double *yd = yp->GetData<double>();
-   double *td = GetData<double>();
-
-   const bool use_target = ComputeOnDevice();
-   const bool use_parallel = (use_target || size > critical_size);
-
-   if (da == 0.0)
-   {
-      if (db == 0.0)
-      {
-         OmpFill(&da);
-      }
-      else
-      {
-         if (td == yd)
-         {
-            // *this *= db
-#pragma omp target teams distribute parallel for        \
-   if (target: use_target)                              \
-   if (parallel: use_parallel) map (to: db)
-            for (std::size_t i = 0; i < size; i++) td[i] *= db;
-         }
-         else
-         {
-            // *this = db * y
-#pragma omp target teams distribute parallel for        \
-   if (target: use_target)                              \
-   if (parallel: use_parallel) map (to: yd, db)
-            for (std::size_t i = 0; i < size; i++) td[i] = yd[i] * db;
-         }
-      }
-   }
-   else
-   {
-      if (db == 0.0)
-      {
-         if (td == xd)
-         {
-            // *this *= da
-#pragma omp target teams distribute parallel for        \
-   if (target: use_target)                              \
-   if (parallel: use_parallel) map (to: da)
-            for (std::size_t i = 0; i < size; i++) td[i] *= da;
-         }
-         else
-         {
-            // *this = da * x
-#pragma omp target teams distribute parallel for        \
-   if (target: use_target)                              \
-   if (parallel: use_parallel) map (to: xd, da)
-            for (std::size_t i = 0; i < size; i++) td[i] = xd[i] * da;
-         }
-      }
-      else
-      {
-         MFEM_ASSERT(xd != yd, "invalid input");
-         if (td == xd)
-         {
-            // *this = da * (*this) + db * y
-#pragma omp target teams distribute parallel for        \
-   if (target: use_target)                              \
-   if (parallel: use_parallel) map (to: da, td, db, yd)
-            for (std::size_t i = 0; i < size; i++) td[i] = da * td[i] + db * yd[i];
-         }
-         else if (td == yd)
-         {
-            // *this = da * x + db * (*this)
-#pragma omp target teams distribute parallel for        \
-   if (target: use_target)                              \
-   if (parallel: use_parallel) map (to: da, xd, db, td)
-            for (std::size_t i = 0; i < size; i++) td[i] = da * xd[i] + db * td[i];
-         }
-         else
-         {
-            // *this = da * x + db * y
-#pragma omp target teams distribute parallel for        \
-   if (target: use_target)                              \
-   if (parallel: use_parallel) map (to: da, xd, db, yd)
-            for (std::size_t i = 0; i < size; i++) td[i] = da * xd[i] + db * yd[i];
-         }
-      }
-   }
-}
-
-mfem::Vector Vector::Wrap()
-{
-   return mfem::Vector(*this);
-}
-
-const mfem::Vector Vector::Wrap() const
-{
-   return mfem::Vector(*const_cast<Vector*>(this));
-}
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)

backends/omp/vector.hpp

@@ -1,71 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BACKENDS_OMP_VECTOR_HPP
-#define MFEM_BACKENDS_OMP_VECTOR_HPP
-
-#include "../../config/config.hpp"
-#if defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#include "../base/vector.hpp"
-#include "array.hpp"
-
-namespace mfem
-{
-
-namespace omp
-{
-
-class Vector : virtual public Array, public mfem::PVector
-{
-protected:
-   //
-   // Inherited fields
-   //
-   // DLayout layout;
-   // char *data;
-   // std::size_t size;
-
-   /**
-      @name Virtual interface
-   */
-   ///@{
-
-   virtual PVector *DoVectorClone(bool copy_data, void **buffer,
-                                  int buffer_type_id) const;
-
-   virtual void DoDotProduct(const PVector &x, void *result,
-                             int result_type_id) const;
-
-   virtual void DoAxpby(const void *a, const PVector &x,
-                        const void *b, const PVector &y,
-                        int ab_type_id);
-
-   ///@}
-   // End: Virtual interface
-
-public:
-   Vector(Layout &lt)
-      : PArray(lt), Array(lt, sizeof(double)), PVector(lt)
-   { }
-
-   mfem::Vector Wrap();
-
-   const mfem::Vector Wrap() const;
-};
-
-} // namespace mfem::omp
-
-} // namespace mfem
-
-#endif // defined(MFEM_USE_BACKENDS) && defined(MFEM_USE_OMP)
-
-#endif // MFEM_BACKENDS_OMP_VECTOR_HPP

config/cmake/MFEMConfig.cmake.in

@@ -39,6 +39,7 @@ set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
+set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
 
 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")

config/cmake/config.hpp.in

@@ -98,6 +98,9 @@
 // Enable MFEM functionality based on Conduit
 #cmakedefine MFEM_USE_CONDUIT
 
+// Enable MFEM functionality based on the PUMI library
+#cmakedefine MFEM_USE_PUMI
+
 // Which library functions to use in class StopWatch for measuring time.
 // For a list of the available options, see INSTALL.
 // If not defined, an option is selected automatically.
@@ -113,4 +116,8 @@
 // Version of HYPRE used for building MFEM.
 #cmakedefine MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
 
+// Macro defined when PUMI is built with support for the Simmetrix SimModSuite
+// library.
+#cmakedefine MFEM_USE_SIMMETRIX
+
 #endif // MFEM_CONFIG_HEADER

config/config.hpp

@@ -36,12 +36,7 @@
 #ifdef MFEM_USE_PETSC
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
-#endif // MFEM_USE_MPI not defined
-
-// Macro that returns its first arg when MFEM_USE_BACKENDS is defined, and its
-// second arg if it is not defined.
-#ifdef MFEM_USE_BACKENDS
-#define MFEM_IF_BACKENDS(x,y) (x)
-#else
-#define MFEM_IF_BACKENDS(x,y) (y)
+#ifdef MFEM_USE_PUMI
+#error Building with PUMI (MFEM_USE_PUMI=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
+#endif // MFEM_USE_MPI not defined

config/config.hpp.in

@@ -33,12 +33,6 @@
 // Description of the git commit used to build MFEM.
 // #define MFEM_GIT_STRING "@MFEM_GIT_STRING@"
 
-// The absolute path of the MFEM source prefix
-// #define MFEM_SOURCE_DIR "@MFEM_SOURCE_DIR@"
-
-// The absolute path of the MFEM installation prefix
-// #define MFEM_INSTALL_DIR "@MFEM_INSTALL_DIR@"
-
 // Build the parallel MFEM library.
 // Requires an MPI compiler, and the libraries HYPRE and METIS.
 // #define MFEM_USE_MPI
@@ -115,20 +109,8 @@
 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR
 
-// Enable the use of MFEM backends.
-// #define MFEM_USE_BACKENDS
-
-// Enable the OCCA backend.
-// #define MFEM_USE_OCCA
-
-// Enable the OMP backend.
-// #define MFEM_USE_OMP
-
-// Enable use of acrotensor in backends.
-// #define MFEM_USE_ACROTENSOR
-
-// Enable use of unified memory.
-// #define MFEM_USE_CUDAUM
+// Enable MFEM functionality based on the PUMI library
+// #define MFEM_USE_PUMI
 
 // Windows specific options
 #ifdef _WIN32
@@ -139,4 +121,8 @@
 // Version of HYPRE used for building MFEM.
 // #define MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
 
+// Macro defined when PUMI is built with support for the Simmetrix SimModSuite
+// library.
+// #define MFEM_USE_SIMMETRIX
+
 #endif // MFEM_CONFIG_HEADER

config/config.mk.in

@@ -13,8 +13,6 @@
 MFEM_VERSION         = @MFEM_VERSION@
 MFEM_VERSION_STRING  = @MFEM_VERSION_STRING@
 MFEM_GIT_STRING      = @MFEM_GIT_STRING@
-MFEM_SOURCE_DIR      = @MFEM_SOURCE_DIR@
-MFEM_INSTALL_DIR     = @MFEM_INSTALL_DIR@
 MFEM_USE_MPI         = @MFEM_USE_MPI@
 MFEM_USE_METIS       = @MFEM_USE_METIS@
 MFEM_USE_METIS_5     = @MFEM_USE_METIS_5@
@@ -39,11 +37,7 @@ MFEM_USE_PETSC       = @MFEM_USE_PETSC@
 MFEM_USE_MPFR        = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE       = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT     = @MFEM_USE_CONDUIT@
-MFEM_USE_BACKENDS    = @MFEM_USE_BACKENDS@
-MFEM_USE_OCCA        = @MFEM_USE_OCCA@
-MFEM_USE_OMP         = @MFEM_USE_OMP@
-MFEM_USE_ACROTENSOR  = @MFEM_USE_ACROTENSOR@
-MFEM_USE_CUDAUM      = @MFEM_USE_CUDAUM@
+MFEM_USE_PUMI        = @MFEM_USE_PUMI@
 
 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@

config/defaults.cmake

@@ -40,6 +40,7 @@ option(MFEM_USE_PETSC "Enable PETSc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
+option(MFEM_USE_PUMI "Enable PUMI" OFF)
 
 # Allow a user to disable testing, examples, and/or miniapps at CONFIGURE TIME
 # if they don't want/need them (e.g. if MFEM is "just a dependency" and all they
@@ -145,6 +146,9 @@ set(AXOM_DIR "${MFEM_DIR}/../axom" CACHE PATH "Path to the Axom library.")
 set(Axom_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
     "Additional packages required by Axom.")
 
+set(PUMI_DIR "${MFEM_DIR}/../pumi-2.1.0" CACHE STRING
+    "Directory where PUMI is installed")
+
 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")

config/defaults.mk

@@ -83,9 +83,6 @@ MFEM_MPI_NP = 4
 # in config.mk and config.hpp.
 
 MFEM_USE_MPI         = NO
-# FIXME: add MFEM_USE_BACKENDS, MFEM_USE_OCCA to the CMake build system
-MFEM_USE_BACKENDS    = YES
-MFEM_USE_OCCA        = YES
 MFEM_USE_METIS       = $(MFEM_USE_MPI)
 MFEM_USE_METIS_5     = NO
 MFEM_DEBUG           = NO
@@ -109,10 +106,7 @@ MFEM_USE_PETSC       = NO
 MFEM_USE_MPFR        = NO
 MFEM_USE_SIDRE       = NO
 MFEM_USE_CONDUIT     = NO
-# FIXME: add MFEM_USE_OMP and MFEM_USE_ACROTENSOR to the CMake build system
-MFEM_USE_OMP         = NO
-MFEM_USE_ACROTENSOR  = NO
-MFEM_USE_CUDAUM      = NO
+MFEM_USE_PUMI        = NO
 
 # Compile and link options for zlib.
 ZLIB_DIR =
@@ -278,30 +272,12 @@ SIDRE_LIB = \
    -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
    -lsidre -lslic -laxom_utils -lconduit -lconduit_relay -lhdf5 $(ZLIB_LIB) -ldl
 
-OCCA_DIR = @MFEM_DIR@/../occa
-OCCA_OPT = -I$(OCCA_DIR)/include
-OCCA_LIB = -Wl,-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca
-
-CUDA_DIR = /usr/local/cuda
-CUDAUM_LIB = -L$(CUDA_DIR)/lib64 -lcudart
-CUDAUM_OPT = -I$(CUDA_DIR)/include
-
-OMP_OPT = -qsmp=omp -qoffload
-
-ACROTENSOR_DIR = @MFEM_DIR@/../acrotensor
-ACROTENSOR_OPT = -std=c++11 -I$(ACROTENSOR_DIR)/inc
-ACROTENSOR_LIB = -Wl,-rpath,$(ACROTENSOR_DIR)/lib/shared -L$(ACROTENSOR_DIR)/lib/shared -lacrotensor
-# If Acrotensor was compile with CUDA support, but MFEM_USE_CUDAUM==NO, then uncomment the lines below
-# ACROTENSOR_OPT += -I$(CUDA_DIR)/include
-# ACROTENSOR_LIB += -L$(CUDA_DIR)/lib64 -lcuda -lcudart -lnvrtc
-ifeq ($(MFEM_USE_CUDAUM),YES)
-ifeq ($(MFEM_USE_MPI),YES)
-# HYPRE needs some extra libraries in parallel on the GPU
-# FIXME: We need another solution for compilers other than XL for the
-#        dlink CUDA step, but fixes need to happen elsewhere as well.
-HYPRE_LIB += -qcuda -lcublas -lcusparse -lnvToolsExt
-endif
-endif
+# PUMI
+# Note that PUMI_DIR is needed -- it is used to check for gmi_sim.h
+PUMI_DIR = @MFEM_DIR@/../pumi-2.1.0
+PUMI_OPT = -I$(PUMI_DIR)/include
+PUMI_LIB = -L$(PUMI_DIR)/lib -lpumi -lcrv -lma -lmds -lapf -lpcu -lgmi -lparma\
+   -llion -lmth -lapf_zoltan -lspr
 
 # If YES, enable some informational messages
 VERBOSE = NO

config/makefile

@@ -38,6 +38,9 @@ all: header config-mk
 MPI = $(MFEM_USE_MPI:NO=)
 GHV = get_hypre_version
 GHV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
+SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)
+SMX_PATH = $(PUMI_DIR)/include/gmi_sim.h
+SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))
 
 $(GHV): $(SRC)$(GHV).cpp
 	$(call mfem-info, Determining HYPRE version ...)
@@ -52,10 +55,16 @@ get-hypre-version: $(GHV).out
 	$(info HYPRE version: $(MFEM_HYPRE_VERSION)),\
 	$(error Unable to determine HYPRE version))
 
-header: $(if $(MPI),get-hypre-version,)
+check-smx:
+	$(call mfem-info, Checking for Simmetrix header [$(SMX_FILE)] ...)
+	$(eval MFEM_USE_SIMMETRIX:=$(if $(wildcard $(SMX_FILE)),YES,NO))
+	$(call mfem-info, MFEM_USE_SIMMETRIX = $(MFEM_USE_SIMMETRIX))
+	$(eval export MFEM_USE_SIMMETRIX)
+
+header: $(if $(MPI),get-hypre-version,) $(if $(SMX),check-smx)
 	$(call mfem-info, Writing $(CONFIG_HPP) ...)
 	@set -- && \
-	for def in $${MFEM_DEFINES} $(if $(MPI),MFEM_HYPRE_VERSION,); do \
+	for def in $${MFEM_DEFINES} $(if $(MPI),MFEM_HYPRE_VERSION) $(SMX); do \
 	   eval var=\$$$$def && \
 	   if [ "NO" != "$${var}" ]; then \
 	      set -- "$$@" -e "s|// \(#define $${def} \)|\1|" && \

config/test.mk

@@ -14,11 +14,13 @@
 # Colors used below:
 # green    '\033[0;32m'
 # red      '\033[0;31m'
+# yellow   '\033[0;33m'
 # no color '\033[0m'
 COLOR_PRINT = if [ -t 1 ]; then \
    printf $(1)$(2)'\033[0m'$(3); else printf $(2)$(3); fi
 PRINT_OK = $(call COLOR_PRINT,'\033[0;32m',OK,"  ($$1 $$2)\n")
 PRINT_FAILED = $(call COLOR_PRINT,'\033[0;31m',FAILED,"  ($$1 $$2)\n")
+PRINT_SKIP = $(call COLOR_PRINT,'\033[0;33m',SKIP,"\n")
 
 # Timing support
 define TIMECMD_detect

data/mixed-ele-meshes/annulus-trans-24.mesh

@@ -0,0 +1,218 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+74
+2 3 0 1 2 3
+2 3 1 5 6 2
+2 3 5 8 9 6
+2 3 8 11 12 9
+2 3 11 14 15 12
+2 3 14 17 18 15
+2 3 17 20 21 18
+2 3 20 23 24 21
+2 3 23 26 27 24
+2 3 26 29 30 27
+2 3 29 32 33 30
+2 3 32 35 36 33
+2 3 35 38 39 36
+2 3 38 41 42 39
+2 3 41 44 45 42
+2 3 44 47 48 45
+2 3 47 50 51 48
+2 3 50 53 54 51
+2 3 53 56 57 54
+2 3 56 59 60 57
+2 3 59 62 63 60
+2 3 62 65 66 63
+2 3 65 68 69 66
+2 3 68 71 72 69
+2 3 71 74 75 72
+1 2 2 3 4
+1 2 6 2 7
+1 2 9 6 10
+1 2 12 9 13
+1 2 15 12 16
+1 2 18 15 19
+1 2 21 18 22
+1 2 24 21 25
+1 2 27 24 28
+1 2 30 27 31
+1 2 33 30 34
+1 2 36 33 37
+1 2 39 36 40
+1 2 42 39 43
+1 2 45 42 46
+1 2 48 45 49
+1 2 51 48 52
+1 2 54 51 55
+1 2 57 54 58
+1 2 60 57 61
+1 2 63 60 64
+1 2 66 63 67
+1 2 69 66 70
+1 2 72 69 73
+1 2 75 72 76
+1 2 2 4 7
+1 2 6 7 10
+1 2 9 10 13
+1 2 12 13 16
+1 2 15 16 19
+1 2 18 19 22
+1 2 21 22 25
+1 2 24 25 28
+1 2 27 28 31
+1 2 30 31 34
+1 2 33 34 37
+1 2 36 37 40
+1 2 39 40 43
+1 2 42 43 46
+1 2 45 46 49
+1 2 48 49 52
+1 2 51 52 55
+1 2 54 55 58
+1 2 57 58 61
+1 2 60 61 64
+1 2 63 64 67
+1 2 66 67 70
+1 2 69 70 73
+1 2 72 73 76
+
+boundary
+53
+1 1 0 1
+1 1 1 5
+1 1 5 8
+1 1 8 11
+1 1 11 14
+1 1 14 17
+1 1 17 20
+1 1 20 23
+1 1 23 26
+1 1 26 29
+1 1 29 32
+1 1 32 35
+1 1 35 38
+1 1 38 41
+1 1 41 44
+1 1 44 47
+1 1 47 50
+1 1 50 53
+1 1 53 56
+1 1 56 59
+1 1 59 62
+1 1 62 65
+1 1 65 68
+1 1 68 71
+1 1 71 74
+1 1 74 75
+1 1 75 76
+1 1 76 73
+1 1 73 70
+1 1 70 67
+1 1 67 64
+1 1 64 61
+1 1 61 58
+1 1 58 55
+1 1 55 52
+1 1 52 49
+1 1 49 46
+1 1 46 43
+1 1 43 40
+1 1 40 37
+1 1 37 34
+1 1 34 31
+1 1 31 28
+1 1 28 25
+1 1 25 22
+1 1 22 19
+1 1 19 16
+1 1 16 13
+1 1 13 10
+1 1 10 7
+1 1 7 4
+1 1 4 3
+1 1 3 0
+
+vertices
+77
+2
+3.9788735773 0.0
+3.84329674785 1.02980825986
+2.88247256089 0.772356194895
+2.98415518297 0.0
+1.97241688113 0.259673608685
+3.44580559639 1.98943678865
+2.58435419729 1.49207759149
+1.83799993026 0.761324498753
+2.81348848799 2.81348848799
+2.11011636599 2.11011636599
+1.57832632157 1.21109238238
+1.98943678865 3.44580559639
+1.49207759149 2.58435419729
+1.21109238238 1.57832632157
+1.02980825986 3.84329674785
+0.772356194895 2.88247256089
+0.761324498753 1.83799993026
+2.43635739532e-16 3.9788735773
+1.82726804649e-16 2.98415518297
+0.259673608685 1.97241688113
+-1.02980825986 3.84329674785
+-0.772356194895 2.88247256089
+-0.259673608685 1.97241688113
+-1.98943678865 3.44580559639
+-1.49207759149 2.58435419729
+-0.761324498753 1.83799993026
+-2.81348848799 2.81348848799
+-2.11011636599 2.11011636599
+-1.21109238238 1.57832632157
+-3.44580559639 1.98943678865
+-2.58435419729 1.49207759149
+-1.57832632157 1.21109238238
+-3.84329674785 1.02980825986
+-2.88247256089 0.772356194895
+-1.83799993026 0.761324498753
+-3.9788735773 4.87271479065e-16
+-2.98415518297 3.65453609299e-16
+-1.97241688113 0.259673608685
+-3.84329674785 -1.02980825986
+-2.88247256089 -0.772356194895
+-1.97241688113 -0.259673608685
+-3.44580559639 -1.98943678865
+-2.58435419729 -1.49207759149
+-1.83799993026 -0.761324498753
+-2.81348848799 -2.81348848799
+-2.11011636599 -2.11011636599
+-1.57832632157 -1.21109238238
+-1.98943678865 -3.44580559639
+-1.49207759149 -2.58435419729
+-1.21109238238 -1.57832632157
+-1.02980825986 -3.84329674785
+-0.772356194895 -2.88247256089
+-0.761324498753 -1.83799993026
+-7.30907218597e-16 -3.9788735773
+-5.48180413948e-16 -2.98415518297
+-0.259673608685 -1.97241688113
+1.02980825986 -3.84329674785
+0.772356194895 -2.88247256089
+0.259673608685 -1.97241688113
+1.98943678865 -3.44580559639
+1.49207759149 -2.58435419729
+0.761324498753 -1.83799993026
+2.81348848799 -2.81348848799
+2.11011636599 -2.11011636599
+1.21109238238 -1.57832632157
+3.44580559639 -1.98943678865
+2.58435419729 -1.49207759149
+1.57832632157 -1.21109238238
+3.84329674785 -1.02980825986
+2.88247256089 -0.772356194895
+1.83799993026 -0.761324498753
+3.9788735773 -9.7454295813e-16
+2.98415518297 -7.30907218597e-16
+1.97241688113 -0.259673608685
+3.84329674785 1.02980825986
+2.88247256089 0.772356194895
+1.97241688113 0.259673608685

data/mixed-ele-meshes/annulus-trans-6.mesh

@@ -0,0 +1,74 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+20
+2 3 0 1 2 3
+2 3 1 5 6 2
+2 3 5 8 9 6
+2 3 8 11 12 9
+2 3 11 14 15 12
+2 3 14 17 18 15
+2 3 17 20 21 18
+1 2 2 3 4
+1 2 6 2 7
+1 2 9 6 10
+1 2 12 9 13
+1 2 15 12 16
+1 2 18 15 19
+1 2 21 18 22
+1 2 2 4 7
+1 2 6 7 10
+1 2 9 10 13
+1 2 12 13 16
+1 2 15 16 19
+1 2 18 19 22
+
+boundary
+17
+1 1 0 1
+1 1 1 5
+1 1 5 8
+1 1 8 11
+1 1 11 14
+1 1 14 17
+1 1 17 20
+1 1 20 21
+1 1 21 22
+1 1 22 19
+1 1 19 16
+1 1 16 13
+1 1 13 10
+1 1 10 7
+1 1 7 4
+1 1 4 3
+1 1 3 0
+
+vertices
+23
+2
+1.11408460164 0.0
+0.557042300822 0.964825566988
+0.417781725616 0.723619175241
+0.835563451232 0.0
+0.482412783494 0.278521150411
+-0.557042300822 0.964825566988
+-0.417781725616 0.723619175241
+3.41090035345e-17 0.557042300822
+-1.11408460164 1.36436014138e-16
+-0.835563451232 1.02327010604e-16
+-0.482412783494 0.278521150411
+-0.557042300822 -0.964825566988
+-0.417781725616 -0.723619175241
+-0.482412783494 -0.278521150411
+0.557042300822 -0.964825566988
+0.417781725616 -0.723619175241
+-1.02327010604e-16 -0.557042300822
+1.11408460164 -2.72872028276e-16
+0.835563451232 -2.04654021207e-16
+0.482412783494 -0.278521150411
+0.557042300822 0.964825566988
+0.417781725616 0.723619175241
+0.482412783494 0.278521150411

data/mixed-ele-meshes/circInSquare.py

@@ -0,0 +1,924 @@
+#Title:circInSquare.py
+#Author:T. M. McManus
+#Date:10-7-18
+#Purpose: Fill a circular sector with triangles and a bounding region,
+#defined by 3 nodes, with quads.  Then reflect/preserve QuadI twice to
+#create a complete disc bounded in a square.
+
+import scipy as sp
+import argparse
+import sys
+import subprocess
+import time
+
+parser=argparse.ArgumentParser(description='Fill a circular sector with triangles and a bounding region,\
+defined by 3 nodes, with quads.  Then reflect/preserve QuadI twice to create a complete disc bounded in a square.'
+,epilog='Sample run: python circInSquare.py -r 1 -e 2 -n 8 -g ../../../glvis/glvis')
+
+parser.add_argument('-r','--circRad', nargs='?',const=1, default = 1.0, type=float, help='Radius of circle')
+parser.add_argument('-e','--edgeLength', nargs='?',const=1,default=2.0,type=float,help='Edge-length of bounding square')
+parser.add_argument('-n','--numEdges',nargs='?',const=1,default=6,type=int,help='n-gon approximation of internal circle')
+parser.add_argument('-o','--outputFile',nargs='?',const=1,default='circInSquare', help='Output file name.')
+parser.add_argument('-g','--glvis',nargs='?',const=1,default='',type=str,help='Abs. or rel. path of glvis binary.')
+args=parser.parse_args()
+
+r=args.circRad 
+edgeLength=args.edgeLength
+numEdges=args.numEdges
+outputName=args.outputFile
+glvis=args.glvis
+
+visMesh=False;
+
+if glvis!='':
+    visMesh=True
+
+if r >= edgeLength:
+    print("Circle radius must be less than bounding square edge length")
+    sys.exit(1)
+
+if sp.mod(numEdges,2) != 0:
+    print("Currently this mixed element generator only supports an even numbers of edges.")
+    sys.exit(1)
+
+    
+#The basic idea:
+#1. Construct topology for regions
+#2. Combine topologies
+#3. Construct boundary
+#4. Construct geometry for regions
+#5. Combine geometries
+#6. Output
+
+def eleMatCirc(numEdges):
+    
+    nNodesSeq=sp.zeros([numEdges])
+    nNodesSeq[0]=3
+    if numEdges != 1:
+        for n in range(1,numEdges):
+            nNodesSeq[n]=nNodesSeq[n-1]+(2+n)
+            
+    numCircNodesTot =int(((numEdges+1)*(numEdges+2))/2)
+    
+    b=range(numCircNodesTot)
+    row_size=1
+    A=sp.zeros([numEdges+1,numEdges+1])
+    start=0;stop=1;
+    for m in range(numEdges+1):
+        if m==0:
+            A[m,range(m+1)]=b[0:1]
+            start=0
+            stop=1
+        else:
+            start=stop
+            stop=stop+m+1
+            A[m,range(m+1)]=b[start:stop]
+
+    M=sp.ones([numEdges**2,5])
+    m_row=0
+    for m in range(numEdges):
+        if m==0:
+            M[0,:]=[1,2,0,1,2]
+            m_row+=1
+        else:
+           holder=sp.size(sp.nonzero(A[m,:]))
+           for n in range(holder):
+               if n!=holder-1:
+                   M[m_row,:]=[1,2,A[m,n],A[m,n+1],A[m+1,n+1]]
+                   m_row+=1
+                   M[m_row,:]=[1,2,A[m,n],A[m+1,n],A[m+1,n+1]]
+                   m_row+=1
+               else:
+                   M[m_row,:]=[1,2,A[m,n],A[m+1,n],A[m+1,n+1]]
+                   m_row+=1
+            
+    return M.astype(int),numCircNodesTot
+
+def eleMatQuad(numEdges):
+    S0=numEdges*(numEdges+1)/(2.0)
+    A=sp.linspace(S0,(S0+(numEdges+1)**2)-1,(numEdges+1)**2)
+    A=A.reshape([numEdges+1,numEdges+1])
+    quadNode=sp.delete(A,-1,1)
+    quadNode=sp.delete(quadNode,-1,0)
+    quadNode=quadNode.flatten()
+    M=sp.zeros([numEdges**2,6])
+    for n in range(numEdges**2):
+        M[n,:]=[2,3,quadNode[n],quadNode[n]+1,quadNode[n]+numEdges+2,quadNode[n]+numEdges+1]
+    return M.astype(int)
+
+def boundMatTot(numEdges):
+    triS1=sp.zeros(numEdges+1)
+    triS3=sp.zeros(numEdges+1)
+    quadS1=sp.zeros(numEdges)
+    quadS2=sp.zeros(numEdges-1)
+    quadS3=sp.zeros(numEdges)
+
+    triS1[0]=0;
+    triS3[0]=0;
+    for n in range(1,numEdges+1):
+        triS1[n]=triS1[n-1]+n
+        triS3[n]=triS1[n]+n
+    ref1=triS3
+    
+    triS3=sp.flipud(triS3)
+    quadS1[0]=triS1[-1]+numEdges+1
+    quadS3[0]=triS1[-1]+2*numEdges+1
+
+    for n in range(1,numEdges):
+        quadS1[n]=quadS1[n-1]+(numEdges+1)
+        quadS3[n]=quadS3[n-1]+(numEdges+1)    
+    ref2=quadS3
+    xAxisRootRef=sp.concatenate([triS1.copy(),quadS1],axis=0)
+    quadS3=sp.flipud(quadS3)
+    quadS2=range(int(quadS1[-1]+1),int(quadS3[0]),1)
+    STOT=sp.concatenate([triS1,quadS1,quadS2,quadS3,triS3],axis=0)
+
+    filler=sp.zeros(1)
+    filler[0]=quadS3[0]
+    fillerFirst=sp.zeros(1)
+    fillerFirst[0]=quadS1[-1]
+    sTotRef=sp.concatenate([triS1,quadS1,quadS2,filler],axis=0)
+    newsTotRef=sp.concatenate([fillerFirst,quadS2,filler],axis=0)
+    boundMat=sp.zeros([STOT.size-1,4])
+    boundMatRef=sp.zeros([sTotRef.size-1,4])
+    new_boundMat_ref=sp.zeros([newsTotRef.size-1,4])
+    
+    for n in range(STOT.size-1):
+        boundMat[n,:]=[1,1,STOT[n],STOT[n+1]]
+    for n in range(sTotRef.size-1):
+        boundMatRef[n,:]=[1,1,sTotRef[n],sTotRef[n+1]]
+    for n in range(newsTotRef.size-1):
+        new_boundMat_ref[n,:]=[1,1,newsTotRef[n],newsTotRef[n+1]]
+        
+    ref=sp.concatenate([ref1,ref2],axis=0).astype(int)
+    return boundMat.astype(int),ref,boundMatRef.astype(int),xAxisRootRef.astype(int),new_boundMat_ref.astype(int)
+
+def vertMatCirc(numEdges):
+    r_o=sp.linspace(0,r,numEdges+1)
+    counter=0
+    vertMat=sp.zeros([numCircNodesTot,2])
+    for m in range(numEdges+1):
+        theta=sp.linspace(0,sp.pi/2.0,m+1)
+        for n in range(sp.size(theta)):
+            vertMat[counter,:]=[r_o[m]*sp.cos(theta[n]),r_o[m]*sp.sin(theta[n])]
+            counter+=1
+    return vertMat
+
+def vertMatQuad(numEdges):
+
+    theta=sp.linspace(0,sp.pi/2.0,numEdges+1)
+    AX=sp.zeros([numEdges+1,numEdges+1])
+    AY=sp.zeros([numEdges+1,numEdges+1])
+    AX[0,:]=r*sp.cos(theta)
+    AY[0,:]=r*sp.sin(theta)
+
+    vertLinSpace=sp.linspace(0,edgeLength,(numEdges/2)+1)
+    horzLineSpace=sp.linspace(edgeLength,0,(numEdges/2)+1)
+
+    #Assigning node locations along the boundary
+    vertCount=0
+    horzCount=1
+    for n in range(numEdges+1):
+        if n < (numEdges/2):
+            AX[-1,n]=edgeLength
+            AY[-1,n]=vertLinSpace[vertCount]
+            vertCount+=1
+        elif n == int(numEdges/2):
+            AX[-1,n]=edgeLength
+            AY[-1,n]=edgeLength
+        else:
+            AX[-1,n]=horzLineSpace[horzCount]
+            AY[-1,n]=edgeLength
+            horzCount+=1
+
+    #Linearly spacing nodes between the inner/outer boundaries
+    #One could then smooth this via r-based adaptivity
+    for col in range(numEdges+1):
+        for row in range(1,numEdges):
+            AX[row,col]=sp.linspace(AX[0,col],AX[-1,col],numEdges+1)[row]
+            AY[row,col]=sp.linspace(AY[0,col],AY[-1,col],numEdges+1)[row]
+
+    AX=sp.delete(AX,0,0)
+    AY=sp.delete(AY,0,0)
+    AX=AX.flatten()
+    AY=AY.flatten()
+    AX_reshape = AX.flatten()
+    
+    numQuadNodesTot=numEdges*(numEdges+1)
+    vertMat=sp.zeros([numQuadNodesTot,2])
+    for n in range(numQuadNodesTot):
+        vertMat[n,:]=[AX[n],AY[n]]    
+    return vertMat
+
+def orient(A):
+    aOrient=sp.zeros([A.shape[0],A.shape[1]])
+    triCounter=0
+    quadCounter=0
+    #Determine the number of triangle and quad elments in the given element matrix
+    for n in range(A.shape[0]):
+        if A[n,1]==2:
+            triCounter+=1
+        else:
+            quadCounter+=1
+    edgeMatTotal=sp.zeros([3*triCounter+4*quadCounter,2])
+    counter=0
+    for n in range(A.shape[0]):
+        detected=0
+        if A[n,1]==2:
+            for m in range(edgeMatTotal.shape[0]):
+                if detected != 1:
+                    if edgeMatTotal[m,0]==A[n,2] and edgeMatTotal[m,1]==A[n,3]:
+                        aOrient[n,:]=[1,2,A[n,2],A[n,4],A[n,3],0]
+                        detected=1
+                        #print("reorder:[{} {} {}] to [{} {} {}]".format(A[n,2],A[n,3],A[n,4],int(aOrient[n,2]),int(aOrient[n,3]),int(aOrient[n,4])))
+                    elif edgeMatTotal[m,0]==A[n,4] and edgeMatTotal[m,1]==A[n,2]:
+                        aOrient[n,:]=[1,2,A[n,2],A[n,4],A[n,3],0]
+                        detected=1
+                    else:
+                        aOrient[n,:]=A[n,:]
+
+            edgeMatTotal[counter,:]=[aOrient[n,2],aOrient[n,3]]
+            counter+=1
+            edgeMatTotal[counter,:]=[aOrient[n,3],aOrient[n,4]]
+            counter+=1
+            edgeMatTotal[counter,:]=[aOrient[n,4],aOrient[n,2]]
+            counter+=1
+        else:
+            for m in range(edgeMatTotal.shape[0]):
+                if detected != 1: 
+                    if edgeMatTotal[m,0]==A[n,2] and edgeMatTotal[m,1]==A[n,3]:
+                        aOrient[n,:]=[2,3,A[n,2],A[n,5],A[n,4],A[n,3]]
+                        detected=1
+                        #print("reorder:[{} {} {} {}] to [{} {} {} {}]".format(A[n,2],A[n,3],A[n,4],A[n,5],int(aOrient[n,2]),int(aOrient[n,3]),int(aOrient[n,4]),int(aOrient[n,5])))
+                    elif edgeMatTotal[m,0]==A[n,5] and edgeMatTotal[m,1]==A[n,2]:
+                        aOrient[n,:]=[2,3,A[n,2],A[n,5],A[n,4],A[n,3]]
+                        detected=1
+                    else:
+                        aOrient[n,:]=A[n,:]
+            edgeMatTotal[counter,:]=[aOrient[n,2],aOrient[n,3]]
+            counter+=1
+            edgeMatTotal[counter,:]=[aOrient[n,3],aOrient[n,4]]
+            counter+=1
+            edgeMatTotal[counter,:]=[aOrient[n,4],aOrient[n,5]]
+            counter+=1
+            edgeMatTotal[counter,:]=[aOrient[n,5],aOrient[n,2]]
+            counter+=1
+    return aOrient.astype(int)
+
+def gVis(_glvis,_meshFile):
+
+    if(_glvis==''):
+        print("Failure: Set glvis location via -g switch")
+        sys.exit(1)
+
+    colFuncFileName=_meshFile.replace('.mesh','.gf')
+    glvsScriptFileName=_meshFile.replace('.mesh','.glvs')
+    imageFileName=_meshFile.replace('.mesh','.png')
+    
+    #Create Coloring Function for mesh
+    _colFuncCommand=_glvis+ ' -m '+ _meshFile +' -sc -k q'
+    args=_colFuncCommand.split()
+    p=subprocess.Popen(args)#Create 'GLVis_coloring.gf'
+
+    _renameCommand='mv GLVis_coloring.gf {}'.format(colFuncFileName)
+    args=_renameCommand.split()
+    p=subprocess.Popen(args)
+
+    #Glvis script template
+    f=open(glvsScriptFileName,'w')
+    f.write('window 0 0 800 800\n'+'\n')
+    f.write('solution {} {}\n'.format(_meshFile,colFuncFileName)+'\n')
+    f.write('{\n'+'perspective off\n'+'zoom 1.5\n'+'keys gAeeRM\n'+'solution {} {} screenshot {}\n'.format(_meshFile,colFuncFileName,imageFileName)+'keys q\n'+'}\n')
+    f.close()
+
+    _runGlvisCommand=_glvis+' -run {}'.format(glvsScriptFileName)
+    args=_runGlvisCommand.split()
+    p=subprocess.Popen(args)
+    p.wait()
+    
+    return 0
+def quadInterDof(_edge,_linEleMat,_linVertMatRound):
+    _state=False
+    for n in range(_linEleMat.shape[0]):
+        if _linEleMat[n,1]==3:
+            if sp.any(_edge[0]==_linEleMat[n,2:6]) and sp.any(_edge[1]==_linEleMat[n,2:6]):
+                print("{} is possibly in {}".format(_edge,_linEleMat[n,2:6]))
+                _n1Loc=sp.where(_edge[0]==_linEleMat[n,2:6])[0][0]
+                _n2Loc=sp.where(_edge[1]==_linEleMat[n,2:6])[0][0]
+                if _n1Loc==sp.mod(_n2Loc+1,4) or _n1Loc==sp.mod(_n2Loc-1,4):
+                    _state=True
+                    xcent=(_linVertMatRound[_linEleMat[n,2],0]+_linVertMatRound[_linEleMat[n,3],0]+_linVertMatRound[_linEleMat[n,4],0]+_linVertMatRound[_linEleMat[n,5],0])/4.0
+                    ycent=(_linVertMatRound[_linEleMat[n,2],1]+_linVertMatRound[_linEleMat[n,3],1]+_linVertMatRound[_linEleMat[n,4],1]+_linVertMatRound[_linEleMat[n,5],1])/4.0
+                    _interDof=sp.zeros(2)
+                    _interDof[0]=sp.round_((_linVertMatRound[_edge[0],0]+_linVertMatRound[_edge[1],0]+xcent)/3.0,5)
+                    _interDof[1]=sp.round_((_linVertMatRound[_edge[0],1]+_linVertMatRound[_edge[1],1]+ycent)/3.0,5)
+                    print("dof loc is {},{}".format(_interDof[0],_interDof[1]))
+                    return(_state,_interDof[0],_interDof[1])
+
+    return(_state,0,0)
+
+[eleMatTriHolder,numCircNodesTot]=eleMatCirc(numEdges) #Construct tri element matrix for the region inside circular sector
+eleMatQuadHolder=eleMatQuad(numEdges) #Construct quad element matrix for region outside the circular sector
+
+#Combining eleMatTriHolder and eleMatQuadHolder
+linEleMat=sp.zeros([eleMatTriHolder.shape[0]+eleMatQuadHolder.shape[0],6])
+counter=0
+for n in range(eleMatTriHolder.shape[0]):
+    linEleMat[n,[0,1,2,3,4]]=eleMatTriHolder[n,:]
+    counter+=1
+for n in range(eleMatQuadHolder.shape[0]):
+    linEleMat[counter+n,:]=eleMatQuadHolder[n,:]
+
+linEleMat=linEleMat.astype(int)
+linBoundMat=boundMatTot(numEdges)[0] #Construct the boundary
+vertMatCircHolder = vertMatCirc(numEdges) #Construct vertex matrix for triang region
+vertMatQuadHolder = vertMatQuad(numEdges) #Construct vertex matrix for the quad region
+
+
+#Combining the two vertex matrices in Quadrant I (q1)
+linVertMat=sp.zeros([vertMatCircHolder.shape[0]+vertMatQuadHolder.shape[0],2])
+counter=0
+for n in range(vertMatCircHolder.shape[0]):
+    linVertMat[n,:]=vertMatCircHolder[n,:]
+    counter+=1
+for n in range(vertMatQuadHolder.shape[0]):
+    linVertMat[counter+n,:]=vertMatQuadHolder[n,:]
+
+#Outputting P1/Q1 mesh to a .mesh file
+g=open(outputName+'Lin.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(linEleMat.shape[0]))
+for n in range(linEleMat.shape[0]):
+    if linEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4],linEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(linBoundMat.shape[0]))
+for n in range(linBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(linBoundMat[n,0],linBoundMat[n,1],linBoundMat[n,2],linBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(linVertMat.shape[0])+'2\n')
+for n in range(linVertMat.shape[0]):
+    g.write('{} {}\n'.format(linVertMat[n,0],linVertMat[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'Lin.mesh')
+
+#Quadratic (P2/Q2) Element Generation
+
+#1.)Create Edge list from previously generated linear elements
+edgeMat=sp.zeros([3*eleMatTriHolder.shape[0]+4*eleMatQuadHolder.shape[0],2])
+linEleMat=orient(linEleMat)#Make sure that element orientation is in agreement with MFEM requirements
+
+counter=0
+for n in range(linEleMat.shape[0]):
+    if linEleMat[n,1]==2:
+        edgeMat[counter,:]=[linEleMat[n,2],linEleMat[n,3]]
+        counter+=1
+        edgeMat[counter,:]=[linEleMat[n,3],linEleMat[n,4]]
+        counter+=1
+        edgeMat[counter,:]=[linEleMat[n,4],linEleMat[n,2]]
+        counter+=1
+    else:
+        edgeMat[counter,:]=[linEleMat[n,2],linEleMat[n,3]]
+        counter+=1
+        edgeMat[counter,:]=[linEleMat[n,3],linEleMat[n,4]]
+        counter+=1
+        edgeMat[counter,:]=[linEleMat[n,4],linEleMat[n,5]]
+        counter+=1
+        edgeMat[counter,:]=[linEleMat[n,5],linEleMat[n,2]]
+        counter+=1
+
+#Remove duplicates
+holder=[]
+for n in range(edgeMat.shape[0]):
+    counter=0
+    for m in range(edgeMat.shape[0]):
+        if edgeMat[n,0]==edgeMat[m,0] and edgeMat[n,1]==edgeMat[m,1] and m!=n:
+            holder.append([n,m])
+        elif edgeMat[n,1]==edgeMat[m,0] and edgeMat[n,0]==edgeMat[m,1] and m!=n:
+            holder.append([n,m])
+
+removeIndices=sp.zeros(len(holder))
+for n in range(len(holder)):
+    if holder[n][0]>holder[n][1]:
+        removeIndices[n]=holder[n][0]
+    else:
+        removeIndices[n]=holder[n][1]
+removeIndices=sp.unique(removeIndices).astype(int)
+edgeMat=sp.delete(edgeMat,removeIndices,0)
+edgeMat=edgeMat.astype(int)
+
+edgeDofMat=sp.zeros([edgeMat.shape[0],2])#These will be the new DoFs that appear after the Element Vertices within the .mesh file
+linVertMatRound=sp.round_(linVertMat,5) 
+ 
+counter=0
+
+for n in edgeMat:
+    if linVertMatRound[n[0],1] == linVertMatRound[n[1],1]:
+        xmid=(linVertMatRound[n[0],0]+linVertMatRound[n[1],0])/2.0
+        ymid=linVertMatRound[n[0],1]
+        edgeDofMat[counter,:]=[xmid,ymid]
+    elif linVertMatRound[n[0],0] == linVertMatRound[n[1],0]:
+        xmid=linVertMatRound[n[0],0]
+        ymid=(linVertMatRound[n[0],1]+linVertMatRound[n[1],1])/2.0
+        edgeDofMat[counter,:]=[xmid,ymid]
+    else:
+        r0=sp.sqrt(linVertMatRound[n[0],0]**2+linVertMatRound[n[0],1]**2)
+        r1=sp.sqrt(linVertMatRound[n[1],0]**2+linVertMatRound[n[1],1]**2)
+        rmid = (r0+r1)/2.0 #should not be needed
+        xmidOld=(linVertMatRound[n[0],0]+linVertMatRound[n[1],0])/2.0
+        ymidOld=(linVertMatRound[n[0],1]+linVertMatRound[n[1],1])/2.0
+        midtheta=sp.arctan(ymidOld/xmidOld)
+        xmid=rmid*sp.cos(midtheta)
+        ymid=rmid*sp.sin(midtheta)
+        edgeDofMat[counter,:]=[xmid,ymid]
+    counter+=1
+edgeDofMat = sp.round_(edgeDofMat,5)
+
+#Determine midpoints of all Q1 elements:
+quadCentroidLoc=sp.zeros([eleMatQuadHolder.shape[0],2])
+for n in range(eleMatQuadHolder.shape[0]):
+    quadCentroidLoc[n,0]=(linVertMatRound[eleMatQuadHolder[n,2],0]+linVertMatRound[eleMatQuadHolder[n,3],0]+linVertMatRound[eleMatQuadHolder[n,4],0]+linVertMatRound[eleMatQuadHolder[n,5],0])/4.0
+    quadCentroidLoc[n,1]=(linVertMatRound[eleMatQuadHolder[n,2],1]+linVertMatRound[eleMatQuadHolder[n,3],1]+linVertMatRound[eleMatQuadHolder[n,4],1]+linVertMatRound[eleMatQuadHolder[n,5],1])/4.0
+
+quadCentroidLoc = sp.round_(quadCentroidLoc,5)
+
+#3.)Populate nodes section
+g=open(outputName+'Quad.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(linEleMat.shape[0]))
+for n in range(linEleMat.shape[0]):
+    if linEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4],linEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(linBoundMat.shape[0]))
+for n in range(linBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(linBoundMat[n,0],linBoundMat[n,1],linBoundMat[n,2],linBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(linVertMat.shape[0]))
+g.write('\n'+'nodes'+'\n'+'FiniteElementSpace'+'\n'+'FiniteElementCollection: H1_2D_P2'+'\n'+'VDim: 2'+'\n'+'Ordering: 1' +'\n\n')
+for n in range(linVertMatRound.shape[0]):
+    g.write('{} {}\n'.format(linVertMatRound[n,0],linVertMatRound[n,1]))
+for n in range(edgeDofMat.shape[0]):
+    g.write('{} {}\n'.format(edgeDofMat[n,0],edgeDofMat[n,1]))
+for n in range(quadCentroidLoc.shape[0]):
+    g.write('{} {}\n'.format(quadCentroidLoc[n,0],quadCentroidLoc[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'Quad.mesh')
+
+#Cubic (P3/Q3) Element Generation
+
+cubeDofMat=sp.zeros([2*edgeMat.shape[0],2])#These will be the new DoFs that appear after the Element Vertices within the .mesh file
+
+counter=0
+for n in edgeMat: #Here DoF ordering matters. 
+    if linVertMatRound[n[0],1] == linVertMatRound[n[1],1]:
+        xmid=(linVertMatRound[n[0],0]+linVertMatRound[n[1],0])/2.0
+        ymid=linVertMatRound[n[0],1]
+        xmid1=(linVertMatRound[n[0],0]+xmid)/2.0
+        ymid1=linVertMatRound[n[0],1]
+        xmid2=(linVertMatRound[n[1],0]+xmid)/2.0
+        ymid2=linVertMatRound[n[0],1]
+        if n[0] > n[1]:
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+        else:
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+            
+    elif linVertMatRound[n[0],0] == linVertMatRound[n[1],0]:
+        xmid=linVertMatRound[n[0],0]
+        ymid=(linVertMatRound[n[0],1]+linVertMatRound[n[1],1])/2.0
+        xmid1=linVertMatRound[n[0],0]
+        ymid1=(linVertMatRound[n[0],1]+ymid)/2.0
+        xmid2=linVertMatRound[n[0],0]
+        ymid2=(linVertMatRound[n[1],1]+ymid)/2.0
+        if n[0] > n[1]:
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+        else:
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+    else:
+        r0=sp.sqrt(linVertMatRound[n[0],0]**2+linVertMatRound[n[0],1]**2)
+        r1=sp.sqrt(linVertMatRound[n[1],0]**2+linVertMatRound[n[1],1]**2)
+        rmid = (r0+r1)/2.0 #should not be needed
+        xmidOld=(linVertMatRound[n[0],0]+linVertMatRound[n[1],0])/2.0
+        ymidOld=(linVertMatRound[n[0],1]+linVertMatRound[n[1],1])/2.0
+        midtheta=sp.arctan(ymidOld/xmidOld)
+        xmid=rmid*sp.cos(midtheta)
+        ymid=rmid*sp.sin(midtheta)
+        xmid1=(linVertMatRound[n[0],0]+xmid)/2.0
+        ymid1=(linVertMatRound[n[0],1]+ymid)/2.0
+        xmid2=(linVertMatRound[n[1],0]+xmid)/2.0
+        ymid2=(linVertMatRound[n[1],1]+ymid)/2.0
+        if n[0] > n[1]:
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+        else:
+            cubeDofMat[counter,:]=[xmid1,ymid1]
+            counter+=1
+            cubeDofMat[counter,:]=[xmid2,ymid2]
+            counter+=1
+            
+cubeDofMat = sp.round_(cubeDofMat,5)
+
+triCentroidLoc=sp.zeros([eleMatTriHolder.shape[0],2])
+
+for n in range(eleMatTriHolder.shape[0]):
+    triCentroidLoc[n,0]=(linVertMatRound[eleMatTriHolder[n,2],0]+linVertMatRound[eleMatTriHolder[n,3],0]+linVertMatRound[eleMatTriHolder[n,4],0])/3.0
+    triCentroidLoc[n,1]=(linVertMatRound[eleMatTriHolder[n,2],1]+linVertMatRound[eleMatTriHolder[n,3],1]+linVertMatRound[eleMatTriHolder[n,4],1])/3.0
+
+quadCentroidLocCubic=sp.zeros([4*eleMatQuadHolder.shape[0],2])
+
+counter=0
+for n in range(eleMatQuadHolder.shape[0]):
+    xcent=quadCentroidLoc[n,0];ycent=quadCentroidLoc[n,1]
+    a=eleMatQuadHolder[n,2:6]
+    aMinIndex=sp.where(a[:]==a.min())[0][0]
+    dof0=0.5*sp.array([xcent+linVertMatRound[a[aMinIndex],0],ycent+linVertMatRound[a[aMinIndex],1]])
+    quadCentroidLocCubic[counter,:]=dof0
+    counter+=1
+    if aMinIndex==0:
+        aLeft=-1
+        aRight=1
+        aLast=2
+    else:
+        aLeft=aMinIndex-1
+        aRight=aMinIndex+1
+        aLast=sp.delete(a,[aMinIndex,aLeft,aRight])[0]
+    edge1=[a[aMinIndex], a[aLeft]]
+    edge2=[a[aMinIndex], a[aRight]]
+    edge1Index=0
+    edge2Index=0
+    edgeCounter=0
+    for edge in edgeMat:
+        if(edge[0]==edge1[0] and edge[1]==edge1[1]) or (edge[1]==edge1[0] and edge[0]==edge1[1]):
+            edge1Index=edgeCounter
+        if(edge[0]==edge2[0] and edge[1]==edge2[1]) or (edge[1]==edge2[0] and edge[0]==edge2[1]):
+            edge2Index=edgeCounter
+        edgeCounter+=1
+
+    if (edge1Index > edge2Index):
+        dof1=0.5*sp.array([xcent+linVertMatRound[a[aLeft],0],ycent+linVertMatRound[a[aLeft],1]])
+        quadCentroidLocCubic[counter,:]=dof1
+        counter+=1
+        dof2=0.5*sp.array([xcent+linVertMatRound[a[aRight],0],ycent+linVertMatRound[a[aRight],1]])
+        quadCentroidLocCubic[counter,:]=dof2
+        counter+=1
+        dof3=0.5*sp.array([xcent+linVertMatRound[a[aLast],0],ycent+linVertMatRound[a[aLast],1]])
+        quadCentroidLocCubic[counter,:]=dof3
+        counter+=1
+    else:
+        dof1=0.5*sp.array([xcent+linVertMatRound[a[aRight],0],ycent+linVertMatRound[a[aRight],1]])
+        quadCentroidLocCubic[counter,:]=dof1
+        counter+=1
+        dof2=0.5*sp.array([xcent+linVertMatRound[a[aLeft],0],ycent+linVertMatRound[a[aLeft],1]])
+        quadCentroidLocCubic[counter,:]=dof2
+        counter+=1
+        dof3=0.5*sp.array([xcent+linVertMatRound[a[aLast],0],ycent+linVertMatRound[a[aLast],1]])
+        quadCentroidLocCubic[counter,:]=dof3
+        counter+=1
+        
+truCentroidLoc=sp.round_(triCentroidLoc,5)
+    
+#3.)Populate nodes section
+g=open(outputName+'Cub.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(linEleMat.shape[0]))
+for n in range(linEleMat.shape[0]):
+    if linEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(linEleMat[n,0],linEleMat[n,1],linEleMat[n,2],linEleMat[n,3],linEleMat[n,4],linEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(linBoundMat.shape[0]))
+for n in range(linBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(linBoundMat[n,0],linBoundMat[n,1],linBoundMat[n,2],linBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(linVertMat.shape[0]))
+g.write('\n'+'nodes'+'\n'+'FiniteElementSpace'+'\n'+'FiniteElementCollection: H1_2D_P3'+'\n'+'VDim: 2'+'\n'+'Ordering: 1' +'\n\n')
+for n in range(linVertMatRound.shape[0]):
+    g.write('{} {}\n'.format(linVertMatRound[n,0],linVertMatRound[n,1]))
+for n in range(cubeDofMat.shape[0]):
+    g.write('{} {}\n'.format(cubeDofMat[n,0],cubeDofMat[n,1]))
+for n in range(triCentroidLoc.shape[0]):
+    g.write('{} {}\n'.format(triCentroidLoc[n,0],triCentroidLoc[n,1]))
+for n in range(quadCentroidLocCubic.shape[0]):
+    g.write('{} {}\n'.format(quadCentroidLocCubic[n,0],quadCentroidLocCubic[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'Cub.mesh')
+#raw_input()
+#'Reflecting' topology about one of its edges and append it to itself
+upperPlaneEleMat = sp.zeros([2*linEleMat.shape[0],6])
+for n in range(linEleMat.shape[0]):
+    upperPlaneEleMat[n,:]=linEleMat[n,:]
+    
+#Create ele_mat_holder.shape[0]x2 matrix for mapping
+refEdge=boundMatTot(numEdges)[1]
+q1NumNodes=linVertMat.shape[0]
+
+mapping = sp.zeros([q1NumNodes])
+counter=0
+for n in range(q1NumNodes):
+     if (sp.any(refEdge == n)):
+         mapping[n]=n
+     else:
+         mapping[n]=counter+q1NumNodes
+         counter+=1
+
+mapping=mapping.astype(int)
+#Implement mapping
+
+counter=0
+for n in range(linEleMat.shape[0],2*linEleMat.shape[0]):
+    upperPlaneEleMat[n,0]=linEleMat[counter,0]
+    upperPlaneEleMat[n,1]=linEleMat[counter,1]
+    upperPlaneEleMat[n,2]=mapping[linEleMat[counter,2]]
+    upperPlaneEleMat[n,3]=mapping[linEleMat[counter,3]]
+    upperPlaneEleMat[n,4]=mapping[linEleMat[counter,4]]
+    upperPlaneEleMat[n,5]=mapping[linEleMat[counter,5]]
+    counter+=1
+    
+upperPlaneEleMat = upperPlaneEleMat.astype(int)
+
+#Reflecting boundary matrix
+origBound=boundMatTot(numEdges)[2]
+upperPlaneBoundMat=sp.zeros([2*origBound.shape[0],4])
+for n in range(origBound.shape[0]):
+    upperPlaneBoundMat[n,:]=origBound[n,:]
+counter=0
+newOrigBound=origBound.copy()
+newOrigBound[:,2]=sp.flipud(origBound[:,3])
+newOrigBound[:,3]=sp.flipud(origBound[:,2])
+for n in range(newOrigBound.shape[0],upperPlaneBoundMat.shape[0]):
+    upperPlaneBoundMat[n,0]=newOrigBound[counter,0]
+    upperPlaneBoundMat[n,1]=newOrigBound[counter,1]
+    upperPlaneBoundMat[n,2]=mapping[newOrigBound[counter,2]]
+    upperPlaneBoundMat[n,3]=mapping[newOrigBound[counter,3]]
+    counter+=1
+upperPlaneBoundMat=upperPlaneBoundMat.astype(int)
+
+#Reflecting vertex matrix about the y-axis and appending it to itself
+upperPlaneNumNodes=q1NumNodes+(q1NumNodes-refEdge.shape[0])
+upperPlaneVertMat = sp.zeros([upperPlaneNumNodes,2])
+for n in range(linVertMat.shape[0]):
+    upperPlaneVertMat[n,:]=linVertMat[n,:]
+counter=0
+for n in range(linVertMat.shape[0],upperPlaneNumNodes):
+        upperPlaneVertMat[n,0]=-1.0*linVertMat[sp.where(mapping==n)[0][0],0]
+        upperPlaneVertMat[n,1]=linVertMat[sp.where(mapping==n)[0][0],1]
+        counter+=1
+
+upperPlaneEleMat=orient(upperPlaneEleMat)
+g=open(outputName+'UpperPlaneLin.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(upperPlaneEleMat.shape[0]))
+for n in range(upperPlaneEleMat.shape[0]):
+    if upperPlaneEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(upperPlaneEleMat[n,0],upperPlaneEleMat[n,1],upperPlaneEleMat[n,2],upperPlaneEleMat[n,3],upperPlaneEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(upperPlaneEleMat[n,0],upperPlaneEleMat[n,1],upperPlaneEleMat[n,2],upperPlaneEleMat[n,3],upperPlaneEleMat[n,4],upperPlaneEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(upperPlaneBoundMat.shape[0]))
+for n in range(upperPlaneBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(upperPlaneBoundMat[n,0],upperPlaneBoundMat[n,1],upperPlaneBoundMat[n,2],upperPlaneBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(upperPlaneVertMat.shape[0])+'2\n')
+for n in range(upperPlaneVertMat.shape[0]):
+    g.write('{} {}\n'.format(upperPlaneVertMat[n,0],upperPlaneVertMat[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'UpperPlaneLin.mesh')
+
+#'Reflecting' topology about one of its edges and append it to itself
+wholePlaneEleMat = sp.zeros([2*upperPlaneEleMat.shape[0],6])
+for n in range(upperPlaneEleMat.shape[0]):
+    wholePlaneEleMat[n,:]=upperPlaneEleMat[n,:]
+
+quad1Edge=boundMatTot(numEdges)[3]
+newRefEdge=sp.zeros(2*quad1Edge.shape[0]-1)
+for n in range(quad1Edge.shape[0]):
+    newRefEdge[n]=quad1Edge[n]
+counter=0
+for n in range(quad1Edge.shape[0],newRefEdge.shape[0]):
+    newRefEdge[n]=mapping[quad1Edge[counter]]
+    counter+=1
+newRefEdge=sp.unique(newRefEdge)
+newRefEdge=newRefEdge.astype(int)
+newTotNumNodes=upperPlaneVertMat.shape[0]
+
+newMapping=sp.zeros([newTotNumNodes])
+counter=0
+for n in range(newTotNumNodes):
+    if (sp.any(newRefEdge == n)):
+        newMapping[n]=n
+    else:
+        newMapping[n]=counter+newTotNumNodes
+        counter+=1
+newMapping=newMapping.astype(int)
+
+counter=0
+for n in range(upperPlaneEleMat.shape[0],2*upperPlaneEleMat.shape[0]):
+    wholePlaneEleMat[n,0]=upperPlaneEleMat[counter,0]
+    wholePlaneEleMat[n,1]=upperPlaneEleMat[counter,1]
+    wholePlaneEleMat[n,2]=newMapping[upperPlaneEleMat[counter,2]]
+    wholePlaneEleMat[n,3]=newMapping[upperPlaneEleMat[counter,3]]
+    wholePlaneEleMat[n,4]=newMapping[upperPlaneEleMat[counter,4]]
+    wholePlaneEleMat[n,5]=newMapping[upperPlaneEleMat[counter,5]]
+    counter+=1
+
+wholePlaneEleMat=wholePlaneEleMat.astype(int)
+
+#Reflecting boundary matrix
+newOrigBoundQuad1=boundMatTot(numEdges)[4]
+newFirstBoundMatHolder=sp.zeros([2*newOrigBoundQuad1.shape[0],4])
+for n in range(newOrigBoundQuad1.shape[0]):
+    newFirstBoundMatHolder[n,:]=newOrigBoundQuad1[n,:]
+
+newNewOrigBoundQuad1=newOrigBoundQuad1.copy()
+newNewOrigBoundQuad1[:,2]=sp.flipud(newOrigBoundQuad1[:,3])
+newNewOrigBoundQuad1[:,3]=sp.flipud(newOrigBoundQuad1[:,2])
+counter=0
+for n in range(newOrigBoundQuad1.shape[0],newFirstBoundMatHolder.shape[0]):
+    newFirstBoundMatHolder[n,0]=newNewOrigBoundQuad1[counter,0]
+    newFirstBoundMatHolder[n,1]=newNewOrigBoundQuad1[counter,1]
+    newFirstBoundMatHolder[n,2]=mapping[newNewOrigBoundQuad1[counter,2]]
+    newFirstBoundMatHolder[n,3]=mapping[newNewOrigBoundQuad1[counter,3]]
+    counter+=1
+
+upperQuadMat=newFirstBoundMatHolder.copy()
+wholePlaneBoundMat=sp.zeros([2*upperQuadMat.shape[0],4])
+for n in range(upperQuadMat.shape[0]):
+    wholePlaneBoundMat[n,:]=upperQuadMat[n,:]
+
+counter=0
+newNewOrigBound=upperQuadMat.copy()
+newNewOrigBound[:,2]=sp.flipud(upperQuadMat[:,3])
+newNewOrigBound[:,3]=sp.flipud(upperQuadMat[:,2])
+newNewOrigBound=newNewOrigBound.astype(int)
+for n in range(newNewOrigBound.shape[0],wholePlaneBoundMat.shape[0]):
+    wholePlaneBoundMat[n,0]=newNewOrigBound[counter,0]
+    wholePlaneBoundMat[n,1]=newNewOrigBound[counter,1]
+    wholePlaneBoundMat[n,2]=newMapping[newNewOrigBound[counter,2]]
+    wholePlaneBoundMat[n,3]=newMapping[newNewOrigBound[counter,3]]
+    counter+=1
+wholePlaneBoundMat=wholePlaneBoundMat.astype(int)
+
+wholePlaneNumNodes=newTotNumNodes+(newTotNumNodes-newRefEdge.shape[0])
+wholePlaneVertMat = sp.zeros([wholePlaneNumNodes,2])
+for n in range(upperPlaneVertMat.shape[0]):
+    wholePlaneVertMat[n,:]=upperPlaneVertMat[n,:]
+counter=0
+for n in range(upperPlaneVertMat.shape[0],wholePlaneNumNodes):
+        wholePlaneVertMat[n,0]=upperPlaneVertMat[sp.where(newMapping==n)[0][0],0]
+        wholePlaneVertMat[n,1]=-1.0*upperPlaneVertMat[sp.where(newMapping==n)[0][0],1]
+        counter+=1
+
+g=open(outputName+'WholePlaneLin.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(wholePlaneEleMat.shape[0]))
+for n in range(wholePlaneEleMat.shape[0]):
+    if wholePlaneEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(wholePlaneEleMat[n,0],wholePlaneEleMat[n,1],wholePlaneEleMat[n,2],wholePlaneEleMat[n,3],wholePlaneEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(wholePlaneEleMat[n,0],wholePlaneEleMat[n,1],wholePlaneEleMat[n,2],wholePlaneEleMat[n,3],wholePlaneEleMat[n,4],wholePlaneEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(wholePlaneBoundMat.shape[0]))
+for n in range(wholePlaneBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(wholePlaneBoundMat[n,0],wholePlaneBoundMat[n,1],wholePlaneBoundMat[n,2],wholePlaneBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(wholePlaneVertMat.shape[0])+'2\n')
+for n in range(wholePlaneVertMat.shape[0]):
+    g.write('{} {}\n'.format(wholePlaneVertMat[n,0],wholePlaneVertMat[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'WholePlaneLin.mesh')
+
+#1.)Create Edge list from elements
+wholePlaneEleMat=orient(wholePlaneEleMat)
+triCounter=0;quadCounter=0;
+for n in range(wholePlaneEleMat.shape[0]):
+    if wholePlaneEleMat[n,1]==2:
+        triCounter+=1
+    else:
+        quadCounter+=1
+edgeMat=sp.zeros([3*triCounter+4*quadCounter,2])
+counter=0
+for n in range(wholePlaneEleMat.shape[0]):
+    if wholePlaneEleMat[n,1]==2:
+        edgeMat[counter,:]=[wholePlaneEleMat[n,2],wholePlaneEleMat[n,3]]
+        counter+=1
+        edgeMat[counter,:]=[wholePlaneEleMat[n,3],wholePlaneEleMat[n,4]]
+        counter+=1
+        edgeMat[counter,:]=[wholePlaneEleMat[n,4],wholePlaneEleMat[n,2]]
+        counter+=1
+    else:
+        edgeMat[counter,:]=[wholePlaneEleMat[n,2],wholePlaneEleMat[n,3]]
+        counter+=1
+        edgeMat[counter,:]=[wholePlaneEleMat[n,3],wholePlaneEleMat[n,4]]
+        counter+=1
+        edgeMat[counter,:]=[wholePlaneEleMat[n,4],wholePlaneEleMat[n,5]]
+        counter+=1
+        edgeMat[counter,:]=[wholePlaneEleMat[n,5],wholePlaneEleMat[n,2]]
+        counter+=1
+
+#Remove duplicates
+holder=[]
+for n in range(edgeMat.shape[0]):
+    counter=0
+    for m in range(edgeMat.shape[0]):
+        if edgeMat[n,0]==edgeMat[m,0] and edgeMat[n,1]==edgeMat[m,1] and m!=n:
+            holder.append([n,m])
+        elif edgeMat[n,1]==edgeMat[m,0] and edgeMat[n,0]==edgeMat[m,1] and m!=n:
+            holder.append([n,m])
+removeIndices=sp.zeros(len(holder))
+for n in range(len(holder)):
+    if holder[n][0]>holder[n][1]:
+        removeIndices[n]=holder[n][0]
+    else:
+        removeIndices[n]=holder[n][1]
+removeIndices=sp.unique(removeIndices).astype(int)
+edgeMat=sp.delete(edgeMat,removeIndices,0)
+edgeMat=edgeMat.astype(int)
+
+edgeDofMat=sp.zeros([edgeMat.shape[0],2])
+
+wholePlaneVertMatRound=sp.round_(wholePlaneVertMat,5)
+
+counter=0
+for n in edgeMat:
+    if wholePlaneVertMatRound[n[0],1] == wholePlaneVertMatRound[n[1],1]:
+        xmid=(wholePlaneVertMatRound[n[0],0]+wholePlaneVertMatRound[n[1],0])/2.0
+        ymid=wholePlaneVertMatRound[n[0],1]
+        edgeDofMat[counter,:]=[xmid,ymid]
+    elif wholePlaneVertMatRound[n[0],0] == wholePlaneVertMatRound[n[1],0]:
+        xmid=wholePlaneVertMatRound[n[0],0]
+        ymid=(wholePlaneVertMatRound[n[0],1]+wholePlaneVertMatRound[n[1],1])/2.0
+        edgeDofMat[counter,:]=[xmid,ymid]
+    else:
+        r0=sp.sqrt(wholePlaneVertMatRound[n[0],0]**2+wholePlaneVertMatRound[n[0],1]**2)
+        r1=sp.sqrt(wholePlaneVertMatRound[n[1],0]**2+wholePlaneVertMatRound[n[1],1]**2)
+        rmid = (r0+r1)/2.0 #should not be needed
+        xmidOld=(wholePlaneVertMatRound[n[0],0]+wholePlaneVertMatRound[n[1],0])/2.0
+        ymidOld=(wholePlaneVertMatRound[n[0],1]+wholePlaneVertMatRound[n[1],1])/2.0
+        midtheta=sp.arctan2(ymidOld,xmidOld)
+        xmid=rmid*sp.cos(midtheta)
+        ymid=rmid*sp.sin(midtheta)
+        edgeDofMat[counter,:]=[xmid,ymid]
+    counter+=1
+edgeDofMat = sp.round_(edgeDofMat,5)
+
+#2.)Create correct dof locations
+#Determine midpoints of all quads:
+quadCentroidLoc=sp.zeros([quadCounter,2])
+counter=0
+for n in range(wholePlaneEleMat.shape[0]):
+    if wholePlaneEleMat[n,1]==3:
+        quadCentroidLoc[counter,0]=(wholePlaneVertMatRound[wholePlaneEleMat[n,2],0]+wholePlaneVertMatRound[wholePlaneEleMat[n,3],0]+wholePlaneVertMatRound[wholePlaneEleMat[n,4],0]+wholePlaneVertMatRound[wholePlaneEleMat[n,5],0])/4.0
+        quadCentroidLoc[counter,1]=(wholePlaneVertMatRound[wholePlaneEleMat[n,2],1]+wholePlaneVertMatRound[wholePlaneEleMat[n,3],1]+wholePlaneVertMatRound[wholePlaneEleMat[n,4],1]+wholePlaneVertMatRound[wholePlaneEleMat[n,5],1])/4.0
+        counter+=1
+
+quadCentroidLoc = sp.round_(quadCentroidLoc,5)
+
+#3.)Populate nodes section
+g=open(outputName+'WholePlaneQuad.mesh','w')
+g.write('MFEM mesh v1.0\n'+'\n')
+g.write('dimension\n'+'2\n'+'\n')
+g.write('elements\n'+'{}\n'.format(wholePlaneEleMat.shape[0]))
+for n in range(wholePlaneEleMat.shape[0]):
+    if wholePlaneEleMat[n,1]==2:
+        g.write('{} {} {} {} {}\n'.format(wholePlaneEleMat[n,0],wholePlaneEleMat[n,1],wholePlaneEleMat[n,2],wholePlaneEleMat[n,3],wholePlaneEleMat[n,4]))
+    else:
+        g.write('{} {} {} {} {} {}\n'.format(wholePlaneEleMat[n,0],wholePlaneEleMat[n,1],wholePlaneEleMat[n,2],wholePlaneEleMat[n,3],wholePlaneEleMat[n,4],wholePlaneEleMat[n,5]))
+g.write('\n'+'boundary\n'+'{}\n'.format(wholePlaneBoundMat.shape[0]))
+for n in range(wholePlaneBoundMat.shape[0]):
+    g.write('{} {} {} {}\n'.format(wholePlaneBoundMat[n,0],wholePlaneBoundMat[n,1],wholePlaneBoundMat[n,2],wholePlaneBoundMat[n,3]))
+g.write('\n'+'vertices\n'+'{}\n'.format(wholePlaneVertMat.shape[0]))
+g.write('\n'+'nodes'+'\n'+'FiniteElementSpace'+'\n'+'FiniteElementCollection: H1_2D_P2'+'\n'+'VDim: 2'+'\n'+'Ordering: 1' +'\n\n')
+for n in range(wholePlaneVertMatRound.shape[0]):
+    g.write('{} {}\n'.format(wholePlaneVertMatRound[n,0],wholePlaneVertMatRound[n,1]))
+for n in range(edgeDofMat.shape[0]):
+    g.write('{} {}\n'.format(edgeDofMat[n,0],edgeDofMat[n,1]))
+for n in range(quadCentroidLoc.shape[0]):
+    g.write('{} {}\n'.format(quadCentroidLoc[n,0],quadCentroidLoc[n,1]))
+g.close()
+
+if(visMesh==True):
+    gVis(glvis,outputName+'WholePlaneQuad.mesh')

data/mixed-ele-meshes/comp_tri_quad_4_edges.mesh

@@ -0,0 +1,264 @@
+MFEM mesh v1.0
+
+dimension
+2
+
+elements
+128
+1 2 0 1 2
+1 2 1 2 4
+1 2 1 3 4
+1 2 2 4 5
+1 2 3 4 7
+1 2 3 6 7
+1 2 4 5 8
+1 2 4 7 8
+1 2 5 8 9
+1 2 6 7 11
+1 2 6 10 11
+1 2 7 8 12
+1 2 7 11 12
+1 2 8 9 13
+1 2 8 12 13
+1 2 9 13 14
+2 3 10 15 16 11
+2 3 11 16 17 12
+2 3 12 17 18 13
+2 3 13 18 19 14
+2 3 15 20 21 16
+2 3 16 21 22 17
+2 3 17 22 23 18
+2 3 18 23 24 19
+2 3 20 25 26 21
+2 3 21 26 27 22
+2 3 22 27 28 23
+2 3 23 28 29 24
+2 3 25 30 31 26
+2 3 26 31 32 27
+2 3 27 32 33 28
+2 3 28 33 34 29
+1 2 0 35 2
+1 2 35 2 37
+1 2 35 36 37
+1 2 2 37 5
+1 2 36 37 39
+1 2 36 38 39
+1 2 37 5 40
+1 2 37 39 40
+1 2 5 40 9
+1 2 38 39 42
+1 2 38 41 42
+1 2 39 40 43
+1 2 39 42 43
+1 2 40 9 44
+1 2 40 43 44
+1 2 9 44 14
+2 3 41 45 46 42
+2 3 42 46 47 43
+2 3 43 47 48 44
+2 3 44 48 19 14
+2 3 45 49 50 46
+2 3 46 50 51 47
+2 3 47 51 52 48
+2 3 48 52 24 19
+2 3 49 53 54 50
+2 3 50 54 55 51
+2 3 51 55 56 52
+2 3 52 56 29 24
+2 3 53 57 58 54
+2 3 54 58 59 55
+2 3 55 59 60 56
+2 3 56 60 34 29
+1 2 0 1 61
+1 2 1 61 62
+1 2 1 3 62
+1 2 61 62 63
+1 2 3 62 64
+1 2 3 6 64
+1 2 62 63 65
+1 2 62 64 65
+1 2 63 65 66
+1 2 6 64 67
+1 2 6 10 67
+1 2 64 65 68
+1 2 64 67 68
+1 2 65 66 69
+1 2 65 68 69
+1 2 66 69 70
+2 3 10 15 71 67
+2 3 67 71 72 68
+2 3 68 72 73 69
+2 3 69 73 74 70
+2 3 15 20 75 71
+2 3 71 75 76 72
+2 3 72 76 77 73
+2 3 73 77 78 74
+2 3 20 25 79 75
+2 3 75 79 80 76
+2 3 76 80 81 77
+2 3 77 81 82 78
+2 3 25 30 83 79
+2 3 79 83 84 80
+2 3 80 84 85 81
+2 3 81 85 86 82
+1 2 0 35 61
+1 2 35 61 87
+1 2 35 36 87
+1 2 61 87 63
+1 2 36 87 88
+1 2 36 38 88
+1 2 87 63 89
+1 2 87 88 89
+1 2 63 89 66
+1 2 38 88 90
+1 2 38 41 90
+1 2 88 89 91
+1 2 88 90 91
+1 2 89 66 92
+1 2 89 91 92
+1 2 66 92 70
+2 3 41 45 93 90
+2 3 90 93 94 91
+2 3 91 94 95 92
+2 3 92 95 74 70
+2 3 45 49 96 93
+2 3 93 96 97 94
+2 3 94 97 98 95
+2 3 95 98 78 74
+2 3 49 53 99 96
+2 3 96 99 100 97
+2 3 97 100 101 98
+2 3 98 101 82 78
+2 3 53 102 103 99
+2 3 99 103 104 100
+2 3 100 104 105 101
+2 3 101 105 86 82
+
+boundary
+16
+1 1 30 31
+1 1 31 32
+1 1 32 33
+1 1 33 34
+1 1 34 60
+1 1 60 59
+1 1 59 58
+1 1 58 57
+1 1 102 103
+1 1 103 104
+1 1 104 105
+1 1 105 86
+1 1 86 85
+1 1 85 84
+1 1 84 83
+1 1 83 30
+
+vertices
+106
+2
+0.0 0.0
+0.125 0.0
+7.65404249467e-18 0.125
+0.25 0.0
+0.176776695297 0.176776695297
+1.53080849893e-17 0.25
+0.375 0.0
+0.324759526419 0.1875
+0.1875 0.324759526419
+2.2962127484e-17 0.375
+0.5 0.0
+0.461939766256 0.191341716183
+0.353553390593 0.353553390593
+0.191341716183 0.461939766256
+3.06161699787e-17 0.5
+0.625 0.0
+0.596454824692 0.268506287137
+0.515165042945 0.515165042945
+0.268506287137 0.596454824692
+2.2962127484e-17 0.625
+0.75 0.0
+0.730969883128 0.345670858091
+0.676776695297 0.676776695297
+0.345670858091 0.730969883128
+1.53080849893e-17 0.75
+0.875 0.0
+0.865484941564 0.422835429046
+0.838388347648 0.838388347648
+0.422835429046 0.865484941564
+7.65404249467e-18 0.875
+1.0 0.0
+1.0 0.5
+1.0 1.0
+0.5 1.0
+0.0 1.0
+-0.125 0.0
+-0.25 0.0
+-0.176776695297 0.176776695297
+-0.375 0.0
+-0.324759526419 0.1875
+-0.1875 0.324759526419
+-0.5 0.0
+-0.461939766256 0.191341716183
+-0.353553390593 0.353553390593
+-0.191341716183 0.461939766256
+-0.625 0.0
+-0.596454824692 0.268506287137
+-0.515165042945 0.515165042945
+-0.268506287137 0.596454824692
+-0.75 0.0
+-0.730969883128 0.345670858091
+-0.676776695297 0.676776695297
+-0.345670858091 0.730969883128
+-0.875 0.0
+-0.865484941564 0.422835429046
+-0.838388347648 0.838388347648
+-0.422835429046 0.865484941564
+-1.0 0.0
+-1.0 0.5
+-1.0 1.0
+-0.5 1.0
+7.65404249467e-18 -0.125
+0.176776695297 -0.176776695297
+1.53080849893e-17 -0.25
+0.324759526419 -0.1875
+0.1875 -0.324759526419
+2.2962127484e-17 -0.375
+0.461939766256 -0.191341716183
+0.353553390593 -0.353553390593
+0.191341716183 -0.461939766256
+3.06161699787e-17 -0.5
+0.596454824692 -0.268506287137
+0.515165042945 -0.515165042945
+0.268506287137 -0.596454824692
+2.2962127484e-17 -0.625
+0.730969883128 -0.345670858091
+0.676776695297 -0.676776695297
+0.345670858091 -0.730969883128
+1.53080849893e-17 -0.75
+0.865484941564 -0.422835429046
+0.838388347648 -0.838388347648
+0.422835429046 -0.865484941564
+7.65404249467e-18 -0.875
+1.0 -0.5
+1.0 -1.0
+0.5 -1.0
+0.0 -1.0
+-0.176776695297 -0.176776695297
+-0.324759526419 -0.1875
+-0.1875 -0.324759526419
+-0.461939766256 -0.191341716183
+-0.353553390593 -0.353553390593
+-0.191341716183 -0.461939766256
+-0.596454824692 -0.268506287137
+-0.515165042945 -0.515165042945
+-0.268506287137 -0.596454824692
+-0.730969883128 -0.345670858091
+-0.676776695297 -0.676776695297
+-0.345670858091 -0.730969883128
+-0.865484941564 -0.422835429046
+-0.838388347648 -0.838388347648
+-0.422835429046 -0.865484941564
+-1.0 -0.0
+-1.0 -0.5
+-1.0 -1.0
+-0.5 -1.0