Adding some king of code for HAM interface and memory structure for it

src/mlpack/methods/ann/augmented/ham_unit.hpp

@@ -0,0 +1,70 @@
+/**
+ * @file ham_unit.hpp
+ * @author Konstantin Sidorov
+ *
+ * Definition of the HAMUnit class, which implements a Hierarchical Attentive
+ * Memory unit as described in https://arxiv.org/abs/1602.03218.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_AUGMENTED_HAM_UNIT_HPP
+#define MLPACK_METHODS_AUGMENTED_HAM_UNIT_HPP
+
+#include "tree_memory.hpp"
+
+namespace mlpack {
+namespace ann /* Artificial Neural Network */ {
+namespace augmented /* Augmented neural network */ {
+
+template<
+  typename EmbedTransformationType,
+  typename JoinTransformationType,
+  typename SearchTransformationType,
+  typename WriteTransformationType
+>
+class HAMUnit {
+public:
+  HAMUnit(int memorySize, Controller& controller,
+          EmbedTransformationType& embed,
+          JoinTransformationType& join,
+          SearchTransformationType& search,
+          WriteTransformationType& write);
+  
+  template<
+    template<typename, typename...> class OptimizerType =
+        mlpack::optimization::StandardSGD,
+    typename... OptimizerTypeArgs
+  >
+  void Train(const MatType& predictors,
+             const MatType& responses,
+             OptimizerType<Controller> optimizer,
+             double gamma);
+
+  void Evaluate(const arma::mat& predictors,
+                const arma::mat& responses);
+private:
+  TreeMemory::iterator Attention() const;
+  
+  arma::mat Output(TreeMemory::iterator memoryCell) const;
+  
+  void Update(Memory::iterator memoryCell);
+
+  void Forward(arma::mat&& input, arma::mat&& output);
+
+  void Backward(arma::mat&& input,
+                arma::mat&& gy,
+                arma::mat&& g);
+
+  void Gradient(arma::mat&& input,
+                arma::mat&& error,
+                arma::mat&& gradient);
+};
+} // namespace augmented
+} // namespace ann
+} // namespace mlpack
+
+#include "ham_unit_impl.hpp"
+#endif

src/mlpack/methods/ann/augmented/tree_memory.cpp

@@ -0,0 +1,50 @@
+/**
+ * @file tree_memory.hpp
+ * @author Konstantin Sidorov
+ *
+ * Definition of the TreeMemory class, which implements a memory structure
+ * for HAMUnit.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_AUGMENTED_TREE_MEMORY_HPP
+#define MLPACK_METHODS_AUGMENTED_TREE_MEMORY_HPP
+
+#include <vector>
+
+using std::vector;
+
+namespace mlpack {
+namespace ann /* Artificial Neural Network */ {
+namespace augmented /* Augmented neural network */ {
+template<typename T, typename J, typename W>
+class TreeMemory {
+public:
+  TreeMemory(size_t size, J joiner, W writer);
+
+  void Initialize(vector<T>& leafValues);
+
+  void Update(size_t pos, T el);
+
+  T Get(size_t index);
+
+  inline size_t Root();
+  inline size_t Left(size_t origin);
+  inline size_t Right(size_t origin);
+  inline size_t Parent(size_t child);
+private:
+  vector<T> memory;
+  J joinFunction;
+  W writeFunction;
+  size_t memorySize;
+  size_t actualMemorySize;
+};
+} // namespace augmented
+} // namespace ann
+} // namespace mlpack
+
+#include "tree_memory_impl.hpp"
+#endif

src/mlpack/methods/ann/augmented/tree_memory_impl.cpp

@@ -0,0 +1,93 @@
+/**
+ * @file tree_memory_impl.hpp
+ * @author Konstantin Sidorov
+ *
+ * Implementation of CopyTask class
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_AUGMENTED_TREE_MEMORY_IMPL_HPP
+#define MLPACK_METHODS_AUGMENTED_TREE_MEMORY_IMPL_HPP
+
+#include <cassert>
+
+#include "tree_memory.hpp"
+
+TreeMemory<T>::TreeMemory(size_t size, J joiner, W writer) {
+  memorySize = size;
+  // Rounding size to the next highest power of 2.
+  // WARNING: this is a hack that works only for 32-bit integers.
+  // In practice we don't really want to 
+  // allocate 1e12 memory cells anyway, though.
+  assert(0 < size && size < static_cast<size_t>(1 << 31));
+  actualMemorySize = size - 1;
+  actualMemorySize |= actualMemorySize >> 1;
+  actualMemorySize |= actualMemorySize >> 2;
+  actualMemorySize |= actualMemorySize >> 4;
+  actualMemorySize |= actualMemorySize >> 8;
+  actualMemorySize |= actualMemorySize >> 16;
+  actualMemorySize |= actualMemorySize >> 32;
+  actualMemorySize++;
+  assert(memorySize <= actualMemorySize);
+  // Allocating enough memory to store all leaf values AND inner node values.
+  memory.resize(2 * actualMemorySize - 1);
+  joinFunction = joiner;
+  writeFunction = writer;
+}
+
+inline T TreeMemory<T>::Get(size_t index) {
+  assert(0 <= index && index < memorySize);
+  return memory[actualMemorySize - 1 + index];
+}
+
+inline size_t TreeMemory<T>::Root() {
+  return 0;
+}
+
+inline size_t TreeMemory<T>::Left(size_t origin) {
+  return res = (origin << 1) + 1;
+}
+
+inline size_t TreeMemory<T>::Right(size_t origin) {
+  return res = (origin << 1) + 2;
+}
+
+inline size_t TreeMemory<T>::Parent(size_t child) {
+  if (child == 0) return actualMemorySize;
+  return ((child + 1) >> 1) - 1;
+}
+
+void TreeMemory<T>::Initialize(vector<T>& leafValues) {
+  assert(leafValues.size() <= memorySize);
+  // First, write in the leaf nodes.
+  for (size_t i = 0; i < leafValues.size(); ++i) {
+    memory[actualMemorySize - 1 + i] = leafValues[i];
+  }
+  size_t lastWrittenIdx = actualMemorySize - 1 + leafValues.size() - 1;
+  // After that, write into inner nodes as prescribed by writeFunction.
+  for (size_t i = actualMemorySize - 2; ; --i) {
+    size_t l = Left(i), r = Right(i);
+    assert(l <= r);
+    if (r > lastWrittenIdx) continue;
+    memory[i] = joinFunction(memory[l], memory[r]);
+
+    if (i == Root()) break;
+  }
+}
+
+void TreeMemory<T>::Update(size_t pos, T el) {
+  assert(pos >= 0 && pos < memorySize);
+  size_t start = actualMemorySize - 1 + pos;
+  memory[start] = writeFunction(memory[start], el);
+  while (true) {
+    start = Parent(start);
+    size_t l = Left(start), r = Right(start);
+    memory[start] = joinFunction(memory[l], memory[r]);
+    if (start == Root()) break;
+  }
+}
+
+#endif