From c2eb5a6b4ff0b03e1737562472dcf950c6416e1e Mon Sep 17 00:00:00 2001 From: Konstantin Sidorov Date: Fri, 30 Jun 2017 13:27:05 +0300 Subject: [PATCH] Adding some king of code for HAM interface and memory structure for it --- src/mlpack/methods/ann/augmented/ham_unit.hpp | 70 ++++++++++++++ .../methods/ann/augmented/tree_memory.cpp | 50 ++++++++++ .../ann/augmented/tree_memory_impl.cpp | 93 +++++++++++++++++++ 3 files changed, 213 insertions(+) create mode 100644 src/mlpack/methods/ann/augmented/ham_unit.hpp create mode 100644 src/mlpack/methods/ann/augmented/tree_memory.cpp create mode 100644 src/mlpack/methods/ann/augmented/tree_memory_impl.cpp diff --git a/src/mlpack/methods/ann/augmented/ham_unit.hpp b/src/mlpack/methods/ann/augmented/ham_unit.hpp new file mode 100644 index 0000000000..67e6bd7bbf --- /dev/null +++ b/src/mlpack/methods/ann/augmented/ham_unit.hpp @@ -0,0 +1,70 @@ +/** + * @file ham_unit.hpp + * @author Konstantin Sidorov + * + * Definition of the HAMUnit class, which implements a Hierarchical Attentive + * Memory unit as described in https://arxiv.org/abs/1602.03218. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_AUGMENTED_HAM_UNIT_HPP +#define MLPACK_METHODS_AUGMENTED_HAM_UNIT_HPP + +#include "tree_memory.hpp" + +namespace mlpack { +namespace ann /* Artificial Neural Network */ { +namespace augmented /* Augmented neural network */ { + +template< + typename EmbedTransformationType, + typename JoinTransformationType, + typename SearchTransformationType, + typename WriteTransformationType +> +class HAMUnit { +public: + HAMUnit(int memorySize, Controller& controller, + EmbedTransformationType& embed, + JoinTransformationType& join, + SearchTransformationType& search, + WriteTransformationType& write); + + template< + template class OptimizerType = + mlpack::optimization::StandardSGD, + typename... OptimizerTypeArgs + > + void Train(const MatType& predictors, + const MatType& responses, + OptimizerType optimizer, + double gamma); + + void Evaluate(const arma::mat& predictors, + const arma::mat& responses); +private: + TreeMemory::iterator Attention() const; + + arma::mat Output(TreeMemory::iterator memoryCell) const; + + void Update(Memory::iterator memoryCell); + + void Forward(arma::mat&& input, arma::mat&& output); + + void Backward(arma::mat&& input, + arma::mat&& gy, + arma::mat&& g); + + void Gradient(arma::mat&& input, + arma::mat&& error, + arma::mat&& gradient); +}; +} // namespace augmented +} // namespace ann +} // namespace mlpack + +#include "ham_unit_impl.hpp" +#endif diff --git a/src/mlpack/methods/ann/augmented/tree_memory.cpp b/src/mlpack/methods/ann/augmented/tree_memory.cpp new file mode 100644 index 0000000000..965a648cd8 --- /dev/null +++ b/src/mlpack/methods/ann/augmented/tree_memory.cpp @@ -0,0 +1,50 @@ +/** + * @file tree_memory.hpp + * @author Konstantin Sidorov + * + * Definition of the TreeMemory class, which implements a memory structure + * for HAMUnit. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_AUGMENTED_TREE_MEMORY_HPP +#define MLPACK_METHODS_AUGMENTED_TREE_MEMORY_HPP + +#include + +using std::vector; + +namespace mlpack { +namespace ann /* Artificial Neural Network */ { +namespace augmented /* Augmented neural network */ { +template +class TreeMemory { +public: + TreeMemory(size_t size, J joiner, W writer); + + void Initialize(vector& leafValues); + + void Update(size_t pos, T el); + + T Get(size_t index); + + inline size_t Root(); + inline size_t Left(size_t origin); + inline size_t Right(size_t origin); + inline size_t Parent(size_t child); +private: + vector memory; + J joinFunction; + W writeFunction; + size_t memorySize; + size_t actualMemorySize; +}; +} // namespace augmented +} // namespace ann +} // namespace mlpack + +#include "tree_memory_impl.hpp" +#endif \ No newline at end of file diff --git a/src/mlpack/methods/ann/augmented/tree_memory_impl.cpp b/src/mlpack/methods/ann/augmented/tree_memory_impl.cpp new file mode 100644 index 0000000000..d665e80526 --- /dev/null +++ b/src/mlpack/methods/ann/augmented/tree_memory_impl.cpp @@ -0,0 +1,93 @@ +/** + * @file tree_memory_impl.hpp + * @author Konstantin Sidorov + * + * Implementation of CopyTask class + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_AUGMENTED_TREE_MEMORY_IMPL_HPP +#define MLPACK_METHODS_AUGMENTED_TREE_MEMORY_IMPL_HPP + +#include + +#include "tree_memory.hpp" + +TreeMemory::TreeMemory(size_t size, J joiner, W writer) { + memorySize = size; + // Rounding size to the next highest power of 2. + // WARNING: this is a hack that works only for 32-bit integers. + // In practice we don't really want to + // allocate 1e12 memory cells anyway, though. + assert(0 < size && size < static_cast(1 << 31)); + actualMemorySize = size - 1; + actualMemorySize |= actualMemorySize >> 1; + actualMemorySize |= actualMemorySize >> 2; + actualMemorySize |= actualMemorySize >> 4; + actualMemorySize |= actualMemorySize >> 8; + actualMemorySize |= actualMemorySize >> 16; + actualMemorySize |= actualMemorySize >> 32; + actualMemorySize++; + assert(memorySize <= actualMemorySize); + // Allocating enough memory to store all leaf values AND inner node values. + memory.resize(2 * actualMemorySize - 1); + joinFunction = joiner; + writeFunction = writer; +} + +inline T TreeMemory::Get(size_t index) { + assert(0 <= index && index < memorySize); + return memory[actualMemorySize - 1 + index]; +} + +inline size_t TreeMemory::Root() { + return 0; +} + +inline size_t TreeMemory::Left(size_t origin) { + return res = (origin << 1) + 1; +} + +inline size_t TreeMemory::Right(size_t origin) { + return res = (origin << 1) + 2; +} + +inline size_t TreeMemory::Parent(size_t child) { + if (child == 0) return actualMemorySize; + return ((child + 1) >> 1) - 1; +} + +void TreeMemory::Initialize(vector& leafValues) { + assert(leafValues.size() <= memorySize); + // First, write in the leaf nodes. + for (size_t i = 0; i < leafValues.size(); ++i) { + memory[actualMemorySize - 1 + i] = leafValues[i]; + } + size_t lastWrittenIdx = actualMemorySize - 1 + leafValues.size() - 1; + // After that, write into inner nodes as prescribed by writeFunction. + for (size_t i = actualMemorySize - 2; ; --i) { + size_t l = Left(i), r = Right(i); + assert(l <= r); + if (r > lastWrittenIdx) continue; + memory[i] = joinFunction(memory[l], memory[r]); + + if (i == Root()) break; + } +} + +void TreeMemory::Update(size_t pos, T el) { + assert(pos >= 0 && pos < memorySize); + size_t start = actualMemorySize - 1 + pos; + memory[start] = writeFunction(memory[start], el); + while (true) { + start = Parent(start); + size_t l = Left(start), r = Right(start); + memory[start] = joinFunction(memory[l], memory[r]); + if (start == Root()) break; + } +} + +#endif \ No newline at end of file