From be70f102a28c31924a228a489c880411362b6469 Mon Sep 17 00:00:00 2001 From: robotcator Date: Wed, 26 Dec 2018 09:58:48 +0800 Subject: [PATCH 001/143] add the skeleton of prioritized replay buffer --- src/mlpack/methods/reinforcement_learning/replay/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mlpack/methods/reinforcement_learning/replay/CMakeLists.txt b/src/mlpack/methods/reinforcement_learning/replay/CMakeLists.txt index 03ff3a5720..381eda1b5d 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/CMakeLists.txt +++ b/src/mlpack/methods/reinforcement_learning/replay/CMakeLists.txt @@ -2,6 +2,8 @@ # Anything not in this list will not be compiled into mlpack. set(SOURCES random_replay.hpp + sumtree.hpp + prioritized_replay.hpp ) # Add directory name to sources. From 082c24b1d44e138f323b26c0d8661c610239b267 Mon Sep 17 00:00:00 2001 From: robotcator Date: Wed, 26 Dec 2018 10:00:16 +0800 Subject: [PATCH 002/143] add the skeleton of prioritized replay buffer --- .../replay/prioritized_replay.hpp | 140 ++++++++++++++++++ .../reinforcement_learning/replay/sumtree.hpp | 59 ++++++++ 2 files changed, 199 insertions(+) create mode 100644 src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp create mode 100644 src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp new file mode 100644 index 0000000000..d481004f12 --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -0,0 +1,140 @@ +/** + * @file prioritized_experience_replay.hpp + * @author Xiaohong + * + * This file is an implementation of prioritized experience repla y. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_RL_PRIORITIZED_REPLAY_HPP +#define MLPACK_METHODS_RL_PRIORITIZED_REPLAY_HPP + +#include +#include "random_replay.hpp" + +namespace mlpack { +namespace rl { + +/** + * Implementation of prioritized experience replay. + * + * + * @code + * @article{schaul2015prioritized, + * title={Prioritized experience replay}, + * author={Schaul, Tom and Quan, John and Antonoglou, Ioannis and Silver, David}, + * journal={arXiv preprint arXiv:1511.05952}, + * year={2015} + * } + * @endcode + * + * @tparam EnvironmentType Desired task. + */ +template +class PrioritizedReplay: public RandomReplay +{ + public: + /** + * Construct an instance of prioritized experience replay class. + * + * @param batchSize Number of examples returned at each sample. + * @param capacity Total memory size in terms of number of examples. + * @param alpha + * @param dimension The dimension of an encoded state. + */ + PrioritizedReplay(const size_t batchSize, + const size_t capacity, + const double alpha, + const size_t dimension = StateType::dimension) : + batchSize(batchSize), + capacity(capacity), + alpha(alpha), + position(0), + states(dimension, capacity), + actions(capacity), + rewards(capacity), + nextStates(dimension, capacity), + isTerminal(capacity), + full(false), + max_priority(1.0) + { + int size = 1; + while (size < capacity) { + size *= 2; + } + idxSum = new SumTree(size); + } + + void Store(const StateType& state, + ActionType action, + double reward, + const StateType& nextState, + bool isEnd) + { + states.col(position) = state.Encode(); + actions(position) = action; + rewards(position) = reward; + nextStates.col(position) = nextState.Encode(); + isTerminal(position) = isEnd; + + idxSum[position] = max_priority * alpha; + + position++; + if (position == capacity) + { + full = true; + position = 0; + } + } + + arma::uvec sampleProportional() + { + arma::uvec idxes(batchSize); + double totalSum = idxSum.sum(0, (full ? capacity : position) - 1); + double sumPerRange = totalSum / batchSize; + for (size_t bt = 0; bt < batchSize; bt ++) { + double mass = arma::randu() * sumPerRange + bt * sumPerRange; + int idx = idxSum.findPrefixSum(mass); + idxes(bt) = idx; + } + return idxes; + } + + void Sample(arma::mat& sampledStates, + arma::icolvec& sampledActions, + arma::colvec& sampledRewards, + arma::mat& sampledNextStates, + arma::icolvec& isTerminal) + { + size_t upperBound = full ? capacity : position; + + arma::uvec sampledIndices = sampleProportional(); + + sampledStates = states.cols(sampledIndices); + sampledActions = actions.elem(sampledIndices); + sampledRewards = rewards.elem(sampledIndices); + sampledNextStates = nextStates.cols(sampledIndices); + isTerminal = this->isTerminal.elem(sampledIndices); + +// btodo: caculate the weights of sampled transitions + + } + + void update_priorities(double beta) + { +// btodo: update priorities of sampled transitions. + } + +private: + double alpha; + double max_priority; + SumTree idxSum; +}; + +} // namespace rl +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp new file mode 100644 index 0000000000..277f7c033d --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp @@ -0,0 +1,59 @@ +/** + * @file sumtree.hpp + * @author Xiaohong + * + * This file is an implementation of sum tree. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_RL_SUMTREE_HPP +#define MLPACK_METHODS_RL_SUMTREE_HPP + +#include + +namespace mlpack { +namespace rl { + template + class SumTree { + SumTree(size_t capacity): + capacity(capacity) + { + auxiliary(capacity, 0); + element(capacity, 0); + } + + void set(size_t idx, T value) + { +// btodo: update the leaf node value + } + + T get(size_t) + { +// btodo: get the leaf node value + return 0; + } + + T sum(size_t _start, size_t _end) + { +// btodo: caculate the sum of contiguous subsequence of the array. + return 0; + } + + size_t findPrefixSum(T mass) + { +// btodo: Find the highest index `idx` in the array such that +// sum(arr[0] + arr[1] + ... + arr[i - i]) <= prefixsum + } + + private: + capacity; + std::vector auxiliary; + std::vector element; + }; +} // namespace rl +} // namespace mlpack + +#endif From 81d8198321f25ac5252f2f466016a3973c0bce16 Mon Sep 17 00:00:00 2001 From: robotcator Date: Mon, 7 Jan 2019 11:07:29 +0800 Subject: [PATCH 003/143] implement the sum tree part --- .../reinforcement_learning/replay/sumtree.hpp | 66 ++++++++++++++++--- 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp index 277f7c033d..2c68d42f1d 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp @@ -2,7 +2,10 @@ * @file sumtree.hpp * @author Xiaohong * - * This file is an implementation of sum tree. + * This file is an implementation of sumtree. + * + * reference: + * [1] https://github.com/openai/baselines/blob/master/baselines/common/segment_tree.py * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -21,36 +24,81 @@ namespace rl { SumTree(size_t capacity): capacity(capacity) { - auxiliary(capacity, 0); - element(capacity, 0); + element(2 * capacity); } void set(size_t idx, T value) { -// btodo: update the leaf node value + idx += capacity; + element[idx] = value; + idx /= 2; + while (idx >= 1) + { + element[idx] = element[2 * idx] + element[2 * idx + 1]; + idx /= 2; + } } - T get(size_t) + T get(size_t idx) { -// btodo: get the leaf node value - return 0; + idx += capacity; + return element[idx]; + } + + T sumHelper(size_t _start, size_t _end, size_t node, size_t node_start, size_t node_end) + { + if (_start == node_start && _end == node_end) + { + return element[node]; + } + size_t mid = (node_start + node_end) / 2; + if (_end <= mid) + { + return sumHelper(_start, _end, 2 * node, node_start, mid); + } + else + { + if (mid + 1 <= _start) + { + return sumHelper(_start, _end, 2 * node + 1, mid + 1 , node_end); + } + else + { + return sumHelper(_start, mid, 2 * node, node_start, mid) + + sumHelper(mid+1, _end, 2 * node + 1, mid + 1 , node_end); + } + } } T sum(size_t _start, size_t _end) { // btodo: caculate the sum of contiguous subsequence of the array. - return 0; + _end -= 1; + return sumHelper(_start, _end, 1, 0, capacity-1); } size_t findPrefixSum(T mass) { // btodo: Find the highest index `idx` in the array such that // sum(arr[0] + arr[1] + ... + arr[i - i]) <= prefixsum + int idx = 1; + while (idx < capacity) + { + if (element[2 * idx] > mass) + { + idx = 2 * idx; + } + else + { + mass -= element[2 * idx]; + idx = 2 * idx + 1; + } + } + return idx - capacity; } private: capacity; - std::vector auxiliary; std::vector element; }; } // namespace rl From 19965ebc855298c9c0f674a8a8c6f9a963ba5f69 Mon Sep 17 00:00:00 2001 From: robotcator Date: Mon, 7 Jan 2019 11:07:56 +0800 Subject: [PATCH 004/143] remove the base class for prioritized_replay --- .../replay/prioritized_replay.hpp | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index d481004f12..36d3a61cbf 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -34,7 +34,7 @@ namespace rl { * @tparam EnvironmentType Desired task. */ template -class PrioritizedReplay: public RandomReplay +class PrioritizedReplay { public: /** @@ -119,7 +119,7 @@ class PrioritizedReplay: public RandomReplay sampledNextStates = nextStates.cols(sampledIndices); isTerminal = this->isTerminal.elem(sampledIndices); -// btodo: caculate the weights of sampled transitions +// btodo: calculate the weights of sampled transitions } @@ -129,9 +129,41 @@ class PrioritizedReplay: public RandomReplay } private: + //! How much prioritization is used. double alpha; + double max_priority; + + //! Locally-stored the prefix sum of prioritization SumTree idxSum; + + //! Locally-stored number of examples of each sample. + size_t batchSize; + + //! Locally-stored total memory limit. + size_t capacity; + + //! Indicate the position to store new transition. + size_t position; + + //! Locally-stored encoded previous states. + arma::mat states; + + //! Locally-stored previous actions. + arma::icolvec actions; + + //! Locally-stored previous rewards. + arma::colvec rewards; + + //! Locally-stored encoded previous next states. + arma::mat nextStates; + + //! Locally-stored termination information of previous experience. + arma::icolvec isTerminal; + + //! Locally-stored indicator that whether the memory is full or not + bool full; + }; } // namespace rl From d6e42f040e34a5c75544408c6a11761c1c818cf2 Mon Sep 17 00:00:00 2001 From: robotcator Date: Sat, 12 Jan 2019 11:39:00 +0800 Subject: [PATCH 005/143] add helper function for `sum`, remove to do tag --- .../methods/reinforcement_learning/replay/sumtree.hpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp index 2c68d42f1d..4fa192d4ad 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp @@ -72,14 +72,19 @@ namespace rl { T sum(size_t _start, size_t _end) { -// btodo: caculate the sum of contiguous subsequence of the array. +// caculate the sum of contiguous subsequence of the array. _end -= 1; return sumHelper(_start, _end, 1, 0, capacity-1); } + T sum() + { + return sum(0, capacity); + } + size_t findPrefixSum(T mass) { -// btodo: Find the highest index `idx` in the array such that +// Find the highest index `idx` in the array such that // sum(arr[0] + arr[1] + ... + arr[i - i]) <= prefixsum int idx = 1; while (idx < capacity) From 924ffcf8d27b432ee5252141d40f5b208be8242a Mon Sep 17 00:00:00 2001 From: robotcator Date: Sat, 12 Jan 2019 11:44:08 +0800 Subject: [PATCH 006/143] remove todo tag --- .../replay/prioritized_replay.hpp | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index 36d3a61cbf..1a2313beea 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -107,11 +107,14 @@ class PrioritizedReplay arma::icolvec& sampledActions, arma::colvec& sampledRewards, arma::mat& sampledNextStates, - arma::icolvec& isTerminal) + arma::icolvec& isTerminal, + arma::uvec& sampledIndices, + arma::rowvec& weights, + double beta) { size_t upperBound = full ? capacity : position; - arma::uvec sampledIndices = sampleProportional(); + sampledIndices = sampleProportional(); sampledStates = states.cols(sampledIndices); sampledActions = actions.elem(sampledIndices); @@ -119,13 +122,26 @@ class PrioritizedReplay sampledNextStates = nextStates.cols(sampledIndices); isTerminal = this->isTerminal.elem(sampledIndices); -// btodo: calculate the weights of sampled transitions +// calculate the weights of sampled transitions + size_t num_sample = full ? capacity : position; + + for (size_t i = 0; i < sampledIndices.n_rows; ++ i) + { + double p_sample = idxSum[sampledIndices[i]] / idxSum.sum(); + weights(i) = pow(num_sample * p_sample, -beta); + } + weights /= weights.max(); } - void update_priorities(double beta) + void update_priorities(arma::uvec& indices, arma::uvec& priorities) { -// btodo: update priorities of sampled transitions. +// update priorities of sampled transitions. + for (sizt_t i = 0; i < indices.n_rows; ++i) + { + idxSum[indices[i]] = alpha * priorities[i]; + max_priority = max(max_priority, priorities[i]); + } } private: From a3c6d590587a424f8fff5cf50519a9c8af646f81 Mon Sep 17 00:00:00 2001 From: robotcator Date: Mon, 14 Jan 2019 15:37:28 +0800 Subject: [PATCH 007/143] update the sumtree implementation --- .../reinforcement_learning/replay/sumtree.hpp | 138 +++++++++--------- 1 file changed, 73 insertions(+), 65 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp index 4fa192d4ad..42bb7345e6 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp @@ -19,93 +19,101 @@ namespace mlpack { namespace rl { - template - class SumTree { - SumTree(size_t capacity): - capacity(capacity) - { - element(2 * capacity); - } - void set(size_t idx, T value) +/** + * Implementation of SumTree. + */ + +template +class SumTree +{ + public: + SumTree(size_t capacity): + capacity(capacity) + { + element = std::vector (2 * capacity); + } + + void set(size_t idx, T value) + { + idx += capacity; + element[idx] = value; + idx /= 2; + while (idx >= 1) { - idx += capacity; - element[idx] = value; + element[idx] = element[2 * idx] + element[2 * idx + 1]; idx /= 2; - while (idx >= 1) - { - element[idx] = element[2 * idx] + element[2 * idx + 1]; - idx /= 2; - } } + } - T get(size_t idx) + T get(size_t idx) + { + idx += capacity; + return element[idx]; + } + + T sumHelper(size_t _start, size_t _end, size_t node, size_t node_start, size_t node_end) + { + if (_start == node_start && _end == node_end) { - idx += capacity; - return element[idx]; + return element[node]; } - - T sumHelper(size_t _start, size_t _end, size_t node, size_t node_start, size_t node_end) + size_t mid = (node_start + node_end) / 2; + if (_end <= mid) { - if (_start == node_start && _end == node_end) + return sumHelper(_start, _end, 2 * node, node_start, mid); + } + else + { + if (mid + 1 <= _start) { - return element[node]; - } - size_t mid = (node_start + node_end) / 2; - if (_end <= mid) - { - return sumHelper(_start, _end, 2 * node, node_start, mid); + return sumHelper(_start, _end, 2 * node + 1, mid + 1 , node_end); } else { - if (mid + 1 <= _start) - { - return sumHelper(_start, _end, 2 * node + 1, mid + 1 , node_end); - } - else - { - return sumHelper(_start, mid, 2 * node, node_start, mid) + - sumHelper(mid+1, _end, 2 * node + 1, mid + 1 , node_end); - } + return sumHelper(_start, mid, 2 * node, node_start, mid) + + sumHelper(mid+1, _end, 2 * node + 1, mid + 1 , node_end); } } + } - T sum(size_t _start, size_t _end) - { -// caculate the sum of contiguous subsequence of the array. - _end -= 1; - return sumHelper(_start, _end, 1, 0, capacity-1); - } + T sum(size_t _start, size_t _end) + { +// caculate the sum of contiguous subsequence of the array. + _end -= 1; + return sumHelper(_start, _end, 1, 0, capacity-1); + } - T sum() - { - return sum(0, capacity); - } + T sum() + { + return sum(0, capacity); + } - size_t findPrefixSum(T mass) - { + size_t findPrefixSum(T mass) + { // Find the highest index `idx` in the array such that -// sum(arr[0] + arr[1] + ... + arr[i - i]) <= prefixsum - int idx = 1; - while (idx < capacity) +// sum(arr[0] + arr[1] + ... + arr[i]) <= mass + int idx = 1; + while (idx < capacity) + { + if (element[2 * idx] > mass) { - if (element[2 * idx] > mass) - { - idx = 2 * idx; - } - else - { - mass -= element[2 * idx]; - idx = 2 * idx + 1; - } + idx = 2 * idx; + } + else + { + mass -= element[2 * idx]; + idx = 2 * idx + 1; } - return idx - capacity; } + return idx - capacity; + } + + private: + size_t capacity; + std::vector element; +}; - private: - capacity; - std::vector element; - }; } // namespace rl } // namespace mlpack From a2d198adb1335d94c285806c2e15fecc89ccf36a Mon Sep 17 00:00:00 2001 From: robotcator Date: Mon, 14 Jan 2019 15:37:46 +0800 Subject: [PATCH 008/143] add unit test for sumtree implementation --- src/mlpack/tests/sumtree_test.cpp | 80 +++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 src/mlpack/tests/sumtree_test.cpp diff --git a/src/mlpack/tests/sumtree_test.cpp b/src/mlpack/tests/sumtree_test.cpp new file mode 100644 index 0000000000..992de78464 --- /dev/null +++ b/src/mlpack/tests/sumtree_test.cpp @@ -0,0 +1,80 @@ +/** + * @file sumtree_test.hpp + * @author Xiaohong + * + * Test for Sumtree implementation + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + + +#include + +#include + +#include +#include "test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::rl; + +BOOST_AUTO_TEST_SUITE(SumTreeTest); + +/** + * Test that we set the element. + */ +BOOST_AUTO_TEST_CASE(SetElement) +{ + SumTree sumtree(4); + sumtree.set(0, 1.0); + sumtree.set(1, 0.8); + sumtree.set(2, 0.6); + sumtree.set(3, 0.4); + + BOOST_CHECK_CLOSE(sumtree.sum(), 2.8, 1e-8); + BOOST_CHECK_CLOSE(sumtree.sum(0, 1), 1.0, 1e-8); + BOOST_CHECK_CLOSE(sumtree.sum(0, 3), 2.4, 1e-8); + BOOST_CHECK_CLOSE(sumtree.sum(1, 4), 1.8, 1e-8); +} + +/** + * Test that we get the element. + */ + +BOOST_AUTO_TEST_CASE(GetElement) +{ + SumTree sumtree(4); + sumtree.set(0, 1.0); + sumtree.set(1, 0.8); + sumtree.set(2, 0.6); + sumtree.set(3, 0.4); + + BOOST_CHECK_CLOSE(sumtree.get(0), 1.0, 1e-8); + BOOST_CHECK_CLOSE(sumtree.get(1), 0.8, 1e-8); + BOOST_CHECK_CLOSE(sumtree.get(2), 0.6, 1e-8); + BOOST_CHECK_CLOSE(sumtree.get(3), 0.4, 1e-8); +} + +/** + * Test that we find the highest index in the array such that + * sum(arr[0] + arr[1] + arr[2] ... + arr[i]) <= mass . + */ + +BOOST_AUTO_TEST_CASE(FindPrefixSum) +{ + SumTree sumtree(4); + sumtree.set(0, 1.0); + sumtree.set(1, 0.8); + sumtree.set(2, 0.6); + sumtree.set(3, 0.4); + + BOOST_CHECK_EQUAL(sumtree.findPrefixSum(0), 0); + BOOST_CHECK_EQUAL(sumtree.findPrefixSum(1), 1); + BOOST_CHECK_EQUAL(sumtree.findPrefixSum(2.8), 3); + BOOST_CHECK_EQUAL(sumtree.findPrefixSum(3.0), 3); +} + +BOOST_AUTO_TEST_SUITE_END(); From 11a0f982d6b2459787f13d11995a942405a54648 Mon Sep 17 00:00:00 2001 From: robotcator Date: Mon, 14 Jan 2019 15:37:54 +0800 Subject: [PATCH 009/143] add unit test for sumtree implementation --- src/mlpack/tests/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 4bc7cbc524..67427aff4f 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -97,6 +97,7 @@ add_executable(mlpack_test sparse_coding_test.cpp spill_tree_test.cpp split_data_test.cpp + sumtree_test.cpp svd_batch_test.cpp svd_incremental_test.cpp svdplusplus_test.cpp From 779531c4d12e6c91b435e35f94a33bd65f99ff03 Mon Sep 17 00:00:00 2001 From: robotcator Date: Wed, 16 Jan 2019 11:30:16 +0800 Subject: [PATCH 010/143] add default constructor --- .../methods/reinforcement_learning/replay/random_replay.hpp | 3 +++ src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp index 163c3de257..4afe6afa84 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp @@ -49,6 +49,9 @@ class RandomReplay //! Convenient typedef for state. using StateType = typename EnvironmentType::State; + RandomReplay() + { /* Nothing to do here. */ } + /** * Construct an instance of random experience replay class. * diff --git a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp index 42bb7345e6..69154a53e5 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp @@ -28,6 +28,10 @@ template class SumTree { public: + + SumTree() + { /* Nothing to do here. */ } + SumTree(size_t capacity): capacity(capacity) { From 3f9435e226f11c3df43307de96d5165f2152c34b Mon Sep 17 00:00:00 2001 From: robotcator Date: Wed, 16 Jan 2019 11:30:55 +0800 Subject: [PATCH 011/143] refactor prioritized replay --- .../replay/prioritized_replay.hpp | 59 +++++++++++++------ 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index 1a2313beea..132a14a84e 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -13,7 +13,7 @@ #define MLPACK_METHODS_RL_PRIORITIZED_REPLAY_HPP #include -#include "random_replay.hpp" +#include "sumtree.hpp" namespace mlpack { namespace rl { @@ -37,6 +37,15 @@ template class PrioritizedReplay { public: + //! Convenient typedef for action. + using ActionType = typename EnvironmentType::Action; + + //! Convenient typedef for state. + using StateType = typename EnvironmentType::State; + + PrioritizedReplay() + { /* Nothing to do here. */ } + /** * Construct an instance of prioritized experience replay class. * @@ -59,13 +68,17 @@ class PrioritizedReplay nextStates(dimension, capacity), isTerminal(capacity), full(false), - max_priority(1.0) + max_priority(1.0), + initial_beta(0.6), + replay_beta_iters(10000) { int size = 1; while (size < capacity) { size *= 2; } - idxSum = new SumTree(size); + + beta = initial_beta; + idxSum = SumTree(size); } void Store(const StateType& state, @@ -80,7 +93,7 @@ class PrioritizedReplay nextStates.col(position) = nextState.Encode(); isTerminal(position) = isEnd; - idxSum[position] = max_priority * alpha; + idxSum.set(position, max_priority * alpha); position++; if (position == capacity) @@ -90,15 +103,15 @@ class PrioritizedReplay } } - arma::uvec sampleProportional() + arma::ucolvec sampleProportional() { - arma::uvec idxes(batchSize); + arma::ucolvec idxes(batchSize); double totalSum = idxSum.sum(0, (full ? capacity : position) - 1); double sumPerRange = totalSum / batchSize; for (size_t bt = 0; bt < batchSize; bt ++) { double mass = arma::randu() * sumPerRange + bt * sumPerRange; - int idx = idxSum.findPrefixSum(mass); - idxes(bt) = idx; + size_t idx = idxSum.findPrefixSum(mass); + idxes[bt] = idx; } return idxes; } @@ -108,9 +121,8 @@ class PrioritizedReplay arma::colvec& sampledRewards, arma::mat& sampledNextStates, arma::icolvec& isTerminal, - arma::uvec& sampledIndices, - arma::rowvec& weights, - double beta) + arma::ucolvec& sampledIndices, + arma::rowvec& weights) { size_t upperBound = full ? capacity : position; @@ -128,30 +140,41 @@ class PrioritizedReplay for (size_t i = 0; i < sampledIndices.n_rows; ++ i) { - double p_sample = idxSum[sampledIndices[i]] / idxSum.sum(); - weights(i) = pow(num_sample * p_sample, -beta); + double p_sample = idxSum.get(sampledIndices[i]) / idxSum.sum(); + weights[i] = pow(num_sample * p_sample, -beta); } weights /= weights.max(); } - void update_priorities(arma::uvec& indices, arma::uvec& priorities) + void update_priorities(arma::ucolvec& indices, arma::colvec& priorities) { // update priorities of sampled transitions. - for (sizt_t i = 0; i < indices.n_rows; ++i) + for (size_t i = 0; i < indices.n_rows; ++i) { - idxSum[indices[i]] = alpha * priorities[i]; - max_priority = max(max_priority, priorities[i]); + idxSum.set(indices[i], alpha * priorities[i]); + max_priority = std::max(max_priority, priorities[i]); } } + void betaAnneal() + { + beta = beta + (1 - initial_beta) * 1.0 / replay_beta_iters; + } + private: //! How much prioritization is used. double alpha; double max_priority; + double initial_beta; + + double beta; + + size_t replay_beta_iters; + //! Locally-stored the prefix sum of prioritization - SumTree idxSum; + SumTree idxSum; //! Locally-stored number of examples of each sample. size_t batchSize; From 75d63c07770f93e9a2204af77744b16836de3cfa Mon Sep 17 00:00:00 2001 From: robotcator Date: Wed, 16 Jan 2019 11:31:09 +0800 Subject: [PATCH 012/143] support prioritized replay --- .../reinforcement_learning/q_learning.hpp | 16 +++- .../q_learning_impl.hpp | 91 ++++++++++++++++--- 2 files changed, 95 insertions(+), 12 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning.hpp b/src/mlpack/methods/reinforcement_learning/q_learning.hpp index 9332c67e39..46914dee71 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning.hpp @@ -16,6 +16,7 @@ #include #include "replay/random_replay.hpp" +#include "replay/prioritized_replay.hpp" #include "training_config.hpp" namespace mlpack { @@ -52,7 +53,8 @@ template < typename NetworkType, typename UpdaterType, typename PolicyType, - typename ReplayType = RandomReplay + typename ReplayType = RandomReplay, + typename PrioritizedReplayType = PrioritizedReplay > class QLearning { @@ -83,6 +85,13 @@ class QLearning UpdaterType updater = UpdaterType(), EnvironmentType environment = EnvironmentType()); + QLearning(TrainingConfig config, + NetworkType network, + PolicyType policy, + PrioritizedReplayType prioritizedReplayMethod, + UpdaterType updater = UpdaterType(), + EnvironmentType environment = EnvironmentType()); + /** * Execute a step in an episode. * @return Reward for the step. @@ -131,6 +140,9 @@ class QLearning //! Locally-stored experience method. ReplayType replayMethod; + //! Locally-stored experience method. + PrioritizedReplayType prioritizedReplayMethod; + //! Locally-stored reinforcement learning task. EnvironmentType environment; @@ -142,6 +154,8 @@ class QLearning //! Locally-stored flag indicating training mode or test mode. bool deterministic; + + bool prioritized_replay; }; } // namespace rl diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 8c84ae261e..7441dcde7a 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -22,14 +22,16 @@ template < typename NetworkType, typename UpdaterType, typename PolicyType, - typename ReplayType + typename ReplayType, + typename PrioritizedReplayType > QLearning< EnvironmentType, NetworkType, UpdaterType, PolicyType, - ReplayType + ReplayType, + PrioritizedReplayType >::QLearning(TrainingConfig config, NetworkType network, PolicyType policy, @@ -43,7 +45,8 @@ QLearning< replayMethod(std::move(replayMethod)), environment(std::move(environment)), totalSteps(0), - deterministic(false) + deterministic(false), + prioritized_replay(false) { if (learningNetwork.Parameters().is_empty()) learningNetwork.ResetParameters(); @@ -57,14 +60,54 @@ template < typename NetworkType, typename UpdaterType, typename PolicyType, - typename ReplayType + typename ReplayType, + typename PrioritizedReplayType +> +QLearning< + EnvironmentType, + NetworkType, + UpdaterType, + PolicyType, + ReplayType, + PrioritizedReplayType +>::QLearning(TrainingConfig config, + NetworkType network, + PolicyType policy, + PrioritizedReplayType prioritizedReplayMethod, + UpdaterType updater, + EnvironmentType environment): + config(std::move(config)), + learningNetwork(std::move(network)), + updater(std::move(updater)), + policy(std::move(policy)), + prioritizedReplayMethod(std::move(prioritizedReplayMethod)), + environment(std::move(environment)), + totalSteps(0), + deterministic(false), + prioritized_replay(true) +{ + if (learningNetwork.Parameters().is_empty()) + learningNetwork.ResetParameters(); + this->updater.Initialize(learningNetwork.Parameters().n_rows, + learningNetwork.Parameters().n_cols); + targetNetwork = learningNetwork; +} + +template < + typename EnvironmentType, + typename NetworkType, + typename UpdaterType, + typename PolicyType, + typename ReplayType, + typename PrioritizedReplayType > arma::Col QLearning< EnvironmentType, NetworkType, UpdaterType, PolicyType, - ReplayType + ReplayType, + PrioritizedReplayType >::BestAction(const arma::mat& actionValues) { arma::Col bestActions(actionValues.n_cols); @@ -82,14 +125,16 @@ template < typename NetworkType, typename UpdaterType, typename BehaviorPolicyType, - typename ReplayType + typename ReplayType, + typename PrioritizedReplayType > double QLearning< EnvironmentType, NetworkType, UpdaterType, BehaviorPolicyType, - ReplayType + ReplayType, + PrioritizedReplayType >::Step() { // Get the action value for each action at current state. @@ -121,8 +166,20 @@ double QLearning< arma::colvec sampledRewards; arma::mat sampledNextStates; arma::icolvec isTerminal; - replayMethod.Sample(sampledStates, sampledActions, sampledRewards, - sampledNextStates, isTerminal); + arma::ucolvec sampledIndices; + arma::rowvec weights; + + if (!prioritized_replay) + { + replayMethod.Sample(sampledStates, sampledActions, sampledRewards, + sampledNextStates, isTerminal); + } + else + { + prioritizedReplayMethod.Sample(sampledStates, sampledActions, sampledRewards, + sampledNextStates, isTerminal, sampledIndices, + weights); + } // Compute action value for next state with target network. arma::mat nextActionValues; @@ -163,6 +220,16 @@ double QLearning< learningNetwork.Backward(target, gradients); updater.Update(learningNetwork.Parameters(), config.StepSize(), gradients); + // if (prioritized_replay) + // { + // arma::colvec td_error(target.n_cols); + // for (size_t i = 0; i < target.n_cols; i ++) + // { + // td_error[i] = nextActionValues(sampledActions[i], i) - target(sampledActions[i], i); + // } + // td_error = arma::abs(td_error); + // prioritizedReplayMethod.update_priorities(sampledIndices, td_error); + // } return reward; } @@ -171,14 +238,16 @@ template < typename NetworkType, typename UpdaterType, typename BehaviorPolicyType, - typename ReplayType + typename ReplayType, + typename PrioritizedReplayType > double QLearning< EnvironmentType, NetworkType, UpdaterType, BehaviorPolicyType, - ReplayType + ReplayType, + PrioritizedReplayType >::Episode() { // Get the initial state from environment. From 1bd6e00e3a45e4c75bcfe9050094261a4909643f Mon Sep 17 00:00:00 2001 From: robotcator Date: Thu, 17 Jan 2019 22:45:39 +0800 Subject: [PATCH 013/143] adding unit test for prioritized replay --- .../q_learning_impl.hpp | 32 +++++---- .../replay/prioritized_replay.hpp | 3 +- src/mlpack/tests/q_learning_test.cpp | 66 +++++++++++++++++++ 3 files changed, 88 insertions(+), 13 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 7441dcde7a..4174f1ecfa 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -149,8 +149,16 @@ double QLearning< double reward = environment.Sample(state, action, nextState); // Store the transition for replay. - replayMethod.Store(state, action, reward, - nextState, environment.IsTerminal(nextState)); + if (prioritized_replay) + { + prioritizedReplayMethod.Store(state, action, reward, + nextState, environment.IsTerminal(nextState)); + } + else + { + replayMethod.Store(state, action, reward, + nextState, environment.IsTerminal(nextState)); + } // Update current state. state = nextState; @@ -220,16 +228,16 @@ double QLearning< learningNetwork.Backward(target, gradients); updater.Update(learningNetwork.Parameters(), config.StepSize(), gradients); - // if (prioritized_replay) - // { - // arma::colvec td_error(target.n_cols); - // for (size_t i = 0; i < target.n_cols; i ++) - // { - // td_error[i] = nextActionValues(sampledActions[i], i) - target(sampledActions[i], i); - // } - // td_error = arma::abs(td_error); - // prioritizedReplayMethod.update_priorities(sampledIndices, td_error); - // } + if (prioritized_replay) + { + arma::colvec td_error(target.n_cols); + for (size_t i = 0; i < target.n_cols; i ++) + { + td_error[i] = nextActionValues(sampledActions[i], i) - target(sampledActions[i], i); + } + td_error = arma::abs(td_error); + prioritizedReplayMethod.update_priorities(sampledIndices, td_error); + } return reward; } diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index 132a14a84e..6575bd68e6 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -106,7 +106,7 @@ class PrioritizedReplay arma::ucolvec sampleProportional() { arma::ucolvec idxes(batchSize); - double totalSum = idxSum.sum(0, (full ? capacity : position) - 1); + double totalSum = idxSum.sum(0, (full ? capacity : position)); double sumPerRange = totalSum / batchSize; for (size_t bt = 0; bt < batchSize; bt ++) { double mass = arma::randu() * sumPerRange + bt * sumPerRange; @@ -137,6 +137,7 @@ class PrioritizedReplay // calculate the weights of sampled transitions size_t num_sample = full ? capacity : position; + weights = arma::rowvec(sampledIndices.n_rows); for (size_t i = 0; i < sampledIndices.n_rows; ++ i) { diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index e3cff2c899..32aba1a40e 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -102,6 +102,72 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDQN) BOOST_REQUIRE(converged); } +//! Test DQN in Cart Pole task. +BOOST_AUTO_TEST_CASE(CartPoleWithDQNPRIORITIZED) + { + // Set up the network. + FFN, GaussianInitialization> model(MeanSquaredError<>(), + GaussianInitialization(0, 0.001)); + model.Add>(4, 128); + model.Add>(); + model.Add>(128, 128); + model.Add>(); + model.Add>(128, 2); + + // Set up the policy and replay method. + GreedyPolicy policy(1.0, 1000, 0.1); + PrioritizedReplay prioritizedReplayMethod(10, 10000, 0.6); + + TrainingConfig config; + config.StepSize() = 0.01; + config.Discount() = 0.9; + config.TargetNetworkSyncInterval() = 100; + config.ExplorationSteps() = 100; + config.DoubleQLearning() = false; + config.StepLimit() = 200; + + // Set up DQN agent. + QLearning + agent(std::move(config), std::move(model), std::move(policy), + std::move(prioritizedReplayMethod)); + + arma::running_stat averageReturn; + size_t episodes = 0; + bool converged = true; + while (true) + { + double episodeReturn = agent.Episode(); + averageReturn(episodeReturn); + episodes += 1; + + if (episodes > 1000) + { + Log::Debug << "Cart Pole with DQN failed." << std::endl; + converged = false; + break; + } + + /** + * Reaching running average return 35 is enough to show it works. + * For the speed of the test case, I didn't set high criterion. + */ + Log::Debug << "Average return: " << averageReturn.mean() + << " Episode return: " << episodeReturn << std::endl; + if (averageReturn.mean() > 35) + { + agent.Deterministic() = true; + arma::running_stat testReturn; + for (size_t i = 0; i < 10; ++i) + testReturn(agent.Episode()); + + Log::Debug << "Average return in deterministic test: " + << testReturn.mean() << std::endl; + break; + } + } + BOOST_REQUIRE(converged); +} + //! Test Double DQN in Cart Pole task. BOOST_AUTO_TEST_CASE(CartPoleWithDoubleDQN) { From 42cceda6ae40b0b6b2939b5dc8f5ebaf324b3247 Mon Sep 17 00:00:00 2001 From: robotcator Date: Mon, 21 Jan 2019 13:56:33 +0800 Subject: [PATCH 014/143] adding document for sumtree class --- .../reinforcement_learning/replay/sumtree.hpp | 55 ++++++++++++++++++- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp index 69154a53e5..588b5c9871 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp @@ -22,6 +22,12 @@ namespace rl { /** * Implementation of SumTree. + * + * Build a Segment Tree like data strucuture. + * https://en.wikipedia.org/wiki/Segment_tree + * + * Used to maintain prefix-sum of an array. + * */ template @@ -29,15 +35,29 @@ class SumTree { public: + /** + * Empty constructor + */ SumTree() { /* Nothing to do here. */ } + /** + * Construct an instance of SumTree class + * + * @param capacity The size of data + */ SumTree(size_t capacity): capacity(capacity) { element = std::vector (2 * capacity); } + /** + * Set the data array with idx + * + * @param idx The array idx to be changed + * @param value The data that array with idx to be + */ void set(size_t idx, T value) { idx += capacity; @@ -50,12 +70,26 @@ class SumTree } } + /** + * Get the data array with idx + * + * @param idx The array idx to get data + */ T get(size_t idx) { idx += capacity; return element[idx]; } + /** + * Help function for the `sum` function + * + * @param _start The starting position of subsequence + * @param _end The end position of subsequence + * @param node Reference position + * @param node_start Starting position of reference segment + * @param node_end End position of reference segment + */ T sumHelper(size_t _start, size_t _end, size_t node, size_t node_start, size_t node_end) { if (_start == node_start && _end == node_end) @@ -81,22 +115,34 @@ class SumTree } } + /** + * Calculate the sum of contiguous subsequence of the array. + * + * @param _start The starting position of subsequence + * @param _end The end position of subsequence + */ T sum(size_t _start, size_t _end) { -// caculate the sum of contiguous subsequence of the array. _end -= 1; return sumHelper(_start, _end, 1, 0, capacity-1); } + /** + * Shortcut for calculating the sum of whole array + */ T sum() { return sum(0, capacity); } + /** + * Find the highest index `idx` in the array such that + * sum(arr[0] + arr[1] + ... + arr[idx]) <= mass + * + * @param mass + * */ size_t findPrefixSum(T mass) { -// Find the highest index `idx` in the array such that -// sum(arr[0] + arr[1] + ... + arr[i]) <= mass int idx = 1; while (idx < capacity) { @@ -114,7 +160,10 @@ class SumTree } private: + //! The capacity of the data array size_t capacity; + + //! double size of capacity, maintain the segment sum of data std::vector element; }; From 534fcf0435bb4681d95829d226aa47e1abf74846 Mon Sep 17 00:00:00 2001 From: robotcator Date: Mon, 21 Jan 2019 14:49:11 +0800 Subject: [PATCH 015/143] adding document for sumtree class --- src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp index 588b5c9871..ff89d10a46 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp @@ -36,7 +36,7 @@ class SumTree public: /** - * Empty constructor + * Default constructor */ SumTree() { /* Nothing to do here. */ } From 1251c4699b15e1b3c1db483c988bcd2c3a282061 Mon Sep 17 00:00:00 2001 From: robotcator Date: Mon, 21 Jan 2019 14:54:28 +0800 Subject: [PATCH 016/143] adding document for PrioritizedReplay class --- .../replay/prioritized_replay.hpp | 59 ++++++++++++++++++- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index 6575bd68e6..2eb26e345d 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -43,6 +43,9 @@ class PrioritizedReplay //! Convenient typedef for state. using StateType = typename EnvironmentType::State; + /** + * Default constructor + */ PrioritizedReplay() { /* Nothing to do here. */ } @@ -51,7 +54,7 @@ class PrioritizedReplay * * @param batchSize Number of examples returned at each sample. * @param capacity Total memory size in terms of number of examples. - * @param alpha + * @param alpha How much prioritization is used * @param dimension The dimension of an encoded state. */ PrioritizedReplay(const size_t batchSize, @@ -81,6 +84,16 @@ class PrioritizedReplay idxSum = SumTree(size); } + /** + * Store the given experience + * Set priorities for the given experience + * + * @param state Given state. + * @param action Given action. + * @param reward Given reward. + * @param nextState Given next state. + * @param isEnd Whether next state is terminal state. + */ void Store(const StateType& state, ActionType action, double reward, @@ -103,6 +116,11 @@ class PrioritizedReplay } } + /** + * Samle some experience accroding to their priorities. + * + * @return The indices to be chosen. + */ arma::ucolvec sampleProportional() { arma::ucolvec idxes(batchSize); @@ -116,6 +134,18 @@ class PrioritizedReplay return idxes; } + /** + * Samle some experience accroding to their priorities. + * + * @param sampledStates Sampled encoded states. + * @param sampledActions Sampled actions. + * @param sampledRewards Sampled rewards. + * @param sampledNextStates Sampled encoded next states. + * @param isTerminal Indicate whether corresponding next state is terminal + * state. + * @param sampledIndices Sampled indices + * @param weights Corresponding weight for updating the loss + */ void Sample(arma::mat& sampledStates, arma::icolvec& sampledActions, arma::colvec& sampledRewards, @@ -127,6 +157,7 @@ class PrioritizedReplay size_t upperBound = full ? capacity : position; sampledIndices = sampleProportional(); + betaAnneal(); sampledStates = states.cols(sampledIndices); sampledActions = actions.elem(sampledIndices); @@ -147,9 +178,14 @@ class PrioritizedReplay weights /= weights.max(); } + /** + * Update priorities of sampled transitions. + * + * @param indices The indices of sample to be updated. + * @param priorities Their corresponding priorities. + */ void update_priorities(arma::ucolvec& indices, arma::colvec& priorities) { -// update priorities of sampled transitions. for (size_t i = 0; i < indices.n_rows; ++i) { idxSum.set(indices[i], alpha * priorities[i]); @@ -157,6 +193,19 @@ class PrioritizedReplay } } + /** + * Get the number of transitions in the memory. + * + * @return Actual used memory size + */ + const size_t& Size() + { + return full ? capacity : position; + } + + /** + * Annealing the beta + */ void betaAnneal() { beta = beta + (1 - initial_beta) * 1.0 / replay_beta_iters; @@ -164,17 +213,21 @@ class PrioritizedReplay private: //! How much prioritization is used. + // (0 - no prioritization, 1 - full prioritization) double alpha; double max_priority; + //! Initial value of beta for prioritized replay buffer. double initial_beta; + //! The value of beta for current sample. double beta; + //! How many iteration for replay beta to decay. size_t replay_beta_iters; - //! Locally-stored the prefix sum of prioritization + //! Locally-stored the prefix sum of prioritization. SumTree idxSum; //! Locally-stored number of examples of each sample. From b83164f3090416c4f68cd4cf1c20c016ef97318d Mon Sep 17 00:00:00 2001 From: robotcator Date: Fri, 25 Jan 2019 20:42:57 +0800 Subject: [PATCH 017/143] add batchUpdate method for prioritized_replay's update_priorities --- .../reinforcement_learning/replay/sumtree.hpp | 22 ++++++++++++++++++- src/mlpack/tests/sumtree_test.cpp | 19 ++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp index ff89d10a46..bc8db58197 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp @@ -23,7 +23,7 @@ namespace rl { /** * Implementation of SumTree. * - * Build a Segment Tree like data strucuture. + * Build a Segment Tree like data structure. * https://en.wikipedia.org/wiki/Segment_tree * * Used to maintain prefix-sum of an array. @@ -70,6 +70,26 @@ class SumTree } } + + /** + * Update the data with batch rather loop over the indices with set method + * + * @param indices The indices of data to be changed + * @param data The data that array with indices to be + */ + void batchUpdate(arma::ucolvec indices, arma::Col data) + { + for (size_t i = 0; i < indices.n_rows; i ++) + { + element[indices[i] + capacity] = data[i]; + } + // update the total tree with bottom-up technique + for (size_t i = capacity-1; i > 0; i --) + { + element[i] = element[2 * i] + element[2 * i + 1]; + } + } + /** * Get the data array with idx * diff --git a/src/mlpack/tests/sumtree_test.cpp b/src/mlpack/tests/sumtree_test.cpp index 992de78464..48942a55cd 100644 --- a/src/mlpack/tests/sumtree_test.cpp +++ b/src/mlpack/tests/sumtree_test.cpp @@ -77,4 +77,23 @@ BOOST_AUTO_TEST_CASE(FindPrefixSum) BOOST_CHECK_EQUAL(sumtree.findPrefixSum(3.0), 3); } +/** + * Test that we find the highest index in the array such that + * sum(arr[0] + arr[1] + arr[2] ... + arr[i]) <= mass . + */ + +BOOST_AUTO_TEST_CASE(BatchUpdate) +{ + SumTree sumtree(4); + arma::ucolvec indices = {0, 1, 2, 3}; + arma::colvec data = {1.0, 0.8, 0.6, 0.4}; + + sumtree.batchUpdate(indices, data); + + BOOST_CHECK_EQUAL(sumtree.findPrefixSum(0), 0); + BOOST_CHECK_EQUAL(sumtree.findPrefixSum(1), 1); + BOOST_CHECK_EQUAL(sumtree.findPrefixSum(2.8), 3); + BOOST_CHECK_EQUAL(sumtree.findPrefixSum(3.0), 3); +} + BOOST_AUTO_TEST_SUITE_END(); From afbbf23aca0d99230c9859bf34833947d454d6b3 Mon Sep 17 00:00:00 2001 From: robotcator Date: Fri, 25 Jan 2019 20:44:00 +0800 Subject: [PATCH 018/143] change update_priorities with batchUpdate --- .../reinforcement_learning/replay/prioritized_replay.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index 2eb26e345d..efb12aa0de 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -186,11 +186,13 @@ class PrioritizedReplay */ void update_priorities(arma::ucolvec& indices, arma::colvec& priorities) { + arma::colvec alphaPri(indices.n_rows); for (size_t i = 0; i < indices.n_rows; ++i) { - idxSum.set(indices[i], alpha * priorities[i]); + alphaPri = alpha * priorities[i]; max_priority = std::max(max_priority, priorities[i]); } + idxSum.batchUpdate(indices, alphaPri); } /** From 65f696e0ccb27f547609871179e08f7e89a23eb8 Mon Sep 17 00:00:00 2001 From: robotcator Date: Fri, 25 Jan 2019 21:51:35 +0800 Subject: [PATCH 019/143] fixed weighted update for prioritized replay --- .../methods/reinforcement_learning/q_learning_impl.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 4174f1ecfa..c0cb382a9a 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -226,6 +226,12 @@ double QLearning< // Learn form experience. arma::mat gradients; learningNetwork.Backward(target, gradients); + + if (prioritized_replay) + { + gradients = arma::mean(weights) * gradients; + } + updater.Update(learningNetwork.Parameters(), config.StepSize(), gradients); if (prioritized_replay) From 0ff4ebe592b7cbce1a8bc661b4ccdd56b4d89e9d Mon Sep 17 00:00:00 2001 From: robotcator Date: Sun, 27 Jan 2019 10:50:26 +0800 Subject: [PATCH 020/143] fix some typo --- .../reinforcement_learning/replay/prioritized_replay.hpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index efb12aa0de..935ff21e45 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -60,6 +60,7 @@ class PrioritizedReplay PrioritizedReplay(const size_t batchSize, const size_t capacity, const double alpha, + const size_t dimension = StateType::dimension) : batchSize(batchSize), capacity(capacity), @@ -74,7 +75,8 @@ class PrioritizedReplay max_priority(1.0), initial_beta(0.6), replay_beta_iters(10000) - { + { + arma_rng::set_value(seed); int size = 1; while (size < capacity) { size *= 2; @@ -117,7 +119,7 @@ class PrioritizedReplay } /** - * Samle some experience accroding to their priorities. + * Sample some experience according to their priorities. * * @return The indices to be chosen. */ @@ -135,7 +137,7 @@ class PrioritizedReplay } /** - * Samle some experience accroding to their priorities. + * Sample some experience according to their priorities. * * @param sampledStates Sampled encoded states. * @param sampledActions Sampled actions. From a0e0bba68445c9fbff3cdb66ca6e441259996a49 Mon Sep 17 00:00:00 2001 From: robotcator Date: Sun, 27 Jan 2019 11:22:02 +0800 Subject: [PATCH 021/143] add random seed parameter --- .../reinforcement_learning/replay/prioritized_replay.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index 935ff21e45..07f66abf4b 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -60,7 +60,7 @@ class PrioritizedReplay PrioritizedReplay(const size_t batchSize, const size_t capacity, const double alpha, - + int seed = 1024, const size_t dimension = StateType::dimension) : batchSize(batchSize), capacity(capacity), @@ -76,7 +76,7 @@ class PrioritizedReplay initial_beta(0.6), replay_beta_iters(10000) { - arma_rng::set_value(seed); + arma::arma_rng::set_seed(seed); int size = 1; while (size < capacity) { size *= 2; From 7a45e3f7607d87ceca76905944abd55e24f9aaa6 Mon Sep 17 00:00:00 2001 From: robotcator Date: Mon, 28 Jan 2019 17:04:35 +0800 Subject: [PATCH 022/143] make consistent with code style --- .../reinforcement_learning/replay/sumtree.hpp | 78 +++++++++---------- src/mlpack/tests/sumtree_test.cpp | 60 +++++++------- 2 files changed, 69 insertions(+), 69 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp index bc8db58197..b2667f84ad 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp @@ -36,15 +36,15 @@ class SumTree public: /** - * Default constructor + * Default constructor. */ SumTree() { /* Nothing to do here. */ } /** - * Construct an instance of SumTree class + * Construct an instance of SumTree class. * - * @param capacity The size of data + * @param capacity Size of data. */ SumTree(size_t capacity): capacity(capacity) @@ -53,12 +53,12 @@ class SumTree } /** - * Set the data array with idx + * Set the data array with idx. * - * @param idx The array idx to be changed - * @param value The data that array with idx to be + * @param idx The array idx to be changed. + * @param value The data that array with idx to be. */ - void set(size_t idx, T value) + void Set(size_t idx, T value) { idx += capacity; element[idx] = value; @@ -72,12 +72,12 @@ class SumTree /** - * Update the data with batch rather loop over the indices with set method + * Update the data with batch rather loop over the indices with set method. * - * @param indices The indices of data to be changed - * @param data The data that array with indices to be + * @param indices The indices of data to be changed. + * @param data The data that array with indices to be. */ - void batchUpdate(arma::ucolvec indices, arma::Col data) + void BatchUpdate(arma::ucolvec indices, arma::Col data) { for (size_t i = 0; i < indices.n_rows; i ++) { @@ -91,11 +91,11 @@ class SumTree } /** - * Get the data array with idx + * Get the data array with idx. * - * @param idx The array idx to get data + * @param idx The array idx to get data. */ - T get(size_t idx) + T Get(size_t idx) { idx += capacity; return element[idx]; @@ -104,33 +104,33 @@ class SumTree /** * Help function for the `sum` function * - * @param _start The starting position of subsequence - * @param _end The end position of subsequence + * @param start The starting position of subsequence. + * @param end The end position of subsequence. * @param node Reference position - * @param node_start Starting position of reference segment - * @param node_end End position of reference segment + * @param node_start Starting position of reference segment. + * @param node_end End position of reference segment. */ - T sumHelper(size_t _start, size_t _end, size_t node, size_t node_start, size_t node_end) + T SumHelper(size_t start, size_t end, size_t node, size_t nodeStart, size_t nodeEnd) { - if (_start == node_start && _end == node_end) + if (start == nodeStart && end == nodeEnd) { return element[node]; } - size_t mid = (node_start + node_end) / 2; - if (_end <= mid) + size_t mid = (nodeStart + nodeEnd) / 2; + if (end <= mid) { - return sumHelper(_start, _end, 2 * node, node_start, mid); + return SumHelper(start, end, 2 * node, nodeStart, mid); } else { - if (mid + 1 <= _start) + if (mid + 1 <= start) { - return sumHelper(_start, _end, 2 * node + 1, mid + 1 , node_end); + return SumHelper(start, end, 2 * node + 1, mid + 1 , nodeEnd); } else { - return sumHelper(_start, mid, 2 * node, node_start, mid) + - sumHelper(mid+1, _end, 2 * node + 1, mid + 1 , node_end); + return SumHelper(start, mid, 2 * node, nodeStart, mid) + + SumHelper(mid+1, end, 2 * node + 1, mid + 1 , nodeEnd); } } } @@ -138,30 +138,30 @@ class SumTree /** * Calculate the sum of contiguous subsequence of the array. * - * @param _start The starting position of subsequence - * @param _end The end position of subsequence + * @param start The starting position of subsequence. + * @param _end The end position of subsequence. */ - T sum(size_t _start, size_t _end) + T Sum(size_t start, size_t end) { - _end -= 1; - return sumHelper(_start, _end, 1, 0, capacity-1); + end -= 1; + return SumHelper(start, end, 1, 0, capacity-1); } /** - * Shortcut for calculating the sum of whole array + * Shortcut for calculating the sum of whole array. */ - T sum() + T Sum() { - return sum(0, capacity); + return Sum(0, capacity); } /** * Find the highest index `idx` in the array such that - * sum(arr[0] + arr[1] + ... + arr[idx]) <= mass + * sum(arr[0] + arr[1] + ... + arr[idx]) <= mass. * * @param mass * */ - size_t findPrefixSum(T mass) + size_t FindPrefixSum(T mass) { int idx = 1; while (idx < capacity) @@ -180,10 +180,10 @@ class SumTree } private: - //! The capacity of the data array + //! The capacity of the data array. size_t capacity; - //! double size of capacity, maintain the segment sum of data + //! double size of capacity, maintain the segment sum of data. std::vector element; }; diff --git a/src/mlpack/tests/sumtree_test.cpp b/src/mlpack/tests/sumtree_test.cpp index 48942a55cd..7cb1c82557 100644 --- a/src/mlpack/tests/sumtree_test.cpp +++ b/src/mlpack/tests/sumtree_test.cpp @@ -29,15 +29,15 @@ BOOST_AUTO_TEST_SUITE(SumTreeTest); BOOST_AUTO_TEST_CASE(SetElement) { SumTree sumtree(4); - sumtree.set(0, 1.0); - sumtree.set(1, 0.8); - sumtree.set(2, 0.6); - sumtree.set(3, 0.4); + sumtree.Set(0, 1.0); + sumtree.Set(1, 0.8); + sumtree.Set(2, 0.6); + sumtree.Set(3, 0.4); - BOOST_CHECK_CLOSE(sumtree.sum(), 2.8, 1e-8); - BOOST_CHECK_CLOSE(sumtree.sum(0, 1), 1.0, 1e-8); - BOOST_CHECK_CLOSE(sumtree.sum(0, 3), 2.4, 1e-8); - BOOST_CHECK_CLOSE(sumtree.sum(1, 4), 1.8, 1e-8); + BOOST_CHECK_CLOSE(sumtree.Sum(), 2.8, 1e-8); + BOOST_CHECK_CLOSE(sumtree.Sum(0, 1), 1.0, 1e-8); + BOOST_CHECK_CLOSE(sumtree.Sum(0, 3), 2.4, 1e-8); + BOOST_CHECK_CLOSE(sumtree.Sum(1, 4), 1.8, 1e-8); } /** @@ -47,34 +47,34 @@ BOOST_AUTO_TEST_CASE(SetElement) BOOST_AUTO_TEST_CASE(GetElement) { SumTree sumtree(4); - sumtree.set(0, 1.0); - sumtree.set(1, 0.8); - sumtree.set(2, 0.6); - sumtree.set(3, 0.4); + sumtree.Set(0, 1.0); + sumtree.Set(1, 0.8); + sumtree.Set(2, 0.6); + sumtree.Set(3, 0.4); - BOOST_CHECK_CLOSE(sumtree.get(0), 1.0, 1e-8); - BOOST_CHECK_CLOSE(sumtree.get(1), 0.8, 1e-8); - BOOST_CHECK_CLOSE(sumtree.get(2), 0.6, 1e-8); - BOOST_CHECK_CLOSE(sumtree.get(3), 0.4, 1e-8); + BOOST_CHECK_CLOSE(sumtree.Get(0), 1.0, 1e-8); + BOOST_CHECK_CLOSE(sumtree.Get(1), 0.8, 1e-8); + BOOST_CHECK_CLOSE(sumtree.Get(2), 0.6, 1e-8); + BOOST_CHECK_CLOSE(sumtree.Get(3), 0.4, 1e-8); } /** * Test that we find the highest index in the array such that - * sum(arr[0] + arr[1] + arr[2] ... + arr[i]) <= mass . + * Sum(arr[0] + arr[1] + arr[2] ... + arr[i]) <= mass . */ BOOST_AUTO_TEST_CASE(FindPrefixSum) { SumTree sumtree(4); - sumtree.set(0, 1.0); - sumtree.set(1, 0.8); - sumtree.set(2, 0.6); - sumtree.set(3, 0.4); + sumtree.Set(0, 1.0); + sumtree.Set(1, 0.8); + sumtree.Set(2, 0.6); + sumtree.Set(3, 0.4); - BOOST_CHECK_EQUAL(sumtree.findPrefixSum(0), 0); - BOOST_CHECK_EQUAL(sumtree.findPrefixSum(1), 1); - BOOST_CHECK_EQUAL(sumtree.findPrefixSum(2.8), 3); - BOOST_CHECK_EQUAL(sumtree.findPrefixSum(3.0), 3); + BOOST_CHECK_EQUAL(sumtree.FindPrefixSum(0), 0); + BOOST_CHECK_EQUAL(sumtree.FindPrefixSum(1), 1); + BOOST_CHECK_EQUAL(sumtree.FindPrefixSum(2.8), 3); + BOOST_CHECK_EQUAL(sumtree.FindPrefixSum(3.0), 3); } /** @@ -88,12 +88,12 @@ BOOST_AUTO_TEST_CASE(BatchUpdate) arma::ucolvec indices = {0, 1, 2, 3}; arma::colvec data = {1.0, 0.8, 0.6, 0.4}; - sumtree.batchUpdate(indices, data); + sumtree.BatchUpdate(indices, data); - BOOST_CHECK_EQUAL(sumtree.findPrefixSum(0), 0); - BOOST_CHECK_EQUAL(sumtree.findPrefixSum(1), 1); - BOOST_CHECK_EQUAL(sumtree.findPrefixSum(2.8), 3); - BOOST_CHECK_EQUAL(sumtree.findPrefixSum(3.0), 3); + BOOST_CHECK_EQUAL(sumtree.FindPrefixSum(0), 0); + BOOST_CHECK_EQUAL(sumtree.FindPrefixSum(1), 1); + BOOST_CHECK_EQUAL(sumtree.FindPrefixSum(2.8), 3); + BOOST_CHECK_EQUAL(sumtree.FindPrefixSum(3.0), 3); } BOOST_AUTO_TEST_SUITE_END(); From 4a45132ff68ea9c5948100cc79640b54676c25ea Mon Sep 17 00:00:00 2001 From: robotcator Date: Mon, 28 Jan 2019 17:04:53 +0800 Subject: [PATCH 023/143] make consistent with code style --- .../replay/prioritized_replay.hpp | 68 +++++++++---------- 1 file changed, 32 insertions(+), 36 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index 07f66abf4b..77d69755eb 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -44,7 +44,7 @@ class PrioritizedReplay using StateType = typename EnvironmentType::State; /** - * Default constructor + * Default constructor. */ PrioritizedReplay() { /* Nothing to do here. */ } @@ -54,7 +54,7 @@ class PrioritizedReplay * * @param batchSize Number of examples returned at each sample. * @param capacity Total memory size in terms of number of examples. - * @param alpha How much prioritization is used + * @param alpha How much prioritization is used. * @param dimension The dimension of an encoded state. */ PrioritizedReplay(const size_t batchSize, @@ -72,23 +72,23 @@ class PrioritizedReplay nextStates(dimension, capacity), isTerminal(capacity), full(false), - max_priority(1.0), - initial_beta(0.6), - replay_beta_iters(10000) + maxPriority(1.0), + initialBeta(0.6), + replayBetaIters(10000) { arma::arma_rng::set_seed(seed); - int size = 1; + size_t size = 1; while (size < capacity) { size *= 2; } - beta = initial_beta; + beta = initialBeta; idxSum = SumTree(size); } /** - * Store the given experience - * Set priorities for the given experience + * Store the given experience. + * Set priorities for the given experience. * * @param state Given state. * @param action Given action. @@ -108,7 +108,7 @@ class PrioritizedReplay nextStates.col(position) = nextState.Encode(); isTerminal(position) = isEnd; - idxSum.set(position, max_priority * alpha); + idxSum.Set(position, maxPriority * alpha); position++; if (position == capacity) @@ -123,14 +123,14 @@ class PrioritizedReplay * * @return The indices to be chosen. */ - arma::ucolvec sampleProportional() + arma::ucolvec SampleProportional() { arma::ucolvec idxes(batchSize); - double totalSum = idxSum.sum(0, (full ? capacity : position)); + double totalSum = idxSum.Sum(0, (full ? capacity : position)); double sumPerRange = totalSum / batchSize; for (size_t bt = 0; bt < batchSize; bt ++) { double mass = arma::randu() * sumPerRange + bt * sumPerRange; - size_t idx = idxSum.findPrefixSum(mass); + size_t idx = idxSum.FindPrefixSum(mass); idxes[bt] = idx; } return idxes; @@ -145,8 +145,8 @@ class PrioritizedReplay * @param sampledNextStates Sampled encoded next states. * @param isTerminal Indicate whether corresponding next state is terminal * state. - * @param sampledIndices Sampled indices - * @param weights Corresponding weight for updating the loss + * @param sampledIndices Sampled indices. + * @param weights Corresponding weight for updating the loss. */ void Sample(arma::mat& sampledStates, arma::icolvec& sampledActions, @@ -158,8 +158,8 @@ class PrioritizedReplay { size_t upperBound = full ? capacity : position; - sampledIndices = sampleProportional(); - betaAnneal(); + sampledIndices = SampleProportional(); + BetaAnneal(); sampledStates = states.cols(sampledIndices); sampledActions = actions.elem(sampledIndices); @@ -167,15 +167,15 @@ class PrioritizedReplay sampledNextStates = nextStates.cols(sampledIndices); isTerminal = this->isTerminal.elem(sampledIndices); -// calculate the weights of sampled transitions + // Calculate the weights of sampled transitions. - size_t num_sample = full ? capacity : position; + size_t numSample = full ? capacity : position; weights = arma::rowvec(sampledIndices.n_rows); for (size_t i = 0; i < sampledIndices.n_rows; ++ i) { - double p_sample = idxSum.get(sampledIndices[i]) / idxSum.sum(); - weights[i] = pow(num_sample * p_sample, -beta); + double p_sample = idxSum.Get(sampledIndices[i]) / idxSum.Sum(); + weights[i] = pow(numSample * p_sample, -beta); } weights /= weights.max(); } @@ -186,21 +186,17 @@ class PrioritizedReplay * @param indices The indices of sample to be updated. * @param priorities Their corresponding priorities. */ - void update_priorities(arma::ucolvec& indices, arma::colvec& priorities) + void UpdatePriorities(arma::ucolvec& indices, arma::colvec& priorities) { - arma::colvec alphaPri(indices.n_rows); - for (size_t i = 0; i < indices.n_rows; ++i) - { - alphaPri = alpha * priorities[i]; - max_priority = std::max(max_priority, priorities[i]); - } - idxSum.batchUpdate(indices, alphaPri); + arma::colvec alphaPri = alpha * priorities; + maxPriority = std::max(maxPriority, arma::max(priorities)); + idxSum.BatchUpdate(indices, alphaPri); } /** * Get the number of transitions in the memory. * - * @return Actual used memory size + * @return Actual used memory size. */ const size_t& Size() { @@ -208,11 +204,11 @@ class PrioritizedReplay } /** - * Annealing the beta + * Annealing the beta. */ - void betaAnneal() + void BetaAnneal() { - beta = beta + (1 - initial_beta) * 1.0 / replay_beta_iters; + beta = beta + (1 - initialBeta) * 1.0 / replayBetaIters; } private: @@ -220,16 +216,16 @@ private: // (0 - no prioritization, 1 - full prioritization) double alpha; - double max_priority; + double maxPriority; //! Initial value of beta for prioritized replay buffer. - double initial_beta; + double initialBeta; //! The value of beta for current sample. double beta; //! How many iteration for replay beta to decay. - size_t replay_beta_iters; + size_t replayBetaIters; //! Locally-stored the prefix sum of prioritization. SumTree idxSum; From 2c835505b1c6c33b156dd3087a169935c6308026 Mon Sep 17 00:00:00 2001 From: robotcator Date: Mon, 28 Jan 2019 17:05:15 +0800 Subject: [PATCH 024/143] make consistent with function name change --- src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index c0cb382a9a..7d2e9283bf 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -242,7 +242,7 @@ double QLearning< td_error[i] = nextActionValues(sampledActions[i], i) - target(sampledActions[i], i); } td_error = arma::abs(td_error); - prioritizedReplayMethod.update_priorities(sampledIndices, td_error); + prioritizedReplayMethod.UpdatePriorities(sampledIndices, td_error); } return reward; } From 834aea3cb4ec9a77e87f1f24c5c3faae5f111378 Mon Sep 17 00:00:00 2001 From: robotcator Date: Mon, 28 Jan 2019 17:08:16 +0800 Subject: [PATCH 025/143] update the for loop style --- .../reinforcement_learning/replay/prioritized_replay.hpp | 4 ++-- src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index 77d69755eb..f49cb14f59 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -128,7 +128,7 @@ class PrioritizedReplay arma::ucolvec idxes(batchSize); double totalSum = idxSum.Sum(0, (full ? capacity : position)); double sumPerRange = totalSum / batchSize; - for (size_t bt = 0; bt < batchSize; bt ++) { + for (size_t bt = 0; bt < batchSize; bt++) { double mass = arma::randu() * sumPerRange + bt * sumPerRange; size_t idx = idxSum.FindPrefixSum(mass); idxes[bt] = idx; @@ -172,7 +172,7 @@ class PrioritizedReplay size_t numSample = full ? capacity : position; weights = arma::rowvec(sampledIndices.n_rows); - for (size_t i = 0; i < sampledIndices.n_rows; ++ i) + for (size_t i = 0; i < sampledIndices.n_rows; i++) { double p_sample = idxSum.Get(sampledIndices[i]) / idxSum.Sum(); weights[i] = pow(numSample * p_sample, -beta); diff --git a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp index b2667f84ad..26ba7be172 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp @@ -79,12 +79,12 @@ class SumTree */ void BatchUpdate(arma::ucolvec indices, arma::Col data) { - for (size_t i = 0; i < indices.n_rows; i ++) + for (size_t i = 0; i < indices.n_rows; i++) { element[indices[i] + capacity] = data[i]; } // update the total tree with bottom-up technique - for (size_t i = capacity-1; i > 0; i --) + for (size_t i = capacity-1; i > 0; i--) { element[i] = element[2 * i] + element[2 * i + 1]; } From 75dafb1ad32cd258d138f02860170230ce77e77b Mon Sep 17 00:00:00 2001 From: robotcator Date: Sat, 9 Feb 2019 12:37:47 +0800 Subject: [PATCH 026/143] add update interface for replay buffer. --- .../reinforcement_learning/q_learning_impl.hpp | 12 +++++------- .../replay/prioritized_replay.hpp | 13 +++++++++++++ .../reinforcement_learning/replay/random_replay.hpp | 6 ++++++ 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 7d2e9283bf..7a4e4a923b 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -236,13 +236,11 @@ double QLearning< if (prioritized_replay) { - arma::colvec td_error(target.n_cols); - for (size_t i = 0; i < target.n_cols; i ++) - { - td_error[i] = nextActionValues(sampledActions[i], i) - target(sampledActions[i], i); - } - td_error = arma::abs(td_error); - prioritizedReplayMethod.UpdatePriorities(sampledIndices, td_error); + prioritizedReplayMethod.Update(target, sampledActions, nextActionValues, sampledIndices); + } + else + { + replayMethod.Update(target, sampledActions, nextActionValues, sampledIndices); } return reward; } diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index f49cb14f59..359eca1cdf 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -211,6 +211,19 @@ class PrioritizedReplay beta = beta + (1 - initialBeta) * 1.0 / replayBetaIters; } + void Update(arma::mat target, arma::icolvec sampledActions, + arma::mat nextActionValues, arma::ucolvec sampledIndices) + { + arma::colvec td_error(target.n_cols); + for (size_t i = 0; i < target.n_cols; i ++) + { + td_error[i] = nextActionValues(sampledActions[i], i) - target(sampledActions[i], i); + } + td_error = arma::abs(td_error); + UpdatePriorities(sampledIndices, td_error); + } + + private: //! How much prioritization is used. // (0 - no prioritization, 1 - full prioritization) diff --git a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp index 4afe6afa84..5e850769c2 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp @@ -138,6 +138,12 @@ class RandomReplay return full ? capacity : position; } + void Update(arma::mat target, arma::icolvec sampledActions, + arma::mat nextActionValues, arma::ucolvec sampledIndices) + { + /* do nothing for random replay*/ + } + private: //! Locally-stored number of examples of each sample. size_t batchSize; From 48f1ac6256d1f2e9d1ccd6b0ebe5c7e0207a07d6 Mon Sep 17 00:00:00 2001 From: robotcator Date: Thu, 14 Feb 2019 10:00:09 +0800 Subject: [PATCH 027/143] fix static code analysis --- .../methods/reinforcement_learning/replay/random_replay.hpp | 4 ++++ src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp | 1 + 2 files changed, 5 insertions(+) diff --git a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp index 5e850769c2..aabac6856b 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp @@ -50,6 +50,10 @@ class RandomReplay using StateType = typename EnvironmentType::State; RandomReplay() + batchSize(0), + capacity(0), + position(0), + full(false) { /* Nothing to do here. */ } /** diff --git a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp index 26ba7be172..955ad7a22c 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp @@ -39,6 +39,7 @@ class SumTree * Default constructor. */ SumTree() + capacity(0) { /* Nothing to do here. */ } /** From 61a065b1c4d303f96808558e2d41ad3b5a988a3d Mon Sep 17 00:00:00 2001 From: robotcator Date: Thu, 14 Feb 2019 10:58:59 +0800 Subject: [PATCH 028/143] fix static code analysis --- .../q_learning_impl.hpp | 26 ++++++++++--------- .../replay/prioritized_replay.hpp | 8 +++--- .../replay/random_replay.hpp | 2 +- .../reinforcement_learning/replay/sumtree.hpp | 6 ++--- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 7a4e4a923b..7431876d0f 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -180,13 +180,13 @@ double QLearning< if (!prioritized_replay) { replayMethod.Sample(sampledStates, sampledActions, sampledRewards, - sampledNextStates, isTerminal); + sampledNextStates, isTerminal); } else { - prioritizedReplayMethod.Sample(sampledStates, sampledActions, sampledRewards, - sampledNextStates, isTerminal, sampledIndices, - weights); + prioritizedReplayMethod.Sample(sampledStates, sampledActions, + sampledRewards, sampledNextStates, isTerminal, + sampledIndices, weights); } // Compute action value for next state with target network. @@ -234,14 +234,16 @@ double QLearning< updater.Update(learningNetwork.Parameters(), config.StepSize(), gradients); - if (prioritized_replay) - { - prioritizedReplayMethod.Update(target, sampledActions, nextActionValues, sampledIndices); - } - else - { - replayMethod.Update(target, sampledActions, nextActionValues, sampledIndices); - } + if (prioritized_replay) + { + prioritizedReplayMethod.Update(target, sampledActions, + nextActionValues, sampledIndices); + } + else + { + replayMethod.Update(target, sampledActions, + nextActionValues, sampledIndices); + } return reward; } diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index 359eca1cdf..4c4699e932 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -217,16 +217,17 @@ class PrioritizedReplay arma::colvec td_error(target.n_cols); for (size_t i = 0; i < target.n_cols; i ++) { - td_error[i] = nextActionValues(sampledActions[i], i) - target(sampledActions[i], i); + td_error[i] = nextActionValues(sampledActions[i], i) - + target(sampledActions[i], i); } td_error = arma::abs(td_error); UpdatePriorities(sampledIndices, td_error); } -private: + private: //! How much prioritization is used. - // (0 - no prioritization, 1 - full prioritization) + //! (0 - no prioritization, 1 - full prioritization) double alpha; double maxPriority; @@ -269,7 +270,6 @@ private: //! Locally-stored indicator that whether the memory is full or not bool full; - }; } // namespace rl diff --git a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp index aabac6856b..588cef36b0 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp @@ -49,7 +49,7 @@ class RandomReplay //! Convenient typedef for state. using StateType = typename EnvironmentType::State; - RandomReplay() + RandomReplay(): batchSize(0), capacity(0), position(0), diff --git a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp index 955ad7a22c..2db7232619 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp @@ -34,11 +34,10 @@ template class SumTree { public: - /** * Default constructor. */ - SumTree() + SumTree(): capacity(0) { /* Nothing to do here. */ } @@ -111,7 +110,8 @@ class SumTree * @param node_start Starting position of reference segment. * @param node_end End position of reference segment. */ - T SumHelper(size_t start, size_t end, size_t node, size_t nodeStart, size_t nodeEnd) + T SumHelper(size_t start, size_t end, size_t node, + size_t nodeStart, size_t nodeEnd) { if (start == nodeStart && end == nodeEnd) { From 7adbe1716d9a32ab50ad9c38acd549a3f0c523a2 Mon Sep 17 00:00:00 2001 From: robotcator Date: Thu, 14 Feb 2019 11:05:10 +0800 Subject: [PATCH 029/143] fix static code analysis --- src/mlpack/tests/q_learning_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index b4c611ba0f..789c014210 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -107,7 +107,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDQNPRIORITIZED) { // Set up the network. FFN, GaussianInitialization> model(MeanSquaredError<>(), - GaussianInitialization(0, 0.001)); + GaussianInitialization(0, 0.001)); model.Add>(4, 128); model.Add>(); model.Add>(128, 128); From 5dd7b93b8ff87c02db492bf9dff1393e1f9c4afd Mon Sep 17 00:00:00 2001 From: robotcator Date: Thu, 14 Feb 2019 16:34:29 +0800 Subject: [PATCH 030/143] fix static code analysis --- src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 7431876d0f..35365c0298 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -152,12 +152,12 @@ double QLearning< if (prioritized_replay) { prioritizedReplayMethod.Store(state, action, reward, - nextState, environment.IsTerminal(nextState)); + nextState, environment.IsTerminal(nextState)); } else { replayMethod.Store(state, action, reward, - nextState, environment.IsTerminal(nextState)); + nextState, environment.IsTerminal(nextState)); } // Update current state. From de097da6cfe3f36a62ad3a4a4046c3e0e383e9c0 Mon Sep 17 00:00:00 2001 From: robotcator Date: Sat, 2 Mar 2019 10:29:46 +0800 Subject: [PATCH 031/143] add comment for the q_learning_test --- src/mlpack/tests/q_learning_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index 789c014210..80d0584f28 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -102,8 +102,8 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDQN) BOOST_REQUIRE(converged); } -//! Test DQN in Cart Pole task. -BOOST_AUTO_TEST_CASE(CartPoleWithDQNPRIORITIZED) +//! Test DQN in Cart Pole task with Prioritized Replay +BOOST_AUTO_TEST_CASE(CartPoleWithDQNPrioritizedReplay) { // Set up the network. FFN, GaussianInitialization> model(MeanSquaredError<>(), From 87cb72eee08ca12f0add933befd556bc00e21c3d Mon Sep 17 00:00:00 2001 From: Yashwant Date: Sun, 19 May 2019 22:57:38 +0530 Subject: [PATCH 032/143] Add Gmm CLI test --- src/mlpack/methods/gmm/gmm_train_main.cpp | 6 ++++++ src/mlpack/tests/CMakeLists.txt | 3 +++ 2 files changed, 9 insertions(+) diff --git a/src/mlpack/methods/gmm/gmm_train_main.cpp b/src/mlpack/methods/gmm/gmm_train_main.cpp index a01adfcb0e..35c6eb8b8f 100644 --- a/src/mlpack/methods/gmm/gmm_train_main.cpp +++ b/src/mlpack/methods/gmm/gmm_train_main.cpp @@ -144,12 +144,18 @@ static void mlpackMain() "number of Gaussians must be positive"); const int gaussians = CLI::GetParam("gaussians"); + RequireParamValue("trials", [](int x) { return x > 0; }, true, + "trials must be greater than 0"); + ReportIgnoredParam({{ "diagonal_covariance", true }}, "no_force_positive"); RequireAtLeastOnePassed({ "output_model" }, false, "no model will be saved"); RequireParamValue("noise", [](double x) { return x >= 0.0; }, true, "variance of noise must be greater than or equal to 0"); + RequireParamValue("max_iterations", [](int x) { return x >= 0; }, true, + "max_iterations must be greater than or equal to 0"); + arma::mat dataPoints = std::move(CLI::GetParam("input")); // Do we need to add noise to the dataset? diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 8354d3423c..d125c31121 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -120,6 +120,9 @@ add_executable(mlpack_test main_tests/det_test.cpp main_tests/decision_tree_test.cpp main_tests/decision_stump_test.cpp + main_tests/gmm_generate_test.cpp + main_tests/gmm_probability_test.cpp + main_tests/gmm_train_test.cpp main_tests/kde_test.cpp main_tests/linear_regression_test.cpp main_tests/logistic_regression_test.cpp From 8cb1e8940fc7b16057413f8fe653c4a40ad3ec1c Mon Sep 17 00:00:00 2001 From: Yashwant Date: Sun, 19 May 2019 23:01:21 +0530 Subject: [PATCH 033/143] Add GMM CLI test --- .../tests/main_tests/gmm_generate_test.cpp | 98 +++++ .../tests/main_tests/gmm_probability_test.cpp | 73 ++++ .../tests/main_tests/gmm_train_test.cpp | 370 ++++++++++++++++++ 3 files changed, 541 insertions(+) create mode 100644 src/mlpack/tests/main_tests/gmm_generate_test.cpp create mode 100644 src/mlpack/tests/main_tests/gmm_probability_test.cpp create mode 100644 src/mlpack/tests/main_tests/gmm_train_test.cpp diff --git a/src/mlpack/tests/main_tests/gmm_generate_test.cpp b/src/mlpack/tests/main_tests/gmm_generate_test.cpp new file mode 100644 index 0000000000..d49538b497 --- /dev/null +++ b/src/mlpack/tests/main_tests/gmm_generate_test.cpp @@ -0,0 +1,98 @@ +/** + * @file gmm_generate_test.cpp + * @author Yashwant Singh + * + * Test mlpackMain() of gmm_generate_main.cpp. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#define BINDING_TYPE BINDING_TYPE_TEST +static const std::string testName = "GmmGenerate"; + +#include +#include +#include + +#include "test_helper.hpp" +#include +#include "../test_tools.hpp" + +using namespace mlpack; + +struct GmmGenerateTestFixture +{ + public: + GmmGenerateTestFixture() + { + // Cache in the options for this program. + CLI::RestoreSettings(testName); + } + + ~GmmGenerateTestFixture() + { + // Clear the settings. + CLI::ClearSettings(); + } +}; + +BOOST_FIXTURE_TEST_SUITE(GmmGenerateMainTest, GmmGenerateTestFixture); + +// Checking that Samples must greater than 0. +BOOST_AUTO_TEST_CASE(GmmGenerateSamplesTest) +{ + arma::mat inputData; + if (!data::Load("vc2.csv", inputData)) + BOOST_FAIL("Unable to load train dataset vc2.csv!"); + + GMM gmm(1, 2); + gmm.Train(inputData, 2); + + SetInputParam("input_model", &gmm); + + Log::Fatal.ignoreInput = true; + SetInputParam("samples", 0);// Invalid + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + +// Making sure samples are provided. +BOOST_AUTO_TEST_CASE(GmmGenerateSamples) +{ + arma::mat inputData; + if (!data::Load("vc2.csv", inputData)) + BOOST_FAIL("Unable to load train dataset vc2.csv!"); + + GMM gmm(1, 2); + gmm.Train(inputData, 2); + + SetInputParam("input_model", &gmm); + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + +// Checking dimensionality of output. +BOOST_AUTO_TEST_CASE(GmmGenerateDimensionality) +{ + arma::mat inputData; + if (!data::Load("vc2.csv", inputData)) + BOOST_FAIL("Unable to load train dataset vc2.csv!"); + + GMM gmm(1, 2); + gmm.Train(inputData,2); + SetInputParam("input_model", &gmm); + SetInputParam("samples", (int) 10); + + mlpackMain(); + + arma::mat output = std::move(CLI::GetParam("output")); + + BOOST_REQUIRE_EQUAL(output.n_rows, gmm.Dimensionality()); + BOOST_REQUIRE_EQUAL(output.n_cols, (int) 10); + } + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/main_tests/gmm_probability_test.cpp b/src/mlpack/tests/main_tests/gmm_probability_test.cpp new file mode 100644 index 0000000000..0fb50b1e3d --- /dev/null +++ b/src/mlpack/tests/main_tests/gmm_probability_test.cpp @@ -0,0 +1,73 @@ +/** + * @file gmm_probability_test.cpp + * @author Yashwant Singh + * + * Test mlpackMain() of gmm_probability_main.cpp. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#define BINDING_TYPE BINDING_TYPE_TEST + +static const std::string testName = "GmmProbability"; + +#include +#include +#include + +#include "test_helper.hpp" + +#include + + +using namespace mlpack; + +struct GmmProbabilityTestFixture +{ + public: + GmmProbabilityTestFixture() + { + // Cache in the options for this program. + CLI::RestoreSettings(testName); + } + + ~GmmProbabilityTestFixture() + { + // Clear the settings. + CLI::ClearSettings(); + } +}; + +void ResetGmmProbabilitySetting() +{ + CLI::ClearSettings(); + CLI::RestoreSettings(testName); +} + +BOOST_FIXTURE_TEST_SUITE(GmmProbabilityMainTest, GmmProbabilityTestFixture); + +// Checking the input and output dimensionality. +BOOST_AUTO_TEST_CASE(GmmProbabilityDimensionality) +{ + arma::mat inputData; + if (!data::Load("vc2.csv", inputData)) + BOOST_FAIL("Unable to load train dataset vc2.csv!"); + + GMM gmm(1, 2); + gmm.Train(std::move(inputData), 2); + + arma::mat inputPoints(1, 8, arma::fill::randu); + + SetInputParam("input", std::move(inputPoints)); + SetInputParam("input_model", &gmm); + + mlpackMain(); + + BOOST_REQUIRE_EQUAL(CLI::GetParam("output").n_cols,8); + BOOST_REQUIRE_EQUAL(CLI::GetParam("output").n_rows,1); +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/main_tests/gmm_train_test.cpp b/src/mlpack/tests/main_tests/gmm_train_test.cpp new file mode 100644 index 0000000000..6b4a455269 --- /dev/null +++ b/src/mlpack/tests/main_tests/gmm_train_test.cpp @@ -0,0 +1,370 @@ +/** + * @file gmm_train_test.cpp + * @author Yashwant Singh + * + * Test mlpackMain() of gmm_train_main.cpp. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#define BINDING_TYPE BINDING_TYPE_TEST +static const std::string testName = "GmmTrain"; + +#include +#include +#include "test_helper.hpp" +#include + +#include +#include "../test_tools.hpp" + +using namespace mlpack; + +struct GmmTrainTestFixture +{ +public: + GmmTrainTestFixture() + { + // Cache in the options for this program. + CLI::RestoreSettings(testName); + } + + ~GmmTrainTestFixture() + { + // Clear the settings. + bindings::tests::CleanMemory(); + CLI::ClearSettings(); + } +}; + +void ResetGmmTrainSetting() +{ + CLI::ClearSettings(); + CLI::RestoreSettings(testName); +} + +BOOST_FIXTURE_TEST_SUITE(GmmTrainMainTest, GmmTrainTestFixture); + +// To check if the gaussian is positive or not. +BOOST_AUTO_TEST_CASE(GmmTrainValidGaussianTest) +{ + arma::mat inputData(5, 10, arma::fill::randu); + + SetInputParam("input", std::move(inputData)); + SetInputParam("gaussians", 0);// Invalid + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + +/** + * To check if the number of gaussians in the output model is same as + * that of input gaussian parameter or not. + **/ +BOOST_AUTO_TEST_CASE(GmmTrainOutputModelGaussianTest) +{ + arma::mat inputData(5, 10, arma::fill::randu); + + SetInputParam("input", std::move(inputData)); + SetInputParam("gaussians", (int) 2); + SetInputParam("trials", (int) 2); + + mlpackMain(); + + GMM* gmm = CLI::GetParam("output_model"); + BOOST_REQUIRE_EQUAL(gmm->Gaussians(), (int) 2); +} + +// Max iterations must be positive. +BOOST_AUTO_TEST_CASE(GmmTrainMaxIterationsTest) +{ + arma::mat inputData(5, 10, arma::fill::randu); + + SetInputParam("input", std::move(inputData)); + SetInputParam("gaussians", (int) 2); + SetInputParam("trials", (int) 1); + SetInputParam("max_iterations", (int)-1);// Invalid. + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + +// Ensure that Trials must be greater than 0. +BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) +{ + arma::mat inputData(5, 10, arma::fill::randu); + + SetInputParam("input", std::move(inputData)); + SetInputParam("gaussians", (int) 2); + SetInputParam("trials", (int) 0);// Invalid. + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + +// Checking that percentage is between 0 and 1. +BOOST_AUTO_TEST_CASE(RefinedStartPercentageTest) +{ + arma::mat inputData(5, 10, arma::fill::randu); + + SetInputParam("input", std::move(inputData)); + SetInputParam("refined_start", true); + + Log::Fatal.ignoreInput = true; + SetInputParam("percentage", (double) 2.0);// Invalid + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + + SetInputParam("percentage", (double) -1.0);// Invalid + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + + Log::Fatal.ignoreInput = false; +} + +// Samplings must be positive. +BOOST_AUTO_TEST_CASE(GmmTrainSamplings) +{ + arma::mat inputData(5, 10, arma::fill::randu); + + SetInputParam("input", std::move(inputData)); + SetInputParam("refined_start", true); + SetInputParam("samplings", (int) 0);// Invalid + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + +// Number of gaussians in the model trained from input model. +BOOST_AUTO_TEST_CASE(GmmTrainNumberOfGaussian) +{ + arma::mat inputData(5, 10, arma::fill::randu); + + SetInputParam("input", inputData); + SetInputParam("gaussians", (int) 2); + + mlpackMain(); + + GMM* gmm = CLI::GetParam("output_model"); + + SetInputParam("input_model", gmm); + + CLI::GetSingleton().Parameters()["input"].wasPassed = false; + + SetInputParam("input", std::move(inputData)); + + mlpackMain(); + + GMM* gmm1 = CLI::GetParam("output_model"); + + BOOST_REQUIRE_EQUAL(gmm1->Gaussians(), (int) 2); +} + +// Ensure that Noise affects the final result. +BOOST_AUTO_TEST_CASE(GmmTrainNoisetest) +{ + arma::mat inputData(5, 10, arma::fill::randu); + + SetInputParam("input", inputData); + SetInputParam("gaussians", (int) 2); + SetInputParam("noise", (double) 0.0); + + mlpackMain(); + + GMM* gmm = CLI::GetParam("output_model"); + + CLI::GetSingleton().Parameters()["input"].wasPassed = false; + CLI::GetSingleton().Parameters()["gaussians"].wasPassed = false; + CLI::GetSingleton().Parameters()["noise"].wasPassed = false; + + SetInputParam("input", std::move(inputData)); + SetInputParam("gaussians", (int) 2); + SetInputParam("noise", (double) 1.5); + mlpackMain(); + + GMM* gmm1 = CLI::GetParam("output_model"); + + arma::uvec sortedIndices = sort_index(gmm->Weights()); + + for(size_t k = 0; k < sortedIndices.n_elem; k++) + CheckMatricesNotEqual(gmm->Component(sortedIndices[k]).Covariance(), + gmm1->Component(sortedIndices[k]).Covariance()); + +} + +// Ensure that Percentage affects the final result when refined_start is true. +BOOST_AUTO_TEST_CASE(GmmTrainPercentageTest) +{ + arma::mat inputData(5, 10, arma::fill::randu); + + SetInputParam("input", inputData); + SetInputParam("gaussians", (int) 2); + SetInputParam("refined_start", true); + SetInputParam("percentage", (double) 0.02); + + mlpackMain(); + + GMM* gmm = CLI::GetParam("output_model"); + + CLI::GetSingleton().Parameters()["input"].wasPassed = false; + CLI::GetSingleton().Parameters()["gaussians"].wasPassed = false; + CLI::GetSingleton().Parameters()["refined_start"].wasPassed = false; + CLI::GetSingleton().Parameters()["percentage"].wasPassed = false; + + SetInputParam("input", std::move(inputData)); + SetInputParam("gaussians", (int) 2); + SetInputParam("refined_start", true); + SetInputParam("percentage", (double) 0.52); + + mlpackMain(); + + GMM* gmm1 = CLI::GetParam("output_model"); + + arma::uvec sortedIndices = sort_index(gmm->Weights()); + + for(size_t k = 0; k < sortedIndices.n_elem; k++) + CheckMatricesNotEqual(gmm->Component(sortedIndices[k]).Covariance(), + gmm1->Component(sortedIndices[k]).Covariance()); +} + +// Ensure that Sampling affects the final result when refined_start is true. +BOOST_AUTO_TEST_CASE(GmmTrainSamplingsTest) +{ + arma::mat inputData(5, 10, arma::fill::randu); + + SetInputParam("input", inputData); + SetInputParam("gaussians", (int) 2); + SetInputParam("refined_start", true); + SetInputParam("percentage", (double) 0.5); + SetInputParam("samplings", (int) 100); + + mlpackMain(); + + GMM* gmm = CLI::GetParam("output_model"); + + CLI::GetSingleton().Parameters()["input"].wasPassed = false; + CLI::GetSingleton().Parameters()["gaussians"].wasPassed = false; + CLI::GetSingleton().Parameters()["refined_start"].wasPassed = false; + CLI::GetSingleton().Parameters()["percentage"].wasPassed = false; + CLI::GetSingleton().Parameters()["samplings"].wasPassed = false; + + SetInputParam("input", std::move(inputData)); + SetInputParam("gaussians", (int) 2); + SetInputParam("refined_start", true); + SetInputParam("percentage", (double) 0.5); + SetInputParam("samplings", (int) 500); + + mlpackMain(); + + GMM* gmm1 = CLI::GetParam("output_model"); + + arma::uvec sortedIndices = sort_index(gmm->Weights()); + + for(size_t k = 0; k < sortedIndices.n_elem; k++) + CheckMatricesNotEqual(gmm->Component(sortedIndices[k]).Covariance(), + gmm1->Component(sortedIndices[k]).Covariance()); + +} + +// Ensure that tolerance affects the final result. +BOOST_AUTO_TEST_CASE(GmmTrainToleranceTest) +{ + arma::mat inputData; + if (!data::Load("vc2.csv", inputData)) + BOOST_FAIL("Unable to load train dataset vc2.csv!"); + + SetInputParam("input", inputData); + SetInputParam("gaussians", (int) 2); + SetInputParam("tolerance", (double) 1e-10); + + mlpackMain(); + + GMM* gmm = CLI::GetParam("output_model"); + + CLI::GetSingleton().Parameters()["input"].wasPassed = false; + CLI::GetSingleton().Parameters()["gaussians"].wasPassed = false; + CLI::GetSingleton().Parameters()["tolerance"].wasPassed = false; + + SetInputParam("input", std::move(inputData)); + SetInputParam("gaussians", (int) 2); + SetInputParam("tolerance", (double) 1e-30); + + mlpackMain(); + + GMM* gmm1 = CLI::GetParam("output_model"); + + arma::uvec sortedIndices = sort_index(gmm->Weights()); + + for(size_t k = 0; k < sortedIndices.n_elem; k++) + CheckMatricesNotEqual(gmm->Component(sortedIndices[k]).Covariance(), + gmm1->Component(sortedIndices[k]).Covariance()); +} + +// Ensure that saved model can be used again. +BOOST_AUTO_TEST_CASE(GmmTrainModelReuseTest) +{ + arma::mat inputData(5, 10, arma::fill::randu); + + SetInputParam("input", inputData); + SetInputParam("gaussians", (int) 2); + + mlpackMain(); + + GMM* gmm = CLI::GetParam("output_model"); + + SetInputParam("input_model", gmm); + + CLI::GetSingleton().Parameters()["input"].wasPassed = false; + + SetInputParam("input", inputData); + + mlpackMain(); + + GMM* gmm1 = CLI::GetParam("output_model"); + + SetInputParam("input_model", gmm1); + + CLI::GetSingleton().Parameters()["input"].wasPassed = false; + + SetInputParam("input", std::move(inputData)); + + mlpackMain(); + + GMM* gmm2 = CLI::GetParam("output_model"); + + BOOST_REQUIRE_EQUAL(gmm1, gmm2); +} + +// Ensure that Gmm's covariances are diagonal when diagonal_covariance is true. +BOOST_AUTO_TEST_CASE(GmmTrainDiagCovariance) +{ + arma::mat inputData(5, 10, arma::fill::randu); + + SetInputParam("input", std::move(inputData)); + SetInputParam("gaussians", (int) 2); + SetInputParam("diagonal_covariance", true); + + mlpackMain(); + + GMM* gmm = CLI::GetParam("output_model"); + + arma::uvec sortedIndices = sort_index(gmm->Weights()); + + for(size_t k = 0; k < sortedIndices.n_elem; k++) + { + arma::mat diagCov(gmm->Component(sortedIndices[k]).Covariance()); + for(size_t i = 0; i < diagCov.n_rows; i++) + for(size_t j = 0; j < diagCov.n_cols; j++) + if (i != j && diagCov(i, j) != (double) 0) + BOOST_FAIL("Covariance Are Not Diagonal"); + } +} + +BOOST_AUTO_TEST_SUITE_END(); + From 45ee51470895d48806929d9a96ce7085426d8f46 Mon Sep 17 00:00:00 2001 From: Yashwant Date: Sun, 19 May 2019 23:27:45 +0530 Subject: [PATCH 034/143] Fix Style Checks --- .../tests/main_tests/gmm_generate_test.cpp | 34 ++++---- .../tests/main_tests/gmm_probability_test.cpp | 28 +++---- .../tests/main_tests/gmm_train_test.cpp | 81 +++++++++---------- 3 files changed, 71 insertions(+), 72 deletions(-) diff --git a/src/mlpack/tests/main_tests/gmm_generate_test.cpp b/src/mlpack/tests/main_tests/gmm_generate_test.cpp index d49538b497..9139549857 100644 --- a/src/mlpack/tests/main_tests/gmm_generate_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_generate_test.cpp @@ -25,17 +25,17 @@ using namespace mlpack; struct GmmGenerateTestFixture { public: - GmmGenerateTestFixture() - { - // Cache in the options for this program. - CLI::RestoreSettings(testName); - } + GmmGenerateTestFixture() + { + // Cache in the options for this program. + CLI::RestoreSettings(testName); + } - ~GmmGenerateTestFixture() - { - // Clear the settings. - CLI::ClearSettings(); - } + ~GmmGenerateTestFixture() + { + // Clear the settings. + CLI::ClearSettings(); + } }; BOOST_FIXTURE_TEST_SUITE(GmmGenerateMainTest, GmmGenerateTestFixture); @@ -46,14 +46,14 @@ BOOST_AUTO_TEST_CASE(GmmGenerateSamplesTest) arma::mat inputData; if (!data::Load("vc2.csv", inputData)) BOOST_FAIL("Unable to load train dataset vc2.csv!"); - + GMM gmm(1, 2); gmm.Train(inputData, 2); SetInputParam("input_model", &gmm); Log::Fatal.ignoreInput = true; - SetInputParam("samples", 0);// Invalid + SetInputParam("samples", 0); // Invalid BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); Log::Fatal.ignoreInput = false; } @@ -64,7 +64,7 @@ BOOST_AUTO_TEST_CASE(GmmGenerateSamples) arma::mat inputData; if (!data::Load("vc2.csv", inputData)) BOOST_FAIL("Unable to load train dataset vc2.csv!"); - + GMM gmm(1, 2); gmm.Train(inputData, 2); @@ -77,13 +77,13 @@ BOOST_AUTO_TEST_CASE(GmmGenerateSamples) // Checking dimensionality of output. BOOST_AUTO_TEST_CASE(GmmGenerateDimensionality) -{ +{ arma::mat inputData; if (!data::Load("vc2.csv", inputData)) BOOST_FAIL("Unable to load train dataset vc2.csv!"); - + GMM gmm(1, 2); - gmm.Train(inputData,2); + gmm.Train(inputData, 2); SetInputParam("input_model", &gmm); SetInputParam("samples", (int) 10); @@ -93,6 +93,6 @@ BOOST_AUTO_TEST_CASE(GmmGenerateDimensionality) BOOST_REQUIRE_EQUAL(output.n_rows, gmm.Dimensionality()); BOOST_REQUIRE_EQUAL(output.n_cols, (int) 10); - } +} BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/main_tests/gmm_probability_test.cpp b/src/mlpack/tests/main_tests/gmm_probability_test.cpp index 0fb50b1e3d..4219a372cf 100644 --- a/src/mlpack/tests/main_tests/gmm_probability_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_probability_test.cpp @@ -9,7 +9,7 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ - + #define BINDING_TYPE BINDING_TYPE_TEST static const std::string testName = "GmmProbability"; @@ -28,17 +28,17 @@ using namespace mlpack; struct GmmProbabilityTestFixture { public: - GmmProbabilityTestFixture() - { - // Cache in the options for this program. - CLI::RestoreSettings(testName); - } + GmmProbabilityTestFixture() + { + // Cache in the options for this program. + CLI::RestoreSettings(testName); + } - ~GmmProbabilityTestFixture() - { - // Clear the settings. - CLI::ClearSettings(); - } + ~GmmProbabilityTestFixture() + { + // Clear the settings. + CLI::ClearSettings(); + } }; void ResetGmmProbabilitySetting() @@ -55,7 +55,7 @@ BOOST_AUTO_TEST_CASE(GmmProbabilityDimensionality) arma::mat inputData; if (!data::Load("vc2.csv", inputData)) BOOST_FAIL("Unable to load train dataset vc2.csv!"); - + GMM gmm(1, 2); gmm.Train(std::move(inputData), 2); @@ -66,8 +66,8 @@ BOOST_AUTO_TEST_CASE(GmmProbabilityDimensionality) mlpackMain(); - BOOST_REQUIRE_EQUAL(CLI::GetParam("output").n_cols,8); - BOOST_REQUIRE_EQUAL(CLI::GetParam("output").n_rows,1); + BOOST_REQUIRE_EQUAL(CLI::GetParam("output").n_cols, 8); + BOOST_REQUIRE_EQUAL(CLI::GetParam("output").n_rows, 1); } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/main_tests/gmm_train_test.cpp b/src/mlpack/tests/main_tests/gmm_train_test.cpp index 6b4a455269..93049763ba 100644 --- a/src/mlpack/tests/main_tests/gmm_train_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_train_test.cpp @@ -26,19 +26,19 @@ using namespace mlpack; struct GmmTrainTestFixture { -public: - GmmTrainTestFixture() - { - // Cache in the options for this program. - CLI::RestoreSettings(testName); - } + public: + GmmTrainTestFixture() + { + // Cache in the options for this program. + CLI::RestoreSettings(testName); + } - ~GmmTrainTestFixture() - { - // Clear the settings. - bindings::tests::CleanMemory(); - CLI::ClearSettings(); - } + ~GmmTrainTestFixture() + { + // Clear the settings. + bindings::tests::CleanMemory(); + CLI::ClearSettings(); + } }; void ResetGmmTrainSetting() @@ -55,7 +55,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainValidGaussianTest) arma::mat inputData(5, 10, arma::fill::randu); SetInputParam("input", std::move(inputData)); - SetInputParam("gaussians", 0);// Invalid + SetInputParam("gaussians", 0); // Invalid Log::Fatal.ignoreInput = true; BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); @@ -73,7 +73,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainOutputModelGaussianTest) SetInputParam("input", std::move(inputData)); SetInputParam("gaussians", (int) 2); SetInputParam("trials", (int) 2); - + mlpackMain(); GMM* gmm = CLI::GetParam("output_model"); @@ -88,7 +88,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainMaxIterationsTest) SetInputParam("input", std::move(inputData)); SetInputParam("gaussians", (int) 2); SetInputParam("trials", (int) 1); - SetInputParam("max_iterations", (int)-1);// Invalid. + SetInputParam("max_iterations", (int)-1); // Invalid. Log::Fatal.ignoreInput = true; BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); @@ -102,8 +102,8 @@ BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) SetInputParam("input", std::move(inputData)); SetInputParam("gaussians", (int) 2); - SetInputParam("trials", (int) 0);// Invalid. - + SetInputParam("trials", (int) 0); // Invalid. + Log::Fatal.ignoreInput = true; BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); Log::Fatal.ignoreInput = false; @@ -118,10 +118,10 @@ BOOST_AUTO_TEST_CASE(RefinedStartPercentageTest) SetInputParam("refined_start", true); Log::Fatal.ignoreInput = true; - SetInputParam("percentage", (double) 2.0);// Invalid + SetInputParam("percentage", (double) 2.0); // Invalid BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); - SetInputParam("percentage", (double) -1.0);// Invalid + SetInputParam("percentage", (double) -1.0); // Invalid BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); Log::Fatal.ignoreInput = false; @@ -134,7 +134,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainSamplings) SetInputParam("input", std::move(inputData)); SetInputParam("refined_start", true); - SetInputParam("samplings", (int) 0);// Invalid + SetInputParam("samplings", (int) 0); // Invalid Log::Fatal.ignoreInput = true; BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); @@ -150,7 +150,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainNumberOfGaussian) SetInputParam("gaussians", (int) 2); mlpackMain(); - + GMM* gmm = CLI::GetParam("output_model"); SetInputParam("input_model", gmm); @@ -176,13 +176,13 @@ BOOST_AUTO_TEST_CASE(GmmTrainNoisetest) SetInputParam("noise", (double) 0.0); mlpackMain(); - + GMM* gmm = CLI::GetParam("output_model"); CLI::GetSingleton().Parameters()["input"].wasPassed = false; CLI::GetSingleton().Parameters()["gaussians"].wasPassed = false; CLI::GetSingleton().Parameters()["noise"].wasPassed = false; - + SetInputParam("input", std::move(inputData)); SetInputParam("gaussians", (int) 2); SetInputParam("noise", (double) 1.5); @@ -191,11 +191,10 @@ BOOST_AUTO_TEST_CASE(GmmTrainNoisetest) GMM* gmm1 = CLI::GetParam("output_model"); arma::uvec sortedIndices = sort_index(gmm->Weights()); - - for(size_t k = 0; k < sortedIndices.n_elem; k++) + + for (size_t k = 0; k < sortedIndices.n_elem; k++) CheckMatricesNotEqual(gmm->Component(sortedIndices[k]).Covariance(), gmm1->Component(sortedIndices[k]).Covariance()); - } // Ensure that Percentage affects the final result when refined_start is true. @@ -209,7 +208,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainPercentageTest) SetInputParam("percentage", (double) 0.02); mlpackMain(); - + GMM* gmm = CLI::GetParam("output_model"); CLI::GetSingleton().Parameters()["input"].wasPassed = false; @@ -227,8 +226,8 @@ BOOST_AUTO_TEST_CASE(GmmTrainPercentageTest) GMM* gmm1 = CLI::GetParam("output_model"); arma::uvec sortedIndices = sort_index(gmm->Weights()); - - for(size_t k = 0; k < sortedIndices.n_elem; k++) + + for (size_t k = 0; k < sortedIndices.n_elem; k++) CheckMatricesNotEqual(gmm->Component(sortedIndices[k]).Covariance(), gmm1->Component(sortedIndices[k]).Covariance()); } @@ -245,7 +244,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainSamplingsTest) SetInputParam("samplings", (int) 100); mlpackMain(); - + GMM* gmm = CLI::GetParam("output_model"); CLI::GetSingleton().Parameters()["input"].wasPassed = false; @@ -265,8 +264,8 @@ BOOST_AUTO_TEST_CASE(GmmTrainSamplingsTest) GMM* gmm1 = CLI::GetParam("output_model"); arma::uvec sortedIndices = sort_index(gmm->Weights()); - - for(size_t k = 0; k < sortedIndices.n_elem; k++) + + for (size_t k = 0; k < sortedIndices.n_elem; k++) CheckMatricesNotEqual(gmm->Component(sortedIndices[k]).Covariance(), gmm1->Component(sortedIndices[k]).Covariance()); @@ -284,7 +283,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainToleranceTest) SetInputParam("tolerance", (double) 1e-10); mlpackMain(); - + GMM* gmm = CLI::GetParam("output_model"); CLI::GetSingleton().Parameters()["input"].wasPassed = false; @@ -300,8 +299,8 @@ BOOST_AUTO_TEST_CASE(GmmTrainToleranceTest) GMM* gmm1 = CLI::GetParam("output_model"); arma::uvec sortedIndices = sort_index(gmm->Weights()); - - for(size_t k = 0; k < sortedIndices.n_elem; k++) + + for (size_t k = 0; k < sortedIndices.n_elem; k++) CheckMatricesNotEqual(gmm->Component(sortedIndices[k]).Covariance(), gmm1->Component(sortedIndices[k]).Covariance()); } @@ -315,7 +314,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainModelReuseTest) SetInputParam("gaussians", (int) 2); mlpackMain(); - + GMM* gmm = CLI::GetParam("output_model"); SetInputParam("input_model", gmm); @@ -327,7 +326,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainModelReuseTest) mlpackMain(); GMM* gmm1 = CLI::GetParam("output_model"); - + SetInputParam("input_model", gmm1); CLI::GetSingleton().Parameters()["input"].wasPassed = false; @@ -351,16 +350,16 @@ BOOST_AUTO_TEST_CASE(GmmTrainDiagCovariance) SetInputParam("diagonal_covariance", true); mlpackMain(); - + GMM* gmm = CLI::GetParam("output_model"); arma::uvec sortedIndices = sort_index(gmm->Weights()); - for(size_t k = 0; k < sortedIndices.n_elem; k++) + for (size_t k = 0; k < sortedIndices.n_elem; k++) { arma::mat diagCov(gmm->Component(sortedIndices[k]).Covariance()); - for(size_t i = 0; i < diagCov.n_rows; i++) - for(size_t j = 0; j < diagCov.n_cols; j++) + for (size_t i = 0; i < diagCov.n_rows; i++) + for (size_t j = 0; j < diagCov.n_cols; j++) if (i != j && diagCov(i, j) != (double) 0) BOOST_FAIL("Covariance Are Not Diagonal"); } From 9820faab532316a552a21173c2aa1f91e894aa34 Mon Sep 17 00:00:00 2001 From: robotcator Date: Tue, 21 May 2019 17:26:31 +0800 Subject: [PATCH 035/143] fix the code review --- .../q_learning_impl.hpp | 19 ++++++------------- .../replay/prioritized_replay.hpp | 17 ++++++++++++----- .../replay/random_replay.hpp | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 35365c0298..eadacd832c 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -174,8 +174,6 @@ double QLearning< arma::colvec sampledRewards; arma::mat sampledNextStates; arma::icolvec isTerminal; - arma::ucolvec sampledIndices; - arma::rowvec weights; if (!prioritized_replay) { @@ -185,8 +183,7 @@ double QLearning< else { prioritizedReplayMethod.Sample(sampledStates, sampledActions, - sampledRewards, sampledNextStates, isTerminal, - sampledIndices, weights); + sampledRewards, sampledNextStates, isTerminal); } // Compute action value for next state with target network. @@ -227,23 +224,19 @@ double QLearning< arma::mat gradients; learningNetwork.Backward(target, gradients); - if (prioritized_replay) - { - gradients = arma::mean(weights) * gradients; - } - - updater.Update(learningNetwork.Parameters(), config.StepSize(), gradients); - if (prioritized_replay) { prioritizedReplayMethod.Update(target, sampledActions, - nextActionValues, sampledIndices); + nextActionValues, gradients); } else { replayMethod.Update(target, sampledActions, - nextActionValues, sampledIndices); + nextActionValues, gradients); } + + updater.Update(learningNetwork.Parameters(), config.StepSize(), gradients); + return reward; } diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index 4c4699e932..dd2e974573 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -152,9 +152,7 @@ class PrioritizedReplay arma::icolvec& sampledActions, arma::colvec& sampledRewards, arma::mat& sampledNextStates, - arma::icolvec& isTerminal, - arma::ucolvec& sampledIndices, - arma::rowvec& weights) + arma::icolvec& isTerminal) { size_t upperBound = full ? capacity : position; @@ -212,7 +210,7 @@ class PrioritizedReplay } void Update(arma::mat target, arma::icolvec sampledActions, - arma::mat nextActionValues, arma::ucolvec sampledIndices) + arma::mat nextActionValues, arma::mat& gradients) { arma::colvec td_error(target.n_cols); for (size_t i = 0; i < target.n_cols; i ++) @@ -222,6 +220,9 @@ class PrioritizedReplay } td_error = arma::abs(td_error); UpdatePriorities(sampledIndices, td_error); + + // Update the gradient + gradients = arma::mean(weights) * gradients; } @@ -268,8 +269,14 @@ class PrioritizedReplay //! Locally-stored termination information of previous experience. arma::icolvec isTerminal; - //! Locally-stored indicator that whether the memory is full or not + //! Locally-stored indicator that whether the memory is full or not. bool full; + + //! Locally-stored the indices of sampled transitions. + arma::ucolvec sampledIndices; + + //! Locally-stored the weights of sampled transitions. + arma::rowvec weights; }; } // namespace rl diff --git a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp index 588cef36b0..79a0ee71cb 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp @@ -143,7 +143,7 @@ class RandomReplay } void Update(arma::mat target, arma::icolvec sampledActions, - arma::mat nextActionValues, arma::ucolvec sampledIndices) + arma::mat nextActionValues, arma::mat& gradients) { /* do nothing for random replay*/ } From 7e3ad76b80b81dfa05e3907734d2030f0004156c Mon Sep 17 00:00:00 2001 From: robotcator Date: Wed, 22 May 2019 10:45:05 +0800 Subject: [PATCH 036/143] add desc for parameters and update the naming --- .../replay/prioritized_replay.hpp | 22 ++++++++++++++----- .../replay/random_replay.hpp | 14 ++++++++++-- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index dd2e974573..f49e6b582c 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -209,17 +209,27 @@ class PrioritizedReplay beta = beta + (1 - initialBeta) * 1.0 / replayBetaIters; } - void Update(arma::mat target, arma::icolvec sampledActions, - arma::mat nextActionValues, arma::mat& gradients) + /** + * Update the priorities of transitions and Update the gradients. + * + * @param target The learned value + * @param sampledActions Agent's sampled action + * @param nextActionValues Agent's next action + * @param gradients The model's gradients + */ + void Update(arma::mat target, + arma::icolvec sampledActions, + arma::mat nextActionValues, + arma::mat& gradients) { - arma::colvec td_error(target.n_cols); + arma::colvec tdError(target.n_cols); for (size_t i = 0; i < target.n_cols; i ++) { - td_error[i] = nextActionValues(sampledActions[i], i) - + tdError[i] = nextActionValues(sampledActions[i], i) - target(sampledActions[i], i); } - td_error = arma::abs(td_error); - UpdatePriorities(sampledIndices, td_error); + tdError = arma::abs(tdError); + UpdatePriorities(sampledIndices, tdError); // Update the gradient gradients = arma::mean(weights) * gradients; diff --git a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp index 79a0ee71cb..cc07701cb9 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp @@ -142,8 +142,18 @@ class RandomReplay return full ? capacity : position; } - void Update(arma::mat target, arma::icolvec sampledActions, - arma::mat nextActionValues, arma::mat& gradients) + /** + * Update the priorities of transitions and Update the gradients. + * + * @param target The learned value + * @param sampledActions Agent's sampled action + * @param nextActionValues Agent's next action + * @param gradients The model's gradients + */ + void Update(arma::mat target, + arma::icolvec sampledActions, + arma::mat nextActionValues, + arma::mat& gradients) { /* do nothing for random replay*/ } From 39dad8e8a174b323e1698aa801f0adb09da6c550 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Thu, 23 May 2019 17:04:30 +0530 Subject: [PATCH 037/143] Added task and tests. --- .../environment/multiple_pole_cart.hpp | 272 ++++++++++++++++++ src/mlpack/tests/rl_components_test.cpp | 20 ++ 2 files changed, 292 insertions(+) create mode 100755 src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp diff --git a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp new file mode 100755 index 0000000000..be08e152be --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp @@ -0,0 +1,272 @@ +/** + * @file multiple_pole_cart.hpp + * @author Rahul Ganesh Prabhu + * + * This file is an implementation of Multiple Pole Cart Balancing Task + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_RL_ENVIRONMENT_MULTIPE_POLE_CART_HPP +#define MLPACK_METHODS_RL_ENVIRONMENT_MULTIPE_POLE_CART_HPP + +#include + +namespace mlpack { +namespace rl { + +/** + * Implementation of Multiple Pole Cart Balancing task. + */ +class MultiplePoleCart +{ + public: + /** + * Implementation of the state of Multiple Pole Cart. Each state is a tuple vector + * (position, velocity, angle, angular velocity). + */ + class State + { + public: + /** + * Construct a state instance. + */ + State(size_t numPoles) + { + data = arma::mat(dimension, numPoles, arma::fill::zeros); + } + + /** + * Construct a state instance from given data. + * + * @param data Data for the position, velocity, angle and angular velocity. + */ + State(const arma::mat& data) : data(data) + { /* Nothing to do here */ } + + //! Modify the internal representation of the state. + arma::mat& Data() { return data; } + + //! Get the position of the cart. + double Position() const { return data(0, 0); } + //! Modify the position of the cart. + double& Position() { return data(0, 0); } + + //! Get the velocity of the cart. + double Velocity() const { return data(1, 0); } + //! Modify the velocity of the cart. + double& Velocity() { return data(1, 0); } + + //! Get the angle of the $i^{th}$ pole with the vertical. + double Angle(size_t i) const { return data(0, i); } + //! Modify the angle of the $i^{th}$ pole with the vertical. + double& Angle(size_t i) { return data(0, i); } + + //! Get the angular velocity of the $i^{th}$ pole. + double AngularVelocity(size_t i) const { return data(1, i); } + //! Modify the angular velocity of the $i^{th}$ pole. + double& AngularVelocity(size_t i) { return data(1, i); } + + //! Get the state of the cart. + arma::colvec CartState() const { return data.col(0); } + + //! Get the state of the $i^{th}$ pole. + arma::colvec PoleState(size_t i) const { return data.col(i); } + + //! Encode the state to a matrix. + const arma::mat& Encode() const { return data; } + + //! Dimension of the encoded state. + size_t dimension = 2; + + private: + //! Locally-stored (position, velocity, angle, angular velocity). + arma::mat data; + }; + + /** + * Implementation of action of Cart Pole. + */ + enum Action + { + backward, + forward, + + // Track the size of the action space. + size + }; + + /** + * Construct a Multiple Pole Cart instance using the given constants. + * + * @param poleNum The number of poles + * @param gravity The gravity constant. + * @param massCart The mass of the cart. + * @param massPole The mass of the pole. + * @param length The length of the pole. + * @param forceMag The magnitude of the applied force. + * @param tau The time interval. + * @param thetaThresholdRadians The maximum angle. + * @param xThreshold The maximum position. + */ + MultiplePoleCart(const size_t poleNum, + const arma::vec poleLengths, + const arma::vec poleMasses, + const double gravity = 9.8, + const double massCart = 1.0, + const double forceMag = 10.0, + const double tau = 0.02, + const double thetaThresholdRadians = 12 * 2 * 3.1416 / 360, + const double xThreshold = 2.4, + const double doneReward = 0.0) : + poleNum(poleNum), + poleLengths(poleLengths), + poleMasses(poleMasses), + gravity(gravity), + massCart(massCart), + forceMag(forceMag), + tau(tau), + thetaThresholdRadians(thetaThresholdRadians), + xThreshold(xThreshold), + doneReward(doneReward) + { + if (poleNum != poleLengths.n_elem) + Log::Fatal << "The number of lengths should be the same as the number of poles." << std::endl; + if (poleNum != poleMasses.n_elem) + Log::Fatal << "The number of masses should be the same as the number of poles." << std::endl; + } + + /** + * Dynamics of Multiple Pole Cart instance. Get reward and next state based on current + * state and current action. + * + * @param state The current state. + * @param action The current action. + * @param nextState The next state. + * @return reward, it's always 1.0. + */ + double Sample(const State& state, + const Action& action, + State& nextState) const + { + // Calculate acceleration. + double totalForce = action ? forceMag : -forceMag; + double totalMass = massCart; + for( size_t i = 0; i < poleNum; i++) + { + double poleOmega = state.AngularVelocity(i); + double sinTheta = sin(state.Angle(i)); + double cosTheta = cos(state.Angle(i)); + totalForce += (poleMasses[i] * poleLengths[i] * poleOmega * poleOmega * sinTheta) + + 0.75 * poleMasses[i] * cosTheta * gravity * sinTheta; + totalMass += poleMasses[i] * (1 - 0.75 * cosTheta * cosTheta); + } + double xAcc = totalForce / totalMass; + + // Update states of the poles. + for( size_t i = 0; i < poleNum; i++) + { + double sinTheta = sin(state.Angle(i)); + double cosTheta = cos(state.Angle(i)); + nextState.Angle(i) = state.Angle(i) + tau * state.AngularVelocity(i); + nextState.AngularVelocity(i) += -tau * 0.75 * (xAcc * cosTheta + + gravity * sinTheta); + } + + // Update state of the cart. + nextState.Position() = state.Position() + tau * state.Velocity(); + nextState.Velocity() = state.Velocity() + tau * xAcc; + + /** + * It is important to note that if the cartpole is falling down, it should + * be penalized. + */ + bool done = IsTerminal(nextState); + if (done) + return doneReward; + /** + * When done is false, it means that the cartpole has fallen down. + * For this case the reward is 1.0. + */ + return 1.0; + } + + /** + * Dynamics of Cart Pole. Get reward based on current state and current + * action. + * + * @param state The current state. + * @param action The current action. + * @return reward, it's always 1.0. + */ + double Sample(const State& state, const Action& action) const + { + State nextState(poleNum); + return Sample(state, action, nextState); + } + + /** + * Initial state representation is randomly generated within [-0.05, 0.05]. + * + * @return Initial state for each episode. + */ + State InitialSample() const + { + return State((arma::randu(2, poleNum) - 0.5) / 10.0); + } + + /** + * Whether given state is a terminal state. + * + * @param state The desired state. + * @return true if state is a terminal state, otherwise false. + */ + bool IsTerminal(const State& state) const + { + for (size_t i = 0; i < poleNum; i++) + if (std::abs(state.Angle(i)) > thetaThresholdRadians) + return true; + return std::abs(state.Position()) > xThreshold; + } + + private: + //! Locally-stored number of poles. + size_t poleNum; + + //! Locally-stored length of poles. + arma::vec poleLengths; + + //! Locally-stored gravity. + double gravity; + + //! Locally-stored mass of the cart. + double massCart; + + //! Locally-stored mass of the pole. + arma::vec poleMasses; + + //! Locally-stored length of the pole. + double length; + + //! Locally-stored magnitude of the applied force. + double forceMag; + + //! Locally-stored time interval. + double tau; + + //! Locally-stored maximum angle. + double thetaThresholdRadians; + + //! Locally-stored maximum position. + double xThreshold; + + //! Locally-stored done reward. + double doneReward; +}; + +} // namespace rl +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 69a7578ec6..7c4ad75876 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -118,6 +119,25 @@ BOOST_AUTO_TEST_CASE(SimpleCartPoleTest) BOOST_REQUIRE_EQUAL(2, CartPole::Action::size); } +/** + * Constructs a MultiplePoleCart instance and check if the main rountine works as + * it should be. + */ +BOOST_AUTO_TEST_CASE(MultiplePoleCartTest) +{ + arma::vec poleLengths = {1,0.5}; + arma::vec poleMasses = {1,1}; + const MultiplePoleCart task = MultiplePoleCart(2, poleLengths, poleMasses); + + MultiplePoleCart::State state = task.InitialSample(); + MultiplePoleCart::Action action = MultiplePoleCart::Action::backward; + double reward = task.Sample(state, action); + + BOOST_REQUIRE_EQUAL(reward, 1.0); + BOOST_REQUIRE(!task.IsTerminal(state)); + BOOST_REQUIRE_EQUAL(2, MultiplePoleCart::Action::size); +} + /** * Construct a random replay instance and check if it works as * it should be. From 030d0cf45b22c6e19fcf491dad33598b2f5d26a9 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Thu, 23 May 2019 18:06:02 +0530 Subject: [PATCH 038/143] Some changes + documentation. --- .../environment/multiple_pole_cart.hpp | 20 +++++++++---------- src/mlpack/tests/rl_components_test.cpp | 4 ++-- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp index be08e152be..ac58bebf48 100755 --- a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp @@ -2,7 +2,7 @@ * @file multiple_pole_cart.hpp * @author Rahul Ganesh Prabhu * - * This file is an implementation of Multiple Pole Cart Balancing Task + * This file is an implementation of Multiple Pole Cart Balancing Task. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -24,14 +24,18 @@ class MultiplePoleCart { public: /** - * Implementation of the state of Multiple Pole Cart. Each state is a tuple vector - * (position, velocity, angle, angular velocity). + * Implementation of the state of Multiple Pole Cart. The state is expressed as + * a matrix where the $0^{th}$ column is the state of the cart, represented by a tuple + * (position, velocity) and the $i^{th}$ column is the state of the $i^{th}$ pole, represented + * by a tuple (angle, angular velocity). */ class State { public: /** * Construct a state instance. + * + * @param numPoles The number of poles. */ State(size_t numPoles) { @@ -69,20 +73,14 @@ class MultiplePoleCart //! Modify the angular velocity of the $i^{th}$ pole. double& AngularVelocity(size_t i) { return data(1, i); } - //! Get the state of the cart. - arma::colvec CartState() const { return data.col(0); } - - //! Get the state of the $i^{th}$ pole. - arma::colvec PoleState(size_t i) const { return data.col(i); } - //! Encode the state to a matrix. const arma::mat& Encode() const { return data; } //! Dimension of the encoded state. - size_t dimension = 2; + const size_t dimension = 2; private: - //! Locally-stored (position, velocity, angle, angular velocity). + //! Locally-stored state data. arma::mat data; }; diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 7c4ad75876..87e4edb6f3 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -103,7 +103,7 @@ BOOST_AUTO_TEST_CASE(SimpleMountainCarTest) } /** - * Constructs a CartPole instance and check if the main rountine works as + * Constructs a CartPole instance and check if the main routine works as * it should be. */ BOOST_AUTO_TEST_CASE(SimpleCartPoleTest) @@ -120,7 +120,7 @@ BOOST_AUTO_TEST_CASE(SimpleCartPoleTest) } /** - * Constructs a MultiplePoleCart instance and check if the main rountine works as + * Constructs a MultiplePoleCart instance and check if the main routine works as * it should be. */ BOOST_AUTO_TEST_CASE(MultiplePoleCartTest) From 63d7f7b1d4436ebea24a0a93e4e2dfafe4e610f5 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Thu, 23 May 2019 18:12:47 +0530 Subject: [PATCH 039/143] Style fixes. --- .../environment/multiple_pole_cart.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp index ac58bebf48..8368913c20 100755 --- a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp @@ -152,7 +152,7 @@ class MultiplePoleCart // Calculate acceleration. double totalForce = action ? forceMag : -forceMag; double totalMass = massCart; - for( size_t i = 0; i < poleNum; i++) + for (size_t i = 0; i < poleNum; i++) { double poleOmega = state.AngularVelocity(i); double sinTheta = sin(state.Angle(i)); @@ -164,7 +164,7 @@ class MultiplePoleCart double xAcc = totalForce / totalMass; // Update states of the poles. - for( size_t i = 0; i < poleNum; i++) + for (size_t i = 0; i < poleNum; i++) { double sinTheta = sin(state.Angle(i)); double cosTheta = cos(state.Angle(i)); @@ -192,7 +192,7 @@ class MultiplePoleCart } /** - * Dynamics of Cart Pole. Get reward based on current state and current + * Dynamics of Multiple Pole Cart. Get reward based on current state and current * action. * * @param state The current state. From 78b2feea20648081b69f43776866f0124569b760 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Thu, 23 May 2019 18:15:38 +0530 Subject: [PATCH 040/143] Fixed angular velocity equation. --- .../reinforcement_learning/environment/multiple_pole_cart.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp index 8368913c20..3f7ae8175c 100755 --- a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp @@ -170,7 +170,7 @@ class MultiplePoleCart double cosTheta = cos(state.Angle(i)); nextState.Angle(i) = state.Angle(i) + tau * state.AngularVelocity(i); nextState.AngularVelocity(i) += -tau * 0.75 * (xAcc * cosTheta + - gravity * sinTheta); + gravity * sinTheta) / poleLengths[i]; } // Update state of the cart. From edb96581dea788caa89db70af538340d34b857a9 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Thu, 23 May 2019 18:22:02 +0530 Subject: [PATCH 041/143] Style fixes. --- .../environment/multiple_pole_cart.hpp | 16 +++++++++------- src/mlpack/tests/rl_components_test.cpp | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp index 3f7ae8175c..c727837459 100755 --- a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp @@ -38,7 +38,7 @@ class MultiplePoleCart * @param numPoles The number of poles. */ State(size_t numPoles) - { + { data = arma::mat(dimension, numPoles, arma::fill::zeros); } @@ -129,11 +129,13 @@ class MultiplePoleCart thetaThresholdRadians(thetaThresholdRadians), xThreshold(xThreshold), doneReward(doneReward) - { + { if (poleNum != poleLengths.n_elem) - Log::Fatal << "The number of lengths should be the same as the number of poles." << std::endl; + Log::Fatal << "The number of lengths should be the same as the number of poles." + << std::endl; if (poleNum != poleMasses.n_elem) - Log::Fatal << "The number of masses should be the same as the number of poles." << std::endl; + Log::Fatal << "The number of masses should be the same as the number of poles." + << std::endl; } /** @@ -157,8 +159,8 @@ class MultiplePoleCart double poleOmega = state.AngularVelocity(i); double sinTheta = sin(state.Angle(i)); double cosTheta = cos(state.Angle(i)); - totalForce += (poleMasses[i] * poleLengths[i] * poleOmega * poleOmega * sinTheta) + - 0.75 * poleMasses[i] * cosTheta * gravity * sinTheta; + totalForce += (poleMasses[i] * poleLengths[i] * poleOmega * poleOmega * + sinTheta) + 0.75 * poleMasses[i] * cosTheta * gravity * sinTheta; totalMass += poleMasses[i] * (1 - 0.75 * cosTheta * cosTheta); } double xAcc = totalForce / totalMass; @@ -169,7 +171,7 @@ class MultiplePoleCart double sinTheta = sin(state.Angle(i)); double cosTheta = cos(state.Angle(i)); nextState.Angle(i) = state.Angle(i) + tau * state.AngularVelocity(i); - nextState.AngularVelocity(i) += -tau * 0.75 * (xAcc * cosTheta + + nextState.AngularVelocity(i) += -tau * 0.75 * (xAcc * cosTheta + gravity * sinTheta) / poleLengths[i]; } diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 87e4edb6f3..3b4b3b2373 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -125,8 +125,8 @@ BOOST_AUTO_TEST_CASE(SimpleCartPoleTest) */ BOOST_AUTO_TEST_CASE(MultiplePoleCartTest) { - arma::vec poleLengths = {1,0.5}; - arma::vec poleMasses = {1,1}; + arma::vec poleLengths = {1, 0.5}; + arma::vec poleMasses = {1, 1}; const MultiplePoleCart task = MultiplePoleCart(2, poleLengths, poleMasses); MultiplePoleCart::State state = task.InitialSample(); From 55814d3ca3f0c3a5ff22268b8735192bb1f0c556 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Thu, 23 May 2019 22:25:54 +0530 Subject: [PATCH 042/143] Style fixes and removed unused variable. --- .../environment/multiple_pole_cart.hpp | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp index c727837459..f5503d2c8b 100755 --- a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp @@ -131,11 +131,11 @@ class MultiplePoleCart doneReward(doneReward) { if (poleNum != poleLengths.n_elem) - Log::Fatal << "The number of lengths should be the same as the number of poles." - << std::endl; + Log::Fatal << "The number of lengths should be the same as the number of" + "poles." << std::endl; if (poleNum != poleMasses.n_elem) - Log::Fatal << "The number of masses should be the same as the number of poles." - << std::endl; + Log::Fatal << "The number of masses should be the same as the number of" + "poles." << std::endl; } /** @@ -238,18 +238,15 @@ class MultiplePoleCart //! Locally-stored length of poles. arma::vec poleLengths; + //! Locally-stored mass of the pole. + arma::vec poleMasses; + //! Locally-stored gravity. double gravity; //! Locally-stored mass of the cart. double massCart; - //! Locally-stored mass of the pole. - arma::vec poleMasses; - - //! Locally-stored length of the pole. - double length; - //! Locally-stored magnitude of the applied force. double forceMag; From de18b388faeac5fc737e2b1be69f94445ef1aba6 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Fri, 24 May 2019 09:16:35 +0530 Subject: [PATCH 043/143] Resolved Marcus' comments. --- .../environment/cart_pole.hpp | 1 + .../environment/multiple_pole_cart.hpp | 23 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp b/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp index 222c8b4190..b5f8230402 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp @@ -104,6 +104,7 @@ class CartPole * @param tau The time interval. * @param thetaThresholdRadians The maximum angle. * @param xThreshold The maximum position. + * @param doneReward Reward recieved on termination. */ CartPole(const double gravity = 9.8, const double massCart = 1.0, diff --git a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp index f5503d2c8b..88d56d05ad 100755 --- a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp @@ -37,9 +37,9 @@ class MultiplePoleCart * * @param numPoles The number of poles. */ - State(size_t numPoles) + State(const size_t numPoles) { - data = arma::mat(dimension, numPoles, arma::fill::zeros); + data = arma::zeros(dimension, numPoles); } /** @@ -108,17 +108,18 @@ class MultiplePoleCart * @param tau The time interval. * @param thetaThresholdRadians The maximum angle. * @param xThreshold The maximum position. + * @param doneReward The reward recieved on termination. */ MultiplePoleCart(const size_t poleNum, - const arma::vec poleLengths, - const arma::vec poleMasses, - const double gravity = 9.8, - const double massCart = 1.0, - const double forceMag = 10.0, - const double tau = 0.02, - const double thetaThresholdRadians = 12 * 2 * 3.1416 / 360, - const double xThreshold = 2.4, - const double doneReward = 0.0) : + const arma::vec& poleLengths, + const arma::vec& poleMasses, + const double gravity = 9.8, + const double massCart = 1.0, + const double forceMag = 10.0, + const double tau = 0.02, + const double thetaThresholdRadians = 12 * 2 * 3.1416 / 360, + const double xThreshold = 2.4, + const double doneReward = 0.0) : poleNum(poleNum), poleLengths(poleLengths), poleMasses(poleMasses), From 2a46e721e191b6575233dc8d01bcf7e8e15d53e9 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Fri, 24 May 2019 17:13:04 +0530 Subject: [PATCH 044/143] Added const prefixes --- .../environment/multiple_pole_cart.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp index 88d56d05ad..5ce577bab9 100755 --- a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp @@ -64,14 +64,14 @@ class MultiplePoleCart double& Velocity() { return data(1, 0); } //! Get the angle of the $i^{th}$ pole with the vertical. - double Angle(size_t i) const { return data(0, i); } + double Angle(const size_t i) const { return data(0, i); } //! Modify the angle of the $i^{th}$ pole with the vertical. - double& Angle(size_t i) { return data(0, i); } + double& Angle(const size_t i) { return data(0, i); } //! Get the angular velocity of the $i^{th}$ pole. - double AngularVelocity(size_t i) const { return data(1, i); } + double AngularVelocity(const size_t i) const { return data(1, i); } //! Modify the angular velocity of the $i^{th}$ pole. - double& AngularVelocity(size_t i) { return data(1, i); } + double& AngularVelocity(const size_t i) { return data(1, i); } //! Encode the state to a matrix. const arma::mat& Encode() const { return data; } From 47434f014b5ece80e500f4d05aace4901a7e2d7b Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Sat, 25 May 2019 08:30:30 +0530 Subject: [PATCH 045/143] Added braces. --- .../reinforcement_learning/environment/multiple_pole_cart.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp index 5ce577bab9..12609a4063 100755 --- a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp @@ -132,11 +132,15 @@ class MultiplePoleCart doneReward(doneReward) { if (poleNum != poleLengths.n_elem) + { Log::Fatal << "The number of lengths should be the same as the number of" "poles." << std::endl; + } if (poleNum != poleMasses.n_elem) + { Log::Fatal << "The number of masses should be the same as the number of" "poles." << std::endl; + } } /** From d6bfc7ed81b241897b08d323305cf2ce8e83ceca Mon Sep 17 00:00:00 2001 From: robotcator Date: Sat, 25 May 2019 16:47:27 +0800 Subject: [PATCH 046/143] change element access to () --- .../methods/reinforcement_learning/q_learning_impl.hpp | 6 +++--- .../replay/prioritized_replay.hpp | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index ba703597de..e87484569f 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -216,10 +216,10 @@ double QLearning< for (size_t i = 0; i < sampledNextStates.n_cols; ++i) { if (isTerminal[i]) - target(sampledActions[i], i) = sampledRewards[i]; + target(sampledActions(i), i) = sampledRewards(i); else - target(sampledActions[i], i) = sampledRewards[i] + config.Discount() * - nextActionValues(bestActions[i], i); + target(sampledActions(i), i) = sampledRewards(i) + config.Discount() * + nextActionValues(bestActions(i), i); } // Learn form experience. diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index f49e6b582c..68891a7883 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -131,7 +131,7 @@ class PrioritizedReplay for (size_t bt = 0; bt < batchSize; bt++) { double mass = arma::randu() * sumPerRange + bt * sumPerRange; size_t idx = idxSum.FindPrefixSum(mass); - idxes[bt] = idx; + idxes(bt) = idx; } return idxes; } @@ -172,8 +172,8 @@ class PrioritizedReplay for (size_t i = 0; i < sampledIndices.n_rows; i++) { - double p_sample = idxSum.Get(sampledIndices[i]) / idxSum.Sum(); - weights[i] = pow(numSample * p_sample, -beta); + double p_sample = idxSum.Get(sampledIndices(i)) / idxSum.Sum(); + weights(i) = pow(numSample * p_sample, -beta); } weights /= weights.max(); } @@ -225,8 +225,8 @@ class PrioritizedReplay arma::colvec tdError(target.n_cols); for (size_t i = 0; i < target.n_cols; i ++) { - tdError[i] = nextActionValues(sampledActions[i], i) - - target(sampledActions[i], i); + tdError(i) = nextActionValues(sampledActions(i), i) - + target(sampledActions(i), i); } tdError = arma::abs(tdError); UpdatePriorities(sampledIndices, tdError); From 25c531577795507034cc334e4d1ebfa81922e62b Mon Sep 17 00:00:00 2001 From: Yashwant Date: Mon, 27 May 2019 08:06:55 +0530 Subject: [PATCH 047/143] Fix Style Checksx --- src/mlpack/tests/main_tests/gmm_generate_test.cpp | 15 +++++---------- .../tests/main_tests/gmm_probability_test.cpp | 7 +++---- src/mlpack/tests/main_tests/gmm_train_test.cpp | 6 +++--- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/mlpack/tests/main_tests/gmm_generate_test.cpp b/src/mlpack/tests/main_tests/gmm_generate_test.cpp index 9139549857..1d725a28e6 100644 --- a/src/mlpack/tests/main_tests/gmm_generate_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_generate_test.cpp @@ -24,7 +24,7 @@ using namespace mlpack; struct GmmGenerateTestFixture { - public: + public: GmmGenerateTestFixture() { // Cache in the options for this program. @@ -43,9 +43,7 @@ BOOST_FIXTURE_TEST_SUITE(GmmGenerateMainTest, GmmGenerateTestFixture); // Checking that Samples must greater than 0. BOOST_AUTO_TEST_CASE(GmmGenerateSamplesTest) { - arma::mat inputData; - if (!data::Load("vc2.csv", inputData)) - BOOST_FAIL("Unable to load train dataset vc2.csv!"); + arma::mat inputData(5, 10, arma::fill::randu); GMM gmm(1, 2); gmm.Train(inputData, 2); @@ -61,9 +59,7 @@ BOOST_AUTO_TEST_CASE(GmmGenerateSamplesTest) // Making sure samples are provided. BOOST_AUTO_TEST_CASE(GmmGenerateSamples) { - arma::mat inputData; - if (!data::Load("vc2.csv", inputData)) - BOOST_FAIL("Unable to load train dataset vc2.csv!"); + arma::mat inputData(5, 10, arma::fill::randu); GMM gmm(1, 2); gmm.Train(inputData, 2); @@ -78,9 +74,7 @@ BOOST_AUTO_TEST_CASE(GmmGenerateSamples) // Checking dimensionality of output. BOOST_AUTO_TEST_CASE(GmmGenerateDimensionality) { - arma::mat inputData; - if (!data::Load("vc2.csv", inputData)) - BOOST_FAIL("Unable to load train dataset vc2.csv!"); + arma::mat inputData(5, 10, arma::fill::randu); GMM gmm(1, 2); gmm.Train(inputData, 2); @@ -96,3 +90,4 @@ BOOST_AUTO_TEST_CASE(GmmGenerateDimensionality) } BOOST_AUTO_TEST_SUITE_END(); + diff --git a/src/mlpack/tests/main_tests/gmm_probability_test.cpp b/src/mlpack/tests/main_tests/gmm_probability_test.cpp index 4219a372cf..f945ea134b 100644 --- a/src/mlpack/tests/main_tests/gmm_probability_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_probability_test.cpp @@ -27,7 +27,7 @@ using namespace mlpack; struct GmmProbabilityTestFixture { - public: + public: GmmProbabilityTestFixture() { // Cache in the options for this program. @@ -52,9 +52,7 @@ BOOST_FIXTURE_TEST_SUITE(GmmProbabilityMainTest, GmmProbabilityTestFixture); // Checking the input and output dimensionality. BOOST_AUTO_TEST_CASE(GmmProbabilityDimensionality) { - arma::mat inputData; - if (!data::Load("vc2.csv", inputData)) - BOOST_FAIL("Unable to load train dataset vc2.csv!"); + arma::mat inputData(5, 10, arma::fill::randu); GMM gmm(1, 2); gmm.Train(std::move(inputData), 2); @@ -71,3 +69,4 @@ BOOST_AUTO_TEST_CASE(GmmProbabilityDimensionality) } BOOST_AUTO_TEST_SUITE_END(); + diff --git a/src/mlpack/tests/main_tests/gmm_train_test.cpp b/src/mlpack/tests/main_tests/gmm_train_test.cpp index 93049763ba..85074fdc62 100644 --- a/src/mlpack/tests/main_tests/gmm_train_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_train_test.cpp @@ -26,7 +26,7 @@ using namespace mlpack; struct GmmTrainTestFixture { - public: + public: GmmTrainTestFixture() { // Cache in the options for this program. @@ -205,7 +205,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainPercentageTest) SetInputParam("input", inputData); SetInputParam("gaussians", (int) 2); SetInputParam("refined_start", true); - SetInputParam("percentage", (double) 0.02); + SetInputParam("percentage", (double) 0.22); mlpackMain(); @@ -219,7 +219,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainPercentageTest) SetInputParam("input", std::move(inputData)); SetInputParam("gaussians", (int) 2); SetInputParam("refined_start", true); - SetInputParam("percentage", (double) 0.52); + SetInputParam("percentage", (double) 0.82); mlpackMain(); From 26a015c8d500c9baec9078eb9955d88c3bf846b3 Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Sun, 31 Mar 2019 06:55:33 +0530 Subject: [PATCH 048/143] Shuffle discriminator predictors of GAN --- src/mlpack/methods/ann/gan/gan_impl.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/gan/gan_impl.hpp b/src/mlpack/methods/ann/gan/gan_impl.hpp index 92a196c125..0d6ce03467 100644 --- a/src/mlpack/methods/ann/gan/gan_impl.hpp +++ b/src/mlpack/methods/ann/gan/gan_impl.hpp @@ -370,7 +370,10 @@ template< > void GAN::Shuffle() { - math::ShuffleData(predictors, responses, predictors, responses); + arma::uvec ordering = arma::shuffle(arma::linspace(0, + numFunctions - 1, numFunctions)); + predictors = predictors.cols(ordering); + discriminator.predictors.cols(0, numFunctions-1) = predictors; } template< From b988e6966f293a65183f5fda04bb416cd4b40394 Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Sun, 31 Mar 2019 23:14:23 +0530 Subject: [PATCH 049/143] Memory sharing b/w predictors of discriminator and GAN --- src/mlpack/methods/ann/gan/gan_impl.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/ann/gan/gan_impl.hpp b/src/mlpack/methods/ann/gan/gan_impl.hpp index 0d6ce03467..57f1cdd491 100644 --- a/src/mlpack/methods/ann/gan/gan_impl.hpp +++ b/src/mlpack/methods/ann/gan/gan_impl.hpp @@ -42,7 +42,6 @@ GAN::GAN( const double multiplier, const double clippingParameter, const double lambda): - predictors(predictors), generator(std::move(generator)), discriminator(std::move(discriminator)), initializeRule(initializeRule), @@ -72,7 +71,8 @@ GAN::GAN( this->discriminator.predictors.set_size(predictors.n_rows, predictors.n_cols + batchSize); this->discriminator.predictors.cols(0, predictors.n_cols - 1) = predictors; - + this->predictors = arma::mat(this->discriminator.predictors.memptr(), + predictors.n_rows, predictors.n_cols, false, false); this->discriminator.responses.set_size(1, predictors.n_cols + batchSize); this->discriminator.responses.ones(); this->discriminator.responses.cols(predictors.n_cols, @@ -372,8 +372,8 @@ void GAN::Shuffle() { arma::uvec ordering = arma::shuffle(arma::linspace(0, numFunctions - 1, numFunctions)); - predictors = predictors.cols(ordering); - discriminator.predictors.cols(0, numFunctions-1) = predictors; + discriminator.predictors.cols(0, numFunctions- 1) = + predictors.cols(ordering); } template< From 8f5a649eb8a85bf8bb89bedbc08a3a0d6b86dcd6 Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Mon, 1 Apr 2019 04:51:41 +0530 Subject: [PATCH 050/143] Change size of predictors and responses --- src/mlpack/methods/ann/gan/gan_impl.hpp | 37 +++++++++++----------- src/mlpack/methods/ann/gan/wgan_impl.hpp | 14 ++++---- src/mlpack/methods/ann/gan/wgangp_impl.hpp | 20 ++++++------ 3 files changed, 35 insertions(+), 36 deletions(-) diff --git a/src/mlpack/methods/ann/gan/gan_impl.hpp b/src/mlpack/methods/ann/gan/gan_impl.hpp index 57f1cdd491..d3b1d6c295 100644 --- a/src/mlpack/methods/ann/gan/gan_impl.hpp +++ b/src/mlpack/methods/ann/gan/gan_impl.hpp @@ -65,18 +65,17 @@ GAN::GAN( this->discriminator.deterministic = this->generator.deterministic = true; - responses.set_size(1, predictors.n_cols); - responses.ones(); + this->predictors.set_size(predictors.n_rows, predictors.n_cols + batchSize); + this->predictors.cols(0, predictors.n_cols - 1) = predictors; + this->discriminator.predictors = arma::mat(this->predictors.memptr(), + this->predictors.n_rows, this->predictors.n_cols, false, false); - this->discriminator.predictors.set_size(predictors.n_rows, - predictors.n_cols + batchSize); - this->discriminator.predictors.cols(0, predictors.n_cols - 1) = predictors; - this->predictors = arma::mat(this->discriminator.predictors.memptr(), - predictors.n_rows, predictors.n_cols, false, false); - this->discriminator.responses.set_size(1, predictors.n_cols + batchSize); - this->discriminator.responses.ones(); - this->discriminator.responses.cols(predictors.n_cols, + responses.set_size(1, predictors.n_cols + batchSize); + responses.ones(); + responses.cols(predictors.n_cols, predictors.n_cols + batchSize - 1) = arma::zeros(1, batchSize); + this->discriminator.responses = arma::mat(this->responses.memptr(), + this->responses.n_rows, this->responses.n_cols, false, false); numFunctions = predictors.n_cols; @@ -232,14 +231,14 @@ GAN::Evaluate( noise.imbue( [&]() { return noiseFunction();} ); generator.Forward(std::move(noise)); - discriminator.predictors.cols(numFunctions, numFunctions + batchSize - 1) = + predictors.cols(numFunctions, numFunctions + batchSize - 1) = boost::apply_visitor(outputParameterVisitor, generator.network.back()); - discriminator.Forward(std::move(discriminator.predictors.cols(numFunctions, + discriminator.Forward(std::move(predictors.cols(numFunctions, numFunctions + batchSize - 1))); - discriminator.responses.cols(numFunctions, numFunctions + batchSize - 1) = + responses.cols(numFunctions, numFunctions + batchSize - 1) = arma::zeros(1, batchSize); - currentTarget = arma::mat(discriminator.responses.memptr() + numFunctions, + currentTarget = arma::mat(responses.memptr() + numFunctions, 1, batchSize, false, false); res += discriminator.outputLayer.Forward( std::move(boost::apply_visitor( @@ -299,9 +298,9 @@ EvaluateWithGradient(const arma::mat& /* parameters */, noise.imbue( [&]() { return noiseFunction();} ); generator.Forward(std::move(noise)); - discriminator.predictors.cols(numFunctions, numFunctions + batchSize - 1) = + predictors.cols(numFunctions, numFunctions + batchSize - 1) = boost::apply_visitor(outputParameterVisitor, generator.network.back()); - discriminator.responses.cols(numFunctions, numFunctions + batchSize - 1) = + responses.cols(numFunctions, numFunctions + batchSize - 1) = arma::zeros(1, batchSize); // Get the gradients of the Generator. @@ -313,7 +312,7 @@ EvaluateWithGradient(const arma::mat& /* parameters */, { // Minimize -log(D(G(noise))). // Pass the error from Discriminator to Generator. - discriminator.responses.cols(numFunctions, numFunctions + batchSize - 1) = + responses.cols(numFunctions, numFunctions + batchSize - 1) = arma::ones(1, batchSize); discriminator.Gradient(discriminator.parameter, numFunctions, noiseGradientDiscriminator, batchSize); @@ -372,8 +371,8 @@ void GAN::Shuffle() { arma::uvec ordering = arma::shuffle(arma::linspace(0, numFunctions - 1, numFunctions)); - discriminator.predictors.cols(0, numFunctions- 1) = - predictors.cols(ordering); + arma::mat temp = predictors.cols(ordering); + predictors.cols(0, numFunctions - 1) = temp; } template< diff --git a/src/mlpack/methods/ann/gan/wgan_impl.hpp b/src/mlpack/methods/ann/gan/wgan_impl.hpp index 9a2cc704f8..dd5f0a8592 100644 --- a/src/mlpack/methods/ann/gan/wgan_impl.hpp +++ b/src/mlpack/methods/ann/gan/wgan_impl.hpp @@ -51,14 +51,14 @@ GAN::Evaluate( noise.imbue( [&]() { return noiseFunction();} ); generator.Forward(std::move(noise)); - discriminator.predictors.cols(numFunctions, numFunctions + batchSize - 1) = + predictors.cols(numFunctions, numFunctions + batchSize - 1) = boost::apply_visitor(outputParameterVisitor, generator.network.back()); - discriminator.Forward(std::move(discriminator.predictors.cols(numFunctions, + discriminator.Forward(std::move(predictors.cols(numFunctions, numFunctions + batchSize - 1))); - discriminator.responses.cols(numFunctions, numFunctions + batchSize - 1) = + responses.cols(numFunctions, numFunctions + batchSize - 1) = -arma::ones(1, batchSize); - currentTarget = arma::mat(discriminator.responses.memptr() + numFunctions, + currentTarget = arma::mat(responses.memptr() + numFunctions, 1, batchSize, false, false); res += discriminator.outputLayer.Forward( std::move(boost::apply_visitor( @@ -117,9 +117,9 @@ EvaluateWithGradient(const arma::mat& /* parameters */, noise.imbue( [&]() { return noiseFunction();} ); generator.Forward(std::move(noise)); - discriminator.predictors.cols(numFunctions, numFunctions + batchSize - 1) = + predictors.cols(numFunctions, numFunctions + batchSize - 1) = boost::apply_visitor(outputParameterVisitor, generator.network.back()); - discriminator.responses.cols(numFunctions, numFunctions + batchSize - 1) = + responses.cols(numFunctions, numFunctions + batchSize - 1) = -arma::ones(1, batchSize); // Get the gradients of the Generator. @@ -133,7 +133,7 @@ EvaluateWithGradient(const arma::mat& /* parameters */, { // Minimize -D(G(noise)). // Pass the error from Discriminator to Generator. - discriminator.responses.cols(numFunctions, numFunctions + batchSize - 1) = + responses.cols(numFunctions, numFunctions + batchSize - 1) = arma::ones(1, batchSize); discriminator.Gradient(discriminator.parameter, numFunctions, noiseGradientDiscriminator, batchSize); diff --git a/src/mlpack/methods/ann/gan/wgangp_impl.hpp b/src/mlpack/methods/ann/gan/wgangp_impl.hpp index 2c56f031fa..6f2027825c 100644 --- a/src/mlpack/methods/ann/gan/wgangp_impl.hpp +++ b/src/mlpack/methods/ann/gan/wgangp_impl.hpp @@ -54,14 +54,14 @@ GAN::Evaluate( arma::mat generatedData = boost::apply_visitor(outputParameterVisitor, generator.network.back()); - discriminator.predictors.cols(numFunctions, numFunctions + batchSize - 1) = + predictors.cols(numFunctions, numFunctions + batchSize - 1) = generatedData; - discriminator.Forward(std::move(discriminator.predictors.cols(numFunctions, + discriminator.Forward(std::move(predictors.cols(numFunctions, numFunctions + batchSize - 1))); - discriminator.responses.cols(numFunctions, numFunctions + batchSize - 1) = + responses.cols(numFunctions, numFunctions + batchSize - 1) = -arma::ones(1, batchSize); - currentTarget = arma::mat(discriminator.responses.memptr() + numFunctions, + currentTarget = arma::mat(responses.memptr() + numFunctions, 1, batchSize, false, false); res += discriminator.outputLayer.Forward( std::move(boost::apply_visitor( @@ -70,9 +70,9 @@ GAN::Evaluate( // Gradient Penalty is calculated here. double epsilon = math::Random(); - discriminator.predictors.cols(numFunctions, numFunctions + batchSize - 1) = + predictors.cols(numFunctions, numFunctions + batchSize - 1) = (epsilon * currentInput) + ((1.0 - epsilon) * generatedData); - discriminator.responses.cols(numFunctions, numFunctions + batchSize - 1) = + responses.cols(numFunctions, numFunctions + batchSize - 1) = -arma::ones(1, batchSize); discriminator.Gradient(discriminator.parameter, numFunctions, normGradientDiscriminator, batchSize); @@ -139,15 +139,15 @@ EvaluateWithGradient(const arma::mat& /* parameters */, // Gradient Penalty is calculated here. double epsilon = math::Random(); - discriminator.predictors.cols(numFunctions, numFunctions + batchSize - 1) = + predictors.cols(numFunctions, numFunctions + batchSize - 1) = (epsilon * currentInput) + ((1.0 - epsilon) * generatedData); - discriminator.responses.cols(numFunctions, numFunctions + batchSize - 1) = + responses.cols(numFunctions, numFunctions + batchSize - 1) = -arma::ones(1, batchSize); discriminator.Gradient(discriminator.parameter, numFunctions, normGradientDiscriminator, batchSize); res += lambda * std::pow(arma::norm(normGradientDiscriminator, 2) - 1, 2); - discriminator.predictors.cols(numFunctions, numFunctions + batchSize - 1) = + predictors.cols(numFunctions, numFunctions + batchSize - 1) = generatedData; res += discriminator.EvaluateWithGradient(discriminator.parameter, numFunctions, noiseGradientDiscriminator, batchSize); @@ -157,7 +157,7 @@ EvaluateWithGradient(const arma::mat& /* parameters */, { // Minimize -D(G(noise)). // Pass the error from Discriminator to Generator. - discriminator.responses.cols(numFunctions, numFunctions + batchSize - 1) = + responses.cols(numFunctions, numFunctions + batchSize - 1) = arma::ones(1, batchSize); discriminator.Gradient(discriminator.parameter, numFunctions, noiseGradientDiscriminator, batchSize); From ca8e8d04cdc0f83c471f34e155cafd7a52f3114c Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Mon, 1 Apr 2019 19:22:37 +0530 Subject: [PATCH 051/143] Remove temporary variable --- src/mlpack/methods/ann/gan/gan_impl.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/gan/gan_impl.hpp b/src/mlpack/methods/ann/gan/gan_impl.hpp index d3b1d6c295..5ca31d8c64 100644 --- a/src/mlpack/methods/ann/gan/gan_impl.hpp +++ b/src/mlpack/methods/ann/gan/gan_impl.hpp @@ -371,8 +371,7 @@ void GAN::Shuffle() { arma::uvec ordering = arma::shuffle(arma::linspace(0, numFunctions - 1, numFunctions)); - arma::mat temp = predictors.cols(ordering); - predictors.cols(0, numFunctions - 1) = temp; + predictors.cols(0, numFunctions - 1) = predictors.cols(ordering); } template< From 7b69a88e031fee8d5a4f48e91760107f5963b78a Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Sat, 6 Apr 2019 20:11:37 +0530 Subject: [PATCH 052/143] Minor changes --- src/mlpack/methods/ann/gan/gan_impl.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/ann/gan/gan_impl.hpp b/src/mlpack/methods/ann/gan/gan_impl.hpp index 5ca31d8c64..cb992f3eed 100644 --- a/src/mlpack/methods/ann/gan/gan_impl.hpp +++ b/src/mlpack/methods/ann/gan/gan_impl.hpp @@ -70,8 +70,7 @@ GAN::GAN( this->discriminator.predictors = arma::mat(this->predictors.memptr(), this->predictors.n_rows, this->predictors.n_cols, false, false); - responses.set_size(1, predictors.n_cols + batchSize); - responses.ones(); + responses.ones(1, predictors.n_cols + batchSize); responses.cols(predictors.n_cols, predictors.n_cols + batchSize - 1) = arma::zeros(1, batchSize); this->discriminator.responses = arma::mat(this->responses.memptr(), @@ -369,7 +368,7 @@ template< > void GAN::Shuffle() { - arma::uvec ordering = arma::shuffle(arma::linspace(0, + const arma::uvec ordering = arma::shuffle(arma::linspace(0, numFunctions - 1, numFunctions)); predictors.cols(0, numFunctions - 1) = predictors.cols(ordering); } From ed22eda6b27be769e4358b945a4acd9a833dad2a Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Mon, 27 May 2019 18:22:08 +0700 Subject: [PATCH 053/143] Add test for memory sharing --- src/mlpack/methods/ann/gan/gan.hpp | 17 +++++++-- src/mlpack/tests/gan_test.cpp | 56 ++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/ann/gan/gan.hpp b/src/mlpack/methods/ann/gan/gan.hpp index ef5f08efee..09b5b142df 100644 --- a/src/mlpack/methods/ann/gan/gan.hpp +++ b/src/mlpack/methods/ann/gan/gan.hpp @@ -210,7 +210,7 @@ class GAN * Gradient function for Standard GAN and DCGAN. * This function passes the gradient based on which network is being * trained, i.e., Generator or Discriminator. - * + * * @param parameters present parameters of the network. * @param i Index of the predictors. * @param gradient Variable to store the present gradient. @@ -228,7 +228,7 @@ class GAN * Gradient function for WGAN. * This function passes the gradient based on which network is being * trained, i.e., Generator or Discriminator. - * + * * @param parameters present parameters of the network. * @param i Index of the predictors. * @param gradient Variable to store the present gradient. @@ -245,7 +245,7 @@ class GAN * Gradient function for WGAN-GP. * This function passes the gradient based on which network is being * trained, i.e., Generator or Discriminator. - * + * * @param parameters present parameters of the network. * @param i Index of the predictors. * @param gradient Variable to store the present gradient. @@ -298,6 +298,17 @@ class GAN //! Return the number of separable functions (the number of predictor points). size_t NumFunctions() const { return numFunctions; } + //! Get the matrix of responses to the input data points. + const arma::mat& Responses() const { return responses; } + //! Modify the matrix of responses to the input data points. + arma::mat& Responses() { return responses; } + + //! Get the matrix of data points (predictors). + const arma::mat& Predictors() const { return predictors; } + //! Modify the matrix of data points (predictors). + arma::mat& Predictors() { return predictors; } + + //! Serialize the model. template void serialize(Archive& ar, const unsigned int /* version */); diff --git a/src/mlpack/tests/gan_test.cpp b/src/mlpack/tests/gan_test.cpp index a89ae67d48..d930c3e8c9 100644 --- a/src/mlpack/tests/gan_test.cpp +++ b/src/mlpack/tests/gan_test.cpp @@ -246,4 +246,60 @@ BOOST_AUTO_TEST_CASE(GANMNISTTest) Log::Info << "Output generated!" << std::endl; } +/* + * Create GAN network and test for memory sharing + * between discriminator and gan predictors. + */ +BOOST_AUTO_TEST_CASE(GANMemorySharingTest) +{ + size_t generatorHiddenLayerSize = 8; + size_t discriminatorHiddenLayerSize = 8; + size_t generatorOutputSize = 1; + size_t discriminatorOutputSize = 1; + size_t discriminatorPreTrain = 0; + size_t batchSize = 8; + size_t noiseDim = 1; + size_t generatorUpdateStep = 1; + double multiplier = 1; + + arma::mat trainData(1, 10000); + trainData.imbue( [&]() { return arma::as_scalar(RandNormal(4, 0.5));}); + trainData = arma::sort(trainData); + + // Create the Discriminator network + FFN > discriminator; + discriminator.Add > ( + generatorOutputSize, discriminatorHiddenLayerSize * 2); + discriminator.Add >(); + discriminator.Add > ( + discriminatorHiddenLayerSize * 2, discriminatorHiddenLayerSize * 2); + discriminator.Add >(); + discriminator.Add > ( + discriminatorHiddenLayerSize * 2, discriminatorHiddenLayerSize * 2); + discriminator.Add >(); + discriminator.Add > ( + discriminatorHiddenLayerSize * 2, discriminatorOutputSize); + + // Create the Generator network + FFN > generator; + generator.Add >(noiseDim, generatorHiddenLayerSize); + generator.Add >(); + generator.Add >(generatorHiddenLayerSize, generatorOutputSize); + + // Create GAN + GaussianInitialization gaussian(0, 0.1); + std::function noiseFunction = [](){ return math::Random(-8, 8) + + math::RandNormal(0, 1) * 0.01;}; + GAN >, + GaussianInitialization, + std::function > + gan(trainData, generator, discriminator, gaussian, noiseFunction, + noiseDim, batchSize, generatorUpdateStep, discriminatorPreTrain, + multiplier); + + CheckMatrices(gan.Predictors(), gan.Discriminator().Predictors()); + gan.Shuffle(); + CheckMatrices(gan.Predictors(), gan.Discriminator().Predictors()); +} + BOOST_AUTO_TEST_SUITE_END(); From 469599cbd64f811f513447a50a037a80006ceff6 Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Mon, 27 May 2019 23:45:40 +0700 Subject: [PATCH 054/143] Remove extra line --- src/mlpack/methods/ann/gan/gan.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mlpack/methods/ann/gan/gan.hpp b/src/mlpack/methods/ann/gan/gan.hpp index 09b5b142df..312bae1b59 100644 --- a/src/mlpack/methods/ann/gan/gan.hpp +++ b/src/mlpack/methods/ann/gan/gan.hpp @@ -308,7 +308,6 @@ class GAN //! Modify the matrix of data points (predictors). arma::mat& Predictors() { return predictors; } - //! Serialize the model. template void serialize(Archive& ar, const unsigned int /* version */); From 0c6c468894b71bad7fa504919a2d0eb95dfa3316 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Tue, 28 May 2019 09:57:14 +0530 Subject: [PATCH 055/143] Optimized expressions. --- .../environment/multiple_pole_cart.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp index 12609a4063..607be6ea51 100755 --- a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp @@ -165,8 +165,10 @@ class MultiplePoleCart double sinTheta = sin(state.Angle(i)); double cosTheta = cos(state.Angle(i)); totalForce += (poleMasses[i] * poleLengths[i] * poleOmega * poleOmega * - sinTheta) + 0.75 * poleMasses[i] * cosTheta * gravity * sinTheta; - totalMass += poleMasses[i] * (1 - 0.75 * cosTheta * cosTheta); + sinTheta) + 0.75 * poleMasses[i] * gravity * sin(2 * state.Angle(i)) + / 2; + totalMass += poleMasses[i] * (1 - 0.75 * (cos(2 * state.Angle(i)) + 1) + / 2); } double xAcc = totalForce / totalMass; From db90db86da14e77dcf5a5e7657d99d0bef26628c Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Tue, 28 May 2019 10:00:49 +0530 Subject: [PATCH 056/143] Fixed style. --- .../reinforcement_learning/environment/multiple_pole_cart.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp index 607be6ea51..97561e136a 100755 --- a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp @@ -167,7 +167,7 @@ class MultiplePoleCart totalForce += (poleMasses[i] * poleLengths[i] * poleOmega * poleOmega * sinTheta) + 0.75 * poleMasses[i] * gravity * sin(2 * state.Angle(i)) / 2; - totalMass += poleMasses[i] * (1 - 0.75 * (cos(2 * state.Angle(i)) + 1) + totalMass += poleMasses[i] * (1 - 0.75 * (cos(2 * state.Angle(i)) + 1) / 2); } double xAcc = totalForce / totalMass; @@ -273,4 +273,4 @@ class MultiplePoleCart } // namespace rl } // namespace mlpack -#endif +#endif \ No newline at end of file From b4750f2e07fd88f0088271701afcf31a6f8a2d29 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Tue, 28 May 2019 10:03:46 +0530 Subject: [PATCH 057/143] Restored newline at end of file. --- .../reinforcement_learning/environment/multiple_pole_cart.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp index 97561e136a..3257ab6432 100755 --- a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp @@ -273,4 +273,4 @@ class MultiplePoleCart } // namespace rl } // namespace mlpack -#endif \ No newline at end of file +#endif From 11033e65d06ec54bae17e3fdfac46fc582481827 Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Tue, 28 May 2019 17:39:19 +0700 Subject: [PATCH 058/143] match with training data --- src/mlpack/tests/gan_test.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/mlpack/tests/gan_test.cpp b/src/mlpack/tests/gan_test.cpp index d930c3e8c9..85c4be5176 100644 --- a/src/mlpack/tests/gan_test.cpp +++ b/src/mlpack/tests/gan_test.cpp @@ -297,9 +297,12 @@ BOOST_AUTO_TEST_CASE(GANMemorySharingTest) noiseDim, batchSize, generatorUpdateStep, discriminatorPreTrain, multiplier); + CheckMatrices(gan.Predictors().head_cols(trainData.n_cols), trainData); CheckMatrices(gan.Predictors(), gan.Discriminator().Predictors()); gan.Shuffle(); CheckMatrices(gan.Predictors(), gan.Discriminator().Predictors()); + CheckMatricesNotEqual(gan.Predictors().head_cols(trainData.n_cols), + trainData); } BOOST_AUTO_TEST_SUITE_END(); From 002466a82dc49eba5724a0809c4ca63858365cf0 Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Fri, 8 Mar 2019 02:55:22 +0530 Subject: [PATCH 059/143] Add implementation --- src/mlpack/methods/ann/layer/CMakeLists.txt | 2 + src/mlpack/methods/ann/layer/highway.hpp | 242 ++++++++++++++++++ src/mlpack/methods/ann/layer/highway_impl.hpp | 239 +++++++++++++++++ src/mlpack/methods/ann/layer/layer.hpp | 1 + src/mlpack/methods/ann/layer/layer_types.hpp | 7 + 5 files changed, 491 insertions(+) create mode 100644 src/mlpack/methods/ann/layer/highway.hpp create mode 100644 src/mlpack/methods/ann/layer/highway_impl.hpp diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index 41558584aa..7c7084c74b 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -40,6 +40,8 @@ set(SOURCES gru_impl.hpp hard_tanh.hpp hard_tanh_impl.hpp + highway.hpp + highway_impl.hpp join.hpp join_impl.hpp layer.hpp diff --git a/src/mlpack/methods/ann/layer/highway.hpp b/src/mlpack/methods/ann/layer/highway.hpp new file mode 100644 index 0000000000..460959a731 --- /dev/null +++ b/src/mlpack/methods/ann/layer/highway.hpp @@ -0,0 +1,242 @@ +/** + * @file highway.hpp + * @author Saksham Bansal + * + * Definition of highway layer first introduced in the paper "Highway networks" + * https://arxiv.org/abs/1505.00387 + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_HIGHWAY_HPP +#define MLPACK_METHODS_ANN_LAYER_HIGHWAY_HPP + +#include + +#include + +#include "../visitor/delete_visitor.hpp" +#include "../visitor/delta_visitor.hpp" +#include "../visitor/output_height_visitor.hpp" +#include "../visitor/output_parameter_visitor.hpp" +#include "../visitor/output_width_visitor.hpp" + +#include "layer_types.hpp" +#include "add_merge.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * The highway layer class. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat, + typename... CustomLayers> +class Highway +{ + public: + //! Create the Highway object. + Highway(); + + /** + * Create the Highway object. + * + * @param inSize The number of input units. + * @param model Expose all the network modules. + */ + Highway(const size_t inSize, const bool model = true); + + //! Destroy the Highway object. + ~Highway(); + + /* + * Destroy all the modules added to the Highway object. + */ + void DeleteModules(); + + /** + * Reset the layer parameter. + */ + void Reset(); + + /** + * Ordinary feed forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + void Forward(arma::Mat&& input, arma::Mat&& output); + + /** + * Ordinary feed backward pass of a neural network, calculating the function + * f(x) by propagating x backwards through f. Using the results from the feed + * forward pass. + * + * @param input The propagated input activation. + * @param gy The backpropagated error. + * @param g The calculated gradient. + */ + template + void Backward(const arma::Mat&& input, + arma::Mat&& gy, + arma::Mat&& g); + + /* + * Calculate the gradient using the output delta and the input activation. + * + * @param input The input parameter used for calculating the gradient. + * @param error The calculated error. + * @param gradient The calculated gradient. + */ + template + void Gradient(arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& gradient); + + /* + * Add a new module to the model. + * + * @param args The layer parameter. + */ + template + void Add(Args... args) { network.push_back(new LayerType(args...)); } + + /* + * Add a new module to the model. + * + * @param layer The Layer to be added to the model. + */ + void Add(LayerTypes layer) { network.push_back(layer); } + + //! Return the model modules. + std::vector >& Model() + { + if (model) + { + return network; + } + + return empty; + } + + //! Get the parameters. + OutputDataType const& Parameters() const { return weights; } + //! Modify the parameters. + OutputDataType& Parameters() { return weights; } + + //! Get the input parameter. + InputDataType const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } + + /** + * Serialize the layer. + */ + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally-stored number of input units. + size_t inSize; + + //! Parameter which indicates if the modules should be exposed. + bool model; + + //! Indicator if we already initialized the model. + bool reset; + + //! Locally-stored network modules. + std::vector > network; + + //! Locally-stored empty list of modules. + std::vector > empty; + + //! Locally-stored weight object. + OutputDataType weights; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Weights for transformation of output. + OutputDataType transformWeight; + + //! Bias for transformation of output. + OutputDataType transformBias; + + //! Locally-stored transform gate parameters + OutputDataType transformGate; + + //! Locally-stored transform gate activation + OutputDataType transformGateActivation; + + //! Locally-stored transform gate activation + OutputDataType transformGateError; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! The input width. + size_t width; + + //! The input height. + size_t height; + + //! The normal output without highway network + OutputDataType networkOutput; + + //! Locally-stored delta visitor. + DeltaVisitor deltaVisitor; + + //! Locally-stored output parameter visitor. + OutputParameterVisitor outputParameterVisitor; + + //! Locally-stored delete visitor. + DeleteVisitor deleteVisitor; + + //! Locally-stored output width visitor. + OutputWidthVisitor outputWidthVisitor; + + //! Locally-stored output height visitor. + OutputHeightVisitor outputHeightVisitor; + +}; // class Highway + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "highway_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/highway_impl.hpp b/src/mlpack/methods/ann/layer/highway_impl.hpp new file mode 100644 index 0000000000..9a6c41f1fa --- /dev/null +++ b/src/mlpack/methods/ann/layer/highway_impl.hpp @@ -0,0 +1,239 @@ +/** + * @file highway_impl.hpp + * @author Saksham Bansal + * + * Implementation of highway layer. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_HIGHWAY_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_HIGHWAY_IMPL_HPP + +// In case it hasn't yet been included. +#include "highway.hpp" + +#include "../visitor/forward_visitor.hpp" +#include "../visitor/backward_visitor.hpp" +#include "../visitor/gradient_visitor.hpp" +#include "../visitor/set_input_height_visitor.hpp" +#include "../visitor/set_input_width_visitor.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +Highway::Highway() +{ + // Nothing to do here. +} + +template< + typename InputDataType, typename OutputDataType, typename... CustomLayers> +Highway::Highway( + const size_t inSize, + const bool model) : + inSize(inSize), + model(model), + reset(false), + width(0), + height(0) +{ + weights.set_size(inSize * inSize + inSize, 1); +} + +template +Highway::~Highway() +{ + if (!model) + { + for (LayerTypes& layer : network) + boost::apply_visitor(deleteVisitor, layer); + } +} + +template +void Highway< + InputDataType, OutputDataType, CustomLayers...>::DeleteModules() +{ + if (model == true) + { + for (LayerTypes& layer : network) + { + boost::apply_visitor(deleteVisitor, layer); + } + } +} + +template +void Highway::Reset() +{ + transformWeight = arma::mat(weights.memptr(), inSize, inSize, false, false); + transformBias = arma::mat(weights.memptr() + transformWeight.n_elem, + inSize, 1, false, false); +} + +template +template +void Highway::Forward( + arma::Mat&& input, arma::Mat&& output) +{ + boost::apply_visitor(ForwardVisitor(std::move(input), std::move( + boost::apply_visitor(outputParameterVisitor, network.front()))), + network.front()); + + if (!reset) + { + if (boost::apply_visitor(outputWidthVisitor, network.front()) != 0) + { + width = boost::apply_visitor(outputWidthVisitor, network.front()); + } + + if (boost::apply_visitor(outputHeightVisitor, network.front()) != 0) + { + height = boost::apply_visitor(outputHeightVisitor, network.front()); + } + } + + for (size_t i = 1; i < network.size(); ++i) + { + if (!reset) + { + // Set the input width. + boost::apply_visitor(SetInputWidthVisitor(width), network[i]); + + // Set the input height. + boost::apply_visitor(SetInputHeightVisitor(height), network[i]); + } + + boost::apply_visitor(ForwardVisitor(std::move(boost::apply_visitor( + outputParameterVisitor, network[i - 1])), std::move( + boost::apply_visitor(outputParameterVisitor, network[i]))), + network[i]); + + if (!reset) + { + // Get the output width. + if (boost::apply_visitor(outputWidthVisitor, network[i]) != 0) + { + width = boost::apply_visitor(outputWidthVisitor, network[i]); + } + + // Get the output height. + if (boost::apply_visitor(outputHeightVisitor, network[i]) != 0) + { + height = boost::apply_visitor(outputHeightVisitor, network[i]); + } + } + } + if (!reset) + { + reset = true; + } + + output = boost::apply_visitor(outputParameterVisitor, network.back()); + + if (arma::size(output) != arma::size(input)){ + Log::Fatal << "The sizes of the output and input matrices of the Highway" + << " network should be equal. Please examine the network layers."; + } + + transformGate = transformWeight * input; + transformGate.each_col() += transformBias; + transformGateActivation = 1.0 /(1 + arma::exp(-transformGate)); + inputParameter = input; + networkOutput = output; + output = (output % transformGateActivation) + + (input % (1 - transformGateActivation)); +} + +template +template +void Highway::Backward( + const arma::Mat&& input, + arma::Mat&& gy, + arma::Mat&& g) +{ + boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( + outputParameterVisitor, network.back())), + std::move(gy % transformGateActivation), + std::move(boost::apply_visitor(deltaVisitor, network.back()))), + network.back()); + + for (size_t i = 2; i < network.size() + 1; ++i) + { + boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( + outputParameterVisitor, network[network.size() - i])), std::move( + boost::apply_visitor(deltaVisitor, network[network.size() - i + 1])), + std::move(boost::apply_visitor(deltaVisitor, + network[network.size() - i]))), network[network.size() - i]); + } + + g = boost::apply_visitor(deltaVisitor, network.front()); + + transformGateError = gy % (networkOutput - inputParameter) % + transformGateActivation % (1.0 - transformGateActivation); + g += transformWeight.t() * transformGateError; + g += gy % (1 - transformGateActivation); +} + +template +template +void Highway::Gradient( + arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& gradient) +{ + boost::apply_visitor(GradientVisitor(std::move(boost::apply_visitor( + outputParameterVisitor, network[network.size() - 2])), + std::move(error % transformGateActivation)), network.back()); + + for (size_t i = 2; i < network.size(); ++i) + { + boost::apply_visitor(GradientVisitor(std::move(boost::apply_visitor( + outputParameterVisitor, network[network.size() - i - 1])), std::move( + boost::apply_visitor(deltaVisitor, network[network.size() - i + 1]))), + network[network.size() - i]); + } + + boost::apply_visitor(GradientVisitor(std::move(input), std::move( + boost::apply_visitor(deltaVisitor, network[1]))), network.front()); + + gradient.submat(0, 0, transformWeight.n_elem - 1, 0) = arma::vectorise( + transformGateError * input.t()); + gradient.submat(transformWeight.n_elem, 0, gradient.n_elem - 1, 0) = + arma::sum(transformGateError, 1); +} + +template +template +void Highway::serialize( + Archive& ar, const unsigned int /* version */) +{ + // If loading, delete the old layers and set size for weights. + if (Archive::is_loading::value) + { + for (LayerTypes& layer : network) + { + boost::apply_visitor(deleteVisitor, layer); + } + weights.set_size(inSize * inSize + inSize, 1); + } + + ar & BOOST_SERIALIZATION_NVP(model); + ar & BOOST_SERIALIZATION_NVP(network); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/layer.hpp b/src/mlpack/methods/ann/layer/layer.hpp index 78abe96cf4..4769e503e1 100644 --- a/src/mlpack/methods/ann/layer/layer.hpp +++ b/src/mlpack/methods/ann/layer/layer.hpp @@ -19,6 +19,7 @@ #include "convolution.hpp" #include "dropconnect.hpp" #include "glimpse.hpp" +#include "highway.hpp" #include "layer_norm.hpp" #include "layer_types.hpp" #include "linear.hpp" diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index fc6bf6f69a..6ac78506d6 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -82,6 +82,12 @@ template class Sequential; +template +class Highway; + template*, Glimpse*, HardTanH*, + Highway*, Join*, LayerNorm*, LeakyReLU*, From 48a80671abea15d7f1af7e849c675c11fa697c4e Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Fri, 8 Mar 2019 03:13:41 +0530 Subject: [PATCH 060/143] Add tests for highway network --- src/mlpack/tests/ann_layer_test.cpp | 90 +++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 4356ac7440..1fc43891f1 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -2338,6 +2338,96 @@ BOOST_AUTO_TEST_CASE(SimpleResidualLayerTest) delete linearB; } +/** + * Simple highway module test. + */ +BOOST_AUTO_TEST_CASE(SimpleHighwayLayerTest) +{ + arma::mat outputA, outputB, input, deltaA, deltaB; + Sequential<>* sequential = new Sequential<>(true); + Highway<>* highway = new Highway<>(10, true); + highway->Parameters().zeros(); + highway->Reset(); + + Linear<>* linearA = new Linear<>(10, 10); + linearA->Parameters().randu(); + linearA->Reset(); + Linear<>* linearB = new Linear<>(10, 10); + linearB->Parameters().randu(); + linearB->Reset(); + + // Add the same layers (with the same parameters) to both Sequential and + // Residual object. + highway->Add(linearA); + highway->Add(linearB); + sequential->Add(linearA); + sequential->Add(linearB); + + // Test the Forward function (pass the same input to both). + input = arma::randu(10, 1); + sequential->Forward(std::move(input), std::move(outputA)); + highway->Forward(std::move(input), std::move(outputB)); + + CheckMatrices(outputB, input * 0.5 + outputA * 0.5); + + delete sequential; + delete highway; + delete linearA; + delete linearB; +} + +/** + * Sequential layer numerical gradient test. + */ +BOOST_AUTO_TEST_CASE(GradientHighwayLayerTest) +{ + // Linear function gradient instantiation. + struct GradientFunction + { + GradientFunction() + { + input = arma::randu(5, 1); + target = arma::mat("1"); + + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(5, 10); + highway = new Highway<>(10); + highway->Add >(10, 10); + highway->Add >(); + highway->Add >(10, 10); + highway->Add >(); + + model->Add(highway); + model->Add >(10, 2); + model->Add >(); + } + + ~GradientFunction() + { + highway->DeleteModules(); + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN, NguyenWidrowInitialization>* model; + Highway<>* highway; + arma::mat input, target; + } function; + + BOOST_REQUIRE_LE(CheckGradient(function), 1e-4); +} + /** * Sequential layer numerical gradient test. */ From b90ceec3444702a9182f9115c5da14c62a9f3d11 Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Fri, 8 Mar 2019 03:20:45 +0530 Subject: [PATCH 061/143] Style fixes --- src/mlpack/methods/ann/layer/highway.hpp | 5 ++--- src/mlpack/methods/ann/layer/highway_impl.hpp | 18 ++++++++++++------ src/mlpack/tests/ann_layer_test.cpp | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/mlpack/methods/ann/layer/highway.hpp b/src/mlpack/methods/ann/layer/highway.hpp index 460959a731..f19707a4b5 100644 --- a/src/mlpack/methods/ann/layer/highway.hpp +++ b/src/mlpack/methods/ann/layer/highway.hpp @@ -162,10 +162,10 @@ class Highway void serialize(Archive& ar, const unsigned int /* version */); private: - //! Locally-stored number of input units. + //! Locally-stored number of input units. size_t inSize; - //! Parameter which indicates if the modules should be exposed. + //! Parameter which indicates if the modules should be exposed. bool model; //! Indicator if we already initialized the model. @@ -230,7 +230,6 @@ class Highway //! Locally-stored output height visitor. OutputHeightVisitor outputHeightVisitor; - }; // class Highway } // namespace ann diff --git a/src/mlpack/methods/ann/layer/highway_impl.hpp b/src/mlpack/methods/ann/layer/highway_impl.hpp index 9a6c41f1fa..7dda61b194 100644 --- a/src/mlpack/methods/ann/layer/highway_impl.hpp +++ b/src/mlpack/methods/ann/layer/highway_impl.hpp @@ -24,8 +24,14 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -Highway::Highway() +template +Highway::Highway() : + inSize(0), + model(true), + reset(false), + width(0), + height(0) { // Nothing to do here. } @@ -44,8 +50,8 @@ Highway::Highway( weights.set_size(inSize * inSize + inSize, 1); } -template +template Highway::~Highway() { if (!model) @@ -55,8 +61,8 @@ Highway::~Highway() } } -template +template void Highway< InputDataType, OutputDataType, CustomLayers...>::DeleteModules() { diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 1fc43891f1..8b4ff96c04 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -2357,7 +2357,7 @@ BOOST_AUTO_TEST_CASE(SimpleHighwayLayerTest) linearB->Reset(); // Add the same layers (with the same parameters) to both Sequential and - // Residual object. + // Highway object. highway->Add(linearA); highway->Add(linearB); sequential->Add(linearA); From 16922821e50c2e3329a6ed204898b1e7ded841ed Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Fri, 8 Mar 2019 21:43:20 +0530 Subject: [PATCH 062/143] Add Konstantin as author --- src/mlpack/methods/ann/layer/highway.hpp | 1 + src/mlpack/methods/ann/layer/highway_impl.hpp | 1 + 2 files changed, 2 insertions(+) diff --git a/src/mlpack/methods/ann/layer/highway.hpp b/src/mlpack/methods/ann/layer/highway.hpp index f19707a4b5..a21b589a12 100644 --- a/src/mlpack/methods/ann/layer/highway.hpp +++ b/src/mlpack/methods/ann/layer/highway.hpp @@ -1,5 +1,6 @@ /** * @file highway.hpp + * @author Konstantin Sidorov * @author Saksham Bansal * * Definition of highway layer first introduced in the paper "Highway networks" diff --git a/src/mlpack/methods/ann/layer/highway_impl.hpp b/src/mlpack/methods/ann/layer/highway_impl.hpp index 7dda61b194..04c66d2fa4 100644 --- a/src/mlpack/methods/ann/layer/highway_impl.hpp +++ b/src/mlpack/methods/ann/layer/highway_impl.hpp @@ -1,5 +1,6 @@ /** * @file highway_impl.hpp + * @author Konstantin Sidorov * @author Saksham Bansal * * Implementation of highway layer. From 187d804c2b82bf437201fb074c87276db629970b Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Sun, 17 Mar 2019 17:43:18 +0530 Subject: [PATCH 063/143] Refractor and add Highway FNN test --- src/mlpack/tests/feedforward_network_test.cpp | 293 ++++++++---------- 1 file changed, 125 insertions(+), 168 deletions(-) diff --git a/src/mlpack/tests/feedforward_network_test.cpp b/src/mlpack/tests/feedforward_network_test.cpp index 1055ff3c08..c6ce1b0b1d 100644 --- a/src/mlpack/tests/feedforward_network_test.cpp +++ b/src/mlpack/tests/feedforward_network_test.cpp @@ -29,47 +29,17 @@ using namespace mlpack::ann; BOOST_AUTO_TEST_SUITE(FeedForwardNetworkTest); /** - * Train and evaluate a vanilla network with the specified structure. + * Train and evaluate a model with the specified structure. */ -template -void BuildVanillaNetwork(MatType& trainData, - MatType& trainLabels, - MatType& testData, - MatType& testLabels, - const size_t outputSize, - const size_t hiddenLayerSize, - const size_t maxEpochs, - const double classificationErrorThreshold) +template +void TestNetwork(ModelType& model, + MatType& trainData, + MatType& trainLabels, + MatType& testData, + MatType& testLabels, + const size_t maxEpochs, + const double classificationErrorThreshold) { - /* - * Construct a feed forward network with trainData.n_rows input nodes, - * hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The - * network structure looks like: - * - * Input Hidden Output - * Layer Layer Layer - * +-----+ +-----+ +-----+ - * | | | | | | - * | +------>| +------>| | - * | | +>| | +>| | - * +-----+ | +--+--+ | +-----+ - * | | - * Bias | Bias | - * Layer | Layer | - * +-----+ | +-----+ | - * | | | | | | - * | +-----+ | +-----+ - * | | | | - * +-----+ +-----+ - */ - - FFN > model; - model.Add >(trainData.n_rows, hiddenLayerSize); - model.Add >(); - model.Add >(hiddenLayerSize, outputSize); - model.Add >(); - - // RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1); ens::RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1); model.Train(trainData, trainLabels, opt); @@ -115,11 +85,39 @@ BOOST_AUTO_TEST_CASE(VanillaNetworkTest) arma::mat testLabels = testData.row(testData.n_rows - 1); testData.shed_row(testData.n_rows - 1); + /* + * Construct a feed forward network with trainData.n_rows input nodes, + * hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The + * network structure looks like: + * + * Input Hidden Output + * Layer Layer Layer + * +-----+ +-----+ +-----+ + * | | | | | | + * | +------>| +------>| | + * | | +>| | +>| | + * +-----+ | +--+--+ | +-----+ + * | | + * Bias | Bias | + * Layer | Layer | + * +-----+ | +-----+ | + * | | | | | | + * | +-----+ | +-----+ + * | | | | + * +-----+ +-----+ + */ + + FFN > model; + model.Add >(trainData.n_rows, 8); + model.Add >(); + model.Add >(8, 3); + model.Add >(); + // Vanilla neural net with logistic activation function. // Because 92 percent of the patients are not hyperthyroid the neural // network must be significant better than 92%. - BuildVanillaNetwork<> - (trainData, trainLabels, testData, testLabels, 3, 8, 10, 0.1); + TestNetwork<> + (model, trainData, trainLabels, testData, testLabels, 10, 0.1); arma::mat dataset; dataset.load("mnist_first250_training_4s_and_9s.arm"); @@ -132,9 +130,14 @@ BOOST_AUTO_TEST_CASE(VanillaNetworkTest) labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); labels += 1; + FFN > model1; + model1.Add >(dataset.n_rows, 10); + model1.Add >(); + model1.Add >(10, 2); + model1.Add >(); // Vanilla neural net with logistic activation function. - BuildVanillaNetwork<> - (dataset, labels, dataset, labels, 2, 10, 10, 0.2); + TestNetwork<> + (model1, dataset, labels, dataset, labels, 10, 0.2); } BOOST_AUTO_TEST_CASE(ForwardBackwardTest) @@ -214,18 +217,23 @@ BOOST_AUTO_TEST_CASE(ForwardBackwardTest) } /** - * Train and evaluate a Dropout network with the specified structure. + * Train the dropout network on a larger dataset. */ -template -void BuildDropoutNetwork(MatType& trainData, - MatType& trainLabels, - MatType& testData, - MatType& testLabels, - const size_t outputSize, - const size_t hiddenLayerSize, - const size_t maxEpochs, - const double classificationErrorThreshold) +BOOST_AUTO_TEST_CASE(DropoutNetworkTest) { + // Load the dataset. + arma::mat trainData; + data::Load("thyroid_train.csv", trainData, true); + + arma::mat trainLabels = trainData.row(trainData.n_rows - 1); + trainData.shed_row(trainData.n_rows - 1); + + arma::mat testData; + data::Load("thyroid_test.csv", testData, true); + + arma::mat testLabels = testData.row(testData.n_rows - 1); + testData.shed_row(testData.n_rows - 1); + /* * Construct a feed forward network with trainData.n_rows input nodes, * hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The @@ -249,44 +257,70 @@ void BuildDropoutNetwork(MatType& trainData, */ FFN > model; - model.Add >(trainData.n_rows, hiddenLayerSize); + model.Add >(trainData.n_rows, 8); model.Add >(); model.Add >(); - model.Add >(hiddenLayerSize, outputSize); + model.Add >(8, 3); model.Add >(); - ens::RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1); + // Vanilla neural net with logistic activation function. + // Because 92 percent of the patients are not hyperthyroid the neural + // network must be significant better than 92%. + TestNetwork<>(model, trainData, trainLabels, testData, testLabels, 10, 0.1); + arma::mat dataset; + dataset.load("mnist_first250_training_4s_and_9s.arm"); - model.Train(trainData, trainLabels, opt); + // Normalize each point since these are images. + for (size_t i = 0; i < dataset.n_cols; ++i) + dataset.col(i) /= norm(dataset.col(i), 2); - MatType predictionTemp; - model.Predict(testData, predictionTemp); - MatType prediction = arma::zeros(1, predictionTemp.n_cols); + arma::mat labels = arma::zeros(1, dataset.n_cols); + labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); + labels += 1; - for (size_t i = 0; i < predictionTemp.n_cols; ++i) - { - prediction(i) = arma::as_scalar(arma::find( - arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1; - } - - size_t error = 0; - for (size_t i = 0; i < testData.n_cols; i++) - { - if (int(arma::as_scalar(prediction.col(i))) == - int(arma::as_scalar(testLabels.col(i)))) - { - error++; - } - } - - double classificationError = 1 - double(error) / testData.n_cols; - BOOST_REQUIRE_LE(classificationError, classificationErrorThreshold); + FFN > model1; + model1.Add >(dataset.n_rows, 10); + model1.Add >(); + model.Add >(); + model1.Add >(10, 2); + model1.Add >(); + // Vanilla neural net with logistic activation function. + TestNetwork<> + (model1, dataset, labels, dataset, labels, 10, 0.2); } /** - * Train the dropout network on a larger dataset. + * Train the highway network on a larger dataset. */ -BOOST_AUTO_TEST_CASE(DropoutNetworkTest) +BOOST_AUTO_TEST_CASE(HighwayNetworkTest) +{ + arma::mat dataset; + dataset.load("mnist_first250_training_4s_and_9s.arm"); + + // Normalize each point since these are images. + for (size_t i = 0; i < dataset.n_cols; ++i) + dataset.col(i) /= norm(dataset.col(i), 2); + + arma::mat labels = arma::zeros(1, dataset.n_cols); + labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); + labels += 1; + + FFN > model; + model.Add >(dataset.n_rows, 10); + Highway<>* highway = new Highway<>(10, true); + highway->Add >(10, 10); + highway->Add >(); + model.Add(highway); + model.Add >(10, 2); + model.Add >(); + TestNetwork<> + (model, dataset, labels, dataset, labels, 10, 0.2); +} + +/** + * Train the dropconnect network on a larger dataset. + */ +BOOST_AUTO_TEST_CASE(DropConnectNetworkTest) { // Load the dataset. arma::mat trainData; @@ -301,42 +335,6 @@ BOOST_AUTO_TEST_CASE(DropoutNetworkTest) arma::mat testLabels = testData.row(testData.n_rows - 1); testData.shed_row(testData.n_rows - 1); - // Vanilla neural net with logistic activation function. - // Because 92 percent of the patients are not hyperthyroid the neural - // network must be significant better than 92%. - BuildDropoutNetwork<> - (trainData, trainLabels, testData, testLabels, 3, 8, 10, 0.1); - - arma::mat dataset; - dataset.load("mnist_first250_training_4s_and_9s.arm"); - - // Normalize each point since these are images. - for (size_t i = 0; i < dataset.n_cols; ++i) - dataset.col(i) /= norm(dataset.col(i), 2); - - arma::mat labels = arma::zeros(1, dataset.n_cols); - labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); - labels += 1; - - // Vanilla neural net with logistic activation function. - BuildDropoutNetwork<> - (dataset, labels, dataset, labels, 2, 10, 10, 0.2); -} - -/** - * Train and evaluate a DropConnect network(with a baselayer) with the - * specified structure. - */ -template -void BuildDropConnectNetwork(MatType& trainData, - MatType& trainLabels, - MatType& testData, - MatType& testLabels, - const size_t outputSize, - const size_t hiddenLayerSize, - const size_t maxEpochs, - const double classificationErrorThreshold) -{ /* * Construct a feed forward network with trainData.n_rows input nodes, * hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The @@ -362,62 +360,16 @@ void BuildDropConnectNetwork(MatType& trainData, */ FFN > model; - model.Add >(trainData.n_rows, hiddenLayerSize); + model.Add >(trainData.n_rows, 8); model.Add >(); - model.Add >(hiddenLayerSize, outputSize); + model.Add >(8, 3); model.Add >(); - ens::RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1); - - model.Train(trainData, trainLabels, opt); - - MatType predictionTemp; - model.Predict(testData, predictionTemp); - MatType prediction = arma::zeros(1, predictionTemp.n_cols); - - for (size_t i = 0; i < predictionTemp.n_cols; ++i) - { - prediction(i) = arma::as_scalar(arma::find( - arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1; - } - - size_t error = 0; - for (size_t i = 0; i < testData.n_cols; i++) - { - if (int(arma::as_scalar(prediction.col(i))) == - int(arma::as_scalar(testLabels.col(i)))) - { - error++; - } - } - - double classificationError = 1 - double(error) / testData.n_cols; - BOOST_REQUIRE_LE(classificationError, classificationErrorThreshold); -} - -/** - * Train the dropconnect network on a larger dataset. - */ -BOOST_AUTO_TEST_CASE(DropConnectNetworkTest) -{ - // Load the dataset. - arma::mat trainData; - data::Load("thyroid_train.csv", trainData, true); - - arma::mat trainLabels = trainData.row(trainData.n_rows - 1); - trainData.shed_row(trainData.n_rows - 1); - - arma::mat testData; - data::Load("thyroid_test.csv", testData, true); - - arma::mat testLabels = testData.row(testData.n_rows - 1); - testData.shed_row(testData.n_rows - 1); - // Vanilla neural net with logistic activation function. // Because 92 percent of the patients are not hyperthyroid the neural // network must be significant better than 92%. - BuildDropConnectNetwork<> - (trainData, trainLabels, testData, testLabels, 3, 8, 10, 0.1); + TestNetwork<> + (model, trainData, trainLabels, testData, testLabels, 10, 0.1); arma::mat dataset; dataset.load("mnist_first250_training_4s_and_9s.arm"); @@ -430,9 +382,14 @@ BOOST_AUTO_TEST_CASE(DropConnectNetworkTest) labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); labels += 1; + FFN > model1; + model1.Add >(dataset.n_rows, 10); + model1.Add >(); + model1.Add >(10, 2); + model1.Add >(); // Vanilla neural net with logistic activation function. - BuildDropConnectNetwork<> - (dataset, labels, dataset, labels, 2, 10, 10, 0.2); + TestNetwork<> + (model1, dataset, labels, dataset, labels, 10, 0.2); } /** From ff80f334286d3525e28e91bc5a57857cce4a8fcc Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Tue, 2 Apr 2019 23:04:27 +0530 Subject: [PATCH 064/143] Add description for highway layer --- .../methods/ann/layer/alpha_dropout.hpp | 1 - src/mlpack/methods/ann/layer/highway.hpp | 20 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/ann/layer/alpha_dropout.hpp b/src/mlpack/methods/ann/layer/alpha_dropout.hpp index dc1640c54f..4edbf3031c 100644 --- a/src/mlpack/methods/ann/layer/alpha_dropout.hpp +++ b/src/mlpack/methods/ann/layer/alpha_dropout.hpp @@ -37,7 +37,6 @@ namespace ann /** Artificial Neural Network. */ { * journal = {Advances in Neural Information Processing Systems}, * year = {2017} * } - * } * @endcode * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, diff --git a/src/mlpack/methods/ann/layer/highway.hpp b/src/mlpack/methods/ann/layer/highway.hpp index a21b589a12..51addb635f 100644 --- a/src/mlpack/methods/ann/layer/highway.hpp +++ b/src/mlpack/methods/ann/layer/highway.hpp @@ -3,8 +3,7 @@ * @author Konstantin Sidorov * @author Saksham Bansal * - * Definition of highway layer first introduced in the paper "Highway networks" - * https://arxiv.org/abs/1505.00387 + * Definition of the highway layer. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -31,7 +30,22 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * The highway layer class. + * Implementation of the Highway layer. The highway class can vary its behavior + * between that of feed-forward fully connected network container and that + * of a layer which simply passes its inputs through depending on the transform + * gate. Note that the size of the input and output matrices of this class + * should be equal. + * + * For more information, refer the following paper. + * + * @code + * @article{Srivastava2015, + * author = {Rupesh Kumar Srivastava, Klaus Greff, Jurgen Schmidhuber}, + * title = {Training Very Deep Networks}, + * journal = {Advances in Neural Information Processing Systems}, + * year = {2015}, + * } + * @endcode * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). From fc51a2fc87d854e53b1998c0df8d08351816995e Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Wed, 29 May 2019 09:43:06 +0530 Subject: [PATCH 065/143] Added Continuous Multiple Pole Cart. Also included Manish's suggestions. --- .../continuous_multiple_pole_cart.hpp | 268 ++++++++++++++++++ .../environment/multiple_pole_cart.hpp | 15 +- src/mlpack/tests/rl_components_test.cpp | 22 ++ 3 files changed, 297 insertions(+), 8 deletions(-) create mode 100644 src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp new file mode 100644 index 0000000000..d7ba2b219e --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp @@ -0,0 +1,268 @@ +/** + * @file continuous_multiple_pole_cart.hpp + * @author Rahul Ganesh Prabhu + * + * This file is an implementation of Continuous Multiple Pole Cart Balancing + * Task. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_METHODS_RL_ENVIRONMENT_CONTINUOUS_MULTIPLE_POLE_CART_HPP +#define MLPACK_METHODS_RL_ENVIRONMENT_CONTINUOUS_MULTIPLE_POLE_CART_HPP + +#include + +namespace mlpack { +namespace rl { + +/** + * Implementation of Continuous Multiple Pole Cart Balancing task. + */ +class ContinuousMultiplePoleCart +{ + public: + /** + * Implementation of the state of Continuous Multiple Pole Cart. The state is expressed as + * a matrix where the $0^{th}$ column is the state of the cart, represented by a tuple + * (position, velocity) and the $i^{th}$ column is the state of the $i^{th}$ pole, represented + * by a tuple (angle, angular velocity). + */ + class State + { + public: + /** + * Construct a state instance. + * + * @param numPoles The number of poles. + */ + State(const size_t numPoles) + { + data = arma::zeros(dimension, numPoles); + } + + /** + * Construct a state instance from given data. + * + * @param data Data for the position, velocity, angle and angular velocity. + */ + State(const arma::mat& data) : data(data) + { /* Nothing to do here */ } + + //! Modify the internal representation of the state. + arma::mat& Data() { return data; } + + //! Get the position of the cart. + double Position() const { return data(0, 0); } + //! Modify the position of the cart. + double& Position() { return data(0, 0); } + + //! Get the velocity of the cart. + double Velocity() const { return data(1, 0); } + //! Modify the velocity of the cart. + double& Velocity() { return data(1, 0); } + + //! Get the angle of the $i^{th}$ pole with the vertical. + double Angle(const size_t i) const { return data(0, i); } + //! Modify the angle of the $i^{th}$ pole with the vertical. + double& Angle(const size_t i) { return data(0, i); } + + //! Get the angular velocity of the $i^{th}$ pole. + double AngularVelocity(const size_t i) const { return data(1, i); } + //! Modify the angular velocity of the $i^{th}$ pole. + double& AngularVelocity(const size_t i) { return data(1, i); } + + //! Encode the state to a matrix. + const arma::mat& Encode() const { return data; } + + //! Dimension of the encoded state. + const size_t dimension = 2; + + private: + //! Locally-stored state data. + arma::mat data; + }; + + /** + * Implementation of action of Continuous Multiple Pole Cart. + */ + struct Action + { + double action[1]; + // Track the size of the action space. + const int size = 1; + }; + + /** + * Construct a Multiple Pole Cart instance using the given constants. + * + * @param poleNum The number of poles + * @param gravity The gravity constant. + * @param massCart The mass of the cart. + * @param massPole The mass of the pole. + * @param length The length of the pole. + * @param tau The time interval. + * @param thetaThresholdRadians The maximum angle. + * @param xThreshold The maximum position. + * @param doneReward The reward recieved on termination. + */ + ContinuousMultiplePoleCart(const size_t poleNum, + const arma::vec& poleLengths, + const arma::vec& poleMasses, + const double gravity = 9.8, + const double massCart = 1.0, + const double tau = 0.02, + const double thetaThresholdRadians = 12 * 2 * 3.1416 / 360, + const double xThreshold = 2.4, + const double doneReward = 0.0) : + poleNum(poleNum), + poleLengths(poleLengths), + poleMasses(poleMasses), + gravity(gravity), + massCart(massCart), + tau(tau), + thetaThresholdRadians(thetaThresholdRadians), + xThreshold(xThreshold), + doneReward(doneReward) + { + if (poleNum != poleLengths.n_elem) + { + Log::Fatal << "The number of lengths should be the same as the number of" + "poles." << std::endl; + } + if (poleNum != poleMasses.n_elem) + { + Log::Fatal << "The number of masses should be the same as the number of" + "poles." << std::endl; + } + } + + /** + * Dynamics of Continuous Multiple Pole Cart instance. Get reward and next state + * based on current state and current action. + * + * @param state The current state. + * @param action The current action. + * @param nextState The next state. + * @return reward, it's always 1.0. + */ + double Sample(const State& state, + const Action& action, + State& nextState) const + { + // Calculate acceleration. + double totalForce = action.action[0]; + double totalMass = massCart; + for (size_t i = 0; i < poleNum; i++) + { + double poleOmega = state.AngularVelocity(i); + double sinTheta = sin(state.Angle(i)); + totalForce += (poleMasses[i] * poleLengths[i] * poleOmega * poleOmega * + sinTheta) + 0.75 * poleMasses[i] * gravity * sin(2 * state.Angle(i)) + / 2; + totalMass += poleMasses[i] * (0.25 + 0.75 * sinTheta * sinTheta); + } + double xAcc = totalForce / totalMass; + + // Update states of the poles. + for (size_t i = 0; i < poleNum; i++) + { + double sinTheta = sin(state.Angle(i)); + double cosTheta = cos(state.Angle(i)); + nextState.Angle(i) = state.Angle(i) + tau * state.AngularVelocity(i); + nextState.AngularVelocity(i) = state.AngularVelocity(i) - tau * 0.75 * + (xAcc * cosTheta + gravity * sinTheta) / poleLengths[i]; + } + + // Update state of the cart. + nextState.Position() = state.Position() + tau * state.Velocity(); + nextState.Velocity() = state.Velocity() + tau * xAcc; + + /** + * It is important to note that if the cartpole is falling down, it should + * be penalized. + */ + bool done = IsTerminal(nextState); + if (done) + return doneReward; + /** + * When done is false, it means that the cartpole has fallen down. + * For this case the reward is 1.0. + */ + return 1.0; + } + + /** + * Dynamics of Continuous Multiple Pole Cart. Get reward based on current + * state and current action. + * + * @param state The current state. + * @param action The current action. + * @return reward, it's always 1.0. + */ + double Sample(const State& state, const Action& action) const + { + State nextState(poleNum); + return Sample(state, action, nextState); + } + + /** + * Initial state representation is randomly generated within [-0.05, 0.05]. + * + * @return Initial state for each episode. + */ + State InitialSample() const + { + return State((arma::randu(2, poleNum) - 0.5) / 10.0); + } + + /** + * Whether given state is a terminal state. + * + * @param state The desired state. + * @return true if state is a terminal state, otherwise false. + */ + bool IsTerminal(const State& state) const + { + for (size_t i = 0; i < poleNum; i++) + if (std::abs(state.Angle(i)) > thetaThresholdRadians) + return true; + return std::abs(state.Position()) > xThreshold; + } + + private: + //! Locally-stored number of poles. + size_t poleNum; + + //! Locally-stored length of poles. + arma::vec poleLengths; + + //! Locally-stored mass of the pole. + arma::vec poleMasses; + + //! Locally-stored gravity. + double gravity; + + //! Locally-stored mass of the cart. + double massCart; + + //! Locally-stored time interval. + double tau; + + //! Locally-stored maximum angle. + double thetaThresholdRadians; + + //! Locally-stored maximum position. + double xThreshold; + + //! Locally-stored done reward. + double doneReward; +}; + +} // namespace rl +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp index 3257ab6432..c591cd1275 100755 --- a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp @@ -9,8 +9,9 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_METHODS_RL_ENVIRONMENT_MULTIPE_POLE_CART_HPP -#define MLPACK_METHODS_RL_ENVIRONMENT_MULTIPE_POLE_CART_HPP + +#ifndef MLPACK_METHODS_RL_ENVIRONMENT_MULTIPLE_POLE_CART_HPP +#define MLPACK_METHODS_RL_ENVIRONMENT_MULTIPLE_POLE_CART_HPP #include @@ -85,7 +86,7 @@ class MultiplePoleCart }; /** - * Implementation of action of Cart Pole. + * Implementation of action of Multiple Pole Cart. */ enum Action { @@ -163,12 +164,10 @@ class MultiplePoleCart { double poleOmega = state.AngularVelocity(i); double sinTheta = sin(state.Angle(i)); - double cosTheta = cos(state.Angle(i)); totalForce += (poleMasses[i] * poleLengths[i] * poleOmega * poleOmega * sinTheta) + 0.75 * poleMasses[i] * gravity * sin(2 * state.Angle(i)) / 2; - totalMass += poleMasses[i] * (1 - 0.75 * (cos(2 * state.Angle(i)) + 1) - / 2); + totalMass += poleMasses[i] * (0.25 + 0.75 * sinTheta * sinTheta); } double xAcc = totalForce / totalMass; @@ -178,8 +177,8 @@ class MultiplePoleCart double sinTheta = sin(state.Angle(i)); double cosTheta = cos(state.Angle(i)); nextState.Angle(i) = state.Angle(i) + tau * state.AngularVelocity(i); - nextState.AngularVelocity(i) += -tau * 0.75 * (xAcc * cosTheta + - gravity * sinTheta) / poleLengths[i]; + nextState.AngularVelocity(i) = state.AngularVelocity(i) - tau * 0.75 * + (xAcc * cosTheta + gravity * sinTheta) / poleLengths[i]; } // Update state of the cart. diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 3b4b3b2373..6d3ba392f6 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -138,6 +139,27 @@ BOOST_AUTO_TEST_CASE(MultiplePoleCartTest) BOOST_REQUIRE_EQUAL(2, MultiplePoleCart::Action::size); } +/** + * Constructs a ContinuousMultiplePoleCart instance and check if the main + * routine works as it should be. + */ +BOOST_AUTO_TEST_CASE(ContinuousMultiplePoleCartTest) +{ + arma::vec poleLengths = {1, 0.5}; + arma::vec poleMasses = {1, 1}; + const ContinuousMultiplePoleCart task = ContinuousMultiplePoleCart(2, + poleLengths, poleMasses); + + ContinuousMultiplePoleCart::State state = task.InitialSample(); + ContinuousMultiplePoleCart::Action action; + action.action[0] = math::Random(-1.0, 1.0); + double reward = task.Sample(state, action); + + BOOST_REQUIRE_EQUAL(reward, 1.0); + BOOST_REQUIRE(!task.IsTerminal(state)); + BOOST_REQUIRE_EQUAL(1, action.size); +} + /** * Construct a random replay instance and check if it works as * it should be. From 863d22cadd86229fc1d31a6c6206ccd8f3b88f2d Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Wed, 29 May 2019 10:26:08 +0530 Subject: [PATCH 066/143] Added to CMakeLists. --- .../methods/reinforcement_learning/environment/CMakeLists.txt | 2 ++ .../environment/continuous_multiple_pole_cart.hpp | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/CMakeLists.txt b/src/mlpack/methods/reinforcement_learning/environment/CMakeLists.txt index 3aabc6373c..626f3408bf 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/CMakeLists.txt +++ b/src/mlpack/methods/reinforcement_learning/environment/CMakeLists.txt @@ -4,6 +4,8 @@ set(SOURCES mountain_car.hpp cart_pole.hpp continuous_mountain_car.hpp + multiple_pole_cart.hpp + continuous_multiple_pole_cart.hpp acrobot.hpp pendulum.hpp reward_clipping.hpp diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp index d7ba2b219e..97eb67ffa5 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp @@ -115,7 +115,8 @@ class ContinuousMultiplePoleCart const double gravity = 9.8, const double massCart = 1.0, const double tau = 0.02, - const double thetaThresholdRadians = 12 * 2 * 3.1416 / 360, + const double thetaThresholdRadians = 12 * 2 * + 3.1416 / 360, const double xThreshold = 2.4, const double doneReward = 0.0) : poleNum(poleNum), From 91578d5ca29d9a8691fd55779531c34676fcc32d Mon Sep 17 00:00:00 2001 From: walragatver Date: Wed, 29 May 2019 15:06:05 +0530 Subject: [PATCH 067/143] Fix serialization of glimpse, maxPooling and meanPooling layers. --- src/mlpack/methods/ann/layer/glimpse_impl.hpp | 3 +++ src/mlpack/methods/ann/layer/max_pooling_impl.hpp | 4 ++++ src/mlpack/methods/ann/layer/mean_pooling_impl.hpp | 4 ++++ 3 files changed, 11 insertions(+) diff --git a/src/mlpack/methods/ann/layer/glimpse_impl.hpp b/src/mlpack/methods/ann/layer/glimpse_impl.hpp index 774cede8ae..a867656895 100644 --- a/src/mlpack/methods/ann/layer/glimpse_impl.hpp +++ b/src/mlpack/methods/ann/layer/glimpse_impl.hpp @@ -224,6 +224,9 @@ void Glimpse::serialize( ar & BOOST_SERIALIZATION_NVP(depth); ar & BOOST_SERIALIZATION_NVP(scale); ar & BOOST_SERIALIZATION_NVP(inputWidth); + ar & BOOST_SERIALIZATION_NVP(inputHeight); + ar & BOOST_SERIALIZATION_NVP(outputWidth); + ar & BOOST_SERIALIZATION_NVP(outputHeight); ar & BOOST_SERIALIZATION_NVP(location); } diff --git a/src/mlpack/methods/ann/layer/max_pooling_impl.hpp b/src/mlpack/methods/ann/layer/max_pooling_impl.hpp index 0afc01d135..d0efb50830 100644 --- a/src/mlpack/methods/ann/layer/max_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/max_pooling_impl.hpp @@ -148,6 +148,10 @@ void MaxPooling::serialize( ar & BOOST_SERIALIZATION_NVP(dW); ar & BOOST_SERIALIZATION_NVP(dH); ar & BOOST_SERIALIZATION_NVP(batchSize); + ar & BOOST_SERIALIZATION_NVP(inputWidth); + ar & BOOST_SERIALIZATION_NVP(inputHeight); + ar & BOOST_SERIALIZATION_NVP(outputWidth); + ar & BOOST_SERIALIZATION_NVP(outputHeight); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp index a5dfedf175..b4dcf282f8 100644 --- a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp @@ -122,6 +122,10 @@ void MeanPooling::serialize( ar & BOOST_SERIALIZATION_NVP(dW); ar & BOOST_SERIALIZATION_NVP(dH); ar & BOOST_SERIALIZATION_NVP(batchSize); + ar & BOOST_SERIALIZATION_NVP(inputWidth); + ar & BOOST_SERIALIZATION_NVP(inputHeight); + ar & BOOST_SERIALIZATION_NVP(outputWidth); + ar & BOOST_SERIALIZATION_NVP(outputHeight); } } // namespace ann From 781655df585f0c8cca864757fa76dfd060b0e735 Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Tue, 28 May 2019 17:45:18 +0700 Subject: [PATCH 068/143] Style fix --- src/mlpack/methods/ann/layer/highway.hpp | 10 +++++----- src/mlpack/methods/ann/layer/highway_impl.hpp | 2 +- src/mlpack/methods/ann/layer/layer_types.hpp | 4 ++-- src/mlpack/tests/ann_layer_test.cpp | 1 + 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/mlpack/methods/ann/layer/highway.hpp b/src/mlpack/methods/ann/layer/highway.hpp index 51addb635f..8c2b09efbc 100644 --- a/src/mlpack/methods/ann/layer/highway.hpp +++ b/src/mlpack/methods/ann/layer/highway.hpp @@ -103,7 +103,7 @@ class Highway * @param g The calculated gradient. */ template - void Backward(const arma::Mat&& input, + void Backward(const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g); @@ -207,13 +207,13 @@ class Highway //! Bias for transformation of output. OutputDataType transformBias; - //! Locally-stored transform gate parameters + //! Locally-stored transform gate parameters. OutputDataType transformGate; - //! Locally-stored transform gate activation + //! Locally-stored transform gate activation. OutputDataType transformGateActivation; - //! Locally-stored transform gate activation + //! Locally-stored transform gate error. OutputDataType transformGateError; //! Locally-stored input parameter object. @@ -228,7 +228,7 @@ class Highway //! The input height. size_t height; - //! The normal output without highway network + //! The normal output without highway network. OutputDataType networkOutput; //! Locally-stored delta visitor. diff --git a/src/mlpack/methods/ann/layer/highway_impl.hpp b/src/mlpack/methods/ann/layer/highway_impl.hpp index 04c66d2fa4..22b1a187e0 100644 --- a/src/mlpack/methods/ann/layer/highway_impl.hpp +++ b/src/mlpack/methods/ann/layer/highway_impl.hpp @@ -164,7 +164,7 @@ template template void Highway::Backward( - const arma::Mat&& input, + const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 6ac78506d6..5e98f11a01 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -198,14 +198,14 @@ using LayerTypes = boost::variant< NegativeLogLikelihood*, PReLU*, Recurrent*, - RecurrentAttention*, + // RecurrentAttention*, ReinforceNormal*, Reparametrization*, Select*, Sequential*, Sequential*, Subview*, - VRClassReward*, + // VRClassReward*, CustomLayers*... >; diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 8b4ff96c04..5d4fd0e701 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -2394,6 +2394,7 @@ BOOST_AUTO_TEST_CASE(GradientHighwayLayerTest) model->Responses() = target; model->Add >(); model->Add >(5, 10); + highway = new Highway<>(10); highway->Add >(10, 10); highway->Add >(); From b116647376d48ca02b8f97d5e97a749e3d5ffd8b Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 29 May 2019 08:17:08 -0400 Subject: [PATCH 069/143] Fix link; thanks rajs123 for pointing it out. --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d8b7b3df44..b377a24e69 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,8 +5,8 @@ contribute to mlpack and join the community! If you would like to make improvements to the library, add new features that are useful to you and others, or have found a bug that you know how to fix, please submit a pull request! -If you would like to learn more about how to get started contributing, see -[Getting Involved](http://www.mlpack.org/involved.html), and if you are +If you would like to learn more about how to get started contributing, see the +[Community](http://www.mlpack.org/community.html) page, and if you are interested in participating in Google Summer of Code, see [mlpack and Google Summer of Code](http://www.mlpack.org/gsoc.html). From f3318fd0f2597e3375a03d630f782b58911b3c3c Mon Sep 17 00:00:00 2001 From: walragatver Date: Thu, 30 May 2019 16:11:10 +0530 Subject: [PATCH 070/143] Serialize floor parameter in mean_pool and max_pool layer. --- src/mlpack/methods/ann/layer/max_pooling_impl.hpp | 1 + src/mlpack/methods/ann/layer/mean_pooling_impl.hpp | 1 + 2 files changed, 2 insertions(+) diff --git a/src/mlpack/methods/ann/layer/max_pooling_impl.hpp b/src/mlpack/methods/ann/layer/max_pooling_impl.hpp index d0efb50830..8006fbd7fb 100644 --- a/src/mlpack/methods/ann/layer/max_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/max_pooling_impl.hpp @@ -148,6 +148,7 @@ void MaxPooling::serialize( ar & BOOST_SERIALIZATION_NVP(dW); ar & BOOST_SERIALIZATION_NVP(dH); ar & BOOST_SERIALIZATION_NVP(batchSize); + ar & BOOST_SERIALIZATION_NVP(floor); ar & BOOST_SERIALIZATION_NVP(inputWidth); ar & BOOST_SERIALIZATION_NVP(inputHeight); ar & BOOST_SERIALIZATION_NVP(outputWidth); diff --git a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp index b4dcf282f8..e7dc4e500a 100644 --- a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp @@ -122,6 +122,7 @@ void MeanPooling::serialize( ar & BOOST_SERIALIZATION_NVP(dW); ar & BOOST_SERIALIZATION_NVP(dH); ar & BOOST_SERIALIZATION_NVP(batchSize); + ar & BOOST_SERIALIZATION_NVP(floor); ar & BOOST_SERIALIZATION_NVP(inputWidth); ar & BOOST_SERIALIZATION_NVP(inputHeight); ar & BOOST_SERIALIZATION_NVP(outputWidth); From 1f77382216f915ee0f1f6ca5d8df13ba802770a2 Mon Sep 17 00:00:00 2001 From: robotcator Date: Fri, 31 May 2019 16:21:01 +0800 Subject: [PATCH 071/143] remove move() to avoid warning --- .../tests/main_tests/hoeffding_tree_test.cpp | 146 +++++++++--------- 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/src/mlpack/tests/main_tests/hoeffding_tree_test.cpp b/src/mlpack/tests/main_tests/hoeffding_tree_test.cpp index 2678a48fdd..30cf035f7d 100644 --- a/src/mlpack/tests/main_tests/hoeffding_tree_test.cpp +++ b/src/mlpack/tests/main_tests/hoeffding_tree_test.cpp @@ -66,25 +66,25 @@ BOOST_AUTO_TEST_CASE(HoeffdingTreeOutputDimensionTest) size_t testSize = testData.n_cols; // Input training data. - SetInputParam("training", std::move(std::make_tuple(info, inputData))); + SetInputParam("training", std::make_tuple(info, inputData)); SetInputParam("labels", std::move(labels)); // Input test data. - SetInputParam("test", std::move(std::make_tuple(info, testData))); + SetInputParam("test", std::make_tuple(info, testData)); mlpackMain(); // Check that number of output points are equal to number of input points. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_cols, testSize); + CLI::GetParam>("predictions").n_cols, testSize); BOOST_REQUIRE_EQUAL(CLI::GetParam("probabilities").n_cols, - testSize); + testSize); // Check number of output rows equals 1 for probabilities and predictions. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_rows, 1); + CLI::GetParam>("predictions").n_rows, 1); BOOST_REQUIRE_EQUAL( - CLI::GetParam("probabilities").n_rows, 1); + CLI::GetParam("probabilities").n_rows, 1); } /** @@ -109,25 +109,25 @@ BOOST_AUTO_TEST_CASE(HoeffdingTreeCategoricalOutputDimensionTest) size_t testSize = testData.n_cols; // Input training data. - SetInputParam("training", std::move(std::make_tuple(info, inputData))); + SetInputParam("training", std::make_tuple(info, inputData)); SetInputParam("labels", std::move(labels)); // Input test data. - SetInputParam("test", std::move(std::make_tuple(info, testData))); + SetInputParam("test", std::make_tuple(info, testData)); mlpackMain(); // Check that number of output points are equal to number of input points. BOOST_REQUIRE_EQUAL(CLI::GetParam> - ("predictions").n_cols, testSize); + ("predictions").n_cols, testSize); BOOST_REQUIRE_EQUAL(CLI::GetParam("probabilities").n_cols, - testSize); + testSize); // Check number of output rows equals 1 for probabilities and predictions. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_rows, 1); + CLI::GetParam>("predictions").n_rows, 1); BOOST_REQUIRE_EQUAL( - CLI::GetParam("probabilities").n_rows, 1); + CLI::GetParam("probabilities").n_rows, 1); } /** @@ -157,25 +157,25 @@ BOOST_AUTO_TEST_CASE(HoeffdingTreeLabelLessTest) size_t testSize = testData.n_cols; // Input training data. - SetInputParam("training", std::move(std::make_tuple(info, inputData))); + SetInputParam("training", std::make_tuple(info, inputData)); // Input test data. - SetInputParam("test", std::move(std::make_tuple(info, testData))); + SetInputParam("test", std::make_tuple(info, testData)); mlpackMain(); // Check that number of output points are equal to number of input points. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_cols, testSize); + CLI::GetParam>("predictions").n_cols, testSize); BOOST_REQUIRE_EQUAL(CLI::GetParam("probabilities").n_cols, - testSize); + testSize); // Check number of output rows equals number of classes in case of // probabilities and 1 for predictions. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_rows, 1); + CLI::GetParam>("predictions").n_rows, 1); BOOST_REQUIRE_EQUAL( - CLI::GetParam("probabilities").n_rows, 1); + CLI::GetParam("probabilities").n_rows, 1); // Reset passed parameters. CLI::GetSingleton().Parameters()["training"].wasPassed = false; @@ -191,8 +191,8 @@ BOOST_AUTO_TEST_CASE(HoeffdingTreeLabelLessTest) inputData.shed_row(inputData.n_rows - 1); // Input training data. - SetInputParam("training", std::move(std::make_tuple(info, inputData))); - SetInputParam("test", std::move(std::make_tuple(info, testData))); + SetInputParam("training", std::make_tuple(info, inputData)); + SetInputParam("test", std::make_tuple(info, testData)); // Pass Labels. SetInputParam("labels", std::move(labels)); @@ -200,21 +200,21 @@ BOOST_AUTO_TEST_CASE(HoeffdingTreeLabelLessTest) // Check that number of output points are equal to number of input points. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_cols, testSize); + CLI::GetParam>("predictions").n_cols, testSize); BOOST_REQUIRE_EQUAL( - CLI::GetParam("probabilities").n_cols, testSize); + CLI::GetParam("probabilities").n_cols, testSize); // Check number of output rows equals 1 for probabilities and predictions. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_rows, 1); + CLI::GetParam>("predictions").n_rows, 1); BOOST_REQUIRE_EQUAL( - CLI::GetParam("probabilities").n_rows, 1); + CLI::GetParam("probabilities").n_rows, 1); // Check that initial and current predictions are same. CheckMatrices( - predictions, CLI::GetParam>("predictions")); + predictions, CLI::GetParam>("predictions")); CheckMatrices( - probabilities, CLI::GetParam("probabilities")); + probabilities, CLI::GetParam("probabilities")); } /** @@ -262,26 +262,26 @@ BOOST_AUTO_TEST_CASE(HoeffdingModelReuseTest) // Input trained model. SetInputParam("test", std::move(std::make_tuple(info, testData))); SetInputParam("input_model", - CLI::GetParam("output_model")); + CLI::GetParam("output_model")); mlpackMain(); // Check that number of output points are equal to number of input points. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_cols, testSize); + CLI::GetParam>("predictions").n_cols, testSize); BOOST_REQUIRE_EQUAL( - CLI::GetParam("probabilities").n_cols, testSize); + CLI::GetParam("probabilities").n_cols, testSize); // Check number of output rows equals 1 for probabilities and predictions. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_rows, 1); + CLI::GetParam>("predictions").n_rows, 1); BOOST_REQUIRE_EQUAL(CLI::GetParam("probabilities").n_rows, 1); // Check that initial predictions and predictions using saved model are same. CheckMatrices( - predictions, CLI::GetParam>("predictions")); + predictions, CLI::GetParam>("predictions")); CheckMatrices( - probabilities, CLI::GetParam("probabilities")); + probabilities, CLI::GetParam("probabilities")); } /** @@ -305,11 +305,11 @@ BOOST_AUTO_TEST_CASE(HoeffdingModelCategoricalReuseTest) size_t testSize = testData.n_cols; // Input training data. - SetInputParam("training", std::move(std::make_tuple(info, inputData))); + SetInputParam("training", std::make_tuple(info, inputData)); SetInputParam("labels", std::move(labels)); // Input test data. - SetInputParam("test", std::move(std::make_tuple(info, testData))); + SetInputParam("test", std::make_tuple(info, testData)); mlpackMain(); @@ -329,27 +329,27 @@ BOOST_AUTO_TEST_CASE(HoeffdingModelCategoricalReuseTest) // Input trained model. SetInputParam("test", std::move(std::make_tuple(info, testData))); SetInputParam("input_model", - CLI::GetParam("output_model")); + CLI::GetParam("output_model")); mlpackMain(); // Check that number of output points are equal to number of input points. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_cols, testSize); + CLI::GetParam>("predictions").n_cols, testSize); BOOST_REQUIRE_EQUAL( - CLI::GetParam("probabilities").n_cols, testSize); + CLI::GetParam("probabilities").n_cols, testSize); // Check number of output rows equals 1 for probabilities and predictions. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_rows, 1); + CLI::GetParam>("predictions").n_rows, 1); BOOST_REQUIRE_EQUAL( - CLI::GetParam("probabilities").n_rows, 1); + CLI::GetParam("probabilities").n_rows, 1); // Check that initial predictions and predictions using saved model are same. CheckMatrices( - predictions, CLI::GetParam>("predictions")); + predictions, CLI::GetParam>("predictions")); CheckMatrices( - probabilities, CLI::GetParam("probabilities")); + probabilities, CLI::GetParam("probabilities")); } /** @@ -372,11 +372,11 @@ BOOST_AUTO_TEST_CASE(HoeffdingMinSamplesTest) BOOST_FAIL("Cannot load test dataset vc2.csv!"); // Input training data. - SetInputParam("training", std::move(std::make_tuple(info, inputData))); + SetInputParam("training", std::make_tuple(info, inputData)); SetInputParam("labels", std::move(labels)); // Input test data. - SetInputParam("test", std::move(std::make_tuple(info, testData))); + SetInputParam("test", std::make_tuple(info, testData)); SetInputParam("min_samples", 10); SetInputParam("confidence", 0.25); @@ -404,11 +404,11 @@ BOOST_AUTO_TEST_CASE(HoeffdingMinSamplesTest) BOOST_FAIL("Cannot load test dataset vc2.csv!"); // Input training data. - SetInputParam("training", std::move(std::make_tuple(info, inputData))); + SetInputParam("training", std::make_tuple(info, inputData)); SetInputParam("labels", std::move(labels)); // Input test data. - SetInputParam("test", std::move(std::make_tuple(info, testData))); + SetInputParam("test", std::make_tuple(info, testData)); SetInputParam("min_samples", 2000); SetInputParam("confidence", 0.25); @@ -417,8 +417,8 @@ BOOST_AUTO_TEST_CASE(HoeffdingMinSamplesTest) // Check that small min_samples creates larger model. BOOST_REQUIRE_LT( - (CLI::GetParam("output_model"))->NumNodes(), - nodes); + (CLI::GetParam("output_model"))->NumNodes(), + nodes); } /** @@ -441,11 +441,11 @@ BOOST_AUTO_TEST_CASE(HoeffdingMaxSamplesTest) BOOST_FAIL("Cannot load test dataset vc2.csv!"); // Input training data. - SetInputParam("training", std::move(std::make_tuple(info, inputData))); + SetInputParam("training", std::make_tuple(info, inputData)); SetInputParam("labels", std::move(labels)); // Input test data. - SetInputParam("test", std::move(std::make_tuple(info, testData))); + SetInputParam("test", std::make_tuple(info, testData)); SetInputParam("max_samples", 50000); SetInputParam("confidence", 0.95); @@ -473,11 +473,11 @@ BOOST_AUTO_TEST_CASE(HoeffdingMaxSamplesTest) BOOST_FAIL("Cannot load test dataset vc2.csv!"); // Input training data. - SetInputParam("training", std::move(std::make_tuple(info, inputData))); + SetInputParam("training", std::make_tuple(info, inputData)); SetInputParam("labels", std::move(labels)); // Input test data. - SetInputParam("test", std::move(std::make_tuple(info, testData))); + SetInputParam("test", std::make_tuple(info, testData)); SetInputParam("max_samples", 5); SetInputParam("confidence", 0.95); @@ -486,7 +486,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingMaxSamplesTest) // Check that large max_samples creates smaller model. BOOST_REQUIRE_LT(nodes, - (CLI::GetParam("output_model"))->NumNodes()); + (CLI::GetParam("output_model"))->NumNodes()); } /** @@ -509,11 +509,11 @@ BOOST_AUTO_TEST_CASE(HoeffdingConfidenceTest) BOOST_FAIL("Cannot load test dataset vc2.csv!"); // Input training data. - SetInputParam("training", std::move(std::make_tuple(info, inputData))); + SetInputParam("training", std::make_tuple(info, inputData)); SetInputParam("labels", std::move(labels)); // Input test data. - SetInputParam("test", std::move(std::make_tuple(info, testData))); + SetInputParam("test", std::make_tuple(info, testData)); SetInputParam("confidence", 0.95); @@ -540,11 +540,11 @@ BOOST_AUTO_TEST_CASE(HoeffdingConfidenceTest) BOOST_FAIL("Cannot load test dataset vc2.csv!"); // Input training data. - SetInputParam("training", std::move(std::make_tuple(info, inputData))); + SetInputParam("training", std::make_tuple(info, inputData)); SetInputParam("labels", std::move(labels)); // Input test data. - SetInputParam("test", std::move(std::make_tuple(info, testData))); + SetInputParam("test", std::make_tuple(info, testData)); // Model with low confidence. SetInputParam("confidence", 0.25); @@ -552,7 +552,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingConfidenceTest) mlpackMain(); // Check that higher confidence creates smaller tree. BOOST_REQUIRE_LT(nodes, - (CLI::GetParam("output_model"))->NumNodes()); + (CLI::GetParam("output_model"))->NumNodes()); } /** @@ -575,11 +575,11 @@ BOOST_AUTO_TEST_CASE(HoeffdingPassesTest) BOOST_FAIL("Cannot load test dataset vc2.csv!"); // Input training data. - SetInputParam("training", std::move(std::make_tuple(info, inputData))); + SetInputParam("training", std::make_tuple(info, inputData)); SetInputParam("labels", std::move(labels)); // Input test data. - SetInputParam("test", std::move(std::make_tuple(info, testData))); + SetInputParam("test", std::make_tuple(info, testData)); SetInputParam("passes", 1); @@ -606,11 +606,11 @@ BOOST_AUTO_TEST_CASE(HoeffdingPassesTest) BOOST_FAIL("Cannot load test dataset vc2.csv!"); // Input training data. - SetInputParam("training", std::move(std::make_tuple(info, inputData))); + SetInputParam("training", std::make_tuple(info, inputData)); SetInputParam("labels", std::move(labels)); // Input test data. - SetInputParam("test", std::move(std::make_tuple(info, testData))); + SetInputParam("test", std::make_tuple(info, testData)); // Model with larger number of passes. SetInputParam("passes", 100); @@ -619,7 +619,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingPassesTest) // Check that model with larger number of passes has greater number of nodes. BOOST_REQUIRE_LT(nodes, - (CLI::GetParam("output_model"))->NumNodes()); + (CLI::GetParam("output_model"))->NumNodes()); } /** @@ -641,11 +641,11 @@ BOOST_AUTO_TEST_CASE(HoeffdingBinarySplittingStrategyTest) BOOST_FAIL("Cannot load test dataset vc2.csv!"); // Input training data. - SetInputParam("training", std::move(std::make_tuple(info, inputData))); + SetInputParam("training", std::make_tuple(info, inputData)); SetInputParam("labels", std::move(labels)); // Input test data. - SetInputParam("test", std::move(std::make_tuple(info, testData))); + SetInputParam("test", std::make_tuple(info, testData)); SetInputParam("numeric_split_strategy", (string) "binary"); SetInputParam("max_samples", 50); @@ -656,7 +656,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingBinarySplittingStrategyTest) // Check that number of children is 2. BOOST_REQUIRE_EQUAL( - (CLI::GetParam("output_model"))->NumNodes()-1, 2); + (CLI::GetParam("output_model"))->NumNodes()-1, 2); } /** @@ -679,11 +679,11 @@ BOOST_AUTO_TEST_CASE(HoeffdingDomingosSplittingStrategyTest) BOOST_FAIL("Cannot load test dataset vc2.csv!"); // Input training data. - SetInputParam("training", std::move(std::make_tuple(info, inputData))); + SetInputParam("training", std::make_tuple(info, inputData)); SetInputParam("labels", std::move(labels)); // Input test data. - SetInputParam("test", std::move(std::make_tuple(info, testData))); + SetInputParam("test", std::make_tuple(info, testData)); SetInputParam("numeric_split_strategy", (string) "domingos"); SetInputParam("max_samples", 50); @@ -714,11 +714,11 @@ BOOST_AUTO_TEST_CASE(HoeffdingDomingosSplittingStrategyTest) BOOST_FAIL("Cannot load test dataset vc2.csv!"); // Input training data. - SetInputParam("training", std::move(std::make_tuple(info, inputData))); + SetInputParam("training", std::make_tuple(info, inputData)); SetInputParam("labels", std::move(labels)); // Input test data. - SetInputParam("test", std::move(std::make_tuple(info, testData))); + SetInputParam("test", std::make_tuple(info, testData)); SetInputParam("numeric_split_strategy", (string) "domingos"); SetInputParam("max_samples", 50); @@ -728,7 +728,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingDomingosSplittingStrategyTest) // Check that both models have different number of nodes. BOOST_CHECK_NE( - (CLI::GetParam("output_model"))->NumNodes(), nodes); + (CLI::GetParam("output_model"))->NumNodes(), nodes); } /** @@ -756,11 +756,11 @@ BOOST_AUTO_TEST_CASE(HoeffdingBinningTest) modLabels = labels.cols(0, 49); // Input training data. - SetInputParam("training", std::move(std::make_tuple(info, modData))); + SetInputParam("training", std::make_tuple(info, modData)); SetInputParam("labels", std::move(modLabels)); // Input test data. - SetInputParam("test", std::move(std::make_tuple(info, testData))); + SetInputParam("test", std::make_tuple(info, testData)); SetInputParam("numeric_split_strategy", (string) "domingos"); SetInputParam("min_samples", 10); @@ -773,7 +773,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingBinningTest) // Check that no splitting has happened. BOOST_REQUIRE_EQUAL( - (CLI::GetParam("output_model"))->NumNodes(), 1); + (CLI::GetParam("output_model"))->NumNodes(), 1); } BOOST_AUTO_TEST_SUITE_END(); From a27a85c634cc9da3b6b3759cc17241aaf839b49a Mon Sep 17 00:00:00 2001 From: robotcator Date: Fri, 31 May 2019 16:50:40 +0800 Subject: [PATCH 072/143] fix code systle --- .../tests/main_tests/hoeffding_tree_test.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/mlpack/tests/main_tests/hoeffding_tree_test.cpp b/src/mlpack/tests/main_tests/hoeffding_tree_test.cpp index 30cf035f7d..8f2fd7e2c8 100644 --- a/src/mlpack/tests/main_tests/hoeffding_tree_test.cpp +++ b/src/mlpack/tests/main_tests/hoeffding_tree_test.cpp @@ -78,7 +78,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingTreeOutputDimensionTest) BOOST_REQUIRE_EQUAL( CLI::GetParam>("predictions").n_cols, testSize); BOOST_REQUIRE_EQUAL(CLI::GetParam("probabilities").n_cols, - testSize); + testSize); // Check number of output rows equals 1 for probabilities and predictions. BOOST_REQUIRE_EQUAL( @@ -119,9 +119,9 @@ BOOST_AUTO_TEST_CASE(HoeffdingTreeCategoricalOutputDimensionTest) // Check that number of output points are equal to number of input points. BOOST_REQUIRE_EQUAL(CLI::GetParam> - ("predictions").n_cols, testSize); + ("predictions").n_cols, testSize); BOOST_REQUIRE_EQUAL(CLI::GetParam("probabilities").n_cols, - testSize); + testSize); // Check number of output rows equals 1 for probabilities and predictions. BOOST_REQUIRE_EQUAL( @@ -168,7 +168,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingTreeLabelLessTest) BOOST_REQUIRE_EQUAL( CLI::GetParam>("predictions").n_cols, testSize); BOOST_REQUIRE_EQUAL(CLI::GetParam("probabilities").n_cols, - testSize); + testSize); // Check number of output rows equals number of classes in case of // probabilities and 1 for predictions. @@ -262,7 +262,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingModelReuseTest) // Input trained model. SetInputParam("test", std::move(std::make_tuple(info, testData))); SetInputParam("input_model", - CLI::GetParam("output_model")); + CLI::GetParam("output_model")); mlpackMain(); @@ -329,7 +329,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingModelCategoricalReuseTest) // Input trained model. SetInputParam("test", std::move(std::make_tuple(info, testData))); SetInputParam("input_model", - CLI::GetParam("output_model")); + CLI::GetParam("output_model")); mlpackMain(); @@ -486,7 +486,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingMaxSamplesTest) // Check that large max_samples creates smaller model. BOOST_REQUIRE_LT(nodes, - (CLI::GetParam("output_model"))->NumNodes()); + (CLI::GetParam("output_model"))->NumNodes()); } /** @@ -552,7 +552,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingConfidenceTest) mlpackMain(); // Check that higher confidence creates smaller tree. BOOST_REQUIRE_LT(nodes, - (CLI::GetParam("output_model"))->NumNodes()); + (CLI::GetParam("output_model"))->NumNodes()); } /** @@ -619,7 +619,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingPassesTest) // Check that model with larger number of passes has greater number of nodes. BOOST_REQUIRE_LT(nodes, - (CLI::GetParam("output_model"))->NumNodes()); + (CLI::GetParam("output_model"))->NumNodes()); } /** From 4b0a41fa64848144c66e8a55f3bc1f565727dbf6 Mon Sep 17 00:00:00 2001 From: Yashwant Date: Fri, 31 May 2019 16:05:57 +0530 Subject: [PATCH 073/143] Resolve Tests --- .../tests/main_tests/gmm_generate_test.cpp | 12 +-- .../tests/main_tests/gmm_probability_test.cpp | 4 +- .../tests/main_tests/gmm_train_test.cpp | 74 ++++++++++--------- 3 files changed, 49 insertions(+), 41 deletions(-) diff --git a/src/mlpack/tests/main_tests/gmm_generate_test.cpp b/src/mlpack/tests/main_tests/gmm_generate_test.cpp index 1d725a28e6..494b99097d 100644 --- a/src/mlpack/tests/main_tests/gmm_generate_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_generate_test.cpp @@ -45,8 +45,8 @@ BOOST_AUTO_TEST_CASE(GmmGenerateSamplesTest) { arma::mat inputData(5, 10, arma::fill::randu); - GMM gmm(1, 2); - gmm.Train(inputData, 2); + GMM gmm(1, 5); + gmm.Train(inputData, 5); SetInputParam("input_model", &gmm); @@ -61,8 +61,8 @@ BOOST_AUTO_TEST_CASE(GmmGenerateSamples) { arma::mat inputData(5, 10, arma::fill::randu); - GMM gmm(1, 2); - gmm.Train(inputData, 2); + GMM gmm(1, 5); + gmm.Train(inputData, 5); SetInputParam("input_model", &gmm); @@ -76,8 +76,8 @@ BOOST_AUTO_TEST_CASE(GmmGenerateDimensionality) { arma::mat inputData(5, 10, arma::fill::randu); - GMM gmm(1, 2); - gmm.Train(inputData, 2); + GMM gmm(1, 5); + gmm.Train(inputData, 5); SetInputParam("input_model", &gmm); SetInputParam("samples", (int) 10); diff --git a/src/mlpack/tests/main_tests/gmm_probability_test.cpp b/src/mlpack/tests/main_tests/gmm_probability_test.cpp index f945ea134b..17d5a8f983 100644 --- a/src/mlpack/tests/main_tests/gmm_probability_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_probability_test.cpp @@ -54,8 +54,8 @@ BOOST_AUTO_TEST_CASE(GmmProbabilityDimensionality) { arma::mat inputData(5, 10, arma::fill::randu); - GMM gmm(1, 2); - gmm.Train(std::move(inputData), 2); + GMM gmm(1, 5); + gmm.Train(std::move(inputData), 5); arma::mat inputPoints(1, 8, arma::fill::randu); diff --git a/src/mlpack/tests/main_tests/gmm_train_test.cpp b/src/mlpack/tests/main_tests/gmm_train_test.cpp index 85074fdc62..2a9d1672a4 100644 --- a/src/mlpack/tests/main_tests/gmm_train_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_train_test.cpp @@ -47,6 +47,7 @@ void ResetGmmTrainSetting() CLI::RestoreSettings(testName); } + BOOST_FIXTURE_TEST_SUITE(GmmTrainMainTest, GmmTrainTestFixture); // To check if the gaussian is positive or not. @@ -175,17 +176,18 @@ BOOST_AUTO_TEST_CASE(GmmTrainNoisetest) SetInputParam("gaussians", (int) 2); SetInputParam("noise", (double) 0.0); + mlpack::math::FixedRandomSeed(); mlpackMain(); GMM* gmm = CLI::GetParam("output_model"); - CLI::GetSingleton().Parameters()["input"].wasPassed = false; - CLI::GetSingleton().Parameters()["gaussians"].wasPassed = false; - CLI::GetSingleton().Parameters()["noise"].wasPassed = false; + ResetGmmTrainSetting(); SetInputParam("input", std::move(inputData)); SetInputParam("gaussians", (int) 2); - SetInputParam("noise", (double) 1.5); + SetInputParam("noise", (double) 100.0); + + mlpack::math::FixedRandomSeed(); mlpackMain(); GMM* gmm1 = CLI::GetParam("output_model"); @@ -193,8 +195,10 @@ BOOST_AUTO_TEST_CASE(GmmTrainNoisetest) arma::uvec sortedIndices = sort_index(gmm->Weights()); for (size_t k = 0; k < sortedIndices.n_elem; k++) - CheckMatricesNotEqual(gmm->Component(sortedIndices[k]).Covariance(), - gmm1->Component(sortedIndices[k]).Covariance()); + BOOST_REQUIRE(arma::norm(gmm->Component(sortedIndices[k]).Mean() - + gmm1->Component(sortedIndices[k]).Mean()) > 1e-50 || + arma::norm(gmm->Component(sortedIndices[k]).Covariance() - + gmm1->Component(sortedIndices[k]).Covariance()) > 1e-50); } // Ensure that Percentage affects the final result when refined_start is true. @@ -205,22 +209,23 @@ BOOST_AUTO_TEST_CASE(GmmTrainPercentageTest) SetInputParam("input", inputData); SetInputParam("gaussians", (int) 2); SetInputParam("refined_start", true); - SetInputParam("percentage", (double) 0.22); + SetInputParam("percentage", (double) 0.01); + SetInputParam("samplings", (int) 200); + mlpack::math::FixedRandomSeed(); mlpackMain(); GMM* gmm = CLI::GetParam("output_model"); - CLI::GetSingleton().Parameters()["input"].wasPassed = false; - CLI::GetSingleton().Parameters()["gaussians"].wasPassed = false; - CLI::GetSingleton().Parameters()["refined_start"].wasPassed = false; - CLI::GetSingleton().Parameters()["percentage"].wasPassed = false; + ResetGmmTrainSetting(); SetInputParam("input", std::move(inputData)); SetInputParam("gaussians", (int) 2); SetInputParam("refined_start", true); - SetInputParam("percentage", (double) 0.82); + SetInputParam("percentage", (double) 0.99); + SetInputParam("samplings", (int) 200); + mlpack::math::FixedRandomSeed(); mlpackMain(); GMM* gmm1 = CLI::GetParam("output_model"); @@ -228,8 +233,10 @@ BOOST_AUTO_TEST_CASE(GmmTrainPercentageTest) arma::uvec sortedIndices = sort_index(gmm->Weights()); for (size_t k = 0; k < sortedIndices.n_elem; k++) - CheckMatricesNotEqual(gmm->Component(sortedIndices[k]).Covariance(), - gmm1->Component(sortedIndices[k]).Covariance()); + BOOST_REQUIRE(arma::norm(gmm->Component(sortedIndices[k]).Mean() - + gmm1->Component(sortedIndices[k]).Mean()) > 1e-50 || + arma::norm(gmm->Component(sortedIndices[k]).Covariance() - + gmm1->Component(sortedIndices[k]).Covariance()) > 1e-50); } // Ensure that Sampling affects the final result when refined_start is true. @@ -240,25 +247,23 @@ BOOST_AUTO_TEST_CASE(GmmTrainSamplingsTest) SetInputParam("input", inputData); SetInputParam("gaussians", (int) 2); SetInputParam("refined_start", true); - SetInputParam("percentage", (double) 0.5); - SetInputParam("samplings", (int) 100); + SetInputParam("percentage", (double) 0.950); + SetInputParam("samplings", (int) 10); + mlpack::math::FixedRandomSeed(); mlpackMain(); GMM* gmm = CLI::GetParam("output_model"); - CLI::GetSingleton().Parameters()["input"].wasPassed = false; - CLI::GetSingleton().Parameters()["gaussians"].wasPassed = false; - CLI::GetSingleton().Parameters()["refined_start"].wasPassed = false; - CLI::GetSingleton().Parameters()["percentage"].wasPassed = false; - CLI::GetSingleton().Parameters()["samplings"].wasPassed = false; + ResetGmmTrainSetting(); SetInputParam("input", std::move(inputData)); SetInputParam("gaussians", (int) 2); SetInputParam("refined_start", true); - SetInputParam("percentage", (double) 0.5); - SetInputParam("samplings", (int) 500); + SetInputParam("percentage", (double) 0.950); + SetInputParam("samplings", (int) 1000); + mlpack::math::FixedRandomSeed(); mlpackMain(); GMM* gmm1 = CLI::GetParam("output_model"); @@ -266,9 +271,10 @@ BOOST_AUTO_TEST_CASE(GmmTrainSamplingsTest) arma::uvec sortedIndices = sort_index(gmm->Weights()); for (size_t k = 0; k < sortedIndices.n_elem; k++) - CheckMatricesNotEqual(gmm->Component(sortedIndices[k]).Covariance(), - gmm1->Component(sortedIndices[k]).Covariance()); - + BOOST_REQUIRE(arma::norm(gmm->Component(sortedIndices[k]).Mean() - + gmm1->Component(sortedIndices[k]).Mean()) > 1e-50 || + arma::norm(gmm->Component(sortedIndices[k]).Covariance() - + gmm1->Component(sortedIndices[k]).Covariance()) > 1e-50); } // Ensure that tolerance affects the final result. @@ -280,20 +286,20 @@ BOOST_AUTO_TEST_CASE(GmmTrainToleranceTest) SetInputParam("input", inputData); SetInputParam("gaussians", (int) 2); - SetInputParam("tolerance", (double) 1e-10); + SetInputParam("tolerance", (double) 1e-8); + mlpack::math::FixedRandomSeed(); mlpackMain(); GMM* gmm = CLI::GetParam("output_model"); - CLI::GetSingleton().Parameters()["input"].wasPassed = false; - CLI::GetSingleton().Parameters()["gaussians"].wasPassed = false; - CLI::GetSingleton().Parameters()["tolerance"].wasPassed = false; + ResetGmmTrainSetting(); SetInputParam("input", std::move(inputData)); SetInputParam("gaussians", (int) 2); - SetInputParam("tolerance", (double) 1e-30); + SetInputParam("tolerance", (double) 10); + mlpack::math::FixedRandomSeed(); mlpackMain(); GMM* gmm1 = CLI::GetParam("output_model"); @@ -301,8 +307,10 @@ BOOST_AUTO_TEST_CASE(GmmTrainToleranceTest) arma::uvec sortedIndices = sort_index(gmm->Weights()); for (size_t k = 0; k < sortedIndices.n_elem; k++) - CheckMatricesNotEqual(gmm->Component(sortedIndices[k]).Covariance(), - gmm1->Component(sortedIndices[k]).Covariance()); + BOOST_REQUIRE(arma::norm(gmm->Component(sortedIndices[k]).Mean() - + gmm1->Component(sortedIndices[k]).Mean()) > 1e-50 || + arma::norm(gmm->Component(sortedIndices[k]).Covariance() - + gmm1->Component(sortedIndices[k]).Covariance()) > 1e-50); } // Ensure that saved model can be used again. From 0a7986a5773bce9bef40262a33114fac4025923d Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Fri, 31 May 2019 21:37:34 +0530 Subject: [PATCH 074/143] Update HISTORY.md --- HISTORY.md | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.md b/HISTORY.md index bfc5b5c3af..bd9a721e15 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,6 @@ ### mlpack 3.1.2 ###### ????-??-?? + * Add Multiple Pole Balancing Environment (#1901). ### mlpack 3.1.1 ###### 2019-05-26 From 459e2a012ff11d0ffe7319c02c8cf95210a4b3ea Mon Sep 17 00:00:00 2001 From: Yashwant Date: Sat, 1 Jun 2019 22:32:23 +0530 Subject: [PATCH 075/143] Add new paramter Maximum_depth --- .../decision_tree/all_categorical_split.hpp | 2 + .../all_categorical_split_impl.hpp | 5 +++ .../best_binary_numeric_split.hpp | 2 + .../best_binary_numeric_split_impl.hpp | 5 +++ .../methods/decision_tree/decision_tree.hpp | 20 +++++++++ .../decision_tree/decision_tree_impl.hpp | 45 ++++++++++++++----- .../decision_tree/decision_tree_main.cpp | 20 ++++++--- .../methods/random_forest/random_forest.hpp | 20 +++++++++ .../random_forest/random_forest_impl.hpp | 38 +++++++++++----- .../random_forest/random_forest_main.cpp | 12 +++-- src/mlpack/tests/decision_tree_test.cpp | 26 ++++++----- .../tests/main_tests/decision_tree_test.cpp | 28 ++++++++++++ .../tests/main_tests/random_forest_test.cpp | 20 +++++++++ src/mlpack/tests/random_forest_test.cpp | 8 ++-- 14 files changed, 204 insertions(+), 47 deletions(-) diff --git a/src/mlpack/methods/decision_tree/all_categorical_split.hpp b/src/mlpack/methods/decision_tree/all_categorical_split.hpp index 2823fa9bb5..87f796e1e1 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split.hpp @@ -47,6 +47,7 @@ class AllCategoricalSplit * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in a leaf node for * splitting. + * @param maximumDepth Maximum Depth Minimum for the tree. * @param classProbabilities Class probabilities vector, which may be filled * with split information a successful split. * @param aux Auxiliary split information, which may be modified on a @@ -62,6 +63,7 @@ class AllCategoricalSplit const WeightVecType& weights, const size_t minimumLeafSize, const double minimumGainSplit, + const size_t maximumDepth, arma::Col& classProbabilities, AuxiliarySplitInfo& aux); diff --git a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp index 97337fc508..ec317b77c6 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp @@ -26,9 +26,14 @@ double AllCategoricalSplit::SplitIfBetter( const WeightVecType& weights, const size_t minimumLeafSize, const double minimumGainSplit, + const size_t maximumDepth, arma::Col& classProbabilities, AuxiliarySplitInfo& /* aux */) { + // First sanity check: if we have reached maximum depth, we can't split. + if (maximumDepth == 1) + return DBL_MAX; + // Count the number of elements in each potential child. const double epsilon = 1e-7; // Tolerance for floating-point errors. arma::Col counts(numCategories, arma::fill::zeros); diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp b/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp index c187e5f08d..74462a75da 100644 --- a/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp +++ b/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp @@ -45,6 +45,7 @@ class BestBinaryNumericSplit * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in a leaf node for * splitting. + * @param maximumDepth Maximum Depth Minimum for the tree. * @param classProbabilities Class probabilities vector, which may be filled * with split information a successful split. * @param aux Auxiliary split information, which may be modified on a @@ -59,6 +60,7 @@ class BestBinaryNumericSplit const WeightVecType& weights, const size_t minimumLeafSize, const double minimumGainSplit, + const size_t maximumDepth, arma::Col& classProbabilities, AuxiliarySplitInfo& aux); diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp index 34cb692dc4..4cefdbd60d 100644 --- a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp @@ -25,9 +25,14 @@ double BestBinaryNumericSplit::SplitIfBetter( const WeightVecType& weights, const size_t minimumLeafSize, const double minimumGainSplit, + const size_t maximumDepth, arma::Col& classProbabilities, AuxiliarySplitInfo& /* aux */) { + // First sanity check: if we have reached maximum depth, we can't split. + if (maximumDepth == 1) + return DBL_MAX; + // First sanity check: if we don't have enough points, we can't split. if (data.n_elem < (minimumLeafSize * 2)) return DBL_MAX; diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp index 3b372c02c3..e544a53d11 100644 --- a/src/mlpack/methods/decision_tree/decision_tree.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree.hpp @@ -64,6 +64,7 @@ class DecisionTree : * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in each leaf node. * @param minimumGainSplit Minimum gain for the node to split. + * @param maximumDepth Maximum depth for the tree. * @param dimensionSelector Instantiated dimension selection policy. */ template @@ -73,6 +74,7 @@ class DecisionTree : const size_t numClasses, const size_t minimumLeafSize = 10, const double minimumGainSplit = 1e-7, + const size_t maximumDepth = 20, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -89,6 +91,7 @@ class DecisionTree : * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in each leaf node. * @param minimumGainSplit Minimum gain for the node to split. + * @param maximumDepth Maximum depth for the tree. * @param dimensionSelector Instantiated dimension selection policy. */ template @@ -97,6 +100,7 @@ class DecisionTree : const size_t numClasses, const size_t minimumLeafSize = 10, const double minimumGainSplit = 1e-7, + const size_t maximumDepth = 20, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -116,6 +120,7 @@ class DecisionTree : * @param weights The weight list of given label. * @param minimumLeafSize Minimum number of points in each leaf node. * @param minimumGainSplit Minimum gain for the node to split. + * @param maximumDepth Maximum depth for the tree. * @param dimensionSelector Instantiated dimension selection policy. */ template @@ -126,6 +131,7 @@ class DecisionTree : WeightsType weights, const size_t minimumLeafSize = 10, const double minimumGainSplit = 1e-7, + const size_t maximumDepth = 20, DimensionSelectionType dimensionSelector = DimensionSelectionType(), const std::enable_if_t @@ -156,6 +163,7 @@ class DecisionTree : WeightsType weights, const size_t minimumLeafSize = 10, const double minimumGainSplit = 1e-7, + const size_t maximumDepth = 20, DimensionSelectionType dimensionSelector = DimensionSelectionType(), const std::enable_if_t @@ -465,6 +482,7 @@ class DecisionTree : arma::rowvec& weights, const size_t minimumLeafSize, const double minimumGainSplit, + const size_t maximumDepth, DimensionSelectionType& dimensionSelector); /** @@ -480,6 +498,7 @@ class DecisionTree : * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in each leaf node. * @param minimumGainSplit Minimum gain for the node to split. + * @param maximumDepth Maximum depth for the tree. * @return The final entropy of decision tree. */ template @@ -491,6 +510,7 @@ class DecisionTree : arma::rowvec& weights, const size_t minimumLeafSize, const double minimumGainSplit, + const size_t maximumDepth, DimensionSelectionType& dimensionSelector); }; diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index 80627541cc..1d65992cb3 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -35,6 +35,7 @@ DecisionTree::type; @@ -50,7 +51,8 @@ DecisionTree(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses, - weights, minimumLeafSize, minimumGainSplit, dimensionSelector); + weights, minimumLeafSize, minimumGainSplit, maximumDepth, + dimensionSelector); } //! Construct and train. @@ -72,6 +74,7 @@ DecisionTree::type; @@ -87,7 +90,7 @@ DecisionTree(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, weights, - minimumLeafSize, minimumGainSplit, dimensionSelector); + minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector); } //! Construct and train with weights. @@ -111,6 +114,7 @@ DecisionTree(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses, - tmpWeights, minimumLeafSize, minimumGainSplit, dimensionSelector); + tmpWeights, minimumLeafSize, minimumGainSplit, maximumDepth, + dimensionSelector); } //! Construct and train with weights. @@ -154,6 +159,7 @@ DecisionTree(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, tmpWeights, - minimumLeafSize, minimumGainSplit, dimensionSelector); + minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector); } //! Construct, don't train. @@ -372,6 +378,7 @@ double DecisionTree(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, - numClasses, weights, minimumLeafSize, minimumGainSplit, + numClasses, weights, minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector); } @@ -420,6 +427,7 @@ double DecisionTree(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, - weights, minimumLeafSize, minimumGainSplit, dimensionSelector); + weights, minimumLeafSize, minimumGainSplit, maximumDepth, + dimensionSelector); } //! Train on the given weighted data. @@ -469,6 +478,7 @@ double DecisionTree(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, - numClasses, tmpWeights, minimumLeafSize, minimumGainSplit, + numClasses, tmpWeights, minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector); } @@ -523,6 +533,7 @@ double DecisionTree(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, - tmpWeights, minimumLeafSize, minimumGainSplit, dimensionSelector); + tmpWeights, minimumLeafSize, minimumGainSplit, maximumDepth, + dimensionSelector); } //! Train on the given data. @@ -579,6 +591,7 @@ double DecisionTreeTrain(data, currentChildBegin, currentCol - currentChildBegin, datasetInfo, labels, numClasses, weights, currentCol - currentChildBegin, minimumGainSplit, - dimensionSelector); + maximumDepth, dimensionSelector); } else { // During recursion entropy of child node may change. double childGain = child->Train(data, currentChildBegin, currentCol - currentChildBegin, datasetInfo, labels, numClasses, - weights, minimumLeafSize, minimumGainSplit, dimensionSelector); + weights, minimumLeafSize, minimumGainSplit, maximumDepth, + dimensionSelector); bestGain += double(childCounts[i]) / double(count) * (-childGain); } children.push_back(child); @@ -757,6 +775,7 @@ double DecisionTreeTrain(data, currentChildBegin, currentCol - currentChildBegin, labels, numClasses, weights, - currentCol - currentChildBegin, minimumGainSplit, + currentCol - currentChildBegin, minimumGainSplit, maximumDepth, dimensionSelector); } else @@ -866,7 +887,7 @@ double DecisionTreeTrain(data, currentChildBegin, currentCol - currentChildBegin, labels, numClasses, weights, - minimumLeafSize, minimumGainSplit, dimensionSelector); + minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector); bestGain += double(childCounts[i]) / double(count) * (-childGain); } children.push_back(child); diff --git a/src/mlpack/methods/decision_tree/decision_tree_main.cpp b/src/mlpack/methods/decision_tree/decision_tree_main.cpp index bd1ac224fa..49c0826b38 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_main.cpp +++ b/src/mlpack/methods/decision_tree/decision_tree_main.cpp @@ -47,7 +47,9 @@ PROGRAM_INFO("Decision tree", " parameter specifies the minimum number of training points that must fall" " into each leaf for it to be split. The " + PRINT_PARAM_STRING("minimum_gain_split") + " parameter specifies " - "the minimum gain that is needed for the node to split. If " + + "the minimum gain that is needed for the node to split. The " + + PRINT_PARAM_STRING("maximum_depth") + " parameter specifies " + "the maximum depth of the tree. If " + PRINT_PARAM_STRING("print_training_error") + " is specified, the training " "error will be printed." "\n\n" @@ -100,6 +102,8 @@ PARAM_INT_IN("minimum_leaf_size", "Minimum number of points in a leaf.", "n", 20); PARAM_DOUBLE_IN("minimum_gain_split", "Minimum gain for node splitting.", "g", 1e-7); +PARAM_INT_IN("maximum_depth", "Maximum Depth of the tree.", "D", + 20); // This is deprecated and should be removed in mlpack 4.0.0. PARAM_FLAG("print_training_error", "Print the training error (deprecated; will " "be removed in mlpack 4.0.0).", "e"); @@ -157,6 +161,9 @@ static void mlpackMain() RequireParamValue("minimum_leaf_size", [](int x) { return x > 0; }, true, "leaf size must be positive"); + RequireParamValue("maximum_depth", [](int x) { return x > 0; }, true, + "depth must be positive"); + RequireParamValue("minimum_gain_split", [](double x) { return (x > 0.0 && x < 1.0); }, true, "gain split must be a fraction in range [0,1]"); @@ -195,6 +202,7 @@ static void mlpackMain() // Now build the tree. const size_t minLeafSize = (size_t) CLI::GetParam("minimum_leaf_size"); + const size_t maxDepth = (size_t) CLI::GetParam("maximum_depth"); const double minimumGainSplit = (double) CLI::GetParam("minimum_gain_split"); @@ -207,13 +215,14 @@ static void mlpackMain() CLI::HasParam("print_training_accuracy")) { model->tree = DecisionTree<>(trainingSet, model->info, labels, - numClasses, std::move(weights), minLeafSize, minimumGainSplit); + numClasses, std::move(weights), minLeafSize, minimumGainSplit, + maxDepth); } else { model->tree = DecisionTree<>(std::move(trainingSet), model->info, std::move(labels), numClasses, std::move(weights), minLeafSize, - minimumGainSplit); + minimumGainSplit, maxDepth); } } else @@ -221,12 +230,13 @@ static void mlpackMain() if (CLI::HasParam("print_training_error")) { model->tree = DecisionTree<>(trainingSet, model->info, labels, - numClasses, minLeafSize, minimumGainSplit); + numClasses, minLeafSize, minimumGainSplit, maxDepth); } else { model->tree = DecisionTree<>(std::move(trainingSet), model->info, - std::move(labels), numClasses, minLeafSize, minimumGainSplit); + std::move(labels), numClasses, minLeafSize, minimumGainSplit, + maxDepth); } } diff --git a/src/mlpack/methods/random_forest/random_forest.hpp b/src/mlpack/methods/random_forest/random_forest.hpp index 3566116043..27167d1964 100644 --- a/src/mlpack/methods/random_forest/random_forest.hpp +++ b/src/mlpack/methods/random_forest/random_forest.hpp @@ -50,6 +50,7 @@ class RandomForest * @param numTrees Number of trees in the forest. * @param minimumLeafSize Minimum number of points in each tree's leaf nodes. * @param minimumGainSplit Minimum gain for splitting a decision tree node. + * @param maximumDepth Maximum depth for the tree. * @param dimensionSelector Instantiated dimension selection policy. */ template @@ -59,6 +60,7 @@ class RandomForest const size_t numTrees = 20, const size_t minimumLeafSize = 1, const double minimumGainSplit = 1e-7, + const size_t maximumDepth = 20, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -77,6 +79,7 @@ class RandomForest * @param numTrees Number of trees in the forest. * @param minimumLeafSize Minimum number of points in each tree's leaf nodes. * @param minimumGainSplit Minimum gain for splitting a decision tree node. + * @param maximumDepth Maximum depth for the tree. * @param dimensionSelector Instantiated dimension selection policy. */ template @@ -87,6 +90,7 @@ class RandomForest const size_t numTrees = 20, const size_t minimumLeafSize = 1, const double minimumGainSplit = 1e-7, + const size_t maximumDepth = 20, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -101,6 +105,9 @@ class RandomForest * @param weights Weights (importances) of each point in the dataset. * @param numTrees Number of trees in the forest. * @param minimumLeafSize Minimum number of points in each tree's leaf nodes. + * @param minimumGainSplit Minimum gain for splitting a decision tree node. + * @param maximumDepth Maximum depth for the tree. + * @param dimensionSelector Instantiated dimension selection policy. */ template RandomForest(const MatType& dataset, @@ -110,6 +117,7 @@ class RandomForest const size_t numTrees = 20, const size_t minimumLeafSize = 1, const double minimumGainSplit = 1e-7, + const size_t maximumDepth = 20, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -129,6 +137,7 @@ class RandomForest * @param numTrees Number of trees in the forest. * @param minimumLeafSize Minimum number of points in each tree's leaf nodes. * @param minimumGainSplit Minimum gain for splitting a decision tree node. + * @param maximumDepth Maximum depth for the tree. * @param dimensionSelector Instantiated dimension selection policy. */ template @@ -140,6 +149,7 @@ class RandomForest const size_t numTrees = 20, const size_t minimumLeafSize = 1, const double minimumGainSplit = 1e-7, + const size_t maximumDepth = 20, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -156,6 +166,7 @@ class RandomForest * @param numTrees Number of trees in the forest. * @param minimumLeafSize Minimum number of points in each tree's leaf nodes. * @param minimumGainSplit Minimum gain for splitting a decision tree node. + * @param maximumDepth Maximum depth for the tree. * @param dimensionSelector Instantiated dimension selection policy. * @return The average entropy of all the decision trees trained under forest. */ @@ -166,6 +177,7 @@ class RandomForest const size_t numTrees = 20, const size_t minimumLeafSize = 1, const double minimumGainSplit = 1e-7, + const size_t maximumDepth = 20, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -185,6 +197,7 @@ class RandomForest * @param numTrees Number of trees in the forest. * @param minimumLeafSize Minimum number of points in each tree's leaf nodes. * @param minimumGainSplit Minimum gain for splitting a decision tree node. + * @param maximumDepth Maximum depth for the tree. * @param dimensionSelector Instantiated dimension selection policy. * @return The average entropy of all the decision trees trained under forest. */ @@ -196,6 +209,7 @@ class RandomForest const size_t numTrees = 20, const size_t minimumLeafSize = 1, const double minimumGainSplit = 1e-7, + const size_t maximumDepth = 20, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -213,6 +227,7 @@ class RandomForest * @param numTrees Number of trees in the forest. * @param minimumLeafSize Minimum number of points in each tree's leaf nodes. * @param minimumGainSplit Minimum gain for splitting a decision tree node. + * @param maximumDepth Maximum depth for the tree. * @param dimensionSelector Instantiated dimension selection policy. * @return The average entropy of all the decision trees trained under forest. */ @@ -224,6 +239,7 @@ class RandomForest const size_t numTrees = 20, const size_t minimumLeafSize = 1, const double minimumGainSplit = 1e-7, + const size_t maximumDepth = 20, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -243,6 +259,7 @@ class RandomForest * @param numTrees Number of trees in the forest. * @param minimumLeafSize Minimum number of points in each tree's leaf nodes. * @param minimumGainSplit Minimum gain for splitting a decision tree node. + * @param maximumDepth Maximum depth for the tree. * @param dimensionSelector Instantiated dimension selection policy. * @return The average entropy of all the decision trees trained under forest. */ @@ -255,6 +272,7 @@ class RandomForest const size_t numTrees = 20, const size_t minimumLeafSize = 1, const double minimumGainSplit = 1e-7, + const size_t maximumDepth = 20, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -334,6 +352,7 @@ class RandomForest * @param numTrees Number of trees in the forest. * @param minimumLeafSize Minimum number of points in each leaf node. * @param minimumGainSplit Minimum gain for splitting a decision tree node. + * @param maximumDepth Maximum depth for the tree. * @param dimensionSelector Instantiated dimension selection policy. * @tparam UseWeights Whether or not to use the weights parameter. * @tparam UseDatasetInfo Whether or not to use the datasetInfo parameter. @@ -349,6 +368,7 @@ class RandomForest const size_t numTrees, const size_t minimumLeafSize, const double minimumGainSplit, + const size_t maximumDepth, DimensionSelectionType& dimensionSelector); //! The trees in the forest. diff --git a/src/mlpack/methods/random_forest/random_forest_impl.hpp b/src/mlpack/methods/random_forest/random_forest_impl.hpp index c52baf060b..d6dbaa7296 100644 --- a/src/mlpack/methods/random_forest/random_forest_impl.hpp +++ b/src/mlpack/methods/random_forest/random_forest_impl.hpp @@ -38,13 +38,14 @@ RandomForest< const size_t numTrees, const size_t minimumLeafSize, const double minimumGainSplit, + const size_t maximumDepth, DimensionSelectionType dimensionSelector) { // Pass off work to the Train() method. data::DatasetInfo info; // Ignored. arma::rowvec weights; // Fake weights, not used. Train(dataset, info, labels, numClasses, weights, numTrees, - minimumLeafSize, minimumGainSplit, dimensionSelector); + minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector); } template< @@ -68,12 +69,14 @@ RandomForest< const size_t numTrees, const size_t minimumLeafSize, const double minimumGainSplit, + const size_t maximumDepth, DimensionSelectionType dimensionSelector) { // Pass off work to the Train() method. arma::rowvec weights; // Fake weights, not used. Train(dataset, datasetInfo, labels, numClasses, weights, - numTrees, minimumLeafSize, minimumGainSplit, dimensionSelector); + numTrees, minimumLeafSize, minimumGainSplit, maximumDepth, + dimensionSelector); } template< @@ -97,12 +100,13 @@ RandomForest< const size_t numTrees, const size_t minimumLeafSize, const double minimumGainSplit, + const size_t maximumDepth, DimensionSelectionType dimensionSelector) { // Pass off work to the Train() method. data::DatasetInfo info; // Ignored by Train(). Train(dataset, info, labels, numClasses, weights, numTrees, - minimumLeafSize, minimumGainSplit, dimensionSelector); + minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector); } template< @@ -127,11 +131,12 @@ RandomForest< const size_t numTrees, const size_t minimumLeafSize, const double minimumGainSplit, + const size_t maximumDepth, DimensionSelectionType dimensionSelector) { // Pass off work to the Train() method. Train(dataset, datasetInfo, labels, numClasses, weights, numTrees, - minimumLeafSize, minimumGainSplit, dimensionSelector); + minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector); } template< @@ -154,13 +159,15 @@ double RandomForest< const size_t numTrees, const size_t minimumLeafSize, const double minimumGainSplit, + const size_t maximumDepth, DimensionSelectionType dimensionSelector) { // Pass off to Train(). data::DatasetInfo info; // Ignored by Train(). arma::rowvec weights; // Ignored by Train(). return Train(dataset, info, labels, numClasses, weights, - numTrees, minimumLeafSize, minimumGainSplit, dimensionSelector); + numTrees, minimumLeafSize, minimumGainSplit, maximumDepth, + dimensionSelector); } template< @@ -184,12 +191,14 @@ double RandomForest< const size_t numTrees, const size_t minimumLeafSize, const double minimumGainSplit, + const size_t maximumDepth, DimensionSelectionType dimensionSelector) { // Pass off to Train(). arma::rowvec weights; // Ignored by Train(). return Train(dataset, datasetInfo, labels, numClasses, weights, - numTrees, minimumLeafSize, minimumGainSplit, dimensionSelector); + numTrees, minimumLeafSize, minimumGainSplit, maximumDepth, + dimensionSelector); } template< @@ -213,12 +222,14 @@ double RandomForest< const size_t numTrees, const size_t minimumLeafSize, const double minimumGainSplit, + const size_t maximumDepth, DimensionSelectionType dimensionSelector) { // Pass off to Train(). data::DatasetInfo info; // Ignored by Train(). return Train(dataset, info, labels, numClasses, weights, - numTrees, minimumLeafSize, minimumGainSplit, dimensionSelector); + numTrees, minimumLeafSize, minimumGainSplit, maximumDepth, + dimensionSelector); } template< @@ -243,11 +254,13 @@ double RandomForest< const size_t numTrees, const size_t minimumLeafSize, const double minimumGainSplit, + const size_t maximumDepth, DimensionSelectionType dimensionSelector) { // Pass off to Train(). return Train(dataset, datasetInfo, labels, numClasses, weights, - numTrees, minimumLeafSize, minimumGainSplit, dimensionSelector); + numTrees, minimumLeafSize, minimumGainSplit, maximumDepth, + dimensionSelector); } template< @@ -447,6 +460,7 @@ double RandomForest< const size_t numTrees, const size_t minimumLeafSize, const double minimumGainSplit, + const size_t maximumDepth, DimensionSelectionType& dimensionSelector) { // Train each tree individually. @@ -472,12 +486,12 @@ double RandomForest< { avgGain += trees[i].Train(bootstrapDataset, datasetInfo, bootstrapLabels, numClasses, bootstrapWeights, minimumLeafSize, - minimumGainSplit, dimensionSelector); + minimumGainSplit, maximumDepth, dimensionSelector); } else { avgGain += trees[i].Train(bootstrapDataset, bootstrapLabels, numClasses, - bootstrapWeights, minimumLeafSize, minimumGainSplit, + bootstrapWeights, minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector); } } @@ -487,12 +501,12 @@ double RandomForest< { avgGain += trees[i].Train(bootstrapDataset, datasetInfo, bootstrapLabels, numClasses, minimumLeafSize, minimumGainSplit, - dimensionSelector); + maximumDepth, dimensionSelector); } else { avgGain += trees[i].Train(bootstrapDataset, bootstrapLabels, numClasses, - minimumLeafSize, minimumGainSplit, dimensionSelector); + minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector); } } Timer::Stop("train_tree"); diff --git a/src/mlpack/methods/random_forest/random_forest_main.cpp b/src/mlpack/methods/random_forest/random_forest_main.cpp index 748289f1e9..492d6568c3 100644 --- a/src/mlpack/methods/random_forest/random_forest_main.cpp +++ b/src/mlpack/methods/random_forest/random_forest_main.cpp @@ -50,7 +50,9 @@ PROGRAM_INFO("Random forests", " controls the number of trees in the random forest. The " + PRINT_PARAM_STRING("minimum_gain_split") + " parameter controls the minimum" " required gain for a decision tree node to split. Larger values will " - "force higher-confidence splits. The " + + "force higher-confidence splits. The " + + PRINT_PARAM_STRING("maximum_depth") + " parameter specifies " + "the maximum depth of the tree. The " + PRINT_PARAM_STRING("subspace_dim") + " parameter is used to control the " "number of random dimensions chosen for an individual node's split. If " + PRINT_PARAM_STRING("print_training_accuracy") + " is specified, the " @@ -105,7 +107,8 @@ PARAM_FLAG("print_training_accuracy", "If set, then the accuracy of the model " PARAM_INT_IN("num_trees", "Number of trees in the random forest.", "N", 10); PARAM_INT_IN("minimum_leaf_size", "Minimum number of points in each leaf " "node.", "n", 1); - +PARAM_INT_IN("maximum_depth", "Maximum Depth of the tree.", "D", + 20); PARAM_MATRIX_OUT("probabilities", "Predicted class probabilities for each " "point in the test set.", "P"); PARAM_UROW_OUT("predictions", "Predicted classes for each point in the test " @@ -177,6 +180,8 @@ static void mlpackMain() RequireParamValue("minimum_leaf_size", [](int x) { return x > 0; }, true, "minimum leaf size must be greater than 0"); + RequireParamValue("maximum_depth", [](int x) { return x > 0; }, true, + "depth must be positive"); RequireParamValue("subspace_dim", [](int x) { return x >= 0; }, true, "subspace dimensionality must be nonnegative"); RequireParamValue("minimum_gain_split", @@ -205,6 +210,7 @@ static void mlpackMain() const size_t numTrees = (size_t) CLI::GetParam("num_trees"); const size_t minimumLeafSize = (size_t) CLI::GetParam("minimum_leaf_size"); + const size_t maxDepth = (size_t) CLI::GetParam("maximum_depth"); const double minimumGainSplit = CLI::GetParam("minimum_gain_split"); const size_t randomDims = (CLI::GetParam("subspace_dim") == 0) ? (size_t) std::sqrt(data.n_rows) : @@ -218,7 +224,7 @@ static void mlpackMain() // Train the model. rfModel->rf.Train(data, labels, numClasses, numTrees, minimumLeafSize, - minimumGainSplit, mrds); + minimumGainSplit, maxDepth, mrds); Timer::Stop("rf_training"); // Did we want training accuracy? diff --git a/src/mlpack/tests/decision_tree_test.cpp b/src/mlpack/tests/decision_tree_test.cpp index 34d645cf54..098ad279f9 100644 --- a/src/mlpack/tests/decision_tree_test.cpp +++ b/src/mlpack/tests/decision_tree_test.cpp @@ -288,10 +288,11 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitSimpleSplitTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 2, weights); const double gain = BestBinaryNumericSplit::SplitIfBetter( - bestGain, values, labels, 2, weights, 3, 1e-7, classProbabilities, aux); + bestGain, values, labels, 2, weights, 3, 1e-7, 20, classProbabilities, + aux); const double weightedGain = BestBinaryNumericSplit::SplitIfBetter(bestGain, values, - labels, 2, weights, 3, 1e-7, classProbabilities, aux); + labels, 2, weights, 3, 1e-7, 20, classProbabilities, aux); // Make sure that a split was made. BOOST_REQUIRE_GT(gain, bestGain); @@ -325,11 +326,12 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitMinSamplesTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 2, weights); const double gain = BestBinaryNumericSplit::SplitIfBetter( - bestGain, values, labels, 2, weights, 8, 1e-7, classProbabilities, aux); + bestGain, values, labels, 2, weights, 8, 1e-7, 20, classProbabilities, + aux); // This should make no difference because it won't split at all. const double weightedGain = BestBinaryNumericSplit::SplitIfBetter(bestGain, values, - labels, 2, weights, 8, 1e-7, classProbabilities, aux); + labels, 2, weights, 8, 1e-7, 20, classProbabilities, aux); // Make sure that no split was made. BOOST_REQUIRE_EQUAL(gain, DBL_MAX); @@ -360,7 +362,8 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitNoGainTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 2, weights); const double gain = BestBinaryNumericSplit::SplitIfBetter( - bestGain, values, labels, 2, weights, 10, 1e-7, classProbabilities, aux); + bestGain, values, labels, 2, weights, 10, 1e-7, 20, classProbabilities, + aux); // Make sure there was no split. BOOST_REQUIRE_EQUAL(gain, DBL_MAX); @@ -384,11 +387,11 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitSimpleSplitTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 4, labels, 3, weights, 3, 1e-7, classProbabilities, + bestGain, values, 4, labels, 3, weights, 3, 1e-7, 20, classProbabilities, aux); const double weightedGain = AllCategoricalSplit::SplitIfBetter(bestGain, values, 4, - labels, 3, weights, 3, 1e-7, classProbabilities, aux); + labels, 3, weights, 3, 1e-7, 20, classProbabilities, aux); // Make sure that a split was made. BOOST_REQUIRE_GT(gain, bestGain); @@ -420,7 +423,7 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitMinSamplesTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 4, labels, 3, weights, 4, 1e-7, classProbabilities, + bestGain, values, 4, labels, 3, weights, 4, 1e-7, 20, classProbabilities, aux); // Make sure it's not split. @@ -453,11 +456,11 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitNoGainTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 10, labels, 3, weights, 10, 1e-7, classProbabilities, - aux); + bestGain, values, 10, labels, 3, weights, 10, 1e-7, 20, + classProbabilities, aux); const double weightedGain = AllCategoricalSplit::SplitIfBetter(bestGain, values, 10, - labels, 3, weights, 10, 1e-7, classProbabilities, aux); + labels, 3, weights, 10, 1e-7, 20, classProbabilities, aux); // Make sure that there was no split. BOOST_REQUIRE_EQUAL(gain, DBL_MAX); @@ -582,6 +585,7 @@ BOOST_AUTO_TEST_CASE(PerfectTrainingSetWithWeight) weights.ones(); // Minimum leaf size of 1. + // Maximum Depth of 1 DecisionTree<> d(dataset, labels, 2, weights, 1, 0.0); // This part of code is dupliacte with no weighted one. diff --git a/src/mlpack/tests/main_tests/decision_tree_test.cpp b/src/mlpack/tests/main_tests/decision_tree_test.cpp index 56ae2c88de..37d7c51e90 100644 --- a/src/mlpack/tests/main_tests/decision_tree_test.cpp +++ b/src/mlpack/tests/main_tests/decision_tree_test.cpp @@ -167,6 +167,34 @@ BOOST_AUTO_TEST_CASE(DecisionTreeMinimumLeafSizeTest) Log::Fatal.ignoreInput = false; } +/** + * Make sure maximum depth size is always a non-negative number. + */ +BOOST_AUTO_TEST_CASE(DecisionTreeMaximumDepthTest) +{ + arma::mat inputData; + DatasetInfo info; + if (!data::Load("braziltourism.arff", inputData, info)) + BOOST_FAIL("Cannot load train dataset braziltourism.arff!"); + + arma::Row labels; + if (!data::Load("braziltourism_labels.txt", labels)) + BOOST_FAIL("Cannot load labels for braziltourism_labels.txt"); + + // Initialize an all-ones weight matrix. + arma::mat weights(1, labels.n_cols, arma::fill::ones); + + // Input training data. + SetInputParam("training", std::move(std::make_tuple(info, inputData))); + SetInputParam("labels", std::move(labels)); + SetInputParam("weights", std::move(weights)); + + SetInputParam("maximum_depth", (int) -1); // Invalid. + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} /** * Make sure minimum gain split is always a fraction in range [0,1]. */ diff --git a/src/mlpack/tests/main_tests/random_forest_test.cpp b/src/mlpack/tests/main_tests/random_forest_test.cpp index a7f1339edb..3c392b0168 100644 --- a/src/mlpack/tests/main_tests/random_forest_test.cpp +++ b/src/mlpack/tests/main_tests/random_forest_test.cpp @@ -186,6 +186,26 @@ BOOST_AUTO_TEST_CASE(RandomForestMinimumLeafSizeTest) Log::Fatal.ignoreInput = false; } +/** + * Make sure maximum depth specified is always a positive number. + */ +BOOST_AUTO_TEST_CASE(RandomForestMaximumDepthTest) +{ + arma::mat inputData; + if (!data::Load("vc2.csv", inputData)) + BOOST_FAIL("Cannot load train dataset vc2.csv!"); + + arma::Row labels; + if (!data::Load("vc2_labels.txt", labels)) + BOOST_FAIL("Cannot load labels for vc2_labels.txt"); + + SetInputParam("maximum_depth", (int) 0); // Invalid. + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + /** * Make sure only one of training data or pre-trained model is passed. */ diff --git a/src/mlpack/tests/random_forest_test.cpp b/src/mlpack/tests/random_forest_test.cpp index 1a7099566e..e015410906 100644 --- a/src/mlpack/tests/random_forest_test.cpp +++ b/src/mlpack/tests/random_forest_test.cpp @@ -223,7 +223,7 @@ BOOST_AUTO_TEST_CASE(UnweightedCategoricalLearningTest) // Train a random forest and a decision tree. RandomForest<> rf(trainingData, di, trainingLabels, 5, 25 /* 25 trees */, 1, - 1e-7, MultipleRandomDimensionSelect(4)); + 1e-7, 20, MultipleRandomDimensionSelect(4)); DecisionTree<> dt(trainingData, di, trainingLabels, 5, 5); // Get performance statistics on test data. @@ -281,7 +281,7 @@ BOOST_AUTO_TEST_CASE(WeightedCategoricalLearningTest) // Build a random forest and a decision tree. RandomForest<> rf(fullData, di, fullLabels, 5, weights, 25 /* 25 trees */, 1, - 1e-7, MultipleRandomDimensionSelect(4)); + 1e-7, 20, MultipleRandomDimensionSelect(4)); DecisionTree<> dt(fullData, di, fullLabels, 5, weights, 5); // Get performance statistics on test data. @@ -445,14 +445,14 @@ BOOST_AUTO_TEST_CASE(RandomForestCategoricalTrainReturnEntropy) // Test random forest on unweighted categorical dataset. RandomForest<> rf; double entropy = rf.Train(fullData, di, fullLabels, 5, 15 /* 15 trees */, 1, - 1e-7, MultipleRandomDimensionSelect(3)); + 1e-7, 20, MultipleRandomDimensionSelect(3)); BOOST_REQUIRE_EQUAL(std::isfinite(entropy), true); // Test random forest on weighted categorical dataset. RandomForest<> wrf; entropy = wrf.Train(fullData, di, fullLabels, 5, weights, 15 /* 15 trees */, - 1, 1e-7, MultipleRandomDimensionSelect(3)); + 1, 1e-7, 20, MultipleRandomDimensionSelect(3)); BOOST_REQUIRE_EQUAL(std::isfinite(entropy), true); } From ece0e6ae0ea4539a1973cbc5aeb1a2e63ba2d3cf Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Sat, 1 Jun 2019 15:14:11 -0400 Subject: [PATCH 076/143] bump version to 3.1.1 --- Doxyfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doxyfile b/Doxyfile index c65f5bfcec..c8605b01be 100644 --- a/Doxyfile +++ b/Doxyfile @@ -4,7 +4,7 @@ # Project related configuration options #--------------------------------------------------------------------------- PROJECT_NAME = mlpack -PROJECT_NUMBER = 3.1.0 +PROJECT_NUMBER = 3.1.1 OUTPUT_DIRECTORY = ./doc CREATE_SUBDIRS = NO OUTPUT_LANGUAGE = English From 62149347e37e3ad0083f889d6ec149933f4dd85d Mon Sep 17 00:00:00 2001 From: robotcator Date: Sun, 2 Jun 2019 10:04:43 +0800 Subject: [PATCH 077/143] fix code style --- .../tests/main_tests/hoeffding_tree_test.cpp | 74 +++++++++---------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/src/mlpack/tests/main_tests/hoeffding_tree_test.cpp b/src/mlpack/tests/main_tests/hoeffding_tree_test.cpp index 8f2fd7e2c8..1c73aa9ab2 100644 --- a/src/mlpack/tests/main_tests/hoeffding_tree_test.cpp +++ b/src/mlpack/tests/main_tests/hoeffding_tree_test.cpp @@ -76,15 +76,15 @@ BOOST_AUTO_TEST_CASE(HoeffdingTreeOutputDimensionTest) // Check that number of output points are equal to number of input points. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_cols, testSize); + CLI::GetParam>("predictions").n_cols, testSize); BOOST_REQUIRE_EQUAL(CLI::GetParam("probabilities").n_cols, - testSize); + testSize); // Check number of output rows equals 1 for probabilities and predictions. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_rows, 1); + CLI::GetParam>("predictions").n_rows, 1); BOOST_REQUIRE_EQUAL( - CLI::GetParam("probabilities").n_rows, 1); + CLI::GetParam("probabilities").n_rows, 1); } /** @@ -119,15 +119,15 @@ BOOST_AUTO_TEST_CASE(HoeffdingTreeCategoricalOutputDimensionTest) // Check that number of output points are equal to number of input points. BOOST_REQUIRE_EQUAL(CLI::GetParam> - ("predictions").n_cols, testSize); + ("predictions").n_cols, testSize); BOOST_REQUIRE_EQUAL(CLI::GetParam("probabilities").n_cols, - testSize); + testSize); // Check number of output rows equals 1 for probabilities and predictions. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_rows, 1); + CLI::GetParam>("predictions").n_rows, 1); BOOST_REQUIRE_EQUAL( - CLI::GetParam("probabilities").n_rows, 1); + CLI::GetParam("probabilities").n_rows, 1); } /** @@ -173,9 +173,9 @@ BOOST_AUTO_TEST_CASE(HoeffdingTreeLabelLessTest) // Check number of output rows equals number of classes in case of // probabilities and 1 for predictions. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_rows, 1); + CLI::GetParam>("predictions").n_rows, 1); BOOST_REQUIRE_EQUAL( - CLI::GetParam("probabilities").n_rows, 1); + CLI::GetParam("probabilities").n_rows, 1); // Reset passed parameters. CLI::GetSingleton().Parameters()["training"].wasPassed = false; @@ -200,21 +200,21 @@ BOOST_AUTO_TEST_CASE(HoeffdingTreeLabelLessTest) // Check that number of output points are equal to number of input points. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_cols, testSize); + CLI::GetParam>("predictions").n_cols, testSize); BOOST_REQUIRE_EQUAL( - CLI::GetParam("probabilities").n_cols, testSize); + CLI::GetParam("probabilities").n_cols, testSize); // Check number of output rows equals 1 for probabilities and predictions. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_rows, 1); + CLI::GetParam>("predictions").n_rows, 1); BOOST_REQUIRE_EQUAL( - CLI::GetParam("probabilities").n_rows, 1); + CLI::GetParam("probabilities").n_rows, 1); // Check that initial and current predictions are same. CheckMatrices( - predictions, CLI::GetParam>("predictions")); + predictions, CLI::GetParam>("predictions")); CheckMatrices( - probabilities, CLI::GetParam("probabilities")); + probabilities, CLI::GetParam("probabilities")); } /** @@ -262,26 +262,26 @@ BOOST_AUTO_TEST_CASE(HoeffdingModelReuseTest) // Input trained model. SetInputParam("test", std::move(std::make_tuple(info, testData))); SetInputParam("input_model", - CLI::GetParam("output_model")); + CLI::GetParam("output_model")); mlpackMain(); // Check that number of output points are equal to number of input points. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_cols, testSize); + CLI::GetParam>("predictions").n_cols, testSize); BOOST_REQUIRE_EQUAL( - CLI::GetParam("probabilities").n_cols, testSize); + CLI::GetParam("probabilities").n_cols, testSize); // Check number of output rows equals 1 for probabilities and predictions. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_rows, 1); + CLI::GetParam>("predictions").n_rows, 1); BOOST_REQUIRE_EQUAL(CLI::GetParam("probabilities").n_rows, 1); // Check that initial predictions and predictions using saved model are same. CheckMatrices( - predictions, CLI::GetParam>("predictions")); + predictions, CLI::GetParam>("predictions")); CheckMatrices( - probabilities, CLI::GetParam("probabilities")); + probabilities, CLI::GetParam("probabilities")); } /** @@ -329,27 +329,27 @@ BOOST_AUTO_TEST_CASE(HoeffdingModelCategoricalReuseTest) // Input trained model. SetInputParam("test", std::move(std::make_tuple(info, testData))); SetInputParam("input_model", - CLI::GetParam("output_model")); + CLI::GetParam("output_model")); mlpackMain(); // Check that number of output points are equal to number of input points. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_cols, testSize); + CLI::GetParam>("predictions").n_cols, testSize); BOOST_REQUIRE_EQUAL( - CLI::GetParam("probabilities").n_cols, testSize); + CLI::GetParam("probabilities").n_cols, testSize); // Check number of output rows equals 1 for probabilities and predictions. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_rows, 1); + CLI::GetParam>("predictions").n_rows, 1); BOOST_REQUIRE_EQUAL( - CLI::GetParam("probabilities").n_rows, 1); + CLI::GetParam("probabilities").n_rows, 1); // Check that initial predictions and predictions using saved model are same. CheckMatrices( - predictions, CLI::GetParam>("predictions")); + predictions, CLI::GetParam>("predictions")); CheckMatrices( - probabilities, CLI::GetParam("probabilities")); + probabilities, CLI::GetParam("probabilities")); } /** @@ -417,8 +417,8 @@ BOOST_AUTO_TEST_CASE(HoeffdingMinSamplesTest) // Check that small min_samples creates larger model. BOOST_REQUIRE_LT( - (CLI::GetParam("output_model"))->NumNodes(), - nodes); + (CLI::GetParam("output_model"))->NumNodes(), + nodes); } /** @@ -486,7 +486,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingMaxSamplesTest) // Check that large max_samples creates smaller model. BOOST_REQUIRE_LT(nodes, - (CLI::GetParam("output_model"))->NumNodes()); + (CLI::GetParam("output_model"))->NumNodes()); } /** @@ -552,7 +552,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingConfidenceTest) mlpackMain(); // Check that higher confidence creates smaller tree. BOOST_REQUIRE_LT(nodes, - (CLI::GetParam("output_model"))->NumNodes()); + (CLI::GetParam("output_model"))->NumNodes()); } /** @@ -619,7 +619,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingPassesTest) // Check that model with larger number of passes has greater number of nodes. BOOST_REQUIRE_LT(nodes, - (CLI::GetParam("output_model"))->NumNodes()); + (CLI::GetParam("output_model"))->NumNodes()); } /** @@ -656,7 +656,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingBinarySplittingStrategyTest) // Check that number of children is 2. BOOST_REQUIRE_EQUAL( - (CLI::GetParam("output_model"))->NumNodes()-1, 2); + (CLI::GetParam("output_model"))->NumNodes()-1, 2); } /** @@ -728,7 +728,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingDomingosSplittingStrategyTest) // Check that both models have different number of nodes. BOOST_CHECK_NE( - (CLI::GetParam("output_model"))->NumNodes(), nodes); + (CLI::GetParam("output_model"))->NumNodes(), nodes); } /** @@ -773,7 +773,7 @@ BOOST_AUTO_TEST_CASE(HoeffdingBinningTest) // Check that no splitting has happened. BOOST_REQUIRE_EQUAL( - (CLI::GetParam("output_model"))->NumNodes(), 1); + (CLI::GetParam("output_model"))->NumNodes(), 1); } BOOST_AUTO_TEST_SUITE_END(); From a1bac97c23e507110b1aa1420002f58973c2c6fd Mon Sep 17 00:00:00 2001 From: robotcator Date: Sun, 2 Jun 2019 10:06:02 +0800 Subject: [PATCH 078/143] fix code style --- src/mlpack/tests/main_tests/hoeffding_tree_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/main_tests/hoeffding_tree_test.cpp b/src/mlpack/tests/main_tests/hoeffding_tree_test.cpp index 1c73aa9ab2..bf3c22d32d 100644 --- a/src/mlpack/tests/main_tests/hoeffding_tree_test.cpp +++ b/src/mlpack/tests/main_tests/hoeffding_tree_test.cpp @@ -166,9 +166,9 @@ BOOST_AUTO_TEST_CASE(HoeffdingTreeLabelLessTest) // Check that number of output points are equal to number of input points. BOOST_REQUIRE_EQUAL( - CLI::GetParam>("predictions").n_cols, testSize); + CLI::GetParam>("predictions").n_cols, testSize); BOOST_REQUIRE_EQUAL(CLI::GetParam("probabilities").n_cols, - testSize); + testSize); // Check number of output rows equals number of classes in case of // probabilities and 1 for predictions. From a3b2bc8c2b87b1f52fdd11eddaaeb3d656c3b7c7 Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Wed, 5 Jun 2019 02:50:17 +0700 Subject: [PATCH 079/143] Style Fix. Adapt to review. --- src/mlpack/methods/ann/layer/highway.hpp | 22 ++++++------- src/mlpack/methods/ann/layer/highway_impl.hpp | 9 ++++-- src/mlpack/tests/ann_layer_test.cpp | 2 +- src/mlpack/tests/feedforward_network_test.cpp | 32 ++++++++----------- 4 files changed, 32 insertions(+), 33 deletions(-) diff --git a/src/mlpack/methods/ann/layer/highway.hpp b/src/mlpack/methods/ann/layer/highway.hpp index 8c2b09efbc..97c10e35ff 100644 --- a/src/mlpack/methods/ann/layer/highway.hpp +++ b/src/mlpack/methods/ann/layer/highway.hpp @@ -3,7 +3,7 @@ * @author Konstantin Sidorov * @author Saksham Bansal * - * Definition of the highway layer. + * Definition of the Highway layer. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -30,7 +30,7 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * Implementation of the Highway layer. The highway class can vary its behavior + * Implementation of the Highway layer. The Highway class can vary its behavior * between that of feed-forward fully connected network container and that * of a layer which simply passes its inputs through depending on the transform * gate. Note that the size of the input and output matrices of this class @@ -73,7 +73,7 @@ class Highway //! Destroy the Highway object. ~Highway(); - /* + /** * Destroy all the modules added to the Highway object. */ void DeleteModules(); @@ -84,7 +84,7 @@ class Highway void Reset(); /** - * Ordinary feed forward pass of a neural network, evaluating the function + * Ordinary feed-forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. * * @param input Input data used for evaluating the specified function. @@ -94,9 +94,9 @@ class Highway void Forward(arma::Mat&& input, arma::Mat&& output); /** - * Ordinary feed backward pass of a neural network, calculating the function - * f(x) by propagating x backwards through f. Using the results from the feed - * forward pass. + * Ordinary feed-backward pass of a neural network, calculating the function + * f(x) by propagating x backwards through f. Using the results from the + * feed-forward pass. * * @param input The propagated input activation. * @param gy The backpropagated error. @@ -107,7 +107,7 @@ class Highway arma::Mat&& gy, arma::Mat&& g); - /* + /** * Calculate the gradient using the output delta and the input activation. * * @param input The input parameter used for calculating the gradient. @@ -119,7 +119,7 @@ class Highway arma::Mat&& error, arma::Mat&& gradient); - /* + /** * Add a new module to the model. * * @param args The layer parameter. @@ -127,14 +127,14 @@ class Highway template void Add(Args... args) { network.push_back(new LayerType(args...)); } - /* + /** * Add a new module to the model. * * @param layer The Layer to be added to the model. */ void Add(LayerTypes layer) { network.push_back(layer); } - //! Return the model modules. + //! Return the modules of the model. std::vector >& Model() { if (model) diff --git a/src/mlpack/methods/ann/layer/highway_impl.hpp b/src/mlpack/methods/ann/layer/highway_impl.hpp index 22b1a187e0..87acf9553d 100644 --- a/src/mlpack/methods/ann/layer/highway_impl.hpp +++ b/src/mlpack/methods/ann/layer/highway_impl.hpp @@ -3,7 +3,7 @@ * @author Konstantin Sidorov * @author Saksham Bansal * - * Implementation of highway layer. + * Implementation of Highway layer. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -58,7 +58,9 @@ Highway::~Highway() if (!model) { for (LayerTypes& layer : network) + { boost::apply_visitor(deleteVisitor, layer); + } } } @@ -67,7 +69,7 @@ template::DeleteModules() { - if (model == true) + if (model) { for (LayerTypes& layer : network) { @@ -146,7 +148,8 @@ void Highway::Forward( output = boost::apply_visitor(outputParameterVisitor, network.back()); - if (arma::size(output) != arma::size(input)){ + if (arma::size(output) != arma::size(input)) + { Log::Fatal << "The sizes of the output and input matrices of the Highway" << " network should be equal. Please examine the network layers."; } diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 5d4fd0e701..1335184d5b 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -2339,7 +2339,7 @@ BOOST_AUTO_TEST_CASE(SimpleResidualLayerTest) } /** - * Simple highway module test. + * Simple Highway module test. */ BOOST_AUTO_TEST_CASE(SimpleHighwayLayerTest) { diff --git a/src/mlpack/tests/feedforward_network_test.cpp b/src/mlpack/tests/feedforward_network_test.cpp index c6ce1b0b1d..3ad3864b92 100644 --- a/src/mlpack/tests/feedforward_network_test.cpp +++ b/src/mlpack/tests/feedforward_network_test.cpp @@ -114,10 +114,9 @@ BOOST_AUTO_TEST_CASE(VanillaNetworkTest) model.Add >(); // Vanilla neural net with logistic activation function. - // Because 92 percent of the patients are not hyperthyroid the neural + // Because 92% of the patients are not hyperthyroid the neural // network must be significant better than 92%. - TestNetwork<> - (model, trainData, trainLabels, testData, testLabels, 10, 0.1); + TestNetwork<>(model, trainData, trainLabels, testData, testLabels, 10, 0.1); arma::mat dataset; dataset.load("mnist_first250_training_4s_and_9s.arm"); @@ -136,8 +135,7 @@ BOOST_AUTO_TEST_CASE(VanillaNetworkTest) model1.Add >(10, 2); model1.Add >(); // Vanilla neural net with logistic activation function. - TestNetwork<> - (model1, dataset, labels, dataset, labels, 10, 0.2); + TestNetwork<>(model1, dataset, labels, dataset, labels, 10, 0.2); } BOOST_AUTO_TEST_CASE(ForwardBackwardTest) @@ -264,7 +262,7 @@ BOOST_AUTO_TEST_CASE(DropoutNetworkTest) model.Add >(); // Vanilla neural net with logistic activation function. - // Because 92 percent of the patients are not hyperthyroid the neural + // Because 92% of the patients are not hyperthyroid the neural // network must be significant better than 92%. TestNetwork<>(model, trainData, trainLabels, testData, testLabels, 10, 0.1); arma::mat dataset; @@ -272,7 +270,9 @@ BOOST_AUTO_TEST_CASE(DropoutNetworkTest) // Normalize each point since these are images. for (size_t i = 0; i < dataset.n_cols; ++i) + { dataset.col(i) /= norm(dataset.col(i), 2); + } arma::mat labels = arma::zeros(1, dataset.n_cols); labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); @@ -285,8 +285,7 @@ BOOST_AUTO_TEST_CASE(DropoutNetworkTest) model1.Add >(10, 2); model1.Add >(); // Vanilla neural net with logistic activation function. - TestNetwork<> - (model1, dataset, labels, dataset, labels, 10, 0.2); + TestNetwork<>(model1, dataset, labels, dataset, labels, 10, 0.2); } /** @@ -313,12 +312,11 @@ BOOST_AUTO_TEST_CASE(HighwayNetworkTest) model.Add(highway); model.Add >(10, 2); model.Add >(); - TestNetwork<> - (model, dataset, labels, dataset, labels, 10, 0.2); + TestNetwork<>(model, dataset, labels, dataset, labels, 10, 0.2); } /** - * Train the dropconnect network on a larger dataset. + * Train the DropConnect network on a larger dataset. */ BOOST_AUTO_TEST_CASE(DropConnectNetworkTest) { @@ -366,10 +364,9 @@ BOOST_AUTO_TEST_CASE(DropConnectNetworkTest) model.Add >(); // Vanilla neural net with logistic activation function. - // Because 92 percent of the patients are not hyperthyroid the neural + // Because 92% of the patients are not hyperthyroid the neural // network must be significant better than 92%. - TestNetwork<> - (model, trainData, trainLabels, testData, testLabels, 10, 0.1); + TestNetwork<>(model, trainData, trainLabels, testData, testLabels, 10, 0.1); arma::mat dataset; dataset.load("mnist_first250_training_4s_and_9s.arm"); @@ -388,8 +385,7 @@ BOOST_AUTO_TEST_CASE(DropConnectNetworkTest) model1.Add >(10, 2); model1.Add >(); // Vanilla neural net with logistic activation function. - TestNetwork<> - (model1, dataset, labels, dataset, labels, 10, 0.2); + TestNetwork<>(model1, dataset, labels, dataset, labels, 10, 0.2); } /** @@ -427,7 +423,7 @@ BOOST_AUTO_TEST_CASE(SerializationTest) testData.shed_row(testData.n_rows - 1); // Vanilla neural net with logistic activation function. - // Because 92 percent of the patients are not hyperthyroid the neural + // Because 92% of the patients are not hyperthyroid the neural // network must be significant better than 92%. FFN > model; model.Add >(trainData.n_rows, 8); @@ -555,7 +551,7 @@ BOOST_AUTO_TEST_CASE(FFNTrainReturnObjective) testData.shed_row(testData.n_rows - 1); // Vanilla neural net with logistic activation function. - // Because 92 percent of the patients are not hyperthyroid the neural + // Because 92% of the patients are not hyperthyroid the neural // network must be significantly better than 92%. FFN > model; model.Add >(trainData.n_rows, 8); From b724be55b06a1ecbc467171044c6a0511071573b Mon Sep 17 00:00:00 2001 From: robotcator Date: Wed, 5 Jun 2019 22:33:12 +0800 Subject: [PATCH 080/143] fix code style --- .../reinforcement_learning/q_learning.hpp | 3 ++- .../reinforcement_learning/q_learning_impl.hpp | 16 ++++++++-------- .../replay/prioritized_replay.hpp | 2 +- .../replay/random_replay.hpp | 8 ++++---- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning.hpp b/src/mlpack/methods/reinforcement_learning/q_learning.hpp index e84128ab9d..3b8944c3d8 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning.hpp @@ -170,7 +170,8 @@ class QLearning //! Locally-stored flag indicating training mode or test mode. bool deterministic; - bool prioritized_replay; + //! Locally-stored flag indicating whether prioritized replay buffer or not. + bool prioritizedReplay; }; } // namespace rl diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index e87484569f..399a9d1021 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -46,7 +46,7 @@ QLearning< environment(std::move(environment)), totalSteps(0), deterministic(false), - prioritized_replay(false) + prioritizedReplay(false) { // Set up q-learning network. if (learningNetwork.Parameters().is_empty()) @@ -85,12 +85,12 @@ QLearning< environment(std::move(environment)), totalSteps(0), deterministic(false), - prioritized_replay(true) + prioritizedReplay(true) { if (learningNetwork.Parameters().is_empty()) learningNetwork.ResetParameters(); this->updater.Initialize(learningNetwork.Parameters().n_rows, - learningNetwork.Parameters().n_cols); + learningNetwork.Parameters().n_cols); targetNetwork = learningNetwork; } @@ -151,7 +151,7 @@ double QLearning< double reward = environment.Sample(state, action, nextState); // Store the transition for replay. - if (prioritized_replay) + if (prioritizedReplay) { prioritizedReplayMethod.Store(state, action, reward, nextState, environment.IsTerminal(nextState)); @@ -177,7 +177,7 @@ double QLearning< arma::mat sampledNextStates; arma::icolvec isTerminal; - if (!prioritized_replay) + if (!prioritizedReplay) { replayMethod.Sample(sampledStates, sampledActions, sampledRewards, sampledNextStates, isTerminal); @@ -226,15 +226,15 @@ double QLearning< arma::mat gradients; learningNetwork.Backward(target, gradients); - if (prioritized_replay) + if (prioritizedReplay) { prioritizedReplayMethod.Update(target, sampledActions, - nextActionValues, gradients); + nextActionValues, gradients); } else { replayMethod.Update(target, sampledActions, - nextActionValues, gradients); + nextActionValues, gradients); } updater.Update(learningNetwork.Parameters(), config.StepSize(), gradients); diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index 68891a7883..7c1f95e2b9 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -154,7 +154,7 @@ class PrioritizedReplay arma::mat& sampledNextStates, arma::icolvec& isTerminal) { - size_t upperBound = full ? capacity : position; +// size_t upperBound = full ? capacity : position; sampledIndices = SampleProportional(); BetaAnneal(); diff --git a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp index cc07701cb9..845cfd70ee 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp @@ -150,10 +150,10 @@ class RandomReplay * @param nextActionValues Agent's next action * @param gradients The model's gradients */ - void Update(arma::mat target, - arma::icolvec sampledActions, - arma::mat nextActionValues, - arma::mat& gradients) + void Update(arma::mat /* target */, + arma::icolvec /* sampledActions */, + arma::mat /* nextActionValues */, + arma::mat& /* gradients */) { /* do nothing for random replay*/ } From 5084c903bee04b60515754c4cdaae3685d9d91ab Mon Sep 17 00:00:00 2001 From: Yashwant Date: Thu, 6 Jun 2019 20:08:18 +0530 Subject: [PATCH 081/143] Add more Test --- .../tests/main_tests/gmm_generate_test.cpp | 15 -- .../tests/main_tests/gmm_probability_test.cpp | 19 ++- .../tests/main_tests/gmm_train_test.cpp | 153 +++++++++++++++--- 3 files changed, 149 insertions(+), 38 deletions(-) diff --git a/src/mlpack/tests/main_tests/gmm_generate_test.cpp b/src/mlpack/tests/main_tests/gmm_generate_test.cpp index 494b99097d..0063c17aeb 100644 --- a/src/mlpack/tests/main_tests/gmm_generate_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_generate_test.cpp @@ -56,21 +56,6 @@ BOOST_AUTO_TEST_CASE(GmmGenerateSamplesTest) Log::Fatal.ignoreInput = false; } -// Making sure samples are provided. -BOOST_AUTO_TEST_CASE(GmmGenerateSamples) -{ - arma::mat inputData(5, 10, arma::fill::randu); - - GMM gmm(1, 5); - gmm.Train(inputData, 5); - - SetInputParam("input_model", &gmm); - - Log::Fatal.ignoreInput = true; - BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); - Log::Fatal.ignoreInput = false; -} - // Checking dimensionality of output. BOOST_AUTO_TEST_CASE(GmmGenerateDimensionality) { diff --git a/src/mlpack/tests/main_tests/gmm_probability_test.cpp b/src/mlpack/tests/main_tests/gmm_probability_test.cpp index 17d5a8f983..7e5853ab7f 100644 --- a/src/mlpack/tests/main_tests/gmm_probability_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_probability_test.cpp @@ -49,6 +49,21 @@ void ResetGmmProbabilitySetting() BOOST_FIXTURE_TEST_SUITE(GmmProbabilityMainTest, GmmProbabilityTestFixture); +// Making sure input_file are provided. +BOOST_AUTO_TEST_CASE(GmmProbabilityInputTest) +{ + arma::mat inputData(5, 10, arma::fill::randu); + + GMM gmm(1, 5); + gmm.Train(inputData, 5); + + SetInputParam("input_model", &gmm); + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + // Checking the input and output dimensionality. BOOST_AUTO_TEST_CASE(GmmProbabilityDimensionality) { @@ -57,14 +72,14 @@ BOOST_AUTO_TEST_CASE(GmmProbabilityDimensionality) GMM gmm(1, 5); gmm.Train(std::move(inputData), 5); - arma::mat inputPoints(1, 8, arma::fill::randu); + arma::mat inputPoints(1, 5, arma::fill::randu); SetInputParam("input", std::move(inputPoints)); SetInputParam("input_model", &gmm); mlpackMain(); - BOOST_REQUIRE_EQUAL(CLI::GetParam("output").n_cols, 8); + BOOST_REQUIRE_EQUAL(CLI::GetParam("output").n_cols, 5); BOOST_REQUIRE_EQUAL(CLI::GetParam("output").n_rows, 1); } diff --git a/src/mlpack/tests/main_tests/gmm_train_test.cpp b/src/mlpack/tests/main_tests/gmm_train_test.cpp index 2a9d1672a4..197c7fc1a6 100644 --- a/src/mlpack/tests/main_tests/gmm_train_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_train_test.cpp @@ -97,7 +97,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainMaxIterationsTest) } // Ensure that Trials must be greater than 0. -BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) +BOOST_AUTO_TEST_CASE(GmmTrainPositiveTrialsTest) { arma::mat inputData(5, 10, arma::fill::randu); @@ -167,16 +167,48 @@ BOOST_AUTO_TEST_CASE(GmmTrainNumberOfGaussian) BOOST_REQUIRE_EQUAL(gmm1->Gaussians(), (int) 2); } -// Ensure that Noise affects the final result. -BOOST_AUTO_TEST_CASE(GmmTrainNoisetest) +// Making sure that enabling no_force_positive doesn't crash. +BOOST_AUTO_TEST_CASE(GmmTrainNoForcePositiveTest) { arma::mat inputData(5, 10, arma::fill::randu); + SetInputParam("input", inputData); + SetInputParam("gaussians", (int) 1); + SetInputParam("no_force_positive", true); + + mlpackMain(); + + GMM* gmm = CLI::GetParam("output_model"); + + SetInputParam("input_model", gmm); + + CLI::GetSingleton().Parameters()["input"].wasPassed = false; + + SetInputParam("input", std::move(inputData)); + + mlpackMain(); + + GMM* gmm1 = CLI::GetParam("output_model"); + + BOOST_REQUIRE_EQUAL(gmm1->Gaussians(), (int) 1); +} + +// Ensure that Noise affects the final result. +BOOST_AUTO_TEST_CASE(GmmTrainNoiseTest) +{ + arma::mat inputData; + if (!data::Load("data_3d_mixed.txt", inputData)) + BOOST_FAIL("Unable to load train dataset data_3d_mixed.txt!"); + SetInputParam("input", inputData); SetInputParam("gaussians", (int) 2); SetInputParam("noise", (double) 0.0); - mlpack::math::FixedRandomSeed(); + size_t seed = std::time(NULL); + mlpack::math::randGen.seed((uint32_t) seed); + srand((unsigned int) seed); + arma::arma_rng::set_seed(seed); + mlpackMain(); GMM* gmm = CLI::GetParam("output_model"); @@ -187,7 +219,10 @@ BOOST_AUTO_TEST_CASE(GmmTrainNoisetest) SetInputParam("gaussians", (int) 2); SetInputParam("noise", (double) 100.0); - mlpack::math::FixedRandomSeed(); + mlpack::math::randGen.seed((uint32_t) seed); + srand((unsigned int) seed); + arma::arma_rng::set_seed(seed); + mlpackMain(); GMM* gmm1 = CLI::GetParam("output_model"); @@ -195,24 +230,75 @@ BOOST_AUTO_TEST_CASE(GmmTrainNoisetest) arma::uvec sortedIndices = sort_index(gmm->Weights()); for (size_t k = 0; k < sortedIndices.n_elem; k++) + { BOOST_REQUIRE(arma::norm(gmm->Component(sortedIndices[k]).Mean() - gmm1->Component(sortedIndices[k]).Mean()) > 1e-50 || arma::norm(gmm->Component(sortedIndices[k]).Covariance() - gmm1->Component(sortedIndices[k]).Covariance()) > 1e-50); + } +} + +// Ensure that Trials affects the final result. +BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) +{ + arma::mat inputData(5, 100, arma::fill::randu); + + SetInputParam("input", inputData); + SetInputParam("gaussians", (int) 2); + SetInputParam("trials", (int) 1); + + size_t seed = std::time(NULL); + mlpack::math::randGen.seed((uint32_t) seed); + srand((unsigned int) seed); + arma::arma_rng::set_seed(seed); + + mlpackMain(); + + GMM* gmm = CLI::GetParam("output_model"); + + ResetGmmTrainSetting(); + + SetInputParam("input", std::move(inputData)); + SetInputParam("gaussians", (int) 2); + SetInputParam("trials", (int) 500); + + mlpack::math::randGen.seed((uint32_t) seed); + srand((unsigned int) seed); + arma::arma_rng::set_seed(seed); + + mlpackMain(); + + GMM* gmm1 = CLI::GetParam("output_model"); + + arma::uvec sortedIndices = sort_index(gmm->Weights()); + + for (size_t k = 0; k < sortedIndices.n_elem; k++) + { + BOOST_REQUIRE(arma::norm(gmm->Component(sortedIndices[k]).Mean() - + gmm1->Component(sortedIndices[k]).Mean()) > 1e-50 || + arma::norm(gmm->Component(sortedIndices[k]).Covariance() - + gmm1->Component(sortedIndices[k]).Covariance()) > 1e-50); + } } // Ensure that Percentage affects the final result when refined_start is true. BOOST_AUTO_TEST_CASE(GmmTrainPercentageTest) { - arma::mat inputData(5, 10, arma::fill::randu); + arma::mat inputData; + if (!data::Load("data_3d_mixed.txt", inputData)) + BOOST_FAIL("Unable to load train dataset data_3d_mixed.txt!"); SetInputParam("input", inputData); SetInputParam("gaussians", (int) 2); SetInputParam("refined_start", true); SetInputParam("percentage", (double) 0.01); - SetInputParam("samplings", (int) 200); + SetInputParam("samplings", (int) 1000); + + size_t seed = std::time(NULL); + mlpack::math::randGen.seed((uint32_t) seed); + srand((unsigned int) seed); + arma::arma_rng::set_seed(seed); - mlpack::math::FixedRandomSeed(); mlpackMain(); GMM* gmm = CLI::GetParam("output_model"); @@ -222,10 +308,13 @@ BOOST_AUTO_TEST_CASE(GmmTrainPercentageTest) SetInputParam("input", std::move(inputData)); SetInputParam("gaussians", (int) 2); SetInputParam("refined_start", true); - SetInputParam("percentage", (double) 0.99); - SetInputParam("samplings", (int) 200); + SetInputParam("percentage", (double) 0.20); + SetInputParam("samplings", (int) 1000); + + mlpack::math::randGen.seed((uint32_t) seed); + srand((unsigned int) seed); + arma::arma_rng::set_seed(seed); - mlpack::math::FixedRandomSeed(); mlpackMain(); GMM* gmm1 = CLI::GetParam("output_model"); @@ -233,24 +322,32 @@ BOOST_AUTO_TEST_CASE(GmmTrainPercentageTest) arma::uvec sortedIndices = sort_index(gmm->Weights()); for (size_t k = 0; k < sortedIndices.n_elem; k++) + { BOOST_REQUIRE(arma::norm(gmm->Component(sortedIndices[k]).Mean() - gmm1->Component(sortedIndices[k]).Mean()) > 1e-50 || arma::norm(gmm->Component(sortedIndices[k]).Covariance() - gmm1->Component(sortedIndices[k]).Covariance()) > 1e-50); + } } // Ensure that Sampling affects the final result when refined_start is true. BOOST_AUTO_TEST_CASE(GmmTrainSamplingsTest) { - arma::mat inputData(5, 10, arma::fill::randu); + arma::mat inputData; + if (!data::Load("data_3d_mixed.txt", inputData)) + BOOST_FAIL("Unable to load train dataset data_3d_mixed.txt!"); SetInputParam("input", inputData); SetInputParam("gaussians", (int) 2); SetInputParam("refined_start", true); - SetInputParam("percentage", (double) 0.950); + SetInputParam("percentage", (double) 0.2); SetInputParam("samplings", (int) 10); - mlpack::math::FixedRandomSeed(); + size_t seed = std::time(NULL); + mlpack::math::randGen.seed((uint32_t) seed); + srand((unsigned int) seed); + arma::arma_rng::set_seed(seed); + mlpackMain(); GMM* gmm = CLI::GetParam("output_model"); @@ -260,10 +357,13 @@ BOOST_AUTO_TEST_CASE(GmmTrainSamplingsTest) SetInputParam("input", std::move(inputData)); SetInputParam("gaussians", (int) 2); SetInputParam("refined_start", true); - SetInputParam("percentage", (double) 0.950); - SetInputParam("samplings", (int) 1000); + SetInputParam("percentage", (double) 0.2); + SetInputParam("samplings", (int) 10000); + + mlpack::math::randGen.seed((uint32_t) seed); + srand((unsigned int) seed); + arma::arma_rng::set_seed(seed); - mlpack::math::FixedRandomSeed(); mlpackMain(); GMM* gmm1 = CLI::GetParam("output_model"); @@ -271,24 +371,30 @@ BOOST_AUTO_TEST_CASE(GmmTrainSamplingsTest) arma::uvec sortedIndices = sort_index(gmm->Weights()); for (size_t k = 0; k < sortedIndices.n_elem; k++) + { BOOST_REQUIRE(arma::norm(gmm->Component(sortedIndices[k]).Mean() - gmm1->Component(sortedIndices[k]).Mean()) > 1e-50 || arma::norm(gmm->Component(sortedIndices[k]).Covariance() - gmm1->Component(sortedIndices[k]).Covariance()) > 1e-50); + } } // Ensure that tolerance affects the final result. BOOST_AUTO_TEST_CASE(GmmTrainToleranceTest) { arma::mat inputData; - if (!data::Load("vc2.csv", inputData)) - BOOST_FAIL("Unable to load train dataset vc2.csv!"); + if (!data::Load("data_3d_mixed.txt", inputData)) + BOOST_FAIL("Unable to load train dataset data_3d_mixed.txt!"); SetInputParam("input", inputData); SetInputParam("gaussians", (int) 2); SetInputParam("tolerance", (double) 1e-8); - mlpack::math::FixedRandomSeed(); + size_t seed = std::time(NULL); + mlpack::math::randGen.seed((uint32_t) seed); + srand((unsigned int) seed); + arma::arma_rng::set_seed(seed); + mlpackMain(); GMM* gmm = CLI::GetParam("output_model"); @@ -299,7 +405,10 @@ BOOST_AUTO_TEST_CASE(GmmTrainToleranceTest) SetInputParam("gaussians", (int) 2); SetInputParam("tolerance", (double) 10); - mlpack::math::FixedRandomSeed(); + mlpack::math::randGen.seed((uint32_t) seed); + srand((unsigned int) seed); + arma::arma_rng::set_seed(seed); + mlpackMain(); GMM* gmm1 = CLI::GetParam("output_model"); @@ -307,10 +416,12 @@ BOOST_AUTO_TEST_CASE(GmmTrainToleranceTest) arma::uvec sortedIndices = sort_index(gmm->Weights()); for (size_t k = 0; k < sortedIndices.n_elem; k++) + { BOOST_REQUIRE(arma::norm(gmm->Component(sortedIndices[k]).Mean() - gmm1->Component(sortedIndices[k]).Mean()) > 1e-50 || arma::norm(gmm->Component(sortedIndices[k]).Covariance() - gmm1->Component(sortedIndices[k]).Covariance()) > 1e-50); + } } // Ensure that saved model can be used again. From e3dc560adc3546b8ed77e64cb084f3c623316d8a Mon Sep 17 00:00:00 2001 From: robotcator Date: Thu, 6 Jun 2019 22:54:35 +0800 Subject: [PATCH 082/143] remove PrioritizedReplayType template parameter --- .../reinforcement_learning/q_learning.hpp | 13 +-- .../q_learning_impl.hpp | 98 +++---------------- .../reinforcement_learning/replay/sumtree.hpp | 2 +- src/mlpack/tests/q_learning_test.cpp | 7 +- 4 files changed, 20 insertions(+), 100 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning.hpp b/src/mlpack/methods/reinforcement_learning/q_learning.hpp index 3b8944c3d8..fa569aabe5 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning.hpp @@ -53,8 +53,7 @@ template < typename NetworkType, typename UpdaterType, typename PolicyType, - typename ReplayType = RandomReplay, - typename PrioritizedReplayType = PrioritizedReplay + typename ReplayType = RandomReplay > class QLearning { @@ -85,13 +84,6 @@ class QLearning UpdaterType updater = UpdaterType(), EnvironmentType environment = EnvironmentType()); - QLearning(TrainingConfig config, - NetworkType network, - PolicyType policy, - PrioritizedReplayType prioritizedReplayMethod, - UpdaterType updater = UpdaterType(), - EnvironmentType environment = EnvironmentType()); - /** * Execute a step in an episode. * @return Reward for the step. @@ -155,9 +147,6 @@ class QLearning //! Locally-stored experience method. ReplayType replayMethod; - //! Locally-stored experience method. - PrioritizedReplayType prioritizedReplayMethod; - //! Locally-stored reinforcement learning task. EnvironmentType environment; diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 399a9d1021..990ae945f3 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -22,16 +22,14 @@ template < typename NetworkType, typename UpdaterType, typename PolicyType, - typename ReplayType, - typename PrioritizedReplayType + typename ReplayType > QLearning< EnvironmentType, NetworkType, UpdaterType, PolicyType, - ReplayType, - PrioritizedReplayType + ReplayType >::QLearning(TrainingConfig config, NetworkType network, PolicyType policy, @@ -61,54 +59,14 @@ template < typename NetworkType, typename UpdaterType, typename PolicyType, - typename ReplayType, - typename PrioritizedReplayType -> -QLearning< - EnvironmentType, - NetworkType, - UpdaterType, - PolicyType, - ReplayType, - PrioritizedReplayType ->::QLearning(TrainingConfig config, - NetworkType network, - PolicyType policy, - PrioritizedReplayType prioritizedReplayMethod, - UpdaterType updater, - EnvironmentType environment): - config(std::move(config)), - learningNetwork(std::move(network)), - updater(std::move(updater)), - policy(std::move(policy)), - prioritizedReplayMethod(std::move(prioritizedReplayMethod)), - environment(std::move(environment)), - totalSteps(0), - deterministic(false), - prioritizedReplay(true) -{ - if (learningNetwork.Parameters().is_empty()) - learningNetwork.ResetParameters(); - this->updater.Initialize(learningNetwork.Parameters().n_rows, - learningNetwork.Parameters().n_cols); - targetNetwork = learningNetwork; -} - -template < - typename EnvironmentType, - typename NetworkType, - typename UpdaterType, - typename PolicyType, - typename ReplayType, - typename PrioritizedReplayType + typename ReplayType > arma::Col QLearning< EnvironmentType, NetworkType, UpdaterType, PolicyType, - ReplayType, - PrioritizedReplayType + ReplayType >::BestAction(const arma::mat& actionValues) { // Take best possible action at a particular instance. @@ -127,16 +85,14 @@ template < typename NetworkType, typename UpdaterType, typename BehaviorPolicyType, - typename ReplayType, - typename PrioritizedReplayType + typename ReplayType > double QLearning< EnvironmentType, NetworkType, UpdaterType, BehaviorPolicyType, - ReplayType, - PrioritizedReplayType + ReplayType >::Step() { // Get the action value for each action at current state. @@ -151,16 +107,8 @@ double QLearning< double reward = environment.Sample(state, action, nextState); // Store the transition for replay. - if (prioritizedReplay) - { - prioritizedReplayMethod.Store(state, action, reward, - nextState, environment.IsTerminal(nextState)); - } - else - { - replayMethod.Store(state, action, reward, - nextState, environment.IsTerminal(nextState)); - } + replayMethod.Store(state, action, reward, + nextState, environment.IsTerminal(nextState)); // Update current state. state = nextState; @@ -177,16 +125,8 @@ double QLearning< arma::mat sampledNextStates; arma::icolvec isTerminal; - if (!prioritizedReplay) - { - replayMethod.Sample(sampledStates, sampledActions, sampledRewards, - sampledNextStates, isTerminal); - } - else - { - prioritizedReplayMethod.Sample(sampledStates, sampledActions, - sampledRewards, sampledNextStates, isTerminal); - } + replayMethod.Sample(sampledStates, sampledActions, sampledRewards, + sampledNextStates, isTerminal); // Compute action value for next state with target network. arma::mat nextActionValues; @@ -226,16 +166,8 @@ double QLearning< arma::mat gradients; learningNetwork.Backward(target, gradients); - if (prioritizedReplay) - { - prioritizedReplayMethod.Update(target, sampledActions, - nextActionValues, gradients); - } - else - { - replayMethod.Update(target, sampledActions, - nextActionValues, gradients); - } + replayMethod.Update(target, sampledActions, + nextActionValues, gradients); updater.Update(learningNetwork.Parameters(), config.StepSize(), gradients); @@ -247,16 +179,14 @@ template < typename NetworkType, typename UpdaterType, typename BehaviorPolicyType, - typename ReplayType, - typename PrioritizedReplayType + typename ReplayType > double QLearning< EnvironmentType, NetworkType, UpdaterType, BehaviorPolicyType, - ReplayType, - PrioritizedReplayType + ReplayType >::Episode() { // Get the initial state from environment. diff --git a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp index 2db7232619..c1d1f3f539 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp @@ -164,7 +164,7 @@ class SumTree * */ size_t FindPrefixSum(T mass) { - int idx = 1; + size_t idx = 1; while (idx < capacity) { if (element[2 * idx] > mass) diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index e6508de92d..affaec6df1 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -116,7 +116,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDQNPrioritizedReplay) // Set up the policy and replay method. GreedyPolicy policy(1.0, 1000, 0.1); - PrioritizedReplay prioritizedReplayMethod(10, 10000, 0.6); + PrioritizedReplay replayMethod(10, 10000, 0.6); TrainingConfig config; config.StepSize() = 0.01; @@ -127,9 +127,10 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDQNPrioritizedReplay) config.StepLimit() = 200; // Set up DQN agent. - QLearning + QLearning agent(std::move(config), std::move(model), std::move(policy), - std::move(prioritizedReplayMethod)); + std::move(replayMethod)); arma::running_stat averageReturn; size_t episodes = 0; From cf12a916498ee998086fdaa88d54852d78badd18 Mon Sep 17 00:00:00 2001 From: robotcator Date: Thu, 6 Jun 2019 23:04:42 +0800 Subject: [PATCH 083/143] reorder the parameter to remove the warning --- .../replay/prioritized_replay.hpp | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index 7c1f95e2b9..0589ec5581 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -64,7 +64,6 @@ class PrioritizedReplay const size_t dimension = StateType::dimension) : batchSize(batchSize), capacity(capacity), - alpha(alpha), position(0), states(dimension, capacity), actions(capacity), @@ -72,6 +71,7 @@ class PrioritizedReplay nextStates(dimension, capacity), isTerminal(capacity), full(false), + alpha(alpha), maxPriority(1.0), initialBeta(0.6), replayBetaIters(10000) @@ -237,24 +237,6 @@ class PrioritizedReplay private: - //! How much prioritization is used. - //! (0 - no prioritization, 1 - full prioritization) - double alpha; - - double maxPriority; - - //! Initial value of beta for prioritized replay buffer. - double initialBeta; - - //! The value of beta for current sample. - double beta; - - //! How many iteration for replay beta to decay. - size_t replayBetaIters; - - //! Locally-stored the prefix sum of prioritization. - SumTree idxSum; - //! Locally-stored number of examples of each sample. size_t batchSize; @@ -282,6 +264,25 @@ class PrioritizedReplay //! Locally-stored indicator that whether the memory is full or not. bool full; + //! How much prioritization is used. + //! (0 - no prioritization, 1 - full prioritization) + double alpha; + + //! Locally-stored the max priority. + double maxPriority; + + //! Initial value of beta for prioritized replay buffer. + double initialBeta; + + //! The value of beta for current sample. + double beta; + + //! How many iteration for replay beta to decay. + size_t replayBetaIters; + + //! Locally-stored the prefix sum of prioritization. + SumTree idxSum; + //! Locally-stored the indices of sampled transitions. arma::ucolvec sampledIndices; From bea5f30536da8ce3cb4ce627664e8e6d7417f35f Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Fri, 7 Jun 2019 00:35:07 +0700 Subject: [PATCH 084/143] Cite paper url --- src/mlpack/methods/ann/layer/highway.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/methods/ann/layer/highway.hpp b/src/mlpack/methods/ann/layer/highway.hpp index 97c10e35ff..f0216b8f0b 100644 --- a/src/mlpack/methods/ann/layer/highway.hpp +++ b/src/mlpack/methods/ann/layer/highway.hpp @@ -44,6 +44,7 @@ namespace ann /** Artificial Neural Network. */ { * title = {Training Very Deep Networks}, * journal = {Advances in Neural Information Processing Systems}, * year = {2015}, + * url = {https://arxiv.org/abs/1507.06228}, * } * @endcode * From 77a964dec01c6f0d6454d83560c654cd2f12c518 Mon Sep 17 00:00:00 2001 From: robotcator Date: Fri, 7 Jun 2019 13:58:59 +0800 Subject: [PATCH 085/143] fix code style according code review --- .../methods/reinforcement_learning/q_learning.hpp | 3 --- .../reinforcement_learning/q_learning_impl.hpp | 3 +-- .../replay/prioritized_replay.hpp | 15 +++++++-------- .../reinforcement_learning/replay/sumtree.hpp | 14 +++++++------- src/mlpack/tests/q_learning_test.cpp | 4 ++-- 5 files changed, 17 insertions(+), 22 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning.hpp b/src/mlpack/methods/reinforcement_learning/q_learning.hpp index fa569aabe5..bf71c93b64 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning.hpp @@ -158,9 +158,6 @@ class QLearning //! Locally-stored flag indicating training mode or test mode. bool deterministic; - - //! Locally-stored flag indicating whether prioritized replay buffer or not. - bool prioritizedReplay; }; } // namespace rl diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 990ae945f3..0caf91cec0 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -43,8 +43,7 @@ QLearning< replayMethod(std::move(replayMethod)), environment(std::move(environment)), totalSteps(0), - deterministic(false), - prioritizedReplay(false) + deterministic(false) { // Set up q-learning network. if (learningNetwork.Parameters().is_empty()) diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index 0589ec5581..e4e551e449 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -2,7 +2,7 @@ * @file prioritized_experience_replay.hpp * @author Xiaohong * - * This file is an implementation of prioritized experience repla y. + * This file is an implementation of prioritized experience replay. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -20,7 +20,8 @@ namespace rl { /** * Implementation of prioritized experience replay. - * + * PER can replay important transitions more frequently by prioritizing + * transitions, and make agent learn more efficiently. * * @code * @article{schaul2015prioritized, @@ -58,10 +59,10 @@ class PrioritizedReplay * @param dimension The dimension of an encoded state. */ PrioritizedReplay(const size_t batchSize, - const size_t capacity, - const double alpha, - int seed = 1024, - const size_t dimension = StateType::dimension) : + const size_t capacity, + const double alpha, + int seed = 1024, + const size_t dimension = StateType::dimension) : batchSize(batchSize), capacity(capacity), position(0), @@ -154,8 +155,6 @@ class PrioritizedReplay arma::mat& sampledNextStates, arma::icolvec& isTerminal) { -// size_t upperBound = full ? capacity : position; - sampledIndices = SampleProportional(); BetaAnneal(); diff --git a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp index c1d1f3f539..92ecbccaea 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp @@ -28,8 +28,8 @@ namespace rl { * * Used to maintain prefix-sum of an array. * + * @tparam T The array's element type. */ - template class SumTree { @@ -83,7 +83,7 @@ class SumTree { element[indices[i] + capacity] = data[i]; } - // update the total tree with bottom-up technique + // update the total tree with bottom-up technique. for (size_t i = capacity-1; i > 0; i--) { element[i] = element[2 * i] + element[2 * i + 1]; @@ -107,8 +107,8 @@ class SumTree * @param start The starting position of subsequence. * @param end The end position of subsequence. * @param node Reference position - * @param node_start Starting position of reference segment. - * @param node_end End position of reference segment. + * @param nodeStart Starting position of reference segment. + * @param nodeEnd End position of reference segment. */ T SumHelper(size_t start, size_t end, size_t node, size_t nodeStart, size_t nodeEnd) @@ -140,7 +140,7 @@ class SumTree * Calculate the sum of contiguous subsequence of the array. * * @param start The starting position of subsequence. - * @param _end The end position of subsequence. + * @param end The end position of subsequence. */ T Sum(size_t start, size_t end) { @@ -160,7 +160,7 @@ class SumTree * Find the highest index `idx` in the array such that * sum(arr[0] + arr[1] + ... + arr[idx]) <= mass. * - * @param mass + * @param mass The upper bound of segment array sum. * */ size_t FindPrefixSum(T mass) { @@ -184,7 +184,7 @@ class SumTree //! The capacity of the data array. size_t capacity; - //! double size of capacity, maintain the segment sum of data. + //! Double size of capacity, maintain the segment sum of data. std::vector element; }; diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index affaec6df1..886fe47e4c 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -102,7 +102,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDQN) BOOST_REQUIRE(converged); } -//! Test DQN in Cart Pole task with Prioritized Replay +//! Test DQN in Cart Pole task with Prioritized Replay. BOOST_AUTO_TEST_CASE(CartPoleWithDQNPrioritizedReplay) { // Set up the network. @@ -129,7 +129,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDQNPrioritizedReplay) // Set up DQN agent. QLearning - agent(std::move(config), std::move(model), std::move(policy), + agent(std::move(config), std::move(model), std::move(policy), std::move(replayMethod)); arma::running_stat averageReturn; From 2ffd1770878db226a341e4a4877ca59d5ab619d7 Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Fri, 7 Jun 2019 14:52:06 +0700 Subject: [PATCH 086/143] Comment for boost::variant TODO --- src/mlpack/methods/ann/layer/layer_types.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 5e98f11a01..48e2d46abd 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -198,6 +198,8 @@ using LayerTypes = boost::variant< NegativeLogLikelihood*, PReLU*, Recurrent*, + // TODO find workaround to support more than 50 types + // as boost::variant can only be used for up to 50 types. // RecurrentAttention*, ReinforceNormal*, Reparametrization*, From 93e4f9d1e9ca218fd1be5c439774cea1931f4573 Mon Sep 17 00:00:00 2001 From: Yashwant Date: Fri, 7 Jun 2019 20:49:24 +0530 Subject: [PATCH 087/143] Add Tests --- .../methods/decision_tree/decision_tree.hpp | 16 ++--- .../decision_tree/decision_tree_impl.hpp | 22 +++---- .../decision_tree/decision_tree_main.cpp | 6 +- .../methods/random_forest/random_forest.hpp | 16 ++--- .../random_forest/random_forest_main.cpp | 7 +-- src/mlpack/tests/decision_tree_test.cpp | 54 +++++++++++++---- .../tests/main_tests/decision_tree_test.cpp | 60 ++++++++++++++++++- src/mlpack/tests/random_forest_test.cpp | 8 +-- 8 files changed, 140 insertions(+), 49 deletions(-) diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp index e544a53d11..b898774656 100644 --- a/src/mlpack/methods/decision_tree/decision_tree.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree.hpp @@ -74,7 +74,7 @@ class DecisionTree : const size_t numClasses, const size_t minimumLeafSize = 10, const double minimumGainSplit = 1e-7, - const size_t maximumDepth = 20, + const size_t maximumDepth = 0, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -100,7 +100,7 @@ class DecisionTree : const size_t numClasses, const size_t minimumLeafSize = 10, const double minimumGainSplit = 1e-7, - const size_t maximumDepth = 20, + const size_t maximumDepth = 0, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -131,7 +131,7 @@ class DecisionTree : WeightsType weights, const size_t minimumLeafSize = 10, const double minimumGainSplit = 1e-7, - const size_t maximumDepth = 20, + const size_t maximumDepth = 0, DimensionSelectionType dimensionSelector = DimensionSelectionType(), const std::enable_if_tTrain(data, currentChildBegin, currentCol - currentChildBegin, datasetInfo, labels, numClasses, weights, currentCol - currentChildBegin, minimumGainSplit, - maximumDepth, dimensionSelector); + maximumDepth - 1, dimensionSelector); } else { // During recursion entropy of child node may change. double childGain = child->Train(data, currentChildBegin, currentCol - currentChildBegin, datasetInfo, labels, numClasses, - weights, minimumLeafSize, minimumGainSplit, maximumDepth, + weights, minimumLeafSize, minimumGainSplit, maximumDepth - 1, dimensionSelector); bestGain += double(childCounts[i]) / double(count) * (-childGain); } @@ -796,7 +795,7 @@ double DecisionTreeTrain(data, currentChildBegin, currentCol - currentChildBegin, labels, numClasses, weights, - currentCol - currentChildBegin, minimumGainSplit, maximumDepth, + currentCol - currentChildBegin, minimumGainSplit, maximumDepth - 1, dimensionSelector); } else @@ -887,7 +886,8 @@ double DecisionTreeTrain(data, currentChildBegin, currentCol - currentChildBegin, labels, numClasses, weights, - minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector); + minimumLeafSize, minimumGainSplit, maximumDepth - 1, + dimensionSelector); bestGain += double(childCounts[i]) / double(count) * (-childGain); } children.push_back(child); diff --git a/src/mlpack/methods/decision_tree/decision_tree_main.cpp b/src/mlpack/methods/decision_tree/decision_tree_main.cpp index 49c0826b38..f3e88e906a 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_main.cpp +++ b/src/mlpack/methods/decision_tree/decision_tree_main.cpp @@ -103,7 +103,7 @@ PARAM_INT_IN("minimum_leaf_size", "Minimum number of points in a leaf.", "n", PARAM_DOUBLE_IN("minimum_gain_split", "Minimum gain for node splitting.", "g", 1e-7); PARAM_INT_IN("maximum_depth", "Maximum Depth of the tree.", "D", - 20); + 0); // This is deprecated and should be removed in mlpack 4.0.0. PARAM_FLAG("print_training_error", "Print the training error (deprecated; will " "be removed in mlpack 4.0.0).", "e"); @@ -161,8 +161,8 @@ static void mlpackMain() RequireParamValue("minimum_leaf_size", [](int x) { return x > 0; }, true, "leaf size must be positive"); - RequireParamValue("maximum_depth", [](int x) { return x > 0; }, true, - "depth must be positive"); + RequireParamValue("maximum_depth", [](int x) { return x >= 0; }, true, + "depth must not be negative"); RequireParamValue("minimum_gain_split", [](double x) { return (x > 0.0 && x < 1.0); }, true, diff --git a/src/mlpack/methods/random_forest/random_forest.hpp b/src/mlpack/methods/random_forest/random_forest.hpp index 27167d1964..c1de286a87 100644 --- a/src/mlpack/methods/random_forest/random_forest.hpp +++ b/src/mlpack/methods/random_forest/random_forest.hpp @@ -60,7 +60,7 @@ class RandomForest const size_t numTrees = 20, const size_t minimumLeafSize = 1, const double minimumGainSplit = 1e-7, - const size_t maximumDepth = 20, + const size_t maximumDepth = 0, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -90,7 +90,7 @@ class RandomForest const size_t numTrees = 20, const size_t minimumLeafSize = 1, const double minimumGainSplit = 1e-7, - const size_t maximumDepth = 20, + const size_t maximumDepth = 0, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -117,7 +117,7 @@ class RandomForest const size_t numTrees = 20, const size_t minimumLeafSize = 1, const double minimumGainSplit = 1e-7, - const size_t maximumDepth = 20, + const size_t maximumDepth = 0, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -149,7 +149,7 @@ class RandomForest const size_t numTrees = 20, const size_t minimumLeafSize = 1, const double minimumGainSplit = 1e-7, - const size_t maximumDepth = 20, + const size_t maximumDepth = 0, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -177,7 +177,7 @@ class RandomForest const size_t numTrees = 20, const size_t minimumLeafSize = 1, const double minimumGainSplit = 1e-7, - const size_t maximumDepth = 20, + const size_t maximumDepth = 0, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -209,7 +209,7 @@ class RandomForest const size_t numTrees = 20, const size_t minimumLeafSize = 1, const double minimumGainSplit = 1e-7, - const size_t maximumDepth = 20, + const size_t maximumDepth = 0, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -239,7 +239,7 @@ class RandomForest const size_t numTrees = 20, const size_t minimumLeafSize = 1, const double minimumGainSplit = 1e-7, - const size_t maximumDepth = 20, + const size_t maximumDepth = 0, DimensionSelectionType dimensionSelector = DimensionSelectionType()); @@ -272,7 +272,7 @@ class RandomForest const size_t numTrees = 20, const size_t minimumLeafSize = 1, const double minimumGainSplit = 1e-7, - const size_t maximumDepth = 20, + const size_t maximumDepth = 0, DimensionSelectionType dimensionSelector = DimensionSelectionType()); diff --git a/src/mlpack/methods/random_forest/random_forest_main.cpp b/src/mlpack/methods/random_forest/random_forest_main.cpp index 492d6568c3..a83cafe5fd 100644 --- a/src/mlpack/methods/random_forest/random_forest_main.cpp +++ b/src/mlpack/methods/random_forest/random_forest_main.cpp @@ -107,8 +107,7 @@ PARAM_FLAG("print_training_accuracy", "If set, then the accuracy of the model " PARAM_INT_IN("num_trees", "Number of trees in the random forest.", "N", 10); PARAM_INT_IN("minimum_leaf_size", "Minimum number of points in each leaf " "node.", "n", 1); -PARAM_INT_IN("maximum_depth", "Maximum Depth of the tree.", "D", - 20); +PARAM_INT_IN("maximum_depth", "Maximum depth of the tree.", "D", 0); PARAM_MATRIX_OUT("probabilities", "Predicted class probabilities for each " "point in the test set.", "P"); PARAM_UROW_OUT("predictions", "Predicted classes for each point in the test " @@ -180,8 +179,8 @@ static void mlpackMain() RequireParamValue("minimum_leaf_size", [](int x) { return x > 0; }, true, "minimum leaf size must be greater than 0"); - RequireParamValue("maximum_depth", [](int x) { return x > 0; }, true, - "depth must be positive"); + RequireParamValue("maximum_depth", [](int x) { return x >= 0; }, true, + "depth must not be negative"); RequireParamValue("subspace_dim", [](int x) { return x >= 0; }, true, "subspace dimensionality must be nonnegative"); RequireParamValue("minimum_gain_split", diff --git a/src/mlpack/tests/decision_tree_test.cpp b/src/mlpack/tests/decision_tree_test.cpp index 098ad279f9..fecb1cadcb 100644 --- a/src/mlpack/tests/decision_tree_test.cpp +++ b/src/mlpack/tests/decision_tree_test.cpp @@ -288,11 +288,11 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitSimpleSplitTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 2, weights); const double gain = BestBinaryNumericSplit::SplitIfBetter( - bestGain, values, labels, 2, weights, 3, 1e-7, 20, classProbabilities, + bestGain, values, labels, 2, weights, 3, 1e-7, 0, classProbabilities, aux); const double weightedGain = BestBinaryNumericSplit::SplitIfBetter(bestGain, values, - labels, 2, weights, 3, 1e-7, 20, classProbabilities, aux); + labels, 2, weights, 3, 1e-7, 0, classProbabilities, aux); // Make sure that a split was made. BOOST_REQUIRE_GT(gain, bestGain); @@ -326,12 +326,12 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitMinSamplesTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 2, weights); const double gain = BestBinaryNumericSplit::SplitIfBetter( - bestGain, values, labels, 2, weights, 8, 1e-7, 20, classProbabilities, + bestGain, values, labels, 2, weights, 8, 1e-7, 0, classProbabilities, aux); // This should make no difference because it won't split at all. const double weightedGain = BestBinaryNumericSplit::SplitIfBetter(bestGain, values, - labels, 2, weights, 8, 1e-7, 20, classProbabilities, aux); + labels, 2, weights, 8, 1e-7, 0, classProbabilities, aux); // Make sure that no split was made. BOOST_REQUIRE_EQUAL(gain, DBL_MAX); @@ -362,7 +362,7 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitNoGainTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 2, weights); const double gain = BestBinaryNumericSplit::SplitIfBetter( - bestGain, values, labels, 2, weights, 10, 1e-7, 20, classProbabilities, + bestGain, values, labels, 2, weights, 10, 1e-7, 0, classProbabilities, aux); // Make sure there was no split. @@ -387,11 +387,11 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitSimpleSplitTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 4, labels, 3, weights, 3, 1e-7, 20, classProbabilities, + bestGain, values, 4, labels, 3, weights, 3, 1e-7, 0, classProbabilities, aux); const double weightedGain = AllCategoricalSplit::SplitIfBetter(bestGain, values, 4, - labels, 3, weights, 3, 1e-7, 20, classProbabilities, aux); + labels, 3, weights, 3, 1e-7, 0, classProbabilities, aux); // Make sure that a split was made. BOOST_REQUIRE_GT(gain, bestGain); @@ -423,7 +423,7 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitMinSamplesTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 4, labels, 3, weights, 4, 1e-7, 20, classProbabilities, + bestGain, values, 4, labels, 3, weights, 4, 1e-7, 0, classProbabilities, aux); // Make sure it's not split. @@ -456,11 +456,11 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitNoGainTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 10, labels, 3, weights, 10, 1e-7, 20, + bestGain, values, 10, labels, 3, weights, 10, 1e-7, 0, classProbabilities, aux); const double weightedGain = AllCategoricalSplit::SplitIfBetter(bestGain, values, 10, - labels, 3, weights, 10, 1e-7, 20, classProbabilities, aux); + labels, 3, weights, 10, 1e-7, 0, classProbabilities, aux); // Make sure that there was no split. BOOST_REQUIRE_EQUAL(gain, DBL_MAX); @@ -1217,4 +1217,38 @@ BOOST_AUTO_TEST_CASE(DecisionTreeCategoricalTrainReturnEntropy) BOOST_REQUIRE_EQUAL(std::isfinite(entropy), true); } +/** + * Make sure different Maximum Depth gives different number of childern. + */ +BOOST_AUTO_TEST_CASE(DifferentMaximumDepthTest) +{ + arma::mat dataset(10, 100, arma::fill::randu); + arma::Row labels(100); + for (size_t i = 0; i < 50; ++i) + { + dataset(3, i) = 0.0; + labels[i] = 0; + } + for (size_t i = 50; i < 100; ++i) + { + dataset(3, i) = 1.0; + labels[i] = 1; + } + + DecisionTree<> d(dataset, labels, 2, 10, 1e-7, 1); + + DecisionTree<> d1(dataset, labels, 2, 10, 1e-7, 2); + + DecisionTree<> d2(dataset, labels, 2, 10, 1e-7, 0); + + // Now require that we have zero children. + BOOST_REQUIRE_EQUAL(d.NumChildren(), 0); + + // Now require that we have zero children. + BOOST_REQUIRE_GT(d1.NumChildren(), 0); + + // Now require that we have zero children. + BOOST_REQUIRE_GT(d2.NumChildren(), 0); +} + BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/main_tests/decision_tree_test.cpp b/src/mlpack/tests/main_tests/decision_tree_test.cpp index 37d7c51e90..159f8cd753 100644 --- a/src/mlpack/tests/main_tests/decision_tree_test.cpp +++ b/src/mlpack/tests/main_tests/decision_tree_test.cpp @@ -41,6 +41,12 @@ struct DecisionTreeTestFixture } }; +void ResetDTSettings() +{ + CLI::ClearSettings(); + CLI::RestoreSettings(testName); +} + BOOST_FIXTURE_TEST_SUITE(DecisionTreeMainTest, DecisionTreeTestFixture); @@ -170,7 +176,7 @@ BOOST_AUTO_TEST_CASE(DecisionTreeMinimumLeafSizeTest) /** * Make sure maximum depth size is always a non-negative number. */ -BOOST_AUTO_TEST_CASE(DecisionTreeMaximumDepthTest) +BOOST_AUTO_TEST_CASE(DecisionTreeNonNegativeMaximumDepthTest) { arma::mat inputData; DatasetInfo info; @@ -447,4 +453,56 @@ BOOST_AUTO_TEST_CASE(DecisionModelCategoricalReuseTest) CheckMatrices(probabilities, CLI::GetParam("probabilities")); } +/** + * Check that different maximum depth gives + * different results. + */ +BOOST_AUTO_TEST_CASE(DecisionTreeMaximumDepthTest) +{ + arma::mat inputData; + DatasetInfo info; + if (!data::Load("vc2.csv", inputData, info)) + BOOST_FAIL("Cannot load train dataset vc2.csv!"); + + arma::Row labels; + if (!data::Load("vc2_labels.txt", labels)) + BOOST_FAIL("Cannot load labels for vc2_labels.txt"); + + // Initialize an all-ones weight matrix. + arma::mat weights(1, labels.n_cols, arma::fill::ones); + + arma::mat testData; + if (!data::Load("vc2_test.csv", testData, info)) + BOOST_FAIL("Cannot load test dataset vc2.csv!"); + + // Input training data. + SetInputParam("training", std::make_tuple(info, inputData)); + SetInputParam("labels", labels); + SetInputParam("weights", weights); + SetInputParam("maximum_depth", (int) 0); + + // Input test data. + SetInputParam("test", std::make_tuple(info, testData)); + + mlpackMain(); + + // Check that number of output points are equal to number of input points. + arma::Row predictions; + predictions = CLI::GetParam>("predictions"); + + ResetDTSettings(); + + // Input training data. + SetInputParam("training", std::make_tuple(info, inputData)); + SetInputParam("labels", std::move(labels)); + SetInputParam("weights", std::move(weights)); + SetInputParam("maximum_depth", (int) 4); + + // Input test data. + SetInputParam("test", std::make_tuple(info, testData)); + + CheckMatricesNotEqual(predictions, + CLI::GetParam>("predictions")); +} + BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/random_forest_test.cpp b/src/mlpack/tests/random_forest_test.cpp index e015410906..68458b9b0d 100644 --- a/src/mlpack/tests/random_forest_test.cpp +++ b/src/mlpack/tests/random_forest_test.cpp @@ -223,7 +223,7 @@ BOOST_AUTO_TEST_CASE(UnweightedCategoricalLearningTest) // Train a random forest and a decision tree. RandomForest<> rf(trainingData, di, trainingLabels, 5, 25 /* 25 trees */, 1, - 1e-7, 20, MultipleRandomDimensionSelect(4)); + 1e-7, 0, MultipleRandomDimensionSelect(4)); DecisionTree<> dt(trainingData, di, trainingLabels, 5, 5); // Get performance statistics on test data. @@ -281,7 +281,7 @@ BOOST_AUTO_TEST_CASE(WeightedCategoricalLearningTest) // Build a random forest and a decision tree. RandomForest<> rf(fullData, di, fullLabels, 5, weights, 25 /* 25 trees */, 1, - 1e-7, 20, MultipleRandomDimensionSelect(4)); + 1e-7, 0, MultipleRandomDimensionSelect(4)); DecisionTree<> dt(fullData, di, fullLabels, 5, weights, 5); // Get performance statistics on test data. @@ -445,14 +445,14 @@ BOOST_AUTO_TEST_CASE(RandomForestCategoricalTrainReturnEntropy) // Test random forest on unweighted categorical dataset. RandomForest<> rf; double entropy = rf.Train(fullData, di, fullLabels, 5, 15 /* 15 trees */, 1, - 1e-7, 20, MultipleRandomDimensionSelect(3)); + 1e-7, 0, MultipleRandomDimensionSelect(3)); BOOST_REQUIRE_EQUAL(std::isfinite(entropy), true); // Test random forest on weighted categorical dataset. RandomForest<> wrf; entropy = wrf.Train(fullData, di, fullLabels, 5, weights, 15 /* 15 trees */, - 1, 1e-7, 20, MultipleRandomDimensionSelect(3)); + 1, 1e-7, 0, MultipleRandomDimensionSelect(3)); BOOST_REQUIRE_EQUAL(std::isfinite(entropy), true); } From 64f31128578245f2434c390836fe352e4b394b41 Mon Sep 17 00:00:00 2001 From: robotcator Date: Sat, 8 Jun 2019 09:56:59 +0800 Subject: [PATCH 088/143] fix code style issue --- .../reinforcement_learning/replay/prioritized_replay.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index e4e551e449..6784745c05 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -61,7 +61,6 @@ class PrioritizedReplay PrioritizedReplay(const size_t batchSize, const size_t capacity, const double alpha, - int seed = 1024, const size_t dimension = StateType::dimension) : batchSize(batchSize), capacity(capacity), @@ -77,9 +76,9 @@ class PrioritizedReplay initialBeta(0.6), replayBetaIters(10000) { - arma::arma_rng::set_seed(seed); size_t size = 1; - while (size < capacity) { + while (size < capacity) + { size *= 2; } @@ -129,7 +128,8 @@ class PrioritizedReplay arma::ucolvec idxes(batchSize); double totalSum = idxSum.Sum(0, (full ? capacity : position)); double sumPerRange = totalSum / batchSize; - for (size_t bt = 0; bt < batchSize; bt++) { + for (size_t bt = 0; bt < batchSize; bt++) + { double mass = arma::randu() * sumPerRange + bt * sumPerRange; size_t idx = idxSum.FindPrefixSum(mass); idxes(bt) = idx; From d54223452151bfb00b1ffde404c4cadad5f6ec4c Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sun, 9 Jun 2019 19:04:04 -0400 Subject: [PATCH 089/143] Increase number of trials. The test does not seem to fail often, but I saw it fail once on Jenkins so let's bump it. I couldn't reproduce any failures locally after ~2000 trials. --- src/mlpack/tests/main_tests/random_forest_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/main_tests/random_forest_test.cpp b/src/mlpack/tests/main_tests/random_forest_test.cpp index a7f1339edb..37a9946476 100644 --- a/src/mlpack/tests/main_tests/random_forest_test.cpp +++ b/src/mlpack/tests/main_tests/random_forest_test.cpp @@ -229,7 +229,7 @@ BOOST_AUTO_TEST_CASE(RandomForestDiffMinLeafSizeTest) BOOST_FAIL("Cannot load labels for vc2_labels.txt"); bool success = false; - for (size_t trial = 0; trial < 3; ++trial) + for (size_t trial = 0; trial < 5; ++trial) { // Input training data. SetInputParam("training", inputData); From fff36379e50e3dfe68c5e61d79776e9422acb720 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sun, 9 Jun 2019 19:04:33 -0400 Subject: [PATCH 090/143] Don't move the data, so that it's available in trial 2. --- src/mlpack/tests/main_tests/random_forest_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/main_tests/random_forest_test.cpp b/src/mlpack/tests/main_tests/random_forest_test.cpp index 37a9946476..dc201611d5 100644 --- a/src/mlpack/tests/main_tests/random_forest_test.cpp +++ b/src/mlpack/tests/main_tests/random_forest_test.cpp @@ -354,8 +354,8 @@ BOOST_AUTO_TEST_CASE(RandomForestDiffNumTreeTest) // Train for num_trees 10. // Input training data. - SetInputParam("training", std::move(inputData)); - SetInputParam("labels", std::move(labels)); + SetInputParam("training", inputData); + SetInputParam("labels", labels); SetInputParam("num_trees", (int) 10); SetInputParam("minimum_leaf_size", (int) 1); From 24f8872a1e34cd5d9f226a836c42043833ebaf83 Mon Sep 17 00:00:00 2001 From: Yashwant Singh Parihar Date: Thu, 13 Jun 2019 20:15:50 +0530 Subject: [PATCH 091/143] Update softmax_regression_main.cpp --- .../methods/softmax_regression/softmax_regression_main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp b/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp index c83b0c3374..26a493374f 100644 --- a/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp +++ b/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp @@ -202,7 +202,7 @@ void TestClassifyAcc(size_t numClasses, const Model& model) // Save predictions, if desired. if (CLI::HasParam("predictions")) - CLI::GetParam>("predictions") = std::move(predictLabels); + CLI::GetParam>("predictions") = predictLabels; // Calculate accuracy, if desired. if (CLI::HasParam("test_labels")) From e08b0c09fe9ba6c570919e000f6777912527dbcd Mon Sep 17 00:00:00 2001 From: Yashwant Singh Parihar Date: Thu, 13 Jun 2019 21:15:36 +0530 Subject: [PATCH 092/143] Update softmax_regression_main.cpp --- .../methods/softmax_regression/softmax_regression_main.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp b/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp index 26a493374f..ed04aaa7dc 100644 --- a/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp +++ b/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp @@ -200,10 +200,6 @@ void TestClassifyAcc(size_t numClasses, const Model& model) arma::Row predictLabels; model.Classify(testData, predictLabels); - // Save predictions, if desired. - if (CLI::HasParam("predictions")) - CLI::GetParam>("predictions") = predictLabels; - // Calculate accuracy, if desired. if (CLI::HasParam("test_labels")) { @@ -242,6 +238,9 @@ void TestClassifyAcc(size_t numClasses, const Model& model) << (totalBingo) / static_cast(predictLabels.n_elem) << " (" << totalBingo << " of " << predictLabels.n_elem << ")." << endl; } + // Save predictions, if desired. + if (CLI::HasParam("predictions")) + CLI::GetParam>("predictions") = std::move(predictLabels); } template From b3a0e811f5232489b11e86eddbb7a404ce22f80e Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Fri, 14 Jun 2019 21:34:36 +0200 Subject: [PATCH 093/143] Add Code of Conduct. --- CODE_OF_CONDUCT.md | 96 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..435b150b99 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,96 @@ +# mlpack Code of Conduct + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and +expression, level of experience, education, socio-economic status, nationality, +personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies within all project spaces, and it also applies when +an individual is representing the project or its community in public spaces. +Examples of representing a project or community include using an official +project e-mail address, posting via an official social media account, or acting +as an appointed representative at an online or offline event. Representation of +a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at conduct@mlpack.org. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Reporting + +If you believe someone is violating the code of conduct we ask that you report +it by emailing conduct@mlpack.org. All reports will be kept confidential. In +some cases we may determine that a public statement will need to be made. If +that's the case, the identities of all victims and reporters will remain +confidential unless those individuals instruct us otherwise. + +If you are unsure whether the incident is a violation, or whether the space +where it happened is covered by this Code of Conduct, we encourage you to still +report it. We would much rather have a few extra reports where we decide to take +no action, rather than miss a report of an actual violation. We do not look +negatively on you if we find the incident is not a violation. And knowing about +incidents that are not violations, or happen outside our spaces, can also help +us to improve the Code of Conduct or the processes surrounding it. + +In your report please include: + +* Your contact info (so we can get in touch with you if we need to follow up) +* Names (real, nicknames, or pseudonyms) of any individuals involved. If there + were other witnesses besides you, please try to include them as well. +* When and where the incident occurred. Please be as specific as possible. +* Your account of what occurred. If there is a publicly available record + (e.g. a mailing list archive or a public IRC logger) please include a link. +* Any extra context you believe existed for the incident. +* If you believe this incident is ongoing. +* Any other information you believe we should have. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 1.4, available at +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html, and +includes some aspects of the Drupal Code of Conduct. From cf1b948ece5cd1ab7efe34b6bac6835c99fff742 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Sat, 15 Jun 2019 19:17:04 +0200 Subject: [PATCH 094/143] Minor style fixes (80 line limit, spaces between operations, comment style). --- .../q_learning_impl.hpp | 5 +++ .../replay/prioritized_replay.hpp | 41 +++++++++---------- .../replay/random_replay.hpp | 16 ++++---- .../reinforcement_learning/replay/sumtree.hpp | 40 +++++++++--------- src/mlpack/tests/q_learning_test.cpp | 5 ++- src/mlpack/tests/sumtree_test.cpp | 11 ++--- 6 files changed, 57 insertions(+), 61 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 0caf91cec0..228ea16807 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -48,6 +48,7 @@ QLearning< // Set up q-learning network. if (learningNetwork.Parameters().is_empty()) learningNetwork.ResetParameters(); + this->updater.Initialize(learningNetwork.Parameters().n_rows, learningNetwork.Parameters().n_cols); targetNetwork = learningNetwork; @@ -155,10 +156,14 @@ double QLearning< for (size_t i = 0; i < sampledNextStates.n_cols; ++i) { if (isTerminal[i]) + { target(sampledActions(i), i) = sampledRewards(i); + } else + { target(sampledActions(i), i) = sampledRewards(i) + config.Discount() * nextActionValues(bestActions(i), i); + } } // Learn form experience. diff --git a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp index 6784745c05..3086abcc42 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/prioritized_replay.hpp @@ -19,16 +19,17 @@ namespace mlpack { namespace rl { /** - * Implementation of prioritized experience replay. - * PER can replay important transitions more frequently by prioritizing + * Implementation of prioritized experience replay. Prioritized experience + * replay can replay important transitions more frequently by prioritizing * transitions, and make agent learn more efficiently. * * @code * @article{schaul2015prioritized, - * title={Prioritized experience replay}, - * author={Schaul, Tom and Quan, John and Antonoglou, Ioannis and Silver, David}, - * journal={arXiv preprint arXiv:1511.05952}, - * year={2015} + * title = {Prioritized experience replay}, + * author = {Schaul, Tom and Quan, John and Antonoglou, + * Ioannis and Silver, David}, + * journal = {arXiv preprint arXiv:1511.05952}, + * year = {2015} * } * @endcode * @@ -87,15 +88,14 @@ class PrioritizedReplay } /** - * Store the given experience. - * Set priorities for the given experience. + * Store the given experience and set the priorities for the given experience. * * @param state Given state. * @param action Given action. * @param reward Given reward. * @param nextState Given next state. * @param isEnd Whether next state is terminal state. - */ + */ void Store(const StateType& state, ActionType action, double reward, @@ -130,9 +130,8 @@ class PrioritizedReplay double sumPerRange = totalSum / batchSize; for (size_t bt = 0; bt < batchSize; bt++) { - double mass = arma::randu() * sumPerRange + bt * sumPerRange; - size_t idx = idxSum.FindPrefixSum(mass); - idxes(bt) = idx; + const double mass = arma::randu() * sumPerRange + bt * sumPerRange; + idxes(bt) = idxSum.FindPrefixSum(mass); } return idxes; } @@ -146,8 +145,6 @@ class PrioritizedReplay * @param sampledNextStates Sampled encoded next states. * @param isTerminal Indicate whether corresponding next state is terminal * state. - * @param sampledIndices Sampled indices. - * @param weights Corresponding weight for updating the loss. */ void Sample(arma::mat& sampledStates, arma::icolvec& sampledActions, @@ -201,7 +198,7 @@ class PrioritizedReplay } /** - * Annealing the beta. + * Annealing the beta. */ void BetaAnneal() { @@ -209,13 +206,13 @@ class PrioritizedReplay } /** - * Update the priorities of transitions and Update the gradients. - * - * @param target The learned value - * @param sampledActions Agent's sampled action - * @param nextActionValues Agent's next action - * @param gradients The model's gradients - */ + * Update the priorities of transitions and Update the gradients. + * + * @param target The learned value. + * @param sampledActions Agent's sampled action. + * @param nextActionValues Agent's next action. + * @param gradients The model's gradients. + */ void Update(arma::mat target, arma::icolvec sampledActions, arma::mat nextActionValues, diff --git a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp index 845cfd70ee..6019b805b6 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp @@ -143,19 +143,19 @@ class RandomReplay } /** - * Update the priorities of transitions and Update the gradients. - * - * @param target The learned value - * @param sampledActions Agent's sampled action - * @param nextActionValues Agent's next action - * @param gradients The model's gradients - */ + * Update the priorities of transitions and Update the gradients. + * + * @param target The learned value + * @param sampledActions Agent's sampled action + * @param nextActionValues Agent's next action + * @param gradients The model's gradients + */ void Update(arma::mat /* target */, arma::icolvec /* sampledActions */, arma::mat /* nextActionValues */, arma::mat& /* gradients */) { - /* do nothing for random replay*/ + /* Do nothing for random replay. */ } private: diff --git a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp index 92ecbccaea..8c577308a7 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/sumtree.hpp @@ -2,10 +2,8 @@ * @file sumtree.hpp * @author Xiaohong * - * This file is an implementation of sumtree. - * - * reference: - * [1] https://github.com/openai/baselines/blob/master/baselines/common/segment_tree.py + * This file is an implementation of sumtree. Based on: + * https://github.com/openai/baselines/blob/master/baselines/common/segment_tree.py * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -37,8 +35,7 @@ class SumTree /** * Default constructor. */ - SumTree(): - capacity(0) + SumTree() : capacity(0) { /* Nothing to do here. */ } /** @@ -46,10 +43,9 @@ class SumTree * * @param capacity Size of data. */ - SumTree(size_t capacity): - capacity(capacity) + SumTree(const size_t capacity) : capacity(capacity) { - element = std::vector (2 * capacity); + element = std::vector(2 * capacity); } /** @@ -58,7 +54,7 @@ class SumTree * @param idx The array idx to be changed. * @param value The data that array with idx to be. */ - void Set(size_t idx, T value) + void Set(size_t idx, const T value) { idx += capacity; element[idx] = value; @@ -70,21 +66,20 @@ class SumTree } } - /** * Update the data with batch rather loop over the indices with set method. * * @param indices The indices of data to be changed. - * @param data The data that array with indices to be. + * @param data The data that array with indices to be. */ - void BatchUpdate(arma::ucolvec indices, arma::Col data) + void BatchUpdate(const arma::ucolvec& indices, const arma::Col& data) { for (size_t i = 0; i < indices.n_rows; i++) { element[indices[i] + capacity] = data[i]; } // update the total tree with bottom-up technique. - for (size_t i = capacity-1; i > 0; i--) + for (size_t i = capacity - 1; i > 0; i--) { element[i] = element[2 * i] + element[2 * i + 1]; } @@ -106,12 +101,15 @@ class SumTree * * @param start The starting position of subsequence. * @param end The end position of subsequence. - * @param node Reference position + * @param node Reference position. * @param nodeStart Starting position of reference segment. * @param nodeEnd End position of reference segment. */ - T SumHelper(size_t start, size_t end, size_t node, - size_t nodeStart, size_t nodeEnd) + T SumHelper(const size_t start, + const size_t end, + const size_t node, + const size_t nodeStart, + const size_t nodeEnd) { if (start == nodeStart && end == nodeEnd) { @@ -131,7 +129,7 @@ class SumTree else { return SumHelper(start, mid, 2 * node, nodeStart, mid) + - SumHelper(mid+1, end, 2 * node + 1, mid + 1 , nodeEnd); + SumHelper(mid + 1, end, 2 * node + 1, mid + 1 , nodeEnd); } } } @@ -142,10 +140,10 @@ class SumTree * @param start The starting position of subsequence. * @param end The end position of subsequence. */ - T Sum(size_t start, size_t end) + T Sum(const size_t start, size_t end) { end -= 1; - return SumHelper(start, end, 1, 0, capacity-1); + return SumHelper(start, end, 1, 0, capacity - 1); } /** @@ -161,7 +159,7 @@ class SumTree * sum(arr[0] + arr[1] + ... + arr[idx]) <= mass. * * @param mass The upper bound of segment array sum. - * */ + */ size_t FindPrefixSum(T mass) { size_t idx = 1; diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index 886fe47e4c..0c34a18cfe 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -153,7 +153,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDQNPrioritizedReplay) * For the speed of the test case, I didn't set high criterion. */ Log::Debug << "Average return: " << averageReturn.mean() - << " Episode return: " << episodeReturn << std::endl; + << " Episode return: " << episodeReturn << std::endl; if (averageReturn.mean() > 35) { agent.Deterministic() = true; @@ -162,10 +162,11 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDQNPrioritizedReplay) testReturn(agent.Episode()); Log::Debug << "Average return in deterministic test: " - << testReturn.mean() << std::endl; + << testReturn.mean() << std::endl; break; } } + BOOST_REQUIRE(converged); } diff --git a/src/mlpack/tests/sumtree_test.cpp b/src/mlpack/tests/sumtree_test.cpp index 7cb1c82557..0cca1aac47 100644 --- a/src/mlpack/tests/sumtree_test.cpp +++ b/src/mlpack/tests/sumtree_test.cpp @@ -2,15 +2,13 @@ * @file sumtree_test.hpp * @author Xiaohong * - * Test for Sumtree implementation + * Test for Sumtree implementation. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ - - #include #include @@ -43,7 +41,6 @@ BOOST_AUTO_TEST_CASE(SetElement) /** * Test that we get the element. */ - BOOST_AUTO_TEST_CASE(GetElement) { SumTree sumtree(4); @@ -60,9 +57,8 @@ BOOST_AUTO_TEST_CASE(GetElement) /** * Test that we find the highest index in the array such that - * Sum(arr[0] + arr[1] + arr[2] ... + arr[i]) <= mass . + * Sum(arr[0] + arr[1] + arr[2] ... + arr[i]) <= mass. */ - BOOST_AUTO_TEST_CASE(FindPrefixSum) { SumTree sumtree(4); @@ -79,9 +75,8 @@ BOOST_AUTO_TEST_CASE(FindPrefixSum) /** * Test that we find the highest index in the array such that - * sum(arr[0] + arr[1] + arr[2] ... + arr[i]) <= mass . + * sum(arr[0] + arr[1] + arr[2] ... + arr[i]) <= mass. */ - BOOST_AUTO_TEST_CASE(BatchUpdate) { SumTree sumtree(4); From 7824ae057c0b00b86fcc35a9c23335723b5eb81a Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sun, 16 Jun 2019 11:03:38 -0400 Subject: [PATCH 095/143] Update history. --- HISTORY.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index bd9a721e15..df77461cd3 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -2,6 +2,9 @@ ###### ????-??-?? * Add Multiple Pole Balancing Environment (#1901). + * Fix prediction output of softmax regression when test set accuracy is + calculated (#1922). + ### mlpack 3.1.1 ###### 2019-05-26 * Fix random forest bug for numerical-only data (#1887). From 1a88085a659f489b1fe9c8952f7eec2b33b3fde2 Mon Sep 17 00:00:00 2001 From: Yashwant Date: Mon, 17 Jun 2019 08:12:55 +0530 Subject: [PATCH 096/143] Imporvement in MaxDepth --- .../best_binary_numeric_split.hpp | 2 - .../best_binary_numeric_split_impl.hpp | 5 - .../decision_tree/decision_tree_impl.hpp | 420 ++++++++++-------- .../decision_tree/decision_tree_main.cpp | 4 +- .../random_forest/random_forest_main.cpp | 3 +- src/mlpack/tests/decision_tree_test.cpp | 55 ++- .../tests/main_tests/random_forest_test.cpp | 2 +- 7 files changed, 254 insertions(+), 237 deletions(-) diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp b/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp index 74462a75da..c187e5f08d 100644 --- a/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp +++ b/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp @@ -45,7 +45,6 @@ class BestBinaryNumericSplit * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in a leaf node for * splitting. - * @param maximumDepth Maximum Depth Minimum for the tree. * @param classProbabilities Class probabilities vector, which may be filled * with split information a successful split. * @param aux Auxiliary split information, which may be modified on a @@ -60,7 +59,6 @@ class BestBinaryNumericSplit const WeightVecType& weights, const size_t minimumLeafSize, const double minimumGainSplit, - const size_t maximumDepth, arma::Col& classProbabilities, AuxiliarySplitInfo& aux); diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp index 4cefdbd60d..34cb692dc4 100644 --- a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp @@ -25,14 +25,9 @@ double BestBinaryNumericSplit::SplitIfBetter( const WeightVecType& weights, const size_t minimumLeafSize, const double minimumGainSplit, - const size_t maximumDepth, arma::Col& classProbabilities, AuxiliarySplitInfo& /* aux */) { - // First sanity check: if we have reached maximum depth, we can't split. - if (maximumDepth == 1) - return DBL_MAX; - // First sanity check: if we don't have enough points, we can't split. if (data.n_elem < (minimumLeafSize * 2)) return DBL_MAX; diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index 76b39d6833..593a6f1157 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -611,130 +611,143 @@ double DecisionTree(bestGain, - data.cols(begin, begin + count - 1).row(i), - datasetInfo.NumMappings(i), - labels.subvec(begin, begin + count - 1), - numClasses, - UseWeights ? weights.subvec(begin, begin + count - 1) : weights, - minimumLeafSize, - minimumGainSplit, - maximumDepth, - classProbabilities, - *this); - } - else if (datasetInfo.Type(i) == data::Datatype::numeric) - { - dimGain = NumericSplit::template SplitIfBetter(bestGain, - data.cols(begin, begin + count - 1).row(i), - labels.subvec(begin, begin + count - 1), - numClasses, - UseWeights ? weights.subvec(begin, begin + count - 1) : weights, - minimumLeafSize, - minimumGainSplit, - maximumDepth, - classProbabilities, - *this); - } - - // If the splitter reported that it did not split, move to the next - // dimension. - if (dimGain == DBL_MAX) - continue; - - // Was there an improvement? If so mark that it's the new best dimension. - bestDim = i; - bestGain = dimGain; - - // If the gain is the best possible, no need to keep looking. - if (bestGain >= 0.0) - break; - } - - // Did we split or not? If so, then split the data and create the children. - if (bestDim != datasetInfo.Dimensionality()) - { - dimensionTypeOrMajorityClass = (size_t) datasetInfo.Type(bestDim); - splitDimension = bestDim; - - // Get the number of children we will have. - size_t numChildren = 0; - if (datasetInfo.Type(bestDim) == data::Datatype::categorical) - numChildren = CategoricalSplit::NumChildren(classProbabilities, *this); - else - numChildren = NumericSplit::NumChildren(classProbabilities, *this); - - // Calculate all child assignments. - arma::Row childAssignments(count); - if (datasetInfo.Type(bestDim) == data::Datatype::categorical) - { - for (size_t j = begin; j < begin + count; ++j) - childAssignments[j - begin] = CategoricalSplit::CalculateDirection( - data(bestDim, j), classProbabilities, *this); - } - else - { - for (size_t j = begin; j < begin + count; ++j) + double dimGain = -DBL_MAX; + if (datasetInfo.Type(i) == data::Datatype::categorical) { - childAssignments[j - begin] = NumericSplit::CalculateDirection( - data(bestDim, j), classProbabilities, *this); + dimGain = CategoricalSplit::template SplitIfBetter(bestGain, + data.cols(begin, begin + count - 1).row(i), + datasetInfo.NumMappings(i), + labels.subvec(begin, begin + count - 1), + numClasses, + UseWeights ? weights.subvec(begin, begin + count - 1) : weights, + minimumLeafSize, + minimumGainSplit, + classProbabilities, + *this); } - } - - // Figure out counts of children. - arma::Row childCounts(numChildren, arma::fill::zeros); - for (size_t i = begin; i < begin + count; ++i) - childCounts[childAssignments[i - begin]]++; - - // Initialize bestGain if recursive split is allowed. - if (!NoRecursion) - { - bestGain = 0.0; - } - - // Split into children. - size_t currentCol = begin; - for (size_t i = 0; i < numChildren; ++i) - { - size_t currentChildBegin = currentCol; - for (size_t j = currentChildBegin; j < begin + count; ++j) + else if (datasetInfo.Type(i) == data::Datatype::numeric) { - if (childAssignments[j - begin] == i) - { - childAssignments.swap_cols(currentCol - begin, j - begin); - data.swap_cols(currentCol, j); - labels.swap_cols(currentCol, j); - if (UseWeights) - weights.swap_cols(currentCol, j); - ++currentCol; - } + dimGain = NumericSplit::template SplitIfBetter(bestGain, + data.cols(begin, begin + count - 1).row(i), + labels.subvec(begin, begin + count - 1), + numClasses, + UseWeights ? weights.subvec(begin, begin + count - 1) : weights, + minimumLeafSize, + minimumGainSplit, + classProbabilities, + *this); } - // Now build the child recursively. - DecisionTree* child = new DecisionTree(); - if (NoRecursion || maximumDepth == 1) + // If the splitter reported that it did not split, move to the next + // dimension. + if (dimGain == DBL_MAX) + continue; + + // Was there an improvement? If so mark that it's the new best dimension. + bestDim = i; + bestGain = dimGain; + + // If the gain is the best possible, no need to keep looking. + if (bestGain >= 0.0) + break; + } + + // Did we split or not? If so, then split the data and create the children. + if (bestDim != datasetInfo.Dimensionality()) + { + dimensionTypeOrMajorityClass = (size_t) datasetInfo.Type(bestDim); + splitDimension = bestDim; + + // Get the number of children we will have. + size_t numChildren = 0; + if (datasetInfo.Type(bestDim) == data::Datatype::categorical) + numChildren = CategoricalSplit::NumChildren(classProbabilities, *this); + else + numChildren = NumericSplit::NumChildren(classProbabilities, *this); + + // Calculate all child assignments. + arma::Row childAssignments(count); + if (datasetInfo.Type(bestDim) == data::Datatype::categorical) { - child->Train(data, currentChildBegin, - currentCol - currentChildBegin, datasetInfo, labels, numClasses, - weights, currentCol - currentChildBegin, minimumGainSplit, - maximumDepth - 1, dimensionSelector); + for (size_t j = begin; j < begin + count; ++j) + childAssignments[j - begin] = CategoricalSplit::CalculateDirection( + data(bestDim, j), classProbabilities, *this); } else { - // During recursion entropy of child node may change. - double childGain = child->Train(data, currentChildBegin, - currentCol - currentChildBegin, datasetInfo, labels, numClasses, - weights, minimumLeafSize, minimumGainSplit, maximumDepth - 1, - dimensionSelector); - bestGain += double(childCounts[i]) / double(count) * (-childGain); + for (size_t j = begin; j < begin + count; ++j) + { + childAssignments[j - begin] = NumericSplit::CalculateDirection( + data(bestDim, j), classProbabilities, *this); + } } - children.push_back(child); + + // Figure out counts of children. + arma::Row childCounts(numChildren, arma::fill::zeros); + for (size_t i = begin; i < begin + count; ++i) + childCounts[childAssignments[i - begin]]++; + + // Initialize bestGain if recursive split is allowed. + if (!NoRecursion) + { + bestGain = 0.0; + } + + // Split into children. + size_t currentCol = begin; + for (size_t i = 0; i < numChildren; ++i) + { + size_t currentChildBegin = currentCol; + for (size_t j = currentChildBegin; j < begin + count; ++j) + { + if (childAssignments[j - begin] == i) + { + childAssignments.swap_cols(currentCol - begin, j - begin); + data.swap_cols(currentCol, j); + labels.swap_cols(currentCol, j); + if (UseWeights) + weights.swap_cols(currentCol, j); + ++currentCol; + } + } + + // Now build the child recursively. + DecisionTree* child = new DecisionTree(); + if (NoRecursion) + { + child->Train(data, currentChildBegin, + currentCol - currentChildBegin, datasetInfo, labels, numClasses, + weights, currentCol - currentChildBegin, minimumGainSplit, + maximumDepth - 1, dimensionSelector); + } + else + { + // During recursion entropy of child node may change. + double childGain = child->Train(data, currentChildBegin, + currentCol - currentChildBegin, datasetInfo, labels, numClasses, + weights, minimumLeafSize, minimumGainSplit, maximumDepth - 1, + dimensionSelector); + bestGain += double(childCounts[i]) / double(count) * (-childGain); + } + children.push_back(child); + } + } + else + { + // Clear auxiliary info objects. + NumericAuxiliarySplitInfo::operator=(NumericAuxiliarySplitInfo()); + CategoricalAuxiliarySplitInfo::operator=(CategoricalAuxiliarySplitInfo()); + + // Calculate class probabilities because we are a leaf. + CalculateClassProbabilities( + labels.subvec(begin, begin + count - 1), + numClasses, + UseWeights ? weights.subvec(begin, begin + count - 1) : weights); } } else @@ -749,6 +762,7 @@ double DecisionTree::template - SplitIfBetter(bestGain, - data.cols(begin, begin + count - 1).row(i), - labels.cols(begin, begin + count - 1), - numClasses, - UseWeights ? - weights.cols(begin, begin + count - 1) : - weights, - minimumLeafSize, - minimumGainSplit, - maximumDepth, - classProbabilities, - *this); - - // If the splitter did not report that it improved, then move to the next - // dimension. - if (dimGain == DBL_MAX) - continue; - - bestDim = i; - bestGain = dimGain; - - // If the gain is the best possible, no need to keep looking. - if (bestGain >= 0.0) - break; - } - - // Did we split or not? If so, then split the data and create the children. - if (bestDim != data.n_rows) - { - // We know that the split is numeric. - size_t numChildren = NumericSplit::NumChildren(classProbabilities, *this); - splitDimension = bestDim; - dimensionTypeOrMajorityClass = (size_t) data::Datatype::numeric; - - // Calculate all child assignments. - arma::Row childAssignments(count); - - for (size_t j = begin; j < begin + count; ++j) + for (size_t i = dimensionSelector.Begin(); i != dimensionSelector.End(); + i = dimensionSelector.Next()) { - childAssignments[j - begin] = NumericSplit::CalculateDirection( - data(bestDim, j), classProbabilities, *this); + const double dimGain = NumericSplitType::template + SplitIfBetter(bestGain, + data.cols(begin, begin + count - 1).row(i), + labels.cols(begin, begin + count - 1), + numClasses, + UseWeights ? + weights.cols(begin, begin + count - 1) : + weights, + minimumLeafSize, + minimumGainSplit, + classProbabilities, + *this); + + // If the splitter did not report that it improved, then move to the next + // dimension. + if (dimGain == DBL_MAX) + continue; + + bestDim = i; + bestGain = dimGain; + + // If the gain is the best possible, no need to keep looking. + if (bestGain >= 0.0) + break; } - // Calculate counts of children in each node. - arma::Row childCounts(numChildren); - childCounts.zeros(); - for (size_t j = begin; j < begin + count; ++j) - childCounts[childAssignments[j - begin]]++; - - // Initialize bestGain if recursive split is allowed. - if (!NoRecursion) + // Did we split or not? If so, then split the data and create the children. + if (bestDim != data.n_rows) { - bestGain = 0.0; - } + // We know that the split is numeric. + size_t numChildren = NumericSplit::NumChildren(classProbabilities, *this); + splitDimension = bestDim; + dimensionTypeOrMajorityClass = (size_t) data::Datatype::numeric; - size_t currentCol = begin; - for (size_t i = 0; i < numChildren; ++i) - { - size_t currentChildBegin = currentCol; - for (size_t j = currentChildBegin; j < begin + count; ++j) + // Calculate all child assignments. + arma::Row childAssignments(count); + + for (size_t j = begin; j < begin + count; ++j) { - if (childAssignments[j - begin] == i) + childAssignments[j - begin] = NumericSplit::CalculateDirection( + data(bestDim, j), classProbabilities, *this); + } + + // Calculate counts of children in each node. + arma::Row childCounts(numChildren); + childCounts.zeros(); + for (size_t j = begin; j < begin + count; ++j) + childCounts[childAssignments[j - begin]]++; + + // Initialize bestGain if recursive split is allowed. + if (!NoRecursion) + { + bestGain = 0.0; + } + + size_t currentCol = begin; + for (size_t i = 0; i < numChildren; ++i) + { + size_t currentChildBegin = currentCol; + for (size_t j = currentChildBegin; j < begin + count; ++j) { - childAssignments.swap_cols(currentCol - begin, j - begin); - data.swap_cols(currentCol, j); - labels.swap_cols(currentCol, j); - if (UseWeights) - weights.swap_cols(currentCol, j); - ++currentCol; + if (childAssignments[j - begin] == i) + { + childAssignments.swap_cols(currentCol - begin, j - begin); + data.swap_cols(currentCol, j); + labels.swap_cols(currentCol, j); + if (UseWeights) + weights.swap_cols(currentCol, j); + ++currentCol; + } } - } - // Now build the child recursively. - DecisionTree* child = new DecisionTree(); - if (NoRecursion || maximumDepth == 1) - { - child->Train(data, currentChildBegin, - currentCol - currentChildBegin, labels, numClasses, weights, - currentCol - currentChildBegin, minimumGainSplit, maximumDepth - 1, - dimensionSelector); + // Now build the child recursively. + DecisionTree* child = new DecisionTree(); + if (NoRecursion) + { + child->Train(data, currentChildBegin, + currentCol - currentChildBegin, labels, numClasses, weights, + currentCol - currentChildBegin, minimumGainSplit, maximumDepth - 1, + dimensionSelector); + } + else + { + // During recursion entropy of child node may change. + double childGain = child->Train(data, currentChildBegin, + currentCol - currentChildBegin, labels, numClasses, weights, + minimumLeafSize, minimumGainSplit, maximumDepth - 1, + dimensionSelector); + bestGain += double(childCounts[i]) / double(count) * (-childGain); + } + children.push_back(child); } - else - { - // During recursion entropy of child node may change. - double childGain = child->Train(data, currentChildBegin, - currentCol - currentChildBegin, labels, numClasses, weights, - minimumLeafSize, minimumGainSplit, maximumDepth - 1, - dimensionSelector); - bestGain += double(childCounts[i]) / double(count) * (-childGain); - } - children.push_back(child); + } + else + { + // We won't be needing these members, so reset them. + NumericAuxiliarySplitInfo::operator=(NumericAuxiliarySplitInfo()); + + // Calculate class probabilities because we are a leaf. + CalculateClassProbabilities( + labels.subvec(begin, begin + count - 1), + numClasses, + UseWeights ? weights.subvec(begin, begin + count - 1) : weights); } } else @@ -904,6 +931,7 @@ double DecisionTree(labels, 2, weights); const double gain = BestBinaryNumericSplit::SplitIfBetter( - bestGain, values, labels, 2, weights, 3, 1e-7, 0, classProbabilities, + bestGain, values, labels, 2, weights, 3, 1e-7, classProbabilities, aux); const double weightedGain = BestBinaryNumericSplit::SplitIfBetter(bestGain, values, - labels, 2, weights, 3, 1e-7, 0, classProbabilities, aux); + labels, 2, weights, 3, 1e-7, classProbabilities, aux); // Make sure that a split was made. BOOST_REQUIRE_GT(gain, bestGain); @@ -326,12 +326,12 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitMinSamplesTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 2, weights); const double gain = BestBinaryNumericSplit::SplitIfBetter( - bestGain, values, labels, 2, weights, 8, 1e-7, 0, classProbabilities, + bestGain, values, labels, 2, weights, 8, 1e-7, classProbabilities, aux); // This should make no difference because it won't split at all. const double weightedGain = BestBinaryNumericSplit::SplitIfBetter(bestGain, values, - labels, 2, weights, 8, 1e-7, 0, classProbabilities, aux); + labels, 2, weights, 8, 1e-7, classProbabilities, aux); // Make sure that no split was made. BOOST_REQUIRE_EQUAL(gain, DBL_MAX); @@ -362,7 +362,7 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitNoGainTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 2, weights); const double gain = BestBinaryNumericSplit::SplitIfBetter( - bestGain, values, labels, 2, weights, 10, 1e-7, 0, classProbabilities, + bestGain, values, labels, 2, weights, 10, 1e-7, classProbabilities, aux); // Make sure there was no split. @@ -387,11 +387,11 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitSimpleSplitTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 4, labels, 3, weights, 3, 1e-7, 0, classProbabilities, + bestGain, values, 4, labels, 3, weights, 3, 1e-7, classProbabilities, aux); const double weightedGain = AllCategoricalSplit::SplitIfBetter(bestGain, values, 4, - labels, 3, weights, 3, 1e-7, 0, classProbabilities, aux); + labels, 3, weights, 3, 1e-7, classProbabilities, aux); // Make sure that a split was made. BOOST_REQUIRE_GT(gain, bestGain); @@ -423,7 +423,7 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitMinSamplesTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 4, labels, 3, weights, 4, 1e-7, 0, classProbabilities, + bestGain, values, 4, labels, 3, weights, 4, 1e-7, classProbabilities, aux); // Make sure it's not split. @@ -456,11 +456,11 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitNoGainTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 10, labels, 3, weights, 10, 1e-7, 0, + bestGain, values, 10, labels, 3, weights, 10, 1e-7, classProbabilities, aux); const double weightedGain = AllCategoricalSplit::SplitIfBetter(bestGain, values, 10, - labels, 3, weights, 10, 1e-7, 0, classProbabilities, aux); + labels, 3, weights, 10, 1e-7, classProbabilities, aux); // Make sure that there was no split. BOOST_REQUIRE_EQUAL(gain, DBL_MAX); @@ -585,7 +585,6 @@ BOOST_AUTO_TEST_CASE(PerfectTrainingSetWithWeight) weights.ones(); // Minimum leaf size of 1. - // Maximum Depth of 1 DecisionTree<> d(dataset, labels, 2, weights, 1, 0.0); // This part of code is dupliacte with no weighted one. @@ -1222,33 +1221,29 @@ BOOST_AUTO_TEST_CASE(DecisionTreeCategoricalTrainReturnEntropy) */ BOOST_AUTO_TEST_CASE(DifferentMaximumDepthTest) { - arma::mat dataset(10, 100, arma::fill::randu); - arma::Row labels(100); - for (size_t i = 0; i < 50; ++i) - { - dataset(3, i) = 0.0; - labels[i] = 0; - } - for (size_t i = 50; i < 100; ++i) - { - dataset(3, i) = 1.0; - labels[i] = 1; - } + arma::mat dataset; + arma::Row labels; + data::Load("vc2.csv", dataset); + data::Load("vc2_labels.txt", labels); - DecisionTree<> d(dataset, labels, 2, 10, 1e-7, 1); + DecisionTree<> d(dataset, labels, 3, 10, 1e-7, 1); - DecisionTree<> d1(dataset, labels, 2, 10, 1e-7, 2); + DecisionTree<> d1(dataset, labels, 3, 10, 1e-7, 2); - DecisionTree<> d2(dataset, labels, 2, 10, 1e-7, 0); + DecisionTree<> d2(dataset, labels, 3, 10, 1e-7); // Now require that we have zero children. BOOST_REQUIRE_EQUAL(d.NumChildren(), 0); - // Now require that we have zero children. - BOOST_REQUIRE_GT(d1.NumChildren(), 0); + // Now require that we have two children. + BOOST_REQUIRE_EQUAL(d1.NumChildren(), 2); + BOOST_REQUIRE_EQUAL(d1.Child(0).NumChildren(), 0); + BOOST_REQUIRE_EQUAL(d1.Child(1).NumChildren(), 0); - // Now require that we have zero children. - BOOST_REQUIRE_GT(d2.NumChildren(), 0); + // Now require that we have two children. + BOOST_REQUIRE_EQUAL(d2.NumChildren(), 2); + BOOST_REQUIRE_EQUAL(d2.Child(0).NumChildren(), 2); + BOOST_REQUIRE_EQUAL(d2.Child(1).NumChildren(), 2); } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/main_tests/random_forest_test.cpp b/src/mlpack/tests/main_tests/random_forest_test.cpp index 3c392b0168..266be0fb1d 100644 --- a/src/mlpack/tests/main_tests/random_forest_test.cpp +++ b/src/mlpack/tests/main_tests/random_forest_test.cpp @@ -199,7 +199,7 @@ BOOST_AUTO_TEST_CASE(RandomForestMaximumDepthTest) if (!data::Load("vc2_labels.txt", labels)) BOOST_FAIL("Cannot load labels for vc2_labels.txt"); - SetInputParam("maximum_depth", (int) 0); // Invalid. + SetInputParam("maximum_depth", (int) -1); // Invalid. Log::Fatal.ignoreInput = true; BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); From c5d219c3ad768f7acb6f91b2dfcbdf8b608a7b72 Mon Sep 17 00:00:00 2001 From: Yashwant Singh Parihar Date: Mon, 17 Jun 2019 08:24:15 +0530 Subject: [PATCH 097/143] Resolve Style Issues --- src/mlpack/methods/decision_tree/decision_tree_impl.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index 593a6f1157..f68392524e 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -611,7 +611,7 @@ double DecisionTreeTrain(data, currentChildBegin, currentCol - currentChildBegin, labels, numClasses, weights, - currentCol - currentChildBegin, minimumGainSplit, maximumDepth - 1, - dimensionSelector); + currentCol - currentChildBegin, minimumGainSplit, + maximumDepth - 1, dimensionSelector); } else { From 48b6c18db7e014a770d9c73b7c0ed1a85bbeb434 Mon Sep 17 00:00:00 2001 From: Yashwant Singh Parihar Date: Mon, 17 Jun 2019 08:50:16 +0530 Subject: [PATCH 098/143] Update all_categorical_split_impl.hpp --- .../methods/decision_tree/all_categorical_split_impl.hpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp index ec317b77c6..97337fc508 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp @@ -26,14 +26,9 @@ double AllCategoricalSplit::SplitIfBetter( const WeightVecType& weights, const size_t minimumLeafSize, const double minimumGainSplit, - const size_t maximumDepth, arma::Col& classProbabilities, AuxiliarySplitInfo& /* aux */) { - // First sanity check: if we have reached maximum depth, we can't split. - if (maximumDepth == 1) - return DBL_MAX; - // Count the number of elements in each potential child. const double epsilon = 1e-7; // Tolerance for floating-point errors. arma::Col counts(numCategories, arma::fill::zeros); From bc37acdb391d219dbed53269fb58a5657241deab Mon Sep 17 00:00:00 2001 From: Yashwant Singh Parihar Date: Mon, 17 Jun 2019 08:51:07 +0530 Subject: [PATCH 099/143] Resolve error --- src/mlpack/methods/decision_tree/all_categorical_split.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/mlpack/methods/decision_tree/all_categorical_split.hpp b/src/mlpack/methods/decision_tree/all_categorical_split.hpp index 87f796e1e1..2823fa9bb5 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split.hpp @@ -47,7 +47,6 @@ class AllCategoricalSplit * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in a leaf node for * splitting. - * @param maximumDepth Maximum Depth Minimum for the tree. * @param classProbabilities Class probabilities vector, which may be filled * with split information a successful split. * @param aux Auxiliary split information, which may be modified on a @@ -63,7 +62,6 @@ class AllCategoricalSplit const WeightVecType& weights, const size_t minimumLeafSize, const double minimumGainSplit, - const size_t maximumDepth, arma::Col& classProbabilities, AuxiliarySplitInfo& aux); From 9a3e3e77006e8339529db535488e0d70e09f654e Mon Sep 17 00:00:00 2001 From: Yashwant Date: Mon, 17 Jun 2019 09:10:35 +0530 Subject: [PATCH 100/143] Remove and upadte Tests as suggested --- .../tests/main_tests/gmm_probability_test.cpp | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/mlpack/tests/main_tests/gmm_probability_test.cpp b/src/mlpack/tests/main_tests/gmm_probability_test.cpp index 7e5853ab7f..0282763aa4 100644 --- a/src/mlpack/tests/main_tests/gmm_probability_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_probability_test.cpp @@ -49,21 +49,6 @@ void ResetGmmProbabilitySetting() BOOST_FIXTURE_TEST_SUITE(GmmProbabilityMainTest, GmmProbabilityTestFixture); -// Making sure input_file are provided. -BOOST_AUTO_TEST_CASE(GmmProbabilityInputTest) -{ - arma::mat inputData(5, 10, arma::fill::randu); - - GMM gmm(1, 5); - gmm.Train(inputData, 5); - - SetInputParam("input_model", &gmm); - - Log::Fatal.ignoreInput = true; - BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); - Log::Fatal.ignoreInput = false; -} - // Checking the input and output dimensionality. BOOST_AUTO_TEST_CASE(GmmProbabilityDimensionality) { @@ -72,7 +57,7 @@ BOOST_AUTO_TEST_CASE(GmmProbabilityDimensionality) GMM gmm(1, 5); gmm.Train(std::move(inputData), 5); - arma::mat inputPoints(1, 5, arma::fill::randu); + arma::mat inputPoints(5, 5, arma::fill::randu); SetInputParam("input", std::move(inputPoints)); SetInputParam("input_model", &gmm); From 5ad9bc10c3fed7c006d1be59a1c3421dbf25fa91 Mon Sep 17 00:00:00 2001 From: Yashwant Singh Parihar Date: Mon, 17 Jun 2019 11:56:22 +0530 Subject: [PATCH 101/143] Fix GMMTrainTest --- src/mlpack/tests/main_tests/gmm_train_test.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/mlpack/tests/main_tests/gmm_train_test.cpp b/src/mlpack/tests/main_tests/gmm_train_test.cpp index 197c7fc1a6..7dc6c1e131 100644 --- a/src/mlpack/tests/main_tests/gmm_train_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_train_test.cpp @@ -284,9 +284,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) // Ensure that Percentage affects the final result when refined_start is true. BOOST_AUTO_TEST_CASE(GmmTrainPercentageTest) { - arma::mat inputData; - if (!data::Load("data_3d_mixed.txt", inputData)) - BOOST_FAIL("Unable to load train dataset data_3d_mixed.txt!"); + arma::mat inputData(50, 100, arma::fill::randu); SetInputParam("input", inputData); SetInputParam("gaussians", (int) 2); @@ -308,7 +306,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainPercentageTest) SetInputParam("input", std::move(inputData)); SetInputParam("gaussians", (int) 2); SetInputParam("refined_start", true); - SetInputParam("percentage", (double) 0.20); + SetInputParam("percentage", (double) 0.35); SetInputParam("samplings", (int) 1000); mlpack::math::randGen.seed((uint32_t) seed); From ed89731921d6c62e027c776761c3aa3ad7ab0c6c Mon Sep 17 00:00:00 2001 From: Yashwant Singh Parihar Date: Mon, 17 Jun 2019 13:07:38 +0530 Subject: [PATCH 102/143] Update gmm_train_test.cpp --- src/mlpack/tests/main_tests/gmm_train_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/main_tests/gmm_train_test.cpp b/src/mlpack/tests/main_tests/gmm_train_test.cpp index 7dc6c1e131..ab71a7c78b 100644 --- a/src/mlpack/tests/main_tests/gmm_train_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_train_test.cpp @@ -306,7 +306,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainPercentageTest) SetInputParam("input", std::move(inputData)); SetInputParam("gaussians", (int) 2); SetInputParam("refined_start", true); - SetInputParam("percentage", (double) 0.35); + SetInputParam("percentage", (double) 0.20); SetInputParam("samplings", (int) 1000); mlpack::math::randGen.seed((uint32_t) seed); From 645cf1b325b32fec6083922055b697d0356b3401 Mon Sep 17 00:00:00 2001 From: Yashwant Singh Parihar Date: Mon, 17 Jun 2019 13:52:38 +0530 Subject: [PATCH 103/143] Fix --- src/mlpack/tests/main_tests/gmm_train_test.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/main_tests/gmm_train_test.cpp b/src/mlpack/tests/main_tests/gmm_train_test.cpp index ab71a7c78b..ffa3ebf544 100644 --- a/src/mlpack/tests/main_tests/gmm_train_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_train_test.cpp @@ -284,7 +284,9 @@ BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) // Ensure that Percentage affects the final result when refined_start is true. BOOST_AUTO_TEST_CASE(GmmTrainPercentageTest) { - arma::mat inputData(50, 100, arma::fill::randu); + arma::mat inputData; + if (!data::Load("data_3d_mixed.txt", inputData)) + BOOST_FAIL("Unable to load train dataset data_3d_mixed.txt!"); SetInputParam("input", inputData); SetInputParam("gaussians", (int) 2); @@ -306,7 +308,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainPercentageTest) SetInputParam("input", std::move(inputData)); SetInputParam("gaussians", (int) 2); SetInputParam("refined_start", true); - SetInputParam("percentage", (double) 0.20); + SetInputParam("percentage", (double) 0.35); SetInputParam("samplings", (int) 1000); mlpack::math::randGen.seed((uint32_t) seed); From 9995041ff78ec0860afd61af42fc9676e690dd45 Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Tue, 18 Jun 2019 00:10:33 +0700 Subject: [PATCH 104/143] Add linear layer in numerical gradient tests --- src/mlpack/methods/ann/layer/batch_norm.hpp | 3 +++ src/mlpack/methods/ann/layer/batch_norm_impl.hpp | 8 ++++---- src/mlpack/tests/ann_layer_test.cpp | 11 ++++++++++- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batch_norm.hpp b/src/mlpack/methods/ann/layer/batch_norm.hpp index fd5729090d..1f0849ab93 100644 --- a/src/mlpack/methods/ann/layer/batch_norm.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm.hpp @@ -195,6 +195,9 @@ class BatchNorm //! Locally-stored normalized input. OutputDataType normalized; + + //! Locally-stored 0 mean input. + OutputDataType inputMean; }; // class BatchNorm } // namespace ann diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index acf7fe2292..037170bee4 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -80,6 +80,7 @@ void BatchNorm::Forward( // Normalize the input. output = input.each_col() - mean; + inputMean = output; output.each_col() /= arma::sqrt(variance + eps); // Use Welford method to compute the sample variance and mean. @@ -87,9 +88,9 @@ void BatchNorm::Forward( { count += 1; - OutputDataType delta = input.col(i) - runningMean; - runningMean = runningMean + delta / count; - runningVariance += delta % (input.col(i) - runningMean); + OutputDataType diff = input.col(i) - runningMean; + runningMean = runningMean + diff / count; + runningVariance += diff % (input.col(i) - runningMean); } // Reused in the backward and gradient step. @@ -106,7 +107,6 @@ template void BatchNorm::Backward( const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) { - const arma::mat inputMean = input.each_col() - mean; const arma::mat stdInv = 1.0 / arma::sqrt(variance + eps); // Step 1: dl / dxhat diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 4356ac7440..06545380e1 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -95,6 +95,7 @@ BOOST_AUTO_TEST_CASE(GradientAddLayerTest) model->Predictors() = input; model->Responses() = target; model->Add >(); + model->Add >(10, 10); model->Add >(10); model->Add >(); } @@ -400,6 +401,7 @@ BOOST_AUTO_TEST_CASE(GradientLinearLayerTest) model->Predictors() = input; model->Responses() = target; model->Add >(); + model->Add >(10, 10); model->Add >(10, 2); model->Add >(); } @@ -483,6 +485,7 @@ BOOST_AUTO_TEST_CASE(GradientLinearNoBiasLayerTest) model->Predictors() = input; model->Responses() = target; model->Add >(); + model->Add >(10, 10); model->Add >(10, 2); model->Add >(); } @@ -585,6 +588,7 @@ BOOST_AUTO_TEST_CASE(GradientFlexibleReLULayerTest) model->Predictors() = input; model->Responses() = target; + model->Add >(2, 2); model->Add >(2, 5); model->Add >(0.05); model->Add >(); @@ -1307,6 +1311,7 @@ BOOST_AUTO_TEST_CASE(GradientConcatLayerTest) model->Predictors() = input; model->Responses() = target; model->Add >(); + model->Add >(10, 10); concat = new Concat<>(true); concat->Add >(10, 2); @@ -1578,6 +1583,7 @@ BOOST_AUTO_TEST_CASE(GradientBatchNormTest) model->Predictors() = input; model->Responses() = target; model->Add >(); + model->Add >(10, 10); model->Add >(10); model->Add >(10, 2); model->Add >(); @@ -1751,6 +1757,7 @@ BOOST_AUTO_TEST_CASE(GradientTransposedConvolutionLayerTest) model = new FFN, RandomInitialization>(); model->Predictors() = input; model->Responses() = target; + model->Add >(36, 36); model->Add >(1, 1, 3, 3, 2, 2, 1, 1, 6, 6); model->Add >(); } @@ -1868,6 +1875,7 @@ BOOST_AUTO_TEST_CASE(GradientAtrousConvolutionLayerTest) model = new FFN, RandomInitialization>(); model->Predictors() = input; model->Responses() = target; + model->Add >(36, 36); model->Add >(1, 1, 3, 3, 1, 1, 0, 0, 6, 6, 2, 2); model->Add >(); } @@ -1947,6 +1955,7 @@ BOOST_AUTO_TEST_CASE(GradientLayerNormTest) model->Predictors() = input; model->Responses() = target; model->Add >(); + model->Add >(10, 10); model->Add >(10); model->Add >(10, 2); model->Add >(); @@ -2355,7 +2364,7 @@ BOOST_AUTO_TEST_CASE(GradientSequentialLayerTest) model->Predictors() = input; model->Responses() = target; model->Add >(); - + model->Add >(10, 10); sequential = new Sequential<>(); sequential->Add >(10, 10); sequential->Add >(); From 33bce014a574a460ec8da2a75ccfcfecb2e54b84 Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Tue, 18 Jun 2019 02:50:41 +0700 Subject: [PATCH 105/143] Fix LayerNorm Implementation --- src/mlpack/methods/ann/layer/layer_norm.hpp | 3 +++ src/mlpack/methods/ann/layer/layer_norm_impl.hpp | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/layer_norm.hpp b/src/mlpack/methods/ann/layer/layer_norm.hpp index 41afac3be9..de182b0d0b 100644 --- a/src/mlpack/methods/ann/layer/layer_norm.hpp +++ b/src/mlpack/methods/ann/layer/layer_norm.hpp @@ -184,6 +184,9 @@ class LayerNorm //! Locally-stored normalized input. OutputDataType normalized; + + //! Locally-stored input with 0 mean. + OutputDataType inputMean; }; // class LayerNorm } // namespace ann diff --git a/src/mlpack/methods/ann/layer/layer_norm_impl.hpp b/src/mlpack/methods/ann/layer/layer_norm_impl.hpp index f0d94e1cf6..bf12916880 100644 --- a/src/mlpack/methods/ann/layer/layer_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/layer_norm_impl.hpp @@ -63,7 +63,7 @@ void LayerNorm::Forward( // Normalize the input. output = input.each_row() - mean; - + inputMean = output; output.each_row() /= arma::sqrt(variance + eps); // Reused in the backward and gradient step. @@ -79,7 +79,6 @@ template void LayerNorm::Backward( const arma::Mat&& input, arma::Mat&& gy, arma::Mat&& g) { - const arma::mat inputMean = input.each_row() - mean; const arma::mat stdInv = 1.0 / arma::sqrt(variance + eps); // dl / dxhat From 2fd5a7fd6812098e5104d478cd81b0cfdabcbd09 Mon Sep 17 00:00:00 2001 From: Yashwant Singh Parihar Date: Tue, 18 Jun 2019 16:30:05 +0530 Subject: [PATCH 106/143] Fix Some Tests --- src/mlpack/tests/main_tests/gmm_train_test.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/mlpack/tests/main_tests/gmm_train_test.cpp b/src/mlpack/tests/main_tests/gmm_train_test.cpp index ffa3ebf544..a8a02b32b9 100644 --- a/src/mlpack/tests/main_tests/gmm_train_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_train_test.cpp @@ -244,8 +244,9 @@ BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) arma::mat inputData(5, 100, arma::fill::randu); SetInputParam("input", inputData); - SetInputParam("gaussians", (int) 2); + SetInputParam("gaussians", (int) 3); SetInputParam("trials", (int) 1); + SetInputParam("max_iterations", (int) 500); size_t seed = std::time(NULL); mlpack::math::randGen.seed((uint32_t) seed); @@ -259,7 +260,8 @@ BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) ResetGmmTrainSetting(); SetInputParam("input", std::move(inputData)); - SetInputParam("gaussians", (int) 2); + SetInputParam("gaussians", (int) 3); + SetInputParam("max_iterations", (int) 500); SetInputParam("trials", (int) 500); mlpack::math::randGen.seed((uint32_t) seed); @@ -308,7 +310,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainPercentageTest) SetInputParam("input", std::move(inputData)); SetInputParam("gaussians", (int) 2); SetInputParam("refined_start", true); - SetInputParam("percentage", (double) 0.35); + SetInputParam("percentage", (double) 0.45); SetInputParam("samplings", (int) 1000); mlpack::math::randGen.seed((uint32_t) seed); @@ -338,9 +340,8 @@ BOOST_AUTO_TEST_CASE(GmmTrainSamplingsTest) BOOST_FAIL("Unable to load train dataset data_3d_mixed.txt!"); SetInputParam("input", inputData); - SetInputParam("gaussians", (int) 2); + SetInputParam("gaussians", (int) 3); SetInputParam("refined_start", true); - SetInputParam("percentage", (double) 0.2); SetInputParam("samplings", (int) 10); size_t seed = std::time(NULL); @@ -355,10 +356,9 @@ BOOST_AUTO_TEST_CASE(GmmTrainSamplingsTest) ResetGmmTrainSetting(); SetInputParam("input", std::move(inputData)); - SetInputParam("gaussians", (int) 2); + SetInputParam("gaussians", (int) 3); SetInputParam("refined_start", true); - SetInputParam("percentage", (double) 0.2); - SetInputParam("samplings", (int) 10000); + SetInputParam("samplings", (int) 25000); mlpack::math::randGen.seed((uint32_t) seed); srand((unsigned int) seed); @@ -485,4 +485,3 @@ BOOST_AUTO_TEST_CASE(GmmTrainDiagCovariance) } BOOST_AUTO_TEST_SUITE_END(); - From 4033487a43281f36bd1ff1c6e44107ce73915ec0 Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Tue, 18 Jun 2019 19:09:14 +0700 Subject: [PATCH 107/143] Style fix --- src/mlpack/methods/ann/layer/batch_norm_impl.hpp | 3 ++- src/mlpack/methods/ann/layer/layer_norm_impl.hpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index 037170bee4..d8ef01c259 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -25,7 +25,8 @@ BatchNorm::BatchNorm() : eps(1e-8), loading(false), deterministic(false), - count(0) + count(0), + size(0) { // Nothing to do here. } diff --git a/src/mlpack/methods/ann/layer/layer_norm_impl.hpp b/src/mlpack/methods/ann/layer/layer_norm_impl.hpp index bf12916880..772b63d40b 100644 --- a/src/mlpack/methods/ann/layer/layer_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/layer_norm_impl.hpp @@ -23,7 +23,8 @@ namespace ann { /** Artificial Neural Network. */ template LayerNorm::LayerNorm() : eps(1e-8), - loading(false) + loading(false), + size(0) { // Nothing to do here. } From d343af3cc86afea7c780bd7b9d1c5335a804d01c Mon Sep 17 00:00:00 2001 From: robotcator Date: Tue, 18 Jun 2019 23:03:39 +0800 Subject: [PATCH 108/143] change the pendulum action type to double --- .../methods/reinforcement_learning/environment/pendulum.hpp | 4 ++-- src/mlpack/tests/reward_clipping_test.cpp | 2 +- src/mlpack/tests/rl_components_test.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index 878b077c20..69e3749faf 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -82,7 +82,7 @@ class Pendulum */ struct Action { - double action[1]; + double action; // Storing degree of freedom const int size = 1; }; @@ -126,7 +126,7 @@ class Pendulum // Get action and clip the values between max and min limits. double torque = std::min( - std::max(action.action[0], -maxTorque), maxTorque); + std::max(action.action, -maxTorque), maxTorque); // Calculate costs of taking this action in the current state. double costs = std::pow(AngleNormalize(theta), 2) + 0.1 * diff --git a/src/mlpack/tests/reward_clipping_test.cpp b/src/mlpack/tests/reward_clipping_test.cpp index a153dea67a..e219b4461e 100644 --- a/src/mlpack/tests/reward_clipping_test.cpp +++ b/src/mlpack/tests/reward_clipping_test.cpp @@ -47,7 +47,7 @@ BOOST_AUTO_TEST_CASE(ClippedRewardTest) RewardClipping::State state = rewardClipping.InitialSample(); RewardClipping::Action action; - action.action[0] = mlpack::math::Random(-1.0, 1.0); + action.action = mlpack::math::Random(-1.0, 1.0); double reward = rewardClipping.Sample(state, action); BOOST_REQUIRE(reward <= 2.0); diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 6d3ba392f6..e3ff6383e4 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -41,7 +41,7 @@ BOOST_AUTO_TEST_CASE(SimplePendulumTest) Pendulum::State state = task.InitialSample(); Pendulum::Action action; - action.action[0] = math::Random(-2.0, 2.0); + action.action = math::Random(-2.0, 2.0); double reward = task.Sample(state, action); // The reward is always negative. Check if not lower than lowest possible. From b21370599be77ac7468a1b0c38c36c62e10d18c4 Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Wed, 19 Jun 2019 17:28:45 +0700 Subject: [PATCH 109/143] Style fix and doc fix --- src/mlpack/methods/ann/layer/batch_norm.hpp | 2 +- src/mlpack/methods/ann/layer/batch_norm_impl.hpp | 4 ++-- src/mlpack/methods/ann/layer/layer_norm.hpp | 2 +- src/mlpack/methods/ann/layer/layer_norm_impl.hpp | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batch_norm.hpp b/src/mlpack/methods/ann/layer/batch_norm.hpp index 1f0849ab93..9d0d1fc012 100644 --- a/src/mlpack/methods/ann/layer/batch_norm.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm.hpp @@ -196,7 +196,7 @@ class BatchNorm //! Locally-stored normalized input. OutputDataType normalized; - //! Locally-stored 0 mean input. + //! Locally-stored zero mean input. OutputDataType inputMean; }; // class BatchNorm diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index d8ef01c259..ec0ea928fb 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -22,11 +22,11 @@ namespace ann { /** Artificial Neural Network. */ template BatchNorm::BatchNorm() : + size(0), eps(1e-8), loading(false), deterministic(false), - count(0), - size(0) + count(0) { // Nothing to do here. } diff --git a/src/mlpack/methods/ann/layer/layer_norm.hpp b/src/mlpack/methods/ann/layer/layer_norm.hpp index de182b0d0b..7bb55e9327 100644 --- a/src/mlpack/methods/ann/layer/layer_norm.hpp +++ b/src/mlpack/methods/ann/layer/layer_norm.hpp @@ -185,7 +185,7 @@ class LayerNorm //! Locally-stored normalized input. OutputDataType normalized; - //! Locally-stored input with 0 mean. + //! Locally-stored zero mean input. OutputDataType inputMean; }; // class LayerNorm diff --git a/src/mlpack/methods/ann/layer/layer_norm_impl.hpp b/src/mlpack/methods/ann/layer/layer_norm_impl.hpp index 772b63d40b..5572478178 100644 --- a/src/mlpack/methods/ann/layer/layer_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/layer_norm_impl.hpp @@ -22,9 +22,9 @@ namespace ann { /** Artificial Neural Network. */ template LayerNorm::LayerNorm() : + size(0), eps(1e-8), - loading(false), - size(0) + loading(false) { // Nothing to do here. } From cedbbc69872a2e9e446fa9770a9631eadf34e756 Mon Sep 17 00:00:00 2001 From: robotcator Date: Wed, 19 Jun 2019 21:25:43 +0800 Subject: [PATCH 110/143] fix the static code warning --- .../methods/reinforcement_learning/environment/pendulum.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index 69e3749faf..bfd4cb1105 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -82,7 +82,7 @@ class Pendulum */ struct Action { - double action; + double action = 0.0; // Storing degree of freedom const int size = 1; }; From 7b0d066fb0c34a1d94e95fbf578ba05009fbcac7 Mon Sep 17 00:00:00 2001 From: Yashwant Date: Thu, 20 Jun 2019 15:35:03 +0530 Subject: [PATCH 111/143] Clean up as Suggested --- COPYRIGHT.txt | 1 + HISTORY.md | 2 + .../decision_tree/all_categorical_split.hpp | 2 - .../all_categorical_split_impl.hpp | 5 - .../decision_tree/decision_tree_impl.hpp | 295 ++++++++---------- src/mlpack/tests/decision_tree_test.cpp | 2 +- 6 files changed, 139 insertions(+), 168 deletions(-) diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index adcbd7fbe8..29e10ee1f4 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -113,6 +113,7 @@ Copyright: Copyright 2019, Dan Timson Copyright 2019, Miguel Canteras Copyright 2019, Bishwa Karki + Copyright 2019, Yashwant Singh Parihar License: BSD-3-clause All rights reserved. diff --git a/HISTORY.md b/HISTORY.md index bd9a721e15..f9e0123919 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -2,6 +2,8 @@ ###### ????-??-?? * Add Multiple Pole Balancing Environment (#1901). + * Add New paramter Maximum_depth to Decision Tree And Random Forest (#1916). + ### mlpack 3.1.1 ###### 2019-05-26 * Fix random forest bug for numerical-only data (#1887). diff --git a/src/mlpack/methods/decision_tree/all_categorical_split.hpp b/src/mlpack/methods/decision_tree/all_categorical_split.hpp index 87f796e1e1..2823fa9bb5 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split.hpp @@ -47,7 +47,6 @@ class AllCategoricalSplit * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in a leaf node for * splitting. - * @param maximumDepth Maximum Depth Minimum for the tree. * @param classProbabilities Class probabilities vector, which may be filled * with split information a successful split. * @param aux Auxiliary split information, which may be modified on a @@ -63,7 +62,6 @@ class AllCategoricalSplit const WeightVecType& weights, const size_t minimumLeafSize, const double minimumGainSplit, - const size_t maximumDepth, arma::Col& classProbabilities, AuxiliarySplitInfo& aux); diff --git a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp index ec317b77c6..97337fc508 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp @@ -26,14 +26,9 @@ double AllCategoricalSplit::SplitIfBetter( const WeightVecType& weights, const size_t minimumLeafSize, const double minimumGainSplit, - const size_t maximumDepth, arma::Col& classProbabilities, AuxiliarySplitInfo& /* aux */) { - // First sanity check: if we have reached maximum depth, we can't split. - if (maximumDepth == 1) - return DBL_MAX; - // Count the number of elements in each potential child. const double epsilon = 1e-7; // Tolerance for floating-point errors. arma::Col counts(numCategories, arma::fill::zeros); diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index 593a6f1157..d31fb26cec 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -656,98 +656,85 @@ double DecisionTree= 0.0) break; } + } + // Did we split or not? If so, then split the data and create the children. + if (bestDim != datasetInfo.Dimensionality()) + { + dimensionTypeOrMajorityClass = (size_t) datasetInfo.Type(bestDim); + splitDimension = bestDim; - // Did we split or not? If so, then split the data and create the children. - if (bestDim != datasetInfo.Dimensionality()) + // Get the number of children we will have. + size_t numChildren = 0; + if (datasetInfo.Type(bestDim) == data::Datatype::categorical) + numChildren = CategoricalSplit::NumChildren(classProbabilities, *this); + else + numChildren = NumericSplit::NumChildren(classProbabilities, *this); + + // Calculate all child assignments. + arma::Row childAssignments(count); + if (datasetInfo.Type(bestDim) == data::Datatype::categorical) { - dimensionTypeOrMajorityClass = (size_t) datasetInfo.Type(bestDim); - splitDimension = bestDim; - - // Get the number of children we will have. - size_t numChildren = 0; - if (datasetInfo.Type(bestDim) == data::Datatype::categorical) - numChildren = CategoricalSplit::NumChildren(classProbabilities, *this); - else - numChildren = NumericSplit::NumChildren(classProbabilities, *this); - - // Calculate all child assignments. - arma::Row childAssignments(count); - if (datasetInfo.Type(bestDim) == data::Datatype::categorical) - { - for (size_t j = begin; j < begin + count; ++j) - childAssignments[j - begin] = CategoricalSplit::CalculateDirection( - data(bestDim, j), classProbabilities, *this); - } - else - { - for (size_t j = begin; j < begin + count; ++j) - { - childAssignments[j - begin] = NumericSplit::CalculateDirection( - data(bestDim, j), classProbabilities, *this); - } - } - - // Figure out counts of children. - arma::Row childCounts(numChildren, arma::fill::zeros); - for (size_t i = begin; i < begin + count; ++i) - childCounts[childAssignments[i - begin]]++; - - // Initialize bestGain if recursive split is allowed. - if (!NoRecursion) - { - bestGain = 0.0; - } - - // Split into children. - size_t currentCol = begin; - for (size_t i = 0; i < numChildren; ++i) - { - size_t currentChildBegin = currentCol; - for (size_t j = currentChildBegin; j < begin + count; ++j) - { - if (childAssignments[j - begin] == i) - { - childAssignments.swap_cols(currentCol - begin, j - begin); - data.swap_cols(currentCol, j); - labels.swap_cols(currentCol, j); - if (UseWeights) - weights.swap_cols(currentCol, j); - ++currentCol; - } - } - - // Now build the child recursively. - DecisionTree* child = new DecisionTree(); - if (NoRecursion) - { - child->Train(data, currentChildBegin, - currentCol - currentChildBegin, datasetInfo, labels, numClasses, - weights, currentCol - currentChildBegin, minimumGainSplit, - maximumDepth - 1, dimensionSelector); - } - else - { - // During recursion entropy of child node may change. - double childGain = child->Train(data, currentChildBegin, - currentCol - currentChildBegin, datasetInfo, labels, numClasses, - weights, minimumLeafSize, minimumGainSplit, maximumDepth - 1, - dimensionSelector); - bestGain += double(childCounts[i]) / double(count) * (-childGain); - } - children.push_back(child); - } + for (size_t j = begin; j < begin + count; ++j) + childAssignments[j - begin] = CategoricalSplit::CalculateDirection( + data(bestDim, j), classProbabilities, *this); } else { - // Clear auxiliary info objects. - NumericAuxiliarySplitInfo::operator=(NumericAuxiliarySplitInfo()); - CategoricalAuxiliarySplitInfo::operator=(CategoricalAuxiliarySplitInfo()); + for (size_t j = begin; j < begin + count; ++j) + { + childAssignments[j - begin] = NumericSplit::CalculateDirection( + data(bestDim, j), classProbabilities, *this); + } + } - // Calculate class probabilities because we are a leaf. - CalculateClassProbabilities( - labels.subvec(begin, begin + count - 1), - numClasses, - UseWeights ? weights.subvec(begin, begin + count - 1) : weights); + // Figure out counts of children. + arma::Row childCounts(numChildren, arma::fill::zeros); + for (size_t i = begin; i < begin + count; ++i) + childCounts[childAssignments[i - begin]]++; + + // Initialize bestGain if recursive split is allowed. + if (!NoRecursion) + { + bestGain = 0.0; + } + + // Split into children. + size_t currentCol = begin; + for (size_t i = 0; i < numChildren; ++i) + { + size_t currentChildBegin = currentCol; + for (size_t j = currentChildBegin; j < begin + count; ++j) + { + if (childAssignments[j - begin] == i) + { + childAssignments.swap_cols(currentCol - begin, j - begin); + data.swap_cols(currentCol, j); + labels.swap_cols(currentCol, j); + if (UseWeights) + weights.swap_cols(currentCol, j); + ++currentCol; + } + } + + // Now build the child recursively. + DecisionTree* child = new DecisionTree(); + if (NoRecursion) + { + child->Train(data, currentChildBegin, + currentCol - currentChildBegin, datasetInfo, labels, numClasses, + weights, currentCol - currentChildBegin, minimumGainSplit, + maximumDepth - 1, dimensionSelector); + } + else + { + // During recursion entropy of child node may change. + double childGain = child->Train(data, currentChildBegin, + currentCol - currentChildBegin, datasetInfo, labels, numClasses, + weights, minimumLeafSize, minimumGainSplit, maximumDepth - 1, + dimensionSelector); + bestGain += double(childCounts[i]) / double(count) * (-childGain); + } + children.push_back(child); } } else @@ -840,84 +827,72 @@ double DecisionTree= 0.0) break; } + } + // Did we split or not? If so, then split the data and create the children. + if (bestDim != data.n_rows) + { + // We know that the split is numeric. + size_t numChildren = NumericSplit::NumChildren(classProbabilities, *this); + splitDimension = bestDim; + dimensionTypeOrMajorityClass = (size_t) data::Datatype::numeric; - // Did we split or not? If so, then split the data and create the children. - if (bestDim != data.n_rows) + // Calculate all child assignments. + arma::Row childAssignments(count); + + for (size_t j = begin; j < begin + count; ++j) { - // We know that the split is numeric. - size_t numChildren = NumericSplit::NumChildren(classProbabilities, *this); - splitDimension = bestDim; - dimensionTypeOrMajorityClass = (size_t) data::Datatype::numeric; - - // Calculate all child assignments. - arma::Row childAssignments(count); - - for (size_t j = begin; j < begin + count; ++j) - { - childAssignments[j - begin] = NumericSplit::CalculateDirection( - data(bestDim, j), classProbabilities, *this); - } - - // Calculate counts of children in each node. - arma::Row childCounts(numChildren); - childCounts.zeros(); - for (size_t j = begin; j < begin + count; ++j) - childCounts[childAssignments[j - begin]]++; - - // Initialize bestGain if recursive split is allowed. - if (!NoRecursion) - { - bestGain = 0.0; - } - - size_t currentCol = begin; - for (size_t i = 0; i < numChildren; ++i) - { - size_t currentChildBegin = currentCol; - for (size_t j = currentChildBegin; j < begin + count; ++j) - { - if (childAssignments[j - begin] == i) - { - childAssignments.swap_cols(currentCol - begin, j - begin); - data.swap_cols(currentCol, j); - labels.swap_cols(currentCol, j); - if (UseWeights) - weights.swap_cols(currentCol, j); - ++currentCol; - } - } - - // Now build the child recursively. - DecisionTree* child = new DecisionTree(); - if (NoRecursion) - { - child->Train(data, currentChildBegin, - currentCol - currentChildBegin, labels, numClasses, weights, - currentCol - currentChildBegin, minimumGainSplit, maximumDepth - 1, - dimensionSelector); - } - else - { - // During recursion entropy of child node may change. - double childGain = child->Train(data, currentChildBegin, - currentCol - currentChildBegin, labels, numClasses, weights, - minimumLeafSize, minimumGainSplit, maximumDepth - 1, - dimensionSelector); - bestGain += double(childCounts[i]) / double(count) * (-childGain); - } - children.push_back(child); - } + childAssignments[j - begin] = NumericSplit::CalculateDirection( + data(bestDim, j), classProbabilities, *this); } - else - { - // We won't be needing these members, so reset them. - NumericAuxiliarySplitInfo::operator=(NumericAuxiliarySplitInfo()); - // Calculate class probabilities because we are a leaf. - CalculateClassProbabilities( - labels.subvec(begin, begin + count - 1), - numClasses, - UseWeights ? weights.subvec(begin, begin + count - 1) : weights); + // Calculate counts of children in each node. + arma::Row childCounts(numChildren); + childCounts.zeros(); + for (size_t j = begin; j < begin + count; ++j) + childCounts[childAssignments[j - begin]]++; + + // Initialize bestGain if recursive split is allowed. + if (!NoRecursion) + { + bestGain = 0.0; + } + + size_t currentCol = begin; + for (size_t i = 0; i < numChildren; ++i) + { + size_t currentChildBegin = currentCol; + for (size_t j = currentChildBegin; j < begin + count; ++j) + { + if (childAssignments[j - begin] == i) + { + childAssignments.swap_cols(currentCol - begin, j - begin); + data.swap_cols(currentCol, j); + labels.swap_cols(currentCol, j); + if (UseWeights) + weights.swap_cols(currentCol, j); + ++currentCol; + } + } + + // Now build the child recursively. + DecisionTree* child = new DecisionTree(); + if (NoRecursion) + { + child->Train(data, currentChildBegin, + currentCol - currentChildBegin, labels, numClasses, weights, + currentCol - currentChildBegin, minimumGainSplit, maximumDepth - 1, + dimensionSelector); + } + else + { + // During recursion entropy of child node may change. + double childGain = child->Train(data, currentChildBegin, + currentCol - currentChildBegin, labels, numClasses, weights, + minimumLeafSize, minimumGainSplit, maximumDepth - 1, + dimensionSelector); + bestGain += double(childCounts[i]) / double(count) * (-childGain); + } + children.push_back(child); } } else diff --git a/src/mlpack/tests/decision_tree_test.cpp b/src/mlpack/tests/decision_tree_test.cpp index 84446014e0..c44f7a7bca 100644 --- a/src/mlpack/tests/decision_tree_test.cpp +++ b/src/mlpack/tests/decision_tree_test.cpp @@ -1217,7 +1217,7 @@ BOOST_AUTO_TEST_CASE(DecisionTreeCategoricalTrainReturnEntropy) } /** - * Make sure different Maximum Depth gives different number of childern. + * Make sure different Maximum Depth gives different number of children. */ BOOST_AUTO_TEST_CASE(DifferentMaximumDepthTest) { From 7ed45b7909da8edac05fe649ee8488cbaf268aa8 Mon Sep 17 00:00:00 2001 From: Yashwant Date: Thu, 20 Jun 2019 17:35:58 +0530 Subject: [PATCH 112/143] Fix SamplingsTest --- src/mlpack/tests/main_tests/gmm_train_test.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/mlpack/tests/main_tests/gmm_train_test.cpp b/src/mlpack/tests/main_tests/gmm_train_test.cpp index a8a02b32b9..00f2160da1 100644 --- a/src/mlpack/tests/main_tests/gmm_train_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_train_test.cpp @@ -244,8 +244,9 @@ BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) arma::mat inputData(5, 100, arma::fill::randu); SetInputParam("input", inputData); - SetInputParam("gaussians", (int) 3); + SetInputParam("gaussians", (int) 8); SetInputParam("trials", (int) 1); + SetInputParam("samplings", (int) 50); SetInputParam("max_iterations", (int) 500); size_t seed = std::time(NULL); @@ -260,8 +261,9 @@ BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) ResetGmmTrainSetting(); SetInputParam("input", std::move(inputData)); - SetInputParam("gaussians", (int) 3); + SetInputParam("gaussians", (int) 8); SetInputParam("max_iterations", (int) 500); + SetInputParam("samplings", (int) 50); SetInputParam("trials", (int) 500); mlpack::math::randGen.seed((uint32_t) seed); @@ -340,8 +342,9 @@ BOOST_AUTO_TEST_CASE(GmmTrainSamplingsTest) BOOST_FAIL("Unable to load train dataset data_3d_mixed.txt!"); SetInputParam("input", inputData); - SetInputParam("gaussians", (int) 3); + SetInputParam("gaussians", (int) 8); SetInputParam("refined_start", true); + SetInputParam("trials", (int) 2); SetInputParam("samplings", (int) 10); size_t seed = std::time(NULL); @@ -356,9 +359,10 @@ BOOST_AUTO_TEST_CASE(GmmTrainSamplingsTest) ResetGmmTrainSetting(); SetInputParam("input", std::move(inputData)); - SetInputParam("gaussians", (int) 3); + SetInputParam("gaussians", (int) 8); SetInputParam("refined_start", true); - SetInputParam("samplings", (int) 25000); + SetInputParam("trials", (int) 2); + SetInputParam("samplings", (int) 5000); mlpack::math::randGen.seed((uint32_t) seed); srand((unsigned int) seed); @@ -485,3 +489,4 @@ BOOST_AUTO_TEST_CASE(GmmTrainDiagCovariance) } BOOST_AUTO_TEST_SUITE_END(); + From 4a791b7b87910cfd594c23f1a5488aeeae517b35 Mon Sep 17 00:00:00 2001 From: Yashwant Singh Parihar Date: Thu, 20 Jun 2019 18:34:28 +0530 Subject: [PATCH 113/143] Fix TrailsTest --- src/mlpack/tests/main_tests/gmm_train_test.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/mlpack/tests/main_tests/gmm_train_test.cpp b/src/mlpack/tests/main_tests/gmm_train_test.cpp index 00f2160da1..e9315fc21f 100644 --- a/src/mlpack/tests/main_tests/gmm_train_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_train_test.cpp @@ -244,9 +244,8 @@ BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) arma::mat inputData(5, 100, arma::fill::randu); SetInputParam("input", inputData); - SetInputParam("gaussians", (int) 8); + SetInputParam("gaussians", (int) 4); SetInputParam("trials", (int) 1); - SetInputParam("samplings", (int) 50); SetInputParam("max_iterations", (int) 500); size_t seed = std::time(NULL); @@ -261,9 +260,8 @@ BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) ResetGmmTrainSetting(); SetInputParam("input", std::move(inputData)); - SetInputParam("gaussians", (int) 8); + SetInputParam("gaussians", (int) 4); SetInputParam("max_iterations", (int) 500); - SetInputParam("samplings", (int) 50); SetInputParam("trials", (int) 500); mlpack::math::randGen.seed((uint32_t) seed); From 29fbf8600daba32a923e030a8d6edba3e33a3019 Mon Sep 17 00:00:00 2001 From: Yashwant Singh Parihar Date: Thu, 20 Jun 2019 19:27:26 +0530 Subject: [PATCH 114/143] Still Failing TrialsTest --- src/mlpack/tests/main_tests/gmm_train_test.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/main_tests/gmm_train_test.cpp b/src/mlpack/tests/main_tests/gmm_train_test.cpp index e9315fc21f..050b756fca 100644 --- a/src/mlpack/tests/main_tests/gmm_train_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_train_test.cpp @@ -244,7 +244,9 @@ BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) arma::mat inputData(5, 100, arma::fill::randu); SetInputParam("input", inputData); - SetInputParam("gaussians", (int) 4); + SetInputParam("gaussians", (int) 3); + SetInputParam("refined_start", true); + SetInputParam("samplings", (int) 1000); SetInputParam("trials", (int) 1); SetInputParam("max_iterations", (int) 500); @@ -260,7 +262,9 @@ BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) ResetGmmTrainSetting(); SetInputParam("input", std::move(inputData)); - SetInputParam("gaussians", (int) 4); + SetInputParam("gaussians", (int) 3); + SetInputParam("refined_start", true); + SetInputParam("samplings", (int) 1000); SetInputParam("max_iterations", (int) 500); SetInputParam("trials", (int) 500); From e3989bd4eb653bf9b0161649f66fadf2a790625c Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 20 Jun 2019 21:03:02 -0400 Subject: [PATCH 115/143] Minor style fixes. --- HISTORY.md | 3 ++- src/mlpack/methods/decision_tree/decision_tree_impl.hpp | 2 ++ src/mlpack/methods/decision_tree/decision_tree_main.cpp | 8 ++++---- src/mlpack/methods/random_forest/random_forest_main.cpp | 6 +++--- src/mlpack/tests/decision_tree_test.cpp | 2 +- src/mlpack/tests/main_tests/decision_tree_test.cpp | 6 +++--- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index b5be1c830d..2e77ec990c 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -2,7 +2,8 @@ ###### ????-??-?? * Add Multiple Pole Balancing Environment (#1901). - * Add New paramter Maximum_depth to Decision Tree And Random Forest (#1916). + * Add new parameter `maximum_depth` to decision tree and random forest + bindings (#1916). * Fix prediction output of softmax regression when test set accuracy is calculated (#1922). diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index 280a3164e5..8d08a74d8b 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -657,6 +657,7 @@ double DecisionTree("maximum_depth", [](int x) { return x >= 0; }, true, - "depth must not be negative"); + "maximum depth must not be negative"); RequireParamValue("minimum_gain_split", [](double x) { return (x > 0.0 && x < 1.0); }, true, diff --git a/src/mlpack/methods/random_forest/random_forest_main.cpp b/src/mlpack/methods/random_forest/random_forest_main.cpp index 00cc38af31..6788e3f296 100644 --- a/src/mlpack/methods/random_forest/random_forest_main.cpp +++ b/src/mlpack/methods/random_forest/random_forest_main.cpp @@ -50,7 +50,7 @@ PROGRAM_INFO("Random forests", " controls the number of trees in the random forest. The " + PRINT_PARAM_STRING("minimum_gain_split") + " parameter controls the minimum" " required gain for a decision tree node to split. Larger values will " - "force higher-confidence splits. The " + + "force higher-confidence splits. The " + PRINT_PARAM_STRING("maximum_depth") + " parameter specifies " "the maximum depth of the tree. The " + PRINT_PARAM_STRING("subspace_dim") + " parameter is used to control the " @@ -107,7 +107,7 @@ PARAM_FLAG("print_training_accuracy", "If set, then the accuracy of the model " PARAM_INT_IN("num_trees", "Number of trees in the random forest.", "N", 10); PARAM_INT_IN("minimum_leaf_size", "Minimum number of points in each leaf " "node.", "n", 1); -PARAM_INT_IN("maximum_depth", "Maximum depth of the tree.(0 means no limit)", +PARAM_INT_IN("maximum_depth", "Maximum depth of the tree (0 means no limit).", "D", 0); PARAM_MATRIX_OUT("probabilities", "Predicted class probabilities for each " "point in the test set.", "P"); @@ -181,7 +181,7 @@ static void mlpackMain() RequireParamValue("minimum_leaf_size", [](int x) { return x > 0; }, true, "minimum leaf size must be greater than 0"); RequireParamValue("maximum_depth", [](int x) { return x >= 0; }, true, - "depth must not be negative"); + "maximum depth must not be negative"); RequireParamValue("subspace_dim", [](int x) { return x >= 0; }, true, "subspace dimensionality must be nonnegative"); RequireParamValue("minimum_gain_split", diff --git a/src/mlpack/tests/decision_tree_test.cpp b/src/mlpack/tests/decision_tree_test.cpp index c44f7a7bca..334fa745a9 100644 --- a/src/mlpack/tests/decision_tree_test.cpp +++ b/src/mlpack/tests/decision_tree_test.cpp @@ -1217,7 +1217,7 @@ BOOST_AUTO_TEST_CASE(DecisionTreeCategoricalTrainReturnEntropy) } /** - * Make sure different Maximum Depth gives different number of children. + * Make sure different maximum depth values give different numbers of children. */ BOOST_AUTO_TEST_CASE(DifferentMaximumDepthTest) { diff --git a/src/mlpack/tests/main_tests/decision_tree_test.cpp b/src/mlpack/tests/main_tests/decision_tree_test.cpp index 159f8cd753..b770d333fd 100644 --- a/src/mlpack/tests/main_tests/decision_tree_test.cpp +++ b/src/mlpack/tests/main_tests/decision_tree_test.cpp @@ -174,7 +174,7 @@ BOOST_AUTO_TEST_CASE(DecisionTreeMinimumLeafSizeTest) } /** - * Make sure maximum depth size is always a non-negative number. + * Make sure maximum depth is always a non-negative number. */ BOOST_AUTO_TEST_CASE(DecisionTreeNonNegativeMaximumDepthTest) { @@ -201,6 +201,7 @@ BOOST_AUTO_TEST_CASE(DecisionTreeNonNegativeMaximumDepthTest) BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); Log::Fatal.ignoreInput = false; } + /** * Make sure minimum gain split is always a fraction in range [0,1]. */ @@ -454,8 +455,7 @@ BOOST_AUTO_TEST_CASE(DecisionModelCategoricalReuseTest) } /** - * Check that different maximum depth gives - * different results. + * Check that different maximum depths give different results. */ BOOST_AUTO_TEST_CASE(DecisionTreeMaximumDepthTest) { From 2f5dc0d8d08838eaeac44d03e385d4a8768eb50b Mon Sep 17 00:00:00 2001 From: Yashwant Singh Parihar Date: Sat, 22 Jun 2019 17:45:10 +0530 Subject: [PATCH 116/143] Fix Trials test --- src/mlpack/tests/main_tests/gmm_train_test.cpp | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/mlpack/tests/main_tests/gmm_train_test.cpp b/src/mlpack/tests/main_tests/gmm_train_test.cpp index 050b756fca..2c38247c30 100644 --- a/src/mlpack/tests/main_tests/gmm_train_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_train_test.cpp @@ -9,6 +9,7 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ + #include #define BINDING_TYPE BINDING_TYPE_TEST @@ -241,14 +242,12 @@ BOOST_AUTO_TEST_CASE(GmmTrainNoiseTest) // Ensure that Trials affects the final result. BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) { - arma::mat inputData(5, 100, arma::fill::randu); + arma::mat inputData(10000, 50, arma::fill::randu); SetInputParam("input", inputData); - SetInputParam("gaussians", (int) 3); - SetInputParam("refined_start", true); - SetInputParam("samplings", (int) 1000); - SetInputParam("trials", (int) 1); + SetInputParam("gaussians", (int) 5); SetInputParam("max_iterations", (int) 500); + SetInputParam("trials", (int) 1); size_t seed = std::time(NULL); mlpack::math::randGen.seed((uint32_t) seed); @@ -262,9 +261,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) ResetGmmTrainSetting(); SetInputParam("input", std::move(inputData)); - SetInputParam("gaussians", (int) 3); - SetInputParam("refined_start", true); - SetInputParam("samplings", (int) 1000); + SetInputParam("gaussians", (int) 5); SetInputParam("max_iterations", (int) 500); SetInputParam("trials", (int) 500); @@ -491,4 +488,3 @@ BOOST_AUTO_TEST_CASE(GmmTrainDiagCovariance) } BOOST_AUTO_TEST_SUITE_END(); - From c53f8481436f443b06f2ffb7e22a72f2844754cf Mon Sep 17 00:00:00 2001 From: Yashwant Singh Parihar Date: Sat, 22 Jun 2019 18:13:45 +0530 Subject: [PATCH 117/143] Fix :( --- src/mlpack/tests/main_tests/gmm_train_test.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/tests/main_tests/gmm_train_test.cpp b/src/mlpack/tests/main_tests/gmm_train_test.cpp index 2c38247c30..4e7380ef1a 100644 --- a/src/mlpack/tests/main_tests/gmm_train_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_train_test.cpp @@ -242,12 +242,12 @@ BOOST_AUTO_TEST_CASE(GmmTrainNoiseTest) // Ensure that Trials affects the final result. BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) { - arma::mat inputData(10000, 50, arma::fill::randu); + arma::mat inputData(5, 1000, arma::fill::randu); SetInputParam("input", inputData); - SetInputParam("gaussians", (int) 5); - SetInputParam("max_iterations", (int) 500); + SetInputParam("gaussians", (int) 3); SetInputParam("trials", (int) 1); + SetInputParam("max_iterations", (int) 500); size_t seed = std::time(NULL); mlpack::math::randGen.seed((uint32_t) seed); @@ -261,7 +261,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) ResetGmmTrainSetting(); SetInputParam("input", std::move(inputData)); - SetInputParam("gaussians", (int) 5); + SetInputParam("gaussians", (int) 3); SetInputParam("max_iterations", (int) 500); SetInputParam("trials", (int) 500); From ecb6daf640888ff6efaf3dec65d06ab5ea7e5f5f Mon Sep 17 00:00:00 2001 From: Yashwant Singh Parihar Date: Sat, 22 Jun 2019 23:21:41 +0530 Subject: [PATCH 118/143] Add gmmtrainmaintest To parallel test --- src/mlpack/tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 8084e29d5b..af1a86c17e 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -188,7 +188,7 @@ set(parallel_tests "SVDIncrementalTest;SVDBatchTest;" "LocalCoordinateCodingTest;FeedForwardNetworkTest;SparseAutoencoderTest;" "GMMTest;CFTest;ConvolutionalNetworkTest;HMMTest;LARSTest;" - "LogisticRegressionTest;" + "LogisticRegressionTest;GmmTrainMainTest;" "LinearSVMTest") # Add tests to the testing framework From ff3d0d8f73ae184ea748b3d89e2acf34dc27ea8d Mon Sep 17 00:00:00 2001 From: Yashwant Singh Parihar Date: Sun, 23 Jun 2019 07:24:47 +0530 Subject: [PATCH 119/143] Update gmm_train_test.cpp --- src/mlpack/tests/main_tests/gmm_train_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/main_tests/gmm_train_test.cpp b/src/mlpack/tests/main_tests/gmm_train_test.cpp index 4e7380ef1a..e049e8538e 100644 --- a/src/mlpack/tests/main_tests/gmm_train_test.cpp +++ b/src/mlpack/tests/main_tests/gmm_train_test.cpp @@ -242,7 +242,7 @@ BOOST_AUTO_TEST_CASE(GmmTrainNoiseTest) // Ensure that Trials affects the final result. BOOST_AUTO_TEST_CASE(GmmTrainTrialsTest) { - arma::mat inputData(5, 1000, arma::fill::randu); + arma::mat inputData(5, 250, arma::fill::randu); SetInputParam("input", inputData); SetInputParam("gaussians", (int) 3); From 9b9e0dd7380d9a3df8671adf4343a551a01bd1e4 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Tue, 25 Jun 2019 14:47:38 +0530 Subject: [PATCH 120/143] Added tests and made the continuous classes uniform --- .../environment/continuous_mountain_car.hpp | 6 ++--- .../continuous_multiple_pole_cart.hpp | 4 ++-- .../environment/pendulum.hpp | 23 +++++++++++++++++-- src/mlpack/tests/rl_components_test.cpp | 6 +++-- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp index 0ab4677fc3..f7fbebc7ce 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp @@ -83,7 +83,7 @@ class ContinuousMountainCar */ struct Action { - double action[1]; + double action = 0.0; // Storing degree of freedom const int size = 1; }; @@ -126,7 +126,7 @@ class ContinuousMountainCar State& nextState) const { // Calculate acceleration. - double force = std::min(std::max(action.action[0], -1.0), 1.0); + double force = std::min(std::max(action.action, -1.0), 1.0); // Update states. nextState.Velocity() = state.Velocity() + force * power - 0.0025 * @@ -144,7 +144,7 @@ class ContinuousMountainCar // If it is a terminal state, add a reward of 100.0 if (IsTerminal(nextState)) reward = 100.0; - reward -= std::pow(action.action[0], 2) * 0.1; + reward -= std::pow(action.action, 2) * 0.1; return reward; } diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp index 97eb67ffa5..46bfe119cf 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp @@ -91,7 +91,7 @@ class ContinuousMultiplePoleCart */ struct Action { - double action[1]; + double action = 0.0; // Track the size of the action space. const int size = 1; }; @@ -155,7 +155,7 @@ class ContinuousMultiplePoleCart State& nextState) const { // Calculate acceleration. - double totalForce = action.action[0]; + double totalForce = action.action; double totalMass = massCart; for (size_t i = 0; i < poleNum; i++) { diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index bfd4cb1105..8b27843e5f 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -93,13 +93,17 @@ class Pendulum * @param maxAngularVelocity Maximum angular velocity. * @param maxTorque Maximum torque. * @param dt The differential value. + * @param angleThreshold The region about the upright position where the + * state is considered terminal. */ Pendulum(const double maxAngularVelocity = 8, const double maxTorque = 2.0, - const double dt = 0.05) : + const double dt = 0.05, + const double angleThreshold = M_PI/6) : maxAngularVelocity(maxAngularVelocity), maxTorque(maxTorque), - dt(dt) + dt(dt), + angleThreshold(angleThreshold) { /* Nothing to do here */ } /** @@ -183,6 +187,18 @@ class Pendulum return double(fmod(theta + M_PI, 2 * M_PI) - M_PI); } + /** + * Whether given state is a terminal state. + * + * @param state desired state. + * @return true if state is a terminal state, otherwise false. + */ + bool isTerminal(const State& state) const + { + return state.Theta() > M_PI - angleThreshold && + state.Theta() < M_PI + angleThreshold; + } + private: //! Locally-stored maximum legal angular velocity. double maxAngularVelocity; @@ -192,6 +208,9 @@ class Pendulum //! Locally-stored dt. double dt; + + //! Locally-stored angle threshold. + double angleThreshold; }; } // namespace rl diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index e3ff6383e4..7bc6ab6d32 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -47,6 +47,8 @@ BOOST_AUTO_TEST_CASE(SimplePendulumTest) // The reward is always negative. Check if not lower than lowest possible. BOOST_REQUIRE(reward >= -(pow(M_PI, 2) + 6.404)); + BOOST_REQUIRE(!task.isTerminal(state)); + // The action is simply the torque. Check if dimension is 1. BOOST_REQUIRE_EQUAL(1, action.size); } @@ -61,7 +63,7 @@ BOOST_AUTO_TEST_CASE(SimpleContinuousMountainCarTest) ContinuousMountainCar::State state = task.InitialSample(); ContinuousMountainCar::Action action; - action.action[0] = math::Random(-1.0, 1.0); + action.action = math::Random(-1.0, 1.0); double reward = task.Sample(state, action); // Maximum reward possible is 100. BOOST_REQUIRE(reward <= 100.0); @@ -152,7 +154,7 @@ BOOST_AUTO_TEST_CASE(ContinuousMultiplePoleCartTest) ContinuousMultiplePoleCart::State state = task.InitialSample(); ContinuousMultiplePoleCart::Action action; - action.action[0] = math::Random(-1.0, 1.0); + action.action = math::Random(-1.0, 1.0); double reward = task.Sample(state, action); BOOST_REQUIRE_EQUAL(reward, 1.0); From aa7b9fb93f550aec390731dcbac29935f30520b7 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Tue, 25 Jun 2019 15:29:38 +0530 Subject: [PATCH 121/143] Fixed bounds --- .../methods/reinforcement_learning/environment/pendulum.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index 8b27843e5f..25d8f38311 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -171,7 +171,7 @@ class Pendulum State InitialSample() const { State state; - state.Theta() = math::Random(-M_PI, M_PI); + state.Theta() = math::Random(-M_PI + angleThreshold, M_PI - angleThreshold); state.AngularVelocity() = math::Random(-1.0, 1.0); return state; } @@ -195,8 +195,8 @@ class Pendulum */ bool isTerminal(const State& state) const { - return state.Theta() > M_PI - angleThreshold && - state.Theta() < M_PI + angleThreshold; + return state.Theta() > M_PI - angleThreshold || + state.Theta() < -M_PI + angleThreshold; } private: From 6b54bbe3d428e8b559e40717ad255131e39a01df Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Tue, 25 Jun 2019 17:54:42 +0530 Subject: [PATCH 122/143] Style fixes + HISTORY.md --- HISTORY.md | 4 ++++ .../methods/reinforcement_learning/environment/pendulum.hpp | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 2e77ec990c..0f0b3031af 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -7,6 +7,10 @@ * Fix prediction output of softmax regression when test set accuracy is calculated (#1922). + + * Add `IsTerminal()` method to the pendulum environment. Action struct in + continuous RL environments now stores the action as a `double` instead + of `double[1]` (#1941, #1931). ### mlpack 3.1.1 ###### 2019-05-26 diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index 25d8f38311..7687bf1e98 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -99,7 +99,7 @@ class Pendulum Pendulum(const double maxAngularVelocity = 8, const double maxTorque = 2.0, const double dt = 0.05, - const double angleThreshold = M_PI/6) : + const double angleThreshold = M_PI / 12) : maxAngularVelocity(maxAngularVelocity), maxTorque(maxTorque), dt(dt), @@ -193,7 +193,7 @@ class Pendulum * @param state desired state. * @return true if state is a terminal state, otherwise false. */ - bool isTerminal(const State& state) const + bool IsTerminal(const State& state) const { return state.Theta() > M_PI - angleThreshold || state.Theta() < -M_PI + angleThreshold; From 2d2730b2e2b27d8974aa092859abc352c3ee9302 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Tue, 25 Jun 2019 18:03:58 +0530 Subject: [PATCH 123/143] Commits not working? --- HISTORY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 0f0b3031af..93d28d2633 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -10,7 +10,7 @@ * Add `IsTerminal()` method to the pendulum environment. Action struct in continuous RL environments now stores the action as a `double` instead - of `double[1]` (#1941, #1931). + of `double[1]` (#1941, #1931) ### mlpack 3.1.1 ###### 2019-05-26 From d9d8a37966429db810b9aba4f7df92e3f5ae32be Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Tue, 25 Jun 2019 18:05:00 +0530 Subject: [PATCH 124/143] Commits working. --- HISTORY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 93d28d2633..0f0b3031af 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -10,7 +10,7 @@ * Add `IsTerminal()` method to the pendulum environment. Action struct in continuous RL environments now stores the action as a `double` instead - of `double[1]` (#1941, #1931) + of `double[1]` (#1941, #1931). ### mlpack 3.1.1 ###### 2019-05-26 From 20dd60b2df4bd0c5d793c10202d068b88dffa9ae Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Wed, 26 Jun 2019 10:07:21 +0530 Subject: [PATCH 125/143] Fixed misspelling. --- src/mlpack/tests/rl_components_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 7bc6ab6d32..7a0e6d66e5 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -47,7 +47,7 @@ BOOST_AUTO_TEST_CASE(SimplePendulumTest) // The reward is always negative. Check if not lower than lowest possible. BOOST_REQUIRE(reward >= -(pow(M_PI, 2) + 6.404)); - BOOST_REQUIRE(!task.isTerminal(state)); + BOOST_REQUIRE(!task.IsTerminal(state)); // The action is simply the torque. Check if dimension is 1. BOOST_REQUIRE_EQUAL(1, action.size); From 8fe8b31d828ce6814012326e58e2059eda331231 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Thu, 27 Jun 2019 14:48:28 +0530 Subject: [PATCH 126/143] Fixed multiple pole cart, changed Pendulum to terminate after a given number of time steps. --- .../environment/multiple_pole_cart.hpp | 18 +++---- .../environment/pendulum.hpp | 54 ++++++++++++++++--- .../environment/reward_clipping.hpp | 4 +- src/mlpack/tests/rl_components_test.cpp | 2 +- 4 files changed, 59 insertions(+), 19 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp index c591cd1275..d765f7c5e7 100755 --- a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp @@ -40,7 +40,7 @@ class MultiplePoleCart */ State(const size_t numPoles) { - data = arma::zeros(dimension, numPoles); + data = arma::zeros(dimension, numPoles + 1); } /** @@ -162,23 +162,23 @@ class MultiplePoleCart double totalMass = massCart; for (size_t i = 0; i < poleNum; i++) { - double poleOmega = state.AngularVelocity(i); - double sinTheta = sin(state.Angle(i)); + double poleOmega = state.AngularVelocity(i + 1); + double sinTheta = sin(state.Angle(i + 1)); totalForce += (poleMasses[i] * poleLengths[i] * poleOmega * poleOmega * - sinTheta) + 0.75 * poleMasses[i] * gravity * sin(2 * state.Angle(i)) - / 2; + sinTheta) + 0.75 * poleMasses[i] * gravity * sin(2 * state.Angle(i + + 1)) / 2; totalMass += poleMasses[i] * (0.25 + 0.75 * sinTheta * sinTheta); } double xAcc = totalForce / totalMass; // Update states of the poles. - for (size_t i = 0; i < poleNum; i++) + for (size_t i = 1; i <= poleNum; i++) { double sinTheta = sin(state.Angle(i)); double cosTheta = cos(state.Angle(i)); nextState.Angle(i) = state.Angle(i) + tau * state.AngularVelocity(i); nextState.AngularVelocity(i) = state.AngularVelocity(i) - tau * 0.75 * - (xAcc * cosTheta + gravity * sinTheta) / poleLengths[i]; + (xAcc * cosTheta + gravity * sinTheta) / poleLengths[i - 1]; } // Update state of the cart. @@ -220,7 +220,7 @@ class MultiplePoleCart */ State InitialSample() const { - return State((arma::randu(2, poleNum) - 0.5) / 10.0); + return State((arma::randu(2, poleNum + 1) - 0.5) / 10.0); } /** @@ -231,7 +231,7 @@ class MultiplePoleCart */ bool IsTerminal(const State& state) const { - for (size_t i = 0; i < poleNum; i++) + for (size_t i = 1; i <= poleNum; i++) if (std::abs(state.Angle(i)) > thetaThresholdRadians) return true; return std::abs(state.Position()) > xThreshold; diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index 7687bf1e98..7394e233c3 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -95,15 +95,22 @@ class Pendulum * @param dt The differential value. * @param angleThreshold The region about the upright position where the * state is considered terminal. + * @param maxTimeSteps The number of time steps after which the episode + * terminates. If the value is 0, there is no limit. */ Pendulum(const double maxAngularVelocity = 8, const double maxTorque = 2.0, const double dt = 0.05, - const double angleThreshold = M_PI / 12) : + const double angleThreshold = M_PI / 12, + const double doneReward = 0.0, + const size_t maxTimeSteps = 0) : maxAngularVelocity(maxAngularVelocity), maxTorque(maxTorque), dt(dt), - angleThreshold(angleThreshold) + angleThreshold(angleThreshold), + doneReward(doneReward), + maxTimeSteps(maxTimeSteps), + timeStepsPerformed(0) { /* Nothing to do here */ } /** @@ -117,8 +124,11 @@ class Pendulum */ double Sample(const State& state, const Action& action, - State& nextState) const + State& nextState) { + // Update the number of time steps performed. + timeStepsPerformed++; + // Get current state. double theta = state.Theta(); double angularVelocity = state.AngularVelocity(); @@ -144,6 +154,15 @@ class Pendulum -maxAngularVelocity), maxAngularVelocity); nextState.Theta() = theta + newAngularVelocity * dt; + // Check if the episode has terminated + bool done = IsTerminal(nextState); + + // Do not reward the agent if time ran out. + if (done && maxTimeSteps != 0 && timeStepsPerformed >= maxTimeSteps) + return 0; + else if (done) + return doneReward; + // Return the reward of taking the action in current state. // The reward is simply the negative of cost incurred for the action. return -costs; @@ -156,7 +175,7 @@ class Pendulum * @param action The current action. * @return reward, The reward. */ - double Sample(const State& state, const Action& action) const + double Sample(const State& state, const Action& action) { State nextState; return Sample(state, action, nextState); @@ -168,11 +187,12 @@ class Pendulum * * @return Initial state for each episode. */ - State InitialSample() const + State InitialSample() { State state; state.Theta() = math::Random(-M_PI + angleThreshold, M_PI - angleThreshold); state.AngularVelocity() = math::Random(-1.0, 1.0); + timeStepsPerformed = 0; return state; } @@ -195,8 +215,19 @@ class Pendulum */ bool IsTerminal(const State& state) const { - return state.Theta() > M_PI - angleThreshold || - state.Theta() < -M_PI + angleThreshold; + if (maxTimeSteps != 0 && timeStepsPerformed >= maxTimeSteps) + { + Log::Info << "Episode terminated due to the maximum number of time steps" + "being taken."; + return true; + } + else if (state.Theta() > M_PI - angleThreshold || + state.Theta() < -M_PI + angleThreshold) + { + Log::Info << "Episode terminated due to agent succeeding."; + return true; + } + return false; } private: @@ -211,6 +242,15 @@ class Pendulum //! Locally-stored angle threshold. double angleThreshold; + + //! Locally-stored done reward. + double doneReward; + + //! Locally-stored maximum number of time steps. + size_t maxTimeSteps; + + //! Locally-stored number of time steps performed. + size_t timeStepsPerformed; }; } // namespace rl diff --git a/src/mlpack/methods/reinforcement_learning/environment/reward_clipping.hpp b/src/mlpack/methods/reinforcement_learning/environment/reward_clipping.hpp index 6e0b09b228..a7ce90038e 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/reward_clipping.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/reward_clipping.hpp @@ -86,7 +86,7 @@ class RewardClipping */ double Sample(const State& state, const Action& action, - State& nextState) const + State& nextState) { // Get original unclipped reward from base environment. double unclippedReward = environment.Sample(state, action, nextState); @@ -102,7 +102,7 @@ class RewardClipping * @param action The current action. * @return clippedReward, Reward clipped between [minReward, maxReward]. */ - double Sample(const State& state, const Action& action) const + double Sample(const State& state, const Action& action) { State nextState; return Sample(state, action, nextState); diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 7a0e6d66e5..0917c0d3c9 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -37,7 +37,7 @@ BOOST_AUTO_TEST_SUITE(RLComponentsTest) */ BOOST_AUTO_TEST_CASE(SimplePendulumTest) { - const Pendulum task = Pendulum(); + Pendulum task = Pendulum(); Pendulum::State state = task.InitialSample(); Pendulum::Action action; From d4edb79047e4ca4ec4123d3917b49c79acb5cdbf Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Thu, 27 Jun 2019 15:04:53 +0530 Subject: [PATCH 127/143] Continuous multiple pole cart also fixed. --- .../continuous_multiple_pole_cart.hpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp index 46bfe119cf..52e975eedf 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp @@ -41,7 +41,7 @@ class ContinuousMultiplePoleCart */ State(const size_t numPoles) { - data = arma::zeros(dimension, numPoles); + data = arma::zeros(dimension, numPoles + 1); } /** @@ -159,23 +159,23 @@ class ContinuousMultiplePoleCart double totalMass = massCart; for (size_t i = 0; i < poleNum; i++) { - double poleOmega = state.AngularVelocity(i); - double sinTheta = sin(state.Angle(i)); + double poleOmega = state.AngularVelocity(i + 1); + double sinTheta = sin(state.Angle(i + 1)); totalForce += (poleMasses[i] * poleLengths[i] * poleOmega * poleOmega * - sinTheta) + 0.75 * poleMasses[i] * gravity * sin(2 * state.Angle(i)) - / 2; + sinTheta) + 0.75 * poleMasses[i] * gravity * sin(2 * state.Angle(i + + 1)) / 2; totalMass += poleMasses[i] * (0.25 + 0.75 * sinTheta * sinTheta); } double xAcc = totalForce / totalMass; // Update states of the poles. - for (size_t i = 0; i < poleNum; i++) + for (size_t i = 1; i <= poleNum; i++) { double sinTheta = sin(state.Angle(i)); double cosTheta = cos(state.Angle(i)); nextState.Angle(i) = state.Angle(i) + tau * state.AngularVelocity(i); nextState.AngularVelocity(i) = state.AngularVelocity(i) - tau * 0.75 * - (xAcc * cosTheta + gravity * sinTheta) / poleLengths[i]; + (xAcc * cosTheta + gravity * sinTheta) / poleLengths[i - 1]; } // Update state of the cart. @@ -217,7 +217,7 @@ class ContinuousMultiplePoleCart */ State InitialSample() const { - return State((arma::randu(2, poleNum) - 0.5) / 10.0); + return State((arma::randu(2, poleNum + 1) - 0.5) / 10.0); } /** @@ -228,7 +228,7 @@ class ContinuousMultiplePoleCart */ bool IsTerminal(const State& state) const { - for (size_t i = 0; i < poleNum; i++) + for (size_t i = 1; i <= poleNum; i++) if (std::abs(state.Angle(i)) > thetaThresholdRadians) return true; return std::abs(state.Position()) > xThreshold; From 2077b5a151aa23d83f8bacc021c7fe81d5eab6dc Mon Sep 17 00:00:00 2001 From: Saksham Bansal <7020962+saksham189@users.noreply.github.com> Date: Thu, 27 Jun 2019 20:50:15 +0700 Subject: [PATCH 128/143] Minor Optimization --- src/mlpack/methods/ann/layer/batch_norm_impl.hpp | 3 +-- src/mlpack/methods/ann/layer/layer_norm_impl.hpp | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp index ec0ea928fb..0c4d76e83a 100644 --- a/src/mlpack/methods/ann/layer/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -124,8 +124,7 @@ void BatchNorm::Backward( // Step 3: sum (dl / dxhat * -1 / stdInv) + variance * // (sum -2 * (x - mu)) / m. - g.each_col() += (arma::sum(norm.each_col() % -stdInv, 1) + (var % - arma::mean(-2 * inputMean, 1))) / input.n_cols; + g.each_col() += arma::sum(norm.each_col() % -stdInv, 1) / input.n_cols; } template diff --git a/src/mlpack/methods/ann/layer/layer_norm_impl.hpp b/src/mlpack/methods/ann/layer/layer_norm_impl.hpp index 5572478178..f5bc562529 100644 --- a/src/mlpack/methods/ann/layer/layer_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/layer_norm_impl.hpp @@ -96,8 +96,7 @@ void LayerNorm::Backward( // sum (dl / dxhat * -1 / stdInv) + variance * // (sum -2 * (x - mu)) / m. - g.each_row() += (arma::sum(norm.each_row() % -stdInv, 0) + (var % - arma::mean(-2 * inputMean, 0))) / input.n_rows; + g.each_row() += arma::sum(norm.each_row() % -stdInv, 0) / input.n_rows; } template From da64d1b3ad1ea88aaaff00f41d62cea33a9233ea Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Fri, 28 Jun 2019 17:26:39 +0530 Subject: [PATCH 129/143] Made all environments consistent. --- .../environment/acrobot.hpp | 99 ++++++++++++------- .../environment/cart_pole.hpp | 59 ++++++++--- .../environment/continuous_mountain_car.hpp | 65 +++++++++--- .../continuous_multiple_pole_cart.hpp | 59 ++++++++--- .../environment/mountain_car.hpp | 63 ++++++++---- .../environment/multiple_pole_cart.hpp | 62 +++++++++--- .../environment/pendulum.hpp | 18 ++-- src/mlpack/tests/rl_components_test.cpp | 12 +-- 8 files changed, 320 insertions(+), 117 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp index 638ef6ee3a..8507a30be1 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp @@ -28,10 +28,10 @@ namespace rl{ class Acrobot { public: - /* - * Implementation of Acrobot State. Each State is a tuple vector - * (theta1, thetha2, angular velocity 1, angular velocity 2). - */ + /* + * Implementation of Acrobot State. Each State is a tuple vector + * (theta1, thetha2, angular velocity 1, angular velocity 2). + */ class State { public: @@ -95,21 +95,24 @@ class Acrobot size }; - /** - * Construct a Acrobot instance using the given constants. - * - * @param gravity The gravity parameter. - * @param linkLength1 The length of link 1. - * @param linkLength2 The length of link 2. - * @param linkMass1 The mass of link 1. - * @param linkMass2 The mass of link 2. - * @param linkCom1 The position of the center of mass of link 1. - * @param linkCom2 The position of the center of mass of link 2. - * @param linkMoi The moments of inertia for both link. - * @param maxVel1 The max angular velocity of link1. - * @param maxVel2 The max angular velocity of link2. - * @param dt The differential value. - */ + /** + * Construct a Acrobot instance using the given constants. + * + * @param gravity The gravity parameter. + * @param linkLength1 The length of link 1. + * @param linkLength2 The length of link 2. + * @param linkMass1 The mass of link 1. + * @param linkMass2 The mass of link 2. + * @param linkCom1 The position of the center of mass of link 1. + * @param linkCom2 The position of the center of mass of link 2. + * @param linkMoi The moments of inertia for both link. + * @param maxVel1 The max angular velocity of link1. + * @param maxVel2 The max angular velocity of link2. + * @param dt The differential value. + * @param doneReward The reward recieved by the agent on success. + * @param maxTimeSteps The number of time steps after which the episode + * terminates. If the value is 0, there is no limit. + */ Acrobot(const double gravity = 9.81, const double linkLength1 = 1.0, const double linkLength2 = 1.0, @@ -121,7 +124,8 @@ class Acrobot const double maxVel1 = 4 * M_PI, const double maxVel2 = 9 * M_PI, const double dt = 0.2, - const double doneReward = 0) : + const double doneReward = 0, + const size_t maxSteps = 0) : gravity(gravity), linkLength1(linkLength1), linkLength2(linkLength2), @@ -133,7 +137,9 @@ class Acrobot maxVel1(maxVel1), maxVel2(maxVel2), dt(dt), - doneReward(doneReward) + doneReward(doneReward), + maxSteps(maxSteps), + timeStepsPerformed(0) { /* Nothing to do here */ } /** @@ -147,8 +153,11 @@ class Acrobot */ double Sample(const State& state, const Action& action, - State& nextState) const + State& nextState) { + // Update the number of time steps performed. + timeStepsPerformed++; + // Make a vector to estimate nextstate. arma::colvec currentState = {state.Theta1(), state.Theta2(), state.AngularVelocity1(), state.AngularVelocity2()}; @@ -158,19 +167,22 @@ class Acrobot nextState.Theta1() = Wrap(currentNextState[0], -M_PI, M_PI); nextState.Theta2() = Wrap(currentNextState[1], -M_PI, M_PI); - //! The value of angular velocity is bounded in min and max value. + //! The value of angular velocity is bounded in min and max value. nextState.AngularVelocity1() = std::min( std::max(currentNextState[2], -maxVel1), maxVel1); nextState.AngularVelocity2() = std::min( std::max(currentNextState[3], -maxVel2), maxVel2); - /** - * If the acrobot reaches a terminal state, it should be given a positive - * reward. This will ensure that the agent learns the goal of the game. - */ + + // Check if the episode has terminated. bool done = IsTerminal(nextState); - if (done) + + // Do not reward the agent if time ran out. + if (done && maxSteps != 0 && timeStepsPerformed >= maxSteps) + return 0; + else if (done) return doneReward; + return -1; }; @@ -183,7 +195,7 @@ class Acrobot * @param action The action taken. * @param nextState The next state. */ - double Sample(const State& state, const Action& action) const + double Sample(const State& state, const Action& action) { State nextState; return Sample(state, action, nextState); @@ -192,8 +204,9 @@ class Acrobot /** * This function does random initialization of state space. */ - State InitialSample() const + State InitialSample() { + timeStepsPerformed = 0; return State((arma::randu(4) - 0.5) / 5.0); } @@ -201,11 +214,23 @@ class Acrobot * This function checks if the acrobot has reached the terminal state. * * @param state The current State. + * @return true if state is a terminal state, otherwise false. * */ bool IsTerminal(const State& state) const { - return bool (-std::cos(state.Theta1())-std::cos(state.Theta1() + - state.Theta2()) > 1.0); + if (maxSteps != 0 && timeStepsPerformed >= maxSteps) + { + Log::Info << "Episode terminated due to the maximum number of time steps" + "being taken."; + return true; + } + else if (bool (-std::cos(state.Theta1())-std::cos(state.Theta1() + + state.Theta2()) > 1.0)) + { + Log::Info << "Episode terminated due to agent succeeding."; + return true; + } + return false; } /** @@ -295,7 +320,6 @@ class Acrobot } /** - * * This function calls the RK4 iterative method to estimate the next state * based on given ordinary differential equation. * @@ -313,6 +337,9 @@ class Acrobot return nextState; }; + //! Get the number of time steps performed + size_t TimeStepsPerformed() const { return timeStepsPerformed; } + private: //! Locally-stored gravity. double gravity; @@ -349,6 +376,12 @@ class Acrobot //! Locally-stored done reward. double doneReward; + + //! Locally-stored maximum number of time steps. + size_t maxSteps; + + //! Locally-stored number of time steps performed. + size_t timeStepsPerformed; }; // class Acrobot /** diff --git a/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp b/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp index b5f8230402..aae32002cd 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp @@ -104,7 +104,9 @@ class CartPole * @param tau The time interval. * @param thetaThresholdRadians The maximum angle. * @param xThreshold The maximum position. - * @param doneReward Reward recieved on termination. + * @param doneReward Reward recieved by agent on success. + * @param maxTimeSteps The number of time steps after which the episode + * terminates. If the value is 0, there is no limit. */ CartPole(const double gravity = 9.8, const double massCart = 1.0, @@ -114,7 +116,8 @@ class CartPole const double tau = 0.02, const double thetaThresholdRadians = 12 * 2 * 3.1416 / 360, const double xThreshold = 2.4, - const double doneReward = 0.0) : + const double doneReward = 0.0, + const size_t maxSteps = 0) : gravity(gravity), massCart(massCart), massPole(massPole), @@ -125,7 +128,9 @@ class CartPole tau(tau), thetaThresholdRadians(thetaThresholdRadians), xThreshold(xThreshold), - doneReward(doneReward) + doneReward(doneReward), + maxSteps(maxSteps), + timeStepsPerformed(0) { /* Nothing to do here */ } /** @@ -139,8 +144,11 @@ class CartPole */ double Sample(const State& state, const Action& action, - State& nextState) const + State& nextState) { + // Update the number of time steps performed. + timeStepsPerformed++; + // Calculate acceleration. double force = action ? forceMag : -forceMag; double cosTheta = std::cos(state.Angle()); @@ -157,13 +165,15 @@ class CartPole nextState.Angle() = state.Angle() + tau * state.AngularVelocity(); nextState.AngularVelocity() = state.AngularVelocity() + tau * thetaAcc; - /** - * It is important to note that if the cartpole is falling down, it should - * be penalized. - */ + // Check if the episode has terminated. bool done = IsTerminal(nextState); - if (done) + + // Do not reward agent if it failed. + if (done && maxSteps != 0 && timeStepsPerformed >= maxSteps) return doneReward; + else if (done) + return 0; + /** * When done is false, it means that the cartpole has fallen down. * For this case the reward is 1.0. @@ -179,7 +189,7 @@ class CartPole * @param action The current action. * @return reward, it's always 1.0. */ - double Sample(const State& state, const Action& action) const + double Sample(const State& state, const Action& action) { State nextState; return Sample(state, action, nextState); @@ -190,23 +200,38 @@ class CartPole * * @return Initial state for each episode. */ - State InitialSample() const + State InitialSample() { + timeStepsPerformed = 0; return State((arma::randu(4) - 0.5) / 10.0); } /** - * Whether given state is a terminal state. + * This function checks if the cart has reached the terminal state. * * @param state The desired state. * @return true if state is a terminal state, otherwise false. */ bool IsTerminal(const State& state) const { - return std::abs(state.Position()) > xThreshold || - std::abs(state.Angle()) > thetaThresholdRadians; + if (maxSteps != 0 && timeStepsPerformed >= maxSteps) + { + Log::Info << "Episode terminated due to the maximum number of time steps" + "being taken."; + return true; + } + else if (std::abs(state.Position()) > xThreshold || + std::abs(state.Angle()) > thetaThresholdRadians) + { + Log::Info << "Episode terminated due to agent failing."; + return true; + } + return false; } + //! Get the number of time steps performed + size_t TimeStepsPerformed() const { return timeStepsPerformed; } + private: //! Locally-stored gravity. double gravity; @@ -240,6 +265,12 @@ class CartPole //! Locally-stored done reward. double doneReward; + + //! Locally-stored maximum number of time steps. + size_t maxSteps; + + //! Locally-stored number of time steps performed. + size_t timeStepsPerformed; }; } // namespace rl diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp index f7fbebc7ce..9933692ef9 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp @@ -97,19 +97,27 @@ class ContinuousMountainCar * @param velocityMin Minimum legal velocity. * @param velocityMax Maximum legal velocity. * @param power Power generated by car. + * @param doneReward Reward recieved by the agent on success. + * @param maxTimeSteps The number of time steps after which the episode + * terminates. If the value is 0, there is no limit. */ ContinuousMountainCar(const double positionMin = -1.2, const double positionMax = 0.6, const double positionGoal = 0.45, const double velocityMin = -0.07, const double velocityMax = 0.07, - const double power = 0.0015) : + const double power = 0.0015, + const double doneReward = 100, + const size_t maxSteps = 0) : positionMin(positionMin), positionMax(positionMax), positionGoal(positionGoal), velocityMin(velocityMin), velocityMax(velocityMax), - power(power) + power(power), + doneReward(doneReward), + maxSteps(maxSteps), + timeStepsPerformed(0) { /* Nothing to do here */ } /** @@ -119,12 +127,14 @@ class ContinuousMountainCar * @param state The current state. * @param action The current action. * @param nextState The next state. - * @return reward, it's always -1.0. */ double Sample(const State& state, const Action& action, - State& nextState) const + State& nextState) { + // Update the number of time steps performed. + timeStepsPerformed++; + // Calculate acceleration. double force = std::min(std::max(action.action, -1.0), 1.0); @@ -139,13 +149,16 @@ class ContinuousMountainCar if (nextState.Position() == positionMin && nextState.Velocity() < 0) nextState.Velocity() = 0.0; - // Calculate reward - double reward = 0.0; - // If it is a terminal state, add a reward of 100.0 - if (IsTerminal(nextState)) - reward = 100.0; - reward -= std::pow(action.action, 2) * 0.1; - return reward; + // Check if the episode has terminated. + bool done = IsTerminal(nextState); + + // Do not reward the agent if time ran out. + if (done && maxSteps != 0 && timeStepsPerformed >= maxSteps) + return 0; + else if (done) + return doneReward; + + return std::pow(action.action, 2) * 0.1; } /** @@ -156,7 +169,7 @@ class ContinuousMountainCar * @param action The current action. * @return reward, it's always -1.0. */ - double Sample(const State& state, const Action& action) const + double Sample(const State& state, const Action& action) { State nextState; return Sample(state, action, nextState); @@ -168,9 +181,10 @@ class ContinuousMountainCar * * @return Initial state for each episode. */ - State InitialSample() const + State InitialSample() { State state; + timeStepsPerformed = 0; state.Velocity() = 0.0; state.Position() = math::Random(-0.6, -0.4); return state; @@ -184,9 +198,23 @@ class ContinuousMountainCar */ bool IsTerminal(const State& state) const { - return state.Position() >= positionGoal; + if (maxSteps != 0 && timeStepsPerformed >= maxSteps) + { + Log::Info << "Episode terminated due to the maximum number of time steps" + "being taken."; + return true; + } + else if (state.Position() >= positionGoal) + { + Log::Info << "Episode terminated due to agent succeeding."; + return true; + } + return false; } + //! Get the number of time steps performed + size_t TimeStepsPerformed() const { return timeStepsPerformed; } + private: //! Locally-stored minimum legal position. double positionMin; @@ -205,6 +233,15 @@ class ContinuousMountainCar //! Locally-stored power. double power; + + //! Locally-stored done reward. + double doneReward; + + //! Locally-stored maximum number of time steps. + size_t maxSteps; + + //! Locally-stored number of time steps performed. + size_t timeStepsPerformed; }; } // namespace rl diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp index 52e975eedf..cff0c5a5b9 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp @@ -107,7 +107,9 @@ class ContinuousMultiplePoleCart * @param tau The time interval. * @param thetaThresholdRadians The maximum angle. * @param xThreshold The maximum position. - * @param doneReward The reward recieved on termination. + * @param doneReward Reward recieved by agent on success. + * @param maxTimeSteps The number of time steps after which the episode + * terminates. If the value is 0, there is no limit. */ ContinuousMultiplePoleCart(const size_t poleNum, const arma::vec& poleLengths, @@ -118,7 +120,8 @@ class ContinuousMultiplePoleCart const double thetaThresholdRadians = 12 * 2 * 3.1416 / 360, const double xThreshold = 2.4, - const double doneReward = 0.0) : + const double doneReward = 0.0, + const size_t maxSteps = 0) : poleNum(poleNum), poleLengths(poleLengths), poleMasses(poleMasses), @@ -127,7 +130,9 @@ class ContinuousMultiplePoleCart tau(tau), thetaThresholdRadians(thetaThresholdRadians), xThreshold(xThreshold), - doneReward(doneReward) + doneReward(doneReward), + maxSteps(maxSteps), + timeStepsPerformed(0) { if (poleNum != poleLengths.n_elem) { @@ -152,7 +157,7 @@ class ContinuousMultiplePoleCart */ double Sample(const State& state, const Action& action, - State& nextState) const + State& nextState) { // Calculate acceleration. double totalForce = action.action; @@ -182,13 +187,15 @@ class ContinuousMultiplePoleCart nextState.Position() = state.Position() + tau * state.Velocity(); nextState.Velocity() = state.Velocity() + tau * xAcc; - /** - * It is important to note that if the cartpole is falling down, it should - * be penalized. - */ + // Check if the episode has terminated. bool done = IsTerminal(nextState); - if (done) + + // Do not reward agent if it failed. + if (done && maxSteps != 0 && timeStepsPerformed >= maxSteps) return doneReward; + else if (done) + return 0; + /** * When done is false, it means that the cartpole has fallen down. * For this case the reward is 1.0. @@ -204,7 +211,7 @@ class ContinuousMultiplePoleCart * @param action The current action. * @return reward, it's always 1.0. */ - double Sample(const State& state, const Action& action) const + double Sample(const State& state, const Action& action) { State nextState(poleNum); return Sample(state, action, nextState); @@ -215,25 +222,45 @@ class ContinuousMultiplePoleCart * * @return Initial state for each episode. */ - State InitialSample() const + State InitialSample() { + timeStepsPerformed = 0; return State((arma::randu(2, poleNum + 1) - 0.5) / 10.0); } /** - * Whether given state is a terminal state. + * This function checks if the cart has reached the terminal state. * * @param state The desired state. * @return true if state is a terminal state, otherwise false. */ bool IsTerminal(const State& state) const { + if (maxSteps != 0 && timeStepsPerformed >= maxSteps) + { + Log::Info << "Episode terminated due to the maximum number of time steps" + "being taken."; + return true; + } + if (std::abs(state.Position()) > xThreshold) + { + Log::Info << "Episode terminated due to cart crossing threshold"; + return true; + } for (size_t i = 1; i <= poleNum; i++) + { if (std::abs(state.Angle(i)) > thetaThresholdRadians) + { + Log::Info << "Episode terminated due to pole falling"; return true; - return std::abs(state.Position()) > xThreshold; + } + } + return false; } + //! Get the number of time steps performed + size_t TimeStepsPerformed() const { return timeStepsPerformed; } + private: //! Locally-stored number of poles. size_t poleNum; @@ -261,6 +288,12 @@ class ContinuousMultiplePoleCart //! Locally-stored done reward. double doneReward; + + //! Locally-stored maximum number of time steps. + size_t maxSteps; + + //! Locally-stored number of time steps performed. + size_t timeStepsPerformed; }; } // namespace rl diff --git a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp index 634d472937..d01a930ce1 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp @@ -43,7 +43,7 @@ class MountainCar /** * Construct a state based on the given data. * - * @param data Data for the velocityand position. + * @param data Data for the velocity and position. */ State(const arma::colvec& data): data(data) { /* Nothing to do here. */ } @@ -93,19 +93,25 @@ class MountainCar * @param positionGoal Final target position. * @param velocityMin Minimum legal velocity. * @param velocityMax Maximum legal velocity. + * @param doneReward The reward recieved by the agent on success. + * @param maxTimeSteps The number of time steps after which the episode + * terminates. If the value is 0, there is no limit. */ MountainCar(const double positionMin = -1.2, const double positionMax = 0.6, const double positionGoal = 0.5, const double velocityMin = -0.07, const double velocityMax = 0.07, - const double doneReward = 0) : + const double doneReward = 0, + const size_t maxSteps = 0) : positionMin(positionMin), positionMax(positionMax), positionGoal(positionGoal), velocityMin(velocityMin), velocityMax(velocityMax), - doneReward(doneReward) + doneReward(doneReward), + maxSteps(maxSteps), + timeStepsPerformed(0) { /* Nothing to do here */ } /** @@ -119,8 +125,11 @@ class MountainCar */ double Sample(const State& state, const Action& action, - State& nextState) const + State& nextState) { + // Update the number of time steps performed. + timeStepsPerformed++; + // Calculate acceleration. int direction = action - 1; nextState.Velocity() = state.Velocity() + 0.001 * direction - 0.0025 * @@ -136,17 +145,16 @@ class MountainCar if (nextState.Position() == positionMin && nextState.Velocity() < 0) nextState.Velocity() = 0.0; + // Check if the episode has terminated. bool done = IsTerminal(nextState); - /** - * If done is true , it means that car has reached its goal. - * To make sure that the agent learns this, we will give some - * positive reward to the agent. If the agent doesn't reach the - * terminal state, then we will give a -1.0 reward to penalize - * the agent to take that step. - */ - if (done) + + // Do not reward the agent if time ran out. + if (done && maxSteps != 0 && timeStepsPerformed >= maxSteps) + return 0; + else if (done) return doneReward; - return -1.0; + + return -1; } /** @@ -157,7 +165,7 @@ class MountainCar * @param action The current action. * @return reward, it's always -1.0. */ - double Sample(const State& state, const Action& action) const + double Sample(const State& state, const Action& action) { State nextState; return Sample(state, action, nextState); @@ -169,25 +177,40 @@ class MountainCar * * @return Initial state for each episode. */ - State InitialSample() const + State InitialSample() { State state; + timeStepsPerformed = 0; state.Velocity() = 0.0; state.Position() = arma::as_scalar(arma::randu(1)) * 0.2 - 0.6; return state; } /** - * Whether given state is a terminal state. + * This function checks if the car has reached the terminal state. * * @param state desired state. * @return true if state is a terminal state, otherwise false. */ bool IsTerminal(const State& state) const { - return state.Position() >= positionGoal; + if (maxSteps != 0 && timeStepsPerformed >= maxSteps) + { + Log::Info << "Episode terminated due to the maximum number of time steps" + "being taken."; + return true; + } + else if (state.Position() >= positionGoal) + { + Log::Info << "Episode terminated due to agent succeeding."; + return true; + } + return false; } + //! Get the number of time steps performed + size_t TimeStepsPerformed() const { return timeStepsPerformed; } + private: //! Locally-stored minimum legal position. double positionMin; @@ -206,6 +229,12 @@ class MountainCar //! Locally-stored done reward. double doneReward; + + //! Locally-stored maximum number of time steps. + size_t maxSteps; + + //! Locally-stored number of time steps performed. + size_t timeStepsPerformed; }; } // namespace rl diff --git a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp index d765f7c5e7..538105c7fd 100755 --- a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp @@ -109,7 +109,9 @@ class MultiplePoleCart * @param tau The time interval. * @param thetaThresholdRadians The maximum angle. * @param xThreshold The maximum position. - * @param doneReward The reward recieved on termination. + * @param doneReward Reward recieved by agent on success. + * @param maxTimeSteps The number of time steps after which the episode + * terminates. If the value is 0, there is no limit. */ MultiplePoleCart(const size_t poleNum, const arma::vec& poleLengths, @@ -120,7 +122,8 @@ class MultiplePoleCart const double tau = 0.02, const double thetaThresholdRadians = 12 * 2 * 3.1416 / 360, const double xThreshold = 2.4, - const double doneReward = 0.0) : + const double doneReward = 0.0, + const size_t maxSteps = 0) : poleNum(poleNum), poleLengths(poleLengths), poleMasses(poleMasses), @@ -130,7 +133,9 @@ class MultiplePoleCart tau(tau), thetaThresholdRadians(thetaThresholdRadians), xThreshold(xThreshold), - doneReward(doneReward) + doneReward(doneReward), + maxSteps(maxSteps), + timeStepsPerformed(0) { if (poleNum != poleLengths.n_elem) { @@ -155,8 +160,11 @@ class MultiplePoleCart */ double Sample(const State& state, const Action& action, - State& nextState) const + State& nextState) { + // Update the number of time steps performed. + timeStepsPerformed++; + // Calculate acceleration. double totalForce = action ? forceMag : -forceMag; double totalMass = massCart; @@ -185,13 +193,15 @@ class MultiplePoleCart nextState.Position() = state.Position() + tau * state.Velocity(); nextState.Velocity() = state.Velocity() + tau * xAcc; - /** - * It is important to note that if the cartpole is falling down, it should - * be penalized. - */ + // Check if the episode has terminated. bool done = IsTerminal(nextState); - if (done) + + // Do not reward agent if it failed. + if (done && maxSteps != 0 && timeStepsPerformed >= maxSteps) return doneReward; + else if (done) + return 0; + /** * When done is false, it means that the cartpole has fallen down. * For this case the reward is 1.0. @@ -207,7 +217,7 @@ class MultiplePoleCart * @param action The current action. * @return reward, it's always 1.0. */ - double Sample(const State& state, const Action& action) const + double Sample(const State& state, const Action& action) { State nextState(poleNum); return Sample(state, action, nextState); @@ -218,25 +228,45 @@ class MultiplePoleCart * * @return Initial state for each episode. */ - State InitialSample() const + State InitialSample() { + timeStepsPerformed = 0; return State((arma::randu(2, poleNum + 1) - 0.5) / 10.0); } /** - * Whether given state is a terminal state. + * This function checks if the car has reached the terminal state. * * @param state The desired state. * @return true if state is a terminal state, otherwise false. */ bool IsTerminal(const State& state) const { + if (maxSteps != 0 && timeStepsPerformed >= maxSteps) + { + Log::Info << "Episode terminated due to the maximum number of time steps" + "being taken."; + return true; + } + if (std::abs(state.Position()) > xThreshold) + { + Log::Info << "Episode terminated due to cart crossing threshold"; + return true; + } for (size_t i = 1; i <= poleNum; i++) + { if (std::abs(state.Angle(i)) > thetaThresholdRadians) + { + Log::Info << "Episode terminated due to pole falling"; return true; - return std::abs(state.Position()) > xThreshold; + } + } + return false; } + //! Get the number of time steps performed + size_t TimeStepsPerformed() const { return timeStepsPerformed; } + private: //! Locally-stored number of poles. size_t poleNum; @@ -267,6 +297,12 @@ class MultiplePoleCart //! Locally-stored done reward. double doneReward; + + //! Locally-stored maximum number of time steps. + size_t maxSteps; + + //! Locally-stored number of time steps performed. + size_t timeStepsPerformed; }; } // namespace rl diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index 7394e233c3..e0a2fca2ab 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -95,6 +95,7 @@ class Pendulum * @param dt The differential value. * @param angleThreshold The region about the upright position where the * state is considered terminal. + * @param doneReward The reward recieved by the agent on success. * @param maxTimeSteps The number of time steps after which the episode * terminates. If the value is 0, there is no limit. */ @@ -103,13 +104,13 @@ class Pendulum const double dt = 0.05, const double angleThreshold = M_PI / 12, const double doneReward = 0.0, - const size_t maxTimeSteps = 0) : + const size_t maxSteps = 0) : maxAngularVelocity(maxAngularVelocity), maxTorque(maxTorque), dt(dt), angleThreshold(angleThreshold), doneReward(doneReward), - maxTimeSteps(maxTimeSteps), + maxSteps(maxSteps), timeStepsPerformed(0) { /* Nothing to do here */ } @@ -158,7 +159,7 @@ class Pendulum bool done = IsTerminal(nextState); // Do not reward the agent if time ran out. - if (done && maxTimeSteps != 0 && timeStepsPerformed >= maxTimeSteps) + if (done && maxSteps != 0 && timeStepsPerformed >= maxSteps) return 0; else if (done) return doneReward; @@ -197,7 +198,7 @@ class Pendulum } /** - * This function calculates the normalized anlge for a particular theta. + * This function calculates the normalized angle for a particular theta. * * @param theta The un-normalized angle. */ @@ -208,14 +209,14 @@ class Pendulum } /** - * Whether given state is a terminal state. + * This function checks if the pendulum has reaches a terminal state * * @param state desired state. * @return true if state is a terminal state, otherwise false. */ bool IsTerminal(const State& state) const { - if (maxTimeSteps != 0 && timeStepsPerformed >= maxTimeSteps) + if (maxSteps != 0 && timeStepsPerformed >= maxSteps) { Log::Info << "Episode terminated due to the maximum number of time steps" "being taken."; @@ -230,6 +231,9 @@ class Pendulum return false; } + //! Get the number of time steps performed + size_t TimeStepsPerformed() const { return timeStepsPerformed; } + private: //! Locally-stored maximum legal angular velocity. double maxAngularVelocity; @@ -247,7 +251,7 @@ class Pendulum double doneReward; //! Locally-stored maximum number of time steps. - size_t maxTimeSteps; + size_t maxSteps; //! Locally-stored number of time steps performed. size_t timeStepsPerformed; diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 0917c0d3c9..1eda86bc79 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -59,7 +59,7 @@ BOOST_AUTO_TEST_CASE(SimplePendulumTest) */ BOOST_AUTO_TEST_CASE(SimpleContinuousMountainCarTest) { - const ContinuousMountainCar task = ContinuousMountainCar(); + ContinuousMountainCar task = ContinuousMountainCar(); ContinuousMountainCar::State state = task.InitialSample(); ContinuousMountainCar::Action action; @@ -77,7 +77,7 @@ BOOST_AUTO_TEST_CASE(SimpleContinuousMountainCarTest) */ BOOST_AUTO_TEST_CASE(SimpleAcrobotTest) { - const Acrobot task = Acrobot(); + Acrobot task = Acrobot(); Acrobot::State state = task.InitialSample(); Acrobot::Action action = Acrobot::Action::negativeTorque; @@ -94,7 +94,7 @@ BOOST_AUTO_TEST_CASE(SimpleAcrobotTest) */ BOOST_AUTO_TEST_CASE(SimpleMountainCarTest) { - const MountainCar task = MountainCar(); + MountainCar task = MountainCar(); MountainCar::State state = task.InitialSample(); MountainCar::Action action = MountainCar::Action::backward; @@ -111,7 +111,7 @@ BOOST_AUTO_TEST_CASE(SimpleMountainCarTest) */ BOOST_AUTO_TEST_CASE(SimpleCartPoleTest) { - const CartPole task = CartPole(); + CartPole task = CartPole(); CartPole::State state = task.InitialSample(); CartPole::Action action = CartPole::Action::backward; @@ -130,7 +130,7 @@ BOOST_AUTO_TEST_CASE(MultiplePoleCartTest) { arma::vec poleLengths = {1, 0.5}; arma::vec poleMasses = {1, 1}; - const MultiplePoleCart task = MultiplePoleCart(2, poleLengths, poleMasses); + MultiplePoleCart task = MultiplePoleCart(2, poleLengths, poleMasses); MultiplePoleCart::State state = task.InitialSample(); MultiplePoleCart::Action action = MultiplePoleCart::Action::backward; @@ -149,7 +149,7 @@ BOOST_AUTO_TEST_CASE(ContinuousMultiplePoleCartTest) { arma::vec poleLengths = {1, 0.5}; arma::vec poleMasses = {1, 1}; - const ContinuousMultiplePoleCart task = ContinuousMultiplePoleCart(2, + ContinuousMultiplePoleCart task = ContinuousMultiplePoleCart(2, poleLengths, poleMasses); ContinuousMultiplePoleCart::State state = task.InitialSample(); From 2b6a2d3d2bbe1737a47ae3120fb37df0e72385c7 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Fri, 28 Jun 2019 18:09:38 +0530 Subject: [PATCH 130/143] Included max steps in tasks. --- src/mlpack/tests/rl_components_test.cpp | 40 ++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 1eda86bc79..65a5f2e598 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -37,7 +37,7 @@ BOOST_AUTO_TEST_SUITE(RLComponentsTest) */ BOOST_AUTO_TEST_CASE(SimplePendulumTest) { - Pendulum task = Pendulum(); + Pendulum task = Pendulum(8, 2, 0.05, M_PI/12, 0, 5); Pendulum::State state = task.InitialSample(); Pendulum::Action action; @@ -49,6 +49,12 @@ BOOST_AUTO_TEST_CASE(SimplePendulumTest) BOOST_REQUIRE(!task.IsTerminal(state)); + while(!task.IsTerminal(state)) + task.Sample(state, action, state); + + // Check if the number of steps performed is the same as the maximum allowed. + BOOST_REQUIRE_EQUAL(task.TimeStepsPerformed(), 5); + // The action is simply the torque. Check if dimension is 1. BOOST_REQUIRE_EQUAL(1, action.size); } @@ -59,7 +65,8 @@ BOOST_AUTO_TEST_CASE(SimplePendulumTest) */ BOOST_AUTO_TEST_CASE(SimpleContinuousMountainCarTest) { - ContinuousMountainCar task = ContinuousMountainCar(); + ContinuousMountainCar task = ContinuousMountainCar(-1.2, 0.6, 0.45, -0.07, + 0.07, 0.0015, 100, 5); ContinuousMountainCar::State state = task.InitialSample(); ContinuousMountainCar::Action action; @@ -68,6 +75,14 @@ BOOST_AUTO_TEST_CASE(SimpleContinuousMountainCarTest) // Maximum reward possible is 100. BOOST_REQUIRE(reward <= 100.0); BOOST_REQUIRE(!task.IsTerminal(state)); + + while(!task.IsTerminal(state)) + task.Sample(state, action, state); + + // Check if the number of steps performed is the same as the maximum allowed. + BOOST_REQUIRE_EQUAL(task.TimeStepsPerformed(), 5); + + // Check if the size of the action space is 1. BOOST_REQUIRE_EQUAL(1, action.size); } @@ -77,7 +92,8 @@ BOOST_AUTO_TEST_CASE(SimpleContinuousMountainCarTest) */ BOOST_AUTO_TEST_CASE(SimpleAcrobotTest) { - Acrobot task = Acrobot(); + Acrobot task = Acrobot(9.81, 1, 1, 1, 1, 0.5, 0.5, 1, 4 * M_PI, 9 * M_PI, + 0.2, 0, 5); Acrobot::State state = task.InitialSample(); Acrobot::Action action = Acrobot::Action::negativeTorque; @@ -85,6 +101,14 @@ BOOST_AUTO_TEST_CASE(SimpleAcrobotTest) BOOST_REQUIRE_EQUAL(reward, -1.0); BOOST_REQUIRE(!task.IsTerminal(state)); + + while(!task.IsTerminal(state)) + task.Sample(state, action, state); + + // Check if the number of steps performed is the same as the maximum allowed. + BOOST_REQUIRE_EQUAL(task.TimeStepsPerformed(), 5); + + // Check if the size of the action space is 3. BOOST_REQUIRE_EQUAL(3, Acrobot::Action::size); } @@ -94,7 +118,7 @@ BOOST_AUTO_TEST_CASE(SimpleAcrobotTest) */ BOOST_AUTO_TEST_CASE(SimpleMountainCarTest) { - MountainCar task = MountainCar(); + MountainCar task = MountainCar(-1.2, 0.6, 0.5, -0.07, 0.07, 0, 5); MountainCar::State state = task.InitialSample(); MountainCar::Action action = MountainCar::Action::backward; @@ -102,6 +126,14 @@ BOOST_AUTO_TEST_CASE(SimpleMountainCarTest) BOOST_REQUIRE_EQUAL(reward, -1.0); BOOST_REQUIRE(!task.IsTerminal(state)); + + while(!task.IsTerminal(state)) + task.Sample(state, action, state); + + // Check if the number of steps performed is the same as the maximum allowed. + BOOST_REQUIRE_EQUAL(task.TimeStepsPerformed(), 5); + + // Check if the size of the action space is 3. BOOST_REQUIRE_EQUAL(3, MountainCar::Action::size); } From ae36a8f46b8098ac4e5838a18c8d69306eb291dd Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Fri, 28 Jun 2019 18:20:48 +0530 Subject: [PATCH 131/143] Updated HISTORY.MD and some documentation. --- HISTORY.md | 11 +++++++---- .../reinforcement_learning/environment/acrobot.hpp | 9 ++------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 0f0b3031af..5b745aab6b 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,4 +1,4 @@ -### mlpack 3.1.2 +### mlpack 4.0.0 ###### ????-??-?? * Add Multiple Pole Balancing Environment (#1901). @@ -8,9 +8,12 @@ * Fix prediction output of softmax regression when test set accuracy is calculated (#1922). - * Add `IsTerminal()` method to the pendulum environment. Action struct in - continuous RL environments now stores the action as a `double` instead - of `double[1]` (#1941, #1931). + * Action struct in continuous RL environments now stores the action as a + `double` instead of `double[1]` (#1941, #1931). + + * Pendulum environment now checks for termination. All RL environments now + have an option to terminate after a set number of time steps (no limit + by default) (#1941). ### mlpack 3.1.1 ###### 2019-05-26 diff --git a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp index 8507a30be1..4b9ab48e1e 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp @@ -105,7 +105,7 @@ class Acrobot * @param linkMass2 The mass of link 2. * @param linkCom1 The position of the center of mass of link 1. * @param linkCom2 The position of the center of mass of link 2. - * @param linkMoi The moments of inertia for both link. + * @param linkMoi The moments of inertia for both links. * @param maxVel1 The max angular velocity of link1. * @param maxVel2 The max angular velocity of link2. * @param dt The differential value. @@ -382,12 +382,7 @@ class Acrobot //! Locally-stored number of time steps performed. size_t timeStepsPerformed; -}; // class Acrobot - -/** - * Add an alias for backward compatibility. - */ -typedef Acrobot Acrobat; +}; } // namespace rl } // namespace mlpack From 7650b4d85596dc4d238307eafe3ad1a01eceaf59 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Fri, 28 Jun 2019 18:26:13 +0530 Subject: [PATCH 132/143] Style fixes. --- src/mlpack/tests/rl_components_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 65a5f2e598..b3dfa74b2a 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -76,7 +76,7 @@ BOOST_AUTO_TEST_CASE(SimpleContinuousMountainCarTest) BOOST_REQUIRE(reward <= 100.0); BOOST_REQUIRE(!task.IsTerminal(state)); - while(!task.IsTerminal(state)) + while (!task.IsTerminal(state)) task.Sample(state, action, state); // Check if the number of steps performed is the same as the maximum allowed. @@ -102,7 +102,7 @@ BOOST_AUTO_TEST_CASE(SimpleAcrobotTest) BOOST_REQUIRE_EQUAL(reward, -1.0); BOOST_REQUIRE(!task.IsTerminal(state)); - while(!task.IsTerminal(state)) + while (!task.IsTerminal(state)) task.Sample(state, action, state); // Check if the number of steps performed is the same as the maximum allowed. @@ -127,7 +127,7 @@ BOOST_AUTO_TEST_CASE(SimpleMountainCarTest) BOOST_REQUIRE_EQUAL(reward, -1.0); BOOST_REQUIRE(!task.IsTerminal(state)); - while(!task.IsTerminal(state)) + while (!task.IsTerminal(state)) task.Sample(state, action, state); // Check if the number of steps performed is the same as the maximum allowed. From a067a73efc528dd44655685cd263a088056eb1df Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Fri, 28 Jun 2019 18:27:10 +0530 Subject: [PATCH 133/143] Oops missed a space. --- src/mlpack/tests/rl_components_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index b3dfa74b2a..bd5879ae91 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -49,7 +49,7 @@ BOOST_AUTO_TEST_CASE(SimplePendulumTest) BOOST_REQUIRE(!task.IsTerminal(state)); - while(!task.IsTerminal(state)) + while (!task.IsTerminal(state)) task.Sample(state, action, state); // Check if the number of steps performed is the same as the maximum allowed. From 7b25daf89d854bbfda1da7c67d23703f6f1b4e84 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Fri, 28 Jun 2019 19:31:40 +0530 Subject: [PATCH 134/143] Fixed continuous multiple pole cart --- .../continuous_multiple_pole_cart.hpp | 3 ++ src/mlpack/tests/rl_components_test.cpp | 31 ++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp index cff0c5a5b9..7179893aad 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp @@ -159,6 +159,9 @@ class ContinuousMultiplePoleCart const Action& action, State& nextState) { + // Update the number of time steps performed. + timeStepsPerformed++; + // Calculate acceleration. double totalForce = action.action; double totalMass = massCart; diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index bd5879ae91..a753a752bc 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -143,7 +143,8 @@ BOOST_AUTO_TEST_CASE(SimpleMountainCarTest) */ BOOST_AUTO_TEST_CASE(SimpleCartPoleTest) { - CartPole task = CartPole(); + CartPole task = CartPole(9.8, 1, 0.1, 0.5, 10, 0.02, 12 * 2 * 3.1416 / 360, + 2.4, 0, 5); CartPole::State state = task.InitialSample(); CartPole::Action action = CartPole::Action::backward; @@ -151,6 +152,13 @@ BOOST_AUTO_TEST_CASE(SimpleCartPoleTest) BOOST_REQUIRE_EQUAL(reward, 1.0); BOOST_REQUIRE(!task.IsTerminal(state)); + + while (!task.IsTerminal(state)) + task.Sample(state, action, state); + + // Check if the number of steps performed is the same as the maximum allowed. + BOOST_REQUIRE_EQUAL(task.TimeStepsPerformed(), 5); + BOOST_REQUIRE_EQUAL(2, CartPole::Action::size); } @@ -162,7 +170,8 @@ BOOST_AUTO_TEST_CASE(MultiplePoleCartTest) { arma::vec poleLengths = {1, 0.5}; arma::vec poleMasses = {1, 1}; - MultiplePoleCart task = MultiplePoleCart(2, poleLengths, poleMasses); + MultiplePoleCart task = MultiplePoleCart(2, poleLengths, poleMasses, 9.8, 1, + 10, 0.02, 12 * 2 * 3.1416 / 360, 2.4, 0, 5); MultiplePoleCart::State state = task.InitialSample(); MultiplePoleCart::Action action = MultiplePoleCart::Action::backward; @@ -170,6 +179,12 @@ BOOST_AUTO_TEST_CASE(MultiplePoleCartTest) BOOST_REQUIRE_EQUAL(reward, 1.0); BOOST_REQUIRE(!task.IsTerminal(state)); + + while (!task.IsTerminal(state)) + task.Sample(state, action, state); + + // Check if the number of steps performed is the same as the maximum allowed. + BOOST_REQUIRE_EQUAL(task.TimeStepsPerformed(), 5); BOOST_REQUIRE_EQUAL(2, MultiplePoleCart::Action::size); } @@ -179,10 +194,12 @@ BOOST_AUTO_TEST_CASE(MultiplePoleCartTest) */ BOOST_AUTO_TEST_CASE(ContinuousMultiplePoleCartTest) { + arma::arma_rng::set_seed_random(); + arma::vec poleLengths = {1, 0.5}; arma::vec poleMasses = {1, 1}; - ContinuousMultiplePoleCart task = ContinuousMultiplePoleCart(2, - poleLengths, poleMasses); + ContinuousMultiplePoleCart task = ContinuousMultiplePoleCart(2, poleLengths, + poleMasses, 9.8, 1, 0.02, 12 * 2 * 3.1416 / 360, 2.4, 0, 5); ContinuousMultiplePoleCart::State state = task.InitialSample(); ContinuousMultiplePoleCart::Action action; @@ -191,6 +208,12 @@ BOOST_AUTO_TEST_CASE(ContinuousMultiplePoleCartTest) BOOST_REQUIRE_EQUAL(reward, 1.0); BOOST_REQUIRE(!task.IsTerminal(state)); + + while (!task.IsTerminal(state)) + task.Sample(state, action, state); + + // Check if the number of steps performed is the same as the maximum allowed. + BOOST_REQUIRE_EQUAL(task.TimeStepsPerformed(), 5); BOOST_REQUIRE_EQUAL(1, action.size); } From 81840f82d13eaa2cad131862cb4f6d53097c8501 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Sun, 30 Jun 2019 16:00:20 +0530 Subject: [PATCH 135/143] Incorporated Marcus' comments. --- .../environment/acrobot.hpp | 37 ++++++++++-------- .../environment/cart_pole.hpp | 31 ++++++++------- .../environment/continuous_mountain_car.hpp | 31 ++++++++------- .../continuous_multiple_pole_cart.hpp | 31 ++++++++------- .../environment/mountain_car.hpp | 31 ++++++++------- .../environment/multiple_pole_cart.hpp | 31 ++++++++------- .../environment/pendulum.hpp | 29 ++++++++------ src/mlpack/tests/rl_components_test.cpp | 39 ++++++++++--------- 8 files changed, 149 insertions(+), 111 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp index 4b9ab48e1e..de10cdd2bc 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp @@ -110,7 +110,7 @@ class Acrobot * @param maxVel2 The max angular velocity of link2. * @param dt The differential value. * @param doneReward The reward recieved by the agent on success. - * @param maxTimeSteps The number of time steps after which the episode + * @param maxSteps The number of steps after which the episode * terminates. If the value is 0, there is no limit. */ Acrobot(const double gravity = 9.81, @@ -139,7 +139,7 @@ class Acrobot dt(dt), doneReward(doneReward), maxSteps(maxSteps), - timeStepsPerformed(0) + stepsPerformed(0) { /* Nothing to do here */ } /** @@ -155,8 +155,8 @@ class Acrobot const Action& action, State& nextState) { - // Update the number of time steps performed. - timeStepsPerformed++; + // Update the number of steps performed. + stepsPerformed++; // Make a vector to estimate nextstate. arma::colvec currentState = {state.Theta1(), state.Theta2(), @@ -178,7 +178,7 @@ class Acrobot bool done = IsTerminal(nextState); // Do not reward the agent if time ran out. - if (done && maxSteps != 0 && timeStepsPerformed >= maxSteps) + if (done && maxSteps != 0 && stepsPerformed >= maxSteps) return 0; else if (done) return doneReward; @@ -206,7 +206,7 @@ class Acrobot */ State InitialSample() { - timeStepsPerformed = 0; + stepsPerformed = 0; return State((arma::randu(4) - 0.5) / 5.0); } @@ -214,18 +214,18 @@ class Acrobot * This function checks if the acrobot has reached the terminal state. * * @param state The current State. - * @return true if state is a terminal state, otherwise false. * + * @return true if state is a terminal state, otherwise false. */ bool IsTerminal(const State& state) const { - if (maxSteps != 0 && timeStepsPerformed >= maxSteps) + if (maxSteps != 0 && stepsPerformed >= maxSteps) { - Log::Info << "Episode terminated due to the maximum number of time steps" + Log::Info << "Episode terminated due to the maximum number of steps" "being taken."; return true; } - else if (bool (-std::cos(state.Theta1())-std::cos(state.Theta1() + - state.Theta2()) > 1.0)) + else if (-std::cos(state.Theta1())-std::cos(state.Theta1() + + state.Theta2()) > 1.0) { Log::Info << "Episode terminated due to agent succeeding."; return true; @@ -337,8 +337,13 @@ class Acrobot return nextState; }; - //! Get the number of time steps performed - size_t TimeStepsPerformed() const { return timeStepsPerformed; } + //! Get the number of steps performed. + size_t StepsPerformed() const { return stepsPerformed; } + + //! Get the maximum number of steps allowed. + size_t MaxSteps() const { return maxSteps; } + //! Set the maximum number of steps allowed. + size_t& MaxSteps() { return maxSteps; } private: //! Locally-stored gravity. @@ -377,11 +382,11 @@ class Acrobot //! Locally-stored done reward. double doneReward; - //! Locally-stored maximum number of time steps. + //! Locally-stored maximum number of steps. size_t maxSteps; - //! Locally-stored number of time steps performed. - size_t timeStepsPerformed; + //! Locally-stored number of steps performed. + size_t stepsPerformed; }; } // namespace rl diff --git a/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp b/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp index aae32002cd..1ef2b6fc6b 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp @@ -105,7 +105,7 @@ class CartPole * @param thetaThresholdRadians The maximum angle. * @param xThreshold The maximum position. * @param doneReward Reward recieved by agent on success. - * @param maxTimeSteps The number of time steps after which the episode + * @param maxSteps The number of steps after which the episode * terminates. If the value is 0, there is no limit. */ CartPole(const double gravity = 9.8, @@ -130,7 +130,7 @@ class CartPole xThreshold(xThreshold), doneReward(doneReward), maxSteps(maxSteps), - timeStepsPerformed(0) + stepsPerformed(0) { /* Nothing to do here */ } /** @@ -146,8 +146,8 @@ class CartPole const Action& action, State& nextState) { - // Update the number of time steps performed. - timeStepsPerformed++; + // Update the number of steps performed. + stepsPerformed++; // Calculate acceleration. double force = action ? forceMag : -forceMag; @@ -169,7 +169,7 @@ class CartPole bool done = IsTerminal(nextState); // Do not reward agent if it failed. - if (done && maxSteps != 0 && timeStepsPerformed >= maxSteps) + if (done && maxSteps != 0 && stepsPerformed >= maxSteps) return doneReward; else if (done) return 0; @@ -202,7 +202,7 @@ class CartPole */ State InitialSample() { - timeStepsPerformed = 0; + stepsPerformed = 0; return State((arma::randu(4) - 0.5) / 10.0); } @@ -214,9 +214,9 @@ class CartPole */ bool IsTerminal(const State& state) const { - if (maxSteps != 0 && timeStepsPerformed >= maxSteps) + if (maxSteps != 0 && stepsPerformed >= maxSteps) { - Log::Info << "Episode terminated due to the maximum number of time steps" + Log::Info << "Episode terminated due to the maximum number of steps" "being taken."; return true; } @@ -229,8 +229,13 @@ class CartPole return false; } - //! Get the number of time steps performed - size_t TimeStepsPerformed() const { return timeStepsPerformed; } + //! Get the number of steps performed. + size_t StepsPerformed() const { return stepsPerformed; } + + //! Get the maximum number of steps allowed. + size_t MaxSteps() const { return maxSteps; } + //! Set the maximum number of steps allowed. + size_t& MaxSteps() { return maxSteps; } private: //! Locally-stored gravity. @@ -266,11 +271,11 @@ class CartPole //! Locally-stored done reward. double doneReward; - //! Locally-stored maximum number of time steps. + //! Locally-stored maximum number of steps. size_t maxSteps; - //! Locally-stored number of time steps performed. - size_t timeStepsPerformed; + //! Locally-stored number of steps performed. + size_t stepsPerformed; }; } // namespace rl diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp index 9933692ef9..711c63d461 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp @@ -98,7 +98,7 @@ class ContinuousMountainCar * @param velocityMax Maximum legal velocity. * @param power Power generated by car. * @param doneReward Reward recieved by the agent on success. - * @param maxTimeSteps The number of time steps after which the episode + * @param maxSteps The number of steps after which the episode * terminates. If the value is 0, there is no limit. */ ContinuousMountainCar(const double positionMin = -1.2, @@ -117,7 +117,7 @@ class ContinuousMountainCar power(power), doneReward(doneReward), maxSteps(maxSteps), - timeStepsPerformed(0) + stepsPerformed(0) { /* Nothing to do here */ } /** @@ -132,8 +132,8 @@ class ContinuousMountainCar const Action& action, State& nextState) { - // Update the number of time steps performed. - timeStepsPerformed++; + // Update the number of steps performed. + stepsPerformed++; // Calculate acceleration. double force = std::min(std::max(action.action, -1.0), 1.0); @@ -153,7 +153,7 @@ class ContinuousMountainCar bool done = IsTerminal(nextState); // Do not reward the agent if time ran out. - if (done && maxSteps != 0 && timeStepsPerformed >= maxSteps) + if (done && maxSteps != 0 && stepsPerformed >= maxSteps) return 0; else if (done) return doneReward; @@ -184,7 +184,7 @@ class ContinuousMountainCar State InitialSample() { State state; - timeStepsPerformed = 0; + stepsPerformed = 0; state.Velocity() = 0.0; state.Position() = math::Random(-0.6, -0.4); return state; @@ -198,9 +198,9 @@ class ContinuousMountainCar */ bool IsTerminal(const State& state) const { - if (maxSteps != 0 && timeStepsPerformed >= maxSteps) + if (maxSteps != 0 && stepsPerformed >= maxSteps) { - Log::Info << "Episode terminated due to the maximum number of time steps" + Log::Info << "Episode terminated due to the maximum number of steps" "being taken."; return true; } @@ -212,8 +212,13 @@ class ContinuousMountainCar return false; } - //! Get the number of time steps performed - size_t TimeStepsPerformed() const { return timeStepsPerformed; } + //! Get the number of steps performed. + size_t StepsPerformed() const { return stepsPerformed; } + + //! Get the maximum number of steps allowed. + size_t MaxSteps() const { return maxSteps; } + //! Set the maximum number of steps allowed. + size_t& MaxSteps() { return maxSteps; } private: //! Locally-stored minimum legal position. @@ -237,11 +242,11 @@ class ContinuousMountainCar //! Locally-stored done reward. double doneReward; - //! Locally-stored maximum number of time steps. + //! Locally-stored maximum number of steps. size_t maxSteps; - //! Locally-stored number of time steps performed. - size_t timeStepsPerformed; + //! Locally-stored number of steps performed. + size_t stepsPerformed; }; } // namespace rl diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp index 7179893aad..4564bf4162 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_multiple_pole_cart.hpp @@ -108,7 +108,7 @@ class ContinuousMultiplePoleCart * @param thetaThresholdRadians The maximum angle. * @param xThreshold The maximum position. * @param doneReward Reward recieved by agent on success. - * @param maxTimeSteps The number of time steps after which the episode + * @param maxSteps The number of steps after which the episode * terminates. If the value is 0, there is no limit. */ ContinuousMultiplePoleCart(const size_t poleNum, @@ -132,7 +132,7 @@ class ContinuousMultiplePoleCart xThreshold(xThreshold), doneReward(doneReward), maxSteps(maxSteps), - timeStepsPerformed(0) + stepsPerformed(0) { if (poleNum != poleLengths.n_elem) { @@ -159,8 +159,8 @@ class ContinuousMultiplePoleCart const Action& action, State& nextState) { - // Update the number of time steps performed. - timeStepsPerformed++; + // Update the number of steps performed. + stepsPerformed++; // Calculate acceleration. double totalForce = action.action; @@ -194,7 +194,7 @@ class ContinuousMultiplePoleCart bool done = IsTerminal(nextState); // Do not reward agent if it failed. - if (done && maxSteps != 0 && timeStepsPerformed >= maxSteps) + if (done && maxSteps != 0 && stepsPerformed >= maxSteps) return doneReward; else if (done) return 0; @@ -227,7 +227,7 @@ class ContinuousMultiplePoleCart */ State InitialSample() { - timeStepsPerformed = 0; + stepsPerformed = 0; return State((arma::randu(2, poleNum + 1) - 0.5) / 10.0); } @@ -239,9 +239,9 @@ class ContinuousMultiplePoleCart */ bool IsTerminal(const State& state) const { - if (maxSteps != 0 && timeStepsPerformed >= maxSteps) + if (maxSteps != 0 && stepsPerformed >= maxSteps) { - Log::Info << "Episode terminated due to the maximum number of time steps" + Log::Info << "Episode terminated due to the maximum number of steps" "being taken."; return true; } @@ -261,8 +261,13 @@ class ContinuousMultiplePoleCart return false; } - //! Get the number of time steps performed - size_t TimeStepsPerformed() const { return timeStepsPerformed; } + //! Get the number of steps performed. + size_t StepsPerformed() const { return stepsPerformed; } + + //! Get the maximum number of steps allowed. + size_t MaxSteps() const { return maxSteps; } + //! Set the maximum number of steps allowed. + size_t& MaxSteps() { return maxSteps; } private: //! Locally-stored number of poles. @@ -292,11 +297,11 @@ class ContinuousMultiplePoleCart //! Locally-stored done reward. double doneReward; - //! Locally-stored maximum number of time steps. + //! Locally-stored maximum number of steps. size_t maxSteps; - //! Locally-stored number of time steps performed. - size_t timeStepsPerformed; + //! Locally-stored number of steps performed. + size_t stepsPerformed; }; } // namespace rl diff --git a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp index d01a930ce1..b02b631cd4 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp @@ -94,7 +94,7 @@ class MountainCar * @param velocityMin Minimum legal velocity. * @param velocityMax Maximum legal velocity. * @param doneReward The reward recieved by the agent on success. - * @param maxTimeSteps The number of time steps after which the episode + * @param maxSteps The number of steps after which the episode * terminates. If the value is 0, there is no limit. */ MountainCar(const double positionMin = -1.2, @@ -111,7 +111,7 @@ class MountainCar velocityMax(velocityMax), doneReward(doneReward), maxSteps(maxSteps), - timeStepsPerformed(0) + stepsPerformed(0) { /* Nothing to do here */ } /** @@ -127,8 +127,8 @@ class MountainCar const Action& action, State& nextState) { - // Update the number of time steps performed. - timeStepsPerformed++; + // Update the number of steps performed. + stepsPerformed++; // Calculate acceleration. int direction = action - 1; @@ -149,7 +149,7 @@ class MountainCar bool done = IsTerminal(nextState); // Do not reward the agent if time ran out. - if (done && maxSteps != 0 && timeStepsPerformed >= maxSteps) + if (done && maxSteps != 0 && stepsPerformed >= maxSteps) return 0; else if (done) return doneReward; @@ -180,7 +180,7 @@ class MountainCar State InitialSample() { State state; - timeStepsPerformed = 0; + stepsPerformed = 0; state.Velocity() = 0.0; state.Position() = arma::as_scalar(arma::randu(1)) * 0.2 - 0.6; return state; @@ -194,9 +194,9 @@ class MountainCar */ bool IsTerminal(const State& state) const { - if (maxSteps != 0 && timeStepsPerformed >= maxSteps) + if (maxSteps != 0 && stepsPerformed >= maxSteps) { - Log::Info << "Episode terminated due to the maximum number of time steps" + Log::Info << "Episode terminated due to the maximum number of steps" "being taken."; return true; } @@ -208,8 +208,13 @@ class MountainCar return false; } - //! Get the number of time steps performed - size_t TimeStepsPerformed() const { return timeStepsPerformed; } + //! Get the number of steps performed. + size_t StepsPerformed() const { return stepsPerformed; } + + //! Get the maximum number of steps allowed. + size_t MaxSteps() const { return maxSteps; } + //! Set the maximum number of steps allowed. + size_t& MaxSteps() { return maxSteps; } private: //! Locally-stored minimum legal position. @@ -230,11 +235,11 @@ class MountainCar //! Locally-stored done reward. double doneReward; - //! Locally-stored maximum number of time steps. + //! Locally-stored maximum number of steps. size_t maxSteps; - //! Locally-stored number of time steps performed. - size_t timeStepsPerformed; + //! Locally-stored number of steps performed. + size_t stepsPerformed; }; } // namespace rl diff --git a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp index 538105c7fd..31f7415500 100755 --- a/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/multiple_pole_cart.hpp @@ -110,7 +110,7 @@ class MultiplePoleCart * @param thetaThresholdRadians The maximum angle. * @param xThreshold The maximum position. * @param doneReward Reward recieved by agent on success. - * @param maxTimeSteps The number of time steps after which the episode + * @param maxSteps The number of steps after which the episode * terminates. If the value is 0, there is no limit. */ MultiplePoleCart(const size_t poleNum, @@ -135,7 +135,7 @@ class MultiplePoleCart xThreshold(xThreshold), doneReward(doneReward), maxSteps(maxSteps), - timeStepsPerformed(0) + stepsPerformed(0) { if (poleNum != poleLengths.n_elem) { @@ -162,8 +162,8 @@ class MultiplePoleCart const Action& action, State& nextState) { - // Update the number of time steps performed. - timeStepsPerformed++; + // Update the number of steps performed. + stepsPerformed++; // Calculate acceleration. double totalForce = action ? forceMag : -forceMag; @@ -197,7 +197,7 @@ class MultiplePoleCart bool done = IsTerminal(nextState); // Do not reward agent if it failed. - if (done && maxSteps != 0 && timeStepsPerformed >= maxSteps) + if (done && maxSteps != 0 && stepsPerformed >= maxSteps) return doneReward; else if (done) return 0; @@ -230,7 +230,7 @@ class MultiplePoleCart */ State InitialSample() { - timeStepsPerformed = 0; + stepsPerformed = 0; return State((arma::randu(2, poleNum + 1) - 0.5) / 10.0); } @@ -242,9 +242,9 @@ class MultiplePoleCart */ bool IsTerminal(const State& state) const { - if (maxSteps != 0 && timeStepsPerformed >= maxSteps) + if (maxSteps != 0 && stepsPerformed >= maxSteps) { - Log::Info << "Episode terminated due to the maximum number of time steps" + Log::Info << "Episode terminated due to the maximum number of steps" "being taken."; return true; } @@ -264,8 +264,13 @@ class MultiplePoleCart return false; } - //! Get the number of time steps performed - size_t TimeStepsPerformed() const { return timeStepsPerformed; } + //! Get the number of steps performed. + size_t StepsPerformed() const { return stepsPerformed; } + + //! Get the maximum number of steps allowed. + size_t MaxSteps() const { return maxSteps; } + //! Set the maximum number of steps allowed. + size_t& MaxSteps() { return maxSteps; } private: //! Locally-stored number of poles. @@ -298,11 +303,11 @@ class MultiplePoleCart //! Locally-stored done reward. double doneReward; - //! Locally-stored maximum number of time steps. + //! Locally-stored maximum number of steps. size_t maxSteps; - //! Locally-stored number of time steps performed. - size_t timeStepsPerformed; + //! Locally-stored number of steps performed. + size_t stepsPerformed; }; } // namespace rl diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index e0a2fca2ab..e71f9809ce 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -96,7 +96,7 @@ class Pendulum * @param angleThreshold The region about the upright position where the * state is considered terminal. * @param doneReward The reward recieved by the agent on success. - * @param maxTimeSteps The number of time steps after which the episode + * @param maxSteps The number of steps after which the episode * terminates. If the value is 0, there is no limit. */ Pendulum(const double maxAngularVelocity = 8, @@ -111,7 +111,7 @@ class Pendulum angleThreshold(angleThreshold), doneReward(doneReward), maxSteps(maxSteps), - timeStepsPerformed(0) + stepsPerformed(0) { /* Nothing to do here */ } /** @@ -127,8 +127,8 @@ class Pendulum const Action& action, State& nextState) { - // Update the number of time steps performed. - timeStepsPerformed++; + // Update the number of steps performed. + stepsPerformed++; // Get current state. double theta = state.Theta(); @@ -159,7 +159,7 @@ class Pendulum bool done = IsTerminal(nextState); // Do not reward the agent if time ran out. - if (done && maxSteps != 0 && timeStepsPerformed >= maxSteps) + if (done && maxSteps != 0 && stepsPerformed >= maxSteps) return 0; else if (done) return doneReward; @@ -193,7 +193,7 @@ class Pendulum State state; state.Theta() = math::Random(-M_PI + angleThreshold, M_PI - angleThreshold); state.AngularVelocity() = math::Random(-1.0, 1.0); - timeStepsPerformed = 0; + stepsPerformed = 0; return state; } @@ -216,7 +216,7 @@ class Pendulum */ bool IsTerminal(const State& state) const { - if (maxSteps != 0 && timeStepsPerformed >= maxSteps) + if (maxSteps != 0 && stepsPerformed >= maxSteps) { Log::Info << "Episode terminated due to the maximum number of time steps" "being taken."; @@ -231,8 +231,13 @@ class Pendulum return false; } - //! Get the number of time steps performed - size_t TimeStepsPerformed() const { return timeStepsPerformed; } + //! Get the number of steps performed. + size_t StepsPerformed() const { return stepsPerformed; } + + //! Get the maximum number of steps allowed. + size_t MaxSteps() const { return maxSteps; } + //! Set the maximum number of steps allowed. + size_t& MaxSteps() { return maxSteps; } private: //! Locally-stored maximum legal angular velocity. @@ -250,11 +255,11 @@ class Pendulum //! Locally-stored done reward. double doneReward; - //! Locally-stored maximum number of time steps. + //! Locally-stored maximum number of steps. size_t maxSteps; - //! Locally-stored number of time steps performed. - size_t timeStepsPerformed; + //! Locally-stored number of steps performed. + size_t stepsPerformed; }; } // namespace rl diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index a753a752bc..c7fbb940cc 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -37,7 +37,8 @@ BOOST_AUTO_TEST_SUITE(RLComponentsTest) */ BOOST_AUTO_TEST_CASE(SimplePendulumTest) { - Pendulum task = Pendulum(8, 2, 0.05, M_PI/12, 0, 5); + Pendulum task = Pendulum(); + task.MaxSteps() = 5; Pendulum::State state = task.InitialSample(); Pendulum::Action action; @@ -53,7 +54,7 @@ BOOST_AUTO_TEST_CASE(SimplePendulumTest) task.Sample(state, action, state); // Check if the number of steps performed is the same as the maximum allowed. - BOOST_REQUIRE_EQUAL(task.TimeStepsPerformed(), 5); + BOOST_REQUIRE_EQUAL(task.StepsPerformed(), 5); // The action is simply the torque. Check if dimension is 1. BOOST_REQUIRE_EQUAL(1, action.size); @@ -65,8 +66,8 @@ BOOST_AUTO_TEST_CASE(SimplePendulumTest) */ BOOST_AUTO_TEST_CASE(SimpleContinuousMountainCarTest) { - ContinuousMountainCar task = ContinuousMountainCar(-1.2, 0.6, 0.45, -0.07, - 0.07, 0.0015, 100, 5); + ContinuousMountainCar task = ContinuousMountainCar(); + task.MaxSteps() = 5; ContinuousMountainCar::State state = task.InitialSample(); ContinuousMountainCar::Action action; @@ -80,7 +81,7 @@ BOOST_AUTO_TEST_CASE(SimpleContinuousMountainCarTest) task.Sample(state, action, state); // Check if the number of steps performed is the same as the maximum allowed. - BOOST_REQUIRE_EQUAL(task.TimeStepsPerformed(), 5); + BOOST_REQUIRE_EQUAL(task.StepsPerformed(), 5); // Check if the size of the action space is 1. BOOST_REQUIRE_EQUAL(1, action.size); @@ -92,8 +93,8 @@ BOOST_AUTO_TEST_CASE(SimpleContinuousMountainCarTest) */ BOOST_AUTO_TEST_CASE(SimpleAcrobotTest) { - Acrobot task = Acrobot(9.81, 1, 1, 1, 1, 0.5, 0.5, 1, 4 * M_PI, 9 * M_PI, - 0.2, 0, 5); + Acrobot task = Acrobot(); + task.MaxSteps() = 5; Acrobot::State state = task.InitialSample(); Acrobot::Action action = Acrobot::Action::negativeTorque; @@ -106,7 +107,7 @@ BOOST_AUTO_TEST_CASE(SimpleAcrobotTest) task.Sample(state, action, state); // Check if the number of steps performed is the same as the maximum allowed. - BOOST_REQUIRE_EQUAL(task.TimeStepsPerformed(), 5); + BOOST_REQUIRE_EQUAL(task.StepsPerformed(), 5); // Check if the size of the action space is 3. BOOST_REQUIRE_EQUAL(3, Acrobot::Action::size); @@ -118,7 +119,8 @@ BOOST_AUTO_TEST_CASE(SimpleAcrobotTest) */ BOOST_AUTO_TEST_CASE(SimpleMountainCarTest) { - MountainCar task = MountainCar(-1.2, 0.6, 0.5, -0.07, 0.07, 0, 5); + MountainCar task = MountainCar(); + task.MaxSteps() = 5; MountainCar::State state = task.InitialSample(); MountainCar::Action action = MountainCar::Action::backward; @@ -131,7 +133,7 @@ BOOST_AUTO_TEST_CASE(SimpleMountainCarTest) task.Sample(state, action, state); // Check if the number of steps performed is the same as the maximum allowed. - BOOST_REQUIRE_EQUAL(task.TimeStepsPerformed(), 5); + BOOST_REQUIRE_EQUAL(task.StepsPerformed(), 5); // Check if the size of the action space is 3. BOOST_REQUIRE_EQUAL(3, MountainCar::Action::size); @@ -143,8 +145,8 @@ BOOST_AUTO_TEST_CASE(SimpleMountainCarTest) */ BOOST_AUTO_TEST_CASE(SimpleCartPoleTest) { - CartPole task = CartPole(9.8, 1, 0.1, 0.5, 10, 0.02, 12 * 2 * 3.1416 / 360, - 2.4, 0, 5); + CartPole task = CartPole(); + task.MaxSteps() = 5; CartPole::State state = task.InitialSample(); CartPole::Action action = CartPole::Action::backward; @@ -157,7 +159,7 @@ BOOST_AUTO_TEST_CASE(SimpleCartPoleTest) task.Sample(state, action, state); // Check if the number of steps performed is the same as the maximum allowed. - BOOST_REQUIRE_EQUAL(task.TimeStepsPerformed(), 5); + BOOST_REQUIRE_EQUAL(task.StepsPerformed(), 5); BOOST_REQUIRE_EQUAL(2, CartPole::Action::size); } @@ -170,8 +172,8 @@ BOOST_AUTO_TEST_CASE(MultiplePoleCartTest) { arma::vec poleLengths = {1, 0.5}; arma::vec poleMasses = {1, 1}; - MultiplePoleCart task = MultiplePoleCart(2, poleLengths, poleMasses, 9.8, 1, - 10, 0.02, 12 * 2 * 3.1416 / 360, 2.4, 0, 5); + MultiplePoleCart task = MultiplePoleCart(2, poleLengths, poleMasses); + task.MaxSteps() = 5; MultiplePoleCart::State state = task.InitialSample(); MultiplePoleCart::Action action = MultiplePoleCart::Action::backward; @@ -184,7 +186,7 @@ BOOST_AUTO_TEST_CASE(MultiplePoleCartTest) task.Sample(state, action, state); // Check if the number of steps performed is the same as the maximum allowed. - BOOST_REQUIRE_EQUAL(task.TimeStepsPerformed(), 5); + BOOST_REQUIRE_EQUAL(task.StepsPerformed(), 5); BOOST_REQUIRE_EQUAL(2, MultiplePoleCart::Action::size); } @@ -199,7 +201,8 @@ BOOST_AUTO_TEST_CASE(ContinuousMultiplePoleCartTest) arma::vec poleLengths = {1, 0.5}; arma::vec poleMasses = {1, 1}; ContinuousMultiplePoleCart task = ContinuousMultiplePoleCart(2, poleLengths, - poleMasses, 9.8, 1, 0.02, 12 * 2 * 3.1416 / 360, 2.4, 0, 5); + poleMasses); + task.MaxSteps() = 5; ContinuousMultiplePoleCart::State state = task.InitialSample(); ContinuousMultiplePoleCart::Action action; @@ -213,7 +216,7 @@ BOOST_AUTO_TEST_CASE(ContinuousMultiplePoleCartTest) task.Sample(state, action, state); // Check if the number of steps performed is the same as the maximum allowed. - BOOST_REQUIRE_EQUAL(task.TimeStepsPerformed(), 5); + BOOST_REQUIRE_EQUAL(task.StepsPerformed(), 5); BOOST_REQUIRE_EQUAL(1, action.size); } From c1877824c59584ea4ec0acd6e5063fa0489c7f67 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Sun, 30 Jun 2019 16:02:42 +0530 Subject: [PATCH 136/143] Missed a comment. --- .../methods/reinforcement_learning/environment/pendulum.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index e71f9809ce..4023d3b9bf 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -218,7 +218,7 @@ class Pendulum { if (maxSteps != 0 && stepsPerformed >= maxSteps) { - Log::Info << "Episode terminated due to the maximum number of time steps" + Log::Info << "Episode terminated due to the maximum number of steps" "being taken."; return true; } From d894f9565a168ff12e64456333fd7a6f56e3bf9a Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Sun, 30 Jun 2019 16:05:31 +0530 Subject: [PATCH 137/143] Style fix. --- .../methods/reinforcement_learning/environment/acrobot.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp index de10cdd2bc..62bb0633fc 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp @@ -224,7 +224,7 @@ class Acrobot "being taken."; return true; } - else if (-std::cos(state.Theta1())-std::cos(state.Theta1() + + else if (-std::cos(state.Theta1()) - std::cos(state.Theta1() + state.Theta2()) > 1.0) { Log::Info << "Episode terminated due to agent succeeding."; From 244fcaf067200c817349108ddd8d2ca69d003f8d Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Sun, 30 Jun 2019 16:22:11 +0530 Subject: [PATCH 138/143] Style fix. --- src/mlpack/tests/rl_components_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index c7fbb940cc..67891ee416 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -94,7 +94,7 @@ BOOST_AUTO_TEST_CASE(SimpleContinuousMountainCarTest) BOOST_AUTO_TEST_CASE(SimpleAcrobotTest) { Acrobot task = Acrobot(); - task.MaxSteps() = 5; + task.MaxSteps() = 5; Acrobot::State state = task.InitialSample(); Acrobot::Action action = Acrobot::Action::negativeTorque; From 7cb700d80504b5b6ba5f1fd71adc995c3779b6e1 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sun, 30 Jun 2019 12:58:22 -0400 Subject: [PATCH 139/143] Fix documentation error. --- doc/guide/python_quickstart.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/guide/python_quickstart.hpp b/doc/guide/python_quickstart.hpp index 59b244b81c..5d21708d71 100644 --- a/doc/guide/python_quickstart.hpp +++ b/doc/guide/python_quickstart.hpp @@ -17,7 +17,7 @@ Installing the mlpack bindings for Python is straightforward. It's easy to use conda or pip to do this: @code{.sh} -pip install mlpack/mlpack3 +pip install mlpack3 @endcode @code{.sh} From 88d8c4fcd2690d3140ff5a8efc88e4c0ec132f3e Mon Sep 17 00:00:00 2001 From: Yashwant Date: Wed, 3 Jul 2019 20:37:12 +0530 Subject: [PATCH 140/143] Resolve Some python-test --- src/mlpack/bindings/python/setup.py.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/bindings/python/setup.py.in b/src/mlpack/bindings/python/setup.py.in index 2263a7041d..b04f60c2f0 100644 --- a/src/mlpack/bindings/python/setup.py.in +++ b/src/mlpack/bindings/python/setup.py.in @@ -108,5 +108,5 @@ setup(name='mlpack', cmdclass={ 'build_ext': build_ext }, ext_modules = modules, setup_requires=['cython', 'pytest-runner'], - tests_require=['pytest'], + tests_require=['pytest', 'more-itertools>=4.0.0,<6.0.0;python_version<="2.7"'], zip_safe = False) From f55e1df4acc6b46bfd86663bc227b13da2fc3678 Mon Sep 17 00:00:00 2001 From: Yashwant Date: Wed, 3 Jul 2019 22:04:58 +0530 Subject: [PATCH 141/143] Add more functionality --- src/mlpack/bindings/python/setup.py.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mlpack/bindings/python/setup.py.in b/src/mlpack/bindings/python/setup.py.in index b04f60c2f0..c88069332c 100644 --- a/src/mlpack/bindings/python/setup.py.in +++ b/src/mlpack/bindings/python/setup.py.in @@ -108,5 +108,7 @@ setup(name='mlpack', cmdclass={ 'build_ext': build_ext }, ext_modules = modules, setup_requires=['cython', 'pytest-runner'], - tests_require=['pytest', 'more-itertools>=4.0.0,<6.0.0;python_version<="2.7"'], + tests_require=['pytest>4.6;python_version>"2.7"', 'pytest<=4.6;python_version<="2.7"', + 'more-itertools>=4.0.0,<6.0.0;python_version<="2.7"', + 'more-itertools>=4.0.0;python_version>"2.7"'], zip_safe = False) From 49cddef1021d85b6f82837dec30eef315eec78e0 Mon Sep 17 00:00:00 2001 From: Yashwant Singh Parihar Date: Fri, 5 Jul 2019 08:47:19 +0530 Subject: [PATCH 142/143] Udpate versions for py-test --- src/mlpack/bindings/python/setup.py.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/bindings/python/setup.py.in b/src/mlpack/bindings/python/setup.py.in index c88069332c..119894c3ac 100644 --- a/src/mlpack/bindings/python/setup.py.in +++ b/src/mlpack/bindings/python/setup.py.in @@ -108,7 +108,7 @@ setup(name='mlpack', cmdclass={ 'build_ext': build_ext }, ext_modules = modules, setup_requires=['cython', 'pytest-runner'], - tests_require=['pytest>4.6;python_version>"2.7"', 'pytest<=4.6;python_version<="2.7"', + tests_require=['pytest>3;python_version>"3.4"', 'pytest>3,<=4.6;python_version<="3.4"' 'more-itertools>=4.0.0,<6.0.0;python_version<="2.7"', 'more-itertools>=4.0.0;python_version>"2.7"'], zip_safe = False) From 99a7007505792702cfd16a9b1c272b024c8273b3 Mon Sep 17 00:00:00 2001 From: Yashwant Singh Parihar Date: Fri, 5 Jul 2019 09:18:22 +0530 Subject: [PATCH 143/143] Sorry for the comma --- src/mlpack/bindings/python/setup.py.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/bindings/python/setup.py.in b/src/mlpack/bindings/python/setup.py.in index 119894c3ac..475afd85f1 100644 --- a/src/mlpack/bindings/python/setup.py.in +++ b/src/mlpack/bindings/python/setup.py.in @@ -108,7 +108,7 @@ setup(name='mlpack', cmdclass={ 'build_ext': build_ext }, ext_modules = modules, setup_requires=['cython', 'pytest-runner'], - tests_require=['pytest>3;python_version>"3.4"', 'pytest>3,<=4.6;python_version<="3.4"' + tests_require=['pytest>3;python_version>"3.4"', 'pytest>3,<=4.6;python_version<="3.4"', 'more-itertools>=4.0.0,<6.0.0;python_version<="2.7"', 'more-itertools>=4.0.0;python_version>"2.7"'], zip_safe = False)