From c475d9aa10d31857650b935ebe24409e769b5b78 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sun, 8 Sep 2019 02:19:39 -0400 Subject: [PATCH 1/3] Handle both ensmallen 1 and 2 API. --- .../reinforcement_learning/q_learning.hpp | 8 + .../q_learning_impl.hpp | 43 ++++- .../worker/n_step_q_learning_worker.hpp | 169 +++++++++++++++++- .../worker/one_step_q_learning_worker.hpp | 169 +++++++++++++++++- .../worker/one_step_sarsa_worker.hpp | 169 +++++++++++++++++- src/mlpack/tests/feedforward_network_test.cpp | 9 + 6 files changed, 557 insertions(+), 10 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning.hpp b/src/mlpack/methods/reinforcement_learning/q_learning.hpp index bf71c93b64..f3a7508a39 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning.hpp @@ -84,6 +84,11 @@ class QLearning UpdaterType updater = UpdaterType(), EnvironmentType environment = EnvironmentType()); + /** + * Clean memory. + */ + ~QLearning(); + /** * Execute a step in an episode. * @return Reward for the step. @@ -140,6 +145,9 @@ class QLearning //! Locally-stored updater. UpdaterType updater; + #if ENS_VERSION_MAJOR >= 2 + typename UpdaterType::template Policy* updatePolicy; + #endif //! Locally-stored behavior policy. PolicyType policy; diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 228ea16807..d90b738fcf 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -39,6 +39,9 @@ QLearning< config(std::move(config)), learningNetwork(std::move(network)), updater(std::move(updater)), + #if ENS_VERSION_MAJOR >= 2 + updatePolicy(NULL), + #endif policy(std::move(policy)), replayMethod(std::move(replayMethod)), environment(std::move(environment)), @@ -49,11 +52,39 @@ QLearning< if (learningNetwork.Parameters().is_empty()) learningNetwork.ResetParameters(); + #if ENS_VERSION_MAJOR == 1 this->updater.Initialize(learningNetwork.Parameters().n_rows, - learningNetwork.Parameters().n_cols); + learningNetwork.Parameters().n_cols); + #else + this->updatePolicy = new typename UpdaterType::template + Policy(this->updater, + learningNetwork.Parameters().n_rows, + learningNetwork.Parameters().n_cols); + #endif + targetNetwork = learningNetwork; } +template < + typename EnvironmentType, + typename NetworkType, + typename UpdaterType, + typename PolicyType, + typename ReplayType +> +QLearning< + EnvironmentType, + NetworkType, + UpdaterType, + PolicyType, + ReplayType +>::~QLearning() +{ + #if ENS_VERSION_MAJOR >= 2 + delete updatePolicy; + #endif +} + template < typename EnvironmentType, typename NetworkType, @@ -166,14 +197,18 @@ double QLearning< } } - // Learn form experience. + // Learn from experience. arma::mat gradients; learningNetwork.Backward(target, gradients); - replayMethod.Update(target, sampledActions, - nextActionValues, gradients); + replayMethod.Update(target, sampledActions, nextActionValues, gradients); + #if ENS_VERSION_MAJOR == 1 updater.Update(learningNetwork.Parameters(), config.StepSize(), gradients); + #else + updatePolicy->Update(learningNetwork.Parameters(), config.StepSize(), + gradients); + #endif return reward; } diff --git a/src/mlpack/methods/reinforcement_learning/worker/n_step_q_learning_worker.hpp b/src/mlpack/methods/reinforcement_learning/worker/n_step_q_learning_worker.hpp index 3aa5041c3f..1ca53315bb 100644 --- a/src/mlpack/methods/reinforcement_learning/worker/n_step_q_learning_worker.hpp +++ b/src/mlpack/methods/reinforcement_learning/worker/n_step_q_learning_worker.hpp @@ -54,20 +54,177 @@ class NStepQLearningWorker const TrainingConfig& config, bool deterministic): updater(updater), + #if ENS_VERSION_MAJOR >= 2 + updatePolicy(NULL), + #endif environment(environment), config(config), deterministic(deterministic), pending(config.UpdateInterval()) { Reset(); } + /** + * Copy another NStepQLearningWorker. + * + * @param other NStepQLearningWorker to copy. + */ + NStepQLearningWorker(const NStepQLearningWorker& other) : + updater(other.updater), + #if ENS_VERSION_MAJOR >= 2 + updatePolicy(NULL), + #endif + environment(other.environment), + config(other.config), + deterministic(other.deterministic), + steps(other.steps), + episodeReturn(other.episodeReturn), + pending(other.pending), + pendingIndex(other.pendingIndex), + network(other.network), + state(other.state) + { + #if ENS_VERSION_MAJOR >= 2 + updatePolicy = new typename UpdaterType::template + Policy(updater, + network.Parameters().n_rows, + network.Parameters().n_cols); + #endif + + Reset(); + } + + /** + * Take ownership of another NStepQLearningWorker. + * + * @param other NStepQLearningWorker to take ownership of. + */ + NStepQLearningWorker(NStepQLearningWorker&& other) : + updater(std::move(other.updater)), + #if ENS_VERSION_MAJOR >= 2 + updatePolicy(NULL), + #endif + environment(std::move(other.environment)), + config(std::move(other.config)), + deterministic(std::move(other.deterministic)), + steps(std::move(other.steps)), + episodeReturn(std::move(other.episodeReturn)), + pending(std::move(other.pending)), + pendingIndex(std::move(other.pendingIndex)), + network(std::move(other.network)), + state(std::move(other.state)) + { + #if ENS_VERSION_MAJOR >= 2 + other.updatePolicy = NULL; + + updatePolicy = new typename UpdaterType::template + Policy(updater, + network.Parameters().n_rows, + network.Parameters().n_cols); + #endif + } + + /** + * Copy another NStepQLearningWorker. + * + * @param other NStepQLearningWorker to copy. + */ + NStepQLearningWorker& operator=(const NStepQLearningWorker& other) + { + if (&other == this) + return *this; + + #if ENS_VERSION_MAJOR >= 2 + delete updatePolicy; + #endif + + updater = other.updater; + environment = other.environment; + config = other.config; + deterministic = other.deterministic; + steps = other.steps; + episodeReturn = other.episodeReturn; + pending = other.pending; + pendingIndex = other.pendingIndex; + network = other.network; + state = other.state; + + #if ENS_VERSION_MAJOR >= 2 + updatePolicy = new typename UpdaterType::template + Policy(updater, + network.Parameters().n_rows, + network.Parameters().n_cols); + #endif + + Reset(); + + return *this; + } + + /** + * Take ownership of another NStepQLearningWorker. + * + * @param other NStepQLearningWorker to take ownership of. + */ + NStepQLearningWorker& operator=(NStepQLearningWorker&& other) + { + if (&other == this) + return *this; + + #if ENS_VERSION_MAJOR >= 2 + delete updatePolicy; + #endif + + updater = std::move(other.updater); + environment = std::move(other.environment); + config = std::move(other.config); + deterministic = std::move(other.deterministic); + steps = std::move(other.steps); + episodeReturn = std::move(other.episodeReturn); + pending = std::move(other.pending); + pendingIndex = std::move(other.pendingIndex); + network = std::move(other.network); + state = std::move(other.state); + + #if ENS_VERSION_MAJOR >= 2 + updatePolicy = new typename UpdaterType::template + Policy(updater, + network.Parameters().n_rows, + network.Parameters().n_cols); + + other.updatePolicy = NULL; + #endif + + return *this; + } + + /** + * Clean memory. + */ + ~NStepQLearningWorker() + { + #if ENS_VERSION_MAJOR >= 2 + delete updatePolicy; + #endif + } + /** * Initialize the worker. * @param learningNetwork The shared network. */ void Initialize(NetworkType& learningNetwork) { + #if ENS_VERSION_MAJOR == 1 updater.Initialize(learningNetwork.Parameters().n_rows, - learningNetwork.Parameters().n_cols); + learningNetwork.Parameters().n_cols); + #else + delete updatePolicy; + + updatePolicy = new typename UpdaterType::template + Policy(updater, + learningNetwork.Parameters().n_rows, + learningNetwork.Parameters().n_cols); + #endif + // Build local network. network = learningNetwork; } @@ -162,8 +319,13 @@ class NStepQLearningWorker config.GradientLimit()); }); // Perform async update of the global network. - updater.Update(learningNetwork.Parameters(), + #if ENS_VERSION_MAJOR == 1 + updater.Update(learningNetwork.Parameters(), config.StepSize(), + totalGradients); + #else + updatePolicy->Update(learningNetwork.Parameters(), config.StepSize(), totalGradients); + #endif // Sync the local network with the global network. network = learningNetwork; @@ -204,6 +366,9 @@ class NStepQLearningWorker //! Locally-stored optimizer. UpdaterType updater; + #if ENS_VERSION_MAJOR >= 2 + typename UpdaterType::template Policy* updatePolicy; + #endif //! Locally-stored task. EnvironmentType environment; diff --git a/src/mlpack/methods/reinforcement_learning/worker/one_step_q_learning_worker.hpp b/src/mlpack/methods/reinforcement_learning/worker/one_step_q_learning_worker.hpp index 198398cedf..02a1c738f8 100644 --- a/src/mlpack/methods/reinforcement_learning/worker/one_step_q_learning_worker.hpp +++ b/src/mlpack/methods/reinforcement_learning/worker/one_step_q_learning_worker.hpp @@ -54,20 +54,177 @@ class OneStepQLearningWorker const TrainingConfig& config, bool deterministic): updater(updater), + #if ENS_VERSION_MAJOR >= 2 + updatePolicy(NULL), + #endif environment(environment), config(config), deterministic(deterministic), pending(config.UpdateInterval()) { Reset(); } + /** + * Copy another OneStepQLearningWorker. + * + * @param other OneStepQLearningWorker to copy. + */ + OneStepQLearningWorker(const OneStepQLearningWorker& other) : + updater(other.updater), + #if ENS_VERSION_MAJOR >= 2 + updatePolicy(NULL), + #endif + environment(other.environment), + config(other.config), + deterministic(other.deterministic), + steps(other.steps), + episodeReturn(other.episodeReturn), + pending(other.pending), + pendingIndex(other.pendingIndex), + network(other.network), + state(other.state) + { + #if ENS_VERSION_MAJOR >= 2 + updatePolicy = new typename UpdaterType::template + Policy(updater, + network.Parameters().n_rows, + network.Parameters().n_cols); + #endif + + Reset(); + } + + /** + * Take ownership of another OneStepQLearningWorker. + * + * @param other OneStepQLearningWorker to take ownership of. + */ + OneStepQLearningWorker(OneStepQLearningWorker&& other) : + updater(std::move(other.updater)), + #if ENS_VERSION_MAJOR >= 2 + updatePolicy(NULL), + #endif + environment(std::move(other.environment)), + config(std::move(other.config)), + deterministic(std::move(other.deterministic)), + steps(std::move(other.steps)), + episodeReturn(std::move(other.episodeReturn)), + pending(std::move(other.pending)), + pendingIndex(std::move(other.pendingIndex)), + network(std::move(other.network)), + state(std::move(other.state)) + { + #if ENS_VERSION_MAJOR >= 2 + other.updatePolicy = NULL; + + updatePolicy = new typename UpdaterType::template + Policy(updater, + network.Parameters().n_rows, + network.Parameters().n_cols); + #endif + } + + /** + * Copy another OneStepQLearningWorker. + * + * @param other OneStepQLearningWorker to copy. + */ + OneStepQLearningWorker& operator=(const OneStepQLearningWorker& other) + { + if (&other == this) + return *this; + + #if ENS_VERSION_MAJOR >= 2 + delete updatePolicy; + #endif + + updater = other.updater; + environment = other.environment; + config = other.config; + deterministic = other.deterministic; + steps = other.steps; + episodeReturn = other.episodeReturn; + pending = other.pending; + pendingIndex = other.pendingIndex; + network = other.network; + state = other.state; + + #if ENS_VERSION_MAJOR >= 2 + updatePolicy = new typename UpdaterType::template + Policy(updater, + network.Parameters().n_rows, + network.Parameters().n_cols); + #endif + + Reset(); + + return *this; + } + + /** + * Take ownership of another OneStepQLearningWorker. + * + * @param other OneStepQLearningWorker to take ownership of. + */ + OneStepQLearningWorker& operator=(OneStepQLearningWorker&& other) + { + if (&other == this) + return *this; + + #if ENS_VERSION_MAJOR >= 2 + delete updatePolicy; + #endif + + updater = std::move(other.updater); + environment = std::move(other.environment); + config = std::move(other.config); + deterministic = std::move(other.deterministic); + steps = std::move(other.steps); + episodeReturn = std::move(other.episodeReturn); + pending = std::move(other.pending); + pendingIndex = std::move(other.pendingIndex); + network = std::move(other.network); + state = std::move(other.state); + + #if ENS_VERSION_MAJOR >= 2 + other.updatePolicy = NULL; + + updatePolicy = new typename UpdaterType::template + Policy(updater, + network.Parameters().n_rows, + network.Parameters().n_cols); + #endif + + return *this; + } + + /** + * Clean memory. + */ + ~OneStepQLearningWorker() + { + #if ENS_VERSION_MAJOR >= 2 + delete updatePolicy; + #endif + } + /** * Initialize the worker. * @param learningNetwork The shared network. */ void Initialize(NetworkType& learningNetwork) { + #if ENS_VERSION_MAJOR == 1 updater.Initialize(learningNetwork.Parameters().n_rows, - learningNetwork.Parameters().n_cols); + learningNetwork.Parameters().n_cols); + #else + delete updatePolicy; + + updatePolicy = new typename UpdaterType::template + Policy(updater, + learningNetwork.Parameters().n_rows, + learningNetwork.Parameters().n_cols); + #endif + // Build local network. network = learningNetwork; } @@ -162,8 +319,13 @@ class OneStepQLearningWorker config.GradientLimit()); }); // Perform async update of the global network. - updater.Update(learningNetwork.Parameters(), + #if ENS_VERSION_MAJOR == 1 + updater.Update(learningNetwork.Parameters(), config.StepSize(), + totalGradients); + #else + updatePolicy->Update(learningNetwork.Parameters(), config.StepSize(), totalGradients); + #endif // Sync the local network with the global network. network = learningNetwork; @@ -204,6 +366,9 @@ class OneStepQLearningWorker //! Locally-stored optimizer. UpdaterType updater; + #if ENS_VERSION_MAJOR >= 2 + typename UpdaterType::template Policy* updatePolicy; + #endif //! Locally-stored task. EnvironmentType environment; diff --git a/src/mlpack/methods/reinforcement_learning/worker/one_step_sarsa_worker.hpp b/src/mlpack/methods/reinforcement_learning/worker/one_step_sarsa_worker.hpp index 3734da95de..417fc3c08a 100644 --- a/src/mlpack/methods/reinforcement_learning/worker/one_step_sarsa_worker.hpp +++ b/src/mlpack/methods/reinforcement_learning/worker/one_step_sarsa_worker.hpp @@ -55,20 +55,177 @@ class OneStepSarsaWorker const TrainingConfig& config, bool deterministic): updater(updater), + #if ENS_VERSION_MAJOR >= 2 + updatePolicy(NULL), + #endif environment(environment), config(config), deterministic(deterministic), pending(config.UpdateInterval()) { Reset(); } + /** + * Copy another OneStepSarsaWorker. + * + * @param other OneStepSarsaWorker to copy. + */ + OneStepSarsaWorker(const OneStepSarsaWorker& other) : + updater(other.updater), + #if ENS_VERSION_MAJOR >= 2 + updatePolicy(NULL), + #endif + environment(other.environment), + config(other.config), + deterministic(other.deterministic), + steps(other.steps), + episodeReturn(other.episodeReturn), + pending(other.pending), + pendingIndex(other.pendingIndex), + network(other.network), + state(other.state) + { + Reset(); + + #if ENS_VERSION_MAJOR >= 2 + updatePolicy = new typename UpdaterType::template + Policy(updater, + network.Parameters().n_rows, + network.Parameters().n_cols); + #endif + } + + /** + * Take ownership of another OneStepSarsaWorker. + * + * @param other OneStepSarsaWorker to take ownership of. + */ + OneStepSarsaWorker(OneStepSarsaWorker&& other) : + updater(std::move(other.updater)), + #if ENS_VERSION_MAJOR >= 2 + updatePolicy(NULL), + #endif + environment(std::move(other.environment)), + config(std::move(other.config)), + deterministic(std::move(other.deterministic)), + steps(std::move(other.steps)), + episodeReturn(std::move(other.episodeReturn)), + pending(std::move(other.pending)), + pendingIndex(std::move(other.pendingIndex)), + network(std::move(other.network)), + state(std::move(other.state)) + { + #if ENS_VERSION_MAJOR >= 2 + other.updatePolicy = NULL; + + updatePolicy = new typename UpdaterType::template + Policy(updater, + network.Parameters().n_rows, + network.Parameters().n_cols); + #endif + } + + /** + * Copy another OneStepSarsaWorker. + * + * @param other OneStepSarsaWorker to copy. + */ + OneStepSarsaWorker& operator=(const OneStepSarsaWorker& other) + { + if (&other == this) + return *this; + + #if ENS_VERSION_MAJOR >= 2 + delete updatePolicy; + #endif + + updater = other.updater; + environment = other.environment; + config = other.config; + deterministic = other.deterministic; + steps = other.steps; + episodeReturn = other.episodeReturn; + pending = other.pending; + pendingIndex = other.pendingIndex; + network = other.network; + state = other.state; + + #if ENS_VERSION_MAJOR >= 2 + updatePolicy = new typename UpdaterType::template + Policy(updater, + network.Parameters().n_rows, + network.Parameters().n_cols); + #endif + + Reset(); + + return *this; + } + + /** + * Take ownership of another OneStepSarsaWorker. + * + * @param other OneStepSarsaWorker to take ownership of. + */ + OneStepSarsaWorker& operator=(OneStepSarsaWorker&& other) + { + if (&other == this) + return *this; + + #if ENS_VERSION_MAJOR >= 2 + delete updatePolicy; + #endif + + updater = std::move(other.updater); + environment = std::move(other.environment); + config = std::move(other.config); + deterministic = std::move(other.deterministic); + steps = std::move(other.steps); + episodeReturn = std::move(other.episodeReturn); + pending = std::move(other.pending); + pendingIndex = std::move(other.pendingIndex); + network = std::move(other.network); + state = std::move(other.state); + + #if ENS_VERSION_MAJOR >= 2 + other.updatePolicy = NULL; + + updatePolicy = new typename UpdaterType::template + Policy(updater, + network.Parameters().n_rows, + network.Parameters().n_cols); + #endif + + return *this; + } + + /** + * Clean memory. + */ + ~OneStepSarsaWorker() + { + #if ENS_VERSION_MAJOR >= 2 + delete updatePolicy; + #endif + } + /** * Initialize the worker. * @param learningNetwork The shared network. */ void Initialize(NetworkType& learningNetwork) { + #if ENS_VERSION_MAJOR == 1 updater.Initialize(learningNetwork.Parameters().n_rows, - learningNetwork.Parameters().n_cols); + learningNetwork.Parameters().n_cols); + #else + delete updatePolicy; + + updatePolicy = new typename UpdaterType::template + Policy(updater, + learningNetwork.Parameters().n_rows, + learningNetwork.Parameters().n_cols); + #endif + // Build local network. network = learningNetwork; } @@ -171,8 +328,13 @@ class OneStepSarsaWorker config.GradientLimit()); }); // Perform async update of the global network. - updater.Update(learningNetwork.Parameters(), + #if ENS_VERSION_MAJOR == 1 + updater.Update(learningNetwork.Parameters(), config.StepSize(), + totalGradients); + #else + updatePolicy->Update(learningNetwork.Parameters(), config.StepSize(), totalGradients); + #endif // Sync the local network with the global network. network = learningNetwork; @@ -215,6 +377,9 @@ class OneStepSarsaWorker //! Locally-stored optimizer. UpdaterType updater; + #if ENS_VERSION_MAJOR >= 2 + typename UpdaterType::template Policy* updatePolicy; + #endif //! Locally-stored task. EnvironmentType environment; diff --git a/src/mlpack/tests/feedforward_network_test.cpp b/src/mlpack/tests/feedforward_network_test.cpp index 3ad3864b92..c47d633ace 100644 --- a/src/mlpack/tests/feedforward_network_test.cpp +++ b/src/mlpack/tests/feedforward_network_test.cpp @@ -159,7 +159,12 @@ BOOST_AUTO_TEST_CASE(ForwardBackwardTest) ens::VanillaUpdate opt; model.ResetParameters(); + #if ENS_VERSION_MAJOR == 1 opt.Initialize(model.Parameters().n_rows, model.Parameters().n_cols); + #else + ens::VanillaUpdate::Policy optPolicy(opt, + model.Parameters().n_rows, model.Parameters().n_cols); + #endif double stepSize = 0.01; size_t batchSize = 10; @@ -179,7 +184,11 @@ BOOST_AUTO_TEST_CASE(ForwardBackwardTest) model.Forward(currentData, currentResuls); arma::mat gradients; model.Backward(currentLabels, gradients); + #if ENS_VERSION_MAJOR == 1 opt.Update(model.Parameters(), stepSize, gradients); + #else + optPolicy.Update(model.Parameters(), stepSize, gradients); + #endif batchStart = batchEnd; arma::mat prediction = arma::zeros(1, currentResuls.n_cols); From f4846ad8bd6b0608635555399d5e1c69e4f3d80b Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sun, 8 Sep 2019 13:44:02 -0400 Subject: [PATCH 2/3] Also initialize action member in copy/move constructors. --- .../worker/one_step_sarsa_worker.hpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/worker/one_step_sarsa_worker.hpp b/src/mlpack/methods/reinforcement_learning/worker/one_step_sarsa_worker.hpp index 417fc3c08a..ef974f0525 100644 --- a/src/mlpack/methods/reinforcement_learning/worker/one_step_sarsa_worker.hpp +++ b/src/mlpack/methods/reinforcement_learning/worker/one_step_sarsa_worker.hpp @@ -82,7 +82,8 @@ class OneStepSarsaWorker pending(other.pending), pendingIndex(other.pendingIndex), network(other.network), - state(other.state) + state(other.state), + action(other.action) { Reset(); @@ -112,7 +113,8 @@ class OneStepSarsaWorker pending(std::move(other.pending)), pendingIndex(std::move(other.pendingIndex)), network(std::move(other.network)), - state(std::move(other.state)) + state(std::move(other.state)), + action(std::move(other.action)) { #if ENS_VERSION_MAJOR >= 2 other.updatePolicy = NULL; @@ -148,6 +150,7 @@ class OneStepSarsaWorker pendingIndex = other.pendingIndex; network = other.network; state = other.state; + action = other.action; #if ENS_VERSION_MAJOR >= 2 updatePolicy = new typename UpdaterType::template @@ -185,6 +188,7 @@ class OneStepSarsaWorker pendingIndex = std::move(other.pendingIndex); network = std::move(other.network); state = std::move(other.state); + action = std::move(other.action); #if ENS_VERSION_MAJOR >= 2 other.updatePolicy = NULL; From 8f3d166bb6d6d9de08bbe320e1508872eea34594 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sun, 8 Sep 2019 15:21:53 -0400 Subject: [PATCH 3/3] Increase travis build timeout. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 62d3ef7851..9c1b4ab0b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,7 +41,7 @@ matrix: - brew install openblas armadillo || brew install openblas armadillo install: - - mkdir build && cd build && cmake $CMAKE_OPTIONS .. && travis_wait 60 make -j2 + - mkdir build && cd build && cmake $CMAKE_OPTIONS .. && travis_wait 75 make -j2 script: - CTEST_OUTPUT_ON_FAILURE=1 travis_wait 30 ctest -j2