From 7bd26a8ba20ad53a0bafe20b5603a8c18e1df133 Mon Sep 17 00:00:00 2001 From: Tarek Date: Sun, 18 Jun 2023 13:24:16 +0300 Subject: [PATCH 1/3] feat(rl): implement Ornstein-Uhlenbeck noise class Signed-off-by: Tarek --- .../reinforcement_learning/noise/noise.hpp | 17 ++++ .../noise/ornstein_uhlenbeck.hpp | 78 +++++++++++++++++++ .../reinforcement_learning.hpp | 1 + src/mlpack/tests/q_learning_test.cpp | 23 ++++++ 4 files changed, 119 insertions(+) create mode 100644 src/mlpack/methods/reinforcement_learning/noise/noise.hpp create mode 100644 src/mlpack/methods/reinforcement_learning/noise/ornstein_uhlenbeck.hpp diff --git a/src/mlpack/methods/reinforcement_learning/noise/noise.hpp b/src/mlpack/methods/reinforcement_learning/noise/noise.hpp new file mode 100644 index 0000000000..efa5998638 --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/noise/noise.hpp @@ -0,0 +1,17 @@ +/** + * @file methods/reinforcement_learning/noise/noise.hpp + * @author Tarek Elsayed + * + * Convenience include for reinforcement learning noises. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_REINFORCEMENT_LEARNING_NOISE_NOISE_HPP +#define MLPACK_METHODS_REINFORCEMENT_LEARNING_NOISE_NOISE_HPP + +#include "ornstein_uhlenbeck.hpp" + +#endif diff --git a/src/mlpack/methods/reinforcement_learning/noise/ornstein_uhlenbeck.hpp b/src/mlpack/methods/reinforcement_learning/noise/ornstein_uhlenbeck.hpp new file mode 100644 index 0000000000..d2b6cf4ab8 --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/noise/ornstein_uhlenbeck.hpp @@ -0,0 +1,78 @@ +/** + * @file methods/reinforcement_learning/noise/ornstein_uhlenbeck.hpp + * @author Tarek Elsayed + * + * This file is the implementation of OUNoise class. + * Ornstein-Uhlenbeck process generates temporally correlated exploration, + * and it effectively copes with physical control problems of inertia. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_RL_NOISE_ORNSTEIN_UHLENBECK_HPP +#define MLPACK_METHODS_RL_NOISE_ORNSTEIN_UHLENBECK_HPP + +#include + +namespace mlpack { +class OUNoise +{ + public: + /** + * @param size The size of the noise vector. + * @param mu The mean of the noise process. + * @param theta The rate of mean reversion. + * @param sigma The standard deviation of the noise. + */ + OUNoise(int size, + double mu = 0.0, + double theta = 0.15, + double sigma = 0.2) : + mu(mu * arma::ones(size)), + theta(theta), + sigma(sigma) + { + reset(); + } + + /** + * Reset the internal state to the mean (mu). + */ + void reset() + { + state = mu; + } + + /** + * Update the internal state and return it as a noise sample. + * + * @return Noise sample. + */ + arma::colvec sample() + { + arma::colvec x = state; + arma::colvec dx = theta * (mu - x) + + sigma * arma::randn(x.n_elem); + state = x + dx; + return state; + } + + private: + //! Locally-stored state of the noise process. + arma::colvec state; + + //! Locally-stored mean of the noise process. + arma::colvec mu; + + //! Locally-stored rate of mean reversion. + double theta; + + //! Locally-stored standard deviation of the noise. + double sigma; +}; + +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/reinforcement_learning/reinforcement_learning.hpp b/src/mlpack/methods/reinforcement_learning/reinforcement_learning.hpp index a9f37471d9..6af0276800 100644 --- a/src/mlpack/methods/reinforcement_learning/reinforcement_learning.hpp +++ b/src/mlpack/methods/reinforcement_learning/reinforcement_learning.hpp @@ -17,6 +17,7 @@ #include "q_networks/q_networks.hpp" #include "replay/replay.hpp" #include "worker/worker.hpp" +#include "noise/noise.hpp" #include "training_config.hpp" #include "async_learning.hpp" diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index b48e7cd6bf..bdbc527e14 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -651,3 +651,26 @@ TEST_CASE("DDPGForMultipleActions", "[QLearningTest]") // If the agent is able to reach this point of the test, it is assured // that the agent can handle multiple actions in continuous space. } + +//! Test Ornstein-Uhlenbeck noise class. +TEST_CASE("OUNoiseTest", "[QLearningTest]") +{ + // Set up the OUNoise parameters. + int size = 3; + double mu = 0.0; + double theta = 0.15; + double sigma = 0.2; + + // Create an instance of the OUNoise class. + OUNoise ouNoise(size, mu, theta, sigma); + + // Test the reset function. + ouNoise.reset(); + arma::colvec state = ouNoise.sample(); + REQUIRE(state.n_elem == size); + + // Verify that the sample is not equal to the reset state. + arma::colvec sample = ouNoise.sample(); + bool isNotEqual = arma::any(sample != state); + REQUIRE(isNotEqual); +} From d8b7431bde34395de6cdf864819a755029361e7a Mon Sep 17 00:00:00 2001 From: Tarek Date: Tue, 20 Jun 2023 18:25:02 +0300 Subject: [PATCH 2/3] feat(rl): modify ddpg to accept a noise instance Signed-off-by: Tarek --- .../methods/reinforcement_learning/ddpg.hpp | 7 +++++ .../reinforcement_learning/ddpg_impl.hpp | 23 +++++++++++++--- src/mlpack/tests/q_learning_test.cpp | 27 ++++++++++++++++--- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/ddpg.hpp b/src/mlpack/methods/reinforcement_learning/ddpg.hpp index 03bf89b870..2bab884a71 100644 --- a/src/mlpack/methods/reinforcement_learning/ddpg.hpp +++ b/src/mlpack/methods/reinforcement_learning/ddpg.hpp @@ -46,6 +46,7 @@ namespace mlpack { * @tparam EnvironmentType The environment of the reinforcement learning task. * @tparam QNetworkType The network used to estimate the critic's Q-values. * @tparam PolicyNetworkType The network to compute action value. + * @tparam NoiseType The noise to add for exploration. * @tparam UpdaterType How to apply gradients when training. * @tparam ReplayType Experience replay method. */ @@ -53,6 +54,7 @@ template < typename EnvironmentType, typename QNetworkType, typename PolicyNetworkType, + typename NoiseType, typename UpdaterType, typename ReplayType = RandomReplay > @@ -75,6 +77,7 @@ class DDPG * @param config Hyper-parameters for training. * @param learningQNetwork The network to compute action value. * @param policyNetwork The network to produce an action given a state. + * @param noise The noise instance for exploration. * @param replayMethod Experience replay method. * @param qNetworkUpdater How to apply gradients to Q network when training. * @param policyNetworkUpdater How to apply gradients to policy network @@ -84,6 +87,7 @@ class DDPG DDPG(TrainingConfig& config, QNetworkType& learningQNetwork, PolicyNetworkType& policyNetwork, + NoiseType& noise, ReplayType& replayMethod, UpdaterType qNetworkUpdater = UpdaterType(), UpdaterType policyNetworkUpdater = UpdaterType(), @@ -150,6 +154,9 @@ class DDPG //! Locally-stored policy network. PolicyNetworkType& policyNetwork; + //! Locally-stored noise instance. + NoiseType& noise; + //! Locally-stored target policy network. PolicyNetworkType targetPNetwork; diff --git a/src/mlpack/methods/reinforcement_learning/ddpg_impl.hpp b/src/mlpack/methods/reinforcement_learning/ddpg_impl.hpp index 380ee9756f..d555aab9ec 100644 --- a/src/mlpack/methods/reinforcement_learning/ddpg_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/ddpg_impl.hpp @@ -23,6 +23,7 @@ template < typename EnvironmentType, typename QNetworkType, typename PolicyNetworkType, + typename NoiseType, typename UpdaterType, typename ReplayType > @@ -30,11 +31,13 @@ DDPG< EnvironmentType, QNetworkType, PolicyNetworkType, + NoiseType, UpdaterType, ReplayType >::DDPG(TrainingConfig& config, QNetworkType& learningQNetwork, PolicyNetworkType& policyNetwork, + NoiseType& noise, ReplayType& replayMethod, UpdaterType qNetworkUpdater, UpdaterType policyNetworkUpdater, @@ -42,6 +45,7 @@ DDPG< config(config), learningQNetwork(learningQNetwork), policyNetwork(policyNetwork), + noise(noise), replayMethod(replayMethod), qNetworkUpdater(std::move(qNetworkUpdater)), #if ENS_VERSION_MAJOR >= 2 @@ -55,6 +59,9 @@ DDPG< totalSteps(0), deterministic(false) { + // Reset the noise instance. + noise.reset(); + // Set up q-learning and policy networks. targetPNetwork = policyNetwork; targetQNetwork = learningQNetwork; @@ -106,6 +113,7 @@ template < typename EnvironmentType, typename QNetworkType, typename PolicyNetworkType, + typename NoiseType, typename UpdaterType, typename ReplayType > @@ -113,6 +121,7 @@ DDPG< EnvironmentType, QNetworkType, PolicyNetworkType, + NoiseType, UpdaterType, ReplayType >::~DDPG() @@ -127,6 +136,7 @@ template < typename EnvironmentType, typename QNetworkType, typename PolicyNetworkType, + typename NoiseType, typename UpdaterType, typename ReplayType > @@ -134,6 +144,7 @@ void DDPG< EnvironmentType, QNetworkType, PolicyNetworkType, + NoiseType, UpdaterType, ReplayType >::SoftUpdate(double rho) @@ -148,6 +159,7 @@ template < typename EnvironmentType, typename QNetworkType, typename PolicyNetworkType, + typename NoiseType, typename UpdaterType, typename ReplayType > @@ -155,6 +167,7 @@ void DDPG< EnvironmentType, QNetworkType, PolicyNetworkType, + NoiseType, UpdaterType, ReplayType >::Update() @@ -255,6 +268,7 @@ template < typename EnvironmentType, typename QNetworkType, typename PolicyNetworkType, + typename NoiseType, typename UpdaterType, typename ReplayType > @@ -262,6 +276,7 @@ void DDPG< EnvironmentType, QNetworkType, PolicyNetworkType, + NoiseType, UpdaterType, ReplayType >::SelectAction() @@ -272,9 +287,9 @@ void DDPG< if (!deterministic) { - arma::colvec noise = arma::randn(outputAction.n_rows) * 0.1; - noise = arma::clamp(noise, -0.25, 0.25); - outputAction = outputAction + noise; + arma::colvec sample = noise.sample() * 0.1; + sample = arma::clamp(sample, -0.25, 0.25); + outputAction = outputAction + sample; } action.action = arma::conv_to>::from(outputAction); } @@ -283,6 +298,7 @@ template < typename EnvironmentType, typename QNetworkType, typename PolicyNetworkType, + typename NoiseType, typename UpdaterType, typename ReplayType > @@ -290,6 +306,7 @@ double DDPG< EnvironmentType, QNetworkType, PolicyNetworkType, + NoiseType, UpdaterType, ReplayType >::Episode() diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index bdbc527e14..650c95017c 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -598,9 +598,19 @@ TEST_CASE("PendulumWithDDPG", "[QLearningTest]") qNetwork.Add(new ReLU()); qNetwork.Add(new Linear(1)); + // Set up the OUNoise parameters. + int size = 1; + double mu = 0.0; + double theta = 1.0; + double sigma = 0.1; + + // Create an instance of the OUNoise class. + OUNoise ouNoise(size, mu, theta, sigma); + // Set up Deep Deterministic Policy Gradient agent. - DDPG - agent(config, qNetwork, policyNetwork, replayMethod); + DDPG + agent(config, qNetwork, policyNetwork, ouNoise, replayMethod); converged = testAgent(agent, -900, 500, 10); if (converged) @@ -633,10 +643,19 @@ TEST_CASE("DDPGForMultipleActions", "[QLearningTest]") config.TargetNetworkSyncInterval() = 1; config.UpdateInterval() = 3; + // Set up the OUNoise parameters. + int size = 4; + double mu = 0.0; + double theta = 1.0; + double sigma = 0.1; + + // Create an instance of the OUNoise class. + OUNoise ouNoise(size, mu, theta, sigma); + // Set up the DDPG agent. DDPG, decltype(qNetwork), decltype(policyNetwork), - AdamUpdate> - agent(config, qNetwork, policyNetwork, replayMethod); + OUNoise, AdamUpdate> + agent(config, qNetwork, policyNetwork, ouNoise, replayMethod); agent.State().Data() = arma::randu (ContinuousActionEnv<3, 4>::State::dimension, 1); From 4646ec122d0be6ea231fd3d6ea417018eadcbf09 Mon Sep 17 00:00:00 2001 From: Tarek Date: Tue, 20 Jun 2023 18:30:24 +0300 Subject: [PATCH 3/3] include Ornstein-Uhlenbeck noise in history Signed-off-by: Tarek --- HISTORY.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 3ba0458995..23bf3050a2 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,7 @@ ### mlpack ?.?.? ###### ????-??-?? + * Reinforcement Learning: Ornstein-Uhlenbeck noise (#3499). + * Reinforcement Learning: Deep Deterministic Policy Gradient (#3494). ### mlpack 4.2.0