From c5177da251fc2f65559fef5489b9fbbc380320f7 Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Tue, 9 Jun 2020 22:29:55 +0530 Subject: [PATCH 01/19] Added Categorical DQN layout --- .../q_networks/CMakeLists.txt | 1 + .../q_networks/categorical_dqn.hpp | 190 ++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp diff --git a/src/mlpack/methods/reinforcement_learning/q_networks/CMakeLists.txt b/src/mlpack/methods/reinforcement_learning/q_networks/CMakeLists.txt index 01a3dcebac..eb5d2014b8 100644 --- a/src/mlpack/methods/reinforcement_learning/q_networks/CMakeLists.txt +++ b/src/mlpack/methods/reinforcement_learning/q_networks/CMakeLists.txt @@ -3,6 +3,7 @@ set(SOURCES simple_dqn.hpp dueling_dqn.hpp + categorical_dqn.hpp ) # Add directory name to sources. diff --git a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp new file mode 100644 index 0000000000..020216ef24 --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp @@ -0,0 +1,190 @@ +/** + * @file methods/reinforcement_learning/q_networks/categorical_dqn.hpp + * @author Nishant Kumar + * + * This file contains the implementation of the categorical deep q network. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_RL_CATEGORICAL_DQN_HPP +#define MLPACK_METHODS_RL_CATEGORICAL_DQN_HPP + +#include +#include +#include +#include +#include +#include + +namespace mlpack { +namespace rl { + +using namespace mlpack::ann; + +/** + * @tparam NetworkType The type of network used for categorical dqn. + */ +template , GaussianInitialization>> +class CategoricalDQN +{ + public: + /** + * Default constructor. + */ + CategoricalDQN() : network(), isNoisy(false), atomSize(0) + { /* Nothing to do here. */ } + + /** + * Construct an instance of CategoricalDQN class. + * + * @param inputDim Number of inputs. + * @param h1 Number of neurons in hiddenlayer-1. + * @param h2 Number of neurons in hiddenlayer-2. + * @param outputDim Number of neurons in output layer. + */ + CategoricalDQN(const int inputDim, + const int h1, + const int h2, + const int outputDim, + const bool isNoisy = false, + const size_t atomSize = 51): + network(EmptyLoss<>(), GaussianInitialization(0, 0.001)), + isNoisy(isNoisy), + atomSize(atomSize) + { + network.Add(new Linear<>(inputDim, h1)); + network.Add(new ReLULayer<>()); + if(isNoisy) + { + noisyLayerIndex.push_back(network.Model().size()); + network.Add(new NoisyLinear<>(h1, h2)); + network.Add(new ReLULayer<>()); + noisyLayerIndex.push_back(network.Model().size()); + network.Add(new NoisyLinear<>(h2, outputDim * atomSize)); + } + else + { + network.Add(new Linear<>(h1, h2)); + network.Add(new ReLULayer<>()); + network.Add(new Linear<>(h2, outputDim * atomSize)); + } + } + + CategoricalDQN(NetworkType network, const bool isNoisy, size_t atomSize): + network(std::move(network)), + isNoisy(isNoisy), + atomSize(atomSize) + { /* Nothing to do here. */ } + + /** + * Predict the responses to a given set of predictors. The responses will + * reflect the output of the given output layer as returned by the + * output layer function. + * + * If you want to pass in a parameter and discard the original parameter + * object, be sure to use std::move to avoid unnecessary copy. + * + * @param state Input state. + * @param actionValue Matrix to put output action values of states input. + */ + void Predict(const arma::mat state, arma::mat& actionValue) + { + arma::mat q_atoms, activations; + network.Predict(state, q_atoms); + activations.set_size(q_atoms.n_rows, q_atoms.n_cols); + actionValue.set_size(q_atoms.n_rows / atomSize, q_atoms.n_cols); + arma::rowvec support = arma::linspace(0, 200, atomSize); + for(size_t i = 0; i < q_atoms.n_rows; i += atomSize) + { + arma::mat activation = arma::mat(activations.memptr() + + i * q_atoms.n_cols, atomSize, q_atoms.n_cols, false, false); + arma::mat input = q_atoms.rows(i, i + atomSize - 1); + softMax.Forward(input, activation); + actionValue.row(i/atomSize) = support * activation; + } + } + + /** + * Perform the forward pass of the states in real batch mode. + * + * @param state The input state. + * @param target The predicted target. + */ + void Forward(const arma::mat state, arma::mat& target) + { + arma::mat q_atoms, activations; + network.Forward(state, q_atoms); + activations.set_size(q_atoms.n_rows, q_atoms.n_cols); + target.set_size(q_atoms.n_rows / atomSize, q_atoms.n_cols); + arma::rowvec support = arma::linspace(0, 200, atomSize); + for(size_t i = 0; i < q_atoms.n_rows; i += atomSize) + { + arma::mat activation = arma::mat(activations.memptr() + + i * q_atoms.n_cols, atomSize, q_atoms.n_cols, false, false); + arma::mat input = q_atoms.rows(i, i + atomSize - 1); + softMax.Forward(input, activation); + target.row(i/atomSize) = support * activation; + } + } + + /** + * Resets the parameters of the network. + */ + void ResetParameters() + { + network.ResetParameters(); + } + + /** + * Resets noise of the network, is the network is of type noisy. + */ + void ResetNoise() + { + for(size_t i = 0; i < noisyLayerIndex.size(); i++) + { + boost::get*> + (network.Model()[noisyLayerIndex[i]])->ResetNoise(); + } + } + + //! Return the Parameters. + const arma::mat& Parameters() const { return network.Parameters(); } + //! Modify the Parameters. + arma::mat& Parameters() { return network.Parameters(); } + + /** + * Perform the backward pass of the state in real batch mode. + * + * @param state The input state. + * @param target The training target. + * @param gradient The gradient. + */ + void Backward(const arma::mat state, arma::mat& target, arma::mat& gradient) + { + network.Backward(state, target, gradient); + } + + private: + //! Locally-stored network. + NetworkType network; + + //! Locally-stored check for noisy network. + bool isNoisy; + + //! Locally-stored number of atoms. + size_t atomSize; + + //! Locally-stored indexes of noisy layers in the network. + std::vector noisyLayerIndex; + + //! Locally-stored softmax activation function. + Softmax<> softMax; +}; + +} // namespace rl +} // namespace mlpack + +#endif From 1ae3a195ce7ed578ffb414757fce1e80cb39dc4a Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Thu, 11 Jun 2020 01:08:33 +0530 Subject: [PATCH 02/19] Separated training functions for simple and categorical dqn, added target dist calculation --- .../reinforcement_learning/q_learning.hpp | 5 + .../q_learning_impl.hpp | 109 +++++++++++++++++- .../q_networks/categorical_dqn.hpp | 16 ++- .../training_config.hpp | 20 +++- 4 files changed, 137 insertions(+), 13 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning.hpp b/src/mlpack/methods/reinforcement_learning/q_learning.hpp index 428bf625fa..72a8dba9c5 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning.hpp @@ -94,6 +94,11 @@ class QLearning */ void TrainAgent(); + /** + * Trains the DQN agent of categorical type. + */ + void TrainCategoricalAgent(); + /** * Execute a step in an episode. * @return Reward for the step. diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index ae4f0ad2a7..2e121b95ee 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -186,6 +186,110 @@ void QLearning< #endif } +template < + typename EnvironmentType, + typename NetworkType, + typename UpdaterType, + typename BehaviorPolicyType, + typename ReplayType +> +void QLearning< + EnvironmentType, + NetworkType, + UpdaterType, + BehaviorPolicyType, + ReplayType +>::TrainCategoricalAgent() +{ + // Start experience replay. + + // Sample from previous experience. + arma::mat sampledStates; + arma::icolvec sampledActions; + arma::colvec sampledRewards; + arma::mat sampledNextStates; + arma::icolvec isTerminal; + + replayMethod.Sample(sampledStates, sampledActions, sampledRewards, + sampledNextStates, isTerminal); + + double vMin = 0, vMax = 200.0; + size_t atomSize = 51; + arma::rowvec support = arma::linspace(vMin, vMax, atomSize); + + size_t batchSize = sampledNextStates.n_cols; + + // Compute action value for next state with target network. + arma::mat nextActionValues; + targetNetwork.Predict(sampledNextStates, nextActionValues); + + arma::Col nextAction; + if (config.DoubleQLearning()) + { + // If use double Q-Learning, use learning network to select the best action. + arma::mat nextActionValues; + learningNetwork.Predict(sampledNextStates, nextActionValues); + nextAction = BestAction(nextActionValues); + } + else + { + nextAction = BestAction(nextActionValues); + } + + arma::mat nextDists, nextDist(atomSize, batchSize); + targetNetwork.Forward(sampledNextStates, nextDists); + for (size_t i = 0; i < batchSize; ++i) + { + nextDist.col(i) = nextDists(nextAction(i) * atomSize, i, + arma::size(atomSize, 1)); + } + + arma::mat tZ = (arma::conv_to::from(config.Discount() * + ((1 - isTerminal) * support)).each_col() + sampledRewards).t(); + tZ = arma::clamp(tZ, vMin, vMax); + arma::mat b = (tZ - vMin) / (vMax - vMin) * (atomSize - 1); + arma::mat l = arma::floor(b); + arma::mat u = arma::ceil(b); + // arma::umat offset(atomSize, batchSize); + // offset.each_row() = arma::linspace(0, (batchSize - 1) * + // atomSize, batchSize); + + arma::mat projDistUpper = nextDist % (u - b); + arma::mat projDistLower = nextDist % (b - l); + + arma::mat projDist = arma::zeros(arma::size(nextDist)); + for (size_t batchNo = 0; batchNo < batchSize; batchNo++) + { + for (size_t j = 0; j < atomSize; j++) + { + projDist(l(j, batchNo), batchNo) += projDistUpper(j, batchNo); + projDist(u(j, batchNo), batchNo) += projDistLower(j, batchNo); + } + } + arma::mat dists; + learningNetwork.Forward(sampledStates, dists); + arma::mat lossGradients = arma::zeros(arma::size(dists)); + for (size_t i = 0; i < batchSize; ++i) + { + lossGradients(sampledActions(i) * atomSize, i, arma::size(atomSize, 1)) = + - (projDist.col(i) / (1e-10 + dists(sampledActions(i) * atomSize, i, + arma::size(atomSize, 1)))); + } + // Learn from experience. + arma::mat gradients; + learningNetwork.Backward(sampledStates, lossGradients, gradients); + + // TODO: verify for PER + replayMethod.Update(lossGradients, sampledActions, nextActionValues, gradients); + + #if ENS_VERSION_MAJOR == 1 + updater.Update(learningNetwork.Parameters(), config.StepSize(), gradients); + #else + updatePolicy->Update(learningNetwork.Parameters(), config.StepSize(), + gradients); + #endif +} + template < typename EnvironmentType, typename NetworkType, @@ -222,7 +326,10 @@ double QLearning< if (deterministic || totalSteps < config.ExplorationSteps()) return reward; - TrainAgent(); + if (config.IsCategorical()) + TrainCategoricalAgent(); + else + TrainAgent(); if (config.NoisyQLearning() == true) { diff --git a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp index 020216ef24..8ad3e3ef7b 100644 --- a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp @@ -51,7 +51,7 @@ class CategoricalDQN const int outputDim, const bool isNoisy = false, const size_t atomSize = 51): - network(EmptyLoss<>(), GaussianInitialization(0, 0.001)), + network(EmptyLoss<>(), GaussianInitialization(0, 0.05)), isNoisy(isNoisy), atomSize(atomSize) { @@ -94,7 +94,7 @@ class CategoricalDQN { arma::mat q_atoms, activations; network.Predict(state, q_atoms); - activations.set_size(q_atoms.n_rows, q_atoms.n_cols); + activations.copy_size(q_atoms); actionValue.set_size(q_atoms.n_rows / atomSize, q_atoms.n_cols); arma::rowvec support = arma::linspace(0, 200, atomSize); for(size_t i = 0; i < q_atoms.n_rows; i += atomSize) @@ -111,22 +111,20 @@ class CategoricalDQN * Perform the forward pass of the states in real batch mode. * * @param state The input state. - * @param target The predicted target. + * @param dist The predicted distributions. */ - void Forward(const arma::mat state, arma::mat& target) + void Forward(const arma::mat state, arma::mat& dist) { - arma::mat q_atoms, activations; + arma::mat q_atoms; network.Forward(state, q_atoms); - activations.set_size(q_atoms.n_rows, q_atoms.n_cols); - target.set_size(q_atoms.n_rows / atomSize, q_atoms.n_cols); + dist.copy_size(q_atoms); arma::rowvec support = arma::linspace(0, 200, atomSize); for(size_t i = 0; i < q_atoms.n_rows; i += atomSize) { - arma::mat activation = arma::mat(activations.memptr() + + arma::mat activation = arma::mat(dist.memptr() + i * q_atoms.n_cols, atomSize, q_atoms.n_cols, false, false); arma::mat input = q_atoms.rows(i, i + atomSize - 1); softMax.Forward(input, activation); - target.row(i/atomSize) = support * activation; } } diff --git a/src/mlpack/methods/reinforcement_learning/training_config.hpp b/src/mlpack/methods/reinforcement_learning/training_config.hpp index e5793f25d6..badb49a4e0 100644 --- a/src/mlpack/methods/reinforcement_learning/training_config.hpp +++ b/src/mlpack/methods/reinforcement_learning/training_config.hpp @@ -29,7 +29,8 @@ class TrainingConfig discount(0.99), gradientLimit(40), doubleQLearning(false), - noisyQLearning(false) + noisyQLearning(false), + isCategorical(false) { /* Nothing to do here. */ } TrainingConfig( @@ -42,7 +43,8 @@ class TrainingConfig double discount, double gradientLimit, bool doubleQLearning, - bool noisyQLearning) : + bool noisyQLearning, + bool isCategorical) : numWorkers(numWorkers), updateInterval(updateInterval), targetNetworkSyncInterval(targetNetworkSyncInterval), @@ -52,7 +54,8 @@ class TrainingConfig discount(discount), gradientLimit(gradientLimit), doubleQLearning(doubleQLearning), - noisyQLearning(noisyQLearning) + noisyQLearning(noisyQLearning), + isCategorical(isCategorical) { /* Nothing to do here. */ } //! Get the amount of workers. @@ -109,6 +112,11 @@ class TrainingConfig //! Modify the indicator of double q-learning. bool& NoisyQLearning() { return noisyQLearning; } + //! Get the indicator of categorical q-learning. + bool IsCategorical() const { return isCategorical; } + //! Modify the indicator of categorical q-learning. + bool& IsCategorical() { return isCategorical; } + private: /** * Locally-stored number of workers. @@ -172,6 +180,12 @@ class TrainingConfig * This is valid only for q-learning agent. */ bool noisyQLearning; + + /** + * Locally-stored indicator for categorical q-learning. + * This is valid only for q-learning agent. + */ + bool isCategorical; }; } // namespace rl From bd9e70acbcda706e7e7ce6bd95b260f5832f06d2 Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Thu, 11 Jun 2020 16:47:00 +0530 Subject: [PATCH 03/19] completed backward and loss calculation functions --- .../q_learning_impl.hpp | 2 +- .../q_networks/categorical_dqn.hpp | 34 +++++++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 2e121b95ee..261bc5c9b2 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -253,7 +253,7 @@ void QLearning< // arma::umat offset(atomSize, batchSize); // offset.each_row() = arma::linspace(0, (batchSize - 1) * // atomSize, batchSize); - + arma::mat projDistUpper = nextDist % (u - b); arma::mat projDistLower = nextDist % (b - l); diff --git a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp index 8ad3e3ef7b..2d6d6cd582 100644 --- a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp @@ -92,17 +92,17 @@ class CategoricalDQN */ void Predict(const arma::mat state, arma::mat& actionValue) { - arma::mat q_atoms, activations; + arma::mat q_atoms; network.Predict(state, q_atoms); activations.copy_size(q_atoms); actionValue.set_size(q_atoms.n_rows / atomSize, q_atoms.n_cols); arma::rowvec support = arma::linspace(0, 200, atomSize); for(size_t i = 0; i < q_atoms.n_rows; i += atomSize) { - arma::mat activation = arma::mat(activations.memptr() + - i * q_atoms.n_cols, atomSize, q_atoms.n_cols, false, false); + arma::mat activation = activations.rows(i, i + atomSize - 1); arma::mat input = q_atoms.rows(i, i + atomSize - 1); softMax.Forward(input, activation); + activations.rows(i, i + atomSize - 1) = activation; actionValue.row(i/atomSize) = support * activation; } } @@ -117,15 +117,15 @@ class CategoricalDQN { arma::mat q_atoms; network.Forward(state, q_atoms); - dist.copy_size(q_atoms); - arma::rowvec support = arma::linspace(0, 200, atomSize); + activations.copy_size(q_atoms); for(size_t i = 0; i < q_atoms.n_rows; i += atomSize) { - arma::mat activation = arma::mat(dist.memptr() + - i * q_atoms.n_cols, atomSize, q_atoms.n_cols, false, false); + arma::mat activation = activations.rows(i, i + atomSize - 1); arma::mat input = q_atoms.rows(i, i + atomSize - 1); softMax.Forward(input, activation); - } + activations.rows(i, i + atomSize - 1) = activation; + } + dist = activations; } /** @@ -157,12 +157,21 @@ class CategoricalDQN * Perform the backward pass of the state in real batch mode. * * @param state The input state. - * @param target The training target. + * @param lossGardients The loss gradients. * @param gradient The gradient. */ - void Backward(const arma::mat state, arma::mat& target, arma::mat& gradient) + void Backward(const arma::mat state, arma::mat& lossGradients, arma::mat& gradient) { - network.Backward(state, target, gradient); + arma::mat activationGradients(arma::size(activations)); + for(size_t i = 0; i < activations.n_rows; i += atomSize) + { + arma::mat activationGrad; + arma::mat lossGrad = lossGradients.rows(i, i + atomSize - 1); + arma::mat activation = activations.rows(i, i + atomSize - 1); + softMax.Backward(activation, lossGrad, activationGrad); + activationGradients.rows(i, i + atomSize - 1) = activationGrad; + } + network.Backward(state, activationGradients, gradient); } private: @@ -180,6 +189,9 @@ class CategoricalDQN //! Locally-stored softmax activation function. Softmax<> softMax; + + //! Locally-stored activations from softMax. + arma::mat activations; }; } // namespace rl From 1066d0cc34b19196b63989e22f9ad585b4daf6c6 Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Thu, 11 Jun 2020 23:16:06 +0530 Subject: [PATCH 04/19] minor fixes --- .../reinforcement_learning/q_networks/categorical_dqn.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp index 2d6d6cd582..30edbfdc99 100644 --- a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp @@ -51,7 +51,7 @@ class CategoricalDQN const int outputDim, const bool isNoisy = false, const size_t atomSize = 51): - network(EmptyLoss<>(), GaussianInitialization(0, 0.05)), + network(EmptyLoss<>(), GaussianInitialization(0, 0.001)), isNoisy(isNoisy), atomSize(atomSize) { @@ -96,7 +96,8 @@ class CategoricalDQN network.Predict(state, q_atoms); activations.copy_size(q_atoms); actionValue.set_size(q_atoms.n_rows / atomSize, q_atoms.n_cols); - arma::rowvec support = arma::linspace(0, 200, atomSize); + double vMin = 0, vMax = 200.0; + arma::rowvec support = arma::linspace(vMin, vMax, atomSize); for(size_t i = 0; i < q_atoms.n_rows; i += atomSize) { arma::mat activation = activations.rows(i, i + atomSize - 1); @@ -104,7 +105,7 @@ class CategoricalDQN softMax.Forward(input, activation); activations.rows(i, i + atomSize - 1) = activation; actionValue.row(i/atomSize) = support * activation; - } + } } /** From 52c0eb58a286174bab1f9ac6a654f6bb9e6b201b Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Tue, 16 Jun 2020 13:24:26 +0530 Subject: [PATCH 05/19] Added tests for categorical dqn --- src/mlpack/tests/q_learning_test.cpp | 37 +++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index 11e08094ed..842c9ac101 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -320,7 +321,6 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDuelingDQN) BOOST_REQUIRE(converged); } - //! Test Dueling DQN in Cart Pole task with Prioritized Replay. BOOST_AUTO_TEST_CASE(CartPoleWithDuelingDQNPrioritizedReplay) { @@ -376,7 +376,6 @@ BOOST_AUTO_TEST_CASE(CartPoleWithNoisyDQN) BOOST_REQUIRE(converged); } - //! Test Dueling-Double-Noisy DQN in Cart Pole task. BOOST_AUTO_TEST_CASE(CartPoleWithDuelingDoubleNoisyDQN) { @@ -465,4 +464,36 @@ BOOST_AUTO_TEST_CASE(CartPoleWithNStepPrioritizedDQN) BOOST_REQUIRE(converged); } -BOOST_AUTO_TEST_SUITE_END(); +//! Test Categorical DQN in Cart Pole task. +BOOST_AUTO_TEST_CASE(CartPoleWithCategoricalDQN) +{ + // It isn't guaranteed that the network will converge in the specified number + // of iterations. + bool converged = false; + for (size_t trial = 0; trial < 3; ++trial) + { + Log::Debug << "Trial number: " << trial << std::endl; + + // Set up the policy and replay method. + GreedyPolicy policy(1.0, 1000, 0.1, 0.99); + RandomReplay replayMethod(32, 2000); + + TrainingConfig config; + config.StepLimit() = 200; + config.IsCategorical() = true; + + // Set up the network with a flag to enable noisy layers. + CategoricalDQN<> network(4, 128, 128, 2); + + // Set up DQN agent. + QLearning + agent(config, network, policy, replayMethod); + + converged = testAgent(agent, 40, 500, 50); + if (converged) + break; + } + BOOST_REQUIRE(converged); +} + +BOOST_AUTO_TEST_SUITE_END(); \ No newline at end of file From 31de8133f05eaea0a77921764bbf0f8343c2f578 Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Tue, 28 Jul 2020 23:20:35 +0530 Subject: [PATCH 06/19] attempt to find bug 1 --- .../q_learning_impl.hpp | 27 ++++++++++++++++--- .../q_networks/categorical_dqn.hpp | 6 ++--- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 09d9b50fde..4c1ee1ab66 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -213,6 +213,12 @@ void QLearning< ReplayType >::TrainCategoricalAgent() { + arma::colvec params = {2.2145,2.2145,2.2145,2.2145,2.2145,2.2145,2.2145,2.2145, + -0.274,-0.274, + 2.2145,2.2145,2.2145,2.2145,2.2145,2.2145,2.2145,2.2145, + -0.274,-0.274, -0.274,-0.274}; + learningNetwork.Parameters() = arma::mat(params); + targetNetwork.Parameters() = arma::mat(params); // Start experience replay. // Sample from previous experience. @@ -226,15 +232,19 @@ void QLearning< sampledNextStates, isTerminal); double vMin = 0, vMax = 200.0; - size_t atomSize = 51; + size_t atomSize = 2; arma::rowvec support = arma::linspace(vMin, vMax, atomSize); + std::cout << "support: " << support << std::endl; + size_t batchSize = sampledNextStates.n_cols; // Compute action value for next state with target network. arma::mat nextActionValues; targetNetwork.Predict(sampledNextStates, nextActionValues); + std::cout << "nextActionValues: " << nextActionValues << std::endl; + arma::Col nextAction; if (config.DoubleQLearning()) { @@ -280,6 +290,9 @@ void QLearning< } arma::mat dists; learningNetwork.Forward(sampledStates, dists); + + std::cout << "dists: " << dists << std::endl; + arma::mat lossGradients = arma::zeros(arma::size(dists)); for (size_t i = 0; i < batchSize; ++i) { @@ -287,10 +300,15 @@ void QLearning< = -(projDist.col(i) / (1e-10 + dists(sampledActions[i].action * atomSize, i, arma::size(atomSize, 1)))); } + + std::cout << "lossGradients: " << lossGradients << std::endl; + // Learn from experience. arma::mat gradients; learningNetwork.Backward(sampledStates, lossGradients, gradients); + std::cout << "gradients: " << gradients << std::endl; + // TODO: verify for PER replayMethod.Update(lossGradients, sampledActions, nextActionValues, gradients); @@ -370,9 +388,12 @@ double QLearning< totalReturn += reward; totalSteps++; - // Store the transition for replay. + state.Data() = {1.2, 0.1, -5, 0.8}; + action.action = CartPole::Action::actions::forward; + reward = 1.7; + nextState.Data() = {0.2, -0.1, -0.5, 1.8}; replayMethod.Store(state, action, reward, nextState, - environment.IsTerminal(nextState), config.Discount()); + false, config.Discount()); // Update current state. state = nextState; diff --git a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp index 30edbfdc99..3f38f30344 100644 --- a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp @@ -67,9 +67,7 @@ class CategoricalDQN } else { - network.Add(new Linear<>(h1, h2)); - network.Add(new ReLULayer<>()); - network.Add(new Linear<>(h2, outputDim * atomSize)); + network.Add(new Linear<>(h1, outputDim * atomSize)); } } @@ -93,7 +91,9 @@ class CategoricalDQN void Predict(const arma::mat state, arma::mat& actionValue) { arma::mat q_atoms; + std::cout << "network params:" << network.Parameters() << std::endl; network.Predict(state, q_atoms); + std::cout << "q_atoms:" << q_atoms << std::endl; activations.copy_size(q_atoms); actionValue.set_size(q_atoms.n_rows / atomSize, q_atoms.n_cols); double vMin = 0, vMax = 200.0; From 6128ce8cd2a2f6b2767828c2303003b38eb0411c Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Fri, 31 Jul 2020 03:24:34 +0530 Subject: [PATCH 07/19] Revert "attempt to find bug 1" This reverts commit 31de8133f05eaea0a77921764bbf0f8343c2f578. --- .../q_learning_impl.hpp | 27 +++---------------- .../q_networks/categorical_dqn.hpp | 6 ++--- 2 files changed, 6 insertions(+), 27 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 4c1ee1ab66..09d9b50fde 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -213,12 +213,6 @@ void QLearning< ReplayType >::TrainCategoricalAgent() { - arma::colvec params = {2.2145,2.2145,2.2145,2.2145,2.2145,2.2145,2.2145,2.2145, - -0.274,-0.274, - 2.2145,2.2145,2.2145,2.2145,2.2145,2.2145,2.2145,2.2145, - -0.274,-0.274, -0.274,-0.274}; - learningNetwork.Parameters() = arma::mat(params); - targetNetwork.Parameters() = arma::mat(params); // Start experience replay. // Sample from previous experience. @@ -232,19 +226,15 @@ void QLearning< sampledNextStates, isTerminal); double vMin = 0, vMax = 200.0; - size_t atomSize = 2; + size_t atomSize = 51; arma::rowvec support = arma::linspace(vMin, vMax, atomSize); - std::cout << "support: " << support << std::endl; - size_t batchSize = sampledNextStates.n_cols; // Compute action value for next state with target network. arma::mat nextActionValues; targetNetwork.Predict(sampledNextStates, nextActionValues); - std::cout << "nextActionValues: " << nextActionValues << std::endl; - arma::Col nextAction; if (config.DoubleQLearning()) { @@ -290,9 +280,6 @@ void QLearning< } arma::mat dists; learningNetwork.Forward(sampledStates, dists); - - std::cout << "dists: " << dists << std::endl; - arma::mat lossGradients = arma::zeros(arma::size(dists)); for (size_t i = 0; i < batchSize; ++i) { @@ -300,15 +287,10 @@ void QLearning< = -(projDist.col(i) / (1e-10 + dists(sampledActions[i].action * atomSize, i, arma::size(atomSize, 1)))); } - - std::cout << "lossGradients: " << lossGradients << std::endl; - // Learn from experience. arma::mat gradients; learningNetwork.Backward(sampledStates, lossGradients, gradients); - std::cout << "gradients: " << gradients << std::endl; - // TODO: verify for PER replayMethod.Update(lossGradients, sampledActions, nextActionValues, gradients); @@ -388,12 +370,9 @@ double QLearning< totalReturn += reward; totalSteps++; - state.Data() = {1.2, 0.1, -5, 0.8}; - action.action = CartPole::Action::actions::forward; - reward = 1.7; - nextState.Data() = {0.2, -0.1, -0.5, 1.8}; + // Store the transition for replay. replayMethod.Store(state, action, reward, nextState, - false, config.Discount()); + environment.IsTerminal(nextState), config.Discount()); // Update current state. state = nextState; diff --git a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp index 3f38f30344..30edbfdc99 100644 --- a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp @@ -67,7 +67,9 @@ class CategoricalDQN } else { - network.Add(new Linear<>(h1, outputDim * atomSize)); + network.Add(new Linear<>(h1, h2)); + network.Add(new ReLULayer<>()); + network.Add(new Linear<>(h2, outputDim * atomSize)); } } @@ -91,9 +93,7 @@ class CategoricalDQN void Predict(const arma::mat state, arma::mat& actionValue) { arma::mat q_atoms; - std::cout << "network params:" << network.Parameters() << std::endl; network.Predict(state, q_atoms); - std::cout << "q_atoms:" << q_atoms << std::endl; activations.copy_size(q_atoms); actionValue.set_size(q_atoms.n_rows / atomSize, q_atoms.n_cols); double vMin = 0, vMax = 200.0; From e8c912c7abe3fd04f72456ca974035bc88565807 Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Fri, 31 Jul 2020 03:35:05 +0530 Subject: [PATCH 08/19] copy constructor changes to q_learning networks --- .../reinforcement_learning/q_learning_impl.hpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 09d9b50fde..be60fb20a9 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -48,9 +48,14 @@ QLearning< totalSteps(0), deterministic(false) { + // To copy over the network structure. + targetNetwork = learningNetwork; + // Set up q-learning network. if (learningNetwork.Parameters().is_empty()) learningNetwork.ResetParameters(); + if (targetNetwork.Parameters().is_empty()) + targetNetwork.ResetParameters(); #if ENS_VERSION_MAJOR == 1 this->updater.Initialize(learningNetwork.Parameters().n_rows, @@ -62,7 +67,8 @@ QLearning< learningNetwork.Parameters().n_cols); #endif - targetNetwork = learningNetwork; + // Initialize the target network with the parameters of learning network. + targetNetwork.Parameters() = learningNetwork.Parameters(); } template < @@ -192,7 +198,7 @@ void QLearning< } // Update target network. if (totalSteps % config.TargetNetworkSyncInterval() == 0) - targetNetwork = learningNetwork; + targetNetwork.Parameters() = learningNetwork.Parameters(); if (totalSteps > config.ExplorationSteps()) policy.Anneal(); @@ -308,7 +314,7 @@ void QLearning< } // Update target network. if (totalSteps % config.TargetNetworkSyncInterval() == 0) - targetNetwork = learningNetwork; + targetNetwork.Parameters() = learningNetwork.Parameters(); if (totalSteps > config.ExplorationSteps()) policy.Anneal(); From cb4cdfe2584a3cd2bae83a5966baec3f678400f4 Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Fri, 31 Jul 2020 03:36:08 +0530 Subject: [PATCH 09/19] Adding docs and minor changes to prebuilt network init for categorical dqn --- .../q_networks/categorical_dqn.hpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp index 30edbfdc99..04889d0ff1 100644 --- a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp @@ -73,7 +73,16 @@ class CategoricalDQN } } - CategoricalDQN(NetworkType network, const bool isNoisy, size_t atomSize): + /** + * Construct an instance of CategoricalDQN class from a pre-constructed network. + * + * @param network The network to be used by CategoricalDQN class. + * @param isNoisy Specifies whether the network needs to be of type noisy. + * @param atomSize Specifies the number of atoms to be used. + */ + CategoricalDQN(NetworkType& network, + const bool isNoisy = false, + size_t atomSize = 51): network(std::move(network)), isNoisy(isNoisy), atomSize(atomSize) From ff8b99afa518c898be72fa281a47f30cad4d92a4 Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Fri, 31 Jul 2020 03:45:47 +0530 Subject: [PATCH 10/19] Added template parameters for outputlayer and init type, style fixes --- .../q_learning_impl.hpp | 11 +++----- .../q_networks/categorical_dqn.hpp | 26 ++++++++++++++----- src/mlpack/tests/q_learning_test.cpp | 2 +- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index be60fb20a9..9b6ae08104 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -289,17 +289,14 @@ void QLearning< arma::mat lossGradients = arma::zeros(arma::size(dists)); for (size_t i = 0; i < batchSize; ++i) { - lossGradients(sampledActions[i].action * atomSize, i, arma::size(atomSize, 1)) - = -(projDist.col(i) / (1e-10 + dists(sampledActions[i].action * atomSize, - i, arma::size(atomSize, 1)))); + lossGradients(sampledActions[i].action * atomSize, i, + arma::size(atomSize, 1)) = -(projDist.col(i) / (1e-10 + dists( + sampledActions[i].action * atomSize, i, arma::size(atomSize, 1)))); } // Learn from experience. arma::mat gradients; learningNetwork.Backward(sampledStates, lossGradients, gradients); - // TODO: verify for PER - replayMethod.Update(lossGradients, sampledActions, nextActionValues, gradients); - #if ENS_VERSION_MAJOR == 1 updater.Update(learningNetwork.Parameters(), config.StepSize(), gradients); #else @@ -387,7 +384,7 @@ double QLearning< if (config.IsCategorical()) TrainCategoricalAgent(); else - TrainAgent(); + TrainAgent(); } return totalReturn; } diff --git a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp index 04889d0ff1..2488b7b476 100644 --- a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp @@ -25,16 +25,22 @@ namespace rl { using namespace mlpack::ann; /** - * @tparam NetworkType The type of network used for categorical dqn. + * @tparam OutputLayerType The output layer type of the network. + * @tparam InitType The initialization type used for the network. + * @tparam NetworkType The type of network used for simple dqn. */ -template , GaussianInitialization>> +template< + typename OutputLayerType = EmptyLoss<>, + typename InitType = GaussianInitialization, + typename NetworkType = FFN +> class CategoricalDQN { public: /** * Default constructor. */ - CategoricalDQN() : network(), isNoisy(false), atomSize(0) + CategoricalDQN() : network(), isNoisy(false), atomSize(0) { /* Nothing to do here. */ } /** @@ -44,20 +50,26 @@ class CategoricalDQN * @param h1 Number of neurons in hiddenlayer-1. * @param h2 Number of neurons in hiddenlayer-2. * @param outputDim Number of neurons in output layer. + * @param isNoisy Specifies whether the network needs to be of type noisy. + * @param init Specifies the initialization rule for the network. + * @param outputLayer Specifies the output layer type for network. + * @param atomSize Specifies the number of atoms to be used. */ CategoricalDQN(const int inputDim, const int h1, const int h2, const int outputDim, const bool isNoisy = false, + InitType init = InitType(), + OutputLayerType outputLayer = OutputLayerType(), const size_t atomSize = 51): - network(EmptyLoss<>(), GaussianInitialization(0, 0.001)), + network(outputLayer, init), isNoisy(isNoisy), atomSize(atomSize) { network.Add(new Linear<>(inputDim, h1)); network.Add(new ReLULayer<>()); - if(isNoisy) + if (isNoisy) { noisyLayerIndex.push_back(network.Model().size()); network.Add(new NoisyLinear<>(h1, h2)); @@ -170,7 +182,9 @@ class CategoricalDQN * @param lossGardients The loss gradients. * @param gradient The gradient. */ - void Backward(const arma::mat state, arma::mat& lossGradients, arma::mat& gradient) + void Backward(const arma::mat state, + arma::mat& lossGradients, + arma::mat& gradient) { arma::mat activationGradients(arma::size(activations)); for(size_t i = 0; i < activations.n_rows; i += atomSize) diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index 7750dea9d1..a788401852 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -496,4 +496,4 @@ BOOST_AUTO_TEST_CASE(CartPoleWithCategoricalDQN) BOOST_REQUIRE(converged); } -BOOST_AUTO_TEST_SUITE_END(); \ No newline at end of file +BOOST_AUTO_TEST_SUITE_END(); From bb2050851c526dd1e1b5e7322147e45a8f8ba783 Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Fri, 31 Jul 2020 09:59:51 +0530 Subject: [PATCH 11/19] Added acrobot test for categorical --- src/mlpack/tests/q_learning_test.cpp | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index a788401852..4f0ffb3e09 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -464,8 +465,8 @@ BOOST_AUTO_TEST_CASE(CartPoleWithNStepPrioritizedDQN) BOOST_REQUIRE(converged); } -//! Test Categorical DQN in Cart Pole task. -BOOST_AUTO_TEST_CASE(CartPoleWithCategoricalDQN) +//! Test Categorical DQN in Acrobot task. +BOOST_AUTO_TEST_CASE(AcrobotWithCategoricalDQN) { // It isn't guaranteed that the network will converge in the specified number // of iterations. @@ -475,21 +476,28 @@ BOOST_AUTO_TEST_CASE(CartPoleWithCategoricalDQN) Log::Debug << "Trial number: " << trial << std::endl; // Set up the policy and replay method. - GreedyPolicy policy(1.0, 1000, 0.1, 0.99); - RandomReplay replayMethod(32, 2000); + GreedyPolicy policy(1.0, 1000, 0.1, 0.99); + RandomReplay replayMethod(32, 10000); TrainingConfig config; - config.StepLimit() = 200; config.IsCategorical() = true; + config.ExplorationSteps() = 64; - // Set up the network with a flag to enable noisy layers. - CategoricalDQN<> network(4, 128, 128, 2); + // Set up the module. Note that we use a custom network here. + FFN, GaussianInitialization> module( + EmptyLoss<>(), GaussianInitialization(0, 0.1)); + module.Add>(4, 128); + module.Add>(); + module.Add>(128, 3 * 51); + + // Adding the module to the CategoricalDQN network. + CategoricalDQN<> network(module); // Set up DQN agent. - QLearning + QLearning agent(config, network, policy, replayMethod); - converged = testAgent(agent, 40, 500, 50); + converged = testAgent(agent, -380, 1000); if (converged) break; } From fbedf281a86d2a2bcfd82b9a6efe0fcdf5e2985f Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Fri, 31 Jul 2020 10:36:21 +0530 Subject: [PATCH 12/19] Doc and style fixes --- .../reinforcement_learning/q_learning_impl.hpp | 5 +---- .../q_networks/categorical_dqn.hpp | 12 ++++++------ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 9b6ae08104..04f3117090 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -268,10 +268,7 @@ void QLearning< arma::mat b = (tZ - vMin) / (vMax - vMin) * (atomSize - 1); arma::mat l = arma::floor(b); arma::mat u = arma::ceil(b); - // arma::umat offset(atomSize, batchSize); - // offset.each_row() = arma::linspace(0, (batchSize - 1) * - // atomSize, batchSize); - + arma::mat projDistUpper = nextDist % (u - b); arma::mat projDistLower = nextDist % (b - l); diff --git a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp index 2488b7b476..14540f7d00 100644 --- a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp @@ -70,7 +70,7 @@ class CategoricalDQN network.Add(new Linear<>(inputDim, h1)); network.Add(new ReLULayer<>()); if (isNoisy) - { + { noisyLayerIndex.push_back(network.Model().size()); network.Add(new NoisyLinear<>(h1, h2)); network.Add(new ReLULayer<>()); @@ -119,7 +119,7 @@ class CategoricalDQN actionValue.set_size(q_atoms.n_rows / atomSize, q_atoms.n_cols); double vMin = 0, vMax = 200.0; arma::rowvec support = arma::linspace(vMin, vMax, atomSize); - for(size_t i = 0; i < q_atoms.n_rows; i += atomSize) + for (size_t i = 0; i < q_atoms.n_rows; i += atomSize) { arma::mat activation = activations.rows(i, i + atomSize - 1); arma::mat input = q_atoms.rows(i, i + atomSize - 1); @@ -140,7 +140,7 @@ class CategoricalDQN arma::mat q_atoms; network.Forward(state, q_atoms); activations.copy_size(q_atoms); - for(size_t i = 0; i < q_atoms.n_rows; i += atomSize) + for (size_t i = 0; i < q_atoms.n_rows; i += atomSize) { arma::mat activation = activations.rows(i, i + atomSize - 1); arma::mat input = q_atoms.rows(i, i + atomSize - 1); @@ -163,7 +163,7 @@ class CategoricalDQN */ void ResetNoise() { - for(size_t i = 0; i < noisyLayerIndex.size(); i++) + for (size_t i = 0; i < noisyLayerIndex.size(); i++) { boost::get*> (network.Model()[noisyLayerIndex[i]])->ResetNoise(); @@ -179,7 +179,7 @@ class CategoricalDQN * Perform the backward pass of the state in real batch mode. * * @param state The input state. - * @param lossGardients The loss gradients. + * @param lossGradients The loss gradients. * @param gradient The gradient. */ void Backward(const arma::mat state, @@ -187,7 +187,7 @@ class CategoricalDQN arma::mat& gradient) { arma::mat activationGradients(arma::size(activations)); - for(size_t i = 0; i < activations.n_rows; i += atomSize) + for (size_t i = 0; i < activations.n_rows; i += atomSize) { arma::mat activationGrad; arma::mat lossGrad = lossGradients.rows(i, i + atomSize - 1); From 151494431681709ef7a251aaca47de793335cb90 Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Sat, 1 Aug 2020 19:29:59 +0530 Subject: [PATCH 13/19] Minor change to the reward function of the environment --- .../methods/reinforcement_learning/environment/cart_pole.hpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp b/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp index 3a30a522fb..b3fe48d375 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp @@ -123,7 +123,7 @@ class CartPole const double tau = 0.02, const double thetaThresholdRadians = 12 * 2 * 3.1416 / 360, const double xThreshold = 2.4, - const double doneReward = 0.0) : + const double doneReward = 1.0) : maxSteps(maxSteps), gravity(gravity), massCart(massCart), @@ -177,8 +177,6 @@ class CartPole // Do not reward agent if it failed. if (done && maxSteps != 0 && stepsPerformed >= maxSteps) return doneReward; - else if (done) - return 0; /** * When done is false, it means that the cartpole has fallen down. From 4137d2d51bd5384a6b6a6543ced7109c00cc08c5 Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Sat, 1 Aug 2020 19:33:34 +0530 Subject: [PATCH 14/19] Added categorical test for cartpole! --- src/mlpack/tests/q_learning_test.cpp | 34 +++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index 4f0ffb3e09..9eec6bd828 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -465,6 +465,38 @@ BOOST_AUTO_TEST_CASE(CartPoleWithNStepPrioritizedDQN) BOOST_REQUIRE(converged); } +//! Test Categorical DQN in Cart Pole task. +BOOST_AUTO_TEST_CASE(CartPoleWithCategoricalDQN) +{ + // It isn't guaranteed that the network will converge in the specified number + // of iterations. + bool converged = false; + for (size_t trial = 0; trial < 3; ++trial) + { + Log::Debug << "Trial number: " << trial << std::endl; + + // Set up the policy and replay method. + GreedyPolicy policy(1.0, 1000, 0.1, 0.99); + RandomReplay replayMethod(32, 10000); + + TrainingConfig config; + config.IsCategorical() = true; + config.ExplorationSteps() = 32; + + // Set up the CategoricalDQN network. + CategoricalDQN<> network(4, 64, 64, 2); + + // Set up DQN agent. + QLearning + agent(config, network, policy, replayMethod); + + converged = testAgent(agent, 60, 500, 20); + if (converged) + break; + } + BOOST_REQUIRE(converged); +} + //! Test Categorical DQN in Acrobot task. BOOST_AUTO_TEST_CASE(AcrobotWithCategoricalDQN) { @@ -497,7 +529,7 @@ BOOST_AUTO_TEST_CASE(AcrobotWithCategoricalDQN) QLearning agent(config, network, policy, replayMethod); - converged = testAgent(agent, -380, 1000); + converged = testAgent(agent, -380, 1000, 20); if (converged) break; } From ca15458de4ea9478bf72c7e15a9dce50cb44cc63 Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Sat, 1 Aug 2020 20:02:23 +0530 Subject: [PATCH 15/19] Updated HISTORY --- HISTORY.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index a941b58491..088a25c748 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,7 @@ ### mlpack ?.?.? ###### ????-??-?? + * Added Categorical DQN to q_networks (#2454). + * Added N-step DQN to q_networks (#2461). * Add Silhoutte Score metric and Pairwise Distances (#2406). From afbe693cb45eaa9931f1e91d763cf2b5a215dca4 Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Sat, 1 Aug 2020 21:31:07 +0530 Subject: [PATCH 16/19] changing tests for categorical --- src/mlpack/tests/q_learning_test.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index 9eec6bd828..aa95dabc71 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -477,20 +477,27 @@ BOOST_AUTO_TEST_CASE(CartPoleWithCategoricalDQN) // Set up the policy and replay method. GreedyPolicy policy(1.0, 1000, 0.1, 0.99); - RandomReplay replayMethod(32, 10000); + RandomReplay replayMethod(32, 4000); TrainingConfig config; config.IsCategorical() = true; config.ExplorationSteps() = 32; - // Set up the CategoricalDQN network. - CategoricalDQN<> network(4, 64, 64, 2); + // Set up the module. Note that we use a custom network here. + FFN, GaussianInitialization> module( + EmptyLoss<>(), GaussianInitialization(0, 0.1)); + module.Add>(4, 128); + module.Add>(); + module.Add>(128, 2 * 51); + + // Adding the module to the CategoricalDQN network. + CategoricalDQN<> network(module); // Set up DQN agent. QLearning agent(config, network, policy, replayMethod); - converged = testAgent(agent, 60, 500, 20); + converged = testAgent(agent, 40, 1000, 20); if (converged) break; } From 4a7c65962df98340af2410484b5632e0a42fce58 Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Mon, 3 Aug 2020 20:25:44 +0530 Subject: [PATCH 17/19] removed acrobot test --- src/mlpack/tests/q_learning_test.cpp | 39 ---------------------------- 1 file changed, 39 deletions(-) diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index aa95dabc71..06e033847a 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -504,43 +504,4 @@ BOOST_AUTO_TEST_CASE(CartPoleWithCategoricalDQN) BOOST_REQUIRE(converged); } -//! Test Categorical DQN in Acrobot task. -BOOST_AUTO_TEST_CASE(AcrobotWithCategoricalDQN) -{ - // It isn't guaranteed that the network will converge in the specified number - // of iterations. - bool converged = false; - for (size_t trial = 0; trial < 3; ++trial) - { - Log::Debug << "Trial number: " << trial << std::endl; - - // Set up the policy and replay method. - GreedyPolicy policy(1.0, 1000, 0.1, 0.99); - RandomReplay replayMethod(32, 10000); - - TrainingConfig config; - config.IsCategorical() = true; - config.ExplorationSteps() = 64; - - // Set up the module. Note that we use a custom network here. - FFN, GaussianInitialization> module( - EmptyLoss<>(), GaussianInitialization(0, 0.1)); - module.Add>(4, 128); - module.Add>(); - module.Add>(128, 3 * 51); - - // Adding the module to the CategoricalDQN network. - CategoricalDQN<> network(module); - - // Set up DQN agent. - QLearning - agent(config, network, policy, replayMethod); - - converged = testAgent(agent, -380, 1000, 20); - if (converged) - break; - } - BOOST_REQUIRE(converged); -} - BOOST_AUTO_TEST_SUITE_END(); From c8e4f08e9cc0c36cde13378efde6067074002148 Mon Sep 17 00:00:00 2001 From: nishantkr18 Date: Tue, 4 Aug 2020 23:37:02 +0530 Subject: [PATCH 18/19] Doc changes and adding categorical parameters to training configs --- .../q_learning_impl.hpp | 17 +++-- .../q_networks/categorical_dqn.hpp | 64 +++++++++++++------ .../training_config.hpp | 48 +++++++++++++- src/mlpack/tests/q_learning_test.cpp | 4 +- 4 files changed, 98 insertions(+), 35 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 04f3117090..f6ef0078a1 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -52,10 +52,8 @@ QLearning< targetNetwork = learningNetwork; // Set up q-learning network. - if (learningNetwork.Parameters().is_empty()) - learningNetwork.ResetParameters(); - if (targetNetwork.Parameters().is_empty()) - targetNetwork.ResetParameters(); + learningNetwork.ResetParameters(); + targetNetwork.ResetParameters(); #if ENS_VERSION_MAJOR == 1 this->updater.Initialize(learningNetwork.Parameters().n_rows, @@ -231,9 +229,9 @@ void QLearning< replayMethod.Sample(sampledStates, sampledActions, sampledRewards, sampledNextStates, isTerminal); - double vMin = 0, vMax = 200.0; - size_t atomSize = 51; - arma::rowvec support = arma::linspace(vMin, vMax, atomSize); + size_t atomSize = config.AtomSize(); + arma::rowvec support = arma::linspace(config.VMin(), + config.VMax(), atomSize); size_t batchSize = sampledNextStates.n_cols; @@ -264,8 +262,9 @@ void QLearning< arma::mat tZ = (arma::conv_to::from(config.Discount() * ((1 - isTerminal) * support)).each_col() + sampledRewards).t(); - tZ = arma::clamp(tZ, vMin, vMax); - arma::mat b = (tZ - vMin) / (vMax - vMin) * (atomSize - 1); + tZ = arma::clamp(tZ, config.VMin(), config.VMax()); + arma::mat b = (tZ - config.VMin()) / (config.VMax() - config.VMin()) * + (atomSize - 1); arma::mat l = arma::floor(b); arma::mat u = arma::ceil(b); diff --git a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp index 14540f7d00..694235fa8e 100644 --- a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp @@ -18,6 +18,7 @@ #include #include #include +#include "../training_config.hpp" namespace mlpack { namespace rl { @@ -25,6 +26,18 @@ namespace rl { using namespace mlpack::ann; /** + * Implementation of the Distributional Deep Q-Learning network. + * For more information, see the following. + * + * @code + * @misc{bellemare2017distributional, + * author = {Marc G. Bellemare, Will Dabney, Rémi Munos}, + * title = {A Distributional Perspective on Reinforcement Learning}, + * year = {2017}, + * url = {http://arxiv.org/abs/1707.06887} + * } + * @endcode + * * @tparam OutputLayerType The output layer type of the network. * @tparam InitType The initialization type used for the network. * @tparam NetworkType The type of network used for simple dqn. @@ -40,7 +53,7 @@ class CategoricalDQN /** * Default constructor. */ - CategoricalDQN() : network(), isNoisy(false), atomSize(0) + CategoricalDQN() : network(), isNoisy(false) { /* Nothing to do here. */ } /** @@ -50,22 +63,24 @@ class CategoricalDQN * @param h1 Number of neurons in hiddenlayer-1. * @param h2 Number of neurons in hiddenlayer-2. * @param outputDim Number of neurons in output layer. + * @param config Hyper-parameters for categorical dqn. * @param isNoisy Specifies whether the network needs to be of type noisy. * @param init Specifies the initialization rule for the network. * @param outputLayer Specifies the output layer type for network. - * @param atomSize Specifies the number of atoms to be used. */ CategoricalDQN(const int inputDim, - const int h1, - const int h2, - const int outputDim, - const bool isNoisy = false, - InitType init = InitType(), - OutputLayerType outputLayer = OutputLayerType(), - const size_t atomSize = 51): + const int h1, + const int h2, + const int outputDim, + TrainingConfig config, + const bool isNoisy = false, + InitType init = InitType(), + OutputLayerType outputLayer = OutputLayerType()): network(outputLayer, init), - isNoisy(isNoisy), - atomSize(atomSize) + atomSize(config.AtomSize()), + vMin(config.VMin()), + vMax(config.VMax()), + isNoisy(isNoisy) { network.Add(new Linear<>(inputDim, h1)); network.Add(new ReLULayer<>()); @@ -89,15 +104,17 @@ class CategoricalDQN * Construct an instance of CategoricalDQN class from a pre-constructed network. * * @param network The network to be used by CategoricalDQN class. + * @param config Hyper-parameters for categorical dqn. * @param isNoisy Specifies whether the network needs to be of type noisy. - * @param atomSize Specifies the number of atoms to be used. */ CategoricalDQN(NetworkType& network, - const bool isNoisy = false, - size_t atomSize = 51): + TrainingConfig config, + const bool isNoisy = false): network(std::move(network)), - isNoisy(isNoisy), - atomSize(atomSize) + atomSize(config.AtomSize()), + vMin(config.VMin()), + vMax(config.VMax()), + isNoisy(isNoisy) { /* Nothing to do here. */ } /** @@ -117,7 +134,6 @@ class CategoricalDQN network.Predict(state, q_atoms); activations.copy_size(q_atoms); actionValue.set_size(q_atoms.n_rows / atomSize, q_atoms.n_cols); - double vMin = 0, vMax = 200.0; arma::rowvec support = arma::linspace(vMin, vMax, atomSize); for (size_t i = 0; i < q_atoms.n_rows; i += atomSize) { @@ -159,7 +175,7 @@ class CategoricalDQN } /** - * Resets noise of the network, is the network is of type noisy. + * Resets noise of the network, if the network is of type noisy. */ void ResetNoise() { @@ -202,12 +218,18 @@ class CategoricalDQN //! Locally-stored network. NetworkType network; - //! Locally-stored check for noisy network. - bool isNoisy; - //! Locally-stored number of atoms. size_t atomSize; + //! Locally-stored minimum value of support. + double vMin; + + //! Locally-stored maximum value of support. + double vMax; + + //! Locally-stored check for noisy network. + bool isNoisy; + //! Locally-stored indexes of noisy layers in the network. std::vector noisyLayerIndex; diff --git a/src/mlpack/methods/reinforcement_learning/training_config.hpp b/src/mlpack/methods/reinforcement_learning/training_config.hpp index 66b4493607..a9dbd8544d 100644 --- a/src/mlpack/methods/reinforcement_learning/training_config.hpp +++ b/src/mlpack/methods/reinforcement_learning/training_config.hpp @@ -30,7 +30,10 @@ class TrainingConfig gradientLimit(40), doubleQLearning(false), noisyQLearning(false), - isCategorical(false) + isCategorical(false), + atomSize(51), + vMin(0), + vMax(200) { /* Nothing to do here. */ } TrainingConfig( @@ -44,7 +47,10 @@ class TrainingConfig double gradientLimit, bool doubleQLearning, bool noisyQLearning, - bool isCategorical) : + bool isCategorical, + size_t atomSize, + double vMin, + double vMax) : numWorkers(numWorkers), updateInterval(updateInterval), targetNetworkSyncInterval(targetNetworkSyncInterval), @@ -55,7 +61,10 @@ class TrainingConfig gradientLimit(gradientLimit), doubleQLearning(doubleQLearning), noisyQLearning(noisyQLearning), - isCategorical(isCategorical) + isCategorical(isCategorical), + atomSize(atomSize), + vMin(vMin), + vMax(vMax) { /* Nothing to do here. */ } //! Get the amount of workers. @@ -117,6 +126,21 @@ class TrainingConfig //! Modify the indicator of categorical q-learning. bool& IsCategorical() { return isCategorical; } + //! Get the number of atoms. + size_t AtomSize() const { return atomSize; } + //! Modify the number of atoms. + size_t& AtomSize() { return atomSize; } + + //! Get the minimum value for support. + double VMin() const { return vMin; } + //! Modify the minimum value for support. + double& VMin() { return vMin; } + + //! Get the maximum value for support. + double VMax() const { return vMax; } + //! Modify the maximum value for support. + double& VMax() { return vMax; } + private: /** * Locally-stored number of workers. @@ -186,6 +210,24 @@ class TrainingConfig * This is valid only for q-learning agent. */ bool isCategorical; + + /** + * Locally-stored number of atoms to be used. + * This is valid only for categorical q-network. + */ + size_t atomSize; + + /** + * Locally-stored minimum value of support. + * This is valid only for categorical q-network. + */ + double vMin; + + /** + * Locally-stored maximum value of support. + * This is valid only for categorical q-network. + */ + double vMax; }; } // namespace rl diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index 06e033847a..308ed2b37f 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -488,10 +488,10 @@ BOOST_AUTO_TEST_CASE(CartPoleWithCategoricalDQN) EmptyLoss<>(), GaussianInitialization(0, 0.1)); module.Add>(4, 128); module.Add>(); - module.Add>(128, 2 * 51); + module.Add>(128, 2 * config.AtomSize()); // Adding the module to the CategoricalDQN network. - CategoricalDQN<> network(module); + CategoricalDQN<> network(module, config); // Set up DQN agent. QLearning From f089e49766753cb49e0b0ededbcc45c5328429cf Mon Sep 17 00:00:00 2001 From: Nishant Kumar Date: Thu, 6 Aug 2020 00:04:58 +0530 Subject: [PATCH 19/19] Update src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp Co-authored-by: favre49 <40389657+favre49@users.noreply.github.com> --- .../reinforcement_learning/q_networks/categorical_dqn.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp index 694235fa8e..b52110d744 100644 --- a/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_networks/categorical_dqn.hpp @@ -26,7 +26,7 @@ namespace rl { using namespace mlpack::ann; /** - * Implementation of the Distributional Deep Q-Learning network. + * Implementation of the Categorical Deep Q-Learning network. * For more information, see the following. * * @code