diff --git a/src/mlpack/methods/reinforcement_learning/q_learning.hpp b/src/mlpack/methods/reinforcement_learning/q_learning.hpp index 3dfbfd3d5e..be737cd748 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning.hpp @@ -77,10 +77,10 @@ class QLearning * @param updater How to apply gradients when training. * @param environment Reinforcement learning task. */ - QLearning(TrainingConfig config, + QLearning(TrainingConfig& config, NetworkType& network, - PolicyType policy, - ReplayType replayMethod, + PolicyType& policy, + ReplayType& replayMethod, UpdaterType updater = UpdaterType(), EnvironmentType environment = EnvironmentType()); @@ -138,7 +138,7 @@ class QLearning arma::Col BestAction(const arma::mat& actionValues); //! Locally-stored hyper-parameters. - TrainingConfig config; + TrainingConfig& config; //! Locally-stored learning network. NetworkType& learningNetwork; @@ -153,10 +153,10 @@ class QLearning #endif //! Locally-stored behavior policy. - PolicyType policy; + PolicyType& policy; //! Locally-stored experience method. - ReplayType replayMethod; + ReplayType& replayMethod; //! Locally-stored reinforcement learning task. EnvironmentType environment; diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 539f5b0f81..0741ff6e58 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -30,20 +30,20 @@ QLearning< UpdaterType, PolicyType, ReplayType ->::QLearning(TrainingConfig config, +>::QLearning(TrainingConfig& config, NetworkType& network, - PolicyType policy, - ReplayType replayMethod, + PolicyType& policy, + ReplayType& replayMethod, UpdaterType updater, EnvironmentType environment): - config(std::move(config)), + config(config), learningNetwork(network), + policy(policy), + replayMethod(replayMethod), updater(std::move(updater)), #if ENS_VERSION_MAJOR >= 2 updatePolicy(NULL), #endif - policy(std::move(policy)), - replayMethod(std::move(replayMethod)), environment(std::move(environment)), totalSteps(0), deterministic(false) diff --git a/src/mlpack/methods/reinforcement_learning/q_networks/dueling_dqn.hpp b/src/mlpack/methods/reinforcement_learning/q_networks/dueling_dqn.hpp index b993d8a563..1c1185be8a 100644 --- a/src/mlpack/methods/reinforcement_learning/q_networks/dueling_dqn.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_networks/dueling_dqn.hpp @@ -77,9 +77,9 @@ class DuelingDQN * @param outputDim Number of neurons in output layer. */ DuelingDQN(const int inputDim, - const int h1, - const int h2, - const int outputDim): + const int h1, + const int h2, + const int outputDim): completeNetwork(EmptyLoss<>(), GaussianInitialization(0, 0.001)) { featureNetwork = new Sequential<>(); diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index 56199e7a04..220202798a 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -43,7 +43,7 @@ BOOST_AUTO_TEST_SUITE(QLearningTest); BOOST_AUTO_TEST_CASE(CartPoleWithDQN) { // Set up the network. - SimpleDQN<> model(4, 128, 128, 2); + SimpleDQN<> network(4, 128, 128, 2); // Set up the policy and replay method. GreedyPolicy policy(1.0, 1000, 0.1, 0.99); @@ -58,9 +58,8 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDQN) config.StepLimit() = 200; // Set up DQN agent. - QLearning - agent(std::move(config), model, std::move(policy), - std::move(replayMethod)); + QLearning + agent(config, network, policy, replayMethod); arma::running_stat averageReturn; size_t episodes = 0; @@ -105,7 +104,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDQN) BOOST_AUTO_TEST_CASE(CartPoleWithDQNPrioritizedReplay) { // Set up the network. - SimpleDQN<> model(4, 128, 128, 2); + SimpleDQN<> network(4, 128, 128, 2); // Set up the policy and replay method. GreedyPolicy policy(1.0, 1000, 0.1); @@ -120,10 +119,9 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDQNPrioritizedReplay) config.StepLimit() = 200; // Set up DQN agent. - QLearning - agent(std::move(config), model, std::move(policy), - std::move(replayMethod)); + agent(config, network, policy, replayMethod); arma::running_stat averageReturn; size_t episodes = 0; @@ -174,7 +172,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDoubleDQN) for (size_t trial = 0; trial < 4; ++trial) { // Set up the network. - SimpleDQN<> model(4, 20, 20, 2); + SimpleDQN<> network(4, 20, 20, 2); // Set up the policy and replay method. GreedyPolicy policy(1.0, 1000, 0.1, 0.99); @@ -189,9 +187,8 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDoubleDQN) config.StepLimit() = 200; // Set up the DQN agent. - QLearning - agent(std::move(config), model, std::move(policy), - std::move(replayMethod)); + QLearning + agent(config, network, policy, replayMethod); arma::running_stat averageReturn; @@ -237,7 +234,7 @@ BOOST_AUTO_TEST_CASE(AcrobotWithDQN) for (size_t trial = 0; trial < 3; ++trial) { // Set up the network. - SimpleDQN<> model(4, 64, 32, 3); + SimpleDQN<> network(4, 64, 32, 3); // Set up the policy and replay method. GreedyPolicy policy(1.0, 1000, 0.1, 0.99); @@ -252,9 +249,8 @@ BOOST_AUTO_TEST_CASE(AcrobotWithDQN) config.StepLimit() = 400; // Set up DQN agent. - QLearning - agent(std::move(config), model, std::move(policy), - std::move(replayMethod)); + QLearning + agent(config, network, policy, replayMethod); arma::running_stat averageReturn; size_t episodes = 0; @@ -308,7 +304,7 @@ BOOST_AUTO_TEST_CASE(MountainCarWithDQN) for (size_t trial = 0; trial < 3; trial++) { // Set up the network. - SimpleDQN<> model(2, 64, 32, 3); + SimpleDQN<> network(2, 64, 32, 3); // Set up the policy and replay method. GreedyPolicy policy(1.0, 1000, 0.1, 0.99); @@ -323,9 +319,8 @@ BOOST_AUTO_TEST_CASE(MountainCarWithDQN) config.StepLimit() = 400; // Set up DQN agent. - QLearning - agent(std::move(config), model, std::move(policy), - std::move(replayMethod)); + QLearning + agent(config, network, policy, replayMethod); arma::running_stat averageReturn; size_t episodes = 0; @@ -378,13 +373,13 @@ BOOST_AUTO_TEST_CASE(DoublePoleCartWithDQN) bool success = false; for (size_t trial = 0; trial < 4; trial++) { - // Set up the network. Note that we use a custom model here, and + // Set up the network. Note that we use a custom network here, and // pass it directly into the agent, without using SimpleDQN. - FFN, GaussianInitialization> model(MeanSquaredError<>(), + FFN, GaussianInitialization> network(MeanSquaredError<>(), GaussianInitialization(0, 0.001)); - model.Add>(6, 256); - model.Add>(); - model.Add>(256, 3); + network.Add>(6, 256); + network.Add>(); + network.Add>(256, 3); // Set up the policy and replay method. GreedyPolicy policy(1.0, 1000, 0.1, 0.99); @@ -399,9 +394,8 @@ BOOST_AUTO_TEST_CASE(DoublePoleCartWithDQN) config.StepLimit() = 600; // Set up DQN agent. - QLearning - agent(std::move(config), model, std::move(policy), - std::move(replayMethod)); + QLearning + agent(config, network, policy, replayMethod); size_t episodes = 0; bool converged = true; @@ -421,7 +415,7 @@ BOOST_AUTO_TEST_CASE(DoublePoleCartWithDQN) break; } - // If the model can solve the environment in two trials this is fine for + // If the network can solve the environment in two trials this is fine for // a simple test. if (episodeSuccesses >= 2) { @@ -446,7 +440,7 @@ BOOST_AUTO_TEST_CASE(DoublePoleCartWithDQN) BOOST_AUTO_TEST_CASE(CartPoleWithDuelingDQN) { // Set up the network. - DuelingDQN<> model(4, 128, 64, 2); + DuelingDQN<> network(4, 128, 64, 2); // Set up the policy and replay method. GreedyPolicy policy(1.0, 1000, 0.1, 0.99); @@ -461,9 +455,8 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDuelingDQN) config.StepLimit() = 200; // Set up DQN agent. - QLearning - agent(std::move(config), model, std::move(policy), - std::move(replayMethod)); + QLearning + agent(config, network, policy, replayMethod); arma::running_stat averageReturn; size_t episodes = 0; @@ -499,8 +492,6 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDuelingDQN) break; } } - // To check if the action returned by the agent is not nan and is finite. - BOOST_REQUIRE(std::isfinite(double(agent.Action()))); BOOST_REQUIRE(converged); } @@ -509,7 +500,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDuelingDQN) BOOST_AUTO_TEST_CASE(CartPoleWithDuelingDQNPrioritizedReplay) { // Set up the network. - DuelingDQN<> model(4, 128, 64, 2); + DuelingDQN<> network(4, 128, 64, 2); // Set up the policy and replay method. GreedyPolicy policy(1.0, 1000, 0.1); @@ -524,10 +515,9 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDuelingDQNPrioritizedReplay) config.StepLimit() = 200; // Set up DQN agent. - QLearning - agent(std::move(config), model, std::move(policy), - std::move(replayMethod)); + agent(config, network, policy, replayMethod); arma::running_stat averageReturn; size_t episodes = 0;