From e5a4c40afb7d3a8858b8b6361c2b883514fb72a9 Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Sun, 1 Sep 2019 10:52:53 +0530 Subject: [PATCH 1/2] Reverted action.action to a double array --- .../environment/continuous_double_pole_cart.hpp | 4 ++-- .../environment/continuous_mountain_car.hpp | 6 +++--- .../methods/reinforcement_learning/environment/pendulum.hpp | 4 ++-- src/mlpack/tests/reward_clipping_test.cpp | 2 +- src/mlpack/tests/rl_components_test.cpp | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_double_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_double_pole_cart.hpp index 046e6a2619..3e87b82ac7 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_double_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_double_pole_cart.hpp @@ -91,7 +91,7 @@ class ContinuousDoublePoleCart */ struct Action { - double action = 0.0; + double action[1]; // Storing degree of freedom const int size = 1; }; @@ -193,7 +193,7 @@ class ContinuousDoublePoleCart const Action& action, arma::vec& dydx) { - double totalForce = action.action; + double totalForce = action.action[0]; double totalMass = massCart; double omega1 = state.AngularVelocity(1); double omega2 = state.AngularVelocity(2); diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp index 711c63d461..5837e1b147 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp @@ -83,7 +83,7 @@ class ContinuousMountainCar */ struct Action { - double action = 0.0; + double action[1]; // Storing degree of freedom const int size = 1; }; @@ -136,7 +136,7 @@ class ContinuousMountainCar stepsPerformed++; // Calculate acceleration. - double force = std::min(std::max(action.action, -1.0), 1.0); + double force = std::min(std::max(action.action[0], -1.0), 1.0); // Update states. nextState.Velocity() = state.Velocity() + force * power - 0.0025 * @@ -158,7 +158,7 @@ class ContinuousMountainCar else if (done) return doneReward; - return std::pow(action.action, 2) * 0.1; + return std::pow(action.action[0], 2) * 0.1; } /** diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index 4023d3b9bf..e7133bdf6b 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -82,7 +82,7 @@ class Pendulum */ struct Action { - double action = 0.0; + double action[1]; // Storing degree of freedom const int size = 1; }; @@ -141,7 +141,7 @@ class Pendulum // Get action and clip the values between max and min limits. double torque = std::min( - std::max(action.action, -maxTorque), maxTorque); + std::max(action.action[0], -maxTorque), maxTorque); // Calculate costs of taking this action in the current state. double costs = std::pow(AngleNormalize(theta), 2) + 0.1 * diff --git a/src/mlpack/tests/reward_clipping_test.cpp b/src/mlpack/tests/reward_clipping_test.cpp index e219b4461e..a153dea67a 100644 --- a/src/mlpack/tests/reward_clipping_test.cpp +++ b/src/mlpack/tests/reward_clipping_test.cpp @@ -47,7 +47,7 @@ BOOST_AUTO_TEST_CASE(ClippedRewardTest) RewardClipping::State state = rewardClipping.InitialSample(); RewardClipping::Action action; - action.action = mlpack::math::Random(-1.0, 1.0); + action.action[0] = mlpack::math::Random(-1.0, 1.0); double reward = rewardClipping.Sample(state, action); BOOST_REQUIRE(reward <= 2.0); diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index aec293e442..b6d3c42aa4 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -42,7 +42,7 @@ BOOST_AUTO_TEST_CASE(SimplePendulumTest) Pendulum::State state = task.InitialSample(); Pendulum::Action action; - action.action = math::Random(-2.0, 2.0); + action.action[0] = math::Random(-2.0, 2.0); double reward = task.Sample(state, action); // The reward is always negative. Check if not lower than lowest possible. @@ -71,7 +71,7 @@ BOOST_AUTO_TEST_CASE(SimpleContinuousMountainCarTest) ContinuousMountainCar::State state = task.InitialSample(); ContinuousMountainCar::Action action; - action.action = math::Random(-1.0, 1.0); + action.action[0] = math::Random(-1.0, 1.0); double reward = task.Sample(state, action); // Maximum reward possible is 100. BOOST_REQUIRE(reward <= 100.0); @@ -199,7 +199,7 @@ BOOST_AUTO_TEST_CASE(ContinuousDoublePoleCartTest) ContinuousDoublePoleCart::State state = task.InitialSample(); ContinuousDoublePoleCart::Action action; - action.action = math::Random(-1.0, 1.0); + action.action[0] = math::Random(-1.0, 1.0); double reward = task.Sample(state, action); BOOST_REQUIRE_EQUAL(reward, 1.0); From 7160d5cfa20f266a0ed05be92c75d093face679e Mon Sep 17 00:00:00 2001 From: Rahul Prabhu Date: Sun, 1 Sep 2019 19:20:11 +0530 Subject: [PATCH 2/2] Fixed HISTORY --- HISTORY.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index a2e38c1056..acd9ff4257 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -17,9 +17,6 @@ * Fix prediction output of softmax regression when test set accuracy is calculated (#1922). - * Action struct in continuous RL environments now stores the action as a - `double` instead of `double[1]` (#1941, #1931). - * Pendulum environment now checks for termination. All RL environments now have an option to terminate after a set number of time steps (no limit by default) (#1941).