From 36eb28eff73e86cf0acd3916879f382234dd8f7e Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Sun, 15 Mar 2020 12:48:46 +0530 Subject: [PATCH 01/12] change variable "power" to "duration" I think that "duration"(indicating time duration in which force is applied) instead of "power" is more appropriate. --- .../environment/continuous_mountain_car.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp index 5837e1b147..4d5eccce56 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp @@ -96,7 +96,7 @@ class ContinuousMountainCar * @param positionGoal Final target position. * @param velocityMin Minimum legal velocity. * @param velocityMax Maximum legal velocity. - * @param power Power generated by car. + * @param duration time duration for which force is apllied on the car. * @param doneReward Reward recieved by the agent on success. * @param maxSteps The number of steps after which the episode * terminates. If the value is 0, there is no limit. @@ -106,7 +106,7 @@ class ContinuousMountainCar const double positionGoal = 0.45, const double velocityMin = -0.07, const double velocityMax = 0.07, - const double power = 0.0015, + const double duration = 0.0015, const double doneReward = 100, const size_t maxSteps = 0) : positionMin(positionMin), @@ -114,7 +114,7 @@ class ContinuousMountainCar positionGoal(positionGoal), velocityMin(velocityMin), velocityMax(velocityMax), - power(power), + duration(duration), doneReward(doneReward), maxSteps(maxSteps), stepsPerformed(0) @@ -139,7 +139,7 @@ class ContinuousMountainCar double force = std::min(std::max(action.action[0], -1.0), 1.0); // Update states. - nextState.Velocity() = state.Velocity() + force * power - 0.0025 * + nextState.Velocity() = state.Velocity() + force * duration - 0.0025 * std::cos(3 * state.Position()); nextState.Velocity() = std::min( std::max(nextState.Velocity(), velocityMin), velocityMax); @@ -236,8 +236,8 @@ class ContinuousMountainCar //! Locally-stored maximum legal velocity. double velocityMax; - //! Locally-stored power. - double power; + //! Locally-stored duration. + double duration; //! Locally-stored done reward. double doneReward; From e22b85d15b8c931c94572f6c5e203c6cbcf4b47c Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Sun, 15 Mar 2020 12:56:05 +0530 Subject: [PATCH 02/12] Update continuous_mountain_car.hpp --- .../environment/continuous_mountain_car.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp index 4d5eccce56..24cdc3cc04 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp @@ -96,7 +96,7 @@ class ContinuousMountainCar * @param positionGoal Final target position. * @param velocityMin Minimum legal velocity. * @param velocityMax Maximum legal velocity. - * @param duration time duration for which force is apllied on the car. + * @param duration Time Duration for which force is applied on the car. * @param doneReward Reward recieved by the agent on success. * @param maxSteps The number of steps after which the episode * terminates. If the value is 0, there is no limit. From 27ceb51793baa28a3576469d3a68970052dc5670 Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Sun, 15 Mar 2020 13:52:13 +0530 Subject: [PATCH 03/12] use the ClampRange() instead of std::max,min mlpack/core/math/clamp.hpp already contains a function to clip values between a minimum value and a maximum value. --- .../environment/continuous_mountain_car.hpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp index 24cdc3cc04..4d59e91d87 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp @@ -18,6 +18,7 @@ #define MLPACK_METHODS_RL_ENVIRONMENT_CONTINUOUS_MOUNTAIN_CAR_HPP #include +#include namespace mlpack { namespace rl { @@ -136,16 +137,14 @@ class ContinuousMountainCar stepsPerformed++; // Calculate acceleration. - double force = std::min(std::max(action.action[0], -1.0), 1.0); + double force = math::ClampRange(action.action[0], -1.0, 1.0); // Update states. nextState.Velocity() = state.Velocity() + force * duration - 0.0025 * std::cos(3 * state.Position()); - nextState.Velocity() = std::min( - std::max(nextState.Velocity(), velocityMin), velocityMax); + nextState.Velocity() = math::ClampRange(nextState.Velocity(), velocityMin, velocityMax); nextState.Position() = state.Position() + nextState.Velocity(); - nextState.Position() = std::min( - std::max(nextState.Position(), positionMin), positionMax); + nextState.Position() = math::ClampRange(nextState.Position(), positionMin, positionMax); if (nextState.Position() == positionMin && nextState.Velocity() < 0) nextState.Velocity() = 0.0; From 924bc562e96d1a3a9b58dc78e06362daf027652a Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Sun, 15 Mar 2020 13:55:40 +0530 Subject: [PATCH 04/12] using clamprange --- .../methods/reinforcement_learning/environment/acrobot.hpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp index 62bb0633fc..7b71b0a996 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp @@ -169,10 +169,8 @@ class Acrobot nextState.Theta2() = Wrap(currentNextState[1], -M_PI, M_PI); //! The value of angular velocity is bounded in min and max value. - nextState.AngularVelocity1() = std::min( - std::max(currentNextState[2], -maxVel1), maxVel1); - nextState.AngularVelocity2() = std::min( - std::max(currentNextState[3], -maxVel2), maxVel2); + nextState.AngularVelocity1() = math::ClampRange(currentNextState[2], -maxVel1, maxVel1); + nextState.AngularVelocity2() = math::ClampRange(currentNextState[3], -maxVel2, maxVel2); // Check if the episode has terminated. bool done = IsTerminal(nextState); From 3daa7e25b8fa5dfb67ed1d609b71a979ce5e7fee Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Sun, 15 Mar 2020 13:58:40 +0530 Subject: [PATCH 05/12] using clamprange --- .../reinforcement_learning/environment/mountain_car.hpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp index b02b631cd4..8f79358e01 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp @@ -17,6 +17,7 @@ #define MLPACK_METHODS_RL_ENVIRONMENT_MOUNTAIN_CAR_HPP #include +#include namespace mlpack { namespace rl { @@ -134,13 +135,11 @@ class MountainCar int direction = action - 1; nextState.Velocity() = state.Velocity() + 0.001 * direction - 0.0025 * std::cos(3 * state.Position()); - nextState.Velocity() = std::min( - std::max(nextState.Velocity(), velocityMin), velocityMax); + nextState.Velocity() = math::ClampRange(nextState.Velocity(), velocityMin, velocityMax); // Update states. nextState.Position() = state.Position() + nextState.Velocity(); - nextState.Position() = std::min( - std::max(nextState.Position(), positionMin), positionMax); + nextState.Position() = math::ClampRange(nextState.Position(), positionMin, positionMax); if (nextState.Position() == positionMin && nextState.Velocity() < 0) nextState.Velocity() = 0.0; From e2a904bac510ca23dc231a48e091cc5948045f21 Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Sun, 15 Mar 2020 14:00:46 +0530 Subject: [PATCH 06/12] using clamprange --- .../reinforcement_learning/environment/pendulum.hpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index e7133bdf6b..1864d711c6 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -17,6 +17,7 @@ #define MLPACK_METHODS_RL_ENVIRONMENT_PENDULUM_HPP #include +#include namespace mlpack { namespace rl { @@ -140,8 +141,7 @@ class Pendulum const double length = 1.0; // Get action and clip the values between max and min limits. - double torque = std::min( - std::max(action.action[0], -maxTorque), maxTorque); + double torque = math::ClampRange(action.action[0], -maxTorque, maxTorque); // Calculate costs of taking this action in the current state. double costs = std::pow(AngleNormalize(theta), 2) + 0.1 * @@ -151,8 +151,7 @@ class Pendulum double newAngularVelocity = angularVelocity + (-3.0 * gravity / (2 * length) * std::sin(theta + M_PI) + 3.0 / std::pow(mass * length, 2) * torque) * dt; - nextState.AngularVelocity() = std::min(std::max(newAngularVelocity, - -maxAngularVelocity), maxAngularVelocity); + nextState.AngularVelocity() = math::ClampRange(newAngularVelocity, -maxAngularVelocity, maxAngularVelocity); nextState.Theta() = theta + newAngularVelocity * dt; // Check if the episode has terminated From f99b2d517ff621e8bef4e6f4a5658cabfa26e68e Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Sun, 15 Mar 2020 14:01:58 +0530 Subject: [PATCH 07/12] using cliprange --- .../reinforcement_learning/environment/reward_clipping.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/reward_clipping.hpp b/src/mlpack/methods/reinforcement_learning/environment/reward_clipping.hpp index a7ce90038e..765519c89e 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/reward_clipping.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/reward_clipping.hpp @@ -13,6 +13,7 @@ #define MLPACK_METHODS_RL_ENVIRONMENT_REWARD_CLIPPING_HPP #include +#include namespace mlpack { namespace rl { @@ -91,7 +92,7 @@ class RewardClipping // Get original unclipped reward from base environment. double unclippedReward = environment.Sample(state, action, nextState); // Clip rewards according to the min and max limit and return. - return std::min(std::max(unclippedReward, minReward), maxReward); + return math::ClampRange(unclippedReward, minReward, maxReward); } /** From f8786d0d6c33ee26dc003c20952f59db46ff2101 Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Sun, 15 Mar 2020 18:40:35 +0530 Subject: [PATCH 08/12] fixed the styling issue --- .../methods/reinforcement_learning/environment/acrobot.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp index 7b71b0a996..51fbe8c603 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp @@ -169,8 +169,10 @@ class Acrobot nextState.Theta2() = Wrap(currentNextState[1], -M_PI, M_PI); //! The value of angular velocity is bounded in min and max value. - nextState.AngularVelocity1() = math::ClampRange(currentNextState[2], -maxVel1, maxVel1); - nextState.AngularVelocity2() = math::ClampRange(currentNextState[3], -maxVel2, maxVel2); + nextState.AngularVelocity1() = math::ClampRange(currentNextState[2], + -maxVel1, maxVel1); + nextState.AngularVelocity2() = math::ClampRange(currentNextState[3], + -maxVel2, maxVel2); // Check if the episode has terminated. bool done = IsTerminal(nextState); From 1243fa0fc1d6594814bece069f4a34680c192595 Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Sun, 15 Mar 2020 18:41:47 +0530 Subject: [PATCH 09/12] Update acrobot.hpp --- .../methods/reinforcement_learning/environment/acrobot.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp index 51fbe8c603..b61cc561e7 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp @@ -170,7 +170,7 @@ class Acrobot //! The value of angular velocity is bounded in min and max value. nextState.AngularVelocity1() = math::ClampRange(currentNextState[2], - -maxVel1, maxVel1); + -maxVel1, maxVel1); nextState.AngularVelocity2() = math::ClampRange(currentNextState[3], -maxVel2, maxVel2); From aa39d630bdecd464d7f56f5bed9af87f955418fe Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Sun, 15 Mar 2020 18:43:13 +0530 Subject: [PATCH 10/12] Update mountain_car.hpp --- .../reinforcement_learning/environment/mountain_car.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp index 8f79358e01..a83e44d4fc 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp @@ -135,11 +135,13 @@ class MountainCar int direction = action - 1; nextState.Velocity() = state.Velocity() + 0.001 * direction - 0.0025 * std::cos(3 * state.Position()); - nextState.Velocity() = math::ClampRange(nextState.Velocity(), velocityMin, velocityMax); + nextState.Velocity() = math::ClampRange(nextState.Velocity(), + velocityMin, velocityMax); // Update states. nextState.Position() = state.Position() + nextState.Velocity(); - nextState.Position() = math::ClampRange(nextState.Position(), positionMin, positionMax); + nextState.Position() = math::ClampRange(nextState.Position(), + positionMin, positionMax); if (nextState.Position() == positionMin && nextState.Velocity() < 0) nextState.Velocity() = 0.0; From 21b7e3240f39ad06ce8eb2821f96b2f67c84d2bf Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Sun, 15 Mar 2020 18:44:35 +0530 Subject: [PATCH 11/12] Update pendulum.hpp --- .../methods/reinforcement_learning/environment/pendulum.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index 1864d711c6..55c02668cd 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -151,7 +151,8 @@ class Pendulum double newAngularVelocity = angularVelocity + (-3.0 * gravity / (2 * length) * std::sin(theta + M_PI) + 3.0 / std::pow(mass * length, 2) * torque) * dt; - nextState.AngularVelocity() = math::ClampRange(newAngularVelocity, -maxAngularVelocity, maxAngularVelocity); + nextState.AngularVelocity() = math::ClampRange(newAngularVelocity, + -maxAngularVelocity, maxAngularVelocity); nextState.Theta() = theta + newAngularVelocity * dt; // Check if the episode has terminated From 9f37d3a5f83407036bb9eb7b3083b59106479f74 Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Mon, 16 Mar 2020 20:22:49 +0530 Subject: [PATCH 12/12] Update continuous_mountain_car.hpp --- .../environment/continuous_mountain_car.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp index 4d59e91d87..c1336428a6 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp @@ -142,9 +142,11 @@ class ContinuousMountainCar // Update states. nextState.Velocity() = state.Velocity() + force * duration - 0.0025 * std::cos(3 * state.Position()); - nextState.Velocity() = math::ClampRange(nextState.Velocity(), velocityMin, velocityMax); + nextState.Velocity() = math::ClampRange(nextState.Velocity(), + velocityMin, velocityMax); nextState.Position() = state.Position() + nextState.Velocity(); - nextState.Position() = math::ClampRange(nextState.Position(), positionMin, positionMax); + nextState.Position() = math::ClampRange(nextState.Position(), + positionMin, positionMax); if (nextState.Position() == positionMin && nextState.Velocity() < 0) nextState.Velocity() = 0.0;