From 69f145d0fae2917be7a2911e25d68cba7bff2733 Mon Sep 17 00:00:00 2001 From: Shashank Shekhar Date: Wed, 9 May 2018 18:56:35 +0530 Subject: [PATCH 1/8] Added Pendulum continuous environment of OpenAI Gym --- .../environment/CMakeLists.txt | 1 + .../environment/pendulum.hpp | 199 ++++++++++++++++++ 2 files changed, 200 insertions(+) create mode 100644 src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp diff --git a/src/mlpack/methods/reinforcement_learning/environment/CMakeLists.txt b/src/mlpack/methods/reinforcement_learning/environment/CMakeLists.txt index 41e677755d..88f5b4849c 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/CMakeLists.txt +++ b/src/mlpack/methods/reinforcement_learning/environment/CMakeLists.txt @@ -5,6 +5,7 @@ set(SOURCES cart_pole.hpp continuous_mountain_car.hpp acrobat.hpp + pendulum.hpp ) # Add directory name to sources. diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp new file mode 100644 index 0000000000..c3492aa210 --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -0,0 +1,199 @@ +/** + * @file pendulum.hpp + * @author Shashank Shekhar + * + * This file is an implementation of Pendulum task: + * https://gym.openai.com/envs/Pendulum-v0/ + * + * TODO: provide an option to use dynamics directly from OpenAI gym. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_METHODS_RL_ENVIRONMENT_PENDULUM_HPP +#define MLPACK_METHODS_RL_ENVIRONMENT_PENDULUM_HPP + +#include + +namespace mlpack { +namespace rl { + +/** + * Implementation of Pendulum task. The inverted pendulum swingup problem is a + * classic problem in the control literature. In this version of the problem, + * the pendulum starts in a random position, and the goal is to swing it up so + * it stays upright + */ +class Pendulum +{ + public: + /** + * Implementation of state of Pendulum. Each state is a + * (theta, angular velocity) vector. + */ + class State + { + public: + /** + * Construct a state instance. + */ + State() : data(dimension, arma::fill::zeros) + { /* Nothing to do here. */ } + + /** + * Construct a state based on the given data. + * + * @param data Data for the theta and angular velocity. + */ + State(const arma::colvec& data): data(data) + { /* Nothing to do here. */ } + + //! Modify the internal representation of the state. + arma::colvec& Data() { return data; } + + //! Get the theta. + double Theta() const { return data[0]; } + //! Modify the value of theta. + double& Theta() { return data[0]; } + + //! Get the angular velocity. + double AngularVelocity() const { return data[1]; } + //! Modify the value of angular velocity. + double& AngularVelocity() { return data[1]; } + + //! Encode the state to a column vector. + const arma::colvec& Encode() const { return data; } + + //! Dimension of the encoded state. + static constexpr size_t dimension = 2; + + private: + //! Locally-stored (theta, angular velocity) vector + arma::colvec data; + }; + + /** + * Implementation of action of Pendulum. + * In Pendulum, the action represents the torque to be applied. + * This value is bounded in range -2.0 to 2.0 by default. + */ + struct Action + { + double action[1]; + }; + + /** + * Construct a Pendulum instance using the given values. + * + * @param maxAngularVelocity Maximum angular velocity. + * @param maxTorque Maximum torque. + * @param dt The differential value. + */ + Pendulum(const double maxAngularVelocity = 8, + const double maxTorque = 2.0, + const double dt = 0.05) : + maxAngularVelocity(maxAngularVelocity), + maxTorque(maxTorque), + dt(dt) + { /* Nothing to do here */ } + + /** + * Dynamics of Pendulum. Get reward and next state based + * on current state and current action. + * + * @param state The current state. + * @param action The current action. + * @param nextState The next state. + * @return reward, The reward for taking the action taken for current state. + */ + double Sample(const State& state, + const Action& action, + State& nextState) const + { + // Get current state + double theta = state.Theta(); + double angular_velocity = state.AngularVelocity(); + + // Define required variables + double gravity = 10.0; + double mass = 1.0; + double length = 1.0; + + // Get action and clip the values + double torque = std::min( + std::max(action.action[0], -maxTorque), maxTorque); + + // Calculate costs + double costs = pow(AngleNormalize(theta),2) + 0.1 * pow(angular_velocity,2) + + 0.001 * pow(torque,2); + + // Calculate new state values + double new_angular_velocity = angular_velocity + (-3.0 * gravity / (2 * + length) * std::sin(theta + M_PI) + 3.0 / (pow(mass * length,2) * + torque) * dt; + double new_theta = theta + new_angular_velocity * dt; + + // Set values for next state + nextState.Theta() = new_theta; + nextState.AngularVelocity() = std::min(std::max(new_angular_velocity, + -maxAngularVelocity), maxAngularVelocity); + + return -costs; + } + + /** + * Dynamics of Pendulum. Get reward based on current state and current action + * + * @param state The current state. + * @param action The current action. + * @return reward, The reward. + */ + double Sample(const State& state, const Action& action) const + { + State nextState; + return Sample(state, action, nextState); + } + + /** + * Initial theta is randomly generated within [-pi, pi]. + * Initial angular velocity is randomly generated within [-1, 1]. + * + * @return Initial state for each episode. + */ + State InitialSample() const + { + State state; + state.Theta() = math::Random(-M_PI, M_PI); + state.AngularVelocity() = math::Random(-1.0, 1.0); + return state; + } + + /** + * This function calculates the normalized anlge for a particular theta. + * + * @param theta The un-normalized angle. + */ + double AngleNormalize(double theta) const + { + // Scale angle within [-pi, pi) + return double((theta + M_PI) % (2 * M_PI) - M_PI); + } + + private: + //! Locally-stored maximum legal angular velocity. + double maxAngularVelocity; + + //! Locally-stored maximum legal torque. + double maxTorque; + + //! Locally-stored dt. + double dt; +}; + +} // namespace rl +} // namespace mlpack + +#endif From f2dfcd1488fb4934b30c9aa9fbb0e86017584165 Mon Sep 17 00:00:00 2001 From: Shashank Shekhar Date: Fri, 11 May 2018 21:17:16 +0530 Subject: [PATCH 2/8] Fixed style errors --- .../reinforcement_learning/environment/pendulum.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index c3492aa210..8ec6fdf49f 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -116,7 +116,7 @@ class Pendulum // Get current state double theta = state.Theta(); double angular_velocity = state.AngularVelocity(); - + // Define required variables double gravity = 10.0; double mass = 1.0; @@ -127,15 +127,15 @@ class Pendulum std::max(action.action[0], -maxTorque), maxTorque); // Calculate costs - double costs = pow(AngleNormalize(theta),2) + 0.1 * pow(angular_velocity,2) - + 0.001 * pow(torque,2); + double costs = pow(AngleNormalize(theta), 2) + 0.1 * + pow(angular_velocity, 2) + 0.001 * pow(torque, 2); // Calculate new state values double new_angular_velocity = angular_velocity + (-3.0 * gravity / (2 * - length) * std::sin(theta + M_PI) + 3.0 / (pow(mass * length,2) * + length) * std::sin(theta + M_PI) + 3.0 / (pow(mass * length, 2) * torque) * dt; double new_theta = theta + new_angular_velocity * dt; - + // Set values for next state nextState.Theta() = new_theta; nextState.AngularVelocity() = std::min(std::max(new_angular_velocity, From e2a657392d7962aed3226f331cd22ae0550cbf27 Mon Sep 17 00:00:00 2001 From: Shashank Shekhar Date: Sat, 12 May 2018 18:36:24 +0530 Subject: [PATCH 3/8] Fixed style and other errors --- .../methods/reinforcement_learning/environment/pendulum.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index 8ec6fdf49f..1ea437ef33 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -83,6 +83,8 @@ class Pendulum struct Action { double action[1]; + // Storing degree of freedom + const int size = 1; }; /** @@ -132,7 +134,7 @@ class Pendulum // Calculate new state values double new_angular_velocity = angular_velocity + (-3.0 * gravity / (2 * - length) * std::sin(theta + M_PI) + 3.0 / (pow(mass * length, 2) * + length) * std::sin(theta + M_PI) + 3.0 / (pow(mass * length, 2)) * torque) * dt; double new_theta = theta + new_angular_velocity * dt; @@ -179,7 +181,7 @@ class Pendulum double AngleNormalize(double theta) const { // Scale angle within [-pi, pi) - return double((theta + M_PI) % (2 * M_PI) - M_PI); + return double(fmod(theta + M_PI, 2 * M_PI) - M_PI); } private: From 8bc94913ab25a3d9833e08624c45277a10fd3e55 Mon Sep 17 00:00:00 2001 From: Shashank Shekhar Date: Sat, 12 May 2018 18:36:52 +0530 Subject: [PATCH 4/8] Added pendulum tests --- src/mlpack/tests/rl_components_test.cpp | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 5d0cae3484..70f37b8d8a 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -28,6 +29,26 @@ using namespace mlpack::rl; BOOST_AUTO_TEST_SUITE(RLComponentsTest) +/** + * Constructs a Pendulum instance and check if the main rountine works as it + * should be working. + */ +BOOST_AUTO_TEST_CASE(SimplePendulumTest) +{ + const Pendulum task = Pendulum(); + + Pendulum::State state = task.InitialSample(); + Pendulum::Action action; + action.action[0] = math::Random(-2.0, 2.0); + double reward = task.Sample(state, action); + + // The reward is always negative. Check if not lower than lowest possible + BOOST_REQUIRE(reward >= -(pow(M_PI, 2) + 6.404)); + + // The reward is simply the torque. Check if dimension is 1 + BOOST_REQUIRE_EQUAL(1, action.size); +} + /** * Constructs a Continuous MountainCar instance and check if the main rountine * works as it should be. From ae2ef99ccd8305c32d3b2c629e3673f89f5ea41f Mon Sep 17 00:00:00 2001 From: Shashank Shekhar Date: Mon, 14 May 2018 10:06:50 +0530 Subject: [PATCH 5/8] Switched to Camel Case and fixed some style issues --- .../environment/pendulum.hpp | 35 +++++++++---------- src/mlpack/tests/rl_components_test.cpp | 4 +-- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index 1ea437ef33..25df22f3da 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -115,34 +115,33 @@ class Pendulum const Action& action, State& nextState) const { - // Get current state + // Get current state. double theta = state.Theta(); - double angular_velocity = state.AngularVelocity(); + double angularVelocity = state.AngularVelocity(); - // Define required variables - double gravity = 10.0; - double mass = 1.0; - double length = 1.0; + // Define constants which specify our pendulum. + const gravity = 10.0; + const mass = 1.0; + const length = 1.0; - // Get action and clip the values + // Get action and clip the values between max and min limits. double torque = std::min( std::max(action.action[0], -maxTorque), maxTorque); - // Calculate costs + // Calculate costs of taking this action in the current state. double costs = pow(AngleNormalize(theta), 2) + 0.1 * - pow(angular_velocity, 2) + 0.001 * pow(torque, 2); + pow(angularVelocity, 2) + 0.001 * pow(torque, 2); - // Calculate new state values - double new_angular_velocity = angular_velocity + (-3.0 * gravity / (2 * - length) * std::sin(theta + M_PI) + 3.0 / (pow(mass * length, 2)) * + // Calculate new state values and assign to the next state. + double newAngularVelocity = angularVelocity + (-3.0 * gravity / (2 * + length) * std::sin(theta + M_PI) + 3.0 / pow(mass * length, 2) * torque) * dt; - double new_theta = theta + new_angular_velocity * dt; - - // Set values for next state - nextState.Theta() = new_theta; - nextState.AngularVelocity() = std::min(std::max(new_angular_velocity, + nextState.AngularVelocity() = std::min(std::max(newAngularVelocity, -maxAngularVelocity), maxAngularVelocity); + nextState.Theta() = theta + newAngularVelocity * dt; + // Return the reward of taking the action in current state. + // The reward is simply the negative of cost incurred for the action. return -costs; } @@ -180,7 +179,7 @@ class Pendulum */ double AngleNormalize(double theta) const { - // Scale angle within [-pi, pi) + // Scale angle within [-pi, pi). return double(fmod(theta + M_PI, 2 * M_PI) - M_PI); } diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 70f37b8d8a..e84875f269 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -42,10 +42,10 @@ BOOST_AUTO_TEST_CASE(SimplePendulumTest) action.action[0] = math::Random(-2.0, 2.0); double reward = task.Sample(state, action); - // The reward is always negative. Check if not lower than lowest possible + // The reward is always negative. Check if not lower than lowest possible. BOOST_REQUIRE(reward >= -(pow(M_PI, 2) + 6.404)); - // The reward is simply the torque. Check if dimension is 1 + // The state is simply the torque. Check if dimension is 1. BOOST_REQUIRE_EQUAL(1, action.size); } From 0d3380551a517eb3c70bac3144d21a4dd354bb3d Mon Sep 17 00:00:00 2001 From: Shashank Shekhar Date: Mon, 14 May 2018 10:29:17 +0530 Subject: [PATCH 6/8] Minor fix for type --- .../methods/reinforcement_learning/environment/pendulum.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index 25df22f3da..ec2bbd174a 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -120,9 +120,9 @@ class Pendulum double angularVelocity = state.AngularVelocity(); // Define constants which specify our pendulum. - const gravity = 10.0; - const mass = 1.0; - const length = 1.0; + const double gravity = 10.0; + const double mass = 1.0; + const double length = 1.0; // Get action and clip the values between max and min limits. double torque = std::min( From da1426368d4a8be5f947d39dba0a0dcd8b54c429 Mon Sep 17 00:00:00 2001 From: Shashank Shekhar Date: Mon, 14 May 2018 12:04:26 +0530 Subject: [PATCH 7/8] Fixed comment mistake --- src/mlpack/tests/rl_components_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index e84875f269..f4c8e63f6f 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -45,7 +45,7 @@ BOOST_AUTO_TEST_CASE(SimplePendulumTest) // The reward is always negative. Check if not lower than lowest possible. BOOST_REQUIRE(reward >= -(pow(M_PI, 2) + 6.404)); - // The state is simply the torque. Check if dimension is 1. + // The action is simply the torque. Check if dimension is 1. BOOST_REQUIRE_EQUAL(1, action.size); } From 1336322b95a251ab94a77f87369c41a7a096a63e Mon Sep 17 00:00:00 2001 From: Shashank Shekhar Date: Tue, 15 May 2018 09:46:16 +0530 Subject: [PATCH 8/8] Fixed the scope of power function and typo in tests --- .../methods/reinforcement_learning/environment/pendulum.hpp | 6 +++--- src/mlpack/tests/rl_components_test.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index ec2bbd174a..878b077c20 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -129,12 +129,12 @@ class Pendulum std::max(action.action[0], -maxTorque), maxTorque); // Calculate costs of taking this action in the current state. - double costs = pow(AngleNormalize(theta), 2) + 0.1 * - pow(angularVelocity, 2) + 0.001 * pow(torque, 2); + double costs = std::pow(AngleNormalize(theta), 2) + 0.1 * + std::pow(angularVelocity, 2) + 0.001 * std::pow(torque, 2); // Calculate new state values and assign to the next state. double newAngularVelocity = angularVelocity + (-3.0 * gravity / (2 * - length) * std::sin(theta + M_PI) + 3.0 / pow(mass * length, 2) * + length) * std::sin(theta + M_PI) + 3.0 / std::pow(mass * length, 2) * torque) * dt; nextState.AngularVelocity() = std::min(std::max(newAngularVelocity, -maxAngularVelocity), maxAngularVelocity); diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index f4c8e63f6f..16d007d623 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -30,7 +30,7 @@ using namespace mlpack::rl; BOOST_AUTO_TEST_SUITE(RLComponentsTest) /** - * Constructs a Pendulum instance and check if the main rountine works as it + * Constructs a Pendulum instance and check if the main routine works as it * should be working. */ BOOST_AUTO_TEST_CASE(SimplePendulumTest)