diff --git a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp index 62bb0633fc..b61cc561e7 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/acrobot.hpp @@ -169,10 +169,10 @@ class Acrobot nextState.Theta2() = Wrap(currentNextState[1], -M_PI, M_PI); //! The value of angular velocity is bounded in min and max value. - nextState.AngularVelocity1() = std::min( - std::max(currentNextState[2], -maxVel1), maxVel1); - nextState.AngularVelocity2() = std::min( - std::max(currentNextState[3], -maxVel2), maxVel2); + nextState.AngularVelocity1() = math::ClampRange(currentNextState[2], + -maxVel1, maxVel1); + nextState.AngularVelocity2() = math::ClampRange(currentNextState[3], + -maxVel2, maxVel2); // Check if the episode has terminated. bool done = IsTerminal(nextState); diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp index 5837e1b147..c1336428a6 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp @@ -18,6 +18,7 @@ #define MLPACK_METHODS_RL_ENVIRONMENT_CONTINUOUS_MOUNTAIN_CAR_HPP #include +#include namespace mlpack { namespace rl { @@ -96,7 +97,7 @@ class ContinuousMountainCar * @param positionGoal Final target position. * @param velocityMin Minimum legal velocity. * @param velocityMax Maximum legal velocity. - * @param power Power generated by car. + * @param duration Time Duration for which force is applied on the car. * @param doneReward Reward recieved by the agent on success. * @param maxSteps The number of steps after which the episode * terminates. If the value is 0, there is no limit. @@ -106,7 +107,7 @@ class ContinuousMountainCar const double positionGoal = 0.45, const double velocityMin = -0.07, const double velocityMax = 0.07, - const double power = 0.0015, + const double duration = 0.0015, const double doneReward = 100, const size_t maxSteps = 0) : positionMin(positionMin), @@ -114,7 +115,7 @@ class ContinuousMountainCar positionGoal(positionGoal), velocityMin(velocityMin), velocityMax(velocityMax), - power(power), + duration(duration), doneReward(doneReward), maxSteps(maxSteps), stepsPerformed(0) @@ -136,16 +137,16 @@ class ContinuousMountainCar stepsPerformed++; // Calculate acceleration. - double force = std::min(std::max(action.action[0], -1.0), 1.0); + double force = math::ClampRange(action.action[0], -1.0, 1.0); // Update states. - nextState.Velocity() = state.Velocity() + force * power - 0.0025 * + nextState.Velocity() = state.Velocity() + force * duration - 0.0025 * std::cos(3 * state.Position()); - nextState.Velocity() = std::min( - std::max(nextState.Velocity(), velocityMin), velocityMax); + nextState.Velocity() = math::ClampRange(nextState.Velocity(), + velocityMin, velocityMax); nextState.Position() = state.Position() + nextState.Velocity(); - nextState.Position() = std::min( - std::max(nextState.Position(), positionMin), positionMax); + nextState.Position() = math::ClampRange(nextState.Position(), + positionMin, positionMax); if (nextState.Position() == positionMin && nextState.Velocity() < 0) nextState.Velocity() = 0.0; @@ -236,8 +237,8 @@ class ContinuousMountainCar //! Locally-stored maximum legal velocity. double velocityMax; - //! Locally-stored power. - double power; + //! Locally-stored duration. + double duration; //! Locally-stored done reward. double doneReward; diff --git a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp index b02b631cd4..a83e44d4fc 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp @@ -17,6 +17,7 @@ #define MLPACK_METHODS_RL_ENVIRONMENT_MOUNTAIN_CAR_HPP #include +#include namespace mlpack { namespace rl { @@ -134,13 +135,13 @@ class MountainCar int direction = action - 1; nextState.Velocity() = state.Velocity() + 0.001 * direction - 0.0025 * std::cos(3 * state.Position()); - nextState.Velocity() = std::min( - std::max(nextState.Velocity(), velocityMin), velocityMax); + nextState.Velocity() = math::ClampRange(nextState.Velocity(), + velocityMin, velocityMax); // Update states. nextState.Position() = state.Position() + nextState.Velocity(); - nextState.Position() = std::min( - std::max(nextState.Position(), positionMin), positionMax); + nextState.Position() = math::ClampRange(nextState.Position(), + positionMin, positionMax); if (nextState.Position() == positionMin && nextState.Velocity() < 0) nextState.Velocity() = 0.0; diff --git a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp index e7133bdf6b..55c02668cd 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp @@ -17,6 +17,7 @@ #define MLPACK_METHODS_RL_ENVIRONMENT_PENDULUM_HPP #include +#include namespace mlpack { namespace rl { @@ -140,8 +141,7 @@ class Pendulum const double length = 1.0; // Get action and clip the values between max and min limits. - double torque = std::min( - std::max(action.action[0], -maxTorque), maxTorque); + double torque = math::ClampRange(action.action[0], -maxTorque, maxTorque); // Calculate costs of taking this action in the current state. double costs = std::pow(AngleNormalize(theta), 2) + 0.1 * @@ -151,8 +151,8 @@ class Pendulum double newAngularVelocity = angularVelocity + (-3.0 * gravity / (2 * length) * std::sin(theta + M_PI) + 3.0 / std::pow(mass * length, 2) * torque) * dt; - nextState.AngularVelocity() = std::min(std::max(newAngularVelocity, - -maxAngularVelocity), maxAngularVelocity); + nextState.AngularVelocity() = math::ClampRange(newAngularVelocity, + -maxAngularVelocity, maxAngularVelocity); nextState.Theta() = theta + newAngularVelocity * dt; // Check if the episode has terminated diff --git a/src/mlpack/methods/reinforcement_learning/environment/reward_clipping.hpp b/src/mlpack/methods/reinforcement_learning/environment/reward_clipping.hpp index a7ce90038e..765519c89e 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/reward_clipping.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/reward_clipping.hpp @@ -13,6 +13,7 @@ #define MLPACK_METHODS_RL_ENVIRONMENT_REWARD_CLIPPING_HPP #include +#include namespace mlpack { namespace rl { @@ -91,7 +92,7 @@ class RewardClipping // Get original unclipped reward from base environment. double unclippedReward = environment.Sample(state, action, nextState); // Clip rewards according to the min and max limit and return. - return std::min(std::max(unclippedReward, minReward), maxReward); + return math::ClampRange(unclippedReward, minReward, maxReward); } /**