Merge pull request #1996 from favre49/RevertRL

Revert Action.action to be a double array

HISTORY.md

@@ -17,9 +17,6 @@
   * Fix prediction output of softmax regression when test set accuracy is
     calculated (#1922).
 
-  * Action struct in continuous RL environments now stores the action as a
-    `double` instead of `double[1]` (#1941, #1931).
-
   * Pendulum environment now checks for termination. All RL environments now
     have an option to terminate after a set number of time steps (no limit
     by default) (#1941).

src/mlpack/methods/reinforcement_learning/environment/continuous_double_pole_cart.hpp

@@ -91,7 +91,7 @@ class ContinuousDoublePoleCart
    */
   struct Action
   {
-    double action = 0.0;
+    double action[1];
     // Storing degree of freedom
     const int size = 1;
   };
@@ -193,7 +193,7 @@ class ContinuousDoublePoleCart
             const Action& action,
             arma::vec& dydx)
   {
-    double totalForce = action.action;
+    double totalForce = action.action[0];
     double totalMass = massCart;
     double omega1 = state.AngularVelocity(1);
     double omega2 = state.AngularVelocity(2);

src/mlpack/methods/reinforcement_learning/environment/continuous_mountain_car.hpp

@@ -83,7 +83,7 @@ class ContinuousMountainCar
    */
   struct Action
   {
-    double action = 0.0;
+    double action[1];
     // Storing degree of freedom
     const int size = 1;
   };
@@ -136,7 +136,7 @@ class ContinuousMountainCar
     stepsPerformed++;
 
     // Calculate acceleration.
-    double force = std::min(std::max(action.action, -1.0), 1.0);
+    double force = std::min(std::max(action.action[0], -1.0), 1.0);
 
     // Update states.
     nextState.Velocity() = state.Velocity() + force * power - 0.0025 *
@@ -158,7 +158,7 @@ class ContinuousMountainCar
     else if (done)
       return doneReward;
 
-    return std::pow(action.action, 2) * 0.1;
+    return std::pow(action.action[0], 2) * 0.1;
   }
 
   /**

src/mlpack/methods/reinforcement_learning/environment/pendulum.hpp

@@ -82,7 +82,7 @@ class Pendulum
    */
   struct Action
   {
-    double action = 0.0;
+    double action[1];
     // Storing degree of freedom
     const int size = 1;
   };
@@ -141,7 +141,7 @@ class Pendulum
 
     // Get action and clip the values between max and min limits.
     double torque = std::min(
-        std::max(action.action, -maxTorque), maxTorque);
+        std::max(action.action[0], -maxTorque), maxTorque);
 
     // Calculate costs of taking this action in the current state.
     double costs = std::pow(AngleNormalize(theta), 2) + 0.1 *

src/mlpack/tests/reward_clipping_test.cpp

@@ -47,7 +47,7 @@ BOOST_AUTO_TEST_CASE(ClippedRewardTest)
 
   RewardClipping<Pendulum>::State state = rewardClipping.InitialSample();
   RewardClipping<Pendulum>::Action action;
-  action.action = mlpack::math::Random(-1.0, 1.0);
+  action.action[0] = mlpack::math::Random(-1.0, 1.0);
   double reward = rewardClipping.Sample(state, action);
 
   BOOST_REQUIRE(reward <= 2.0);

src/mlpack/tests/rl_components_test.cpp

@@ -42,7 +42,7 @@ BOOST_AUTO_TEST_CASE(SimplePendulumTest)
 
   Pendulum::State state = task.InitialSample();
   Pendulum::Action action;
-  action.action = math::Random(-2.0, 2.0);
+  action.action[0] = math::Random(-2.0, 2.0);
   double reward = task.Sample(state, action);
 
   // The reward is always negative. Check if not lower than lowest possible.
@@ -73,7 +73,7 @@ BOOST_AUTO_TEST_CASE(SimpleContinuousMountainCarTest)
 
   ContinuousMountainCar::State state = task.InitialSample();
   ContinuousMountainCar::Action action;
-  action.action = math::Random(-1.0, 1.0);
+  action.action[0] = math::Random(-1.0, 1.0);
   double reward = task.Sample(state, action);
   // Maximum reward possible is 100.
   BOOST_REQUIRE(reward <= 100.0);
@@ -201,7 +201,7 @@ BOOST_AUTO_TEST_CASE(ContinuousDoublePoleCartTest)
 
   ContinuousDoublePoleCart::State state = task.InitialSample();
   ContinuousDoublePoleCart::Action action;
-  action.action = math::Random(-1.0, 1.0);
+  action.action[0] = math::Random(-1.0, 1.0);
   double reward = task.Sample(state, action);
 
   BOOST_REQUIRE_EQUAL(reward, 1.0);