diff --git a/src/mlpack/methods/reinforcement_learning/sac_impl.hpp b/src/mlpack/methods/reinforcement_learning/sac_impl.hpp index a12dafb759..7c77a8d0e2 100644 --- a/src/mlpack/methods/reinforcement_learning/sac_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/sac_impl.hpp @@ -294,7 +294,7 @@ void SAC< noise = arma::clamp(noise, -0.25, 0.25); outputAction = outputAction + noise; } - action.action = arma::conv_to>::from(outputAction); + action.action = arma::conv_to>::from(outputAction); } template < diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index 33ab7ea8a1..da7767ab3b 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -22,8 +22,9 @@ #include #include #include -#include #include +#include +#include #include #include #include @@ -515,7 +516,7 @@ BOOST_AUTO_TEST_CASE(PendulumWithSAC) for (size_t trial = 0; trial < 3; ++trial) { Log::Debug << "Trial number: " << trial << std::endl; - // Set up the policy and replay method. + // Set up the replay method. RandomReplay replayMethod(32, 10000); TrainingConfig config; @@ -547,4 +548,52 @@ BOOST_AUTO_TEST_CASE(PendulumWithSAC) BOOST_REQUIRE(converged); } +//! A test to ensure SAC works with multiple actions in action space. +BOOST_AUTO_TEST_CASE(SACForMultipleActions) +{ + ContinuousActionEnv::State::dimension = 3; + ContinuousActionEnv::Action::size = 4; + + FFN, GaussianInitialization> + policyNetwork(EmptyLoss<>(), GaussianInitialization(0, 0.1)); + policyNetwork.Add(new Linear<>(ContinuousActionEnv::State::dimension, 128)); + policyNetwork.Add(new ReLULayer<>()); + policyNetwork.Add(new Linear<>(128, ContinuousActionEnv::Action::size)); + policyNetwork.Add(new TanHLayer<>()); + + FFN, GaussianInitialization> + qNetwork(EmptyLoss<>(), GaussianInitialization(0, 0.1)); + qNetwork.Add(new Linear<>(ContinuousActionEnv::State::dimension + + ContinuousActionEnv::Action::size, 128)); + qNetwork.Add(new ReLULayer<>()); + qNetwork.Add(new Linear<>(128, 1)); + + // Set up the replay method. + RandomReplay replayMethod(32, 10000); + + TrainingConfig config; + config.StepSize() = 0.001; + config.TargetNetworkSyncInterval() = 1; + config.UpdateInterval() = 3; + + // Set up Soft actor-critic agent. + SAC + agent(config, qNetwork, policyNetwork, replayMethod); + + agent.State().Data() = arma::randu + (ContinuousActionEnv::State::dimension, 1); + agent.SelectAction(); + + // Test to check if the action dimension given by the agent is correct. + BOOST_REQUIRE_EQUAL(agent.Action().action.size(), + ContinuousActionEnv::Action::size); + + replayMethod.Store(agent.State(), agent.Action(), 1, agent.State(), 1, 0.99); + agent.TotalSteps()++; + agent.Update(); + // If the agent is able to reach till this point of the test, it is assured + // that the agent can handle multiple actions in continuous space. +} + BOOST_AUTO_TEST_SUITE_END();