Merge branch 'master' into bisigmoid

.ci/linux-steps.yaml

@@ -27,7 +27,7 @@ steps:
     if [ "$(binding)" == "python" ]; then
       export PYBIN=$(which python)
       $PYBIN -m pip install --upgrade pip
-      $PYBIN -m pip install --upgrade --ignore-installed setuptools cython pandas
+      $PYBIN -m pip install --upgrade --ignore-installed setuptools cython pandas wheel
     fi
 
     if [ "a$(julia.version)" != "a" ]; then

HISTORY.md

@@ -1,8 +1,18 @@
 ### mlpack ?.?.?
 ###### ????-??-??
+  * Reinforcement Learning: Twin Delayed Deep Deterministic 
+    Policy Gradient (#3512).
+
+  * Reinforcement Learning: Ornstein-Uhlenbeck noise (#3499).
+
   * Reinforcement Learning: Deep Deterministic Policy Gradient (#3494).
-  * Bipolar sigmoid activation function added and invertible functions fixed (#3506).
+
+  * Add `ClassProbabilities()` member to `DecisionTree` so that the internal
+    details of trees can be more easily inspected (#3511).
   
+  * Bipolar sigmoid activation function added and invertible functions 
+    fixed (#3506).
+
 ### mlpack 4.2.0
 ###### 2023-06-14
   * Adapt C_ReLU, ReLU6, FlexibleReLU layer for new neural network API (#3445).

README.md

@@ -264,6 +264,7 @@ With that in mind, if you would still like to manually build the mlpack Python
 bindings, first make sure that the following Python packages are installed:
 
  - setuptools
+ - wheel
  - cython >= 0.24
  - numpy
  - pandas >= 0.15.0

src/mlpack/bindings/python/CMakeLists.txt

@@ -70,18 +70,22 @@ find_python_module(pandas 0.15.0)
 if (NOT PY_PANDAS)
   set(PY_NOT_FOUND_MSG "${PY_NOT_FOUND_MSG}\n    - pandas")
 endif ()
+find_python_module(wheel)
+if (NOT PY_WHEEL)
+  set(PY_NOT_FOUND_MSG "${PY_NOT_FOUND_MSG}\n    - wheel")
+endif ()
 
 ## We need to check here if Python and other dependencies is even available, as
 ## it is require to build python-bindings.
 if (FORCE_BUILD_PYTHON_BINDINGS)
   if (NOT PYTHON_EXECUTABLE OR NOT PY_DISTUTILS OR NOT PY_CYTHON OR NOT PY_NUMPY
-      OR NOT PY_PANDAS)
+      OR NOT PY_PANDAS OR NOT PY_WHEEL)
     unset(BUILD_PYTHON_BINDINGS CACHE)
     message(FATAL_ERROR "\nCould not Build Python Bindings; the following modules are not available: ${PY_NOT_FOUND_MSG}")
   endif()
 else()
   if (NOT PYTHON_EXECUTABLE OR NOT PY_DISTUTILS OR NOT PY_CYTHON OR NOT PY_NUMPY
-      OR NOT PY_PANDAS)
+      OR NOT PY_PANDAS OR NOT PY_WHEEL)
     unset(BUILD_PYTHON_BINDINGS CACHE)
     not_found_return("Not building Python bindings; the following modules are not available: ${PY_NOT_FOUND_MSG}")
   endif()
@@ -203,6 +207,9 @@ execute_process(COMMAND ${PYTHON_EXECUTABLE}
     "${PYTHON_INSTALL_PREFIX}"
     OUTPUT_VARIABLE CMAKE_PYTHON_PATH)
 string(STRIP "${CMAKE_PYTHON_PATH}" CMAKE_PYTHON_PATH)
+if (DEFINED ENV{PYTHONPATH})
+  string(APPEND CMAKE_PYTHON_PATH : $ENV{PYTHONPATH})
+endif ()
 install(CODE "set(ENV{PYTHONPATH} ${CMAKE_PYTHON_PATH})")
 install(CODE "set(PYTHON_EXECUTABLE \"${PYTHON_EXECUTABLE}\")")
 install(CODE "set(CMAKE_BINARY_DIR \"${CMAKE_BINARY_DIR}\")")

src/mlpack/bindings/python/PythonInstall.cmake

@@ -5,16 +5,19 @@
 if (DEFINED ENV{DESTDIR})
   execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install
           --prefix=${PYTHON_INSTALL_PREFIX}
-          --root=$ENV{DESTDIR} .
+          --root=$ENV{DESTDIR}
+          --no-index --no-deps --no-build-isolation -f ./ .
       WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/src/mlpack/bindings/python/"
       RESULT_VARIABLE setup_res)
 elseif (WIN32)
-  execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install .
+  execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install
+          --no-index --no-deps --no-build-isolation -f ./ .
       WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/src/mlpack/bindings/python/"
       RESULT_VARIABLE setup_res)
 else ()
   execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install
-          --prefix=${PYTHON_INSTALL_PREFIX} .
+          --prefix=${PYTHON_INSTALL_PREFIX}
+          --no-index --no-deps --no-build-isolation -f ./ .
       WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/src/mlpack/bindings/python/"
       RESULT_VARIABLE setup_res)
 endif ()

src/mlpack/methods/decision_tree/decision_tree.hpp

@@ -470,6 +470,11 @@ class DecisionTree :
   //! trained tree).
   size_t SplitDimension() const { return splitDimension; }
 
+  //! Get the class probabilities, if this is a leaf node in the trained tree.
+  //! Note that if this is not a leaf, then this may contain arbitrary
+  //! information used by the split in the tree!
+  const arma::vec& ClassProbabilities() const { return classProbabilities; }
+
   /**
    * Given a point and that this node is not a leaf, calculate the index of the
    * child node this point would go towards.  This method is primarily used by

src/mlpack/methods/reinforcement_learning/ddpg.hpp

@@ -46,6 +46,7 @@ namespace mlpack {
  * @tparam EnvironmentType The environment of the reinforcement learning task.
  * @tparam QNetworkType The network used to estimate the critic's Q-values.
  * @tparam PolicyNetworkType The network to compute action value.
+ * @tparam NoiseType The noise to add for exploration.
  * @tparam UpdaterType How to apply gradients when training.
  * @tparam ReplayType Experience replay method.
  */
@@ -53,6 +54,7 @@ template <
   typename EnvironmentType,
   typename QNetworkType,
   typename PolicyNetworkType,
+  typename NoiseType,
   typename UpdaterType,
   typename ReplayType = RandomReplay<EnvironmentType>
 >
@@ -75,6 +77,7 @@ class DDPG
    * @param config Hyper-parameters for training.
    * @param learningQNetwork The network to compute action value.
    * @param policyNetwork The network to produce an action given a state.
+   * @param noise The noise instance for exploration.
    * @param replayMethod Experience replay method.
    * @param qNetworkUpdater How to apply gradients to Q network when training.
    * @param policyNetworkUpdater How to apply gradients to policy network
@@ -84,6 +87,7 @@ class DDPG
   DDPG(TrainingConfig& config,
       QNetworkType& learningQNetwork,
       PolicyNetworkType& policyNetwork,
+      NoiseType& noise,
       ReplayType& replayMethod,
       UpdaterType qNetworkUpdater = UpdaterType(),
       UpdaterType policyNetworkUpdater = UpdaterType(),
@@ -150,6 +154,9 @@ class DDPG
   //! Locally-stored policy network.
   PolicyNetworkType& policyNetwork;
 
+  //! Locally-stored noise instance.
+  NoiseType& noise;
+
   //! Locally-stored target policy network.
   PolicyNetworkType targetPNetwork;
 

src/mlpack/methods/reinforcement_learning/ddpg_impl.hpp

@@ -23,6 +23,7 @@ template <
   typename EnvironmentType,
   typename QNetworkType,
   typename PolicyNetworkType,
+  typename NoiseType,
   typename UpdaterType,
   typename ReplayType
 >
@@ -30,11 +31,13 @@ DDPG<
   EnvironmentType,
   QNetworkType,
   PolicyNetworkType,
+  NoiseType, 
   UpdaterType,
   ReplayType
 >::DDPG(TrainingConfig& config,
        QNetworkType& learningQNetwork,
        PolicyNetworkType& policyNetwork,
+       NoiseType& noise,
        ReplayType& replayMethod,
        UpdaterType qNetworkUpdater,
        UpdaterType policyNetworkUpdater,
@@ -42,6 +45,7 @@ DDPG<
   config(config),
   learningQNetwork(learningQNetwork),
   policyNetwork(policyNetwork),
+  noise(noise),
   replayMethod(replayMethod),
   qNetworkUpdater(std::move(qNetworkUpdater)),
   #if ENS_VERSION_MAJOR >= 2
@@ -55,6 +59,9 @@ DDPG<
   totalSteps(0),
   deterministic(false)
 {
+  // Reset the noise instance.
+  noise.reset();
+
   // Set up q-learning and policy networks.
   targetPNetwork = policyNetwork;
   targetQNetwork = learningQNetwork;
@@ -106,6 +113,7 @@ template <
   typename EnvironmentType,
   typename QNetworkType,
   typename PolicyNetworkType,
+  typename NoiseType,
   typename UpdaterType,
   typename ReplayType
 >
@@ -113,6 +121,7 @@ DDPG<
   EnvironmentType,
   QNetworkType,
   PolicyNetworkType,
+  NoiseType,
   UpdaterType,
   ReplayType
 >::~DDPG()
@@ -127,6 +136,7 @@ template <
   typename EnvironmentType,
   typename QNetworkType,
   typename PolicyNetworkType,
+  typename NoiseType,
   typename UpdaterType,
   typename ReplayType
 >
@@ -134,6 +144,7 @@ void DDPG<
   EnvironmentType,
   QNetworkType,
   PolicyNetworkType,
+  NoiseType,
   UpdaterType,
   ReplayType
 >::SoftUpdate(double rho)
@@ -148,6 +159,7 @@ template <
   typename EnvironmentType,
   typename QNetworkType,
   typename PolicyNetworkType,
+  typename NoiseType,
   typename UpdaterType,
   typename ReplayType
 >
@@ -155,6 +167,7 @@ void DDPG<
   EnvironmentType,
   QNetworkType,
   PolicyNetworkType,
+  NoiseType,
   UpdaterType,
   ReplayType
 >::Update()
@@ -255,6 +268,7 @@ template <
   typename EnvironmentType,
   typename QNetworkType,
   typename PolicyNetworkType,
+  typename NoiseType,
   typename UpdaterType,
   typename ReplayType
 >
@@ -262,6 +276,7 @@ void DDPG<
   EnvironmentType,
   QNetworkType,
   PolicyNetworkType,
+  NoiseType,
   UpdaterType,
   ReplayType
 >::SelectAction()
@@ -272,9 +287,9 @@ void DDPG<
 
   if (!deterministic)
   {
-    arma::colvec noise = arma::randn<arma::colvec>(outputAction.n_rows) * 0.1;
-    noise = arma::clamp(noise, -0.25, 0.25);
-    outputAction = outputAction + noise;
+    arma::colvec sample = noise.sample() * 0.1;
+    sample = arma::clamp(sample, -0.25, 0.25);
+    outputAction = outputAction + sample;
   }
   action.action = arma::conv_to<std::vector<double>>::from(outputAction);
 }
@@ -283,6 +298,7 @@ template <
   typename EnvironmentType,
   typename QNetworkType,
   typename PolicyNetworkType,
+  typename NoiseType,
   typename UpdaterType,
   typename ReplayType
 >
@@ -290,6 +306,7 @@ double DDPG<
   EnvironmentType,
   QNetworkType,
   PolicyNetworkType,
+  NoiseType,
   UpdaterType,
   ReplayType
 >::Episode()

src/mlpack/methods/reinforcement_learning/environment/cont_double_pole_cart.hpp

@@ -1,5 +1,5 @@
 /**
- * @file methods/reinforcement_learning/environment/continuous_double_pole_cart.hpp
+ * @file methods/reinforcement_learning/environment/cont_double_pole_cart.hpp
  * @author Rahul Ganesh Prabhu
  *
  * This file is an implementation of Continuous Double Pole Cart Balancing
@@ -11,8 +11,8 @@
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
 
-#ifndef MLPACK_METHODS_RL_ENVIRONMENT_CONTINUOUS_DOUBLE_POLE_CART_HPP
-#define MLPACK_METHODS_RL_ENVIRONMENT_CONTINUOUS_DOUBLE_POLE_CART_HPP
+#ifndef MLPACK_METHODS_RL_ENVIRONMENT_CONT_DOUBLE_POLE_CART_HPP
+#define MLPACK_METHODS_RL_ENVIRONMENT_CONT_DOUBLE_POLE_CART_HPP
 
 #include <mlpack/prereqs.hpp>
 

src/mlpack/methods/reinforcement_learning/environment/environment.hpp

@@ -15,7 +15,7 @@
 #include "env_type.hpp"
 #include "acrobot.hpp"
 #include "cart_pole.hpp"
-#include "continuous_double_pole_cart.hpp"
+#include "cont_double_pole_cart.hpp"
 #include "continuous_mountain_car.hpp"
 #include "double_pole_cart.hpp"
 #include "ftn.hpp"

src/mlpack/methods/reinforcement_learning/noise/noise.hpp

@@ -0,0 +1,17 @@
+/**
+ * @file methods/reinforcement_learning/noise/noise.hpp
+ * @author Tarek Elsayed
+ *
+ * Convenience include for reinforcement learning noises.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_REINFORCEMENT_LEARNING_NOISE_NOISE_HPP
+#define MLPACK_METHODS_REINFORCEMENT_LEARNING_NOISE_NOISE_HPP
+
+#include "ornstein_uhlenbeck.hpp"
+
+#endif

src/mlpack/methods/reinforcement_learning/noise/ornstein_uhlenbeck.hpp

@@ -0,0 +1,78 @@
+/**
+ * @file methods/reinforcement_learning/noise/ornstein_uhlenbeck.hpp
+ * @author Tarek Elsayed
+ *
+ * This file is the implementation of OUNoise class.
+ * Ornstein-Uhlenbeck process generates temporally correlated exploration,
+ * and it effectively copes with physical control problems of inertia.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_RL_NOISE_ORNSTEIN_UHLENBECK_HPP
+#define MLPACK_METHODS_RL_NOISE_ORNSTEIN_UHLENBECK_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+class OUNoise
+{
+ public:
+  /**
+   * @param size The size of the noise vector.
+   * @param mu The mean of the noise process.
+   * @param theta The rate of mean reversion.
+   * @param sigma The standard deviation of the noise.
+   */
+  OUNoise(int size,
+          double mu = 0.0,
+          double theta = 0.15,
+          double sigma = 0.2) :
+      mu(mu * arma::ones<arma::colvec>(size)),
+      theta(theta),
+      sigma(sigma)
+  {
+    reset();
+  }
+
+  /**
+   * Reset the internal state to the mean (mu).
+   */
+  void reset()
+  {
+    state = mu;
+  }
+
+  /**
+   * Update the internal state and return it as a noise sample.
+   * 
+   * @return Noise sample.
+   */
+  arma::colvec sample()
+  {
+    arma::colvec x = state;
+    arma::colvec dx = theta * (mu - x) + 
+        sigma * arma::randn<arma::colvec>(x.n_elem);
+    state = x + dx;
+    return state;
+  }
+
+ private:
+  //! Locally-stored state of the noise process.
+  arma::colvec state;
+
+  //! Locally-stored mean of the noise process.
+  arma::colvec mu;
+
+  //! Locally-stored rate of mean reversion.
+  double theta;
+
+  //! Locally-stored standard deviation of the noise.
+  double sigma;
+};
+
+} // namespace mlpack
+
+#endif

src/mlpack/methods/reinforcement_learning/reinforcement_learning.hpp

@@ -17,11 +17,13 @@
 #include "q_networks/q_networks.hpp"
 #include "replay/replay.hpp"
 #include "worker/worker.hpp"
+#include "noise/noise.hpp"
 
 #include "training_config.hpp"
 #include "async_learning.hpp"
 #include "q_learning.hpp"
-#include "sac.hpp"
 #include "ddpg.hpp"
+#include "td3.hpp"
+#include "sac.hpp"
 
 #endif

src/mlpack/methods/reinforcement_learning/td3.hpp

@@ -0,0 +1,194 @@
+/**
+ * @file methods/reinforcement_learning/td3.hpp
+ * @author Tarek Elsayed
+ *
+ * This file is the definition of TD3 class, which implements the
+ * Twin Delayed Deep Deterministic Policy Gradient algorithm.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_RL_TD3_HPP
+#define MLPACK_METHODS_RL_TD3_HPP
+
+#include <mlpack/core.hpp>
+#include <ensmallen.hpp>
+#include <mlpack/methods/ann/ann.hpp>
+
+#include "replay/replay.hpp"
+#include "training_config.hpp"
+
+namespace mlpack {
+
+/**
+ * Implementation of Twin Delayed Deep Deterministic policy gradient, 
+ * a model-free off-policy actor-critic based algorithm.
+ *
+ * For more details, see the following:
+ * @code
+ * @misc{Fujimoto et al, 2018,
+ *  author    = {Scott Fujimoto,
+ *               Herke van Hoof,
+ *               David Meger},
+ *  title     = {Addressing Function Approximation Error in 
+ *               Actor-Critic Methods},
+ *  year      = {2018},
+ *  url       = {https://arxiv.org/abs/1802.09477}
+ * }
+ * @endcode
+ *
+ * @tparam EnvironmentType The environment of the reinforcement learning task.
+ * @tparam QNetworkType The network used to estimate the critic's Q-values.
+ * @tparam PolicyNetworkType The network to compute action value.
+ * @tparam UpdaterType How to apply gradients when training.
+ * @tparam ReplayType Experience replay method.
+ */
+template <
+  typename EnvironmentType,
+  typename QNetworkType,
+  typename PolicyNetworkType,
+  typename UpdaterType,
+  typename ReplayType = RandomReplay<EnvironmentType>
+>
+class TD3
+{
+ public:
+  //! Convenient typedef for state.
+  using StateType = typename EnvironmentType::State;
+
+  //! Convenient typedef for action.
+  using ActionType = typename EnvironmentType::Action;
+
+  /**
+   * Create the TD3 object with given settings.
+   *
+   * If you want to pass in a parameter and discard the original parameter
+   * object, you can directly pass the parameter, as the constructor takes
+   * a reference. This avoids unnecessary copy.
+   *
+   * @param config Hyper-parameters for training.
+   * @param learningQNetwork The network to compute action value.
+   * @param policyNetwork The network to produce an action given a state.
+   * @param replayMethod Experience replay method.
+   * @param qNetworkUpdater How to apply gradients to Q network when training.
+   * @param policyNetworkUpdater How to apply gradients to policy network
+   *        when training.
+   * @param environment Reinforcement learning task.
+   */
+  TD3(TrainingConfig& config,
+      QNetworkType& learningQNetwork,
+      PolicyNetworkType& policyNetwork,
+      ReplayType& replayMethod,
+      UpdaterType qNetworkUpdater = UpdaterType(),
+      UpdaterType policyNetworkUpdater = UpdaterType(),
+      EnvironmentType environment = EnvironmentType());
+
+  /**
+    * Clean memory.
+    */
+  ~TD3();
+
+  /**
+   * Softly update the target networks` parameters from the learning networks`
+   * parameters.
+   * 
+   * @param rho How "softly" should the parameters be copied.
+   * */
+  void SoftUpdate(double rho);
+
+  /**
+   * Update the Q and policy networks.
+   * */
+  void Update();
+
+  /**
+   * Select an action, given an agent.
+   */
+  void SelectAction();
+
+  /**
+   * Execute an episode.
+   * @return Return of the episode.
+   */
+  double Episode();
+
+  //! Modify total steps from beginning.
+  size_t& TotalSteps() { return totalSteps; }
+  //! Get total steps from beginning.
+  const size_t& TotalSteps() const { return totalSteps; }
+
+  //! Modify the state of the agent.
+  StateType& State() { return state; }
+  //! Get the state of the agent.
+  const StateType& State() const { return state; }
+
+  //! Get the action of the agent.
+  const ActionType& Action() const { return action; }
+
+  //! Modify the training mode / test mode indicator.
+  bool& Deterministic() { return deterministic; }
+  //! Get the indicator of training mode / test mode.
+  const bool& Deterministic() const { return deterministic; }
+
+
+ private:
+  //! Locally-stored hyper-parameters.
+  TrainingConfig& config;
+
+  //! Locally-stored learning Q1 and Q2 network.
+  QNetworkType& learningQ1Network;
+  QNetworkType learningQ2Network;
+
+  //! Locally-stored target Q1 and Q2 network.
+  QNetworkType targetQ1Network;
+  QNetworkType targetQ2Network;
+
+  //! Locally-stored policy network.
+  PolicyNetworkType& policyNetwork;
+
+  //! Locally-stored target policy network.
+  PolicyNetworkType targetPNetwork;
+
+  //! Locally-stored experience method.
+  ReplayType& replayMethod;
+
+  //! Locally-stored updater.
+  UpdaterType qNetworkUpdater;
+  #if ENS_VERSION_MAJOR >= 2
+  typename UpdaterType::template Policy<arma::mat, arma::mat>*
+      qNetworkUpdatePolicy;
+  #endif
+
+  //! Locally-stored updater.
+  UpdaterType policyNetworkUpdater;
+  #if ENS_VERSION_MAJOR >= 2
+  typename UpdaterType::template Policy<arma::mat, arma::mat>*
+      policyNetworkUpdatePolicy;
+  #endif
+
+  //! Locally-stored reinforcement learning task.
+  EnvironmentType environment;
+
+  //! Total steps from the beginning of the task.
+  size_t totalSteps;
+
+  //! Locally-stored current state of the agent.
+  StateType state;
+
+  //! Locally-stored action of the agent.
+  ActionType action;
+
+  //! Locally-stored flag indicating training mode or test mode.
+  bool deterministic;
+
+  //! Locally-stored loss function.
+  MeanSquaredError lossFunction;
+};
+
+} // namespace mlpack
+
+// Include implementation
+#include "td3_impl.hpp"
+#endif

src/mlpack/methods/reinforcement_learning/td3_impl.hpp

@@ -0,0 +1,362 @@
+/**
+ * @file methods/reinforcement_learning/td3_impl.hpp
+ * @author Tarek Elsayed
+ *
+ * This file is the implementation of TD3 class, which implements the
+ * Twin Delayed Deep Deterministic Policy Gradient algorithm.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_RL_td3_IMPL_HPP
+#define MLPACK_METHODS_RL_td3_IMPL_HPP
+
+#include <mlpack/prereqs.hpp>
+
+#include "td3.hpp"
+
+namespace mlpack {
+
+template <
+  typename EnvironmentType,
+  typename QNetworkType,
+  typename PolicyNetworkType,
+  typename UpdaterType,
+  typename ReplayType
+>
+TD3<
+  EnvironmentType,
+  QNetworkType,
+  PolicyNetworkType,
+  UpdaterType,
+  ReplayType
+>::TD3(TrainingConfig& config,
+       QNetworkType& learningQ1Network,
+       PolicyNetworkType& policyNetwork,
+       ReplayType& replayMethod,
+       UpdaterType qNetworkUpdater,
+       UpdaterType policyNetworkUpdater,
+       EnvironmentType environment):
+  config(config),
+  learningQ1Network(learningQ1Network),
+  policyNetwork(policyNetwork),
+  replayMethod(replayMethod),
+  qNetworkUpdater(std::move(qNetworkUpdater)),
+  #if ENS_VERSION_MAJOR >= 2
+  qNetworkUpdatePolicy(NULL),
+  #endif
+  policyNetworkUpdater(std::move(policyNetworkUpdater)),
+  #if ENS_VERSION_MAJOR >= 2
+  policyNetworkUpdatePolicy(NULL),
+  #endif
+  environment(std::move(environment)),
+  totalSteps(0),
+  deterministic(false)
+{ 
+  // Set up q-learning and policy networks.
+  targetPNetwork = policyNetwork;
+  targetQ1Network = learningQ1Network;
+  learningQ2Network = learningQ1Network;
+  targetQ2Network = learningQ2Network;
+
+  // Reset all the networks.
+  // Note: the q and policy networks have an if condition before reset.
+  // This is because we don't want to reset a loaded(possibly pretrained) model
+  // passed using this constructor.
+  const size_t envSampleSize = environment.InitialSample().Encode().n_elem;
+  if (policyNetwork.Parameters().n_elem != envSampleSize)
+    policyNetwork.Reset(envSampleSize);
+
+  targetPNetwork.Reset(envSampleSize);
+
+  const size_t networkSize = envSampleSize +
+      policyNetwork.Network()[policyNetwork.Network().size() - 1]->OutputSize();
+
+  if (learningQ1Network.Parameters().n_elem != networkSize)
+  {
+    learningQ1Network.Reset(networkSize);
+    learningQ2Network.Reset(networkSize);
+  }
+  targetQ1Network.Reset(networkSize);
+  targetQ2Network.Reset(networkSize);
+
+  #if ENS_VERSION_MAJOR == 1
+  this->qNetworkUpdater.Initialize(learningQ1Network.Parameters().n_rows,
+                                   learningQ1Network.Parameters().n_cols);
+  #else
+  this->qNetworkUpdatePolicy = new typename UpdaterType::template
+      Policy<arma::mat, arma::mat>(this->qNetworkUpdater,
+                                   learningQ1Network.Parameters().n_rows,
+                                   learningQ1Network.Parameters().n_cols);
+  #endif
+
+  #if ENS_VERSION_MAJOR == 1
+  this->policyNetworkUpdater.Initialize(policyNetwork.Parameters().n_rows,
+                                        policyNetwork.Parameters().n_cols);
+  #else
+  this->policyNetworkUpdatePolicy = new typename UpdaterType::template
+      Policy<arma::mat, arma::mat>(this->policyNetworkUpdater,
+                                   policyNetwork.Parameters().n_rows,
+                                   policyNetwork.Parameters().n_cols);
+  #endif
+
+  // Copy over the learning networks to their respective target networks.
+  targetQ1Network.Parameters() = learningQ1Network.Parameters();
+  targetQ2Network.Parameters() = learningQ2Network.Parameters();
+  targetPNetwork.Parameters() = policyNetwork.Parameters();
+}
+
+template <
+  typename EnvironmentType,
+  typename QNetworkType,
+  typename PolicyNetworkType,
+  typename UpdaterType,
+  typename ReplayType
+>
+TD3<
+  EnvironmentType,
+  QNetworkType,
+  PolicyNetworkType,
+  UpdaterType,
+  ReplayType
+>::~TD3()
+{
+  #if ENS_VERSION_MAJOR >= 2
+  delete qNetworkUpdatePolicy;
+  delete policyNetworkUpdatePolicy;
+  #endif
+}
+
+template <
+  typename EnvironmentType,
+  typename QNetworkType,
+  typename PolicyNetworkType,
+  typename UpdaterType,
+  typename ReplayType
+>
+void TD3<
+  EnvironmentType,
+  QNetworkType,
+  PolicyNetworkType,
+  UpdaterType,
+  ReplayType
+>::SoftUpdate(double rho)
+{
+  targetQ1Network.Parameters() = (1 - rho) * targetQ1Network.Parameters() +
+      rho * learningQ1Network.Parameters();
+  targetQ2Network.Parameters() = (1 - rho) * targetQ2Network.Parameters() +
+      rho * learningQ2Network.Parameters();
+  targetPNetwork.Parameters() = (1 - rho) * targetPNetwork.Parameters() +
+      rho * policyNetwork.Parameters();
+}
+
+template <
+  typename EnvironmentType,
+  typename QNetworkType,
+  typename PolicyNetworkType,
+  typename UpdaterType,
+  typename ReplayType
+>
+void TD3<
+  EnvironmentType,
+  QNetworkType,
+  PolicyNetworkType,
+  UpdaterType,
+  ReplayType
+>::Update()
+{
+  // Sample from previous experience.
+  arma::mat sampledStates;
+  std::vector<ActionType> sampledActions;
+  arma::rowvec sampledRewards;
+  arma::mat sampledNextStates;
+  arma::irowvec isTerminal;
+
+  replayMethod.Sample(sampledStates, sampledActions, sampledRewards,
+      sampledNextStates, isTerminal);
+
+  // Critic network update.
+
+  // Use the target actor to obtain the next actions.
+  arma::mat nextStateActions;
+  targetPNetwork.Predict(sampledNextStates, nextStateActions);
+
+  // Compute the estimated next Q-values using the target Q-networks.
+  arma::mat targetQInput = arma::join_vert(nextStateActions,
+      sampledNextStates);
+  arma::rowvec Q1, Q2;
+  targetQ1Network.Predict(targetQInput, Q1);
+  targetQ2Network.Predict(targetQInput, Q2);
+  arma::rowvec nextQ = sampledRewards + config.Discount() * ((1 - isTerminal)
+      % arma::min(Q1, Q2));
+
+  arma::mat sampledActionValues(action.size, sampledActions.size());
+  for (size_t i = 0; i < sampledActions.size(); i++)
+    sampledActionValues.col(i) = arma::conv_to<arma::colvec>::from
+                                 (sampledActions[i].action);
+  arma::mat learningQInput = arma::join_vert(sampledActionValues,
+      sampledStates);
+  learningQ1Network.Forward(learningQInput, Q1);
+  learningQ2Network.Forward(learningQInput, Q2);
+
+  arma::mat gradQ1Loss, gradQ2Loss;
+  lossFunction.Backward(Q1, nextQ, gradQ1Loss);
+  lossFunction.Backward(Q2, nextQ, gradQ2Loss);
+
+  // Sum both losses
+  arma::mat combinedLoss = gradQ1Loss + gradQ2Loss;
+
+  // Update the critic networks.
+  arma::mat gradientQ1, gradientQ2;
+  learningQ1Network.Backward(learningQInput, combinedLoss, gradientQ1);
+  learningQ2Network.Backward(learningQInput, combinedLoss, gradientQ2);
+  #if ENS_VERSION_MAJOR == 1
+  qNetworkUpdater.Update(learningQ1Network.Parameters(), config.StepSize(),
+      gradientQ1);
+  #else
+  qNetworkUpdatePolicy->Update(learningQ1Network.Parameters(),
+      config.StepSize(), gradientQ1);
+  #endif
+  #if ENS_VERSION_MAJOR == 1
+  qNetworkUpdater.Update(learningQ2Network.Parameters(), config.StepSize(),
+      gradientQ2);
+  #else
+  qNetworkUpdatePolicy->Update(learningQ2Network.Parameters(),
+      config.StepSize(), gradientQ2);
+  #endif
+
+  // Actor network update.
+
+  if (totalSteps % config.TargetNetworkSyncInterval() == 0)
+  {
+    // Get the size of the first hidden layer in the Q network.
+    size_t hidden1 = learningQ1Network.Network()[0]->OutputSize();
+
+    arma::mat gradient;
+    for (size_t i = 0; i < sampledStates.n_cols; i++)
+    {
+        arma::mat grad, gradQ, q;
+        arma::colvec singleState = sampledStates.col(i);
+        arma::colvec singlePi;
+        policyNetwork.Forward(singleState, singlePi);
+        arma::colvec input = arma::join_vert(singlePi, singleState);
+        arma::mat weightLastLayer;
+
+        // Note that we can use an empty matrix for the backwards pass, since the
+        // networks use EmptyLoss.
+        learningQ1Network.Forward(input, q);
+        learningQ1Network.Backward(input, arma::mat("-1"), gradQ);
+        weightLastLayer = arma::reshape(learningQ1Network.Parameters().
+            rows(0, hidden1 * singlePi.n_rows - 1), hidden1, singlePi.n_rows);
+
+        arma::colvec gradQBias = gradQ(input.n_rows * hidden1, 0,
+            arma::size(hidden1, 1));
+        arma::mat gradPolicy = weightLastLayer.t() * gradQBias;
+        policyNetwork.Backward(singleState, gradPolicy, grad);
+        if (i == 0)
+        {
+            gradient.copy_size(grad);
+            gradient.fill(0.0);
+        }
+        gradient += grad;
+    }
+    gradient /= sampledStates.n_cols;
+
+    #if ENS_VERSION_MAJOR == 1
+    policyUpdater.Update(policyNetwork.Parameters(), config.StepSize(), gradient);
+    #else
+    policyNetworkUpdatePolicy->Update(policyNetwork.Parameters(),
+        config.StepSize(), gradient);
+    #endif
+
+    // Update target networks
+    SoftUpdate(config.Rho());
+  }
+}
+
+template <
+  typename EnvironmentType,
+  typename QNetworkType,
+  typename PolicyNetworkType,
+  typename UpdaterType,
+  typename ReplayType
+>
+void TD3<
+  EnvironmentType,
+  QNetworkType,
+  PolicyNetworkType,
+  UpdaterType,
+  ReplayType
+>::SelectAction()
+{
+  // Get the action at current state, from policy.
+  arma::colvec outputAction;
+  policyNetwork.Predict(state.Encode(), outputAction);
+
+  if (!deterministic)
+  {
+    arma::colvec noise = arma::randn<arma::colvec>(outputAction.n_rows) * 0.1;
+    noise = arma::clamp(noise, -0.25, 0.25);
+    outputAction = outputAction + noise;
+  }
+  action.action = arma::conv_to<std::vector<double>>::from(outputAction);
+}
+
+template <
+  typename EnvironmentType,
+  typename QNetworkType,
+  typename PolicyNetworkType,
+  typename UpdaterType,
+  typename ReplayType
+>
+double TD3<
+  EnvironmentType,
+  QNetworkType,
+  PolicyNetworkType,
+  UpdaterType,
+  ReplayType
+>::Episode()
+{
+  // Get the initial state from environment.
+  state = environment.InitialSample();
+
+  // Track the steps in this episode.
+  size_t steps = 0;
+
+  // Track the return of this episode.
+  double totalReturn = 0.0;
+
+  // Running until get to the terminal state.
+  while (!environment.IsTerminal(state))
+  {
+    if (config.StepLimit() && steps >= config.StepLimit())
+      break;
+    SelectAction();
+
+    // Interact with the environment to advance to next state.
+    StateType nextState;
+    double reward = environment.Sample(state, action, nextState);
+
+    totalReturn += reward;
+    steps++;
+    totalSteps++;
+
+    // Store the transition for replay.
+    replayMethod.Store(state, action, reward, nextState,
+        environment.IsTerminal(nextState), config.Discount());
+
+    // Update current state.
+    state = nextState;
+
+    if (deterministic || totalSteps < config.ExplorationSteps())
+      continue;
+    for (size_t i = 0; i < config.UpdateInterval(); i++)
+      Update();
+  }
+  return totalReturn;
+}
+
+} // namespace mlpack
+#endif

src/mlpack/tests/q_learning_test.cpp

@@ -598,9 +598,19 @@ TEST_CASE("PendulumWithDDPG", "[QLearningTest]")
     qNetwork.Add(new ReLU());
     qNetwork.Add(new Linear(1));
 
+    // Set up the OUNoise parameters.
+    int size = 1; 
+    double mu = 0.0;
+    double theta = 1.0; 
+    double sigma = 0.1;  
+
+    // Create an instance of the OUNoise class.
+    OUNoise ouNoise(size, mu, theta, sigma);
+
     // Set up Deep Deterministic Policy Gradient agent.
-    DDPG<Pendulum, decltype(qNetwork), decltype(policyNetwork), AdamUpdate>
-        agent(config, qNetwork, policyNetwork, replayMethod);
+    DDPG<Pendulum, decltype(qNetwork), decltype(policyNetwork), 
+        OUNoise, AdamUpdate>
+        agent(config, qNetwork, policyNetwork, ouNoise, replayMethod);
 
     converged = testAgent<decltype(agent)>(agent, -900, 500, 10);
     if (converged)
@@ -633,8 +643,126 @@ TEST_CASE("DDPGForMultipleActions", "[QLearningTest]")
   config.TargetNetworkSyncInterval() = 1;
   config.UpdateInterval() = 3;
 
+  // Set up the OUNoise parameters.
+  int size = 4;
+  double mu = 0.0;
+  double theta = 1.0;  
+  double sigma = 0.1; 
+
+  // Create an instance of the OUNoise class.
+  OUNoise ouNoise(size, mu, theta, sigma);
+
   // Set up the DDPG agent.
   DDPG<ContinuousActionEnv<3, 4>, decltype(qNetwork), decltype(policyNetwork),
+      OUNoise, AdamUpdate>
+      agent(config, qNetwork, policyNetwork, ouNoise, replayMethod);
+
+  agent.State().Data() = arma::randu<arma::colvec>
+      (ContinuousActionEnv<3, 4>::State::dimension, 1);
+  agent.SelectAction();
+
+  // Test to check if the action dimension given by the agent is correct.
+  REQUIRE(agent.Action().action.size() == 4);
+
+  replayMethod.Store(agent.State(), agent.Action(), 1, agent.State(), 1, 0.99);
+  agent.TotalSteps()++;
+  agent.Update();
+  // If the agent is able to reach this point of the test, it is assured
+  // that the agent can handle multiple actions in continuous space.
+}
+
+//! Test Ornstein-Uhlenbeck noise class.
+TEST_CASE("OUNoiseTest", "[QLearningTest]")
+{
+  // Set up the OUNoise parameters.
+  int size = 3;
+  double mu = 0.0;
+  double theta = 0.15;
+  double sigma = 0.2;
+
+  // Create an instance of the OUNoise class.
+  OUNoise ouNoise(size, mu, theta, sigma);
+
+  // Test the reset function.
+  ouNoise.reset();
+  arma::colvec state = ouNoise.sample();
+  REQUIRE(state.n_elem == size);
+
+  // Verify that the sample is not equal to the reset state.
+  arma::colvec sample = ouNoise.sample();
+  bool isNotEqual = arma::any(sample != state);
+  REQUIRE(isNotEqual);
+}
+
+//! Test TD3 on Pendulum task.
+TEST_CASE("PendulumWithTD3", "[QLearningTest]")
+{
+  // It isn't guaranteed that the network will converge in the specified number
+  // of iterations using random weights.
+  bool converged = false;
+  for (size_t trial = 0; trial < 8; ++trial)
+  {
+    Log::Debug << "Trial number: " << trial << std::endl;
+    // Set up the replay method.
+    RandomReplay<Pendulum> replayMethod(32, 10000);
+
+    TrainingConfig config;
+    config.StepSize() = 0.001;
+    config.TargetNetworkSyncInterval() = 2;
+    config.UpdateInterval() = 3;
+
+    // Set up Actor network.
+    FFN<EmptyLoss, GaussianInitialization>
+        policyNetwork(EmptyLoss(), GaussianInitialization(0, 0.1));
+    policyNetwork.Add(new Linear(128));
+    policyNetwork.Add(new ReLU());
+    policyNetwork.Add(new Linear(1));
+    policyNetwork.Add(new TanH());
+
+    // Set up Critic network.
+    FFN<EmptyLoss, GaussianInitialization>
+        qNetwork(EmptyLoss(), GaussianInitialization(0, 0.1));
+    qNetwork.Add(new Linear(128));
+    qNetwork.Add(new ReLU());
+    qNetwork.Add(new Linear(1));
+
+    // Set up Twin Delayed Deep Deterministic policy gradient agent.
+    TD3<Pendulum, decltype(qNetwork), decltype(policyNetwork), AdamUpdate>
+        agent(config, qNetwork, policyNetwork, replayMethod);
+
+    converged = testAgent<decltype(agent)>(agent, -900, 500, 10);
+    if (converged)
+      break;
+  }
+  REQUIRE(converged);
+}
+
+//! A test to ensure TD3 works with multiple actions in action space.
+TEST_CASE("TD3ForMultipleActions", "[QLearningTest]")
+{
+  FFN<EmptyLoss, GaussianInitialization>
+      policyNetwork(EmptyLoss(), GaussianInitialization(0, 0.1));
+  policyNetwork.Add(new Linear(128));
+  policyNetwork.Add(new ReLU());
+  policyNetwork.Add(new Linear(4));
+  policyNetwork.Add(new TanH());
+
+  FFN<EmptyLoss, GaussianInitialization>
+      qNetwork(EmptyLoss(), GaussianInitialization(0, 0.1));
+  qNetwork.Add(new Linear(128));
+  qNetwork.Add(new ReLU());
+  qNetwork.Add(new Linear(1));
+
+  // Set up the replay method.
+  RandomReplay<ContinuousActionEnv<3, 4>> replayMethod(32, 10000);
+
+  TrainingConfig config;
+  config.StepSize() = 0.001;
+  config.TargetNetworkSyncInterval() = 2;
+  config.UpdateInterval() = 3;
+
+  // Set up the TD3 agent.
+  TD3<ContinuousActionEnv<3, 4>, decltype(qNetwork), decltype(policyNetwork),
       AdamUpdate>
       agent(config, qNetwork, policyNetwork, replayMethod);