From 902a7f2a88aba276e89fbfc33353038536ab5aff Mon Sep 17 00:00:00 2001
From: nishantkr18 <nniishantkumar@gmail.com>
Date: Sat, 8 Aug 2020 11:50:51 +0530
Subject: [PATCH] Added rho as a training parameter for soft actor critic

---
 .../reinforcement_learning/sac_impl.hpp       |  2 +-
 .../training_config.hpp                       | 20 ++++++++++++++++---
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/src/mlpack/methods/reinforcement_learning/sac_impl.hpp b/src/mlpack/methods/reinforcement_learning/sac_impl.hpp
index c36570aa31..f8577b8714 100644
--- a/src/mlpack/methods/reinforcement_learning/sac_impl.hpp
+++ b/src/mlpack/methods/reinforcement_learning/sac_impl.hpp
@@ -265,7 +265,7 @@ void SAC<
 
   // Update target network
   if (totalSteps % config.TargetNetworkSyncInterval() == 0)
-    SoftUpdate(0.005);
+    SoftUpdate(config.Rho());
 }
 
 template <
diff --git a/src/mlpack/methods/reinforcement_learning/training_config.hpp b/src/mlpack/methods/reinforcement_learning/training_config.hpp
index a9dbd8544d..5c83fb0985 100644
--- a/src/mlpack/methods/reinforcement_learning/training_config.hpp
+++ b/src/mlpack/methods/reinforcement_learning/training_config.hpp
@@ -33,7 +33,8 @@ class TrainingConfig
       isCategorical(false),
       atomSize(51),
       vMin(0),
-      vMax(200)
+      vMax(200),
+      rho(0.005)
   { /* Nothing to do here. */ }
 
   TrainingConfig(
@@ -50,7 +51,8 @@ class TrainingConfig
       bool isCategorical,
       size_t atomSize,
       double vMin,
-      double vMax) :
+      double vMax,
+      double rho) :
       numWorkers(numWorkers),
       updateInterval(updateInterval),
       targetNetworkSyncInterval(targetNetworkSyncInterval),
@@ -64,7 +66,8 @@ class TrainingConfig
       isCategorical(isCategorical),
       atomSize(atomSize),
       vMin(vMin),
-      vMax(vMax)
+      vMax(vMax),
+      rho(rho)
   { /* Nothing to do here. */ }
 
   //! Get the amount of workers.
@@ -141,6 +144,11 @@ class TrainingConfig
   //! Modify the maximum value for support.
   double& VMax() { return vMax; }
 
+  //! Get the rho value for sac.
+  double Rho() const { return rho; }
+  //! Modify the rho value for sac.
+  double& Rho() { return rho; }
+
  private:
   /**
    * Locally-stored number of workers.
@@ -228,6 +236,12 @@ class TrainingConfig
    * This is valid only for categorical q-network.
    */
   double vMax;
+
+  /**
+   * Locally-stored parameter for softly updating q networks.
+   * This is valid only for Soft Actor-Critic.
+   */
+  double rho;
 };
 
 } // namespace rl