diff --git a/src/mlpack/core/optimizers/sgd/decay_policies/no_decay.hpp b/src/mlpack/core/optimizers/sgd/decay_policies/no_decay.hpp
index 44f77a8ee5..a26dfb625c 100644
--- a/src/mlpack/core/optimizers/sgd/decay_policies/no_decay.hpp
+++ b/src/mlpack/core/optimizers/sgd/decay_policies/no_decay.hpp
@@ -42,6 +42,25 @@ class NoDecay
   {
     // Nothing to do here.
   }
+
+  /**
+   * This function is called in each iteration after the Update step for SVRG.
+   *
+   * @param iterate Parameters that minimize the function.
+   * @param iterate0 The last function parameters at time t - 1.
+   * @param gradient The current gradient matrix at time t.
+   * @param fullGradient The computed full gradient.
+   * @param stepSize Step size to be used for the given iteration.
+   */
+  void Update(const arma::mat& /* iterate */,
+              const arma::mat& /*iterate0 */,
+              const arma::mat& /* gradient */,
+              const arma::mat& /* fullGradient */,
+              const size_t /* numBatches */,
+              double& /* stepSize */)
+  {
+    // Nothing to do here.
+  }
 };
 
 } // namespace optimization
diff --git a/src/mlpack/core/optimizers/svrg/CMakeLists.txt b/src/mlpack/core/optimizers/svrg/CMakeLists.txt
index f40b45eabe..71f918f019 100644
--- a/src/mlpack/core/optimizers/svrg/CMakeLists.txt
+++ b/src/mlpack/core/optimizers/svrg/CMakeLists.txt
@@ -1,7 +1,6 @@
 set(SOURCES
   svrg.hpp
   svrg_impl.hpp
-  no_decay.hpp
   barzilai_borwein_decay.hpp
   svrg_update.hpp
 )
diff --git a/src/mlpack/core/optimizers/svrg/no_decay.hpp b/src/mlpack/core/optimizers/svrg/no_decay.hpp
deleted file mode 100644
index bf954d6e28..0000000000
--- a/src/mlpack/core/optimizers/svrg/no_decay.hpp
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * @file no_decay.hpp
- * @author Marcus Edel
- *
- * Definition of the policy type for the decay class.
- *
- * You should define your own decay update that looks like NoDecay.
- *
- * mlpack is free software; you may redistribute it and/or modify it under the
- * terms of the 3-clause BSD license.  You should have received a copy of the
- * 3-clause BSD license along with mlpack.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-
-#ifndef MLPACK_CORE_OPTIMIZERS_SVRG_NO_DECAY_HPP
-#define MLPACK_CORE_OPTIMIZERS_SVRG_NO_DECAY_HPP
-
-#include <mlpack/prereqs.hpp>
-
-namespace mlpack {
-namespace optimization {
-
-/**
- * Definition of the NoDecay class. Use this as a template for your own.
- */
-class NoDecay
-{
- public:
-  /**
-   * This constructor is called before the first iteration.
-   */
-  NoDecay() { }
-
-  /**
-   * Update step for SVRG. The function parameters are updated in the negative
-   * direction of the gradient.
-   *
-   * @param iterate Parameters that minimize the function.
-   * @param iterate0 The last function parameters at time t - 1.
-   * @param gradient The current gradient matrix at time t.
-   * @param fullGradient The computed full gradient.
-   * @param stepSize Step size to be used for the given iteration.
-   */
-  void Update(const arma::mat& /* iterate */,
-              const arma::mat& /*iterate0 */,
-              const arma::mat& /* gradient */,
-              const arma::mat& /* fullGradient */,
-              const size_t /* numBatches */,
-              double& /* stepSize */)
-  {
-    // Nothing to do here.
-  }
-};
-
-} // namespace optimization
-} // namespace mlpack
-
-#endif // MLPACK_CORE_OPTIMIZERS_SVRG_NO_DECAY_HPP
diff --git a/src/mlpack/core/optimizers/svrg/svrg.hpp b/src/mlpack/core/optimizers/svrg/svrg.hpp
index fdc2282a91..c4a6ac61ea 100644
--- a/src/mlpack/core/optimizers/svrg/svrg.hpp
+++ b/src/mlpack/core/optimizers/svrg/svrg.hpp
@@ -13,9 +13,9 @@
 #define MLPACK_CORE_OPTIMIZERS_SVRG_SVRG_HPP
 
 #include <mlpack/prereqs.hpp>
+#include <mlpack/core/optimizers/sgd/decay_policies/no_decay.hpp>
 
 #include "svrg_update.hpp"
-#include "no_decay.hpp"
 #include "barzilai_borwein_decay.hpp"
 
 namespace mlpack {