diff --git a/src/mlpack/methods/ann/cnn.hpp b/src/mlpack/methods/ann/cnn.hpp
index 7e6158c983..e1872e4fb1 100644
--- a/src/mlpack/methods/ann/cnn.hpp
+++ b/src/mlpack/methods/ann/cnn.hpp
@@ -182,7 +182,7 @@ class CNN
    */
   double Evaluate(const arma::mat& parameters,
                   const size_t i,
-                  const bool deterministic = false);
+                  const bool deterministic = true);
 
   /**
    * Evaluate the gradient of the convolutional neural network with the given
diff --git a/src/mlpack/methods/ann/cnn_impl.hpp b/src/mlpack/methods/ann/cnn_impl.hpp
index 5950fdaf9f..1494f9c9a9 100644
--- a/src/mlpack/methods/ann/cnn_impl.hpp
+++ b/src/mlpack/methods/ann/cnn_impl.hpp
@@ -245,9 +245,11 @@ template<typename LayerTypes,
 void CNN<
 LayerTypes, OutputLayerType, InitializationRuleType, PerformanceFunction
 >::Gradient(const arma::mat& /* unused */,
-            const size_t /* unused */,
+            const size_t i,
             arma::mat& gradient)
 {
+  Evaluate(parameter, i, false);
+
   NetworkGradients(gradient, network);
 
   Backward<>(error, network);
diff --git a/src/mlpack/methods/ann/ffn.hpp b/src/mlpack/methods/ann/ffn.hpp
index d4cd35d321..2bed68aaca 100644
--- a/src/mlpack/methods/ann/ffn.hpp
+++ b/src/mlpack/methods/ann/ffn.hpp
@@ -183,7 +183,7 @@ class FFN
    */
   double Evaluate(const arma::mat& parameters,
                   const size_t i,
-                  const bool deterministic = false);
+                  const bool deterministic = true);
 
   /**
    * Evaluate the gradient of the feedforward network with the given parameters,
diff --git a/src/mlpack/methods/ann/ffn_impl.hpp b/src/mlpack/methods/ann/ffn_impl.hpp
index a057f6bfc2..ca48c307a2 100644
--- a/src/mlpack/methods/ann/ffn_impl.hpp
+++ b/src/mlpack/methods/ann/ffn_impl.hpp
@@ -249,9 +249,11 @@ template<typename LayerTypes,
 void FFN<
 LayerTypes, OutputLayerType, InitializationRuleType, PerformanceFunction
 >::Gradient(const arma::mat& /* unused */,
-            const size_t /* unused */,
+            const size_t i,
             arma::mat& gradient)
 {
+  Evaluate(parameter, i, false);
+
   NetworkGradients(gradient, network);
 
   Backward<>(error, network);
diff --git a/src/mlpack/methods/ann/rnn.hpp b/src/mlpack/methods/ann/rnn.hpp
index 9d1d21bd36..1389a87433 100644
--- a/src/mlpack/methods/ann/rnn.hpp
+++ b/src/mlpack/methods/ann/rnn.hpp
@@ -185,7 +185,7 @@ class RNN
    */
   double Evaluate(const arma::mat& parameters,
                   const size_t i,
-                  const bool deterministic = false);
+                  const bool deterministic = true);
 
   /**
    * Evaluate the gradient of the recurrent neural network with the given
diff --git a/src/mlpack/methods/ann/rnn_impl.hpp b/src/mlpack/methods/ann/rnn_impl.hpp
index a01a3abebf..f5b519a6ce 100644
--- a/src/mlpack/methods/ann/rnn_impl.hpp
+++ b/src/mlpack/methods/ann/rnn_impl.hpp
@@ -281,6 +281,8 @@ LayerTypes, OutputLayerType, InitializationRuleType, PerformanceFunction
             const size_t i,
             arma::mat& gradient)
 {
+  Evaluate(parameter, i, false);
+
   gradient.zeros();
   arma::mat currentGradient = arma::mat(gradient.n_rows, gradient.n_cols);
   NetworkGradients(currentGradient, network);