diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp
index 72a1e5ebd6..b3ec8e33a6 100644
--- a/src/mlpack/tests/recurrent_network_test.cpp
+++ b/src/mlpack/tests/recurrent_network_test.cpp
@@ -17,6 +17,7 @@
 #include <mlpack/core/optimizers/sgd/sgd.hpp>
 #include <mlpack/methods/ann/activation_functions/logistic_function.hpp>
 #include <mlpack/methods/ann/init_rules/random_init.hpp>
+ #include <mlpack/methods/ann/init_rules/nguyen_widrow_init.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "old_boost_test_definitions.hpp"
@@ -105,7 +106,7 @@ BOOST_AUTO_TEST_CASE(SequenceClassificationTest)
   RNN<decltype(modules), BinaryClassificationLayer, RandomInitialization,
       MeanSquaredErrorFunction> net(modules, classOutputLayer);
 
-  SGD<decltype(net)> opt(net, 0.5, 400 * input.n_cols, -100);
+  SGD<decltype(net)> opt(net, 0.5, 500 * input.n_cols, -100);
 
   net.Train(input, labels, opt);
 
@@ -533,13 +534,13 @@ void DistractedSequenceRecallTestNetwork(HiddenLayerType& hiddenLayer0)
   auto modules = std::tie(linearLayer0, recurrentLayer0, hiddenLayer0,
                           hiddenLayer, hiddenBaseLayer);
 
-  RNN<decltype(modules), BinaryClassificationLayer, RandomInitialization,
+  RNN<decltype(modules), BinaryClassificationLayer, NguyenWidrowInitialization,
       MeanSquaredErrorFunction> net(modules, classOutputLayer);
 
-  SGD<decltype(net)> opt(net, 0.05, 2, -200);
+  SGD<decltype(net)> opt(net, 0.04, 2, -200);
 
   arma::mat inputTemp, labelsTemp;
-  for (size_t i = 0; i < 30; i++)
+  for (size_t i = 0; i < 40; i++)
   {
     for (size_t j = 0; j < trainDistractedSequenceCount; j++)
     {
diff --git a/src/mlpack/tests/rmsprop_test.cpp b/src/mlpack/tests/rmsprop_test.cpp
index f1115925c0..62f8cc001f 100644
--- a/src/mlpack/tests/rmsprop_test.cpp
+++ b/src/mlpack/tests/rmsprop_test.cpp
@@ -45,9 +45,9 @@ BOOST_AUTO_TEST_CASE(SimpleRMSpropTestFunction)
   arma::mat coordinates = f.GetInitialPoint();
   optimizer.Optimize(coordinates);
 
-  BOOST_REQUIRE_SMALL(coordinates[0], 1e-3);
-  BOOST_REQUIRE_SMALL(coordinates[1], 1e-3);
-  BOOST_REQUIRE_SMALL(coordinates[2], 1e-3);
+  BOOST_REQUIRE_SMALL(coordinates[0], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[1], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
 }
 
 /**
@@ -141,8 +141,8 @@ BOOST_AUTO_TEST_CASE(FeedforwardTest)
   FFN<decltype(modules), decltype(classOutputLayer), RandomInitialization,
       MeanSquaredErrorFunction> net(modules, classOutputLayer);
 
-  RMSprop<decltype(net)> opt(net, 0.1, 0.88, 1e-15,
-      300 * input.n_cols, 1e-18);
+  RMSprop<decltype(net)> opt(net, 0.03, 0.88, 1e-15,
+      300 * input.n_cols, -10);
 
   net.Train(input, labels, opt);