Refactor tests to eliminate random failures.

The tolerance is looser, but the tests now checks whether or not using momentum gives close to the same results as without momentum (s/momentum/regularization/, too).
Fixes #417.

src/mlpack/tests/svd_batch_test.cpp

@@ -33,6 +33,29 @@ BOOST_AUTO_TEST_CASE(SVDBatchConvergenceElementTest)
                    amf.TerminationPolicy().MaxIterations());
 }
 
+//! This is used to ensure we start from the same initial point.
+class SpecificRandomInitialization
+{
+ public:
+  SpecificRandomInitialization(const size_t n, const size_t r, const size_t m) :
+      W(arma::randu<arma::mat>(n, r)),
+      H(arma::randu<arma::mat>(r, m)) { }
+
+  template<typename MatType>
+  inline void Initialize(const MatType& /* V */,
+                         const size_t /* r */,
+                         arma::mat& W,
+                         arma::mat& H)
+  {
+    W = this->W;
+    H = this->H;
+  }
+
+ private:
+  arma::mat W;
+  arma::mat H;
+};
+
 /**
  * Make sure the momentum is working okay.
  */
@@ -60,32 +83,24 @@ BOOST_AUTO_TEST_CASE(SVDBatchMomentumTest)
   // Fill sparse matrix.
   sp_mat cleanedData = arma::sp_mat(locations, values, maxUserID, maxItemID);
 
-  // Explicitly setting the random seed forces the random initialization to be
-  // the same.  There may be a better way to do this.
-  mlpack::math::RandomSeed(10);
+  // Create the initial matrices.
+  SpecificRandomInitialization sri(cleanedData.n_rows, 2, cleanedData.n_cols);
+
   ValidationRMSETermination<sp_mat> vrt(cleanedData, 2000);
   AMF<ValidationRMSETermination<sp_mat>,
-      RandomInitialization,
-      SVDBatchLearning> amf_1(vrt,
-                              RandomInitialization(),
-                              SVDBatchLearning(0.0009, 0, 0, 0));
+      SpecificRandomInitialization,
+      SVDBatchLearning> amf1(vrt, sri, SVDBatchLearning(0.0009, 0, 0, 0));
 
-  mat m1,m2;
-  double RMSE_1 = amf_1.Apply(cleanedData, 2, m1, m2);
-  size_t iter_1 = amf_1.TerminationPolicy().Iteration();
+  mat m1, m2;
+  const double regularRMSE = amf1.Apply(cleanedData, 2, m1, m2);
 
-  mlpack::math::RandomSeed(10);
   AMF<ValidationRMSETermination<sp_mat>,
-      RandomInitialization,
-      SVDBatchLearning> amf_2(vrt,
-                              RandomInitialization(),
-                              SVDBatchLearning(0.0009, 0, 0, 0.8));
+      SpecificRandomInitialization,
+      SVDBatchLearning> amf2(vrt, sri, SVDBatchLearning(0.0009, 0, 0, 0.8));
 
-  double RMSE_2 = amf_2.Apply(cleanedData, 2, m1, m2);
-  size_t iter_2 = amf_2.TerminationPolicy().Iteration();
+  const double momentumRMSE = amf2.Apply(cleanedData, 2, m1, m2);
 
-  BOOST_REQUIRE_LE(RMSE_2, RMSE_1);
-  BOOST_REQUIRE_LE(iter_2, iter_1);
+  BOOST_REQUIRE_LE(momentumRMSE, regularRMSE + 0.05);
 }
 
 /**
@@ -115,27 +130,24 @@ BOOST_AUTO_TEST_CASE(SVDBatchRegularizationTest)
   // Fill sparse matrix.
   sp_mat cleanedData = arma::sp_mat(locations, values, maxUserID, maxItemID);
 
-  mlpack::math::RandomSeed(10);
+  // Create the initial matrices.
+  SpecificRandomInitialization sri(cleanedData.n_rows, 2, cleanedData.n_cols);
+
   ValidationRMSETermination<sp_mat> vrt(cleanedData, 2000);
   AMF<ValidationRMSETermination<sp_mat>,
-      RandomInitialization,
-      SVDBatchLearning> amf_1(vrt,
-                              RandomInitialization(),
-                              SVDBatchLearning(0.0009, 0, 0, 0));
+      SpecificRandomInitialization,
+      SVDBatchLearning> amf1(vrt, sri, SVDBatchLearning(0.0009, 0, 0, 0));
 
   mat m1, m2;
-  double RMSE_1 = amf_1.Apply(cleanedData, 2, m1, m2);
+  double regularRMSE = amf1.Apply(cleanedData, 2, m1, m2);
 
-  mlpack::math::RandomSeed(10);
   AMF<ValidationRMSETermination<sp_mat>,
-      RandomInitialization,
-      SVDBatchLearning> amf_2(vrt,
-                              RandomInitialization(),
-                              SVDBatchLearning(0.0009, 0.5, 0.5, 0.8));
+      SpecificRandomInitialization,
+      SVDBatchLearning> amf2(vrt, sri, SVDBatchLearning(0.0009, 0.5, 0.5, 0.8));
 
-  double RMSE_2 = amf_2.Apply(cleanedData, 2, m1, m2);
+  double momentumRMSE = amf2.Apply(cleanedData, 2, m1, m2);
 
-  BOOST_REQUIRE_LE(RMSE_2, RMSE_1);
+  BOOST_REQUIRE_LE(momentumRMSE, regularRMSE + 0.05);
 }
 
 /**
@@ -143,7 +155,6 @@ BOOST_AUTO_TEST_CASE(SVDBatchRegularizationTest)
  */
 BOOST_AUTO_TEST_CASE(SVDBatchNegativeElementTest)
 {
-  mlpack::math::RandomSeed(std::time(NULL));
   // Create two 5x3 matrices that we should be able to recover.
   mat testLeft;
   testLeft.randu(5, 3);

src/mlpack/tests/svd_incremental_test.cpp

@@ -23,7 +23,6 @@ using namespace arma;
  */
 BOOST_AUTO_TEST_CASE(SVDIncompleteIncrementalConvergenceTest)
 {
-  mlpack::math::RandomSeed(10);
   sp_mat data;
   data.sprandn(1000, 1000, 0.2);
 
@@ -38,7 +37,7 @@ BOOST_AUTO_TEST_CASE(SVDIncompleteIncrementalConvergenceTest)
   amf.Apply(data, 2, m1, m2);
 
   BOOST_REQUIRE_NE(amf.TerminationPolicy().Iteration(),
-                    amf.TerminationPolicy().MaxIterations());
+                   amf.TerminationPolicy().MaxIterations());
 }
 
 /**
@@ -46,7 +45,6 @@ BOOST_AUTO_TEST_CASE(SVDIncompleteIncrementalConvergenceTest)
  */
 BOOST_AUTO_TEST_CASE(SVDCompleteIncrementalConvergenceTest)
 {
-  mlpack::math::RandomSeed(10);
   sp_mat data;
   data.sprandn(1000, 1000, 0.2);
 
@@ -62,9 +60,31 @@ BOOST_AUTO_TEST_CASE(SVDCompleteIncrementalConvergenceTest)
   amf.Apply(data, 2, m1, m2);
 
   BOOST_REQUIRE_NE(amf.TerminationPolicy().Iteration(),
-                    amf.TerminationPolicy().MaxIterations());
+                   amf.TerminationPolicy().MaxIterations());
 }
 
+//! This is used to ensure we start from the same initial point.
+class SpecificRandomInitialization
+{
+ public:
+  SpecificRandomInitialization(const size_t n, const size_t r, const size_t m) :
+      W(arma::randu<arma::mat>(n, r)),
+      H(arma::randu<arma::mat>(r, m)) { }
+
+  template<typename MatType>
+  inline void Initialize(const MatType& /* V */,
+                         const size_t /* r */,
+                         arma::mat& W,
+                         arma::mat& H)
+  {
+    W = this->W;
+    H = this->H;
+  }
+
+ private:
+  arma::mat W;
+  arma::mat H;
+};
 
 BOOST_AUTO_TEST_CASE(SVDIncompleteIncrementalRegularizationTest)
 {
@@ -91,29 +111,27 @@ BOOST_AUTO_TEST_CASE(SVDIncompleteIncrementalRegularizationTest)
   sp_mat cleanedData = arma::sp_mat(locations, values, maxUserID, maxItemID);
   sp_mat cleanedData2 = cleanedData;
 
-  mlpack::math::RandomSeed(10);
+  SpecificRandomInitialization sri(cleanedData.n_rows, 2, cleanedData.n_cols);
+
   ValidationRMSETermination<sp_mat> vrt(cleanedData, 2000);
   AMF<IncompleteIncrementalTermination<ValidationRMSETermination<sp_mat> >,
-      RandomInitialization,
-      SVDIncompleteIncrementalLearning> amf_1(vrt,
-                              RandomInitialization(),
-                              SVDIncompleteIncrementalLearning(0.001, 0, 0));
+      SpecificRandomInitialization,
+      SVDIncompleteIncrementalLearning> amf1(vrt, sri,
+      SVDIncompleteIncrementalLearning(0.001, 0, 0));
 
-  mat m1,m2;
-  double RMSE_1 = amf_1.Apply(cleanedData, 2, m1, m2);
+  mat m1, m2;
+  double regularRMSE = amf1.Apply(cleanedData, 2, m1, m2);
 
-  mlpack::math::RandomSeed(10);
   ValidationRMSETermination<sp_mat> vrt2(cleanedData2, 2000);
   AMF<IncompleteIncrementalTermination<ValidationRMSETermination<sp_mat> >,
-      RandomInitialization,
-      SVDIncompleteIncrementalLearning> amf_2(vrt2,
-                              RandomInitialization(),
-                              SVDIncompleteIncrementalLearning(0.001, 0.01, 0.01));
+      SpecificRandomInitialization,
+      SVDIncompleteIncrementalLearning> amf2(vrt2, sri,
+      SVDIncompleteIncrementalLearning(0.001, 0.01, 0.01));
 
   mat m3, m4;
-  double RMSE_2 = amf_2.Apply(cleanedData2, 2, m3, m4);
+  double regularizedRMSE = amf2.Apply(cleanedData2, 2, m3, m4);
 
-  BOOST_REQUIRE_LT(RMSE_2, RMSE_1);
+  BOOST_REQUIRE_LT(regularizedRMSE, regularRMSE + 0.075);
 }
 
 BOOST_AUTO_TEST_SUITE_END();